Created a tokenizer and the supporting code, currently processes text that is not a string literal.

includes/token.h

@@ -1,11 +1,17 @@
 #ifndef TOKEN_H
 #define TOKEN_H
 
-enum TokenType {
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdbool.h>
+
+typedef enum {
     Empty,
     String,
     Number,
     Symbol,
+    LineEnd = '\n',
     Plus = '+',
     Minus = '-',
     Star = '*',
@@ -13,23 +19,25 @@ enum TokenType {
     Carot = '^',
     OpenParen = '(',
     CloseParen = ')',
+    OpenBracket = '[',
+    CloseBracket = ']',
     FileEnd
-};
+} TokenType;
 
 typedef struct _token {
     TokenType Type;
-    char* Name;
     union {
+        char Operator[8];
         char* String;
         int Number;
     } Value;
     int LineNumber;
     int ColumnNumber;
-    int Index;
-    int Length;
-    int Resolved;
+    bool Resolved;
 } Token;
 
-Token* TokenCreate(char* name, TokenType type, int lineNumber, int columnNumber, int index, int length, int resolved);
+Token* TokenCreate(TokenType type, int lineNumber, int columnNumber, bool resolved);
+char* TokenStringify(const Token* token, bool valueOnly);
+int StringifyTokenType(TokenType type, char buffer[32]);
 
 #endif