Created a tokenizer and the supporting code, currently processes text that is not a string literal.

includes/array.h

@@ -0,0 +1,13 @@
+#ifndef ARRAY_H
+#define ARRAY_H
+
+typedef struct _array {
+    int Capacity;
+    int Size;
+    void** Items;
+} Array;
+
+Array* ArrayCreate(void);
+int ArrayAdd(Array* array, void* item);
+
+#endif

includes/opcodes.h

@@ -10,13 +10,13 @@ typedef enum {
 } Registers;
 
 typedef enum {
-    ADD = 0x01, SUB, MUL, DIV, MOV, AND, OR, XOR, NOT, SHL, SHR, NOP, CMP, JMP, JG, JL, 
-    OUTB, INB, HLT, ENI, INT, LIVT, PUSHA, POPA, CALL, RET, PUSH, POP
+    ADD = 0x01, SUB, MUL, DIV, MOV, AND, OR, XOR, NOT, SHL, SHR, NOP, CMP, JMP, JZ, JG, JL, 
+    OUTB, INB, HLT, CLI, ENI, INT, LIVT, PUSHA, POPA, CALL, RET, PUSH, POP
 } Mnemonic;
 
 int IsOpcode(const char*, Mnemonic*);
 int IsRegister(const char*, Registers*);
 void GetMnemonicText(Mnemonic mnemonic, char buffer[12]);
-void GetRegisterText(Registers reg, char buffer[3]);
+void GetRegisterText(Registers reg, char buffer[4]);
 
 #endif

includes/token.h

@@ -1,11 +1,17 @@
 #ifndef TOKEN_H
 #define TOKEN_H
 
-enum TokenType {
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdbool.h>
+
+typedef enum {
     Empty,
     String,
     Number,
     Symbol,
+    LineEnd = '\n',
     Plus = '+',
     Minus = '-',
     Star = '*',
@@ -13,23 +19,25 @@ enum TokenType {
     Carot = '^',
     OpenParen = '(',
     CloseParen = ')',
+    OpenBracket = '[',
+    CloseBracket = ']',
     FileEnd
-};
+} TokenType;
 
 typedef struct _token {
     TokenType Type;
-    char* Name;
     union {
+        char Operator[8];
         char* String;
         int Number;
     } Value;
     int LineNumber;
     int ColumnNumber;
-    int Index;
-    int Length;
-    int Resolved;
+    bool Resolved;
 } Token;
 
-Token* TokenCreate(char* name, TokenType type, int lineNumber, int columnNumber, int index, int length, int resolved);
+Token* TokenCreate(TokenType type, int lineNumber, int columnNumber, bool resolved);
+char* TokenStringify(const Token* token, bool valueOnly);
+int StringifyTokenType(TokenType type, char buffer[32]);
 
 #endif

includes/tokenizer.h

@@ -0,0 +1,10 @@
+#ifndef TOKENIZER_H
+#define TOKENIZER_H
+
+#include "array.h"
+#include "dictionary.h"
+#include "token.h"
+
+Array* Tokenize(const char* text, Dictionary** variables);
+
+#endif