Split up some code to more in line with what its actually doing.

includes/lexer.h

@@ -0,0 +1,36 @@
+#ifndef LEXER_H
+#define LEXER_H
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+#include "list.h"
+#include "tokenizer.h"
+
+typedef enum {
+    TK_Add,
+    TK_Sub,
+    TK_Mul,
+    TK_Div,
+    TK_Power,
+    TK_LParam,
+    TK_RParam,
+    TK_LBracket,
+    TK_RBracket,
+    TK_String,
+    TK_Number,
+    TK_Hex,
+    TK_Opcode,
+    TK_Register,
+    TK_Label
+} TokenType;
+
+typedef struct {
+    TokenType type;
+    char* value;
+} Token;
+
+List* GenerateTokensFromFile(const char*);
+
+#endif

includes/tokenizer.h

@@ -6,31 +6,6 @@
 #include <string.h>
 #include "list.h"
 
-typedef enum {
-    TK_Add,
-    TK_Sub,
-    TK_Mul,
-    TK_Div,
-    TK_Power,
-    TK_LParam,
-    TK_RParam,
-    TK_Comma,
-    TK_LBracket,
-    TK_RBracket,
-    TK_Colon,
-    TK_Text,
-    TK_Number,
-    TK_Hex,
-    TK_Opcode,
-    TK_Register,
-    TK_Invalid
-} TokenType;
-
-typedef struct {
-    TokenType type;
-    char* value;
-} Token;
-
-List* TokenizeString(const char *);
+char* SplitOnBasicGrammar(char*);
 
 #endif