From 375fd46045f574848387299623a54ed0bcde37ed Mon Sep 17 00:00:00 2001
From: Garritt McCune <glmccune@gmail.com>
Date: Thu, 27 Jan 2022 18:20:09 +0000
Subject: [PATCH] Split up some code to more in line with what its actually
 doing.

---
 includes/lexer.h     |  36 +++++++++
 includes/tokenizer.h |  27 +------
 src/lexer.c          | 138 ++++++++++++++++++++++++++++++++
 src/main.c           |   4 +-
 src/tokenizer.c      | 182 ++++++-------------------------------------
 5 files changed, 202 insertions(+), 185 deletions(-)
 create mode 100644 includes/lexer.h
 create mode 100644 src/lexer.c

diff --git a/includes/lexer.h b/includes/lexer.h
new file mode 100644
index 0000000..84b1f12
--- /dev/null
+++ b/includes/lexer.h
@@ -0,0 +1,36 @@
+#ifndef LEXER_H
+#define LEXER_H
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+#include "list.h"
+#include "tokenizer.h"
+
+typedef enum {
+    TK_Add,
+    TK_Sub,
+    TK_Mul,
+    TK_Div,
+    TK_Power,
+    TK_LParam,
+    TK_RParam,
+    TK_LBracket,
+    TK_RBracket,
+    TK_String,
+    TK_Number,
+    TK_Hex,
+    TK_Opcode,
+    TK_Register,
+    TK_Label
+} TokenType;
+
+typedef struct {
+    TokenType type;
+    char* value;
+} Token;
+
+List* GenerateTokensFromFile(const char*);
+
+#endif
\ No newline at end of file
diff --git a/includes/tokenizer.h b/includes/tokenizer.h
index 7be1f97..30ce8e9 100644
--- a/includes/tokenizer.h
+++ b/includes/tokenizer.h
@@ -6,31 +6,6 @@
 #include <string.h>
 #include "list.h"
 
-typedef enum {
-    TK_Add,
-    TK_Sub,
-    TK_Mul,
-    TK_Div,
-    TK_Power,
-    TK_LParam,
-    TK_RParam,
-    TK_Comma,
-    TK_LBracket,
-    TK_RBracket,
-    TK_Colon,
-    TK_Text,
-    TK_Number,
-    TK_Hex,
-    TK_Opcode,
-    TK_Register,
-    TK_Invalid
-} TokenType;
-
-typedef struct {
-    TokenType type;
-    char* value;
-} Token;
-
-List* TokenizeString(const char *);
+char* SplitOnBasicGrammar(char*);
 
 #endif
\ No newline at end of file
diff --git a/src/lexer.c b/src/lexer.c
new file mode 100644
index 0000000..fc5a1e6
--- /dev/null
+++ b/src/lexer.c
@@ -0,0 +1,138 @@
+#include "../includes/lexer.h"
+#include "../includes/opcodes.h"
+#include <stdio.h>
+#include <stdlib.h>
+
+Token* CreateToken(TokenType, char*);
+Token* GetNextToken(char *);
+int TokenIsNumeric(const char*, int *);
+
+List* GenerateTokensFromFile(const char* file_path) {
+    FILE *file;
+    int line_count = 1;
+    char* line = NULL;
+    size_t len = 0;
+    ssize_t bytes_read;
+
+    file = fopen(file_path, "r");
+
+    if (!file) {
+        printf("Failed to open '%s' for reading.\n", file_path);
+        return NULL;
+    }
+
+    //char* context;// = malloc(sizeof(char *));
+
+    while((bytes_read = getline(&line, &len, file)) != -1) {
+        Token* token = GetNextToken(line);
+
+        while (token) {
+
+            if (!token) break;
+
+            if(token->type == TK_Number)  {
+                printf("Found number: '%s'\n", token->value);
+            }
+            else if (token->type == TK_Hex) {
+                printf("Hex number: '%s'\n", token->value);
+            }
+            else if (token->type == TK_Opcode) {
+                printf("Found op: '%s'\n", token->value);
+            }
+            else if (token->type == TK_Register) {
+                printf("Found Reg: '%s'\n", token->value);
+            }
+            else if(token->type == TK_Label) {
+                printf("Label: '%s'\n", token->value);
+            }
+            else {
+                printf("Found text: '%s'\n", token->value);
+            }
+
+            free(token->value);
+            free(token);
+
+            token = GetNextToken(NULL);
+        }
+
+        line_count++;
+    }
+
+    fclose(file);
+
+    if (line) free(line);
+
+    return NULL;
+}
+
+Token* GetNextToken(char *string) {
+    char* string_token = SplitOnBasicGrammar(string);
+    int base = 0;
+    
+    while (strlen(string_token) != 0) {
+        
+        if (TokenIsNumeric(string_token, &base)) {
+            if (base == 10) return CreateToken(TK_Number, string_token);
+            if (base == 16) return CreateToken(TK_Hex, string_token);
+        }
+
+        //if (strcmp(string_token, ":") == 0) return CreateToken(TK_Colon, string_token);
+        //if (strcmp(string_token, ",") == 0) return CreateToken(TK_Comma, string_token);
+        if (IsOpcode(string_token)) return CreateToken(TK_Opcode, string_token);
+        if (IsRegister(string_token)) return CreateToken(TK_Register, string_token);
+
+        char* next = SplitOnBasicGrammar(NULL);
+
+        if (next && strcmp(next, ":") == 0) {
+            free(next);
+            return CreateToken(TK_Label, string_token);
+        }
+
+        if (next) free(next);
+
+        return CreateToken(TK_String, string_token);
+    }
+
+    return NULL;
+}
+
+Token* CreateToken(TokenType type, char* value) {
+    Token* token = calloc(1, sizeof(Token));
+
+    if (!token) return NULL;
+
+    token->type = type;
+    token->value = value;
+
+    return token;
+}
+
+int TokenIsNumeric(const char* token, int *base) {
+    *base = 0;
+
+    if (!token) return 0;
+    
+    unsigned long length = strlen(token);
+    int i = 0;
+
+    if (length == 0) return 0;
+
+    *base = 10;
+
+    if (length > 2) {
+        if (token[0] == '0' && token[1] == 'x') { 
+            *base = 16;
+            i = 2;
+        }
+    }
+
+    for(; i < length; i++) {
+        if (!isdigit(token[i])) { 
+            if (*base == 16 && token[i] >= 'A' && token[i] <= 'F') continue;
+
+            return 0;
+        }
+    }
+
+    return 1;
+}
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index 2e0b020..7c748ef 100644
--- a/src/main.c
+++ b/src/main.c
@@ -2,7 +2,7 @@
 #include <stdio.h>
 #include <string.h>
 #include "../includes/list.h"
-#include "../includes/tokenizer.h"
+#include "../includes/lexer.h"
 
 typedef struct {
     char *opcode;
@@ -22,7 +22,7 @@ int main(int argc, char* args[]) {
     }
     // char* test = "user_id";
     // printf("Key for '%s' is %d for size %d\n", test, GetKeyIndex(test, HASHTABLEDEFAULTSIZE), HASHTABLEDEFAULTSIZE);
-    TokenizeString(args[1]);
+    GenerateTokensFromFile(args[1]);
 
     // for (int i = 1; i < argc; i++) {
     //     input_files[i - 1] = args[i];
diff --git a/src/tokenizer.c b/src/tokenizer.c
index 319efe5..85fa6d7 100644
--- a/src/tokenizer.c
+++ b/src/tokenizer.c
@@ -1,136 +1,4 @@
 #include "../includes/tokenizer.h"
-#include "../includes/opcodes.h"
-#include <stddef.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <ctype.h>
-#include <string.h>
-
-Token* GetNextToken(char *);
-TokenType GetOperatorType(char);
-char* SplitOnBasicGrammar(char*);
-int TokenIsNumeric(const char*, int *);
-Token* CreateToken(TokenType, char*);
-
-List* TokenizeString(const char *file_path) {
-    FILE *file;
-    char* line = NULL;
-    size_t len = 0;
-    ssize_t bytes_read;
-
-    file = fopen(file_path, "r");
-
-    if (!file) {
-        printf("Failed to open '%s' for reading.\n", file_path);
-        return NULL;
-    }
-
-    while((bytes_read = getline(&line, &len, file)) != -1) {
-        Token* token = GetNextToken(line);
-
-        while (token) {
-            if (!token) break;
-
-            if(token->type == TK_Number)  {
-                printf("Found number: '%s'\n", token->value);
-            }
-            else if (token->type == TK_Hex) {
-                printf("Hex number: '%s'\n", token->value);
-            }
-            else if(token->type == TK_Colon) {
-                printf("Colon found\n");
-            }
-            else if (token->type == TK_Comma) {
-                printf("Comma\n");
-            }
-            else if (token->type == TK_Opcode) {
-                printf("Found op: '%s'\n", token->value);
-            }
-            else if (token->type == TK_Register) {
-                printf("Found Reg: '%s'\n", token->value);
-            }
-            else {
-                printf("Found text: '%s'\n", token->value);
-            }
-
-            free(token->value);
-            free(token);
-
-            token = GetNextToken(NULL);
-        }
-    }
-
-    fclose(file);
-
-    if (line) free(line);
-
-    return NULL;
-}
-
-Token* GetNextToken(char *string) {
-    char* string_token = SplitOnBasicGrammar(string);
-    int base = 0;
-    
-    while (strlen(string_token) != 0) {
-        
-        if (TokenIsNumeric(string_token, &base)) {
-            if (base == 10) return CreateToken(TK_Number, string_token);
-            if (base == 16) return CreateToken(TK_Hex, string_token);
-        }
-
-        if (strcmp(string_token, ":") == 0) return CreateToken(TK_Colon, string_token);
-        if (strcmp(string_token, ",") == 0) return CreateToken(TK_Comma, string_token);
-        if (IsOpcode(string_token)) return CreateToken(TK_Opcode, string_token);
-        if (IsRegister(string_token)) return CreateToken(TK_Register, string_token);
-
-        return CreateToken(TK_Text, string_token);
-    }
-    
-    free(string_token);
-
-    return NULL;
-}
-
-Token* CreateToken(TokenType type, char* value) {
-    Token* token = calloc(1, sizeof(Token));
-
-    if (!token) return NULL;
-
-    token->type = type;
-    token->value = value;
-
-    return token;
-}
-
-int TokenIsNumeric(const char* token, int *base) {
-    *base = 0;
-
-    if (!token) return 0;
-    
-    unsigned long length = strlen(token);
-    int i = 0;
-
-    if (length == 0) return 0;
-
-    *base = 10;
-
-    if (length > 2) {
-        if (token[0] == '0' && token[1] == 'x') { 
-            *base = 16;
-            i = 2;
-        }
-    }
-
-    for(; i < length; i++) {
-        if (!isdigit(token[i])) { 
-            if (*base == 16 && token[i] >= 'A' && token[i] <= 'F') continue;
-
-            return 0;
-        }
-    }
-
-    return 1;
-}
 
 char* SplitOnBasicGrammar(char* line) {
     static char* string;
@@ -195,29 +63,29 @@ char* SplitOnBasicGrammar(char* line) {
     return token;
 }
 
-TokenType GetOperatorType(char c) {
-    switch (c){
-        case '+':
-        return TK_Add;
-        case '-':
-        return TK_Sub;
-        case '*':
-        return TK_Mul;
-        case '/':
-        return TK_Div;
-        case '^':
-        return TK_Power;
-        case ':':
-        return TK_Colon;
-        case '(':
-        return TK_LParam;
-        case ')':
-        return TK_RParam;
-        case '[':
-        return TK_LBracket;
-        case ']':
-        return TK_RBracket;
-    };
+// TokenType GetOperatorType(char c) {
+//     switch (c){
+//         case '+':
+//         return TK_Add;
+//         case '-':
+//         return TK_Sub;
+//         case '*':
+//         return TK_Mul;
+//         case '/':
+//         return TK_Div;
+//         case '^':
+//         return TK_Power;
+//         case ':':
+//         return TK_Colon;
+//         case '(':
+//         return TK_LParam;
+//         case ')':
+//         return TK_RParam;
+//         case '[':
+//         return TK_LBracket;
+//         case ']':
+//         return TK_RBracket;
+//     };
 
-    return TK_Invalid;
-}
\ No newline at end of file
+//     return TK_Invalid;
+// }
\ No newline at end of file