Cleaned up the tokenizer and setting the stage for more granular token types.

src/tokenizer.c

@@ -9,13 +9,13 @@ Token* GetNextToken(char *);
 TokenType GetOperatorType(char);
 char* SplitOnBasicGrammar(char*);
 int TokenIsNumeric(const char*);
+Token* CreateToken(TokenType, char*);
 
 List* TokenizeString(const char *file_path) {
     FILE *file;
     char* line = NULL;
     size_t len = 0;
     ssize_t bytes_read;
-    //List* tokens = CreateList();
 
     file = fopen(file_path, "r");
 
@@ -27,15 +27,19 @@ List* TokenizeString(const char *file_path) {
     while((bytes_read = getline(&line, &len, file)) != -1) {
         Token* token = GetNextToken(line);
 
-        // for(int i = 0; i < tokens->size; i++) {
-        //     printf("Token type '%d' with value '%s'\n", ((Token* ) tokens->content[i])->type, ((Token*) tokens->content[i])->value);
-        // }
         while (token) {
-            if(token)  {
+            if (!token) break;
+
+            if(token->type == TK_Number)  {
                 printf("Found number: '%s'\n", token->value);
-                free(token->value);
-                free(token);
             }
+            else {
+                printf("Found text: '%s'\n", token->value);
+            }
+
+            free(token->value);
+            free(token);
+
             token = GetNextToken(NULL);
         }
     }
@@ -48,48 +52,31 @@ List* TokenizeString(const char *file_path) {
 }
 
 Token* GetNextToken(char *string) {
-    // static char* text;
-    // static int position;
-    // static unsigned long length;
-
-    // if (string) {
-    //     length = strlen(string);
-
-    //     if (length == 0) return NULL;
-
-    //     text = string;
-    //     position = 0;
-    // }
-
     char* string_token = SplitOnBasicGrammar(string);
-    //Token* token = malloc(sizeof(Token));
     
     while (strlen(string_token) != 0) {
-        //if (token[0] == '"') printf("String token: '%s'\n", token);
-        // if (strcmp(token, ".db") == 0) {
-        //     free(token);
-        //     token = SplitOnBasicGrammar(NULL);
-        //     printf("String named '%s' declared. ", token);
-        //     free(token);
-        //     token = SplitOnBasicGrammar(NULL);
-        //     printf("Value: '%s'\n", token);
-        // }
-        if (TokenIsNumeric(string_token)) {
-            Token* token = malloc(sizeof(Token));
-            token->type = TK_Number;
-            token->value = string_token;
-            return token;
-        }
+        
+        if (TokenIsNumeric(string_token)) return CreateToken(TK_Number, string_token);
 
-        free(string_token);
-        string_token = SplitOnBasicGrammar(NULL);
+        return CreateToken(TK_Text, string_token);
     }
-    //free(currentWord);
+    
     free(string_token);
 
     return NULL;
 }
 
+Token* CreateToken(TokenType type, char* value) {
+    Token* token = calloc(1, sizeof(Token));
+
+    if (!token) return NULL;
+
+    token->type = type;
+    token->value = value;
+
+    return token;
+}
+
 int TokenIsNumeric(const char* token) {
     if (!token) return 0;