Starting to get a basic structure for tokenizing. Currently TK_Number tokens are being generated successfully.

src/tokenizer.c

@@ -5,16 +5,17 @@
 #include <ctype.h>
 #include <string.h>
 
-List* TokenizeLine(char *);
+Token* GetNextToken(char *);
 TokenType GetOperatorType(char);
-char* SplitOnWhiteSpace(char*);
+char* SplitOnBasicGrammar(char*);
+int TokenIsNumeric(const char*);
 
 List* TokenizeString(const char *file_path) {
     FILE *file;
     char* line = NULL;
     size_t len = 0;
     ssize_t bytes_read;
-    List* tokens = CreateList();
+    //List* tokens = CreateList();
 
     file = fopen(file_path, "r");
 
@@ -24,49 +25,86 @@ List* TokenizeString(const char *file_path) {
     }
 
     while((bytes_read = getline(&line, &len, file)) != -1) {
-        List* tokens = TokenizeLine(line);
+        Token* token = GetNextToken(line);
 
         // for(int i = 0; i < tokens->size; i++) {
         //     printf("Token type '%d' with value '%s'\n", ((Token* ) tokens->content[i])->type, ((Token*) tokens->content[i])->value);
         // }
+        while (token) {
+            if(token)  {
+                printf("Found number: '%s'\n", token->value);
+                free(token->value);
+                free(token);
+            }
+            token = GetNextToken(NULL);
+        }
     }
 
     fclose(file);
 
     if (line) free(line);
 
-    return tokens;
+    return NULL;
 }
 
-List* TokenizeLine(char *line) {
-    if (!line) return NULL;
-    if (strlen(line) == 0) return NULL;
-    if (line[0] == ';') return NULL;
+Token* GetNextToken(char *string) {
+    // static char* text;
+    // static int position;
+    // static unsigned long length;
 
-    char* token = SplitOnWhiteSpace(line);
+    // if (string) {
+    //     length = strlen(string);
+
+    //     if (length == 0) return NULL;
+
+    //     text = string;
+    //     position = 0;
+    // }
+
+    char* string_token = SplitOnBasicGrammar(string);
+    //Token* token = malloc(sizeof(Token));
     
-    while (strlen(token) != 0) {
-        //printf("'%s' ", token);
+    while (strlen(string_token) != 0) {
         //if (token[0] == '"') printf("String token: '%s'\n", token);
-        if (strcmp(token, ".db") == 0) {
-            free(token);
-            token = SplitOnWhiteSpace(NULL);
-            printf("String named '%s' declared. ", token);
-            free(token);
-            token = SplitOnWhiteSpace(NULL);
-            printf("Value: '%s'\n", token);
+        // if (strcmp(token, ".db") == 0) {
+        //     free(token);
+        //     token = SplitOnBasicGrammar(NULL);
+        //     printf("String named '%s' declared. ", token);
+        //     free(token);
+        //     token = SplitOnBasicGrammar(NULL);
+        //     printf("Value: '%s'\n", token);
+        // }
+        if (TokenIsNumeric(string_token)) {
+            Token* token = malloc(sizeof(Token));
+            token->type = TK_Number;
+            token->value = string_token;
+            return token;
         }
 
-        free(token);
-        token = SplitOnWhiteSpace(NULL);
+        free(string_token);
+        string_token = SplitOnBasicGrammar(NULL);
     }
     //free(currentWord);
-    free(token);
+    free(string_token);
 
     return NULL;
 }
 
-char* SplitOnWhiteSpace(char* line) {
+int TokenIsNumeric(const char* token) {
+    if (!token) return 0;
+    
+    unsigned long length = strlen(token);
+
+    if (length == 0) return 0;
+
+    for(int i = 0; i < length; i++) {
+        if (!isdigit(token[i])) return 0;
+    }
+
+    return 1;
+}
+
+char* SplitOnBasicGrammar(char* line) {
     static char* string;
     static unsigned long position;
 
@@ -117,8 +155,6 @@ char* SplitOnWhiteSpace(char* line) {
         if (string[position] != '\n') token[i] = string[position];
     }
 
-    //token[strlen(token)] = '\0';
-
     return token;
 }