Added a peek function to the tokenizer.

includes/tokenizer.h

@@ -6,6 +6,7 @@
 #include <string.h>
 #include "list.h"
 
-char* SplitOnBasicGrammar(char*);
+char* GetToken(char*);
+char* PeekNextToken(void);
 
 #endif

src/lexer.c

@@ -66,7 +66,7 @@ List* GenerateTokensFromFile(const char* file_path) {
 }
 
 Token* GetNextToken(char *string) {
-    char* string_token = SplitOnBasicGrammar(string);
+    char* string_token = GetToken(string);
     int base = 0;
     
     while (strlen(string_token) != 0) {
@@ -90,6 +90,15 @@ Token* GetNextToken(char *string) {
 
         // if (next) free(next);
 
+        if (strcmp(".db", string_token) == 0) {
+            char* next = PeekNextToken();
+            if (next) {
+                printf("String variable name: '%s'\n", next);
+                free(next);
+            }
+        }
+
+
         return CreateToken(TK_String, string_token);
     }
 

src/tokenizer.c

@@ -2,58 +2,76 @@
 #include <stdio.h>
 #include <string.h>
 
+char* SplitOnBasicGrammar(char*, unsigned long *);
 void GetStringLiteral(char*, unsigned long, char**, unsigned long*);
+static char* string;
+static unsigned long position;
 
-char* SplitOnBasicGrammar(char* line) {
+char* GetToken(char* line) {
-    static char* string;
-    static unsigned long position;
-
     if (line) {
         string = line;
         position = 0;
     }
 
+    return SplitOnBasicGrammar(string, &position);
+}
+
+char* PeekNextToken() {
+    if (!string) return NULL;
+    if (strlen(string) == 0) return NULL;
+
+    unsigned long pos = position;
+
+    return SplitOnBasicGrammar(string, &pos);
+}
+
+char* SplitOnBasicGrammar(char* line, unsigned long *string_index) {
+    // if (line) {
+    //     string = line;
+    //     position = 0;
+    // }
+
     unsigned long length = strlen(string);
     char* token = calloc(1, length + 1);
     //int parsing_string = 0;
 
-    for (int i = 0; position < length; i++, position++) {
+    for (int i = 0; *string_index < length; i++, (*string_index)++) {
         
-        if (string[position] == ';') break;
+        if (string[*string_index] == ';') break;
 
-        if (string[position] == ':') {
+        if (string[*string_index] == ':') {
             if (strlen(token) == 0) {
                 token[0] = ':';
-                position++;
+                (*string_index)++;
             }
 
             break;
         }
 
-        if (string[position] == '"') {
+        if (string[*string_index] == '"') {
-            position++;
+            (*string_index)++;
-            GetStringLiteral(string, length, &token, &position);
+            GetStringLiteral(string, length, &token, string_index);
             break;
         }
 
-        if (string[position] == ' ') {
+        if (string[*string_index] == ' ') {
-            while(string[position] == ' ') {
+            while(string[*string_index] == ' ') {
-                position++;
+                (*string_index)++;
             }
 
             if (strlen(token) != 0) break;
         }
 
-        if (string[position] == ',') {
+        if (string[*string_index] == ',') {
             if (strlen(token) == 0) {
-                token[0] = string[position];
+                token[0] = string[*string_index];
-                position++;
+                (*string_index)++;
             }
 
             break;
         }
         
-        if (string[position] != '\n') token[i] = string[position];
+        if (string[*string_index] != '\n') token[i] = string[*string_index];
     }
 
     return token;