From ebc1d30c1c957e55ed3c30e25448ef154736f90c Mon Sep 17 00:00:00 2001
From: Garritt McCune <glmccune@gmail.com>
Date: Mon, 31 Jan 2022 22:06:52 +0000
Subject: [PATCH] Added a peek function to the tokenizer.

---
 includes/tokenizer.h |  3 ++-
 src/lexer.c          | 11 ++++++++-
 src/tokenizer.c      | 54 +++++++++++++++++++++++++++++---------------
 3 files changed, 48 insertions(+), 20 deletions(-)

diff --git a/includes/tokenizer.h b/includes/tokenizer.h
index 30ce8e9..a074cc6 100644
--- a/includes/tokenizer.h
+++ b/includes/tokenizer.h
@@ -6,6 +6,7 @@
 #include <string.h>
 #include "list.h"
 
-char* SplitOnBasicGrammar(char*);
+char* GetToken(char*);
+char* PeekNextToken(void);
 
 #endif
\ No newline at end of file
diff --git a/src/lexer.c b/src/lexer.c
index 530d74f..6724d21 100644
--- a/src/lexer.c
+++ b/src/lexer.c
@@ -66,7 +66,7 @@ List* GenerateTokensFromFile(const char* file_path) {
 }
 
 Token* GetNextToken(char *string) {
-    char* string_token = SplitOnBasicGrammar(string);
+    char* string_token = GetToken(string);
     int base = 0;
     
     while (strlen(string_token) != 0) {
@@ -90,6 +90,15 @@ Token* GetNextToken(char *string) {
 
         // if (next) free(next);
 
+        if (strcmp(".db", string_token) == 0) {
+            char* next = PeekNextToken();
+            if (next) {
+                printf("String variable name: '%s'\n", next);
+                free(next);
+            }
+        }
+
+
         return CreateToken(TK_String, string_token);
     }
 
diff --git a/src/tokenizer.c b/src/tokenizer.c
index ebf02c8..c9902fd 100644
--- a/src/tokenizer.c
+++ b/src/tokenizer.c
@@ -2,58 +2,76 @@
 #include <stdio.h>
 #include <string.h>
 
+char* SplitOnBasicGrammar(char*, unsigned long *);
 void GetStringLiteral(char*, unsigned long, char**, unsigned long*);
+static char* string;
+static unsigned long position;
 
-char* SplitOnBasicGrammar(char* line) {
-    static char* string;
-    static unsigned long position;
-
+char* GetToken(char* line) {
     if (line) {
         string = line;
         position = 0;
     }
 
+    return SplitOnBasicGrammar(string, &position);
+}
+
+char* PeekNextToken() {
+    if (!string) return NULL;
+    if (strlen(string) == 0) return NULL;
+
+    unsigned long pos = position;
+
+    return SplitOnBasicGrammar(string, &pos);
+}
+
+char* SplitOnBasicGrammar(char* line, unsigned long *string_index) {
+    // if (line) {
+    //     string = line;
+    //     position = 0;
+    // }
+
     unsigned long length = strlen(string);
     char* token = calloc(1, length + 1);
     //int parsing_string = 0;
 
-    for (int i = 0; position < length; i++, position++) {
+    for (int i = 0; *string_index < length; i++, (*string_index)++) {
         
-        if (string[position] == ';') break;
+        if (string[*string_index] == ';') break;
 
-        if (string[position] == ':') {
+        if (string[*string_index] == ':') {
             if (strlen(token) == 0) {
                 token[0] = ':';
-                position++;
+                (*string_index)++;
             }
 
             break;
         }
 
-        if (string[position] == '"') {
-            position++;
-            GetStringLiteral(string, length, &token, &position);
+        if (string[*string_index] == '"') {
+            (*string_index)++;
+            GetStringLiteral(string, length, &token, string_index);
             break;
         }
 
-        if (string[position] == ' ') {
-            while(string[position] == ' ') {
-                position++;
+        if (string[*string_index] == ' ') {
+            while(string[*string_index] == ' ') {
+                (*string_index)++;
             }
 
             if (strlen(token) != 0) break;
         }
 
-        if (string[position] == ',') {
+        if (string[*string_index] == ',') {
             if (strlen(token) == 0) {
-                token[0] = string[position];
-                position++;
+                token[0] = string[*string_index];
+                (*string_index)++;
             }
 
             break;
         }
         
-        if (string[position] != '\n') token[i] = string[position];
+        if (string[*string_index] != '\n') token[i] = string[*string_index];
     }
 
     return token;