From b32e1bad7c31c6583255b5081528c708d9ffa88c Mon Sep 17 00:00:00 2001
From: Garritt McCune <glmccune@gmail.com>
Date: Thu, 20 Jan 2022 21:21:18 +0000
Subject: [PATCH] Fixed some bugs in the string tokenizer.

---
 src/tokenizer.c | 31 +++++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/src/tokenizer.c b/src/tokenizer.c
index bb7bdf3..5bd2210 100644
--- a/src/tokenizer.c
+++ b/src/tokenizer.c
@@ -44,13 +44,14 @@ List* TokenizeLine(char *line) {
     if (line[0] == ';') return NULL;
 
     char* token = SplitOnWhiteSpace(line);
+    
     while (strlen(token) != 0) {
-        printf("Token: '%s'\n", token);
+        printf("'%s' ", token);
         //if (token[0] == '"') printf("String token: '%s'\n", token);
         free(token);
         token = SplitOnWhiteSpace(NULL);
-
     }
+    printf("\n");
     //free(currentWord);
     free(token);
 
@@ -70,16 +71,30 @@ char* SplitOnWhiteSpace(char* line) {
     int parsing_string = 0;
 
     for (int i = 0; position < strlen(string); i++, position++) {
+        
+        if (parsing_string) {
+            if (string[position] == '\n') break;
+
+            token[i] = string[position];
+
+            continue;
+        }
+
+        if (string[position] == ';') break;
+
         if (string[position] == '"') {
             parsing_string = 1;
         }
 
-        if (string[position] == ' ' && !parsing_string) {
-            position++;
-            break;
+        if (string[position] == ' ') {
+            while(string[position] == ' ') {
+                position++;
+            }
+
+            if (strlen(token) != 0) break;
         }
 
-        if (string[position] == ',' && !parsing_string) {
+        if (string[position] == ',') {
             if (strlen(token) == 0) {
                 token[0] = string[position];
                 position++;
@@ -87,11 +102,11 @@ char* SplitOnWhiteSpace(char* line) {
 
             break;
         }
-
+        
         if (string[position] != '\n') token[i] = string[position];
     }
 
-    token[strlen(token)] = '\0';
+    //token[strlen(token)] = '\0';
 
     return token;
 }