Fixed some bugs in the string tokenizer.

src/tokenizer.c

@@ -44,13 +44,14 @@ List* TokenizeLine(char *line) {
     if (line[0] == ';') return NULL;
 
     char* token = SplitOnWhiteSpace(line);
+    
     while (strlen(token) != 0) {
-        printf("Token: '%s'\n", token);
+        printf("'%s' ", token);
         //if (token[0] == '"') printf("String token: '%s'\n", token);
         free(token);
         token = SplitOnWhiteSpace(NULL);
-
     }
+    printf("\n");
     //free(currentWord);
     free(token);
 
@@ -70,16 +71,30 @@ char* SplitOnWhiteSpace(char* line) {
     int parsing_string = 0;
 
     for (int i = 0; position < strlen(string); i++, position++) {
+        
+        if (parsing_string) {
+            if (string[position] == '\n') break;
+
+            token[i] = string[position];
+
+            continue;
+        }
+
+        if (string[position] == ';') break;
+
         if (string[position] == '"') {
             parsing_string = 1;
         }
 
-        if (string[position] == ' ' && !parsing_string) {
-            position++;
-            break;
+        if (string[position] == ' ') {
+            while(string[position] == ' ') {
+                position++;
+            }
+
+            if (strlen(token) != 0) break;
         }
 
-        if (string[position] == ',' && !parsing_string) {
+        if (string[position] == ',') {
             if (strlen(token) == 0) {
                 token[0] = string[position];
                 position++;
@@ -87,11 +102,11 @@ char* SplitOnWhiteSpace(char* line) {
 
             break;
         }
-
+        
         if (string[position] != '\n') token[i] = string[position];
     }
 
-    token[strlen(token)] = '\0';
+    //token[strlen(token)] = '\0';
 
     return token;
 }