Experimenting with tokenizing strings. This commit just focuses on splitting strings up by whitespace.

includes/tokenizer.h

@@ -0,0 +1,33 @@
+#ifndef TOKENIZER_H
+#define TOKENIZER_H
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include "list.h"
+
+typedef enum {
+    TK_Add,
+    TK_Sub,
+    TK_Mul,
+    TK_Div,
+    TK_Power,
+    TK_LParam,
+    TK_RParam,
+    TK_Comma,
+    TK_LBracket,
+    TK_RBracket,
+    TK_Colon,
+    TK_Text,
+    TK_Number,
+    TK_Invalid
+} TokenType;
+
+typedef struct {
+    TokenType type;
+    char* value;
+} Token;
+
+List* TokenizeString(const char *);
+
+#endif

src/main.c

@@ -2,6 +2,8 @@
 #include <stdio.h>
 #include <string.h>
 #include "../includes/list.h"
+#include "../includes/hash_table.h"
+#include "../includes/tokenizer.h"
 
 typedef struct {
     char *opcode;
@@ -11,7 +13,6 @@ typedef struct {
 
 void print_file(char*);
 List* get_strings(const char*);
-void PrintList(const List*);
 
 int main(int argc, char* args[]) {
     const char *input_files[argc - 1];
@@ -20,6 +21,9 @@ int main(int argc, char* args[]) {
         printf("Input files required.\n");
         return -1;
     }
+    // char* test = "user_id";
+    // printf("Key for '%s' is %d for size %d\n", test, GetKeyIndex(test, HASHTABLEDEFAULTSIZE), HASHTABLEDEFAULTSIZE);
+    TokenizeString(args[1]);
 
     // for (int i = 1; i < argc; i++) {
     //     input_files[i - 1] = args[i];
@@ -27,29 +31,29 @@ int main(int argc, char* args[]) {
     // }
 
     //print_file(args[1]);
-    struct test {
+    // struct test {
-        int SomeValue;
+    //     int SomeValue;
-        char* SomeText;
+    //     char* SomeText;
-    };
+    // };
-    struct test* thing = malloc(sizeof(struct test));
+    // struct test* thing = malloc(sizeof(struct test));
-    thing->SomeText = calloc(1, 16);
+    // thing->SomeText = calloc(1, 16);
-    List *list = CreateList();
+    // List *list = CreateList();
 
-    for (int i = 0; i < 64; i++) {
+    // for (int i = 0; i < 64; i++) {
-        sprintf(thing->SomeText, "%d", i);
+    //     sprintf(thing->SomeText, "%d", i);
-        thing->SomeValue = i;
+    //     thing->SomeValue = i;
-        AddListItem(thing, sizeof(struct test), list);
+    //     AddListItem(thing, sizeof(struct test), list);
 
-        if (i % 4 == 0) printf("Size: %d; Capacity: %d\n", list->size, list->capacity);
+    //     if (i % 4 == 0) printf("Size: %d; Capacity: %d\n", list->size, list->capacity);
 
-        printf("Some Value: %d Some Text: '%s'\n", ((struct test*) list->content[list->size - 1])->SomeValue, ((struct test*) list->content[list->size - 1])->SomeText);
+    //     printf("Some Value: %d Some Text: '%s'\n", ((struct test*) list->content[list->size - 1])->SomeValue, ((struct test*) list->content[list->size - 1])->SomeText);
-    }
+    // }
 
-    //PrintList(list);
+    // //PrintList(list);
-    free(thing->SomeText);
+    // free(thing->SomeText);
-    free(thing);
+    // free(thing);
     
-    free(list);
+    //free(list);
 }
 
 void print_file(char* file_path) {
@@ -86,7 +90,7 @@ void print_file(char* file_path) {
         // free(inst->parameters);
         // free(inst);
 
-        PrintList(list);
+        //PrintList(list);
         DestroyList(list);
     }
 
@@ -108,7 +112,7 @@ List* get_strings(const char* line) {
         if (line[i] == ' ' || line[i] == ',') {
             if (index > 0) {
                 currentWord[index] = '\0';
-                AddListItem(currentWord, strlen(currentWord) + 1, list);
+                PushListItem(currentWord, strlen(currentWord) + 1, list);
             }
             index = 0;
             continue;
@@ -121,7 +125,7 @@ List* get_strings(const char* line) {
                     index++;
                 }
                 currentWord[index] = '\0';
-                AddListItem(currentWord, strlen(currentWord) + 1, list);
+                PushListItem(currentWord, strlen(currentWord) + 1, list);
             }
             break;
         }

src/tokenizer.c

@@ -0,0 +1,144 @@
+#include "../includes/tokenizer.h"
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <string.h>
+
+List* TokenizeLine(char *);
+TokenType GetOperatorType(char);
+char* SplitOnWhiteSpace(char*);
+unsigned long position = 0;
+
+List* TokenizeString(const char *file_path) {
+    FILE *file;
+    char* line = NULL;
+    size_t len = 0;
+    ssize_t bytes_read;
+    List* tokens = CreateList();
+
+    file = fopen(file_path, "r");
+
+    if (!file) {
+        printf("Failed to open '%s' for reading.\n", file_path);
+        return NULL;
+    }
+
+    while((bytes_read = getline(&line, &len, file)) != -1) {
+        List* tokens = TokenizeLine(line);
+
+        // for(int i = 0; i < tokens->size; i++) {
+        //     printf("Token type '%d' with value '%s'\n", ((Token* ) tokens->content[i])->type, ((Token*) tokens->content[i])->value);
+        // }
+    }
+
+    fclose(file);
+
+    if (line) free(line);
+
+    return tokens;
+}
+
+List* TokenizeLine(char *line) {
+    if (!line) return NULL;
+    if (strlen(line) == 0) return NULL;
+    if (line[0] == ';') return NULL;
+
+    //List* tokens = CreateList();
+    //Token* token = malloc(sizeof(Token));
+    //unsigned long length = strlen(line) + 1;
+    //token->value = calloc(1, length);
+    //char* currentWord = malloc(length);
+    //char* token = strtok(line, " ");
+
+    //while(token != NULL) {
+        // if (token[0] == '"') {
+        //     strcpy(currentWord, token);
+        //     token = strtok(NULL, " ");
+        //     if (token == NULL) {
+        //         printf("Expect double quote.\n");
+        //         break;
+        //     }
+        //     strcat(currentWord, token);
+        //     printf("String token '%s' found\n", currentWord);
+        // }
+
+        // token = strtok(NULL, " ");
+        //token
+    //}
+    char* token = SplitOnWhiteSpace(line);
+    while (strlen(token) != 0) {
+        printf("Token: '%s'\n", token);
+        token = SplitOnWhiteSpace(line);
+    }
+    position = 0;
+    //free(currentWord);
+    free(token);
+    //int index = 0;
+
+    // for (int i = 0; i < length; i++) {
+    //     if (line[i] == ' ') {
+    //         if (index == 0) continue;
+    //         printf("Found a spcae\n");
+    //         currentWord[index + 1] = '\0';
+
+    //         token->type = TK_Text;
+    //         strcpy(token->value, currentWord);
+    //         PushListItem(token, sizeof(Token), tokens);
+    //     }
+
+    //     currentWord[index] = line[i];
+    //     index++;
+    // }
+
+    return NULL;
+}
+
+char* SplitOnWhiteSpace(char* line) {
+    char* token = calloc(1, strlen(line) + 1);
+    int parsing_string = 0;
+
+    for (int i = 0; position < strlen(line); i++, position++) {
+        if (line[position] == '"') {
+            parsing_string = 1;
+        }
+
+        if (line[position] == ' ' && !parsing_string) {
+            position++;
+            break;
+        }
+
+        if (line[position] != '\n') token[i] = line[position];
+    }
+
+    token[strlen(token)] = '\0';
+
+    return token;
+}
+
+TokenType GetOperatorType(char c) {
+    switch (c){
+        case '+':
+        return TK_Add;
+        case '-':
+        return TK_Sub;
+        case '*':
+        return TK_Mul;
+        case '/':
+        return TK_Div;
+        case '^':
+        return TK_Power;
+        case ':':
+        return TK_Colon;
+        case '(':
+        return TK_LParam;
+        case ')':
+        return TK_RParam;
+        case '[':
+        return TK_LBracket;
+        case ']':
+        return TK_RBracket;
+    };
+
+    return TK_Invalid;
+}