Experimenting with tokenizing strings. This commit just focuses on splitting strings up by whitespace.

src/tokenizer.c

@@ -0,0 +1,144 @@
+#include "../includes/tokenizer.h"
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <string.h>
+
+List* TokenizeLine(char *);
+TokenType GetOperatorType(char);
+char* SplitOnWhiteSpace(char*);
+unsigned long position = 0;
+
+List* TokenizeString(const char *file_path) {
+    FILE *file;
+    char* line = NULL;
+    size_t len = 0;
+    ssize_t bytes_read;
+    List* tokens = CreateList();
+
+    file = fopen(file_path, "r");
+
+    if (!file) {
+        printf("Failed to open '%s' for reading.\n", file_path);
+        return NULL;
+    }
+
+    while((bytes_read = getline(&line, &len, file)) != -1) {
+        List* tokens = TokenizeLine(line);
+
+        // for(int i = 0; i < tokens->size; i++) {
+        //     printf("Token type '%d' with value '%s'\n", ((Token* ) tokens->content[i])->type, ((Token*) tokens->content[i])->value);
+        // }
+    }
+
+    fclose(file);
+
+    if (line) free(line);
+
+    return tokens;
+}
+
+List* TokenizeLine(char *line) {
+    if (!line) return NULL;
+    if (strlen(line) == 0) return NULL;
+    if (line[0] == ';') return NULL;
+
+    //List* tokens = CreateList();
+    //Token* token = malloc(sizeof(Token));
+    //unsigned long length = strlen(line) + 1;
+    //token->value = calloc(1, length);
+    //char* currentWord = malloc(length);
+    //char* token = strtok(line, " ");
+
+    //while(token != NULL) {
+        // if (token[0] == '"') {
+        //     strcpy(currentWord, token);
+        //     token = strtok(NULL, " ");
+        //     if (token == NULL) {
+        //         printf("Expect double quote.\n");
+        //         break;
+        //     }
+        //     strcat(currentWord, token);
+        //     printf("String token '%s' found\n", currentWord);
+        // }
+
+        // token = strtok(NULL, " ");
+        //token
+    //}
+    char* token = SplitOnWhiteSpace(line);
+    while (strlen(token) != 0) {
+        printf("Token: '%s'\n", token);
+        token = SplitOnWhiteSpace(line);
+    }
+    position = 0;
+    //free(currentWord);
+    free(token);
+    //int index = 0;
+
+    // for (int i = 0; i < length; i++) {
+    //     if (line[i] == ' ') {
+    //         if (index == 0) continue;
+    //         printf("Found a spcae\n");
+    //         currentWord[index + 1] = '\0';
+
+    //         token->type = TK_Text;
+    //         strcpy(token->value, currentWord);
+    //         PushListItem(token, sizeof(Token), tokens);
+    //     }
+
+    //     currentWord[index] = line[i];
+    //     index++;
+    // }
+
+    return NULL;
+}
+
+char* SplitOnWhiteSpace(char* line) {
+    char* token = calloc(1, strlen(line) + 1);
+    int parsing_string = 0;
+
+    for (int i = 0; position < strlen(line); i++, position++) {
+        if (line[position] == '"') {
+            parsing_string = 1;
+        }
+
+        if (line[position] == ' ' && !parsing_string) {
+            position++;
+            break;
+        }
+
+        if (line[position] != '\n') token[i] = line[position];
+    }
+
+    token[strlen(token)] = '\0';
+
+    return token;
+}
+
+TokenType GetOperatorType(char c) {
+    switch (c){
+        case '+':
+        return TK_Add;
+        case '-':
+        return TK_Sub;
+        case '*':
+        return TK_Mul;
+        case '/':
+        return TK_Div;
+        case '^':
+        return TK_Power;
+        case ':':
+        return TK_Colon;
+        case '(':
+        return TK_LParam;
+        case ')':
+        return TK_RParam;
+        case '[':
+        return TK_LBracket;
+        case ']':
+        return TK_RBracket;
+    };
+
+    return TK_Invalid;
+}