From 99ebd8299604a21e8cfc809e24f9c244d5b7656d Mon Sep 17 00:00:00 2001 From: Garritt McCune Date: Thu, 20 Jan 2022 18:17:58 +0000 Subject: [PATCH] Experimenting with tokenizing strings. This commit just focuses on splitting strings up by whitespace. --- includes/tokenizer.h | 33 ++++++++++ src/main.c | 48 ++++++++------- src/tokenizer.c | 144 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 203 insertions(+), 22 deletions(-) create mode 100644 includes/tokenizer.h create mode 100644 src/tokenizer.c diff --git a/includes/tokenizer.h b/includes/tokenizer.h new file mode 100644 index 0000000..02ff6a8 --- /dev/null +++ b/includes/tokenizer.h @@ -0,0 +1,33 @@ +#ifndef TOKENIZER_H +#define TOKENIZER_H + +#include +#include +#include +#include "list.h" + +typedef enum { + TK_Add, + TK_Sub, + TK_Mul, + TK_Div, + TK_Power, + TK_LParam, + TK_RParam, + TK_Comma, + TK_LBracket, + TK_RBracket, + TK_Colon, + TK_Text, + TK_Number, + TK_Invalid +} TokenType; + +typedef struct { + TokenType type; + char* value; +} Token; + +List* TokenizeString(const char *); + +#endif \ No newline at end of file diff --git a/src/main.c b/src/main.c index 4558389..0efffba 100644 --- a/src/main.c +++ b/src/main.c @@ -2,6 +2,8 @@ #include #include #include "../includes/list.h" +#include "../includes/hash_table.h" +#include "../includes/tokenizer.h" typedef struct { char *opcode; @@ -11,7 +13,6 @@ typedef struct { void print_file(char*); List* get_strings(const char*); -void PrintList(const List*); int main(int argc, char* args[]) { const char *input_files[argc - 1]; @@ -20,6 +21,9 @@ int main(int argc, char* args[]) { printf("Input files required.\n"); return -1; } + // char* test = "user_id"; + // printf("Key for '%s' is %d for size %d\n", test, GetKeyIndex(test, HASHTABLEDEFAULTSIZE), HASHTABLEDEFAULTSIZE); + TokenizeString(args[1]); // for (int i = 1; i < argc; i++) { // input_files[i - 1] = args[i]; @@ -27,29 +31,29 @@ int main(int argc, char* args[]) { // } //print_file(args[1]); - struct test { - int SomeValue; - char* SomeText; - }; - struct test* thing = malloc(sizeof(struct test)); - thing->SomeText = calloc(1, 16); - List *list = CreateList(); + // struct test { + // int SomeValue; + // char* SomeText; + // }; + // struct test* thing = malloc(sizeof(struct test)); + // thing->SomeText = calloc(1, 16); + // List *list = CreateList(); - for (int i = 0; i < 64; i++) { - sprintf(thing->SomeText, "%d", i); - thing->SomeValue = i; - AddListItem(thing, sizeof(struct test), list); + // for (int i = 0; i < 64; i++) { + // sprintf(thing->SomeText, "%d", i); + // thing->SomeValue = i; + // AddListItem(thing, sizeof(struct test), list); - if (i % 4 == 0) printf("Size: %d; Capacity: %d\n", list->size, list->capacity); + // if (i % 4 == 0) printf("Size: %d; Capacity: %d\n", list->size, list->capacity); - printf("Some Value: %d Some Text: '%s'\n", ((struct test*) list->content[list->size - 1])->SomeValue, ((struct test*) list->content[list->size - 1])->SomeText); - } + // printf("Some Value: %d Some Text: '%s'\n", ((struct test*) list->content[list->size - 1])->SomeValue, ((struct test*) list->content[list->size - 1])->SomeText); + // } - //PrintList(list); - free(thing->SomeText); - free(thing); + // //PrintList(list); + // free(thing->SomeText); + // free(thing); - free(list); + //free(list); } void print_file(char* file_path) { @@ -86,7 +90,7 @@ void print_file(char* file_path) { // free(inst->parameters); // free(inst); - PrintList(list); + //PrintList(list); DestroyList(list); } @@ -108,7 +112,7 @@ List* get_strings(const char* line) { if (line[i] == ' ' || line[i] == ',') { if (index > 0) { currentWord[index] = '\0'; - AddListItem(currentWord, strlen(currentWord) + 1, list); + PushListItem(currentWord, strlen(currentWord) + 1, list); } index = 0; continue; @@ -121,7 +125,7 @@ List* get_strings(const char* line) { index++; } currentWord[index] = '\0'; - AddListItem(currentWord, strlen(currentWord) + 1, list); + PushListItem(currentWord, strlen(currentWord) + 1, list); } break; } diff --git a/src/tokenizer.c b/src/tokenizer.c new file mode 100644 index 0000000..d49ac44 --- /dev/null +++ b/src/tokenizer.c @@ -0,0 +1,144 @@ +#include "../includes/tokenizer.h" +#include +#include +#include +#include +#include + +List* TokenizeLine(char *); +TokenType GetOperatorType(char); +char* SplitOnWhiteSpace(char*); +unsigned long position = 0; + +List* TokenizeString(const char *file_path) { + FILE *file; + char* line = NULL; + size_t len = 0; + ssize_t bytes_read; + List* tokens = CreateList(); + + file = fopen(file_path, "r"); + + if (!file) { + printf("Failed to open '%s' for reading.\n", file_path); + return NULL; + } + + while((bytes_read = getline(&line, &len, file)) != -1) { + List* tokens = TokenizeLine(line); + + // for(int i = 0; i < tokens->size; i++) { + // printf("Token type '%d' with value '%s'\n", ((Token* ) tokens->content[i])->type, ((Token*) tokens->content[i])->value); + // } + } + + fclose(file); + + if (line) free(line); + + return tokens; +} + +List* TokenizeLine(char *line) { + if (!line) return NULL; + if (strlen(line) == 0) return NULL; + if (line[0] == ';') return NULL; + + //List* tokens = CreateList(); + //Token* token = malloc(sizeof(Token)); + //unsigned long length = strlen(line) + 1; + //token->value = calloc(1, length); + //char* currentWord = malloc(length); + //char* token = strtok(line, " "); + + //while(token != NULL) { + // if (token[0] == '"') { + // strcpy(currentWord, token); + // token = strtok(NULL, " "); + // if (token == NULL) { + // printf("Expect double quote.\n"); + // break; + // } + // strcat(currentWord, token); + // printf("String token '%s' found\n", currentWord); + // } + + // token = strtok(NULL, " "); + //token + //} + char* token = SplitOnWhiteSpace(line); + while (strlen(token) != 0) { + printf("Token: '%s'\n", token); + token = SplitOnWhiteSpace(line); + } + position = 0; + //free(currentWord); + free(token); + //int index = 0; + + // for (int i = 0; i < length; i++) { + // if (line[i] == ' ') { + // if (index == 0) continue; + // printf("Found a spcae\n"); + // currentWord[index + 1] = '\0'; + + // token->type = TK_Text; + // strcpy(token->value, currentWord); + // PushListItem(token, sizeof(Token), tokens); + // } + + // currentWord[index] = line[i]; + // index++; + // } + + return NULL; +} + +char* SplitOnWhiteSpace(char* line) { + char* token = calloc(1, strlen(line) + 1); + int parsing_string = 0; + + for (int i = 0; position < strlen(line); i++, position++) { + if (line[position] == '"') { + parsing_string = 1; + } + + if (line[position] == ' ' && !parsing_string) { + position++; + break; + } + + if (line[position] != '\n') token[i] = line[position]; + } + + token[strlen(token)] = '\0'; + + return token; +} + +TokenType GetOperatorType(char c) { + switch (c){ + case '+': + return TK_Add; + case '-': + return TK_Sub; + case '*': + return TK_Mul; + case '/': + return TK_Div; + case '^': + return TK_Power; + case ':': + return TK_Colon; + case '(': + return TK_LParam; + case ')': + return TK_RParam; + case '[': + return TK_LBracket; + case ']': + return TK_RBracket; + }; + + return TK_Invalid; +} \ No newline at end of file