diff --git a/src/tokenizer.c b/src/tokenizer.c index 480af03..3b3ad71 100644 --- a/src/tokenizer.c +++ b/src/tokenizer.c @@ -5,16 +5,17 @@ #include #include -List* TokenizeLine(char *); +Token* GetNextToken(char *); TokenType GetOperatorType(char); -char* SplitOnWhiteSpace(char*); +char* SplitOnBasicGrammar(char*); +int TokenIsNumeric(const char*); List* TokenizeString(const char *file_path) { FILE *file; char* line = NULL; size_t len = 0; ssize_t bytes_read; - List* tokens = CreateList(); + //List* tokens = CreateList(); file = fopen(file_path, "r"); @@ -24,49 +25,86 @@ List* TokenizeString(const char *file_path) { } while((bytes_read = getline(&line, &len, file)) != -1) { - List* tokens = TokenizeLine(line); + Token* token = GetNextToken(line); // for(int i = 0; i < tokens->size; i++) { // printf("Token type '%d' with value '%s'\n", ((Token* ) tokens->content[i])->type, ((Token*) tokens->content[i])->value); // } + while (token) { + if(token) { + printf("Found number: '%s'\n", token->value); + free(token->value); + free(token); + } + token = GetNextToken(NULL); + } } fclose(file); if (line) free(line); - return tokens; + return NULL; } -List* TokenizeLine(char *line) { - if (!line) return NULL; - if (strlen(line) == 0) return NULL; - if (line[0] == ';') return NULL; +Token* GetNextToken(char *string) { + // static char* text; + // static int position; + // static unsigned long length; - char* token = SplitOnWhiteSpace(line); + // if (string) { + // length = strlen(string); + + // if (length == 0) return NULL; + + // text = string; + // position = 0; + // } + + char* string_token = SplitOnBasicGrammar(string); + //Token* token = malloc(sizeof(Token)); - while (strlen(token) != 0) { - //printf("'%s' ", token); + while (strlen(string_token) != 0) { //if (token[0] == '"') printf("String token: '%s'\n", token); - if (strcmp(token, ".db") == 0) { - free(token); - token = SplitOnWhiteSpace(NULL); - printf("String named '%s' declared. ", token); - free(token); - token = SplitOnWhiteSpace(NULL); - printf("Value: '%s'\n", token); + // if (strcmp(token, ".db") == 0) { + // free(token); + // token = SplitOnBasicGrammar(NULL); + // printf("String named '%s' declared. ", token); + // free(token); + // token = SplitOnBasicGrammar(NULL); + // printf("Value: '%s'\n", token); + // } + if (TokenIsNumeric(string_token)) { + Token* token = malloc(sizeof(Token)); + token->type = TK_Number; + token->value = string_token; + return token; } - free(token); - token = SplitOnWhiteSpace(NULL); + free(string_token); + string_token = SplitOnBasicGrammar(NULL); } //free(currentWord); - free(token); + free(string_token); return NULL; } -char* SplitOnWhiteSpace(char* line) { +int TokenIsNumeric(const char* token) { + if (!token) return 0; + + unsigned long length = strlen(token); + + if (length == 0) return 0; + + for(int i = 0; i < length; i++) { + if (!isdigit(token[i])) return 0; + } + + return 1; +} + +char* SplitOnBasicGrammar(char* line) { static char* string; static unsigned long position; @@ -117,8 +155,6 @@ char* SplitOnWhiteSpace(char* line) { if (string[position] != '\n') token[i] = string[position]; } - //token[strlen(token)] = '\0'; - return token; }