diff --git a/src/tokenizer.c b/src/tokenizer.c index d49ac44..bb7bdf3 100644 --- a/src/tokenizer.c +++ b/src/tokenizer.c @@ -8,7 +8,6 @@ List* TokenizeLine(char *); TokenType GetOperatorType(char); char* SplitOnWhiteSpace(char*); -unsigned long position = 0; List* TokenizeString(const char *file_path) { FILE *file; @@ -44,71 +43,52 @@ List* TokenizeLine(char *line) { if (strlen(line) == 0) return NULL; if (line[0] == ';') return NULL; - //List* tokens = CreateList(); - //Token* token = malloc(sizeof(Token)); - //unsigned long length = strlen(line) + 1; - //token->value = calloc(1, length); - //char* currentWord = malloc(length); - //char* token = strtok(line, " "); - - //while(token != NULL) { - // if (token[0] == '"') { - // strcpy(currentWord, token); - // token = strtok(NULL, " "); - // if (token == NULL) { - // printf("Expect double quote.\n"); - // break; - // } - // strcat(currentWord, token); - // printf("String token '%s' found\n", currentWord); - // } - - // token = strtok(NULL, " "); - //token - //} char* token = SplitOnWhiteSpace(line); while (strlen(token) != 0) { printf("Token: '%s'\n", token); - token = SplitOnWhiteSpace(line); + //if (token[0] == '"') printf("String token: '%s'\n", token); + free(token); + token = SplitOnWhiteSpace(NULL); + } - position = 0; //free(currentWord); free(token); - //int index = 0; - - // for (int i = 0; i < length; i++) { - // if (line[i] == ' ') { - // if (index == 0) continue; - // printf("Found a spcae\n"); - // currentWord[index + 1] = '\0'; - - // token->type = TK_Text; - // strcpy(token->value, currentWord); - // PushListItem(token, sizeof(Token), tokens); - // } - - // currentWord[index] = line[i]; - // index++; - // } return NULL; } char* SplitOnWhiteSpace(char* line) { - char* token = calloc(1, strlen(line) + 1); + static char* string; + static unsigned long position; + + if (line) { + string = line; + position = 0; + } + + char* token = calloc(1, strlen(string) + 1); int parsing_string = 0; - for (int i = 0; position < strlen(line); i++, position++) { - if (line[position] == '"') { + for (int i = 0; position < strlen(string); i++, position++) { + if (string[position] == '"') { parsing_string = 1; } - if (line[position] == ' ' && !parsing_string) { + if (string[position] == ' ' && !parsing_string) { position++; break; } - if (line[position] != '\n') token[i] = line[position]; + if (string[position] == ',' && !parsing_string) { + if (strlen(token) == 0) { + token[0] = string[position]; + position++; + } + + break; + } + + if (string[position] != '\n') token[i] = string[position]; } token[strlen(token)] = '\0';