#include "../includes/tokenizer.h" #include #include #include #include #include Token* GetNextToken(char *); TokenType GetOperatorType(char); char* SplitOnBasicGrammar(char*); int TokenIsNumeric(const char*); Token* CreateToken(TokenType, char*); List* TokenizeString(const char *file_path) { FILE *file; char* line = NULL; size_t len = 0; ssize_t bytes_read; file = fopen(file_path, "r"); if (!file) { printf("Failed to open '%s' for reading.\n", file_path); return NULL; } while((bytes_read = getline(&line, &len, file)) != -1) { Token* token = GetNextToken(line); while (token) { if (!token) break; if(token->type == TK_Number) { printf("Found number: '%s'\n", token->value); } else { printf("Found text: '%s'\n", token->value); } free(token->value); free(token); token = GetNextToken(NULL); } } fclose(file); if (line) free(line); return NULL; } Token* GetNextToken(char *string) { char* string_token = SplitOnBasicGrammar(string); while (strlen(string_token) != 0) { if (TokenIsNumeric(string_token)) return CreateToken(TK_Number, string_token); return CreateToken(TK_Text, string_token); } free(string_token); return NULL; } Token* CreateToken(TokenType type, char* value) { Token* token = calloc(1, sizeof(Token)); if (!token) return NULL; token->type = type; token->value = value; return token; } int TokenIsNumeric(const char* token) { if (!token) return 0; unsigned long length = strlen(token); if (length == 0) return 0; for(int i = 0; i < length; i++) { if (!isdigit(token[i])) return 0; } return 1; } char* SplitOnBasicGrammar(char* line) { static char* string; static unsigned long position; if (line) { string = line; position = 0; } char* token = calloc(1, strlen(string) + 1); int parsing_string = 0; for (int i = 0; position < strlen(string); i++, position++) { if (parsing_string) { if (string[position] == '\n') break; if (string[position] == '"') break; token[i] = string[position]; continue; } if (string[position] == ';') break; if (string[position] == '"') { parsing_string = 1; i--; continue; } if (string[position] == ' ') { while(string[position] == ' ') { position++; } if (strlen(token) != 0) break; } if (string[position] == ',') { if (strlen(token) == 0) { token[0] = string[position]; position++; } break; } if (string[position] != '\n') token[i] = string[position]; } return token; } TokenType GetOperatorType(char c) { switch (c){ case '+': return TK_Add; case '-': return TK_Sub; case '*': return TK_Mul; case '/': return TK_Div; case '^': return TK_Power; case ':': return TK_Colon; case '(': return TK_LParam; case ')': return TK_RParam; case '[': return TK_LBracket; case ']': return TK_RBracket; }; return TK_Invalid; }