173 lines
3.5 KiB
C
173 lines
3.5 KiB
C
#include "../includes/tokenizer.h"
|
|
#include <stddef.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <ctype.h>
|
|
#include <string.h>
|
|
|
|
Token* GetNextToken(char *);
|
|
TokenType GetOperatorType(char);
|
|
char* SplitOnBasicGrammar(char*);
|
|
int TokenIsNumeric(const char*);
|
|
Token* CreateToken(TokenType, char*);
|
|
|
|
List* TokenizeString(const char *file_path) {
|
|
FILE *file;
|
|
char* line = NULL;
|
|
size_t len = 0;
|
|
ssize_t bytes_read;
|
|
|
|
file = fopen(file_path, "r");
|
|
|
|
if (!file) {
|
|
printf("Failed to open '%s' for reading.\n", file_path);
|
|
return NULL;
|
|
}
|
|
|
|
while((bytes_read = getline(&line, &len, file)) != -1) {
|
|
Token* token = GetNextToken(line);
|
|
|
|
while (token) {
|
|
if (!token) break;
|
|
|
|
if(token->type == TK_Number) {
|
|
printf("Found number: '%s'\n", token->value);
|
|
}
|
|
else {
|
|
printf("Found text: '%s'\n", token->value);
|
|
}
|
|
|
|
free(token->value);
|
|
free(token);
|
|
|
|
token = GetNextToken(NULL);
|
|
}
|
|
}
|
|
|
|
fclose(file);
|
|
|
|
if (line) free(line);
|
|
|
|
return NULL;
|
|
}
|
|
|
|
Token* GetNextToken(char *string) {
|
|
char* string_token = SplitOnBasicGrammar(string);
|
|
|
|
while (strlen(string_token) != 0) {
|
|
|
|
if (TokenIsNumeric(string_token)) return CreateToken(TK_Number, string_token);
|
|
|
|
return CreateToken(TK_Text, string_token);
|
|
}
|
|
|
|
free(string_token);
|
|
|
|
return NULL;
|
|
}
|
|
|
|
Token* CreateToken(TokenType type, char* value) {
|
|
Token* token = calloc(1, sizeof(Token));
|
|
|
|
if (!token) return NULL;
|
|
|
|
token->type = type;
|
|
token->value = value;
|
|
|
|
return token;
|
|
}
|
|
|
|
int TokenIsNumeric(const char* token) {
|
|
if (!token) return 0;
|
|
|
|
unsigned long length = strlen(token);
|
|
|
|
if (length == 0) return 0;
|
|
|
|
for(int i = 0; i < length; i++) {
|
|
if (!isdigit(token[i])) return 0;
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
|
|
char* SplitOnBasicGrammar(char* line) {
|
|
static char* string;
|
|
static unsigned long position;
|
|
|
|
if (line) {
|
|
string = line;
|
|
position = 0;
|
|
}
|
|
|
|
char* token = calloc(1, strlen(string) + 1);
|
|
int parsing_string = 0;
|
|
|
|
for (int i = 0; position < strlen(string); i++, position++) {
|
|
|
|
if (parsing_string) {
|
|
if (string[position] == '\n') break;
|
|
if (string[position] == '"') break;
|
|
|
|
token[i] = string[position];
|
|
|
|
continue;
|
|
}
|
|
|
|
if (string[position] == ';') break;
|
|
|
|
if (string[position] == '"') {
|
|
parsing_string = 1;
|
|
i--;
|
|
continue;
|
|
}
|
|
|
|
if (string[position] == ' ') {
|
|
while(string[position] == ' ') {
|
|
position++;
|
|
}
|
|
|
|
if (strlen(token) != 0) break;
|
|
}
|
|
|
|
if (string[position] == ',') {
|
|
if (strlen(token) == 0) {
|
|
token[0] = string[position];
|
|
position++;
|
|
}
|
|
|
|
break;
|
|
}
|
|
|
|
if (string[position] != '\n') token[i] = string[position];
|
|
}
|
|
|
|
return token;
|
|
}
|
|
|
|
TokenType GetOperatorType(char c) {
|
|
switch (c){
|
|
case '+':
|
|
return TK_Add;
|
|
case '-':
|
|
return TK_Sub;
|
|
case '*':
|
|
return TK_Mul;
|
|
case '/':
|
|
return TK_Div;
|
|
case '^':
|
|
return TK_Power;
|
|
case ':':
|
|
return TK_Colon;
|
|
case '(':
|
|
return TK_LParam;
|
|
case ')':
|
|
return TK_RParam;
|
|
case '[':
|
|
return TK_LBracket;
|
|
case ']':
|
|
return TK_RBracket;
|
|
};
|
|
|
|
return TK_Invalid;
|
|
} |