glm94/assm-test
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Cleaned up the tokenizer and setting the stage for more granular token types.

Browse Source
This commit is contained in:
glm94
2022-01-21 21:17:37 +00:00
parent 0636315166
commit 019a7f44a8
2 changed files with 89 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main.c
+67 -68
View File
@@ -2,7 +2,6 @@
#include <stdio.h> #include <stdio.h>
#include <string.h> #include <string.h>
#include "../includes/list.h" #include "../includes/list.h"
#include "../includes/hash_table.h"
#include "../includes/tokenizer.h" #include "../includes/tokenizer.h"
typedef struct { typedef struct {
@@ -56,83 +55,83 @@ int main(int argc, char* args[]) {
//free(list); //free(list);
} }
void print_file(char* file_path) { // void print_file(char* file_path) {
FILE *file; // FILE *file;
char* line = NULL; // char* line = NULL;
size_t len = 0; // size_t len = 0;
ssize_t bytes_read; // ssize_t bytes_read;
file = fopen(file_path, "r"); // file = fopen(file_path, "r");
if (!file) { // if (!file) {
printf("Failed to open '%s' for reading.\n", file_path); // printf("Failed to open '%s' for reading.\n", file_path);
return; // return;
}
while((bytes_read = getline(&line, &len, file)) != -1) {
List* list = get_strings(line);
if (!list) continue;
if (list->size == 0) continue;
// Instruction *inst = malloc(sizeof(Instruction));
// inst->parameters = malloc(sizeof(char*) * 2);
// if (strcmp(list->root[0], "mov") == 0) {
// inst->opcode = "mov";
// inst->count = 2;
// inst->parameters[0] = list->root[1];
// inst->parameters[1] = list->root[2];
// printf("Instruction '%s' with %d params ('%s', '%s')\n", inst->opcode, inst->count, inst->parameters[0], inst->parameters[1]);
// } // }
// free(inst->parameters); // while((bytes_read = getline(&line, &len, file)) != -1) {
// free(inst); // List* list = get_strings(line);
//PrintList(list); // if (!list) continue;
DestroyList(list); // if (list->size == 0) continue;
}
fclose(file); // // Instruction *inst = malloc(sizeof(Instruction));
// // inst->parameters = malloc(sizeof(char*) * 2);
if (line) free(line); // // if (strcmp(list->root[0], "mov") == 0) {
} // // inst->opcode = "mov";
// // inst->count = 2;
// // inst->parameters[0] = list->root[1];
// // inst->parameters[1] = list->root[2];
List* get_strings(const char* line) { // // printf("Instruction '%s' with %d params ('%s', '%s')\n", inst->opcode, inst->count, inst->parameters[0], inst->parameters[1]);
if (strlen(line) == 0 ) return NULL; // // }
if (line[0] == ';') return NULL; //If the line starts with a semi-colon its just a comment, so ignore it.
List *list = CreateList(); // // free(inst->parameters);
unsigned long lineLength = strlen(line); // // free(inst);
char* currentWord = malloc(lineLength + 1);
int index = 0;
for (int i = 0; i < lineLength; i++) { // //PrintList(list);
if (line[i] == ' ' || line[i] == ',') { // DestroyList(list);
if (index > 0) { // }
currentWord[index] = '\0';
PushListItem(currentWord, strlen(currentWord) + 1, list);
}
index = 0;
continue;
}
if (line[i] == ';' || i + 1 == lineLength) { // fclose(file);
if (index > 0) {
if (line[i] != ';') {
currentWord[index] = line[i];
index++;
}
currentWord[index] = '\0';
PushListItem(currentWord, strlen(currentWord) + 1, list);
}
break;
}
currentWord[index] = line[i]; // if (line) free(line);
index++; // }
}
return list; // List* get_strings(const char* line) {
} // if (strlen(line) == 0 ) return NULL;
// if (line[0] == ';') return NULL; //If the line starts with a semi-colon its just a comment, so ignore it.
// List *list = CreateList();
// unsigned long lineLength = strlen(line);
// char* currentWord = malloc(lineLength + 1);
// int index = 0;
// for (int i = 0; i < lineLength; i++) {
// if (line[i] == ' ' || line[i] == ',') {
// if (index > 0) {
// currentWord[index] = '\0';
// PushListItem(currentWord, strlen(currentWord) + 1, list);
// }
// index = 0;
// continue;
// }
// if (line[i] == ';' || i + 1 == lineLength) {
// if (index > 0) {
// if (line[i] != ';') {
// currentWord[index] = line[i];
// index++;
// }
// currentWord[index] = '\0';
// PushListItem(currentWord, strlen(currentWord) + 1, list);
// }
// break;
// }
// currentWord[index] = line[i];
// index++;
// }
// return list;
// }
src/tokenizer.c
+25 -38
View File
@@ -9,13 +9,13 @@ Token* GetNextToken(char *);
TokenType GetOperatorType(char); TokenType GetOperatorType(char);
char* SplitOnBasicGrammar(char*); char* SplitOnBasicGrammar(char*);
int TokenIsNumeric(const char*); int TokenIsNumeric(const char*);
Token* CreateToken(TokenType, char*);
List* TokenizeString(const char *file_path) { List* TokenizeString(const char *file_path) {
FILE *file; FILE *file;
char* line = NULL; char* line = NULL;
size_t len = 0; size_t len = 0;
ssize_t bytes_read; ssize_t bytes_read;
//List* tokens = CreateList();
file = fopen(file_path, "r"); file = fopen(file_path, "r");
@@ -27,15 +27,19 @@ List* TokenizeString(const char *file_path) {
while((bytes_read = getline(&line, &len, file)) != -1) { while((bytes_read = getline(&line, &len, file)) != -1) {
Token* token = GetNextToken(line); Token* token = GetNextToken(line);
// for(int i = 0; i < tokens->size; i++) {
// printf("Token type '%d' with value '%s'\n", ((Token* ) tokens->content[i])->type, ((Token*) tokens->content[i])->value);
// }
while (token) { while (token) {
if(token) { if (!token) break;
if(token->type == TK_Number) {
printf("Found number: '%s'\n", token->value); printf("Found number: '%s'\n", token->value);
}
else {
printf("Found text: '%s'\n", token->value);
}
free(token->value); free(token->value);
free(token); free(token);
}
token = GetNextToken(NULL); token = GetNextToken(NULL);
} }
} }
@@ -48,48 +52,31 @@ List* TokenizeString(const char *file_path) {
} }
Token* GetNextToken(char *string) { Token* GetNextToken(char *string) {
// static char* text;
// static int position;
// static unsigned long length;
// if (string) {
// length = strlen(string);
// if (length == 0) return NULL;
// text = string;
// position = 0;
// }
char* string_token = SplitOnBasicGrammar(string); char* string_token = SplitOnBasicGrammar(string);
//Token* token = malloc(sizeof(Token));
while (strlen(string_token) != 0) { while (strlen(string_token) != 0) {
//if (token[0] == '"') printf("String token: '%s'\n", token);
// if (strcmp(token, ".db") == 0) { if (TokenIsNumeric(string_token)) return CreateToken(TK_Number, string_token);
// free(token);
// token = SplitOnBasicGrammar(NULL); return CreateToken(TK_Text, string_token);
// printf("String named '%s' declared. ", token);
// free(token);
// token = SplitOnBasicGrammar(NULL);
// printf("Value: '%s'\n", token);
// }
if (TokenIsNumeric(string_token)) {
Token* token = malloc(sizeof(Token));
token->type = TK_Number;
token->value = string_token;
return token;
} }
free(string_token);
string_token = SplitOnBasicGrammar(NULL);
}
//free(currentWord);
free(string_token); free(string_token);
return NULL; return NULL;
} }
Token* CreateToken(TokenType type, char* value) {
Token* token = calloc(1, sizeof(Token));
if (!token) return NULL;
token->type = type;
token->value = value;
return token;
}
int TokenIsNumeric(const char* token) { int TokenIsNumeric(const char* token) {
if (!token) return 0; if (!token) return 0;