glm94/assm-test
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Cleaned up the tokenizer and setting the stage for more granular token types.

Browse Source
This commit is contained in:
glm94
2022-01-21 21:17:37 +00:00
parent 0636315166
commit 019a7f44a8
2 changed files with 89 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main.c
+63 -64
View File
@@ -2,7 +2,6 @@
#include <stdio.h>
#include <string.h>
#include "../includes/list.h"
#include "../includes/hash_table.h"
#include "../includes/tokenizer.h"
typedef struct {
@@ -56,83 +55,83 @@ int main(int argc, char* args[]) {
//free(list);
}
void print_file(char* file_path) {
FILE *file;
char* line = NULL;
size_t len = 0;
ssize_t bytes_read;
// void print_file(char* file_path) {
// FILE *file;
// char* line = NULL;
// size_t len = 0;
// ssize_t bytes_read;
file = fopen(file_path, "r");
// file = fopen(file_path, "r");
if (!file) {
printf("Failed to open '%s' for reading.\n", file_path);
return;
}
// if (!file) {
// printf("Failed to open '%s' for reading.\n", file_path);
// return;
// }
while((bytes_read = getline(&line, &len, file)) != -1) {
List* list = get_strings(line);
// while((bytes_read = getline(&line, &len, file)) != -1) {
// List* list = get_strings(line);
if (!list) continue;
if (list->size == 0) continue;
// if (!list) continue;
// if (list->size == 0) continue;
// Instruction *inst = malloc(sizeof(Instruction));
// inst->parameters = malloc(sizeof(char*) * 2);
// // Instruction *inst = malloc(sizeof(Instruction));
// // inst->parameters = malloc(sizeof(char*) * 2);
// if (strcmp(list->root[0], "mov") == 0) {
// inst->opcode = "mov";
// inst->count = 2;
// inst->parameters[0] = list->root[1];
// inst->parameters[1] = list->root[2];
// // if (strcmp(list->root[0], "mov") == 0) {
// // inst->opcode = "mov";
// // inst->count = 2;
// // inst->parameters[0] = list->root[1];
// // inst->parameters[1] = list->root[2];
// printf("Instruction '%s' with %d params ('%s', '%s')\n", inst->opcode, inst->count, inst->parameters[0], inst->parameters[1]);
// }
// // printf("Instruction '%s' with %d params ('%s', '%s')\n", inst->opcode, inst->count, inst->parameters[0], inst->parameters[1]);
// // }
// free(inst->parameters);
// free(inst);
// // free(inst->parameters);
// // free(inst);
//PrintList(list);
DestroyList(list);
}
// //PrintList(list);
// DestroyList(list);
// }
fclose(file);
// fclose(file);
if (line) free(line);
}
// if (line) free(line);
// }
List* get_strings(const char* line) {
if (strlen(line) == 0 ) return NULL;
if (line[0] == ';') return NULL; //If the line starts with a semi-colon its just a comment, so ignore it.
// List* get_strings(const char* line) {
// if (strlen(line) == 0 ) return NULL;
// if (line[0] == ';') return NULL; //If the line starts with a semi-colon its just a comment, so ignore it.
List *list = CreateList();
unsigned long lineLength = strlen(line);
char* currentWord = malloc(lineLength + 1);
int index = 0;
// List *list = CreateList();
// unsigned long lineLength = strlen(line);
// char* currentWord = malloc(lineLength + 1);
// int index = 0;
for (int i = 0; i < lineLength; i++) {
if (line[i] == ' ' || line[i] == ',') {
if (index > 0) {
currentWord[index] = '\0';
PushListItem(currentWord, strlen(currentWord) + 1, list);
}
index = 0;
continue;
}
// for (int i = 0; i < lineLength; i++) {
// if (line[i] == ' ' || line[i] == ',') {
// if (index > 0) {
// currentWord[index] = '\0';
// PushListItem(currentWord, strlen(currentWord) + 1, list);
// }
// index = 0;
// continue;
// }
if (line[i] == ';' || i + 1 == lineLength) {
if (index > 0) {
if (line[i] != ';') {
currentWord[index] = line[i];
index++;
}
currentWord[index] = '\0';
PushListItem(currentWord, strlen(currentWord) + 1, list);
}
break;
}
// if (line[i] == ';' || i + 1 == lineLength) {
// if (index > 0) {
// if (line[i] != ';') {
// currentWord[index] = line[i];
// index++;
// }
// currentWord[index] = '\0';
// PushListItem(currentWord, strlen(currentWord) + 1, list);
// }
// break;
// }
currentWord[index] = line[i];
index++;
}
// currentWord[index] = line[i];
// index++;
// }
return list;
}
// return list;
// }
src/tokenizer.c
+26 -39
View File
@@ -9,13 +9,13 @@ Token* GetNextToken(char *);
TokenType GetOperatorType(char);
char* SplitOnBasicGrammar(char*);
int TokenIsNumeric(const char*);
Token* CreateToken(TokenType, char*);
List* TokenizeString(const char *file_path) {
FILE *file;
char* line = NULL;
size_t len = 0;
ssize_t bytes_read;
//List* tokens = CreateList();
file = fopen(file_path, "r");
@@ -27,15 +27,19 @@ List* TokenizeString(const char *file_path) {
while((bytes_read = getline(&line, &len, file)) != -1) {
Token* token = GetNextToken(line);
// for(int i = 0; i < tokens->size; i++) {
// printf("Token type '%d' with value '%s'\n", ((Token* ) tokens->content[i])->type, ((Token*) tokens->content[i])->value);
// }
while (token) {
if(token) {
if (!token) break;
if(token->type == TK_Number) {
printf("Found number: '%s'\n", token->value);
free(token->value);
free(token);
}
else {
printf("Found text: '%s'\n", token->value);
}
free(token->value);
free(token);
token = GetNextToken(NULL);
}
}
@@ -48,48 +52,31 @@ List* TokenizeString(const char *file_path) {
}
Token* GetNextToken(char *string) {
// static char* text;
// static int position;
// static unsigned long length;
// if (string) {
// length = strlen(string);
// if (length == 0) return NULL;
// text = string;
// position = 0;
// }
char* string_token = SplitOnBasicGrammar(string);
//Token* token = malloc(sizeof(Token));
while (strlen(string_token) != 0) {
//if (token[0] == '"') printf("String token: '%s'\n", token);
// if (strcmp(token, ".db") == 0) {
// free(token);
// token = SplitOnBasicGrammar(NULL);
// printf("String named '%s' declared. ", token);
// free(token);
// token = SplitOnBasicGrammar(NULL);
// printf("Value: '%s'\n", token);
// }
if (TokenIsNumeric(string_token)) {
Token* token = malloc(sizeof(Token));
token->type = TK_Number;
token->value = string_token;
return token;
}
if (TokenIsNumeric(string_token)) return CreateToken(TK_Number, string_token);
free(string_token);
string_token = SplitOnBasicGrammar(NULL);
return CreateToken(TK_Text, string_token);
}
//free(currentWord);
free(string_token);
return NULL;
}
Token* CreateToken(TokenType type, char* value) {
Token* token = calloc(1, sizeof(Token));
if (!token) return NULL;
token->type = type;
token->value = value;
return token;
}
int TokenIsNumeric(const char* token) {
if (!token) return 0;