diff --git a/includes/lexer.h b/includes/lexer.h deleted file mode 100644 index 81f7af2..0000000 --- a/includes/lexer.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef LEXER_H -#define LEXER_H - -#include -#include -#include -#include -#include "list.h" -#include "tokenizer.h" -#include "token.h" - -List* GetTokensFromLine(char*); - -#endif \ No newline at end of file diff --git a/includes/opcodes.h b/includes/opcodes.h index 2efde03..a2e04c6 100644 --- a/includes/opcodes.h +++ b/includes/opcodes.h @@ -2,7 +2,7 @@ #define OPCODES_H #include -#include "lexer.h" +#include "token.h" #define OPCODECOUNT 10 #define REGISTERCOUNT 8 diff --git a/includes/scanner.h b/includes/scanner.h index 086b16e..5b59df1 100644 --- a/includes/scanner.h +++ b/includes/scanner.h @@ -4,6 +4,7 @@ #include #include #include +#include #include "stdlib.h" #include "token.h" #include "list.h" diff --git a/includes/token.h b/includes/token.h index e01efa6..ae3313c 100644 --- a/includes/token.h +++ b/includes/token.h @@ -2,6 +2,9 @@ #define TOKEN_H #include "stdlib.h" +#include +#include +#include typedef enum { PLUS, @@ -30,10 +33,12 @@ typedef enum { typedef struct { TokenType type; - char* value; + char* lexeme; + void* value; + int line; } Token; -Token* CreateToken(TokenType type, char* value); +Token* CreateToken(char*, void*, int, TokenType); void FreeToken(Token*); #endif \ No newline at end of file diff --git a/includes/tokenizer.h b/includes/tokenizer.h deleted file mode 100644 index a853fac..0000000 --- a/includes/tokenizer.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef TOKENIZER_H -#define TOKENIZER_H - -#include -#include -#include -#include "list.h" - -List* GetTokens(const char *); -char* GetToken(char*); -char* PeekNextToken(void); - -#endif \ No newline at end of file diff --git a/src/lexer.c b/src/lexer.c deleted file mode 100644 index dca4b04..0000000 --- a/src/lexer.c +++ /dev/null @@ -1,104 +0,0 @@ -#include "../includes/lexer.h" -#include "../includes/opcodes.h" -#include -#include -#include - -Token* GetNextToken(char *); -int TokenIsNumeric(const char*, int *); - -List* GetTokensFromLine(char* line) { - List* tokens = CreateList(); - - Token* token = GetNextToken(line); - - while (token) { - - if (!token) break; - - AddListItem(token, sizeof(Token), tokens); - - token = GetNextToken(NULL); - } - - return tokens; -} - -Token* GetNextToken(char *string) { - char* string_token = GetToken(string); - int base = 0; - TokenType type; - - while (strlen(string_token) != 0) { - - if (strcmp(string_token, ".db") == 0) { - return CreateToken(DB, string_token); - } - - if (TokenIsNumeric(string_token, &base)) { - if (base == 10) return CreateToken(NUMBER, string_token); - if (base == 16) return CreateToken(HEX, string_token); - } - - char* next = PeekNextToken(); - - if (next) { - if (strcmp(":", next) == 0) { - free(next); - free(GetToken(NULL)); - return CreateToken(IDENTIFIER, string_token); - } - - free(next); - } - - if (IsOpcode(string_token, &type)) { - return CreateToken(type, string_token); - } - - if (IsRegister(string_token, &type)) { - return CreateToken(type, string_token); - } - - // if (op) return CreateToken(op->op, op->lexeme); - // if (reg) return CreateToken(reg->type, reg->lexeme); - - if (strcmp(",", string_token) == 0) return CreateToken(COMMA, string_token); - - return CreateToken(IDENTIFIER, string_token); - } - - free(string_token); - - return NULL; -} - -int TokenIsNumeric(const char* token, int *base) { - *base = 0; - - if (!token) return 0; - - unsigned long length = strlen(token); - int i = 0; - - if (length == 0) return 0; - - *base = 10; - - if (length > 2) { - if (token[0] == '0' && token[1] == 'x') { - *base = 16; - i = 2; - } - } - - for(; i < length; i++) { - if (!isdigit(token[i])) { - if (*base == 16 && token[i] >= 'A' && token[i] <= 'F') continue; - - return 0; - } - } - - return 1; -} \ No newline at end of file diff --git a/src/main.c b/src/main.c index 7114b94..e3aa244 100644 --- a/src/main.c +++ b/src/main.c @@ -3,7 +3,6 @@ #include #include #include "../includes/list.h" -#include "../includes/lexer.h" #include "../includes/parser.h" #include "../includes/futil.h" #include "../includes/scanner.h" @@ -26,7 +25,9 @@ int main(int argc, char* args[]) { for(int i = 0; i < list->size; i++) { Token* t = (Token*) list->content[i]; - printf("[%i] '%s'\n", t->type, t->value); + printf("[%i] '%s'", t->type, t->lexeme); + if (t->type == NUMBER || t->type == HEX) printf(" NUM: %ld", (long) t->value); + printf("\n"); } free(source_code); diff --git a/src/parser.c b/src/parser.c index a320468..47d7d27 100644 --- a/src/parser.c +++ b/src/parser.c @@ -1,10 +1,10 @@ #include "../includes/parser.h" -#include "../includes/lexer.h" +#include "../includes/token.h" #include #include -static List* SymbolsTable; -static unsigned int program_counter = 0; +//static List* SymbolsTable; +//static unsigned int program_counter = 0; void ProcessDirective(List*); void ProcessVariableDeclaration(List*); @@ -51,5 +51,5 @@ void ProcessVariableDeclaration(List* tokens) { Token* string_literal = (Token*) tokens->content[2]; printf("Found string literal.\n"); - printf("Name '%s' value: '%s'\n", symbol_token->value, string_literal->value); + printf("Name '%s' value: '%s'\n", symbol_token->lexeme, string_literal->lexeme); } \ No newline at end of file diff --git a/src/scanner.c b/src/scanner.c index f842619..1b92524 100644 --- a/src/scanner.c +++ b/src/scanner.c @@ -1,8 +1,4 @@ #include "../includes/scanner.h" -#include -#include -#include -#include const char* SourceCode; int Line = 0; @@ -96,18 +92,21 @@ Token* ParseNumber(void) { if (length == 0) return NULL; - char* lexeme = calloc(sizeof(char), length); + char* lexeme = calloc(sizeof(char), length + 1); if (!lexeme) { - fprintf(stderr, "Failed to calloc space for number.\n"); + fprintf(stderr, "Failed to calloc space for number. %s.\n", strerror(errno)); return NULL; } memcpy(lexeme, &SourceCode[start], length); + //Setting the base to zero means the function will detect the base. + //https://pubs.opengroup.org/onlinepubs/7908799/xsh/strtol.html + printf("Number Parsed: %s (%ld)\n", lexeme, strtol(lexeme, NULL, 0)); - if (base == 10) return CreateToken(NUMBER, lexeme); + if (base == 10) return CreateToken(lexeme, lexeme, Line, NUMBER); - return CreateToken(HEX, lexeme); + return CreateToken(lexeme, lexeme, Line, HEX); } Token* ParseDirective(void) { @@ -119,17 +118,17 @@ Token* ParseDirective(void) { if (length == 0) return NULL; - char* directive = calloc(sizeof(char), length); + char* directive = calloc(sizeof(char), length + 1); if (!directive) { - fprintf(stderr, "Failed to calloc for the assmebler directive.\n"); + fprintf(stderr, "Failed to calloc for the assmebler directive. %s.\n", strerror(errno)); return NULL; } memcpy(directive, &SourceCode[start], length); if (strcmp(directive, ".db") == 0) { - return CreateToken(DB, directive); + return CreateToken(directive, directive, Line, DB); } else if (strcmp(directive, ".org") == 0) { fprintf(stderr, "[Warning] Org is not a supported directive.\n"); @@ -164,12 +163,18 @@ Token* ParseString(void) { if (length == 0) return NULL; - char* lexeme = calloc(sizeof(char), length); + char* lexeme = calloc(sizeof(char), length + 1); + + if (!lexeme) { + fprintf(stderr, "Failed to calloc memory for string. %s.\n", strerror(errno)); + return NULL; + } + memcpy(lexeme, &SourceCode[start], length); AdvanceScanner(); //Consume the trailing double quote. - Token* token = CreateToken(STRING, lexeme); + Token* token = CreateToken(lexeme, lexeme, Line, STRING); return token; } @@ -185,15 +190,21 @@ Token* ParseIdentifier(void) { if (length == 0) return NULL; - TokenType type; - char* lexeme = calloc(sizeof(char), length); + TokenType type; + char* lexeme = calloc(sizeof(char), length + 1); + + if (!lexeme) { + fprintf(stderr, "Failed to calloc memory for identifier. %s.\n", strerror(errno)); + return NULL; + } + memcpy(lexeme, &SourceCode[start], length); - if (IsOpcode(lexeme, &type)) return CreateToken(type, lexeme); - if (IsRegister(lexeme, &type)) return CreateToken(type, lexeme); - if (lexeme[length - 1] == ':') return CreateToken(LABEL, lexeme); + if (IsOpcode(lexeme, &type)) return CreateToken(lexeme, lexeme, Line, type); + if (IsRegister(lexeme, &type)) return CreateToken(lexeme, lexeme, Line, type); + if (lexeme[length - 1] == ':') return CreateToken(lexeme, lexeme, Line, LABEL); - return CreateToken(IDENTIFIER, lexeme); + return CreateToken(lexeme, lexeme, Line, IDENTIFIER); } char PeekScanner(void) { @@ -234,15 +245,15 @@ int IsPunctuation(char c) { Token* ParsePunctuation(char c) { switch(c) { case '[': - return CreateToken(LBRACKET, "["); + return CreateToken("[", NULL, Line, LBRACKET); case ']': - return CreateToken(RBracket, "]"); + return CreateToken("]", NULL, Line, RBracket); case '(': - return CreateToken(LPARAM, "("); + return CreateToken("(", NULL, Line, LPARAM); case ')': - return CreateToken(RPARAM, ")"); + return CreateToken(")", NULL, Line, RPARAM); case ',': - return CreateToken(COMMA, ","); + return CreateToken(",", NULL, Line, COMMA); default: return NULL; } diff --git a/src/token.c b/src/token.c index 97d05ff..382ed8a 100644 --- a/src/token.c +++ b/src/token.c @@ -1,11 +1,16 @@ #include "../includes/token.h" -Token* CreateToken(TokenType type, char* value) { +Token* CreateToken(char* lexeme, void* value, int lineNumber, TokenType type) { Token* token = calloc(1, sizeof(Token)); - if (!token) return NULL; + if (!token) { + fprintf(stderr, "Failed to calloc memory for Token. %s.\n", strerror(errno)); + return NULL; + } token->type = type; + token->line = lineNumber; + token->lexeme = lexeme; token->value = value; return token; @@ -14,7 +19,7 @@ Token* CreateToken(TokenType type, char* value) { void FreeToken(Token* token) { if (!token) return; - if (token->value) free(token->value); + if (token->value && token->type >= STRING) free(token->value); free(token); } \ No newline at end of file diff --git a/src/tokenizer.c b/src/tokenizer.c deleted file mode 100644 index e542a28..0000000 --- a/src/tokenizer.c +++ /dev/null @@ -1,116 +0,0 @@ -#include "../includes/tokenizer.h" -#include -#include - -char* SplitOnBasicGrammar(char*, unsigned long *); -void GetStringLiteral(char*, unsigned long, char**, unsigned long*); -static char* string; -static unsigned long current_position; - -char* GetToken(char* line) { - if (line) { - string = line; - current_position = 0; - } - - return SplitOnBasicGrammar(string, ¤t_position); -} - -char* PeekNextToken() { - if (!string) return NULL; - if (strlen(string) == 0) return NULL; - - unsigned long pos = current_position; - - return SplitOnBasicGrammar(string, &pos); -} - -//TODO: this function should probably return NULL when its run out of tokens -//instead of sending back a valid pointer with memory that must be free()ed by the caller. -char* SplitOnBasicGrammar(char* line, unsigned long *string_index) { - - unsigned long length = strlen(string); - char* token = calloc(1, length + 1); - - for (int i = 0; *string_index < length; i++, (*string_index)++) { - - if (string[*string_index] == ';') break; - - if (string[*string_index] == ':') { - if (strlen(token) == 0) { - token[0] = ':'; - (*string_index)++; - } - - break; - } - - if (string[*string_index] == '"') { - (*string_index)++; - GetStringLiteral(string, length, &token, string_index); - break; - } - - if (string[*string_index] == ' ') { - while(string[*string_index] == ' ') { - (*string_index)++; - } - - if (strlen(token) != 0) break; - } - - if (string[*string_index] == ',') { - if (strlen(token) == 0) { - token[0] = string[*string_index]; - (*string_index)++; - } - - break; - } - - if (string[*string_index] != '\n') token[i] = string[*string_index]; - } - - return token; -} - -void GetStringLiteral(char* text, unsigned long length, char** token, unsigned long* position) { - if (!text) return; - if (!token) return; - - for(int i = 0; *position < length; (*position)++, i++) { - if (text[*position] == '"') { - (*position)++; - break; - } - - (*token)[i] = text[*position]; - } -} - -// TokenType GetOperatorType(char c) { -// switch (c){ -// case '+': -// return TK_Add; -// case '-': -// return TK_Sub; -// case '*': -// return TK_Mul; -// case '/': -// return TK_Div; -// case '^': -// return TK_Power; -// case ':': -// return TK_Colon; -// case '(': -// return TK_LParam; -// case ')': -// return TK_RParam; -// case '[': -// return TK_LBracket; -// case ']': -// return TK_RBracket; -// }; - -// return TK_Invalid; -// } \ No newline at end of file