Code cleanup. Converting a number lexeme to a proper integer setup but not done yet. So number tokens don't have a valid value.

includes/lexer.h

@@ -1,14 +0,0 @@
-#ifndef LEXER_H
-#define LEXER_H
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <ctype.h>
-#include "list.h"
-#include "tokenizer.h"
-#include "token.h"
-
-List* GetTokensFromLine(char*);
-
-#endif

includes/opcodes.h

@@ -2,7 +2,7 @@
 #define OPCODES_H
 
 #include <string.h>
-#include "lexer.h"
+#include "token.h"
 
 #define OPCODECOUNT 10
 #define REGISTERCOUNT 8

includes/scanner.h

@@ -4,6 +4,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <ctype.h>
+#include <errno.h>
 #include "stdlib.h"
 #include "token.h"
 #include "list.h"

includes/token.h

@@ -2,6 +2,9 @@
 #define TOKEN_H
 
 #include "stdlib.h"
+#include <string.h>
+#include <errno.h>
+#include <stdio.h>
 
 typedef enum {
     PLUS,
@@ -30,10 +33,12 @@ typedef enum {
 
 typedef struct {
     TokenType type;
-    char* value;
+    char* lexeme;
+    void* value;
+    int line;
 } Token;
 
-Token* CreateToken(TokenType type, char* value);
+Token* CreateToken(char*, void*, int, TokenType);
 void FreeToken(Token*);
 
 #endif

includes/tokenizer.h

@@ -1,13 +0,0 @@
-#ifndef TOKENIZER_H
-#define TOKENIZER_H
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include "list.h"
-
-List* GetTokens(const char *);
-char* GetToken(char*);
-char* PeekNextToken(void);
-
-#endif

src/lexer.c

@@ -1,104 +0,0 @@
-#include "../includes/lexer.h"
-#include "../includes/opcodes.h"
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-Token* GetNextToken(char *);
-int TokenIsNumeric(const char*, int *);
-
-List* GetTokensFromLine(char* line) {
-    List* tokens = CreateList();
-    
-    Token* token = GetNextToken(line);
-
-    while (token) {
-
-        if (!token) break;
-        
-        AddListItem(token, sizeof(Token), tokens);
-
-        token = GetNextToken(NULL);
-    }
-
-    return tokens;
-}
-
-Token* GetNextToken(char *string) {
-    char* string_token = GetToken(string);
-    int base = 0;
-    TokenType type;
-    
-    while (strlen(string_token) != 0) {
-
-        if (strcmp(string_token, ".db") == 0) {
-            return CreateToken(DB, string_token);
-        }
-        
-        if (TokenIsNumeric(string_token, &base)) {
-            if (base == 10) return CreateToken(NUMBER, string_token);
-            if (base == 16) return CreateToken(HEX, string_token);
-        }
-
-        char* next = PeekNextToken();
-
-        if (next) {
-            if (strcmp(":", next) == 0) {
-                free(next);
-                free(GetToken(NULL));
-                return CreateToken(IDENTIFIER, string_token);
-            }
-
-            free(next);
-        }
-
-        if (IsOpcode(string_token, &type)) {
-            return CreateToken(type, string_token);
-        }
-
-        if (IsRegister(string_token, &type)) {
-            return CreateToken(type, string_token);
-        }
-
-        // if (op) return CreateToken(op->op, op->lexeme);
-        // if (reg) return CreateToken(reg->type, reg->lexeme);
-
-        if (strcmp(",", string_token) == 0) return CreateToken(COMMA, string_token);
-
-        return CreateToken(IDENTIFIER, string_token);
-    }
-
-    free(string_token);
-
-    return NULL;
-}
-
-int TokenIsNumeric(const char* token, int *base) {
-    *base = 0;
-
-    if (!token) return 0;
-    
-    unsigned long length = strlen(token);
-    int i = 0;
-
-    if (length == 0) return 0;
-
-    *base = 10;
-
-    if (length > 2) {
-        if (token[0] == '0' && token[1] == 'x') { 
-            *base = 16;
-            i = 2;
-        }
-    }
-
-    for(; i < length; i++) {
-        if (!isdigit(token[i])) { 
-            if (*base == 16 && token[i] >= 'A' && token[i] <= 'F') continue;
-
-            return 0;
-        }
-    }
-
-    return 1;
-}

src/main.c

@@ -3,7 +3,6 @@
 #include <string.h>
 #include <sysexits.h>
 #include "../includes/list.h"
-#include "../includes/lexer.h"
 #include "../includes/parser.h"
 #include "../includes/futil.h"
 #include "../includes/scanner.h"
@@ -26,7 +25,9 @@ int main(int argc, char* args[]) {
     for(int i = 0; i < list->size; i++) {
         Token* t = (Token*) list->content[i];
 
-        printf("[%i] '%s'\n", t->type, t->value);
+        printf("[%i] '%s'", t->type, t->lexeme);
+        if (t->type == NUMBER || t->type == HEX) printf(" NUM: %ld", (long) t->value);
+        printf("\n");
     }
 
     free(source_code);

src/parser.c

@@ -1,10 +1,10 @@
 #include "../includes/parser.h"
-#include "../includes/lexer.h"
+#include "../includes/token.h"
 #include <stdio.h>
 #include <stdlib.h>
 
-static List* SymbolsTable;
+//static List* SymbolsTable;
-static unsigned int program_counter = 0;
+//static unsigned int program_counter = 0;
 void ProcessDirective(List*);
 void ProcessVariableDeclaration(List*);
 
@@ -51,5 +51,5 @@ void ProcessVariableDeclaration(List* tokens) {
     Token* string_literal = (Token*) tokens->content[2];
 
     printf("Found string literal.\n");
-    printf("Name '%s' value: '%s'\n", symbol_token->value, string_literal->value);
+    printf("Name '%s' value: '%s'\n", symbol_token->lexeme, string_literal->lexeme);
 }

src/scanner.c

@@ -1,8 +1,4 @@
 #include "../includes/scanner.h"
-#include <ctype.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
 
 const char* SourceCode;
 int Line = 0;
@@ -96,18 +92,21 @@ Token* ParseNumber(void) {
 
     if (length == 0) return NULL;
 
-    char* lexeme = calloc(sizeof(char), length);
+    char* lexeme = calloc(sizeof(char), length + 1);
 
     if (!lexeme) {
-        fprintf(stderr, "Failed to calloc space for number.\n");
+        fprintf(stderr, "Failed to calloc space for number. %s.\n", strerror(errno));
         return NULL;
     }
 
     memcpy(lexeme, &SourceCode[start], length);
+    //Setting the base to zero means the function will detect the base.
+    //https://pubs.opengroup.org/onlinepubs/7908799/xsh/strtol.html
+    printf("Number Parsed: %s (%ld)\n", lexeme, strtol(lexeme, NULL, 0));
 
-    if (base == 10) return CreateToken(NUMBER, lexeme);
+    if (base == 10) return CreateToken(lexeme, lexeme, Line, NUMBER);
 
-    return CreateToken(HEX, lexeme);
+    return CreateToken(lexeme, lexeme, Line, HEX);
 }
 
 Token* ParseDirective(void) {
@@ -119,17 +118,17 @@ Token* ParseDirective(void) {
 
     if (length == 0) return NULL;
 
-    char* directive = calloc(sizeof(char), length);
+    char* directive = calloc(sizeof(char), length + 1);
 
     if (!directive) {
-        fprintf(stderr, "Failed to calloc for the assmebler directive.\n");
+        fprintf(stderr, "Failed to calloc for the assmebler directive. %s.\n", strerror(errno));
         return NULL;
     }
 
     memcpy(directive, &SourceCode[start], length);
 
     if (strcmp(directive, ".db") == 0) {
-        return CreateToken(DB, directive);
+        return CreateToken(directive, directive, Line, DB);
     }
     else if (strcmp(directive, ".org") == 0) {
         fprintf(stderr, "[Warning] Org is not a supported directive.\n");
@@ -164,12 +163,18 @@ Token* ParseString(void) {
 
     if (length == 0) return NULL;
 
-    char* lexeme = calloc(sizeof(char), length);
+    char* lexeme = calloc(sizeof(char), length + 1);
+
+    if (!lexeme) {
+        fprintf(stderr, "Failed to calloc memory for string. %s.\n", strerror(errno));
+        return NULL;
+    }
+
     memcpy(lexeme, &SourceCode[start], length);
 
     AdvanceScanner(); //Consume the trailing double quote.
 
-    Token* token = CreateToken(STRING, lexeme);
+    Token* token = CreateToken(lexeme, lexeme, Line, STRING);
 
     return token;
 }
@@ -185,15 +190,21 @@ Token* ParseIdentifier(void) {
 
     if (length == 0) return NULL;
     
-    TokenType type;    
+    TokenType type;
-    char* lexeme = calloc(sizeof(char), length);
+    char* lexeme = calloc(sizeof(char), length + 1);
+
+    if (!lexeme) {
+        fprintf(stderr, "Failed to calloc memory for identifier. %s.\n", strerror(errno));
+        return NULL;
+    }
+
     memcpy(lexeme, &SourceCode[start], length);
 
-    if (IsOpcode(lexeme, &type)) return CreateToken(type, lexeme);
+    if (IsOpcode(lexeme, &type)) return CreateToken(lexeme, lexeme, Line, type);
-    if (IsRegister(lexeme, &type)) return CreateToken(type, lexeme);
+    if (IsRegister(lexeme, &type)) return CreateToken(lexeme, lexeme, Line, type);
-    if (lexeme[length - 1] == ':') return CreateToken(LABEL, lexeme);
+    if (lexeme[length - 1] == ':') return CreateToken(lexeme, lexeme, Line, LABEL);
 
-    return CreateToken(IDENTIFIER, lexeme);
+    return CreateToken(lexeme, lexeme, Line, IDENTIFIER);
 }
 
 char PeekScanner(void) {
@@ -234,15 +245,15 @@ int IsPunctuation(char c) {
 Token* ParsePunctuation(char c) {
     switch(c) {
         case '[':
-            return CreateToken(LBRACKET, "[");
+            return CreateToken("[", NULL, Line, LBRACKET);
         case ']':
-            return CreateToken(RBracket, "]");
+            return CreateToken("]", NULL, Line, RBracket);
         case '(':
-            return CreateToken(LPARAM, "(");
+            return CreateToken("(", NULL, Line, LPARAM);
         case ')':
-            return CreateToken(RPARAM, ")");
+            return CreateToken(")", NULL, Line, RPARAM);
         case ',':
-            return CreateToken(COMMA, ",");
+            return CreateToken(",", NULL, Line, COMMA);
         default:
             return NULL;
     }

src/token.c

@@ -1,11 +1,16 @@
 #include "../includes/token.h"
 
-Token* CreateToken(TokenType type, char* value) {
+Token* CreateToken(char* lexeme, void* value, int lineNumber, TokenType type) {
     Token* token = calloc(1, sizeof(Token));
 
-    if (!token) return NULL;
+    if (!token) { 
+        fprintf(stderr, "Failed to calloc memory for Token. %s.\n", strerror(errno));
+        return NULL;
+    }
 
     token->type = type;
+    token->line = lineNumber;
+    token->lexeme = lexeme;
     token->value = value;
 
     return token;
@@ -14,7 +19,7 @@ Token* CreateToken(TokenType type, char* value) {
 void FreeToken(Token* token) {
     if (!token) return;
 
-    if (token->value) free(token->value);
+    if (token->value && token->type >= STRING) free(token->value);
 
     free(token);
 }

src/tokenizer.c

@@ -1,116 +0,0 @@
-#include "../includes/tokenizer.h"
-#include <stdio.h>
-#include <string.h>
-
-char* SplitOnBasicGrammar(char*, unsigned long *);
-void GetStringLiteral(char*, unsigned long, char**, unsigned long*);
-static char* string;
-static unsigned long current_position;
-
-char* GetToken(char* line) {
-    if (line) {
-        string = line;
-        current_position = 0;
-    }
-
-    return SplitOnBasicGrammar(string, &current_position);
-}
-
-char* PeekNextToken() {
-    if (!string) return NULL;
-    if (strlen(string) == 0) return NULL;
-
-    unsigned long pos = current_position;
-
-    return SplitOnBasicGrammar(string, &pos);
-}
-
-//TODO: this function should probably return NULL when its run out of tokens
-//instead of sending back a valid pointer with memory that must be free()ed by the caller.
-char* SplitOnBasicGrammar(char* line, unsigned long *string_index) {
-
-    unsigned long length = strlen(string);
-    char* token = calloc(1, length + 1);
-
-    for (int i = 0; *string_index < length; i++, (*string_index)++) {
-        
-        if (string[*string_index] == ';') break;
-
-        if (string[*string_index] == ':') {
-            if (strlen(token) == 0) {
-                token[0] = ':';
-                (*string_index)++;
-            }
-
-            break;
-        }
-
-        if (string[*string_index] == '"') {
-            (*string_index)++;
-            GetStringLiteral(string, length, &token, string_index);
-            break;
-        }
-
-        if (string[*string_index] == ' ') {
-            while(string[*string_index] == ' ') {
-                (*string_index)++;
-            }
-
-            if (strlen(token) != 0) break;
-        }
-
-        if (string[*string_index] == ',') {
-            if (strlen(token) == 0) {
-                token[0] = string[*string_index];
-                (*string_index)++;
-            }
-
-            break;
-        }
-        
-        if (string[*string_index] != '\n') token[i] = string[*string_index];
-    }
-
-    return token;
-}
-
-void GetStringLiteral(char* text, unsigned long length, char** token, unsigned long* position) {
-    if (!text) return;
-    if (!token) return;
-
-    for(int i = 0; *position < length; (*position)++, i++) {
-        if (text[*position] == '"') {
-            (*position)++;
-            break;
-        }
-        
-        (*token)[i] = text[*position];
-    }
-}
-
-// TokenType GetOperatorType(char c) {
-//     switch (c){
-//         case '+':
-//         return TK_Add;
-//         case '-':
-//         return TK_Sub;
-//         case '*':
-//         return TK_Mul;
-//         case '/':
-//         return TK_Div;
-//         case '^':
-//         return TK_Power;
-//         case ':':
-//         return TK_Colon;
-//         case '(':
-//         return TK_LParam;
-//         case ')':
-//         return TK_RParam;
-//         case '[':
-//         return TK_LBracket;
-//         case ']':
-//         return TK_RBracket;
-//     };
-
-//     return TK_Invalid;
-// }