Updated the scanner to parse registers, assembly keywords and identifiers.

includes/lexer.h

@@ -7,34 +7,7 @@
 #include <ctype.h>
 #include "list.h"
 #include "tokenizer.h"
-
+#include "token.h"
-typedef enum {
-    PLUS,
-    MINUS,
-    STAR,
-    SLASH,
-    POWER,
-    LPARAM,
-    RPARAM,
-    LBRACKET,
-    RBracket,
-    COMMA,
-    IDENTIFIER,
-    NUMBER,
-    HEX,
-    //Keywords
-    DB, ORG,    
-    //Opcodes
-    COPY, ADD, SUB, JZ, INT, YLD, RET,
-    CMP, IN, OUT,
-    //Registers
-    R1, R2, R3, R4, R5, R6, R7, R8
-} TokenType;
-
-typedef struct {
-    TokenType type;
-    char* value;
-} Token;
 
 List* GetTokensFromLine(char*);
 

includes/scanner.h

@@ -3,9 +3,11 @@
 
 #include <stdio.h>
 #include <string.h>
+#include <ctype.h>
 #include "stdlib.h"
 #include "token.h"
 #include "list.h"
+#include "opcodes.h"
 
 List* GenerateTokenList(const char*);
 

src/lexer.c

@@ -4,7 +4,6 @@
 #include <stdlib.h>
 #include <string.h>
 
-Token* CreateToken(TokenType, char*);
 Token* GetNextToken(char *);
 int TokenIsNumeric(const char*, int *);
 
@@ -74,17 +73,6 @@ Token* GetNextToken(char *string) {
     return NULL;
 }
 
-Token* CreateToken(TokenType type, char* value) {
-    Token* token = calloc(1, sizeof(Token));
-
-    if (!token) return NULL;
-
-    token->type = type;
-    token->value = value;
-
-    return token;
-}
-
 int TokenIsNumeric(const char* token, int *base) {
     *base = 0;
 

src/main.c

@@ -6,6 +6,7 @@
 #include "../includes/lexer.h"
 #include "../includes/parser.h"
 #include "../includes/futil.h"
+#include "../includes/scanner.h"
 
 List* get_strings(const char*);
 
@@ -20,30 +21,12 @@ int main(int argc, char* args[]) {
 
     if (!ReadAllString(args[1], &source_code, &bytes_read)) return EX_IOERR;
 
-    char* token = strtok(source_code, "\n");
+    List* list = GenerateTokenList(source_code);
-    int line_number = 1;
 
-    while(token != NULL) {
+    for(int i = 0; i < list->size; i++) {
+        Token* t = (Token*) list->content[i];
 
-        List* tokens = GetTokensFromLine(token);
+        printf("[%i] '%s'\n", t->type, t->value);
-
-        if (line_number == 1 || line_number == 2) {
-            ParseTokens(tokens);
-        }
-
-        for (int i = 0; i < tokens->size; i++) {
-            Token* t = (Token*) tokens->content[i];
-
-            printf("Token Type: '%d' Value: '%s' (%d)\n", t->type, t->value, line_number);
-
-            if (t->type <= ORG)  free(t->value);
-        }
-
-        DestroyList(tokens);
-
-        token = strtok(NULL, "\n");
-
-        line_number++;
     }
 
     free(source_code);

src/scanner.c

@@ -1,9 +1,11 @@
 #include "../includes/scanner.h"
+#include <ctype.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
 const char* SourceCode;
+int Line = 0;
 int Position = 0;
 int SourceLength = 0;
 int ScannerAtEnd(void);
@@ -12,6 +14,8 @@ char AdvanceScanner(void);
 Token* ParseString(void);
 Token* ParseDirective(void);
 Token* ParseIdentifier(void);
+Token* ParseNumber(void);
+void IgnoreLine(void);
 
 List* GenerateTokenList(const char* source) {
     List* tokens = CreateList();
@@ -23,15 +27,20 @@ List* GenerateTokenList(const char* source) {
     SourceLength = strlen(source);
 
     for(int i = 0; i < SourceLength; i++) {
-        char c = AdvanceScanner();
+        char c = PeekScanner();
+        
         switch(c) {
             case ' ':
             case '\r':
             case '\t':
+                AdvanceScanner();
                 break; //Ignore whitespace
+            case '\n':
+                AdvanceScanner();
+                Line++;
+                break;
             case ';':
-                while(!ScannerAtEnd() && PeekScanner() != '\n') AdvanceScanner();
+                IgnoreLine();
-
                 break;
             case '.': //directive like ".org" or ".db"
                 token = ParseDirective();
@@ -42,15 +51,15 @@ List* GenerateTokenList(const char* source) {
                 if (token) AddListItem(token, sizeof(Token), tokens);
                 break;
             case ',':
+                AdvanceScanner();
                 AddListItem(CreateToken(COMMA, ","), sizeof(Token), tokens);
                 break;
             default:
-                // if (c >= '0' && c <= '9') {
+                if (c >= '0' && c <= '9') {
-                //     //Number
+                    ParseNumber();
-
-                //     break;
-                // }
 
+                    break;
+                }
                 token = ParseIdentifier();
 
                 if (token) AddListItem(token, sizeof(Token), tokens);
@@ -64,31 +73,66 @@ List* GenerateTokenList(const char* source) {
     return tokens;
 }
 
-Token* ParseDirective(void) {
+Token* ParseNumber(void) {
-    char directive[10] = { 0 };
+    int start = Position;
-    int length = 0;
 
-    for(; length < sizeof directive - 1; length++) {
+    while(!ScannerAtEnd() && PeekScanner() >= '0' && PeekScanner() <= '9')
-        if (PeekScanner() == ' ') break;
+        AdvanceScanner();
 
-        directive[length] = AdvanceScanner();
+    char* lexeme = calloc(Position - start + 1, sizeof(char));
-    }
 
-    if (strcmp(directive, ".db") == 0) {
+    if (!lexeme) {
-        return CreateToken(DB, ".db");
+        fprintf(stderr, "Failed to calloc space for number.\n");
-    } else if (strcmp(directive, ".org") == 0) {
-        //fprintf(stderr, "[Warning] Org is not a supported directive.\n");
         return NULL;
     }
 
-    fprintf(stderr, "[Error] Unknown assembler directive '%s'\n", directive);
+    memcpy(lexeme, &SourceCode[start], Position - start);
-    //Ignore the line 
-    while(PeekScanner() != '\n') AdvanceScanner();
 
+    return CreateToken(NUMBER, lexeme);
+}
+
+Token* ParseDirective(void) {
+    int start = Position;
+
+    while(!ScannerAtEnd() && PeekScanner() != ' ' && PeekScanner() != '\n') AdvanceScanner();
+        
+    if (start == Position) return NULL;
+
+    char* directive = calloc(Position - start + 1, sizeof(char));
+
+    if (!directive) {
+        fprintf(stderr, "Failed to calloc for the assmebler directive.\n");
+        return NULL;
+    }
+
+    memcpy(directive, &SourceCode[start], Position - start);
+
+    if (strcmp(directive, ".db") == 0) {
+        return CreateToken(DB, directive);
+    } 
+    // else if (strcmp(directive, ".org") == 0) {
+    //     fprintf(stderr, "[Warning] Org is not a supported directive.\n");
+    //     IgnoreLine();
+    //     return NULL;
+    // }
+
+    fprintf(stderr, "[Error] Unknown assembler directive '%s'\n", directive);
+
+    free(directive);
+    
+    //Ignore the line 
+    IgnoreLine();
+    
     return NULL;
 }
 
+void IgnoreLine(void) {
+    while(!ScannerAtEnd() && PeekScanner() != '\n') AdvanceScanner();
+}
+
 Token* ParseString(void) {
+    AdvanceScanner(); //Consume the first double quote.
+
     int start = Position;
 
     while(!ScannerAtEnd() && PeekScanner() != '"') {
@@ -96,9 +140,9 @@ Token* ParseString(void) {
     }
 
     char* lexeme = calloc(Position - start, sizeof(char));
-    memccpy(lexeme, &SourceCode[start], sizeof(char), Position - start);
+    memcpy(lexeme, &SourceCode[start], Position - start);
 
-    AdvanceScanner();
+    AdvanceScanner(); //Consume the trailing double quote.
 
     Token* token = CreateToken(STRING, lexeme);
 
@@ -108,20 +152,24 @@ Token* ParseString(void) {
 Token* ParseIdentifier(void) {
     int start = Position;
 
-    while(!ScannerAtEnd() && PeekScanner() != ' ' && PeekScanner() != '\n') {
+    while(!ScannerAtEnd() && PeekScanner() != ',' && PeekScanner() != ' ' && PeekScanner() != '\n') {
         AdvanceScanner();
     }
 
     if (Position - start == 0) return NULL;
     
     char* lexeme = calloc(Position - start, sizeof(char));
-    memccpy(lexeme, &SourceCode[start], sizeof(char), Position - start);
+    TokenType type;
+    memcpy(lexeme, &SourceCode[start], Position - start);
+
+    if (IsOpcode(lexeme, &type)) return CreateToken(type, lexeme);
+    if (IsRegister(lexeme, &type)) return CreateToken(type, lexeme);
 
     return CreateToken(IDENTIFIER, lexeme);
 }
 
 char PeekScanner(void) {
-    if (Position > SourceLength) return '\0';
+    if (Position >= SourceLength) return '\0';
 
     return SourceCode[Position];
 }