Attempting to declutter the lexer and tokenizer by rereolling them into the 'Scanner' object.

includes/scanner.h

@@ -0,0 +1,12 @@
+#ifndef SCANNER_H
+#define SCANNER_H
+
+#include <stdio.h>
+#include <string.h>
+#include "stdlib.h"
+#include "token.h"
+#include "list.h"
+
+List* GenerateTokenList(const char*);
+
+#endif

includes/token.h

@@ -0,0 +1,38 @@
+#ifndef TOKEN_H
+#define TOKEN_H
+
+#include "stdlib.h"
+
+typedef enum {
+    PLUS,
+    MINUS,
+    STAR,
+    SLASH,
+    POWER,
+    LPARAM,
+    RPARAM,
+    LBRACKET,
+    RBracket,
+    COMMA,
+    STRING,
+    IDENTIFIER,
+    NUMBER,
+    HEX,
+    //Keywords
+    DB, ORG,    
+    //Opcodes
+    COPY, ADD, SUB, JZ, INT, YLD, RET,
+    CMP, IN, OUT,
+    //Registers
+    R1, R2, R3, R4, R5, R6, R7, R8
+} TokenType;
+
+typedef struct {
+    TokenType type;
+    char* value;
+} Token;
+
+Token* CreateToken(TokenType type, char* value);
+void FreeToken(Token*);
+
+#endif

src/scanner.c

@@ -0,0 +1,137 @@
+#include "../includes/scanner.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+const char* SourceCode;
+int Position = 0;
+int SourceLength = 0;
+int ScannerAtEnd(void);
+char PeekScanner(void);
+char AdvanceScanner(void);
+Token* ParseString(void);
+Token* ParseDirective(void);
+Token* ParseIdentifier(void);
+
+List* GenerateTokenList(const char* source) {
+    List* tokens = CreateList();
+    Token* token = NULL;
+
+    if (!tokens) return NULL;
+
+    SourceCode = source;
+    SourceLength = strlen(source);
+
+    for(int i = 0; i < SourceLength; i++) {
+        char c = AdvanceScanner();
+        switch(c) {
+            case ' ':
+            case '\r':
+            case '\t':
+                break; //Ignore whitespace
+            case ';':
+                while(!ScannerAtEnd() && PeekScanner() != '\n') AdvanceScanner();
+
+                break;
+            case '.': //directive like ".org" or ".db"
+                token = ParseDirective();
+                if (token) AddListItem(token, sizeof(Token), tokens);
+                break;
+            case '"':
+                token = ParseString();
+                if (token) AddListItem(token, sizeof(Token), tokens);
+                break;
+            case ',':
+                AddListItem(CreateToken(COMMA, ","), sizeof(Token), tokens);
+                break;
+            default:
+                // if (c >= '0' && c <= '9') {
+                //     //Number
+
+                //     break;
+                // }
+
+                token = ParseIdentifier();
+
+                if (token) AddListItem(token, sizeof(Token), tokens);
+
+                break;
+        }
+
+        token = NULL;
+    }
+
+    return tokens;
+}
+
+Token* ParseDirective(void) {
+    char directive[10] = { 0 };
+    int length = 0;
+
+    for(; length < sizeof directive - 1; length++) {
+        if (PeekScanner() == ' ') break;
+
+        directive[length] = AdvanceScanner();
+    }
+
+    if (strcmp(directive, ".db") == 0) {
+        return CreateToken(DB, ".db");
+    } else if (strcmp(directive, ".org") == 0) {
+        //fprintf(stderr, "[Warning] Org is not a supported directive.\n");
+        return NULL;
+    }
+
+    fprintf(stderr, "[Error] Unknown assembler directive '%s'\n", directive);
+    //Ignore the line 
+    while(PeekScanner() != '\n') AdvanceScanner();
+
+    return NULL;
+}
+
+Token* ParseString(void) {
+    int start = Position;
+
+    while(!ScannerAtEnd() && PeekScanner() != '"') {
+        AdvanceScanner();
+    }
+
+    char* lexeme = calloc(Position - start, sizeof(char));
+    memccpy(lexeme, &SourceCode[start], sizeof(char), Position - start);
+
+    AdvanceScanner();
+
+    Token* token = CreateToken(STRING, lexeme);
+
+    return token;
+}
+
+Token* ParseIdentifier(void) {
+    int start = Position;
+
+    while(!ScannerAtEnd() && PeekScanner() != ' ' && PeekScanner() != '\n') {
+        AdvanceScanner();
+    }
+
+    if (Position - start == 0) return NULL;
+    
+    char* lexeme = calloc(Position - start, sizeof(char));
+    memccpy(lexeme, &SourceCode[start], sizeof(char), Position - start);
+
+    return CreateToken(IDENTIFIER, lexeme);
+}
+
+char PeekScanner(void) {
+    if (Position > SourceLength) return '\0';
+
+    return SourceCode[Position];
+}
+
+char AdvanceScanner(void) {
+    if (ScannerAtEnd()) return '\0';
+
+    return SourceCode[Position++];
+}
+
+int ScannerAtEnd(void) {
+    return Position >= SourceLength;
+}

src/token.c

@@ -0,0 +1,20 @@
+#include "../includes/token.h"
+
+Token* CreateToken(TokenType type, char* value) {
+    Token* token = calloc(1, sizeof(Token));
+
+    if (!token) return NULL;
+
+    token->type = type;
+    token->value = value;
+
+    return token;
+}
+
+void FreeToken(Token* token) {
+    if (!token) return;
+
+    if (token->value) free(token->value);
+
+    free(token);
+}