Created a tokenizer and the supporting code, currently processes text that is not a string literal.

Makefile

@@ -0,0 +1,41 @@
+CC = gcc
+CFLAGS=-g -Wall -DDEBUG -Wpedantic -Wextra -Wunused-result -std=c99 -pedantic-errors
+SRCDIR=src
+OBJDIR=obj
+SRCS=$(wildcard $(SRCDIR)/*.c)
+# Substitute all .c with .o from SRCS
+OBJS=$(patsubst $(SRCDIR)/%.c, $(OBJDIR)/%.o, $(SRCS))
+
+BINDIR=bin
+BIN=$(BINDIR)/assm
+
+all: $(BIN)
+
+release: CFLAGS=-Wall -Wpedantic -std=c99 -pedantic-errors -O2
+release: clean
+release: $(BIN)
+
+$(BIN): $(OBJS) $(BINDIR)
+	$(CC) $(CFLAGS) $(OBJS) -o $@
+
+$(OBJDIR)/%.o: $(SRCDIR)/%.c $(OBJDIR)
+	$(CC) $(CFLAGS) -c $< -o $@
+
+$(BINDIR):
+	mkdir $@
+
+$(OBJDIR):
+	mkdir $@
+
+.PHONY: clean
+.PHONY: test
+.PHONY: disass
+
+clean:
+	rm -rf $(BINDIR)/* $(OBJDIR)/*
+
+test:
+	$(BIN) misc/another_test.asm
+
+disass:
+	objdump -S --disassemble $(OBJDIR)/$(FILE).o > $(OBJDIR)/$(FILE).s

includes/array.h

@@ -0,0 +1,13 @@
+#ifndef ARRAY_H
+#define ARRAY_H
+
+typedef struct _array {
+    int Capacity;
+    int Size;
+    void** Items;
+} Array;
+
+Array* ArrayCreate(void);
+int ArrayAdd(Array* array, void* item);
+
+#endif

includes/opcodes.h

@@ -10,13 +10,13 @@ typedef enum {
 } Registers;
 
 typedef enum {
-    ADD = 0x01, SUB, MUL, DIV, MOV, AND, OR, XOR, NOT, SHL, SHR, NOP, CMP, JMP, JG, JL, 
-    OUTB, INB, HLT, ENI, INT, LIVT, PUSHA, POPA, CALL, RET, PUSH, POP
+    ADD = 0x01, SUB, MUL, DIV, MOV, AND, OR, XOR, NOT, SHL, SHR, NOP, CMP, JMP, JZ, JG, JL, 
+    OUTB, INB, HLT, CLI, ENI, INT, LIVT, PUSHA, POPA, CALL, RET, PUSH, POP
 } Mnemonic;
 
 int IsOpcode(const char*, Mnemonic*);
 int IsRegister(const char*, Registers*);
 void GetMnemonicText(Mnemonic mnemonic, char buffer[12]);
-void GetRegisterText(Registers reg, char buffer[3]);
+void GetRegisterText(Registers reg, char buffer[4]);
 
 #endif

includes/token.h

@@ -1,11 +1,17 @@
 #ifndef TOKEN_H
 #define TOKEN_H
 
-enum TokenType {
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdbool.h>
+
+typedef enum {
     Empty,
     String,
     Number,
     Symbol,
+    LineEnd = '\n',
     Plus = '+',
     Minus = '-',
     Star = '*',
@@ -13,23 +19,25 @@ enum TokenType {
     Carot = '^',
     OpenParen = '(',
     CloseParen = ')',
+    OpenBracket = '[',
+    CloseBracket = ']',
     FileEnd
-};
+} TokenType;
 
 typedef struct _token {
     TokenType Type;
-    char* Name;
     union {
+        char Operator[8];
         char* String;
         int Number;
     } Value;
     int LineNumber;
     int ColumnNumber;
-    int Index;
-    int Length;
-    int Resolved;
+    bool Resolved;
 } Token;
 
-Token* TokenCreate(char* name, TokenType type, int lineNumber, int columnNumber, int index, int length, int resolved);
+Token* TokenCreate(TokenType type, int lineNumber, int columnNumber, bool resolved);
+char* TokenStringify(const Token* token, bool valueOnly);
+int StringifyTokenType(TokenType type, char buffer[32]);
 
 #endif

includes/tokenizer.h

@@ -0,0 +1,10 @@
+#ifndef TOKENIZER_H
+#define TOKENIZER_H
+
+#include "array.h"
+#include "dictionary.h"
+#include "token.h"
+
+Array* Tokenize(const char* text, Dictionary** variables);
+
+#endif

src/array.c

@@ -0,0 +1,28 @@
+#include "../includes/array.h"
+#include <stdlib.h>
+
+#define ARRAY_DEFAULT_CAPACITY 32
+
+struct _array* ArrayCreate(void) {
+    struct _array* array = calloc(1, sizeof(Array));
+
+    array->Items = calloc(ARRAY_DEFAULT_CAPACITY, sizeof(void*));
+    array->Capacity = ARRAY_DEFAULT_CAPACITY;
+
+    return array;
+}
+
+int ArrayAdd(struct _array* array, void* item) {
+    if (!array) return 0;
+
+    if (array->Size == array->Capacity) {
+        array->Items = realloc(array->Items, array->Capacity * 2 * sizeof(void*));
+
+        array->Capacity *= 2;
+    }
+
+    array->Items[array->Size] = item;
+    array->Size++;
+
+    return 1;
+}

src/dictionary.c

@@ -32,8 +32,10 @@ KeyValPair* KeyValPairCreate(const char* key, void* value) {
 int DictionaryAdd(struct _dictionary* dict, const char* key, void* value) {
     if (!dict) return 0;
 
-    if (dict->Size = dict->Capacity) {
+    if (dict->Size == dict->Capacity) {
         dict->Pairs = realloc(dict->Pairs, sizeof(KeyValPair*) * dict->Capacity * 2);
+
+        dict->Capacity *= 2;
     }
 
     dict->Pairs[dict->Size] = KeyValPairCreate(key, value);

src/main.c

@@ -1,10 +1,13 @@
 #include <stdio.h>
 #include "../includes/futil.h"
+#include "../includes/tokenizer.h"
 
 int main(int argc, char** argv) {
     char* file;
     size_t count;
 
+    //gcc -o test src/main.c src/futil.c src/dictionary.c src/tokenizer.c src/token.c src/array.c -g
+
     if (argc <= 1) {
         printf("Usage: assm <file.asm>\n");
 
@@ -17,5 +20,20 @@ int main(int argc, char** argv) {
         return 1;
     }
 
-    printf(file);
+    Dictionary* variables = DictionaryCreate();
+    
+    Array* tokens = Tokenize(file, &variables);
+
+    if (tokens->Size == 0) {
+        printf("No tokens\n");
+
+        return 0;
+    }
+    else printf("Found %d tokens.", tokens->Size);
+
+    for(int i = 0; i < tokens->Size; i++) {
+        Token* token = tokens->Items[i];
+
+        printf(TokenStringify(token, false));
+    }
 }

src/token.c

@@ -0,0 +1,85 @@
+#include "../includes/token.h"
+
+struct _token* TokenCreate(TokenType type, int lineNumber, int columnNumber, bool resolved) {
+    struct _token* token = calloc(1, sizeof(Token));
+
+    token->Type = type;
+    token->LineNumber = lineNumber;
+    token->ColumnNumber = columnNumber;
+    token->Resolved = resolved;
+
+    return token;
+}
+
+char* TokenStringify(const struct _token* token, bool valueOnly) {
+    if (!token) return NULL;
+    char type[32];
+    char number[32];
+
+    int length = StringifyTokenType(token->Type, type);
+
+    char* result = calloc(512, sizeof(char));
+    #ifdef DEBUG
+    
+    #endif
+
+    switch(token->Type) {
+        case Empty:
+            snprintf(result, length + 1, "%s ", type);
+            break;
+        case String:
+            snprintf(result, 511, "%s \"%s\" ", type, token->Value.String);
+            break;
+        case Number:
+            snprintf(number, 32, "%d", token->Value.Number);
+            snprintf(result, 511, "%s %s", type, number);
+            break;
+        case Symbol:
+            snprintf(result, 511, "%s %s", type, token->Value.String);
+            break;
+        case LineEnd:
+            snprintf(result, 511, "%s\n", type);
+            break;
+        case FileEnd:
+            snprintf(result, length + 1, "%s ", type);
+            break;
+        default:
+            //snprintf(buffer, length, "<%c>", type);
+            break;
+    }
+
+    return result;
+}
+
+int StringifyTokenType(TokenType type, char buffer[32]) {
+    if (!buffer) return 0;
+    
+    memset(buffer, '\0', 32);
+
+    switch(type) {
+        case Empty:
+            strncpy(buffer, "<empty >", 8);
+            break;
+        case String:
+            strncpy(buffer, "<string>", 8);
+            break;
+        case Number:
+            strncpy(buffer, "<number>", 8);
+            break;
+        case Symbol:
+            strncpy(buffer, "<symbol>", 8);
+            break;
+        case LineEnd:
+            strncpy(buffer, "<LnEnd >", 8);
+            break;
+        case FileEnd:
+            strncpy(buffer, "<filend>", 8);
+            break;
+        default:
+            snprintf(buffer, 8, "<%c    >", type);
+
+            break;
+    }
+
+    return 8;
+}

src/tokenizer.c

@@ -0,0 +1,134 @@
+#include "../includes/tokenizer.h"
+#include "../includes/keywords.h"
+#include "../includes/string_builder.h"
+#include <ctype.h>
+#include <string.h>
+#include <stdbool.h>
+
+#define MAXWORDLENGTH 1024
+
+//char Word[MAXWORDLENGTH] = { 0 };
+Array* Tokens; 
+
+const char* SourceCode;
+size_t SourceCodeLength = 0;
+size_t SourceCodeIndex = 0;
+int SourceLineNumber = 0;
+int SourceColumnNumber = 0;
+
+bool TryPeekTokenizer(char* c);
+bool TryPopTokenizer(char* c);
+bool TokenizerAtEnd(void);
+char AdvanceTokenizer(void);
+bool BackTokenzier(void);
+char TokenizerLookAhead(void);
+
+void TokenizerExpectWord(void);
+void TokenizerIgnoreLine(void);
+
+Array* Tokenize(const char* sourceCode, Dictionary** variables) {
+    Tokens = ArrayCreate();
+    
+    SourceCodeLength = strlen(sourceCode) - 1;
+    SourceCode = sourceCode;
+
+    while(!TokenizerAtEnd()) {
+        char c;
+
+        if (!TryPeekTokenizer(&c)) break;
+
+        if (isalpha(c)) {
+            TokenizerExpectWord();
+
+            continue;
+        }
+
+        if (c == ';') {
+            TokenizerIgnoreLine();
+
+            continue;
+        }
+
+        AdvanceTokenizer();
+    }
+
+    ArrayAdd(Tokens, TokenCreate(FileEnd, SourceLineNumber + 1, 0, true));
+
+    return Tokens;
+}
+
+void TokenizerExpectWord() {
+    char word[256] = { 0 };
+    int wordLength = 0;
+    char c;
+
+    while(TryPopTokenizer(&c)) {
+        if (isspace(c)) {
+            break;
+        }
+
+        if (ispunct(c)) {
+            BackTokenzier();
+
+            break;
+        }
+
+        if (wordLength == 255) break; //Sync, error etc here at some point.
+
+        word[wordLength] = c;
+        wordLength++;
+    }
+
+    if (wordLength > 0) {
+        Token* token = TokenCreate(Symbol, SourceLineNumber, SourceColumnNumber, false);
+
+        token->Value.String = calloc(wordLength + 1, sizeof(char));
+
+        strncpy(token->Value.String, word, wordLength);
+
+        ArrayAdd(Tokens, token);
+    }
+}
+
+void TokenizerIgnoreLine(void) {
+    char c;
+
+    while(TryPopTokenizer(&c)) {
+        if (c == '\n') break;
+    }
+}
+
+bool TryPeekTokenizer(char* c) {
+    *c = SourceCode[SourceCodeIndex];
+
+    return *c != '\0';
+}
+
+bool TryPopTokenizer(char* c) {
+    *c = AdvanceTokenizer();
+
+    return *c != '\0';
+}
+
+char AdvanceTokenizer() {
+    if (TokenizerAtEnd()) return '\0';
+    if (SourceCodeIndex + 1 > SourceCodeLength) return '\0';
+
+    char c = SourceCode[SourceCodeIndex];
+
+    SourceCodeIndex++;
+
+    return c;
+}
+
+bool BackTokenzier(void) {
+    if (SourceCodeIndex == 0) return false;
+
+    SourceCodeIndex--;
+
+    return true;
+}
+
+bool TokenizerAtEnd() {
+    return SourceCodeLength == SourceCodeIndex;
+}