Created a tokenizer and the supporting code, currently processes text that is not a string literal.

src/array.c

@@ -0,0 +1,28 @@
+#include "../includes/array.h"
+#include <stdlib.h>
+
+#define ARRAY_DEFAULT_CAPACITY 32
+
+struct _array* ArrayCreate(void) {
+    struct _array* array = calloc(1, sizeof(Array));
+
+    array->Items = calloc(ARRAY_DEFAULT_CAPACITY, sizeof(void*));
+    array->Capacity = ARRAY_DEFAULT_CAPACITY;
+
+    return array;
+}
+
+int ArrayAdd(struct _array* array, void* item) {
+    if (!array) return 0;
+
+    if (array->Size == array->Capacity) {
+        array->Items = realloc(array->Items, array->Capacity * 2 * sizeof(void*));
+
+        array->Capacity *= 2;
+    }
+
+    array->Items[array->Size] = item;
+    array->Size++;
+
+    return 1;
+}

src/dictionary.c

@@ -32,8 +32,10 @@ KeyValPair* KeyValPairCreate(const char* key, void* value) {
 int DictionaryAdd(struct _dictionary* dict, const char* key, void* value) {
     if (!dict) return 0;
 
-    if (dict->Size = dict->Capacity) {
+    if (dict->Size == dict->Capacity) {
         dict->Pairs = realloc(dict->Pairs, sizeof(KeyValPair*) * dict->Capacity * 2);
+
+        dict->Capacity *= 2;
     }
 
     dict->Pairs[dict->Size] = KeyValPairCreate(key, value);

src/main.c

@@ -1,10 +1,13 @@
 #include <stdio.h>
 #include "../includes/futil.h"
+#include "../includes/tokenizer.h"
 
 int main(int argc, char** argv) {
     char* file;
     size_t count;
 
+    //gcc -o test src/main.c src/futil.c src/dictionary.c src/tokenizer.c src/token.c src/array.c -g
+
     if (argc <= 1) {
         printf("Usage: assm <file.asm>\n");
 
@@ -17,5 +20,20 @@ int main(int argc, char** argv) {
         return 1;
     }
 
-    printf(file);
+    Dictionary* variables = DictionaryCreate();
+    
+    Array* tokens = Tokenize(file, &variables);
+
+    if (tokens->Size == 0) {
+        printf("No tokens\n");
+
+        return 0;
+    }
+    else printf("Found %d tokens.", tokens->Size);
+
+    for(int i = 0; i < tokens->Size; i++) {
+        Token* token = tokens->Items[i];
+
+        printf(TokenStringify(token, false));
+    }
 }

src/token.c

@@ -0,0 +1,85 @@
+#include "../includes/token.h"
+
+struct _token* TokenCreate(TokenType type, int lineNumber, int columnNumber, bool resolved) {
+    struct _token* token = calloc(1, sizeof(Token));
+
+    token->Type = type;
+    token->LineNumber = lineNumber;
+    token->ColumnNumber = columnNumber;
+    token->Resolved = resolved;
+
+    return token;
+}
+
+char* TokenStringify(const struct _token* token, bool valueOnly) {
+    if (!token) return NULL;
+    char type[32];
+    char number[32];
+
+    int length = StringifyTokenType(token->Type, type);
+
+    char* result = calloc(512, sizeof(char));
+    #ifdef DEBUG
+    
+    #endif
+
+    switch(token->Type) {
+        case Empty:
+            snprintf(result, length + 1, "%s ", type);
+            break;
+        case String:
+            snprintf(result, 511, "%s \"%s\" ", type, token->Value.String);
+            break;
+        case Number:
+            snprintf(number, 32, "%d", token->Value.Number);
+            snprintf(result, 511, "%s %s", type, number);
+            break;
+        case Symbol:
+            snprintf(result, 511, "%s %s", type, token->Value.String);
+            break;
+        case LineEnd:
+            snprintf(result, 511, "%s\n", type);
+            break;
+        case FileEnd:
+            snprintf(result, length + 1, "%s ", type);
+            break;
+        default:
+            //snprintf(buffer, length, "<%c>", type);
+            break;
+    }
+
+    return result;
+}
+
+int StringifyTokenType(TokenType type, char buffer[32]) {
+    if (!buffer) return 0;
+    
+    memset(buffer, '\0', 32);
+
+    switch(type) {
+        case Empty:
+            strncpy(buffer, "<empty >", 8);
+            break;
+        case String:
+            strncpy(buffer, "<string>", 8);
+            break;
+        case Number:
+            strncpy(buffer, "<number>", 8);
+            break;
+        case Symbol:
+            strncpy(buffer, "<symbol>", 8);
+            break;
+        case LineEnd:
+            strncpy(buffer, "<LnEnd >", 8);
+            break;
+        case FileEnd:
+            strncpy(buffer, "<filend>", 8);
+            break;
+        default:
+            snprintf(buffer, 8, "<%c    >", type);
+
+            break;
+    }
+
+    return 8;
+}

src/tokenizer.c

@@ -0,0 +1,134 @@
+#include "../includes/tokenizer.h"
+#include "../includes/keywords.h"
+#include "../includes/string_builder.h"
+#include <ctype.h>
+#include <string.h>
+#include <stdbool.h>
+
+#define MAXWORDLENGTH 1024
+
+//char Word[MAXWORDLENGTH] = { 0 };
+Array* Tokens; 
+
+const char* SourceCode;
+size_t SourceCodeLength = 0;
+size_t SourceCodeIndex = 0;
+int SourceLineNumber = 0;
+int SourceColumnNumber = 0;
+
+bool TryPeekTokenizer(char* c);
+bool TryPopTokenizer(char* c);
+bool TokenizerAtEnd(void);
+char AdvanceTokenizer(void);
+bool BackTokenzier(void);
+char TokenizerLookAhead(void);
+
+void TokenizerExpectWord(void);
+void TokenizerIgnoreLine(void);
+
+Array* Tokenize(const char* sourceCode, Dictionary** variables) {
+    Tokens = ArrayCreate();
+    
+    SourceCodeLength = strlen(sourceCode) - 1;
+    SourceCode = sourceCode;
+
+    while(!TokenizerAtEnd()) {
+        char c;
+
+        if (!TryPeekTokenizer(&c)) break;
+
+        if (isalpha(c)) {
+            TokenizerExpectWord();
+
+            continue;
+        }
+
+        if (c == ';') {
+            TokenizerIgnoreLine();
+
+            continue;
+        }
+
+        AdvanceTokenizer();
+    }
+
+    ArrayAdd(Tokens, TokenCreate(FileEnd, SourceLineNumber + 1, 0, true));
+
+    return Tokens;
+}
+
+void TokenizerExpectWord() {
+    char word[256] = { 0 };
+    int wordLength = 0;
+    char c;
+
+    while(TryPopTokenizer(&c)) {
+        if (isspace(c)) {
+            break;
+        }
+
+        if (ispunct(c)) {
+            BackTokenzier();
+
+            break;
+        }
+
+        if (wordLength == 255) break; //Sync, error etc here at some point.
+
+        word[wordLength] = c;
+        wordLength++;
+    }
+
+    if (wordLength > 0) {
+        Token* token = TokenCreate(Symbol, SourceLineNumber, SourceColumnNumber, false);
+
+        token->Value.String = calloc(wordLength + 1, sizeof(char));
+
+        strncpy(token->Value.String, word, wordLength);
+
+        ArrayAdd(Tokens, token);
+    }
+}
+
+void TokenizerIgnoreLine(void) {
+    char c;
+
+    while(TryPopTokenizer(&c)) {
+        if (c == '\n') break;
+    }
+}
+
+bool TryPeekTokenizer(char* c) {
+    *c = SourceCode[SourceCodeIndex];
+
+    return *c != '\0';
+}
+
+bool TryPopTokenizer(char* c) {
+    *c = AdvanceTokenizer();
+
+    return *c != '\0';
+}
+
+char AdvanceTokenizer() {
+    if (TokenizerAtEnd()) return '\0';
+    if (SourceCodeIndex + 1 > SourceCodeLength) return '\0';
+
+    char c = SourceCode[SourceCodeIndex];
+
+    SourceCodeIndex++;
+
+    return c;
+}
+
+bool BackTokenzier(void) {
+    if (SourceCodeIndex == 0) return false;
+
+    SourceCodeIndex--;
+
+    return true;
+}
+
+bool TokenizerAtEnd() {
+    return SourceCodeLength == SourceCodeIndex;
+}