Fixed a bug where the tokenizer wasn't parsing hexadecimal numbers correctly. Added support for tokenizing the colon and comma characters.

includes/array.h

@@ -1,6 +1,8 @@
 #ifndef ARRAY_H
 #define ARRAY_H
 
+#include <stdbool.h>
+
 typedef struct _array {
     int Capacity;
     int Size;
@@ -8,6 +10,7 @@ typedef struct _array {
 } Array;
 
 Array* ArrayCreate(void);
-int ArrayAdd(Array* array, void* item);
+bool ArrayAdd(Array* array, void* item);
+void* ArrayPeek(Array* array);
 
 #endif

includes/token.h

@@ -12,6 +12,8 @@ typedef enum {
     Number,
     Symbol,
     LineEnd = '\n',
+    Colon = ':',
+    Comma = ',',
     Plus = '+',
     Minus = '-',
     Star = '*',
@@ -37,7 +39,7 @@ typedef struct _token {
 } Token;
 
 Token* TokenCreate(TokenType type, int lineNumber, int columnNumber, bool resolved);
-char* TokenStringify(const Token* token, bool valueOnly);
+char* TokenStringify(const Token* token);
 int StringifyTokenType(TokenType type, char buffer[32]);
 
 #endif

src/array.c

@@ -12,8 +12,8 @@ struct _array* ArrayCreate(void) {
     return array;
 }
 
-int ArrayAdd(struct _array* array, void* item) {
-    if (!array) return 0;
+bool ArrayAdd(struct _array* array, void* item) {
+    if (!array) return false;
 
     if (array->Size == array->Capacity) {
         array->Items = realloc(array->Items, array->Capacity * 2 * sizeof(void*));
@@ -24,5 +24,13 @@ int ArrayAdd(struct _array* array, void* item) {
     array->Items[array->Size] = item;
     array->Size++;
 
-    return 1;
+    return true;
+}
+
+void* ArrayPeek(struct _array* array) {
+    if (!array) return NULL;
+
+    if (array->Size == 0 || array->Capacity == 0) return NULL;
+
+    return array->Items[array->Size - 1];
 }

src/main.c

@@ -30,10 +30,24 @@ int main(int argc, char** argv) {
         return 0;
     }
 
+    char* buffer;
+
     for(int i = 0; i < tokens->Size; i++) {
         Token* token = tokens->Items[i];
 
-        printf("%s ", TokenStringify(token, false));
+        if (token->Type == LineEnd) {
+            printf("\n");
+
+            continue;
+        }
+
+        buffer = TokenStringify(token);
+
+        if (buffer) {
+            printf("%s ", buffer);
+
+            free(buffer);
+        }
     }
 
     printf("\n");

src/token.c

@@ -11,26 +11,43 @@ struct _token* TokenCreate(TokenType type, int lineNumber, int columnNumber, boo
     return token;
 }
 
-char* TokenStringify(const struct _token* token, bool valueOnly) {
-    if (!token) return NULL;
-    char buffer[32];
+char* TokenStringify(const struct _token* token) {
+    if (!token) return false;
+
+    char* buffer = NULL;
 
     switch(token->Type) {
-        case Empty:
-            return "<>";
         case String:
         case Symbol:
-            return token->Value.String;
+            buffer = calloc(strlen(token->Value.String) + 1, sizeof(char));
+
+            strncpy(buffer, token->Value.String, strlen(token->Value.String));
+
+            break;
+        case Empty:
+            buffer = calloc(8, sizeof(char));
+            
+            strncpy(buffer, "<>", sizeof(char) * 2);
+
+            break;
         case Number:
+            buffer = calloc(32, sizeof(char));
+            
             sprintf(buffer, "%d", token->Value.Number);
 
             break;
         case LineEnd:
-            return "\n";
+            buffer = calloc(8, sizeof(char));
+            
+            strncpy(buffer, "\n", sizeof(char));
+
+            break;
         case FileEnd:
-            return NULL;
+            break;
         default:
-            strncpy(buffer, token->Value.Operator, 8);
+            buffer = calloc(16, sizeof(char));
+            
+            strncpy(buffer, token->Value.Operator, sizeof(char));
 
             break;
     }

src/tokenizer.c

@@ -64,8 +64,13 @@ Array* Tokenize(const char* sourceCode, Dictionary** variables) {
             continue;
         }
 
+        Token* token = NULL;
+
         switch(c) {
             case '\n':
+                token = (Token*) ArrayPeek(Tokens);
+                //Is there a previous token and if so, was it a line break?
+                if (token && token->Type == LineEnd) break; //Discard empty line.
             case '+':
             case '-':
             case '*':
@@ -76,7 +81,8 @@ Array* Tokenize(const char* sourceCode, Dictionary** variables) {
             case '[':
             case ']':
             case ':':
-                Token* token = TokenCreate((TokenType)c, SourceLineNumber, SourceColumnNumber, true);
+            case ',':
+                token = TokenCreate((TokenType)c, SourceLineNumber, SourceColumnNumber, true);
 
                 token->Value.Operator[0] = c;
 
@@ -106,18 +112,18 @@ void TokenizerExpectNumber(void) {
             break;
         }
 
-        if (!isdigit(c)) break;
-
         if (PeekTokenizer() == 'x' || PeekTokenizer() == 'X')
         {
             AdvanceTokenizer();
 
             word[wordLength] = c;
-            word[wordLength + 1] = PeekTokenizer();
+            word[wordLength + 1] = 'x';
             wordLength += 2;
 
             continue;
         }
+
+        if (!isxdigit(c)) break;
         
         if (wordLength == 255) break; //Sync, error etc here at some point.
 
@@ -190,7 +196,7 @@ void TokenizerExpectString(void) {
 
             break;
         }
-        
+
         if (c == '"') break;
 
         if (c == '\\') {
@@ -267,7 +273,22 @@ char AdvanceTokenizer() {
 bool BackTokenzier(void) {
     if (SourceCodeIndex == 0) return false;
 
-    SourceColumnNumber--;
+    if (SourceCode[SourceCodeIndex - 1] == '\n') {
+        SourceColumnNumber = 1;
+
+        for(int i = SourceCodeIndex - 1; i >= 0; i--) {
+            char c = SourceCode[i];
+
+            if (c == '\n') break;
+
+            SourceColumnNumber++;
+        }
+        
+        SourceLineNumber--;
+    }
+    else
+        SourceColumnNumber--;
+
     SourceCodeIndex--;
 
     return true;