Cleaned up the TokenStringify function a bit (ignore the returning of a local variable) and updated the tokenizer to better handle new lines.

src/main.c

@@ -29,7 +29,6 @@ int main(int argc, char** argv) {
 
         return 0;
     }
-    else printf("Found %d tokens.", tokens->Size);
 
     for(int i = 0; i < tokens->Size; i++) {
         Token* token = tokens->Items[i];

src/token.c

@@ -13,43 +13,29 @@ struct _token* TokenCreate(TokenType type, int lineNumber, int columnNumber, boo
 
 char* TokenStringify(const struct _token* token, bool valueOnly) {
     if (!token) return NULL;
-    char type[32];
+    char buffer[32];
-    char number[32];
-
-    int length = StringifyTokenType(token->Type, type);
-
-    char* result = calloc(512, sizeof(char));
-    #ifdef DEBUG
-    
-    #endif
 
     switch(token->Type) {
         case Empty:
-            snprintf(result, length + 1, "%s ", type);
+            return "<>";
-            break;
         case String:
-            snprintf(result, 511, "%s \"%s\" ", type, token->Value.String);
-            break;
-        case Number:
-            snprintf(number, 32, "%d", token->Value.Number);
-            snprintf(result, 511, "%s %s", type, number);
-            break;
         case Symbol:
-            snprintf(result, 511, "%s %s", type, token->Value.String);
+            return token->Value.String;
+        case Number:
+            sprintf(buffer, "%d", token->Value.Number);
+
             break;
         case LineEnd:
-            snprintf(result, 511, "%s\n", type);
+            return "\n";
-            break;
         case FileEnd:
-            snprintf(result, length + 1, "%s ", type);
+            return NULL;
-            break;
         default:
-            snprintf(result, length + 2, "%s %c", type, token->Value.Operator[0]);
+            strncpy(buffer, token->Value.Operator, 8);
-            //snprintf(buffer, length, "<%c>", type);
+
             break;
     }
 
-    return result;
+    return buffer;
 }
 
 int StringifyTokenType(TokenType type, char buffer[32]) {

src/tokenizer.c

@@ -75,6 +75,7 @@ Array* Tokenize(const char* sourceCode, Dictionary** variables) {
             case ')':
             case '[':
             case ']':
+            case ':':
                 Token* token = TokenCreate((TokenType)c, SourceLineNumber, SourceColumnNumber, true);
 
                 token->Value.Operator[0] = c;
@@ -99,6 +100,24 @@ void TokenizerExpectNumber(void) {
     char c;
 
     while(TryPopTokenizer(&c)) {
+        if (c == '\n') {
+            BackTokenzier();
+
+            break;
+        }
+
+        if (!isdigit(c)) break;
+
+        if (PeekTokenizer() == 'x' || PeekTokenizer() == 'X')
+        {
+            AdvanceTokenizer();
+
+            word[wordLength] = c;
+            word[wordLength + 1] = PeekTokenizer();
+            wordLength += 2;
+
+            continue;
+        }
         
         if (wordLength == 255) break; //Sync, error etc here at some point.
 
@@ -121,6 +140,12 @@ void TokenizerExpectWord() {
     char c;
 
     while(TryPopTokenizer(&c)) {
+        if (c == '\n') {
+            BackTokenzier();
+
+            break;
+        }
+
         if (isspace(c)) {
             break;
         }
@@ -160,6 +185,12 @@ void TokenizerExpectString(void) {
     char c;
 
     while(TryPopTokenizer(&c)) {
+        if (c == '\n') {
+            BackTokenzier();
+
+            break;
+        }
+        
         if (c == '"') break;
 
         if (c == '\\') {
@@ -201,7 +232,7 @@ void TokenizerIgnoreLine(void) {
     char c;
 
     while(TryPopTokenizer(&c)) {
-        if (c == '\n') break;
+        if (PeekTokenizer() == '\n') break;
     }
 }