Added support for number literals and punctuation.

.gitignore

@@ -1,2 +1,4 @@
+bin/
+obj/
 .vscode/
 tags

src/main.c

@@ -34,7 +34,7 @@ int main(int argc, char** argv) {
     for(int i = 0; i < tokens->Size; i++) {
         Token* token = tokens->Items[i];
 
-        printf("%s ", token->Value.String);//TokenStringify(token, false));
+        printf("%s ", TokenStringify(token, false));
     }
 
     printf("\n");

src/token.c

@@ -44,6 +44,7 @@ char* TokenStringify(const struct _token* token, bool valueOnly) {
             snprintf(result, length + 1, "%s ", type);
             break;
         default:
+            snprintf(result, length + 2, "%s %c", type, token->Value.Operator[0]);
             //snprintf(buffer, length, "<%c>", type);
             break;
     }

src/tokenizer.c

@@ -13,8 +13,8 @@ Array* Tokens;
 const char* SourceCode;
 size_t SourceCodeLength = 0;
 size_t SourceCodeIndex = 0;
-int SourceLineNumber = 0;
+int SourceLineNumber = 1;
-int SourceColumnNumber = 0;
+int SourceColumnNumber = 1;
 
 char PeekTokenizer(void);
 bool TryPopTokenizer(char* c);
@@ -24,6 +24,7 @@ bool BackTokenzier(void);
 
 void TokenizerExpectWord(void);
 void TokenizerExpectString(void);
+void TokenizerExpectNumber(void);
 void TokenizerIgnoreLine(void);
 
 Array* Tokenize(const char* sourceCode, Dictionary** variables) {
@@ -43,6 +44,12 @@ Array* Tokenize(const char* sourceCode, Dictionary** variables) {
             continue;
         }
 
+        if (isdigit(c)) {
+            TokenizerExpectNumber();
+
+            continue;
+        }
+
         if (c == '"') {
             AdvanceTokenizer();
 
@@ -57,6 +64,27 @@ Array* Tokenize(const char* sourceCode, Dictionary** variables) {
             continue;
         }
 
+        switch(c) {
+            case '\n':
+            case '+':
+            case '-':
+            case '*':
+            case '/':
+            case '^':
+            case '(':
+            case ')':
+            case '[':
+            case ']':
+                Token* token = TokenCreate((TokenType)c, SourceLineNumber, SourceColumnNumber, true);
+
+                token->Value.Operator[0] = c;
+
+                ArrayAdd(Tokens, token);
+                break;
+            default:
+                break;
+        }
+
         AdvanceTokenizer();
     }
 
@@ -65,6 +93,28 @@ Array* Tokenize(const char* sourceCode, Dictionary** variables) {
     return Tokens;
 }
 
+void TokenizerExpectNumber(void) {
+    char word[256] = { 0 };
+    int wordLength = 0;
+    char c;
+
+    while(TryPopTokenizer(&c)) {
+        
+        if (wordLength == 255) break; //Sync, error etc here at some point.
+
+        word[wordLength] = c;
+        wordLength++;
+    }
+
+    if (wordLength > 0) {
+        Token* token = TokenCreate(Number, SourceLineNumber, SourceColumnNumber, false);
+
+        token->Value.Number = (int) strtol(word, NULL, 0);
+
+        ArrayAdd(Tokens, token);
+    }
+}
+
 void TokenizerExpectWord() {
     char word[256] = { 0 };
     int wordLength = 0;
@@ -171,6 +221,13 @@ char AdvanceTokenizer() {
 
     char c = SourceCode[SourceCodeIndex];
 
+    if (c == '\n') 
+    {
+        SourceColumnNumber = 1;
+        SourceLineNumber++;
+    }
+
+    SourceColumnNumber++;
     SourceCodeIndex++;
 
     return c;
@@ -179,6 +236,7 @@ char AdvanceTokenizer() {
 bool BackTokenzier(void) {
     if (SourceCodeIndex == 0) return false;
 
+    SourceColumnNumber--;
     SourceCodeIndex--;
 
     return true;