Made some chanegs to the tokenizer to understand hexadecimal formatted numbers, along with some more language grammar primitives.

includes/tokenizer.h

@@ -20,6 +20,7 @@ typedef enum {
     TK_Colon,
     TK_Text,
     TK_Number,
+    TK_Hex,
     TK_Invalid
 } TokenType;
 

src/tokenizer.c

@@ -8,7 +8,7 @@
 Token* GetNextToken(char *);
 TokenType GetOperatorType(char);
 char* SplitOnBasicGrammar(char*);
-int TokenIsNumeric(const char*);
+int TokenIsNumeric(const char*, int *);
 Token* CreateToken(TokenType, char*);
 
 List* TokenizeString(const char *file_path) {
@@ -33,6 +33,15 @@ List* TokenizeString(const char *file_path) {
             if(token->type == TK_Number)  {
                 printf("Found number: '%s'\n", token->value);
             }
+            else if (token->type == TK_Hex) {
+                printf("Hex number: '%s'\n", token->value);
+            }
+            else if(token->type == TK_Colon) {
+                printf("Colon found\n");
+            }
+            else if (token->type == TK_Comma) {
+                printf("Comma\n");
+            }
             else {
                 printf("Found text: '%s'\n", token->value);
             }
@@ -53,10 +62,17 @@ List* TokenizeString(const char *file_path) {
 
 Token* GetNextToken(char *string) {
     char* string_token = SplitOnBasicGrammar(string);
+    int base = 0;
     
     while (strlen(string_token) != 0) {
         
-        if (TokenIsNumeric(string_token)) return CreateToken(TK_Number, string_token);
+        if (TokenIsNumeric(string_token, &base)) {
+            if (base == 10) return CreateToken(TK_Number, string_token);
+            if (base == 16) return CreateToken(TK_Hex, string_token);
+        }
+
+        if (strcmp(string_token, ":") == 0) return CreateToken(TK_Colon, string_token);
+        if (strcmp(string_token, ",") == 0) return CreateToken(TK_Comma, string_token);
 
         return CreateToken(TK_Text, string_token);
     }
@@ -77,15 +93,31 @@ Token* CreateToken(TokenType type, char* value) {
     return token;
 }
 
-int TokenIsNumeric(const char* token) {
+int TokenIsNumeric(const char* token, int *base) {
+    *base = 0;
+
     if (!token) return 0;
     
     unsigned long length = strlen(token);
+    int i = 0;
 
     if (length == 0) return 0;
 
-    for(int i = 0; i < length; i++) {
-        if (!isdigit(token[i])) return 0;
+    *base = 10;
+
+    if (length > 2) {
+        if (token[0] == '0' && token[1] == 'x') { 
+            *base = 16;
+            i = 2;
+        }
+    }
+
+    for(; i < length; i++) {
+        if (!isdigit(token[i])) { 
+            if (*base == 16 && token[i] >= 'A' && token[i] <= 'F') continue;
+
+            return 0;
+        }
     }
 
     return 1;
@@ -116,6 +148,15 @@ char* SplitOnBasicGrammar(char* line) {
 
         if (string[position] == ';') break;
 
+        if (string[position] == ':') {
+            if (strlen(token) == 0) {
+                token[0] = ':';
+                position++;
+            }
+
+            break;
+        }
+
         if (string[position] == '"') {
             parsing_string = 1;
             i--;