From 2570c5cac0713bab01dfbde0eb364ed1d21c10d9 Mon Sep 17 00:00:00 2001
From: Garritt McCune <glmccune@gmail.com>
Date: Sun, 23 Jan 2022 21:54:06 -0600
Subject: [PATCH] Made some chanegs to the tokenizer to understand hexadecimal
 formatted numbers, along with some more language grammar primitives.

---
 includes/tokenizer.h |  1 +
 src/tokenizer.c      | 51 +++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 47 insertions(+), 5 deletions(-)

diff --git a/includes/tokenizer.h b/includes/tokenizer.h
index 02ff6a8..4e3887e 100644
--- a/includes/tokenizer.h
+++ b/includes/tokenizer.h
@@ -20,6 +20,7 @@ typedef enum {
     TK_Colon,
     TK_Text,
     TK_Number,
+    TK_Hex,
     TK_Invalid
 } TokenType;
 
diff --git a/src/tokenizer.c b/src/tokenizer.c
index e35339c..1f4d4c9 100644
--- a/src/tokenizer.c
+++ b/src/tokenizer.c
@@ -8,7 +8,7 @@
 Token* GetNextToken(char *);
 TokenType GetOperatorType(char);
 char* SplitOnBasicGrammar(char*);
-int TokenIsNumeric(const char*);
+int TokenIsNumeric(const char*, int *);
 Token* CreateToken(TokenType, char*);
 
 List* TokenizeString(const char *file_path) {
@@ -33,6 +33,15 @@ List* TokenizeString(const char *file_path) {
             if(token->type == TK_Number)  {
                 printf("Found number: '%s'\n", token->value);
             }
+            else if (token->type == TK_Hex) {
+                printf("Hex number: '%s'\n", token->value);
+            }
+            else if(token->type == TK_Colon) {
+                printf("Colon found\n");
+            }
+            else if (token->type == TK_Comma) {
+                printf("Comma\n");
+            }
             else {
                 printf("Found text: '%s'\n", token->value);
             }
@@ -53,10 +62,17 @@ List* TokenizeString(const char *file_path) {
 
 Token* GetNextToken(char *string) {
     char* string_token = SplitOnBasicGrammar(string);
+    int base = 0;
     
     while (strlen(string_token) != 0) {
         
-        if (TokenIsNumeric(string_token)) return CreateToken(TK_Number, string_token);
+        if (TokenIsNumeric(string_token, &base)) {
+            if (base == 10) return CreateToken(TK_Number, string_token);
+            if (base == 16) return CreateToken(TK_Hex, string_token);
+        }
+
+        if (strcmp(string_token, ":") == 0) return CreateToken(TK_Colon, string_token);
+        if (strcmp(string_token, ",") == 0) return CreateToken(TK_Comma, string_token);
 
         return CreateToken(TK_Text, string_token);
     }
@@ -77,15 +93,31 @@ Token* CreateToken(TokenType type, char* value) {
     return token;
 }
 
-int TokenIsNumeric(const char* token) {
+int TokenIsNumeric(const char* token, int *base) {
+    *base = 0;
+
     if (!token) return 0;
     
     unsigned long length = strlen(token);
+    int i = 0;
 
     if (length == 0) return 0;
 
-    for(int i = 0; i < length; i++) {
-        if (!isdigit(token[i])) return 0;
+    *base = 10;
+
+    if (length > 2) {
+        if (token[0] == '0' && token[1] == 'x') { 
+            *base = 16;
+            i = 2;
+        }
+    }
+
+    for(; i < length; i++) {
+        if (!isdigit(token[i])) { 
+            if (*base == 16 && token[i] >= 'A' && token[i] <= 'F') continue;
+
+            return 0;
+        }
     }
 
     return 1;
@@ -116,6 +148,15 @@ char* SplitOnBasicGrammar(char* line) {
 
         if (string[position] == ';') break;
 
+        if (string[position] == ':') {
+            if (strlen(token) == 0) {
+                token[0] = ':';
+                position++;
+            }
+
+            break;
+        }
+
         if (string[position] == '"') {
             parsing_string = 1;
             i--;