Created a tokenizer and the supporting code, currently processes text that is not a string literal.

2025-01-07 23:52:03 -06:00
parent 2c85077031
commit 7e70c6fd19
10 changed files with 351 additions and 12 deletions
@@ -0,0 +1,85 @@
+#include "../includes/token.h"
+
+struct _token* TokenCreate(TokenType type, int lineNumber, int columnNumber, bool resolved) {
+    struct _token* token = calloc(1, sizeof(Token));
+
+    token->Type = type;
+    token->LineNumber = lineNumber;
+    token->ColumnNumber = columnNumber;
+    token->Resolved = resolved;
+
+    return token;
+}
+
+char* TokenStringify(const struct _token* token, bool valueOnly) {
+    if (!token) return NULL;
+    char type[32];
+    char number[32];
+
+    int length = StringifyTokenType(token->Type, type);
+
+    char* result = calloc(512, sizeof(char));
+    #ifdef DEBUG
+    
+    #endif
+
+    switch(token->Type) {
+        case Empty:
+            snprintf(result, length + 1, "%s ", type);
+            break;
+        case String:
+            snprintf(result, 511, "%s \"%s\" ", type, token->Value.String);
+            break;
+        case Number:
+            snprintf(number, 32, "%d", token->Value.Number);
+            snprintf(result, 511, "%s %s", type, number);
+            break;
+        case Symbol:
+            snprintf(result, 511, "%s %s", type, token->Value.String);
+            break;
+        case LineEnd:
+            snprintf(result, 511, "%s\n", type);
+            break;
+        case FileEnd:
+            snprintf(result, length + 1, "%s ", type);
+            break;
+        default:
+            //snprintf(buffer, length, "<%c>", type);
+            break;
+    }
+
+    return result;
+}
+
+int StringifyTokenType(TokenType type, char buffer[32]) {
+    if (!buffer) return 0;
+    
+    memset(buffer, '\0', 32);
+
+    switch(type) {
+        case Empty:
+            strncpy(buffer, "<empty >", 8);
+            break;
+        case String:
+            strncpy(buffer, "<string>", 8);
+            break;
+        case Number:
+            strncpy(buffer, "<number>", 8);
+            break;
+        case Symbol:
+            strncpy(buffer, "<symbol>", 8);
+            break;
+        case LineEnd:
+            strncpy(buffer, "<LnEnd >", 8);
+            break;
+        case FileEnd:
+            strncpy(buffer, "<filend>", 8);
+            break;
+        default:
+            snprintf(buffer, 8, "<%c    >", type);
+
+            break;
+    }
+
+    return 8;
+}