Fixed a bug where the tokenizer wasn't parsing hexadecimal numbers correctly. Added support for tokenizing the colon and comma characters.

This commit is contained in:
2025-01-08 23:22:43 -06:00
parent 428d674f54
commit ebfd7cf94c
6 changed files with 86 additions and 21 deletions
+15 -1
View File
@@ -30,10 +30,24 @@ int main(int argc, char** argv) {
return 0;
}
char* buffer;
for(int i = 0; i < tokens->Size; i++) {
Token* token = tokens->Items[i];
printf("%s ", TokenStringify(token, false));
if (token->Type == LineEnd) {
printf("\n");
continue;
}
buffer = TokenStringify(token);
if (buffer) {
printf("%s ", buffer);
free(buffer);
}
}
printf("\n");