From 428556a7a1a1310f1cf29bdb6917e3a62f18276d Mon Sep 17 00:00:00 2001 From: Garritt McCune Date: Thu, 26 Jun 2025 23:24:03 -0500 Subject: [PATCH] Updated the parser to be able to correctly sync via a longjmp. Fixed a few bugs in the tokenizer, like not allowing underscores in symbol / label names and taught it how to detect a label declaration. Fixed the code that gets the register name. --- examples/strlen.asm | 24 ++++++------- includes/parser.h | 22 +++++++++++- includes/token.h | 1 + includes/tokenizer.h | 2 +- src/main.c | 24 ++++--------- src/opcodes.c | 27 ++++++++++++--- src/parser.c | 9 +++-- src/token.c | 15 +++++++-- src/tokenizer.c | 80 ++++++++++++++++++++++++++++++++------------ 9 files changed, 141 insertions(+), 63 deletions(-) diff --git a/examples/strlen.asm b/examples/strlen.asm index c19e755..7982b66 100644 --- a/examples/strlen.asm +++ b/examples/strlen.asm @@ -1,22 +1,22 @@ namespace string 67; my string namespace here -namespace text "some string in the middle of nowhere" +namespace text "some string in the middle of nowhere";do you do this? word FRAMEBUFFER 0x00200500 asciiz Msg "Hello World!" ; fn effectively is an "Append to Namespace" command, ie string + "." + _start -fn _start: - mov FRAMEBUFFER, r32 ; imm, r +fn _start:;do you do this? + mov FRAMEBUFFER, r32; imm, r mov msg, r1 ; imm, r call strlen ; result into r2 mov 0, r4 ; counter loop: mov byte [r1], r3 ; read char into r3 - cmp r3, 0 ; NUL byte? - je end - mov byte r3, [r32] ; Write char to frame buffer - inc r32 ; Next space in framebuffer - inc r4 ; inc counter - inc r1 ; inc pointer to msg + cmp r3, 0; NUL byte? + jz end + mov byte r3, [r32]; Write char to frame buffer + add 1, r32 ; Next space in framebuffer + add 1, r4 ; inc counter + add 1, r1 ; inc pointer to msg jmp loop end: ;At the moment this just writes Msg to the framebuffer. ret @@ -29,9 +29,9 @@ fn strlen: loop: mov byte [r1], r3 ; mov char into r3 cmp r3, 0 ; NUL byte? - je end - inc r1 ; next char - inc r2 ; length++ + jz end + add 1, r1 ; next char + add 1, r2 ; length++ jmp loop end: ret \ No newline at end of file diff --git a/includes/parser.h b/includes/parser.h index c391eb0..211d816 100644 --- a/includes/parser.h +++ b/includes/parser.h @@ -3,15 +3,35 @@ #include #include +#include #include "dictionary.h" #include "opcodes.h" #include "token.h" #include "array.h" #include "keywords.h" -typedef struct _instruction Instruction; typedef Array Instructions; +typedef struct parameter { + bool Pointer; + TokenType Type; + union { + char* Name; + Registers Register; + uint32_t Number; + } Value; +} Parameter; + +typedef struct instruction { + bool IsOpcode; + union { + Mnemonics Opcode; + Keywords Directive; + } Inst; + int ParameterCount; + Parameter Parameters[2]; +} Instruction; + Instructions* ParseTokens(Array* tokens); #endif \ No newline at end of file diff --git a/includes/token.h b/includes/token.h index b8f19d2..dc3e6a1 100644 --- a/includes/token.h +++ b/includes/token.h @@ -16,6 +16,7 @@ typedef enum { Register, Mnemonic, Keyword, + Label, LineEnd = '\n', Colon = ':', Comma = ',', diff --git a/includes/tokenizer.h b/includes/tokenizer.h index 0d16edb..06c69cb 100644 --- a/includes/tokenizer.h +++ b/includes/tokenizer.h @@ -5,6 +5,6 @@ #include "dictionary.h" #include "token.h" -Array* Tokenize(const char* text, Dictionary** variables); +Array* Tokenize(const char* text); #endif \ No newline at end of file diff --git a/src/main.c b/src/main.c index e96fabf..555351f 100644 --- a/src/main.c +++ b/src/main.c @@ -8,8 +8,6 @@ int main(int argc, char** argv) { char* file; size_t count; - //gcc -o test src/main.c src/futil.c src/dictionary.c src/tokenizer.c src/token.c src/array.c -g - if (argc <= 1) { printf("Usage: assm \n"); @@ -22,20 +20,15 @@ int main(int argc, char** argv) { return 1; } - Dictionary* variables = DictionaryCreate(); - - Array* tokens = Tokenize(file, &variables); + Array* tokens = Tokenize(file); if (tokens->Size == 0) { printf("No tokens\n"); return 0; } -/* - char* buffer; - char type[32]; - for(int i = 0; i < tokens->Size; i++) { + for(size_t i = 0; i < tokens->Size; i++) { Token* token = tokens->Items[i]; if (token->Type == LineEnd) { @@ -44,17 +37,14 @@ int main(int argc, char** argv) { continue; } - //StringifyTokenType(token->Type, type); - buffer = TokenStringify(token); + char* str = TokenStringify(token); - if (buffer) { - printf("%s ", buffer); + printf("%s ", str); - free(buffer); - } + free(str); } printf("\n"); -*/ - (void) ParseTokens(tokens); + + //(void) ParseTokens(tokens); } \ No newline at end of file diff --git a/src/opcodes.c b/src/opcodes.c index 97f51e3..3373e03 100644 --- a/src/opcodes.c +++ b/src/opcodes.c @@ -3,7 +3,7 @@ #include #include -#define OPCODE_COUNT 29 +#define OPCODE_COUNT 30 struct _instruction { char* Name; @@ -24,6 +24,7 @@ struct _instruction Instructions[OPCODE_COUNT] = { { "shr", SHR }, { "nop", NOP }, { "cmp", CMP }, + { "jmp", JMP }, { "jz", JZ }, { "jg", JG }, { "jl", JL }, @@ -56,10 +57,26 @@ void GetMnemonicText(Mnemonics mnemonic, char buffer[12]) { void GetRegisterText(Registers reg, char buffer[5]) { memset(buffer, '\0', 5); - if (reg < R1 || reg > R32) return; - - buffer[0] = 'r'; - buffer[1] = reg + 49; + switch(reg) { + case RBP: + buffer[0] = 'r'; + buffer[1] = 'b'; + buffer[2] = 'p'; + break; + case RSP: + buffer[0] = 'r'; + buffer[1] = 's'; + buffer[2] = 'p'; + break; + case RIP: + buffer[0] = 'r'; + buffer[1] = 'i'; + buffer[2] = 'p'; + break; + default: + snprintf(buffer, 5, "r%d", reg); + break; + } } int IsOpcode(const char* text, Mnemonics* opcode) { diff --git a/src/parser.c b/src/parser.c index 8977b8a..ca54690 100644 --- a/src/parser.c +++ b/src/parser.c @@ -6,6 +6,8 @@ size_t ParserIndex = 0; const size_t MessageBufferLength = 2048 * sizeof(char); char* MessageBuffer; +jmp_buf ParserSyncPointBuffer; + Token* ParserAdvance(void); void ParserIgnoreLine(void); bool ParserAtEnd(void); @@ -20,6 +22,8 @@ Instructions* ParseTokens(Array* tokens) { Tokens = tokens; + (void) setjmp(ParserSyncPointBuffer); + while(!ParserAtEnd()) { Token* token = ParserAdvance(); @@ -43,6 +47,8 @@ void ParserSync(Token* got, char* message) { free(str); ParserIgnoreLine(); + + longjmp(ParserSyncPointBuffer, 0); } void ParserHandleKeyword(Keywords keyword) { @@ -54,9 +60,6 @@ void ParserHandleKeyword(Keywords keyword) { namespace = ParserExpect(Symbol); ParserExpectLineEnd(); - - printf("Namespace `%s` seen\n", namespace->Value.String); - break; } default: diff --git a/src/token.c b/src/token.c index df09940..8561016 100644 --- a/src/token.c +++ b/src/token.c @@ -30,10 +30,20 @@ char* TokenStringify(const struct _token* token) { return str; } else if (token->Type == Symbol) { - bufferSize = strlen(token->Value.String) + 1; + // Add three for 2 backticks and a NUL byte. + bufferSize = strlen(token->Value.String) + 3; str = calloc(bufferSize, sizeof(char)); - strncpy(str, token->Value.String, bufferSize); + snprintf(str, bufferSize, "`%s`", token->Value.String); + + return str; + } + else if (token->Type == Label) { + // Add 2 for the colon and Nul byte. + bufferSize = strlen(token->Value.String) + 2; + str = calloc(bufferSize, sizeof(char)); + + snprintf(str, bufferSize, "%s:", token->Value.String); return str; } @@ -63,6 +73,7 @@ char* TokenStringify(const struct _token* token) { strcpy(str, "(LineEnd)"); break; case FileEnd: + strcpy(str, "(FileEnd)"); break; default: sprintf(str, "%s", token->Value.Operator); diff --git a/src/tokenizer.c b/src/tokenizer.c index 09d99a8..3634df9 100644 --- a/src/tokenizer.c +++ b/src/tokenizer.c @@ -15,6 +15,7 @@ int SourceLineNumber = 1; int SourceColumnNumber = 1; char PeekTokenizer(void); +char TokenizerLookAhead(void); bool TryPopTokenizer(char* c); bool TokenizerAtEnd(void); char AdvanceTokenizer(void); @@ -23,9 +24,9 @@ bool BackTokenzier(void); void TokenizerExpectWord(void); void TokenizerExpectString(void); void TokenizerExpectNumber(void); -void TokenizerIgnoreLine(void); +void TokenizerIgnoreLine(); -Array* Tokenize(const char* sourceCode, Dictionary** variables) { +Array* Tokenize(const char* sourceCode) { Tokens = ArrayCreate(); SourceCodeLength = strlen(sourceCode); @@ -36,6 +37,16 @@ Array* Tokenize(const char* sourceCode, Dictionary** variables) { if (c == '\0') break; + if (c == ';') { + while(TryPopTokenizer(&c)) { + if (PeekTokenizer() != '\n') continue; + + break; + } + + continue; + } + if (isalpha(c)) { TokenizerExpectWord(); @@ -56,21 +67,32 @@ Array* Tokenize(const char* sourceCode, Dictionary** variables) { continue; } - if (c == ';') { - TokenizerIgnoreLine(); + if (c == '_') { + c = TokenizerLookAhead(); - continue; + if (isdigit(c) || isalpha(c) || c == '_') + TokenizerExpectWord(); + else + c = '_'; } Token* token = NULL; switch(c) { case '\n': - token = (Token*) ArrayPeek(Tokens); - //Is there a previous token and if so, was it a line break? - if (token && token->Type == LineEnd) - break; //Discard empty line. - __attribute__ ((fallthrough)); + { + Token* previousToken = (Token*) ArrayPeek(Tokens); + + if (previousToken->Type == LineEnd) break; + + Token* token = TokenCreate(LineEnd, SourceLineNumber, SourceColumnNumber, true); + + token->Value.Operator[0] = '\n'; + + ArrayAdd(Tokens, token); + + break; + } case '+': case '-': case '*': @@ -80,7 +102,7 @@ Array* Tokenize(const char* sourceCode, Dictionary** variables) { case ')': case '[': case ']': - case ':': + //case ':': case ',': token = TokenCreate((TokenType)c, SourceLineNumber, SourceColumnNumber, true); @@ -106,7 +128,7 @@ void TokenizerExpectNumber(void) { char c; while(TryPopTokenizer(&c)) { - if (c == '\n') { + if (c == '\n' || c == ';') { BackTokenzier(); break; @@ -123,7 +145,11 @@ void TokenizerExpectNumber(void) { continue; } - if (!isxdigit(c)) break; + if (!isxdigit(c)) { + BackTokenzier(); + + break; + } if (wordLength == 255) break; //Sync, error etc here at some point. @@ -144,9 +170,10 @@ void TokenizerExpectWord() { char word[256] = { 0 }; int wordLength = 0; char c; + bool isLabel = false; while(TryPopTokenizer(&c)) { - if (c == '\n') { + if (c == '\n' || c == ';') { BackTokenzier(); break; @@ -156,13 +183,14 @@ void TokenizerExpectWord() { break; } - if (c == ';') { - TokenizerIgnoreLine(); + if (c == ':') { + BackTokenzier(); + isLabel = true; break; } - if (ispunct(c)) { + if (ispunct(c) && c != '_') { BackTokenzier(); break; @@ -197,7 +225,7 @@ void TokenizerExpectWord() { token->Value.Keyword = keyword; } else { - token = TokenCreate(Symbol, SourceLineNumber, SourceColumnNumber, false); + token = TokenCreate(isLabel ? Label : Symbol, SourceLineNumber, SourceColumnNumber, false); token->Value.String = calloc(wordLength + 1, sizeof(char)); @@ -213,7 +241,7 @@ void TokenizerExpectString(void) { char c; while(TryPopTokenizer(&c)) { - if (c == '\n') { + if (c == '\n' || c == ';') { BackTokenzier(); break; @@ -256,11 +284,12 @@ void TokenizerExpectString(void) { } } -void TokenizerIgnoreLine(void) { +void TokenizerIgnoreLine() { char c; - while(TryPopTokenizer(&c)) { - if (PeekTokenizer() == '\n') break; + if (PeekTokenizer() != '\n') continue; + + break; } } @@ -268,6 +297,13 @@ char PeekTokenizer(void) { return SourceCode[SourceCodeIndex]; } +char TokenizerLookAhead(void) { + if (TokenizerAtEnd()) return '\0'; + if (SourceCodeIndex + 1 > SourceCodeLength) return '\0'; + + return SourceCode[SourceCodeIndex + 1]; +} + bool TryPopTokenizer(char* c) { *c = AdvanceTokenizer();