diff --git a/src/main.c b/src/main.c index 5683ba1..570fa53 100644 --- a/src/main.c +++ b/src/main.c @@ -29,7 +29,6 @@ int main(int argc, char** argv) { return 0; } - else printf("Found %d tokens.", tokens->Size); for(int i = 0; i < tokens->Size; i++) { Token* token = tokens->Items[i]; diff --git a/src/token.c b/src/token.c index 2c4faba..307dac1 100644 --- a/src/token.c +++ b/src/token.c @@ -13,43 +13,29 @@ struct _token* TokenCreate(TokenType type, int lineNumber, int columnNumber, boo char* TokenStringify(const struct _token* token, bool valueOnly) { if (!token) return NULL; - char type[32]; - char number[32]; - - int length = StringifyTokenType(token->Type, type); - - char* result = calloc(512, sizeof(char)); - #ifdef DEBUG - - #endif + char buffer[32]; switch(token->Type) { case Empty: - snprintf(result, length + 1, "%s ", type); - break; + return "<>"; case String: - snprintf(result, 511, "%s \"%s\" ", type, token->Value.String); - break; - case Number: - snprintf(number, 32, "%d", token->Value.Number); - snprintf(result, 511, "%s %s", type, number); - break; case Symbol: - snprintf(result, 511, "%s %s", type, token->Value.String); + return token->Value.String; + case Number: + sprintf(buffer, "%d", token->Value.Number); + break; case LineEnd: - snprintf(result, 511, "%s\n", type); - break; + return "\n"; case FileEnd: - snprintf(result, length + 1, "%s ", type); - break; + return NULL; default: - snprintf(result, length + 2, "%s %c", type, token->Value.Operator[0]); - //snprintf(buffer, length, "<%c>", type); + strncpy(buffer, token->Value.Operator, 8); + break; } - return result; + return buffer; } int StringifyTokenType(TokenType type, char buffer[32]) { diff --git a/src/tokenizer.c b/src/tokenizer.c index bfa434f..3e5d507 100644 --- a/src/tokenizer.c +++ b/src/tokenizer.c @@ -75,6 +75,7 @@ Array* Tokenize(const char* sourceCode, Dictionary** variables) { case ')': case '[': case ']': + case ':': Token* token = TokenCreate((TokenType)c, SourceLineNumber, SourceColumnNumber, true); token->Value.Operator[0] = c; @@ -99,6 +100,24 @@ void TokenizerExpectNumber(void) { char c; while(TryPopTokenizer(&c)) { + if (c == '\n') { + BackTokenzier(); + + break; + } + + if (!isdigit(c)) break; + + if (PeekTokenizer() == 'x' || PeekTokenizer() == 'X') + { + AdvanceTokenizer(); + + word[wordLength] = c; + word[wordLength + 1] = PeekTokenizer(); + wordLength += 2; + + continue; + } if (wordLength == 255) break; //Sync, error etc here at some point. @@ -121,6 +140,12 @@ void TokenizerExpectWord() { char c; while(TryPopTokenizer(&c)) { + if (c == '\n') { + BackTokenzier(); + + break; + } + if (isspace(c)) { break; } @@ -160,6 +185,12 @@ void TokenizerExpectString(void) { char c; while(TryPopTokenizer(&c)) { + if (c == '\n') { + BackTokenzier(); + + break; + } + if (c == '"') break; if (c == '\\') { @@ -201,7 +232,7 @@ void TokenizerIgnoreLine(void) { char c; while(TryPopTokenizer(&c)) { - if (c == '\n') break; + if (PeekTokenizer() == '\n') break; } }