#include "scanner.h" #include "token.h" const char* source_code; //Start and Current hold the offsets that index into the string source_code. int start; //Points to the first character in the lexeme being scanned. int current; //points to the character currently being considered. int length; int line = 1; const char* AdvanceScanner(void); int ScannerAtEnd(void); void ScanToken(TokenList*); TokenList* CreateList(void); int AddToTokenList(Token*, TokenList*); int ScannerMatch(char); char ScannerPeek(void); char PeekNext(void); void ParseString(TokenList *); void ParseNumber(TokenList *); void ParseIdentifier(TokenList *); int IsAlpha(char c); KeyValuePair* Get(const char*); TokenList* ScanTokens(const char* source) { if (!source) return NULL; source_code = source; length = strlen(source); if (length == 0) return NULL; TokenList* tokens = CreateList(); while(!ScannerAtEnd()) { start = current; ScanToken(tokens); } AddToTokenList(CreateToken(NULL, line, EndOF), tokens); return tokens; } TokenList* CreateList() { TokenList* list = calloc(1, sizeof(TokenList)); if (!list) { fprintf(stderr, "Failed to calloc TokenList.\n"); return NULL; } list->tokens = calloc(DEFAULT_TOKENLIST_SIZE, sizeof(Token*)); if (!list->tokens) { free(list); fprintf(stderr, "Failed to calloc tokens.\n"); return NULL; } list->capacity = DEFAULT_TOKENLIST_SIZE; list->size = 0; return list; } void DestroyTokenList(TokenList* list) { if (!list) return; for(int i = 0; i < list->size; i++) { FreeToken(list->tokens[i]); } free(list->tokens); free(list); } int AddToTokenList(Token* token, TokenList* list) { if (!list) return 0; if (!token) return 0; if ((list->size + 1) > list->capacity) { void* new_ptr = realloc(list->tokens, sizeof(Token*) * list->capacity * 2); if (!new_ptr) { fprintf(stderr, "Failed to realloc TokenList to size %d.\n", list->capacity * 2); return 0; } list->tokens = new_ptr; list->capacity = list->capacity * 2; } list->tokens[list->size] = token; list->size++; return 1; } void ScanToken(TokenList* tokens) { const char* c = AdvanceScanner(); switch (*c) { case '(': AddToTokenList(CreateToken(NULL, line, LParen), tokens); break; case ')': AddToTokenList(CreateToken(NULL, line, LParen), tokens); break; case '{': AddToTokenList(CreateToken(NULL, line, LParen), tokens); break; case '}': AddToTokenList(CreateToken(NULL, line, RParen), tokens); break; case ',': AddToTokenList(CreateToken(NULL, line, Comma), tokens); break; case '.': AddToTokenList(CreateToken(NULL, line, Dot), tokens); break; case '-': AddToTokenList(CreateToken(NULL, line, Minus), tokens); break; case '+': AddToTokenList(CreateToken(NULL, line, Plus), tokens); break; case ';': AddToTokenList(CreateToken(NULL, line, Semicolon), tokens); break; case '*': AddToTokenList(CreateToken(NULL, line, Star), tokens); break; case '!': if (ScannerMatch('=')) AddToTokenList(CreateToken(NULL, line, Bang_Equal), tokens); else AddToTokenList(CreateToken(NULL, line, Bang), tokens); break; case '=': if (ScannerMatch('=')) AddToTokenList(CreateToken(NULL, line, Equal_Equal), tokens); else AddToTokenList(CreateToken(NULL, line, Equal), tokens); break; case '<': if (ScannerMatch('=')) AddToTokenList(CreateToken(NULL, line, Less_Equal), tokens); else AddToTokenList(CreateToken(NULL, line, Less), tokens); break; case '>': if (ScannerMatch('=')) AddToTokenList(CreateToken(NULL, line, Greater_Equal), tokens); else AddToTokenList(CreateToken(NULL, line, Greater), tokens); break; case '/': if (ScannerMatch('/')) while(ScannerPeek() != '\n' && !ScannerAtEnd()) { AdvanceScanner(); } else AddToTokenList(CreateToken(NULL, line, Slash), tokens); break; case '"': ParseString(tokens); break; case ' ': case '\r': case '\t': break; //Ignore whitespace case '\n': line++; break; default: if (isdigit(*c)) { ParseNumber(tokens); break; } else if (IsAlpha(*c)) { ParseIdentifier(tokens); break; } fprintf(stderr, "Unexcpedted character %c\n", *c); break; } } int ScannerAtEnd() { return current >= length; } const char* AdvanceScanner() { return &source_code[current++]; } int ScannerMatch(char expected) { if (ScannerAtEnd()) return 0; if (source_code[current] != expected) return 0; current++; return 1; } char ScannerPeek() { if (ScannerAtEnd()) return '\0'; return source_code[current]; } void ParseString(TokenList *list) { start = current; //The start is currently pointing to the first double quote so we need to move it //to the next (first character) of the string literal. while(ScannerPeek() != '"' && !ScannerAtEnd()) { if (ScannerPeek() == '\n') line++; AdvanceScanner(); } if (ScannerAtEnd()) { fprintf(stderr, "Unterminated string.\n"); return; } char* lexeme = calloc(current - start + 1, sizeof(char)); if (!lexeme) { fprintf(stderr, "Failed to calloc for string lexeme. %s\n", strerror(errno)); return; } snprintf(lexeme, current - start, "%s", &source_code[start]); AddToTokenList(CreateToken(lexeme, line, String), list); AdvanceScanner(); // The closing ". } void ParseNumber(TokenList* list) { while(isdigit(ScannerPeek())) AdvanceScanner(); if (ScannerPeek() == '.' && isdigit(PeekNext())) { AdvanceScanner(); while(isdigit(ScannerPeek())) AdvanceScanner(); } char* lexeme = calloc(current - start + 2, sizeof(char)); if (!lexeme) { fprintf(stderr, "Failed to calloc for number lexeme. %s\n", strerror(errno)); return; } snprintf(lexeme, current - start + 1, "%s", &source_code[start]); AddToTokenList(CreateToken(lexeme, line, Number), list); } char PeekNext() { if (current + 1 >= length) return '\0'; return source_code[current + 1]; } void ParseIdentifier(TokenList * list) { while(IsAlpha(ScannerPeek())) AdvanceScanner(); char* lexeme = calloc(current - start + 2, sizeof(char)); snprintf(lexeme, current - start + 1, "%s", &source_code[start]); KeyValuePair* result = Get(lexeme); if (result) { free(lexeme); AddToTokenList(CreateToken(NULL, line, result->type), list); } else AddToTokenList(CreateToken(lexeme, line, Identifier), list); } int IsAlpha(char c) { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c == '_'); } KeyValuePair* Get(const char* text) { if (!text) return NULL; for (int i = 0; i < TOKENTYPE_MAPPINGS_COUNT; i++) if (strcmp(text, TokenTypeMappings[i].lexeme) == 0) return &TokenTypeMappings[i]; return NULL; }