glm94/clox
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

The Scanner has been refactored and some responsibilities were pulled from it. Confirmed working, the Parser still needs to be touched on however.

Browse Source
This commit is contained in:
glm94
2022-03-02 20:52:40 +00:00
parent 1b33468343
commit e20ccb8cfb
7 changed files with 238 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scanner.c
+71 -89
View File
@@ -1,32 +1,5 @@
#include "scanner.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
typedef struct {
char* keyword;
TokenType type;
} KeywordPair;
KeywordPair keywords[KEYWORD_COUNT] = {
{ "and", AND },
{ "class", CLASS },
{ "else", ELSE },
{ "false", FALSE },
{ "for", FOR },
{ "fun", FUN },
{ "if", IF },
{ "nil", NIL },
{ "or", OR },
{ "print", PRINT },
{ "return", RETURN },
{ "super", SUPER },
{ "this", THIS },
{ "true", TRUE },
{ "var", VAR },
{ "while", WHILE }
};
#include "token.h"
const char* source_code;
//Start and Current hold the offsets that index into the string source_code.
@@ -39,7 +12,7 @@ const char* SAdvance(void);
int SIsAtEnd(void);
void ScanToken(TokenList*);
TokenList* CreateList(void);
int AddTokenToList(TokenType, const char*, int, TokenList*);
int AddToTokenList(Token*, TokenList*);
int SMatch(char);
char SPeek(void);
char PeekNext(void);
@@ -47,7 +20,7 @@ void ParseString(TokenList *);
void ParseNumber(TokenList *);
void ParseIdentifier(TokenList *);
int IsAlpha(char c);
KeywordPair* Get(char*);
KeyValuePair* Get(const char*);
TokenList* ScanTokens(const char* source) {
if (!source) return NULL;
@@ -64,8 +37,7 @@ TokenList* ScanTokens(const char* source) {
ScanToken(tokens);
}
//Add EOF token and return list once that's set up.
AddTokenToList(EndOF, NULL, 0, tokens);
AddToTokenList(CreateToken(NULL, line, EndOF), tokens);
return tokens;
}
@@ -96,7 +68,10 @@ void DestroyTokenList(TokenList* list) {
if (!list) return;
for(int i = 0; i < list->size; i++) {
//free(list->tokens[i]->lexeme); This shouldn't be needed since the lexeme is a pointer into the source code.
//Only free objects that required allocation, i.e. not in the TokenTypeMappings
if (list->tokens[i]->type == String || list->tokens[i]->type == Number || list->tokens[i]->type == Identifier)
free((void *) list->tokens[i]->lexeme);
free(list->tokens[i]);
}
@@ -104,34 +79,24 @@ void DestroyTokenList(TokenList* list) {
free(list);
}
int AddTokenToList(TokenType type, const char* lexeme, int length, TokenList* tokens) {
if (!tokens) return 0;
Token* token = calloc(1, sizeof(Token));
int AddToTokenList(Token* token, TokenList* list) {
if (!list) return 0;
if (!token) return 0;
if (!token) {
fprintf(stderr, "Failed to calloc memory for new Token.\n");
return 0;
}
token->lexeme = lexeme;
token->type = type;
token->length = length; //Set the length of the lexeme (which is just a pointer into the complete source listing).
token->line = line;
if ((tokens->size + 1) > tokens->capacity) {
void* new_ptr = realloc(tokens->tokens, sizeof(Token*) * tokens->capacity * 2);
if ((list->size + 1) > list->capacity) {
void* new_ptr = realloc(list->tokens, sizeof(Token*) * list->capacity * 2);
if (!new_ptr) {
fprintf(stderr, "Failed to realloc TokenList to size %d.\n", tokens->capacity * 2);
fprintf(stderr, "Failed to realloc TokenList to size %d.\n", list->capacity * 2);
return 0;
}
tokens->tokens = new_ptr;
tokens->capacity = tokens->capacity * 2;
list->tokens = new_ptr;
list->capacity = list->capacity * 2;
}
tokens->tokens[tokens->size] = token;
tokens->size++;
list->tokens[list->size] = token;
list->size++;
return 1;
}
@@ -141,58 +106,55 @@ void ScanToken(TokenList* tokens) {
switch (*c) {
case '(':
AddTokenToList(LParen, c, 1, tokens);
AddToTokenList(CreateToken(NULL, line, LParen), tokens);
break;
case ')':
AddTokenToList(RParen, c, 1, tokens);
AddToTokenList(CreateToken(NULL, line, LParen), tokens);
break;
case '{':
AddTokenToList(LBrace, c, 1, tokens);
AddToTokenList(CreateToken(NULL, line, LParen), tokens);
break;
case '}':
AddTokenToList(RBrace, c, 1, tokens);
AddToTokenList(CreateToken(NULL, line, RParen), tokens);
break;
case ',':
AddTokenToList(Comma, c, 1, tokens);
AddToTokenList(CreateToken(NULL, line, Comma), tokens);
break;
case '.':
AddTokenToList(Dot, c, 1, tokens);
AddToTokenList(CreateToken(NULL, line, Dot), tokens);
break;
case '-':
AddTokenToList(Minus, c, 1, tokens);
AddToTokenList(CreateToken(NULL, line, Minus), tokens);
break;
case '+':
AddTokenToList(Plus, c, 1, tokens);
AddToTokenList(CreateToken(NULL, line, Plus), tokens);
break;
case ';':
AddTokenToList(Semicolon, c, 1, tokens);
AddToTokenList(CreateToken(NULL, line, Semicolon), tokens);
break;
case '*':
AddTokenToList(Star, c, 1, tokens);
AddToTokenList(CreateToken(NULL, line, Star), tokens);
break;
case '!':
if (SMatch('=')) AddTokenToList(Bang_Equal, "!=", 2, tokens);
else AddTokenToList(Bang, c, 1, tokens);
if (SMatch('=')) AddToTokenList(CreateToken(NULL, line, Bang_Equal), tokens);
else AddToTokenList(CreateToken(NULL, line, Bang), tokens);
break;
case '=':
if (SMatch('=')) AddTokenToList(Equal_Equal, "==", 2, tokens);
else AddTokenToList(Equal, c, 1, tokens);
if (SMatch('=')) AddToTokenList(CreateToken(NULL, line, Equal_Equal), tokens);
else AddToTokenList(CreateToken(NULL, line, Equal), tokens);
break;
case '<':
if (SMatch('=')) AddTokenToList(Less_Equal, "<=", 2, tokens);
else AddTokenToList(Less, c, 1, tokens);
if (SMatch('=')) AddToTokenList(CreateToken(NULL, line, Less_Equal), tokens);
else AddToTokenList(CreateToken(NULL, line, Less), tokens);
break;
case '>':
if (SMatch('=')) AddTokenToList(Greater_Equal, ">=", 2, tokens);
else AddTokenToList(Greater, c, 1, tokens);
if (SMatch('=')) AddToTokenList(CreateToken(NULL, line, Greater_Equal), tokens);
else AddToTokenList(CreateToken(NULL, line, Greater), tokens);
break;
case '/':
if (SMatch('/')) {
while(SPeek() != '\n' && !SIsAtEnd()) SAdvance();
}
else {
AddTokenToList(Slash, c, 1, tokens);
}
if (SMatch('/')) while(SPeek() != '\n' && !SIsAtEnd()) { SAdvance(); }
else AddToTokenList(CreateToken(NULL, line, Slash), tokens);
break;
case '"':
ParseString(tokens);
@@ -254,7 +216,16 @@ void ParseString(TokenList *list) {
return;
}
AddTokenToList(String, &source_code[start], current - start, list);
char* lexeme = calloc(current - start + 1, sizeof(char));
if (!lexeme) {
fprintf(stderr, "Failed to calloc for string lexeme. %s\n", strerror(errno));
return;
}
snprintf(lexeme, current - start, "%s", &source_code[start]);
AddToTokenList(CreateToken(lexeme, line, String), list);
SAdvance(); // The closing ".
}
@@ -268,7 +239,16 @@ void ParseNumber(TokenList* list) {
while(isdigit(SPeek())) SAdvance();
}
AddTokenToList(Number, &source_code[start], current - start, list);
char* lexeme = calloc(current - start + 2, sizeof(char));
if (!lexeme) {
fprintf(stderr, "Failed to calloc for number lexeme. %s\n", strerror(errno));
return;
}
snprintf(lexeme, current - start + 1, "%s", &source_code[start]);
AddToTokenList(CreateToken(lexeme, line, Number), list);
}
char PeekNext() {
@@ -280,16 +260,18 @@ char PeekNext() {
void ParseIdentifier(TokenList * list) {
while(IsAlpha(SPeek())) SAdvance();
char* lexeme = calloc(sizeof(char*), (current - start + 1));
char* lexeme = calloc(current - start + 2, sizeof(char));
snprintf(lexeme, current - start + 1, "%s", &source_code[start]);
printf("%s\n", lexeme);
KeyValuePair* result = Get(lexeme);
KeywordPair* result = Get(lexeme);
if (result) {
free(lexeme);
if (result) AddTokenToList(result->type, &source_code[start], current - start, list);
else AddTokenToList(Identifier, &source_code[start], current - start, list);
free(lexeme);
AddToTokenList(CreateToken(NULL, line, result->type), list);
}
else AddToTokenList(CreateToken(lexeme, line, Identifier), list);
}
int IsAlpha(char c) {
@@ -298,11 +280,11 @@ int IsAlpha(char c) {
(c == '_');
}
KeywordPair* Get(char* text) {
KeyValuePair* Get(const char* text) {
if (!text) return NULL;
for (int i = 0; i < KEYWORD_COUNT; i++)
if (strcmp(text, keywords[i].keyword) == 0) return &keywords[i];
for (int i = 0; i < TOKENTYPE_MAPPINGS_COUNT; i++)
if (strcmp(text, TokenTypeMappings[i].lexeme) == 0) return &TokenTypeMappings[i];
return NULL;
}