glm94/clox
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

The Scanner has been refactored and some responsibilities were pulled from it. Confirmed working, the Parser still needs to be touched on however.

Browse Source
This commit is contained in:
glm94
2022-03-02 20:52:40 +00:00
parent 1b33468343
commit e20ccb8cfb
7 changed files with 238 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lox.c
+2 -8
View File
@@ -21,19 +21,13 @@ int main(int argc, char** argv) {
} }
void Print(TokenList* list) { void Print(TokenList* list) {
char lexeme[100];
for(int i = 0; i < list->size; i++) { for(int i = 0; i < list->size; i++) {
if (list->tokens[i]->type == EndOF) { if (list->tokens[i]->type == EndOF) {
printf("[Line %d] EOF\n", list->tokens[i]->line);\ printf("[Line %d] EOF\n", list->tokens[i]->line);\
return; return;
} }
printf("[Line %d] ", list->tokens[i]->line); printf("[Line %d] %s\n", list->tokens[i]->line, list->tokens[i]->lexeme);
for(int j = 0; j < list->tokens[i]->length; j++) {
printf("%c", list->tokens[i]->lexeme[j]);
}
printf("\n");
} }
} }
@@ -41,10 +35,10 @@ void RunFile(const char* path) {
printf("Running '%s'\n", path); printf("Running '%s'\n", path);
char* contents = GetFileContents(path); char* contents = GetFileContents(path);
TokenList* tokens = ScanTokens(contents); TokenList* tokens = ScanTokens(contents);
free(contents);
Print(tokens); Print(tokens);
DestroyTokenList(tokens); DestroyTokenList(tokens);
free(contents);
} }
char* GetFileContents(const char* path) { char* GetFileContents(const char* path) {
parser.c
+47 -46
View File
@@ -1,9 +1,4 @@
#include "parser.h" #include "parser.h"
#include "expr.h"
#include "scanner.h"
#include <stdarg.h>
#include <stdlib.h>
#include <string.h>
Expr* Expression(void); Expr* Expression(void);
Expr* Equality(void); Expr* Equality(void);
@@ -12,17 +7,23 @@ Expr* Term(void);
Expr* Factor(void); Expr* Factor(void);
Expr* Unary(void); Expr* Unary(void);
Expr* Primary(void); Expr* Primary(void);
int Match(int, ...); int SMatch(int, ...);
int Check(TokenType); int Check(TokenType);
int IsAtEnd(void); int SIsAtEnd(void);
Token* Peek(void); Token* SPeek(void);
Token* Previous(void); Token* Previous(void);
Token* Advance(void); Token* SAdvance(void);
Token* CreateToken(char*, TokenType); //Token* CreateToken(char*, TokenType);
const TokenList* tokens; const TokenList* ListOfTokens;
int Current = 0; int Current = 0;
Expr* GenerateExpressionTree(const TokenList* list) {
ListOfTokens = list;
return Expression();
}
//Simply expands the equality rule //Simply expands the equality rule
Expr* Expression() { Expr* Expression() {
return Equality(); return Equality();
@@ -31,7 +32,7 @@ Expr* Expression() {
Expr* Equality() { Expr* Equality() {
Expr* expr = Comparison(); Expr* expr = Comparison();
while(Match(2, Bang_Equal, Equal_Equal)) { while(SMatch(2, Bang_Equal, Equal_Equal)) {
Token* operator = Previous(); Token* operator = Previous();
Expr* right = Comparison(); Expr* right = Comparison();
Expr* temp = calloc(1, sizeof(Expr)); Expr* temp = calloc(1, sizeof(Expr));
@@ -48,7 +49,7 @@ Expr* Equality() {
Expr* Comparison() { Expr* Comparison() {
Expr* expr = Term(); Expr* expr = Term();
while(Match(4, Greater, Greater_Equal, Less, Less_Equal)) { while(SMatch(4, Greater, Greater_Equal, Less, Less_Equal)) {
Token* operator = Previous(); Token* operator = Previous();
Expr* right = Term(); Expr* right = Term();
Expr* temp = calloc(1, sizeof(Expr)); Expr* temp = calloc(1, sizeof(Expr));
@@ -65,7 +66,7 @@ Expr* Comparison() {
Expr* Term() { Expr* Term() {
Expr* expr = Factor(); Expr* expr = Factor();
while(Match(2, Minus, Plus)) { while(SMatch(2, Minus, Plus)) {
Token* operator = Previous(); Token* operator = Previous();
Expr* right = Factor(); Expr* right = Factor();
Expr* temp = calloc(1, sizeof(Expr)); Expr* temp = calloc(1, sizeof(Expr));
@@ -82,7 +83,7 @@ Expr* Term() {
Expr* Factor() { Expr* Factor() {
Expr* expr = Unary(); Expr* expr = Unary();
while(Match(2, Slash, Star)) { while(SMatch(2, Slash, Star)) {
Token* operator = Previous(); Token* operator = Previous();
Expr* right = Unary(); Expr* right = Unary();
Expr* temp = calloc(1, sizeof(Expr)); Expr* temp = calloc(1, sizeof(Expr));
@@ -97,7 +98,7 @@ Expr* Factor() {
} }
Expr* Unary() { Expr* Unary() {
if (Match(2, Bang, Minus)) { if (SMatch(2, Bang, Minus)) {
Token* operator = Previous(); Token* operator = Previous();
Expr* right = Unary(); Expr* right = Unary();
Expr* expr = calloc(1, sizeof(Expr)); Expr* expr = calloc(1, sizeof(Expr));
@@ -114,25 +115,25 @@ Expr* Primary() {
Expr* expr = calloc(1, sizeof(Expr)); Expr* expr = calloc(1, sizeof(Expr));
expr->type = LITERAL; expr->type = LITERAL;
if (Match(1, FALSE)) { if (SMatch(3, FALSE, TRUE, NIL)) {
expr->expression.Literal.type = CreateToken("false", FALSE); expr->expression.Literal.type = SPeek();//CreateToken("false", FALSE);
return expr;
}
if (Match(1, TRUE)) {
expr->expression.Literal.type = CreateToken("true", TRUE);
return expr;
}
if (Match(1, NIL)) {
expr->expression.Literal.type = CreateToken("nil", NIL);
return expr; return expr;
} }
// if (SMatch(1, TRUE)) {
// expr->expression.Literal.type = //CreateToken("true", TRUE);
// return expr;
// }
// if (SMatch(1, NIL)) {
// expr->expression.Literal.type = //CreateToken("nil", NIL);
// return expr;
// }
if (Match(2, Number, String)) { if (SMatch(2, Number, String)) {
expr->expression.Literal.type = Previous(); expr->expression.Literal.type = Previous();
return expr; return expr;
} }
if (Match(1, LParen)) { if (SMatch(1, LParen)) {
free(expr); free(expr);
expr = Expression(); expr = Expression();
//Consume(RParen, "Expect ')' after expression."); //Consume(RParen, "Expect ')' after expression.");
@@ -143,13 +144,13 @@ Expr* Primary() {
return expr; return expr;
} }
int Match(int count, ...) { int SMatch(int count, ...) {
va_list list; va_list list;
va_start(list, count); va_start(list, count);
for(int i = 0; i < count; i++) { for(int i = 0; i < count; i++) {
if(Check(va_arg(list, TokenType))) { if(Check(va_arg(list, TokenType))) {
Advance(); SAdvance();
return 1; return 1;
} }
} }
@@ -158,33 +159,33 @@ int Match(int count, ...) {
} }
int Check(TokenType type) { int Check(TokenType type) {
if (IsAtEnd()) return 0; if (SIsAtEnd()) return 0;
return Peek()->type == type; return SPeek()->type == type;
} }
int IsAtEnd() { int SIsAtEnd() {
return Peek()->type == EndOF; return SPeek()->type == EndOF;
} }
Token* Peek() { Token* SPeek() {
return tokens->tokens[Current]; return ListOfTokens->tokens[Current];
} }
Token* Previous() { Token* Previous() {
return tokens->tokens[Current - 1]; return ListOfTokens->tokens[Current - 1];
} }
Token* Advance() { Token* SAdvance() {
if (!IsAtEnd()) Current++; if (!SIsAtEnd()) Current++;
return Previous(); return Previous();
} }
Token* CreateToken(char* lexeme, TokenType type) { // Token* CreateToken(char* lexeme, TokenType type) {
Token* token = calloc(1, sizeof(Token)); // Token* token = calloc(1, sizeof(Token));
token->type = type; // token->type = type;
token->lexeme = lexeme; // token->lexeme = lexeme;
token->length = strlen(lexeme); // token->length = strlen(lexeme);
return token; // return token;
} // }
parser.h
+7
View File
@@ -1,6 +1,13 @@
#ifndef PARSER_H #ifndef PARSER_H
#define PARSER_H #define PARSER_H
#include "expr.h"
#include "scanner.h"
#include "token.h"
#include <stdarg.h>
#include <stdlib.h>
#include <string.h>
Expr* GenerateExpressionTree(const TokenList* list);
#endif #endif
scanner.c
+70 -88
View File
@@ -1,32 +1,5 @@
#include "scanner.h" #include "scanner.h"
#include <stdio.h> #include "token.h"
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
typedef struct {
char* keyword;
TokenType type;
} KeywordPair;
KeywordPair keywords[KEYWORD_COUNT] = {
{ "and", AND },
{ "class", CLASS },
{ "else", ELSE },
{ "false", FALSE },
{ "for", FOR },
{ "fun", FUN },
{ "if", IF },
{ "nil", NIL },
{ "or", OR },
{ "print", PRINT },
{ "return", RETURN },
{ "super", SUPER },
{ "this", THIS },
{ "true", TRUE },
{ "var", VAR },
{ "while", WHILE }
};
const char* source_code; const char* source_code;
//Start and Current hold the offsets that index into the string source_code. //Start and Current hold the offsets that index into the string source_code.
@@ -39,7 +12,7 @@ const char* SAdvance(void);
int SIsAtEnd(void); int SIsAtEnd(void);
void ScanToken(TokenList*); void ScanToken(TokenList*);
TokenList* CreateList(void); TokenList* CreateList(void);
int AddTokenToList(TokenType, const char*, int, TokenList*); int AddToTokenList(Token*, TokenList*);
int SMatch(char); int SMatch(char);
char SPeek(void); char SPeek(void);
char PeekNext(void); char PeekNext(void);
@@ -47,7 +20,7 @@ void ParseString(TokenList *);
void ParseNumber(TokenList *); void ParseNumber(TokenList *);
void ParseIdentifier(TokenList *); void ParseIdentifier(TokenList *);
int IsAlpha(char c); int IsAlpha(char c);
KeywordPair* Get(char*); KeyValuePair* Get(const char*);
TokenList* ScanTokens(const char* source) { TokenList* ScanTokens(const char* source) {
if (!source) return NULL; if (!source) return NULL;
@@ -64,8 +37,7 @@ TokenList* ScanTokens(const char* source) {
ScanToken(tokens); ScanToken(tokens);
} }
//Add EOF token and return list once that's set up. AddToTokenList(CreateToken(NULL, line, EndOF), tokens);
AddTokenToList(EndOF, NULL, 0, tokens);
return tokens; return tokens;
} }
@@ -96,7 +68,10 @@ void DestroyTokenList(TokenList* list) {
if (!list) return; if (!list) return;
for(int i = 0; i < list->size; i++) { for(int i = 0; i < list->size; i++) {
//free(list->tokens[i]->lexeme); This shouldn't be needed since the lexeme is a pointer into the source code. //Only free objects that required allocation, i.e. not in the TokenTypeMappings
if (list->tokens[i]->type == String || list->tokens[i]->type == Number || list->tokens[i]->type == Identifier)
free((void *) list->tokens[i]->lexeme);
free(list->tokens[i]); free(list->tokens[i]);
} }
@@ -104,34 +79,24 @@ void DestroyTokenList(TokenList* list) {
free(list); free(list);
} }
int AddTokenToList(TokenType type, const char* lexeme, int length, TokenList* tokens) { int AddToTokenList(Token* token, TokenList* list) {
if (!tokens) return 0; if (!list) return 0;
Token* token = calloc(1, sizeof(Token)); if (!token) return 0;
if (!token) { if ((list->size + 1) > list->capacity) {
fprintf(stderr, "Failed to calloc memory for new Token.\n"); void* new_ptr = realloc(list->tokens, sizeof(Token*) * list->capacity * 2);
return 0;
}
token->lexeme = lexeme;
token->type = type;
token->length = length; //Set the length of the lexeme (which is just a pointer into the complete source listing).
token->line = line;
if ((tokens->size + 1) > tokens->capacity) {
void* new_ptr = realloc(tokens->tokens, sizeof(Token*) * tokens->capacity * 2);
if (!new_ptr) { if (!new_ptr) {
fprintf(stderr, "Failed to realloc TokenList to size %d.\n", tokens->capacity * 2); fprintf(stderr, "Failed to realloc TokenList to size %d.\n", list->capacity * 2);
return 0; return 0;
} }
tokens->tokens = new_ptr; list->tokens = new_ptr;
tokens->capacity = tokens->capacity * 2; list->capacity = list->capacity * 2;
} }
tokens->tokens[tokens->size] = token; list->tokens[list->size] = token;
tokens->size++; list->size++;
return 1; return 1;
} }
@@ -141,58 +106,55 @@ void ScanToken(TokenList* tokens) {
switch (*c) { switch (*c) {
case '(': case '(':
AddTokenToList(LParen, c, 1, tokens); AddToTokenList(CreateToken(NULL, line, LParen), tokens);
break; break;
case ')': case ')':
AddTokenToList(RParen, c, 1, tokens); AddToTokenList(CreateToken(NULL, line, LParen), tokens);
break; break;
case '{': case '{':
AddTokenToList(LBrace, c, 1, tokens); AddToTokenList(CreateToken(NULL, line, LParen), tokens);
break; break;
case '}': case '}':
AddTokenToList(RBrace, c, 1, tokens); AddToTokenList(CreateToken(NULL, line, RParen), tokens);
break; break;
case ',': case ',':
AddTokenToList(Comma, c, 1, tokens); AddToTokenList(CreateToken(NULL, line, Comma), tokens);
break; break;
case '.': case '.':
AddTokenToList(Dot, c, 1, tokens); AddToTokenList(CreateToken(NULL, line, Dot), tokens);
break; break;
case '-': case '-':
AddTokenToList(Minus, c, 1, tokens); AddToTokenList(CreateToken(NULL, line, Minus), tokens);
break; break;
case '+': case '+':
AddTokenToList(Plus, c, 1, tokens); AddToTokenList(CreateToken(NULL, line, Plus), tokens);
break; break;
case ';': case ';':
AddTokenToList(Semicolon, c, 1, tokens); AddToTokenList(CreateToken(NULL, line, Semicolon), tokens);
break; break;
case '*': case '*':
AddTokenToList(Star, c, 1, tokens); AddToTokenList(CreateToken(NULL, line, Star), tokens);
break; break;
case '!': case '!':
if (SMatch('=')) AddTokenToList(Bang_Equal, "!=", 2, tokens); if (SMatch('=')) AddToTokenList(CreateToken(NULL, line, Bang_Equal), tokens);
else AddTokenToList(Bang, c, 1, tokens); else AddToTokenList(CreateToken(NULL, line, Bang), tokens);
break; break;
case '=': case '=':
if (SMatch('=')) AddTokenToList(Equal_Equal, "==", 2, tokens); if (SMatch('=')) AddToTokenList(CreateToken(NULL, line, Equal_Equal), tokens);
else AddTokenToList(Equal, c, 1, tokens); else AddToTokenList(CreateToken(NULL, line, Equal), tokens);
break; break;
case '<': case '<':
if (SMatch('=')) AddTokenToList(Less_Equal, "<=", 2, tokens); if (SMatch('=')) AddToTokenList(CreateToken(NULL, line, Less_Equal), tokens);
else AddTokenToList(Less, c, 1, tokens); else AddToTokenList(CreateToken(NULL, line, Less), tokens);
break; break;
case '>': case '>':
if (SMatch('=')) AddTokenToList(Greater_Equal, ">=", 2, tokens); if (SMatch('=')) AddToTokenList(CreateToken(NULL, line, Greater_Equal), tokens);
else AddTokenToList(Greater, c, 1, tokens); else AddToTokenList(CreateToken(NULL, line, Greater), tokens);
break; break;
case '/': case '/':
if (SMatch('/')) { if (SMatch('/')) while(SPeek() != '\n' && !SIsAtEnd()) { SAdvance(); }
while(SPeek() != '\n' && !SIsAtEnd()) SAdvance(); else AddToTokenList(CreateToken(NULL, line, Slash), tokens);
}
else {
AddTokenToList(Slash, c, 1, tokens);
}
break; break;
case '"': case '"':
ParseString(tokens); ParseString(tokens);
@@ -254,7 +216,16 @@ void ParseString(TokenList *list) {
return; return;
} }
AddTokenToList(String, &source_code[start], current - start, list); char* lexeme = calloc(current - start + 1, sizeof(char));
if (!lexeme) {
fprintf(stderr, "Failed to calloc for string lexeme. %s\n", strerror(errno));
return;
}
snprintf(lexeme, current - start, "%s", &source_code[start]);
AddToTokenList(CreateToken(lexeme, line, String), list);
SAdvance(); // The closing ". SAdvance(); // The closing ".
} }
@@ -268,7 +239,16 @@ void ParseNumber(TokenList* list) {
while(isdigit(SPeek())) SAdvance(); while(isdigit(SPeek())) SAdvance();
} }
AddTokenToList(Number, &source_code[start], current - start, list); char* lexeme = calloc(current - start + 2, sizeof(char));
if (!lexeme) {
fprintf(stderr, "Failed to calloc for number lexeme. %s\n", strerror(errno));
return;
}
snprintf(lexeme, current - start + 1, "%s", &source_code[start]);
AddToTokenList(CreateToken(lexeme, line, Number), list);
} }
char PeekNext() { char PeekNext() {
@@ -280,16 +260,18 @@ char PeekNext() {
void ParseIdentifier(TokenList * list) { void ParseIdentifier(TokenList * list) {
while(IsAlpha(SPeek())) SAdvance(); while(IsAlpha(SPeek())) SAdvance();
char* lexeme = calloc(sizeof(char*), (current - start + 1)); char* lexeme = calloc(current - start + 2, sizeof(char));
snprintf(lexeme, current - start + 1, "%s", &source_code[start]); snprintf(lexeme, current - start + 1, "%s", &source_code[start]);
printf("%s\n", lexeme);
KeyValuePair* result = Get(lexeme);
KeywordPair* result = Get(lexeme); if (result) {
free(lexeme);
if (result) AddTokenToList(result->type, &source_code[start], current - start, list); AddToTokenList(CreateToken(NULL, line, result->type), list);
else AddTokenToList(Identifier, &source_code[start], current - start, list); }
else AddToTokenList(CreateToken(lexeme, line, Identifier), list);
free(lexeme);
} }
int IsAlpha(char c) { int IsAlpha(char c) {
@@ -298,11 +280,11 @@ int IsAlpha(char c) {
(c == '_'); (c == '_');
} }
KeywordPair* Get(char* text) { KeyValuePair* Get(const char* text) {
if (!text) return NULL; if (!text) return NULL;
for (int i = 0; i < KEYWORD_COUNT; i++) for (int i = 0; i < TOKENTYPE_MAPPINGS_COUNT; i++)
if (strcmp(text, keywords[i].keyword) == 0) return &keywords[i]; if (strcmp(text, TokenTypeMappings[i].lexeme) == 0) return &TokenTypeMappings[i];
return NULL; return NULL;
} }
scanner.h
+6 -34
View File
@@ -1,41 +1,13 @@
#ifndef SCANNER_H #ifndef SCANNER_H
#define SCANNER_H #define SCANNER_H
#include "token.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#define DEFAULT_TOKENLIST_SIZE 32 #define DEFAULT_TOKENLIST_SIZE 32
#define KEYWORD_COUNT 16
typedef enum {
//Single-character tokens
LParen, RParen,
LBrace, RBrace,
Comma,
Dot,
Minus, Plus,
Semicolon,
Slash, Star,
//One or two character tokens
Bang, Bang_Equal,
Equal, Equal_Equal,
Greater, Greater_Equal,
Less, Less_Equal,
//Literals
Identifier,
String,
Number,
//Keywords
AND, CLASS, ELSE, FALSE, FUN,
FOR, IF, NIL, OR, PRINT, RETURN,
SUPER, THIS, TRUE, VAR, WHILE,
EndOF
} TokenType;
typedef struct {
TokenType type;
const char* lexeme;
int line;
int length;
} Token;
typedef struct { typedef struct {
Token** tokens; Token** tokens;
token.c
+51
View File
@@ -0,0 +1,51 @@
#include "token.h"
KeyValuePair TokenTypeMappings[TOKENTYPE_MAPPINGS_COUNT] = {
{ "(", LParen}, { ")", RParen}, { ",", Comma }, { ".", Dot },
{ "-", Minus }, { "+", Plus }, { ";", Semicolon }, { "/", Slash}, { "*", Star },
{ "!", Bang }, { "!=", Bang_Equal }, { "=", Equal }, { "==", Equal_Equal },
{ ">", Greater }, { ">=", Greater_Equal }, { "<", Less }, { "<=", Less_Equal },
{ "and", AND }, { "class", CLASS }, { "else", ELSE },
{ "false", FALSE }, { "for", FOR }, { "fun", FUN },
{ "if", IF }, { "nil", NIL }, { "or", OR },
{ "print", PRINT }, { "return", RETURN }, { "super", SUPER },
{ "this", THIS }, { "true", TRUE }, { "var", VAR }, { "while", WHILE },
{ "", EndOF }
};
const char* GetLexemeMapping(TokenType);
Token* CreateToken(const char* lexeme, int line, TokenType type) {
Token* token = calloc(1, sizeof(Token));
if (!token) {
fprintf(stderr, "Failed to calloc token. %s", strerror(errno));
return NULL;
}
if (lexeme) token->lexeme = lexeme;
else {
const char* mapping_result = GetLexemeMapping(type);
if (!mapping_result) {
fprintf(stderr, "Failed to get the mapping for %s\n", lexeme);
free(token);
return NULL;
}
token->lexeme = mapping_result;
}
token->line = line;
token->type = type;
return token;
}
const char* GetLexemeMapping(TokenType type) {
for (int i = 0; i < TOKENTYPE_MAPPINGS_COUNT; i++) {
if (TokenTypeMappings[i].type == type) return TokenTypeMappings[i].lexeme;
}
return NULL;
}
token.h
+54
View File
@@ -0,0 +1,54 @@
#ifndef TOKEN_H
#define TOKEN_H
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#define TOKENTYPE_MAPPINGS_COUNT 34
typedef enum {
//Single-character tokens
LParen, RParen,
LBrace, RBrace,
Comma,
Dot,
Minus, Plus,
Semicolon,
Slash, Star,
//One or two character tokens
Bang, Bang_Equal,
Equal, Equal_Equal,
Greater, Greater_Equal,
Less, Less_Equal,
//Literals
Identifier,
String,
Number,
//Keywords
AND, CLASS, ELSE, FALSE, FUN,
FOR, IF, NIL, OR, PRINT, RETURN,
SUPER, THIS, TRUE, VAR, WHILE,
EndOF
} TokenType;
typedef struct {
const char* lexeme;
TokenType type;
} KeyValuePair;
typedef struct {
TokenType type;
const char* lexeme;
int line;
int length;
} Token;
extern KeyValuePair TokenTypeMappings[TOKENTYPE_MAPPINGS_COUNT];
//If the first parameter is NULL, the token creation will attempt to infer the lexeme from the TokenType.
Token* CreateToken(const char*, int, TokenType);
#endif