Updated the token to more reflect what was implemented in the book. Maybe not the right way to handle it but perhaps this setup will come in handy in later chapters.

expr.c

@@ -2,5 +2,5 @@
 
 void VisitBinary(struct binary);
 void VisitGrouping(struct grouping);
-void VisitLiteral(struct literal);
+void VisitLiteral(Token);
 void VisitUnary(struct unary);

expr.h

@@ -24,11 +24,6 @@ struct grouping {
     struct Expr* expression;
 };
 
-struct literal {
-    Token* type;
-    void* object;
-};
-
 struct unary {
     Token* op;
     struct Expr* right;
@@ -39,14 +34,14 @@ struct Expr {
     union ex {
         struct binary Binary;
         struct grouping Grouping;
-        struct literal Literal;
+        Token* Literal;
         struct unary Unary;
     } expression;
 };
 
 void VisitBinary(struct binary);
 void VisitGrouping(struct grouping);
-void VisitLiteral(struct literal);
+void VisitLiteral(Token);
 void VisitUnary(struct unary);
 
 #endif

lox.c

@@ -1,4 +1,6 @@
 #include "scanner.h"
+#include "parser.h"
+#include "expr.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <sysexits.h>
@@ -21,19 +23,13 @@ int main(int argc, char** argv) {
 }
 
 void Print(TokenList* list) {
-    char lexeme[100];
-
     for(int i = 0; i < list->size; i++) {
         if (list->tokens[i]->type == EndOF) {
             printf("[Line %d] EOF\n", list->tokens[i]->line);\
             return;
         }
 
-        printf("[Line %d] ", list->tokens[i]->line);
+        printf("[Line %d] %s\n", list->tokens[i]->line, list->tokens[i]->lexeme);
-        for(int j = 0; j < list->tokens[i]->length; j++) {
-            printf("%c", list->tokens[i]->lexeme[j]);
-        }
-        printf("\n");
     }
 }
 
@@ -41,10 +37,14 @@ void RunFile(const char* path) {
     printf("Running '%s'\n", path);
     char* contents = GetFileContents(path);
     TokenList* tokens = ScanTokens(contents);
-    Print(tokens);
-
-    DestroyTokenList(tokens);
     free(contents);
+    Print(tokens);
+    printf("TREE:\n");
+    Expr* tree = GenerateExpressionTree(tokens);
+    PrintExpressionTree(tree);
+    printf("\n");
+    FreeExpressionTree(tree);
+    DestroyTokenList(tokens);
 }
 
 char* GetFileContents(const char* path) {

parser.c

@@ -1,9 +1,7 @@
 #include "parser.h"
 #include "expr.h"
-#include "scanner.h"
+#include "token.h"
-#include <stdarg.h>
+#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
 
 Expr* Expression(void);
 Expr* Equality(void);
@@ -14,15 +12,21 @@ Expr* Unary(void);
 Expr* Primary(void);
 int Match(int, ...);
 int Check(TokenType);
-int IsAtEnd(void);
+int ParserAtEnd(void);
-Token* Peek(void);
+Token* ParserPeek(void);
 Token* Previous(void);
-Token* Advance(void);
+Token* AdvanceParser(void);
-Token* CreateToken(char*, TokenType);
+void SynchronizeParser(void);
 
-const TokenList* tokens;
+const TokenList* ListOfTokens;
 int Current = 0;
 
+Expr* GenerateExpressionTree(const TokenList* list) {
+    ListOfTokens = list;
+
+    return Expression();
+}
+
 //Simply expands the equality rule
 Expr* Expression() {
     return Equality();
@@ -114,33 +118,33 @@ Expr* Primary() {
     Expr* expr = calloc(1, sizeof(Expr));
     expr->type = LITERAL;
 
-    if (Match(1, FALSE)) {
+    if (Match(3, FALSE, TRUE, NIL)) {
-        expr->expression.Literal.type = CreateToken("false", FALSE);
+        expr->expression.Literal = ParserPeek();
-        return expr;
-    }
-    if (Match(1, TRUE)) {
-        expr->expression.Literal.type = CreateToken("true", TRUE);
-        return expr;
-    }
-    if (Match(1, NIL)) {
-        expr->expression.Literal.type = CreateToken("nil", NIL);
         return expr;
     }
 
     if (Match(2, Number, String)) {
-        expr->expression.Literal.type = Previous();
+        expr->expression.Literal = Previous();
         return expr;
     }
 
+    free(expr);
+
     if (Match(1, LParen)) {
-        free(expr);
         expr = Expression();
+        
+        Expr* temp = calloc(1, sizeof(Expr));
+        if (!Check(RParen)) printf("Unbalanced\n"); //Todo: something or another...
         //Consume(RParen, "Expect ')' after expression.");
-        expr->type = GROUPING;
+        temp->type = GROUPING;
-        return expr;
+        temp->expression.Grouping.expression = expr;
+
+        return temp;
     }
 
-    return expr;
+    printf("Bad expression\n");
+
+    return NULL;
 }
 
 int Match(int count, ...) {
@@ -149,7 +153,7 @@ int Match(int count, ...) {
 
     for(int i = 0; i < count; i++) {
         if(Check(va_arg(list, TokenType))) {
-            Advance();
+            AdvanceParser();
             return 1;
         }
     }
@@ -158,33 +162,84 @@ int Match(int count, ...) {
 }
 
 int Check(TokenType type) {
-    if (IsAtEnd()) return 0;
+    if (ParserAtEnd()) return 0;
-    return Peek()->type == type;
+    return ParserPeek()->type == type;
 }
 
-int IsAtEnd() {
+int ParserAtEnd() {
-    return Peek()->type == EndOF;
+    return ParserPeek()->type == EndOF;
 }
 
-Token* Peek() {
+Token* ParserPeek() {
-    return tokens->tokens[Current];
+    return ListOfTokens->tokens[Current];
 }
 
 Token* Previous() {
-    return tokens->tokens[Current - 1];
+    return ListOfTokens->tokens[Current - 1];
 }
 
-Token* Advance() {
+Token* AdvanceParser() {
-    if (!IsAtEnd()) Current++;
+    if (!ParserAtEnd()) Current++;
 
     return Previous();
 }
 
-Token* CreateToken(char* lexeme, TokenType type) {
+void SynchronizeParser(void) {
-    Token* token = calloc(1, sizeof(Token));
+    AdvanceParser();
-    token->type = type;
+    //Discard tokens until we find a statement boundary, or at least something that looks like one.
-    token->lexeme = lexeme;
+    while(!ParserAtEnd()) {
-    token->length = strlen(lexeme);
+        if (Previous()->type == Semicolon) return;
 
-    return token;
+        switch(ParserPeek()->type) {
+            case CLASS:
+            case FOR:
+            case FUN:
+            case IF:
+            case PRINT:
+            case RETURN:
+            case VAR:
+            case WHILE:
+                return;
+            default:
+            break;
+        }
+
+        AdvanceParser();
+    }
+}
+
+void PrintExpressionTree(const Expr* tree) {
+    if (!tree) return;
+    
+    if (tree->type == BINARY) {
+        printf("(");
+        PrintExpressionTree(tree->expression.Binary.left);
+        PrintExpressionTree(tree->expression.Binary.right);
+        printf("%s", tree->expression.Binary.op->lexeme);
+        printf(")");
+    }
+    else if (tree->type == UNARY) {
+        printf("(");
+        printf("%s", tree->expression.Unary.op->lexeme);
+        PrintExpressionTree(tree->expression.Unary.right);
+        printf(")");
+    }
+    else if (tree->type == GROUPING) {
+        PrintExpressionTree(tree->expression.Grouping.expression);
+    }
+    else if (tree->type == LITERAL) printf("%s", tree->expression.Literal->lexeme);
+}
+
+void FreeExpressionTree(Expr* tree) {
+    if (!tree) return;
+
+    if (tree->type == BINARY) {
+        FreeExpressionTree(tree->expression.Binary.left);
+        FreeExpressionTree(tree->expression.Binary.right);
+    }
+    else if (tree->type == UNARY) {
+        FreeExpressionTree(tree->expression.Unary.right);
+    }
+
+    free(tree);
 }

parser.h

@@ -1,6 +1,15 @@
 #ifndef PARSER_H
 #define PARSER_H
 
+#include "expr.h"
+#include "scanner.h"
+#include "token.h"
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
 
+Expr* GenerateExpressionTree(const TokenList* list);
+void PrintExpressionTree(const Expr*);
+void FreeExpressionTree(Expr*);
 
 #endif

scanner.c

@@ -1,32 +1,5 @@
 #include "scanner.h"
-#include <stdio.h>
+#include "token.h"
-#include <stdlib.h>
-#include <string.h>
-#include <ctype.h>
-
-typedef struct {
-    char* keyword;
-    TokenType type;
-} KeywordPair;
-
-KeywordPair keywords[KEYWORD_COUNT] = {
-    { "and", AND },
-    { "class", CLASS },
-    { "else", ELSE },
-    { "false", FALSE },
-    { "for", FOR },
-    { "fun", FUN },
-    { "if", IF },
-    { "nil", NIL },
-    { "or", OR },
-    { "print", PRINT },
-    { "return", RETURN },
-    { "super", SUPER },
-    { "this", THIS },
-    { "true", TRUE },
-    { "var", VAR },
-    { "while", WHILE }
-};
 
 const char* source_code;
 //Start and Current hold the offsets that index into the string source_code.
@@ -35,19 +8,19 @@ int current; //points to the character currently being considered.
 int length;
 int line = 1;
 
-const char* SAdvance(void);
+const char* AdvanceScanner(void);
-int SIsAtEnd(void);
+int ScannerAtEnd(void);
 void ScanToken(TokenList*);
 TokenList* CreateList(void);
-int AddTokenToList(TokenType, const char*, int, TokenList*);
+int AddToTokenList(Token*, TokenList*);
-int SMatch(char);
+int ScannerMatch(char);
-char SPeek(void);
+char ScannerPeek(void);
 char PeekNext(void);
 void ParseString(TokenList *);
 void ParseNumber(TokenList *);
 void ParseIdentifier(TokenList *);
 int IsAlpha(char c);
-KeywordPair* Get(char*);
+KeyValuePair* Get(const char*);
 
 TokenList* ScanTokens(const char* source) {
     if (!source) return NULL;
@@ -59,13 +32,12 @@ TokenList* ScanTokens(const char* source) {
 
     TokenList* tokens = CreateList();
 
-    while(!SIsAtEnd()) {
+    while(!ScannerAtEnd()) {
         start = current;
         ScanToken(tokens);
     }
 
-    //Add EOF token and return list once that's set up.
+    AddToTokenList(CreateToken(NULL, NULL, line, EndOF), tokens);
-    AddTokenToList(EndOF, NULL, 0, tokens);
     
     return tokens;
 }
@@ -96,103 +68,89 @@ void DestroyTokenList(TokenList* list) {
     if (!list) return;
 
     for(int i = 0; i < list->size; i++) {
-        //free(list->tokens[i]->lexeme); This shouldn't be needed since the lexeme is a pointer into the source code.
+        FreeToken(list->tokens[i]);
-        free(list->tokens[i]);
     }
 
     free(list->tokens);
     free(list);
 }
 
-int AddTokenToList(TokenType type, const char* lexeme, int length, TokenList* tokens) {
+int AddToTokenList(Token* token, TokenList* list) {
-    if (!tokens) return 0;
+    if (!list) return 0;
-    Token* token = calloc(1, sizeof(Token));
+    if (!token) return 0;
 
-    if (!token) {
+    if ((list->size + 1) > list->capacity) {
-        fprintf(stderr, "Failed to calloc memory for new Token.\n");
+        void* new_ptr = realloc(list->tokens, sizeof(Token*) * list->capacity * 2);
-        return 0;
-    }
-
-    token->lexeme = lexeme;
-    token->type = type;
-    token->length = length; //Set the length of the lexeme (which is just a pointer into the complete source listing).
-    token->line = line;
-
-    if ((tokens->size + 1) > tokens->capacity) {
-        void* new_ptr = realloc(tokens->tokens, sizeof(Token*) * tokens->capacity * 2);
 
         if (!new_ptr) {
-            fprintf(stderr, "Failed to realloc TokenList to size %d.\n", tokens->capacity * 2);
+            fprintf(stderr, "Failed to realloc TokenList to size %d.\n", list->capacity * 2);
             return 0;
         }
 
-        tokens->tokens = new_ptr;
+        list->tokens = new_ptr;
-        tokens->capacity = tokens->capacity * 2;
+        list->capacity = list->capacity * 2;
     }
 
-    tokens->tokens[tokens->size] = token;
+    list->tokens[list->size] = token;
-    tokens->size++;
+    list->size++;
 
     return 1;
 }
 
 void ScanToken(TokenList* tokens) {
-    const char* c = SAdvance();
+    const char* c = AdvanceScanner();
     
     switch (*c) {
     case '(':
-        AddTokenToList(LParen, c, 1, tokens);
+        AddToTokenList(CreateToken(NULL, NULL, line, LParen), tokens);
         break;
     case ')':
-        AddTokenToList(RParen, c, 1, tokens);
+        AddToTokenList(CreateToken(NULL, NULL, line, RParen), tokens);
         break;
     case '{':
-        AddTokenToList(LBrace, c, 1, tokens);
+        AddToTokenList(CreateToken(NULL, NULL, line, LBrace), tokens);
         break;
     case '}':
-        AddTokenToList(RBrace, c, 1, tokens);
+        AddToTokenList(CreateToken(NULL, NULL, line, RBrace), tokens);
         break;
     case ',':
-        AddTokenToList(Comma, c, 1, tokens);
+        AddToTokenList(CreateToken(NULL, NULL, line, Comma), tokens);
         break;
     case '.':
-        AddTokenToList(Dot, c, 1, tokens);
+        AddToTokenList(CreateToken(NULL, NULL, line, Dot), tokens);
         break;
     case '-':
-        AddTokenToList(Minus, c, 1, tokens);
+        AddToTokenList(CreateToken(NULL, NULL, line, Minus), tokens);
         break;
     case '+':
-        AddTokenToList(Plus, c, 1, tokens);
+        AddToTokenList(CreateToken(NULL, NULL, line, Plus), tokens);
         break;
     case ';':
-        AddTokenToList(Semicolon, c, 1, tokens);
+        AddToTokenList(CreateToken(NULL, NULL, line, Semicolon), tokens);
         break;
     case '*':
-        AddTokenToList(Star, c, 1, tokens);
+        AddToTokenList(CreateToken(NULL, NULL, line, Star), tokens);
         break;
     case '!':
-        if (SMatch('=')) AddTokenToList(Bang_Equal, "!=", 2, tokens);
+        if (ScannerMatch('=')) AddToTokenList(CreateToken(NULL, NULL, line, Bang_Equal), tokens);
-        else AddTokenToList(Bang, c, 1, tokens);
+        else AddToTokenList(CreateToken(NULL, NULL, line, Bang), tokens);
         break;
     case '=':
-        if (SMatch('=')) AddTokenToList(Equal_Equal, "==", 2, tokens);
+        if (ScannerMatch('=')) AddToTokenList(CreateToken(NULL, NULL, line, Equal_Equal), tokens);
-        else AddTokenToList(Equal, c, 1, tokens);
+        else AddToTokenList(CreateToken(NULL, NULL, line, Equal), tokens);
         break;
     case '<':
-        if (SMatch('=')) AddTokenToList(Less_Equal, "<=", 2, tokens);
+        if (ScannerMatch('=')) AddToTokenList(CreateToken(NULL, NULL, line, Less_Equal), tokens);
-        else AddTokenToList(Less, c, 1, tokens);
+        else AddToTokenList(CreateToken(NULL, NULL, line, Less), tokens);
         break;
     case '>':
-        if (SMatch('=')) AddTokenToList(Greater_Equal, ">=", 2, tokens);
+        if (ScannerMatch('=')) AddToTokenList(CreateToken(NULL, NULL, line, Greater_Equal), tokens);
-        else AddTokenToList(Greater, c, 1, tokens);
+        else AddToTokenList(CreateToken(NULL, NULL, line, Greater), tokens);
         break;
     case '/':
-        if (SMatch('/')) {
+        if (ScannerMatch('/')) while(ScannerPeek() != '\n' && !ScannerAtEnd()) { AdvanceScanner(); }
-            while(SPeek() != '\n' && !SIsAtEnd()) SAdvance();
+        else AddToTokenList(CreateToken(NULL, NULL, line, Slash), tokens);
-        }
+
-        else {
-            AddTokenToList(Slash, c, 1, tokens);
-        }
         break;
     case '"':
         ParseString(tokens);
@@ -218,16 +176,16 @@ void ScanToken(TokenList* tokens) {
     }
 }
 
-int SIsAtEnd() {
+int ScannerAtEnd() {
     return current >= length;
 }
 
-const char* SAdvance() {
+const char* AdvanceScanner() {
     return &source_code[current++];
 }
 
-int SMatch(char expected) {
+int ScannerMatch(char expected) {
-    if (SIsAtEnd()) return 0;
+    if (ScannerAtEnd()) return 0;
     if (source_code[current] != expected) return 0;
 
     current++;
@@ -235,8 +193,8 @@ int SMatch(char expected) {
     return 1;
 }
 
-char SPeek() {
+char ScannerPeek() {
-    if (SIsAtEnd()) return '\0';
+    if (ScannerAtEnd()) return '\0';
 
     return source_code[current];
 }
@@ -244,31 +202,50 @@ char SPeek() {
 void ParseString(TokenList *list) {
     start = current;    //The start is currently pointing to the first double quote so we need to move it
                         //to the next (first character) of the string literal.
-    while(SPeek() != '"' && !SIsAtEnd()) {
+    while(ScannerPeek() != '"' && !ScannerAtEnd()) {
-        if (SPeek() == '\n') line++;
+        if (ScannerPeek() == '\n') line++;
-        SAdvance();
+        AdvanceScanner();
     }
 
-    if (SIsAtEnd()) {
+    if (ScannerAtEnd()) {
         fprintf(stderr, "Unterminated string.\n");
         return;
     }
 
-    AddTokenToList(String, &source_code[start], current - start, list);
+    char* lexeme = calloc(current - start + 1, sizeof(char));
 
-    SAdvance(); // The closing ".
+    if (!lexeme) {
+        fprintf(stderr, "Failed to calloc for string lexeme. %s\n", strerror(errno));
+        return;
+    }
+
+    snprintf(lexeme, current - start, "%s", &source_code[start]);
+    
+    AddToTokenList(CreateToken(lexeme, lexeme, line, String), list);
+
+    AdvanceScanner(); // The closing ".
 }
 
 void ParseNumber(TokenList* list) {
-    while(isdigit(SPeek())) SAdvance();
+    while(isdigit(ScannerPeek())) AdvanceScanner();
 
-    if (SPeek() == '.' && isdigit(PeekNext())) {
+    if (ScannerPeek() == '.' && isdigit(PeekNext())) {
-        SAdvance();
+        AdvanceScanner();
 
-        while(isdigit(SPeek())) SAdvance();
+        while(isdigit(ScannerPeek())) AdvanceScanner();
     }
 
-    AddTokenToList(Number, &source_code[start], current - start, list);
+    char* lexeme = calloc(current - start + 2, sizeof(char));
+
+    if (!lexeme) {
+        fprintf(stderr, "Failed to calloc for number lexeme. %s\n", strerror(errno));
+        return;
+    }
+
+    snprintf(lexeme, current - start + 1, "%s", &source_code[start]);
+    double value = atof(lexeme); //Will this reference become stale on return?
+    
+    AddToTokenList(CreateToken(lexeme, &value, line, Number), list);
 }
 
 char PeekNext() {
@@ -278,18 +255,20 @@ char PeekNext() {
 }
 
 void ParseIdentifier(TokenList * list) {
-    while(IsAlpha(SPeek())) SAdvance();
+    while(IsAlpha(ScannerPeek())) AdvanceScanner();
 
-    char* lexeme = calloc(sizeof(char*), (current - start + 1));
+    char* lexeme = calloc(current - start + 2, sizeof(char));
     
     snprintf(lexeme, current - start + 1, "%s", &source_code[start]);
     
-    KeywordPair* result = Get(lexeme);
+    KeyValuePair* result = Get(lexeme);
-
-    if (result) AddTokenToList(result->type, &source_code[start], current - start, list);
-    else AddTokenToList(Identifier, &source_code[start], current - start, list);
 
+    if (result) {
         free(lexeme);
+
+        AddToTokenList(CreateToken(NULL, NULL, line, result->type), list);
+    }
+    else AddToTokenList(CreateToken(lexeme, lexeme, line, Identifier), list);
 }
 
 int IsAlpha(char c) {
@@ -298,11 +277,11 @@ int IsAlpha(char c) {
             (c == '_');
 }
 
-KeywordPair* Get(char* text) {
+KeyValuePair* Get(const char* text) {
     if (!text) return NULL;
     
-    for (int i = 0; i < KEYWORD_COUNT; i++) 
+    for (int i = 0; i < TOKENTYPE_MAPPINGS_COUNT; i++) 
-        if (strcmp(text, keywords[i].keyword) == 0) return &keywords[i];
+        if (strcmp(text, TokenTypeMappings[i].lexeme) == 0) return &TokenTypeMappings[i];
 
     return NULL;
 }

scanner.h

@@ -1,41 +1,13 @@
 #ifndef SCANNER_H
 #define SCANNER_H
 
+#include "token.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+
 #define DEFAULT_TOKENLIST_SIZE 32
-#define KEYWORD_COUNT 16
-
-typedef enum {
-    //Single-character tokens
-    LParen, RParen,
-    LBrace, RBrace,
-    Comma,
-    Dot,
-    Minus, Plus,
-    Semicolon,
-    Slash, Star,
-    //One or two character tokens
-    Bang, Bang_Equal,
-    Equal, Equal_Equal,
-    Greater, Greater_Equal,
-    Less, Less_Equal,
-    //Literals
-    Identifier,
-    String,
-    Number,
-    //Keywords
-    AND, CLASS, ELSE, FALSE, FUN,
-    FOR, IF, NIL, OR, PRINT, RETURN,
-    SUPER, THIS, TRUE, VAR, WHILE,
-
-    EndOF
-} TokenType;
-
-typedef struct {
-    TokenType type;
-    const char* lexeme;
-    int line;
-    int length;
-} Token;
 
 typedef struct {
     Token** tokens;

token.c

@@ -0,0 +1,63 @@
+#include "token.h"
+
+KeyValuePair TokenTypeMappings[TOKENTYPE_MAPPINGS_COUNT] = {
+    { "(", LParen}, { ")", RParen}, { ",", Comma }, { ".", Dot },
+    { "-", Minus }, { "+", Plus }, { ";", Semicolon }, { "/", Slash}, { "*", Star },
+    { "!", Bang }, { "!=", Bang_Equal }, { "=", Equal }, { "==", Equal_Equal },
+    { ">", Greater }, { ">=", Greater_Equal }, { "<", Less }, { "<=", Less_Equal },
+    { "and", AND }, { "class", CLASS }, { "else", ELSE },
+    { "false", FALSE }, { "for", FOR }, { "fun", FUN },
+    { "if", IF }, { "nil", NIL }, { "or", OR },
+    { "print", PRINT }, { "return", RETURN }, { "super", SUPER },
+    { "this", THIS }, { "true", TRUE }, { "var", VAR }, { "while", WHILE },
+    { "", EndOF }
+};
+
+const char* GetLexemeMapping(TokenType);
+
+Token* CreateToken(const char* lexeme, void* literal, int line, TokenType type) {
+    Token* token = calloc(1, sizeof(Token));
+    
+    if (!token) {
+        fprintf(stderr, "Failed to calloc token. %s", strerror(errno));
+        return NULL;
+    }
+
+    if (lexeme) {
+        token->literal = literal;
+        token->lexeme = lexeme;
+    }
+    else { 
+        const char* mapping_result = GetLexemeMapping(type);
+        
+        if (!mapping_result) {
+            fprintf(stderr, "Failed to get the mapping for %s\n", lexeme);
+            free(token);
+            return NULL;
+        }
+
+        token->literal = mapping_result;
+        token->lexeme = mapping_result;
+    }
+
+    token->line = line;
+    token->type = type;
+
+    return token;
+}
+
+const char* GetLexemeMapping(TokenType type) {
+    for (int i = 0; i < TOKENTYPE_MAPPINGS_COUNT; i++) {
+        if (TokenTypeMappings[i].type == type) return TokenTypeMappings[i].lexeme;
+    }
+
+    return NULL;
+}
+
+void FreeToken(Token* token) {
+    if (!token) return;
+
+    if (token->type == String || token->type == Number || token->type == Identifier) free((void *) token->lexeme);
+
+    free(token);
+}

token.h

@@ -0,0 +1,55 @@
+#ifndef TOKEN_H
+#define TOKEN_H
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+#define TOKENTYPE_MAPPINGS_COUNT 34
+
+typedef enum {
+    //Single-character tokens
+    LParen, RParen,
+    LBrace, RBrace,
+    Comma,
+    Dot,
+    Minus, Plus,
+    Semicolon,
+    Slash, Star,
+    //One or two character tokens
+    Bang, Bang_Equal,
+    Equal, Equal_Equal,
+    Greater, Greater_Equal,
+    Less, Less_Equal,
+    //Literals
+    Identifier,
+    String,
+    Number,
+    //Keywords
+    AND, CLASS, ELSE, FALSE, FUN,
+    FOR, IF, NIL, OR, PRINT, RETURN,
+    SUPER, THIS, TRUE, VAR, WHILE,
+
+    EndOF
+} TokenType;
+
+typedef struct {
+    const char* lexeme;
+    TokenType type;
+} KeyValuePair;
+
+typedef struct {
+    TokenType type;
+    const char* lexeme;
+    const void* literal;
+    int line;
+} Token;
+
+extern KeyValuePair TokenTypeMappings[TOKENTYPE_MAPPINGS_COUNT];
+
+//If the first parameter is NULL, the token creation will attempt to infer the lexeme from the TokenType.
+Token* CreateToken(const char*, void*, int, TokenType);
+void FreeToken(Token*);
+
+#endif