clox/scanner.c

#include "scanner.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>

typedef struct {
    char* keyword;
    TokenType type;
} KeywordPair;

KeywordPair keywords[KEYWORD_COUNT] = {
    { "and", AND },
    { "class", CLASS },
    { "else", ELSE },
    { "false", FALSE },
    { "for", FOR },
    { "fun", FUN },
    { "if", IF },
    { "nil", NIL },
    { "or", OR },
    { "print", PRINT },
    { "return", RETURN },
    { "super", SUPER },
    { "this", THIS },
    { "true", TRUE },
    { "var", VAR },
    { "while", WHILE }
};

const char* source_code;
//Start and Current hold the offsets that index into the string source_code.
int start;   //Points to the first character in the lexeme being scanned.
int current; //points to the character currently being considered.
int length;
int line = 1;

const char* Advance(void);
int IsAtEnd(void);
void ScanToken(TokenList*);
TokenList* CreateList(void);
int AddTokenToList(TokenType, const char*, int, TokenList*);
int Match(char);
char Peek(void);
char PeekNext(void);
void ParseString(TokenList *);
void ParseNumber(TokenList *);
void ParseIdentifier(TokenList *);
int IsAlpha(char c);
KeywordPair* Get(char*);

TokenList* ScanTokens(const char* source) {
    if (!source) return NULL;
    
    source_code = source;
    length = strlen(source);

    if (length == 0) return NULL;

    TokenList* tokens = CreateList();

    while(!IsAtEnd()) {
        start = current;
        ScanToken(tokens);
    }

    //Add EOF token and return list once that's set up.
    AddTokenToList(EndOF, NULL, 0, tokens);
    
    return tokens;
}

TokenList* CreateList() {
    TokenList* list = calloc(1, sizeof(TokenList));

    if (!list) {
        fprintf(stderr, "Failed to calloc TokenList.\n");
        return NULL;
    }

    list->tokens = calloc(DEFAULT_TOKENLIST_SIZE, sizeof(Token*));

    if (!list->tokens) {
        free(list);
        fprintf(stderr, "Failed to calloc tokens.\n");
        return NULL;
    }

    list->capacity = DEFAULT_TOKENLIST_SIZE;
    list->size = 0;

    return list;
}

void DestroyTokenList(TokenList* list) {
    if (!list) return;

    for(int i = 0; i < list->size; i++) {
        //free(list->tokens[i]->lexeme); This shouldn't be needed since the lexeme is a pointer into the source code.
        free(list->tokens[i]);
    }

    free(list->tokens);
    free(list);
}

int AddTokenToList(TokenType type, const char* lexeme, int length, TokenList* tokens) {
    if (!tokens) return 0;
    Token* token = calloc(1, sizeof(Token));

    if (!token) {
        fprintf(stderr, "Failed to calloc memory for new Token.\n");
        return 0;
    }

    token->lexeme = lexeme;
    token->type = type;
    token->length = length; //Set the length of the lexeme (which is just a pointer into the complete source listing).
    token->line = line;

    if ((tokens->size + 1) > tokens->capacity) {
        void* new_ptr = realloc(tokens->tokens, sizeof(Token*) * tokens->capacity * 2);

        if (!new_ptr) {
            fprintf(stderr, "Failed to realloc TokenList to size %d.\n", tokens->capacity * 2);
            return 0;
        }

        tokens->tokens = new_ptr;
        tokens->capacity = tokens->capacity * 2;
    }

    tokens->tokens[tokens->size] = token;
    tokens->size++;

    return 1;
}

void ScanToken(TokenList* tokens) {
    const char* c = Advance();
    
    switch (*c) {
    case '(':
        AddTokenToList(LParen, c, 1, tokens);
        break;
    case ')':
        AddTokenToList(RParen, c, 1, tokens);
        break;
    case '{':
        AddTokenToList(LBrace, c, 1, tokens);
        break;
    case '}':
        AddTokenToList(RBrace, c, 1, tokens);
        break;
    case ',':
        AddTokenToList(Comma, c, 1, tokens);
        break;
    case '.':
        AddTokenToList(Dot, c, 1, tokens);
        break;
    case '-':
        AddTokenToList(Minus, c, 1, tokens);
        break;
    case '+':
        AddTokenToList(Plus, c, 1, tokens);
        break;
    case ';':
        AddTokenToList(Semicolon, c, 1, tokens);
        break;
    case '*':
        AddTokenToList(Star, c, 1, tokens);
        break;
    case '!':
        if (Match('=')) AddTokenToList(Bang_Equal, "!=", 2, tokens);
        else AddTokenToList(Bang, c, 1, tokens);
        break;
    case '=':
        if (Match('=')) AddTokenToList(Equal_Equal, "==", 2, tokens);
        else AddTokenToList(Equal, c, 1, tokens);
        break;
    case '<':
        if (Match('=')) AddTokenToList(Less_Equal, "<=", 2, tokens);
        else AddTokenToList(Less, c, 1, tokens);
        break;
    case '>':
        if (Match('=')) AddTokenToList(Greater_Equal, ">=", 2, tokens);
        else AddTokenToList(Greater, c, 1, tokens);
        break;
    case '/':
        if (Match('/')) {
            while(Peek() != '\n' && !IsAtEnd()) Advance();
        }
        else {
            AddTokenToList(Slash, c, 1, tokens);
        }
        break;
    case '"':
        ParseString(tokens);
        break;
    case ' ':
    case '\r':
    case '\t':
        break; //Ignore whitespace
    case '\n':
        line++;
        break;
    default:
        if (isdigit(*c)) {
            ParseNumber(tokens);
            break;
        } else if (IsAlpha(*c)) {
            ParseIdentifier(tokens);
            break;
        }

        fprintf(stderr, "Unexcpedted character %c\n", *c);
        break;
    }
}

int IsAtEnd() {
    return current >= length;
}

const char* Advance() {
    return &source_code[current++];
}

int Match(char expected) {
    if (IsAtEnd()) return 0;
    if (source_code[current] != expected) return 0;

    current++;

    return 1;
}

char Peek() {
    if (IsAtEnd()) return '\0';

    return source_code[current];
}

void ParseString(TokenList *list) {
    start = current;    //The start is currently pointing to the first double quote so we need to move it
                        //to the next (first character) of the string literal.
    while(Peek() != '"' && !IsAtEnd()) {
        if (Peek() == '\n') line++;
        Advance();
    }

    if (IsAtEnd()) {
        fprintf(stderr, "Unterminated string.\n");
        return;
    }

    AddTokenToList(String, &source_code[start], current - start, list);

    Advance(); // The closing ".
}

void ParseNumber(TokenList* list) {
    while(isdigit(Peek())) Advance();

    if (Peek() == '.' && isdigit(PeekNext())) {
        Advance();

        while(isdigit(Peek())) Advance();
    }

    AddTokenToList(Number, &source_code[start], current - start, list);
}

char PeekNext() {
    if (current + 1 >= length) return '\0';

    return source_code[current + 1];
}

void ParseIdentifier(TokenList * list) {
    while(IsAlpha(Peek())) Advance();

    char* lexeme = calloc(sizeof(char*), (current - start + 1));
    
    snprintf(lexeme, current - start + 1, "%s", &source_code[start]);

    KeywordPair* result = Get(lexeme);

    if (result) AddTokenToList(result->type, &source_code[start], current - start, list);
    else AddTokenToList(Identifier, &source_code[start], current - start, list);

    free(lexeme);
}

int IsAlpha(char c) {
    return  (c >= 'a' && c <= 'z') ||
            (c >= 'A' && c <= 'Z') ||
            (c == '_');
}

KeywordPair* Get(char* text) {
    if (!text) return NULL;

    for (int i = 0; i < KEYWORD_COUNT; i++) 
        if (strcmp(text, keywords[i].keyword) == 0) return &keywords[i];

    return NULL;
}