Added support for string literals in the tokenizer.

includes/keywords.h

@@ -2,7 +2,7 @@
 #define KEYWORDS_H
 
 typedef enum {
-    FN, ORG
+    FN, ORG, NAMESPACE, DB, CONST
 } Keyword;
 
 int IsKeyword(const char* text, Keyword* keyword);

src/keywords.c

@@ -1,6 +1,7 @@
 #include "../includes/keywords.h"
+#include <string.h>
 
-#define KEYWORD_COUNT 2
+#define KEYWORD_COUNT 5
 
 struct _keyword {
     char* Name;
@@ -9,15 +10,18 @@ struct _keyword {
 
 struct _keyword keywords[KEYWORD_COUNT] = {
     { "fn", FN },
-    { "org", ORG }
-}
+    { "org", ORG },
+    { "namespace", NAMESPACE },
+    { "db", DB },
+    { "cosnt", CONST }
+};
 
 int IsKeyword(const char* text, Keyword* keyword) {
     if (!text) return 0;
 
     for(unsigned long i = 0; i < KEYWORD_COUNT; i++) {
         if (strcmp(keywords[i].Name, text) == 0) {
-            if (keyword) *keyword = instructions[i].Word;
+            if (keyword) *keyword = keywords[i].Word;
 
             return 1;
         }
@@ -27,11 +31,13 @@ int IsKeyword(const char* text, Keyword* keyword) {
 }
 
 void GetKeywordText(Keyword keyword, char buffer[16]) {
-    memset(buffer, '\0', sizeof(buffer));
+    if (!buffer) return;
+    
+    memset(buffer, '\0', 16);
 
     for(int i = 0; i < KEYWORD_COUNT; i++) {
-        if (instructions[i].Word == keyword) { 
-            strncpy(buffer, instructions[i].Name, sizeof(buffer));
+        if (keywords[i].Word == keyword) { 
+            strncpy(buffer, keywords[i].Name, 16);
             
             break;
         }

src/main.c

@@ -34,6 +34,8 @@ int main(int argc, char** argv) {
     for(int i = 0; i < tokens->Size; i++) {
         Token* token = tokens->Items[i];
 
-        printf(TokenStringify(token, false));
+        printf("%s ", token->Value.String);//TokenStringify(token, false));
     }
+
+    printf("\n");
 }

src/tokenizer.c

@@ -16,26 +16,26 @@ size_t SourceCodeIndex = 0;
 int SourceLineNumber = 0;
 int SourceColumnNumber = 0;
 
-bool TryPeekTokenizer(char* c);
+char PeekTokenizer(void);
 bool TryPopTokenizer(char* c);
 bool TokenizerAtEnd(void);
 char AdvanceTokenizer(void);
 bool BackTokenzier(void);
-char TokenizerLookAhead(void);
 
 void TokenizerExpectWord(void);
+void TokenizerExpectString(void);
 void TokenizerIgnoreLine(void);
 
 Array* Tokenize(const char* sourceCode, Dictionary** variables) {
     Tokens = ArrayCreate();
     
-    SourceCodeLength = strlen(sourceCode) - 1;
+    SourceCodeLength = strlen(sourceCode);
     SourceCode = sourceCode;
 
     while(!TokenizerAtEnd()) {
-        char c;
+        char c = PeekTokenizer();
 
-        if (!TryPeekTokenizer(&c)) break;
+        if (c == '\0') break;
 
         if (isalpha(c)) {
             TokenizerExpectWord();
@@ -43,6 +43,14 @@ Array* Tokenize(const char* sourceCode, Dictionary** variables) {
             continue;
         }
 
+        if (c == '"') {
+            AdvanceTokenizer();
+
+            TokenizerExpectString();
+
+            continue;
+        }
+
         if (c == ';') {
             TokenizerIgnoreLine();
 
@@ -67,6 +75,12 @@ void TokenizerExpectWord() {
             break;
         }
 
+        if (c == ';') {
+            TokenizerIgnoreLine();
+
+            break;
+        }
+
         if (ispunct(c)) {
             BackTokenzier();
 
@@ -90,6 +104,49 @@ void TokenizerExpectWord() {
     }
 }
 
+void TokenizerExpectString(void) {
+    char word[256] = { 0 };
+    int wordLength = 0;
+    char c;
+
+    while(TryPopTokenizer(&c)) {
+        if (c == '"') break;
+
+        if (c == '\\') {
+            c = PeekTokenizer();
+
+            switch(c) {
+                case 'n':
+                    c = '\n';
+                    break;
+                case 't':
+                    c = '\t';
+                    break;
+                default:
+                    c = '\\';
+                    break;
+            }
+
+            if (c != '\\') AdvanceTokenizer();
+        }
+
+        if (wordLength == 255) break; //Sync, error etc here at some point.
+
+        word[wordLength] = c;
+        wordLength++;
+    }
+
+    if (wordLength > 0) {
+        Token* token = TokenCreate(String, SourceLineNumber, SourceColumnNumber, true);
+
+        token->Value.String = calloc(wordLength + 1, sizeof(char));
+
+        strncpy(token->Value.String, word, wordLength);
+
+        ArrayAdd(Tokens, token);
+    }
+}
+
 void TokenizerIgnoreLine(void) {
     char c;
 
@@ -98,10 +155,8 @@ void TokenizerIgnoreLine(void) {
     }
 }
 
-bool TryPeekTokenizer(char* c) {
-    *c = SourceCode[SourceCodeIndex];
-
-    return *c != '\0';
+char PeekTokenizer(void) {
+    return SourceCode[SourceCodeIndex];
 }
 
 bool TryPopTokenizer(char* c) {