Last bit of clean up before building the parser.

src/lexer.c

@@ -7,40 +7,20 @@ Token* CreateToken(TokenType, char*);
 Token* GetNextToken(char *);
 int TokenIsNumeric(const char*, int *);
 
-List* GenerateTokensFromFile(const char* file_path) {
-    FILE *file;
-    int line_count = 1;
-    char* line = NULL;
-    size_t len = 0;
-    ssize_t bytes_read;
+List* GetTokensFromLine(char* line) {
     List* tokens = CreateList();
+    
+    Token* token = GetNextToken(line);
 
-    file = fopen(file_path, "r");
+    while (token) {
 
-    if (!file) {
-        printf("Failed to open '%s' for reading.\n", file_path);
-        return NULL;
+        if (!token) break;
+        
+        AddListItem(token, sizeof(Token), tokens);
+
+        token = GetNextToken(NULL);
     }
 
-    while((bytes_read = getline(&line, &len, file)) != -1) {
-        Token* token = GetNextToken(line);
-
-        while (token) {
-
-            if (!token) break;
-            
-            AddListItem(token, sizeof(Token), tokens);
-
-            token = GetNextToken(NULL);
-        }
-
-        line_count++;
-    }
-
-    fclose(file);
-
-    if (line) free(line);
-
     return tokens;
 }
 

src/main.c

@@ -10,26 +10,48 @@ typedef struct {
     char** parameters;
 } Instruction;
 
-void print_file(char*);
 List* get_strings(const char*);
 
 int main(int argc, char* args[]) {
-    const char *input_files[argc - 1];
-
     if (argc == 1) {
         printf("Input files required.\n");
         return -1;
     }
+
+    FILE *file;
+    int line_count = 1;
+    char* line = NULL;
+    size_t len = 0;
+    ssize_t bytes_read;
+    int line_number = 1;
+
+    file = fopen(args[1], "r");
+
+    if (!file) {
+        printf("Failed to open '%s' for reading.\n", args[1]);
+        return 1;
+    }
     
-    List* tokens = GenerateTokensFromFile(args[1]);
+    //getline() uses realloc() for the line parameter, so there shouldn't
+    //be any issue with memory leaks as long as the last time this function
+    //is called, line gets free()ed.
+    while((bytes_read = getline(&line, &len, file)) != -1) {
+        List* tokens = GetTokensFromLine(line);
 
-    for (int i = 0; i < tokens->size; i++) {
-        Token* t = (Token*) tokens->content[i];
+        for (int i = 0; i < tokens->size; i++) {
+            Token* t = (Token*) tokens->content[i];
 
-        printf("Token Type: '%d' Value: '%s'\n", t->type, t->value);
+            printf("Token Type: '%d' Value: '%s' (%d)\n", t->type, t->value, line_number);
 
-        free(t->value);
+            free(t->value);
+        }
+
+        DestroyList(tokens);
+
+        line_number++;
     }
 
-    DestroyList(tokens);
+    fclose(file);
+
+    if (line) free(line);
 }

src/tokenizer.c

@@ -25,6 +25,8 @@ char* PeekNextToken() {
     return SplitOnBasicGrammar(string, &pos);
 }
 
+//TODO: this function should probably return NULL when its run out of tokens
+//instead of sending back a valid pointer with memory that must be free()ed by the caller.
 char* SplitOnBasicGrammar(char* line, unsigned long *string_index) {
 
     unsigned long length = strlen(string);