glm94/assm-test
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Code cleanup. Converting a number lexeme to a proper integer setup but not done yet. So number tokens don't have a valid value.

Browse Source
This commit is contained in:
glm94
2022-05-05 16:12:33 +00:00
parent 9fa48520c7
commit 8ec0b9b216
11 changed files with 59 additions and 283 deletions
Download Patch File Download Diff File Expand all files Collapse all files
includes/lexer.h
-14
View File
@@ -1,14 +0,0 @@
#ifndef LEXER_H
#define LEXER_H
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include "list.h"
#include "tokenizer.h"
#include "token.h"
List* GetTokensFromLine(char*);
#endif
includes/opcodes.h
+1 -1
View File
@@ -2,7 +2,7 @@
#define OPCODES_H
#include <string.h>
#include "lexer.h"
#include "token.h"
#define OPCODECOUNT 10
#define REGISTERCOUNT 8
includes/scanner.h
+1
View File
@@ -4,6 +4,7 @@
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <errno.h>
#include "stdlib.h"
#include "token.h"
#include "list.h"
includes/token.h
+7 -2
View File
@@ -2,6 +2,9 @@
#define TOKEN_H
#include "stdlib.h"
#include <string.h>
#include <errno.h>
#include <stdio.h>
typedef enum {
PLUS,
@@ -30,10 +33,12 @@ typedef enum {
typedef struct {
TokenType type;
char* value;
char* lexeme;
void* value;
int line;
} Token;
Token* CreateToken(TokenType type, char* value);
Token* CreateToken(char*, void*, int, TokenType);
void FreeToken(Token*);
#endif
includes/tokenizer.h
-13
View File
@@ -1,13 +0,0 @@
#ifndef TOKENIZER_H
#define TOKENIZER_H
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "list.h"
List* GetTokens(const char *);
char* GetToken(char*);
char* PeekNextToken(void);
#endif
src/lexer.c
-104
View File
@@ -1,104 +0,0 @@
#include "../includes/lexer.h"
#include "../includes/opcodes.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
Token* GetNextToken(char *);
int TokenIsNumeric(const char*, int *);
List* GetTokensFromLine(char* line) {
List* tokens = CreateList();
Token* token = GetNextToken(line);
while (token) {
if (!token) break;
AddListItem(token, sizeof(Token), tokens);
token = GetNextToken(NULL);
}
return tokens;
}
Token* GetNextToken(char *string) {
char* string_token = GetToken(string);
int base = 0;
TokenType type;
while (strlen(string_token) != 0) {
if (strcmp(string_token, ".db") == 0) {
return CreateToken(DB, string_token);
}
if (TokenIsNumeric(string_token, &base)) {
if (base == 10) return CreateToken(NUMBER, string_token);
if (base == 16) return CreateToken(HEX, string_token);
}
char* next = PeekNextToken();
if (next) {
if (strcmp(":", next) == 0) {
free(next);
free(GetToken(NULL));
return CreateToken(IDENTIFIER, string_token);
}
free(next);
}
if (IsOpcode(string_token, &type)) {
return CreateToken(type, string_token);
}
if (IsRegister(string_token, &type)) {
return CreateToken(type, string_token);
}
// if (op) return CreateToken(op->op, op->lexeme);
// if (reg) return CreateToken(reg->type, reg->lexeme);
if (strcmp(",", string_token) == 0) return CreateToken(COMMA, string_token);
return CreateToken(IDENTIFIER, string_token);
}
free(string_token);
return NULL;
}
int TokenIsNumeric(const char* token, int *base) {
*base = 0;
if (!token) return 0;
unsigned long length = strlen(token);
int i = 0;
if (length == 0) return 0;
*base = 10;
if (length > 2) {
if (token[0] == '0' && token[1] == 'x') {
*base = 16;
i = 2;
}
}
for(; i < length; i++) {
if (!isdigit(token[i])) {
if (*base == 16 && token[i] >= 'A' && token[i] <= 'F') continue;
return 0;
}
}
return 1;
}
src/main.c
+3 -2
View File
@@ -3,7 +3,6 @@
#include <string.h>
#include <sysexits.h>
#include "../includes/list.h"
#include "../includes/lexer.h"
#include "../includes/parser.h"
#include "../includes/futil.h"
#include "../includes/scanner.h"
@@ -26,7 +25,9 @@ int main(int argc, char* args[]) {
for(int i = 0; i < list->size; i++) {
Token* t = (Token*) list->content[i];
printf("[%i] '%s'\n", t->type, t->value);
printf("[%i] '%s'", t->type, t->lexeme);
if (t->type == NUMBER || t->type == HEX) printf(" NUM: %ld", (long) t->value);
printf("\n");
}
free(source_code);
src/parser.c
+4 -4
View File
@@ -1,10 +1,10 @@
#include "../includes/parser.h"
#include "../includes/lexer.h"
#include "../includes/token.h"
#include <stdio.h>
#include <stdlib.h>
static List* SymbolsTable;
static unsigned int program_counter = 0;
//static List* SymbolsTable;
//static unsigned int program_counter = 0;
void ProcessDirective(List*);
void ProcessVariableDeclaration(List*);
@@ -51,5 +51,5 @@ void ProcessVariableDeclaration(List* tokens) {
Token* string_literal = (Token*) tokens->content[2];
printf("Found string literal.\n");
printf("Name '%s' value: '%s'\n", symbol_token->value, string_literal->value);
printf("Name '%s' value: '%s'\n", symbol_token->lexeme, string_literal->lexeme);
}
src/scanner.c
+35 -24
View File
@@ -1,8 +1,4 @@
#include "../includes/scanner.h"
#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
const char* SourceCode;
int Line = 0;
@@ -96,18 +92,21 @@ Token* ParseNumber(void) {
if (length == 0) return NULL;
char* lexeme = calloc(sizeof(char), length);
char* lexeme = calloc(sizeof(char), length + 1);
if (!lexeme) {
fprintf(stderr, "Failed to calloc space for number.\n");
fprintf(stderr, "Failed to calloc space for number. %s.\n", strerror(errno));
return NULL;
}
memcpy(lexeme, &SourceCode[start], length);
//Setting the base to zero means the function will detect the base.
//https://pubs.opengroup.org/onlinepubs/7908799/xsh/strtol.html
printf("Number Parsed: %s (%ld)\n", lexeme, strtol(lexeme, NULL, 0));
if (base == 10) return CreateToken(NUMBER, lexeme);
if (base == 10) return CreateToken(lexeme, lexeme, Line, NUMBER);
return CreateToken(HEX, lexeme);
return CreateToken(lexeme, lexeme, Line, HEX);
}
Token* ParseDirective(void) {
@@ -119,17 +118,17 @@ Token* ParseDirective(void) {
if (length == 0) return NULL;
char* directive = calloc(sizeof(char), length);
char* directive = calloc(sizeof(char), length + 1);
if (!directive) {
fprintf(stderr, "Failed to calloc for the assmebler directive.\n");
fprintf(stderr, "Failed to calloc for the assmebler directive. %s.\n", strerror(errno));
return NULL;
}
memcpy(directive, &SourceCode[start], length);
if (strcmp(directive, ".db") == 0) {
return CreateToken(DB, directive);
return CreateToken(directive, directive, Line, DB);
}
else if (strcmp(directive, ".org") == 0) {
fprintf(stderr, "[Warning] Org is not a supported directive.\n");
@@ -164,12 +163,18 @@ Token* ParseString(void) {
if (length == 0) return NULL;
char* lexeme = calloc(sizeof(char), length);
char* lexeme = calloc(sizeof(char), length + 1);
if (!lexeme) {
fprintf(stderr, "Failed to calloc memory for string. %s.\n", strerror(errno));
return NULL;
}
memcpy(lexeme, &SourceCode[start], length);
AdvanceScanner(); //Consume the trailing double quote.
Token* token = CreateToken(STRING, lexeme);
Token* token = CreateToken(lexeme, lexeme, Line, STRING);
return token;
}
@@ -185,15 +190,21 @@ Token* ParseIdentifier(void) {
if (length == 0) return NULL;
TokenType type;
char* lexeme = calloc(sizeof(char), length);
TokenType type;
char* lexeme = calloc(sizeof(char), length + 1);
if (!lexeme) {
fprintf(stderr, "Failed to calloc memory for identifier. %s.\n", strerror(errno));
return NULL;
}
memcpy(lexeme, &SourceCode[start], length);
if (IsOpcode(lexeme, &type)) return CreateToken(type, lexeme);
if (IsRegister(lexeme, &type)) return CreateToken(type, lexeme);
if (lexeme[length - 1] == ':') return CreateToken(LABEL, lexeme);
if (IsOpcode(lexeme, &type)) return CreateToken(lexeme, lexeme, Line, type);
if (IsRegister(lexeme, &type)) return CreateToken(lexeme, lexeme, Line, type);
if (lexeme[length - 1] == ':') return CreateToken(lexeme, lexeme, Line, LABEL);
return CreateToken(IDENTIFIER, lexeme);
return CreateToken(lexeme, lexeme, Line, IDENTIFIER);
}
char PeekScanner(void) {
@@ -234,15 +245,15 @@ int IsPunctuation(char c) {
Token* ParsePunctuation(char c) {
switch(c) {
case '[':
return CreateToken(LBRACKET, "[");
return CreateToken("[", NULL, Line, LBRACKET);
case ']':
return CreateToken(RBracket, "]");
return CreateToken("]", NULL, Line, RBracket);
case '(':
return CreateToken(LPARAM, "(");
return CreateToken("(", NULL, Line, LPARAM);
case ')':
return CreateToken(RPARAM, ")");
return CreateToken(")", NULL, Line, RPARAM);
case ',':
return CreateToken(COMMA, ",");
return CreateToken(",", NULL, Line, COMMA);
default:
return NULL;
}
src/token.c
+8 -3
View File
@@ -1,11 +1,16 @@
#include "../includes/token.h"
Token* CreateToken(TokenType type, char* value) {
Token* CreateToken(char* lexeme, void* value, int lineNumber, TokenType type) {
Token* token = calloc(1, sizeof(Token));
if (!token) return NULL;
if (!token) {
fprintf(stderr, "Failed to calloc memory for Token. %s.\n", strerror(errno));
return NULL;
}
token->type = type;
token->line = lineNumber;
token->lexeme = lexeme;
token->value = value;
return token;
@@ -14,7 +19,7 @@ Token* CreateToken(TokenType type, char* value) {
void FreeToken(Token* token) {
if (!token) return;
if (token->value) free(token->value);
if (token->value && token->type >= STRING) free(token->value);
free(token);
}
src/tokenizer.c
-116
View File
@@ -1,116 +0,0 @@
#include "../includes/tokenizer.h"
#include <stdio.h>
#include <string.h>
char* SplitOnBasicGrammar(char*, unsigned long *);
void GetStringLiteral(char*, unsigned long, char**, unsigned long*);
static char* string;
static unsigned long current_position;
char* GetToken(char* line) {
if (line) {
string = line;
current_position = 0;
}
return SplitOnBasicGrammar(string, &current_position);
}
char* PeekNextToken() {
if (!string) return NULL;
if (strlen(string) == 0) return NULL;
unsigned long pos = current_position;
return SplitOnBasicGrammar(string, &pos);
}
//TODO: this function should probably return NULL when its run out of tokens
//instead of sending back a valid pointer with memory that must be free()ed by the caller.
char* SplitOnBasicGrammar(char* line, unsigned long *string_index) {
unsigned long length = strlen(string);
char* token = calloc(1, length + 1);
for (int i = 0; *string_index < length; i++, (*string_index)++) {
if (string[*string_index] == ';') break;
if (string[*string_index] == ':') {
if (strlen(token) == 0) {
token[0] = ':';
(*string_index)++;
}
break;
}
if (string[*string_index] == '"') {
(*string_index)++;
GetStringLiteral(string, length, &token, string_index);
break;
}
if (string[*string_index] == ' ') {
while(string[*string_index] == ' ') {
(*string_index)++;
}
if (strlen(token) != 0) break;
}
if (string[*string_index] == ',') {
if (strlen(token) == 0) {
token[0] = string[*string_index];
(*string_index)++;
}
break;
}
if (string[*string_index] != '\n') token[i] = string[*string_index];
}
return token;
}
void GetStringLiteral(char* text, unsigned long length, char** token, unsigned long* position) {
if (!text) return;
if (!token) return;
for(int i = 0; *position < length; (*position)++, i++) {
if (text[*position] == '"') {
(*position)++;
break;
}
(*token)[i] = text[*position];
}
}
// TokenType GetOperatorType(char c) {
// switch (c){
// case '+':
// return TK_Add;
// case '-':
// return TK_Sub;
// case '*':
// return TK_Mul;
// case '/':
// return TK_Div;
// case '^':
// return TK_Power;
// case ':':
// return TK_Colon;
// case '(':
// return TK_LParam;
// case ')':
// return TK_RParam;
// case '[':
// return TK_LBracket;
// case ']':
// return TK_RBracket;
// };
// return TK_Invalid;
// }