glm94/assm-test
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Included a check for commas in the whitespace splitter. Might rename that function but this is all just testing before refactoring so I won't rename things just yet.

Browse Source
This commit is contained in:
glm94
2022-01-20 19:48:24 +00:00
parent 99ebd82996
commit 23d4bd8089
1 changed files with 26 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/tokenizer.c
+26 -46
View File
@@ -8,7 +8,6 @@
List* TokenizeLine(char *); List* TokenizeLine(char *);
TokenType GetOperatorType(char); TokenType GetOperatorType(char);
char* SplitOnWhiteSpace(char*); char* SplitOnWhiteSpace(char*);
unsigned long position = 0;
List* TokenizeString(const char *file_path) { List* TokenizeString(const char *file_path) {
FILE *file; FILE *file;
@@ -44,71 +43,52 @@ List* TokenizeLine(char *line) {
if (strlen(line) == 0) return NULL; if (strlen(line) == 0) return NULL;
if (line[0] == ';') return NULL; if (line[0] == ';') return NULL;
//List* tokens = CreateList();
//Token* token = malloc(sizeof(Token));
//unsigned long length = strlen(line) + 1;
//token->value = calloc(1, length);
//char* currentWord = malloc(length);
//char* token = strtok(line, " ");
//while(token != NULL) {
// if (token[0] == '"') {
// strcpy(currentWord, token);
// token = strtok(NULL, " ");
// if (token == NULL) {
// printf("Expect double quote.\n");
// break;
// }
// strcat(currentWord, token);
// printf("String token '%s' found\n", currentWord);
// }
// token = strtok(NULL, " ");
//token
//}
char* token = SplitOnWhiteSpace(line); char* token = SplitOnWhiteSpace(line);
while (strlen(token) != 0) { while (strlen(token) != 0) {
printf("Token: '%s'\n", token); printf("Token: '%s'\n", token);
token = SplitOnWhiteSpace(line); //if (token[0] == '"') printf("String token: '%s'\n", token);
free(token);
token = SplitOnWhiteSpace(NULL);
} }
position = 0;
//free(currentWord); //free(currentWord);
free(token); free(token);
//int index = 0;
// for (int i = 0; i < length; i++) {
// if (line[i] == ' ') {
// if (index == 0) continue;
// printf("Found a spcae\n");
// currentWord[index + 1] = '\0';
// token->type = TK_Text;
// strcpy(token->value, currentWord);
// PushListItem(token, sizeof(Token), tokens);
// }
// currentWord[index] = line[i];
// index++;
// }
return NULL; return NULL;
} }
char* SplitOnWhiteSpace(char* line) { char* SplitOnWhiteSpace(char* line) {
char* token = calloc(1, strlen(line) + 1); static char* string;
static unsigned long position;
if (line) {
string = line;
position = 0;
}
char* token = calloc(1, strlen(string) + 1);
int parsing_string = 0; int parsing_string = 0;
for (int i = 0; position < strlen(line); i++, position++) { for (int i = 0; position < strlen(string); i++, position++) {
if (line[position] == '"') { if (string[position] == '"') {
parsing_string = 1; parsing_string = 1;
} }
if (line[position] == ' ' && !parsing_string) { if (string[position] == ' ' && !parsing_string) {
position++; position++;
break; break;
} }
if (line[position] != '\n') token[i] = line[position]; if (string[position] == ',' && !parsing_string) {
if (strlen(token) == 0) {
token[0] = string[position];
position++;
}
break;
}
if (string[position] != '\n') token[i] = string[position];
} }
token[strlen(token)] = '\0'; token[strlen(token)] = '\0';