Repo clean up and changed the tokenizer so it doesn't stringify tokens with a line break.

2020-07-26 02:11:04 -05:00
parent fc7f1e0dbf
commit 9c30eea95f
9 changed files with 2 additions and 530 deletions
@@ -133,5 +133,4 @@ def stringify_token_list(tokens):
 	text = ''
 	for tk in tokens:
 		text += str(tk.value)
-	#text += '\n'
 	return text