Created a tokenizer and the supporting code, currently processes text that is not a string literal.

Makefile

@@ -0,0 +1,41 @@
+CC = gcc
+CFLAGS=-g -Wall -DDEBUG -Wpedantic -Wextra -Wunused-result -std=c99 -pedantic-errors
+SRCDIR=src
+OBJDIR=obj
+SRCS=$(wildcard $(SRCDIR)/*.c)
+# Substitute all .c with .o from SRCS
+OBJS=$(patsubst $(SRCDIR)/%.c, $(OBJDIR)/%.o, $(SRCS))
+
+BINDIR=bin
+BIN=$(BINDIR)/assm
+
+all: $(BIN)
+
+release: CFLAGS=-Wall -Wpedantic -std=c99 -pedantic-errors -O2
+release: clean
+release: $(BIN)
+
+$(BIN): $(OBJS) $(BINDIR)
+	$(CC) $(CFLAGS) $(OBJS) -o $@
+
+$(OBJDIR)/%.o: $(SRCDIR)/%.c $(OBJDIR)
+	$(CC) $(CFLAGS) -c $< -o $@
+
+$(BINDIR):
+	mkdir $@
+
+$(OBJDIR):
+	mkdir $@
+
+.PHONY: clean
+.PHONY: test
+.PHONY: disass
+
+clean:
+	rm -rf $(BINDIR)/* $(OBJDIR)/*
+
+test:
+	$(BIN) misc/another_test.asm
+
+disass:
+	objdump -S --disassemble $(OBJDIR)/$(FILE).o > $(OBJDIR)/$(FILE).s