Ported Lahiru's tokneizer code over. Appears to work fine. Also did some house keeping.

2020-04-07 23:54:55 -05:00
parent d0e36ff2ce
commit c760dc04b5
4 changed files with 153 additions and 85 deletions
@@ -0,0 +1 @@
+__pycache__
@@ -1,47 +1,8 @@
 import token
+import tokenizer
+#userInput = input('Enter your formula: ')
+tokens = tokenizer.get_tokens_from_expression_string("1 + 2 / 3 (3 * 4) 5")

-def flatten_input(input):
-    tokens = []
-    tmp = ""
-    parsing_number = False
-
-    for i in range(0, len(input)):
-        c = input[i]
-        if c == '.' or c.isdigit():
-            if len(tokens) > 1:
-                lookBehind = tokens[len(tokens) - 1]
-
-                if lookBehind == token.TokenType.exp_start:
-                    tokens.append(token.Token("^", token.TokenType.power))
-            if (i + 1) == len(input):
-                tokens.append(token.Token(tmp + c, token.TokenType.constant))
-
-            parsing_number = True
-            tmp = tmp + c
-            continue
-
-    print(tokens)
-
-def operate(n1, n2, tokenType):
-    if tokenType == token.TokenType.add:
-        print("Adding %d and %d to get %d." %(n1, n2, n1 + n2))
-    elif tokenType == token.TokenType.subtract:
-        print("Subtracting %d and %d to get %d." %(n1, n2, n1 - n2))
-    elif tokenType == token.TokenType.multiply:
-        print("Multiplying %d and %d to get %d" %(n1, n2, n1 * n2))
-    elif tokenType == token.TokenType.divide:
-        print("Dividing %d and %d to get %d" %(n1, n2, n1 / n2))
-    elif tokenType == token.TokenType.power:
-        print("Raising %d to the power of %d to get %d" %(n1, n2, n1**n2))
-    else:
-        raise ValueError(f"Invalid operator value.")
-
-#t1 = token.TokenType()
-#t = token.Token("+", token.TokenType.add)
-#print(t1.get_operator_verb(t.type))
-operate(10, 2, 4)
-operate(10, 2, 0)
-operate(10, 2, 1)
-operate(10, 2, 2)
-operate(10, 2, 3)
-flatten_input("THis is 96.")
+for t in tokens:
+	print("%s (%s) " %(t.value, token.TokenType.get_operator_name(t.type)), end = "")
+print()
@@ -1,46 +1,64 @@
-class Token:
-    def __init__(self, value, type):
-        self.value = value
-        self.type = type
+class CToken:
+	def __init__(self, value, type):
+		self.value = value
+		self.type = type

 class TokenType:
-    add = 0
-    subtract = 1
-    multiply = 2
-    divide = 3
-    power = 4
-    variable = 5
-    constant = 6
-    exp_start = 7
-    exp_end = 8
-    unknown = 9
+	add = 0
+	subtract = 1
+	multiply = 2
+	divide = 3
+	power = 4
+	variable = 5
+	constant = 6
+	exp_start = 7
+	exp_end = 8
+	unknown = 9

-    OPERATOR_VERBS = {
-        0 : "Adding",
-        1 : "Subtracting",
-        2 : "Multiplying",
-        3 : "Dividing",
-        4 : "Raising"
-    }
+	OPERATOR_VERBS = {
+		0 : "Adding",
+		1 : "Subtracting",
+		2 : "Multiplying",
+		3 : "Dividing",
+		4 : "Raising"
+	}
+	
+	TOKEN_NAMES = {
+		0 : "Addition",
+		1 : "Subtraction",
+		2 : "Multiplication",
+		3 : "Division",
+		4 : "Power",
+		5 : "Variable",
+		6 : "Constant",
+		7 : "Expression Start",
+		8 : "Expression End",
+		9 : "Unknown"
+	}

-    OPERATORS = {
-        '+' : add,
-        '-' : subtract,
-        '*' : multiply,
-        '^' : power,
-        '/' : divide
-    }
+	OPERATORS = {
+		'+' : add,
+		'-' : subtract,
+		'*' : multiply,
+		'^' : power,
+		'/' : divide
+	}

-    def get_operator_verb(self, token):
-        if not token in self.OPERATOR_VERBS:
-            raise ValueError("An operator verb could not be found.", "token")
-        return self.OPERATOR_VERBS[token]
+	@staticmethod
+	def get_operator_verb(token):
+		if not token in TokenType.OPERATOR_VERBS:
+			raise ValueError("An operator verb could not be found.", "token")
+		return TokenType.OPERATOR_VERBS[token]

-    def get_operator(self, character):
-        if not character in self.OPERATORS:
-            raise ValueError(f"Invalid operator '{character}' could not be found in the operators list.", "character")
-        return self.OPERATORS[character]
-
-#t = TokenType()
-#t1 = Token("$", TokenType.Unknown)
-#print(t.GetOperator(t1.value))
+	@staticmethod
+	def get_operator(character):
+		"""
+        Accepts an individual character and returns either the math operator TokenType or unknown.
+        """
+		if not character in TokenType.OPERATORS:
+			return TokenType.unknown
+		return TokenType.OPERATORS[character]
+	
+	@staticmethod
+	def get_operator_name(tokenType):
+		return TokenType.TOKEN_NAMES[tokenType]
@@ -0,0 +1,88 @@
+import token
+
+def get_tokens_from_expression_string(expression_string):
+	"""
+	Takes user input of an actuarial formula and parses the formula to id its components.
+	:return: tokens (List of objects of CToken class).
+	"""
+	tokens = []  # List of objects of CToken class
+	tmp = ""
+	parsing_number = False
+	symbols_dic = {}  # Dictionary to keep track of the no. of times each variable appears.
+
+	for i in range(0, len(expression_string)):
+		c = expression_string[i]
+		if c == '.' or c.isdigit():
+			if tmp.count('.') == 1:
+				raise ValueError("Invalid grammar, to many periods in number!")
+			if len(tokens) > 1:
+				lookBehind = tokens[-1]
+				if lookBehind.type == token.TokenType.exp_end:
+					tokens.append(token.CToken("^", token.TokenType.power))
+			if (i + 1) == len(expression_string):
+				tokens.append(token.CToken(tmp + c, token.TokenType.constant))
+			parsing_number = True
+			tmp = tmp + c
+			continue
+		elif c == '(':
+			if parsing_number:
+				tokens.append(token.CToken(tmp, token.TokenType.constant))
+				tokens.append(token.CToken("*", token.TokenType.multiply))
+			tokens.append(token.CToken("(", token.TokenType.exp_start))
+			parsing_number = False
+			tmp = ""
+			continue
+		elif c == ')':
+			if parsing_number:				
+				tokens.append(token.CToken(tmp, token.TokenType.constant))
+			tokens.append(token.CToken(")", token.TokenType.exp_end))
+			parsing_number = False
+			tmp = ""
+			continue
+		elif token.TokenType.get_operator(c) != token.TokenType.unknown:
+			if parsing_number:
+				tokens.append(token.CToken(tmp, token.TokenType.constant))
+			tokens.append(token.CToken(c, token.TokenType.get_operator(c)))
+			parsing_number = False
+			tmp = ""
+			continue
+		elif c != ' ':
+			if len(tokens) > 1:
+				lookBehind = tokens[-1]
+				if lookBehind.type == TokenType.exp_end:
+					tokens.append(token.CToken("^", token.TokenType.power))
+			if (i + 1) < len(expression_string):
+				lookAHead = expression_string[i + 1]
+				if lookAHead == '(':
+					tokens.append(token.CToken(c, token.TokenType.variable))
+					tokens.append(token.CToken("*", token.TokenType.multiply))
+					tmp = ""
+					parsing_number = False
+					continue
+			if parsing_number:
+				tokens.append(token.CToken(tmp, token.TokenType.constant))
+				tokens.append(token.CToken("*", token.TokenType.multiply))
+			if c in symbols_dic.keys():
+				symbols_dic[c] += 1
+			else:
+				symbols_dic[c] = 1
+			tokens.append(token.CToken(c + str(symbols_dic[c]), token.TokenType.variable))
+			parsing_number = False
+			tmp = ""
+
+	return tokens
+
+def operate(n1, n2, tokenType):
+	if tokenType == token.TokenType.add:
+		print("Adding %d and %d to get %d." %(n1, n2, n1 + n2))
+	elif tokenType == token.TokenType.subtract:
+		print("Subtracting %d and %d to get %d." %(n1, n2, n1 - n2))
+	elif tokenType == token.TokenType.multiply:
+		print("Multiplying %d and %d to get %d" %(n1, n2, n1 * n2))
+	elif tokenType == token.TokenType.divide:
+		print("Dividing %d and %d to get %d" %(n1, n2, n1 / n2))
+	elif tokenType == token.TokenType.power:
+		print("Raising %d to the power of %d to get %d" %(n1, n2, n1**n2))
+	else:
+		raise ValueError("Invalid operator value.")
+