From c760dc04b54904d466d4cee6b798dafefd1259dc Mon Sep 17 00:00:00 2001 From: Garritt McCune Date: Tue, 7 Apr 2020 23:54:55 -0500 Subject: [PATCH] Ported Lahiru's tokneizer code over. Appears to work fine. Also did some house keeping. --- .gitignore | 1 + main.py | 51 ++++----------------------- token.py | 98 +++++++++++++++++++++++++++++++--------------------- tokenizer.py | 88 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 153 insertions(+), 85 deletions(-) create mode 100644 .gitignore create mode 100644 tokenizer.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bee8a64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/main.py b/main.py index a0a7a24..c40b03f 100755 --- a/main.py +++ b/main.py @@ -1,47 +1,8 @@ import token +import tokenizer +#userInput = input('Enter your formula: ') +tokens = tokenizer.get_tokens_from_expression_string("1 + 2 / 3 (3 * 4) 5") -def flatten_input(input): - tokens = [] - tmp = "" - parsing_number = False - - for i in range(0, len(input)): - c = input[i] - if c == '.' or c.isdigit(): - if len(tokens) > 1: - lookBehind = tokens[len(tokens) - 1] - - if lookBehind == token.TokenType.exp_start: - tokens.append(token.Token("^", token.TokenType.power)) - if (i + 1) == len(input): - tokens.append(token.Token(tmp + c, token.TokenType.constant)) - - parsing_number = True - tmp = tmp + c - continue - - print(tokens) - -def operate(n1, n2, tokenType): - if tokenType == token.TokenType.add: - print("Adding %d and %d to get %d." %(n1, n2, n1 + n2)) - elif tokenType == token.TokenType.subtract: - print("Subtracting %d and %d to get %d." %(n1, n2, n1 - n2)) - elif tokenType == token.TokenType.multiply: - print("Multiplying %d and %d to get %d" %(n1, n2, n1 * n2)) - elif tokenType == token.TokenType.divide: - print("Dividing %d and %d to get %d" %(n1, n2, n1 / n2)) - elif tokenType == token.TokenType.power: - print("Raising %d to the power of %d to get %d" %(n1, n2, n1**n2)) - else: - raise ValueError(f"Invalid operator value.") - -#t1 = token.TokenType() -#t = token.Token("+", token.TokenType.add) -#print(t1.get_operator_verb(t.type)) -operate(10, 2, 4) -operate(10, 2, 0) -operate(10, 2, 1) -operate(10, 2, 2) -operate(10, 2, 3) -flatten_input("THis is 96.") +for t in tokens: + print("%s (%s) " %(t.value, token.TokenType.get_operator_name(t.type)), end = "") +print() diff --git a/token.py b/token.py index 5822fc0..f538c60 100755 --- a/token.py +++ b/token.py @@ -1,46 +1,64 @@ -class Token: - def __init__(self, value, type): - self.value = value - self.type = type +class CToken: + def __init__(self, value, type): + self.value = value + self.type = type class TokenType: - add = 0 - subtract = 1 - multiply = 2 - divide = 3 - power = 4 - variable = 5 - constant = 6 - exp_start = 7 - exp_end = 8 - unknown = 9 + add = 0 + subtract = 1 + multiply = 2 + divide = 3 + power = 4 + variable = 5 + constant = 6 + exp_start = 7 + exp_end = 8 + unknown = 9 - OPERATOR_VERBS = { - 0 : "Adding", - 1 : "Subtracting", - 2 : "Multiplying", - 3 : "Dividing", - 4 : "Raising" - } + OPERATOR_VERBS = { + 0 : "Adding", + 1 : "Subtracting", + 2 : "Multiplying", + 3 : "Dividing", + 4 : "Raising" + } + + TOKEN_NAMES = { + 0 : "Addition", + 1 : "Subtraction", + 2 : "Multiplication", + 3 : "Division", + 4 : "Power", + 5 : "Variable", + 6 : "Constant", + 7 : "Expression Start", + 8 : "Expression End", + 9 : "Unknown" + } - OPERATORS = { - '+' : add, - '-' : subtract, - '*' : multiply, - '^' : power, - '/' : divide - } + OPERATORS = { + '+' : add, + '-' : subtract, + '*' : multiply, + '^' : power, + '/' : divide + } - def get_operator_verb(self, token): - if not token in self.OPERATOR_VERBS: - raise ValueError("An operator verb could not be found.", "token") - return self.OPERATOR_VERBS[token] + @staticmethod + def get_operator_verb(token): + if not token in TokenType.OPERATOR_VERBS: + raise ValueError("An operator verb could not be found.", "token") + return TokenType.OPERATOR_VERBS[token] - def get_operator(self, character): - if not character in self.OPERATORS: - raise ValueError(f"Invalid operator '{character}' could not be found in the operators list.", "character") - return self.OPERATORS[character] - -#t = TokenType() -#t1 = Token("$", TokenType.Unknown) -#print(t.GetOperator(t1.value)) + @staticmethod + def get_operator(character): + """ + Accepts an individual character and returns either the math operator TokenType or unknown. + """ + if not character in TokenType.OPERATORS: + return TokenType.unknown + return TokenType.OPERATORS[character] + + @staticmethod + def get_operator_name(tokenType): + return TokenType.TOKEN_NAMES[tokenType] diff --git a/tokenizer.py b/tokenizer.py new file mode 100644 index 0000000..2ea9810 --- /dev/null +++ b/tokenizer.py @@ -0,0 +1,88 @@ +import token + +def get_tokens_from_expression_string(expression_string): + """ + Takes user input of an actuarial formula and parses the formula to id its components. + :return: tokens (List of objects of CToken class). + """ + tokens = [] # List of objects of CToken class + tmp = "" + parsing_number = False + symbols_dic = {} # Dictionary to keep track of the no. of times each variable appears. + + for i in range(0, len(expression_string)): + c = expression_string[i] + if c == '.' or c.isdigit(): + if tmp.count('.') == 1: + raise ValueError("Invalid grammar, to many periods in number!") + if len(tokens) > 1: + lookBehind = tokens[-1] + if lookBehind.type == token.TokenType.exp_end: + tokens.append(token.CToken("^", token.TokenType.power)) + if (i + 1) == len(expression_string): + tokens.append(token.CToken(tmp + c, token.TokenType.constant)) + parsing_number = True + tmp = tmp + c + continue + elif c == '(': + if parsing_number: + tokens.append(token.CToken(tmp, token.TokenType.constant)) + tokens.append(token.CToken("*", token.TokenType.multiply)) + tokens.append(token.CToken("(", token.TokenType.exp_start)) + parsing_number = False + tmp = "" + continue + elif c == ')': + if parsing_number: + tokens.append(token.CToken(tmp, token.TokenType.constant)) + tokens.append(token.CToken(")", token.TokenType.exp_end)) + parsing_number = False + tmp = "" + continue + elif token.TokenType.get_operator(c) != token.TokenType.unknown: + if parsing_number: + tokens.append(token.CToken(tmp, token.TokenType.constant)) + tokens.append(token.CToken(c, token.TokenType.get_operator(c))) + parsing_number = False + tmp = "" + continue + elif c != ' ': + if len(tokens) > 1: + lookBehind = tokens[-1] + if lookBehind.type == TokenType.exp_end: + tokens.append(token.CToken("^", token.TokenType.power)) + if (i + 1) < len(expression_string): + lookAHead = expression_string[i + 1] + if lookAHead == '(': + tokens.append(token.CToken(c, token.TokenType.variable)) + tokens.append(token.CToken("*", token.TokenType.multiply)) + tmp = "" + parsing_number = False + continue + if parsing_number: + tokens.append(token.CToken(tmp, token.TokenType.constant)) + tokens.append(token.CToken("*", token.TokenType.multiply)) + if c in symbols_dic.keys(): + symbols_dic[c] += 1 + else: + symbols_dic[c] = 1 + tokens.append(token.CToken(c + str(symbols_dic[c]), token.TokenType.variable)) + parsing_number = False + tmp = "" + + return tokens + +def operate(n1, n2, tokenType): + if tokenType == token.TokenType.add: + print("Adding %d and %d to get %d." %(n1, n2, n1 + n2)) + elif tokenType == token.TokenType.subtract: + print("Subtracting %d and %d to get %d." %(n1, n2, n1 - n2)) + elif tokenType == token.TokenType.multiply: + print("Multiplying %d and %d to get %d" %(n1, n2, n1 * n2)) + elif tokenType == token.TokenType.divide: + print("Dividing %d and %d to get %d" %(n1, n2, n1 / n2)) + elif tokenType == token.TokenType.power: + print("Raising %d to the power of %d to get %d" %(n1, n2, n1**n2)) + else: + raise ValueError("Invalid operator value.") +