diff --git a/app.py b/app.py index a1f08a3..7c98061 100644 --- a/app.py +++ b/app.py @@ -1,11 +1,11 @@ +import configparser from datetime import datetime from os import getenv import json import logging import logging.config -from flask import Flask, request, render_template, abort -import configparser +from flask import Flask, request, render_template from db_client import DbClient as TransitClient from db_client_transform import DbClient as TransformClient @@ -21,10 +21,10 @@ def read_config() -> configparser.ConfigParser: - conf = configparser.ConfigParser() - with open("config/config.ini") as f: - conf.read_file(f) - return conf + config = configparser.ConfigParser() + with open("config/config.ini", encoding="utf-8") as f: + config.read_file(f) + return config def read_vault_token(): @@ -43,7 +43,7 @@ def health(): @app.route("/customers", methods=["GET"]) def get_customers(): customers = dbc.get_customer_records() - logger.debug("Customers: {}".format(customers)) + logger.debug(f"Customers: {customers}") return json.dumps(customers) @@ -62,26 +62,26 @@ def get_customer(): @app.route("/customers", methods=["POST"]) def create_customer(): - logging.debug("Form Data: {}".format(dict(request.form))) - customer = {k: v for (k, v) in dict(request.form).items()} + logging.debug(f"Form Data: {dict(request.form)}") + customer = dict(dict(request.form).items()) for k, v in customer.items(): - if type(v) is list: + if isinstance(v, list): customer[k] = v[0] - logging.debug("Customer: {}".format(customer)) + logging.debug(f"Customer: {customer}") if "create_date" not in customer.keys(): customer["create_date"] = datetime.now().isoformat() new_record = dbc.insert_customer_record(customer) - logging.debug("New Record: {}".format(new_record)) + logging.debug(f"New Record: {new_record}") return json.dumps(new_record) @app.route("/customers", methods=["PUT"]) def update_customer(): - logging.debug("Form Data: {}".format(dict(request.form))) - customer = {k: v for (k, v) in dict(request.form).items()} - logging.debug("Customer: {}".format(customer)) + logging.debug(f"Form Data: {dict(request.form)}") + customer = dict(dict(request.form).items()) + logging.debug(f"Customer: {customer}") new_record = dbc.update_customer_record(customer) - logging.debug("New Record: {}".format(new_record)) + logging.debug(f"New Record: {new_record}") return json.dumps(new_record) @@ -91,7 +91,7 @@ def index(): @app.route("/records", methods=["GET"]) -def records(): +def get_records(): records = json.loads(get_customers()) return render_template("records.html", results=records) @@ -128,18 +128,16 @@ def update_submit(): ) -def init_vault(conf): - global dbc - +def init_vault(conf) -> TransitClient: + client = TransitClient() if not conf.has_section("VAULT") or conf["VAULT"]["Enabled"].lower() == "false": - return + return client - dbc = TransitClient() if ( - conf.has_option("VAULT","Transform") + conf.has_option("VAULT", "Transform") and conf["VAULT"]["Transform"].lower() == "true" ): - dbc = TransformClient() + client = TransformClient() vault_token = "" if conf["VAULT"]["InjectToken"].lower() == "true": @@ -148,7 +146,7 @@ def init_vault(conf): else: vault_token = conf["VAULT"]["Token"] - dbc.init_vault( + client.init_vault( addr=conf["VAULT"]["Address"], token=vault_token, namespace=conf["VAULT"]["Namespace"], @@ -157,50 +155,49 @@ def init_vault(conf): ) if ( - conf.has_option("VAULT","Transform") + conf.has_option("VAULT", "Transform") and conf["VAULT"]["Transform"].lower() == "true" ): logger.info("Using Transform database client...") - dbc.init_transform( + client.init_transform( transform_path=conf["VAULT"]["TransformPath"], ssn_role=conf["VAULT"]["SSNRole"], transform_masking_path=conf["VAULT"]["TransformMaskingPath"], ccn_role=conf["VAULT"]["CCNRole"], ) + if ( + conf.has_option("VAULT", "database_auth") + and conf["VAULT"]["database_auth"] != "" + ): + client.vault_db_auth(conf["VAULT"]["database_auth"]) - - if conf.has_option("VAULT", "database_auth") and conf["VAULT"]["database_auth"] != "": - dbc.vault_db_auth(conf["VAULT"]["database_auth"]) + return client if __name__ == "__main__": logger.warning("In Main...") - conf = read_config() - + app_config = read_config() logging.basicConfig( - level=log_level[conf["DEFAULT"]["LogLevel"]], + level=log_level[app_config["DEFAULT"]["LogLevel"]], format="%(asctime)s - %(levelname)8s - %(name)9s - %(funcName)15s - %(message)s", ) try: - init_vault(conf) + dbc = init_vault(app_config) if not dbc.is_initialized: logger.info("Using DB credentials from config.ini...") dbc.init_db( - uri=conf["DATABASE"]["Address"], - prt=conf["DATABASE"]["Port"], - uname=conf["DATABASE"]["User"], - pw=conf["DATABASE"]["Password"], - db=conf["DATABASE"]["Database"], - ) - appPort = conf["DEFAULT"]["port"] - logger.info( - "Starting Flask server on {} listening on port {}".format( - "0.0.0.0", appPort + uri=app_config["DATABASE"]["Address"], + prt=app_config["DATABASE"]["Port"], + uname=app_config["DATABASE"]["User"], + pw=app_config["DATABASE"]["Password"], + db=app_config["DATABASE"]["Database"], ) - ) - app.run(host="0.0.0.0", port=appPort) + APP_HOST = "0.0.0.0" + appPort = app_config["DEFAULT"]["port"] + logger.info(f"Starting Flask server on {APP_HOST} listening on port {appPort}") + app.run(host=APP_HOST, port=appPort) except Exception as e: - logging.error("There was an error starting the server: {}".format(e)) + logging.error(f"There was an error starting the server: {e}") diff --git a/db_client.py b/db_client.py index 6f149d0..afca1e9 100644 --- a/db_client.py +++ b/db_client.py @@ -6,7 +6,7 @@ from mysql.connector import errorcode import hvac -customer_table = """ +CUSTOMER_TABLE = """ CREATE TABLE IF NOT EXISTS `customers` ( `cust_no` int(11) NOT NULL AUTO_INCREMENT, `birth_date` varchar(255) NOT NULL, @@ -20,7 +20,7 @@ PRIMARY KEY (`cust_no`) ) ENGINE=InnoDB;""" -seed_customers = """ +SEED_CUSTOMERS = """ INSERT IGNORE into customers VALUES (2, "3/14/69", "Larry", "Johnson", "2020-01-01T14:49:12.301977", "360-56-6750", "3600-5600-6750-0000", "Tyler, Texas", "7000000"), (40, "11/26/69", "Shawn", "Kemp", "2020-02-21T10:24:55.985726", "235-32-8091", "2350-3200-8091-0001", "Elkhart, Indiana", "15000000"), @@ -31,14 +31,18 @@ class DbClient: - conn = None + conn: mysql.connector.MySQLConnection = None + uri: str = None + port: int = None + username: str = None + password: str = None + db: str = None + vault_client: hvac.Client = None - key_name = None - mount_point = None - username = None - password = None - namespace = None - is_initialized = False + key_name: str = None + mount_point: str = None + namespace: str = None + is_initialized: bool = False def init_db(self, uri, prt, uname, pw, db): self.connect_db(uri, prt, uname, pw) @@ -49,9 +53,7 @@ def connect_db(self, uri, prt, uname, pw): for i in range(0, 10): try: logger.debug( - "Connecting to {}:{} with username {} and password {}".format( - uri, prt, uname, pw - ) + f"Connecting to {uri}:{prt} with username {uname} and password {pw}" ) self.conn = mysql.connector.connect( user=uname, password=pw, host=uri, port=prt @@ -77,17 +79,17 @@ def connect_db(self, uri, prt, uname, pw): def _init_database(self, db): cursor = self.conn.cursor() - logger.info("Preparing database {}...".format(db)) - cursor.execute("CREATE DATABASE IF NOT EXISTS `{}`".format(db)) - cursor.execute("USE `{}`".format(db)) + logger.info(f"Preparing database {db}...") + cursor.execute(f"CREATE DATABASE IF NOT EXISTS `{db}`") + cursor.execute(f"USE `{db}`") logger.info("Preparing customer table...") - cursor.execute(customer_table) - cursor.execute(seed_customers) + cursor.execute(CUSTOMER_TABLE) + cursor.execute(SEED_CUSTOMERS) self.conn.commit() cursor.close() self.db = db self.is_initialized = True - + def get_namespace(self): return self.namespace @@ -96,7 +98,7 @@ def init_vault(self, addr, token, namespace, path, key_name): if not addr or not token: logger.warning("Skipping initialization...") return - logger.warning("Connecting to vault server: {}".format(addr)) + logger.warning(f"Connecting to vault server: {addr}") self.vault_client = hvac.Client( url=addr, token=token, namespace=namespace, verify=False ) @@ -109,7 +111,7 @@ def init_vault(self, addr, token, namespace, path, key_name): key_name = None self.key_name = key_name self.mount_point = path - logger.debug("Initialized vault_client: {}".format(self.vault_client)) + logger.debug(f"Initialized vault_client: {self.vault_client}") def vault_db_auth(self, path): try: @@ -117,15 +119,11 @@ def vault_db_auth(self, path): self.username = resp["data"]["username"] self.password = resp["data"]["password"] logger.debug( - "Retrieved username {} and password {} from Vault.".format( - self.username, self.password - ) + f"Retrieved username {self.username} and password {self.password} from Vault." ) except Exception as e: logger.error( - "An error occurred reading DB creds from path {}. Error: {}".format( - path, e - ) + f"An error occurred reading DB creds from path {path}. Error: {e}" ) # the data must be base64ed before being passed to encrypt @@ -136,45 +134,44 @@ def encrypt(self, value): name=self.key_name, plaintext=base64.b64encode(value.encode()).decode("ascii"), ) - logger.debug("Response: {}".format(response)) + logger.debug(f"Response: {response}") return response["data"]["ciphertext"] except Exception as e: - logger.error("There was an error encrypting the data: {}".format(e)) + logger.error(f"There was an error encrypting the data: {e}") raise e # The data returned from Transit is base64 encoded so we decode it before returning def decrypt(self, value): # support unencrypted messages on first read - logger.debug("Decrypting {}".format(value)) + logger.debug(f"Decrypting {value}") if not value.startswith("vault:v"): return value - else: - try: - response = self.vault_client.secrets.transit.decrypt_data( - mount_point=self.mount_point, name=self.key_name, ciphertext=value - ) - logger.debug("Response: {}".format(response)) - plaintext = response["data"]["plaintext"] - logger.debug("Plaintext (base64 encoded): {}".format(plaintext)) - decoded = base64.b64decode(plaintext).decode() - logger.debug("Decoded: {}".format(decoded)) - return decoded - except Exception as e: - logger.error("There was an error encrypting the data: {}".format(e)) - raise e + try: + response = self.vault_client.secrets.transit.decrypt_data( + mount_point=self.mount_point, name=self.key_name, ciphertext=value + ) + logger.debug(f"Response: {response}") + plaintext = response["data"]["plaintext"] + logger.debug(f"Plaintext (base64 encoded): {plaintext}") + decoded = base64.b64decode(plaintext).decode() + logger.debug(f"Decoded: {decoded}") + return decoded + except Exception as e: + logger.error(f"There was an error encrypting the data: {e}") + raise e # Long running apps may expire the DB connection def _execute_sql(self, sql, cursor): try: cursor.execute(sql) - return 1 except mysql.connector.errors.OperationalError as error: if error[0] == 2006: - logger.error("Error encountered: {}. Reconnecting db...".format(error)) + logger.error(f"Error encountered: {error}. Reconnecting db...") self.init_db(self.uri, self.port, self.username, self.password, self.db) cursor = self.conn.cursor() cursor.execute(sql) return 0 + return 1 def process_customer(self, row, raw=None): r = {} @@ -198,7 +195,7 @@ def process_customer(self, row, raw=None): def get_customer_records(self, num=None, raw=None): if num is None: num = 50 - statement = "SELECT * FROM `customers` LIMIT {}".format(num) + statement = f"SELECT * FROM `customers` LIMIT {num}" cursor = self.conn.cursor() self._execute_sql(statement, cursor) results = [] @@ -207,11 +204,11 @@ def get_customer_records(self, num=None, raw=None): r = self.process_customer(row, raw) results.append(r) except Exception as e: - logger.error("There was an error retrieving the record: {}".format(e)) + logger.error(f"There was an error retrieving the record: {e}") return results - def get_customer_record(self, id): - statement = "SELECT * FROM `customers` WHERE cust_no = {}".format(id) + def get_customer_record(self, cid): + statement = f"SELECT * FROM `customers` WHERE cust_no = {cid}" cursor = self.conn.cursor() self._execute_sql(statement, cursor) results = [] @@ -220,39 +217,35 @@ def get_customer_record(self, id): r = self.process_customer(row) results.append(r) except Exception as e: - logger.error("There was an error retrieving the record: {}".format(e)) + logger.error(f"There was an error retrieving the record: {e}") return results def get_insert_sql(self, record) -> str: - if self.vault_client is None and self.key_name == None: - return """INSERT INTO `customers` (`birth_date`, `first_name`, `last_name`, `create_date`, `social_security_number`, `credit_card_number`, `address`, `salary`) - VALUES ("{}", "{}", "{}", "{}", "{}", "{}", "{}", "{}");""".format( - record["birth_date"], - record["first_name"], - record["last_name"], - record["create_date"], - record["ssn"], - record["ccn"], - record["address"], - record["salary"], - ) + if self.vault_client is None and self.key_name is None: + return f"""INSERT INTO `customers` (`birth_date`, `first_name`, `last_name`, `create_date`, `social_security_number`, `credit_card_number`, `address`, `salary`) + VALUES ("{record["birth_date"]}", + "{record["first_name"]}", + "{record["last_name"]}", + "{record["create_date"]}", + "{record["ssn"]}", + "{record["ccn"]}", + "{record["address"]}", + "{record["salary"]}");""" - return """INSERT INTO `customers` (`birth_date`, `first_name`, `last_name`, `create_date`, `social_security_number`, `credit_card_number`, `address`, `salary`) - VALUES ("{}", "{}", "{}", "{}", "{}", "{}", "{}","{}");""".format( - self.encrypt(record["birth_date"]), - record["first_name"], - record["last_name"], - record["create_date"], - self.encrypt(record["ssn"]), - self.encrypt(record["ccn"]), - self.encrypt(record["address"]), - self.encrypt(record["salary"]), - ) + return f"""INSERT INTO `customers` (`birth_date`, `first_name`, `last_name`, `create_date`, `social_security_number`, `credit_card_number`, `address`, `salary`) + VALUES ("{self.encrypt(record["birth_date"])}", + "{record["first_name"]}", + "{record["last_name"]}", + "{record["create_date"]}", + "{self.encrypt(record["ssn"])}", + "{self.encrypt(record["ccn"])}", + "{self.encrypt(record["address"])}", + "{self.encrypt(record["salary"])}");""" def insert_customer_record(self, record): statement = self.get_insert_sql(record) - logger.debug("SQL Statement: {}".format(statement)) + logger.debug(f"SQL Statement: {statement}") cursor = self.conn.cursor() self._execute_sql(statement, cursor) self.conn.commit() @@ -260,34 +253,29 @@ def insert_customer_record(self, record): def get_update_sql(self, record) -> str: if self.vault_client is None: - return """UPDATE `customers` - SET birth_date = "{}", first_name = "{}", last_name = "{}", social_security_number = "{}", credit_card_number = "{}", address = "{}", salary = "{}" - WHERE cust_no = {};""".format( - record["birth_date"], - record["first_name"], - record["last_name"], - record["ssn"], - record["ccn"], - record["address"], - record["salary"], - record["cust_no"], - ) - return """UPDATE `customers` - SET birth_date = "{}", first_name = "{}", last_name = "{}", social_security_number = "{}", credit_card_number = "{}", address = "{}", salary = "{}" - WHERE cust_no = {};""".format( - self.encrypt(record["birth_date"]), - record["first_name"], - record["last_name"], - self.encrypt(record["ssn"]), - self.encrypt(record["ccn"]), - self.encrypt(record["address"]), - self.encrypt(record["salary"]), - record["cust_no"], - ) + return f"""UPDATE `customers` + SET birth_date = "{record["birth_date"]}", + first_name = "{record["first_name"]}", + last_name = "{record["last_name"]}", + social_security_number = "{record["ssn"]}", + credit_card_number = "{record["ccn"]}", + address = "{record["address"]}", + salary = "{record["salary"]}" + WHERE cust_no = {record["cust_no"]};""" + + return f"""UPDATE `customers` + SET birth_date = "{self.encrypt(record["birth_date"])}", + first_name = "{record["first_name"]}", + last_name = "{record["last_name"]}", + social_security_number = "{self.encrypt(record["ssn"])}", + credit_card_number = "{self.encrypt(record["ccn"])}", + address = "{self.encrypt(record["address"])}", + salary = "{self.encrypt(record["salary"])}" + WHERE cust_no = {record["cust_no"]};""" def update_customer_record(self, record): statement = self.get_update_sql(record) - logger.debug("Sql Statement: {}".format(statement)) + logger.debug(f"Sql Statement: {statement}") cursor = self.conn.cursor() self._execute_sql(statement, cursor) self.conn.commit() diff --git a/db_client_transform.py b/db_client_transform.py index 6f69d6a..2616413 100644 --- a/db_client_transform.py +++ b/db_client_transform.py @@ -23,7 +23,7 @@ def init_transform( self.transform_masking_mount_point = transform_masking_path self.ssn_role = ssn_role self.ccn_role = ccn_role - logger.debug("Initialized transform: {}".format(self.vault_client)) + logger.debug(f"Initialized transform: {self.vault_client}") def encode_ssn(self, value): try: @@ -49,11 +49,13 @@ def encode_ssn(self, value): "cache-control": "no-cache", } - response = requests.request("POST", url, data=payload, headers=headers) - logger.debug("Response: {}".format(response.text)) + response = requests.request("POST", url, data=payload, headers=headers, timeout=300) + logger.debug(f"Response: {response.text}") return response.json()["data"]["encoded_value"] except Exception as e: - logger.error("There was an error encrypting the data: {}".format(e)) + logger.error(f"There was an error encrypting the data: {e}") + + return "" def encode_ccn(self, value): try: @@ -79,15 +81,17 @@ def encode_ccn(self, value): "cache-control": "no-cache", } - response = requests.request("POST", url, data=payload, headers=headers) - logger.debug("Response: {}".format(response.text)) + response = requests.request("POST", url, data=payload, headers=headers, timeout=300) + logger.debug(f"Response: {response.text}") return response.json()["data"]["encoded_value"] except Exception as e: - logger.error("There was an error encrypting the data: {}".format(e)) + logger.error(f"There was an error encrypting the data: {e}") + + return "" def decode_ssn(self, value): # we're going to have funny stuff if ProtectRecords is false - logger.debug("Decoding {}".format(value)) + logger.debug(f"Decoding {value}") try: # transform not available in hvac, raw api call url = ( @@ -111,11 +115,13 @@ def decode_ssn(self, value): "cache-control": "no-cache", } - response = requests.request("POST", url, data=payload, headers=headers) - logger.debug("Response: {}".format(response.text)) + response = requests.request( + "POST", url, data=payload, headers=headers, timeout=300 + ) + logger.debug(f"Response: {response.text}") return response.json()["data"]["decoded_value"] except Exception as e: - logger.error("There was an error decoding the data: {}".format(e)) + logger.error(f"There was an error decoding the data: {e}") return None def process_customer(self, row, raw=None): @@ -140,31 +146,27 @@ def get_insert_sql(self, record) -> str: if self.vault_client is None: return super().get_insert_sql(record) - return """INSERT INTO `customers` (`birth_date`, `first_name`, `last_name`, `create_date`, `social_security_number`, `credit_card_number`, `address`, `salary`) - VALUES ("{}", "{}", "{}", "{}", "{}", "{}", "{}","{}");""".format( - self.encrypt(record["birth_date"]), - record["first_name"], - record["last_name"], - record["create_date"], - self.encode_ssn(record["ssn"]), - self.encode_ccn(record["ccn"]), - self.encrypt(record["address"]), - self.encrypt(record["salary"]), - ) + return f"""INSERT INTO `customers` (`birth_date`, `first_name`, `last_name`, `create_date`, + `social_security_number`, `credit_card_number`, `address`, `salary`) + VALUES ("{self.encrypt(record["birth_date"])}", + "{record["first_name"]}", + "{record["last_name"]}", + "{record["create_date"]}", + "{self.encode_ssn(record["ssn"])}", + "{self.encode_ccn(record["ccn"])}", + "{self.encrypt(record["address"])}", + "{self.encrypt(record["salary"])}");""" def get_update_sql(self, record) -> str: if self.vault_client is None: return super().get_update_sql(record) - return """UPDATE `customers` - SET birth_date = "{}", first_name = "{}", last_name = "{}", social_security_number = "{}", credit_card_number = "{}", address = "{}", salary = "{}" - WHERE cust_no = {};""".format( - self.encrypt(record["birth_date"]), - record["first_name"], - record["last_name"], - self.encode_ssn(record["ssn"]), - self.encode_ccn(record["ccn"]), - self.encrypt(record["address"]), - self.encrypt(record["salary"]), - record["cust_no"], - ) + return f"""UPDATE `customers` + SET birth_date = "{self.encrypt(record["birth_date"])}", f + irst_name = "{record["first_name"]}", + last_name = "{record["last_name"]}", + social_security_number = "{self.encode_ssn(record["ssn"])}", + credit_card_number = "{self.encode_ccn(record["ccn"])}", + address = "{self.encrypt(record["address"])}", + salary = "{self.encrypt(record["salary"])}" + WHERE cust_no = {record["cust_no"]};"""