From 1d1dfea7d3682feb1ffc87fbd659e1f798313741 Mon Sep 17 00:00:00 2001 From: Joey <17505625+jmctune@users.noreply.github.com> Date: Sat, 30 Nov 2024 11:29:46 -0600 Subject: [PATCH] refactor: config; strip some windows deps (#278) - Creates a `UserConfig` class that makes it easier to interact with the user_settings.ini file. - Removes windows pop up messages - Working on trying to strip out windows dependencies where appropriate - Typing - Add tests for `UserConfig` class --- app/common/config.py | 102 ++++++++++++++++++++++ app/common/errors.py | 11 --- app/common/translate.py | 169 +++--------------------------------- app/common/update.py | 128 ++++++++++++--------------- app/main.py | 9 +- app/tests/test_config.py | 54 ++++++++++++ app/tests/test_translate.py | 28 ------ 7 files changed, 225 insertions(+), 276 deletions(-) create mode 100644 app/common/config.py create mode 100644 app/tests/test_config.py diff --git a/app/common/config.py b/app/common/config.py new file mode 100644 index 0000000..b3c7f06 --- /dev/null +++ b/app/common/config.py @@ -0,0 +1,102 @@ +from common.lib import get_project_root +from loguru import logger as log + +import configparser +import os + + +class UserConfig: + def __init__(self, filepath: str = None, warnings: bool = False) -> dict: + if not filepath: + settings_file = get_project_root("user_settings.ini") + else: + settings_file = f"{filepath}/user_settings.ini" + + self.filepath = filepath + self.file = settings_file + self.warnings = warnings + self.config = self.read() + self.service = self.eval_translation_service() + + if self.service: + self.key = self.eval_translation_key() + + self.game_path = self.config['config'].get('installdirectory') + + + def read(self) -> dict: + base_config = configparser.ConfigParser() + base_config["translation"] = { + "enabledeepltranslate": False, + "deepltranslatekey": "", + "enablegoogletranslate": False, + "googletranslatekey": "", + } + base_config["config"] = { + "installdirectory": "" + } + + # Create the config if it doesn't exist. + if not os.path.exists(self.file): + with open(self.file, "w+") as configfile: + base_config.write(configfile) + + if self.warnings: + log.warning( + "user_settings.ini was not found, so one was created for you. " + "You will need to fill in the appropriate values and restart this program " + "to pick up your changes." + ) + + # Compare user's config with base config to ensure all sections and keys exist. + user_config = configparser.ConfigParser() + user_config.read(self.file) + + for section in base_config.sections(): + if section not in user_config.sections(): + log.exception(f"{section} section missing from user_settings.ini.") + for key, _ in base_config.items(section): + if key not in user_config[section]: + log.exception(f"{key} missing from {section} in user_settings.ini.") + + return user_config + + + def reinit(self) -> dict: + # update class instance with new values read from file written by this method. + return self.__init__(self.filepath) + + + def update(self, section: str, key: str, value: str) -> None: + config = configparser.ConfigParser() + config.read(self.file) + config.set(section, key, value) + with open(self.file, "w+") as configfile: + config.write(configfile) + + + def eval_translation_service(self) -> str: + if self.config['translation'].getboolean('enabledeepltranslate'): + return "deepl" + if self.config['translation'].getboolean('enablegoogletranslate'): + return "google" + + if self.warnings: + log.warning("You did not enable a translation service, so no live translation will be performed.") + + return "" + + + def eval_translation_key(self) -> str: + service = self.eval_translation_service() + if service == "deepl": + if key := self.config['translation'].get('deepltranslatekey'): + return key + if service == "google": + if key := self.config['translation'].get('googletranslatekey'): + return key + + if self.warnings: + log.exception(f"You enabled {service}, but did not specify a key.") + + return "" diff --git a/app/common/errors.py b/app/common/errors.py index 8491508..5913ff1 100644 --- a/app/common/errors.py +++ b/app/common/errors.py @@ -1,14 +1,3 @@ -import ctypes -import sys - - -def message_box(title: str, message: str, exit_prog: bool = False): - """Generate a topmost message box.""" - ctypes.windll.user32.MessageBoxW(0, message, f"[dqxclarity] {title}", 0x1000) # 0x1000 ensures a topmost window - if exit_prog: - sys.exit() - - class ClarityError(Exception): """Base class for exceptions.""" diff --git a/app/common/translate.py b/app/common/translate.py index 63f3b3c..cea60cd 100644 --- a/app/common/translate.py +++ b/app/common/translate.py @@ -1,15 +1,11 @@ +from common.config import UserConfig from common.db_ops import generate_glossary_dict, generate_m00_dict, init_db -from common.errors import message_box -from common.lib import get_project_root from googleapiclient.discovery import build -import configparser import deepl import langdetect -import os import pykakasi import re -import shutil import textwrap import unicodedata @@ -17,25 +13,20 @@ class Translate(): service = None api_key = None - region_code = None glossary = None def __init__(self): if Translate.service is None: - self.user_settings = load_user_config() - self.translation_settings = determine_translation_service() - Translate.service = self.translation_settings["TranslateService"] - Translate.api_key = self.translation_settings["TranslateKey"] - Translate.region_code = self.translation_settings["RegionCode"] + self.user_settings = UserConfig() + Translate.service = self.user_settings.service + Translate.api_key = self.user_settings.key if Translate.glossary is None: Translate.glossary = generate_glossary_dict() - def deepl(self, text: list): - region_code = Translate.region_code - if region_code.lower() == "en": - region_code = "en-us" + def deepl(self, text: list) -> list: + region_code = "en-us" translator = deepl.Translator(Translate.api_key) response = translator.translate_text( text=text, @@ -49,9 +40,10 @@ def deepl(self, text: list): return text_results - def google(self, text: list): + def google(self, text: list) -> list: + region_code = "en" service = build("translate", "v2", developerKey=Translate.api_key) - response = service.translations().list(source="ja", target="en", format="text", q=text).execute() + response = service.translations().list(source="ja", target=region_code, format="text", q=text).execute() text_results = [] for result in response["translations"]: text_results.append(result["translatedText"]) @@ -227,7 +219,7 @@ def __add_line_endings(self, text: str) -> str: return output - def translate(self, text: list): + def translate(self, text: list) -> list: """Translates a list of strings, passing them through our glossary first. @@ -243,7 +235,6 @@ def translate(self, text: list): return self.deepl(text) if Translate.service == "google": return self.google(text) - return None def sanitize_and_translate(self, text: str, wrap_width: int, max_lines=None, add_brs=True): @@ -465,146 +456,6 @@ def sanitize_and_translate(self, text: str, wrap_width: int, max_lines=None, add return pristine_str -def load_user_config(filepath: str = None): - """Returns a user's config settings. If the config doesn't exist, a default - config is generated. If the user's config is missing values, we back up the - old config and generate a new default one for them. - - :param filepath: Path to the user_settings.ini file. Don't include - the filename or trailing forward slash. - :returns: Dict of config. - """ - if not filepath: - filepath = get_project_root("user_settings.ini") - else: - filepath = f"{filepath}/user_settings.ini" - base_config = configparser.ConfigParser() - base_config["translation"] = { - "enabledeepltranslate": False, - "deepltranslatekey": "", - "enablegoogletranslate": False, - "googletranslatekey": "", - "regioncode": "en", - } - base_config["config"] = {"installdirectory": ""} - - def create_base_config(): - with open(filepath, "w+") as configfile: - base_config.write(configfile) - - # Create the config if it doesn't exist - if not os.path.exists(filepath): - create_base_config() - - # Verify the integrity of the config. If a key is missing, - # trigger user_config_state and create a new one, backing - # up the old config. - user_config = configparser.ConfigParser() - user_config_state = 0 - user_config.read(filepath) - for section in base_config.sections(): - if section not in user_config.sections(): - user_config_state = 1 - break - for key, val in base_config.items(section): - if key not in user_config[section]: - user_config_state = 1 - break - - # Notify user their config is busted - if user_config_state == 1: - shutil.copyfile(filepath, "user_settings.invalid") - create_base_config() - message_box( - title="New config created", - message=f"We found a missing config value in your user_settings.ini.\n\nYour old config has been renamed to user_settings.invalid in case you need to reference it.\n\nPlease relaunch dqxclarity.", - exit_prog=True, - ) - - config_dict = {} - good_config = configparser.ConfigParser() - good_config.read(filepath) - for section in good_config.sections(): - config_dict[section] = {} - for key, val in good_config.items(section): - config_dict[section][key] = val - - return config_dict - - -def update_user_config(section: str, key: str, value: str, filename="user_settings.ini"): - """Updates an existing configuration option in a user's config. - - :param section: Section of the config - :param key: Key in the section's config - :param value: Value of the key - :param filename: Filename of the user's config settings. - """ - config = configparser.ConfigParser() - config.read(filename) - config.set(section, key, value) - with open(filename, "w+") as configfile: - config.write(configfile) - - -def determine_translation_service(communication_window_enabled=False): - """Parses the user_settings file to get information needed to make - translation calls. - - :param communication_window_enabled: If True, will verify that a - service is enabled and a key is entered. - """ - config = load_user_config() - enabledeepltranslate = eval(config["translation"]["enabledeepltranslate"]) - deepltranslatekey = config["translation"]["deepltranslatekey"] - enablegoogletranslate = eval(config["translation"]["enablegoogletranslate"]) - googletranslatekey = config["translation"]["googletranslatekey"] - regioncode = config["translation"]["regioncode"] - - reiterate = "Either open the user_settings.ini file in Notepad or use the API settings button in the DQXClarity launcher to set it up." - - if enabledeepltranslate and enablegoogletranslate: - message_box( - title="Too many translation services enabled", - message=f"Only enable one translation service. {reiterate}\n\nCurrent values:\n\nenabledeepltranslate: {enabledeepltranslate}\nenablegoogletranslate: {enablegoogletranslate}", - exit_prog=True, - ) - - if enabledeepltranslate and deepltranslatekey == "": - message_box( - title="No DeepL key specified", - message=f"DeepL is enabled, but no key was provided. {reiterate}", - exit_prog=True, - ) - - if enablegoogletranslate and googletranslatekey == "": - message_box( - title="No Google API key specified", - message=f"Google API is enabled, but no key was provided. {reiterate}", - exit_prog=True, - ) - - if communication_window_enabled: - if not enabledeepltranslate and not enablegoogletranslate: - message_box( - title="No translation service is configured", - message=f"You enabled API translation, but didn't enable a service. Please configure a service and relaunch. {reiterate}", - exit_prog=True, - ) - - dic = {} - if enabledeepltranslate: - dic["TranslateService"] = "deepl" - dic["TranslateKey"] = deepltranslatekey - elif enablegoogletranslate: - dic["TranslateService"] = "google" - dic["TranslateKey"] = googletranslatekey - - dic["RegionCode"] = regioncode - - return dic - - def clean_up_and_return_items(text: str) -> str: """Cleans up unnecessary text from item strings and searches for the name in items.json. diff --git a/app/common/update.py b/app/common/update.py index e89d339..d895082 100644 --- a/app/common/update.py +++ b/app/common/update.py @@ -1,3 +1,4 @@ +from common.config import UserConfig from common.constants import ( GITHUB_CLARITY_CUTSCENE_JSON_URL, GITHUB_CLARITY_DAT1_URL, @@ -11,9 +12,7 @@ GITHUB_CUSTOM_TRANSLATIONS_ZIP_URL, ) from common.db_ops import db_query -from common.errors import message_box from common.process import check_if_running_as_admin, is_dqx_process_running -from common.translate import load_user_config, update_user_config from io import BytesIO from loguru import logger as log from openpyxl import load_workbook @@ -24,12 +23,11 @@ import json import os import requests -import shutil import sys import winreg -def download_custom_files(): +def download_custom_files() -> None: log.info("Downloading custom translation files from dqx-translation-project/dqx-custom-translations.") response = requests.get(GITHUB_CUSTOM_TRANSLATIONS_ZIP_URL, timeout=15) @@ -65,7 +63,7 @@ def download_custom_files(): log.exception(f"Status Code: {response.status_code}. Reason: {response.reason}") -def read_custom_json_and_import(name: str, data: str): +def read_custom_json_and_import(name: str, data: str) -> None: content = json.loads(data) query_list = [] @@ -81,7 +79,7 @@ def read_custom_json_and_import(name: str, data: str): db_query(query) -def download_game_jsons(): +def download_game_jsons() -> None: log.info("Downloading translation files from dqx-translation-project/dqx_translations.") # dqx_translations is roughly 17MB~ right now. we only need these files from that repository. @@ -103,7 +101,7 @@ def download_game_jsons(): log.exception(f"Status Code: {response.status_code}. Reason: {response.reason}") -def check_for_updates(update: bool): +def check_for_updates(update: bool) -> None: """Checks to see if Clarity is running the latest version of itself. If not, will launch updater.py and exit. @@ -147,7 +145,7 @@ def check_for_updates(update: bool): return -def read_xlsx_and_import(data: str): +def read_xlsx_and_import(data: str) -> None: """We manage a file outside of this repository called merge.xlsx in dqx- custom-translations. @@ -166,7 +164,7 @@ def read_xlsx_and_import(data: str): db_query("DELETE FROM fixed_dialog_template") values = [] - for i, row in enumerate(ws_dialogue, start=2): + for i, _ in enumerate(ws_dialogue, start=2): source_text = ws_dialogue.cell(row=i, column=1).value en_text = ws_dialogue.cell(row=i, column=3).value notes = ws_dialogue.cell(row=i, column=4).value @@ -196,7 +194,7 @@ def read_xlsx_and_import(data: str): # Walkthrough worksheet values = [] - for i, row in enumerate(ws_walkthrough, start=2): + for i, _ in enumerate(ws_walkthrough, start=2): source_text = ws_walkthrough.cell(row=i, column=1).value en_text = ws_walkthrough.cell(row=i, column=3).value @@ -211,7 +209,7 @@ def read_xlsx_and_import(data: str): # Quests worksheet values = [] - for i, row in enumerate(ws_quests, start=2): + for i, _ in enumerate(ws_quests, start=2): source_text = ws_quests.cell(row=i, column=1).value en_text = ws_quests.cell(row=i, column=3).value @@ -226,7 +224,7 @@ def read_xlsx_and_import(data: str): # Story So Far worksheet values = [] - for i, row in enumerate(ws_story_so_far, start=2): + for i, _ in enumerate(ws_story_so_far, start=2): source_text = ws_story_so_far.cell(row=i, column=1).value deepl_text = ws_story_so_far.cell(row=i, column=2).value fixed_en_text = ws_story_so_far.cell(row=i, column=3).value @@ -244,7 +242,7 @@ def read_xlsx_and_import(data: str): db_query(query) -def read_glossary_and_import(data: str): +def read_glossary_and_import(data: str) -> None: decoded_data = data.decode('utf-8') query_list = [] @@ -266,94 +264,76 @@ def read_glossary_and_import(data: str): db_query(insert_query) -def download_dat_files(): - """Verifies the user's DQX install location and prompts them to locate it - if not found. - - Uses this location to download the latest data files from the - dqxclarity repo. - """ +def download_dat_files() -> None: + """Downloads and applies the dat translation mod to the user's DQX + directory.""" if is_dqx_process_running(): - message = "Please close DQX before attempting to update the translated DAT/IDX file." - log.error(message) - message_box( - title="DQXGame.exe is open", - message=message - ) + log.exception("Please close DQX before attempting to update the translated DAT/IDX file.") if not check_if_running_as_admin(): - message = "dqxclarity must be running as an administrator in order to update the translated DAT/IDX file. Please re-launch dqxclarity as an administrator and try again." - log.error(message) - message_box( - title="Program not elevated", - message=message, - exit_prog=True + log.exception( + "dqxclarity must be running as an administrator in order to apply the dat translation mod. " + "Please re-launch dqxclarity as an administrator and try again." ) - config = load_user_config() - dat0_file = "data00000000.win32.dat0" - idx0_file = "data00000000.win32.idx" + config = UserConfig() + read_game_path = "/".join([config.game_path, "Game/Content/Data", "data00000000.win32.dat0"]) - install_directory = config["config"]["installdirectory"] + if not os.path.exists(read_game_path): + default_game_path = "C:/Program Files (x86)/SquareEnix/DRAGON QUEST X" - valid_path = False - if install_directory: - if os.path.isdir(install_directory): - log.success("DQX game path is valid.") - valid_path = True - - if not valid_path: - default_path = 'C:/Program Files (x86)/SquareEnix/DRAGON QUEST X' - if os.path.exists(default_path): - update_user_config('config', 'installdirectory', default_path) + if os.path.exists(default_game_path): + config.update(section='config', key='installdirectory', value=default_game_path) else: - message_box( - title="Could not find DQX directory", - message="Browse to the path where you installed the game and select the \"DRAGON QUEST X\" folder. \ - \n\nMake sure you didn't move the data00000000.dat0 file outside of the game directory or rename it." + log.warning( + "Could not verify DRAGON QUEST X directory. " + "Browse to the path where you installed the game and select the \"DRAGON QUEST X\" folder. " + "Make sure you didn't move the data00000000.dat0 file outside of the game directory or rename it. " + "If the file is missing, make sure you patch the game first before running this program." ) while True: dqx_path = askdirectory() + if not dqx_path: - message_box( - title="Canceled", - message="Operation has been canceled. dqxclarity will exit.", - exit_prog=True - ) - dat0_path = "/".join([dqx_path, "Game/Content/Data", dat0_file]) + log.error("You did not select a directory or closed the window. Program will exit.") + + dat0_path = "/".join([dqx_path, "Game/Content/Data", "data00000000.win32.dat0"]) if os.path.isfile(dat0_path): - update_user_config('config', 'installdirectory', dqx_path) - log.success("DQX path verified.") + config.update(section='config', key='installdirectory', value=dqx_path) + log.success("DRAGON QUEST X path verified.") break else: - message_box( - title="Invalid Directory", - message="The path you provided is not a valid DQX path.\nBrowse to the path where you installed the game and select the \"DRAGON QUEST X\" folder." + log.warning( + "The path you provided is not a valid path. " + "Browse to the path where you installed the game and select the \"DRAGON QUEST X\" folder." ) - config = load_user_config() # call this again in case we made changes above - dqx_path = "/".join([config['config']['installdirectory'], "Game/Content/Data"]) - idx0_path = "/".join([dqx_path, idx0_file]) - - if not os.path.isfile(f"{idx0_path}.bak"): - log.info(f"Did not find a backup of existing idx file. Backing up and renaming to {idx0_file}.bak") - shutil.copy(idx0_path, f"{idx0_path}.bak") + config.reinit() # re-read config in case we changed it above. + dqx_path = "/".join([config.game_path, "Game/Content/Data"]) try: log.info("Downloading DAT1 and IDX files.") - dat_request = requests.get(GITHUB_CLARITY_DAT1_URL, timeout=10) - idx_request = requests.get(GITHUB_CLARITY_IDX_URL, timeout=10) + dat_request = requests.get(GITHUB_CLARITY_DAT1_URL, timeout=60) + idx_request = requests.get(GITHUB_CLARITY_IDX_URL, timeout=60) # Make sure both requests are good before we write the files - if dat_request.status_code == 200 and idx_request.status_code == 200: + if ( + (dat_request.status_code == 200 and len(dat_request.content) != 0) and + (idx_request.status_code == 200 and len(idx_request.content) != 0) + ): with open(dqx_path + "/data00000000.win32.dat1", "w+b") as f: f.write(dat_request.content) + with open(dqx_path + "/data00000000.win32.idx", "w+b") as f: f.write(idx_request.content) - log.success("Translation files downloaded.") + + log.success("Game dat translation mod applied.") else: - log.error("Failed to download translation files. Clarity will continue without updating translation files") + log.error( + "Failed to download translation files. " + "dqxclarity will continue without updating dat translation mod." + ) except Exception as e: - log.error(f"Failed to download data files. Error: {e}") + log.error(f"Failed to download dat translation mod files. Error: {e}") diff --git a/app/main.py b/app/main.py index 393eedc..6a88e96 100644 --- a/app/main.py +++ b/app/main.py @@ -1,4 +1,5 @@ from clarity import loop_scan_for_walkthrough, run_scans +from common.config import UserConfig from common.db_ops import ( create_db_schema, fix_m00_tables_schema, @@ -6,7 +7,6 @@ ) from common.lib import get_project_root, setup_logging from common.process import wait_for_dqx_to_launch -from common.translate import determine_translation_service from common.update import ( check_for_updates, download_custom_files, @@ -59,6 +59,10 @@ def blast_off( create_db_schema() sync_existing_tables() + # we don't do anything with the config here, but this will validate the config is ok before running. + log.info("Checking user_settings.ini.") + UserConfig(warnings=True) + if update_dat: log.info("Updating DAT mod.") download_dat_files() @@ -68,9 +72,6 @@ def blast_off( download_custom_files() download_game_jsons() - log.info("Checking user_settings.ini.") - determine_translation_service(communication_window_enabled=communication_window) - try: wait_for_dqx_to_launch() diff --git a/app/tests/test_config.py b/app/tests/test_config.py new file mode 100644 index 0000000..8ea289a --- /dev/null +++ b/app/tests/test_config.py @@ -0,0 +1,54 @@ +import os +import shutil +import sys + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) +from common.config import * + +import unittest + + +class TestConfig(unittest.TestCase): + @classmethod + def tearDownClass(cls) -> None: + if os.path.exists('user_settings.ini'): + os.remove('user_settings.ini') + + + def test_init_user_config(self) -> None: + _ = UserConfig('.') + self.assertTrue(os.path.exists('user_settings.ini')) + + + def test_update_user_config(self) -> None: + config = UserConfig('.') + config.update(section='translation', key='deepltranslatekey', value='abcd1234') + config.update(section='translation', key='enabledeepltranslate', value='True') + + config.reinit() + + self.assertTrue(config.config._sections['translation']['enabledeepltranslate'] == 'True') + self.assertTrue(config.config._sections['translation']['deepltranslatekey'] == 'abcd1234') + + + def test_eval_translation_service(self) -> None: + config = UserConfig('.') + config.update(section='translation', key='enabledeepltranslate', value='True') + + config.reinit() + + self.assertTrue(config.service == 'deepl') + + + def test_eval_translation_key(self) -> None: + config = UserConfig('.') + config.update(section='translation', key='enabledeepltranslate', value='True') + config.update(section='translation', key='deepltranslatekey', value='abcd1234') + + config.reinit() + + self.assertTrue(config.key == 'abcd1234') + + +if __name__ == '__main__': + unittest.main() diff --git a/app/tests/test_translate.py b/app/tests/test_translate.py index a4fa649..6540450 100644 --- a/app/tests/test_translate.py +++ b/app/tests/test_translate.py @@ -1,5 +1,4 @@ import os -import shutil import sys sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) @@ -9,11 +8,6 @@ class TestTranslate(unittest.TestCase): - @classmethod - def tearDownClass(cls) -> None: - if os.path.exists('user_settings.ini'): - os.remove('user_settings.ini') - def test_detect_lang(self): ja_str = 'ショウブは　近くのものを指差した！' @@ -39,28 +33,6 @@ def test_transliterate_player_name(self): self.assertTrue(result == 'Fuanshii') - def test_load_user_config(self): - config = load_user_config('.') - - # test that a new file was created - self.assertTrue(os.path.exists('user_settings.ini')) - - # test that keys were created - self.assertTrue(config['translation']) - self.assertTrue(config['config']) - - - def test_load_update_user_config(self): - shutil.copy('../../user_settings.ini', '.') - update_user_config(section='translation', key='deepltranslatekey', value='abcd1234') - config = load_user_config('.') - self.assertTrue(config['translation']['deepltranslatekey'] == 'abcd1234') - - - def test_determine_translation_service(self): - pass # tbd - - def test_clean_up_and_return_items(self): pass # tbd