From e774623eb7ad67618a3cacae6e4ef6a065cc1201 Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Sat, 18 Mar 2023 12:24:09 +1100 Subject: [PATCH 1/9] Update docs to match new translation output behavior - See #12 --- README.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 2b3a291..73c3f42 100644 --- a/README.md +++ b/README.md @@ -83,11 +83,11 @@ which should show an error message and list available chapters Then run -```python build.py onikakushi --translation``` +```python build.py onikakushi``` Then the output files will be located in the `output/translation` folder. You can then merge the `HigurashiEp0X_Data` folder with the one in your release. **Please include all the files (not just the `sharedassets0.assets` file), so the installer can select the correct file at install time.** -If you want to rebuild all chapters, run `python build.py all --translation` to build all chapters. +If you want to rebuild all chapters, run `python build.py all` to build all chapters. ### Common Problems @@ -99,15 +99,11 @@ You may encounter the following problems: **NOTE: The script should automatically detect if the vanilla assets or UABE has changed, and re-download them. But if that doesn't work, use the '--force-download' option like so:** -```python build.py rei --translation --force-download``` +```python build.py rei --force-download``` ## Instructions for Dev Team -For our dev team, the instructions are nearly the same, just remove the `--translation` argument. - -```python build.py onikakushi``` - -Archive files will be automatically created in the `output` folder +Instructions are the same as for translators, but archive files will be automatically created in the `output` folder ---- @@ -133,6 +129,8 @@ Click on the 'Actions' tab to observe the build process. Once the build is complete, go to the 'Releases' page, and a new draft release should appear. You can check everything is OK before publishing the release, or just download the files without publishing the release. +Note that doing this will build both the `translation.7z` file for translators to use, and also the individual archives for the 07th-mod developers to use. + ### Building `ui-compiler.exe` using Github Actions To build just the `ui-compiler.exe` using Github Actions (on Github's server), push any tag to the repository. From 0c383072568e5da779d939cbca424b63e2d6b24a Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Sun, 9 Jul 2023 12:00:57 +1000 Subject: [PATCH 2/9] Rpevent running script on python 2.7, add warning on python 3.11 --- build.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build.py b/build.py index 086c069..28da706 100644 --- a/build.py +++ b/build.py @@ -240,6 +240,13 @@ def save(self): with open(LastModifiedManager.savePath, 'w') as handle: json.dump(self.lastModifiedDict, handle) +if sys.version_info < (2, 7): + print(">>>> ERROR: This script does not work on Python 2.7") + exit(-1) + +if sys.version_info > (3, 10): + print(">>>> WARNING: This script probably does not work on Python 3.11 because unitypack uses old version of decrunch which does not build. Use Python 3.10 or below if you have this error.") + lastModifiedManager = LastModifiedManager() # Parse command line arguments From 8f3128ff86167976a7b03508a4538f7bf63bd1c7 Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Sun, 9 Jul 2023 12:03:34 +1000 Subject: [PATCH 3/9] Add note about Unix/MacOS sharedassets compatability --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 73c3f42..bf909c0 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,12 @@ The following information is only used when adding support for a new episode. Please look through the detailed documentation, especially if you're working on a new chapter, new language, or using UABE - this file does not contain information on those topics. +### WARNING about Unix/MacOS `sharedassets0.assets` + +We've found that the MacOS sharedassets can be used on Linux, **but the Linux sharedassets CANNOT be used on MacOS in certain cases**, giving you the "purple text" issue. + +For this reason, whenever a new chapter is prepared, the 'vanilla' unix `sharedassets0.assets` should be taken from the MacOS version. + ### Preparing font files You'll need to extract the 'msgothic' font files from the stock `.assets` file before starting: From 5f4039370646fb29455e00049ad65518d5ad2a4e Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Sun, 9 Jul 2023 17:00:08 +1000 Subject: [PATCH 4/9] Remove unused Ch.8 Matsuribayashi sharedassets variant - I re-tested the "GOG Linux" mod with the normal linux sharedassets and it works, so the special variant is not is not required. - I think this is because we upgrade the GOG/MG Linux variant anyway, which means no special sharedassets is required (maybe it was required in the past, but not now) - If there are still issues, we should fix it by upgrading the variant, rather than creating new sharedassets - See https://github.com/07th-mod/ui-editing-scripts/issues/16 - See https://github.com/orian34/matsuribayashi/issues/9 - See https://github.com/07th-mod/python-patcher/issues/128 --- build.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index 28da706..77e29f5 100644 --- a/build.py +++ b/build.py @@ -113,11 +113,19 @@ def get_translation_sharedassets_name(self) -> str: # 'matsuribayashi 5.6.7f1 unix' ], "matsuribayashi": [ - BuildVariant("M_GOG-M_MG-Steam", "matsuribayashi", "2017.2.5", "unix"), + # Based on the GOG MacOS sharedassets, but works on Linux too. + # Working on: + # - Linux Steam (2023-07-09) + # - Linux GOG (2023-07-09) + # - MacOS GOG (2023-07-09) + BuildVariant("GOG-MG-Steam", "matsuribayashi", "2017.2.5", "unix"), + + # NOTE: I'm 99% certain this file is no longer used, as we just upgrade the entire GOG/Mangagamer game # Special version for GOG/Mangagamer Linux with SHA256: # A200EC2A85349BC03B59C8E2F106B99ED0CBAAA25FC50928BB8BA2E2AA90FCE9 # CRC32L 51100D6D - BuildVariant("L_GOG-L_MG", "matsuribayashi", "2017.2.5", "unix", "51100D6D"), + # BuildVariant("GOG-MG", "matsuribayashi", "2017.2.5", "unix", "51100D6D"), # TO BE REMOVED + BuildVariant("GOG-MG-Steam", "matsuribayashi", "2017.2.5", "win", translation_default=True), ], 'rei': [ From d7ab0edcb70bf6e6ef88d88e34e79291bebbfd20 Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Sun, 9 Jul 2023 20:09:38 +1000 Subject: [PATCH 5/9] Update old compileall.sh to remove unused sharedassets configurations --- compileall.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/compileall.sh b/compileall.sh index 765a530..091b0b7 100644 --- a/compileall.sh +++ b/compileall.sh @@ -15,13 +15,10 @@ cargo run meakashi 5.5.3p1 win && \ cargo run meakashi 5.5.3p1 unix && \ cargo run tsumihoroboshi 5.5.3p3 win && \ cargo run tsumihoroboshi 5.5.3p3 unix && \ -cargo run tsumihoroboshi 5.6.7f1 win && \ cargo run minagoroshi 5.6.7f1 win && \ cargo run minagoroshi 5.6.7f1 unix && \ cargo run matsuribayashi 5.6.7f1 win && \ cargo run matsuribayashi 5.6.7f1 unix && \ -cargo run matsuribayashi 2017.2.5 unix && \ -cargo run matsuribayashi 2017.2.5 win && \ cargo run rei 2019.4.3 win && \ cargo run rei 2019.4.4 win && \ cargo run rei 2019.4.3 unix && \ From 0306beb4b54a09422e6a8bb9bac8773bf735aa71 Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Fri, 14 Jul 2023 20:04:24 +1000 Subject: [PATCH 6/9] Fix double exception if discriminator provided when it is not needed --- scripts/UnityTextModifier.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/UnityTextModifier.py b/scripts/UnityTextModifier.py index 10084ad..69119c1 100644 --- a/scripts/UnityTextModifier.py +++ b/scripts/UnityTextModifier.py @@ -72,14 +72,14 @@ def findInAssetBundle(self, bundle): offsets.append(offset) start = offset + 1 if len(offsets) == 0: - raise IndexError(f"No asset found for {self.shortString}") + raise IndexError(f"WARNING: No asset found for {self.shortString}") if self.discriminator == None: if len(offsets) > 1: - raise IndexError(f"Multiple assets found for {self.shortString}, candidates are " + ", ".join(f"{index}: 0x{offset:x}" for index, offset in enumerate(offsets)) + ". Please add a field like 'Discriminator: 0' to indicate which block should apply to which asset (do NOT use quotes around the number, do not use the raw address). For an example, see https://github.com/07th-mod/higurashi-dev-guides/wiki/UI-editing-scripts#unitytextmodifier") + raise IndexError(f"WARNING: Multiple assets found for {self.shortString}, candidates are " + ", ".join(f"{index}: 0x{offset:x}" for index, offset in enumerate(offsets)) + ". Please add a field like 'Discriminator: 0' to indicate which block should apply to which asset (do NOT use quotes around the number, do not use the raw address). For an example, see https://github.com/07th-mod/higurashi-dev-guides/wiki/UI-editing-scripts#unitytextmodifier") self.offset = offsets[0] else: if len(offsets) <= self.discriminator: - raise IndexError(f"Not enough offsets found for ${self.trimmedE} / {self.trimmedJ} to meet request for #{self.discriminator}, there were only {len(offsets)}") + raise IndexError(f"WARNING: Not enough offsets found for {self.shortString} to meet request for #{self.discriminator}, there were only {len(offsets)}") self.offset = offsets[self.discriminator] def checkObject(self, id, object, bundle): @@ -113,6 +113,7 @@ def __str__(self): try: return f"" except: return "" +warning_count = 0 with open(sys.argv[2], encoding="utf-8") as jsonFile: edits = [ScriptEdit.fromJSON(x) for x in json.load(jsonFile)] @@ -127,6 +128,8 @@ def __str__(self): print(f"Found {edit.shortString} at offset 0x{edit.offset:x}") except IndexError as e: print(e) + warning_count =+ 1 + edits = newEdits assetsFile.seek(0) @@ -136,3 +139,6 @@ def __str__(self): edit.checkObject(id, obj, bundle) for edit in edits: edit.write(sys.argv[3]) + +if warning_count > 0: + print(">>>>>>>> ONE OR MORE WARNINGS OCCURRED, please check logs! <<<<<<<<<") From a91dac2a3c8cd0b154c2ee88f6b7470b0aa68914 Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Fri, 14 Jul 2023 20:10:39 +1000 Subject: [PATCH 7/9] Add script for merging multiple text-edits together --- .../MergeMultipleTextEdits.py | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 scripts/Manually Run Scripts/MergeMultipleTextEdits.py diff --git a/scripts/Manually Run Scripts/MergeMultipleTextEdits.py b/scripts/Manually Run Scripts/MergeMultipleTextEdits.py new file mode 100644 index 0000000..fc5bacf --- /dev/null +++ b/scripts/Manually Run Scripts/MergeMultipleTextEdits.py @@ -0,0 +1,100 @@ +# Use this script to merge together multiple text-edits.json files together +# It expects all .json files to be in a folder called 'text-edits' next to this script +# It outputs to a file called merged-translations.json +# Files will be output in alphabetical order, so it is recommended to prepend each file with the chapter number +# If there are any conflicts, and exception will be raised. +# +# This script is usually never needed, unless translators have been swapping out the text-edits.json for each chapter +# rather than maintaining a common text-edits.json for all chapters. + +import os +import json + + +class Fragment: + order = 0 + + def __init__(self, fragment_as_dictionary: dict[str, str]): + self.current_english = fragment_as_dictionary['CurrentEnglish'] + self.current_japanese = fragment_as_dictionary['CurrentJapanese'] + self.new_english = fragment_as_dictionary['NewEnglish'] + self.new_japanese = fragment_as_dictionary['NewJapanese'] + self.discriminator = fragment_as_dictionary.get('Discriminator') + self.order = Fragment.order + Fragment.order += 1 + + # Generate a key for this Fragment, used later to check for conflicting fragments + self.key = self.current_english + self.current_japanese + if self.discriminator is not None: + self.key += str(self.discriminator) + + + def equals(self, other: 'Fragment'): + return ( + self.current_english == other.current_english and + self.current_japanese == other.current_japanese and + self.new_english == other.new_english and + self.new_japanese == other.new_japanese and + self.discriminator == other.discriminator + ) + + def __repr__(self) -> str: + return f"{self.order} ce: {self.current_english} cj: {self.current_japanese} ne: {self.new_english} nj: {self.new_japanese} d: {self.discriminator}" + + def as_dict(self): + retval = { + 'CurrentEnglish': self.current_english, + 'CurrentJapanese': self.current_japanese, + 'NewEnglish': self.new_english, + 'NewJapanese': self.new_japanese, + } + + if self.discriminator is not None: + retval['Discriminator'] = self.discriminator + + return retval + +def merge(all_translations: dict[str, Fragment], fragment: Fragment): + + if not fragment.key in all_translations: + all_translations[fragment.key] = fragment + else: + existing_item = all_translations[fragment.key] + + if existing_item.equals(fragment): + print(f"Skipping duplicate item {fragment}") + else: + raise Exception(f"Warning: non duplicate item existing:{existing_item} new: {fragment}") + + + +in_folder = "text-edits" + +files = os.listdir(in_folder) + +all_fragments = [] # type: list[Fragment] + +for filename in files: + path = os.path.join(in_folder, filename) + print(f"Parsing {path}") + + with open(path, encoding='utf-8') as f: + chapter_list_dict = json.loads(f.read()) + + all_fragments.extend(Fragment(f) for f in chapter_list_dict) + +all_translations = {} + +# Merge all fragments into one dict, ignoring duplicates +for f in all_fragments: + print(f.current_english) + merge(all_translations, f) + print() + +# Convert to list and sort by 'order' which is the order the fragments were loaded +sorted_translations = list(sorted(all_translations.values(), key=lambda f: f.order)) +for item in sorted_translations: + print(item) + +with open("merged-translations.json", 'w', encoding='utf-8') as out: + out.write(json.dumps([f.as_dict() for f in sorted_translations], indent=4, ensure_ascii=False)) \ No newline at end of file From a8d7b4b710b5df132e5451a94d50f337a5a61ba3 Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Fri, 14 Jul 2023 20:22:31 +1000 Subject: [PATCH 8/9] Change arg to disable translation, rather than to have no effect - translation output is enabled by default --- build.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/build.py b/build.py index 77e29f5..0fd7287 100644 --- a/build.py +++ b/build.py @@ -267,14 +267,11 @@ def save(self): choices=["all", "github_actions"] + list(chapter_to_build_variants.keys()), ) parser.add_argument("--force-download", default=False, action='store_true') -parser.add_argument("--translation", default=False, action='store_true') +parser.add_argument("--disable-translation", default=False, action='store_true') args = parser.parse_args() force_download = args.force_download -# NOTE: For now, translation archive output is always enabled, as most of the time this script will be used for translators -translation = args.translation - # Get chapter name from git tag if "github_actions" specified as the chapter chapter_name = args.chapter if chapter_name == "github_actions": @@ -285,13 +282,21 @@ def save(self): ) exit(0) -# NOTE: For now, translation archive output is always enabled, as most of the time this script will be used for translators +# NOTE: For now, translation archive output is enabled by default, as most of the time this script will be used for translators translation = True +if args.disable_translation: + translation = False + # Get a list of build variants (like 'onikakushi 5.2.2f1 win') depending on commmand line arguments build_variants = get_build_variants(chapter_name) build_variants_list = "\n - ".join([b.get_build_command() for b in build_variants]) -print(f"For chapter '{chapter_name}' building:\n - {build_variants_list}") +print(f"-------- Build Started --------") +print(f"Chapter: [{chapter_name}] | Translation Archive Output: [{('Enabled' if translation else 'Disabled')}]") +print(f"Variants:") +print(f" - {build_variants_list}") +print(f"-------------------------------") +print() # Install python dependencies print("Installing python dependencies") From 5adde5b725ce1426b81953637b19180a81114782 Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Fri, 14 Jul 2023 20:23:24 +1000 Subject: [PATCH 9/9] Fix python 3.11 warning showing on Python 3.10.X --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index 0fd7287..7be89e0 100644 --- a/build.py +++ b/build.py @@ -252,7 +252,7 @@ def save(self): print(">>>> ERROR: This script does not work on Python 2.7") exit(-1) -if sys.version_info > (3, 10): +if not (sys.version_info < (3, 11)): print(">>>> WARNING: This script probably does not work on Python 3.11 because unitypack uses old version of decrunch which does not build. Use Python 3.10 or below if you have this error.") lastModifiedManager = LastModifiedManager()