From 025d13c93443e4700e1389832c800c20ad65692d Mon Sep 17 00:00:00 2001 From: Matt Dallmeyer Date: Fri, 27 Sep 2024 19:39:05 -0700 Subject: [PATCH] move mod bundling into modbase (#5) --- .github/ISSUE_TEMPLATE/config.yml | 8 - .github/ISSUE_TEMPLATE/jak1-bug-report.yml | 74 --- .github/ISSUE_TEMPLATE/jak2-bug-report.yml | 74 --- .github/schemas/README.md | 7 + .github/schemas/mods/v2/mod-schema.v2.json | 43 ++ .github/schemas/mods/v2/types.ts | 21 + .../create-mod-release/bundle-linux.py | 27 ++ .../create-mod-release/bundle-macos.py | 27 ++ .../create-mod-release/bundle-windows.py | 27 ++ .github/scripts/create-mod-release/common.py | 191 ++++++++ .../create-mod-release/emit-metadata.py | 36 ++ .../scripts/releases/extract_build_unix.sh | 0 .../scripts/releases/extract_build_windows.sh | 0 .github/workflows/build-matrix.yaml | 67 --- .github/workflows/compiler-output-check.yaml | 98 ---- .github/workflows/cut-release.yaml | 8 +- .github/workflows/draft-new-release.yaml | 35 -- .github/workflows/mod-release-pipeline.yml | 430 ++++++++++++++++++ 18 files changed, 813 insertions(+), 360 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/config.yml delete mode 100644 .github/ISSUE_TEMPLATE/jak1-bug-report.yml delete mode 100644 .github/ISSUE_TEMPLATE/jak2-bug-report.yml create mode 100644 .github/schemas/README.md create mode 100644 .github/schemas/mods/v2/mod-schema.v2.json create mode 100644 .github/schemas/mods/v2/types.ts create mode 100644 .github/scripts/create-mod-release/bundle-linux.py create mode 100644 .github/scripts/create-mod-release/bundle-macos.py create mode 100644 .github/scripts/create-mod-release/bundle-windows.py create mode 100644 .github/scripts/create-mod-release/common.py create mode 100644 .github/scripts/create-mod-release/emit-metadata.py mode change 100755 => 100644 .github/scripts/releases/extract_build_unix.sh mode change 100755 => 100644 .github/scripts/releases/extract_build_windows.sh delete mode 100644 .github/workflows/build-matrix.yaml delete mode 100644 .github/workflows/compiler-output-check.yaml delete mode 100644 .github/workflows/draft-new-release.yaml create mode 100644 .github/workflows/mod-release-pipeline.yml diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml deleted file mode 100644 index 5e3e4b989f..0000000000 --- a/.github/ISSUE_TEMPLATE/config.yml +++ /dev/null @@ -1,8 +0,0 @@ -blank_issues_enabled: true -contact_links: - - name: Feature Requests and Ideas - url: https://github.com/open-goal/jak-project/discussions/new/choose - about: Please use the discussion board to bring up feature requests and ideas. If there is interest in them, they can then be converted to an issue(s). - - name: Discord Help Channel - url: https://discord.gg/dPRCfsju3N - about: The best place for general help and asking questions. diff --git a/.github/ISSUE_TEMPLATE/jak1-bug-report.yml b/.github/ISSUE_TEMPLATE/jak1-bug-report.yml deleted file mode 100644 index c9f71ae66d..0000000000 --- a/.github/ISSUE_TEMPLATE/jak1-bug-report.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: "\U0001F41B Jak 1 Bug Report" -description: Create a bug report for Jak 1. -labels: ["bug", "jak1"] -body: - - type: checkboxes - attributes: - label: Acknowledgements - description: Take a couple minutes to help our maintainers work faster. - options: - - label: I have [searched](https://github.com/open-goal/jak-project/issues?q=is%3Aissue+is%3Aopen+label%3Abug+label%3Ajak1+sort%3Aupdated-desc+) for duplicate or closed bug reports - required: true - - label: I understand that I am supposed to provide my own legitimately obtained copy of the game - required: true - - - type: textarea - attributes: - label: Describe the Bug - description: A clear and concise description of what the bug is. You may post screenshots or videos of the bug here. - validations: - required: true - - - type: textarea - attributes: - label: How To Reproduce - description: Steps to reproduce the behavior. You can also post a video of it here. - validations: - required: true - - - type: dropdown - attributes: - label: Does this problem occur on original hardware or PCSX2? - description: Some things that may seem like bugs are actually exactly how the original game behaved. - options: - - Yes, it's unique to OpenGOAL - - Didn't check - - Not needed, bug is obvious - validations: - required: true - - - type: textarea - attributes: - label: Expected Behavior - description: A clear and concise description of the expected behavior. - placeholder: When I do X, Y should happen. - validations: - required: true - - - type: textarea - attributes: - label: Environment Information - description: "You can upload the [Support Package](https://github.com/open-goal/launcher#asking-for-help) provided by the Launcher here, or you can provide the following information: CPU, GPU, OS Version, OpenGOAL Version (found in the window's title bar)" - validations: - required: true - - - type: dropdown - attributes: - label: Game Version - options: - - NTSC 1.0 (black label) - - NTSC Greatest Hits version (red label) - - PAL - - JP - validations: - required: true - - - type: dropdown - attributes: - label: Have you set the game to something other than `60fps`? - options: - - "No" - - "Yes" - validations: - required: true - diff --git a/.github/ISSUE_TEMPLATE/jak2-bug-report.yml b/.github/ISSUE_TEMPLATE/jak2-bug-report.yml deleted file mode 100644 index 69005637ed..0000000000 --- a/.github/ISSUE_TEMPLATE/jak2-bug-report.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: "\U0001F41B Jak 2 Bug Report" -description: Create a bug report for Jak 2. -labels: ["bug", "jak2"] -body: - - type: checkboxes - attributes: - label: Acknowledgements - description: Take a couple minutes to help our maintainers work faster. - options: - - label: I have [searched](https://github.com/open-goal/jak-project/issues?q=is%3Aissue+is%3Aopen+label%3Abug+label%3Ajak2+sort%3Aupdated-desc+) for duplicate or closed bug reports - required: true - - label: I understand that I am supposed to provide my own legitimately obtained copy of the game - required: true - - - type: textarea - attributes: - label: Describe the Bug - description: A clear and concise description of what the bug is. You may post screenshots or videos of the bug here. - validations: - required: true - - - type: textarea - attributes: - label: How To Reproduce - description: Steps to reproduce the behavior. You can also post a video of it here. - validations: - required: true - - - type: dropdown - attributes: - label: Does this problem occur on original hardware or PCSX2? - description: Some things that may seem like bugs are actually exactly how the original game behaved. - options: - - Yes, it's unique to OpenGOAL - - Didn't check - - Not needed, bug is obvious - validations: - required: true - - - type: textarea - attributes: - label: Expected Behavior - description: A clear and concise description of the expected behavior. - placeholder: When I do X, Y should happen. - validations: - required: true - - - type: textarea - attributes: - label: Environment Information - description: "You can upload the [Support Package](https://github.com/open-goal/launcher#asking-for-help) provided by the Launcher here, or you can provide the following information: CPU, GPU, OS Version, OpenGOAL Version (found in the window's title bar)" - validations: - required: true - - - type: dropdown - attributes: - label: Game Version - options: - - NTSC 1.0 (black label) - - NTSC Greatest Hits version (red label) - - PAL - - JP - validations: - required: true - - - type: dropdown - attributes: - label: Have you set the game to something other than `60fps`? - options: - - "No" - - "Yes" - validations: - required: true - diff --git a/.github/schemas/README.md b/.github/schemas/README.md new file mode 100644 index 0000000000..d392ea9434 --- /dev/null +++ b/.github/schemas/README.md @@ -0,0 +1,7 @@ +Generated via typescript definitions: + +```bash +npx ts-json-schema-generator --path './*.ts' --type 'ModMetadata' > mod-schema.v1.json +``` + +And then some value validations added on mostly for valid semver checking \ No newline at end of file diff --git a/.github/schemas/mods/v2/mod-schema.v2.json b/.github/schemas/mods/v2/mod-schema.v2.json new file mode 100644 index 0000000000..dfa1f90827 --- /dev/null +++ b/.github/schemas/mods/v2/mod-schema.v2.json @@ -0,0 +1,43 @@ +{ + "$ref": "#/definitions/ModMetadata", + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "ModMetadata": { + "additionalProperties": false, + "properties": { + "publishedDate": { + "type": "string" + }, + "schemaVersion": { + "description": "Semantic Version", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "type": "string" + }, + "supportedGames": { + "items": { + "enum": [ + "jak1", + "jak2", + "jak3", + "jakx" + ], + "type": "string" + }, + "type": "array" + }, + "version": { + "description": "Semantic Version", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "type": "string" + } + }, + "required": [ + "schemaVersion", + "version", + "publishedDate", + "supportedGames" + ], + "type": "object" + } + } +} diff --git a/.github/schemas/mods/v2/types.ts b/.github/schemas/mods/v2/types.ts new file mode 100644 index 0000000000..aef931ea6f --- /dev/null +++ b/.github/schemas/mods/v2/types.ts @@ -0,0 +1,21 @@ +// interface ChangelogEntry { +// summary: string, +// timestamp: string, +// authors?: string[], +// description?: string, +// } + +/** + * Semantic Version + * @pattern ^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$ + */ +type Semver = string; + +type SupportedGame = "jak1" | "jak2" | "jak3" | "jakx"; + +export interface ModMetadata { + schemaVersion: Semver; + version: Semver; + publishedDate: string; + supportedGames: SupportedGame[]; +} diff --git a/.github/scripts/create-mod-release/bundle-linux.py b/.github/scripts/create-mod-release/bundle-linux.py new file mode 100644 index 0000000000..ae458e838a --- /dev/null +++ b/.github/scripts/create-mod-release/bundle-linux.py @@ -0,0 +1,27 @@ +from common import ( + create_output_dir, + download_release, + finalize_bundle, + get_args, + override_binaries_and_assets, + patch_mod_timestamp_and_version_info, +) + +args = get_args("linux") + +print(args) + +# Create our output directory +create_output_dir(args, "linux") + +# Download the Release +download_release(args, "linux", is_zip=False) + +# Override any files +override_binaries_and_assets(args, "linux") + +# Replace placeholder text with mod version and timestamp +patch_mod_timestamp_and_version_info(args, "linux") + +# Rezip it up and prepare it for upload +finalize_bundle(args, "linux", is_zip=False) diff --git a/.github/scripts/create-mod-release/bundle-macos.py b/.github/scripts/create-mod-release/bundle-macos.py new file mode 100644 index 0000000000..07dc32730e --- /dev/null +++ b/.github/scripts/create-mod-release/bundle-macos.py @@ -0,0 +1,27 @@ +from common import ( + create_output_dir, + download_release, + finalize_bundle, + get_args, + override_binaries_and_assets, + patch_mod_timestamp_and_version_info, +) + +args = get_args("macos-intel") + +print(args) + +# Create our output directory +create_output_dir(args, "macos-intel") + +# Download the Release +download_release(args, "macos-intel", is_zip=False) + +# Override any files +override_binaries_and_assets(args, "macos-intel") + +# Replace placeholder text with mod version and timestamp +patch_mod_timestamp_and_version_info(args, "macos-intel") + +# Rezip it up and prepare it for upload +finalize_bundle(args, "macos-intel", is_zip=False) diff --git a/.github/scripts/create-mod-release/bundle-windows.py b/.github/scripts/create-mod-release/bundle-windows.py new file mode 100644 index 0000000000..38df6da79d --- /dev/null +++ b/.github/scripts/create-mod-release/bundle-windows.py @@ -0,0 +1,27 @@ +from common import ( + create_output_dir, + download_release, + finalize_bundle, + get_args, + override_binaries_and_assets, + patch_mod_timestamp_and_version_info, +) + +args = get_args("windows") + +print(args) + +# Create our output directory +create_output_dir(args, "windows") + +# Download or Build the Release +download_release(args, "windows", is_zip=True) + +# Override any files +override_binaries_and_assets(args, "windows") + +# Replace placeholder text with mod version and timestamp +patch_mod_timestamp_and_version_info(args, "windows") + +# Rezip it up and prepare it for upload +finalize_bundle(args, "windows", is_zip=True) diff --git a/.github/scripts/create-mod-release/common.py b/.github/scripts/create-mod-release/common.py new file mode 100644 index 0000000000..e8eefa3ede --- /dev/null +++ b/.github/scripts/create-mod-release/common.py @@ -0,0 +1,191 @@ +import datetime +import json +import os +import glob +import shutil +import tarfile +import urllib.request +import zipfile + + +def default_asset_prefix(platform): + if platform == "windows": + return "opengoal-windows" + elif platform == "linux": + return "opengoal-linux" + elif platform == "macos-intel": + return "opengoal-macos-intel" + elif platform == "macos-arm": + return "opengoal-macos-arm" + else: + return "opengoal-unknown" + + +def get_args(platform): + return { + "outputDir": os.getenv("outputDir"), + "versionName": os.getenv("versionName"), + "toolingRepo": os.getenv("toolingRepo"), + "toolingReleaseAssetPrefix": os.getenv( + "toolingReleaseAssetPrefix", default_asset_prefix(platform) + ), + "toolingVersion": os.getenv("toolingVersion") + } + + +def create_output_dir(args, dir_name): + if os.path.exists(os.path.join(args["outputDir"], dir_name)): + print( + "Expected output directory already exists, clearing it - {}".format( + os.path.join(args["outputDir"], dir_name) + ) + ) + os.rmdir(os.path.join(args["outputDir"], dir_name)) + + os.makedirs(os.path.join(args["outputDir"], dir_name), exist_ok=True) + +def download_release(args, out_folder, is_zip=True): + toolingRepo = args["toolingRepo"] + tooling_version = args["toolingVersion"] + if tooling_version == "latest": + # Get the latest release + with urllib.request.urlopen( + f"https://api.github.com/repos/{toolingRepo}/releases/latest" + ) as response: + data = json.loads(response.read().decode()) + tooling_version = data["tag_name"] + + if is_zip: + releaseAssetUrl = f"https://github.com/{toolingRepo}/releases/download/{tooling_version}/{args['toolingReleaseAssetPrefix']}-{tooling_version}.zip" + urllib.request.urlretrieve( + releaseAssetUrl, os.path.join(args["outputDir"], out_folder, "release.zip") + ) + # Extract it + with zipfile.ZipFile( + os.path.join(args["outputDir"], out_folder, "release.zip"), "r" + ) as zip_ref: + zip_ref.extractall(os.path.join(args["outputDir"], out_folder)) + os.remove(os.path.join(args["outputDir"], out_folder, "release.zip")) + else: + releaseAssetUrl = f"https://github.com/{toolingRepo}/releases/download/{tooling_version}/{args['toolingReleaseAssetPrefix']}-{tooling_version}.tar.gz" + urllib.request.urlretrieve( + releaseAssetUrl, + os.path.join(args["outputDir"], out_folder, "release.tar.gz"), + ) + # Extract it + with tarfile.open( + os.path.join(args["outputDir"], out_folder, "release.tar.gz") + ) as tar_ball: + tar_ball.extractall(os.path.join(args["outputDir"], out_folder)) + os.remove(os.path.join(args["outputDir"], out_folder, "release.tar.gz")) + + +def override_binaries_and_assets(args, out_folder): + # Copy-in Mod Assets + customAssetsDir = "./custom_assets" + if os.path.exists(customAssetsDir): + shutil.copytree( + customAssetsDir, + os.path.join(args["outputDir"], out_folder, "data", "custom_assets"), + dirs_exist_ok=True, + ) + + goalSourceDir = "./goal_src" + if not os.path.exists(goalSourceDir): + print( + "Goal source directory not found at {}, not much of a mod without that!".format( + goalSourceDir + ) + ) + exit(1) + shutil.copytree( + goalSourceDir, + os.path.join(args["outputDir"], out_folder, "data", "goal_src"), + dirs_exist_ok=True, + ) + + gameAssetsDir = "./game/assets" + if not os.path.exists(gameAssetsDir): + print("Game assets directory not found at {}!".format(gameAssetsDir)) + exit(1) + shutil.copytree( + gameAssetsDir, + os.path.join(args["outputDir"], out_folder, "data", "game", "assets"), + dirs_exist_ok=True, + ) + + decompilerConfigDir = "./decompiler/config" + if os.path.exists(decompilerConfigDir): + shutil.copytree( + decompilerConfigDir, + os.path.join(args["outputDir"], out_folder, "data", "decompiler", "config"), + dirs_exist_ok=True, + ) + else: + print( + "Decompiler config directory not found at {}, skipping.".format( + decompilerConfigDir + ) + ) + + +def patch_mod_timestamp_and_version_info(args, out_folder): + try: + mod_settings_files = glob.glob(f"{args['outputDir']}/{out_folder}/data/goal_src/**/mod-settings.gc", recursive=True) + for settings_file_path in mod_settings_files: + file = open(settings_file_path, "r") + file_data = file.read() + file.close() + # Check if the placeholder string is present in the file + if "%MODVERSIONPLACEHOLDER%" in file_data: + # Replace the placeholder string with the version and date string + version_str = ( + args["versionName"] + + " " + + datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + ) + file_data = file_data.replace("%MODVERSIONPLACEHOLDER%", version_str) + # Write the updated content back to the mod-settings + file = open(settings_file_path, "w") + file.write(file_data) + file.close() + print( + f"String %MODVERSIONPLACEHOLDER% replaced with '{version_str}' in the file." + ) + else: + print(f"Couldn't find %MODVERSIONPLACEHOLDER% in the file.") + except Exception as e: + print( + f"Something went wrong trying to replace placeholder text with mod version info:" + ) + print(e) + +def finalize_bundle(args, out_folder, is_zip=True): + if is_zip: + shutil.make_archive( + f"{out_folder}-{args['versionName']}", + "zip", + os.path.join(args["outputDir"], out_folder), + ) + os.makedirs(os.path.join(args["outputDir"], "dist"), exist_ok=True) + shutil.move( + f"{out_folder}-{args['versionName']}.zip", + os.path.join( + args["outputDir"], "dist", f"{out_folder}-{args['versionName']}.zip" + ), + ) + else: + shutil.make_archive( + f"{out_folder}-{args['versionName']}", + "gztar", + os.path.join(args["outputDir"], out_folder), + ) + os.makedirs(os.path.join(args["outputDir"], "dist"), exist_ok=True) + shutil.move( + f"{out_folder}-{args['versionName']}.tar.gz", + os.path.join( + args["outputDir"], "dist", f"{out_folder}-{args['versionName']}.tar.gz" + ), + ) + # Cleanup + shutil.rmtree(os.path.join(args["outputDir"], out_folder)) diff --git a/.github/scripts/create-mod-release/emit-metadata.py b/.github/scripts/create-mod-release/emit-metadata.py new file mode 100644 index 0000000000..2acbd967f7 --- /dev/null +++ b/.github/scripts/create-mod-release/emit-metadata.py @@ -0,0 +1,36 @@ +# Simple script to emit a metadata.json file with the relevant mod information +# Values are passed in as environment variables +from datetime import datetime +import json +import os + +def split_comma_sep_val(str): + if "," not in str: + return [str] + return str.split(",") + +# Validate supported games list +games = split_comma_sep_val(os.getenv("SUPPORTED_GAMES")) +if len(games) == 0: + print("SUPPORTED_GAMES list is empty") + exit(1) +for game in games: + if game != "jak1" and game != "jak2" and game != "jak3" and game != "jakx": + print("SUPPORTED_GAMES contains invalid game: ", game) + exit(1) + +# Input has been validated, create metadata.json +metadata = { + "schemaVersion": os.getenv("SCHEMA_VERSION"), + "version": os.getenv("VERSION").removeprefix("v"), + "supportedGames": games, + "publishedDate": datetime.now().isoformat(), +} + +with open("{}/metadata.json".format(os.getenv("OUT_DIR")), "w", encoding="utf-8") as f: + print( + "Writing the following metadata: {}".format( + json.dumps(metadata, indent=2, ensure_ascii=False) + ) + ) + f.write(json.dumps(metadata, indent=2, ensure_ascii=False)) diff --git a/.github/scripts/releases/extract_build_unix.sh b/.github/scripts/releases/extract_build_unix.sh old mode 100755 new mode 100644 diff --git a/.github/scripts/releases/extract_build_windows.sh b/.github/scripts/releases/extract_build_windows.sh old mode 100755 new mode 100644 diff --git a/.github/workflows/build-matrix.yaml b/.github/workflows/build-matrix.yaml deleted file mode 100644 index 1d3c4d7e2e..0000000000 --- a/.github/workflows/build-matrix.yaml +++ /dev/null @@ -1,67 +0,0 @@ -name: Build - -on: - push: - branches: - - main - pull_request: - branches: - - main - merge_group: {} - -jobs: - # Windows - build_windows_clang: - name: "🖥️ Windows (Clang)" - uses: ./.github/workflows/windows-build-clang.yaml - if: github.repository == 'OpenGOAL-Mods/OG-Mod-Base' - with: - cmakePreset: "Release-windows-clang" - cachePrefix: "" - secrets: inherit - - build_windows_msvc: - name: "🖥️ Windows (MSVC)" - uses: ./.github/workflows/windows-build-msvc.yaml - if: github.repository == 'OpenGOAL-Mods/OG-Mod-Base' - with: - cmakePreset: "Release-windows-msvc" - cachePrefix: "" - secrets: inherit - - # Linux - build_linux_clang: - name: "🐧 Linux (Clang)" - uses: ./.github/workflows/linux-build-clang.yaml - if: github.repository == 'OpenGOAL-Mods/OG-Mod-Base' - with: - cmakePreset: "Release-linux-clang-asan" - cachePrefix: "" - secrets: inherit - - build_linux_gcc: - name: "🐧 Linux (GCC)" - uses: ./.github/workflows/linux-build-gcc.yaml - if: github.repository == 'OpenGOAL-Mods/OG-Mod-Base' - with: - cmakePreset: "Release-linux-gcc" - cachePrefix: "" - secrets: inherit - - # MacOS - build_macos_intel: - name: "🍎 MacOS" - uses: ./.github/workflows/macos-build.yaml - if: github.repository == 'OpenGOAL-Mods/OG-Mod-Base' - with: - cmakePreset: "Release-macos-clang" - cachePrefix: "" - - # Q4 2023 there will hopefully be native arm64 runners - # https://github.com/github/roadmap/issues/528 - # build_macos_arm: - # name: "🍎 MacOS" - # uses: ./.github/workflows/macos-build-arm.yaml - # with: - # cmakePreset: "Release-macos-clang" - # cachePrefix: "" diff --git a/.github/workflows/compiler-output-check.yaml b/.github/workflows/compiler-output-check.yaml deleted file mode 100644 index 7daab005da..0000000000 --- a/.github/workflows/compiler-output-check.yaml +++ /dev/null @@ -1,98 +0,0 @@ -name: Compilation Check - -on: - pull_request: - branches: - - master - -jobs: - build: - name: Compare - runs-on: ubuntu-20.04 - timeout-minutes: 60 - - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - with: - ref: master - - - name: Install Package Dependencies - run: | - sudo apt update - sudo apt install build-essential cmake \ - clang gcc g++ lcov make nasm libxrandr-dev \ - libxinerama-dev libxcursor-dev libpulse-dev \ - libxi-dev zip ninja-build libgl1-mesa-dev libssl-dev - - - name: Setup sccache - uses: hendrikmuhs/ccache-action@v1.2.14 - with: - variant: sccache - key: linux-ubuntu-20.04--Release-linux-clang-asan-${{ github.sha }} - restore-keys: linux-ubuntu-20.04--Release-linux-clang-asan - max-size: 1000M - - - name: CMake Generation (master) - env: - CC: clang - CXX: clang++ - run: | - cmake -B build --preset=Release-linux-clang-asan \ - -DCMAKE_C_COMPILER_LAUNCHER=sccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=sccache - - - name: Build goalc (master) - run: | - cmake --build build --parallel $((`nproc`)) --target goalc - mv ./build ./build.master - - - name: Checkout PR - uses: actions/checkout@v4 - with: - clean: "false" - - - name: CMake Generation (PR) - env: - CC: clang - CXX: clang++ - run: | - cmake -B build --preset=Release-linux-clang-asan \ - -DCMAKE_C_COMPILER_LAUNCHER=sccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=sccache - - - name: Build goalc (PR) - run: cmake --build build --parallel $((`nproc`)) --target goalc - - - name: Compile and preserve (master) - run: | - ./build.master/goalc/goalc --game jak1 --cmd "(make-group \"all-code\")" - ./build.master/goalc/goalc --game jak2 --cmd "(make-group \"all-code\")" - ./build.master/goalc/goalc --game jak3 --cmd "(make-group \"all-code\")" - mv ./out/jak1/obj ./out/jak1/obj.master - mv ./out/jak2/obj ./out/jak2/obj.master - mv ./out/jak3/obj ./out/jak3/obj.master - - - name: Compile and preserve (PR) - run: | - ./build/goalc/goalc --game jak1 --cmd "(make-group \"all-code\")" - ./build/goalc/goalc --game jak2 --cmd "(make-group \"all-code\")" - ./build/goalc/goalc --game jak3 --cmd "(make-group \"all-code\")" - mv ./out/jak1/obj ./out/jak1/obj.pr - mv ./out/jak2/obj ./out/jak2/obj.pr - mv ./out/jak3/obj ./out/jak3/obj.pr - - - name: Compare Results and Produce Report - run: | - ls -l ./out/jak1 - ls -l ./out/jak2 - ls -l ./out/jak3 - set +e - python ./scripts/gsrc/compare-compilation-outputs.py --base "./out/jak1/obj.master,./out/jak2/obj.master,./out/jak3/obj.master" --compare "./out/jak1/obj.pr,./out/jak2/obj.pr,./out/jak3/obj.pr" --markdown - SCRIPT_EXIT_CODE=$? - cat ./comp-diff-report.md >> $GITHUB_STEP_SUMMARY - if [ "$SCRIPT_EXIT_CODE" -ne 0 ]; then - exit 1 - fi - - diff --git a/.github/workflows/cut-release.yaml b/.github/workflows/cut-release.yaml index f95a1f21bc..00e10c8d25 100644 --- a/.github/workflows/cut-release.yaml +++ b/.github/workflows/cut-release.yaml @@ -1,4 +1,4 @@ -name: 🏭 Cut Mod Release +name: ⭐ Cut Mod Release ⭐ on: workflow_dispatch: @@ -67,10 +67,10 @@ jobs: if [[ ${{ inputs.binary_source == 'vanilla' }} ]]; then REPO=open-goal/jak-project; fi echo "REPO=$REPO" >> $GITHUB_OUTPUT - cut_release: - name: "Cut Release" + cut_mod_release: + name: "Cut Mod Release" needs: prep_vars - uses: open-goal/mod-bundling-tools/.github/workflows/create-mod-release.yml@v1 + uses: .github/workflows/mod-release-pipeline.yml with: semverBump: ${{ inputs.bump }} metadataSupportedGames: ${{ needs.prep_vars.outputs.supported_games }} diff --git a/.github/workflows/draft-new-release.yaml b/.github/workflows/draft-new-release.yaml deleted file mode 100644 index dffc0f4f64..0000000000 --- a/.github/workflows/draft-new-release.yaml +++ /dev/null @@ -1,35 +0,0 @@ -name: 🏭 Draft Release - -on: - workflow_dispatch: - inputs: - bump: - description: 'Semver Bump Type' - required: true - default: 'patch' - type: choice - options: - - patch - - minor - - major - -jobs: - cut-release: - runs-on: ubuntu-latest - steps: - # Docs - https://github.com/mathieudutour/github-tag-action - # Workflows cannot trigger other workflows implicitly - # - https://github.community/t/github-actions-workflow-not-triggering-with-tag-push/17053/7 - - name: Bump Version and Push Tag - if: github.repository == 'open-goal/jak-project' - id: tag_version - uses: mathieudutour/github-tag-action@v6.2 - with: - github_token: ${{ secrets.BOT_PAT }} - tag_prefix: v - default_bump: ${{ github.event.inputs.bump }} - - - name: Create Release - env: - GITHUB_TOKEN: ${{ secrets.BOT_PAT }} - run: gh release create ${{ steps.tag_version.outputs.new_tag }} --generate-notes --draft --repo open-goal/jak-project diff --git a/.github/workflows/mod-release-pipeline.yml b/.github/workflows/mod-release-pipeline.yml new file mode 100644 index 0000000000..6959f0bff2 --- /dev/null +++ b/.github/workflows/mod-release-pipeline.yml @@ -0,0 +1,430 @@ +name: "Mod Release Pipeline" +on: + workflow_call: + inputs: + metadataSupportedGames: + description: "The supported games for the mod, can be comma-separated, supports `jak1|jak2|jak3|jakx`" + required: true + default: "jak4" + type: "string" + outputDir: + description: "The directory that the releases assets are created and temporarily stored in. Defaults to ./bundler" + required: false + default: "./bundler" + type: "string" + semverBump: + description: "What semver bump to use - patch|minor|major. Defaults to patch" + required: false + default: "patch" + type: "string" + releaseBranches: + description: "Comma separated list of branches (JavaScript regular expression accepted) that will generate the release tags. You probably want your default branch in this list." + required: false + default: "master,main" + type: "string" + buildBinaries: + description: "Whether to build binaries from source. Defaults to `false`, pulling binaries from `toolingRepo` instead." + required: false + default: false + type: "boolean" + toolingRepo: + description: "The repository from which the tooling is taken for the bundle. Defaults to open-goal/jak-project." + required: false + default: "open-goal/jak-project" + type: "string" + toolingVersion: + description: "The version of `toolingRepo` to bundle. Defaults to latest version." + required: false + default: "latest" + type: "string" + skipWindows: + description: "Whether to skip Windows builds, defaults to `false`" + required: false + default: false + type: "boolean" + skipLinux: + description: "Whether to skip Linux builds, defaults to `false`" + required: false + default: false + type: "boolean" + skipMacOS: + description: "Whether to skip macOS builds, defaults to `false`" + required: false + default: false + type: "boolean" + secrets: + token: + description: "GitHub token used to create the release and push assets to it." + required: true + outputs: + taggedVersion: + description: "The version that was tagged and pushed for the mod" + value: ${{ jobs.create_release.outputs.bundleTagName }} + +permissions: + contents: write + +jobs: + validate_metadata: + name: "Validate Metadata" + runs-on: ubuntu-latest + steps: + - name: "Checkout Repository" + uses: actions/checkout@v4 + + - name: "Create metadata.json" + env: + SCHEMA_VERSION: "0.1.0" + VERSION: "v0.0.0" + SUPPORTED_GAMES: ${{ inputs.metadataSupportedGames }} + OUT_DIR: "/tmp" + run: | + python ./.github/scripts/create-mod-release/emit-metadata.py + + - name: "Validating Metadata" + run: | + npm install -g ajv-cli + ajv validate -s ./.github/schemas/mods/v2/mod-schema.v2.json -d /tmp/metadata.json + + create_release: + name: "Create Release" + needs: + - validate_metadata + runs-on: ubuntu-latest + outputs: + bundleTagName: ${{ steps.tag_version.outputs.new_tag }} + steps: + - name: Bump Version and Push Tag + id: tag_version + uses: mathieudutour/github-tag-action@v6.1 + with: + github_token: ${{ secrets.token }} + tag_prefix: v + default_bump: ${{ inputs.semverBump }} + release_branches: ${{ inputs.releaseBranches }} + + - name: Create Release + env: + GITHUB_TOKEN: ${{ secrets.token }} + run: gh release create ${{ steps.tag_version.outputs.new_tag }} --generate-notes --draft --repo ${{ github.repository }} + + # if `buildBinaries`, run the workflow to build and generate artifacts + + build_windows_clang: + name: "Windows Build" + needs: create_release + if: ${{ !inputs.skipWindows && inputs.buildBinaries }} + # assumes that this file is defined in your mod repo + uses: ./.github/workflows/windows-build-clang.yaml + with: + cmakePreset: "Release-windows-clang-static" + cachePrefix: "static" + uploadArtifacts: true + secrets: inherit + + bundle_windows_build: + name: "Bundle Windows Build" + needs: + - create_release + - build_windows_clang + if: ${{ !inputs.skipWindows && inputs.buildBinaries }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.create_release.outputs.bundleTagName }} + + - name: "Checkout Repository" + uses: actions/checkout@v4 + + - name: Prepare Artifact Folder + run: mkdir -p ./ci-artifacts + + - uses: actions/download-artifact@v4 + name: Download all Artifacts + with: + path: ./ci-artifacts/ + + - name: Display structure of downloaded files + run: ls -Rl ./ci-artifacts/ + + - name: Prepare Windows Build Assets + run: | + mkdir -p ./ci-artifacts/windows + mkdir -p ${{ inputs.outputDir }}/dist + chmod +x ./.github/scripts/releases/extract_build_windows.sh + ./.github/scripts/releases/extract_build_windows.sh ./ci-artifacts/windows ./ci-artifacts/opengoal-windows-static ./ + TAG_VAL=${{ needs.create_release.outputs.bundleTagName }} + 7z a -tzip ${{ inputs.outputDir }}/dist/windows-${TAG_VAL}.zip ./ci-artifacts/windows/* + + - name: Upload Bundle + uses: actions/upload-artifact@v4 + with: + name: windows + if-no-files-found: error + path: ${{ inputs.outputDir }}/dist + + build_linux_clang: + name: "Linux Build" + needs: create_release + if: ${{ !inputs.skipLinux && inputs.buildBinaries }} + # assumes that this file is defined in your mod repo + uses: ./.github/workflows/linux-build-clang.yaml + with: + cmakePreset: "Release-linux-clang-static" + cachePrefix: "static" + uploadArtifacts: true + secrets: inherit + + bundle_linux_build: + name: "Bundle Linux Build" + needs: + - create_release + - build_linux_clang + if: ${{ !inputs.skipLinux && inputs.buildBinaries }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.create_release.outputs.bundleTagName }} + + - name: "Checkout Repository" + uses: actions/checkout@v4 + + - name: Prepare Artifact Folder + run: mkdir -p ./ci-artifacts + + - uses: actions/download-artifact@v4 + name: Download all Artifacts + with: + path: ./ci-artifacts/ + + - name: Display structure of downloaded files + run: ls -Rl ./ci-artifacts/ + + - name: Prepare Linux Build Assets + run: | + mkdir -p ./ci-artifacts/linux + mkdir -p ${{ inputs.outputDir }}/dist + chmod +x ./.github/scripts/releases/extract_build_unix.sh + ./.github/scripts/releases/extract_build_unix.sh ./ci-artifacts/linux ./ci-artifacts/opengoal-linux-static ./ + TAG_VAL=${{ needs.create_release.outputs.bundleTagName }} + tar czf ${{ inputs.outputDir }}/dist/linux-${TAG_VAL}.tar.gz ./ci-artifacts/linux + + - name: Upload Bundle + uses: actions/upload-artifact@v4 + with: + name: linux + if-no-files-found: error + path: ${{ inputs.outputDir }}/dist + + build_macos_intel: + name: "MacOS Build" + needs: create_release + if: ${{ !inputs.skipMacOS && inputs.buildBinaries }} + # assumes that this file is defined in your mod repo + uses: ./.github/workflows/macos-build.yaml + with: + cmakePreset: "Release-macos-clang-static" + cachePrefix: "static" + uploadArtifacts: true + secrets: inherit + + bundle_macos_build: + name: "Bundle MacOS Build" + needs: + - create_release + - build_macos_intel + if: ${{ !inputs.skipMacOS && inputs.buildBinaries }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.create_release.outputs.bundleTagName }} + + - name: "Checkout Repository" + uses: actions/checkout@v4 + + - name: Prepare Artifact Folder + run: mkdir -p ./ci-artifacts + + - uses: actions/download-artifact@v4 + name: Download all Artifacts + with: + path: ./ci-artifacts/ + + - name: Display structure of downloaded files + run: ls -Rl ./ci-artifacts/ + + - name: Prepare MacOS Build Assets + run: | + mkdir -p ./ci-artifacts/macos-intel + mkdir -p ${{ inputs.outputDir }}/dist + chmod +x ./.github/scripts/releases/extract_build_unix.sh + ./.github/scripts/releases/extract_build_unix.sh ./ci-artifacts/macos-intel ./ci-artifacts/opengoal-macos-static ./ + TAG_VAL=${{ needs.create_release.outputs.bundleTagName }} + tar czf ${{ inputs.outputDir }}/dist/macos-intel-${TAG_VAL}.tar.gz ./ci-artifacts/macos-intel + + - name: Upload Bundle + uses: actions/upload-artifact@v4 + with: + name: macos-intel + if-no-files-found: error + path: ${{ inputs.outputDir }}/dist + + # if not `buildBinaries`, run the steps to build release assets + + bundle_windows_no_build: + name: "Bundle Windows (no build)" + needs: create_release + if: ${{ !inputs.skipWindows && !inputs.buildBinaries }} + runs-on: ubuntu-latest + steps: + - name: "Checkout Repository" + uses: actions/checkout@v4 + + - name: "Create Windows Release" + env: + outputDir: ${{ inputs.outputDir }} + versionName: ${{ needs.create_release.outputs.bundleTagName }} + toolingRepo: ${{ inputs.toolingRepo }} + toolingVersion: ${{ inputs.toolingVersion }} + run: python ./.github/scripts/create-mod-release/bundle-windows.py + + - name: Upload Bundle + uses: actions/upload-artifact@v4 + with: + name: windows + if-no-files-found: error + path: ${{ inputs.outputDir }}/dist + + bundle_linux_no_build: + name: "Bundle Linux (no build)" + needs: create_release + if: ${{ !inputs.skipLinux && !inputs.buildBinaries }} + runs-on: ubuntu-latest + steps: + - name: "Checkout Repository" + uses: actions/checkout@v4 + + - name: "Checkout Workflow Scripts" + uses: actions/checkout@v4 + with: + repository: "open-goal/mod-bundling-tools" + ref: ${{ github.event.inputs.ref }} + path: ".github" + + - name: "Create Linux Release" + env: + outputDir: ${{ inputs.outputDir }} + versionName: ${{ needs.create_release.outputs.bundleTagName }} + toolingRepo: ${{ inputs.toolingRepo }} + toolingVersion: ${{ inputs.toolingVersion }} + run: python ./.github/scripts/create-mod-release/bundle-linux.py + + - name: Upload Bundle + uses: actions/upload-artifact@v4 + with: + name: linux + if-no-files-found: error + path: ${{ inputs.outputDir }}/dist + + bundle_macos_no_build: + name: "Bundle macOS Intel (no build)" + needs: create_release + if: ${{ !inputs.skipMacOS && !inputs.buildBinaries }} + runs-on: ubuntu-latest + steps: + - name: "Checkout Repository" + uses: actions/checkout@v4 + + - name: "Create MacOS Intel Release" + env: + outputDir: ${{ inputs.outputDir }} + versionName: ${{ needs.create_release.outputs.bundleTagName }} + toolingRepo: ${{ inputs.toolingRepo }} + toolingVersion: ${{ inputs.toolingVersion }} + run: python ./.github/scripts/create-mod-release/bundle-macos.py + + - name: Upload Bundle + uses: actions/upload-artifact@v4 + with: + name: macos-intel + if-no-files-found: error + path: ${{ inputs.outputDir }}/dist + + finalize_release: + name: "Finalize Release" + needs: + - create_release + - bundle_windows_build + - bundle_linux_build + - bundle_macos_build + - bundle_windows_no_build + - bundle_linux_no_build + - bundle_macos_no_build + if: | + always() && + needs.create_release.result == 'success' && + (needs.bundle_windows_build.result == 'skipped' || needs.bundle_windows_build.result == 'success') && + (needs.bundle_linux_build.result == 'skipped' || needs.bundle_linux_build.result == 'success') && + (needs.bundle_macos_build.result == 'skipped' || needs.bundle_macos_build.result == 'success') && + (needs.bundle_windows_no_build.result == 'skipped' || needs.bundle_windows_no_build.result == 'success') && + (needs.bundle_linux_no_build.result == 'skipped' || needs.bundle_linux_no_build.result == 'success') && + (needs.bundle_macos_no_build.result == 'skipped' || needs.bundle_macos_no_build.result == 'success') + runs-on: ubuntu-latest + steps: + - name: "Checkout Repository" + uses: actions/checkout@v4 + + - name: Prepare Output Folder + run: mkdir -p ${{ inputs.outputDir }}/dist + + - name: Download all Artifacts + uses: actions/download-artifact@v4 + with: + path: ${{ inputs.outputDir }}/artifacts + + - name: Display structure of downloaded files + run: ls -Rl ${{ inputs.outputDir }} + + - name: Move Linux Assets + if: ${{ !inputs.skipLinux }} + run: mv ${{ inputs.outputDir }}/artifacts/linux/* ${{ inputs.outputDir }}/dist + + - name: Move Windows Assets + if: ${{ !inputs.skipWindows }} + run: mv ${{ inputs.outputDir }}/artifacts/windows/* ${{ inputs.outputDir }}/dist + + - name: Move MacOS Assets + if: ${{ !inputs.skipMacOS }} + run: mv ${{ inputs.outputDir }}/artifacts/macos-intel/* ${{ inputs.outputDir }}/dist + + - name: Prepare Release Metadata + env: + SCHEMA_VERSION: "0.1.0" + VERSION: ${{ needs.create_release.outputs.bundleTagName }} + SUPPORTED_GAMES: ${{ inputs.metadataSupportedGames }} + OUT_DIR: "${{ inputs.outputDir }}/dist" + run: python ./.github/scripts/create-mod-release/emit-metadata.py + + - name: Validating Metadata + run: | + npm install -g ajv-cli + ajv validate -s ./.github/schemas/mods/v2/mod-schema.v2.json -d ${{ inputs.outputDir }}/dist/metadata.json + + - name: Upload Assets + env: + GITHUB_TOKEN: ${{ secrets.token }} + run: | + TAG_VAL=${{ needs.create_release.outputs.bundleTagName }} + echo $TAG_VAL + gh release upload "${TAG_VAL}" ${{ github.WORKSPACE }}/${{ inputs.outputDir }}/dist/* --repo ${{ github.repository }} --clobber + + - name: Publish Release + env: + GITHUB_TOKEN: ${{ secrets.token }} + run: | + TAG_VAL=${{ needs.create_release.outputs.bundleTagName }} + echo $TAG_VAL + gh release edit ${TAG_VAL} --draft=false --repo ${{ github.repository }}