diff --git a/.github/workflows/build_packages.yml b/.github/workflows/build_packages.yml index e0f6501e1..9530cf4d6 100644 --- a/.github/workflows/build_packages.yml +++ b/.github/workflows/build_packages.yml @@ -7,7 +7,21 @@ name: Build packages on: + # Trigger from another workflow (typically to build dev packages and then test them) + workflow_call: + inputs: + build_type: + description: The type of build version to produce ("stable", "rc", or "dev") + type: string + default: "dev" + # Trigger manually (typically to test the workflow or manually build a release [candidate]) workflow_dispatch: + inputs: + build_type: + description: The type of build version to produce ("stable", "rc", or "dev") + type: string + default: "rc" + # Trigger on a schedule to build nightly release candidates. schedule: # Runs at 11:00 AM UTC, which is 3:00 AM PST (UTC-8) - cron: '0 11 * * *' @@ -16,14 +30,12 @@ permissions: contents: read jobs: - # Note: metadata generation could happen in a separate trigger/schedule - # workflow. For cross platform builds, it's useful to just generate the - # metadata on Linux and pass that to later jobs using artifacts. + # Generate metadata on Linux and pass to later jobs. setup_metadata: if: ${{ github.repository_owner == 'nod-ai' || github.event_name != 'schedule' }} runs-on: ubuntu-24.04 outputs: - version_suffix: ${{ steps.version_rc.outputs.version_suffix }} + version_suffix: ${{ steps.version_local.outputs.version_suffix }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Setup Python @@ -34,18 +46,35 @@ jobs: - name: Install Python packages run: pip install packaging - - name: Generate release candidate versions - id: version_rc + # Compute version suffix based on inputs (default to 'rc') + - name: Compute stable version suffix + if: ${{ inputs.build_type == 'stable' }} + run: | + version_suffix="" + echo "version_suffix=${version_suffix}" >> $GITHUB_ENV + - name: Compute rc version suffix + if: ${{ inputs.build_type == 'rc' || inputs.build_type == '' }} run: | version_suffix="$(printf 'rc%(%Y%m%d)T')" + echo "version_suffix=${version_suffix}" >> $GITHUB_ENV + - name: Compute dev version suffix + if: ${{ inputs.build_type == 'dev' }} + run: | + version_suffix=".dev0+${{ github.sha }}" + echo "version_suffix=${version_suffix}" >> $GITHUB_ENV + + - name: Write version local files + id: version_local + run: | echo "version_suffix=${version_suffix}" >> $GITHUB_OUTPUT python3 build_tools/python_deploy/compute_local_version.py --version-suffix=${version_suffix} sharktank python3 build_tools/python_deploy/compute_local_version.py --version-suffix=${version_suffix} shortfin python3 build_tools/python_deploy/compute_common_version.py -rc --version-suffix=${version_suffix} --write-json - - name: Upload version_local.json + + - name: Upload version_local.json files uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: - name: version_local + name: version_local_files path: | sharktank/version_local.json shortfin/version_local.json @@ -57,6 +86,8 @@ jobs: permissions: contents: write needs: [setup_metadata] + env: + OUTPUT_DIR: "${{ github.workspace }}/bindist" strategy: fail-fast: false matrix: @@ -101,47 +132,59 @@ jobs: path: "c" # Windows can hit path length limits, so use a short path. submodules: false - - name: Download version_local.json + - name: Download version_local.json files uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - name: version_local + name: version_local_files path: ./c/ merge-multiple: true + # Dev builds can use a cache and disable tracing to halve the build time. + # Other build use no cache for supply chain security and keep tracing enabled. + - name: Apply dev settings + if: ${{ inputs.build_type == 'dev' }} + run: | + echo "CACHE_DIR=${{ github.workspace }}/.shark-ai-cache" >> $GITHUB_ENV + echo "SHORTFIN_ENABLE_TRACING=OFF" >> $GITHUB_ENV + - name: Setup cache + if: ${{ inputs.build_type == 'dev' }} + uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + with: + path: ${{ env.CACHE_DIR }} + key: build-packages-${{ matrix.package }}-${{ matrix.platform }}-${{ matrix.python-version }}-v1-${{ github.sha }} + restore-keys: | + build-packages-${{ matrix.package }}-${{ matrix.platform }}-${{ matrix.python-version }}-v1- + + # Build packages. - name: Build shark-ai (Linux x86_64) if: "matrix.package == 'shark-ai' && matrix.platform == 'linux-x86_64'" - env: - OUTPUT_DIR: "${{ github.workspace }}/bindist" run: | [ -e ./bindist/* ] && rm ./bindist/* ./c/build_tools/python_deploy/write_requirements.py --version-suffix=${{ needs.setup_metadata.outputs.version_suffix }} ./c/shark-ai/build_tools/build_linux_package.sh - - name: Build sharktank (Linux x86_64) if: "matrix.package == 'sharktank' && matrix.platform == 'linux-x86_64'" - env: - OUTPUT_DIR: "${{ github.workspace }}/bindist" run: | [ -e ./bindist/* ] && rm ./bindist/* ./c/sharktank/build_tools/build_linux_package.sh - - name: Build shortfin (Linux x86_64, ${{ matrix.python-version }}) if: "matrix.package == 'shortfin' && matrix.platform == 'linux-x86_64'" env: - OUTPUT_DIR: "${{ github.workspace }}/bindist" OVERRIDE_PYTHON_VERSIONS: "${{ matrix.python-version }}" run: | [ -e ./bindist/* ] && rm ./bindist/* ./c/shortfin/build_tools/build_linux_package.sh + # Always upload to GitHub artifacts. - name: Upload python wheels uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: if-no-files-found: error name: snapshot-${{ matrix.package }}-${{ matrix.platform }}-${{ matrix.python-version }} path: bindist - - - name: Release python wheels + # Upload release candidate versions to a 'dev-wheels' GitHub release. + - name: Release rc python wheels + if: ${{ inputs.build_type == 'rc' || inputs.build_type == '' }} uses: ncipollo/release-action@2c591bcc8ecdcd2db72b97d6147f871fcd833ba5 # v1.14.0 with: artifacts: bindist/*.whl