Build Python Package #504
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Python Package | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| incoming_ref: | |
| description: > | |
| The ref from Cantera/cantera to be built. Can be a tag, commit hash, | |
| or branch name. | |
| required: true | |
| default: "main" | |
| upload: | |
| description: Attempt to upload to PyPI | |
| required: true | |
| default: "false" | |
| concurrency: | |
| group: ${{ github.ref }}-${{ github.event.inputs.incoming_ref}} | |
| cancel-in-progress: true | |
| env: | |
| ACTION_URL: "https://github.com/Cantera/pypi-packages/actions/runs/${{ github.run_id }}" | |
| jobs: | |
| dump: | |
| name: Dump the input parameters for the workflow | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Dump Event Payload | |
| run: jq . "$GITHUB_EVENT_PATH" | |
| - name: Echo the input variables | |
| run: | | |
| echo "${{ github.event.inputs.incoming_ref }}" | |
| echo "${{ github.event.inputs.upload }}" | |
| post-pending-status: | |
| name: Post a pending workflow status to Cantera/cantera | |
| runs-on: ubuntu-latest | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.CANTERA_REPO_STATUS }} | |
| outputs: | |
| incoming-sha: ${{ steps.get-incoming-sha.outputs.incoming-sha }} | |
| tag-ref: ${{ steps.munge-incoming-ref.outputs.tag-ref }} | |
| steps: | |
| - name: Munge the incoming ref | |
| id: munge-incoming-ref | |
| run: | | |
| import os | |
| import re | |
| from pathlib import Path | |
| INCOMING_REF = "${{ github.event.inputs.incoming_ref }}" | |
| INCOMING_SHA = "" | |
| if INCOMING_REF.startswith("refs/"): | |
| INCOMING_REF = INCOMING_REF.replace("refs/", "") | |
| elif re.match(r"^v\d\.\d\.\d.*$", INCOMING_REF) is not None: | |
| INCOMING_REF = f"tags/{INCOMING_REF}" | |
| elif re.match(r"^[a-f0-9]{6,40}", INCOMING_REF) is not None: | |
| INCOMING_SHA = INCOMING_REF | |
| else: | |
| INCOMING_REF = f"heads/{INCOMING_REF}" | |
| TAG_REF = "false" | |
| if INCOMING_REF.startswith("tags"): | |
| TAG_REF = "true" | |
| Path(os.environ["GITHUB_ENV"]).write_text( | |
| f"INCOMING_REF={INCOMING_REF}\n" | |
| f"TAG_REF={TAG_REF}\n" | |
| f"INCOMING_SHA={INCOMING_SHA}" | |
| ) | |
| Path(os.environ["GITHUB_OUTPUT"]).write_text( | |
| f"tag-ref={TAG_REF}" | |
| ) | |
| shell: python | |
| - name: Get the SHA associated with the incoming ref | |
| id: get-incoming-sha | |
| run: | | |
| if [[ "${INCOMING_SHA}" == "" ]]; then | |
| INCOMING_SHA=$(gh api repos/cantera/cantera/git/matching-refs/${INCOMING_REF} \ | |
| -H "Accept: application/vnd.github.v3+json" --jq ".[0].object.sha") | |
| echo "INCOMING_SHA=${INCOMING_SHA}" >> $GITHUB_ENV | |
| fi | |
| # This needs to be in this step to be output to other jobs. | |
| echo "incoming-sha=${INCOMING_SHA}" >> $GITHUB_OUTPUT | |
| - name: Post the status to the upstream commit | |
| id: set-the-status | |
| if: env.TAG_REF == 'false' | |
| run: | | |
| gh api repos/cantera/cantera/statuses/${INCOMING_SHA} \ | |
| -H "Accept: application/vnd.github.v3+json" \ | |
| --field state='pending' \ | |
| --field target_url=$ACTION_URL \ | |
| --field context='PyPI Package Build' \ | |
| --field description="Pending build" \ | |
| --silent | |
| sdist: | |
| name: Build the sdist | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - "post-pending-status" | |
| outputs: | |
| job-status: ${{ job.status }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| name: Checkout the repository | |
| with: | |
| repository: "Cantera/cantera" | |
| submodules: recursive | |
| ref: ${{ github.event.inputs.incoming_ref }} | |
| - name: Set Up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache: 'pip' | |
| cache-dependency-path: interfaces/python_sdist/pyproject.toml.in | |
| - name: Install dependencies | |
| run: python3 -m pip install scons build | |
| - name: Build the sdist | |
| run: | | |
| python3 `which scons` sdist | |
| - name: Archive the built sdist | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: ./build/python_sdist/dist/*.tar.gz | |
| name: cibw-sdist | |
| if-no-files-found: error | |
| # Copied from https://github.com/hynek/build-and-inspect-python-package/ | |
| - name: Show SDist contents hierarchically, including metadata. | |
| shell: bash | |
| run: | | |
| mkdir -p /tmp/out/sdist | |
| cp build/python_sdist/dist/*.tar.gz /tmp/ | |
| cd /tmp | |
| tar xf *.tar.gz -C out/sdist | |
| echo -e '\n<details><summary>SDist contents</summary>\n' >> $GITHUB_STEP_SUMMARY | |
| (cd /tmp/out/sdist && tree -Da --timefmt="%Y-%m-%dT%H:%M:%SZ" * | sed 's/^/ /' | tee -a $GITHUB_STEP_SUMMARY) | |
| echo -e '\n</details>\n' >> $GITHUB_STEP_SUMMARY | |
| echo ----- Metadata Follows ----- | |
| echo -e '\n<details><summary>Metadata</summary>\n' >> $GITHUB_STEP_SUMMARY | |
| cat out/sdist/*/PKG-INFO | sed 's/^/ /' | tee -a $GITHUB_STEP_SUMMARY | |
| echo -e '\n</details>\n' >> $GITHUB_STEP_SUMMARY | |
| echo ----- End of Metadata ----- | |
| build-wheels: | |
| name: Build ${{ matrix.os }} ${{ matrix.arch }} for py${{ matrix.python || '-all' }} | |
| runs-on: ${{ matrix.os }} | |
| needs: ["sdist", "post-pending-status"] | |
| outputs: | |
| job-status: ${{ job.status }} | |
| strategy: | |
| matrix: | |
| # Wheel builds are fast except for aarch64, so split that into multiple jobs, | |
| # one for each Python version | |
| os: [ubuntu-latest] | |
| arch: [aarch64] | |
| python: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| include: | |
| - os: ubuntu-latest | |
| arch: x86_64 | |
| - os: windows-2022 | |
| arch: AMD64 | |
| boost-arch: x86 | |
| boost-toolset: msvc | |
| boost-platform-version: 2022 | |
| boost-version: "1.85.0" | |
| - os: macos-14 | |
| arch: arm64 | |
| boost-arch: aarch64 | |
| boost-toolset: clang | |
| # Since we only use the headers, we can use the platform version for this | |
| # macos version | |
| boost-platform-version: "14" | |
| boost-version: "1.85.0" | |
| - os: macos-13 | |
| arch: x86_64 | |
| boost-arch: x86 | |
| boost-toolset: clang | |
| # Since we only use the headers, we can use the platform version for this | |
| # macos version | |
| boost-platform-version: "13" | |
| boost-version: "1.85.0" | |
| fail-fast: false | |
| steps: | |
| - name: Checkout the repository | |
| uses: actions/checkout@v4 | |
| - name: Download pre-built sdist | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: cibw-sdist | |
| - name: Extract the sdist tarball | |
| run: tar -xvf *.tar.gz --strip-components=1 | |
| shell: bash | |
| - name: Download test files | |
| id: download-test-files | |
| uses: ./.github/actions/download-cantera-test | |
| with: | |
| incoming-sha: ${{ needs.post-pending-status.outputs.incoming-sha }} | |
| # Linux steps | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: all | |
| if: matrix.arch != 'x86_64' && runner.os == 'Linux' | |
| - name: Set up CIBW environment | |
| run: | | |
| PYTHON="${{ matrix.python }}" | |
| if [[ $PYTHON == "" ]]; then PYTHON="*"; fi | |
| CIBW_BUILD="cp${PYTHON//./}-*_${{ matrix.arch }}" | |
| echo "CIBW_BUILD=${CIBW_BUILD}" | tee -a $GITHUB_ENV | |
| if: runner.os == 'Linux' | |
| - name: Install boost | |
| # Our custom manylinux images already have Boost installed | |
| if: runner.os != 'Linux' | |
| uses: MarkusJx/[email protected] | |
| id: install-boost | |
| with: | |
| # REQUIRED: Specify the required boost version | |
| # A list of supported versions can be found here: | |
| # https://github.com/MarkusJx/prebuilt-boost/blob/main/versions-manifest.json | |
| boost_version: ${{ matrix.boost-version }} | |
| # OPTIONAL: Specify a custon install location | |
| boost_install_dir: ${{ runner.temp }} | |
| toolset: ${{ matrix.boost-toolset }} | |
| platform_version: ${{ matrix.boost-platform-version }} | |
| arch: ${{ matrix.boost-arch }} | |
| - name: Cache built libraries | |
| id: cache-built-libraries | |
| # Our custom manylinux images already have hdf5 installed | |
| if: runner.os != 'Linux' | |
| uses: actions/cache@v4 | |
| with: | |
| key: ${{ matrix.os }}-${{ matrix.arch }}-0 | |
| path: ${{ runner.temp }}/cache | |
| # Windows-only steps | |
| - name: Set Up Nuget | |
| uses: nuget/setup-nuget@v2 | |
| if: runner.os == 'Windows' | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.12 | |
| if: runner.os == 'Windows' | |
| - run: bash ./cibw_before_all_windows.sh "${{ runner.temp }}" | |
| if: runner.os == 'Windows' | |
| - name: Set up CIBW environment | |
| # On Windows, Boost_ROOT needs to have \ replaced by / because that's what | |
| # cibuildwheel says. CANTERA_TEST_DIR doesn't need the replacement because | |
| # it will be substituted in a cmd or pwsh session. | |
| run: | | |
| $BOOST_ROOT = "${{ steps.install-boost.outputs.BOOST_ROOT }}" -replace "\\", "/" | |
| echo "Boost_ROOT=$BOOST_ROOT" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append | |
| echo "CANTERA_TEST_DIR=${{ steps.download-test-files.outputs.test-root }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append | |
| echo "CIBW_BUILD=cp*-*" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append | |
| shell: pwsh | |
| if: runner.os == 'Windows' | |
| # macOS-only steps | |
| - name: Build required libraries | |
| run: bash ./cibw_before_all_macos.sh "${{ runner.temp }}" | |
| if: runner.os == 'macOS' | |
| # Force installation to resolve Python 3.8 bug (https://github.com/pypa/cibuildwheel/pull/1871#issuecomment-2161613619) | |
| - name: Hack for 3.8 bug | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.8 | |
| if: runner.os == 'macOS' && matrix.arch == 'arm64' | |
| - name: Set up CIBW environment | |
| run: | | |
| echo "Boost_ROOT=${{ steps.install-boost.outputs.BOOST_ROOT }}" >> $GITHUB_ENV | |
| echo "CANTERA_TEST_DIR=${{ steps.download-test-files.outputs.test-root }}" >> $GITHUB_ENV | |
| echo "CIBW_BUILD=cp*-*" >> $GITHUB_ENV | |
| if: runner.os == 'macOS' | |
| - name: Build wheels | |
| uses: pypa/[email protected] | |
| env: | |
| CIBW_ENVIRONMENT_LINUX: CT_SKIP_SLOW=1 CANTERA_TEST_DIR=/host/${{ steps.download-test-files.outputs.test-root }} | |
| CIBW_ENVIRONMENT_WINDOWS: CT_SKIP_SLOW=1 CMAKE_BUILD_PARALLEL_LEVEL=4 | |
| CIBW_ENVIRONMNET_MACOS: CT_SKIP_SLOW=1 | |
| CIBW_BUILD: ${{ env.CIBW_BUILD }} | |
| CIBW_ARCHS: ${{ matrix.arch }} | |
| - name: Archive the built wheels | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: ./wheelhouse/*.whl | |
| name: cibw-wheels-${{ runner.os }}-${{ strategy.job-index }} | |
| publish-files-to-pypi: | |
| name: Publish distribution files to PyPI | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| job-status: ${{ job.status }} | |
| needs: | |
| - "sdist" | |
| - "build-wheels" | |
| if: github.event.inputs.upload == 'true' | |
| permissions: | |
| id-token: write | |
| environment: pypi | |
| steps: | |
| - name: Download pre-built wheels | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: dist | |
| pattern: cibw-* | |
| merge-multiple: true | |
| - name: pypi-publish | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| send_status_to_cantera: | |
| name: Send jobs status to Cantera/cantera | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - "post-pending-status" | |
| - "sdist" | |
| - "build-wheels" | |
| - "publish-files-to-pypi" | |
| if: always() | |
| steps: | |
| - name: Collect statuses | |
| run: | | |
| from collections import Counter | |
| import os | |
| statuses = { | |
| "sdist": "${{needs.sdist.outputs.job-status}}", | |
| "wheels": "${{needs.build-wheels.outputs.job-status}}", | |
| "publish": "${{needs.publish-files-to-pypi.outputs.job-status}}", | |
| } | |
| # This is a deliberate comparison to the empty string. | |
| if statuses["publish"] == "" and "${{ github.event.inputs.upload }}" == "false": | |
| publish = statuses.pop("publish") | |
| else: | |
| publish = "" | |
| if all(v == "success" for v in statuses.values()): | |
| overall_status = "success" | |
| elif any(v in ("cancelled", "") for v in statuses.values()): | |
| overall_status = "error" | |
| elif any(v == "failure" for v in statuses.values()): | |
| overall_status = "failure" | |
| status_counts = Counter(statuses.values()) | |
| status_counts.update([publish]) | |
| description = [] | |
| if overall_status in ("error", "failure"): | |
| if status_counts.get("success") is not None: | |
| description.append(f"{status_counts['success']} succeeded") | |
| if status_counts.get("cancelled") is not None: | |
| description.append(f"{status_counts['cancelled']} cancelled") | |
| if status_counts.get("failure") is not None: | |
| description.append(f"{status_counts['failure']} failed") | |
| if status_counts.get("") is not None: | |
| description.append(f"{status_counts['']} skipped") | |
| description = ", ".join(description) | |
| else: | |
| description = "Successfully built Python wheels!" | |
| with open(os.environ["GITHUB_ENV"], "a") as gh_env: | |
| gh_env.write(f"OVERALL_STATUS={overall_status}\nDESCRIPTION={description}") | |
| shell: python | |
| - name: Post the status to the upstream commit | |
| if: needs.post-pending-status.outputs.tag-ref == 'false' | |
| run: | | |
| INCOMING_SHA=${{ needs.post-pending-status.outputs.incoming-sha }} | |
| gh api repos/cantera/cantera/statuses/${INCOMING_SHA} \ | |
| -H "Accept: application/vnd.github.v3+json" \ | |
| --field state="${OVERALL_STATUS}" \ | |
| --field target_url=$ACTION_URL \ | |
| --field context='PyPI Package Build' \ | |
| --field description="${DESCRIPTION}" \ | |
| --silent | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.CANTERA_REPO_STATUS }} |