Skip to content
Build Python Package

Build Python Package #361

Sign in to view logs

Build Python Package

Build Python Package #361

Workflow file for this run

.github/workflows/python-package.yml at eefb667
name: Build Python Package
on:
workflow_dispatch:
inputs:
incoming_ref:
description: >
The ref from Cantera/cantera to be built. Can be a tag, commit hash,
or branch name.
required: true
default: "main"
upload:
description: Attempt to upload to PyPI
required: true
default: "false"
concurrency:
group: ${{ github.ref }}-${{ github.event.inputs.incoming_ref}}
cancel-in-progress: true
env:
CIBW_BUILD_FRONTEND: build
CIBW_TEST_EXTRAS: pandas,units
CIBW_TEST_REQUIRES: pytest
ACTION_URL: "https://github.com/Cantera/pypi-packages/actions/runs/${{ github.run_id }}"
jobs:
dump:
name: Dump the input parameters for the workflow
runs-on: ubuntu-22.04
steps:
- name: Dump Event Payload
run: jq . "$GITHUB_EVENT_PATH"
- name: Echo the input variables
run: |
echo "${{ github.event.inputs.incoming_ref }}"
echo "${{ github.event.inputs.upload }}"
post-pending-status:
name: Post a pending workflow status to Cantera/cantera
runs-on: ubuntu-22.04
env:
GITHUB_TOKEN: ${{ secrets.CANTERA_REPO_STATUS }}
outputs:
incoming-sha: ${{ steps.get-incoming-sha.outputs.incoming-sha }}
tag-ref: ${{ steps.munge-incoming-ref.outputs.tag-ref }}
steps:
- name: Munge the incoming ref
id: munge-incoming-ref
run: |
import os
import re
from pathlib import Path
INCOMING_REF = "${{ github.event.inputs.incoming_ref }}"
INCOMING_SHA = ""
if INCOMING_REF.startswith("refs/"):
INCOMING_REF = INCOMING_REF.replace("refs/", "")
elif re.match(r"^v\d\.\d\.\d.*$", INCOMING_REF) is not None:
INCOMING_REF = f"tags/{INCOMING_REF}"
elif re.match(r"^[a-f0-9]{6,40}", INCOMING_REF) is not None:
INCOMING_SHA = INCOMING_REF
else:
INCOMING_REF = f"heads/{INCOMING_REF}"
TAG_REF = "false"
if INCOMING_REF.startswith("tags"):
TAG_REF = "true"
Path(os.environ["GITHUB_ENV"]).write_text(
f"INCOMING_REF={INCOMING_REF}\n"
f"TAG_REF={TAG_REF}\n"
f"INCOMING_SHA={INCOMING_SHA}"
)
Path(os.environ["GITHUB_OUTPUT"]).write_text(
f"tag-ref={TAG_REF}"
)
shell: python
- name: Get the SHA associated with the incoming ref
id: get-incoming-sha
run: |
if [[ "${INCOMING_SHA}" == "" ]]; then
INCOMING_SHA=$(gh api repos/cantera/cantera/git/matching-refs/${INCOMING_REF} \
-H "Accept: application/vnd.github.v3+json" --jq ".[0].object.sha")
echo "INCOMING_SHA=${INCOMING_SHA}" >> $GITHUB_ENV
fi
# This needs to be in this step to be output to other jobs.
echo "incoming-sha=${INCOMING_SHA}" >> $GITHUB_OUTPUT
- name: Post the status to the upstream commit
id: set-the-status
if: env.TAG_REF == 'false'
run: |
gh api repos/cantera/cantera/statuses/${INCOMING_SHA} \
-H "Accept: application/vnd.github.v3+json" \
--field state='pending' \
--field target_url=$ACTION_URL \
--field context='PyPI Package Build' \
--field description="Pending build" \
--silent
sdist:
name: Build the sdist
runs-on: ubuntu-22.04
needs:
- "post-pending-status"
outputs:
job-status: ${{ job.status }}
steps:
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install libboost-dev
- uses: actions/checkout@v3
name: Checkout the repository
with:
repository: "Cantera/cantera"
submodules: recursive
ref: ${{ github.event.inputs.incoming_ref }}
- name: Set Up Python 3.10
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Install dependencies
run: python3 -m pip install -U pip scons build
- name: Build the sdist
run: |
python3 `which scons` sdist f90_interface=n python_package='none' \
system_blas_lapack=n system_sundials=n system_eigen=n system_fmt=n \
system_yamlcpp=n googletest=none
- name: Archive the built sdist
uses: actions/upload-artifact@v3
with:
path: ./build/python_sdist/dist/*.tar.gz
name: sdist
if-no-files-found: error
# Copied from https://github.com/hynek/build-and-inspect-python-package/
- name: Show SDist contents hierarchically, including metadata.
shell: bash
run: |
mkdir -p /tmp/out/sdist
cp build/python_sdist/dist/*.tar.gz /tmp/
cd /tmp
tar xf *.tar.gz -C out/sdist
echo -e '\n<details><summary>SDist contents</summary>\n' >> $GITHUB_STEP_SUMMARY
(cd /tmp/out/sdist && tree -Da --timefmt="%Y-%m-%dT%H:%M:%SZ" * | sed 's/^/ /' | tee -a $GITHUB_STEP_SUMMARY)
echo -e '\n</details>\n' >> $GITHUB_STEP_SUMMARY
echo ----- Metadata Follows -----
echo -e '\n<details><summary>Metadata</summary>\n' >> $GITHUB_STEP_SUMMARY
cat out/sdist/*/PKG-INFO | sed 's/^/ /' | tee -a $GITHUB_STEP_SUMMARY
echo -e '\n</details>\n' >> $GITHUB_STEP_SUMMARY
echo ----- End of Metadata -----
linux-wheel:
name: Build ${{ matrix.libc }}linux_${{ matrix.arch }} for py${{ matrix.py }}
runs-on: ubuntu-20.04
needs: ["sdist", "post-pending-status"]
outputs:
job-status: ${{ job.status }}
strategy:
matrix:
py: ["38", "39", "310", "311"]
arch: ["x86_64", "i686"]
libc: ["many", "musl"]
include:
- py: "311"
arch: "aarch64"
libc: "many"
- py: "311"
arch: "ppc64le"
libc: "many"
- py: "311"
arch: "s390x"
libc: "many"
- py: "310"
arch: "aarch64"
libc: "many"
- py: "310"
arch: "ppc64le"
libc: "many"
- py: "310"
arch: "s390x"
libc: "many"
- py: "39"
arch: "aarch64"
libc: "many"
- py: "38"
arch: "aarch64"
libc: "many"
fail-fast: true
env:
BOOST_INCLUDE: include
BOOST_URL: https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.7z
steps:
- name: Download pre-built sdist
uses: actions/download-artifact@v3
with:
name: sdist
- name: Extract the sdist tarball
run: tar -xvf *.tar.gz --strip-components=1
- name: Restore Boost cache
uses: actions/cache@v3
id: cache-boost
with:
path: ${{ env.BOOST_INCLUDE }}/boost
key: boost-${{env.BOOST_URL}}
- name: Install Boost Headers
if: steps.cache-boost.outputs.cache-hit != 'true'
run: |
mkdir -p $BOOST_INCLUDE
curl --progress-bar --location --output $BOOST_INCLUDE/download.7z $BOOST_URL
7z -o$BOOST_INCLUDE x $BOOST_INCLUDE/download.7z -y -bd boost_1_78_0/boost
mv $BOOST_INCLUDE/boost_1_78_0/boost $BOOST_INCLUDE/boost
rm $BOOST_INCLUDE/download.7z
rm -r $BOOST_INCLUDE/boost_1_78_0
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
with:
platforms: all
- name: Build wheels
uses: pypa/[email protected]
env:
CIBW_ENVIRONMENT: BOOST_INCLUDE=${{ env.BOOST_INCLUDE }} CT_SKIP_SLOW=1
CIBW_BUILD: cp${{ matrix.py }}-${{ matrix.libc }}linux*
CIBW_ARCHS: ${{ matrix.arch }}
# cibuildwheel on Linux uses a Docker container to run the build, so
# runner.temp is not available. cibuildwheel also uses the /tmp folder, so
# we should be pretty safe to also use that.
CIBW_TEST_COMMAND: pytest -vv --durations=100 /tmp/test/python
CIBW_BEFORE_TEST: |
curl -sL "https://github.com/cantera/cantera/archive/${{ needs.post-pending-status.outputs.incoming-sha }}.tar.gz" -o /tmp/cantera.tar.gz \
&& tar -xzf /tmp/cantera.tar.gz --strip-components=1 -C /tmp "cantera-${{ needs.post-pending-status.outputs.incoming-sha }}/test"
# NumPy is generally not available for these platforms so testing takes a
# while. This just skips the tests on these
# combinations, the wheels are still built and uploaded.
CIBW_TEST_SKIP: "*-manylinux_{i686,ppc64le,s390x} *musl*"
- name: Archive the built wheels
uses: actions/upload-artifact@v3
with:
path: ./wheelhouse/*.whl
name: wheels
windows-wheel:
name: Build ${{ matrix.arch }} Windows Wheels for py${{ matrix.py }}
runs-on: windows-2019
needs: ["sdist", "post-pending-status"]
outputs:
job-status: ${{ job.status }}
strategy:
matrix:
py: ["38", "39", "310", "311"]
arch: ["AMD64", "x86"]
fail-fast: true
env:
BOOST_ROOT: ${{ github.workspace }}/3rdparty/boost
BOOST_URL: https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.7z
steps:
- name: Download pre-built sdist
uses: actions/download-artifact@v3
with:
name: sdist
- name: Extract the sdist tarball
run: tar -xvf *.tar.gz --strip-components=1
shell: bash
- name: Restore Boost cache
uses: actions/cache@v3
id: cache-boost
with:
path: ${{env.BOOST_ROOT}}
key: boost-${{env.BOOST_URL}}
- name: Install Boost Headers
if: steps.cache-boost.outputs.cache-hit != 'true'
run: |
BOOST_ROOT=$(echo $BOOST_ROOT | sed 's/\\/\//g')
mkdir -p $BOOST_ROOT
curl --progress-bar --location --output $BOOST_ROOT/download.7z $BOOST_URL
7z -o$BOOST_ROOT x $BOOST_ROOT/download.7z -y -bd boost_1_78_0/boost
mv $BOOST_ROOT/boost_1_78_0/boost $BOOST_ROOT/boost
rm $BOOST_ROOT/download.7z
shell: bash
- name: Build wheels
uses: pypa/[email protected]
env:
CIBW_ENVIRONMENT: BOOST_INCLUDE=${BOOST_ROOT} CT_SKIP_SLOW=1
CIBW_ARCHS: ${{ matrix.arch }}
CIBW_BUILD: cp${{ matrix.py }}-*
CIBW_TEST_COMMAND: pytest -vv --durations=100 ${{ runner.temp }}/test/python
CIBW_BEFORE_TEST: |
curl -sL "https://github.com/cantera/cantera/archive/${{ needs.post-pending-status.outputs.incoming-sha }}.tar.gz" -o ${{ runner.temp }}/cantera.tar.gz && tar -xzf ${{ runner.temp }}/cantera.tar.gz --strip-components=1 -C ${{ runner.temp }} "cantera-${{ needs.post-pending-status.outputs.incoming-sha }}/test"
- name: Archive the built wheels
uses: actions/upload-artifact@v3
with:
path: ./wheelhouse/*.whl
name: wheels
macos-wheel:
name: Build ${{ matrix.arch }} macOS Wheels for py${{ matrix.py }}
runs-on: macos-11
needs: ["sdist", "post-pending-status"]
outputs:
job-status: ${{ job.status }}
strategy:
matrix:
py: ["39", "310", "311"]
arch: ["x86_64", "arm64"]
deployment_target: ["11.0"]
include:
- py: "38"
deployment_target: "10.15"
arch: "x86_64"
- py: "38"
deployment_target: "10.15"
arch: "arm64"
fail-fast: true
env:
MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target }}
steps:
- name: Download pre-built sdist
uses: actions/download-artifact@v3
with:
name: sdist
- name: Extract the sdist tarball
run: tar -xvf *.tar.gz --strip-components=1
- name: Install Brew dependencies
run: brew install boost
- name: Build wheels
uses: pypa/[email protected]
env:
CIBW_ENVIRONMENT: BOOST_INCLUDE="$(brew --prefix)/include" RUNNER_TEMP=${{ runner.temp }} CT_SKIP_SLOW=1
CIBW_BUILD: cp${{ matrix.py }}-*
CIBW_ARCHS_MACOS: ${{ matrix.arch }}
# Testing won't be available for macOS ARM until native ARM runners are available.
CIBW_TEST_SKIP: "*-macosx_arm64"
CIBW_TEST_COMMAND: pytest -vv --durations=100 ${RUNNER_TEMP}/test/python
CIBW_BEFORE_TEST: |
curl -sL "https://github.com/cantera/cantera/archive/${{ needs.post-pending-status.outputs.incoming-sha }}.tar.gz" -o ${{ runner.temp }}/cantera.tar.gz && tar -xzf ${{ runner.temp }}/cantera.tar.gz --strip-components=1 -C ${{ runner.temp }} "cantera-${{ needs.post-pending-status.outputs.incoming-sha }}/test"
- name: Archive the built wheels
uses: actions/upload-artifact@v3
with:
path: ./wheelhouse/*.whl
name: wheels
publish-files-to-pypi:
name: Publish distribution files to PyPI
runs-on: ubuntu-22.04
outputs:
job-status: ${{ job.status }}
needs:
- "sdist"
- "linux-wheel"
- "windows-wheel"
- "macos-wheel"
if: github.event.inputs.upload == 'true'
steps:
- name: Download pre-built wheels
uses: actions/download-artifact@v3
with:
path: dist/
name: wheels
- name: Download pre-build sdist
uses: actions/download-artifact@v3
with:
path: dist/
name: sdist
- name: pypi-publish
uses: pypa/gh-action-pypi-publish@release/v1
with:
user: __token__
password: ${{ secrets.PYPI_TOKEN }}
send_status_to_cantera:
name: Send jobs status to Cantera/cantera
runs-on: ubuntu-22.04
needs:
- "post-pending-status"
- "sdist"
- "linux-wheel"
- "windows-wheel"
- "macos-wheel"
- "publish-files-to-pypi"
if: always()
steps:
- name: Collect statuses
run: |
from collections import Counter
import os
statuses = {
"sdist": "${{needs.sdist.outputs.job-status}}",
"linux": "${{needs.linux-wheel.outputs.job-status}}",
"windows": "${{needs.windows-wheel.outputs.job-status}}",
"macos": "${{needs.macos-wheel.outputs.job-status}}",
"publish": "${{needs.publish-files-to-pypi.outputs.job-status}}",
}
# This is a deliberate comparison to the empty string.
if statuses["publish"] == "" and "${{ github.event.inputs.upload }}" == "false":
publish = statuses.pop("publish")
else:
publish = ""
if all(v == "success" for v in statuses.values()):
overall_status = "success"
elif any(v in ("cancelled", "") for v in statuses.values()):
overall_status = "error"
elif any(v == "failure" for v in statuses.values()):
overall_status = "failure"
status_counts = Counter(statuses.values())
status_counts.update([publish])
description = []
if overall_status in ("error", "failure"):
if status_counts.get("success") is not None:
description.append(f"{status_counts['success']} succeeded")
if status_counts.get("cancelled") is not None:
description.append(f"{status_counts['cancelled']} cancelled")
if status_counts.get("failure") is not None:
description.append(f"{status_counts['failure']} failed")
if status_counts.get("") is not None:
description.append(f"{status_counts['']} skipped")
description = ", ".join(description)
else:
description = "Successfully built Python wheels!"
with open(os.environ["GITHUB_ENV"], "a") as gh_env:
gh_env.write(f"OVERALL_STATUS={overall_status}\nDESCRIPTION={description}")
shell: python
- name: Post the status to the upstream commit
if: needs.post-pending-status.outputs.tag-ref == 'false'
run: |
INCOMING_SHA=${{ needs.post-pending-status.outputs.incoming-sha }}
gh api repos/cantera/cantera/statuses/${INCOMING_SHA} \
-H "Accept: application/vnd.github.v3+json" \
--field state="${OVERALL_STATUS}" \
--field target_url=$ACTION_URL \
--field context='PyPI Package Build' \
--field description="${DESCRIPTION}" \
--silent
env:
GITHUB_TOKEN: ${{ secrets.CANTERA_REPO_STATUS }}