Skip to content
pyopenms-wheels-and-packages

pyopenms-wheels-and-packages #1

Sign in to view logs

pyopenms-wheels-and-packages

pyopenms-wheels-and-packages #1

Workflow file for this run

.github/workflows/pyopenms-wheels.yml at c1dccb3
# This builds python wheels on manylinux, macOS and Windows and uploads them
# to public pypi for release and for nightlies also to pypi.cs.uni-tuebingen.de
name: pyopenms-wheels-and-packages
# Controls when the action will run. Triggers the workflow on push or pull request
# events but only for the master branch
on:
workflow_dispatch:
inputs:
upload-to-pypi:
type: boolean
description: actually upload the release package to pypi.org
default: false
push:
tags:
- 'Release*'
branches: [ nightly, py_nightly_test ]
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
build-win:
runs-on: windows-latest
steps:
- uses: actions/checkout@v4
with:
path: OpenMS
- name: Setup cmake
uses: jwlawson/actions-setup-cmake@v2
with:
cmake-version: '3.25.x'
- name: Reduce PATH # This fixes a bug described in https://github.com/facebook/watchman/commit/59bcfbf91d2a24ab580b8a78e384f133cc202383
run: echo "DISTUTILS_USE_SDK=1"
- name: Get number of CPU cores
uses: SimenB/github-actions-cpu-cores@v2
id: cpu-cores
- name: Install Qt
uses: jurplel/install-qt-action@v3
with:
version: '5.12.6' # 5.12.7 is broken https://bugreports.qt.io/browse/QTBUG-81715
host: 'windows' # default: win64_msvc2017_64
target: 'desktop'
install-deps: 'true'
archives: 'qtsvg qtbase'
cache: 'false'
- name: Install Miniconda
shell: bash # uses git bash on windows
run: |
MINICONDA_FILENAME=Miniconda3-latest-Windows-x86_64.exe
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
./Miniconda3-latest-Windows-x86_64.exe //InstallationType=JustMe //RegisterPython=0 //S //D=$HOME/miniconda3
- name: Download contrib build from archive (Windows)
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
cd OpenMS/contrib
# Download the file using the URL fetched from GitHub
gh release download -R OpenMS/contrib --pattern 'contrib_build-Windows.tar.gz'
# Extract the archive
7z x -so contrib_build-Windows.tar.gz | 7z x -si -ttar
rm contrib_build-Windows.tar.gz
ls -la
# https://github.com/marketplace/actions/visual-studio-shell
- name: Set up Visual Studio shell
uses: egor-tensin/vs-shell@v2
with:
arch: x64
- name: Setup conda paths
shell: bash
run: |
echo "$HOME/miniconda3/bin" >> $GITHUB_PATH
echo "$HOME/miniconda3/Scripts" >> $GITHUB_PATH
- name: Build
shell: bash
run: |
PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json)
mkdir bld
pushd bld
# TODO: set generator via variable, then we can share this step
cmake --version
cmake -G "Visual Studio 17 2022" -A x64 -DOPENMS_CONTRIB_LIBS="$GITHUB_WORKSPACE/OpenMS/contrib" -DCMAKE_PREFIX_PATH="$(echo $Qt5_Dir)/lib/cmake;${Qt5_Dir}" ../OpenMS
# Note: multiple --targets only supported by CMake 3.15+
cmake --build . --config Release --target OpenMS
mkdir pyopenms_whls
# The growing PATH leads to a failure in the last iteration of the pyopenms build process.
# We will store the PATH variable in the beginning and reset it with every iteration.
RESET_PATH=$PATH
for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do
# reset the path
PATH=$RESET_PATH
# create and activate conda environment
py=$(echo "$py" | tr -d " \n")
pynodot=$(echo "$py" | tr -d ".")
conda create -n pyoms-bld-"${pynodot}" python="${py}"
source activate pyoms-bld-"${pynodot}"
# set current python executable
CURRENT_PYTHON_EXECUTABLE=$(which python)
${CURRENT_PYTHON_EXECUTABLE} -m pip install --upgrade pip
# pip install all the stuff
pip install -U setuptools
pip install -U pip
pip install -U autowrap
pip install -U pytest
pip install 'numpy<=1.26.4'
pip install -U wheel
echo $CURRENT_PYTHON_EXECUTABLE
# build pyopenms distribution
cmake --version
cmake -DPYTHON_EXECUTABLE:FILEPATH=$CURRENT_PYTHON_EXECUTABLE -DPYOPENMS=ON -DPY_NUM_THREADS=${{ steps.cpu-cores.outputs.count }} .
cmake --build . --config Release --target pyopenms
# copy to directory
cp pyOpenMS/dist/*.whl pyopenms_whls/
# clean up
conda deactivate
conda remove --name "pyoms-bld-${pynodot}" --all
# clean previous pyopenms libs
find . -name "pyopenms*.so" -exec rm -rf {} \;
done
- uses: actions/upload-artifact@v4
name: Upload artifacts
with:
name: ${{ runner.os }}-${{ runner.arch }}-wheels
path: bld/pyopenms_whls
build-macos-arm:
runs-on: macos-latest
env:
SYSTEM_VERSION_COMPAT: 0 # workaround for mac see https://github.com/actions/setup-python/issues/469 for details.
steps:
- uses: actions/checkout@v4
with:
path: OpenMS
- name: Get number of CPU cores
uses: SimenB/github-actions-cpu-cores@v2
id: cpu-cores
- name: Setup cmake
uses: jwlawson/actions-setup-cmake@v2
with:
cmake-version: '3.25.x'
- name: Install Miniconda
shell: bash
run: |
MINICONDA_FILENAME=Miniconda3-latest-MacOSX-arm64.sh
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3
- name: Setup conda paths
shell: bash
run: |
echo "$HOME/miniconda3/bin" >> $GITHUB_PATH
echo "$HOME/miniconda3/Scripts" >> $GITHUB_PATH
- name: Install build tools
run: brew install autoconf automake libtool
- name: Install contrib packages from brew
run: |
## Needed for Qt. Install before to overwrite the default softlinks on the GH runners
brew install python3 --force --overwrite
brew install --quiet ccache autoconf automake libtool ninja && brew link --overwrite ccache
brew install libsvm xerces-c boost eigen sqlite coinutils cbc cgl clp qt@5 libomp
echo "cmake_prefix=$(brew --prefix qt@5)/lib/cmake;$(brew --prefix qt@5)" >> $GITHUB_OUTPUT
echo "Qt5_DIR=$(brew --prefix qt@5)/lib/cmake/Qt5" >> $GITHUB_ENV
if [[ "${{ steps.set-vars.outputs.pkg_type }}" != "none" ]]; then
brew install --quiet doxygen ghostscript graphviz
fi
- name: Build
run: |
PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json)
export CC=clang
export CXX=clang++
export MACOSX_DEPLOYMENT_TARGET=12
# Unfortunately, on macOS due to the inofficial way of enabling OpenMP on AppleClang, passing the following
# options to setup.py extra_compile_args does not work. See also https://gist.github.com/andyfaff/084005bee32aee83d6b59e843278ab3e
export CFLAGS="-Xpreprocessor -fopenmp $CFLAGS"
export CXXFLAGS="-Xpreprocessor -fopenmp $CXXFLAGS"
mkdir bld
pushd bld
# Use -DCMAKE_FIND_DEBUG_MODE=ON for debug
cmake -DCMAKE_BUILD_TYPE="Release" -DCMAKE_PREFIX_PATH="$(brew --prefix libomp);$(echo $Qt5_Dir)/lib/cmake;${Qt5_Dir}" -DCMAKE_OSX_DEPLOYMENT_TARGET=12 -DBOOST_USE_STATIC=OFF ../OpenMS
make -j${{ steps.cpu-cores.outputs.count }} OpenMS
mkdir pyopenms_whls
for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do
py=$(echo "$py" | tr -d " \n")
pynodot=$(echo "$py" | tr -d ".")
conda create -n pyoms-bld-"${pynodot}" python="${py}"
source activate pyoms-bld-"${pynodot}"
# set current python executable
CURRENT_PYTHON_EXECUTABLE=$(which python)
# pip install all the stuff
pip install -U setuptools
pip install -U pip
pip install -U autowrap
pip install -U pytest
pip install 'numpy<=1.26.4'
pip install -U wheel
pip install -U pandas
# build pyopenms distribution (macOS)
cmake -DPYTHON_EXECUTABLE:FILEPATH=$CURRENT_PYTHON_EXECUTABLE -DPYOPENMS=ON -DPY_NUM_THREADS=${{ steps.cpu-cores.outputs.count }} .
make -j${{ steps.cpu-cores.outputs.count }} pyopenms
# copy to directory
cp pyOpenMS/dist/*.whl pyopenms_whls/
# clean up
conda deactivate
conda remove --name pyoms-bld-"${pynodot}" --all
# clean previous pyopenms libs
find . -name "pyopenms*.so" -exec rm -rf {} \;
done
- uses: actions/upload-artifact@v4
name: Upload artifacts
if: always()
with:
name: ${{ runner.os }}-buildfiles
path: |
bld/CMakeCache.txt
bld/pyOpenMS/env.py
- uses: actions/upload-artifact@v4
name: Upload wheels
with:
name: ${{ runner.os }}-${{ runner.arch }}-wheels
path: bld/pyopenms_whls
build-macos:
runs-on: macos-13
env:
SYSTEM_VERSION_COMPAT: 0 # workaround for mac see https://github.com/actions/setup-python/issues/469 for details.
steps:
- uses: actions/checkout@v4
with:
path: OpenMS
- name: Get number of CPU cores
uses: SimenB/github-actions-cpu-cores@v2
id: cpu-cores
- name: Setup cmake
uses: jwlawson/actions-setup-cmake@v2
with:
cmake-version: '3.25.x'
- name: Install Miniconda
shell: bash
run: |
MINICONDA_FILENAME=Miniconda3-latest-MacOSX-x86_64.sh
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3
- name: Setup conda paths
shell: bash
run: |
echo "$HOME/miniconda3/bin" >> $GITHUB_PATH
echo "$HOME/miniconda3/Scripts" >> $GITHUB_PATH
- name: Install build tools
run: brew install autoconf automake libtool
- name: Install contrib packages from brew
run: |
## Needed for Qt. Install before to overwrite the default softlinks on the GH runners
brew install python3 --force --overwrite
brew install --quiet ccache autoconf automake libtool ninja && brew link --overwrite ccache
brew install libsvm xerces-c boost eigen sqlite coinutils cbc cgl clp qt@5 libomp
echo "cmake_prefix=$(brew --prefix qt@5)/lib/cmake;$(brew --prefix qt@5)" >> $GITHUB_OUTPUT
echo "Qt5_DIR=$(brew --prefix qt@5)/lib/cmake/Qt5" >> $GITHUB_ENV
if [[ "${{ steps.set-vars.outputs.pkg_type }}" != "none" ]]; then
brew install --quiet doxygen ghostscript graphviz
fi
- name: Build
run: |
PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json)
export CC=clang
export CXX=clang++
export MACOSX_DEPLOYMENT_TARGET=12
# Unfortunately, on macOS due to the inofficial way of enabling OpenMP on AppleClang, passing the following
# options to setup.py extra_compile_args does not work. See also https://gist.github.com/andyfaff/084005bee32aee83d6b59e843278ab3e
export CFLAGS="-Xpreprocessor -fopenmp $CFLAGS"
export CXXFLAGS="-Xpreprocessor -fopenmp $CXXFLAGS"
mkdir bld
pushd bld
# Use -DCMAKE_FIND_DEBUG_MODE=ON for debug
cmake -DCMAKE_BUILD_TYPE="Release" -DCMAKE_PREFIX_PATH="$(brew --prefix libomp);$(echo $Qt5_Dir)/lib/cmake;${Qt5_Dir}" -DCMAKE_OSX_DEPLOYMENT_TARGET=12 -DBOOST_USE_STATIC=OFF ../OpenMS
make -j${{ steps.cpu-cores.outputs.count }} OpenMS
mkdir pyopenms_whls
for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do
py=$(echo "$py" | tr -d " \n")
pynodot=$(echo "$py" | tr -d ".")
conda create -n pyoms-bld-"${pynodot}" python="${py}"
source activate pyoms-bld-"${pynodot}"
# set current python executable
CURRENT_PYTHON_EXECUTABLE=$(which python)
# pip install all the stuff
pip install -U setuptools
pip install -U pip
pip install -U autowrap
pip install -U pytest
pip install 'numpy<=1.26.4'
pip install -U wheel
pip install -U pandas
# build pyopenms distribution (macOS)
cmake -DPYTHON_EXECUTABLE:FILEPATH=$CURRENT_PYTHON_EXECUTABLE -DPYOPENMS=ON -DPY_NUM_THREADS=${{ steps.cpu-cores.outputs.count }} .
make -j${{ steps.cpu-cores.outputs.count }} pyopenms
# copy to directory
cp pyOpenMS/dist/*.whl pyopenms_whls/
# clean up
conda deactivate
conda remove --name pyoms-bld-"${pynodot}" --all
# clean previous pyopenms libs
find . -name "pyopenms*.so" -exec rm -rf {} \;
done
- uses: actions/upload-artifact@v4
name: Upload wheels
with:
name: ${{ runner.os }}-${{ runner.arch }}-wheels
path: bld/pyopenms_whls
build-lnx:
runs-on: ubuntu-latest
container: ghcr.io/openms/contrib_manylinux_2_28:latest
steps:
# Cancels older builds if still running
- uses: rokroskar/workflow-run-cleanup-action@master
env:
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/develop'"
- uses: actions/checkout@v4
name: Checkout sources
with:
path: OpenMS
- name: Get number of CPU cores
uses: SimenB/github-actions-cpu-cores@v2
id: cpu-cores
- name: Setup cmake
uses: jwlawson/actions-setup-cmake@v2
with:
cmake-version: '3.25.x'
- name: Build on manylinux2014
shell: bash
run: |
PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json)
yum install -y jq
cmake --version
mkdir -p $GITHUB_WORKSPACE/data/wheelhouse/
mkdir -p $GITHUB_WORKSPACE/data/wheelhouse/before_fix/
LD_OLD_LIBRARY_PATH=$LD_LIBRARY_PATH
mkdir openms-build
cd openms-build
cmake -DCMAKE_BUILD_TYPE="Release" -DOPENMS_CONTRIB_LIBS="/contrib-build/" -DCMAKE_PREFIX_PATH="/contrib-build/" $GITHUB_WORKSPACE/OpenMS
make -j${{ steps.cpu-cores.outputs.count }} OpenMS
# compile and configure OpenMS
for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do
py=$(echo "$py" | tr -d " \n")
pynodot=$(echo "$py" | tr -d ".")
PYBIN=$(echo /opt/python/cp*${pynodot})
# Create a venv for this python version
"$PYBIN/bin/python" -m venv $pynodot
source $pynodot/bin/activate
"$PYBIN/bin/pip" install -U pip
"$PYBIN/bin/pip" install -U Cython
"$PYBIN/bin/pip" install -U setuptools
"$PYBIN/bin/pip" install -U wheel
"$PYBIN/bin/pip" install 'numpy<=1.26.4'
"$PYBIN/bin/pip" install -U pytest
"$PYBIN/bin/pip" install -U autowrap
# configure (don't copy deps since we use auditwheel)
cmake -DNO_DEPENDENCIES=ON -DOPENMS_CONTRIB_LIBS="/contrib-build/" -DCMAKE_PREFIX_PATH="/contrib-build/" -DPYOPENMS=On -DPython_ROOT_DIR=$PYBIN -DPython_FIND_STRATEGY="LOCATION" -DPY_NUM_THREADS=${{ steps.cpu-cores.outputs.count }} $GITHUB_WORKSPACE/OpenMS
make -j${{ steps.cpu-cores.outputs.count }} pyopenms
# ensure auditwheel can find the libraries
export LD_LIBRARY_PATH=$LD_OLD_LIBRARY_PATH:`pwd`/lib
# strip the libraries before repairing
strip --strip-all lib/libOpenMS.so
strip --strip-all lib/libOpenSwathAlgo.so
pushd pyOpenMS
# Bundle stripped plus external shared libraries into the wheels
for whl in dist/pyopenms*.whl; do
auditwheel repair "$whl" -w wheelhouse/
done
mv wheelhouse/* $GITHUB_WORKSPACE/data/wheelhouse/
popd
export LD_LIBRARY_PATH=$LD_OLD_LIBRARY_PATH
# clean previous pyopenms libs
rm pyOpenMS/pyopenms/_pyopenms*.so
# deactivate the venv
deactivate
done
- uses: actions/upload-artifact@v4
name: Upload artifacts
with:
name: ${{ runner.os }}-${{ runner.arch }}-wheels
path: data/wheelhouse/
# Tests the build workflows
test:
needs: [build-win, build-macos, build-lnx, build-macos-arm]
runs-on: ${{ matrix.os }}
env:
SYSTEM_VERSION_COMPAT: 0 #courtesy of https://github.com/actions/setup-python/issues/469 in lieu of an actual solution.
strategy:
matrix:
os: [ubuntu-latest, macos-13, windows-latest, macos-latest]
steps:
- uses: actions/checkout@v4
name: Checkout sources
with:
path: OpenMS
- name: Install Miniconda
shell: bash
run: |
if [[ "${{ runner.os }}" == "Linux" ]]; then
MINICONDA_FILENAME=Miniconda3-latest-Linux-x86_64.sh
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3
elif [[ "${{ runner.os }}" == "macOS" && "${{ runner.arch }}" == "X64" ]]; then
MINICONDA_FILENAME=Miniconda3-latest-MacOSX-x86_64.sh
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3
elif [[ "${{ runner.os }}" == "macOS" && "${{ runner.arch }}" == "ARM64" ]]; then
MINICONDA_FILENAME=Miniconda3-latest-MacOSX-arm64.sh
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3
elif [[ "${{ runner.os }}" == "Windows" ]]; then
MINICONDA_FILENAME=Miniconda3-latest-Windows-x86_64.exe
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
./Miniconda3-latest-Windows-x86_64.exe //InstallationType=JustMe //RegisterPython=0 //S
fi
- name: Setup conda paths
shell: bash
run: |
echo "$HOME/miniconda3/bin" >> $GITHUB_PATH
echo "$HOME/miniconda3/Scripts" >> $GITHUB_PATH
- name: Download artifacts
uses: actions/download-artifact@v4
with:
name: ${{ runner.os }}-${{ runner.arch }}-wheels
- name: Test
shell: bash
run: |
PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json)
echo $PYTHON_VERSIONS
for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do
py=$(echo "$py" | tr -d " \n")
pynodot=$(echo "$py" | tr -d ".")
conda create -n pyoms-bld-$pynodot python=$py
source activate pyoms-bld-$pynodot
# Find .whl for current python version
CURRENT_VERSION=$(echo $py | sed "s/\.//g") # remove "." for find
echo "Testing python version $CURRENT_VERSION"
CURRENT_WHL=$(find $GITHUB_WORKSPACE -name "*cp$CURRENT_VERSION*.whl") # Probably $GITHUB_WORKSPACE
if [[ "$CURRENT_WHL" == '' ]]; then
continue
fi
echo "Found wheel $CURRENT_WHL : Testing.."
# pip install all the stuff
python -m pip install --upgrade pip
python -m pip install $CURRENT_WHL
# check if package was installed
python -c "import importlib.util; p_name = 'pyopenms'; package = importlib.util.find_spec(p_name); print(p_name + ' was sucessfully installed! Nice one!') if package is not None else print('Something seems to be wrong!')"
# run test
python -c "exec(\"import pyopenms\nprint(pyopenms.EmpiricalFormula(\'C6H12O6\').getMonoWeight())\")"
conda deactivate
# clean up
conda remove --name pyoms-bld-$pynodot --all
done
# Upload the wheels (the action only runs on ubuntu slaves)
publish:
needs: [test] # doesn't really NEED it but we want the tests to pass first
runs-on: ubuntu-latest
steps:
# haha github is just ridiculous.. not getting their actions to do basic stuff for years
- name: Download artifacts
uses: actions/download-artifact@v4
with:
name: Linux-X64-wheels
path: /home/runner/work/OpenMS/OpenMS/Linux-X64-wheels
- name: Download artifacts
uses: actions/download-artifact@v4
with:
name: Windows-X64-wheels
path: /home/runner/work/OpenMS/OpenMS/Windows-X64-wheels
- name: Download artifacts
uses: actions/download-artifact@v4
with:
name: macOS-X64-wheels
path: /home/runner/work/OpenMS/OpenMS/macOS-X64-wheels
- name: Download artifacts
uses: actions/download-artifact@v4
with:
name: macOS-ARM64-wheels
path: /home/runner/work/OpenMS/OpenMS/macOS-ARM64-wheels
- name: Merge all artifacts
run: |
mkdir wheels
cp -lr *-wheels/* wheels/
pwd
ls -la wheels/
- name: Publish package to nightly PyPI
# Upload the wheels to our pypi server for develop, nightly and release branches (unless it's explictly stated to upload it to pypi.org)
if: contains(github.ref, 'develop') || contains(github.ref, 'nightly') || (contains(github.ref, 'Release') && inputs.upload-to-pypi == 'false')
uses: pypa/gh-action-pypi-publish@release/v1
with:
repository-url: https://pypi.cs.uni-tuebingen.de/
user: openms
password: ${{ secrets.abiservices_pypi_pw }}
packages-dir: ${{ github.workspace }}/wheels
- name: Publish package to PyPI
if: contains(github.ref, 'Release') && inputs.upload-to-pypi
uses: pypa/gh-action-pypi-publish@release/v1
with:
user: __token__
repository-url: https://test.pypi.org/
password: ${{ secrets.pypi_api_token_release }}
packages-dir: ${{ github.workspace }}/wheels