Generate conda packages #369
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Generate conda packages | |
| # This action automatically generate conda packages for the packages in the robotology-superbuild | |
| # Check doc/conda-recipe-generation.md for more info | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| upload_conda_binaries: | |
| description: 'If true, upload conda binaries, for any other value only generate them without uploading.' | |
| required: true | |
| default: 'true' | |
| test_metapackages_generation: | |
| description: 'If true, also test metapackages generation. Not compatible with upload_conda_binaries set to true.' | |
| required: true | |
| default: 'false' | |
| schedule: | |
| # Run the job once a week | |
| - cron: '0 0 * * 2' | |
| release: | |
| types: [published] | |
| jobs: | |
| # Regardless of the branch on which this action is trigged, | |
| # the CONDA_BUILD_NUMBER CMake option needs to be read from the | |
| # master branch, to avoid that a conda package generation from the | |
| # master branch and one from a releases/YYYY.MM branch use the same | |
| # CONDA_BUILD_NUMBER | |
| get-conda-build-number: | |
| name: "Read Conda Build number from master branch" | |
| runs-on: ubuntu-latest | |
| # Define outputs (see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-defining-outputs-for-a-job) | |
| outputs: | |
| conda_build_number: ${{ steps.step1.outputs.conda_build_number }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: 'master' | |
| - id: step1 | |
| name: Get CONDA_BUILD_NUMBER and set it as output | |
| shell: bash | |
| run: | | |
| # Get CONDA_BUILD_NUMBER via grep and set it to an environment variable | |
| export CONDA_BUILD_NUMBER=`grep "CONDA_BUILD_NUMBER" ./conda/cmake/CondaGenerationOptions.cmake | grep -oe '\([0-9.]*\)'` | |
| echo "conda_build_number=${CONDA_BUILD_NUMBER}" >> $GITHUB_OUTPUT | |
| generate-conda-packages: | |
| name: "Generate conda packages @${{ matrix.os }}" | |
| runs-on: ${{ matrix.os }} | |
| needs: get-conda-build-number | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: ubuntu-20.04 | |
| conda_platform: linux-64 | |
| - os: macos-11.0 | |
| conda_platform: osx-64 | |
| - os: windows-2019 | |
| conda_platform: win-64 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| miniforge-variant: Mambaforge | |
| miniforge-version: latest | |
| - name: Install files to enable compilation of mex files [Conda/Linux] | |
| if: contains(matrix.os, 'ubuntu') | |
| run: | | |
| curl -L -O https://github.com/robotology/robotology-vcpkg-ports/releases/download/storage/msdk_R2020b_mexa64.zip | |
| unzip msdk_R2020b_mexa64.zip | |
| rm msdk_R2020b_mexa64.zip | |
| echo "GHA_Matlab_ROOT_DIR=${GITHUB_WORKSPACE}/msdk_R2020b_mexa64" >> $GITHUB_ENV | |
| echo "GHA_Matlab_MEX_EXTENSION=mexa64" >> $GITHUB_ENV | |
| - name: Install files to enable compilation of mex files [Conda/macOS] | |
| if: contains(matrix.os, 'macos') | |
| run: | | |
| curl -L -O https://github.com/robotology/robotology-vcpkg-ports/releases/download/storage/msdk_R2020a_mexmaci64.zip | |
| unzip msdk_R2020a_mexmaci64.zip | |
| rm msdk_R2020a_mexmaci64.zip | |
| echo "GHA_Matlab_ROOT_DIR=${GITHUB_WORKSPACE}/msdk_R2020a_mexmaci64" >> $GITHUB_ENV | |
| echo "GHA_Matlab_MEX_EXTENSION=mexmaci64" >> $GITHUB_ENV | |
| - name: Install files to enable compilation of mex files [Conda/Windows] | |
| if: contains(matrix.os, 'windows') | |
| shell: bash | |
| run: | | |
| curl -L -O https://github.com/robotology/robotology-vcpkg-ports/releases/download/storage/msdk_R2020a_mexw64.zip | |
| unzip msdk_R2020a_mexw64.zip | |
| rm msdk_R2020a_mexw64.zip | |
| echo "GHA_Matlab_ROOT_DIR=${GITHUB_WORKSPACE}/msdk_R2020a_mexw64" >> $GITHUB_ENV | |
| echo "GHA_Matlab_MEX_EXTENSION=mexw64" >> $GITHUB_ENV | |
| - name: Dependencies for conda recipes generation and upload | |
| shell: bash -l {0} | |
| run: | | |
| mamba install pyyaml jinja2 conda-build ninja anaconda-client conda-forge-pinning mamba boa multisheller | |
| - name: Print used environment | |
| shell: bash -l {0} | |
| run: | | |
| mamba list | |
| env | |
| - name: Generate recipes [Linux&macOS] | |
| if: contains(matrix.os, 'macos') || contains(matrix.os, 'ubuntu') | |
| shell: bash -l {0} | |
| run: | | |
| mkdir build | |
| cd build | |
| cmake -GNinja -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON -DCONDA_BUILD_NUMBER=${{needs.get-conda-build-number.outputs.conda_build_number}} .. | |
| - name: Generate recipes [Windows] | |
| if: contains(matrix.os, 'windows') | |
| shell: bash -l {0} | |
| run: | | |
| mkdir build | |
| cd build | |
| cmake -G"Visual Studio 16 2019" -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON -DCONDA_BUILD_NUMBER=${{needs.get-conda-build-number.outputs.conda_build_number}} .. | |
| - name: Specify additional option if we are in a release and we need to generate robotology-distro metapackages | |
| if: github.event_name == 'release' | |
| shell: bash -l {0} | |
| run: | | |
| cd build | |
| cmake -DROBOTOLOGY_PROJECT_TAGS=Custom -DCONDA_GENERATE_ROBOTOLOGY_METAPACKAGES:BOOL=ON . | |
| - name: Specify additional option if we want generate robotology-distro metapackages | |
| if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true' | |
| shell: bash -l {0} | |
| run: | | |
| cd build | |
| cmake -DCONDA_GENERATE_ROBOTOLOGY_METAPACKAGES:BOOL=ON . | |
| - name: Fail if both upload_conda_binaries and test_metapackages_generation are passed | |
| if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true' && github.event.inputs.upload_conda_binaries == 'true' | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| core.setFailed('Build failed. It is not possible to set to true both upload_conda_binaries and test_metapackages_generation.') | |
| - name: If test_metapackages_generation is enabled set an arbitrary version number | |
| if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true' | |
| shell: bash -l {0} | |
| run: | | |
| sed -i -e 's/set(INSTALLER_VERSION "")/set(INSTALLER_VERSION 1234.0)/g' ./packaging/windows/CMakeLists.txt | |
| sed -i -e 's/set(CONDA_ROBOTOLOGY_SUPERBUILD_VERSION "")/set(CONDA_ROBOTOLOGY_SUPERBUILD_VERSION 1234.0)/g' ./conda/cmake/CondaGenerationOptions.cmake | |
| - name: Build conda packages | |
| shell: bash -l {0} | |
| run: | | |
| conda config --remove channels defaults | |
| conda config --env --set channel_priority strict | |
| cd build/conda/generated_recipes | |
| # Debug generated recipes | |
| cat */meta.yaml | |
| cat */bld.bat | |
| cat */build.sh | |
| # We use the conda_build_config from conda-forge-pinning, and our local one | |
| # See https://docs.conda.io/projects/conda-build/en/latest/resources/variants.html#creating-conda-build-variant-config-files | |
| # We manually specify the build order as conda build is too slow, and conda mambabuild does not support correctly multiple recipes | |
| # see https://github.com/mamba-org/boa/issues/117 | |
| # We also manually delete the recipe directory to avoid that it is built again, | |
| # specifically because we can't use --skip-existing as these packages are part of the robotology channel | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml icub-contrib-common | |
| rm -rf icub-contrib-common | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml icub-firmware-shared | |
| rm -rf icub-firmware-shared | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml robots-configuration | |
| rm -rf robots-configuration | |
| # icub-main needs robotology channel as it also depends on esdcan, and local to avoid that robotology packages are used instead of local built one | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge -c robotology icub-main | |
| rm -rf icub-main | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml wearables | |
| rm -rf wearables | |
| # Channels specified explicitly due to https://github.com/robotology/robotology-superbuild/issues/1140 | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge human-dynamics-estimation | |
| rm -rf human-dynamics-estimation | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml robot-log-visualizer | |
| rm -rf robot-log-visualizer | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml idyntree-matlab-bindings | |
| rm -rf idyntree-matlab-bindings | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml blockfactory | |
| rm -rf blockfactory | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml yarp-matlab-bindings | |
| rm -rf yarp-matlab-bindings | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml wb-toolbox | |
| rm -rf wb-toolbox | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml blocktest | |
| rm -rf blocktest | |
| conda build -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml . | |
| - name: Build conda metapackages (if necessary) | |
| if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true') | |
| shell: bash -l {0} | |
| run: | | |
| cd build/conda/generated_recipes_metapackages | |
| # Debug generated recipes | |
| cat */meta.yaml | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge -c robotology robotology-distro | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge -c robotology robotology-distro-all | |
| - name: Upload conda packages | |
| shell: bash -l {0} | |
| # Upload by default on schedule events, and on workflow dispatch only if input upload_conda_binaries is 'true' | |
| if: github.event_name == 'schedule' || github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true') | |
| env: | |
| ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} | |
| run: | | |
| cd ${CONDA_PREFIX}/conda-bld/${{ matrix.conda_platform}}/ | |
| ls *.tar.bz2 | |
| anaconda upload --skip-existing *.tar.bz2 | |
| # Only on linux-64 we upload noarch packages (noarch packages need to be uploaded only once, as they can be reused across platforms) | |
| - name: Upload conda packages (noarch) | |
| shell: bash -l {0} | |
| # Upload by default on schedule events, and on workflow dispatch only if input upload_conda_binaries is 'true' | |
| if: matrix.conda_platform == 'linux-64' && (github.event_name == 'schedule' || github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true')) | |
| env: | |
| ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} | |
| run: | | |
| cd ${CONDA_PREFIX}/conda-bld/noarch/ | |
| ls *.tar.bz2 | |
| anaconda upload --skip-existing *.tar.bz2 | |
| # If the generate-conda-packages completed correctly and binaries are uploaded, | |
| # bump automatically the CONDA_BUILD_NUMBER in conda/cmake/CondaGenerationOptions.cmake | |
| # of master branch for future builds | |
| # the master branch is always used in case the action is triggered by a release on | |
| # a release/vYYYY.MM branch | |
| bump-conda-build-number: | |
| name: "Bump Conda Build number for future builds" | |
| runs-on: ubuntu-latest | |
| needs: generate-conda-packages | |
| if: github.event_name == 'schedule' || github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true') | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: 'master' | |
| - name: Bump Conda Build number for future builds | |
| shell: bash | |
| run: | | |
| sh ./scripts/robotologyBumpCondaBuildNumber.sh | |
| - uses: EndBug/[email protected] | |
| with: | |
| default_author: github_actions | |
| message: 'Bump CONDA_BUILD_NUMBER after successful Conda packages build and upload' |