feat: simplify unify containers #115
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Build all container images. | |
| # | |
| name: Build images | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| pull_request: | |
| paths-ignore: | |
| - '.circleci/**' | |
| - 'docs/**' | |
| - 'test/**' | |
| env: | |
| # Used to override BIOCONDA_UTILS_VERSION in images/versions.sh | |
| BIOCONDA_UTILS_VERSION: ${{ github.event.release && github.event.release.tag_name || github.head_ref || github.ref_name }} | |
| jobs: | |
| # JOBS FOR BUILDING IMAGES | |
| # ---------------------------------------------------------------------- | |
| # These jobs will build images for archs, put them into a manifest, and push | |
| # that to GitHub Container Registry. Later, the testing jobs will test and | |
| # push to quay.io. | |
| build-base-debian: | |
| name: Build base-debian | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| # A note on these TAG_EXISTS_* outputs: these allow subsequent jobs to | |
| # change behavior (e.g., skip building or skip pushing to ghcr) depending | |
| # on whether an image has already been created. | |
| TAG_EXISTS_base-debian: ${{ steps.base-debian.outputs.TAG_EXISTS_base-debian }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install qemu dependency | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y qemu-user-static | |
| - name: base-debian | |
| id: base-debian | |
| run: | | |
| source images/versions.sh | |
| if [ $(tag_exists $BASE_DEBIAN_IMAGE_NAME $BASE_TAG) ]; then | |
| echo "TAG_EXISTS_base-debian=true" >> $GITHUB_OUTPUT | |
| else | |
| cd images && bash build.sh base-glibc-debian-bash | |
| fi | |
| - name: push to ghcr | |
| if: '${{ ! steps.base-debian.outputs.TAG_EXISTS_base-debian }}' | |
| run: | | |
| echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin | |
| source images/versions.sh | |
| push_to_ghcr $BASE_DEBIAN_IMAGE_NAME $BASE_TAG | |
| build-base-busybox: | |
| name: Build base-busybox | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| TAG_EXISTS_base-busybox: ${{ steps.base-busybox.outputs.TAG_EXISTS_base-busybox }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install qemu dependency | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y qemu-user-static | |
| - name: base-busybox | |
| id: base-busybox | |
| run: | | |
| source images/versions.sh | |
| if [ $(tag_exists $BASE_BUSYBOX_IMAGE_NAME $BASE_TAG) ]; then | |
| echo "TAG_EXISTS_base-busybox=true" >> $GITHUB_OUTPUT | |
| else | |
| cd images && bash build.sh base-glibc-busybox-bash | |
| fi | |
| - name: push to ghcr | |
| if: '${{ ! steps.base-busybox.outputs.TAG_EXISTS_base-busybox }}' | |
| run: | | |
| echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin | |
| source images/versions.sh | |
| push_to_ghcr $BASE_BUSYBOX_IMAGE_NAME $BASE_TAG | |
| build-build-env: | |
| name: Build build-env | |
| outputs: | |
| TAG_EXISTS_build-env: ${{ steps.build-env.outputs.TAG_EXISTS_build-env }} | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install qemu dependency | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y qemu-user-static | |
| - name: build-env | |
| id: build-env | |
| run: | | |
| source images/versions.sh | |
| if [ $(tag_exists $BUILD_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG) ]; then | |
| echo "TAG_EXISTS_build-env=true" >> $GITHUB_OUTPUT | |
| else | |
| cd images && bash build.sh bioconda-utils-build-env-cos7 | |
| fi | |
| - name: push to ghcr | |
| if: '${{ ! steps.build-env.outputs.TAG_EXISTS_build-env }}' | |
| run: | | |
| echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin | |
| source images/versions.sh | |
| push_to_ghcr $BUILD_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG | |
| build-create-env: | |
| name: Build create-env | |
| needs: [build-build-env, build-base-busybox] | |
| outputs: | |
| TAG_EXISTS_create-env: ${{ steps.create-env.outputs.TAG_EXISTS_create-env }} | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install qemu dependency | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y qemu-user-static | |
| - name: Build create-env | |
| id: create-env | |
| run: | | |
| source images/versions.sh | |
| echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin | |
| if [ $(tag_exists $CREATE_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG) ]; then | |
| echo "TAG_EXISTS_create-env=true" >> $GITHUB_OUTPUT | |
| else | |
| cd images && bash build.sh create-env | |
| fi | |
| - name: push to ghcr | |
| if: '${{ ! steps.create-env.outputs.TAG_EXISTS_create-env }}' | |
| run: | | |
| echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin | |
| source images/versions.sh | |
| push_to_ghcr $CREATE_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG | |
| # END OF BUILDING IMAGES | |
| # ---------------------------------------------------------------------- | |
| # START TESTING | |
| # These testing jobs will run the respective Dockerfile.test in each image | |
| # directory. | |
| test: | |
| name: test bioconda-utils with images | |
| runs-on: ubuntu-20.04 | |
| needs: [build-base-debian, build-base-busybox, build-build-env, build-create-env] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # # Clone bioconda-recipes to use as part of the tests. | |
| # - uses: actions/checkout@v4 | |
| # with: | |
| # fetch-depth: 0 | |
| # repository: bioconda/bioconda-recipes | |
| # path: recipes | |
| # - name: set path | |
| # run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH | |
| - name: Install bioconda-utils | |
| run: | | |
| export BIOCONDA_DISABLE_BUILD_PREP=1 | |
| BRANCH=simplify-unify-containers | |
| wget https://raw.githubusercontent.com/bioconda/bioconda-common/${BRANCH}/{common,install-and-set-up-conda,configure-conda}.sh | |
| source images/versions.sh | |
| # Ensure install-and-set-up-conda uses same version as in the container | |
| # (which uses images/versions.sh | |
| export BIOCONDA_UTILS_TAG=$BIOCONDA_UTILS_VERSION | |
| bash install-and-set-up-conda.sh | |
| eval "$(conda shell.bash hook)" | |
| conda create -n bioconda -y --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt | |
| conda activate bioconda | |
| python setup.py install | |
| - name: test | |
| run: | | |
| eval "$(conda shell.bash hook)" | |
| conda activate bioconda | |
| source images/versions.sh | |
| # Figure out which registry to use for each image, based on what was built. | |
| [ ${{ needs.build-build-env.outputs.TAG_EXISTS_build-env }} ] && BUILD_ENV_REGISTRY='quay.io/bioconda' || BUILD_ENV_REGISTRY="ghcr.io/bioconda" | |
| [ ${{ needs.build-create-env.outputs.TAG_EXISTS_create-env }} ] && CREATE_ENV_REGISTRY='quay.io/bioconda' || CREATE_ENV_REGISTRY="ghcr.io/bioconda" | |
| [ ${{ needs.build-base-busybox.outputs.TAG_EXISTS_base_busybox }} ] && DEST_BASE_REGISTRY='quay.io/bioconda' || DEST_BASE_REGISTRY="ghcr.io/bioconda" | |
| [ ${{ needs.build-base-debian.outputs.TAG_EXISTS_base_debian }} ] && DEST_EXTENDED_BASE_REGISTRY='quay.io/bioconda' || DEST_EXTENDED_BASE_REGISTRY="ghcr.io/bioconda" | |
| # Tell mulled-build which image to use | |
| export DEST_BASE_IMAGE="${DEST_BASE_IMAGE_REGISTRY}/${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG}" | |
| export DEFAULT_BASE_IMAGE="${DEST_BASE_REGISTRY}/${BASE_BUSYBOX_IMAGE_NAME}" | |
| export DEFAULT_EXTENDED_BASE_IMAGE="${DEST_EXTENDED_BASE_REGISTRY}/${BASE_DEBIAN_IMAGE_NAME}" | |
| export BUILD_ENV_IMAGE="${BUILD_ENV_REGISTRY}/${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" | |
| export CREATE_ENV_IMAGE="${CREATE_ENV_REGISTRY}/${CREATE_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" | |
| # # Build a package with containers. | |
| # cd recipes | |
| # bioconda-utils build \ | |
| # --docker-base-image "${BUILD_ENV_REGISTRY}/${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" \ | |
| # --mulled-conda-image "${CREATE_ENV_REGISTRY}/${CREATE_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" \ | |
| # --packages seqtk \ | |
| # --docker \ | |
| # --mulled-test \ | |
| # --force | |
| py.test --durations=0 test/ -v --log-level=DEBUG -k "docker" --tb=native | |
| # END TESTING | |
| # ------------------------------------------------------------------------ | |
| # START PUSHING IMAGES | |
| # For these push steps, a repository must first exist on quay.io/bioconda | |
| # AND that repository must also be configured to allow write access for the | |
| # appropriate service account. This must be done by a user with admin | |
| # access to quay.io/bioconda. | |
| # | |
| # This uses the TAG_EXISTS_* outputs from previous jobs to determine if | |
| # a push to quay.io should happen. | |
| push: | |
| name: push images | |
| runs-on: ubuntu-20.04 | |
| needs: [build-base-debian, build-base-busybox, build-build-env, build-create-env, test] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: push base-debian | |
| if: ${{ ! needs.base-debian.outputs.TAG_EXISTS_base-debian }} | |
| run: | | |
| echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin | |
| source images/versions.sh | |
| move_from_ghcr_to_quay ${BASE_DEBIAN_IMAGE_NAME} ${BASE_TAG} | |
| - name: push base-busybox | |
| if: ${{ ! needs.base-busybox.outputs.TAG_EXISTS_base-busybox }} | |
| run: | | |
| echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin | |
| source images/versions.sh | |
| move_from_ghcr_to_quay ${BASE_BUSYBOX_IMAGE_NAME} ${BASE_TAG} | |
| - name: push create-env | |
| if: ${{ ! needs.create-env.outputs.TAG_EXISTS_create-env }} | |
| run: | | |
| echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin | |
| source images/versions.sh | |
| move_from_ghcr_to_quay ${CREATE_ENV_IMAGE_NAME} ${BIOCONDA_IMAGE_TAG} | |
| - name: push build-env | |
| if: ${{ ! needs.build-env.outputs.TAG_EXISTS_build-env }} | |
| run: | | |
| echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin | |
| source images/versions.sh | |
| move_from_ghcr_to_quay ${BUILD_ENV_IMAGE_NAME} ${BIOCONDA_IMAGE_TAG} |