Build worker container with custom conda channel #2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| on: | |
| push: | |
| branches: ["main"] | |
| pull_request: | |
| branches: ["*"] | |
| jobs: | |
| set-deps-hash: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| deps-hash: ${{ steps.set-hash.outputs.deps-hash }} | |
| short-deps-hash: ${{ steps.set-hash.outputs.short-deps-hash }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set dependency files hash | |
| id: set-hash | |
| run: | | |
| export DEPS_HASH=${{ hashFiles('.rattler-build/recipes/**.yaml', '.rattler-build/build.sh', 'pyproject.toml', 'docker/Dockerfile.only-deps') }} | |
| echo "deps-hash=$DEPS_HASH" >> $GITHUB_OUTPUT | |
| echo "short-deps-hash=$(echo $DEPS_HASH | cut -c1-7)" >> $GITHUB_OUTPUT | |
| set-commit-sha: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| commit-sha: ${{ steps.set-sha.outputs.commit-sha }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set commit sha | |
| id: set-sha | |
| run: | | |
| if [ "${{ github.event_name }}" == "pull_request" ]; then | |
| export COMMIT_SHA=${{ github.event.pull_request.head.sha }} | |
| else | |
| export COMMIT_SHA=${{ github.sha }} | |
| fi | |
| echo "commit-sha=$(echo $COMMIT_SHA | cut -c1-7)" >> $GITHUB_OUTPUT | |
| build-conda-channel: | |
| runs-on: ubuntu-latest | |
| needs: set-deps-hash | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup micromamba env for rattler build | |
| uses: mamba-org/setup-micromamba@v1 | |
| with: | |
| environment-name: rattler | |
| init-shell: bash | |
| - uses: actions/cache@v4 | |
| id: restore-cache | |
| with: | |
| path: .rattler-build/artifacts/ # this path is gitignored, so it won't be in the checkout, only the cache | |
| key: ${{ runner.os }}-${{ needs.set-deps-hash.outputs.deps-hash }} | |
| - name: Install rattler-build & build artifacts (i.e. local conda channel) | |
| if: steps.restore-cache.outputs.cache-hit != 'true' | |
| shell: bash -leo pipefail {0} | |
| run: | | |
| micromamba install -c conda-forge rattler-build --yes | |
| chmod +x .rattler-build/build.sh | |
| ./.rattler-build/build.sh | |
| - name: Upload conda channel | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: rattler-artifacts | |
| path: | | |
| .rattler-build/artifacts/ | |
| !.rattler-build/artifacts/bld | |
| !.rattler-build/artifacts/src_cache | |
| if-no-files-found: error | |
| compression-level: 0 | |
| unit-test: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-conda-channel | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [ | |
| "3.10", | |
| "3.11", | |
| "3.12", | |
| ] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download conda channel | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: rattler-artifacts | |
| path: tmp/conda-channel | |
| - name: Log conda channel contents | |
| run: ls -lR tmp/conda-channel | |
| - name: Setup Micromamba | |
| uses: mamba-org/setup-micromamba@v1 | |
| with: | |
| cache-environment: true # todo how to handle caching in this case? | |
| cache-downloads: true | |
| init-shell: bash | |
| - name: Install all dependencies with conda | |
| shell: bash -leo pipefail {0} | |
| run: | # todo: ecoscope-workflows build variants (e.g. "compile-only" w/out ecoscope-core-tasks, etc.) | |
| micromamba install python=${{ matrix.python-version }} -c conda-forge --yes \ | |
| micromamba install ecoscope-workflows \ | |
| -c file://$(pwd)/tmp/conda-channel \ | |
| -c conda-forge \ | |
| --only-deps \ | |
| --yes | |
| - name: Install our package from current head ref | |
| shell: bash -leo pipefail {0} | |
| run: | | |
| python -m pip install . --no-deps | |
| - name: Run doctests | |
| shell: bash -leo pipefail {0} | |
| run: | | |
| python -m pytest -v ecoscope_workflows/ --doctest-modules \ | |
| --ignore=ecoscope_workflows/visualize.py | |
| - name: Export shell name and conda env name to env | |
| run: | | |
| echo 'SHELL="bash -leo pipefail"' >> $GITHUB_ENV | |
| echo "CONDA_ENV_NAME=${{ matrix.environment-name }}" >> $GITHUB_ENV | |
| - name: Test with pytest | |
| env: | |
| EARTHRANGER_SERVER: ${{ secrets.EARTHRANGER_SERVER }} | |
| EARTHRANGER_USERNAME: ${{ secrets.EARTHRANGER_USERNAME }} | |
| EARTHRANGER_PASSWORD: ${{ secrets.EARTHRANGER_PASSWORD }} | |
| shell: bash -leo pipefail {0} | |
| run: | # FIXME: run IO tests separately since they require credentials which could expire and fail the tests | |
| python -m pytest -m "not io" tests -vvv | |
| build-only-deps: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - set-deps-hash | |
| - build-conda-channel | |
| outputs: | |
| image-tag: ${{ steps.set-tag.outputs.image-tag }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download conda channel | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: rattler-artifacts | |
| path: tmp/conda-channel | |
| - name: Log conda channel contents | |
| run: ls -lR tmp/conda-channel | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set only deps image tag on env and output for downstream jobs | |
| id: set-tag | |
| run: | # note: tag is based on deps hash, so we can avoid rebuilding if deps haven't changed | |
| export IMAGE_NAME=ecoscope-workflows-only-deps | |
| echo "IMAGE_NAME=$IMAGE_NAME" >> $GITHUB_ENV | |
| echo "IMAGE_TAG=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME:${{ needs.set-deps-hash.outputs.short-deps-hash }}" >> $GITHUB_ENV | |
| echo "image-tag=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME:${{ needs.set-deps-hash.outputs.short-deps-hash }}" >> $GITHUB_OUTPUT | |
| - name: Check if tag already exists on ghcr | |
| id: check-tag | |
| run: | | |
| export has_tag=$(\ | |
| curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ | |
| https://api.github.com/orgs/${{ github.repository_owner }}/packages/container/${{ env.IMAGE_NAME }}/versions | \ | |
| jq '[ .[] | select(.metadata.container.tags[] == "${{ needs.set-deps-hash.outputs.short-deps-hash }}") ]') | |
| echo $has_tag | |
| if [ "$has_tag" != "[]" ]; then | |
| echo "exists=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "exists=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Build and push only deps image | |
| if: steps.check-tag.outputs.exists != 'true' # todo: some way to force rebuild if needed | |
| run: | | |
| docker buildx build \ | |
| --tag ${{ env.IMAGE_TAG }} \ | |
| --platform=linux/amd64 \ | |
| --file docker/Dockerfile.only-deps \ | |
| --build-arg CHANNEL_MOUNT=tmp/conda-channel \ | |
| --cache-from=type=gha \ | |
| --cache-to=type=gha \ | |
| . | |
| docker push ${{ env.IMAGE_TAG }} | |
| build-testable: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - set-deps-hash | |
| - set-commit-sha | |
| - build-conda-channel | |
| - build-only-deps | |
| outputs: | |
| image-tag: ${{ steps.set-tag.outputs.image-tag }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download conda channel | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: rattler-artifacts | |
| path: tmp/conda-channel | |
| - name: Log conda channel contents | |
| run: ls -lR tmp/conda-channel | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set testable image tag on env and output for downstream jobs | |
| id: set-tag | |
| run: | | |
| export NAME=wildlife-dynamics/ecoscope-workflows-testable | |
| export TAG=${{ needs.set-deps-hash.outputs.short-deps-hash }}-${{ needs.set-commit-sha.outputs.commit-sha }} | |
| echo "IMAGE_TAG=ghcr.io/$NAME:$TAG" >> $GITHUB_ENV | |
| echo "image-tag=ghcr.io/$NAME:$TAG" >> $GITHUB_OUTPUT | |
| - name: Build testable container | |
| run: | | |
| docker buildx build \ | |
| --tag ${{ env.IMAGE_TAG }} \ | |
| --platform=linux/amd64 \ | |
| --file ./docker/Dockerfile.base \ | |
| --build-arg ONLY_DEPS_IMAGE_TAG=${{ needs.build-only-deps.outputs.image-tag }} \ | |
| --cache-from=type=gha \ | |
| --cache-to=type=gha \ | |
| . | |
| docker push ${{ env.IMAGE_TAG }} | |
| local-e2e-test: | |
| runs-on: ubuntu-latest | |
| needs: build-testable | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Test worker container - sequential end-to-end | |
| shell: bash -leo pipefail {0} | |
| run: | | |
| docker run \ | |
| -v `pwd`/tests:/opt/tests \ | |
| -v `pwd`/examples:/opt/examples \ | |
| ${{ needs.build-testable.outputs.image-tag }} \ | |
| /env/bin/python -m pytest -v /opt/tests/test_examples.py \ | |
| -k "end_to_end and sequential" | |
| - name: Test worker container - async end-to-end | |
| shell: bash -leo pipefail {0} | |
| run: | | |
| docker run \ | |
| -v `pwd`/tests:/opt/tests \ | |
| -v `pwd`/examples:/opt/examples \ | |
| ${{ needs.build-testable.outputs.image-tag }} \ | |
| /env/bin/python -m pytest -v /opt/tests/test_examples.py \ | |
| -k "end_to_end and async" | |
| remote-e2e-test: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - set-commit-sha | |
| - build-conda-channel | |
| - build-testable | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Setup build environment | |
| uses: mamba-org/setup-micromamba@v1 | |
| with: | |
| environment-name: rattler | |
| init-shell: bash | |
| - name: Download conda channel | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: rattler-artifacts | |
| path: tmp/conda-channel | |
| - name: Build & install lithops | |
| shell: bash -leo pipefail {0} | |
| run: | | |
| micromamba install -c file://$(pwd)/tmp/conda-channel lithops --yes | |
| - name: Google auth | |
| id: google-auth | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| # TODO: change to workload identity | |
| credentials_json: '${{ secrets.GCP_CLOUDRUN_LITHOPS_TESTING_SERVICE_ACCOUNT_KEY }}' | |
| - name: Set lithops runtime name on env | |
| run: | | |
| echo "LITHOPS_RUNTIME_NAME=lithops-runtime-${{ github.event_name }}-${{ needs.set-commit-sha.outputs.commit-sha }}" >> $GITHUB_ENV | |
| - name: Create lithops config | |
| run: | | |
| cat <<EOF > .lithops-config.yaml | |
| lithops: | |
| backend: gcp_cloudrun | |
| storage: gcp_storage | |
| log_level: INFO | |
| data_limit: 500 | |
| gcp: | |
| region: us-central1 | |
| gcp_cloudrun: | |
| runtime: us.gcr.io/${{ steps.google-auth.outputs.project_id }}/${{ env.LITHOPS_RUNTIME_NAME }} | |
| runtime_cpu: 2 | |
| runtime_memory: 1000 | |
| EOF | |
| - name: Lithops build | |
| shell: bash -leo pipefail {0} | |
| run: | # note: pull + re-tag is bc iiuc we cannot pass a --build-arg to lithops runtime build? | |
| docker pull ${{ needs.build-testable.outputs.image-tag }} | |
| docker tag ${{ needs.build-testable.outputs.image-tag }} ecoscope-workflows-base:latest | |
| LITHOPS_CONFIG_FILE=.lithops-config.yaml \ | |
| lithops runtime build -b gcp_cloudrun \ | |
| -f docker/Dockerfile.worker \ | |
| ${{ env.LITHOPS_RUNTIME_NAME }} | |
| - name: Lithops deploy | |
| shell: bash -leo pipefail {0} | |
| run: | | |
| LITHOPS_CONFIG_FILE=.lithops-config.yaml \ | |
| lithops runtime deploy ${{ env.LITHOPS_RUNTIME_NAME }} | |
| - name: Test remote worker container - async end-to-end | |
| shell: bash -leo pipefail {0} | |
| run: | | |
| docker run \ | |
| -v `pwd`/tests:/opt/tests \ | |
| -v `pwd`/examples:/opt/examples \ | |
| ${{ needs.build-testable.outputs.image-tag }} /env/bin/python -m pytest -v /opt/tests/test_examples.py \ | |
| -k "end_to_end and async" | |
| - name: Post Lithops deploy (teardown) | |
| if: always() # always run teardown even if previous steps fail | |
| shell: bash -leo pipefail {0} | |
| run: | | |
| LITHOPS_CONFIG_FILE=.lithops-config.yaml \ | |
| lithops runtime delete ${{ env.LITHOPS_RUNTIME_NAME }} |