Use pixi.sh instead of micromamba
#15
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build (conda) | ||
| on: | ||
| push: | ||
| branches: ["main"] | ||
| pull_request: | ||
| branches: ["*"] | ||
| types: [ opened, reopened, synchronize, labeled ] | ||
| schedule: | ||
| - cron: '0 4 * * *' # run once a day at 4 AM | ||
| jobs: | ||
| docker-switch: | ||
| # run if: | ||
| # - all pushes to main | ||
| # - schedule defined above | ||
| # - a PR was just labeled 'test-docker' | ||
| # - a PR with labeled 'test-docker' label was opened, reopened, or synchronized | ||
| # if at least one of these conditions is met, then the job will not run, thereby skipping all docker-based jobs | ||
| if: | | ||
| github.event_name == 'push' || | ||
| github.event_name == 'schedule' || | ||
| github.event.label.name == 'test-docker' || | ||
| contains( github.event.pull_request.labels.*.name, 'test-docker') | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Switch | ||
| run: | | ||
| echo "🐳 Docker tests are switched on, enjoy!" | ||
| set-deps-hash: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| deps-hash: ${{ steps.set-hash.outputs.deps-hash }} | ||
| short-deps-hash: ${{ steps.set-hash.outputs.short-deps-hash }} | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Set dependency files hash | ||
| id: set-hash | ||
| run: | | ||
| export DEPS_HASH=${{ hashFiles('.rattler-build/recipes/**.yaml', '.rattler-build/build.sh', 'pyproject.toml', 'docker/Dockerfile.only-deps') }} | ||
| echo "deps-hash=$DEPS_HASH" >> $GITHUB_OUTPUT | ||
| echo "short-deps-hash=$(echo $DEPS_HASH | cut -c1-7)" >> $GITHUB_OUTPUT | ||
| set-commit-sha: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| commit-sha: ${{ steps.set-sha.outputs.commit-sha }} | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Set commit sha | ||
| id: set-sha | ||
| run: | | ||
| if [ "${{ github.event_name }}" == "pull_request" ]; then | ||
| export COMMIT_SHA=${{ github.event.pull_request.head.sha }} | ||
| else | ||
| export COMMIT_SHA=${{ github.sha }} | ||
| fi | ||
| echo "commit-sha=$(echo $COMMIT_SHA | cut -c1-7)" >> $GITHUB_OUTPUT | ||
| build-conda-channel: | ||
| runs-on: ubuntu-latest | ||
| needs: set-deps-hash | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Setup micromamba env for rattler build | ||
| uses: mamba-org/setup-micromamba@v1 | ||
| with: | ||
| environment-name: rattler | ||
| init-shell: bash | ||
| - uses: actions/cache@v4 | ||
| id: restore-cache | ||
| with: | ||
| path: .rattler-build/artifacts/ # this path is gitignored, so it won't be in the checkout, only the cache | ||
| key: ${{ runner.os }}-${{ needs.set-deps-hash.outputs.deps-hash }} | ||
| - name: Install rattler-build & build artifacts (i.e. local conda channel) | ||
| if: steps.restore-cache.outputs.cache-hit != 'true' | ||
| shell: bash -leo pipefail {0} | ||
| run: | | ||
| micromamba install -c conda-forge rattler-build --yes | ||
| chmod +x .rattler-build/build.sh | ||
| ./.rattler-build/build.sh | ||
| - name: Upload conda channel | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: rattler-artifacts | ||
| path: | | ||
| .rattler-build/artifacts/ | ||
| !.rattler-build/artifacts/bld | ||
| !.rattler-build/artifacts/src_cache | ||
| if-no-files-found: error | ||
| compression-level: 0 | ||
| unit-test: | ||
| runs-on: ubuntu-latest | ||
| needs: | ||
| - build-conda-channel | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| python-version: [ | ||
| "3.10", | ||
| "3.11", | ||
| "3.12", | ||
| ] | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Download conda channel | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: rattler-artifacts | ||
| path: tmp/conda-channel | ||
| - name: Log conda channel contents | ||
| run: ls -lR tmp/conda-channel | ||
| - name: Setup Micromamba | ||
| uses: mamba-org/setup-micromamba@v1 | ||
| with: | ||
| environment-name: unit-test | ||
| init-shell: bash | ||
| - name: Install all dependencies with conda | ||
| shell: bash -leo pipefail {0} | ||
| run: | # todo: ecoscope-workflows build variants (e.g. "compile-only" w/out ecoscope-core, etc.) | ||
| micromamba install python=${{ matrix.python-version }} pytest -c conda-forge --yes | ||
| micromamba install ecoscope-workflows \ | ||
| -c file://$(pwd)/tmp/conda-channel \ | ||
| -c conda-forge \ | ||
| --only-deps \ | ||
| --yes | ||
| - name: Install our package from current head ref | ||
| shell: bash -leo pipefail {0} | ||
| run: | | ||
| python -m pip install . --no-deps | ||
| - name: Run doctests | ||
| shell: bash -leo pipefail {0} | ||
| run: | | ||
| python -m pytest -v ecoscope_workflows/ --doctest-modules \ | ||
| --ignore=ecoscope_workflows/visualize.py | ||
| - name: Test with pytest (not io) | ||
| shell: bash -leo pipefail {0} | ||
| run: | | ||
| python -m pytest -m "not io" tests -vvv | ||
| - name: Test with pytest (io) | ||
| env: | ||
| EARTHRANGER_SERVER: ${{ secrets.EARTHRANGER_SERVER }} | ||
| EARTHRANGER_USERNAME: ${{ secrets.EARTHRANGER_USERNAME }} | ||
| EARTHRANGER_PASSWORD: ${{ secrets.EARTHRANGER_PASSWORD }} | ||
| shell: bash -leo pipefail {0} | ||
| run: | | ||
| python -m pytest -m "io" tests -vvv | ||
| build-testable: | ||
| runs-on: ubuntu-latest | ||
| needs: | ||
| - docker-switch | ||
| - set-deps-hash | ||
| - set-commit-sha | ||
| - build-conda-channel | ||
| - build-only-deps | ||
|
Check failure on line 162 in .github/workflows/test-build.yaml
|
||
| outputs: | ||
| image-tag: ${{ steps.set-tag.outputs.image-tag }} | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Download conda channel | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: rattler-artifacts | ||
| path: tmp/conda-channel | ||
| - name: Log conda channel contents | ||
| run: ls -lR tmp/conda-channel | ||
| - name: Login to GitHub Container Registry | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| registry: ghcr.io | ||
| username: ${{ github.actor }} | ||
| password: ${{ secrets.GITHUB_TOKEN }} | ||
| - name: Set testable image tag on env and output for downstream jobs | ||
| id: set-tag | ||
| run: | | ||
| export NAME=wildlife-dynamics/ecoscope-workflows-testable | ||
| export TAG=${{ needs.set-deps-hash.outputs.short-deps-hash }}-${{ needs.set-commit-sha.outputs.commit-sha }} | ||
| echo "IMAGE_TAG=ghcr.io/$NAME:$TAG" >> $GITHUB_ENV | ||
| echo "image-tag=ghcr.io/$NAME:$TAG" >> $GITHUB_OUTPUT | ||
| - name: Build testable container | ||
| run: | | ||
| docker buildx build \ | ||
| --tag ${{ env.IMAGE_TAG }} \ | ||
| --platform=linux/amd64 \ | ||
| --file ./docker/Dockerfile.pixi \ | ||
| --build-arg CHANNEL_MOUNT=tmp/conda-channel \ | ||
| --cache-from=type=gha \ | ||
| --cache-to=type=gha \ | ||
| . | ||
| docker push ${{ env.IMAGE_TAG }} | ||
| local-e2e-test: | ||
| runs-on: ubuntu-latest | ||
| needs: build-testable | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Login to GitHub Container Registry | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| registry: ghcr.io | ||
| username: ${{ github.actor }} | ||
| password: ${{ secrets.GITHUB_TOKEN }} | ||
| - name: Test worker container - sequential end-to-end | ||
| shell: bash -leo pipefail {0} | ||
| run: | | ||
| docker run \ | ||
| -v `pwd`/tests:/opt/tests \ | ||
| -v `pwd`/examples:/opt/examples \ | ||
| ${{ needs.build-testable.outputs.image-tag }} \ | ||
| python -m pytest -v /opt/tests/test_examples.py \ | ||
| -k "end_to_end and sequential" | ||
| - name: Test worker container - async end-to-end | ||
| shell: bash -leo pipefail {0} | ||
| run: | | ||
| docker run \ | ||
| -v `pwd`/tests:/opt/tests \ | ||
| -v `pwd`/examples:/opt/examples \ | ||
| ${{ needs.build-testable.outputs.image-tag }} \ | ||
| python -m pytest -v /opt/tests/test_examples.py \ | ||
| -k "end_to_end and async" | ||
| remote-e2e-test: | ||
| runs-on: ubuntu-latest | ||
| needs: | ||
| - set-commit-sha | ||
| - build-conda-channel | ||
| - build-testable | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Login to GitHub Container Registry | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| registry: ghcr.io | ||
| username: ${{ github.actor }} | ||
| password: ${{ secrets.GITHUB_TOKEN }} | ||
| - name: Setup build environment | ||
| uses: mamba-org/setup-micromamba@v1 | ||
| with: | ||
| environment-name: rattler | ||
| init-shell: bash | ||
| - name: Download conda channel | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: rattler-artifacts | ||
| path: tmp/conda-channel | ||
| - name: Build & install lithops | ||
| shell: bash -leo pipefail {0} | ||
| run: | | ||
| micromamba install -c file://$(pwd)/tmp/conda-channel lithops --yes | ||
| - name: Google auth | ||
| id: google-auth | ||
| uses: google-github-actions/auth@v2 | ||
| with: | ||
| # TODO: change to workload identity | ||
| credentials_json: '${{ secrets.GCP_CLOUDRUN_LITHOPS_TESTING_SERVICE_ACCOUNT_KEY }}' | ||
| - name: Set lithops runtime name on env | ||
| run: | | ||
| echo "LITHOPS_RUNTIME_NAME=lithops-runtime-${{ github.event_name }}-${{ needs.set-commit-sha.outputs.commit-sha }}" >> $GITHUB_ENV | ||
| - name: Create lithops config | ||
| run: | | ||
| cat <<EOF > .lithops-config.yaml | ||
| lithops: | ||
| backend: gcp_cloudrun | ||
| storage: gcp_storage | ||
| log_level: INFO | ||
| data_limit: 500 | ||
| gcp: | ||
| region: us-central1 | ||
| gcp_cloudrun: | ||
| runtime: us.gcr.io/${{ steps.google-auth.outputs.project_id }}/${{ env.LITHOPS_RUNTIME_NAME }} | ||
| runtime_cpu: 2 | ||
| runtime_memory: 1000 | ||
| EOF | ||
| - name: Lithops build | ||
| shell: bash -leo pipefail {0} | ||
| run: | # note: pull + re-tag is bc iiuc we cannot pass a --build-arg to lithops runtime build? | ||
| docker pull ${{ needs.build-testable.outputs.image-tag }} | ||
| docker tag ${{ needs.build-testable.outputs.image-tag }} ecoscope-workflows-base:latest | ||
| LITHOPS_CONFIG_FILE=.lithops-config.yaml \ | ||
| lithops runtime build -b gcp_cloudrun \ | ||
| -f docker/Dockerfile.worker \ | ||
| ${{ env.LITHOPS_RUNTIME_NAME }} | ||
| - name: Lithops deploy | ||
| shell: bash -leo pipefail {0} | ||
| run: | | ||
| LITHOPS_CONFIG_FILE=.lithops-config.yaml \ | ||
| lithops runtime deploy ${{ env.LITHOPS_RUNTIME_NAME }} | ||
| - name: Test remote worker container - async end-to-end | ||
| shell: bash -leo pipefail {0} | ||
| run: | | ||
| docker run \ | ||
| -v `pwd`/tests:/opt/tests \ | ||
| -v `pwd`/examples:/opt/examples \ | ||
| ${{ needs.build-testable.outputs.image-tag }} python -m pytest -v /opt/tests/test_examples.py \ | ||
| -k "end_to_end and async" | ||
| - name: Post Lithops deploy (teardown) | ||
| if: always() # always run teardown even if previous steps fail | ||
| shell: bash -leo pipefail {0} | ||
| run: | | ||
| LITHOPS_CONFIG_FILE=.lithops-config.yaml \ | ||
| lithops runtime delete ${{ env.LITHOPS_RUNTIME_NAME }} | ||
