From 3ad0e8c9d443b9adedeee7165e1fe2112558d631 Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Mon, 16 Oct 2023 23:10:07 +0200 Subject: [PATCH 1/4] feat: image with conda packages in homedir --- .../workflows/build-and-push-to-docker.yml | 66 +++++++++++++++++++ docker/py/Dockerfile | 2 + 2 files changed, 68 insertions(+) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index 58e2be97..63352745 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -114,6 +114,72 @@ jobs: tags: ${{ steps.meta.outputs.tags }} provenance: false + build-py-homedir-env: + needs: lint + runs-on: ubuntu-latest + + strategy: + fail-fast: true + matrix: + BASE_PYTHON_VERSION: + - "3.10" + - "3.9" + - "3.8" + steps: + - name: Docker Login + uses: Azure/docker-login@v1 + with: + username: ${{ secrets.RENKU_DOCKER_USERNAME }} + password: ${{ secrets.RENKU_DOCKER_PASSWORD }} + - uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2.0.0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.0.0 + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.DOCKER_PREFIX }}-py-homedir-envs + tags: | + type=sha,prefix=${{ matrix.BASE_PYTHON_VERSION }}- + - name: Build and load + uses: docker/build-push-action@v4 + with: + build-args: | + BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }} + CONDA_ENVS_DIRS=/home/jovyan/work/envs + cache-from: type=gha + cache-to: type=gha,mode=max + context: docker/py + labels: ${{ steps.meta.outputs.labels }} + load: true + tags: ${{ steps.meta.outputs.tags }} + provenance: false + - name: Image Acceptance Tests + uses: cypress-io/github-action@v4 + env: + TEST_IMAGE_NAME: ${{ env.DOCKER_PREFIX }}-py-homedir-envs:${{ steps.meta.outputs.version }} + TEST_USER_NAME: jovyan + TEST_SPEC: jupyterlab.cy.ts + with: + working-directory: tests + command: npx mocha -r ts-node/register index.ts + # export the default base image for other jobs to use + - name: Build and export + if: matrix.BASE_PYTHON_VERSION == env.DEFAULT_PYTHON_VERSION + uses: docker/build-push-action@v4 + with: + build-args: | + BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }} + CONDA_ENVS_DIRS=/home/jovyan/work/envs + cache-from: type=gha + context: docker/py + labels: ${{ steps.meta.outputs.labels }} + push: true + tags: ${{ steps.meta.outputs.tags }} + provenance: false + build-py-multiarch: needs: build-py runs-on: ubuntu-latest diff --git a/docker/py/Dockerfile b/docker/py/Dockerfile index bdf1b99f..57650343 100644 --- a/docker/py/Dockerfile +++ b/docker/py/Dockerfile @@ -105,3 +105,5 @@ COPY --chown=1000:100 --from=builder /opt/conda /opt/conda COPY --chown=1000:100 --from=builder "$HOME/.renku" "$HOME/.renku" RUN ln -s "$HOME/.renku/venv/bin/renku" "$HOME/.renku/bin/renku" && \ ln -s "$HOME/.renku/venv/bin/_toil_worker" "$HOME/.renku/bin/" +ARG CONDA_ENVS_DIRS=/opt/conda/envs +ENV CONDA_ENVS_DIRS=${CONDA_ENVS_DIRS} From d051c1368734821dd53214dd2f9823ff3c121f26 Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Tue, 17 Oct 2023 11:01:53 +0200 Subject: [PATCH 2/4] squashme: minor fix --- .github/workflows/build-and-push-to-docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index 63352745..575dbe9f 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -591,6 +591,7 @@ jobs: TEST_SPEC: rstudio.cy.ts with: working-directory: tests + start: docker images command: npx mocha -r ts-node/register index.ts - name: Build and push uses: docker/build-push-action@v4 From 48961fc00f346e53fe89b9ab87384d16539d8389 Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Tue, 17 Oct 2023 12:03:27 +0200 Subject: [PATCH 3/4] squashme: update docker build actions --- .../workflows/build-and-push-to-docker.yml | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index 575dbe9f..97ccd4a4 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -71,7 +71,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v2.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2.0.0 + uses: docker/setup-buildx-action@v3 - name: Docker meta id: meta uses: docker/metadata-action@v4 @@ -80,7 +80,7 @@ jobs: tags: | type=sha,prefix=${{ matrix.BASE_PYTHON_VERSION }}- - name: Build and load - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }} @@ -103,7 +103,7 @@ jobs: # export the default base image for other jobs to use - name: Build and export if: matrix.BASE_PYTHON_VERSION == env.DEFAULT_PYTHON_VERSION - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }} @@ -135,7 +135,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v2.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2.0.0 + uses: docker/setup-buildx-action@v3 - name: Docker meta id: meta uses: docker/metadata-action@v4 @@ -144,7 +144,7 @@ jobs: tags: | type=sha,prefix=${{ matrix.BASE_PYTHON_VERSION }}- - name: Build and load - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }} @@ -168,7 +168,7 @@ jobs: # export the default base image for other jobs to use - name: Build and export if: matrix.BASE_PYTHON_VERSION == env.DEFAULT_PYTHON_VERSION - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }} @@ -201,7 +201,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v2.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2.0.0 + uses: docker/setup-buildx-action@v3 - name: Docker meta id: meta uses: docker/metadata-action@v4 @@ -212,7 +212,7 @@ jobs: type=semver,pattern={{version}},prefix=${{ matrix.BASE_PYTHON_VERSION }}- type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' && matrix.BASE_PYTHON_VERSION == env.DEFAULT_PYTHON_VERSION }} - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: docker/py push: true @@ -279,7 +279,7 @@ jobs: echo "sha_short=$sha_short" >> $GITHUB_OUTPUT echo "renku_base=${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${sha_short}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2.0.0 + uses: docker/setup-buildx-action@v3 - name: Docker meta id: meta uses: docker/metadata-action@v4 @@ -290,7 +290,7 @@ jobs: type=semver,pattern={{version}},prefix=${{ matrix.CUDA_VERSION }}- type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' && matrix.CUDA_VERSION == '11.7' }} - name: Build and load - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | RENKU_BASE=${{ steps.vars.outputs.renku_base }} @@ -318,7 +318,7 @@ jobs: working-directory: tests command: npx mocha -r ts-node/register index.ts - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | RENKU_BASE=${{ steps.vars.outputs.renku_base }} @@ -361,7 +361,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v2.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2.0.0 + uses: docker/setup-buildx-action@v3 - name: Docker meta id: meta uses: docker/metadata-action@v4 @@ -372,7 +372,7 @@ jobs: type=semver,pattern={{version}},prefix= type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }},prefix= - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | RENKU_BASE=${{ steps.vars.outputs.renku_base }} @@ -399,7 +399,7 @@ jobs: username: ${{ secrets.RENKU_DOCKER_USERNAME }} password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2.0.0 + uses: docker/setup-buildx-action@v3 - uses: actions/checkout@v3 - name: Set outputs id: vars @@ -417,7 +417,7 @@ jobs: type=semver,pattern={{version}},prefix= type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }},prefix= - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | BASE_IMAGE=${{ env.DOCKER_PREFIX }}-vnc:${{ steps.vars.outputs.sha_short }} @@ -454,7 +454,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v2.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2.0.0 + uses: docker/setup-buildx-action@v3 - name: Set outputs id: vars run: | @@ -471,7 +471,7 @@ jobs: type=semver,pattern={{version}},prefix=${{ matrix.JULIA_VERSION }}- type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' && matrix.JULIA_VERSION == '1.9.0' }} - name: Build and load - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | RENKU_BASE=${{ steps.vars.outputs.renku_base }} @@ -494,7 +494,7 @@ jobs: working-directory: tests command: npx mocha -r ts-node/register index.ts - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | RENKU_BASE=${{ steps.vars.outputs.renku_base }} @@ -553,7 +553,7 @@ jobs: - uses: actions/checkout@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2.0.0 + uses: docker/setup-buildx-action@v3 - name: Set outputs id: vars run: | @@ -570,7 +570,7 @@ jobs: type=semver,pattern={{version}},prefix=${{ matrix.RVERSION }}- type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' && matrix.RVERSION == 'devel' }} - name: Build and load - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | RENKU_BASE=${{ steps.vars.outputs.renku_base }} @@ -594,7 +594,7 @@ jobs: start: docker images command: npx mocha -r ts-node/register index.ts - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | RENKU_BASE=${{ steps.vars.outputs.renku_base }} From dc9f02fdc6e9582ea91573d0caec33e5d508b29e Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Tue, 17 Oct 2023 15:58:16 +0200 Subject: [PATCH 4/4] chore: make disk space --- .github/workflows/build-and-push-to-docker.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index 97ccd4a4..5a2682ea 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -569,6 +569,11 @@ jobs: type=sha,prefix=${{ matrix.RVERSION }}- type=semver,pattern={{version}},prefix=${{ matrix.RVERSION }}- type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' && matrix.RVERSION == 'devel' }} + - name: Clean cache + run: | + df -h + rm -rf /opt/hostedtoolcache/* + df -h - name: Build and load uses: docker/build-push-action@v5 with: