From 421c73b53536c2741900523fcb47a7ef9f2c6a1e Mon Sep 17 00:00:00 2001 From: cmdoret Date: Thu, 24 Oct 2024 12:34:37 +0200 Subject: [PATCH 01/11] feat(config): template dockerfile --- config/template.Dockerfile | 69 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 config/template.Dockerfile diff --git a/config/template.Dockerfile b/config/template.Dockerfile new file mode 100644 index 0000000..e96ec44 --- /dev/null +++ b/config/template.Dockerfile @@ -0,0 +1,69 @@ +# syntax=docker/dockerfile:1 + +ARG PYTHON_VERSION=3.12 +FROM python:${PYTHON_VERSION}-slim as base + +# Read aws creds +ENV AWS_ACCESS_KEY +ENV AWS_SECRET_KEY +ENV AWS_BUCKET + +# Prevents Python from writing pyc files. +ENV PYTHONDONTWRITEBYTECODE=1 +# Keeps Python from buffering stdout and stderr to avoid situations where +# the application crashes without emitting any logs due to buffering. +ENV PYTHONUNBUFFERED=1 + +WORKDIR /app + +# Create a non-privileged user that the app will run under. +ARG UID=10001 +RUN adduser \ + --disabled-password \ + --gecos "" \ + --home "/nonexistent" \ + --shell "/sbin/nologin" \ + --no-create-home \ + --uid "${UID}" \ + appuser + +# Install S3 dependencies +RUN apt-get update -y && \ + apt-get install -y \ + s3fs \ + libfuse-dev \ + libcurl4-openssl-dev \ + libxml2-dev \ + libssl-dev \ + mime-support \ + automake \ + libtool \ + wget \ + tar \ + git \ + unzip && + apt-get clean + +RUN pip --no-cache-dir install --upgrade awscli +RUN mkdir -p /mnt/s3 + + +# Download dependencies as a separate step to take advantage of Docker's caching. +# Leverage a cache mount to /root/.cache/pip to speed up subsequent builds. +# Leverage a bind mount to requirements.txt to avoid having to copy them into +# into this layer. +RUN --mount=type=cache,target=/root/.cache/pip \ + --mount=type=bind,source=algo/,target=algo/ \ + python -m pip install ./algo + +# Switch to the non-privileged user to run the application. +USER appuser + +VOLUME ["/output"] + +# Define input / output files +ENV INPUT_FILE="" +ENV OUTPUT_FILE="" +# Run the application +# NOTE: edit the second command +CMD s3fs ${AWS_BUCKET} /mnt/s3; python3 -m algo "/mnt/s3/input/${INPUT_FILE}" "/output/${OUTPUT_FILE}" From 577c34a7f274c41c5080de58b1d49ecd540637f0 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Thu, 24 Oct 2024 13:02:25 +0200 Subject: [PATCH 02/11] ci: docker publish evaluator --- .../workflows/docker-publish-evaluator.yml | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 .github/workflows/docker-publish-evaluator.yml diff --git a/.github/workflows/docker-publish-evaluator.yml b/.github/workflows/docker-publish-evaluator.yml new file mode 100644 index 0000000..b982299 --- /dev/null +++ b/.github/workflows/docker-publish-evaluator.yml @@ -0,0 +1,96 @@ +name: Docker + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + + +jobs: + build: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Install the cosign tool except on PR + # https://github.com/sigstore/cosign-installer + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0 + with: + cosign-release: 'v2.2.4' + + # Set up BuildKit Docker container builder to be able to build + # multi-platform images and export cache + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ evaluator }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: ./evaluation + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + file: config/evaluator.Dockerfile + + # Sign the resulting Docker image digest except on PRs. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + # https://github.com/sigstore/cosign + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} + From f244ee35c6d1b98717232b5f07afd231d3433975 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Thu, 24 Oct 2024 14:16:17 +0200 Subject: [PATCH 03/11] ci: docker publish site-builder --- .../workflows/docker-publish-site-builder.yml | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 .github/workflows/docker-publish-site-builder.yml diff --git a/.github/workflows/docker-publish-site-builder.yml b/.github/workflows/docker-publish-site-builder.yml new file mode 100644 index 0000000..9d818b4 --- /dev/null +++ b/.github/workflows/docker-publish-site-builder.yml @@ -0,0 +1,96 @@ +name: Docker + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + + +jobs: + build: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Install the cosign tool except on PR + # https://github.com/sigstore/cosign-installer + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0 + with: + cosign-release: 'v2.2.4' + + # Set up BuildKit Docker container builder to be able to build + # multi-platform images and export cache + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ site-builder }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: ./website + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + file: config/site-builder.Dockerfile + + # Sign the resulting Docker image digest except on PRs. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + # https://github.com/sigstore/cosign + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} + From 2b225a065f747326edc96bbe5fb9ba4a5c7b6059 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Thu, 24 Oct 2024 14:26:34 +0200 Subject: [PATCH 04/11] ci: fix img paths --- .github/workflows/docker-publish-evaluator.yml | 4 ++-- .github/workflows/docker-publish-site-builder.yml | 4 ++-- .github/workflows/pr-check.yml | 9 +++++++++ 3 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/pr-check.yml diff --git a/.github/workflows/docker-publish-evaluator.yml b/.github/workflows/docker-publish-evaluator.yml index b982299..31dbb71 100644 --- a/.github/workflows/docker-publish-evaluator.yml +++ b/.github/workflows/docker-publish-evaluator.yml @@ -15,7 +15,7 @@ env: # Use docker.io for Docker Hub if empty REGISTRY: ghcr.io # github.repository as / - IMAGE_NAME: ${{ github.repository }} + IMAGE_NAME: ${{ github.repository }}-evaluator jobs: @@ -63,7 +63,7 @@ jobs: id: meta uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 with: - images: ${{ env.REGISTRY }}/${{ evaluator }} + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} # Build and push Docker image with Buildx (don't push on PR) # https://github.com/docker/build-push-action diff --git a/.github/workflows/docker-publish-site-builder.yml b/.github/workflows/docker-publish-site-builder.yml index 9d818b4..2186c75 100644 --- a/.github/workflows/docker-publish-site-builder.yml +++ b/.github/workflows/docker-publish-site-builder.yml @@ -15,7 +15,7 @@ env: # Use docker.io for Docker Hub if empty REGISTRY: ghcr.io # github.repository as / - IMAGE_NAME: ${{ github.repository }} + IMAGE_NAME: ${{ github.repository }}-site-builder jobs: @@ -63,7 +63,7 @@ jobs: id: meta uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 with: - images: ${{ env.REGISTRY }}/${{ site-builder }} + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} # Build and push Docker image with Buildx (don't push on PR) # https://github.com/docker/build-push-action diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml new file mode 100644 index 0000000..c417f06 --- /dev/null +++ b/.github/workflows/pr-check.yml @@ -0,0 +1,9 @@ +name: PR checks + +on: + pull_request: + paths: + - 'algorithms/*.yaml' + +env: + From 1a4b6330b56c1c381aa60830a8acbde3d5dd2d64 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Thu, 24 Oct 2024 15:25:43 +0200 Subject: [PATCH 05/11] ci: add pr check Dockerfile --- config/algo-checks.Dockerfile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 config/algo-checks.Dockerfile diff --git a/config/algo-checks.Dockerfile b/config/algo-checks.Dockerfile new file mode 100644 index 0000000..db57479 --- /dev/null +++ b/config/algo-checks.Dockerfile @@ -0,0 +1,14 @@ +# Provides dependencies to verify algorithm definition files +# and validate them against schema + +FROM alpine:3.20 + +# install dependencies +RUN apk add --no-cache \ + curl \ + cargo \ + yq + +ENV PATH="${PATH}:/root/.cargo/bin" + +RUN cargo install jsonschema-cli From 93f75710d81e5f436cfb6c0f9592b354a1e1dc44 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Thu, 24 Oct 2024 15:42:59 +0200 Subject: [PATCH 06/11] ci: better workflow titles --- .github/workflows/docker-publish-evaluator.yml | 2 +- .github/workflows/docker-publish-site-builder.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish-evaluator.yml b/.github/workflows/docker-publish-evaluator.yml index 31dbb71..e1a5d92 100644 --- a/.github/workflows/docker-publish-evaluator.yml +++ b/.github/workflows/docker-publish-evaluator.yml @@ -1,4 +1,4 @@ -name: Docker +name: Publish evaluator container # This workflow uses actions that are not certified by GitHub. # They are provided by a third-party and are governed by diff --git a/.github/workflows/docker-publish-site-builder.yml b/.github/workflows/docker-publish-site-builder.yml index 2186c75..90acff4 100644 --- a/.github/workflows/docker-publish-site-builder.yml +++ b/.github/workflows/docker-publish-site-builder.yml @@ -1,4 +1,4 @@ -name: Docker +name: Publish website builder container # This workflow uses actions that are not certified by GitHub. # They are provided by a third-party and are governed by From 959479e812ea262eb58ca8663024e42ac5ac0d5b Mon Sep 17 00:00:00 2001 From: cmdoret Date: Thu, 24 Oct 2024 15:43:12 +0200 Subject: [PATCH 07/11] ci: pr check workflow --- .github/workflows/pr-check.yml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index c417f06..5bddf0e 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -1,4 +1,4 @@ -name: PR checks +name: PR checks for algorithm submission on: pull_request: @@ -6,4 +6,26 @@ on: - 'algorithms/*.yaml' env: + IMAGE_NAME: ${{ github.repository }}-evaluator + +jobs: + + check_yaml: + container: + image: ghcr.io/${{ env.IMAGE_NAME }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.github_token }} + steps: + + - name: Convert yaml + run: yq -o json FILE.yaml > FILE.json + + - name: Validate jsonschema + run: jsonschema-cli -i FILE.json SCHEMA.json + + check_image: + steps: + - name: Pull image + run: docker pull $(sed 's/^image: \(.*\)$' FILE.yaml) From 5edf6e83449f2d3d8eae74b07fbd8fb727e51f50 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Thu, 24 Oct 2024 15:51:41 +0200 Subject: [PATCH 08/11] ci: website builder dockerfile --- website/Dockerfile | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 website/Dockerfile diff --git a/website/Dockerfile b/website/Dockerfile new file mode 100644 index 0000000..55f4c6e --- /dev/null +++ b/website/Dockerfile @@ -0,0 +1,17 @@ +# Use an official Python runtime as a parent image +FROM python:3.12-slim + +# Set the working directory in the container +WORKDIR . + +# Copy the requirements.txt file into the container +COPY requirements.txt . + +# Install any needed packages specified in requirements.txt +RUN pip install --no-cache-dir -r requirements.txt + +# Copy the rest of the application code into the container +COPY . . + +# Command to run the evaluator.py script +CMD ["python", "script.py"] From 1e20259df1fa62d72066dee586cbbda7157f98d9 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Thu, 24 Oct 2024 15:52:35 +0200 Subject: [PATCH 09/11] ci: update dockerfile paths --- .github/workflows/docker-publish-evaluator.yml | 2 +- .github/workflows/docker-publish-site-builder.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish-evaluator.yml b/.github/workflows/docker-publish-evaluator.yml index e1a5d92..38f1c6d 100644 --- a/.github/workflows/docker-publish-evaluator.yml +++ b/.github/workflows/docker-publish-evaluator.yml @@ -77,7 +77,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max - file: config/evaluator.Dockerfile + file: evaluation/Dockerfile # Sign the resulting Docker image digest except on PRs. # This will only write to the public Rekor transparency log when the Docker diff --git a/.github/workflows/docker-publish-site-builder.yml b/.github/workflows/docker-publish-site-builder.yml index 90acff4..12b525c 100644 --- a/.github/workflows/docker-publish-site-builder.yml +++ b/.github/workflows/docker-publish-site-builder.yml @@ -77,7 +77,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max - file: config/site-builder.Dockerfile + file: website/Dockerfile # Sign the resulting Docker image digest except on PRs. # This will only write to the public Rekor transparency log when the Docker From 8cf8fe21f5d7af9ff58f80a6b364e71cffdad018 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Thu, 24 Oct 2024 15:52:44 +0200 Subject: [PATCH 10/11] ci: site-builder dockerfile --- evaluation/Dockerfile | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 evaluation/Dockerfile diff --git a/evaluation/Dockerfile b/evaluation/Dockerfile new file mode 100644 index 0000000..a040586 --- /dev/null +++ b/evaluation/Dockerfile @@ -0,0 +1,17 @@ +# Use an official Python runtime as a parent image +FROM python:3.12-slim + +# Set the working directory in the container +WORKDIR . + +# Copy the requirements.txt file into the container +COPY requirements.txt . + +# Install any needed packages specified in requirements.txt +RUN pip install --no-cache-dir -r requirements.txt + +# Copy the rest of the application code into the container +COPY . . + +# Command to run the evaluator.py script +CMD ["python", "__main__.py"] From 843ea6660e6b81948da886ea0516fa5bc9900867 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Thu, 24 Oct 2024 15:56:47 +0200 Subject: [PATCH 11/11] ci: placeholder requirements files --- evaluation/requirements.txt | 1 + website/requirements.txt | 1 + 2 files changed, 2 insertions(+) create mode 100644 evaluation/requirements.txt create mode 100644 website/requirements.txt diff --git a/evaluation/requirements.txt b/evaluation/requirements.txt new file mode 100644 index 0000000..24ce15a --- /dev/null +++ b/evaluation/requirements.txt @@ -0,0 +1 @@ +numpy diff --git a/website/requirements.txt b/website/requirements.txt new file mode 100644 index 0000000..6966869 --- /dev/null +++ b/website/requirements.txt @@ -0,0 +1 @@ +sphinx