From ca20f63b60a452d76be1ee7095e60c59cc2b6772 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Mon, 19 Aug 2024 13:46:25 +0100 Subject: [PATCH 1/9] preview docs --- .github/workflows/docs-image.yml | 46 ++++++++++++ docs/Dockerfile | 16 +++++ .../loculus/templates/docs-preview.yaml | 71 +++++++++++++++++++ kubernetes/loculus/values_e2e_and_dev.yaml | 1 + 4 files changed, 134 insertions(+) create mode 100644 .github/workflows/docs-image.yml create mode 100644 docs/Dockerfile create mode 100644 kubernetes/loculus/templates/docs-preview.yaml diff --git a/.github/workflows/docs-image.yml b/.github/workflows/docs-image.yml new file mode 100644 index 000000000..12d184dbe --- /dev/null +++ b/.github/workflows/docs-image.yml @@ -0,0 +1,46 @@ +name: Build and Push Docker Image for Docs + +on: + workflow_dispatch: + push: + branches: + - main + +env: + DOCKER_IMAGE: ghcr.io/${{ github.repository }}/docs + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and Push Docker Image + uses: docker/build-push-action@v5 + with: + context: ./docs + file: ./docs/Dockerfile + push: true + tags: | + ${{ env.DOCKER_IMAGE }}:${{ github.sha }} + ${{ env.DOCKER_IMAGE }}:latest + + - name: Echo Image URL + run: | + echo "Docker image pushed to ${{ env.DOCKER_IMAGE }}:${{ github.sha }}" + echo "Docker image pushed to ${{ env.DOCKER_IMAGE }}:latest" \ No newline at end of file diff --git a/docs/Dockerfile b/docs/Dockerfile new file mode 100644 index 000000000..75c3d32db --- /dev/null +++ b/docs/Dockerfile @@ -0,0 +1,16 @@ +FROM node:20-alpine +WORKDIR /app + +COPY package*.json ./ + +RUN npm ci + +COPY . . + +RUN npm run build + +RUN npm install -g http-server + +EXPOSE 8080 + +CMD ["http-server", "dist", "-p", "8080"] \ No newline at end of file diff --git a/kubernetes/loculus/templates/docs-preview.yaml b/kubernetes/loculus/templates/docs-preview.yaml new file mode 100644 index 000000000..7f9ebfcca --- /dev/null +++ b/kubernetes/loculus/templates/docs-preview.yaml @@ -0,0 +1,71 @@ +{{- $dockerTag := include "loculus.dockerTag" .Values }} +{{- $docsHost := printf "docs%s%s" .Values.subdomainSeparator .Values.host }} +{{- if .Values.previewDocs }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: loculus-docs + annotations: + argocd.argoproj.io/sync-options: Replace=true + reloader.stakater.com/auto: "true" +spec: + replicas: 1 + selector: + matchLabels: + app: loculus + component: docs + template: + metadata: + labels: + app: loculus + component: docs + spec: + containers: + - name: docs + image: "ghcr.io/loculus-project/docs:{{ $dockerTag }}" + imagePullPolicy: Always + resources: + requests: + memory: "200Mi" + {{ include "loculus.standardCPURequest" $ }} + limits: + memory: "1Gi" + ports: + - containerPort: 8080 + +--- +apiVersion: v1 +kind: Service +metadata: + name: loculus-docs +spec: + selector: + app: loculus + component: docs + ports: + - protocol: TCP + port: 80 + targetPort: 8080 + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: loculus-docs + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/ssl-redirect: "false" +spec: + rules: + - host: {{ $docsHost | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: loculus-docs + port: + number: 80 +{{- end }} \ No newline at end of file diff --git a/kubernetes/loculus/values_e2e_and_dev.yaml b/kubernetes/loculus/values_e2e_and_dev.yaml index a68bf2077..70bca8c6e 100644 --- a/kubernetes/loculus/values_e2e_and_dev.yaml +++ b/kubernetes/loculus/values_e2e_and_dev.yaml @@ -11,3 +11,4 @@ createTestAccounts: true backendExtraArgs: - "--loculus.debug-mode=true" reduceResourceRequest: true +previewDocs: true \ No newline at end of file From f1346e862b5a8a9ffd25e4d779c0aa96d09cd169 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Mon, 19 Aug 2024 14:02:51 +0100 Subject: [PATCH 2/9] put in the right values file --- kubernetes/loculus/values_e2e_and_dev.yaml | 3 +-- kubernetes/loculus/values_preview_server.yaml | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/loculus/values_e2e_and_dev.yaml b/kubernetes/loculus/values_e2e_and_dev.yaml index 70bca8c6e..336310385 100644 --- a/kubernetes/loculus/values_e2e_and_dev.yaml +++ b/kubernetes/loculus/values_e2e_and_dev.yaml @@ -10,5 +10,4 @@ secrets: createTestAccounts: true backendExtraArgs: - "--loculus.debug-mode=true" -reduceResourceRequest: true -previewDocs: true \ No newline at end of file +reduceResourceRequest: true \ No newline at end of file diff --git a/kubernetes/loculus/values_preview_server.yaml b/kubernetes/loculus/values_preview_server.yaml index 199ba7e27..11872a7f9 100644 --- a/kubernetes/loculus/values_preview_server.yaml +++ b/kubernetes/loculus/values_preview_server.yaml @@ -34,3 +34,4 @@ secrets: github-username: AgBtyXeimJmI7e1di9ubkzNODtAAlm0LiXEJJGfzzR/KExSrVMYsWcIIQOCfS/bbLvvvSyddKPGeFYhozG6CK943oTJ2rZHeMBjqtXX0AEw655KfJNgHWoWWk1xdmOzITtzKNlHi4cPcxswQte84NhJqRZPJ1sNhQfOm/AwL0NplQXD6xyl7UGAUKOW8rSLgH+gTUsTzmz2prgwiF3SmwwHCSClL/q9H4/nqkMycMQ5NcQK+5cpUUeWidzL3LjANCJS815S8oMdUWbDFwB3NPPfUhYdRuXM19MAHxq2hhAkn6rKGNI8tQhahi8dfmH2QDMyS1KVRof6taBXJIVMxeejL8nOJbNhPBdtyM+3hVasm/frLycUJxzvQpPPv1ictprM5K/r2sJDIqyXVOZSspyXQ09gFz2D13QsOSfaERbKUMtn9L0bfsG3N/zwo614N8YNCXf+dvIjVCoFwMD5RQ5IkqHLPEQiJ77++feSO+4fbfItjMI8qP4mM6YRI7VqweVKufwdSSropRpxhytnjqoWfvHFyDgzsw4ZuJwlyZs01yuJelog4yqG8WszWCv6Ae3fRM3gaquCj2UlFfxTBxN+aZId/lV3LCG1l06g82J6+YTL5wdCnYuYcpZmp21651J9jWwWzCumRWaQgbY6Ou0VjHKZ9if1uDdYA3S7SJJx/ZDZK4cixoS9VXgsPWFgmHbZNHqOFzOd2Dl9kRA== github-pat: AgBa5wjYBw3GWXihWdoedx3oEbR5z808WoeIGyk5aAZPQ1MTMbWm4ZMdvJUZVPlkRacQXSt+A2pniYMjzl0bqHkc5guVQOoHu6jZgtyfaOyqT4Ergz1+L9VUzyn6E1NwqV2BtdKQTN7A1VvtLw0BLVG0E01SUcHeoU9MWlF+9BWFtXWWYxWL/sTLKoz0tnj8yu6LLHMMYeCnABAoZwZbJ3kf1UVp3FwcJbuSqsrUTPaeS6z5g90jL/zwJ150VUh9kInHs+kwV1imZp06ZUm3gpdY1f/EQZnxHTPUoaYLOo91cEhG7bSD9KQz1B6m07VzjIlnrwduPUCFQs1gk1hU7gO+6MYlMqCIb8NUQbuX5d5cHiENJm9DXAJdd8oNW02JQ371gulVYvXagAdrYgYv75Mr3YjejHZeinZrJ/ZxeZI0+fL4SLkh+77RyuN1R6ab1vGBoUG+LgHsUEBT7mw/wXtrzRmNBwxNBXSfTrrKM+EzM9KUsrSe490sK3Rf4TExjynL/9biS9ZFw601tdBh8luO3xFhCr8Bn4q7sWnKNmvSu+gMZklyhR65Hf0LHTxUdDcK8WZKFyds6foHTAv9kX4+NK7upVfAdjx9Tpf0ONad3nSboi/+1vYoGtAZcL6BZoNL+1Xu2WAlVa+WollJSV2k4KeTEswE1vmPAhoapyxpry9Mz+8CGkohvSPjHGdfXSNSW7K9zKCCubqzHV2+EsnaDWJ9+cwghAQFOpKdbJuHLwalMpAkTztRjr5wHfD6KSo1apCHZl84S0B2u8floEzU1XRiW8zq10m8cb3/xmdteNL3FNMlZlEieD2nsiU= reduceResourceRequest: true +previewDocs: true \ No newline at end of file From 704d7b30c0ccb9d6316c0fedf31cdb55c2717b0e Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Mon, 19 Aug 2024 14:15:02 +0100 Subject: [PATCH 3/9] don't just run on main --- .github/workflows/docs-image.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/docs-image.yml b/.github/workflows/docs-image.yml index 12d184dbe..6361a1ece 100644 --- a/.github/workflows/docs-image.yml +++ b/.github/workflows/docs-image.yml @@ -2,9 +2,7 @@ name: Build and Push Docker Image for Docs on: workflow_dispatch: - push: - branches: - - main + pull_request: env: DOCKER_IMAGE: ghcr.io/${{ github.repository }}/docs From 0d4f7fe4888700e0ea64e8377c59cd5e075b3bcb Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Mon, 19 Aug 2024 14:15:47 +0100 Subject: [PATCH 4/9] fix image name --- .github/workflows/docs-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-image.yml b/.github/workflows/docs-image.yml index 6361a1ece..a58c271f6 100644 --- a/.github/workflows/docs-image.yml +++ b/.github/workflows/docs-image.yml @@ -5,7 +5,7 @@ on: pull_request: env: - DOCKER_IMAGE: ghcr.io/${{ github.repository }}/docs + DOCKER_IMAGE: ghcr.io/loculus-project/docs jobs: build-and-push: From bd7592b5ca1938e1166ff047e1bacfeb5e610360 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Mon, 19 Aug 2024 14:22:35 +0100 Subject: [PATCH 5/9] cannot import SVG from parent dir --- docs/src/content/docs/introduction/architectureOverview.svg | 1 + docs/src/content/docs/introduction/system-overview.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 docs/src/content/docs/introduction/architectureOverview.svg diff --git a/docs/src/content/docs/introduction/architectureOverview.svg b/docs/src/content/docs/introduction/architectureOverview.svg new file mode 100644 index 000000000..06e8a3c66 --- /dev/null +++ b/docs/src/content/docs/introduction/architectureOverview.svg @@ -0,0 +1 @@ +SILO-LAPISSILOLAPISLoculus WebsiteLoculus BackendPostgres DBPreprocessing PipelineIngest PipelineKeycloakauthenticateauthenticateupload sequence datapersist sequence entriesfetch new sequence entriessubmit enriched sequence entriesnew Genbank sequencesprovide sequence entriessequence query resultssequence query results \ No newline at end of file diff --git a/docs/src/content/docs/introduction/system-overview.md b/docs/src/content/docs/introduction/system-overview.md index ffba5ec75..9e03c90eb 100644 --- a/docs/src/content/docs/introduction/system-overview.md +++ b/docs/src/content/docs/introduction/system-overview.md @@ -16,7 +16,7 @@ Loculus has a modular architecture and consists of several sub-services: - **Website:** The frontend application of Loculus accesses the APIs of the backend server and LAPIS. It uses the backend server for everything related to data submission and LAPIS for searching and downloading released data. For logins and registrations, users are redirected to Keycloak. - **Preprocessing pipeline(s):** A preprocessing pipeline fetches [unprocessed/user-submitted data](../glossary#unprocessed-data) from the backend server, processes them (which usually includes cleaning, alignment and adding annotations), and sends [processed data](../glossary#processed-data) back to the backend server. The pipeline contains [organism](../glossary#organism)-specific logic, thus, there is a separate pipeline for each organism. We maintain a customizeable preprocessing pipeline that uses [Nextclade](https://github.com/nextstrain/nextclade) for alignment, quality checks and annotations but it is easy to write a new one by following the [preprocessing pipeline specifications](https://github.com/loculus-project/loculus/blob/main/preprocessing/specification.md). -![Architecture overview](../../../../../backend/docs/plantuml/architectureOverview.svg) +![Architecture overview](./architectureOverview.svg) ## Motivation From eda2d20ce82b1edd4c1220e3b5aee7f848269475 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Mon, 19 Aug 2024 14:30:42 +0100 Subject: [PATCH 6/9] update CI --- .github/workflows/docs-image.yml | 101 +++++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 26 deletions(-) diff --git a/.github/workflows/docs-image.yml b/.github/workflows/docs-image.yml index a58c271f6..e7da1bae7 100644 --- a/.github/workflows/docs-image.yml +++ b/.github/workflows/docs-image.yml @@ -1,44 +1,93 @@ -name: Build and Push Docker Image for Docs - +name: docs-image on: - workflow_dispatch: pull_request: - + push: + branches: + - main + workflow_dispatch: + inputs: + build_arm: + type: boolean + description: "Build for ARM as well" + default: false + required: false + workflow_call: + inputs: + build_arm: + type: boolean + description: "Build for ARM as well" + default: false + required: false env: - DOCKER_IMAGE: ghcr.io/loculus-project/docs - + DOCKER_IMAGE_NAME: ghcr.io/loculus-project/docs + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + BUILD_ARM: ${{ github.event.inputs.build_arm || inputs.build_arm || github.ref == 'refs/heads/main' }} + sha: ${{ github.event.pull_request.head.sha || github.sha }} +concurrency: + group: ci-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}-docs-${{github.event.inputs.build_arm}} + cancel-in-progress: true jobs: - build-and-push: + docs-docker: + name: Build Docs Docker Image runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: read packages: write - + checks: read steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to GitHub Container Registry + - name: Shorten sha + run: echo "sha=${sha::7}" >> $GITHUB_ENV + - uses: actions/checkout@v4 + - name: Generate files hash + id: files-hash + run: | + DIR_HASH=$(echo -n ${{ hashFiles('docs/**', '.github/workflows/docs-image.yml') }}) + echo "DIR_HASH=$DIR_HASH${{ env.BUILD_ARM == 'true' && '-arm' || '' }}" >> $GITHUB_ENV + - name: Setup Docker metadata + id: dockerMetadata + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_IMAGE_NAME }} + tags: | + type=raw,value=${{ env.DIR_HASH }} + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + type=raw,value=${{ env.BRANCH_NAME }} + type=raw,value=commit-${{ env.sha }} + type=raw,value=${{ env.BRANCH_NAME }}-arm,enable=${{ env.BUILD_ARM }} + - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and Push Docker Image - uses: docker/build-push-action@v5 + - name: Check if image exists + id: check-image + run: | + EXISTS=$(docker manifest inspect ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false") + echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Get node version build arg + if: env.CACHE_HIT == 'false' + run: | + NODE_VERSION=$(cat docs/.nvmrc | tr -cd [:digit:].) + echo "NODE_VERSION=$NODE_VERSION" >> $GITHUB_ENV + - name: Build and push image + if: env.CACHE_HIT == 'false' + uses: docker/build-push-action@v6 with: context: ./docs - file: ./docs/Dockerfile push: true - tags: | - ${{ env.DOCKER_IMAGE }}:${{ github.sha }} - ${{ env.DOCKER_IMAGE }}:latest - - - name: Echo Image URL + tags: ${{ steps.dockerMetadata.outputs.tags }} + cache-from: type=gha,scope=docs-${{ github.ref }} + cache-to: type=gha,mode=max,scope=docs-${{ github.ref }} + platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} + build-args: NODE_VERSION=${{ env.NODE_VERSION }} + - name: Retag and push existing image if cache hit + if: env.CACHE_HIT == 'true' run: | - echo "Docker image pushed to ${{ env.DOCKER_IMAGE }}:${{ github.sha }}" - echo "Docker image pushed to ${{ env.DOCKER_IMAGE }}:latest" \ No newline at end of file + TAGS=(${{ steps.dockerMetadata.outputs.tags }}) + for TAG in "${TAGS[@]}"; do + docker buildx imagetools create --tag $TAG ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} + done From 7085fa38e018c38ffb7ce57c0138bac07d46c252 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Mon, 19 Aug 2024 14:52:35 +0100 Subject: [PATCH 7/9] update --- kubernetes/loculus/templates/docs-preview.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/kubernetes/loculus/templates/docs-preview.yaml b/kubernetes/loculus/templates/docs-preview.yaml index 7f9ebfcca..fdcf14264 100644 --- a/kubernetes/loculus/templates/docs-preview.yaml +++ b/kubernetes/loculus/templates/docs-preview.yaml @@ -52,13 +52,10 @@ spec: apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: loculus-docs - annotations: - kubernetes.io/ingress.class: nginx - nginx.ingress.kubernetes.io/ssl-redirect: "false" + name: loculus-docs-ingress spec: rules: - - host: {{ $docsHost | quote }} + - host: "{{ $docsHost }}" http: paths: - path: / @@ -66,6 +63,10 @@ spec: backend: service: name: loculus-docs - port: + port: number: 80 + tls: + - hosts: + - "{{ $docsHost }}" +--- {{- end }} \ No newline at end of file From 5dbafff7447a990cd527571dd39853f80677f310 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Mon, 19 Aug 2024 15:13:53 +0100 Subject: [PATCH 8/9] Update Dockerfile --- docs/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/Dockerfile b/docs/Dockerfile index 75c3d32db..5369bbffd 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -8,9 +8,10 @@ RUN npm ci COPY . . RUN npm run build +RUN mkdir -p dist/loculus && mv dist/* dist/loculus/ RUN npm install -g http-server EXPOSE 8080 -CMD ["http-server", "dist", "-p", "8080"] \ No newline at end of file +CMD ["http-server", "dist", "-p", "8080"] From 3394ed8b8fb9a66d0b331ce43efc4d8795559ff0 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Mon, 19 Aug 2024 15:18:22 +0100 Subject: [PATCH 9/9] update --- docs/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/Dockerfile b/docs/Dockerfile index 5369bbffd..b93c86c80 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -8,10 +8,10 @@ RUN npm ci COPY . . RUN npm run build -RUN mkdir -p dist/loculus && mv dist/* dist/loculus/ +RUN mkdir -p server && mv dist server/loculus RUN npm install -g http-server EXPOSE 8080 -CMD ["http-server", "dist", "-p", "8080"] +CMD ["http-server", "server", "-p", "8080"]