diff --git a/.github/workflows/docs-image.yml b/.github/workflows/docs-image.yml new file mode 100644 index 000000000..e7da1bae7 --- /dev/null +++ b/.github/workflows/docs-image.yml @@ -0,0 +1,93 @@ +name: docs-image +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + inputs: + build_arm: + type: boolean + description: "Build for ARM as well" + default: false + required: false + workflow_call: + inputs: + build_arm: + type: boolean + description: "Build for ARM as well" + default: false + required: false +env: + DOCKER_IMAGE_NAME: ghcr.io/loculus-project/docs + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + BUILD_ARM: ${{ github.event.inputs.build_arm || inputs.build_arm || github.ref == 'refs/heads/main' }} + sha: ${{ github.event.pull_request.head.sha || github.sha }} +concurrency: + group: ci-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}-docs-${{github.event.inputs.build_arm}} + cancel-in-progress: true +jobs: + docs-docker: + name: Build Docs Docker Image + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + packages: write + checks: read + steps: + - name: Shorten sha + run: echo "sha=${sha::7}" >> $GITHUB_ENV + - uses: actions/checkout@v4 + - name: Generate files hash + id: files-hash + run: | + DIR_HASH=$(echo -n ${{ hashFiles('docs/**', '.github/workflows/docs-image.yml') }}) + echo "DIR_HASH=$DIR_HASH${{ env.BUILD_ARM == 'true' && '-arm' || '' }}" >> $GITHUB_ENV + - name: Setup Docker metadata + id: dockerMetadata + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_IMAGE_NAME }} + tags: | + type=raw,value=${{ env.DIR_HASH }} + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + type=raw,value=${{ env.BRANCH_NAME }} + type=raw,value=commit-${{ env.sha }} + type=raw,value=${{ env.BRANCH_NAME }}-arm,enable=${{ env.BUILD_ARM }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Check if image exists + id: check-image + run: | + EXISTS=$(docker manifest inspect ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false") + echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Get node version build arg + if: env.CACHE_HIT == 'false' + run: | + NODE_VERSION=$(cat docs/.nvmrc | tr -cd [:digit:].) + echo "NODE_VERSION=$NODE_VERSION" >> $GITHUB_ENV + - name: Build and push image + if: env.CACHE_HIT == 'false' + uses: docker/build-push-action@v6 + with: + context: ./docs + push: true + tags: ${{ steps.dockerMetadata.outputs.tags }} + cache-from: type=gha,scope=docs-${{ github.ref }} + cache-to: type=gha,mode=max,scope=docs-${{ github.ref }} + platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} + build-args: NODE_VERSION=${{ env.NODE_VERSION }} + - name: Retag and push existing image if cache hit + if: env.CACHE_HIT == 'true' + run: | + TAGS=(${{ steps.dockerMetadata.outputs.tags }}) + for TAG in "${TAGS[@]}"; do + docker buildx imagetools create --tag $TAG ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} + done diff --git a/docs/Dockerfile b/docs/Dockerfile new file mode 100644 index 000000000..b93c86c80 --- /dev/null +++ b/docs/Dockerfile @@ -0,0 +1,17 @@ +FROM node:20-alpine +WORKDIR /app + +COPY package*.json ./ + +RUN npm ci + +COPY . . + +RUN npm run build +RUN mkdir -p server && mv dist server/loculus + +RUN npm install -g http-server + +EXPOSE 8080 + +CMD ["http-server", "server", "-p", "8080"] diff --git a/docs/src/content/docs/introduction/architectureOverview.svg b/docs/src/content/docs/introduction/architectureOverview.svg new file mode 100644 index 000000000..06e8a3c66 --- /dev/null +++ b/docs/src/content/docs/introduction/architectureOverview.svg @@ -0,0 +1 @@ +SILO-LAPISSILOLAPISLoculus WebsiteLoculus BackendPostgres DBPreprocessing PipelineIngest PipelineKeycloakauthenticateauthenticateupload sequence datapersist sequence entriesfetch new sequence entriessubmit enriched sequence entriesnew Genbank sequencesprovide sequence entriessequence query resultssequence query results \ No newline at end of file diff --git a/docs/src/content/docs/introduction/system-overview.md b/docs/src/content/docs/introduction/system-overview.md index ffba5ec75..9e03c90eb 100644 --- a/docs/src/content/docs/introduction/system-overview.md +++ b/docs/src/content/docs/introduction/system-overview.md @@ -16,7 +16,7 @@ Loculus has a modular architecture and consists of several sub-services: - **Website:** The frontend application of Loculus accesses the APIs of the backend server and LAPIS. It uses the backend server for everything related to data submission and LAPIS for searching and downloading released data. For logins and registrations, users are redirected to Keycloak. - **Preprocessing pipeline(s):** A preprocessing pipeline fetches [unprocessed/user-submitted data](../glossary#unprocessed-data) from the backend server, processes them (which usually includes cleaning, alignment and adding annotations), and sends [processed data](../glossary#processed-data) back to the backend server. The pipeline contains [organism](../glossary#organism)-specific logic, thus, there is a separate pipeline for each organism. We maintain a customizeable preprocessing pipeline that uses [Nextclade](https://github.com/nextstrain/nextclade) for alignment, quality checks and annotations but it is easy to write a new one by following the [preprocessing pipeline specifications](https://github.com/loculus-project/loculus/blob/main/preprocessing/specification.md). -![Architecture overview](../../../../../backend/docs/plantuml/architectureOverview.svg) +![Architecture overview](./architectureOverview.svg) ## Motivation diff --git a/kubernetes/loculus/templates/docs-preview.yaml b/kubernetes/loculus/templates/docs-preview.yaml new file mode 100644 index 000000000..fdcf14264 --- /dev/null +++ b/kubernetes/loculus/templates/docs-preview.yaml @@ -0,0 +1,72 @@ +{{- $dockerTag := include "loculus.dockerTag" .Values }} +{{- $docsHost := printf "docs%s%s" .Values.subdomainSeparator .Values.host }} +{{- if .Values.previewDocs }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: loculus-docs + annotations: + argocd.argoproj.io/sync-options: Replace=true + reloader.stakater.com/auto: "true" +spec: + replicas: 1 + selector: + matchLabels: + app: loculus + component: docs + template: + metadata: + labels: + app: loculus + component: docs + spec: + containers: + - name: docs + image: "ghcr.io/loculus-project/docs:{{ $dockerTag }}" + imagePullPolicy: Always + resources: + requests: + memory: "200Mi" + {{ include "loculus.standardCPURequest" $ }} + limits: + memory: "1Gi" + ports: + - containerPort: 8080 + +--- +apiVersion: v1 +kind: Service +metadata: + name: loculus-docs +spec: + selector: + app: loculus + component: docs + ports: + - protocol: TCP + port: 80 + targetPort: 8080 + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: loculus-docs-ingress +spec: + rules: + - host: "{{ $docsHost }}" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: loculus-docs + port: + number: 80 + tls: + - hosts: + - "{{ $docsHost }}" +--- +{{- end }} \ No newline at end of file diff --git a/kubernetes/loculus/values_e2e_and_dev.yaml b/kubernetes/loculus/values_e2e_and_dev.yaml index a68bf2077..336310385 100644 --- a/kubernetes/loculus/values_e2e_and_dev.yaml +++ b/kubernetes/loculus/values_e2e_and_dev.yaml @@ -10,4 +10,4 @@ secrets: createTestAccounts: true backendExtraArgs: - "--loculus.debug-mode=true" -reduceResourceRequest: true +reduceResourceRequest: true \ No newline at end of file diff --git a/kubernetes/loculus/values_preview_server.yaml b/kubernetes/loculus/values_preview_server.yaml index 199ba7e27..11872a7f9 100644 --- a/kubernetes/loculus/values_preview_server.yaml +++ b/kubernetes/loculus/values_preview_server.yaml @@ -34,3 +34,4 @@ secrets: github-username: AgBtyXeimJmI7e1di9ubkzNODtAAlm0LiXEJJGfzzR/KExSrVMYsWcIIQOCfS/bbLvvvSyddKPGeFYhozG6CK943oTJ2rZHeMBjqtXX0AEw655KfJNgHWoWWk1xdmOzITtzKNlHi4cPcxswQte84NhJqRZPJ1sNhQfOm/AwL0NplQXD6xyl7UGAUKOW8rSLgH+gTUsTzmz2prgwiF3SmwwHCSClL/q9H4/nqkMycMQ5NcQK+5cpUUeWidzL3LjANCJS815S8oMdUWbDFwB3NPPfUhYdRuXM19MAHxq2hhAkn6rKGNI8tQhahi8dfmH2QDMyS1KVRof6taBXJIVMxeejL8nOJbNhPBdtyM+3hVasm/frLycUJxzvQpPPv1ictprM5K/r2sJDIqyXVOZSspyXQ09gFz2D13QsOSfaERbKUMtn9L0bfsG3N/zwo614N8YNCXf+dvIjVCoFwMD5RQ5IkqHLPEQiJ77++feSO+4fbfItjMI8qP4mM6YRI7VqweVKufwdSSropRpxhytnjqoWfvHFyDgzsw4ZuJwlyZs01yuJelog4yqG8WszWCv6Ae3fRM3gaquCj2UlFfxTBxN+aZId/lV3LCG1l06g82J6+YTL5wdCnYuYcpZmp21651J9jWwWzCumRWaQgbY6Ou0VjHKZ9if1uDdYA3S7SJJx/ZDZK4cixoS9VXgsPWFgmHbZNHqOFzOd2Dl9kRA== github-pat: AgBa5wjYBw3GWXihWdoedx3oEbR5z808WoeIGyk5aAZPQ1MTMbWm4ZMdvJUZVPlkRacQXSt+A2pniYMjzl0bqHkc5guVQOoHu6jZgtyfaOyqT4Ergz1+L9VUzyn6E1NwqV2BtdKQTN7A1VvtLw0BLVG0E01SUcHeoU9MWlF+9BWFtXWWYxWL/sTLKoz0tnj8yu6LLHMMYeCnABAoZwZbJ3kf1UVp3FwcJbuSqsrUTPaeS6z5g90jL/zwJ150VUh9kInHs+kwV1imZp06ZUm3gpdY1f/EQZnxHTPUoaYLOo91cEhG7bSD9KQz1B6m07VzjIlnrwduPUCFQs1gk1hU7gO+6MYlMqCIb8NUQbuX5d5cHiENJm9DXAJdd8oNW02JQ371gulVYvXagAdrYgYv75Mr3YjejHZeinZrJ/ZxeZI0+fL4SLkh+77RyuN1R6ab1vGBoUG+LgHsUEBT7mw/wXtrzRmNBwxNBXSfTrrKM+EzM9KUsrSe490sK3Rf4TExjynL/9biS9ZFw601tdBh8luO3xFhCr8Bn4q7sWnKNmvSu+gMZklyhR65Hf0LHTxUdDcK8WZKFyds6foHTAv9kX4+NK7upVfAdjx9Tpf0ONad3nSboi/+1vYoGtAZcL6BZoNL+1Xu2WAlVa+WollJSV2k4KeTEswE1vmPAhoapyxpry9Mz+8CGkohvSPjHGdfXSNSW7K9zKCCubqzHV2+EsnaDWJ9+cwghAQFOpKdbJuHLwalMpAkTztRjr5wHfD6KSo1apCHZl84S0B2u8floEzU1XRiW8zq10m8cb3/xmdteNL3FNMlZlEieD2nsiU= reduceResourceRequest: true +previewDocs: true \ No newline at end of file