From 06fd713dc88dc1993c30d422747d7234289e120c Mon Sep 17 00:00:00 2001 From: "Felix T.J. Dietrich" Date: Fri, 6 Dec 2024 15:49:09 +0100 Subject: [PATCH 1/2] Docker deployment via GitHub Actions (#115) * create prod compose * add workflows * add deploy workflow * change port * Format README, remove trailing backslash from example DEPLOYMENT_URL * modify workflow * fix image name * yolo * fix error * remove needed file * update * fix compose file * fix casing * remove cache * map to host * fix redis * remove arg * fix deployment url * fix sed * fix url * rename file * generalize docker compose deploy * use workflow_dispatch * revert readme --------- Co-authored-by: Faris Demirovic --- .github/workflows/build-and-push-shared.yml | 94 ++++++++++++++++ .github/workflows/build-and-push.yml | 16 +++ .github/workflows/deploy-dev.yml | 21 ++++ .../deploy-docker-compose-shared.yml | 106 ++++++++++++++++++ Dockerfile.redis | 8 +- docker-compose.prod.yml | 36 ++++++ packages/server/src/main/server.ts | 11 ++ packages/webapp/webpack/webpack.common.js | 2 +- packages/webapp/webpack/webpack.dev.js | 2 +- 9 files changed, 289 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/build-and-push-shared.yml create mode 100644 .github/workflows/build-and-push.yml create mode 100644 .github/workflows/deploy-dev.yml create mode 100644 .github/workflows/deploy-docker-compose-shared.yml create mode 100644 docker-compose.prod.yml diff --git a/.github/workflows/build-and-push-shared.yml b/.github/workflows/build-and-push-shared.yml new file mode 100644 index 00000000..7a7047ec --- /dev/null +++ b/.github/workflows/build-and-push-shared.yml @@ -0,0 +1,94 @@ +# Move to ls1intum/.github/.github/workflows/build-and-push-docker-image.yml@main in the future +name: Build and Push Docker Image + +on: + workflow_call: + inputs: + image-name: + type: string + default: ${{ github.repository }} + description: "The name for the docker image (Default: Repository name)" + docker-file: + type: string + default: Dockerfile + description: "The path to the Dockerfile (Default: ./Dockerfile)" + docker-context: + type: string + default: . + description: "The context for the Docker build (Default: .)" + build-args: + type: string + description: "List of additional build contexts (e.g., name=path)" + required: false + platforms: + type: string + description: "List of platforms for which to build the image" + default: linux/amd64,linux/arm64 + registry: + type: string + default: ghcr.io + description: "The registry to push the image to (Default: ghcr.io)" + + secrets: + registry-user: + required: false + registry-password: + required: false + + outputs: + image-tag: + description: "The tag of the pushed image" + value: ${{ jobs.build.outputs.image-tag }} +jobs: + build: + name: Build Docker Image for ${{ inputs.image-name }} + runs-on: ubuntu-latest + outputs: + image-tag: ${{ steps.set-tag.outputs.image-tag }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: all + + - name: Install Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ secrets.registry-user || github.actor }} + password: ${{ secrets.registry-password || secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ inputs.registry }}/${{ inputs.image-name }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=sha,prefix=,format=long + + - name: Set image tag output + id: set-tag + run: echo "::set-output name=image-tag::${{ steps.meta.outputs.version }}" + + - name: Build and push Docker Image + uses: docker/build-push-action@v6 + with: + context: ${{ inputs.docker-context }} + file: ${{ inputs.docker-file }} + platforms: ${{ inputs.platforms }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: ${{ inputs.build-args }} + push: true \ No newline at end of file diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml new file mode 100644 index 00000000..093fa920 --- /dev/null +++ b/.github/workflows/build-and-push.yml @@ -0,0 +1,16 @@ +name: Build Docker Image + +on: + pull_request: + push: + branches: [main] + +jobs: + build-and-push-workflow: + name: Build and Push Docker Image + # TODO: uses: ls1intum/.github/.github/workflows/build-and-push-docker-image.yml@main + uses: ./.github/workflows/build-and-push-shared.yml + with: + image-name: ls1intum/apollon_standalone + docker-file: Dockerfile.redis + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml new file mode 100644 index 00000000..59c1b95a --- /dev/null +++ b/.github/workflows/deploy-dev.yml @@ -0,0 +1,21 @@ +name: Deploy Dev + +on: + workflow_dispatch: + inputs: + image-tag: + type: string + description: "The tag of the docker images to deploy" + required: true +jobs: + deploy: + # TODO: uses: ls1intum/.github/.github/workflows/deploy-docker-compose.yml@main + uses: ./.github/workflows/deploy-docker-compose-shared.yml + with: + environment: Dev + docker-compose-file: "./docker-compose.prod.yml" + image-tag: ${{ inputs.image-tag }} + env-vars: | + DEPLOYMENT_URL=${{ vars.DEPLOYMENT_URL }} + APOLLON_REDIS_DIAGRAM_TTL=${{ vars.APOLLON_REDIS_DIAGRAM_TTL }} + secrets: inherit diff --git a/.github/workflows/deploy-docker-compose-shared.yml b/.github/workflows/deploy-docker-compose-shared.yml new file mode 100644 index 00000000..a5e2827c --- /dev/null +++ b/.github/workflows/deploy-docker-compose-shared.yml @@ -0,0 +1,106 @@ +# ls1intum/.github/workflows/deploy-docker-compose.yml +name: Deploy Docker Compose + +on: + workflow_call: + inputs: + environment: + type: string + description: "The deployment environment (e.g., production, staging)" + required: true + docker-compose-file: + type: string + default: "./docker-compose.yml" + description: "Path to the Docker Compose file (Default: ./docker-compose.yml)" + image-tag: + type: string + default: latest + description: "Tag of the Docker images to deploy (Default: latest)" + env-vars: + type: string + description: "Additional environment variables in KEY=VALUE format, separated by newlines" + required: false + +jobs: + deploy: + runs-on: ubuntu-latest + environment: + name: ${{ inputs.environment }} + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: SSH to VM and Execute Docker-Compose Down (if exists) + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ vars.VM_HOST }} + username: ${{ vars.VM_USERNAME }} + key: ${{ secrets.VM_SSH_PRIVATE_KEY }} + proxy_host: ${{ vars.DEPLOYMENT_GATEWAY_HOST }} + proxy_username: ${{ vars.DEPLOYMENT_GATEWAY_USER }} + proxy_key: ${{ secrets.DEPLOYMENT_GATEWAY_SSH_KEY }} + proxy_port: ${{ vars.DEPLOYMENT_GATEWAY_PORT }} + script: | + #!/bin/bash + set -e # Exit immediately if a command exits with a non-zero status + + COMPOSE_FILE="${{ inputs.docker-compose-file }}" + + # Check if docker-compose.prod.yml exists + if [ -f "$COMPOSE_FILE" ]; then + echo "$COMPOSE_FILE found." + + # Check if .env exists + if [ -f ".env" ]; then + docker compose -f "$COMPOSE_FILE" --env-file=".env" down --remove-orphans --rmi all + else + docker compose -f "$COMPOSE_FILE" down --remove-orphans --rmi all + fi + else + echo "$COMPOSE_FILE does not exist. Skipping docker compose down." + fi + + - name: Copy Docker Compose File to VM Host + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ vars.VM_HOST }} + username: ${{ vars.VM_USERNAME }} + key: ${{ secrets.VM_SSH_PRIVATE_KEY }} + proxy_host: ${{ vars.DEPLOYMENT_GATEWAY_HOST }} + proxy_username: ${{ vars.DEPLOYMENT_GATEWAY_USER }} + proxy_key: ${{ secrets.DEPLOYMENT_GATEWAY_SSH_KEY }} + proxy_port: ${{ vars.DEPLOYMENT_GATEWAY_PORT }} + source: ${{ inputs.docker-compose-file }} + target: /home/${{ vars.VM_USERNAME }} + + - name: SSH to VM and Create .env File + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ vars.VM_HOST }} + username: ${{ vars.VM_USERNAME }} + key: ${{ secrets.VM_SSH_PRIVATE_KEY }} + proxy_host: ${{ vars.DEPLOYMENT_GATEWAY_HOST }} + proxy_username: ${{ vars.DEPLOYMENT_GATEWAY_USER }} + proxy_key: ${{ secrets.DEPLOYMENT_GATEWAY_SSH_KEY }} + proxy_port: ${{ vars.DEPLOYMENT_GATEWAY_PORT }} + script: | + touch .env + + echo "ENVIRONMENT=${{ inputs.environment }}" > .env + echo "IMAGE_TAG=${{ inputs.image-tag }}" >> .env + if [ "${{ inputs.env-vars }}" != "" ]; then + echo "${{ inputs.env-vars }}" >> .env + fi + + - name: SSH to VM and Execute Docker Compose Up + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ vars.VM_HOST }} + username: ${{ vars.VM_USERNAME }} + key: ${{ secrets.VM_SSH_PRIVATE_KEY }} + proxy_host: ${{ vars.DEPLOYMENT_GATEWAY_HOST }} + proxy_username: ${{ vars.DEPLOYMENT_GATEWAY_USER }} + proxy_key: ${{ secrets.DEPLOYMENT_GATEWAY_SSH_KEY }} + proxy_port: ${{ vars.DEPLOYMENT_GATEWAY_PORT }} + script: | + docker compose -f ${{ inputs.docker-compose-file }} --env-file=.env up --pull=always -d \ No newline at end of file diff --git a/Dockerfile.redis b/Dockerfile.redis index 09d8f44c..bad7eb50 100644 --- a/Dockerfile.redis +++ b/Dockerfile.redis @@ -7,11 +7,9 @@ RUN apk add --no-cache \ pango-dev \ giflib-dev - -ARG DEPLOYMENT_URL="http://localhost:8080" - -ENV APOLLON_REDIS_URL="" -ENV DEPLOYMENT_URL=${DEPLOYMENT_URL} +ENV DEPLOYMENT_URL="http://localhost:8080" +ENV APOLLON_REDIS_URL="redis://localhost:6379" +ENV APOLLON_REDIS_DIAGRAM_TTL="30d" WORKDIR /app diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 00000000..54ac0cba --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,36 @@ +version: '3.8' + +services: + redis: + image: redis/redis-stack-server:7.4.0-v1 + container_name: apollon-redis + volumes: + - ./redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 5 + restart: unless-stopped + networks: + - apollon-network + + apollon-standalone: + image: "ghcr.io/ls1intum/apollon_standalone:${IMAGE_TAG}" + container_name: apollon-standalone + environment: + - APOLLON_REDIS_URL=redis://apollon-redis:6379 + - APOLLON_REDIS_DIAGRAM_TTL=${APOLLON_REDIS_DIAGRAM_TTL} + - DEPLOYMENT_URL=${DEPLOYMENT_URL} + restart: unless-stopped + ports: + - "8080:8080" + depends_on: + redis: + condition: service_healthy + networks: + - apollon-network + +networks: + apollon-network: + driver: bridge diff --git a/packages/server/src/main/server.ts b/packages/server/src/main/server.ts index 21143e1c..281f32d2 100644 --- a/packages/server/src/main/server.ts +++ b/packages/server/src/main/server.ts @@ -1,3 +1,5 @@ +import fs from 'fs'; +import path from 'path'; import bodyParser from 'body-parser'; import express, { RequestHandler } from 'express'; import * as Sentry from '@sentry/node'; @@ -19,6 +21,15 @@ if (process.env.SENTRY_DSN) { Sentry.setTag('package', 'server'); } +// Replace http://localhost:8080 with the actual process.env.DEPLOYMENT_URL +const jsFiles = fs.readdirSync(webappPath).filter((file) => file.endsWith('.js')); +jsFiles.forEach((file) => { + const filePath = path.join(webappPath, file); + const content = fs.readFileSync(filePath, 'utf8') + .replace(/http:\/\/localhost:8080/g, process.env.DEPLOYMENT_URL || 'http://localhost:8080'); + fs.writeFileSync(filePath, content); +}); + app.use('/', express.static(webappPath)); app.use(bodyParser.json() as RequestHandler); app.use( diff --git a/packages/webapp/webpack/webpack.common.js b/packages/webapp/webpack/webpack.common.js index 875fdfc2..14c8b68c 100644 --- a/packages/webapp/webpack/webpack.common.js +++ b/packages/webapp/webpack/webpack.common.js @@ -77,7 +77,7 @@ module.exports = { }), new webpack.DefinePlugin({ 'process.env.APPLICATION_SERVER_VERSION': JSON.stringify(process.env.APPLICATION_SERVER_VERSION || true), - 'process.env.DEPLOYMENT_URL': JSON.stringify(process.env.DEPLOYMENT_URL || 'http://localhost:8888'), + 'process.env.DEPLOYMENT_URL': JSON.stringify(process.env.DEPLOYMENT_URL || 'http://localhost:8080'), 'process.env.SENTRY_DSN': JSON.stringify(process.env.SENTRY_DSN || null), 'process.env.POSTHOG_HOST': JSON.stringify(process.env.POSTHOG_HOST || null), 'process.env.POSTHOG_KEY': JSON.stringify(process.env.POSTHOG_KEY || null), diff --git a/packages/webapp/webpack/webpack.dev.js b/packages/webapp/webpack/webpack.dev.js index e72e66cb..ea22ad23 100644 --- a/packages/webapp/webpack/webpack.dev.js +++ b/packages/webapp/webpack/webpack.dev.js @@ -17,7 +17,7 @@ module.exports = merge(common, { devServer: { static: path.join(__dirname, '../../build/webapp'), host: '0.0.0.0', - port: 8888, + port: 8080, proxy: [ { context: ['/'], From 123552442e2f350293b9e799ef66ec063cf459c5 Mon Sep 17 00:00:00 2001 From: "Felix T.J. Dietrich" Date: Fri, 6 Dec 2024 21:29:18 +0100 Subject: [PATCH 2/2] Improve deploy workflow (#119) * check for image * log payload * improve workflow * update workflow * fix set-output * fix ref * log inputs * fix workflow * add debug log * try fix? * fix env * fix description * ignore env * generalize workflow * fix env * fix env * remove log * restore env * fix name * mount volumes --- .github/workflows/build-and-push-shared.yml | 2 +- .github/workflows/deploy-dev.yml | 25 +++++-- .../deploy-docker-compose-shared.yml | 70 ++++++++++++++++--- docker-compose.prod.yml | 4 +- docker-compose.yml | 2 + 5 files changed, 84 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build-and-push-shared.yml b/.github/workflows/build-and-push-shared.yml index 7a7047ec..99813bac 100644 --- a/.github/workflows/build-and-push-shared.yml +++ b/.github/workflows/build-and-push-shared.yml @@ -80,7 +80,7 @@ jobs: - name: Set image tag output id: set-tag - run: echo "::set-output name=image-tag::${{ steps.meta.outputs.version }}" + run: echo "image-tag=${{ steps.meta.outputs.version }}" >> $GITHUB_OUTPUT - name: Build and push Docker Image uses: docker/build-push-action@v6 diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index 59c1b95a..5b2a4b9d 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -1,21 +1,32 @@ -name: Deploy Dev +name: Deploy to Dev on: workflow_dispatch: inputs: image-tag: type: string - description: "The tag of the docker images to deploy" - required: true -jobs: + description: "Image tag to deploy (default: pr- if PR exists, latest for default branch)" + +jobs: + prepare-env: + runs-on: ubuntu-latest + environment: Dev + outputs: + env-vars: | + DEPLOYMENT_URL=${{ vars.DEPLOYMENT_URL }} + APOLLON_REDIS_DIAGRAM_TTL=${{ vars.APOLLON_REDIS_DIAGRAM_TTL }} + steps: + - name: Do nothing + run: echo "Nothing to do here" + deploy: + needs: prepare-env # TODO: uses: ls1intum/.github/.github/workflows/deploy-docker-compose.yml@main uses: ./.github/workflows/deploy-docker-compose-shared.yml with: environment: Dev docker-compose-file: "./docker-compose.prod.yml" + main-image-name: ls1intum/apollon_standalone image-tag: ${{ inputs.image-tag }} - env-vars: | - DEPLOYMENT_URL=${{ vars.DEPLOYMENT_URL }} - APOLLON_REDIS_DIAGRAM_TTL=${{ vars.APOLLON_REDIS_DIAGRAM_TTL }} + env-vars: ${{ needs.prepare-env.outputs.env-vars }} secrets: inherit diff --git a/.github/workflows/deploy-docker-compose-shared.yml b/.github/workflows/deploy-docker-compose-shared.yml index a5e2827c..76821c14 100644 --- a/.github/workflows/deploy-docker-compose-shared.yml +++ b/.github/workflows/deploy-docker-compose-shared.yml @@ -12,25 +12,77 @@ on: type: string default: "./docker-compose.yml" description: "Path to the Docker Compose file (Default: ./docker-compose.yml)" + main-image-name: + type: string + description: "The name of the main image for checking if it exists with the given tag" + required: true image-tag: type: string - default: latest - description: "Tag of the Docker images to deploy (Default: latest)" + description: "Image tag to deploy (default: pr- if PR exists, latest for default branch)" env-vars: type: string description: "Additional environment variables in KEY=VALUE format, separated by newlines" required: false jobs: + prepare-deploy: + runs-on: ubuntu-latest + environment: Dev + outputs: + image-tag-to-deploy: ${{ steps.retrieve-image-tag.outputs.image-tag-to-deploy }} + + steps: + - name: Retrieve image tag to deploy + id: retrieve-image-tag + run: | + if [ -n "${{ inputs.image-tag }}" ]; then + echo "image-tag-to-deploy=${{ inputs.image-tag }}" >> $GITHUB_OUTPUT + exit 0 + fi + + REF=$(echo "${{ github.event.ref }}" | sed -n 's#^refs/heads/##p') + if [ "$REF" = "${{ github.event.repository.default_branch }}" ]; then + echo "image-tag-to-deploy=latest" >> $GITHUB_OUTPUT + fi + + PULLS=$(curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls?head=${{ github.repository_owner }}:${REF}") + PR_NUMBER=$(echo "$PULLS" | jq -r '.[0].number') + + if [ -z "$PR_NUMBER" ]; then + echo "No PR found for branch $REF." + exit 1 + else + echo "PR #$PR_NUMBER found for branch $REF." + echo "image-tag-to-deploy=pr-$PR_NUMBER" >> $GITHUB_OUTPUT + fi + + - name: Check if image exists + run: | + IMAGE_NAME="${{ inputs.main-image-name }}" + IMAGE_TAG="${{ steps.retrieve-image-tag.outputs.image-tag-to-deploy }}" + + ENCODED_TOKEN=$(echo -n "${{ secrets.GITHUB_TOKEN }}" | base64) + TAG_EXISTS=$(curl -s -H "Authorization: Bearer ${ENCODED_TOKEN}" \ + https://ghcr.io/v2/${IMAGE_NAME}/tags/list \ + | jq -r --arg TAG "${IMAGE_TAG}" '.tags[] | select(. == $TAG)') + + if [ -z "$TAG_EXISTS" ]; then + echo "Image ${IMAGE_NAME}:${IMAGE_TAG} does not exist." + exit 1 + else + echo "Image ${IMAGE_NAME}:${IMAGE_TAG} exists." + fi + deploy: + needs: prepare-deploy runs-on: ubuntu-latest environment: name: ${{ inputs.environment }} steps: - - name: Checkout Repository + - name: Checkout repository uses: actions/checkout@v4 - - name: SSH to VM and Execute Docker-Compose Down (if exists) + - name: SSH to VM and execute docker compose down (if exists) uses: appleboy/ssh-action@v1.0.3 with: host: ${{ vars.VM_HOST }} @@ -49,7 +101,7 @@ jobs: # Check if docker-compose.prod.yml exists if [ -f "$COMPOSE_FILE" ]; then echo "$COMPOSE_FILE found." - + # Check if .env exists if [ -f ".env" ]; then docker compose -f "$COMPOSE_FILE" --env-file=".env" down --remove-orphans --rmi all @@ -60,7 +112,7 @@ jobs: echo "$COMPOSE_FILE does not exist. Skipping docker compose down." fi - - name: Copy Docker Compose File to VM Host + - name: Copy docker compose file to VM host uses: appleboy/scp-action@v0.1.7 with: host: ${{ vars.VM_HOST }} @@ -73,7 +125,7 @@ jobs: source: ${{ inputs.docker-compose-file }} target: /home/${{ vars.VM_USERNAME }} - - name: SSH to VM and Create .env File + - name: SSH to VM and create .env file uses: appleboy/ssh-action@v1.0.3 with: host: ${{ vars.VM_HOST }} @@ -87,12 +139,12 @@ jobs: touch .env echo "ENVIRONMENT=${{ inputs.environment }}" > .env - echo "IMAGE_TAG=${{ inputs.image-tag }}" >> .env + echo "IMAGE_TAG=${{ needs.prepare-deploy.outputs.image-tag-to-deploy }}" >> .env if [ "${{ inputs.env-vars }}" != "" ]; then echo "${{ inputs.env-vars }}" >> .env fi - - name: SSH to VM and Execute Docker Compose Up + - name: SSH to VM and execute docker compose up uses: appleboy/ssh-action@v1.0.3 with: host: ${{ vars.VM_HOST }} diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 54ac0cba..90b8ff7b 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: redis: image: redis/redis-stack-server:7.4.0-v1 @@ -22,6 +20,8 @@ services: - APOLLON_REDIS_URL=redis://apollon-redis:6379 - APOLLON_REDIS_DIAGRAM_TTL=${APOLLON_REDIS_DIAGRAM_TTL} - DEPLOYMENT_URL=${DEPLOYMENT_URL} + volumes: + - /opt/apollon/diagrams:/app/diagrams restart: unless-stopped ports: - "8080:8080" diff --git a/docker-compose.yml b/docker-compose.yml index 92b0edc4..6bc6114e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,8 @@ services: - APOLLON_REDIS_URL=redis://apollon_redis:6379 - APOLLON_REDIS_DIAGRAM_TTL=${APOLLON_REDIS_DIAGRAM_TTL} - DEPLOYMENT_URL=${DEPLOYMENT_URL} + volumes: + - ./diagrams:/app/diagrams restart: always networks: - apollon_network