From 1d448cad884d46425403268ef1efefa08b2a1c07 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 6 Mar 2023 14:50:04 +0000 Subject: [PATCH 01/11] FIX: Fix path to app in Dockerfile --- Dockerfile | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index fe21829..ac2e153 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,4 +32,4 @@ ARG GUNICORN_THREADS=8 ENV GUNICORN_THREADS=$GUNICORN_THREADS # Timeout is set to 0 to disable the timeouts of the workers to allow Cloud Run to handle instance scaling. -CMD exec gunicorn --bind :$PORT --workers $GUNICORN_WORKERS --threads $GUNICORN_THREADS --timeout 0 app:app +CMD exec gunicorn --bind :$PORT --workers $GUNICORN_WORKERS --threads $GUNICORN_THREADS --timeout 0 dashboard.app:app diff --git a/pyproject.toml b/pyproject.toml index 884b463..7db12c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ [tool.poetry] name = "aerosense-dashboard" -version = "0.4.0" +version = "0.4.1" description = "High-level visualisation for aerosense" authors = ["Tom Clark", "Marcus Lugg", "Yuriy Marykovsky"] license = "BSD-3" From 8317c0ec6fbb61e1fd86070b968bc1c5313b1507 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 6 Mar 2023 14:50:37 +0000 Subject: [PATCH 02/11] OPS: Use `octue/get-deployment-info` action in CD workflow --- .github/workflows/cd.yml | 149 +++++++++++++++++++++++ .github/workflows/cloud_run_deploy.yml | 158 ------------------------- 2 files changed, 149 insertions(+), 158 deletions(-) create mode 100644 .github/workflows/cd.yml delete mode 100644 .github/workflows/cloud_run_deploy.yml diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 0000000..35bcadb --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,149 @@ +name: cd + +on: + push: + branches: + - main + + workflow_dispatch: + inputs: + debug_enabled: + description: "Enable tmate debug" + type: boolean + default: "false" + +jobs: + info: + runs-on: ubuntu-latest + + timeout-minutes: 5 + + permissions: + contents: read + + outputs: + branch_tag_kebab: ${{ steps.get-deployment-info.outputs.branch_tag_kebab }} + branch_tag_screaming: ${{ steps.get-deployment-info.outputs.branch_tag_screaming}} + image_latest_artifact: ${{ steps.get-deployment-info.outputs.image_latest_artifact}} + image_latest_tag: ${{ steps.get-deployment-info.outputs.image_latest_tag }} + image_version_artifact: ${{ steps.get-deployment-info.outputs.image_version_artifact}} + image_version_tag: ${{ steps.get-deployment-info.outputs.image_version_tag }} + short_sha: ${{ steps.get-deployment-info.outputs.short_sha }} + gcp_project_name: ${{ steps.get-deployment-info.outputs.gcp_project_name}} + gcp_project_number: ${{ steps.get-deployment-info.outputs.gcp_project_number}} + gcp_region: ${{ steps.get-deployment-info.outputs.gcp_region}} + gcp_resource_affix: ${{ steps.get-deployment-info.outputs.gcp_resource_affix}} + gcp_service_name: ${{ steps.get-deployment-info.outputs.gcp_service_name}} + gcp_environment: ${{ steps.get-deployment-info.outputs.gcp_environment}} + version: ${{ steps.get-deployment-info.outputs.version }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install poetry + uses: snok/install-poetry@v1 + + - name: Get deployment info + id: get-deployment-info + uses: octue/get-deployment-info@0.1.3 + with: + gcp_project_name: aerosense-twined + gcp_project_number: 885434704038 + gcp_region: europe-west1 + gcp_resource_affix: aerosense + gcp_service_name: dashboard + gcp_environment: main + + build: + runs-on: ubuntu-latest + timeout-minutes: 60 + + permissions: + id-token: write + contents: read + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Authenticate with GCP Workload Identity + id: auth + uses: google-github-actions/auth@v0 + with: + # NOTE: If setting create_credentials_file=true, .dockerignore file must include `gha-creds-*.json` to avoid baking these credentials into build + create_credentials_file: true + workload_identity_provider: "projects/${{ needs.info.outputs.gcp_project_number }}/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider" + service_account: "dashboard-github-actions@${{ needs.info.outputs.gcp_project_name }}.iam.gserviceaccount.com" + + - name: Setup gcloud + uses: "google-github-actions/setup-gcloud@v0" + + - name: Configure Docker for GCP + run: gcloud auth configure-docker ${{ needs.info.outputs.gcp_region }}-docker.pkg.dev + + - name: Build and push artifact with latest and version tags + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64 + file: Dockerfile + load: false + push: true + cache-from: type=gha + cache-to: type=gha,mode=max + tags: | + ${{ needs.info.outputs.image_version_artifact }} + ${{ needs.info.outputs.image_latest_artifact }} + + deploy: + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 60 + needs: + - info + - build + + permissions: + id-token: write + contents: read + + steps: + - name: Checkout + # Shouldn't be necessary since we pull code in an image, but it's required by google-github-actions/auth + uses: actions/checkout@v3 + + - name: Authenticate with GCP Workload Identity + id: auth + uses: google-github-actions/auth@v0 + with: + # NOTE: If setting create_credentials_file=true, .dockerignore file must include `gha-creds-*.json` to avoid baking these credentials into build + create_credentials_file: true + workload_identity_provider: "projects/${{ needs.info.outputs.gcp_project_number }}/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider" + service_account: "dashboard-github-actions@${{ needs.info.outputs.gcp_project_name }}.iam.gserviceaccount.com" + + - name: Setup gcloud + uses: "google-github-actions/setup-gcloud@v0" + + - name: Setup tmate session [DEBUG] + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true'}} + uses: mxschmitt/action-tmate@v3 + + - name: Deploy to Cloud Run Server + id: deploy_dashboard + uses: google-github-actions/deploy-cloudrun@v0 + with: + image: ${{ needs.info.outputs.image_version_artifact }} + region: ${{ needs.info.outputs.gcp_region }} + service: ${{ needs.info.outputs.gcp_resource_affix }}-${{ needs.info.outputs.gcp_service_name }}-${{ needs.info.outputs.branch_tag_kebab }} + tag: sha${{ needs.info.outputs.short_sha }} + flags: --service-account=dashboard@{{ needs.info.outputs.gcp_project_name }}.iam.gserviceaccount.com + + - name: Show Cloud Run Deployment URL + run: echo "${{ steps.deploy_dashboard.outputs.url }}" diff --git a/.github/workflows/cloud_run_deploy.yml b/.github/workflows/cloud_run_deploy.yml deleted file mode 100644 index c75e954..0000000 --- a/.github/workflows/cloud_run_deploy.yml +++ /dev/null @@ -1,158 +0,0 @@ -on: - push: - branches: - - main - - workflow_dispatch: - inputs: - debug_enabled: - description: "Enable tmate debug" - type: boolean - default: "false" - -jobs: - build: - runs-on: ubuntu-latest - timeout-minutes: 60 - - permissions: - id-token: write - contents: read - - outputs: - version: ${{ steps.get_version.outputs.version }} - branch_tag: ${{ steps.tags.outputs.branch_tag }} - short_sha: ${{ steps.tags.outputs.short_sha }} - environment_label: ${{ steps.tags.outputs.environment_label}} - image_version_tag: ${{ steps.tags.outputs.image_version_tag }} - image_latest_tag: ${{ steps.tags.outputs.image_latest_tag }} - image_version_artefact: ${{ steps.artefacts.outputs.image_version_artefact}} - image_latest_artefact: ${{ steps.artefacts.outputs.image_latest_artefact}} - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Install poetry - uses: snok/install-poetry@v1 - - - name: Get version - id: get_version - run: | - PACKAGE_VERSION=$(poetry version -s) - echo "::set-output name=version::${PACKAGE_VERSION}" - - - name: Get slugified branch name and docker image tags - id: tags - run: | - echo "::set-output name=short_sha::$(git rev-parse --short HEAD)" - BRANCH_TAG=$(echo ${GITHUB_REF#refs/heads/} | iconv -c -t ascii//TRANSLIT | sed -E 's/[~^]+//g' | sed -E 's/[^a-zA-Z0-9]+/-/g' | sed -E 's/^-+|-+$//g' | tr A-Z a-z) - if [ "$BRANCH_TAG" = "main" ]; then - TAG_VERSION=${{steps.get_version.outputs.version}} - ENVIRONMENT_LABEL=production - else - TAG_VERSION="unreleased" - ENVIRONMENT_LABEL=staging - fi - echo "::set-output name=branch_tag::$BRANCH_TAG" - echo "::set-output name=tag_version::$TAG_VERSION" - echo "::set-output name=environment_label::$ENVIRONMENT_LABEL" - echo "::set-output name=image_version_tag::$BRANCH_TAG-$TAG_VERSION" - echo "::set-output name=image_latest_tag::$BRANCH_TAG-latest" - echo "Output: branch_tag = $BRANCH_TAG" - echo "Output: tag_version = $TAG_VERSION" - echo "Output: environment_label = $ENVIRONMENT_LABEL" - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Authenticate with GCP Workload Identity - id: auth - uses: google-github-actions/auth@v0 - with: - # NOTE: If setting create_credentials_file=true, .dockerignore file must include `gha-creds-*.json` to avoid baking these credentials into build - create_credentials_file: true - workload_identity_provider: "projects/885434704038/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider" - service_account: "dashboard-github-actions@aerosense-twined.iam.gserviceaccount.com" - - - name: Setup gcloud - uses: "google-github-actions/setup-gcloud@v0" - - - name: Configure Docker for GCP - run: gcloud auth configure-docker europe-west1-docker.pkg.dev - - - name: Set image artefact address - id: artefacts - run: | - echo "::set-output name=image_version_artefact::europe-west1-docker.pkg.dev/aerosense-twined/dashboard/dashboard:${{ steps.tags.outputs.image_version_tag}}" - echo "::set-output name=image_latest_artefact::europe-west1-docker.pkg.dev/aerosense-twined/dashboard/dashboard:${{ steps.tags.outputs.image_latest_tag}}" - - - name: Build and push container - # Note: We don't push containers with shas because we'd end up with terabytes in storage (an image for every commit) - uses: docker/build-push-action@v2 - with: - context: . - platforms: linux/amd64 - file: Dockerfile - push: true - cache-from: type=gha - cache-to: type=gha,mode=max - tags: | - ${{ steps.artefacts.outputs.image_version_artefact}} - ${{ steps.artefacts.outputs.image_latest_artefact}} - - deploy: - runs-on: ubuntu-latest - timeout-minutes: 60 - needs: build - - permissions: - id-token: write - contents: read - - steps: - - name: Checkout - # Shouldn't be necessary since we pull code in an image, but it's required by google-github-actions/auth - uses: actions/checkout@v3 - - - name: Show input values - run: | - echo "GitHub event_name: ${{ github.event_name }}" - echo "GitHub event inputs debug_enabled: ${{ github.event.inputs.debug_enabled }}" - echo "Slugified branch_tag: ${{ needs.build.outputs.branch_tag }}" - echo "Docker image_version_artefact: ${{ needs.build.outputs.image_version_artefact }}" - echo "Docker image_latest_artefact: ${{ needs.build.outputs.image_latest_artefact }}" - echo "Repository short_sha: ${{ needs.build.outputs.short_sha }}" - echo "Environment environment_label: ${{ needs.build.outputs.environment_label }}" - - - name: Authenticate with GCP Workload Identity - id: auth - uses: google-github-actions/auth@v0 - with: - # NOTE: If setting create_credentials_file=true, .dockerignore file must include `gha-creds-*.json` to avoid baking these credentials into build - create_credentials_file: true - workload_identity_provider: "projects/885434704038/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider" - service_account: "dashboard-github-actions@aerosense-twined.iam.gserviceaccount.com" - - - name: Setup gcloud - uses: "google-github-actions/setup-gcloud@v0" - - - name: Setup tmate session [DEBUG] - if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true'}} - uses: mxschmitt/action-tmate@v3 - - - name: Deploy to Cloud Run Server - id: deploy_dashboard - uses: google-github-actions/deploy-cloudrun@v0 - with: - image: ${{ needs.build.outputs.image_version_artefact }} - region: europe-west1 - service: dashboard-${{ needs.build.outputs.branch_tag }} - tag: sha${{ needs.build.outputs.short_sha }} - flags: --service-account=dashboard@aerosense-twined.iam.gserviceaccount.com - - - name: Show Cloud Run Deployment URL - run: echo "${{ steps.deploy_dashboard.outputs.url }}" From f280627d1fa603e862ce549ab280451486b58476 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 6 Mar 2023 14:51:38 +0000 Subject: [PATCH 03/11] WIP: Temporarily run CD workflow on all branches --- .github/workflows/cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 35bcadb..5d6a900 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -3,7 +3,7 @@ name: cd on: push: branches: - - main + # - main workflow_dispatch: inputs: From cc2f00b1bc516dfdc518006f37124ea327ce22a0 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 6 Mar 2023 14:55:26 +0000 Subject: [PATCH 04/11] OPS: Add missing needs field to CD workflow --- .github/workflows/cd.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 5d6a900..f57fd81 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -58,6 +58,7 @@ jobs: build: runs-on: ubuntu-latest timeout-minutes: 60 + needs: info permissions: id-token: write @@ -106,9 +107,7 @@ jobs: if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest timeout-minutes: 60 - needs: - - info - - build + needs: build permissions: id-token: write From e257635ab1e5a6313a511232614724b1d26c8377 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 6 Mar 2023 14:58:59 +0000 Subject: [PATCH 05/11] CHO: Work around incorrect container being served by GHA --- .github/workflows/cd.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index f57fd81..0e36823 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -21,12 +21,13 @@ jobs: permissions: contents: read + # TODO: Update spelling of "artefact" to "artifact" when GitHub actions manages to run the version of the `octue/get-deployment-info` action we're asking it to. outputs: branch_tag_kebab: ${{ steps.get-deployment-info.outputs.branch_tag_kebab }} branch_tag_screaming: ${{ steps.get-deployment-info.outputs.branch_tag_screaming}} - image_latest_artifact: ${{ steps.get-deployment-info.outputs.image_latest_artifact}} + image_latest_artifact: ${{ steps.get-deployment-info.outputs.image_latest_artefact}} image_latest_tag: ${{ steps.get-deployment-info.outputs.image_latest_tag }} - image_version_artifact: ${{ steps.get-deployment-info.outputs.image_version_artifact}} + image_version_artifact: ${{ steps.get-deployment-info.outputs.image_version_artefact}} image_version_tag: ${{ steps.get-deployment-info.outputs.image_version_tag }} short_sha: ${{ steps.get-deployment-info.outputs.short_sha }} gcp_project_name: ${{ steps.get-deployment-info.outputs.gcp_project_name}} From 4eae017a0d36ba21745f86412cd335ed578fea40 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 6 Mar 2023 15:06:40 +0000 Subject: [PATCH 06/11] OPS: Use Zurich region for hosting docker images --- .github/workflows/cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 0e36823..27add8d 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -51,7 +51,7 @@ jobs: with: gcp_project_name: aerosense-twined gcp_project_number: 885434704038 - gcp_region: europe-west1 + gcp_region: europe-west6 gcp_resource_affix: aerosense gcp_service_name: dashboard gcp_environment: main From 563ce0e8ac881b09e0a762df0a0e1519db9bcf68 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 6 Mar 2023 15:12:24 +0000 Subject: [PATCH 07/11] WIP: Temporarily deploy from non-main branches --- .github/workflows/cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 27add8d..f15e44e 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -105,7 +105,7 @@ jobs: ${{ needs.info.outputs.image_latest_artifact }} deploy: - if: github.ref == 'refs/heads/main' + # if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest timeout-minutes: 60 needs: build From a88d9998a0bd8574ec42a85249cd262a63ff7cfc Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 6 Mar 2023 15:18:30 +0000 Subject: [PATCH 08/11] OPS: Add missing needs element --- .github/workflows/cd.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index f15e44e..a2773e4 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -108,7 +108,9 @@ jobs: # if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest timeout-minutes: 60 - needs: build + needs: + - info + - build permissions: id-token: write From 5dabb5f067f9790af84259dd75a1d35b9754fda4 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 6 Mar 2023 15:28:26 +0000 Subject: [PATCH 09/11] OPS: Add missing dollar sign --- .github/workflows/cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index a2773e4..f4ece9a 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -145,7 +145,7 @@ jobs: region: ${{ needs.info.outputs.gcp_region }} service: ${{ needs.info.outputs.gcp_resource_affix }}-${{ needs.info.outputs.gcp_service_name }}-${{ needs.info.outputs.branch_tag_kebab }} tag: sha${{ needs.info.outputs.short_sha }} - flags: --service-account=dashboard@{{ needs.info.outputs.gcp_project_name }}.iam.gserviceaccount.com + flags: --service-account=dashboard@${{ needs.info.outputs.gcp_project_name }}.iam.gserviceaccount.com - name: Show Cloud Run Deployment URL run: echo "${{ steps.deploy_dashboard.outputs.url }}" From edd69d8ff928f52f935a604849a233cfab6bd395 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 6 Mar 2023 15:34:12 +0000 Subject: [PATCH 10/11] REV: Revert "WIP: Temporarily deploy from non-main branches" This reverts commit 563ce0e8ac881b09e0a762df0a0e1519db9bcf68. --- .github/workflows/cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index f4ece9a..837695a 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -105,7 +105,7 @@ jobs: ${{ needs.info.outputs.image_latest_artifact }} deploy: - # if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest timeout-minutes: 60 needs: From f4c56ee03752679ea6b0ee025e0c3c59c077caf5 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 6 Mar 2023 15:34:20 +0000 Subject: [PATCH 11/11] REV: Revert "WIP: Temporarily run CD workflow on all branches" This reverts commit f280627d1fa603e862ce549ab280451486b58476. --- .github/workflows/cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 837695a..99bb553 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -3,7 +3,7 @@ name: cd on: push: branches: - # - main + - main workflow_dispatch: inputs: