From 6cf40031888cc40408612f4384b27e53490b61cf Mon Sep 17 00:00:00 2001 From: Andrew Bolyachevets Date: Sat, 15 Jun 2024 20:05:19 -0700 Subject: [PATCH] Clean up cicd (#1547) * clean up cicd --- .github/workflows/emailer-cd-gcp.yml | 34 ++++++ api/.env.sample | 4 +- api/Makefile | 1 + api/config.py | 9 +- jobs/nr-day-job/Makefile | 1 + jobs/nr-day-job/config.py | 4 +- jobs/nro-extractor/Makefile | 1 + jobs/nro-extractor/config.py | 4 +- services/emailer/config.py | 2 +- .../devops/gcp/clouddeploy-targets.yaml | 100 ++++++++++++++++++ services/emailer/devops/vaults.gcp.env | 2 +- services/namex-pay/.env.sample | 8 +- services/namex-pay/Makefile | 1 + services/namex-pay/config.py | 10 +- services/pubsub/tests/test_queue.py | 2 +- services/solr-names-updater/Makefile | 1 + services/solr-names-updater/config.py | 9 +- 17 files changed, 165 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/emailer-cd-gcp.yml create mode 100644 services/emailer/devops/gcp/clouddeploy-targets.yaml diff --git a/.github/workflows/emailer-cd-gcp.yml b/.github/workflows/emailer-cd-gcp.yml new file mode 100644 index 000000000..746217d6f --- /dev/null +++ b/.github/workflows/emailer-cd-gcp.yml @@ -0,0 +1,34 @@ +name: Namex Emailer CD GCP + +on: + push: + branches: + - main + paths: + - "services/emailer/**" + workflow_dispatch: + inputs: + target: + description: "Deploy To" + required: true + type: choice + options: + - dev + - test + - sandbox + - prod + +jobs: + namex-emailer-cd: + permissions: + id-token: write + contents: write + + uses: bcgov/bcregistry-sre/.github/workflows/cloud-run-service-cd.yaml@main + with: + target: ${{ github.event.inputs.target }} # Corrected the input reference + app_name: "namex-emailer" + working_directory: "./services/emailer" + secrets: + WORKLOAD_IDENTIFY_POOLS_PROVIDER: ${{ secrets.WORKLOAD_IDENTIFY_POOLS_PROVIDER }} + GCP_SERVICE_ACCOUNT: ${{ secrets.GCP_SERVICE_ACCOUNT }} diff --git a/api/.env.sample b/api/.env.sample index f2ac64735..b3de0396e 100644 --- a/api/.env.sample +++ b/api/.env.sample @@ -71,8 +71,8 @@ JWT_OIDC_CACHING_ENABLED=True JWT_OIDC_JWKS_CACHE_TIMEOUT=300 # PUBSUB -GCP_AUTH_KEY= -EMAILER_TOPIC= +BUSINESS_GCP_AUTH_KEY= +NAMEX_MAILER_TOPIC= NAMEX_NR_STATE_TOPIC= # MARS diff --git a/api/Makefile b/api/Makefile index 7976a59c0..9e91ec385 100644 --- a/api/Makefile +++ b/api/Makefile @@ -24,6 +24,7 @@ setup: clean install ## Setup the project clean: clean-build clean-pyc clean-test ## Clean the project rm -rf .venv/ + rm -rf poetry.lock clean-build: ## Clean build files rm -fr build/ diff --git a/api/config.py b/api/config.py index c173305c3..66775854a 100644 --- a/api/config.py +++ b/api/config.py @@ -89,7 +89,6 @@ class Config(object): JWT_OIDC_JWKS_CACHE_TIMEOUT = int(os.getenv('JWT_OIDC_JWKS_CACHE_TIMEOUT', '300')) JWT_OIDC_AUDIENCE = os.getenv('JWT_OIDC_AUDIENCE') - JWT_OIDC_CLIENT_SECRET = os.getenv('JWT_OIDC_CLIENT_SECRET') SBC_SVC_AUTH_URL = os.getenv('KEYCLOAK_AUTH_TOKEN_URL', '') SBC_SVC_AUTH_CLIENT_ID = os.getenv('NAMEX_SBC_SERVICE_ACCOUNT_CLIENT_ID', '') @@ -107,9 +106,9 @@ class Config(object): DISABLE_NAMEREQUEST_SOLR_UPDATES = int(os.getenv('DISABLE_NAMEREQUEST_SOLR_UPDATES', 0)) - GCP_AUTH_KEY = os.getenv("GCP_AUTH_KEY", None) - NAMEX_NR_STATE_TOPIC = os.getenv("NAMEX_NR_STATE_TOPIC", "namex-pay-dev") - EMAILER_TOPIC = os.getenv("EMAILER_TOPIC", "namex-receipt-dev") + GCP_AUTH_KEY = os.getenv('BUSINESS_GCP_AUTH_KEY', None) + NAMEX_NR_STATE_TOPIC = os.getenv('NAMEX_NR_STATE_TOPIC', '') + EMAILER_TOPIC = os.getenv('NAMEX_MAILER_TOPIC', '') AUDIENCE = os.getenv("AUDIENCE", "https://pubsub.googleapis.com/google.pubsub.v1.Subscriber") PUBLISHER_AUDIENCE = os.getenv("PUBLISHER_AUDIENCE", "https://pubsub.googleapis.com/google.pubsub.v1.Publisher") @@ -154,7 +153,7 @@ class TestConfig(Config): port=int(DB_PORT), name=DB_NAME ) - EMAILER_TOPIC = os.getenv("EMAILER_TOPIC", "namex-receipt-dev") + EMAILER_TOPIC = os.getenv('NAMEX_MAILER_TOPIC', '') # We can't run NRO locally for running our tests DISABLE_NAMEREQUEST_NRO_UPDATES = int(os.getenv('DISABLE_NAMEREQUEST_NRO_UPDATES', 1)) diff --git a/jobs/nr-day-job/Makefile b/jobs/nr-day-job/Makefile index 01dc7e283..63da3a4cb 100644 --- a/jobs/nr-day-job/Makefile +++ b/jobs/nr-day-job/Makefile @@ -16,6 +16,7 @@ setup: install ## Setup the project clean: clean-build clean-pyc clean-test ## Clean the project rm -rf .venv/ + rm -rf poetry.lock clean-build: ## Clean build files rm -fr build/ diff --git a/jobs/nr-day-job/config.py b/jobs/nr-day-job/config.py index b484f5b8f..f111ad749 100644 --- a/jobs/nr-day-job/config.py +++ b/jobs/nr-day-job/config.py @@ -74,8 +74,8 @@ class Config(): # pylint: disable=too-few-public-methods name=DB_NAME, ) - GCP_AUTH_KEY = os.getenv('GCP_AUTH_KEY', None) - EMAILER_TOPIC = os.getenv('EMAILER_TOPIC', '') + GCP_AUTH_KEY = os.getenv('BUSINESS_GCP_AUTH_KEY', None) + EMAILER_TOPIC = os.getenv('NAMEX_MAILER_TOPIC', '') NAMEX_NR_STATE_TOPIC = os.getenv('NAMEX_NR_STATE_TOPIC', '') class DevConfig(Config): # pylint: disable=too-few-public-methods diff --git a/jobs/nro-extractor/Makefile b/jobs/nro-extractor/Makefile index 9ec32045b..b5766fa72 100644 --- a/jobs/nro-extractor/Makefile +++ b/jobs/nro-extractor/Makefile @@ -22,6 +22,7 @@ setup: install ## Setup the project clean: clean-build clean-pyc clean-test ## Clean the project rm -rf .venv/ + rm -rf poetry.lock clean-build: ## Clean build files rm -fr build/ diff --git a/jobs/nro-extractor/config.py b/jobs/nro-extractor/config.py index 9a7b18a4d..d1ab4f6cc 100644 --- a/jobs/nro-extractor/config.py +++ b/jobs/nro-extractor/config.py @@ -38,9 +38,9 @@ class BaseConfig(object): NRO_HOST = os.getenv('NRO_HOST', '') NRO_PORT = int(os.getenv('NRO_PORT', '1521')) - GCP_AUTH_KEY = os.getenv('GCP_AUTH_KEY', None) + GCP_AUTH_KEY = os.getenv('BUSINESS_GCP_AUTH_KEY', None) NAMEX_NR_STATE_TOPIC = os.getenv('NAMEX_NR_STATE_TOPIC', '') - EMAILER_TOPIC = os.getenv('EMAILER_TOPIC', '') + EMAILER_TOPIC = os.getenv('NAMEX_MAILER_TOPIC', '') class Config(BaseConfig): diff --git a/services/emailer/config.py b/services/emailer/config.py index 3b765e99c..29b94e2c2 100644 --- a/services/emailer/config.py +++ b/services/emailer/config.py @@ -102,7 +102,7 @@ class Config: # pylint: disable=too-few-public-methods ACCOUNT_SVC_CLIENT_SECRET = os.getenv("KEYCLOAK_CLIENT_SECRET") ACCOUNT_SVC_TIMEOUT = os.getenv("KEYCLOAK_TIMEOUT") - SUB_AUDIENCE = os.getenv('SUB_AUDIENCE', '') + SUB_AUDIENCE = os.getenv('EMAILER_SUB_AUDIENCE', '') SUB_SERVICE_ACCOUNT = os.getenv('SUB_SERVICE_ACCOUNT', '') NAME_REQUEST_URL = os.getenv("NAME_REQUEST_URL", "") diff --git a/services/emailer/devops/gcp/clouddeploy-targets.yaml b/services/emailer/devops/gcp/clouddeploy-targets.yaml new file mode 100644 index 000000000..4bdfd7530 --- /dev/null +++ b/services/emailer/devops/gcp/clouddeploy-targets.yaml @@ -0,0 +1,100 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: deploy.cloud.google.com/v1 +kind: Target +metadata: + name: dev +description: Dev Environment +deployParameters: + deploy-env: "development" + deploy-project-id: "a083gt-dev" + service-name: "namex-emailer-dev" + container-name: "namex-emailer-dev" + app-env: "dev" + service-account: "sa-api@a083gt-dev.iam.gserviceaccount.com" +run: + location: projects/a083gt-dev/locations/northamerica-northeast1 +executionConfigs: +- usages: [DEPLOY, RENDER] + artifactStorage: 'gs://c4hnrd-tools_clouddeploy/history' +--- + +apiVersion: deploy.cloud.google.com/v1 +kind: Target +metadata: + name: test +description: Test Environment +deployParameters: + deploy-env: "development" + deploy-project-id: "a083gt-test" + service-name: "namex-emailer-test" + container-name: "namex-emailer-test" + app-env: "test" + service-account: "sa-api@a083gt-test.iam.gserviceaccount.com" +run: + location: projects/a083gt-test/locations/northamerica-northeast1 +executionConfigs: +- usages: [DEPLOY, RENDER] + artifactStorage: 'gs://c4hnrd-tools_clouddeploy/history' +--- + +apiVersion: deploy.cloud.google.com/v1 +kind: Target +metadata: + name: sandbox +description: Sandbox Environment +requireApproval: true +deployParameters: + deploy-env: "production" + deploy-project-id: "a083gt-tools" + service-name: "namex-emailer-sandbox" + container-name: "namex-emailer-sandbox" + app-env: "sandbox" + service-account: "sa-api@a083gt-tools.iam.gserviceaccount.com" + max-scale: "50" + container-concurrency: "20" + container-port: "8080" + resources-cpu: 4000m + resources-memory: 8Gi +run: + location: projects/a083gt-tools/locations/northamerica-northeast1 +executionConfigs: +- usages: [DEPLOY, RENDER] + artifactStorage: 'gs://c4hnrd-tools_clouddeploy/history' +--- + +apiVersion: deploy.cloud.google.com/v1 +kind: Target +metadata: + name: prod +description: Production Environment +requireApproval: true +deployParameters: + deploy-env: "production" + deploy-project-id: "a083gt-prod" + service-name: "namex-emailer-prod" + container-name: "namex-emailer-prod" + app-env: "production" + service-account: "sa-api@a083gt-prod.iam.gserviceaccount.com" + max-scale: "50" + container-concurrency: "20" + container-port: "8080" + resources-cpu: 4000m + resources-memory: 8Gi +run: + location: projects/a083gt-prod/locations/northamerica-northeast1 +executionConfigs: +- usages: [DEPLOY, RENDER] + artifactStorage: 'gs://c4hnrd-tools_clouddeploy/history' \ No newline at end of file diff --git a/services/emailer/devops/vaults.gcp.env b/services/emailer/devops/vaults.gcp.env index bde932d86..5980694e1 100644 --- a/services/emailer/devops/vaults.gcp.env +++ b/services/emailer/devops/vaults.gcp.env @@ -13,5 +13,5 @@ NAMEX_API_VERSION="op://API/$APP_ENV/namex-api/NAMEX_API_VERSION" AUDIENCE="op://gcp-queue/$APP_ENV/payment/AUDIENCE" TOPIC_NAME="op://gcp-queue/$APP_ENV/payment/TOPIC_NAME" PUBLISHER_AUDIENCE="op://gcp-queue/$APP_ENV/payment/PUBLISHER_AUDIENCE" -GCP_AUTH_KEY="op://gcp-queue/$APP_ENV/payment/GCP_AUTH_KEY" +GCP_AUTH_KEY="op://gcp-queue/$APP_ENV/payment/BUSINESS_GCP_AUTH_KEY" SENTRY_DSN="" \ No newline at end of file diff --git a/services/namex-pay/.env.sample b/services/namex-pay/.env.sample index 640d3867d..76ae0976a 100644 --- a/services/namex-pay/.env.sample +++ b/services/namex-pay/.env.sample @@ -28,8 +28,8 @@ ORACLE_PORT=1521 PAY_API_URL= PAY_API_VERSION= -GCP_AUTH_KEY= -EMAILER_TOPIC= -SUB_AUDIENCE= -SUB_SERVICE_ACCOUNT= +BUSINESS_GCP_AUTH_KEY= +NAMEX_MAILER_TOPIC= +PAY_SUB_AUDIENCE= +AUTHPAY_SERVICE_ACCOUNT= DEBUG_REQUEST= \ No newline at end of file diff --git a/services/namex-pay/Makefile b/services/namex-pay/Makefile index 9571a2bbb..211aec087 100644 --- a/services/namex-pay/Makefile +++ b/services/namex-pay/Makefile @@ -16,6 +16,7 @@ setup: install ## Setup the project clean: clean-build clean-pyc clean-test ## Clean the project rm -rf .venv/ + rm -rf poetry.lock clean-build: ## Clean build files rm -fr build/ diff --git a/services/namex-pay/config.py b/services/namex-pay/config.py index 1da07571b..a9d69f226 100644 --- a/services/namex-pay/config.py +++ b/services/namex-pay/config.py @@ -82,13 +82,13 @@ class Config(): # pylint: disable=too-few-public-methods NRO_HOST = os.getenv('ORACLE_HOST', '') NRO_PORT = int(os.getenv('ORACLE_PORT', '1521')) - GCP_AUTH_KEY = os.getenv('GCP_AUTH_KEY', None) - EMAILER_TOPIC = os.getenv('EMAILER_TOPIC', '') + GCP_AUTH_KEY = os.getenv('BUSINESS_GCP_AUTH_KEY', None) + EMAILER_TOPIC = os.getenv('NAMEX_MAILER_TOPIC', '') NAMEX_NR_STATE_TOPIC = os.getenv('NAMEX_NR_STATE_TOPIC', '') AUDIENCE = os.getenv('AUDIENCE', 'https://pubsub.googleapis.com/google.pubsub.v1.Subscriber') PUBLISHER_AUDIENCE = os.getenv('PUBLISHER_AUDIENCE', 'https://pubsub.googleapis.com/google.pubsub.v1.Publisher') - SUB_AUDIENCE = os.getenv('SUB_AUDIENCE', '') - SUB_SERVICE_ACCOUNT = os.getenv('SUB_SERVICE_ACCOUNT', '') + SUB_AUDIENCE = os.getenv('PAY_SUB_AUDIENCE', '') + SUB_SERVICE_ACCOUNT = os.getenv('AUTHPAY_SERVICE_ACCOUNT', '') DEBUG_REQUEST = os.getenv('DEBUG_REQUEST', False) ENVIRONMENT = os.getenv('ENVIRONMENT', 'prod') @@ -116,7 +116,7 @@ class TestConfig(Config): # pylint: disable=too-few-public-methods DB_HOST = os.getenv('DATABASE_TEST_HOST', '127.0.0.1') DB_PORT = os.getenv('DATABASE_TEST_PORT', '5432') SQLALCHEMY_DATABASE_URI = f'postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}' - EMAILER_TOPIC = os.getenv("EMAILER_TOPIC", "namex-receipt-dev") + EMAILER_TOPIC = os.getenv('NAMEX_MAILER_TOPIC', '') class ProdConfig(Config): # pylint: disable=too-few-public-methods """Production environment configuration.""" diff --git a/services/pubsub/tests/test_queue.py b/services/pubsub/tests/test_queue.py index abfd841c4..2acd2bb24 100644 --- a/services/pubsub/tests/test_queue.py +++ b/services/pubsub/tests/test_queue.py @@ -104,7 +104,7 @@ def test_gcp_pubsub_connectivity(): load_dotenv(".env") app = flask.Flask(__name__) - app.config['GCP_AUTH_KEY'] = os.getenv('GCP_AUTH_KEY') + app.config['GCP_AUTH_KEY'] = os.getenv('BUSINESS_GCP_AUTH_KEY') queue = GcpQueue() queue.init_app(app) diff --git a/services/solr-names-updater/Makefile b/services/solr-names-updater/Makefile index 8dc030247..eac0ecb11 100644 --- a/services/solr-names-updater/Makefile +++ b/services/solr-names-updater/Makefile @@ -16,6 +16,7 @@ setup: install ## Setup the project clean: clean-build clean-pyc clean-test ## Clean the project rm -rf .venv/ + rm -rf poetry.lock clean-build: ## Clean build files rm -fr build/ diff --git a/services/solr-names-updater/config.py b/services/solr-names-updater/config.py index 3cb4d0991..68c816b23 100644 --- a/services/solr-names-updater/config.py +++ b/services/solr-names-updater/config.py @@ -82,13 +82,13 @@ class Config(): # pylint: disable=too-few-public-methods name=DB_NAME, ) - GCP_AUTH_KEY = os.getenv('GCP_AUTH_KEY', None) - EMAILER_TOPIC = os.getenv('EMAILER_TOPIC', 'namex-receipt-dev') + GCP_AUTH_KEY = os.getenv('BUSINESS_GCP_AUTH_KEY', None) + EMAILER_TOPIC = os.getenv('NAMEX_MAILER_TOPIC', '') AUDIENCE = os.getenv('AUDIENCE', 'https://pubsub.googleapis.com/google.pubsub.v1.Subscriber') PUBLISHER_AUDIENCE = os.getenv('PUBLISHER_AUDIENCE', 'https://pubsub.googleapis.com/google.pubsub.v1.Publisher') DEBUG_REQUEST = os.getenv('DEBUG_REQUEST', False) - SUB_AUDIENCE = os.getenv('SUB_AUDIENCE', '') - SUB_SERVICE_ACCOUNT = os.getenv('SUB_SERVICE_ACCOUNT', '') + SUB_AUDIENCE = os.getenv('SOLR_SUB_AUDIENCE', '') + SUB_SERVICE_ACCOUNT = os.getenv('BUSINESS_SERVICE_ACCOUNT', '') SOLR_FEEDER_API_URL = os.getenv('SOLR_FEEDER_API_URL', None) @@ -100,7 +100,6 @@ class Config(): # pylint: disable=too-few-public-methods JWT_OIDC_JWKS_URI = os.getenv('JWT_OIDC_JWKS_URI') JWT_OIDC_ISSUER = os.getenv('JWT_OIDC_ISSUER') JWT_OIDC_AUDIENCE = os.getenv('JWT_OIDC_AUDIENCE') - JWT_OIDC_CLIENT_SECRET = os.getenv('JWT_OIDC_CLIENT_SECRET') JWT_OIDC_CACHING_ENABLED = os.getenv('JWT_OIDC_CACHING_ENABLED') JWT_OIDC_JWKS_CACHE_TIMEOUT = int(os.getenv('JWT_OIDC_JWKS_CACHE_TIMEOUT', '300'))