From 755e05c0560add7fdec29a0f39cf9dc6485485c8 Mon Sep 17 00:00:00 2001 From: Jasper Ginn Date: Sun, 18 Feb 2024 11:16:11 +0100 Subject: [PATCH] docs: add scripts --- ...d referencing them in code deployments.md" | 158 +++++++++++++++++- 1 file changed, 157 insertions(+), 1 deletion(-) diff --git "a/docs/ADRs/\360\237\244\253 Adding secrets to dagster GKE deployment and referencing them in code deployments.md" "b/docs/ADRs/\360\237\244\253 Adding secrets to dagster GKE deployment and referencing them in code deployments.md" index db14c4d..d82c496 100644 --- "a/docs/ADRs/\360\237\244\253 Adding secrets to dagster GKE deployment and referencing them in code deployments.md" +++ "b/docs/ADRs/\360\237\244\253 Adding secrets to dagster GKE deployment and referencing them in code deployments.md" @@ -91,7 +91,163 @@ resource "kubernetes_secret" "credentials" { } ``` ### Adding secrets not generated by Terraform -To add a secret and make it available to code locations, we have created a [GitHub Actions pipeline](https://github.com/JasperHG90/dagster-infra/blob/main/.github/workflows/add_secrets.yml). A user can add secrets to the GKE deployment by adding them to the 'dagster-infra' [Secrets and variables](https://github.com/JasperHG90/dagster-infra/settings/secrets/actions) GitHub settings. +To add a secret and make it available to code locations, we have created a [GitHub Actions pipeline](https://github.com/JasperHG90/dagster-infra/blob/main/.github/workflows/add_secrets.yml). + +```yaml +# dagster-infra/.github/workflows/add_secrets.yml +name: 'Add secrets' + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + add_secrets: + name: 'Add secrets' + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'pip' + - id: 'auth' + uses: 'google-github-actions/auth@v2' + with: + credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}' + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v2' + - name: 'Set project' + run: gcloud config set project jasperg-dagster + - name: 'Install dependencies' + run: pip install -r requirements_scripts.txt + - name: 'Add secrets' + run: python scripts/add_secret.py from-prefix DAGSTER_SECRET + env: + DAGSTER_SECRET_SLACK_BOT_OAUTH_TOKEN: '${{ secrets.DAGSTER_SECRET_SLACK_BOT_OAUTH_TOKEN }}' +``` + +The GitHub action uses the following python script: + +```python +""" +Add a secret from an environment variable to the gcp secrets manager. + +Required: gcloud installed and authenticated. +""" + +import os +import json +import logging +import tempfile +import pathlib as plb +from dataclasses import dataclass + +import typer +import sh + +logger = logging.getLogger("add_secret") +handler = logging.StreamHandler() +format = logging.Formatter("%(name)s - %(levelname)s - %(message)s") +handler.setFormatter(format) +logger.addHandler(handler) +logger.setLevel(logging.DEBUG) + +app = typer.Typer() + + +@dataclass +class ProjectConfig: + region: str + account: str + project: str + + def __post_init__(self): + if self.project is None: + raise ValueError( + "Project name not found. Set it using 'gcloud config set project '" + ) + + def log(self): + logger.debug(f"Project: {self.project}") + logger.debug(f"Account: {self.account}") + logger.debug(f"Region: {self.region}") + + +def _get_project_config(): + project_config = json.loads(sh.gcloud.config.list("--format", "json")) + compute = project_config.get("compute") + core = project_config.get("core") + return ProjectConfig( + region=None if compute is None else compute.get("region"), + account=None if core is None else core.get("account"), + project=None if core is None else core.get("project"), + ) + + +def _get_secret(secret_name: str): + logger.debug(f"Filtering for secret with name='{secret_name}'.") + secrets = json.loads(sh.gcloud.secrets.list("--format", "json", "--filter", secret_name)) + logger.debug(f"Found {len(secrets)} secrets.") + return secrets + + +def _create_secret(secret_name: str): + sh.gcloud.secrets.create(secret_name, "--replication-policy", "automatic") + logger.debug(f"Created secret with name='{secret_name}'.") + + +def _add_secret_version(secret_name: str, secret_value: str): + with tempfile.TemporaryDirectory() as tmpdir: + _secret_file = plb.Path(tmpdir) / "secret.txt" + with _secret_file.open("w") as f: + f.write(secret_value) + sh.gcloud.secrets.versions.add(secret_name, "--data-file", str(_secret_file)) + logger.debug(f"Added secret version to secret with name='{secret_name}'.") + + +def _from_env_var(env_var_name: str): + """Add a secret from an environment variable to the gcp secrets manager.""" + logger.debug(f"Looking for environment variable '{env_var_name}'.") + _value = os.getenv(env_var_name) + if _value is None: + raise KeyError(f"Environment variable {env_var_name} not found.") + _project_config = _get_project_config() + _project_config.log() + if len(_get_secret(env_var_name)) == 0: + _create_secret(env_var_name) + _add_secret_version(env_var_name, _value) + + +@app.command() +def from_env_var( + env_var_name: str = typer.Argument( + ..., help="Name of the environment variable to add to the secrets manager." + ) +): + _from_env_var(env_var_name) + + +@app.command() +def from_prefix( + prefix: str = typer.Argument( + ..., help="Prefix of the environment variables to add to the secrets manager." + ) +): + for k in os.environ.keys(): + if k.startswith(prefix): + _from_env_var(k) + + +if __name__ == "__main__": + app() +``` + +A user can add secrets to the GKE deployment by adding them to the 'dagster-infra' [Secrets and variables](https://github.com/JasperHG90/dagster-infra/settings/secrets/actions) GitHub settings. Secrets must be prefixed with "DAGSTER_SECRET", and will be added to the GKE 'dagster-prd' namespace as an 'Opaque' secret. The name of the kubernetes secret will be in lower-case, and underscores are replaced with hyphens.