Setup workload identity via terraform

.github/ISSUE_TEMPLATE/01_new-issue.yml

@@ -2,55 +2,31 @@ name: 💡 General Issue
 description: A general template for many kinds of issues.
 body:
   - type: textarea
-    id: description
+    id: context
     attributes:
-      label: Background and proposal
+      label: Context
       description: |
-        - **What is the problem** that needs to be solved?
-        - **What is the solution** that would resolve it?
-        - **What is the opportunity / value** in solving this problem? And for who?
-
-        _below is a suggested structure_
-      value: |
-        **Context**
-        Background information: When ___ kind of person is trying to do ___, then ___ happens.
-        Problem or opportunity:  This is a problem because ___. It would be better if ___.
-
-        **Proposed solution**
-        Action to take: We should make it possible to ___ by doing ___.
-        Value and who benefits: This would allow ___ users to ___.
+        Any background information that helps others understand this issue and why it is important.
 
     validations:
       required: true
 
   - type: textarea
-    id: implementation
+    id: proposal
     attributes:
-      label: Implementation guide and constraints
+      label: Proposal
       description: |
-
-        _Anything that will help lower the uncertainty in doing this._
-
-        - Suggestions to reduce risk and guide others who may want to implement a solution.
-        - Constraints and "out of scope" ideas that shouldn't be addressed here.
-        - Time boxes and work planning.
-      placeholder: |
-        - The best way to do this would be...
-        - This work should *not* include...
-        - We should try to do ___ in a 2-week time box before moving further.
-
+        (optional) A clear and concise description of what we should do, if we have a next step in mind.
+        Add any guidance that will lower our uncertainty in resolving this (e.g., instructions, constraints to follow, red flags to avoid).
     validations:
       required: false
 
   - type: textarea
     id: tasks
     attributes:
-      label: Updates and ongoing work
+      label: Updates and actions
       description: |
-        Provide updates as we start to plan and do work.
-        - Sub-issues and tasks to work on
-        - Links to project boards
-        - Updates over time
+        (optional) A place to track ongoing work items or tasks, as we figure them out.
 
     validations:
       required: false

.github/workflows/test-deployer-code.yaml

@@ -0,0 +1,34 @@
+name: Run tests on the deployer code
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - deployer/**
+      - tests/**
+    tags:
+      - "**"
+  pull_request:
+    branches:
+      - master
+    paths:
+      - deployer/**
+      - tests/**
+  workflow_dispatch:
+
+jobs:
+  test-deployer:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.9"
+      - name: Install dependencies
+        run: |
+          pip install -U pip
+          pip install -r requirements.txt
+      - name: Run tests
+        run: |
+          python -m pytest -vvv --color=yes

config/clusters/2i2c/catalyst-cooperative.values.yaml

@@ -1,4 +1,8 @@
 basehub:
+  userServiceAccount:
+    annotations:
+      iam.gke.io/gcp-service-account: [email protected]
+
   jupyterhub:
     singleuser:
       image:

config/clusters/2i2c/dask-staging.values.yaml

@@ -1,4 +1,7 @@
 basehub:
+  userServiceAccount:
+    annotations:
+      iam.gke.io/gcp-service-account: [email protected]
   jupyterhub:
     custom:
       cloudResources:

config/clusters/2i2c/ohw.values.yaml

@@ -1,4 +1,8 @@
 basehub:
+  userServiceAccount:
+    annotations:
+      iam.gke.io/gcp-service-account: [email protected]
+
   jupyterhub:
     prePuller:
       continuous:

config/clusters/leap/common.values.yaml

@@ -17,12 +17,6 @@ basehub:
       2i2c:
         add_staff_user_ids_to_admin_users: true
         add_staff_user_ids_of_type: "github"
-      cloudResources:
-        provider: gcp
-        gcp:
-          projectId: leap-pangeo
-        scratchBucket:
-          enabled: false
       homepage:
         templateVars:
           org:

config/clusters/leap/prod.values.yaml

@@ -1,5 +1,11 @@
 basehub:
+  userServiceAccount:
+    annotations:
+      iam.gke.io/gcp-service-account: [email protected]
   jupyterhub:
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: gcs://leap-scratch/$(JUPYTERHUB_USER)
     hub:
       config:
         GitHubOAuthenticator:

config/clusters/leap/staging.values.yaml

@@ -1,5 +1,11 @@
 basehub:
+  userServiceAccount:
+    annotations:
+      iam.gke.io/gcp-service-account: [email protected]
   jupyterhub:
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: gcs://leap-scratch-staging/$(JUPYTERHUB_USER)
     hub:
       config:
         GitHubOAuthenticator:

config/clusters/meom-ige/cluster.yaml

@@ -19,6 +19,7 @@ hubs:
       # to the helm upgrade command in, and that has meaning. Please check
       # that you intend for these files to be applied in this order.
       - common.values.yaml
+      - staging.values.yaml
   - name: prod
     display_name: "SWOT Ocean Pangeo Team (prod)"
     domain: meom-ige.2i2c.cloud
@@ -32,3 +33,4 @@ hubs:
       # to the helm upgrade command in, and that has meaning. Please check
       # that you intend for these files to be applied in this order.
       - common.values.yaml
+      - prod.values.yaml

config/clusters/meom-ige/prod.values.yaml

@@ -0,0 +1,4 @@
+basehub:
+  userServiceAccount:
+    annotations:
+      iam.gke.io/gcp-service-account: [email protected]

config/clusters/meom-ige/staging.values.yaml

@@ -0,0 +1,4 @@
+basehub:
+  userServiceAccount:
+    annotations:
+      iam.gke.io/gcp-service-account: meom-ige-staging-workload-sa@meom-ige-cnrs.iam.gserviceaccount.com

config/clusters/pangeo-hubs/prod.values.yaml

@@ -1,4 +1,7 @@
 basehub:
+  userServiceAccount:
+    annotations:
+      iam.gke.io/gcp-service-account: [email protected]
   jupyterhub:
     hub:
       config:

config/clusters/pangeo-hubs/staging.values.yaml

@@ -1,4 +1,7 @@
 basehub:
+  userServiceAccount:
+    annotations:
+      iam.gke.io/gcp-service-account: staging-user-sa@pangeo-integration-te-3eea.iam.gserviceaccount.com
   jupyterhub:
     hub:
       config:

deployer/cli.py

@@ -8,6 +8,7 @@
     deploy_support,
     deploy_grafana_dashboards,
     use_cluster_credentials,
+    generate_helm_upgrade_jobs,
 )
 from config_validation import (
     validate_cluster_config,
@@ -16,6 +17,17 @@
 )
 
 
+def _converted_string_to_list(full_str: str) -> list:
+    """
+    Take a SPACE-DELIMITED string and split it into a list.
+
+    This function is used by the generate-helm-upgrade-jobs subcommand to ensure that
+    the list os added or modified files parsed from the command line is transformed
+    into a list of strings instead of one long string with spaces between the elements
+    """
+    return full_str.split(" ")
+
+
 def main():
     argparser = argparse.ArgumentParser(
         description="""A command line tool to perform various functions related
@@ -92,6 +104,24 @@ def main():
         parents=[base_parser],
         help="Modify the current kubeconfig with the deployer's access credentials for the named cluster",
     )
+
+    # generate-helm-upgrade-jobs subcommand
+    # This subparser does not depend on the base parser.
+    generate_helm_upgrade_jobs_parser = subparsers.add_parser(
+        "generate-helm-upgrade-jobs",
+        help="Generate a set of matrix jobs to perform a helm upgrade in parallel across clusters and hubs. Emit JSON to stdout that can be read by the strategy.matrix field of a GitHub Actions workflow.",
+    )
+    generate_helm_upgrade_jobs_parser.add_argument(
+        "filepaths",
+        nargs="?",
+        type=_converted_string_to_list,
+        help="A singular or space-delimited list of added or modified filepaths in the repo",
+    )
+    generate_helm_upgrade_jobs_parser.add_argument(
+        "--pretty-print",
+        action="store_true",
+        help="Pretty print the generated matrix jobs as tables using rich",
+    )
     # === End section ===#
 
     args = argparser.parse_args()
@@ -113,3 +143,5 @@ def main():
         deploy_grafana_dashboards(args.cluster_name)
     elif args.action == "use-cluster-credentials":
         use_cluster_credentials(args.cluster_name)
+    elif args.action == "generate-helm-upgrade-jobs":
+        generate_helm_upgrade_jobs(args.filepaths, pretty_print=args.pretty_print)

deployer/config_validation.py

@@ -152,3 +152,53 @@ def validate_support_config(cluster_name):
                 sys.exit(1)
     else:
         print_colour(f"No support defined for {cluster_name}. Nothing to validate!")
+
+
+def assert_single_auth_method_enabled(cluster_name, hub_name):
+    """
+    For each hub of a specific cluster, it asserts that only a single auth
+    method is enabled. An error is raised when an authenticator
+    other than Auth0 is enabled and `auth0` is not explicitly disabled.
+    """
+    _prepare_helm_charts_dependencies_and_schemas()
+
+    config_file_path = find_absolute_path_to_cluster_file(cluster_name)
+    with open(config_file_path) as f:
+        cluster = Cluster(yaml.load(f), config_file_path.parent)
+
+    hubs = []
+    if hub_name:
+        hubs = [h for h in cluster.hubs if h.spec["name"] == hub_name]
+    else:
+        hubs = cluster.hubs
+
+    for i, hub in enumerate(hubs):
+        print_colour(
+            f"{i+1} / {len(hubs)}: Validating authenticator config for {hub.spec['name']}..."
+        )
+
+        authenticator_class = "auth0"
+        for values_file_name in hub.spec["helm_chart_values_files"]:
+            if "secret" not in os.path.basename(values_file_name):
+                values_file = config_file_path.parent.joinpath(values_file_name)
+                # Load the hub extra config from its specific values files
+                config = yaml.load(values_file)
+                # Check if there's config that specifies an authenticator class
+                try:
+                    if hub.spec["helm_chart"] != "basehub":
+                        authenticator_class = config["basehub"]["jupyterhub"]["hub"][
+                            "config"
+                        ]["JupyterHub"]["authenticator_class"]
+                    else:
+                        authenticator_class = config["jupyterhub"]["hub"]["config"][
+                            "JupyterHub"
+                        ]["authenticator_class"]
+                except KeyError:
+                    pass
+
+        # If the authenticator class is other than auth0, then raise an error
+        # if auth0 is not explicitly disabled from the cluster config
+        if authenticator_class != "auth0" and hub.spec["auth0"].get("enabled", True):
+            raise ValueError(
+                f"Please disable auth0 for {hub.spec['name']} hub before using another authenticator class!"
+            )