From 69ec676479ac7aa132b07c9d6bf685deced1bfea Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 28 Mar 2022 19:56:33 -0700 Subject: [PATCH] Move KSA creation to helm - Write docs on how to setup workload-identity - Create features/ subsection of the docs - Remove older instructions --- config/clusters/meom-ige/cluster.yaml | 2 + config/clusters/meom-ige/common.values.yaml | 3 + config/clusters/meom-ige/prod.values.yaml | 4 + config/clusters/meom-ige/staging.values.yaml | 4 + docs/howto/configure/data-access.md | 125 ---------------- docs/howto/configure/index.md | 1 - docs/howto/features/cloud-access.md | 148 +++++++++++++++++++ docs/howto/features/index.md | 5 + docs/index.md | 1 + docs/topic/features.md | 43 ++++++ helm-charts/basehub/templates/user-sa.yaml | 9 +- helm-charts/basehub/values.schema.yaml | 24 +++ helm-charts/basehub/values.yaml | 4 + terraform/gcp/buckets.tf | 2 +- terraform/gcp/projects/meom-ige.tfvars | 4 +- terraform/gcp/variables.tf | 4 +- terraform/gcp/workload-identity.tf | 26 ++-- 17 files changed, 259 insertions(+), 150 deletions(-) create mode 100644 config/clusters/meom-ige/prod.values.yaml create mode 100644 config/clusters/meom-ige/staging.values.yaml delete mode 100644 docs/howto/configure/data-access.md create mode 100644 docs/howto/features/cloud-access.md create mode 100644 docs/howto/features/index.md create mode 100644 docs/topic/features.md diff --git a/config/clusters/meom-ige/cluster.yaml b/config/clusters/meom-ige/cluster.yaml index a4da5f22d1..b835833e8e 100644 --- a/config/clusters/meom-ige/cluster.yaml +++ b/config/clusters/meom-ige/cluster.yaml @@ -19,6 +19,7 @@ hubs: # to the helm upgrade command in, and that has meaning. Please check # that you intend for these files to be applied in this order. - common.values.yaml + - staging.values.yaml - name: prod display_name: "SWOT Ocean Pangeo Team (prod)" domain: meom-ige.2i2c.cloud @@ -32,3 +33,4 @@ hubs: # to the helm upgrade command in, and that has meaning. Please check # that you intend for these files to be applied in this order. - common.values.yaml + - prod.values.yaml diff --git a/config/clusters/meom-ige/common.values.yaml b/config/clusters/meom-ige/common.values.yaml index ca3f4ce9c2..096cd5d6cf 100644 --- a/config/clusters/meom-ige/common.values.yaml +++ b/config/clusters/meom-ige/common.values.yaml @@ -1,4 +1,7 @@ basehub: + userServiceAccount: + annotations: + iam.gke.io/gcp-service-account: meom-ige-staging-workload-sa@meom-ige-cnrs.iam.gserviceaccount.com nfs: pv: # from https://docs.aws.amazon.com/efs/latest/ug/mounting-fs-nfs-mount-settings.html diff --git a/config/clusters/meom-ige/prod.values.yaml b/config/clusters/meom-ige/prod.values.yaml new file mode 100644 index 0000000000..f1e0bc13ea --- /dev/null +++ b/config/clusters/meom-ige/prod.values.yaml @@ -0,0 +1,4 @@ +basehub: + userServiceAccount: + annotations: + iam.gke.io/gcp-service-account: meom-ige-prod@meom-ige-cnrs.iam.gserviceaccount.com diff --git a/config/clusters/meom-ige/staging.values.yaml b/config/clusters/meom-ige/staging.values.yaml new file mode 100644 index 0000000000..6d80ac0bb1 --- /dev/null +++ b/config/clusters/meom-ige/staging.values.yaml @@ -0,0 +1,4 @@ +basehub: + userServiceAccount: + annotations: + iam.gke.io/gcp-service-account: meom-ige-staging-workload-sa@meom-ige-cnrs.iam.gserviceaccount.com diff --git a/docs/howto/configure/data-access.md b/docs/howto/configure/data-access.md deleted file mode 100644 index 691d2fa7b1..0000000000 --- a/docs/howto/configure/data-access.md +++ /dev/null @@ -1,125 +0,0 @@ -# Data Access - -Here we will document various ways to grant hubs access to external data. - -## Data Access via Requester Pays - -For some hubs, such as our Pangeo deployments, the communities they serve require access to data stored in other projects. -Accessing data normally comes with a charge that the folks _hosting_ the data have to take care of. -However, there is a method by which those making the request are responsible for the charges instead: [Requester Pays](https://cloud.google.com/storage/docs/requester-pays). -This section demonstrates the steps required to setup this method. - -### Setting up Requester Pays Access on GCP - -```{note} -We may automate these steps in the future. -``` - -Make sure you are logged into the `gcloud` CLI and have set the default project to be the one you wish to work with. - -```{note} -These steps should be run every time a new hub is added to a cluster, to avoid sharing of credentials. -``` - -1. Create a new Service Account - -```bash -gcloud iam service-accounts create {{ NAMESPACE }}-user-sa \ - --description="Service Account to allow access to external data stored elsewhere in the cloud" \ - --display-name="Requester Pays Service Account" -``` - -where: - -- `{{ NAMESPACE }}-user-sa` will be the name of the Service Account, and; -- `{{ NAMESPACE }}` is the name of the deployment, e.g. `staging`. - -```{note} -We create a separate service account for this so as to avoid granting excessive permissions to any single service account. -We may change this policy in the future. -``` - -2. Grant the Service Account roles on the project - -We will need to grant the [Service Usage Consumer](https://cloud.google.com/iam/docs/understanding-roles#service-usage-roles) and [Storage Object Viewer](https://cloud.google.com/iam/docs/understanding-roles#cloud-storage-roles) roles on the project to the new service account. - -```bash -gcloud projects add-iam-policy-binding \ - --role roles/serviceusage.serviceUsageConsumer \ - --member "serviceAccount:{{ NAMESPACE }}-user-sa@{{ PROJECT_ID }}.iam.gserviceaccount.com" \ - {{ PROJECT_ID }} - -gcloud projects add-iam-policy-binding \ - --role roles/storage.objectViewer \ - --member "serviceAccount:{{ NAMESPACE }}-user-sa@{{ PROJECT_ID }}.iam.gserviceaccount.com" \ - {{ PROJECT_ID }} -``` - -where: - -- `{{ PROJECT_ID }}` is the ID of the Google Cloud project, **not** the display name! -- `{{ NAMESPACE }}` is the deployment namespace - -````{note} -If you're not sure what `{{ PROJECT_ID }}` should be, you can run: - -```bash -gcloud config get-value project -``` -```` - -3. Grant the Service Account the `workloadIdentityUser` role on the cluster - -We will now grant the [Workload Identity User](https://cloud.google.com/iam/docs/understanding-roles#service-accounts-roles) role to the cluster to act on behalf of the users. - -```bash -gcloud iam service-accounts add-iam-policy-binding \ - --role roles/iam.workloadIdentityUser \ - --member "serviceAccount:{{ PROJECT_ID }}.svc.id.goog[{{ NAMESPACE }}/{{ SERVICE_ACCOUNT }}]" \ - {{ NAMESPACE }}-user-sa@{{ PROJECT_ID }}.iam.gserviceaccount.com -``` - -Where: - -- `{{ PROJECT_ID }}` is the project ID of the Google Cloud Project. - Note: this is the **ID**, not the display name! -- `{{ NAMESPACE }}` is the Kubernetes namespace/deployment to grant access to -- `{{ SERVICE_ACCOUNT }}` is the _Kubernetes_ service account to grant access to. - Usually, this is `user-sa`. - Run `kubectl --namespace {{ NAMESPACE }} get serviceaccount` if you're not sure. - -4. Link the Google Service Account to the Kubernetes Service Account - -We now link the two service accounts together so Kubernetes can use the Google API. - -```bash -kubectl annotate serviceaccount \ - --namespace {{ NAMESPACE }} \ - {{ SERVICE_ACCOUNT }} \ - iam.gke.io/gcp-service-account={{ NAMESPACE }}-user-sa@{{ PROJECT_ID }}.iam.gserviceaccount.com -``` - -Where: - -- `{{ NAMESPACE }}` is the target Kubernetes namespace -- `{{ SERVICE_ACCOUNT }}` is the target Kubernetes service account name. - Usually, this is `user-sa`. - Run `kubectl --namespace {{ NAMESPACE }} get serviceaccount` if you're not sure. -- `{{ PROJECT_ID }}` is the project ID of the Google Cloud Project. - Note: this is the **ID**, not the display name! - -5. RESTART THE HUB - -This is a very important step. -If you don't do this you won't see the changes applied. - -You can restart the hub by heading to `https://{{ hub_url }}/hub/admin` (you need to be logged in as admin), clicking the "Shutdown Hub" button, and waiting for it to come back up. - -You can now test the requester pays access by starting a server on the hub and running the below code in a script or Notebook. - -```python -from intake import open_catalog - -cat = open_catalog("https://raw.githubusercontent.com/pangeo-data/pangeo-datastore/master/intake-catalogs/ocean/altimetry.yaml") -ds = cat['j3'].to_dask() -``` diff --git a/docs/howto/configure/index.md b/docs/howto/configure/index.md index ea18ef179d..db88f65373 100644 --- a/docs/howto/configure/index.md +++ b/docs/howto/configure/index.md @@ -4,5 +4,4 @@ auth-management.md update-env.md culling.md -data-access.md ``` diff --git a/docs/howto/features/cloud-access.md b/docs/howto/features/cloud-access.md new file mode 100644 index 0000000000..768dcbc649 --- /dev/null +++ b/docs/howto/features/cloud-access.md @@ -0,0 +1,148 @@ +# Enable user access to cloud features + +Users of our hubs often need to be granted specific cloud permissions +so they can use features of the cloud provider they are on, without +having to do a bunch of cloud-provider specific setup themselves. This +helps keep code cloud provider agnostic as much as possible, while also +improving the security posture of our hubs. + +This page lists various features we offer around access to cloud resources, +and how to enable them. + +## GCP + +### How it works + +On Google Cloud Platform, we use [Workload Identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) +to map a particular [Kubernetes Service Account](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) +to a particular [Google Cloud Service Account](https://cloud.google.com/iam/docs/service-accounts). +All pods using the Kubernetes Service Account (user's jupyter notebook pods +as well as dask worker pods) +will have the permissions assigned to the Google Cloud Service Account. +This Google Cloud Service Account is managed via terraform. + +(howto:features:cloud-access:gcp:access-perms)= +### Enabling specific cloud access permissions + +1. In the `.tfvars` file for the project in which this hub is based off + create (or modify) the `hub_cloud_permissions` variable. The config is + like: + + ```terraform + hub_cloud_permissions = { + "": { + requestor_pays : true, + bucket_admin_access : ["bucket-1", "bucket-2"] + hub_namespace : "" + } + } + ``` + + where: + + 1. `` is the name of the hub, but restricted in length. This + and the cluster name together can't be more than 29 characters. `terraform` + will complain if you go over this limit, so in general just use the name + of the hub and shorten it only if `terraform` complains. + 2. `requestor_pays` enables permissions for user pods and dask worker + pods to identify as the project while making requests to Google Cloud Storage + buckets marked as 'requestor pays'. More details [here](topic:features:cloud:gcp:requestor-pays). + 3. `bucket_admin_access` lists bucket names (as specified in `user_buckets` + terraform variable) all users on this hub should have full read/write + access to. Used along with the [user_buckets](howto:features:cloud-access:gcp:storage-buckets) + terraform variable to enable the [scratch buckets](topic:features:cloud:gcp:scratch-buckets) + feature. + 3. `hub_namespace` is the full name of the hub, as hubs are put in Kubernetes + Namespaces that are the same as their names. This is explicitly specified here + because `` could possibly be truncated. + +2. Run `terraform apply -var-file=projects/.tfvars`, and look at the + plan carefully. It should only be creating or modifying IAM related objects (such as roles + and service accounts), and not really touch anything else. When it looks good, accept + the changes and apply it. This provisions a Google Cloud Service Account (if needed) + and grants it the appropriate permissions. + +3. We will need to connect the Kubernetes Service Account used by the jupyter and dask pods + with this Google Cloud Service Account. This is done by setting an annotation on the + Kubernetes Service Account. + +4. Run `terraform output kubernetes_sa_annotations`, this should + show you a list of hubs and the annotation required to be set on them: + + ``` + $ terraform output kubernetes_sa_annotations + { + "prod" = "iam.gke.io/gcp-service-account: meom-ige-prod@meom-ige-cnrs.iam.gserviceaccount.com" + "staging" = "iam.gke.io/gcp-service-account: meom-ige-staging@meom-ige-cnrs.iam.gserviceaccount.com" + } + ``` + + This shows all the annotations for all the hubs configured to provide cloud access + in this cluster. You only need to care about the hub you are currently dealing with. + +5. (If needed) create a `.values.yaml` file specific to this hub under `config/clusters/`, + and add it under `helm_chart_values_files` for the appropriate hub in `config/clusters//cluster.yaml`. + +6. Specify the annotation from step 4, nested under `userServiceAccount.annotations`. + + ```yaml + userServiceAccount: + annotations: + iam.gke.io/gcp-service-account: meom-ige-staging@meom-ige-cnrs.iam.gserviceaccount.com" + ``` + + ```{note} + If the hub is a `daskhub`, nest the config under a `basehub` key + ``` + +7. Get this change deployed, and users should now be able to use the requestor pays feature! + Currently running users might have to restart their pods for the change to take effect. + +(howto:features:cloud-access:gcp:storage-buckets)= +### Creating storage buckets for use with the hub + +See [the relevant topic page](topic:features:cloud:gcp:scratch-buckets) for +users want this! + +1. In the `.tfvars` file for the project in which this hub is based off + create (or modify) the `user_buckets` variable. The config is + like: + + ```terraform + user_buckets = ["bucket1", "bucket2"] + ``` + + Since storage buckets need to be globally unique across all of Google Cloud, + the actual created names are `-`, where `` is + set by the `prefi` variable in the `.tfvars` file + +2. Enable access to these buckets from the hub by [editing `hub_cloud_permissions`](howto:features:cloud-access:gcp:access-perms) + in the same `.tfvars` file. Follow all the steps listed there - this + should create the storage buckets and provide all users access to them! + +3. You can set the `SCRATCH_BUCKET` (and the deprecated `PANGEO_SCRATCH`) + env vars on all user pods so users can use the created bucket without + having to hard-code the bucket name in their code. In the hub-specific + `.values.yaml` file in `config/clusters//.values.yaml`, + set: + + ```yaml + jupyterhub: + singleuser: + extraEnv: + SCRATCH_BUCKET: gcs:///$(JUPYTERHUB_USER) + ``` + + ```{note} + If the hub is a `daskhub`, nest the config under a `basehub` key + ``` + + The `$(JUPYTERHUB_USER)` expands to the name of the current user for + each user, so everyone gets a little prefix inside the bucket to store + their own stuff without stepping on other people's objects. But this is + **not a security mechanism** - everyone can access everyone else's objects! + + You can also add other env vars pointing to other buckets users requested. + +4. Get this change deployed, and users should now be able to use the buckets! + Currently running users might have to restart their pods for the change to take effect. diff --git a/docs/howto/features/index.md b/docs/howto/features/index.md new file mode 100644 index 0000000000..3002e67e4d --- /dev/null +++ b/docs/howto/features/index.md @@ -0,0 +1,5 @@ +# Hub Features + +```{toctree} +cloud-access.md +``` \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 6464d6a1bd..d9a33252e6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -24,6 +24,7 @@ How-To guides answer the question 'How do I...?' for a lot of topics. :maxdepth: 2 :caption: How-to guides howto/configure/index +howto/features/index howto/customize/index howto/operate/index ``` diff --git a/docs/topic/features.md b/docs/topic/features.md new file mode 100644 index 0000000000..e04faa07eb --- /dev/null +++ b/docs/topic/features.md @@ -0,0 +1,43 @@ +# Features available on the hubs + +This document is a concise description of various features we can +optionally enable on a given JupyterHub. Explicit instructions on how to +do so should be provided in a linked how-to document. + +## Cloud Permissions + +Users of our hubs often need to be granted specific cloud permissions +so they can use features of the cloud provider they are on, without +having to do a bunch of cloud-provider specific setup themselves. This +helps keep code cloud provider agnostic as much as possible, while also +improving the security posture of our hubs. + +### GCP + +(topic:features:cloud:gcp:requestor-pays)= +#### 'Requestor Pays' access to Google Cloud Storage buckets + +By default, the organization *hosting* data on Google Cloud pays for both +storage and bandwidth costs of the data. However, Google Cloud also offers +a [requestor pays](https://cloud.google.com/storage/docs/requester-pays) +option, where the bandwidth costs are paid for by the organization *requesting* +the data. This is very commonly used by organizations that provide big datasets +on Google Cloud storage, to sustainably share costs of maintaining the data. + +When this feature is enabled, users on a hub accessing cloud buckets from +other organizations marked as 'requestor pays' will increase our cloud bill. +Hence, this is an opt-in feature. + +(topic:features:cloud:gcp:scratch-buckets)= +#### 'Scratch' Buckets on Google Cloud Storage + +Users often want one or more Google Cloud Storage [buckets](https://cloud.google.com/storage/docs/json_api/v1/buckets) +to store intermediate results, share big files with other users, or +to store raw data that should be accessible to everyone within the hub. +We can create one more more buckets and provide *all* users on the hub +*equal* access to these buckets, allowing users to create objects in them. +A single bucket can also be designated as as *scratch bucket*, which will +set a `SCRATCH_BUCKET` (and a deprecated `PANGEO_SCRATCH`) environment variable +of the form `gcs:///`. This can be used by individual +users to store objects temporarily for their own use, although there is nothing +preventing other users from accessing these objects! \ No newline at end of file diff --git a/helm-charts/basehub/templates/user-sa.yaml b/helm-charts/basehub/templates/user-sa.yaml index 7b3bd83f9b..21255905a7 100644 --- a/helm-charts/basehub/templates/user-sa.yaml +++ b/helm-charts/basehub/templates/user-sa.yaml @@ -1,10 +1,7 @@ +{{ if .Values.userServiceAccount.enabled -}} apiVersion: v1 kind: ServiceAccount metadata: - annotations: - {{- if .Values.jupyterhub.custom.cloudResources.scratchBucket.enabled }} - {{- if eq .Values.jupyterhub.custom.cloudResources.provider "gcp" }} - iam.gke.io/gcp-service-account: {{ include "cloudResources.gcp.serviceAccountName" .}}@{{ .Values.jupyterhub.custom.cloudResources.gcp.projectId }}.iam.gserviceaccount.com - {{- end }} - {{- end }} + annotations: {{ .Values.userServiceAccount.annotations | toJson}} name: user-sa +{{- end }} \ No newline at end of file diff --git a/helm-charts/basehub/values.schema.yaml b/helm-charts/basehub/values.schema.yaml index 4c174fe32c..987fb29e18 100644 --- a/helm-charts/basehub/values.schema.yaml +++ b/helm-charts/basehub/values.schema.yaml @@ -16,7 +16,31 @@ required: - inClusterNFS - global - jupyterhub + - userServiceAccount properties: + userServiceAccount: + type: object + additionalProperties: false + required: + - enabled + properties: + enabled: + type: boolean + description: | + Enables creation of a Service Account for use by notebook & dask pods. + + Config must still be set for notebook and dask pods to actually use + this service account, which is named user-sa. + annotations: + type: object + additionalProperties: true + description: | + Dictionary of annotations that can be applied to the service account. + + When used with GKE and Workload Identity, you need to set + the annotation with key "iam.gke.io/gcp-service-account" to the + email address of the Google Service Account whose credentials it + should have. azureFile: type: object additionalProperties: false diff --git a/helm-charts/basehub/values.yaml b/helm-charts/basehub/values.yaml index f24b1db842..efb516bf4a 100644 --- a/helm-charts/basehub/values.yaml +++ b/helm-charts/basehub/values.yaml @@ -1,3 +1,7 @@ +userServiceAccount: + enabled: true + annotations: {} + azureFile: enabled: false pv: diff --git a/terraform/gcp/buckets.tf b/terraform/gcp/buckets.tf index d34f1e2b90..efb84d2950 100644 --- a/terraform/gcp/buckets.tf +++ b/terraform/gcp/buckets.tf @@ -16,7 +16,7 @@ locals { # Nested for loop, thanks to https://www.daveperrett.com/articles/2021/08/19/nested-for-each-with-terraform/ bucket_permissions = distinct(flatten([ for hub_name, permissions in var.hub_cloud_permissions : [ - for bucket_name in permissions.bucket_admin : { + for bucket_name in permissions.bucket_admin_access : { hub_name = hub_name bucket_name = bucket_name } diff --git a/terraform/gcp/projects/meom-ige.tfvars b/terraform/gcp/projects/meom-ige.tfvars index 0b6b65e951..76e23e049f 100644 --- a/terraform/gcp/projects/meom-ige.tfvars +++ b/terraform/gcp/projects/meom-ige.tfvars @@ -92,12 +92,12 @@ user_buckets = [ hub_cloud_permissions = { "staging" : { requestor_pays : true, - bucket_admin: ["scratch", "data"], + bucket_admin_access: ["scratch", "data"], hub_namespace: "staging" }, "prod" : { requestor_pays : true, - bucket_admin: ["scratch", "data"], + bucket_admin_access: ["scratch", "data"], hub_namespace: "prod" } } \ No newline at end of file diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf index f36678a518..a26ce64f95 100644 --- a/terraform/gcp/variables.tf +++ b/terraform/gcp/variables.tf @@ -257,7 +257,7 @@ variable "max_cpu" { } variable "hub_cloud_permissions" { - type = map(object({ requestor_pays : bool, bucket_admin : set(string), hub_namespace : string })) + type = map(object({ requestor_pays : bool, bucket_admin_access : set(string), hub_namespace : string })) default = {} description = <<-EOT Map of cloud permissions given to a particular hub @@ -268,7 +268,7 @@ variable "hub_cloud_permissions" { 1. requestor_pays: Identify as coming from the google cloud project when accessing storage buckets marked as https://cloud.google.com/storage/docs/requester-pays. This *potentially* incurs cost for us, the originating project, so opt-in. - 2. bucket_admin: List of GCS storage buckets that users on this hub should have read + 2. bucket_admin_access: List of GCS storage buckets that users on this hub should have read and write permissions for. EOT } diff --git a/terraform/gcp/workload-identity.tf b/terraform/gcp/workload-identity.tf index 6154b2037a..b898cfa32d 100644 --- a/terraform/gcp/workload-identity.tf +++ b/terraform/gcp/workload-identity.tf @@ -14,7 +14,8 @@ # Create the service account if there is an entry for the hub, regardless of what # kind of permissions it wants. resource "google_service_account" "workload_sa" { - for_each = var.hub_cloud_permissions + for_each = var.hub_cloud_permissions + # Service account IDs are limited to 30 chars, so use key not hub namespace account_id = "${var.prefix}-${each.key}" display_name = "Service account for user pods in hub ${each.key} in ${var.prefix}" project = var.project_id @@ -52,17 +53,16 @@ resource "google_project_iam_member" "requestor_pays_binding" { member = "serviceAccount:${google_service_account.workload_sa[each.value].email}" } -# Create the Service Account in the Kubernetes Namespace -# FIXME: We might need to create the k8s namespace here some of the time, but then who is -# responsible for that - terraform or helm (via our deployer?) -resource "kubernetes_service_account" "workload_kubernetes_sa" { - for_each = var.hub_cloud_permissions +output "kubernetes_sa_annotations" { + value = { for k, v in var.hub_cloud_permissions : v.hub_namespace => "iam.gke.io/gcp-service-account: ${google_service_account.workload_sa[k].email}" } + description = <<-EOT + Annotations to apply to userServiceAccount in each hub to enable cloud permissions for them. - metadata { - name = "user-sa" - namespace = each.value.hub_namespace - annotations = { - "iam.gke.io/gcp-service-account" = google_service_account.workload_sa[each.key].email - } - } + Helm, not terraform, control namespace creation for us. This makes it quite difficult + to create the appropriate kubernetes service account attached to the Google Cloud Service + Account in the appropriate namespace. Instead, this output provides the list of annotations + to be applied to the kubernetes service account used by jupyter and dask pods in a given hub. + This should be specified under userServiceAccount.annotations (or basehub.userServiceAccount.annotations + in case of daskhub) on a values file created specifically for that hub. + EOT }