From 040d3732d6aaed83c541dce5780b71fdbd8d4b2a Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 6 Dec 2024 13:55:21 +0000 Subject: [PATCH 1/5] Remove feature to assign difference service accounts based on admin status --- .../clusters/opensci/sciencecore.values.yaml | 6 ----- .../templates/serviceaccount-admin.yaml | 7 ------ helm-charts/basehub/values.schema.yaml | 24 ------------------- helm-charts/basehub/values.yaml | 4 ---- terraform/aws/bucket-access.tf | 8 ------- terraform/aws/irsa.tf | 2 +- terraform/aws/projects/opensci.tfvars | 3 --- terraform/aws/variables.tf | 5 +--- 8 files changed, 2 insertions(+), 57 deletions(-) delete mode 100644 helm-charts/basehub/templates/serviceaccount-admin.yaml diff --git a/config/clusters/opensci/sciencecore.values.yaml b/config/clusters/opensci/sciencecore.values.yaml index ce67dcd42a..e972ecc0ed 100644 --- a/config/clusters/opensci/sciencecore.values.yaml +++ b/config/clusters/opensci/sciencecore.values.yaml @@ -2,10 +2,6 @@ userServiceAccount: enabled: true annotations: eks.amazonaws.com/role-arn: arn:aws:iam::211125293633:role/opensci-sciencecore -adminServiceAccount: - enabled: true - annotations: - eks.amazonaws.com/role-arn: arn:aws:iam::211125293633:role/opensci-sciencecore-admin-sa jupyterhub: ingress: @@ -36,8 +32,6 @@ jupyterhub: funded_by: name: "" url: "" - singleuserAdmin: - serviceAccountName: admin-sa singleuser: nodeSelector: 2i2c/hub-name: sciencecore diff --git a/helm-charts/basehub/templates/serviceaccount-admin.yaml b/helm-charts/basehub/templates/serviceaccount-admin.yaml deleted file mode 100644 index c189b57e61..0000000000 --- a/helm-charts/basehub/templates/serviceaccount-admin.yaml +++ /dev/null @@ -1,7 +0,0 @@ -{{ if .Values.adminServiceAccount.enabled -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: admin-sa - annotations: {{ .Values.adminServiceAccount.annotations | toJson }} -{{- end }} diff --git a/helm-charts/basehub/values.schema.yaml b/helm-charts/basehub/values.schema.yaml index b654e8a9a8..67810df18a 100644 --- a/helm-charts/basehub/values.schema.yaml +++ b/helm-charts/basehub/values.schema.yaml @@ -16,7 +16,6 @@ required: - global - jupyterhub - userServiceAccount - - adminServiceAccount - dex - staticWebsite - ingressBasicAuth @@ -150,27 +149,6 @@ properties: description: | Dictionary of annotations that can be applied to the service account. - When used with GKE and Workload Identity, you need to set - the annotation with key "iam.gke.io/gcp-service-account" to the - email address of the Google Service Account whose credentials it - should have. - adminServiceAccount: - type: object - additionalProperties: false - required: - - enabled - properties: - enabled: - type: boolean - description: | - Enables creation of a Service Account named admin-sa for opt-in use - via jupyterhub.custom.singleuserAdmin.serviceAccountName. - annotations: - type: object - additionalProperties: true - description: | - Dictionary of annotations that can be applied to the service account. - When used with GKE and Workload Identity, you need to set the annotation with key "iam.gke.io/gcp-service-account" to the email address of the Google Service Account whose credentials it @@ -526,8 +504,6 @@ properties: additionalProperties: true extraEnv: type: object - serviceAccountName: - type: string 2i2c: type: object additionalProperties: false diff --git a/helm-charts/basehub/values.yaml b/helm-charts/basehub/values.yaml index 1afe453309..314a20cddd 100644 --- a/helm-charts/basehub/values.yaml +++ b/helm-charts/basehub/values.yaml @@ -5,10 +5,6 @@ userServiceAccount: enabled: true annotations: {} -adminServiceAccount: - enabled: false - annotations: {} - binderhub-service: enabled: false ingress: diff --git a/terraform/aws/bucket-access.tf b/terraform/aws/bucket-access.tf index f50ce5ae6c..707cc339d3 100644 --- a/terraform/aws/bucket-access.tf +++ b/terraform/aws/bucket-access.tf @@ -23,8 +23,6 @@ locals { user-sa: bucket_admin_access: [scratch-sciencecore] bucket_readonly_access: [persistent-sciencecore] - admin-sa: - bucket_admin_access: [scratch-sciencecore, persistent-sciencecore] Then, the `local.bucket_role_actions` will look like below, with one list item for each element in all `bucket_admin/readonly_access` lists: @@ -36,9 +34,6 @@ locals { - bucket: scratch-sciencecore role: sciencecore actions: ["s3:*"] - - bucket: scratch-sciencecore - role: sciencecore-admin-sa - actions: ["s3:*"] - bucket: persistent-sciencecore role: sciencecore actions: ["s3:ListBucket", "s3:GetObject", "s3:GetObjectVersion"] @@ -94,9 +89,6 @@ locals { - bucket: scratch-sciencecore role: sciencecore actions: ["s3:*"] - - bucket: scratch-sciencecore - role: sciencecore-admin-sa - actions: ["s3:*"] persistent-sciencecore: - bucket: persistent-sciencecore role: sciencecore diff --git a/terraform/aws/irsa.tf b/terraform/aws/irsa.tf index 986e9f7315..0cd36e86a5 100644 --- a/terraform/aws/irsa.tf +++ b/terraform/aws/irsa.tf @@ -2,7 +2,7 @@ This file provides resources _per hub and role_. Each role is tied to a specific k8s ServiceAccount allowed to assume the role. - - Role - for use by k8s ServiceAccount (user-sa, admin-sa) + - Role - for use by k8s ServiceAccount (user-sa) - Policy - if extra_iam_policy is declared - RolePolicyAttachment - if extra_iam_policy is declared */ diff --git a/terraform/aws/projects/opensci.tfvars b/terraform/aws/projects/opensci.tfvars index b69eefbce2..4d90a5e25c 100644 --- a/terraform/aws/projects/opensci.tfvars +++ b/terraform/aws/projects/opensci.tfvars @@ -31,8 +31,5 @@ hub_cloud_permissions = { bucket_admin_access : ["scratch-sciencecore"], bucket_readonly_access : ["persistent-sciencecore"], }, - "admin-sa" : { - bucket_admin_access : ["scratch-sciencecore", "persistent-sciencecore"], - }, }, } diff --git a/terraform/aws/variables.tf b/terraform/aws/variables.tf index 6411b74e17..a5746033ce 100644 --- a/terraform/aws/variables.tf +++ b/terraform/aws/variables.tf @@ -68,8 +68,7 @@ variable "hub_cloud_permissions" { Accounts attached to it, for different kinds of users. The key is the name of the Kubernetes Service Account. By convention, basehub currently only supports creation of Kubernetes Service Accounts `user-sa` (for non-admin - users on the hub) and `admin-sa` (for admin users on the hub). The value - can be one of: + users on the hub). The value can be one of: 1. bucket_admin_access: List of S3 storage buckets to grant full read & write permissions to. @@ -78,8 +77,6 @@ variable "hub_cloud_permissions" { 3. extra_iam_policy: An AWS IAM Policy document that grants additional rights to this Kubernetes Service Account. - Note that these are independent of each other - so if you want both admins - and non-admins to have a set of permissions, you may need to repeat them. EOT } From 24c3a03bbe28cf04ae90cf62e80223f3006f8b3d Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 6 Dec 2024 14:35:49 +0000 Subject: [PATCH 2/5] Remove ability to create different Service Accounts per hub --- terraform/aws/bucket-access.tf | 145 --------------------------------- terraform/aws/buckets.tf | 42 ++++++++++ terraform/aws/irsa.tf | 76 ++++++++--------- terraform/aws/variables.tf | 21 ++--- 4 files changed, 84 insertions(+), 200 deletions(-) delete mode 100644 terraform/aws/bucket-access.tf diff --git a/terraform/aws/bucket-access.tf b/terraform/aws/bucket-access.tf deleted file mode 100644 index 707cc339d3..0000000000 --- a/terraform/aws/bucket-access.tf +++ /dev/null @@ -1,145 +0,0 @@ -/* - Creates one aws_s3_bucket_policy per bucket - there can't be more than one as - they otherwise replace each other when applied. - - The bucket policies grant bucket specific permissions to specific IAM Roles - based on them having `bucket_admin_access` or `bucket_readonly_access` - referencing the bucket via `var.hub_cloud_permissions`. -*/ - -locals { - /* - The bucket_role_actions local variable defined below is a list of objects - generated from `var.hub_cloud_permissions` roles and their respective - bucket_admin_access and bucket_readonly_access lists. - - If for example `var.hub_cloud_permissions` is: - - hub_cloud_permissions: - staging: - user-sa: - bucket_admin_access: [scratch-staging] - sciencecore: - user-sa: - bucket_admin_access: [scratch-sciencecore] - bucket_readonly_access: [persistent-sciencecore] - - Then, the `local.bucket_role_actions` will look like below, with one list - item for each element in all `bucket_admin/readonly_access` lists: - - bucket_role_actions: - - bucket: scratch-staging - role: staging - actions: ["s3:*"] - - bucket: scratch-sciencecore - role: sciencecore - actions: ["s3:*"] - - bucket: persistent-sciencecore - role: sciencecore - actions: ["s3:ListBucket", "s3:GetObject", "s3:GetObjectVersion"] - - bucket: persistent-sciencecore - role: sciencecore - actions: ["s3:*"] - */ - bucket_role_actions = flatten([ - for hub, hub_value in var.hub_cloud_permissions : [ - for role, role_value in hub_value : flatten([ - [ - for bucket in role_value.bucket_admin_access : { - bucket = bucket - // role should match the id set in irsa.tf - role = role == "user-sa" ? hub : "${hub}-${role}" - actions = ["s3:*"] - } - ], - [ - for bucket in role_value.bucket_readonly_access : { - bucket = bucket - // role should match the id set in irsa.tf - role = role == "user-sa" ? hub : "${hub}-${role}" - actions = [ - "s3:ListBucket", - "s3:GetObject", - "s3:GetObjectVersion", - ] - } - ], - ]) - ] - ]) -} - -locals { - /* - The `local.bucket_role_actions_lists` variable defined below is reprocessing - `local.bucket_role_actions` to a dictionary with one key per bucket with - associated permissions. - - Below is an example value `local.bucket_role_actions_lists` could take. This - example value is aligned with the example value for - `var.hub_cloud_permissions` and `local.bucket_role_actions` in the code - block above. - - bucket_role_actions_lists: - scratch-staging: - - bucket: scratch-staging - role: staging - actions: ["s3:*"] - scratch-sciencecore: - - bucket: scratch-sciencecore - role: sciencecore - actions: ["s3:*"] - persistent-sciencecore: - - bucket: persistent-sciencecore - role: sciencecore - actions: ["s3:ListBucket", "s3:GetObject", "s3:GetObjectVersion"] - - bucket: persistent-sciencecore - role: sciencecore - actions: ["s3:*"] - */ - bucket_role_actions_lists = { - for bucket, _ in var.user_buckets : - bucket => [for bra in local.bucket_role_actions : bra if bra.bucket == bucket] - // Filter out user_buckets not mentioned in hub_cloud_permissions - if length([for bra in local.bucket_role_actions : bra if bra.bucket == bucket]) != 0 - } -} - - - -# ref: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document -data "aws_iam_policy_document" "bucket_policy" { - for_each = local.bucket_role_actions_lists - - # Only one policy document can be declared per bucket, so we provide multiple - # "statement" in this policy. - dynamic "statement" { - for_each = { for index, bra in each.value : "${bra.bucket}.${bra.role}" => bra } - - content { - effect = "Allow" - actions = statement.value.actions - principals { - type = "AWS" - identifiers = [ - aws_iam_role.irsa_role[statement.value.role].arn - ] - } - resources = [ - # Grant access only to the bucket and its contents - aws_s3_bucket.user_buckets[statement.value.bucket].arn, - "${aws_s3_bucket.user_buckets[statement.value.bucket].arn}/*", - ] - } - } -} - -# There can only be one of these per bucket, if more are defined they will end -# up replacing each other without terraform indicating there is trouble. -# -# ref: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_policy -resource "aws_s3_bucket_policy" "user_bucket_access" { - for_each = local.bucket_role_actions_lists - bucket = aws_s3_bucket.user_buckets[each.key].id - policy = data.aws_iam_policy_document.bucket_policy[each.key].json -} diff --git a/terraform/aws/buckets.tf b/terraform/aws/buckets.tf index 8734245074..99425bb5d6 100644 --- a/terraform/aws/buckets.tf +++ b/terraform/aws/buckets.tf @@ -38,6 +38,48 @@ resource "aws_s3_bucket_lifecycle_configuration" "user_bucket_expiry" { } } +locals { + # Nested for loop, thanks to https://www.daveperrett.com/articles/2021/08/19/nested-for-each-with-terraform/ + bucket_permissions = distinct(flatten([ + for hub_name, permissions in var.hub_cloud_permissions : [ + for bucket_name in permissions.bucket_admin_access : { + hub_name = hub_name + bucket_name = bucket_name + } + ] + ])) +} + +# ref: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document +data "aws_iam_policy_document" "bucket_access" { + for_each = { for bp in local.bucket_permissions : "${bp.hub_name}.${bp.bucket_name}" => bp } + statement { + effect = "Allow" + actions = ["s3:*"] + principals { + type = "AWS" + identifiers = [ + aws_iam_role.irsa_role[each.value.hub_name].arn + ] + } + resources = [ + # Grant access only to the bucket and its contents + aws_s3_bucket.user_buckets[each.value.bucket_name].arn, + "${aws_s3_bucket.user_buckets[each.value.bucket_name].arn}/*" + ] + } +} + +# There can only be one of these per bucket, if more are defined they will end +# up replacing each other without terraform indicating there is trouble. +# +# ref: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_policy +resource "aws_s3_bucket_policy" "user_bucket_access" { + for_each = { for bp in local.bucket_permissions : "${bp.hub_name}.${bp.bucket_name}" => bp } + bucket = aws_s3_bucket.user_buckets[each.value.bucket_name].id + policy = data.aws_iam_policy_document.bucket_access[each.key].json +} + output "buckets" { value = { for b, _ in var.user_buckets : b => aws_s3_bucket.user_buckets[b].id } description = <<-EOT diff --git a/terraform/aws/irsa.tf b/terraform/aws/irsa.tf index 0cd36e86a5..ef0fc5462f 100644 --- a/terraform/aws/irsa.tf +++ b/terraform/aws/irsa.tf @@ -1,43 +1,37 @@ -/* - This file provides resources _per hub and role_. Each role is tied to a - specific k8s ServiceAccount allowed to assume the role. - - - Role - for use by k8s ServiceAccount (user-sa) - - Policy - if extra_iam_policy is declared - - RolePolicyAttachment - if extra_iam_policy is declared -*/ - # ref: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity data "aws_caller_identity" "current" {} # ref: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition data "aws_partition" "current" {} +# locals { +# # Nested for loop, thanks to https://www.daveperrett.com/articles/2021/08/19/nested-for-each-with-terraform/ +# hub_to_role_mapping = flatten([ +# for hub, hub_value in var.hub_cloud_permissions : [ +# for ksa_name, cloud_permissions in hub_value : { +# // Most hubs only use `user-sa`, so we use just the hub name for the IAM +# // role for user-sa. `user-sa` was also the only service account supported +# // for a long time, so this special casing reduces the amount of work +# // we needed to do to introduce other service accounts. +# iam_role_name = ksa_name == "user-sa" ? hub : "${hub}-${ksa_name}" +# hub = hub +# ksa_name = ksa_name +# cloud_permissions = cloud_permissions +# } +# ] +# ]) +# } +resource "aws_iam_role" "irsa_role" { + for_each = var.hub_cloud_permissions + name = "${var.cluster_name}-${each.key}" -locals { - # Nested for loop, thanks to https://www.daveperrett.com/articles/2021/08/19/nested-for-each-with-terraform/ - hub_to_role_mapping = flatten([ - for hub, hub_value in var.hub_cloud_permissions : [ - for ksa_name, cloud_permissions in hub_value : { - // Most hubs only use `user-sa`, so we use just the hub name for the IAM - // role for user-sa. `user-sa` was also the only service account supported - // for a long time, so this special casing reduces the amount of work - // we needed to do to introduce other service accounts. - iam_role_name = ksa_name == "user-sa" ? hub : "${hub}-${ksa_name}" - hub = hub - ksa_name = ksa_name - cloud_permissions = cloud_permissions - } - ] - ]) + assume_role_policy = data.aws_iam_policy_document.irsa_role_assume[each.key].json } - - # ref: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document data "aws_iam_policy_document" "irsa_role_assume" { - for_each = { for index, hr in local.hub_to_role_mapping : hr.iam_role_name => hr } + for_each = var.hub_cloud_permissions statement { effect = "Allow" actions = ["sts:AssumeRoleWithWebIdentity"] @@ -53,42 +47,40 @@ data "aws_iam_policy_document" "irsa_role_assume" { test = "StringEquals" variable = "${replace(data.aws_eks_cluster.cluster.identity[0].oidc[0].issuer, "https://", "")}:sub" values = [ - "system:serviceaccount:${each.value.hub}:${each.value.ksa_name}" + "system:serviceaccount:${each.key}:user-sa" ] } } } -# ref: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role -resource "aws_iam_role" "irsa_role" { - for_each = { for index, hr in local.hub_to_role_mapping : hr.iam_role_name => hr } - name = "${var.cluster_name}-${each.key}" +# # ref: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role +# resource "aws_iam_role" "irsa_role" { +# for_each = { for index, hr in local.hub_to_role_mapping : hr.iam_role_name => hr } +# name = "${var.cluster_name}-${each.key}" - assume_role_policy = data.aws_iam_policy_document.irsa_role_assume[each.key].json -} +# assume_role_policy = data.aws_iam_policy_document.irsa_role_assume[each.key].json +# } # ref: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy resource "aws_iam_policy" "extra_user_policy" { - for_each = { for index, hr in local.hub_to_role_mapping : hr.iam_role_name => hr if hr.cloud_permissions.extra_iam_policy != "" } + for_each = { for hub_name, value in var.hub_cloud_permissions : hub_name => value if value.extra_iam_policy != "" } name = "${var.cluster_name}-${each.key}-extra-user-policy" description = "Extra permissions granted to users on hub ${each.key} on ${var.cluster_name}" - policy = each.value.cloud_permissions.extra_iam_policy + policy = each.value.extra_iam_policy } # ref: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment resource "aws_iam_role_policy_attachment" "extra_user_policy" { - for_each = { for index, hr in local.hub_to_role_mapping : hr.iam_role_name => hr if hr.cloud_permissions.extra_iam_policy != "" } + for_each = { for hub_name, value in var.hub_cloud_permissions : hub_name => value if value.extra_iam_policy != "" } role = aws_iam_role.irsa_role[each.key].name policy_arn = aws_iam_policy.extra_user_policy[each.key].arn } - - output "kubernetes_sa_annotations" { value = { - for index, hr in local.hub_to_role_mapping : - hr.iam_role_name => "eks.amazonaws.com/role-arn: ${aws_iam_role.irsa_role[hr.iam_role_name].arn}" + for k, v in var.hub_cloud_permissions : + k => "eks.amazonaws.com/role-arn: ${aws_iam_role.irsa_role[k].arn}" } description = <<-EOT Annotations to apply to userServiceAccount in each hub to enable cloud permissions for them. diff --git a/terraform/aws/variables.tf b/terraform/aws/variables.tf index a5746033ce..90954e5efe 100644 --- a/terraform/aws/variables.tf +++ b/terraform/aws/variables.tf @@ -48,27 +48,22 @@ variable "user_buckets" { variable "hub_cloud_permissions" { type = map( - map( - object({ - bucket_admin_access : optional(set(string), []) - bucket_readonly_access : optional(set(string), []) - extra_iam_policy : optional(string, "") - }) - ) + object({ + bucket_admin_access : optional(set(string), []) + bucket_readonly_access : optional(set(string), []) + extra_iam_policy : optional(string, "") + }) ) default = {} description = <<-EOT - Cloud permissions attached to Kubernetes Service Accounts in a particular + Cloud permissions attached to a Kubernetes Service Account in a particular k8s namespace (hub) in this cluster. The key is a Kubernetes namespace, which by convention in 2i2c clusters is also the name of the hub. - The value is itself a map, as each hub can have multiple Kubernetes Service - Accounts attached to it, for different kinds of users. The key is the name - of the Kubernetes Service Account. By convention, basehub currently only - supports creation of Kubernetes Service Accounts `user-sa` (for non-admin - users on the hub). The value can be one of: + The value is itself a map describing particular permissions users running on + those hubs should have. Currently supported are: 1. bucket_admin_access: List of S3 storage buckets to grant full read & write permissions to. From 4f400a25e35517c6284e668fd12370c221432a89 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 6 Dec 2024 14:48:52 +0000 Subject: [PATCH 3/5] Update terraform template file --- terraform/aws/projects/template.tfvars | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/terraform/aws/projects/template.tfvars b/terraform/aws/projects/template.tfvars index 6f12f067b5..3bd7d48acf 100644 --- a/terraform/aws/projects/template.tfvars +++ b/terraform/aws/projects/template.tfvars @@ -35,9 +35,7 @@ filestores = { # hub_cloud_permissions = { {% for hub in hubs %} # "{{ hub }}" : { -# "user-sa" : { -# bucket_admin_access : ["scratch-{{ hub }}"], -# }, +# bucket_admin_access : ["scratch-{{ hub }}"], # }, {% endfor %} From 49de060d364b5efd6fb1cd65d5166c0b9d1d9c69 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 6 Dec 2024 16:49:30 +0000 Subject: [PATCH 4/5] Update cluster config to remove user-sa key, no longer required --- terraform/aws/projects/2i2c-aws-us.tfvars | 28 +- .../projects/catalystproject-africa.tfvars | 14 +- terraform/aws/projects/earthscope.tfvars | 12 +- terraform/aws/projects/gridsst.tfvars | 10 +- terraform/aws/projects/jupyter-health.tfvars | 10 +- .../projects/jupyter-meets-the-earth.tfvars | 66 ++-- terraform/aws/projects/kitware.tfvars | 9 +- terraform/aws/projects/maap.tfvars | 9 +- terraform/aws/projects/nasa-cryo.tfvars | 136 ++++--- terraform/aws/projects/nasa-ghg.tfvars | 210 +++++----- terraform/aws/projects/nasa-veda.tfvars | 363 +++++++++--------- terraform/aws/projects/nmfs-openscapes.tfvars | 8 +- terraform/aws/projects/openscapes.tfvars | 32 +- terraform/aws/projects/opensci.tfvars | 12 +- terraform/aws/projects/projectpythia.tfvars | 5 +- terraform/aws/projects/smithsonian.tfvars | 9 +- terraform/aws/projects/strudel.tfvars | 1 + terraform/aws/projects/ubc-eoas.tfvars | 9 +- terraform/aws/projects/victor.tfvars | 10 +- 19 files changed, 438 insertions(+), 515 deletions(-) diff --git a/terraform/aws/projects/2i2c-aws-us.tfvars b/terraform/aws/projects/2i2c-aws-us.tfvars index 12a809ee95..748134d635 100644 --- a/terraform/aws/projects/2i2c-aws-us.tfvars +++ b/terraform/aws/projects/2i2c-aws-us.tfvars @@ -3,6 +3,7 @@ cluster_name = "2i2c-aws-us" cluster_nodes_location = "us-west-2a" enable_aws_ce_grafana_backend_iam = true +disable_cluster_wide_filestore = false user_buckets = { "scratch-staging" : { @@ -31,34 +32,23 @@ user_buckets = { }, } - hub_cloud_permissions = { "staging" : { - "user-sa" : { - bucket_admin_access : ["scratch-staging"], - }, + bucket_admin_access : ["scratch-staging"], }, "dask-staging" : { - "user-sa" : { - bucket_admin_access : ["scratch-dask-staging"], - }, + bucket_admin_access : ["scratch-dask-staging"], }, "showcase" : { - "user-sa" : { - bucket_admin_access : [ - "scratch-showcase", - "persistent-showcase", - ], - }, + bucket_admin_access : [ + "scratch-showcase", + "persistent-showcase", + ], }, "ncar-cisl" : { - "user-sa" : { - bucket_admin_access : ["scratch-ncar-cisl"], - }, + bucket_admin_access : ["scratch-ncar-cisl"], }, "itcoocean" : { - "user-sa" : { - bucket_admin_access : ["scratch-itcoocean"], - }, + bucket_admin_access : ["scratch-itcoocean"], }, } diff --git a/terraform/aws/projects/catalystproject-africa.tfvars b/terraform/aws/projects/catalystproject-africa.tfvars index 28df57bade..d69bd2efdc 100644 --- a/terraform/aws/projects/catalystproject-africa.tfvars +++ b/terraform/aws/projects/catalystproject-africa.tfvars @@ -3,6 +3,7 @@ cluster_name = "catalystproject-africa" cluster_nodes_location = "af-south-1a" enable_aws_ce_grafana_backend_iam = true +disable_cluster_wide_filestore = false user_buckets = { "scratch-staging" : { @@ -19,21 +20,14 @@ user_buckets = { }, } - hub_cloud_permissions = { "staging" : { - "user-sa" : { - bucket_admin_access : ["scratch-staging"], - }, + bucket_admin_access : ["scratch-staging"], }, "prod" : { - "user-sa" : { - bucket_admin_access : ["scratch"], - }, + bucket_admin_access : ["scratch"], }, "bhki" : { - "user-sa" : { - bucket_admin_access : ["persistent-bhki"], - }, + bucket_admin_access : ["persistent-bhki"], }, } diff --git a/terraform/aws/projects/earthscope.tfvars b/terraform/aws/projects/earthscope.tfvars index 54c8cd5e44..f6eadf1060 100644 --- a/terraform/aws/projects/earthscope.tfvars +++ b/terraform/aws/projects/earthscope.tfvars @@ -3,7 +3,7 @@ cluster_name = "earthscope" cluster_nodes_location = "us-east-2a" default_tags = { - "2i2c.org/cluster-name" : "{var_cluster_name}", + "2i2c.org/cluster-name" : "earthscope", "ManagedBy" : "2i2c", # Requested by the community in https://2i2c.freshdesk.com/a/tickets/1460 "earthscope:application:name" : "geolab", @@ -15,6 +15,7 @@ default_budget_alert = { } enable_aws_ce_grafana_backend_iam = true +disable_cluster_wide_filestore = false user_buckets = { "scratch-staging" : { @@ -27,16 +28,11 @@ user_buckets = { }, } - hub_cloud_permissions = { "staging" : { - "user-sa" : { - bucket_admin_access : ["scratch-staging"], - }, + bucket_admin_access : ["scratch-staging"], }, "prod" : { - "user-sa" : { - bucket_admin_access : ["scratch"], - }, + bucket_admin_access : ["scratch"], }, } diff --git a/terraform/aws/projects/gridsst.tfvars b/terraform/aws/projects/gridsst.tfvars index f655343706..e3303e0af2 100644 --- a/terraform/aws/projects/gridsst.tfvars +++ b/terraform/aws/projects/gridsst.tfvars @@ -3,6 +3,7 @@ cluster_name = "gridsst" cluster_nodes_location = "us-west-2a" enable_aws_ce_grafana_backend_iam = true +disable_cluster_wide_filestore = false user_buckets = { "scratch-staging" : { @@ -15,16 +16,11 @@ user_buckets = { }, } - hub_cloud_permissions = { "staging" : { - "user-sa" : { - bucket_admin_access : ["scratch-staging"], - }, + bucket_admin_access : ["scratch-staging"], }, "prod" : { - "user-sa" : { - bucket_admin_access : ["scratch"], - }, + bucket_admin_access : ["scratch"], }, } diff --git a/terraform/aws/projects/jupyter-health.tfvars b/terraform/aws/projects/jupyter-health.tfvars index 851ae36211..0544133cc4 100644 --- a/terraform/aws/projects/jupyter-health.tfvars +++ b/terraform/aws/projects/jupyter-health.tfvars @@ -3,6 +3,7 @@ cluster_name = "jupyter-health" cluster_nodes_location = "us-east-2a" enable_aws_ce_grafana_backend_iam = true +disable_cluster_wide_filestore = false user_buckets = { "scratch-staging" : { @@ -15,16 +16,11 @@ user_buckets = { }, } - hub_cloud_permissions = { "staging" : { - "user-sa" : { - bucket_admin_access : ["scratch-staging"], - }, + bucket_admin_access : ["scratch-staging"], }, "prod" : { - "user-sa" : { - bucket_admin_access : ["scratch"], - }, + bucket_admin_access : ["scratch"], }, } diff --git a/terraform/aws/projects/jupyter-meets-the-earth.tfvars b/terraform/aws/projects/jupyter-meets-the-earth.tfvars index 32c4bf1f33..a1db6fb96c 100644 --- a/terraform/aws/projects/jupyter-meets-the-earth.tfvars +++ b/terraform/aws/projects/jupyter-meets-the-earth.tfvars @@ -7,6 +7,7 @@ default_budget_alert = { } enable_aws_ce_grafana_backend_iam = true +disable_cluster_wide_filestore = false user_buckets = { "scratch-staging" : { @@ -22,44 +23,39 @@ user_buckets = { }, } - hub_cloud_permissions = { "staging" : { - "user-sa" : { - bucket_admin_access : ["scratch-staging"], - # FIXME: Previously, users were granted full S3 permissions. - # Keep it the same for now - extra_iam_policy : <<-EOT - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["s3:*"], - "Resource": ["arn:aws:s3:::*"] - } - ] - } - EOT - }, + bucket_admin_access : ["scratch-staging"], + # FIXME: Previously, users were granted full S3 permissions. + # Keep it the same for now + extra_iam_policy : <<-EOT + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:*"], + "Resource": ["arn:aws:s3:::*"] + } + ] + } + EOT }, "prod" : { - "user-sa" : { - bucket_admin_access : ["scratch"], - # FIXME: Previously, users were granted full S3 permissions. - # Keep it the same for now - extra_iam_policy : <<-EOT - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["s3:*"], - "Resource": ["arn:aws:s3:::*"] - } - ] - } - EOT - }, + bucket_admin_access : ["scratch"], + # FIXME: Previously, users were granted full S3 permissions. + # Keep it the same for now + extra_iam_policy : <<-EOT + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:*"], + "Resource": ["arn:aws:s3:::*"] + } + ] + } + EOT }, } diff --git a/terraform/aws/projects/kitware.tfvars b/terraform/aws/projects/kitware.tfvars index 8714f56661..4ac530c042 100644 --- a/terraform/aws/projects/kitware.tfvars +++ b/terraform/aws/projects/kitware.tfvars @@ -3,6 +3,7 @@ cluster_name = "kitware" cluster_nodes_location = "us-west-2a" enable_aws_ce_grafana_backend_iam = true +disable_cluster_wide_filestore = false user_buckets = { "scratch-staging" : { @@ -17,13 +18,9 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - "user-sa" : { - bucket_admin_access : ["scratch-staging"], - }, + bucket_admin_access : ["scratch-staging"], }, "prod" : { - "user-sa" : { - bucket_admin_access : ["scratch"], - }, + bucket_admin_access : ["scratch"], }, } diff --git a/terraform/aws/projects/maap.tfvars b/terraform/aws/projects/maap.tfvars index c6235fcf7c..a713a3c2aa 100644 --- a/terraform/aws/projects/maap.tfvars +++ b/terraform/aws/projects/maap.tfvars @@ -33,14 +33,9 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - "user-sa" : { - bucket_admin_access : ["scratch-staging"], - }, + bucket_admin_access : ["scratch-staging"], }, - "prod" : { - "user-sa" : { - bucket_admin_access : ["scratch-prod"], - }, + bucket_admin_access : ["scratch-prod"], }, } diff --git a/terraform/aws/projects/nasa-cryo.tfvars b/terraform/aws/projects/nasa-cryo.tfvars index 9ed162823e..1d05d6687c 100644 --- a/terraform/aws/projects/nasa-cryo.tfvars +++ b/terraform/aws/projects/nasa-cryo.tfvars @@ -36,78 +36,74 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - "user-sa" : { - bucket_admin_access : ["scratch-staging", "persistent-staging"], - # Provides readonly requestor-pays access to usgs-landsat bucket, - # veda bucket (https://2i2c.freshdesk.com/a/tickets/1547) and sliderule - # bucket (https://2i2c.freshdesk.com/a/tickets/1508). - # FIXME: We should find a way to allow access to *all* requester pays - # buckets, without having to explicitly list them. However, we don't want - # to give access to all *internal* s3 buckets willy-nilly - this can be - # a massive security hole, especially if terraform state is also here. - # As a temporary measure, we allow-list buckets here. - extra_iam_policy : <<-EOT - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:*" - ], - "Resource": [ - "arn:aws:s3:::usgs-landsat", - "arn:aws:s3:::usgs-landsat/*", - "arn:aws:s3:::sliderule-public", - "arn:aws:s3:::sliderule-public/*", - "arn:aws:s3:::veda-data-store", - "arn:aws:s3:::veda-data-store/*", - "arn:aws:s3:::veda-data-store-staging", - "arn:aws:s3:::veda-data-store-staging/*", - "arn:aws:s3:::ghgc-data-store", - "arn:aws:s3:::ghgc-data-store/*" + bucket_admin_access : ["scratch-staging", "persistent-staging"], + # Provides readonly requestor-pays access to usgs-landsat bucket, + # veda bucket (https://2i2c.freshdesk.com/a/tickets/1547) and sliderule + # bucket (https://2i2c.freshdesk.com/a/tickets/1508). + # FIXME: We should find a way to allow access to *all* requester pays + # buckets, without having to explicitly list them. However, we don't want + # to give access to all *internal* s3 buckets willy-nilly - this can be + # a massive security hole, especially if terraform state is also here. + # As a temporary measure, we allow-list buckets here. + extra_iam_policy : <<-EOT + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:*" + ], + "Resource": [ + "arn:aws:s3:::usgs-landsat", + "arn:aws:s3:::usgs-landsat/*", + "arn:aws:s3:::sliderule-public", + "arn:aws:s3:::sliderule-public/*", + "arn:aws:s3:::veda-data-store", + "arn:aws:s3:::veda-data-store/*", + "arn:aws:s3:::veda-data-store-staging", + "arn:aws:s3:::veda-data-store-staging/*", + "arn:aws:s3:::ghgc-data-store", + "arn:aws:s3:::ghgc-data-store/*" - ] - } - ] - } - EOT - }, + ] + } + ] + } + EOT }, "prod" : { - "user-sa" : { - bucket_admin_access : ["scratch", "persistent"], - # Provides readonly requestor-pays access to usgs-landsat bucket - # FIXME: We should find a way to allow access to *all* requester pays - # buckets, without having to explicitly list them. However, we don't want - # to give access to all *internal* s3 buckets willy-nilly - this can be - # a massive security hole, especially if terraform state is also here. - # As a temporary measure, we allow-list buckets here. - extra_iam_policy : <<-EOT - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:*" - ], - "Resource": [ - "arn:aws:s3:::usgs-landsat", - "arn:aws:s3:::usgs-landsat/*", - "arn:aws:s3:::sliderule-public", - "arn:aws:s3:::sliderule-public/*", - "arn:aws:s3:::veda-data-store", - "arn:aws:s3:::veda-data-store/*", - "arn:aws:s3:::veda-data-store-staging", - "arn:aws:s3:::veda-data-store-staging/*", - "arn:aws:s3:::ghgc-data-store", - "arn:aws:s3:::ghgc-data-store/*" - ] - } - ] - } - EOT - }, + bucket_admin_access : ["scratch", "persistent"], + # Provides readonly requestor-pays access to usgs-landsat bucket + # FIXME: We should find a way to allow access to *all* requester pays + # buckets, without having to explicitly list them. However, we don't want + # to give access to all *internal* s3 buckets willy-nilly - this can be + # a massive security hole, especially if terraform state is also here. + # As a temporary measure, we allow-list buckets here. + extra_iam_policy : <<-EOT + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:*" + ], + "Resource": [ + "arn:aws:s3:::usgs-landsat", + "arn:aws:s3:::usgs-landsat/*", + "arn:aws:s3:::sliderule-public", + "arn:aws:s3:::sliderule-public/*", + "arn:aws:s3:::veda-data-store", + "arn:aws:s3:::veda-data-store/*", + "arn:aws:s3:::veda-data-store-staging", + "arn:aws:s3:::veda-data-store-staging/*", + "arn:aws:s3:::ghgc-data-store", + "arn:aws:s3:::ghgc-data-store/*" + ] + } + ] + } + EOT }, } diff --git a/terraform/aws/projects/nasa-ghg.tfvars b/terraform/aws/projects/nasa-ghg.tfvars index c2b7780157..7dd3f0f6fa 100644 --- a/terraform/aws/projects/nasa-ghg.tfvars +++ b/terraform/aws/projects/nasa-ghg.tfvars @@ -7,6 +7,7 @@ default_budget_alert = { } enable_aws_ce_grafana_backend_iam = true +disable_cluster_wide_filestore = false user_buckets = { "scratch-staging" : { @@ -19,116 +20,111 @@ user_buckets = { }, } - hub_cloud_permissions = { "staging" : { - "user-sa" : { - bucket_admin_access : ["scratch-staging"], - extra_iam_policy : <<-EOT - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:PutObject", - "s3:GetObject", - "s3:ListBucketMultipartUploads", - "s3:AbortMultipartUpload", - "s3:ListBucketVersions", - "s3:ListBucket", - "s3:DeleteObject", - "s3:GetBucketLocation", - "s3:ListMultipartUploadParts" - ], - "Resource": [ - "arn:aws:s3:::ghgc-data-staging", - "arn:aws:s3:::ghgc-data-staging/*", - "arn:aws:s3:::ghgc-data-store-dev", - "arn:aws:s3:::ghgc-data-store-dev/*", - "arn:aws:s3:::ghgc-data-store", - "arn:aws:s3:::ghgc-data-store/*", - "arn:aws:s3:::ghgc-data-store-staging", - "arn:aws:s3:::ghgc-data-store-staging/*", - "arn:aws:s3:::veda-data-store", - "arn:aws:s3:::veda-data-store/*", - "arn:aws:s3:::veda-data-store-staging", - "arn:aws:s3:::veda-data-store-staging/*", - "arn:aws:s3:::lp-prod-protected", - "arn:aws:s3:::lp-prod-protected/*", - "arn:aws:s3:::gesdisc-cumulus-prod-protected", - "arn:aws:s3:::gesdisc-cumulus-prod-protected/*", - "arn:aws:s3:::nsidc-cumulus-prod-protected", - "arn:aws:s3:::nsidc-cumulus-prod-protected/*", - "arn:aws:s3:::ornl-cumulus-prod-protected", - "arn:aws:s3:::ornl-cumulus-prod-protected/*", - "arn:aws:s3:::podaac-ops-cumulus-public", - "arn:aws:s3:::podaac-ops-cumulus-public/*", - "arn:aws:s3:::podaac-ops-cumulus-protected", - "arn:aws:s3:::podaac-ops-cumulus-protected/*" - ] - }, - { - "Effect": "Allow", - "Action": "s3:ListAllMyBuckets", - "Resource": "*" - } - ] - } - EOT - }, + bucket_admin_access : ["scratch-staging"], + extra_iam_policy : <<-EOT + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucketMultipartUploads", + "s3:AbortMultipartUpload", + "s3:ListBucketVersions", + "s3:ListBucket", + "s3:DeleteObject", + "s3:GetBucketLocation", + "s3:ListMultipartUploadParts" + ], + "Resource": [ + "arn:aws:s3:::ghgc-data-staging", + "arn:aws:s3:::ghgc-data-staging/*", + "arn:aws:s3:::ghgc-data-store-dev", + "arn:aws:s3:::ghgc-data-store-dev/*", + "arn:aws:s3:::ghgc-data-store", + "arn:aws:s3:::ghgc-data-store/*", + "arn:aws:s3:::ghgc-data-store-staging", + "arn:aws:s3:::ghgc-data-store-staging/*", + "arn:aws:s3:::veda-data-store", + "arn:aws:s3:::veda-data-store/*", + "arn:aws:s3:::veda-data-store-staging", + "arn:aws:s3:::veda-data-store-staging/*", + "arn:aws:s3:::lp-prod-protected", + "arn:aws:s3:::lp-prod-protected/*", + "arn:aws:s3:::gesdisc-cumulus-prod-protected", + "arn:aws:s3:::gesdisc-cumulus-prod-protected/*", + "arn:aws:s3:::nsidc-cumulus-prod-protected", + "arn:aws:s3:::nsidc-cumulus-prod-protected/*", + "arn:aws:s3:::ornl-cumulus-prod-protected", + "arn:aws:s3:::ornl-cumulus-prod-protected/*", + "arn:aws:s3:::podaac-ops-cumulus-public", + "arn:aws:s3:::podaac-ops-cumulus-public/*", + "arn:aws:s3:::podaac-ops-cumulus-protected", + "arn:aws:s3:::podaac-ops-cumulus-protected/*" + ] + }, + { + "Effect": "Allow", + "Action": "s3:ListAllMyBuckets", + "Resource": "*" + } + ] + } + EOT }, "prod" : { - "user-sa" : { - bucket_admin_access : ["scratch"], - extra_iam_policy : <<-EOT - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:PutObject", - "s3:GetObject", - "s3:ListBucketMultipartUploads", - "s3:AbortMultipartUpload", - "s3:ListBucketVersions", - "s3:ListBucket", - "s3:DeleteObject", - "s3:GetBucketLocation", - "s3:ListMultipartUploadParts" - ], - "Resource": [ - "arn:aws:s3:::ghgc-data-staging", - "arn:aws:s3:::ghgc-data-staging/*", - "arn:aws:s3:::ghgc-data-store-dev", - "arn:aws:s3:::ghgc-data-store-dev/*", - "arn:aws:s3:::ghgc-data-store", - "arn:aws:s3:::ghgc-data-store/*", - "arn:aws:s3:::ghgc-data-store-staging", - "arn:aws:s3:::ghgc-data-store-staging/*", - "arn:aws:s3:::veda-data-store", - "arn:aws:s3:::veda-data-store/*", - "arn:aws:s3:::veda-data-store-staging", - "arn:aws:s3:::veda-data-store-staging/*", - "arn:aws:s3:::lp-prod-protected", - "arn:aws:s3:::lp-prod-protected/*", - "arn:aws:s3:::gesdisc-cumulus-prod-protected", - "arn:aws:s3:::gesdisc-cumulus-prod-protected/*", - "arn:aws:s3:::nsidc-cumulus-prod-protected", - "arn:aws:s3:::nsidc-cumulus-prod-protected/*", - "arn:aws:s3:::ornl-cumulus-prod-protected", - "arn:aws:s3:::ornl-cumulus-prod-protected/*" - ] - }, - { - "Effect": "Allow", - "Action": "s3:ListAllMyBuckets", - "Resource": "*" - } - ] - } - EOT - }, + bucket_admin_access : ["scratch"], + extra_iam_policy : <<-EOT + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucketMultipartUploads", + "s3:AbortMultipartUpload", + "s3:ListBucketVersions", + "s3:ListBucket", + "s3:DeleteObject", + "s3:GetBucketLocation", + "s3:ListMultipartUploadParts" + ], + "Resource": [ + "arn:aws:s3:::ghgc-data-staging", + "arn:aws:s3:::ghgc-data-staging/*", + "arn:aws:s3:::ghgc-data-store-dev", + "arn:aws:s3:::ghgc-data-store-dev/*", + "arn:aws:s3:::ghgc-data-store", + "arn:aws:s3:::ghgc-data-store/*", + "arn:aws:s3:::ghgc-data-store-staging", + "arn:aws:s3:::ghgc-data-store-staging/*", + "arn:aws:s3:::veda-data-store", + "arn:aws:s3:::veda-data-store/*", + "arn:aws:s3:::veda-data-store-staging", + "arn:aws:s3:::veda-data-store-staging/*", + "arn:aws:s3:::lp-prod-protected", + "arn:aws:s3:::lp-prod-protected/*", + "arn:aws:s3:::gesdisc-cumulus-prod-protected", + "arn:aws:s3:::gesdisc-cumulus-prod-protected/*", + "arn:aws:s3:::nsidc-cumulus-prod-protected", + "arn:aws:s3:::nsidc-cumulus-prod-protected/*", + "arn:aws:s3:::ornl-cumulus-prod-protected", + "arn:aws:s3:::ornl-cumulus-prod-protected/*" + ] + }, + { + "Effect": "Allow", + "Action": "s3:ListAllMyBuckets", + "Resource": "*" + } + ] + } + EOT }, } diff --git a/terraform/aws/projects/nasa-veda.tfvars b/terraform/aws/projects/nasa-veda.tfvars index 480c632573..c2ddb105f7 100644 --- a/terraform/aws/projects/nasa-veda.tfvars +++ b/terraform/aws/projects/nasa-veda.tfvars @@ -26,197 +26,192 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - "user-sa" : { - bucket_admin_access : ["scratch-staging"], - extra_iam_policy : <<-EOT - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:PutObject", - "s3:GetObject", - "s3:ListBucketMultipartUploads", - "s3:AbortMultipartUpload", - "s3:ListBucketVersions", - "s3:CreateBucket", - "s3:ListBucket", - "s3:DeleteObject", - "s3:GetBucketLocation", - "s3:ListMultipartUploadParts" - ], - "Resource": [ - "arn:aws:s3:::veda-data-store", - "arn:aws:s3:::veda-data-store/*", - "arn:aws:s3:::veda-data-store-staging", - "arn:aws:s3:::veda-data-store-staging/*", - "arn:aws:s3:::veda-nex-gddp-cmip6-public", - "arn:aws:s3:::veda-nex-gddp-cmip6-public/*", - "arn:aws:s3:::cmip6-staging", - "arn:aws:s3:::cmip6-staging/*", - "arn:aws:s3:::lp-prod-protected", - "arn:aws:s3:::lp-prod-protected/*", - "arn:aws:s3:::gesdisc-cumulus-prod-protected", - "arn:aws:s3:::gesdisc-cumulus-prod-protected/*", - "arn:aws:s3:::nsidc-cumulus-prod-protected", - "arn:aws:s3:::nsidc-cumulus-prod-protected/*", - "arn:aws:s3:::ornl-cumulus-prod-protected", - "arn:aws:s3:::ornl-cumulus-prod-protected/*", - "arn:aws:s3:::pangeo-forge-veda-output", - "arn:aws:s3:::pangeo-forge-veda-output/*", - "arn:aws:s3:::podaac-ops-cumulus-public", - "arn:aws:s3:::podaac-ops-cumulus-public/*", - "arn:aws:s3:::podaac-ops-cumulus-protected", - "arn:aws:s3:::podaac-ops-cumulus-protected/*", - "arn:aws:s3:::maap-ops-workspace", - "arn:aws:s3:::maap-ops-workspace/*", - "arn:aws:s3:::nasa-maap-data-store", - "arn:aws:s3:::nasa-maap-data-store/*", - "arn:aws:s3:::sdap-dev-zarr", - "arn:aws:s3:::sdap-dev-zarr/*", - "arn:aws:s3:::usgs-landsat", - "arn:aws:s3:::usgs-landsat/*", - "arn:aws:s3:::sentinel-cogs", - "arn:aws:s3:::sentinel-cogs/*" - ] - }, - { - "Effect": "Allow", - "Action": "s3:ListAllMyBuckets", - "Resource": "*" - } - ] - } - EOT - }, + bucket_admin_access : ["scratch-staging"], + extra_iam_policy : <<-EOT + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucketMultipartUploads", + "s3:AbortMultipartUpload", + "s3:ListBucketVersions", + "s3:CreateBucket", + "s3:ListBucket", + "s3:DeleteObject", + "s3:GetBucketLocation", + "s3:ListMultipartUploadParts" + ], + "Resource": [ + "arn:aws:s3:::veda-data-store", + "arn:aws:s3:::veda-data-store/*", + "arn:aws:s3:::veda-data-store-staging", + "arn:aws:s3:::veda-data-store-staging/*", + "arn:aws:s3:::veda-nex-gddp-cmip6-public", + "arn:aws:s3:::veda-nex-gddp-cmip6-public/*", + "arn:aws:s3:::cmip6-staging", + "arn:aws:s3:::cmip6-staging/*", + "arn:aws:s3:::lp-prod-protected", + "arn:aws:s3:::lp-prod-protected/*", + "arn:aws:s3:::gesdisc-cumulus-prod-protected", + "arn:aws:s3:::gesdisc-cumulus-prod-protected/*", + "arn:aws:s3:::nsidc-cumulus-prod-protected", + "arn:aws:s3:::nsidc-cumulus-prod-protected/*", + "arn:aws:s3:::ornl-cumulus-prod-protected", + "arn:aws:s3:::ornl-cumulus-prod-protected/*", + "arn:aws:s3:::pangeo-forge-veda-output", + "arn:aws:s3:::pangeo-forge-veda-output/*", + "arn:aws:s3:::podaac-ops-cumulus-public", + "arn:aws:s3:::podaac-ops-cumulus-public/*", + "arn:aws:s3:::podaac-ops-cumulus-protected", + "arn:aws:s3:::podaac-ops-cumulus-protected/*", + "arn:aws:s3:::maap-ops-workspace", + "arn:aws:s3:::maap-ops-workspace/*", + "arn:aws:s3:::nasa-maap-data-store", + "arn:aws:s3:::nasa-maap-data-store/*", + "arn:aws:s3:::sdap-dev-zarr", + "arn:aws:s3:::sdap-dev-zarr/*", + "arn:aws:s3:::usgs-landsat", + "arn:aws:s3:::usgs-landsat/*", + "arn:aws:s3:::sentinel-cogs", + "arn:aws:s3:::sentinel-cogs/*" + ] + }, + { + "Effect": "Allow", + "Action": "s3:ListAllMyBuckets", + "Resource": "*" + } + ] + } + EOT }, "prod" : { - "user-sa" : { - bucket_admin_access : ["scratch"], - extra_iam_policy : <<-EOT - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:PutObject", - "s3:GetObject", - "s3:ListBucketMultipartUploads", - "s3:AbortMultipartUpload", - "s3:ListBucketVersions", - "s3:CreateBucket", - "s3:ListBucket", - "s3:DeleteObject", - "s3:GetBucketLocation", - "s3:ListMultipartUploadParts" - ], - "Resource": [ - "arn:aws:s3:::veda-data-store", - "arn:aws:s3:::veda-data-store/*", - "arn:aws:s3:::veda-data-store-staging", - "arn:aws:s3:::veda-data-store-staging/*", - "arn:aws:s3:::veda-nex-gddp-cmip6-public", - "arn:aws:s3:::veda-nex-gddp-cmip6-public/*", - "arn:aws:s3:::cmip6-staging", - "arn:aws:s3:::cmip6-staging/*", - "arn:aws:s3:::lp-prod-protected", - "arn:aws:s3:::lp-prod-protected/*", - "arn:aws:s3:::gesdisc-cumulus-prod-protected", - "arn:aws:s3:::gesdisc-cumulus-prod-protected/*", - "arn:aws:s3:::nsidc-cumulus-prod-protected", - "arn:aws:s3:::nsidc-cumulus-prod-protected/*", - "arn:aws:s3:::ornl-cumulus-prod-protected", - "arn:aws:s3:::ornl-cumulus-prod-protected/*", - "arn:aws:s3:::pangeo-forge-veda-output", - "arn:aws:s3:::pangeo-forge-veda-output/*", - "arn:aws:s3:::podaac-ops-cumulus-public", - "arn:aws:s3:::podaac-ops-cumulus-public/*", - "arn:aws:s3:::podaac-ops-cumulus-protected", - "arn:aws:s3:::podaac-ops-cumulus-protected/*", - "arn:aws:s3:::maap-ops-workspace", - "arn:aws:s3:::maap-ops-workspace/*", - "arn:aws:s3:::nasa-maap-data-store", - "arn:aws:s3:::nasa-maap-data-store/*", - "arn:aws:s3:::sdap-dev-zarr", - "arn:aws:s3:::sdap-dev-zarr/*", - "arn:aws:s3:::usgs-landsat", - "arn:aws:s3:::usgs-landsat/*" - ] - }, - { - "Effect": "Allow", - "Action": "s3:ListAllMyBuckets", - "Resource": "*" - } - ] - } - EOT - }, + bucket_admin_access : ["scratch"], + extra_iam_policy : <<-EOT + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucketMultipartUploads", + "s3:AbortMultipartUpload", + "s3:ListBucketVersions", + "s3:CreateBucket", + "s3:ListBucket", + "s3:DeleteObject", + "s3:GetBucketLocation", + "s3:ListMultipartUploadParts" + ], + "Resource": [ + "arn:aws:s3:::veda-data-store", + "arn:aws:s3:::veda-data-store/*", + "arn:aws:s3:::veda-data-store-staging", + "arn:aws:s3:::veda-data-store-staging/*", + "arn:aws:s3:::veda-nex-gddp-cmip6-public", + "arn:aws:s3:::veda-nex-gddp-cmip6-public/*", + "arn:aws:s3:::cmip6-staging", + "arn:aws:s3:::cmip6-staging/*", + "arn:aws:s3:::lp-prod-protected", + "arn:aws:s3:::lp-prod-protected/*", + "arn:aws:s3:::gesdisc-cumulus-prod-protected", + "arn:aws:s3:::gesdisc-cumulus-prod-protected/*", + "arn:aws:s3:::nsidc-cumulus-prod-protected", + "arn:aws:s3:::nsidc-cumulus-prod-protected/*", + "arn:aws:s3:::ornl-cumulus-prod-protected", + "arn:aws:s3:::ornl-cumulus-prod-protected/*", + "arn:aws:s3:::pangeo-forge-veda-output", + "arn:aws:s3:::pangeo-forge-veda-output/*", + "arn:aws:s3:::podaac-ops-cumulus-public", + "arn:aws:s3:::podaac-ops-cumulus-public/*", + "arn:aws:s3:::podaac-ops-cumulus-protected", + "arn:aws:s3:::podaac-ops-cumulus-protected/*", + "arn:aws:s3:::maap-ops-workspace", + "arn:aws:s3:::maap-ops-workspace/*", + "arn:aws:s3:::nasa-maap-data-store", + "arn:aws:s3:::nasa-maap-data-store/*", + "arn:aws:s3:::sdap-dev-zarr", + "arn:aws:s3:::sdap-dev-zarr/*", + "arn:aws:s3:::usgs-landsat", + "arn:aws:s3:::usgs-landsat/*" + ] + }, + { + "Effect": "Allow", + "Action": "s3:ListAllMyBuckets", + "Resource": "*" + } + ] + } + EOT }, "binder" : { - "user-sa" : { - bucket_admin_access : ["scratch-binder"], - extra_iam_policy : <<-EOT - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:GetObject", - "s3:ListBucketVersions", - "s3:ListBucket", - "s3:GetBucketLocation" - ], - "Resource": [ - "arn:aws:s3:::veda-data-store", - "arn:aws:s3:::veda-data-store/*", - "arn:aws:s3:::veda-data-store-staging", - "arn:aws:s3:::veda-data-store-staging/*", - "arn:aws:s3:::veda-nex-gddp-cmip6-public", - "arn:aws:s3:::veda-nex-gddp-cmip6-public/*", - "arn:aws:s3:::cmip6-staging", - "arn:aws:s3:::cmip6-staging/*", - "arn:aws:s3:::lp-prod-protected", - "arn:aws:s3:::lp-prod-protected/*", - "arn:aws:s3:::gesdisc-cumulus-prod-protected", - "arn:aws:s3:::gesdisc-cumulus-prod-protected/*", - "arn:aws:s3:::nsidc-cumulus-prod-protected", - "arn:aws:s3:::nsidc-cumulus-prod-protected/*", - "arn:aws:s3:::ornl-cumulus-prod-protected", - "arn:aws:s3:::ornl-cumulus-prod-protected/*", - "arn:aws:s3:::pangeo-forge-veda-output", - "arn:aws:s3:::pangeo-forge-veda-output/*", - "arn:aws:s3:::podaac-ops-cumulus-public", - "arn:aws:s3:::podaac-ops-cumulus-public/*", - "arn:aws:s3:::podaac-ops-cumulus-protected", - "arn:aws:s3:::podaac-ops-cumulus-protected/*", - "arn:aws:s3:::maap-ops-workspace", - "arn:aws:s3:::maap-ops-workspace/*", - "arn:aws:s3:::nasa-maap-data-store", - "arn:aws:s3:::nasa-maap-data-store/*", - "arn:aws:s3:::sdap-dev-zarr", - "arn:aws:s3:::sdap-dev-zarr/*", - "arn:aws:s3:::usgs-landsat", - "arn:aws:s3:::usgs-landsat/*", - "arn:aws:s3:::sentinel-cogs", - "arn:aws:s3:::sentinel-cogs/*" - ] - }, - { - "Effect": "Allow", - "Action": "s3:ListAllMyBuckets", - "Resource": "*" - } - ] - } - EOT - }, + bucket_admin_access : ["scratch-binder"], + extra_iam_policy : <<-EOT + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:ListBucketVersions", + "s3:ListBucket", + "s3:GetBucketLocation" + ], + "Resource": [ + "arn:aws:s3:::veda-data-store", + "arn:aws:s3:::veda-data-store/*", + "arn:aws:s3:::veda-data-store-staging", + "arn:aws:s3:::veda-data-store-staging/*", + "arn:aws:s3:::veda-nex-gddp-cmip6-public", + "arn:aws:s3:::veda-nex-gddp-cmip6-public/*", + "arn:aws:s3:::cmip6-staging", + "arn:aws:s3:::cmip6-staging/*", + "arn:aws:s3:::lp-prod-protected", + "arn:aws:s3:::lp-prod-protected/*", + "arn:aws:s3:::gesdisc-cumulus-prod-protected", + "arn:aws:s3:::gesdisc-cumulus-prod-protected/*", + "arn:aws:s3:::nsidc-cumulus-prod-protected", + "arn:aws:s3:::nsidc-cumulus-prod-protected/*", + "arn:aws:s3:::ornl-cumulus-prod-protected", + "arn:aws:s3:::ornl-cumulus-prod-protected/*", + "arn:aws:s3:::pangeo-forge-veda-output", + "arn:aws:s3:::pangeo-forge-veda-output/*", + "arn:aws:s3:::podaac-ops-cumulus-public", + "arn:aws:s3:::podaac-ops-cumulus-public/*", + "arn:aws:s3:::podaac-ops-cumulus-protected", + "arn:aws:s3:::podaac-ops-cumulus-protected/*", + "arn:aws:s3:::maap-ops-workspace", + "arn:aws:s3:::maap-ops-workspace/*", + "arn:aws:s3:::nasa-maap-data-store", + "arn:aws:s3:::nasa-maap-data-store/*", + "arn:aws:s3:::sdap-dev-zarr", + "arn:aws:s3:::sdap-dev-zarr/*", + "arn:aws:s3:::usgs-landsat", + "arn:aws:s3:::usgs-landsat/*", + "arn:aws:s3:::sentinel-cogs", + "arn:aws:s3:::sentinel-cogs/*" + ] + }, + { + "Effect": "Allow", + "Action": "s3:ListAllMyBuckets", + "Resource": "*" + } + ] + } + EOT }, } +disable_cluster_wide_filestore = false ebs_volumes = { "staging" = { size = 100 diff --git a/terraform/aws/projects/nmfs-openscapes.tfvars b/terraform/aws/projects/nmfs-openscapes.tfvars index 540fa26def..9187c4d269 100644 --- a/terraform/aws/projects/nmfs-openscapes.tfvars +++ b/terraform/aws/projects/nmfs-openscapes.tfvars @@ -43,14 +43,10 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - "user-sa" : { - bucket_admin_access : ["scratch-staging", "persistent-staging"], - }, + bucket_admin_access : ["scratch-staging", "persistent-staging"], }, "prod" : { - "user-sa" : { - bucket_admin_access : ["scratch", "persistent"], - }, + bucket_admin_access : ["scratch", "persistent"], }, } diff --git a/terraform/aws/projects/openscapes.tfvars b/terraform/aws/projects/openscapes.tfvars index 7b176a8f82..0b88dd4547 100644 --- a/terraform/aws/projects/openscapes.tfvars +++ b/terraform/aws/projects/openscapes.tfvars @@ -7,6 +7,7 @@ default_budget_alert = { } enable_aws_ce_grafana_backend_iam = true +disable_cluster_wide_filestore = false # The initial EFS is now used by the prod hub only # So we tag it appropriately for costs purposes @@ -58,31 +59,24 @@ user_buckets = { }, } - hub_cloud_permissions = { "staging" : { - "user-sa" : { - bucket_admin_access : [ - "scratch-staging", - "persistent-staging", - ], - }, + bucket_admin_access : [ + "scratch-staging", + "persistent-staging", + ], }, "prod" : { - "user-sa" : { - bucket_admin_access : [ - "scratch", - "persistent", - ], - } + bucket_admin_access : [ + "scratch", + "persistent", + ], }, "workshop" : { - "user-sa" : { - bucket_admin_access : [ - "scratch-workshop", - "persistent-workshop", - ], - } + bucket_admin_access : [ + "scratch-workshop", + "persistent-workshop", + ], }, } diff --git a/terraform/aws/projects/opensci.tfvars b/terraform/aws/projects/opensci.tfvars index 4d90a5e25c..f4a5e144e9 100644 --- a/terraform/aws/projects/opensci.tfvars +++ b/terraform/aws/projects/opensci.tfvars @@ -3,6 +3,7 @@ cluster_name = "opensci" cluster_nodes_location = "us-west-2a" enable_aws_ce_grafana_backend_iam = true +disable_cluster_wide_filestore = false user_buckets = { "scratch-staging" : { @@ -19,17 +20,12 @@ user_buckets = { }, } - hub_cloud_permissions = { "staging" : { - "user-sa" : { - bucket_admin_access : ["scratch-staging"], - }, + bucket_admin_access : ["scratch-staging"], }, "sciencecore" : { - "user-sa" : { - bucket_admin_access : ["scratch-sciencecore"], - bucket_readonly_access : ["persistent-sciencecore"], - }, + bucket_admin_access : ["scratch-sciencecore"], + bucket_readonly_access : ["persistent-sciencecore"], }, } diff --git a/terraform/aws/projects/projectpythia.tfvars b/terraform/aws/projects/projectpythia.tfvars index 45f8ea1629..da5f008e32 100644 --- a/terraform/aws/projects/projectpythia.tfvars +++ b/terraform/aws/projects/projectpythia.tfvars @@ -7,6 +7,7 @@ default_budget_alert = { } enable_aws_ce_grafana_backend_iam = true +disable_cluster_wide_filestore = false # FIXME: placeholder bucket to get the 2i2c:hub-name tag in place # so the community cand enable it for cost allocation purposes @@ -20,8 +21,6 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - "user-sa" : { - bucket_admin_access : ["placeholder-bucket-staging"], - }, + bucket_admin_access : ["placeholder-bucket-staging"], }, } \ No newline at end of file diff --git a/terraform/aws/projects/smithsonian.tfvars b/terraform/aws/projects/smithsonian.tfvars index f56b29de89..88c5f892cd 100644 --- a/terraform/aws/projects/smithsonian.tfvars +++ b/terraform/aws/projects/smithsonian.tfvars @@ -3,6 +3,7 @@ cluster_name = "smithsonian" cluster_nodes_location = "us-east-2b" enable_aws_ce_grafana_backend_iam = true +disable_cluster_wide_filestore = false user_buckets = { "scratch-staging" : { @@ -17,13 +18,9 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - "user-sa" : { - bucket_admin_access : ["scratch-staging"], - }, + bucket_admin_access : ["scratch-staging"], }, "prod" : { - "user-sa" : { - bucket_admin_access : ["scratch"], - }, + bucket_admin_access : ["scratch"], }, } diff --git a/terraform/aws/projects/strudel.tfvars b/terraform/aws/projects/strudel.tfvars index 8cd8483e23..dcaf796d3e 100644 --- a/terraform/aws/projects/strudel.tfvars +++ b/terraform/aws/projects/strudel.tfvars @@ -8,6 +8,7 @@ cluster_name = "strudel" cluster_nodes_location = "us-west-2a" enable_aws_ce_grafana_backend_iam = true +disable_cluster_wide_filestore = false # Tip: uncomment and fill the missing info in the lines below if you want # to setup scratch buckets for the hubs on this cluster. diff --git a/terraform/aws/projects/ubc-eoas.tfvars b/terraform/aws/projects/ubc-eoas.tfvars index 76f8b0dd1e..7641179499 100644 --- a/terraform/aws/projects/ubc-eoas.tfvars +++ b/terraform/aws/projects/ubc-eoas.tfvars @@ -3,6 +3,7 @@ cluster_name = "ubc-eoas" cluster_nodes_location = "ca-central-1a" enable_aws_ce_grafana_backend_iam = true +disable_cluster_wide_filestore = false user_buckets = { "scratch-staging" : { @@ -17,13 +18,9 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - "user-sa" : { - bucket_admin_access : ["scratch-staging"], - }, + bucket_admin_access : ["scratch-staging"], }, "prod" : { - "user-sa" : { - bucket_admin_access : ["scratch"], - }, + bucket_admin_access : ["scratch"], }, } diff --git a/terraform/aws/projects/victor.tfvars b/terraform/aws/projects/victor.tfvars index 9a3c18a101..c8270ea362 100644 --- a/terraform/aws/projects/victor.tfvars +++ b/terraform/aws/projects/victor.tfvars @@ -3,6 +3,7 @@ cluster_name = "victor" cluster_nodes_location = "us-west-2a" enable_aws_ce_grafana_backend_iam = true +disable_cluster_wide_filestore = false user_buckets = { "scratch-staging" : { @@ -15,16 +16,11 @@ user_buckets = { }, } - hub_cloud_permissions = { "staging" : { - "user-sa" : { - bucket_admin_access : ["scratch-staging"], - }, + bucket_admin_access : ["scratch-staging"], }, "prod" : { - "user-sa" : { - bucket_admin_access : ["scratch"], - }, + bucket_admin_access : ["scratch"], }, } From 9ca25108f46cb3866376184c0056e2fd05800d24 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 6 Dec 2024 16:54:36 +0000 Subject: [PATCH 5/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- terraform/aws/projects/openscapes.tfvars | 2 +- terraform/aws/projects/projectpythia.tfvars | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/aws/projects/openscapes.tfvars b/terraform/aws/projects/openscapes.tfvars index 0b88dd4547..4992fc5c11 100644 --- a/terraform/aws/projects/openscapes.tfvars +++ b/terraform/aws/projects/openscapes.tfvars @@ -7,7 +7,7 @@ default_budget_alert = { } enable_aws_ce_grafana_backend_iam = true -disable_cluster_wide_filestore = false +disable_cluster_wide_filestore = false # The initial EFS is now used by the prod hub only # So we tag it appropriately for costs purposes diff --git a/terraform/aws/projects/projectpythia.tfvars b/terraform/aws/projects/projectpythia.tfvars index da5f008e32..cb1c95df0c 100644 --- a/terraform/aws/projects/projectpythia.tfvars +++ b/terraform/aws/projects/projectpythia.tfvars @@ -7,7 +7,7 @@ default_budget_alert = { } enable_aws_ce_grafana_backend_iam = true -disable_cluster_wide_filestore = false +disable_cluster_wide_filestore = false # FIXME: placeholder bucket to get the 2i2c:hub-name tag in place # so the community cand enable it for cost allocation purposes