From 735caa333a2ff1d26b3f605652572cf3a784bfd4 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 22 Apr 2024 20:52:48 +0200 Subject: [PATCH] aws hubs: consistent setup of cloud permissions and bucket envs --- config/clusters/2i2c-aws-us/dask-staging.values.yaml | 2 ++ config/clusters/2i2c-aws-us/itcoocean.values.yaml | 3 +-- config/clusters/2i2c-aws-us/ncar-cisl.values.yaml | 2 ++ config/clusters/2i2c-aws-us/showcase.values.yaml | 3 +-- config/clusters/2i2c-aws-us/staging.values.yaml | 3 +++ config/clusters/gridsst/prod.values.yaml | 3 +++ config/clusters/gridsst/staging.values.yaml | 3 +++ config/clusters/jupyter-health/prod.values.yaml | 3 +++ config/clusters/jupyter-health/staging.values.yaml | 3 +++ .../jupyter-meets-the-earth/staging.values.yaml | 5 +---- config/clusters/kitware/prod.values.yaml | 3 +++ config/clusters/kitware/staging.values.yaml | 3 +++ config/clusters/nasa-ghg/prod.values.yaml | 3 +++ config/clusters/nasa-ghg/staging.values.yaml | 3 +++ config/clusters/nasa-veda/prod.values.yaml | 3 +++ config/clusters/nasa-veda/staging.values.yaml | 2 ++ config/clusters/opensci/staging.values.yaml | 7 +++++++ config/clusters/smithsonian/prod.values.yaml | 12 ++++++++++++ config/clusters/smithsonian/staging.values.yaml | 12 ++++++++++++ config/clusters/ubc-eoas/prod.values.yaml | 8 ++++++++ config/clusters/ubc-eoas/staging.values.yaml | 8 ++++++++ config/clusters/victor/prod.values.yaml | 3 +++ config/clusters/victor/staging.values.yaml | 2 ++ terraform/aws/projects/2i2c-aws-us.tfvars | 4 ++-- .../aws/projects/jupyter-meets-the-earth.tfvars | 3 +++ 25 files changed, 96 insertions(+), 10 deletions(-) diff --git a/config/clusters/2i2c-aws-us/dask-staging.values.yaml b/config/clusters/2i2c-aws-us/dask-staging.values.yaml index 1a5af779fd..9b82f68bf7 100644 --- a/config/clusters/2i2c-aws-us/dask-staging.values.yaml +++ b/config/clusters/2i2c-aws-us/dask-staging.values.yaml @@ -31,6 +31,8 @@ basehub: image: name: pangeo/pangeo-notebook tag: "latest" + extraEnv: + SCRATCH_BUCKET: s3://2i2c-aws-us-scratch-dask-staging/$(JUPYTERHUB_USER) hub: config: JupyterHub: diff --git a/config/clusters/2i2c-aws-us/itcoocean.values.yaml b/config/clusters/2i2c-aws-us/itcoocean.values.yaml index 2959236a55..d1cea4e473 100644 --- a/config/clusters/2i2c-aws-us/itcoocean.values.yaml +++ b/config/clusters/2i2c-aws-us/itcoocean.values.yaml @@ -81,8 +81,7 @@ jupyterhub: mountPath: /home/jovyan/shared-public subPath: _shared-public extraEnv: - SCRATCH_BUCKET: s3://scratch-itcoocean/$(JUPYTERHUB_USER) - PANGEO_SCRATCH: s3://scratch-itcoocean/$(JUPYTERHUB_USER) + SCRATCH_BUCKET: s3://2i2c-aws-us-scratch-itcoocean/$(JUPYTERHUB_USER) profileList: # NOTE: About node sharing # diff --git a/config/clusters/2i2c-aws-us/ncar-cisl.values.yaml b/config/clusters/2i2c-aws-us/ncar-cisl.values.yaml index 7e9ee9d2f2..def98a8208 100644 --- a/config/clusters/2i2c-aws-us/ncar-cisl.values.yaml +++ b/config/clusters/2i2c-aws-us/ncar-cisl.values.yaml @@ -52,6 +52,8 @@ basehub: # pangeo/pangeo-notebook is maintained at: https://github.com/pangeo-data/pangeo-docker-images name: pangeo/pangeo-notebook tag: "2023.05.18" + extraEnv: + SCRATCH_BUCKET: s3://2i2c-aws-us-scratch-ncar-cisl/$(JUPYTERHUB_USER) profileList: # NOTE: About node sharing # diff --git a/config/clusters/2i2c-aws-us/showcase.values.yaml b/config/clusters/2i2c-aws-us/showcase.values.yaml index d27c1de38b..4651311502 100644 --- a/config/clusters/2i2c-aws-us/showcase.values.yaml +++ b/config/clusters/2i2c-aws-us/showcase.values.yaml @@ -50,8 +50,7 @@ basehub: enable_auth_state: true singleuser: extraEnv: - SCRATCH_BUCKET: s3://2i2c-aws-us-scratch-researchdelight/$(JUPYTERHUB_USER) - PANGEO_SCRATCH: s3://2i2c-aws-us-scratch-researchdelight/$(JUPYTERHUB_USER) + SCRATCH_BUCKET: s3://2i2c-aws-us-scratch-showcase/$(JUPYTERHUB_USER) PERSISTENT_BUCKET: s3://2i2c-aws-us-persistent-showcase/$(JUPYTERHUB_USER) GH_SCOPED_CREDS_CLIENT_ID: Iv1.f9261c4c78b4dfdd GH_SCOPED_CREDS_APP_URL: https://github.com/apps/2i2c-community-showcase-hub diff --git a/config/clusters/2i2c-aws-us/staging.values.yaml b/config/clusters/2i2c-aws-us/staging.values.yaml index 0ba53adf1e..2020c91af1 100644 --- a/config/clusters/2i2c-aws-us/staging.values.yaml +++ b/config/clusters/2i2c-aws-us/staging.values.yaml @@ -32,3 +32,6 @@ jupyterhub: authenticator_class: "github" GitHubOAuthenticator: oauth_callback_url: "https://staging.aws.2i2c.cloud/hub/oauth_callback" + singleuser: + extraEnv: + SCRATCH_BUCKET: s3://2i2c-aws-us-scratch-staging/$(JUPYTERHUB_USER) diff --git a/config/clusters/gridsst/prod.values.yaml b/config/clusters/gridsst/prod.values.yaml index 2e9ce2a766..3de86a45ee 100644 --- a/config/clusters/gridsst/prod.values.yaml +++ b/config/clusters/gridsst/prod.values.yaml @@ -12,3 +12,6 @@ basehub: config: GitHubOAuthenticator: oauth_callback_url: https://gridsst.2i2c.cloud/hub/oauth_callback + singleuser: + extraEnv: + SCRATCH_BUCKET: s3://gridsst-scratch/$(JUPYTERHUB_USER) diff --git a/config/clusters/gridsst/staging.values.yaml b/config/clusters/gridsst/staging.values.yaml index cdf619a813..0ab3596e22 100644 --- a/config/clusters/gridsst/staging.values.yaml +++ b/config/clusters/gridsst/staging.values.yaml @@ -12,3 +12,6 @@ basehub: config: GitHubOAuthenticator: oauth_callback_url: https://staging.gridsst.2i2c.cloud/hub/oauth_callback + singleuser: + extraEnv: + SCRATCH_BUCKET: s3://gridsst-scratch-staging/$(JUPYTERHUB_USER) diff --git a/config/clusters/jupyter-health/prod.values.yaml b/config/clusters/jupyter-health/prod.values.yaml index b1072a659c..cc3d05e92c 100644 --- a/config/clusters/jupyter-health/prod.values.yaml +++ b/config/clusters/jupyter-health/prod.values.yaml @@ -11,3 +11,6 @@ jupyterhub: config: GitHubOAuthenticator: oauth_callback_url: https://jupyter-health.2i2c.cloud/hub/oauth_callback + singleuser: + extraEnv: + SCRATCH_BUCKET: s3://jupyter-health-scratch/$(JUPYTERHUB_USER) diff --git a/config/clusters/jupyter-health/staging.values.yaml b/config/clusters/jupyter-health/staging.values.yaml index 0548bc2591..75f464af5c 100644 --- a/config/clusters/jupyter-health/staging.values.yaml +++ b/config/clusters/jupyter-health/staging.values.yaml @@ -11,3 +11,6 @@ jupyterhub: config: GitHubOAuthenticator: oauth_callback_url: https://staging.jupyter-health.2i2c.cloud/hub/oauth_callback + singleuser: + extraEnv: + SCRATCH_BUCKET: s3://jupyter-health-scratch-staging/$(JUPYTERHUB_USER) diff --git a/config/clusters/jupyter-meets-the-earth/staging.values.yaml b/config/clusters/jupyter-meets-the-earth/staging.values.yaml index 496a20e452..26aa468678 100644 --- a/config/clusters/jupyter-meets-the-earth/staging.values.yaml +++ b/config/clusters/jupyter-meets-the-earth/staging.values.yaml @@ -12,9 +12,6 @@ basehub: tls: - hosts: [staging.jmte.2i2c.cloud] secretName: https-auto-tls - singleuser: extraEnv: - # This bucket is created via terraform. - SCRATCH_BUCKET: s3://jupyter-meets-the-earth-staging-scratch/$(JUPYTERHUB_USER) - PANGEO_SCRATCH: s3://jupyter-meets-the-earth-staging-scratch/$(JUPYTERHUB_USER) + SCRATCH_BUCKET: s3://jupyter-meets-the-earth-scratch-staging/$(JUPYTERHUB_USER) diff --git a/config/clusters/kitware/prod.values.yaml b/config/clusters/kitware/prod.values.yaml index 0a678505e9..6691d494be 100644 --- a/config/clusters/kitware/prod.values.yaml +++ b/config/clusters/kitware/prod.values.yaml @@ -12,3 +12,6 @@ jupyterhub: config: GitHubOAuthenticator: oauth_callback_url: https://kitware.2i2c.cloud/hub/oauth_callback + singleuser: + extraEnv: + SCRATCH_BUCKET: s3://kitware-scratch/$(JUPYTERHUB_USER) diff --git a/config/clusters/kitware/staging.values.yaml b/config/clusters/kitware/staging.values.yaml index 1da48d161c..f25ae98680 100644 --- a/config/clusters/kitware/staging.values.yaml +++ b/config/clusters/kitware/staging.values.yaml @@ -12,3 +12,6 @@ jupyterhub: config: GitHubOAuthenticator: oauth_callback_url: https://staging.kitware.2i2c.cloud/hub/oauth_callback + singleuser: + extraEnv: + SCRATCH_BUCKET: s3://kitware-scratch-staging/$(JUPYTERHUB_USER) diff --git a/config/clusters/nasa-ghg/prod.values.yaml b/config/clusters/nasa-ghg/prod.values.yaml index f0444fcb98..8ac2a44230 100644 --- a/config/clusters/nasa-ghg/prod.values.yaml +++ b/config/clusters/nasa-ghg/prod.values.yaml @@ -16,3 +16,6 @@ basehub: config: GitHubOAuthenticator: oauth_callback_url: https://hub.ghg.center/hub/oauth_callback + singleuser: + extraEnv: + SCRATCH_BUCKET: s3://nasa-ghg-hub-scratch/$(JUPYTERHUB_USER) diff --git a/config/clusters/nasa-ghg/staging.values.yaml b/config/clusters/nasa-ghg/staging.values.yaml index c0da76fc18..ae1915a722 100644 --- a/config/clusters/nasa-ghg/staging.values.yaml +++ b/config/clusters/nasa-ghg/staging.values.yaml @@ -16,3 +16,6 @@ basehub: config: GitHubOAuthenticator: oauth_callback_url: https://staging.ghg.2i2c.cloud/hub/oauth_callback + singleuser: + extraEnv: + SCRATCH_BUCKET: s3://nasa-ghg-hub-scratch-staging/$(JUPYTERHUB_USER) diff --git a/config/clusters/nasa-veda/prod.values.yaml b/config/clusters/nasa-veda/prod.values.yaml index 24d6bc29f5..414d36a39c 100644 --- a/config/clusters/nasa-veda/prod.values.yaml +++ b/config/clusters/nasa-veda/prod.values.yaml @@ -16,3 +16,6 @@ basehub: config: GitHubOAuthenticator: oauth_callback_url: https://hub.openveda.cloud/hub/oauth_callback + singleuser: + extraEnv: + SCRATCH_BUCKET: s3://nasa-veda-scratch/$(JUPYTERHUB_USER) diff --git a/config/clusters/nasa-veda/staging.values.yaml b/config/clusters/nasa-veda/staging.values.yaml index 3c052540c0..e3c5e89718 100644 --- a/config/clusters/nasa-veda/staging.values.yaml +++ b/config/clusters/nasa-veda/staging.values.yaml @@ -4,6 +4,8 @@ basehub: eks.amazonaws.com/role-arn: arn:aws:iam::444055461661:role/nasa-veda-staging jupyterhub: singleuser: + extraEnv: + SCRATCH_BUCKET: s3://nasa-veda-scratch-staging/$(JUPYTERHUB_USER) initContainers: - &volume_ownership_fix_initcontainer name: volume-mount-ownership-fix diff --git a/config/clusters/opensci/staging.values.yaml b/config/clusters/opensci/staging.values.yaml index ec2d10d73c..55862c5b10 100644 --- a/config/clusters/opensci/staging.values.yaml +++ b/config/clusters/opensci/staging.values.yaml @@ -1,3 +1,8 @@ +userServiceAccount: + enabled: true + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::211125293633:role/opensci-staging + jupyterhub: ingress: hosts: @@ -28,6 +33,8 @@ jupyterhub: name: "" url: "" singleuser: + extraEnv: + SCRATCH_BUCKET: s3://opensci-scratch-staging/$(JUPYTERHUB_USER) profileList: - display_name: "Only Profile Available, this info is not shown in the UI" slug: only-choice diff --git a/config/clusters/smithsonian/prod.values.yaml b/config/clusters/smithsonian/prod.values.yaml index 3dcd0fca27..11d1b132f8 100644 --- a/config/clusters/smithsonian/prod.values.yaml +++ b/config/clusters/smithsonian/prod.values.yaml @@ -1,7 +1,19 @@ basehub: + userServiceAccount: + enabled: true + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::969396938818:role/smithsonian-prod + jupyterhub: ingress: hosts: [smithsonian.2i2c.cloud] tls: - hosts: [smithsonian.2i2c.cloud] secretName: https-auto-tls + hub: + config: + GitHubOAuthenticator: + oauth_callback_url: https://smithsonian.2i2c.cloud/hub/oauth_callback + singleuser: + extraEnv: + SCRATCH_BUCKET: s3://smithsonian-scratch/$(JUPYTERHUB_USER) diff --git a/config/clusters/smithsonian/staging.values.yaml b/config/clusters/smithsonian/staging.values.yaml index ec0105a61f..99ab548c44 100644 --- a/config/clusters/smithsonian/staging.values.yaml +++ b/config/clusters/smithsonian/staging.values.yaml @@ -1,7 +1,19 @@ basehub: + userServiceAccount: + enabled: true + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::969396938818:role/smithsonian-staging + jupyterhub: ingress: hosts: [staging.smithsonian.2i2c.cloud] tls: - hosts: [staging.smithsonian.2i2c.cloud] secretName: https-auto-tls + hub: + config: + GitHubOAuthenticator: + oauth_callback_url: https://staging.smithsonian.2i2c.cloud/hub/oauth_callback + singleuser: + extraEnv: + SCRATCH_BUCKET: s3://smithsonian-scratch-staging/$(JUPYTERHUB_USER) diff --git a/config/clusters/ubc-eoas/prod.values.yaml b/config/clusters/ubc-eoas/prod.values.yaml index cd7601b9d8..9feba0f89f 100644 --- a/config/clusters/ubc-eoas/prod.values.yaml +++ b/config/clusters/ubc-eoas/prod.values.yaml @@ -1,3 +1,8 @@ +userServiceAccount: + enabled: true + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::259060176665:role/ubc-eoas-prod + jupyterhub: ingress: hosts: [ubc-eoas.2i2c.cloud] @@ -8,3 +13,6 @@ jupyterhub: config: CILogonOAuthenticator: oauth_callback_url: https://ubc-eoas.2i2c.cloud/hub/oauth_callback + singleuser: + extraEnv: + SCRATCH_BUCKET: s3://ubc-eoas-scratch/$(JUPYTERHUB_USER) diff --git a/config/clusters/ubc-eoas/staging.values.yaml b/config/clusters/ubc-eoas/staging.values.yaml index 91e2f1f7ab..6e39d8637a 100644 --- a/config/clusters/ubc-eoas/staging.values.yaml +++ b/config/clusters/ubc-eoas/staging.values.yaml @@ -1,3 +1,8 @@ +userServiceAccount: + enabled: true + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::259060176665:role/ubc-eoas-staging + jupyterhub: ingress: hosts: [staging.ubc-eoas.2i2c.cloud] @@ -8,3 +13,6 @@ jupyterhub: config: CILogonOAuthenticator: oauth_callback_url: https://staging.ubc-eoas.2i2c.cloud/hub/oauth_callback + singleuser: + extraEnv: + SCRATCH_BUCKET: s3://ubc-eoas-scratch-staging/$(JUPYTERHUB_USER) diff --git a/config/clusters/victor/prod.values.yaml b/config/clusters/victor/prod.values.yaml index 1e4a2cf50e..420f90f729 100644 --- a/config/clusters/victor/prod.values.yaml +++ b/config/clusters/victor/prod.values.yaml @@ -13,3 +13,6 @@ basehub: config: GitHubOAuthenticator: oauth_callback_url: https://victor.2i2c.cloud/hub/oauth_callback + singleuser: + extraEnv: + SCRATCH_BUCKET: s3://victor-scratch/$(JUPYTERHUB_USER) diff --git a/config/clusters/victor/staging.values.yaml b/config/clusters/victor/staging.values.yaml index c0e824090a..6595284a66 100644 --- a/config/clusters/victor/staging.values.yaml +++ b/config/clusters/victor/staging.values.yaml @@ -14,6 +14,8 @@ basehub: GitHubOAuthenticator: oauth_callback_url: https://staging.victor.2i2c.cloud/hub/oauth_callback singleuser: + extraEnv: + SCRATCH_BUCKET: s3://victor-scratch-staging/$(JUPYTERHUB_USER) profileList: # Create a small instance that can launch a custom image - display_name: "Bring your own image - Small: m5.large" diff --git a/terraform/aws/projects/2i2c-aws-us.tfvars b/terraform/aws/projects/2i2c-aws-us.tfvars index ab53710a46..d55e3347c6 100644 --- a/terraform/aws/projects/2i2c-aws-us.tfvars +++ b/terraform/aws/projects/2i2c-aws-us.tfvars @@ -14,7 +14,7 @@ user_buckets = { "scratch-dask-staging" : { "delete_after" : 7 }, - "scratch-researchdelight" : { + "scratch-showcase" : { "delete_after" : 7 }, "persistent-showcase" : { @@ -46,7 +46,7 @@ hub_cloud_permissions = { "showcase" : { "user-sa" : { bucket_admin_access : [ - "scratch-researchdelight", + "scratch-showcase", "persistent-showcase", ], }, diff --git a/terraform/aws/projects/jupyter-meets-the-earth.tfvars b/terraform/aws/projects/jupyter-meets-the-earth.tfvars index 85fe727fbe..1564f499f3 100644 --- a/terraform/aws/projects/jupyter-meets-the-earth.tfvars +++ b/terraform/aws/projects/jupyter-meets-the-earth.tfvars @@ -11,6 +11,9 @@ user_buckets = { "scratch-staging" : { "delete_after" : 7 }, + // IMPORTANT: This bucket isn't used, they are instead using s3://jmte-scratch + // that doesn't have a delete_after policy setup etc, but maybe + // they want to have. "scratch" : { "delete_after" : 7 },