diff --git a/config/clusters/2i2c/binderhub-ui-demo.values.yaml b/config/clusters/2i2c/binderhub-ui-demo.values.yaml index 046d976226..33bf4519dc 100644 --- a/config/clusters/2i2c/binderhub-ui-demo.values.yaml +++ b/config/clusters/2i2c/binderhub-ui-demo.values.yaml @@ -33,6 +33,10 @@ jupyterhub: singleuserAdmin: extraVolumeMounts: [] singleuser: + nodeSelector: + # Schedule users on the smallest instance + # https://github.com/2i2c-org/infrastructure/issues/4241 + node.kubernetes.io/instance-type: n2-highmem-4 storage: type: none extraVolumeMounts: [] @@ -68,6 +72,11 @@ jupyterhub: username_claim: "email" binderhub-service: enabled: true + dockerApi: + nodeSelector: + # Schedule dockerApi pods to run on the smallest user nodes only + # https://github.com/2i2c-org/infrastructure/issues/4241 + node.kubernetes.io/instance-type: n2-highmem-4 ingress: enabled: true hosts: @@ -77,6 +86,11 @@ binderhub-service: hosts: - binderhub-ui-demo.2i2c.cloud config: + KubernetesBuildExecutor: + nodeSelector: + # Schedule builder pods to run on the smallest user nodes only + # https://github.com/2i2c-org/infrastructure/issues/4241 + node.kubernetes.io/instance-type: n2-highmem-4 BinderHub: base_url: / hub_url: https://hub.binderhub-ui-demo.2i2c.cloud diff --git a/config/clusters/hhmi/binder.values.yaml b/config/clusters/hhmi/binder.values.yaml index ae514b81df..262f4c7083 100644 --- a/config/clusters/hhmi/binder.values.yaml +++ b/config/clusters/hhmi/binder.values.yaml @@ -20,6 +20,10 @@ jupyterhub: templateVars: enabled: false singleuser: + nodeSelector: + # Schedule users on the smallest instance + # https://github.com/2i2c-org/infrastructure/issues/4241 + node.kubernetes.io/instance-type: n2-highmem-4 cloudMetadata: blockWithIptables: false cpu: @@ -81,6 +85,11 @@ jupyterhub: - access:services!service=binder binderhub-service: enabled: true + dockerApi: + nodeSelector: + # Schedule dockerApi pods to run on the smallest user nodes only + # https://github.com/2i2c-org/infrastructure/issues/4241 + node.kubernetes.io/instance-type: n2-highmem-4 ingress: enabled: true hosts: [binder.hhmi.2i2c.cloud] @@ -88,6 +97,11 @@ binderhub-service: - hosts: [binder.hhmi.2i2c.cloud] secretName: binder-https-auto-tls config: + KubernetesBuildExecutor: + nodeSelector: + # Schedule builder pods to run on the smallest user nodes only + # https://github.com/2i2c-org/infrastructure/issues/4241 + node.kubernetes.io/instance-type: n2-highmem-4 BinderHub: base_url: / hub_url: https://hub.binder.hhmi.2i2c.cloud diff --git a/config/clusters/opensci/big-binder.values.yaml b/config/clusters/opensci/big-binder.values.yaml index e58af60fa8..77229d26da 100644 --- a/config/clusters/opensci/big-binder.values.yaml +++ b/config/clusters/opensci/big-binder.values.yaml @@ -48,6 +48,10 @@ jupyterhub: singleuserAdmin: extraVolumeMounts: [] singleuser: + nodeSelector: + # Schedule users on the smallest instance + # https://github.com/2i2c-org/infrastructure/issues/4241 + node.kubernetes.io/instance-type: r5.xlarge cloudMetadata: blockWithIptables: false cpu: @@ -89,6 +93,11 @@ jupyterhub: - access:services!service=binder binderhub-service: enabled: true + dockerApi: + nodeSelector: + # Schedule dockerApi pods to run on the smallest user nodes only + # https://github.com/2i2c-org/infrastructure/issues/4241 + node.kubernetes.io/instance-type: r5.xlarge ingress: enabled: true hosts: [big.binder.opensci.2i2c.cloud] @@ -96,6 +105,11 @@ binderhub-service: - hosts: [big.binder.opensci.2i2c.cloud] secretName: binder-https-auto-tls config: + KubernetesBuildExecutor: + nodeSelector: + # Schedule builder pods to run on the smallest user nodes only + # https://github.com/2i2c-org/infrastructure/issues/4241 + node.kubernetes.io/instance-type: r5.xlarge BinderHub: base_url: / hub_url: https://hub.big.binder.opensci.2i2c.cloud diff --git a/config/clusters/opensci/small-binder.values.yaml b/config/clusters/opensci/small-binder.values.yaml index 040850e347..8882804b5f 100644 --- a/config/clusters/opensci/small-binder.values.yaml +++ b/config/clusters/opensci/small-binder.values.yaml @@ -17,6 +17,10 @@ jupyterhub: templateVars: enabled: false singleuser: + nodeSelector: + # Schedule users on the smallest instance + # https://github.com/2i2c-org/infrastructure/issues/4241 + node.kubernetes.io/instance-type: r5.xlarge cpu: limit: 2 memory: @@ -76,6 +80,11 @@ jupyterhub: - access:services!service=binder binderhub-service: enabled: true + dockerApi: + nodeSelector: + # Schedule dockerApi pods to run on the smallest user nodes only + # https://github.com/2i2c-org/infrastructure/issues/4241 + node.kubernetes.io/instance-type: r5.xlarge ingress: enabled: true hosts: [binder.opensci.2i2c.cloud] @@ -83,6 +92,11 @@ binderhub-service: - hosts: [binder.opensci.2i2c.cloud] secretName: binder-https-auto-tls config: + KubernetesBuildExecutor: + nodeSelector: + # Schedule builder pods to run on the smallest user nodes only + # https://github.com/2i2c-org/infrastructure/issues/4241 + node.kubernetes.io/instance-type: r5.xlarge BinderHub: base_url: / hub_url: https://hub.binder.opensci.2i2c.cloud diff --git a/config/clusters/projectpythia/pythia-binder.values.yaml b/config/clusters/projectpythia/pythia-binder.values.yaml index 1f7426e057..69f533adf3 100644 --- a/config/clusters/projectpythia/pythia-binder.values.yaml +++ b/config/clusters/projectpythia/pythia-binder.values.yaml @@ -16,6 +16,10 @@ jupyterhub: singleuserAdmin: extraVolumeMounts: [] singleuser: + nodeSelector: + # Schedule users on the smallest instance + # https://github.com/2i2c-org/infrastructure/issues/4241 + node.kubernetes.io/instance-type: r5.xlarge storage: type: none extraVolumeMounts: [] @@ -74,6 +78,11 @@ jupyterhub: - access:services!service=binder binderhub-service: enabled: true + dockerApi: + nodeSelector: + # Schedule dockerApi pods to run on the smallest user nodes only + # https://github.com/2i2c-org/infrastructure/issues/4241 + node.kubernetes.io/instance-type: r5.xlarge ingress: enabled: true hosts: [binder.pythia.2i2c.cloud] @@ -81,6 +90,11 @@ binderhub-service: - hosts: [binder.pythia.2i2c.cloud] secretName: binder-https-auto-tls config: + KubernetesBuildExecutor: + nodeSelector: + # Schedule builder pods to run on the smallest user nodes only + # https://github.com/2i2c-org/infrastructure/issues/4241 + node.kubernetes.io/instance-type: r5.xlarge GitHubRepoProvider: allowed_specs: - ^ProjectPythia/.*$ diff --git a/config/clusters/projectpythia/testing.values.yaml b/config/clusters/projectpythia/testing.values.yaml deleted file mode 100644 index 3b9feaed07..0000000000 --- a/config/clusters/projectpythia/testing.values.yaml +++ /dev/null @@ -1,67 +0,0 @@ -jupyterhub: - ingress: - hosts: [hub.binder.pythia.2i2c.cloud] - tls: - - hosts: [hub.binder.pythia.2i2c.cloud] - secretName: https-auto-tls - custom: - binderhubUI: - enabled: true - 2i2c: - add_staff_user_ids_to_admin_users: false - jupyterhubConfigurator: - enabled: false - singleuserAdmin: - extraVolumeMounts: [] - singleuser: - storage: - type: none - extraVolumeMounts: [] - initContainers: [] - profileList: [] - hub: - redirectToServer: false - loadRoles: - binder: - services: - - binder - scopes: - - servers - - admin:users - user: - scopes: - - self - # Admin users will by default have access:services, so this is only - # observed to be required for non-admin users. - - access:services!service=binder -binderhub-service: - enabled: true - ingress: - enabled: true - hosts: [binder.pythia.2i2c.cloud] - tls: - - hosts: [binder.pythia.2i2c.cloud] - secretName: binder-https-auto-tls - config: - BinderHub: - base_url: / - hub_url: https://hub.binder.pythia.2i2c.cloud - badge_base_url: https://binder.pythia.2i2c.cloud - auth_enabled: false - enable_api_only_mode: false - banner_message: "" - about_message: Binder for use with Project Pythia - extraEnv: - - name: JUPYTERHUB_API_TOKEN - valueFrom: - secretKeyRef: - name: hub - key: hub.services.binder.apiToken - - name: JUPYTERHUB_CLIENT_ID - value: "service-binder" - - name: JUPYTERHUB_API_URL - value: "https://hub.binder.pythia.2i2c.cloud/hub/api" - # Without this, the redirect URL to /hub/api/... gets - # appended to binderhub's URL instead of the hub's - - name: JUPYTERHUB_BASE_URL - value: "https://hub.binder.pythia.2i2c.cloud/" diff --git a/config/clusters/templates/common/binderhub-ui-hub.values.yaml b/config/clusters/templates/common/binderhub-ui-hub.values.yaml index 59f0046908..23dac60e7e 100644 --- a/config/clusters/templates/common/binderhub-ui-hub.values.yaml +++ b/config/clusters/templates/common/binderhub-ui-hub.values.yaml @@ -35,6 +35,14 @@ jupyterhub: singleuserAdmin: extraVolumeMounts: [] singleuser: + nodeSelector: + # Schedule users on the smallest instance + # https://github.com/2i2c-org/infrastructure/issues/4241 + {% if provider == "gcp" %} + node.kubernetes.io/instance-type: n2-highmem-4 + {% else %} + node.kubernetes.io/instance-type: r5.xlarge + {% endif %} storage: type: none extraVolumeMounts: [] @@ -119,6 +127,15 @@ jupyterhub: - access:services!service=binder binderhub-service: enabled: true + dockerApi: + nodeSelector: + # Schedule dockerApi pods to run on the smallest user nodes only + # https://github.com/2i2c-org/infrastructure/issues/4241 + {% if provider == "gcp" %} + node.kubernetes.io/instance-type: n2-highmem-4 + {% else %} + node.kubernetes.io/instance-type: r5.xlarge + {% endif %} ingress: enabled: true hosts: [{{ binderhub_domain }}] @@ -126,6 +143,15 @@ binderhub-service: - hosts: [{{ binderhub_domain }}] secretName: binder-https-auto-tls config: + KubernetesBuildExecutor: + nodeSelector: + # Schedule builder pods to run on the smallest user nodes only + # https://github.com/2i2c-org/infrastructure/issues/4241 + {% if provider == "gcp" %} + node.kubernetes.io/instance-type: n2-highmem-4 + {% else %} + node.kubernetes.io/instance-type: r5.xlarge + {% endif %} BinderHub: base_url: / hub_url: https://{{ jupyterhub_domain }} diff --git a/deployer/commands/generate/hub_asset/hub_files.py b/deployer/commands/generate/hub_asset/hub_files.py index cef8fb7cb3..d2fd01e0df 100644 --- a/deployer/commands/generate/hub_asset/hub_files.py +++ b/deployer/commands/generate/hub_asset/hub_files.py @@ -86,6 +86,9 @@ def binderhub_ui_values_file( cluster_name: str = typer.Option( ..., prompt="Name of the cluster where the hub will live" ), + provider: str = typer.Option( + ..., prompt="Name of the provider where the hub will live (gcp/aws)" + ), jupyterhub_domain: str = typer.Option( ..., prompt="Domain where jupyterhub will run (ex. hub.binder.community.2i2c.cloud)", @@ -127,6 +130,7 @@ def binderhub_ui_values_file( vars = { "cluster_name": cluster_name, + "provider": provider, "jupyterhub_domain": jupyterhub_domain, "binderhub_domain": binderhub_domain, "authenticator": authenticator, diff --git a/docs/howto/features/binderhub-ui.md b/docs/howto/features/binderhub-ui.md index 471a63d64e..283ffef54c 100644 --- a/docs/howto/features/binderhub-ui.md +++ b/docs/howto/features/binderhub-ui.md @@ -68,7 +68,7 @@ Some of the configuration that gets inherited either from the `basehub` defaults initContainers: [] profileList: [] ``` -#### 2. Check jupyterhub and binderhub domains setup +#### 1. Check jupyterhub and binderhub domains setup Having separate domains for both jupyterhub and binderhub will help with having clean and correct sharing URLs without having them be based off the `hub/services/:name` path. @@ -94,7 +94,7 @@ binderhub-service: secretName: binder-https-auto-tls ``` -#### 3. Check that binderhubUI is enabled +#### 1. Check that binderhubUI is enabled Enable `jupyterhub.custom.binderhubUI` which will in turn enable the hub to use [BinderSpawnerMixin](https://github.com/jupyterhub/binderhub/blob/bd297b2c3f713cf46b0b22cfabc86d8140bbed41/helm-chart/binderhub/values.yaml#L115-L207) that allows converting JupyterHub container spawners into BinderHub spawners @@ -105,7 +105,7 @@ jupyterhub: enabled: true ``` -#### 4. Check that the binderhub-service chart is enabled +#### 1. Check that the binderhub-service chart is enabled We will use the [binderhub-service](https://github.com/2i2c-org/binderhub-service/) Helm chart to run BinderHub, the Python software, as a standalone service to build and push images with [repo2docker](https://github.com/jupyterhub/repo2docker), next to JupyterHub so we need to enable it. @@ -114,7 +114,7 @@ binderhub-service: enabled: true ``` -#### 5. Check that BinderHub is configured correctly +#### 1. Check that BinderHub is configured correctly We need to configure BinderHub so that: @@ -131,7 +131,33 @@ binderhub-service: enable_api_only_mode: false ``` -#### 6. Check the binderhub extra env variables +#### 1. Check that the builder docker api and user pods are scheduled on the smallest available instance + +In general, for GCP, they should run on `n2-highmem-4` and on AWS they should be placed on `r5.xlarge` machines. But it's best to double-check the cluster's terraform or eksctl configuration files to make sure this is the smallest instance and not another one. + + +```yaml +binderhub-service: + dockerApi: + nodeSelector: + # Schedule dockerApi pods to run on the smallest user nodes only + # https://github.com/2i2c-org/infrastructure/issues/4241 + node.kubernetes.io/instance-type: n2-highmem-4 + config: + KubernetesBuildExecutor: + nodeSelector: + # Schedule builder pods to run on the smallest user nodes only + # https://github.com/2i2c-org/infrastructure/issues/4241 + node.kubernetes.io/instance-type: n2-highmem-4 +jupyterhub: + singleuser: + nodeSelector: + # Schedule users on the smallest instance + # https://github.com/2i2c-org/infrastructure/issues/4241 + node.kubernetes.io/instance-type: n2-highmem-4 +``` + +#### 1. Check the binderhub extra env variables These are needed by the jupyterhub software bits that the binderhub software uses.