From 91bb1217cbe8fe1647d65fb0f34f7eaffd1ef6f5 Mon Sep 17 00:00:00 2001 From: Silva Alejandro Ismael Date: Sun, 10 Dec 2023 10:29:14 -0300 Subject: [PATCH 1/3] Add earthscope cluster --- .../earthscope/enc-support.secret.values.yaml | 17 +++ .../clusters/earthscope/support.values.yaml | 28 ++++ eksctl/earthscope.jsonnet | 140 ++++++++++++++++++ eksctl/ssh-keys/earthscope.key.pub | 1 + eksctl/ssh-keys/secret/earthscope.key | 21 +++ terraform/aws/projects/earthscope.tfvars | 28 ++++ 6 files changed, 235 insertions(+) create mode 100644 config/clusters/earthscope/enc-support.secret.values.yaml create mode 100644 config/clusters/earthscope/support.values.yaml create mode 100644 eksctl/earthscope.jsonnet create mode 100644 eksctl/ssh-keys/earthscope.key.pub create mode 100644 eksctl/ssh-keys/secret/earthscope.key create mode 100644 terraform/aws/projects/earthscope.tfvars diff --git a/config/clusters/earthscope/enc-support.secret.values.yaml b/config/clusters/earthscope/enc-support.secret.values.yaml new file mode 100644 index 0000000000..5326c1c7e6 --- /dev/null +++ b/config/clusters/earthscope/enc-support.secret.values.yaml @@ -0,0 +1,17 @@ +prometheusIngressAuthSecret: + username: ENC[AES256_GCM,data:IyOeJ2ltWgjhRzUrspPpfSpvkgFfAGwrmW2tlBxwoWYWK97hqSn6TAs5M+lT/kp7AICAMflPj1k3IazNiSdvXw==,iv:2vLBAANHd3T+gyYoBVs69ryC4cRQvnc7WufaF3ZliDs=,tag:+Tbpy2ezJTV4xuhA6WXWXg==,type:str] + password: ENC[AES256_GCM,data:+LSkp7SQSr6vW4b9rGPgj3I495DquL7uXGo6zbZhfqqqgCTsKgVKNpPFt5Y05TqbOw0fl5qnB6jezNzR05+SBQ==,iv:mtrFlfCbvb4CNjlopzSKL2Q9pLnNjXZw5kB238RFfXQ=,tag:fXhkUIVZ02P0fr1jyx08fg==,type:str] +sops: + kms: [] + gcp_kms: + - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs + created_at: "2023-12-10T12:56:11Z" + enc: CiUA4OM7eKM2uAp93NZo20W2g59kkkg+f9lIdX3X3C5yMr18KqNWEkkAjTWv+u4Qk+RFc9++zwDFIdNUIJglHKdQjyrPgmKu7Y2GjH8cCYQZnmRy5hABlKEuILmRwdNPqjdfiqRsCwGUbPBZhpIbwUjO + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2023-12-10T12:56:13Z" + mac: ENC[AES256_GCM,data:c4cVSM6GfV9ysrVxx8SYtLHuub9qJc4i8S4XsRrbkJONZgledsuFm65Qe7/yfpypNQd1cJMgHK2kSjEbVfBgO+FMUPHc3dPgG0UjyzkFjTgmzQpDlcyZ0sPrSkCm2RAVLBB0y3qGA8/4jq4gRp+hVgNlhW5q8QKNDYMmR8IK+MU=,iv:+tqnpit70z5KQwFjYU8EdunSFBWx41seakSDDTLtpaI=,tag:HXboDkt6hPrzsX2xY4roiw==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.8.1 diff --git a/config/clusters/earthscope/support.values.yaml b/config/clusters/earthscope/support.values.yaml new file mode 100644 index 0000000000..a1060d493e --- /dev/null +++ b/config/clusters/earthscope/support.values.yaml @@ -0,0 +1,28 @@ +prometheusIngressAuthSecret: + enabled: true + +prometheus: + server: + ingress: + enabled: true + hosts: + - prometheus.earthscope.2i2c.cloud + tls: + - secretName: prometheus-tls + hosts: + - prometheus.earthscope.2i2c.cloud + +grafana: + grafana.ini: + server: + root_url: https://grafana.earthscope.2i2c.cloud/ + auth.github: + enabled: true + allowed_organizations: 2i2c-org + ingress: + hosts: + - grafana.earthscope.2i2c.cloud + tls: + - secretName: grafana-tls + hosts: + - grafana.earthscope.2i2c.cloud diff --git a/eksctl/earthscope.jsonnet b/eksctl/earthscope.jsonnet new file mode 100644 index 0000000000..90926c0cef --- /dev/null +++ b/eksctl/earthscope.jsonnet @@ -0,0 +1,140 @@ +/* + This file is a jsonnet template of a eksctl's cluster configuration file, + that is used with the eksctl CLI to both update and initialize an AWS EKS + based cluster. + + This file has in turn been generated from eksctl/template.jsonnet which is + relevant to compare with for changes over time. + + To use jsonnet to generate an eksctl configuration file from this, do: + + jsonnet earthscope.jsonnet > earthscope.eksctl.yaml + + References: + - https://eksctl.io/usage/schema/ +*/ +local ng = import "./libsonnet/nodegroup.jsonnet"; + +// place all cluster nodes here +local clusterRegion = "us-east-2"; +local masterAzs = ["us-east-2a", "us-east-2b", "us-east-2c"]; +local nodeAz = "us-east-2a"; + +// Node definitions for notebook nodes. Config here is merged +// with our notebook node definition. +// A `node.kubernetes.io/instance-type label is added, so pods +// can request a particular kind of node with a nodeSelector +local notebookNodes = [ + { instanceType: "r5.xlarge" }, + { instanceType: "r5.4xlarge" }, + { instanceType: "r5.16xlarge" }, +]; +local daskNodes = [ + // Node definitions for dask worker nodes. Config here is merged + // with our dask worker node definition, which uses spot instances. + // A `node.kubernetes.io/instance-type label is set to the name of the + // *first* item in instanceDistribution.instanceTypes, to match + // what we do with notebook nodes. Pods can request a particular + // kind of node with a nodeSelector + // + // A not yet fully established policy is being developed about using a single + // node pool, see https://github.com/2i2c-org/infrastructure/issues/2687. + // + { instancesDistribution+: { instanceTypes: ["r5.4xlarge"] }}, +]; + + +{ + apiVersion: 'eksctl.io/v1alpha5', + kind: 'ClusterConfig', + metadata+: { + name: "earthscope", + region: clusterRegion, + version: "1.27", + }, + availabilityZones: masterAzs, + iam: { + withOIDC: true, + }, + // If you add an addon to this config, run the create addon command. + // + // eksctl create addon --config-file=earthscope.eksctl.yaml + // + addons: [ + { + // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that + // couple to AWS EBS based storage, without it expect to see pods + // mounting a PVC failing to schedule and PVC resources that are + // unbound. + // + // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html + // + name: 'aws-ebs-csi-driver', + version: "latest", + wellKnownPolicies: { + ebsCSIController: true, + }, + }, + ], + nodeGroups: [ + ng + { + namePrefix: 'core', + nameSuffix: 'a', + nameIncludeInstanceType: false, + availabilityZones: [nodeAz], + ssh: { + publicKeyPath: 'ssh-keys/earthscope.key.pub' + }, + instanceType: "r5.xlarge", + minSize: 1, + maxSize: 6, + labels+: { + "hub.jupyter.org/node-purpose": "core", + "k8s.dask.org/node-purpose": "core" + }, + }, + ] + [ + ng + { + namePrefix: 'nb', + availabilityZones: [nodeAz], + minSize: 0, + maxSize: 500, + instanceType: n.instanceType, + ssh: { + publicKeyPath: 'ssh-keys/earthscope.key.pub' + }, + labels+: { + "hub.jupyter.org/node-purpose": "user", + "k8s.dask.org/node-purpose": "scheduler" + }, + taints+: { + "hub.jupyter.org_dedicated": "user:NoSchedule", + "hub.jupyter.org/dedicated": "user:NoSchedule" + }, + } + n for n in notebookNodes + ] + ( if daskNodes != null then + [ + ng + { + namePrefix: 'dask', + availabilityZones: [nodeAz], + minSize: 0, + maxSize: 500, + ssh: { + publicKeyPath: 'ssh-keys/earthscope.key.pub' + }, + labels+: { + "k8s.dask.org/node-purpose": "worker" + }, + taints+: { + "k8s.dask.org_dedicated" : "worker:NoSchedule", + "k8s.dask.org/dedicated" : "worker:NoSchedule" + }, + instancesDistribution+: { + onDemandBaseCapacity: 0, + onDemandPercentageAboveBaseCapacity: 0, + spotAllocationStrategy: "capacity-optimized", + }, + } + n for n in daskNodes + ] else [] + ) +} \ No newline at end of file diff --git a/eksctl/ssh-keys/earthscope.key.pub b/eksctl/ssh-keys/earthscope.key.pub new file mode 100644 index 0000000000..a8f5253c5b --- /dev/null +++ b/eksctl/ssh-keys/earthscope.key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChGAZs8MNn5tVcfwFl6xsSBJYDE/Vm+RvxuehIlDzj3S9BRZvSp2beHMLgPZ0tJmooxlmAz0vbOrCjCzGcUiegyKT5elecQZPY8bewsLCG5YaZ+YYDPIEEftVHHH9UGYeLA59+WocviRJkbiODGxQuY6As9z1cCtNzCV47HiHOGPIz4fePvPxH0USwQmZncUasp9QGOwKkoT6iExfZzT7o94H7xYe5i29pe4x4NMPdgw6xPYbl9XSAidG9LkV0PHb+vv4sVeBkp50lIeRKmg4hZ6wqEsrG2+y8DMU9YPq0OJLck3lzxl0j0qDQanSzkbV02aHqw/nUmg8vqDazcUXFP8Q8s9SHLfgb6u2kchNGHaX3o3MSsU95/X3xgwPD4ZtdcUvM9GgJD0n5m3SznmxFeBv69fhXGBqwIILem8FjTbAkDa27dnvJJ1jUnYIOsYdPuA+nmcS/Om6UKEB0ChG7obU0/QN0lWjZciR77dVggOgP56A5XbwmaeDELt3KStk= starlord@genos diff --git a/eksctl/ssh-keys/secret/earthscope.key b/eksctl/ssh-keys/secret/earthscope.key new file mode 100644 index 0000000000..196adaaf2a --- /dev/null +++ b/eksctl/ssh-keys/secret/earthscope.key @@ -0,0 +1,21 @@ +{ + "data": "ENC[AES256_GCM,data:gceWUhWoo1TdUSxNgTMpasRydoUKvWqoauIR6sQxCIkdzEjTV6/8VpZPxxiESNXwYWN9VwNujEBM7ZS3iCSjOtVEvUud61u6B1DHQ5mi14gwcWiZfiiACaKsxfQ2u9UDYEPuiQmczzK3JuzakWndR7f5Mx85ylipMy3gTkc9GkvBHnbiQt03B+7Dqjiz8Cz+OtCYlteRo8UjXrEAB3TB+sNBXANOLoXjaDwS6hC/sBgorm3WvPyNdBqmSKswCb5R4AkdR+kzdWYV+vfjLSbYZGKDmTxrEHGP494fO0YG3XmConI+X1rwasHZ+V0FZfdIVpUp85cIC2JhM0OyaDpkvRynXO8MJggzZdbvj/BEPnTA+TINE/oNph73qOelpmVlfDUVGD973WYyJLz7oKbZTxsiCEVh80q14OemUxMsx7CPwdXRxN43A5afQ+Xd+exHuE6xEHYVrW2xa2dsqqdr7ztysvuWPYuNufJC+P5tCjhgzzZG/HNJm3bay0Mbvq6+ytKdZ2Qn7VYwXm9iHlYaGj1L3MV4n1gLpg+Gd1oH0hSmywtp4nI8yCLUOUIXXDYKo6U74PA7rdBgQD248zdP4WbZNfLlSrsdwtPZnB/mlo5pzx9qDsFbPjmuQyrv978w+lK09biR9tqck9AU031jZPmz7Xw+srJKONL/4hdxAdnIWYVl05ocPMl1330/Bg3QXEQTwpx7fuH2bVY7Hm6xxveM1r6GuA/ycMgX/b4xGuhtTXZnt/5CrVWeHsF2KDTL4O+cnlYC4J4ERFvx4I8qGEScZliYrXKzC07yBE2X2jXXB5Sawlz8bn0PF2KgHy0ySp3cMOfCgmbiEL89sliuSmqeMdrn3xkYl7MrmNYYLpS9OJu+ZkMiUN+f/0ojr9PqylHHhMrO9jwK2FqOuUzRRgezi6WDcWQHAjhH0FUZgowLDD6hprolIhNgYWIN3OnxYvVW85adawxigv6QbJY3FSJQXZfsE3qtxDw5/R4298NGSmn9PBhbQwVp86lNoKf1OE+Md33bDJJ1bsKMBLdSC9FBeJcMrntfa5msO4LcaORuGFPZ9iNZnUS8tc8fZVY4UT/PADjxPExEAndQIrPQwiOcs8YF9IaUa8zRTaOmmGfwLalWHcOLFloKGbEUo4YEr6nMu6A5zEmSHq3JqO4G93NMZn+qeQhJWjgkc8KU8k3jkH+D/mt5Ie+V2gVfFbCRpuiaVnqWrTLh15/y5McyxxiYP0Q1kJBJMEIDKWOsFZZb+b7XALLjAIY8o+5v/f9G2HeYavb+4gWqBVUvNKTYl0FguqhBxSvMBvC5zZF6rkeqLBstepeg1is8gqJV3A/cdCLcqHKynLDWCt5L88UI2OpPLH80jdcA5VkyCCXxl5UxWWfcM/X+jPdU+I/sfiyla7Azrk3J4a+84YjRw2to7jhHnZdZGW7YPqdKZOu8hzqq2mHfR0j0fD03U7TNO4Q6WfIcTRX5dKvwhupuPTFkcfREi6bKZSDyixyyQgRu1X372Cut+WfUcKJg40W39R4N6qeRcHBTNh9CrCokptBP9StvMtyt/FVJEkBTJZzbHEa/HBcjXFRGjXC1vSwWvRN8rePJOhsHo3aDZbSDTXIXUWHcU+0TCE+/b/+TQMUZ+VTIAM9gBR1JakXWVFWxm9d17Zr7M6Fk3UoRZ/3IpDFCBky+q0I05+4H2Zix6OPk8nSDF6WjkmW623UhsdHYC5KJQasylXRINbLYh/YfWqUYOMQw/WHxnZimgeJzY6Qw7DoxCWkuGdFTmRgyHRTK578bQnTqewZ8G6EB/nxJYTDNfptXAPmN/8A+Iw4Ha2Wejd3+VKniGVEgVrecvYTtqKQjWiraf515Tva0/mVfg1Orjz/8uRXAFqFX44b9eooQB1xEpEjhZrd9j2fjVpe3wfa0C/LQvhEAFB7V/sG3DemGY7GBsMmk2sWBsEO/O2j1Rs63oae4rWOg46jyQJ9hiS4D206KXcZbBBY6+qcpGtyVn9139n7qG4IXdCH+q5XEJ5mx07qPac8AC9V4OaeG+KGi/WqVTFOqU/BuiP3Aws2vW+a84ic9A4tSLA5v6c8uHr/SwxwQrjWLVF3Ktl2GFGi+nL6PnarJN80Efmrw4clY4NmyI7hzt9MBGJfrErHK5vatHfApMGb9tSnsOSXN7v4W71gKU/S4euYkEZGmuWYe/j0nTb1VMJZzg8LM2F+qWUxn4ph71UCiuFtmA3S22EvKuPghG8ALa6109kidj0RluKPxghnDrkFPGZE70YRrVqVqvjelBWEt0jIWy066ga7KdfhxnVtJ8BJ5Zzb3U8VPb7muKcAKlRAx/LX+yZYAmn/k9xLi5NOQ2CG56hWnMOAYpJ+FFtdF8485VIY7xzTf2pXxegvTM4iIUwSI8ITK8MnnefN2o6k9/npSz2HSKg6UaMt6Ua2TkmbtUkuL4q2JQmhM+2TVfSMds4wH7WTdSMKgCWiz+sIgI4BMbf2fnjPeUusobTIRuvKHkd4Yy3VDct/ihT1bYMz67l+MNPGv71V4PcqzmM/OyaRSSqxMEdkYo26C8E2eaQWV+Q2/fUje1BtYBoarJddzb5eRyGh3EXyZxomGa9dqPiuGL6KJez3L1GSn12I+TUmCEUF3hecc5YVYwVXVG/WciWeERH01cbMaZHcrkYEzETVweaKhjY6Gu/Vf8oU36505E48fO7+z087d2l6p+bvRTvlfj9PbuKEJ536uDjC9CMNKUIP4T3sUEZkxRTLk/eNuuOegTYCS110pXGqISREfAsya8lCjnCFttTR7kjx+rSnWRpi7uoYaAvzM/RAt/2lKYUpnXVe7gEktfkT2pI7aDkprX5U8T/cL6EzcpQFQYbpDf5YVHie0kRV/TVlyqaRSmElNw1LxzFF1EK75gbmIwZaTVy1EVNBWZMc8j3Gxp8vTayXOc6+95yhzWD+20Gv4IMt9mpsKxx93xbGpk6FnPlSWZa9VVW/XED/Neu4ipncyCLRSJEdrnGFOuyOHa+MTPSxGSSN3fTbWnlSHVFyn2w+WUhek2pnh4zkoXxn6LqOMWU8biNnTusoGXuihbox/3CUuUh58uMya5b3ljqt4iSjALqGkO/9k5pADNIM9+18A64ddyRqcLEyz6O+foHh62M9KWuVcrTAo2gxVWFTx8V4UiMu1j6u28O3MK1RPGO/6ji9W433XI6FGtBBMz35SGm4GOgnN8Cq3rFpNEuVyNY7QuftMUnCnXN5D9HSiTx4mAqQqpoZWn8kEOqqWCdMHe3mLUG+mazZHEQ9NybKG7ew7WcZPy4EwmrVKoUpIYJ7Y+an8ixkFgWaNXhWHE8R8hnLbO6RTiwSgj3ysh5j2qT2tr91TUIPsz7jBd0rPUY5uJyt6Z1Keuf8gHKqKTM/BUJbQZefsXxXwbcsfXbT9CUMY6gNaUHX5ENeTUapQtyZvFY1IqnFxrCtPTbRkNx+EBA==,iv:DuDPWeW38lHht+scY99sRCJ2qrJaYxOSmFPseuKtwXc=,tag:prH5Z4kNf+1daY5X12Ea/w==,type:str]", + "sops": { + "kms": null, + "gcp_kms": [ + { + "resource_id": "projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs", + "created_at": "2023-12-10T12:56:10Z", + "enc": "CiUA4OM7eI6hovem8F7/tJN3UDN94q8DTbmkxSwi1Qox3BMYpEBbEkkAjTWv+nupMoEKc6FE/9OPFjUyKXoJkBs7hmlFZ0KtBjJCmK3i/6coxoac5vApqqV2lTMIAclIqrrnNVKz7WzGhT3hyIHfKQjw" + } + ], + "azure_kv": null, + "hc_vault": null, + "age": null, + "lastmodified": "2023-12-10T12:56:11Z", + "mac": "ENC[AES256_GCM,data:iJ49V58dnR15SGZL0kpcPln2SaVq0JFDrdDRsfjuII7BurCfvK1ZFgufLH3ZLDCS0X99XnYok7uSHrwwh7NEeNU0AUYdJfZ2tntO5rFjS1TjpoKvh0+hQy8l91rCrl4oJGisFKHejXDGDod4/D+NtKkne0im2WkafFPy9adaQ20=,iv:GgjXbURrSp+9HrEtpJrtIKYPp2Extc9A7HnW/dPppSc=,tag:oLDkZ7pgf/tJXxT/hGQunA==,type:str]", + "pgp": null, + "unencrypted_suffix": "_unencrypted", + "version": "3.8.1" + } +} \ No newline at end of file diff --git a/terraform/aws/projects/earthscope.tfvars b/terraform/aws/projects/earthscope.tfvars new file mode 100644 index 0000000000..57aeb6fbf9 --- /dev/null +++ b/terraform/aws/projects/earthscope.tfvars @@ -0,0 +1,28 @@ +region = "us-east-2" + +cluster_name = "earthscope" + +cluster_nodes_location = "us-east-2a" + +user_buckets = { + "scratch-staging" : { + "delete_after" : 7 + }, + "scratch" : { + "delete_after" : 7 + }, +} + + +hub_cloud_permissions = { + "staging" : { + requestor_pays : true, + bucket_admin_access : ["scratch-staging"], + extra_iam_policy : "" + }, + "prod" : { + requestor_pays : true, + bucket_admin_access : ["scratch"], + extra_iam_policy : "" + }, +} \ No newline at end of file From ec631715913e986443356820d14ec973bd2d9b3a Mon Sep 17 00:00:00 2001 From: Silva Alejandro Ismael Date: Wed, 13 Dec 2023 13:05:55 -0300 Subject: [PATCH 2/3] Add support chart to earthscope cluster --- .github/workflows/deploy-hubs.yaml | 1 + config/clusters/earthscope/cluster.yaml | 12 +++++++++ config/clusters/earthscope/common.values.yaml | 12 +++++++++ .../enc-deployer-credentials.secret.json | 25 +++++++++++++++++++ .../earthscope/enc-grafana-token.secret.yaml | 15 +++++++++++ .../clusters/earthscope/support.values.yaml | 6 +++++ eksctl/earthscope.jsonnet | 2 +- 7 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 config/clusters/earthscope/cluster.yaml create mode 100644 config/clusters/earthscope/common.values.yaml create mode 100644 config/clusters/earthscope/enc-deployer-credentials.secret.json create mode 100644 config/clusters/earthscope/enc-grafana-token.secret.yaml diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index f91acf5e3e..f417ef0c71 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -206,6 +206,7 @@ jobs: failure_catalystproject-africa: "${{ env.failure_catalystproject-africa }}" failure_hhmi: "${{ env.failure_hhmi }}" failure_nasa-esdis: "${{ env.failure_nasa-esdis }}" + failure_earthscope: "${{ env.failure_earthscope }}" # Only run this job on pushes to the default branch and when the job output is not # an empty list diff --git a/config/clusters/earthscope/cluster.yaml b/config/clusters/earthscope/cluster.yaml new file mode 100644 index 0000000000..6c87d543ed --- /dev/null +++ b/config/clusters/earthscope/cluster.yaml @@ -0,0 +1,12 @@ +name: earthscope +provider: aws +aws: + key: enc-deployer-credentials.secret.json + clusterType: eks + clusterName: earthscope + region: us-east-2 +support: + helm_chart_values_files: + - support.values.yaml + - enc-support.secret.values.yaml +hubs: [] diff --git a/config/clusters/earthscope/common.values.yaml b/config/clusters/earthscope/common.values.yaml new file mode 100644 index 0000000000..09fddb90ea --- /dev/null +++ b/config/clusters/earthscope/common.values.yaml @@ -0,0 +1,12 @@ +nfs: + pv: + # from https://docs.aws.amazon.com/efs/latest/ug/mounting-fs-nfs-mount-settings.html + mountOptions: + - rsize=1048576 + - wsize=1048576 + - timeo=600 + - soft # We pick soft over hard, so NFS lockups don't lead to hung processes + - retrans=2 + - noresvport + serverIP: fs-08e7747330d833d82.efs.us-east-2.amazonaws.com + baseShareName: / diff --git a/config/clusters/earthscope/enc-deployer-credentials.secret.json b/config/clusters/earthscope/enc-deployer-credentials.secret.json new file mode 100644 index 0000000000..97a86ef90e --- /dev/null +++ b/config/clusters/earthscope/enc-deployer-credentials.secret.json @@ -0,0 +1,25 @@ +{ + "AccessKey": { + "AccessKeyId": "ENC[AES256_GCM,data:3/SbbOw+wTiuR/pAQX7YlzqZ+A0=,iv:xLjEyZGkbaQ3eVcsfsoFInazu52pT++HtqmbMYFO830=,tag:h8u13XufpQEQzuntIERpfQ==,type:str]", + "SecretAccessKey": "ENC[AES256_GCM,data:PAVaW/JJ93Lp3Fx+SzN7ZWZ311NuXjK4E7Fc9kRPxXDmQ/oVScCNYQ==,iv:8BEyHh9q+quWTQGpFYcTaXQcu1G9dSH6xhxLded9ht8=,tag:ut13RYce6EJ8QzBXw5oz1w==,type:str]", + "UserName": "ENC[AES256_GCM,data:K5C/1DqLJQrmifgZdWirxa3ws5vmV4M=,iv:raMPP4vlfmLsiLkgeyzBmZjffNLnVKFrS2A9S5DfNLM=,tag:wCMDinJkFGAnvIIIvliF0w==,type:str]" + }, + "sops": { + "kms": null, + "gcp_kms": [ + { + "resource_id": "projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs", + "created_at": "2023-12-12T16:24:08Z", + "enc": "CiUA4OM7eOXl1aeA440/OPv/B9ma2zG5unKWGkDFcWuyjc5s3zg6EkkAjTWv+nwxhn+vpS6YKnVaaH/RXfstRskGPc3dp2i0uPgew3epepkKiLgOgC83b49xR6hlUIWYaShh0sj756PXEXi9O4HFGAtx" + } + ], + "azure_kv": null, + "hc_vault": null, + "age": null, + "lastmodified": "2023-12-12T16:24:12Z", + "mac": "ENC[AES256_GCM,data:6wtwcF2rP6HQzbhaPv3hjpMD9nQbgOeFcuUkFa8pUk1xJy64ISB5VBLop3PgMMXO5kODMsPSC+z3Jmlo8BGtWLb73IvEPQvDUWDxDD9cpN4GRY5et2SD7S9xj7sG1pAoW89xPojsaRBJjGxsNvNMZfE91wSgZMw5hP06SLczbTs=,iv:lxMzrM4Q703SrMfb7DgJxN+tqSU6X+qhBPlURibAOJg=,tag:WnpMif7eXtfO++fy4q3xyQ==,type:str]", + "pgp": null, + "unencrypted_suffix": "_unencrypted", + "version": "3.8.1" + } +} \ No newline at end of file diff --git a/config/clusters/earthscope/enc-grafana-token.secret.yaml b/config/clusters/earthscope/enc-grafana-token.secret.yaml new file mode 100644 index 0000000000..4b5bb816c8 --- /dev/null +++ b/config/clusters/earthscope/enc-grafana-token.secret.yaml @@ -0,0 +1,15 @@ +grafana_token: ENC[AES256_GCM,data:BMfj2nbCJsDHhMx2o2yEoTturNCXOc+jBwKa9Qc/ExO33kAimGZ+e4K43XFoCg==,iv:mhLg7G468X5xcBfqZrO3L5O5VR2sQ8kX87pPg6JU6UI=,tag:Ih/2tuZrBStr42ytEG6kXw==,type:str] +sops: + kms: [] + gcp_kms: + - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs + created_at: "2023-12-14T14:57:49Z" + enc: CiUA4OM7eBNA5OdxpRPD7yDKzvTB3I+oOlGepXn6Kc6VVzU6kcvJEkkAjTWv+oydSi4AiBKtVMhOCs/7vIG7DGwzvOy+kmFq5jJt1FJrhJ4FA2L9x0XygvC+xFI0P7RKQjWEaolkkmJy7chYLVZ1OE42 + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2023-12-14T14:57:50Z" + mac: ENC[AES256_GCM,data:Tmm/phrp21xChffGJl9Y5bONlhrC8cHEb8it/iQXv1BwN03vOT1vbeVmGCOm5QN9sEeISsMlKOO7F90xrndF1DX2B7S8i82DYlKoAN14KPj5pLZycu1aEWVYKjNXE2kq0g6P9CFm4YTE0np+wfFrXbjnGN3klqeh/XhmANmVbAI=,iv:JB2FZHMHIssG7PLzFz0Hdde7bMD/jTtGanJab7i7CsU=,tag:4l3bmrsw+QykwoB38ECMAQ==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.8.1 diff --git a/config/clusters/earthscope/support.values.yaml b/config/clusters/earthscope/support.values.yaml index a1060d493e..97b0aae8fa 100644 --- a/config/clusters/earthscope/support.values.yaml +++ b/config/clusters/earthscope/support.values.yaml @@ -26,3 +26,9 @@ grafana: - secretName: grafana-tls hosts: - grafana.earthscope.2i2c.cloud + +cluster-autoscaler: + enabled: true + autoDiscovery: + clusterName: earthscope + awsRegion: us-east-2 diff --git a/eksctl/earthscope.jsonnet b/eksctl/earthscope.jsonnet index 90926c0cef..07b42121fa 100644 --- a/eksctl/earthscope.jsonnet +++ b/eksctl/earthscope.jsonnet @@ -50,7 +50,7 @@ local daskNodes = [ metadata+: { name: "earthscope", region: clusterRegion, - version: "1.27", + version: "1.28", }, availabilityZones: masterAzs, iam: { From be1d57ce26804edebaf5c3606d417ff0e638267c Mon Sep 17 00:00:00 2001 From: Silva Alejandro Ismael Date: Thu, 14 Dec 2023 13:24:08 -0300 Subject: [PATCH 3/3] Add staging and prod hubs to earthscope cluster --- config/clusters/earthscope/cluster.yaml | 18 ++- config/clusters/earthscope/common.values.yaml | 127 ++++++++++++++++-- .../earthscope/enc-prod.secret.values.yaml | 21 +++ .../earthscope/enc-staging.secret.values.yaml | 21 +++ config/clusters/earthscope/prod.values.yaml | 16 +++ .../clusters/earthscope/staging.values.yaml | 16 +++ 6 files changed, 206 insertions(+), 13 deletions(-) create mode 100644 config/clusters/earthscope/enc-prod.secret.values.yaml create mode 100644 config/clusters/earthscope/enc-staging.secret.values.yaml create mode 100644 config/clusters/earthscope/prod.values.yaml create mode 100644 config/clusters/earthscope/staging.values.yaml diff --git a/config/clusters/earthscope/cluster.yaml b/config/clusters/earthscope/cluster.yaml index 6c87d543ed..0ba2dc1848 100644 --- a/config/clusters/earthscope/cluster.yaml +++ b/config/clusters/earthscope/cluster.yaml @@ -9,4 +9,20 @@ support: helm_chart_values_files: - support.values.yaml - enc-support.secret.values.yaml -hubs: [] +hubs: + - name: staging + display_name: "EarthScope (staging)" + domain: staging.earthscope.2i2c.cloud + helm_chart: daskhub + helm_chart_values_files: + - common.values.yaml + - staging.values.yaml + - enc-staging.secret.values.yaml + - name: prod + display_name: "EarthScope (prod)" + domain: earthscope.2i2c.cloud + helm_chart: daskhub + helm_chart_values_files: + - common.values.yaml + - prod.values.yaml + - enc-prod.secret.values.yaml diff --git a/config/clusters/earthscope/common.values.yaml b/config/clusters/earthscope/common.values.yaml index 09fddb90ea..f1a527c3ab 100644 --- a/config/clusters/earthscope/common.values.yaml +++ b/config/clusters/earthscope/common.values.yaml @@ -1,12 +1,115 @@ -nfs: - pv: - # from https://docs.aws.amazon.com/efs/latest/ug/mounting-fs-nfs-mount-settings.html - mountOptions: - - rsize=1048576 - - wsize=1048576 - - timeo=600 - - soft # We pick soft over hard, so NFS lockups don't lead to hung processes - - retrans=2 - - noresvport - serverIP: fs-08e7747330d833d82.efs.us-east-2.amazonaws.com - baseShareName: / +basehub: + nfs: + pv: + # from https://docs.aws.amazon.com/efs/latest/ug/mounting-fs-nfs-mount-settings.html + mountOptions: + - rsize=1048576 + - wsize=1048576 + - timeo=600 + - soft # We pick soft over hard, so NFS lockups don't lead to hung processes + - retrans=2 + - noresvport + serverIP: fs-08e7747330d833d82.efs.us-east-2.amazonaws.com + baseShareName: / + jupyterhub: + custom: + 2i2c: + add_staff_user_ids_to_admin_users: true + add_staff_user_ids_of_type: "google" + homepage: + templateVars: + org: + url: https://www.earthscope.org/ + logo_url: https://drive.google.com/uc?export=view&id=1UUStqv7PBcxiIkzECUFKIdQKKIU8mXeb + designed_by: + name: "2i2c" + url: https://2i2c.org + operated_by: + name: "2i2c" + url: https://2i2c.org + funded_by: + name: "EarthScope Consortium" + url: https://www.earthscope.org/ + hub: + config: + JupyterHub: + authenticator_class: cilogon + CILogonOAuthenticator: + allowed_idps: + http://github.com/login/oauth/authorize: + default: true + username_derivation: + username_claim: "preferred_username" + http://google.com/accounts/o8/id: + username_derivation: + username_claim: email + Authenticator: + admin_users: + - timdittmann + - chad-earthscope + singleuser: + profileList: + - display_name: "Shared Small: 1-4 CPU, 8-32 GB" + description: "A shared machine, the recommended option until you experience a limitation." + profile_options: &profile_options + image: + display_name: Image + unlisted_choice: + enabled: True + display_name: "Custom image" + validation_regex: "^.+:.+$" + validation_message: "Must be a publicly available docker image, of form :" + kubespawner_override: + image: "{value}" + choices: + jupyter-scipy: + display_name: Jupyter + slug: jupyter-scipy + kubespawner_override: + image: jupyter/scipy-notebook:2023-06-27 + rocker-geospatial: + display_name: RStudio + slug: rocker-geospatial + kubespawner_override: + image: rocker/binder:4.3 + # Launch into RStudio after the user logs in + default_url: /rstudio + # Ensures container working dir is homedir + # https://github.com/2i2c-org/infrastructure/issues/2559 + working_dir: /home/rstudio + kubespawner_override: + mem_guarantee: 7.234G + cpu_guarantee: 0.1 + mem_limit: null + node_selector: + node.kubernetes.io/instance-type: r5.xlarge + + - display_name: "Small: 4 CPU, 32 GB" + description: "A dedicated machine for you." + profile_options: *profile_options + kubespawner_override: + mem_guarantee: 28.937G + cpu_guarantee: 0.4 + mem_limit: null + node_selector: + node.kubernetes.io/instance-type: r5.xlarge + + - display_name: "Medium: 16 CPU, 128 GB" + description: "A dedicated machine for you." + profile_options: *profile_options + kubespawner_override: + mem_guarantee: 120.513G + cpu_guarantee: 1.6 + mem_limit: null + node_selector: + node.kubernetes.io/instance-type: r5.4xlarge + + - display_name: "Large: 64 CPU, 512 GB" + description: "A dedicated machine for you" + profile_options: *profile_options + kubespawner_override: + mem_guarantee: 489.13G + cpu_guarantee: 6.4 + mem_limit: null + node_selector: + node.kubernetes.io/instance-type: r5.16xlarge diff --git a/config/clusters/earthscope/enc-prod.secret.values.yaml b/config/clusters/earthscope/enc-prod.secret.values.yaml new file mode 100644 index 0000000000..323551895b --- /dev/null +++ b/config/clusters/earthscope/enc-prod.secret.values.yaml @@ -0,0 +1,21 @@ +basehub: + jupyterhub: + hub: + config: + CILogonOAuthenticator: + client_id: ENC[AES256_GCM,data:1C0ercYZjjc63vTPPcVa7B0Y1bnuawg854Yf3Kl4UnJ0gYuqem+zuv1lQfOzU8zKXy5L,iv:2IZjb7WzomJg8I9uDDXINjULJPXUBfJCldMOxH+B8tA=,tag:Dv1xaVkDCpI7/GLuGv6GzA==,type:str] + client_secret: ENC[AES256_GCM,data:2mGbTTnKcVZp57ZX2Tj2o+j2y0NfABPtTiV6sw3oWlR/t7w4fiFkSK9cyArnJwQfRjWc6M6NNB50A3zWZrKaoPLRj8Afiq8pFTjtRZnZGe5g4h2mXYg=,iv:xmJEHc2V0aG1KEh2eAPj80tZoNzFnBz42QdCSmzO2mc=,tag:+48SuYVdlJCizaYVMn9hrA==,type:str] +sops: + kms: [] + gcp_kms: + - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs + created_at: "2023-12-14T18:24:33Z" + enc: CiUA4OM7eKr6IJS10QOinx3kZfdMUdO4HmyV6U+JIe+s7IqTAm1DEkkAjTWv+ifsDVRYeh6Gdg0+tLE53DfZfJP0xHGuo6yxdoREE2FGKpodr42/SaaTUt2k5zKlg6k9tOe4FgmRGCT5JjIltibg5hwU + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2023-12-14T18:24:34Z" + mac: ENC[AES256_GCM,data:0Kde6XE/A7k9CwhxQFsa3I61ohr9WN7AO2haWkFETpDG+jXtU5MYkrScbwnlayLa0vM6vk2OfUxR6LrB9jPcxTx8+n2Pqx6kPTzgr8a8ORhG4xc6Lqj0a1KyDMdnGi5beqoXSxolPyd1mnSTAFAVIGwle37Gg0fIr0VFii9lsfQ=,iv:gPVYPvyTEriA9sxbmtMRo611b5dB5idYa0J+DtEYcaY=,tag:RNOItHNKtUGZ/UgfT1Ea2Q==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.8.1 diff --git a/config/clusters/earthscope/enc-staging.secret.values.yaml b/config/clusters/earthscope/enc-staging.secret.values.yaml new file mode 100644 index 0000000000..bab6d99775 --- /dev/null +++ b/config/clusters/earthscope/enc-staging.secret.values.yaml @@ -0,0 +1,21 @@ +basehub: + jupyterhub: + hub: + config: + CILogonOAuthenticator: + client_id: ENC[AES256_GCM,data:Lv/25K0A8CZs6dK20mujkn536hpreimP/MUqGOJ4cpXLTFnJNRmGkN7mYPC2klalEKcn,iv:nj4b7Y75A9wgg+w2XBas17Cs8Az3AzDkeO9u1ZwI1Jo=,tag:gCMMoa3iQWVRQvTQkCIkAg==,type:str] + client_secret: ENC[AES256_GCM,data:EAD3iQGXs7soD4VxRXol2YuuJBmOpDBbX5Cg+VyTk7xA7Jn715vZMNBeOKtal1a6kzyds3tuw+h+DWsF3Dod2MxHS7H4FARHLopP9xuAvS6Tw3mZZ28=,iv:F8CqwLYz7WR5qge0Yj91aU/w5pj6fiEaBvndVe4zvG4=,tag:60BekNlkRhf2a3Nkvo1kWg==,type:str] +sops: + kms: [] + gcp_kms: + - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs + created_at: "2023-12-14T18:42:48Z" + enc: CiUA4OM7eGqP+F9UNcdWkWcEANT1YIeSiFyzogRgfD+PMhJISk+lEkkAjTWv+sk2C+z/gAjXwaTvoJEJKeuCyiegMLu8QTkJ1KCtcQEU52qv/gm6HvBAlQAnTUKxpQFejxzGOp/8+FNCZiAuaT2hHq1D + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2023-12-14T18:42:52Z" + mac: ENC[AES256_GCM,data:DWO/hv47PbcFx8NATfOJrLUMkOV3dTUzr53nUtpDge+NseEOSoMKeEWz1L7jWYhM+Iga05csm78BT9c3gI921dKlOXRJ6fn1e5guxqKPOAuZugbWUeEqGa8Z26sAwuSRXIZyWiWDJZJThsNk4+s0s7vZmXcrGHGjWA3eCEvTwxE=,iv:9QDeyrmE0euFgqcvZMCuubNA44YB8x2Sa1CqEGJjKjM=,tag:qj5ZnX1TS2Lt4QbXuJFB0Q==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.8.1 diff --git a/config/clusters/earthscope/prod.values.yaml b/config/clusters/earthscope/prod.values.yaml new file mode 100644 index 0000000000..a850dd41e8 --- /dev/null +++ b/config/clusters/earthscope/prod.values.yaml @@ -0,0 +1,16 @@ +basehub: + jupyterhub: + ingress: + hosts: [earthscope.2i2c.cloud] + tls: + - hosts: [earthscope.2i2c.cloud] + secretName: https-auto-tls + custom: + homepage: + templateVars: + org: + name: "EarthScope" + hub: + config: + CILogonOAuthenticator: + oauth_callback_url: https://earthscope.2i2c.cloud/hub/oauth_callback diff --git a/config/clusters/earthscope/staging.values.yaml b/config/clusters/earthscope/staging.values.yaml new file mode 100644 index 0000000000..bb621d8433 --- /dev/null +++ b/config/clusters/earthscope/staging.values.yaml @@ -0,0 +1,16 @@ +basehub: + jupyterhub: + ingress: + hosts: [staging.earthscope.2i2c.cloud] + tls: + - hosts: [staging.earthscope.2i2c.cloud] + secretName: https-auto-tls + custom: + homepage: + templateVars: + org: + name: "EarthScope staging" + hub: + config: + CILogonOAuthenticator: + oauth_callback_url: https://staging.earthscope.2i2c.cloud/hub/oauth_callback