Merge pull request #558 from yuvipanda/carbonplan-eksctl

Move carbonplan hub to eksctl

.sops.yaml

@@ -3,6 +3,8 @@ creation_rules:
     gcp_kms: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs
   - path_regex: .*/kops/ssh-keys/.*
     gcp_kms: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs
+  - path_regex: .*/eksctl/ssh-keys/.*
+    gcp_kms: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs
   - path_regex: config/secrets.yaml$
     gcp_kms: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs
   - path_regex: support/secrets.yaml$

config/hubs/carbonplan.cluster.yaml

@@ -13,6 +13,11 @@ support:
           limits:
             cpu: 4
             memory: 8Gi
+    cluster-autoscaler:
+      enabled: true
+      autoDiscovery:
+        clusterName: carbonplanhub
+      awsRegion: us-west-2
     grafana:
       ingress:
         hosts:

eksctl/carbonplan.jsonnet

@@ -0,0 +1,87 @@
+// Exports an eksctl config file for carbonplan cluster
+local cluster = import "./libsonnet/cluster.jsonnet";
+local ng = import "./libsonnet/nodegroup.jsonnet";
+
+// place all cluster nodes here
+local clusterRegion = "us-west-2";
+local masterAzs = ["us-west-2a", "us-west-2b", "us-west-2c"];
+local nodeAz = "us-west-2a";
+
+// Node definitions for use with dask and notebook nodes
+// These are merged in with the defaults for either node type,
+// and so can contain any overrides.
+local nodes = [
+    { instanceType: "r5.large" },
+    { instanceType: "r5.xlarge" },
+    { instanceType: "r5.2xlarge" },
+    { instanceType: "r5.8xlarge" },
+    { instanceType: "x1.16xlarge" },
+    { instanceType: "x1.32xlarge" }
+];
+
+cluster {
+    metadata+: {
+        name: "carbonplanhub",
+        region: clusterRegion
+    },
+    availabilityZones: masterAzs,
+    nodeGroups: [
+        ng {
+            name: 'core-a',
+            availabilityZones: [nodeAz],
+            ssh: {
+                publicKeyPath: 'ssh-keys/carbonplan.key.pub'
+            },
+            instanceType: "m5.xlarge",
+            minSize: 1,
+            maxSize: 6,
+            labels+: {
+                "hub.jupyter.org/node-purpose": "core",
+                "k8s.dask.org/node-purpose": "core"
+            },
+        },
+    ] + [
+        ng {
+            // NodeGroup names can't have a '.' in them, while
+            // instanceTypes always have a .
+            name: "nb-%s" % std.strReplace(self.instanceType, ".", "-"),
+            availabilityZones: [nodeAz],
+            minSize: 0,
+            maxSize: 500,
+            ssh: {
+                publicKeyPath: 'ssh-keys/carbonplan.key.pub'
+            },
+            labels+: {
+                "hub.jupyter.org/node-purpose": "user",
+                "k8s.dask.org/node-purpose": "scheduler"
+            },
+            taints+: {
+                "hub.jupyter.org_dedicated": "user:NoSchedule",
+                "hub.jupyter.org/dedicated": "user:NoSchedule"
+            },
+
+        } + n for n in nodes
+    ] + [
+        ng {
+            // NodeGroup names can't have a '.' in them, while
+            // instanceTypes always have a .
+            name: "dask-%s" % std.strReplace(self.instanceType, ".", "-"),
+            availabilityZones: [nodeAz],
+            minSize: 0,
+            maxSize: 500,
+            ssh: {
+                publicKeyPath: 'ssh-keys/carbonplan.key.pub'
+            },
+            labels+: {
+                "k8s.dask.org/node-purpose": "worker"
+            },
+            taints+: {
+                "k8s.dask.org_dedicated" : "worker:NoSchedule",
+                "k8s.dask.org/dedicated" : "worker:NoSchedule"
+            },
+
+        } + n for n in nodes
+    ]
+
+
+}

eksctl/libsonnet/cluster.jsonnet

@@ -0,0 +1,19 @@
+// Exports a customizable eksctl cluster object
+// https://eksctl.io/usage/schema/ lists the config
+//
+// The default configuration is pretty bare, and only
+// sets the default k8s version. Everything else must
+// be merged in by the jsonnet file for each cluster
+{
+  apiVersion: 'eksctl.io/v1alpha5',
+  kind: 'ClusterConfig',
+  metadata: {
+    name: '',
+    region: '',
+    version: '1.19',
+  },
+  availabilityZones: [],
+  iam: {
+    withOIDC: true,
+  },
+}

eksctl/libsonnet/nodegroup.jsonnet

@@ -0,0 +1,32 @@
+// Make ASG tags for given set of k8s labels
+local makeCloudLabels(labels) = {
+  ['k8s.io/cluster-autoscaler/node-template/label/%s' % key]: labels[key]
+  for key in std.objectFields(labels)
+};
+
+# Make asg tags for given set of k8s taints
+local makeCloudTaints(taints) = {
+  ['k8s.io/cluster-autoscaler/node-template/taint/%s' % key]: taints[key]
+  for key in std.objectFields(taints)
+};
+
+{
+  name: '',
+  availabilityZones: [],
+  minSize: 0,
+  desiredCapacity: self.minSize,
+  instanceType: '',
+  volumeSize: 80,
+  labels+: {
+	  // Add instance type as label to nodegroups, so they
+	  // can be picked up by the autoscaler
+    'node.kubernetes.io/instance-type': $.instanceType,
+  },
+  taints+: {},
+  tags: makeCloudLabels(self.labels) + makeCloudTaints(self.taints),
+  iam: {
+    withAddonPolicies: {
+      autoScaler: true,
+    },
+  },
+}

eksctl/ssh-keys/carbonplan.key

@@ -0,0 +1,21 @@
+{
+	"data": "ENC[AES256_GCM,data:BZazQGKXm86j1Pb/RRhmGdJGSoUUed7GY6Fo6xrZEh73qyNS13IJop9eGmeMI7UsuOCwPungwW++pJX7MDQ5wa4ol9E4m/+1FoO56paLwGnCexsx0no9s+3y9UZVN4TCXkd3ISaq92FdUc2FKuLkUiKMOC78uYtaSDyIOWOpba0MxbLdScITbSBGwmP+nEoWt52wGokqrPMGKre9WMpRaUycPSB+rL0A13rnH/nwYdQhRHEWmCIulPAMNe5aeYUQ/NMPJaoxGhL7A7TmeFcX7nWedymRn/WGTS3AIo8iPXQkBe7lurPQrS1pzP5lG6IX6pNJ3STbHYRkWl+xTaoBjH29r5+VPuHsBfMr1jwW8j3dCqOB/oODbz0cSW7qlb0sLNgJ+JuCyCBFJNzie6pBHORLjhhv1ifhMCuLXs0Fk5TJVnh7ODwWaRUuD1DgGz2XnOOiWOW1hIb2hqt0dojEbgSC2iPuUi1bCu5UnNZ8XgD27A5xKwRasYtCJuRJjJ+WGiQnzSNSTW4A/qJ6tPk8oVLgxYvGlmgot6KtCSIOsc9EbrYIrBKXZIl34B/X954F/1HkIi1qTtho3c6QSyvXB1pmjEBg7Trl5gbqGQ3aFLqKu4Li1Uv3wo5P1yKubfwI7gz7OMwRKl+x0VGQcL+isYugcwu//ABPYWgiyXx/UEW7rMxsWQS9OHXBCig97g+4+JZH+VeJimov6qOmCTGzpJh7ubkbYhw/OnvNZC5jRX75372RsYp00iMb9kH/txEQit+d/2T+9Sck4nVE+iOyQuyeHzAo2uNG5MQQM3V3WbppZ7ITRzIv6Zve26+HtdZj+ukle+NpmFPnwe1kAt5q+d769wI0q0G/lU0S3HFSGMEIBTm9cDBDg9lxRHEGLylAgvfpBYwJGRW3mF0BmgmxAUygA9w6YzFuEWd4S/jj4aWqkDEXK0VpdosaamxmfVVIU7crq/gI9i1bjW/GdEGjruh5eqiQCscXPfOziV6bD9sliXAETE4ViMW2sLNK/0jdq0n2GAT0rtGnL86OJ7unaTAMnpjQT9JpriYByAEqzwyYwBmIzexCfvIiX+Q3XWCPpXFoArOmpfM8bNfui8j3BGTy96hk4Bn54s2r+JdK0TLxPaKBlG9YHyhhbW5speZjqhhqQk1trTJKTnQrTDWb2zeT8U2KFP71ErU3IPxgK1oy2unBealNPV9ozAXzW6xliCd0ZBoJCrROrWNxeI9Wtpta0ZgyRLHjs7vZduzNdEDsQ3+O+XNhIHJd1Yb4RwoKaA4aedk82Yy3smxCYvxcwMMTx4b27L/xcAWBxMzzLE/rUeJ+YIn8IllaEWHv7ud5DjBjvv2MIL8GWDOykH982HtSYuvs6NnZ0K5kc+As0AUAuFoAGwsW/PSxCXRNHlFTOXnQ//zLKYL+s6X5pGCbKWjRGXoo/jd+mhJyy2/UKPbAc2q+aGEAliZjvWH9BWX8WSEKCa1Far3kZMOB01WLu5oDXSCbuqCoV882u7mhcvoAsxccm7MXuQUBlohP6DLamW4Le6wzy2zXGo9QnfcNsHhminp0gJW1WknCUR4BoZnRo0CzVDdOXbVazYQGTI0x2FzQWDievape27ccbOeLMI4PjMf2m/LlLLgCAXp3Emrzf1DfhUc/s0I8XAM3HmuUWeFGyZkjJlXomGowtbyCKoAxqiy7VRMDkHYo87s48uvruJIJQdKwpv8WPJmd0xprCE6eAK+LkqkoTTfOBPpOoUwEkUC8DgQH7g2JJv6aryFcxM7vw4m7hniRM0v2yXhSPCpQ0ThQ2oY8SHc0R3afyfOd5958PpRZl4wixOIp2p9TVR8HhQ3KpNbIAY+t2ft3EMpDHFI0SBWw+Ns2dgxJEo3/xGYUndb/88CR9WRITO+xLsBkYOmsXZihXe2XG4RDQ5iYCcVVfrqz6yMy9rWqEaSwEgiZq/9Y/b7dvCs9soLl0PMfkDlm7thiTGh1Bq+7LwJRvaufJTO2LyJqHxiITppazzm12FW0x7DACcBbyQRNhpbIwUUOdoVgjRiknGHmP/J8KrxEq9WuDQLdMt9quXCli5aTtYNkY+Ja1mxLe/WjLIorCyYfzravveDZbcP4TVJkjrLIWrcIz88JEf1HWRU1erhMHnAQ63aRgRzk6Ztle6f9TF4XfnM14GnmhBeWV55HQLsFcpq34OR5AsH4+fbAHv6Vc6vbDlTBZkU0g3IJTq/YYnOqDBLOjy0ifxI1/TEkAXtcKevhCsOqZVfCKO+pUkmnGg++5VI4dsS+h2/7KHpj0SFYmbdggDnTiRlVAU4/mbpLcSTX6+KYBlfAydawDBita6fzFKX+dHeMOBdRgCrEXfsf5xgRk2u5B0JppdbrJXCJ7J+b+TM4y8P7Mr6USKxFH3tGEWEETbA1/hz0QcUf+Gvk+lNB/RN9FGmPNt59GvSTGWALeUKLCclbzmmu0UnOeW5BZteFIaRpJ6Mya9ATvbjKyX8J3hb20Db00MTKLc7EoD8iQ9qQO6kDbhHDzpHFDfs/ytSN47G637xrIOFO064uB2N3Y5mGFp5NjETgrjemGjRqrzBGwIzsxciGgLjSlG2FWTXBLP8rh/ROXKGhDvJ/qkTIwX55RAzg2e/UpK8isaPJwQTa0B2ef8obMiwib5jnH0y0p3iVCv7mEMKTQF1vI5hV8wi+l8FhjDN8IcLQFFxOxXK0rQuK8EsEzk57UGW2s8mKwmUClpq6rxOQ+VokDzDm2MeDuD9ZA+u7rv7d+QJu59NGvfDHOx21HSZS4r8UsEO45LBy27saQgFY1b100Q8j4G641xAip1uTY98hi2fIyQi79p1NZ6hVB/ia65xb728uOzI2VvBjiOphbGTawVQBSqDtpbOjc4Fo8Xrw2xlqaQPsRDnn6baMWCXfFt4+lgMI8D75lQtQ/stV7ZRZOM+qUI7Qf/k5gjIKT0LcjGGFLRRrbg4GiTbnGR8H10PiYTFyTAfYhaQFN9VKfVY50ulCJlI2QC8uJy5o40R3Ye6jJFYu7mjOWNJF7eY51FkexiTmvVt9SerR4UqqcM7qE++IpI+xu2qCmY8TA08SiZ0qfKBT/VrcBlRC3gYQvaEVSX+GZlKUfzXn++pY/SeSReOpf2TRy01/vZwlOe+9yDBX8GSw7KKxCW5qGFUSWUSBM8D1mjsTM74kWw7LAlLJnpIRXxns7GmwRiGyhSo5R8gyZg0tiNDuegazCW0JOLUJRX4bL4HDOdTCMvjK1dPe0oCqG0AG25v9TWWfp1Vnm5EDcdUVFQAyIu+w5wy68NPt5RIsKQPGK2gcP46btKR/janEwmPvh+Bzwm/onPOLFAMhqiPU0kD7tvOW7tpZgo1kbaZzz3n6X5mPevhffHIj9QMor66Kb0x24+6h9KdwHDvJ9aAsS9VO8qnUVJWuPoA2eHWS+4rxaHFSCVDlOGLqmdhzd4nt5btHB/DcBzOJssQJHqVV+jyHa6TGouONaDVPTFSFfq6D,iv:6L89V9Nycfneb4bU0nCvA0i+qxy9JMqwRIr/H5T3PQ4=,tag:yCmxcl0ZwMLQ0zU0CmZ6FA==,type:str]",
+	"sops": {
+		"kms": null,
+		"gcp_kms": [
+			{
+				"resource_id": "projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs",
+				"created_at": "2021-07-27T12:23:33Z",
+				"enc": "CiQA4OM7eICUZva/D+yx43fQXwFmI54mOdn4+aT3TdnTqIWaI9USSQB6TpsYaJh4GNAW1aI3J+N9fokpWYybO8SUIQRebDpmNUpeMh+xqQG2lRnpjnGt/2D2JGnMzUn076Nd/0CIUSlVoq5/W57jsk8="
+			}
+		],
+		"azure_kv": null,
+		"hc_vault": null,
+		"age": null,
+		"lastmodified": "2021-07-27T12:23:34Z",
+		"mac": "ENC[AES256_GCM,data:8KbxNB7+PYDE79iHXrw7aLvt3DE07MMRhAqhrbdNeKAVE+iMpQyxk4twjG/hEJSYXq3Q17VjIJNqDtFF4EE+/cveOH4A9yNtXlne0mEacFuRD2Av4GHUt7i1EjFNyXiPhirUovu1RiG9pcIKu5zkOhMOMk2lYv6l4i2GVmxDjLQ=,iv:WMM/f3kGv6gdoxwiPMPs2/8VHYDfkmZSp8hgZ+hnOzo=,tag:KfQa2YWEzDRJPHc9oqXyIA==,type:str]",
+		"pgp": null,
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.7.1"
+	}
+}

eksctl/ssh-keys/carbonplan.key.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDaDc3nWE5ndKpmeIg1vbYnCmG4UZ/ZYMKXAdKaVy/p0EHec99o2WqqoqIjgJzeZBGSC/v57igWWbxjlvtnAMbd1M7JXvz1RTjzsw2ankBCq96UOklQoOR341ekxt/Zomp8TC+J7neXYHdSzQ280HNHQpX0MhwxsHLTs49kYOmauiW4uiXuTgRL5nvCeHzeIXFvhLuC9QmeQpaFoPgJrHS5AGER5C7fcRdybyMVfm/EDvc27c9S6NYZ98fcjyR9fmIeOB+krNWlB3UIQG1IwxR1Yhy9f/c8R2RCg63aWFMgGbF70MHGi9m3ntFkq9HMPoEHzLDB20vd/9Om+vIqq9nykBYiS1tjzGKv/jbvr7d6c81POv6UgMaVcDZYnAx/AHv4zzS0KDtnNLd/OAk2r1Ov9v104J7U17WjNBJjNMxtMe8skj/JqXPJJ/6FDI3/7S+zKX1tszMqi+XD/jUAisiZh/VvarJGzkyI09JyC6M6w507/HBjzuMePFtroeCf270= [email protected]

kops/libsonnet/cluster.jsonnet

@@ -1,3 +1,4 @@
+
 // Exports a customizable kops Cluster object.
 // https://kops.sigs.k8s.io/cluster_spec/ lists available properties.
 //