From 799e4ac1346a08f2d370fa51a10c4924818f05cc Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 6 Feb 2023 14:15:39 +0100 Subject: [PATCH] aws clusters: systematically add ebs-csi-driver addon before k8s 1.23+ upgrade --- eksctl/2i2c-aws-us.jsonnet | 50 ++++++++++++++++++++++++++++++++------ eksctl/carbonplan.jsonnet | 30 ++++++++++++++--------- eksctl/gridsst.jsonnet | 48 ++++++++++++++++++++++++++++++------ eksctl/nasa-cryo.jsonnet | 48 ++++++++++++++++++++++++++++++------ eksctl/nasa-veda.jsonnet | 34 ++++++++++++++++---------- eksctl/ubc-eoas.jsonnet | 43 +++++++++++++++++++++----------- eksctl/uwhackweeks.jsonnet | 48 ++++++++++++++++++++++++++++++------ eksctl/victor.jsonnet | 48 ++++++++++++++++++++++++++++++------ 8 files changed, 275 insertions(+), 74 deletions(-) diff --git a/eksctl/2i2c-aws-us.jsonnet b/eksctl/2i2c-aws-us.jsonnet index 249250abc8..9661348f71 100644 --- a/eksctl/2i2c-aws-us.jsonnet +++ b/eksctl/2i2c-aws-us.jsonnet @@ -1,4 +1,18 @@ -// Exports an eksctl config file for carbonplan cluster +/* + This file is a jsonnet template of a eksctl's cluster configuration file, + that is used with the eksctl CLI to both update and initialize an AWS EKS + based cluster. + + This file has in turn been generated from eksctl/template.jsonnet which is + relevant to compare with for changes over time. + + To use jsonnet to generate an eksctl configuration file from this, do: + + jsonnet 2i2c-aws-us.jsonnet > eksctl-config.yaml + + References: + - https://eksctl.io/usage/schema/ +*/ local ng = import "./libsonnet/nodegroup.jsonnet"; // place all cluster nodes here @@ -23,19 +37,20 @@ local notebookNodes = [ }, ]; -local daskNodes = - if "daskhub" == "daskhub" then [ +local daskNodes = [ // Node definitions for dask worker nodes. Config here is merged // with our dask worker node definition, which uses spot instances. // A `node.kubernetes.io/instance-type label is set to the name of the // *first* item in instanceDistribution.instanceTypes, to match // what we do with notebook nodes. Pods can request a particular // kind of node with a nodeSelector - { instancesDistribution+: { instanceTypes: ["m5.large"] }}, - { instancesDistribution+: { instanceTypes: ["m5.xlarge"] }}, - { instancesDistribution+: { instanceTypes: ["m5.2xlarge"] }}, - { instancesDistribution+: { instanceTypes: ["m5.8xlarge"] }}, - ]; + { instancesDistribution+: { instanceTypes: ["m5.large"] }}, + { instancesDistribution+: { instanceTypes: ["m5.xlarge"] }}, + { instancesDistribution+: { instanceTypes: ["m5.2xlarge"] }}, + { instancesDistribution+: { instanceTypes: ["m5.8xlarge"] }}, +]; + + { apiVersion: 'eksctl.io/v1alpha5', kind: 'ClusterConfig', @@ -51,6 +66,25 @@ local daskNodes = iam: { withOIDC: true, }, + // If you add an addon to this config, run the create addon command. + // + // eksctl create addon --config-file=eksctl-config.yaml + // + addons: [ + { + // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that + // couple to AWS EBS based storage, without it expect to see pods + // mounting a PVC failing to schedule and PVC resources that are + // unbound. + // + // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html + // + name: 'aws-ebs-csi-driver', + wellKnownPolicies: { + ebsCSIController: true, + }, + }, + ], nodeGroups: [ ng { name: 'core-a', diff --git a/eksctl/carbonplan.jsonnet b/eksctl/carbonplan.jsonnet index b0f6487521..3f44912810 100644 --- a/eksctl/carbonplan.jsonnet +++ b/eksctl/carbonplan.jsonnet @@ -1,14 +1,18 @@ -// This file is a jinja2 template of a jsonnet template of a eksctl's cluster -// configuration file, which is in turn can be used with the `eksctl` CLI to both -// update and initialize a AWS EKS based cluster. -// -// This jinja2 template is only used by the deployer script as part of creating -// new clusters. If a relevant change is made here or the dependent file -// libsonnet/nodegroup.jsonnet, one may consider if we should manually update -// already generated jsonnet files in this folder. -// -// Configuration reference: https://eksctl.io/usage/schema/ -// +/* + This file is a jsonnet template of a eksctl's cluster configuration file, + that is used with the eksctl CLI to both update and initialize an AWS EKS + based cluster. + + This file has in turn been generated from eksctl/template.jsonnet which is + relevant to compare with for changes over time. + + To use jsonnet to generate an eksctl configuration file from this, do: + + jsonnet carbonplan.jsonnet > eksctl-config.yaml + + References: + - https://eksctl.io/usage/schema/ +*/ local ng = import "./libsonnet/nodegroup.jsonnet"; // place all cluster nodes here @@ -69,6 +73,10 @@ local daskNodes = [ ], } for namespace in namespaces], }, + // If you add an addon to this config, run the create addon command. + // + // eksctl create addon --config-file=eksctl-config.yaml + // addons: [ { // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that diff --git a/eksctl/gridsst.jsonnet b/eksctl/gridsst.jsonnet index 5dc045a4b1..b728146ee6 100644 --- a/eksctl/gridsst.jsonnet +++ b/eksctl/gridsst.jsonnet @@ -1,4 +1,18 @@ -// Exports an eksctl config file for carbonplan cluster +/* + This file is a jsonnet template of a eksctl's cluster configuration file, + that is used with the eksctl CLI to both update and initialize an AWS EKS + based cluster. + + This file has in turn been generated from eksctl/template.jsonnet which is + relevant to compare with for changes over time. + + To use jsonnet to generate an eksctl configuration file from this, do: + + jsonnet gridsst.jsonnet > eksctl-config.yaml + + References: + - https://eksctl.io/usage/schema/ +*/ local ng = import "./libsonnet/nodegroup.jsonnet"; // place all cluster nodes here @@ -23,19 +37,20 @@ local notebookNodes = [ }, ]; -// Node definitions for dask worker nodes. Config here is merged -// with our dask worker node definition, which uses spot instances. -// A `node.kubernetes.io/instance-type label is set to the name of the -// *first* item in instanceDistribution.instanceTypes, to match -// what we do with notebook nodes. Pods can request a particular -// kind of node with a nodeSelector local daskNodes = [ + // Node definitions for dask worker nodes. Config here is merged + // with our dask worker node definition, which uses spot instances. + // A `node.kubernetes.io/instance-type label is set to the name of the + // *first* item in instanceDistribution.instanceTypes, to match + // what we do with notebook nodes. Pods can request a particular + // kind of node with a nodeSelector { instancesDistribution+: { instanceTypes: ["m5.large"] }}, { instancesDistribution+: { instanceTypes: ["m5.xlarge"] }}, { instancesDistribution+: { instanceTypes: ["m5.2xlarge"] }}, { instancesDistribution+: { instanceTypes: ["m5.8xlarge"] }}, ]; + { apiVersion: 'eksctl.io/v1alpha5', kind: 'ClusterConfig', @@ -48,6 +63,25 @@ local daskNodes = [ iam: { withOIDC: true, }, + // If you add an addon to this config, run the create addon command. + // + // eksctl create addon --config-file=eksctl-config.yaml + // + addons: [ + { + // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that + // couple to AWS EBS based storage, without it expect to see pods + // mounting a PVC failing to schedule and PVC resources that are + // unbound. + // + // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html + // + name: 'aws-ebs-csi-driver', + wellKnownPolicies: { + ebsCSIController: true, + }, + }, + ], nodeGroups: [ ng { name: 'core-a', diff --git a/eksctl/nasa-cryo.jsonnet b/eksctl/nasa-cryo.jsonnet index 384a79ec26..37e24dc6b9 100644 --- a/eksctl/nasa-cryo.jsonnet +++ b/eksctl/nasa-cryo.jsonnet @@ -1,4 +1,18 @@ -// Exports an eksctl config file for carbonplan cluster +/* + This file is a jsonnet template of a eksctl's cluster configuration file, + that is used with the eksctl CLI to both update and initialize an AWS EKS + based cluster. + + This file has in turn been generated from eksctl/template.jsonnet which is + relevant to compare with for changes over time. + + To use jsonnet to generate an eksctl configuration file from this, do: + + jsonnet nasa-cryo.jsonnet > eksctl-config.yaml + + References: + - https://eksctl.io/usage/schema/ +*/ local ng = import "./libsonnet/nodegroup.jsonnet"; // place all cluster nodes here @@ -17,19 +31,20 @@ local notebookNodes = [ { instanceType: "m5.8xlarge" }, ]; -// Node definitions for dask worker nodes. Config here is merged -// with our dask worker node definition, which uses spot instances. -// A `node.kubernetes.io/instance-type label is set to the name of the -// *first* item in instanceDistribution.instanceTypes, to match -// what we do with notebook nodes. Pods can request a particular -// kind of node with a nodeSelector local daskNodes = [ + // Node definitions for dask worker nodes. Config here is merged + // with our dask worker node definition, which uses spot instances. + // A `node.kubernetes.io/instance-type label is set to the name of the + // *first* item in instanceDistribution.instanceTypes, to match + // what we do with notebook nodes. Pods can request a particular + // kind of node with a nodeSelector { instancesDistribution+: { instanceTypes: ["m5.large"] }}, { instancesDistribution+: { instanceTypes: ["m5.xlarge"] }}, { instancesDistribution+: { instanceTypes: ["m5.2xlarge"] }}, { instancesDistribution+: { instanceTypes: ["m5.8xlarge"] }}, ]; + { apiVersion: 'eksctl.io/v1alpha5', kind: 'ClusterConfig', @@ -42,6 +57,25 @@ local daskNodes = [ iam: { withOIDC: true, }, + // If you add an addon to this config, run the create addon command. + // + // eksctl create addon --config-file=eksctl-config.yaml + // + addons: [ + { + // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that + // couple to AWS EBS based storage, without it expect to see pods + // mounting a PVC failing to schedule and PVC resources that are + // unbound. + // + // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html + // + name: 'aws-ebs-csi-driver', + wellKnownPolicies: { + ebsCSIController: true, + }, + }, + ], nodeGroups: [ ng { name: 'core-a', diff --git a/eksctl/nasa-veda.jsonnet b/eksctl/nasa-veda.jsonnet index 51f5082f53..fa6c5cd243 100644 --- a/eksctl/nasa-veda.jsonnet +++ b/eksctl/nasa-veda.jsonnet @@ -1,14 +1,18 @@ -// This file is a jinja2 template of a jsonnet template of a eksctl's cluster -// configuration file, which is in turn can be used with the `eksctl` CLI to both -// update and initialize a AWS EKS based cluster. -// -// This jinja2 template is only used by the deployer script as part of creating -// new clusters. If a relevant change is made here or the dependent file -// libsonnet/nodegroup.jsonnet, one may consider if we should manually update -// already generated jsonnet files in this folder. -// -// Configuration reference: https://eksctl.io/usage/schema/ -// +/* + This file is a jsonnet template of a eksctl's cluster configuration file, + that is used with the eksctl CLI to both update and initialize an AWS EKS + based cluster. + + This file has in turn been generated from eksctl/template.jsonnet which is + relevant to compare with for changes over time. + + To use jsonnet to generate an eksctl configuration file from this, do: + + jsonnet nasa-veda.jsonnet > eksctl-config.yaml + + References: + - https://eksctl.io/usage/schema/ +*/ local ng = import "./libsonnet/nodegroup.jsonnet"; // place all cluster nodes here @@ -26,6 +30,7 @@ local notebookNodes = [ { instanceType: "m5.2xlarge" }, { instanceType: "m5.8xlarge" }, ]; + local daskNodes = [ // Node definitions for dask worker nodes. Config here is merged // with our dask worker node definition, which uses spot instances. @@ -38,9 +43,8 @@ local daskNodes = [ { instancesDistribution+: { instanceTypes: ["m5.2xlarge"] }}, { instancesDistribution+: { instanceTypes: ["m5.8xlarge"] }}, ]; - - + { apiVersion: 'eksctl.io/v1alpha5', kind: 'ClusterConfig', @@ -53,6 +57,10 @@ local daskNodes = [ iam: { withOIDC: true, }, + // If you add an addon to this config, run the create addon command. + // + // eksctl create addon --config-file=eksctl-config.yaml + // addons: [ { // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that diff --git a/eksctl/ubc-eoas.jsonnet b/eksctl/ubc-eoas.jsonnet index c9a13883e9..fe05f8eddb 100644 --- a/eksctl/ubc-eoas.jsonnet +++ b/eksctl/ubc-eoas.jsonnet @@ -1,4 +1,18 @@ -// Exports an eksctl config file for carbonplan cluster +/* + This file is a jsonnet template of a eksctl's cluster configuration file, + that is used with the eksctl CLI to both update and initialize an AWS EKS + based cluster. + + This file has in turn been generated from eksctl/template.jsonnet which is + relevant to compare with for changes over time. + + To use jsonnet to generate an eksctl configuration file from this, do: + + jsonnet ubc-eoas.jsonnet > eksctl-config.yaml + + References: + - https://eksctl.io/usage/schema/ +*/ local ng = import "./libsonnet/nodegroup.jsonnet"; // place all cluster nodes here @@ -17,19 +31,9 @@ local notebookNodes = [ { instanceType: "m5.8xlarge" }, ]; -local daskNodes = - if "basehub" == "daskhub" then [ - // Node definitions for dask worker nodes. Config here is merged - // with our dask worker node definition, which uses spot instances. - // A `node.kubernetes.io/instance-type label is set to the name of the - // *first* item in instanceDistribution.instanceTypes, to match - // what we do with notebook nodes. Pods can request a particular - // kind of node with a nodeSelector - { instancesDistribution+: { instanceTypes: ["m5.large"] }}, - { instancesDistribution+: { instanceTypes: ["m5.xlarge"] }}, - { instancesDistribution+: { instanceTypes: ["m5.2xlarge"] }}, - { instancesDistribution+: { instanceTypes: ["m5.8xlarge"] }}, - ]; +local daskNodes = [] + + { apiVersion: 'eksctl.io/v1alpha5', kind: 'ClusterConfig', @@ -42,8 +46,19 @@ local daskNodes = iam: { withOIDC: true, }, + // If you add an addon to this config, run the create addon command. + // + // eksctl create addon --config-file=eksctl-config.yaml + // addons: [ { + // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that + // couple to AWS EBS based storage, without it expect to see pods + // mounting a PVC failing to schedule and PVC resources that are + // unbound. + // + // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html + // name: 'aws-ebs-csi-driver', wellKnownPolicies: { ebsCSIController: true, diff --git a/eksctl/uwhackweeks.jsonnet b/eksctl/uwhackweeks.jsonnet index f4ed65a96c..19c2fcae9b 100644 --- a/eksctl/uwhackweeks.jsonnet +++ b/eksctl/uwhackweeks.jsonnet @@ -1,4 +1,18 @@ -// Exports an eksctl config file for uwhackweeks cluster +/* + This file is a jsonnet template of a eksctl's cluster configuration file, + that is used with the eksctl CLI to both update and initialize an AWS EKS + based cluster. + + This file has in turn been generated from eksctl/template.jsonnet which is + relevant to compare with for changes over time. + + To use jsonnet to generate an eksctl configuration file from this, do: + + jsonnet uwhackweeks.jsonnet > eksctl-config.yaml + + References: + - https://eksctl.io/usage/schema/ +*/ local ng = import "./libsonnet/nodegroup.jsonnet"; // place all cluster nodes here @@ -28,19 +42,20 @@ local notebookNodes = [ }, ]; -// Node definitions for dask worker nodes. Config here is merged -// with our dask worker node definition, which uses spot instances. -// A `node.kubernetes.io/instance-type label is set to the name of the -// *first* item in instanceDistribution.instanceTypes, to match -// what we do with notebook nodes. Pods can request a particular -// kind of node with a nodeSelector local daskNodes = [ + // Node definitions for dask worker nodes. Config here is merged + // with our dask worker node definition, which uses spot instances. + // A `node.kubernetes.io/instance-type label is set to the name of the + // *first* item in instanceDistribution.instanceTypes, to match + // what we do with notebook nodes. Pods can request a particular + // kind of node with a nodeSelector { instancesDistribution+: { instanceTypes: ["m5.large"] }}, { instancesDistribution+: { instanceTypes: ["m5.xlarge"] }}, { instancesDistribution+: { instanceTypes: ["m5.2xlarge"] }}, { instancesDistribution+: { instanceTypes: ["m5.8xlarge"] }}, ]; + { apiVersion: 'eksctl.io/v1alpha5', kind: 'ClusterConfig', @@ -63,6 +78,25 @@ local daskNodes = [ ], } for namespace in namespaces], }, + // If you add an addon to this config, run the create addon command. + // + // eksctl create addon --config-file=eksctl-config.yaml + // + addons: [ + { + // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that + // couple to AWS EBS based storage, without it expect to see pods + // mounting a PVC failing to schedule and PVC resources that are + // unbound. + // + // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html + // + name: 'aws-ebs-csi-driver', + wellKnownPolicies: { + ebsCSIController: true, + }, + }, + ], nodeGroups: [n + {clusterName:: $.metadata.name} for n in [ ng { name: 'core-b', diff --git a/eksctl/victor.jsonnet b/eksctl/victor.jsonnet index bc4a7a8010..8041ad54b1 100644 --- a/eksctl/victor.jsonnet +++ b/eksctl/victor.jsonnet @@ -1,4 +1,18 @@ -// Exports an eksctl config file for carbonplan cluster +/* + This file is a jsonnet template of a eksctl's cluster configuration file, + that is used with the eksctl CLI to both update and initialize an AWS EKS + based cluster. + + This file has in turn been generated from eksctl/template.jsonnet which is + relevant to compare with for changes over time. + + To use jsonnet to generate an eksctl configuration file from this, do: + + jsonnet victor.jsonnet > eksctl-config.yaml + + References: + - https://eksctl.io/usage/schema/ +*/ local ng = import "./libsonnet/nodegroup.jsonnet"; // place all cluster nodes here @@ -17,19 +31,20 @@ local notebookNodes = [ { instanceType: "m5.8xlarge" }, ]; -// Node definitions for dask worker nodes. Config here is merged -// with our dask worker node definition, which uses spot instances. -// A `node.kubernetes.io/instance-type label is set to the name of the -// *first* item in instanceDistribution.instanceTypes, to match -// what we do with notebook nodes. Pods can request a particular -// kind of node with a nodeSelector local daskNodes = [ + // Node definitions for dask worker nodes. Config here is merged + // with our dask worker node definition, which uses spot instances. + // A `node.kubernetes.io/instance-type label is set to the name of the + // *first* item in instanceDistribution.instanceTypes, to match + // what we do with notebook nodes. Pods can request a particular + // kind of node with a nodeSelector { instancesDistribution+: { instanceTypes: ["m5.large"] }}, { instancesDistribution+: { instanceTypes: ["m5.xlarge"] }}, { instancesDistribution+: { instanceTypes: ["m5.2xlarge"] }}, { instancesDistribution+: { instanceTypes: ["m5.8xlarge"] }}, ]; + { apiVersion: 'eksctl.io/v1alpha5', kind: 'ClusterConfig', @@ -45,6 +60,25 @@ local daskNodes = [ iam: { withOIDC: true, }, + // If you add an addon to this config, run the create addon command. + // + // eksctl create addon --config-file=eksctl-config.yaml + // + addons: [ + { + // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that + // couple to AWS EBS based storage, without it expect to see pods + // mounting a PVC failing to schedule and PVC resources that are + // unbound. + // + // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html + // + name: 'aws-ebs-csi-driver', + wellKnownPolicies: { + ebsCSIController: true, + }, + }, + ], nodeGroups: [ ng { name: 'core-a',