Merge pull request #2149 from consideRatio/pr/add-ebs-addon-systemati…

…cally aws clusters: systematically add ebs-csi-driver addon before k8s 1.23+ upgrade
2i2c-org · Feb 6, 2023 · 8f2cd3e · 8f2cd3e
2 parents 175a229 + 799e4ac
commit 8f2cd3e
Show file tree

Hide file tree

Showing 8 changed files with 275 additions and 74 deletions.
diff --git a/eksctl/2i2c-aws-us.jsonnet b/eksctl/2i2c-aws-us.jsonnet
@@ -1,4 +1,18 @@
-// Exports an eksctl config file for carbonplan cluster
+/*
+    This file is a jsonnet template of a eksctl's cluster configuration file,
+    that is used with the eksctl CLI to both update and initialize an AWS EKS
+    based cluster.
+
+    This file has in turn been generated from eksctl/template.jsonnet which is
+    relevant to compare with for changes over time.
+
+    To use jsonnet to generate an eksctl configuration file from this, do:
+
+        jsonnet 2i2c-aws-us.jsonnet > eksctl-config.yaml
+
+    References:
+    - https://eksctl.io/usage/schema/
+*/
 local ng = import "./libsonnet/nodegroup.jsonnet";
 
 // place all cluster nodes here
@@ -23,19 +37,20 @@ local notebookNodes = [
     },
 ];
 
-local daskNodes =
-    if "daskhub" == "daskhub" then [
+local daskNodes = [
     // Node definitions for dask worker nodes. Config here is merged
     // with our dask worker node definition, which uses spot instances.
     // A `node.kubernetes.io/instance-type label is set to the name of the
     // *first* item in instanceDistribution.instanceTypes, to match
     // what we do with notebook nodes. Pods can request a particular
     // kind of node with a nodeSelector
-        { instancesDistribution+: { instanceTypes: ["m5.large"] }},
-        { instancesDistribution+: { instanceTypes: ["m5.xlarge"] }},
-        { instancesDistribution+: { instanceTypes: ["m5.2xlarge"] }},
-        { instancesDistribution+: { instanceTypes: ["m5.8xlarge"] }},
-    ];
+    { instancesDistribution+: { instanceTypes: ["m5.large"] }},
+    { instancesDistribution+: { instanceTypes: ["m5.xlarge"] }},
+    { instancesDistribution+: { instanceTypes: ["m5.2xlarge"] }},
+    { instancesDistribution+: { instanceTypes: ["m5.8xlarge"] }},
+];
+
+
 {
     apiVersion: 'eksctl.io/v1alpha5',
     kind: 'ClusterConfig',
@@ -51,6 +66,25 @@ local daskNodes =
     iam: {
         withOIDC: true,
     },
+    // If you add an addon to this config, run the create addon command.
+    //
+    //    eksctl create addon --config-file=eksctl-config.yaml
+    //
+    addons: [
+        {
+            // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that
+            // couple to AWS EBS based storage, without it expect to see pods
+            // mounting a PVC failing to schedule and PVC resources that are
+            // unbound.
+            //
+            // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html
+            //
+            name: 'aws-ebs-csi-driver',
+            wellKnownPolicies: {
+                ebsCSIController: true,
+            },
+        },
+    ],
     nodeGroups: [
         ng {
             name: 'core-a',

diff --git a/eksctl/carbonplan.jsonnet b/eksctl/carbonplan.jsonnet
@@ -1,14 +1,18 @@
-// This file is a jinja2 template of a jsonnet template of a eksctl's cluster
-// configuration file, which is in turn can be used with the `eksctl` CLI to both
-// update and initialize a AWS EKS based cluster.
-//
-// This jinja2 template is only used by the deployer script as part of creating
-// new clusters. If a relevant change is made here or the dependent file
-// libsonnet/nodegroup.jsonnet, one may consider if we should manually update
-// already generated jsonnet files in this folder.
-//
-// Configuration reference: https://eksctl.io/usage/schema/
-//
+/*
+    This file is a jsonnet template of a eksctl's cluster configuration file,
+    that is used with the eksctl CLI to both update and initialize an AWS EKS
+    based cluster.
+
+    This file has in turn been generated from eksctl/template.jsonnet which is
+    relevant to compare with for changes over time.
+
+    To use jsonnet to generate an eksctl configuration file from this, do:
+
+        jsonnet carbonplan.jsonnet > eksctl-config.yaml
+
+    References:
+    - https://eksctl.io/usage/schema/
+*/
 local ng = import "./libsonnet/nodegroup.jsonnet";
 
 // place all cluster nodes here
@@ -69,6 +73,10 @@ local daskNodes = [
             ],
         } for namespace in namespaces],
     },
+    // If you add an addon to this config, run the create addon command.
+    //
+    //    eksctl create addon --config-file=eksctl-config.yaml
+    //
     addons: [
         {
             // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that

diff --git a/eksctl/gridsst.jsonnet b/eksctl/gridsst.jsonnet
@@ -1,4 +1,18 @@
-// Exports an eksctl config file for carbonplan cluster
+/*
+    This file is a jsonnet template of a eksctl's cluster configuration file,
+    that is used with the eksctl CLI to both update and initialize an AWS EKS
+    based cluster.
+
+    This file has in turn been generated from eksctl/template.jsonnet which is
+    relevant to compare with for changes over time.
+
+    To use jsonnet to generate an eksctl configuration file from this, do:
+
+        jsonnet gridsst.jsonnet > eksctl-config.yaml
+
+    References:
+    - https://eksctl.io/usage/schema/
+*/
 local ng = import "./libsonnet/nodegroup.jsonnet";
 
 // place all cluster nodes here
@@ -23,19 +37,20 @@ local notebookNodes = [
     },
 ];
 
-// Node definitions for dask worker nodes. Config here is merged
-// with our dask worker node definition, which uses spot instances.
-// A `node.kubernetes.io/instance-type label is set to the name of the
-// *first* item in instanceDistribution.instanceTypes, to match
-// what we do with notebook nodes. Pods can request a particular
-// kind of node with a nodeSelector
 local daskNodes = [
+    // Node definitions for dask worker nodes. Config here is merged
+    // with our dask worker node definition, which uses spot instances.
+    // A `node.kubernetes.io/instance-type label is set to the name of the
+    // *first* item in instanceDistribution.instanceTypes, to match
+    // what we do with notebook nodes. Pods can request a particular
+    // kind of node with a nodeSelector
     { instancesDistribution+: { instanceTypes: ["m5.large"] }},
     { instancesDistribution+: { instanceTypes: ["m5.xlarge"] }},
     { instancesDistribution+: { instanceTypes: ["m5.2xlarge"] }},
     { instancesDistribution+: { instanceTypes: ["m5.8xlarge"] }},
 ];
 
+
 {
     apiVersion: 'eksctl.io/v1alpha5',
     kind: 'ClusterConfig',
@@ -48,6 +63,25 @@ local daskNodes = [
     iam: {
         withOIDC: true,
     },
+    // If you add an addon to this config, run the create addon command.
+    //
+    //    eksctl create addon --config-file=eksctl-config.yaml
+    //
+    addons: [
+        {
+            // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that
+            // couple to AWS EBS based storage, without it expect to see pods
+            // mounting a PVC failing to schedule and PVC resources that are
+            // unbound.
+            //
+            // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html
+            //
+            name: 'aws-ebs-csi-driver',
+            wellKnownPolicies: {
+                ebsCSIController: true,
+            },
+        },
+    ],
     nodeGroups: [
         ng {
             name: 'core-a',

diff --git a/eksctl/nasa-cryo.jsonnet b/eksctl/nasa-cryo.jsonnet
@@ -1,4 +1,18 @@
-// Exports an eksctl config file for carbonplan cluster
+/*
+    This file is a jsonnet template of a eksctl's cluster configuration file,
+    that is used with the eksctl CLI to both update and initialize an AWS EKS
+    based cluster.
+
+    This file has in turn been generated from eksctl/template.jsonnet which is
+    relevant to compare with for changes over time.
+
+    To use jsonnet to generate an eksctl configuration file from this, do:
+
+        jsonnet nasa-cryo.jsonnet > eksctl-config.yaml
+
+    References:
+    - https://eksctl.io/usage/schema/
+*/
 local ng = import "./libsonnet/nodegroup.jsonnet";
 
 // place all cluster nodes here
@@ -17,19 +31,20 @@ local notebookNodes = [
     { instanceType: "m5.8xlarge" },
 ];
 
-// Node definitions for dask worker nodes. Config here is merged
-// with our dask worker node definition, which uses spot instances.
-// A `node.kubernetes.io/instance-type label is set to the name of the
-// *first* item in instanceDistribution.instanceTypes, to match
-// what we do with notebook nodes. Pods can request a particular
-// kind of node with a nodeSelector
 local daskNodes = [
+    // Node definitions for dask worker nodes. Config here is merged
+    // with our dask worker node definition, which uses spot instances.
+    // A `node.kubernetes.io/instance-type label is set to the name of the
+    // *first* item in instanceDistribution.instanceTypes, to match
+    // what we do with notebook nodes. Pods can request a particular
+    // kind of node with a nodeSelector
     { instancesDistribution+: { instanceTypes: ["m5.large"] }},
     { instancesDistribution+: { instanceTypes: ["m5.xlarge"] }},
     { instancesDistribution+: { instanceTypes: ["m5.2xlarge"] }},
     { instancesDistribution+: { instanceTypes: ["m5.8xlarge"] }},
 ];
 
+
 {
     apiVersion: 'eksctl.io/v1alpha5',
     kind: 'ClusterConfig',
@@ -42,6 +57,25 @@ local daskNodes = [
     iam: {
         withOIDC: true,
     },
+    // If you add an addon to this config, run the create addon command.
+    //
+    //    eksctl create addon --config-file=eksctl-config.yaml
+    //
+    addons: [
+        {
+            // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that
+            // couple to AWS EBS based storage, without it expect to see pods
+            // mounting a PVC failing to schedule and PVC resources that are
+            // unbound.
+            //
+            // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html
+            //
+            name: 'aws-ebs-csi-driver',
+            wellKnownPolicies: {
+                ebsCSIController: true,
+            },
+        },
+    ],
     nodeGroups: [
         ng {
             name: 'core-a',

diff --git a/eksctl/nasa-veda.jsonnet b/eksctl/nasa-veda.jsonnet
@@ -1,14 +1,18 @@
-// This file is a jinja2 template of a jsonnet template of a eksctl's cluster
-// configuration file, which is in turn can be used with the `eksctl` CLI to both
-// update and initialize a AWS EKS based cluster.
-//
-// This jinja2 template is only used by the deployer script as part of creating
-// new clusters. If a relevant change is made here or the dependent file
-// libsonnet/nodegroup.jsonnet, one may consider if we should manually update
-// already generated jsonnet files in this folder.
-//
-// Configuration reference: https://eksctl.io/usage/schema/
-//
+/*
+    This file is a jsonnet template of a eksctl's cluster configuration file,
+    that is used with the eksctl CLI to both update and initialize an AWS EKS
+    based cluster.
+
+    This file has in turn been generated from eksctl/template.jsonnet which is
+    relevant to compare with for changes over time.
+
+    To use jsonnet to generate an eksctl configuration file from this, do:
+
+        jsonnet nasa-veda.jsonnet > eksctl-config.yaml
+
+    References:
+    - https://eksctl.io/usage/schema/
+*/
 local ng = import "./libsonnet/nodegroup.jsonnet";
 
 // place all cluster nodes here
@@ -26,6 +30,7 @@ local notebookNodes = [
     { instanceType: "m5.2xlarge" },
     { instanceType: "m5.8xlarge" },
 ];
+
 local daskNodes = [
     // Node definitions for dask worker nodes. Config here is merged
     // with our dask worker node definition, which uses spot instances.
@@ -38,9 +43,8 @@ local daskNodes = [
     { instancesDistribution+: { instanceTypes: ["m5.2xlarge"] }},
     { instancesDistribution+: { instanceTypes: ["m5.8xlarge"] }},
 ];
-
 
-                       
+
 {
     apiVersion: 'eksctl.io/v1alpha5',
     kind: 'ClusterConfig',
@@ -53,6 +57,10 @@ local daskNodes = [
     iam: {
         withOIDC: true,
     },
+    // If you add an addon to this config, run the create addon command.
+    //
+    //    eksctl create addon --config-file=eksctl-config.yaml
+    //
     addons: [
         {
             // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that

diff --git a/eksctl/ubc-eoas.jsonnet b/eksctl/ubc-eoas.jsonnet
@@ -1,4 +1,18 @@
-// Exports an eksctl config file for carbonplan cluster
+/*
+    This file is a jsonnet template of a eksctl's cluster configuration file,
+    that is used with the eksctl CLI to both update and initialize an AWS EKS
+    based cluster.
+
+    This file has in turn been generated from eksctl/template.jsonnet which is
+    relevant to compare with for changes over time.
+
+    To use jsonnet to generate an eksctl configuration file from this, do:
+
+        jsonnet ubc-eoas.jsonnet > eksctl-config.yaml
+
+    References:
+    - https://eksctl.io/usage/schema/
+*/
 local ng = import "./libsonnet/nodegroup.jsonnet";
 
 // place all cluster nodes here
@@ -17,19 +31,9 @@ local notebookNodes = [
     { instanceType: "m5.8xlarge" },
 ];
 
-local daskNodes =
-    if "basehub" == "daskhub" then [
-    // Node definitions for dask worker nodes. Config here is merged
-    // with our dask worker node definition, which uses spot instances.
-    // A `node.kubernetes.io/instance-type label is set to the name of the
-    // *first* item in instanceDistribution.instanceTypes, to match
-    // what we do with notebook nodes. Pods can request a particular
-    // kind of node with a nodeSelector
-        { instancesDistribution+: { instanceTypes: ["m5.large"] }},
-        { instancesDistribution+: { instanceTypes: ["m5.xlarge"] }},
-        { instancesDistribution+: { instanceTypes: ["m5.2xlarge"] }},
-        { instancesDistribution+: { instanceTypes: ["m5.8xlarge"] }},
-    ];
+local daskNodes = []
+
+
 {
     apiVersion: 'eksctl.io/v1alpha5',
     kind: 'ClusterConfig',
@@ -42,8 +46,19 @@ local daskNodes =
     iam: {
         withOIDC: true,
     },
+    // If you add an addon to this config, run the create addon command.
+    //
+    //    eksctl create addon --config-file=eksctl-config.yaml
+    //
     addons: [
         {
+            // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that
+            // couple to AWS EBS based storage, without it expect to see pods
+            // mounting a PVC failing to schedule and PVC resources that are
+            // unbound.
+            //
+            // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html
+            //
             name: 'aws-ebs-csi-driver',
             wellKnownPolicies: {
                 ebsCSIController: true,