From 9531454bba75c6d149d1a531f507cdb2d406187c Mon Sep 17 00:00:00 2001 From: Mike Norgate Date: Wed, 10 Apr 2024 17:04:45 +0100 Subject: [PATCH 1/6] feat: add support for setting node affinity selectors for dedicated nodes --- castai/resource_node_template.go | 73 +++++++++++++++++++++++++++++++- docs/resources/node_template.md | 14 ++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/castai/resource_node_template.go b/castai/resource_node_template.go index 20fc1749..155980c9 100644 --- a/castai/resource_node_template.go +++ b/castai/resource_node_template.go @@ -59,6 +59,10 @@ const ( FieldNodeTemplateDedicatedNodeAffinity = "dedicated_node_affinity" FieldNodeTemplateAzName = "az_name" FieldNodeTemplateInstanceTypes = "instance_types" + FieldNodeTemplateAffinityName = "affinity" + FieldNodeTemplateAffinityKeyName = "key" + FieldNodeTemplateAffinityOperatorName = "operator" + FieldNodeTemplateAffinityValuesName = "values" ) const ( @@ -73,9 +77,19 @@ const ( OsWindows = "windows" ) +const ( + NodeSelectorOperationIn = "In" + NodeSelectorOperationNotIn = "NotIn" + NodeSelectorOperationExists = "Exists" + NodeSelectorOperationDoesNot = "DoesNotExist" + NodeSelectorOperationGt = "Gt" + NodeSelectorOperationLt = "Lt" +) + func resourceNodeTemplate() *schema.Resource { supportedArchitectures := []string{ArchAMD64, ArchARM64} supportedOs := []string{OsLinux, OsWindows} + supportedSelectorOperations := []string{NodeSelectorOperationIn, NodeSelectorOperationNotIn, NodeSelectorOperationExists, NodeSelectorOperationDoesNot, NodeSelectorOperationGt, NodeSelectorOperationLt} return &schema.Resource{ CreateContext: resourceNodeTemplateCreate, @@ -368,10 +382,37 @@ func resourceNodeTemplate() *schema.Resource { Description: "Availability zone name.", }, FieldNodeTemplateName: { - Required: true, + Optional: true, Type: schema.TypeString, Description: "Name of node group.", }, + FieldNodeTemplateAffinityName: { + Optional: true, + Type: schema.TypeList, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + FieldNodeTemplateAffinityKeyName: { + Required: true, + Type: schema.TypeString, + Description: "Key of the node affinity selector.", + }, + FieldNodeTemplateAffinityOperatorName: { + Required: true, + Type: schema.TypeString, + ValidateDiagFunc: validation.ToDiagFunc(validation.StringInSlice(supportedSelectorOperations, false)), + Description: fmt.Sprintf("Operator of the node affinity selector. Allowed values: %s.", strings.Join(supportedSelectorOperations, ", ")), + }, + FieldNodeTemplateAffinityValuesName: { + Required: true, + Type: schema.TypeList, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + Description: "Values of the node affinity selector.", + }, + }, + }, + }, }, }, }, @@ -638,6 +679,17 @@ func flattenNodeAffinity(affinities []sdk.NodetemplatesV1TemplateConstraintsDedi result[FieldNodeTemplateName] = lo.FromPtr(item.Name) result[FieldNodeTemplateAzName] = lo.FromPtr(item.AzName) + + if item.Affinity != nil && len(*item.Affinity) > 0 { + result[FieldNodeTemplateAffinityName] = lo.Map(*item.Affinity, func(affinity sdk.K8sSelectorV1KubernetesNodeAffinity, index int) map[string]any { + return map[string]any{ + FieldNodeTemplateAffinityKeyName: affinity.Key, + FieldNodeTemplateAffinityOperatorName: affinity.Operator, + FieldNodeTemplateAffinityValuesName: affinity.Values, + } + }) + } + return result }) } @@ -1160,6 +1212,25 @@ func toTemplateConstraintsNodeAffinity(o map[string]any) *sdk.NodetemplatesV1Tem if v, ok := o[FieldNodeTemplateInstanceTypes].([]any); ok { out.InstanceTypes = toPtr(toStringList(v)) } + if v, ok := o[FieldNodeTemplateAffinityName].([]any); ok { + out.Affinity = toPtr(lo.Map(v, func(item any, _ int) sdk.K8sSelectorV1KubernetesNodeAffinity { + val, ok := item.(map[string]any) + if !ok { + return sdk.K8sSelectorV1KubernetesNodeAffinity{} + } + out := sdk.K8sSelectorV1KubernetesNodeAffinity{} + if v, ok := val[FieldNodeTemplateAffinityKeyName].(string); ok { + out.Key = v + } + if v, ok := val[FieldNodeTemplateAffinityOperatorName].(string); ok { + out.Operator = sdk.K8sSelectorV1Operator(v) + } + if v, ok := val[FieldNodeTemplateAffinityValuesName].([]any); ok { + out.Values = toStringList(v) + } + return out + })) + } return &out } diff --git a/docs/resources/node_template.md b/docs/resources/node_template.md index 7936f15a..a2afa978 100644 --- a/docs/resources/node_template.md +++ b/docs/resources/node_template.md @@ -87,8 +87,22 @@ Required: - `az_name` (String) Availability zone name. - `instance_types` (List of String) Instance/node types in this node group. + +Optional: + +- `affinity` (Block List) (see [below for nested schema](#nestedblock--constraints--dedicated_node_affinity--affinity)) - `name` (String) Name of node group. + +### Nested Schema for `constraints.dedicated_node_affinity.affinity` + +Required: + +- `key` (String) Key of the node affinity selector. +- `operator` (String) Operator of the node affinity selector. Allowed values: In, NotIn, Exists, DoesNotExist, Gt, Lt. +- `values` (List of String) Values of the node affinity selector. + + ### Nested Schema for `constraints.gpu` From a58d03837c6d7f3bdd929d2d4264980286c7d2e8 Mon Sep 17 00:00:00 2001 From: Mike Norgate Date: Wed, 10 Apr 2024 17:06:58 +0100 Subject: [PATCH 2/6] rebase master --- castai/sdk/api.gen.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/castai/sdk/api.gen.go b/castai/sdk/api.gen.go index 36f7d3a1..968536a2 100644 --- a/castai/sdk/api.gen.go +++ b/castai/sdk/api.gen.go @@ -87,8 +87,8 @@ const ( In K8sSelectorV1Operator = "in" Lt K8sSelectorV1Operator = "Lt" Lt1 K8sSelectorV1Operator = "lt" - NotIn K8sSelectorV1Operator = "notIn" - NotInt K8sSelectorV1Operator = "NotInt" + NotIn K8sSelectorV1Operator = "NotIn" + NotIn1 K8sSelectorV1Operator = "notIn" ) // Defines values for NodeconfigV1AKSConfigOsDiskType. @@ -1577,7 +1577,7 @@ type K8sSelectorV1KubernetesNodeAffinity struct { Key string `json:"key"` // - IN: In values - // - NotInt: Not int values + // - NotIn: Not in values // - Exists: Just exist // - DoesNotExist: Values does not exist // - Gt: Greater then @@ -1587,7 +1587,7 @@ type K8sSelectorV1KubernetesNodeAffinity struct { } // - IN: In values -// - NotInt: Not int values +// - NotIn: Not in values // - Exists: Just exist // - DoesNotExist: Values does not exist // - Gt: Greater then @@ -2117,9 +2117,11 @@ type NodetemplatesV1TemplateConstraintsCustomPriority struct { // NodetemplatesV1TemplateConstraintsDedicatedNodeAffinity defines model for nodetemplates.v1.TemplateConstraints.DedicatedNodeAffinity. type NodetemplatesV1TemplateConstraintsDedicatedNodeAffinity struct { - AzName *string `json:"azName,omitempty"` - InstanceTypes *[]string `json:"instanceTypes,omitempty"` - Name *string `json:"name,omitempty"` + // The affinity rules required for choosing the node. + Affinity *[]K8sSelectorV1KubernetesNodeAffinity `json:"affinity,omitempty"` + AzName *string `json:"azName,omitempty"` + InstanceTypes *[]string `json:"instanceTypes,omitempty"` + Name *string `json:"name,omitempty"` } // NodetemplatesV1TemplateConstraintsGPUConstraints defines model for nodetemplates.v1.TemplateConstraints.GPUConstraints. From fe4b5460eb610d55d56e3932ba94efc9892e5571 Mon Sep 17 00:00:00 2001 From: Mike Norgate Date: Wed, 10 Apr 2024 17:13:15 +0100 Subject: [PATCH 3/6] generate latest SDK --- castai/resource_node_template_test.go | 1 + castai/sdk/api.gen.go | 21 +++++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/castai/resource_node_template_test.go b/castai/resource_node_template_test.go index 1dc79907..731a7435 100644 --- a/castai/resource_node_template_test.go +++ b/castai/resource_node_template_test.go @@ -179,6 +179,7 @@ constraints.0.max_memory = 0 constraints.0.min_cpu = 10 constraints.0.min_memory = 0 constraints.0.dedicated_node_affinity.# = 1 +constraints.0.dedicated_node_affinity.0.affinity.# = 0 constraints.0.dedicated_node_affinity.0.az_name = eu-central-1a constraints.0.dedicated_node_affinity.0.instance_types.# = 1 constraints.0.dedicated_node_affinity.0.instance_types.0 = m5.24xlarge diff --git a/castai/sdk/api.gen.go b/castai/sdk/api.gen.go index 968536a2..513b17db 100644 --- a/castai/sdk/api.gen.go +++ b/castai/sdk/api.gen.go @@ -1685,8 +1685,11 @@ type NodeconfigV1NewNodeConfiguration struct { // Optional docker daemon configuration properties. Provide only properties that you want to override. Available values https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-configuration-file DockerConfig *map[string]interface{} `json:"dockerConfig,omitempty"` - Eks *NodeconfigV1EKSConfig `json:"eks,omitempty"` - Gke *NodeconfigV1GKEConfig `json:"gke,omitempty"` + + // Drain timeout in seconds. Defaults to 0. + DrainTimeoutSec *int32 `json:"drainTimeoutSec"` + Eks *NodeconfigV1EKSConfig `json:"eks,omitempty"` + Gke *NodeconfigV1GKEConfig `json:"gke,omitempty"` // Image to be used while provisioning the node. If nothing is provided will be resolved to latest available image based on Kubernetes version if possible. Image *string `json:"image"` @@ -1737,8 +1740,11 @@ type NodeconfigV1NodeConfiguration struct { // Optional docker daemon configuration properties. Applicable for EKS only. DockerConfig *map[string]interface{} `json:"dockerConfig"` - Eks *NodeconfigV1EKSConfig `json:"eks,omitempty"` - Gke *NodeconfigV1GKEConfig `json:"gke,omitempty"` + + // Drain timeout in seconds. Defaults to 0. + DrainTimeoutSec *int32 `json:"drainTimeoutSec"` + Eks *NodeconfigV1EKSConfig `json:"eks,omitempty"` + Gke *NodeconfigV1GKEConfig `json:"gke,omitempty"` // The node configuration ID. Id *string `json:"id,omitempty"` @@ -1792,8 +1798,11 @@ type NodeconfigV1NodeConfigurationUpdate struct { // Optional docker daemon configuration properties. Provide only properties that you want to override. Available values https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-configuration-file DockerConfig *map[string]interface{} `json:"dockerConfig,omitempty"` - Eks *NodeconfigV1EKSConfig `json:"eks,omitempty"` - Gke *NodeconfigV1GKEConfig `json:"gke,omitempty"` + + // Drain timeout in seconds. Defaults to 0. + DrainTimeoutSec *int32 `json:"drainTimeoutSec"` + Eks *NodeconfigV1EKSConfig `json:"eks,omitempty"` + Gke *NodeconfigV1GKEConfig `json:"gke,omitempty"` // Image to be used while provisioning the node. If nothing is provided will be resolved to latest available image based on Kubernetes version if possible. Image *string `json:"image"` From 07ac6b085a157a63c73728c5d4880954fa8d0e5c Mon Sep 17 00:00:00 2001 From: Mike Norgate Date: Thu, 11 Apr 2024 09:02:07 +0100 Subject: [PATCH 4/6] Add affinity to test --- castai/resource_node_template_test.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/castai/resource_node_template_test.go b/castai/resource_node_template_test.go index 731a7435..ce970169 100644 --- a/castai/resource_node_template_test.go +++ b/castai/resource_node_template_test.go @@ -92,7 +92,14 @@ func TestNodeTemplateResourceReadContext(t *testing.T) { { "name": "foo", "azName": "eu-central-1a", - "instanceTypes": ["m5.24xlarge"] + "instanceTypes": ["m5.24xlarge"], + "affinity": [ + { + "key": "gke.io/gcp-nodepool", + "operator": "In", + "values": ["foo"] + } + ] } ] }, @@ -179,7 +186,11 @@ constraints.0.max_memory = 0 constraints.0.min_cpu = 10 constraints.0.min_memory = 0 constraints.0.dedicated_node_affinity.# = 1 -constraints.0.dedicated_node_affinity.0.affinity.# = 0 +constraints.0.dedicated_node_affinity.0.affinity.# = 1 +constraints.0.dedicated_node_affinity.0.affinity.0.key = gke.io/gcp-nodepool +constraints.0.dedicated_node_affinity.0.affinity.0.operator = In +constraints.0.dedicated_node_affinity.0.affinity.0.values.# = 1 +constraints.0.dedicated_node_affinity.0.affinity.0.values.0 = foo constraints.0.dedicated_node_affinity.0.az_name = eu-central-1a constraints.0.dedicated_node_affinity.0.instance_types.# = 1 constraints.0.dedicated_node_affinity.0.instance_types.0 = m5.24xlarge From efd5d95ba86b89da5d56eef66e5a1d457a343320 Mon Sep 17 00:00:00 2001 From: Mike Norgate Date: Thu, 11 Apr 2024 09:22:06 +0100 Subject: [PATCH 5/6] change affinity node name to required --- castai/resource_node_template.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/castai/resource_node_template.go b/castai/resource_node_template.go index 155980c9..b4bdab92 100644 --- a/castai/resource_node_template.go +++ b/castai/resource_node_template.go @@ -382,7 +382,7 @@ func resourceNodeTemplate() *schema.Resource { Description: "Availability zone name.", }, FieldNodeTemplateName: { - Optional: true, + Required: true, Type: schema.TypeString, Description: "Name of node group.", }, From ea444430d8e96cf1657e8c95008b22a90b7601bd Mon Sep 17 00:00:00 2001 From: Mike Norgate Date: Thu, 11 Apr 2024 09:29:16 +0100 Subject: [PATCH 6/6] re-generate docs --- docs/resources/node_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/resources/node_template.md b/docs/resources/node_template.md index a2afa978..f8a72911 100644 --- a/docs/resources/node_template.md +++ b/docs/resources/node_template.md @@ -87,11 +87,11 @@ Required: - `az_name` (String) Availability zone name. - `instance_types` (List of String) Instance/node types in this node group. +- `name` (String) Name of node group. Optional: - `affinity` (Block List) (see [below for nested schema](#nestedblock--constraints--dedicated_node_affinity--affinity)) -- `name` (String) Name of node group. ### Nested Schema for `constraints.dedicated_node_affinity.affinity`