update nodepool(s): generic volume handling

banzaicloud · Aug 23, 2021 · fd43b4f · fd43b4f
1 parent 3d0f8c0
commit fd43b4f
Show file tree

Hide file tree

Showing 12 changed files with 531 additions and 262 deletions.
diff --git a/internal/cluster/distribution/eks/eksadapter/node_pool_manager.go b/internal/cluster/distribution/eks/eksadapter/node_pool_manager.go
@@ -235,6 +235,31 @@ func (n nodePoolManager) UpdateNodePool(
 		ExecutionStartToCloseTimeout: 30 * 24 * 60 * time.Minute,
 	}
 
+	if nodePoolUpdate.Volumes == nil {
+		nodePoolUpdate.Volumes = &eks.NodePoolVolumes{
+			InstanceRoot: &eks.NodePoolVolume{
+				Type:    "gp3",
+				Storage: eks.EBS_STORAGE,
+			},
+		}
+
+		// copy deprecated property values
+		if nodePoolUpdate.VolumeSize > 0 {
+			nodePoolUpdate.Volumes.InstanceRoot.Size = nodePoolUpdate.VolumeSize
+		}
+		if nodePoolUpdate.VolumeType != "" {
+			nodePoolUpdate.Volumes.InstanceRoot.Type = nodePoolUpdate.VolumeType
+		}
+		if nodePoolUpdate.VolumeEncryption != nil {
+			nodePoolUpdate.Volumes.InstanceRoot.Encryption = nodePoolUpdate.VolumeEncryption
+		}
+		if nodePoolUpdate.UseInstanceStore != nil && *nodePoolUpdate.UseInstanceStore {
+			nodePoolUpdate.Volumes.KubeletRoot = &eks.NodePoolVolume{
+				Storage: eks.INSTANCE_STORE_STORAGE,
+			}
+		}
+	}
+
 	input := eksworkflow.UpdateNodePoolWorkflowInput{
 		ProviderSecretID: c.SecretID.String(),
 		Region:           c.Location,
@@ -247,12 +272,9 @@ func (n nodePoolManager) UpdateNodePool(
 		NodePoolName:    nodePoolName,
 		OrganizationID:  c.OrganizationID,
 
-		NodeVolumeEncryption: nodePoolUpdate.VolumeEncryption,
-		NodeVolumeSize:       nodePoolUpdate.VolumeSize,
-		NodeVolumeType:       nodePoolUpdate.VolumeType,
-		NodeImage:            nodePoolUpdate.Image,
-		SecurityGroups:       nodePoolUpdate.SecurityGroups,
-		UseInstanceStore:     nodePoolUpdate.UseInstanceStore,
+		NodeVolumes:    nodePoolUpdate.Volumes,
+		NodeImage:      nodePoolUpdate.Image,
+		SecurityGroups: nodePoolUpdate.SecurityGroups,
 
 		Options: eks.NodePoolUpdateOptions{
 			MaxSurge:       nodePoolUpdate.Options.MaxSurge,

diff --git a/internal/cluster/distribution/eks/eksadapter/node_pool_processor.go b/internal/cluster/distribution/eks/eksadapter/node_pool_processor.go
@@ -123,7 +123,7 @@ func (p nodePoolProcessor) ProcessNewNodePool(
 	}
 
 	// default kubelet root EBS size to 50GB
-	if nodePool.Volumes.KubeletRoot != nil && nodePool.Volumes.KubeletRoot.Storage == eks.INSTANCE_STORE_STORAGE &&
+	if nodePool.Volumes.KubeletRoot != nil && nodePool.Volumes.KubeletRoot.Storage == eks.EBS_STORAGE &&
 		nodePool.Volumes.KubeletRoot.Size == 0 {
 		nodePool.Volumes.KubeletRoot.Size = 50
 	}

diff --git a/internal/cluster/distribution/eks/ekscluster/eks.go b/internal/cluster/distribution/eks/ekscluster/eks.go
@@ -413,7 +413,7 @@ func (eks *CreateClusterEKS) AddDefaults(location string) error {
 		}
 
 		// default kubelet root EBS size to 50GB
-		if np.Volumes.KubeletRoot != nil && np.Volumes.KubeletRoot.Storage == INSTANCE_STORE_STORAGE &&
+		if np.Volumes.KubeletRoot != nil && np.Volumes.KubeletRoot.Storage == EBS_STORAGE &&
 			np.Volumes.KubeletRoot.Size == 0 {
 			np.Volumes.KubeletRoot.Size = 50
 		}

diff --git a/internal/cluster/distribution/eks/eksprovider/driver/cluster_creator.go b/internal/cluster/distribution/eks/eksprovider/driver/cluster_creator.go
@@ -307,8 +307,9 @@ func getVolumeParams(volume *pkgEks.NodePoolVolume) *eks.NodePoolVolume {
 	}
 
 	newVolume := &eks.NodePoolVolume{
-		Size: volume.Size,
-		Type: volume.Type,
+		Storage: eks.EBS_STORAGE,
+		Size:    volume.Size,
+		Type:    volume.Type,
 	}
 	if volume.Encryption != nil {
 		newVolume.Encryption = &eks.NodePoolVolumeEncryption{

diff --git a/internal/cluster/distribution/eks/eksprovider/driver/cluster_updater.go b/internal/cluster/distribution/eks/eksprovider/driver/cluster_updater.go
@@ -222,32 +222,32 @@ func newASGsFromRequestedUpdatedNodePools(
 	}
 
 	for nodePoolName, nodePool := range requestedUpdatedNodePools {
-		var volumeEncryption *eks.NodePoolVolumeEncryption
-		if nodePool.VolumeEncryption != nil {
-			volumeEncryption = &eks.NodePoolVolumeEncryption{
-				Enabled:          nodePool.VolumeEncryption.Enabled,
-				EncryptionKeyARN: nodePool.VolumeEncryption.EncryptionKeyARN,
+		var nodePoolVolumes *eks.NodePoolVolumes
+		if nodePool.Volumes != nil {
+			nodePoolVolumes = &eks.NodePoolVolumes{}
+			if nodePool.Volumes.KubeletRoot != nil {
+				nodePoolVolumes.KubeletRoot = getVolumeParams(nodePool.Volumes.KubeletRoot)
+			}
+			if nodePool.Volumes.InstanceRoot != nil {
+				nodePoolVolumes.InstanceRoot = getVolumeParams(nodePool.Volumes.InstanceRoot)
 			}
 		}
 
 		updatedNodePools = append(updatedNodePools, workflow.AutoscaleGroup{
-			Name:                 nodePoolName,
-			NodeSpotPrice:        nodePool.SpotPrice,
-			Autoscaling:          nodePool.Autoscaling,
-			NodeMinCount:         nodePool.MinCount,
-			NodeMaxCount:         nodePool.MaxCount,
-			Count:                nodePool.Count,
-			NodeVolumeEncryption: volumeEncryption,
-			NodeVolumeSize:       nodePool.VolumeSize,
-			NodeVolumeType:       nodePool.VolumeType,
-			NodeImage:            nodePool.Image,
-			NodeInstanceType:     nodePool.InstanceType,
-			SecurityGroups:       nodePool.SecurityGroups,
-			UseInstanceStore:     nodePool.UseInstanceStore,
-			Labels:               nodePool.Labels,
-			Delete:               false,
-			Create:               false,
-			CreatedBy:            creators[nodePoolName],
+			Name:             nodePoolName,
+			NodeSpotPrice:    nodePool.SpotPrice,
+			Autoscaling:      nodePool.Autoscaling,
+			NodeMinCount:     nodePool.MinCount,
+			NodeMaxCount:     nodePool.MaxCount,
+			Count:            nodePool.Count,
+			NodeImage:        nodePool.Image,
+			NodeInstanceType: nodePool.InstanceType,
+			Volumes:          nodePoolVolumes,
+			SecurityGroups:   nodePool.SecurityGroups,
+			Labels:           nodePool.Labels,
+			Delete:           false,
+			Create:           false,
+			CreatedBy:        creators[nodePoolName],
 		})
 	}
 
@@ -410,6 +410,42 @@ func newNodePoolsFromUpdateRequest(
 				nodePool.SpotPrice = eks.DefaultSpotPrice
 			}
 
+			// convert deprecated params in case no Volumes struct is specified
+			if nodePool.Volumes == nil {
+				volumes := pkgEks.NodePoolVolumes{}
+				instanceRootVolume := pkgEks.NodePoolVolume{}
+				isInstanceRoot := false
+
+				if nodePool.VolumeSize > 0 {
+					isInstanceRoot = true
+					instanceRootVolume.Size = nodePool.VolumeSize
+				}
+				if nodePool.VolumeType != "" {
+					isInstanceRoot = true
+					instanceRootVolume.Type = nodePool.VolumeType
+				}
+				if nodePool.VolumeEncryption != nil {
+					isInstanceRoot = true
+					instanceRootVolume.Encryption = nodePool.VolumeEncryption
+				}
+
+				if isInstanceRoot {
+					volumes.InstanceRoot = &instanceRootVolume
+				}
+
+				isKubeletRoot := false
+				if nodePool.UseInstanceStore != nil && *nodePool.UseInstanceStore {
+					isKubeletRoot = true
+					volumes.KubeletRoot = &pkgEks.NodePoolVolume{
+						Storage: pkgEks.INSTANCE_STORE_STORAGE,
+					}
+				}
+
+				if isInstanceRoot || isKubeletRoot {
+					nodePool.Volumes = &volumes
+				}
+			}
+
 			requestedNewNodePools[nodePoolName] = nodePool
 		}
 	}

diff --git a/internal/cluster/distribution/eks/eksprovider/workflow/amazon.go b/internal/cluster/distribution/eks/eksprovider/workflow/amazon.go
@@ -220,22 +220,30 @@ func NewSubnetsFromEKSSubnets(
 // TODO: remove when UpdateNodePoolWorkflow is refactored and this is not needed
 // anymore.
 type AutoscaleGroup struct {
-	Name                 string
-	NodeSpotPrice        string
-	Autoscaling          bool
-	NodeMinCount         int
-	NodeMaxCount         int
-	Count                int
+	Name          string
+	NodeSpotPrice string
+	Autoscaling   bool
+	NodeMinCount  int
+	NodeMaxCount  int
+	Count         int
+
+	Volumes *eks.NodePoolVolumes `json:"volumes,omitempty"`
+
+	// deprecated, property replaced with Volumes.InstanceRoot.Encryption
 	NodeVolumeEncryption *eks.NodePoolVolumeEncryption
-	NodeVolumeSize       int
-	NodeVolumeType       string
-	NodeImage            string
-	NodeInstanceType     string
+	// deprecated, property replaced with Volumes.InstanceRoot.Size
+	NodeVolumeSize int
+	// deprecated, property replaced with Volumes.InstanceRoot.Type
+	NodeVolumeType string
+	// deprecated, property replaced with Volumes.KubeletRoot.Type="instance-storage"
+	UseInstanceStore *bool `json:"useInstanceStore,omitempty" yaml:"useInstanceStore,omitempty"`
+
+	NodeImage        string
+	NodeInstanceType string
 
 	// SecurityGroups collects the user specified custom node security group
 	// IDs.
-	SecurityGroups   []string
-	UseInstanceStore *bool
+	SecurityGroups []string
 
 	Labels    map[string]string
 	Delete    bool

diff --git a/internal/cluster/distribution/eks/eksprovider/workflow/create_asg_activity.go b/internal/cluster/distribution/eks/eksprovider/workflow/create_asg_activity.go
@@ -139,8 +139,9 @@ func (a *CreateAsgActivity) Execute(ctx context.Context, input CreateAsgActivity
 	}
 
 	nodeVolumeStorageStorage, nodeVolumeSize, nodeVolumeEncryptionEnabled, nodeVolumeEncryptionKeyARN, nodeVolumeType :=
-		a.getDefaultedVolumeParams(input.NodeVolumes.InstanceRoot)
-	kubeletRootVolumeStorage, kubeletRootVolumeSize, kubeletRootVolumeEncryptionEnabled, kubeletRootVolumeEncryptionKeyARN, kubeletRootVolumeType := a.getDefaultedVolumeParams(input.NodeVolumes.KubeletRoot)
+		getDefaultedTemplateVolumeParams(input.NodeVolumes.InstanceRoot, a.defaultNodeVolumeEncryption)
+	kubeletRootVolumeStorage, kubeletRootVolumeSize, kubeletRootVolumeEncryptionEnabled, kubeletRootVolumeEncryptionKeyARN, kubeletRootVolumeType :=
+		getDefaultedTemplateVolumeParams(input.NodeVolumes.KubeletRoot, a.defaultNodeVolumeEncryption)
 
 	var stackTagsBuilder strings.Builder
 	for tagIndex, tag := range tags {
@@ -326,38 +327,43 @@ func (a *CreateAsgActivity) Execute(ctx context.Context, input CreateAsgActivity
 	return &outParams, nil
 }
 
-func (a *CreateAsgActivity) getDefaultedVolumeParams(volume *eks.NodePoolVolume) (string, int, string, string, string) {
-	storageType := "none"
+func getDefaultedTemplateVolumeParams(volume *eks.NodePoolVolume, defaultNodeVolumeEncryption *eks.NodePoolVolumeEncryption) (string, int, string, string, string) {
+	storageType := ""
 	size := 0
 	nodeVolumeEncryptionEnabled := "" // Note: defaulting to AWS account default encryption settings.
 	nodeVolumeEncryptionKeyARN := ""
-	nodeVolumeType := "gp3"
+	nodeVolumeType := ""
 
 	if volume == nil {
 		return storageType, size, nodeVolumeEncryptionEnabled, nodeVolumeEncryptionKeyARN, nodeVolumeType
 	}
 
 	storageType = volume.Storage
-	size = volume.Size
 
-	if volume.Encryption != nil {
-		nodeVolumeEncryptionEnabled = strconv.FormatBool(volume.Encryption.Enabled)
-	} else if a.defaultNodeVolumeEncryption != nil {
-		nodeVolumeEncryptionEnabled = strconv.FormatBool(a.defaultNodeVolumeEncryption.Enabled)
-	}
+	if eks.EBS_STORAGE == storageType {
+		size = volume.Size
 
-	if nodeVolumeEncryptionEnabled == "true" &&
-		volume.Encryption != nil &&
-		volume.Encryption.EncryptionKeyARN != "" {
-		nodeVolumeEncryptionKeyARN = volume.Encryption.EncryptionKeyARN
-	} else if nodeVolumeEncryptionEnabled == "true" &&
-		a.defaultNodeVolumeEncryption != nil &&
-		a.defaultNodeVolumeEncryption.EncryptionKeyARN != "" {
-		nodeVolumeEncryptionKeyARN = a.defaultNodeVolumeEncryption.EncryptionKeyARN
-	}
+		if volume.Encryption != nil {
+			nodeVolumeEncryptionEnabled = strconv.FormatBool(volume.Encryption.Enabled)
+		} else if defaultNodeVolumeEncryption != nil {
+			nodeVolumeEncryptionEnabled = strconv.FormatBool(defaultNodeVolumeEncryption.Enabled)
+		}
 
-	if volume.Type != "" {
-		nodeVolumeType = volume.Type
+		if nodeVolumeEncryptionEnabled == "true" &&
+			volume.Encryption != nil &&
+			volume.Encryption.EncryptionKeyARN != "" {
+			nodeVolumeEncryptionKeyARN = volume.Encryption.EncryptionKeyARN
+		} else if nodeVolumeEncryptionEnabled == "true" &&
+			defaultNodeVolumeEncryption != nil &&
+			defaultNodeVolumeEncryption.EncryptionKeyARN != "" {
+			nodeVolumeEncryptionKeyARN = defaultNodeVolumeEncryption.EncryptionKeyARN
+		}
+
+		if volume.Type != "" {
+			nodeVolumeType = volume.Type
+		} else {
+			nodeVolumeType = "gp3"
+		}
 	}
 	return storageType, size, nodeVolumeEncryptionEnabled, nodeVolumeEncryptionKeyARN, nodeVolumeType
 }