From fd43b4f702d90f297b17297dceff2aeecc98af7e Mon Sep 17 00:00:00 2001 From: Magyari Sandor Szilard Date: Mon, 23 Aug 2021 19:09:32 +0200 Subject: [PATCH] update nodepool(s): generic volume handling --- .../eks/eksadapter/node_pool_manager.go | 34 ++++- .../eks/eksadapter/node_pool_processor.go | 2 +- .../distribution/eks/ekscluster/eks.go | 2 +- .../eks/eksprovider/driver/cluster_creator.go | 5 +- .../eks/eksprovider/driver/cluster_updater.go | 80 +++++++--- .../eks/eksprovider/workflow/amazon.go | 32 ++-- .../workflow/create_asg_activity.go | 50 ++++--- .../workflow/update_asg_activity.go | 123 ++++++++-------- .../eksworkflow/activity_update_node_group.go | 137 +++++++++++++----- .../eksworkflow/workflow_update_node_pool.go | 103 +++++++------ internal/cluster/distribution/eks/service.go | 136 ++++++++++++----- src/cluster/eks_update_cluster.go | 89 ++++++++---- 12 files changed, 531 insertions(+), 262 deletions(-) diff --git a/internal/cluster/distribution/eks/eksadapter/node_pool_manager.go b/internal/cluster/distribution/eks/eksadapter/node_pool_manager.go index 9361da7bf0..45441c1756 100644 --- a/internal/cluster/distribution/eks/eksadapter/node_pool_manager.go +++ b/internal/cluster/distribution/eks/eksadapter/node_pool_manager.go @@ -235,6 +235,31 @@ func (n nodePoolManager) UpdateNodePool( ExecutionStartToCloseTimeout: 30 * 24 * 60 * time.Minute, } + if nodePoolUpdate.Volumes == nil { + nodePoolUpdate.Volumes = &eks.NodePoolVolumes{ + InstanceRoot: &eks.NodePoolVolume{ + Type: "gp3", + Storage: eks.EBS_STORAGE, + }, + } + + // copy deprecated property values + if nodePoolUpdate.VolumeSize > 0 { + nodePoolUpdate.Volumes.InstanceRoot.Size = nodePoolUpdate.VolumeSize + } + if nodePoolUpdate.VolumeType != "" { + nodePoolUpdate.Volumes.InstanceRoot.Type = nodePoolUpdate.VolumeType + } + if nodePoolUpdate.VolumeEncryption != nil { + nodePoolUpdate.Volumes.InstanceRoot.Encryption = nodePoolUpdate.VolumeEncryption + } + if nodePoolUpdate.UseInstanceStore != nil && *nodePoolUpdate.UseInstanceStore { + nodePoolUpdate.Volumes.KubeletRoot = &eks.NodePoolVolume{ + Storage: eks.INSTANCE_STORE_STORAGE, + } + } + } + input := eksworkflow.UpdateNodePoolWorkflowInput{ ProviderSecretID: c.SecretID.String(), Region: c.Location, @@ -247,12 +272,9 @@ func (n nodePoolManager) UpdateNodePool( NodePoolName: nodePoolName, OrganizationID: c.OrganizationID, - NodeVolumeEncryption: nodePoolUpdate.VolumeEncryption, - NodeVolumeSize: nodePoolUpdate.VolumeSize, - NodeVolumeType: nodePoolUpdate.VolumeType, - NodeImage: nodePoolUpdate.Image, - SecurityGroups: nodePoolUpdate.SecurityGroups, - UseInstanceStore: nodePoolUpdate.UseInstanceStore, + NodeVolumes: nodePoolUpdate.Volumes, + NodeImage: nodePoolUpdate.Image, + SecurityGroups: nodePoolUpdate.SecurityGroups, Options: eks.NodePoolUpdateOptions{ MaxSurge: nodePoolUpdate.Options.MaxSurge, diff --git a/internal/cluster/distribution/eks/eksadapter/node_pool_processor.go b/internal/cluster/distribution/eks/eksadapter/node_pool_processor.go index 9ee86e8a55..c917424f1d 100644 --- a/internal/cluster/distribution/eks/eksadapter/node_pool_processor.go +++ b/internal/cluster/distribution/eks/eksadapter/node_pool_processor.go @@ -123,7 +123,7 @@ func (p nodePoolProcessor) ProcessNewNodePool( } // default kubelet root EBS size to 50GB - if nodePool.Volumes.KubeletRoot != nil && nodePool.Volumes.KubeletRoot.Storage == eks.INSTANCE_STORE_STORAGE && + if nodePool.Volumes.KubeletRoot != nil && nodePool.Volumes.KubeletRoot.Storage == eks.EBS_STORAGE && nodePool.Volumes.KubeletRoot.Size == 0 { nodePool.Volumes.KubeletRoot.Size = 50 } diff --git a/internal/cluster/distribution/eks/ekscluster/eks.go b/internal/cluster/distribution/eks/ekscluster/eks.go index 1d7e11aa64..3a2aa1047f 100644 --- a/internal/cluster/distribution/eks/ekscluster/eks.go +++ b/internal/cluster/distribution/eks/ekscluster/eks.go @@ -413,7 +413,7 @@ func (eks *CreateClusterEKS) AddDefaults(location string) error { } // default kubelet root EBS size to 50GB - if np.Volumes.KubeletRoot != nil && np.Volumes.KubeletRoot.Storage == INSTANCE_STORE_STORAGE && + if np.Volumes.KubeletRoot != nil && np.Volumes.KubeletRoot.Storage == EBS_STORAGE && np.Volumes.KubeletRoot.Size == 0 { np.Volumes.KubeletRoot.Size = 50 } diff --git a/internal/cluster/distribution/eks/eksprovider/driver/cluster_creator.go b/internal/cluster/distribution/eks/eksprovider/driver/cluster_creator.go index 2822aef159..42b59f78b4 100644 --- a/internal/cluster/distribution/eks/eksprovider/driver/cluster_creator.go +++ b/internal/cluster/distribution/eks/eksprovider/driver/cluster_creator.go @@ -307,8 +307,9 @@ func getVolumeParams(volume *pkgEks.NodePoolVolume) *eks.NodePoolVolume { } newVolume := &eks.NodePoolVolume{ - Size: volume.Size, - Type: volume.Type, + Storage: eks.EBS_STORAGE, + Size: volume.Size, + Type: volume.Type, } if volume.Encryption != nil { newVolume.Encryption = &eks.NodePoolVolumeEncryption{ diff --git a/internal/cluster/distribution/eks/eksprovider/driver/cluster_updater.go b/internal/cluster/distribution/eks/eksprovider/driver/cluster_updater.go index 34350665a8..d4308eb24b 100644 --- a/internal/cluster/distribution/eks/eksprovider/driver/cluster_updater.go +++ b/internal/cluster/distribution/eks/eksprovider/driver/cluster_updater.go @@ -222,32 +222,32 @@ func newASGsFromRequestedUpdatedNodePools( } for nodePoolName, nodePool := range requestedUpdatedNodePools { - var volumeEncryption *eks.NodePoolVolumeEncryption - if nodePool.VolumeEncryption != nil { - volumeEncryption = &eks.NodePoolVolumeEncryption{ - Enabled: nodePool.VolumeEncryption.Enabled, - EncryptionKeyARN: nodePool.VolumeEncryption.EncryptionKeyARN, + var nodePoolVolumes *eks.NodePoolVolumes + if nodePool.Volumes != nil { + nodePoolVolumes = &eks.NodePoolVolumes{} + if nodePool.Volumes.KubeletRoot != nil { + nodePoolVolumes.KubeletRoot = getVolumeParams(nodePool.Volumes.KubeletRoot) + } + if nodePool.Volumes.InstanceRoot != nil { + nodePoolVolumes.InstanceRoot = getVolumeParams(nodePool.Volumes.InstanceRoot) } } updatedNodePools = append(updatedNodePools, workflow.AutoscaleGroup{ - Name: nodePoolName, - NodeSpotPrice: nodePool.SpotPrice, - Autoscaling: nodePool.Autoscaling, - NodeMinCount: nodePool.MinCount, - NodeMaxCount: nodePool.MaxCount, - Count: nodePool.Count, - NodeVolumeEncryption: volumeEncryption, - NodeVolumeSize: nodePool.VolumeSize, - NodeVolumeType: nodePool.VolumeType, - NodeImage: nodePool.Image, - NodeInstanceType: nodePool.InstanceType, - SecurityGroups: nodePool.SecurityGroups, - UseInstanceStore: nodePool.UseInstanceStore, - Labels: nodePool.Labels, - Delete: false, - Create: false, - CreatedBy: creators[nodePoolName], + Name: nodePoolName, + NodeSpotPrice: nodePool.SpotPrice, + Autoscaling: nodePool.Autoscaling, + NodeMinCount: nodePool.MinCount, + NodeMaxCount: nodePool.MaxCount, + Count: nodePool.Count, + NodeImage: nodePool.Image, + NodeInstanceType: nodePool.InstanceType, + Volumes: nodePoolVolumes, + SecurityGroups: nodePool.SecurityGroups, + Labels: nodePool.Labels, + Delete: false, + Create: false, + CreatedBy: creators[nodePoolName], }) } @@ -410,6 +410,42 @@ func newNodePoolsFromUpdateRequest( nodePool.SpotPrice = eks.DefaultSpotPrice } + // convert deprecated params in case no Volumes struct is specified + if nodePool.Volumes == nil { + volumes := pkgEks.NodePoolVolumes{} + instanceRootVolume := pkgEks.NodePoolVolume{} + isInstanceRoot := false + + if nodePool.VolumeSize > 0 { + isInstanceRoot = true + instanceRootVolume.Size = nodePool.VolumeSize + } + if nodePool.VolumeType != "" { + isInstanceRoot = true + instanceRootVolume.Type = nodePool.VolumeType + } + if nodePool.VolumeEncryption != nil { + isInstanceRoot = true + instanceRootVolume.Encryption = nodePool.VolumeEncryption + } + + if isInstanceRoot { + volumes.InstanceRoot = &instanceRootVolume + } + + isKubeletRoot := false + if nodePool.UseInstanceStore != nil && *nodePool.UseInstanceStore { + isKubeletRoot = true + volumes.KubeletRoot = &pkgEks.NodePoolVolume{ + Storage: pkgEks.INSTANCE_STORE_STORAGE, + } + } + + if isInstanceRoot || isKubeletRoot { + nodePool.Volumes = &volumes + } + } + requestedNewNodePools[nodePoolName] = nodePool } } diff --git a/internal/cluster/distribution/eks/eksprovider/workflow/amazon.go b/internal/cluster/distribution/eks/eksprovider/workflow/amazon.go index b5b31f7b1d..44984215f5 100644 --- a/internal/cluster/distribution/eks/eksprovider/workflow/amazon.go +++ b/internal/cluster/distribution/eks/eksprovider/workflow/amazon.go @@ -220,22 +220,30 @@ func NewSubnetsFromEKSSubnets( // TODO: remove when UpdateNodePoolWorkflow is refactored and this is not needed // anymore. type AutoscaleGroup struct { - Name string - NodeSpotPrice string - Autoscaling bool - NodeMinCount int - NodeMaxCount int - Count int + Name string + NodeSpotPrice string + Autoscaling bool + NodeMinCount int + NodeMaxCount int + Count int + + Volumes *eks.NodePoolVolumes `json:"volumes,omitempty"` + + // deprecated, property replaced with Volumes.InstanceRoot.Encryption NodeVolumeEncryption *eks.NodePoolVolumeEncryption - NodeVolumeSize int - NodeVolumeType string - NodeImage string - NodeInstanceType string + // deprecated, property replaced with Volumes.InstanceRoot.Size + NodeVolumeSize int + // deprecated, property replaced with Volumes.InstanceRoot.Type + NodeVolumeType string + // deprecated, property replaced with Volumes.KubeletRoot.Type="instance-storage" + UseInstanceStore *bool `json:"useInstanceStore,omitempty" yaml:"useInstanceStore,omitempty"` + + NodeImage string + NodeInstanceType string // SecurityGroups collects the user specified custom node security group // IDs. - SecurityGroups []string - UseInstanceStore *bool + SecurityGroups []string Labels map[string]string Delete bool diff --git a/internal/cluster/distribution/eks/eksprovider/workflow/create_asg_activity.go b/internal/cluster/distribution/eks/eksprovider/workflow/create_asg_activity.go index 88c78859aa..12c014bd29 100644 --- a/internal/cluster/distribution/eks/eksprovider/workflow/create_asg_activity.go +++ b/internal/cluster/distribution/eks/eksprovider/workflow/create_asg_activity.go @@ -139,8 +139,9 @@ func (a *CreateAsgActivity) Execute(ctx context.Context, input CreateAsgActivity } nodeVolumeStorageStorage, nodeVolumeSize, nodeVolumeEncryptionEnabled, nodeVolumeEncryptionKeyARN, nodeVolumeType := - a.getDefaultedVolumeParams(input.NodeVolumes.InstanceRoot) - kubeletRootVolumeStorage, kubeletRootVolumeSize, kubeletRootVolumeEncryptionEnabled, kubeletRootVolumeEncryptionKeyARN, kubeletRootVolumeType := a.getDefaultedVolumeParams(input.NodeVolumes.KubeletRoot) + getDefaultedTemplateVolumeParams(input.NodeVolumes.InstanceRoot, a.defaultNodeVolumeEncryption) + kubeletRootVolumeStorage, kubeletRootVolumeSize, kubeletRootVolumeEncryptionEnabled, kubeletRootVolumeEncryptionKeyARN, kubeletRootVolumeType := + getDefaultedTemplateVolumeParams(input.NodeVolumes.KubeletRoot, a.defaultNodeVolumeEncryption) var stackTagsBuilder strings.Builder for tagIndex, tag := range tags { @@ -326,38 +327,43 @@ func (a *CreateAsgActivity) Execute(ctx context.Context, input CreateAsgActivity return &outParams, nil } -func (a *CreateAsgActivity) getDefaultedVolumeParams(volume *eks.NodePoolVolume) (string, int, string, string, string) { - storageType := "none" +func getDefaultedTemplateVolumeParams(volume *eks.NodePoolVolume, defaultNodeVolumeEncryption *eks.NodePoolVolumeEncryption) (string, int, string, string, string) { + storageType := "" size := 0 nodeVolumeEncryptionEnabled := "" // Note: defaulting to AWS account default encryption settings. nodeVolumeEncryptionKeyARN := "" - nodeVolumeType := "gp3" + nodeVolumeType := "" if volume == nil { return storageType, size, nodeVolumeEncryptionEnabled, nodeVolumeEncryptionKeyARN, nodeVolumeType } storageType = volume.Storage - size = volume.Size - if volume.Encryption != nil { - nodeVolumeEncryptionEnabled = strconv.FormatBool(volume.Encryption.Enabled) - } else if a.defaultNodeVolumeEncryption != nil { - nodeVolumeEncryptionEnabled = strconv.FormatBool(a.defaultNodeVolumeEncryption.Enabled) - } + if eks.EBS_STORAGE == storageType { + size = volume.Size - if nodeVolumeEncryptionEnabled == "true" && - volume.Encryption != nil && - volume.Encryption.EncryptionKeyARN != "" { - nodeVolumeEncryptionKeyARN = volume.Encryption.EncryptionKeyARN - } else if nodeVolumeEncryptionEnabled == "true" && - a.defaultNodeVolumeEncryption != nil && - a.defaultNodeVolumeEncryption.EncryptionKeyARN != "" { - nodeVolumeEncryptionKeyARN = a.defaultNodeVolumeEncryption.EncryptionKeyARN - } + if volume.Encryption != nil { + nodeVolumeEncryptionEnabled = strconv.FormatBool(volume.Encryption.Enabled) + } else if defaultNodeVolumeEncryption != nil { + nodeVolumeEncryptionEnabled = strconv.FormatBool(defaultNodeVolumeEncryption.Enabled) + } - if volume.Type != "" { - nodeVolumeType = volume.Type + if nodeVolumeEncryptionEnabled == "true" && + volume.Encryption != nil && + volume.Encryption.EncryptionKeyARN != "" { + nodeVolumeEncryptionKeyARN = volume.Encryption.EncryptionKeyARN + } else if nodeVolumeEncryptionEnabled == "true" && + defaultNodeVolumeEncryption != nil && + defaultNodeVolumeEncryption.EncryptionKeyARN != "" { + nodeVolumeEncryptionKeyARN = defaultNodeVolumeEncryption.EncryptionKeyARN + } + + if volume.Type != "" { + nodeVolumeType = volume.Type + } else { + nodeVolumeType = "gp3" + } } return storageType, size, nodeVolumeEncryptionEnabled, nodeVolumeEncryptionKeyARN, nodeVolumeType } diff --git a/internal/cluster/distribution/eks/eksprovider/workflow/update_asg_activity.go b/internal/cluster/distribution/eks/eksprovider/workflow/update_asg_activity.go index 47ba902112..91af1a4b99 100644 --- a/internal/cluster/distribution/eks/eksprovider/workflow/update_asg_activity.go +++ b/internal/cluster/distribution/eks/eksprovider/workflow/update_asg_activity.go @@ -17,7 +17,6 @@ package workflow import ( "context" "fmt" - "strconv" "strings" "time" @@ -58,24 +57,21 @@ type UpdateAsgActivityInput struct { EKSActivityInput // name of the cloud formation template stack - StackName string - Name string - Version string - NodeSpotPrice string - Autoscaling bool - NodeMinCount int - NodeMaxCount int - Count int - NodeVolumeEncryption *eks.NodePoolVolumeEncryption - NodeVolumeSize int - NodeVolumeType string - NodeImage string - NodeInstanceType string + StackName string + Name string + Version string + NodeSpotPrice string + Autoscaling bool + NodeMinCount int + NodeMaxCount int + Count int + NodeVolumes *eks.NodePoolVolumes + NodeImage string + NodeInstanceType string // SecurityGroups collects the user specified custom node security group // IDs. - SecurityGroups []string - UseInstanceStore *bool + SecurityGroups []string Labels map[string]string Tags map[string]string @@ -184,28 +180,10 @@ func (a *UpdateAsgActivity) Execute(ctx context.Context, input UpdateAsgActivity fmt.Sprintf("%v=%v", cluster.NodePoolNameLabelKey, input.Name), } - nodeVolumeEncryptionEnabled := "" - if input.NodeVolumeEncryption != nil { - nodeVolumeEncryptionEnabled = strconv.FormatBool(input.NodeVolumeEncryption.Enabled) - } else if a.defaultNodeVolumeEncryption != nil { - nodeVolumeEncryptionEnabled = strconv.FormatBool(a.defaultNodeVolumeEncryption.Enabled) - } - - nodeVolumeEncryptionKeyARN := "" - if nodeVolumeEncryptionEnabled == "true" && - input.NodeVolumeEncryption != nil && - input.NodeVolumeEncryption.EncryptionKeyARN != "" { - nodeVolumeEncryptionKeyARN = input.NodeVolumeEncryption.EncryptionKeyARN - } else if nodeVolumeEncryptionEnabled == "true" && - a.defaultNodeVolumeEncryption != nil && - a.defaultNodeVolumeEncryption.EncryptionKeyARN != "" { - nodeVolumeEncryptionKeyARN = a.defaultNodeVolumeEncryption.EncryptionKeyARN - } - - nodeVolumeType := "gp3" - if input.NodeVolumeType != "" { - nodeVolumeType = input.NodeVolumeType - } + nodeVolumeStorage, nodeVolumeSize, nodeVolumeEncryptionEnabled, nodeVolumeEncryptionKeyARN, nodeVolumeType := + getDefaultedTemplateVolumeParams(input.NodeVolumes.InstanceRoot, a.defaultNodeVolumeEncryption) + kubeletRootVolumeStorage, kubeletRootVolumeSize, kubeletRootVolumeEncryptionEnabled, kubeletRootVolumeEncryptionKeyARN, kubeletRootVolumeType := + getDefaultedTemplateVolumeParams(input.NodeVolumes.KubeletRoot, a.defaultNodeVolumeEncryption) var stackTagsBuilder strings.Builder for tagIndex, tag := range tags { @@ -249,23 +227,59 @@ func (a *UpdateAsgActivity) Execute(ctx context.Context, input UpdateAsgActivity ParameterKey: aws.String("NodeAutoScalingInitSize"), ParameterValue: aws.String(fmt.Sprintf("%d", input.Count)), }, - { - ParameterKey: aws.String("NodeVolumeEncryptionEnabled"), - ParameterValue: aws.String(nodeVolumeEncryptionEnabled), - }, - { - ParameterKey: aws.String("NodeVolumeEncryptionKeyARN"), - ParameterValue: aws.String(nodeVolumeEncryptionKeyARN), - }, + + sdkCloudformation.NewOptionalStackParameter( + "NodeVolumeStorage", + nodeVolumeStorage != "" || input.CurrentTemplateVersion.IsLessThan("2.5.0"), + nodeVolumeStorage, + ), + sdkCloudformation.NewOptionalStackParameter( + "NodeVolumeEncryptionEnabled", + nodeVolumeEncryptionEnabled != "" || input.CurrentTemplateVersion.IsLessThan("2.1.0"), // Note: older templates cannot use non-existing previous value. + nodeVolumeEncryptionEnabled, + ), + sdkCloudformation.NewOptionalStackParameter( + "NodeVolumeEncryptionKeyARN", + nodeVolumeEncryptionEnabled != "" || input.CurrentTemplateVersion.IsLessThan("2.1.0"), // Note: when enablement is set, key ARN should be updated. // Note: older templates cannot use non-existing previous value. + nodeVolumeEncryptionKeyARN, + ), sdkCloudformation.NewOptionalStackParameter( "NodeVolumeSize", - input.NodeVolumeSize > 0, - fmt.Sprintf("%d", input.NodeVolumeSize), + nodeVolumeSize > 0, + fmt.Sprintf("%d", nodeVolumeSize), ), - { - ParameterKey: aws.String("NodeVolumeType"), - ParameterValue: aws.String(nodeVolumeType), - }, + sdkCloudformation.NewOptionalStackParameter( + "NodeVolumeType", + input.NodeVolumes.InstanceRoot.Type != "" || input.CurrentTemplateVersion.IsLessThan("2.4.0"), // Note: older templates cannot use non-existing previous value. + nodeVolumeType, + ), + + sdkCloudformation.NewOptionalStackParameter( + "KubeletRootVolumeStorage", + kubeletRootVolumeStorage != "" || input.CurrentTemplateVersion.IsLessThan("2.5.0"), + kubeletRootVolumeStorage, + ), + sdkCloudformation.NewOptionalStackParameter( + "KubeletRootVolumeEncryptionEnabled", + kubeletRootVolumeEncryptionEnabled != "" || input.CurrentTemplateVersion.IsLessThan("2.5.0"), // Note: older templates cannot use non-existing previous value. + kubeletRootVolumeEncryptionEnabled, + ), + sdkCloudformation.NewOptionalStackParameter( + "KubeletRootVolumeEncryptionKeyARN", + kubeletRootVolumeEncryptionEnabled != "" || input.CurrentTemplateVersion.IsLessThan("2.5.0"), // Note: when enablement is set, key ARN should be updated. // Note: older templates cannot use non-existing previous value. + kubeletRootVolumeEncryptionKeyARN, + ), + sdkCloudformation.NewOptionalStackParameter( + "KubeletRootVolumeSize", + kubeletRootVolumeSize > 0 || input.CurrentTemplateVersion.IsLessThan("2.5.0"), + fmt.Sprintf("%d", kubeletRootVolumeSize), + ), + sdkCloudformation.NewOptionalStackParameter( + "KubeletRootVolumeType", + input.NodeVolumes.KubeletRoot.Type != "" || input.CurrentTemplateVersion.IsLessThan("2.5.0"), // Note: older templates cannot use non-existing previous value. + kubeletRootVolumeType, + ), + { ParameterKey: aws.String("StackTags"), ParameterValue: aws.String(stackTagsBuilder.String()), @@ -314,11 +328,6 @@ func (a *UpdateAsgActivity) Execute(ctx context.Context, input UpdateAsgActivity ParameterKey: aws.String("KubeletExtraArguments"), ParameterValue: aws.String(fmt.Sprintf("--node-labels %v", strings.Join(nodeLabels, ","))), }, - sdkCloudformation.NewOptionalStackParameter( - "UseInstanceStore", - input.UseInstanceStore != nil || input.CurrentTemplateVersion.IsLessThan("2.2.0"), - strconv.FormatBool(aws.BoolValue(input.UseInstanceStore)), // Note: false default value for old stacks. - ), } requestToken := aws.String(sdkAmazon.NewNormalizedClientRequestToken(activity.GetInfo(ctx).WorkflowExecution.ID)) diff --git a/internal/cluster/distribution/eks/eksworkflow/activity_update_node_group.go b/internal/cluster/distribution/eks/eksworkflow/activity_update_node_group.go index 00f14b8836..abe32d7363 100644 --- a/internal/cluster/distribution/eks/eksworkflow/activity_update_node_group.go +++ b/internal/cluster/distribution/eks/eksworkflow/activity_update_node_group.go @@ -62,13 +62,10 @@ type UpdateNodeGroupActivityInput struct { NodePoolName string NodePoolVersion string - NodeVolumeEncryption *eks.NodePoolVolumeEncryption - NodeVolumeSize int - NodeVolumeType string - NodeImage string - DesiredCapacity int64 - SecurityGroups []string - UseInstanceStore *bool + NodeVolumes *eks.NodePoolVolumes + NodeImage string + DesiredCapacity int64 + SecurityGroups []string MaxBatchSize int MinInstancesInService int @@ -118,30 +115,70 @@ func (a UpdateNodeGroupActivity) Execute(ctx context.Context, input UpdateNodeGr } nodeVolumeEncryptionEnabled := "" - if input.NodeVolumeEncryption != nil { - nodeVolumeEncryptionEnabled = strconv.FormatBool(input.NodeVolumeEncryption.Enabled) - } else if input.CurrentTemplateVersion.IsLessThan("2.1.0") && - a.defaultNodeVolumeEncryption != nil { // Note: old stack, Pipeline default should take precedence over AWS default. - nodeVolumeEncryptionEnabled = strconv.FormatBool(a.defaultNodeVolumeEncryption.Enabled) - } - nodeVolumeEncryptionKeyARN := "" - if nodeVolumeEncryptionEnabled == "true" && - input.NodeVolumeEncryption != nil && - input.NodeVolumeEncryption.EncryptionKeyARN != "" { - nodeVolumeEncryptionKeyARN = input.NodeVolumeEncryption.EncryptionKeyARN - } else if nodeVolumeEncryptionEnabled == "true" && - a.defaultNodeVolumeEncryption != nil && - a.defaultNodeVolumeEncryption.EncryptionKeyARN != "" { - nodeVolumeEncryptionKeyARN = a.defaultNodeVolumeEncryption.EncryptionKeyARN - } else if input.CurrentTemplateVersion.IsLessThan("2.1.0") && - a.defaultNodeVolumeEncryption != nil { // Note: old stack, Pipeline default should take precedence over AWS default. - nodeVolumeEncryptionKeyARN = a.defaultNodeVolumeEncryption.EncryptionKeyARN + nodeVolumeType := "" + nodeVolumeSize := 0 + + if eks.EBS_STORAGE == input.NodeVolumes.InstanceRoot.Storage { + if input.NodeVolumes.InstanceRoot.Encryption != nil { + nodeVolumeEncryptionEnabled = strconv.FormatBool(input.NodeVolumes.InstanceRoot.Encryption.Enabled) + } else if input.CurrentTemplateVersion.IsLessThan("2.1.0") && + a.defaultNodeVolumeEncryption != nil { // Note: old stack, Pipeline default should take precedence over AWS default. + nodeVolumeEncryptionEnabled = strconv.FormatBool(a.defaultNodeVolumeEncryption.Enabled) + } + + if nodeVolumeEncryptionEnabled == "true" && + input.NodeVolumes.InstanceRoot.Encryption != nil && + input.NodeVolumes.InstanceRoot.Encryption.EncryptionKeyARN != "" { + nodeVolumeEncryptionKeyARN = input.NodeVolumes.InstanceRoot.Encryption.EncryptionKeyARN + } else if nodeVolumeEncryptionEnabled == "true" && + a.defaultNodeVolumeEncryption != nil && + a.defaultNodeVolumeEncryption.EncryptionKeyARN != "" { + nodeVolumeEncryptionKeyARN = a.defaultNodeVolumeEncryption.EncryptionKeyARN + } else if input.CurrentTemplateVersion.IsLessThan("2.1.0") && + a.defaultNodeVolumeEncryption != nil { // Note: old stack, Pipeline default should take precedence over AWS default. + nodeVolumeEncryptionKeyARN = a.defaultNodeVolumeEncryption.EncryptionKeyARN + } + + nodeVolumeType = "gp3" + if input.NodeVolumes.InstanceRoot.Type != "" { + nodeVolumeType = input.NodeVolumes.InstanceRoot.Type + } + + nodeVolumeSize = input.NodeVolumes.InstanceRoot.Size } - nodeVolumeType := "gp3" - if input.NodeVolumeType != "" { - nodeVolumeType = input.NodeVolumeType + kubeletRootVolumeEncryptionEnabled := "" + kubeletRootVolumeEncryptionKeyARN := "" + kubeletRootVolumeType := "" + kubeletRootVolumeSize := 0 + if eks.EBS_STORAGE == input.NodeVolumes.KubeletRoot.Storage { + if input.NodeVolumes.KubeletRoot.Encryption != nil { + kubeletRootVolumeEncryptionEnabled = strconv.FormatBool(input.NodeVolumes.KubeletRoot.Encryption.Enabled) + } else if input.CurrentTemplateVersion.IsLessThan("2.5.0") && + a.defaultNodeVolumeEncryption != nil { // Note: old stack, Pipeline default should take precedence over AWS default. + kubeletRootVolumeEncryptionEnabled = strconv.FormatBool(a.defaultNodeVolumeEncryption.Enabled) + } + + if kubeletRootVolumeEncryptionEnabled == "true" && + input.NodeVolumes.KubeletRoot.Encryption != nil && + input.NodeVolumes.KubeletRoot.Encryption.EncryptionKeyARN != "" { + kubeletRootVolumeEncryptionKeyARN = input.NodeVolumes.KubeletRoot.Encryption.EncryptionKeyARN + } else if kubeletRootVolumeEncryptionEnabled == "true" && + a.defaultNodeVolumeEncryption != nil && + a.defaultNodeVolumeEncryption.EncryptionKeyARN != "" { + kubeletRootVolumeEncryptionKeyARN = a.defaultNodeVolumeEncryption.EncryptionKeyARN + } else if input.CurrentTemplateVersion.IsLessThan("2.5.0") && + a.defaultNodeVolumeEncryption != nil { // Note: old stack, Pipeline default should take precedence over AWS default. + kubeletRootVolumeEncryptionKeyARN = a.defaultNodeVolumeEncryption.EncryptionKeyARN + } + + kubeletRootVolumeType = "gp3" + if input.NodeVolumes.KubeletRoot.Type != "" { + kubeletRootVolumeType = input.NodeVolumes.KubeletRoot.Type + } + + kubeletRootVolumeSize = input.NodeVolumes.KubeletRoot.Size } tags := getNodePoolStackTags(input.ClusterName, input.ClusterTags) @@ -200,6 +237,12 @@ func (a UpdateNodeGroupActivity) Execute(ctx context.Context, input UpdateNodeGr input.DesiredCapacity > 0, fmt.Sprintf("%d", input.DesiredCapacity), ), + // InstanceRoot / Node volume params + sdkCloudFormation.NewOptionalStackParameter( + "NodeVolumeStorage", + input.NodeVolumes.InstanceRoot.Storage != "" || input.CurrentTemplateVersion.IsLessThan("2.5.0"), + input.NodeVolumes.InstanceRoot.Storage, + ), sdkCloudFormation.NewOptionalStackParameter( "NodeVolumeEncryptionEnabled", nodeVolumeEncryptionEnabled != "" || input.CurrentTemplateVersion.IsLessThan("2.1.0"), // Note: older templates cannot use non-existing previous value. @@ -212,14 +255,41 @@ func (a UpdateNodeGroupActivity) Execute(ctx context.Context, input UpdateNodeGr ), sdkCloudFormation.NewOptionalStackParameter( "NodeVolumeSize", - input.NodeVolumeSize > 0, - fmt.Sprintf("%d", input.NodeVolumeSize), + nodeVolumeSize > 0, + fmt.Sprintf("%d", nodeVolumeSize), ), sdkCloudFormation.NewOptionalStackParameter( "NodeVolumeType", - input.NodeVolumeType != "" || input.CurrentTemplateVersion.IsLessThan("2.4.0"), // Note: older templates cannot use non-existing previous value. + input.NodeVolumes.InstanceRoot.Type != "" || input.CurrentTemplateVersion.IsLessThan("2.4.0"), // Note: older templates cannot use non-existing previous value. nodeVolumeType, ), + // KubeletRoot volume params + sdkCloudFormation.NewOptionalStackParameter( + "KubeletRootVolumeStorage", + input.NodeVolumes.KubeletRoot.Storage != "" || input.CurrentTemplateVersion.IsLessThan("2.5.0"), + input.NodeVolumes.KubeletRoot.Storage, + ), + sdkCloudFormation.NewOptionalStackParameter( + "KubeletRootVolumeEncryptionEnabled", + kubeletRootVolumeEncryptionEnabled != "" || input.CurrentTemplateVersion.IsLessThan("2.5.0"), // Note: older templates cannot use non-existing previous value. + kubeletRootVolumeEncryptionEnabled, + ), + sdkCloudFormation.NewOptionalStackParameter( + "KubeletRootVolumeEncryptionKeyARN", + kubeletRootVolumeEncryptionEnabled != "" || input.CurrentTemplateVersion.IsLessThan("2.5.0"), // Note: when enablement is set, key ARN should be updated. // Note: older templates cannot use non-existing previous value. + kubeletRootVolumeEncryptionKeyARN, + ), + sdkCloudFormation.NewOptionalStackParameter( + "KubeletRootVolumeSize", + kubeletRootVolumeSize > 0 || input.CurrentTemplateVersion.IsLessThan("2.5.0"), + fmt.Sprintf("%d", kubeletRootVolumeSize), + ), + sdkCloudFormation.NewOptionalStackParameter( + "KubeletRootVolumeType", + input.NodeVolumes.KubeletRoot.Type != "" || input.CurrentTemplateVersion.IsLessThan("2.5.0"), // Note: older templates cannot use non-existing previous value. + kubeletRootVolumeType, + ), + { ParameterKey: aws.String("StackTags"), ParameterValue: aws.String(stackTagsBuilder.String()), @@ -264,11 +334,6 @@ func (a UpdateNodeGroupActivity) Execute(ctx context.Context, input UpdateNodeGr ParameterKey: aws.String("KubeletExtraArguments"), ParameterValue: aws.String(fmt.Sprintf("--node-labels %v", strings.Join(nodeLabels, ","))), }, - sdkCloudFormation.NewOptionalStackParameter( - "UseInstanceStore", - input.UseInstanceStore != nil || input.CurrentTemplateVersion.IsLessThan("2.2.0"), - strconv.FormatBool(aws.BoolValue(input.UseInstanceStore)), // Note: false default value for old stacks. - ), } // we don't reuse the creation time template, since it may have changed diff --git a/internal/cluster/distribution/eks/eksworkflow/workflow_update_node_pool.go b/internal/cluster/distribution/eks/eksworkflow/workflow_update_node_pool.go index 28cc1d0443..1d4f3a97e5 100644 --- a/internal/cluster/distribution/eks/eksworkflow/workflow_update_node_pool.go +++ b/internal/cluster/distribution/eks/eksworkflow/workflow_update_node_pool.go @@ -70,12 +70,9 @@ type UpdateNodePoolWorkflowInput struct { ClusterName string NodePoolName string - NodeVolumeEncryption *eks.NodePoolVolumeEncryption - NodeVolumeSize int - NodeVolumeType string - NodeImage string - SecurityGroups []string - UseInstanceStore *bool + NodeVolumes *eks.NodePoolVolumes + NodeImage string + SecurityGroups []string Options eks.NodePoolUpdateOptions @@ -136,10 +133,19 @@ func (w UpdateNodePoolWorkflow) Execute(ctx workflow.Context, input UpdateNodePo var currentTemplateVersion semver.Version effectiveImage := input.NodeImage - effectiveVolumeEncryption := input.NodeVolumeEncryption - effectiveVolumeSize := input.NodeVolumeSize + effectiveSecurityGroups := input.SecurityGroups - effectiveUseInstanceStore := input.UseInstanceStore + + effectiveVolumes := input.NodeVolumes + if effectiveVolumes == nil { + effectiveVolumes = &eks.NodePoolVolumes{} + } + if effectiveVolumes.InstanceRoot == nil { + effectiveVolumes.InstanceRoot = &eks.NodePoolVolume{} + } + if effectiveVolumes.KubeletRoot == nil { + effectiveVolumes.KubeletRoot = &eks.NodePoolVolume{} + } { // Note: needing CF stack for template version and possibly node pool version. getCFStackInput := eksWorkflow.GetCFStackActivityInput{ @@ -155,14 +161,17 @@ func (w UpdateNodePoolWorkflow) Execute(ctx workflow.Context, input UpdateNodePo } var parameters struct { - CustomNodeSecurityGroups string `mapstructure:"CustomNodeSecurityGroups,omitempty"` // Note: CustomNodeSecurityGroups is only available from template version 2.0.0. - NodeImageID string `mapstructure:"NodeImageId"` - NodeVolumeEncryptionEnabled string `mapstructure:"NodeVolumeEncryptionEnabled,omitempty"` // Note: NodeVolumeEncryptionEnabled is only available from template version 2.1.0. - NodeVolumeEncryptionKeyARN string `mapstructure:"NodeVolumeEncryptionKeyARN,omitempty"` // Note: NodeVolumeEncryptionKeyARN is only available from template version 2.1.0. - NodeVolumeSize int `mapstructure:"NodeVolumeSize"` - TemplateVersion semver.Version `mapstructure:"TemplateVersion,omitempty"` // Note: TemplateVersion is only available from template version 2.0.0. - UseInstanceStore string `mapstructure:"UseInstanceStore,omitempty"` // Note: TemplateVersion is only available from template version 2.2.0. - + CustomNodeSecurityGroups string `mapstructure:"CustomNodeSecurityGroups,omitempty"` // Note: CustomNodeSecurityGroups is only available from template version 2.0.0. + NodeImageID string `mapstructure:"NodeImageId"` + NodeVolumeStorage string `mapstructure:"NodeVolumeStorage,omitempty"` // Note: NodeVolumeStorage is only available from template version 2.5.0. + NodeVolumeEncryptionEnabled string `mapstructure:"NodeVolumeEncryptionEnabled,omitempty"` // Note: NodeVolumeEncryptionEnabled is only available from template version 2.1.0. + NodeVolumeEncryptionKeyARN string `mapstructure:"NodeVolumeEncryptionKeyARN,omitempty"` // Note: NodeVolumeEncryptionKeyARN is only available from template version 2.1.0. + NodeVolumeSize int `mapstructure:"NodeVolumeSize"` + KubeletRootVolumeStorage string `mapstructure:"KubeletRootVolumeStorage,omitempty"` // Note: KubeletRootVolumeStorage is only available from template version 2.5.0. + KubeletRootVolumeEncryptionEnabled string `mapstructure:"KubeletRootVolumeEncryptionEnabled,omitempty"` // Note: KubeletRootVolumeEncryptionEnabled is only available from template version 2.5.0. + KubeletRootVolumeEncryptionKeyARN string `mapstructure:"KubeletRootVolumeEncryptionKeyARN,omitempty"` // Note: KubeletRootVolumeEncryptionKeyARN is only available from template version 2.5.0. + KubeletRootVolumeSize int `mapstructure:"KubeletRootVolumeSize,omitempty"` // Note: KubeletRootVolumeSize is only available from template version 2.5.0. + TemplateVersion semver.Version `mapstructure:"TemplateVersion,omitempty"` // Note: TemplateVersion is only available from template version 2.0.0. } err = sdkCloudFormation.ParseStackParameters(getCFStackOutput.Stack.Parameters, ¶meters) if err != nil { @@ -175,41 +184,57 @@ func (w UpdateNodePoolWorkflow) Execute(ctx workflow.Context, input UpdateNodePo effectiveImage = parameters.NodeImageID } - if effectiveVolumeEncryption == nil && + if effectiveVolumes.InstanceRoot.Storage == "" { + effectiveVolumes.InstanceRoot.Storage = parameters.NodeVolumeStorage + } + + if effectiveVolumes.InstanceRoot.Encryption == nil && parameters.NodeVolumeEncryptionEnabled != "" { isNodeVolumeEncryptionEnabled, err := strconv.ParseBool(parameters.NodeVolumeEncryptionEnabled) if err != nil { return errors.WrapIf(err, "invalid node volume encryption enabled value") } - effectiveVolumeEncryption = &eks.NodePoolVolumeEncryption{ + effectiveVolumes.InstanceRoot.Encryption = &eks.NodePoolVolumeEncryption{ Enabled: isNodeVolumeEncryptionEnabled, EncryptionKeyARN: parameters.NodeVolumeEncryptionKeyARN, } } - if effectiveVolumeSize == 0 { - effectiveVolumeSize = parameters.NodeVolumeSize + if effectiveVolumes.InstanceRoot.Size == 0 { + effectiveVolumes.InstanceRoot.Size = parameters.NodeVolumeSize } - if effectiveSecurityGroups == nil && - parameters.CustomNodeSecurityGroups != "" { - effectiveSecurityGroups = strings.Split(parameters.CustomNodeSecurityGroups, ",") - sort.Strings(effectiveSecurityGroups) + if effectiveVolumes.KubeletRoot.Storage == "" { + effectiveVolumes.KubeletRoot.Storage = parameters.KubeletRootVolumeStorage } - if effectiveUseInstanceStore == nil && - parameters.UseInstanceStore != "" { - useInstanceStore, err := strconv.ParseBool(parameters.UseInstanceStore) + if effectiveVolumes.KubeletRoot.Encryption == nil && + parameters.KubeletRootVolumeEncryptionEnabled != "" { + isVolumeEncryptionEnabled, err := strconv.ParseBool(parameters.KubeletRootVolumeEncryptionEnabled) if err != nil { - return errors.WrapIf(err, "invalid UseInstanceStore parameter value") + return errors.WrapIf(err, "invalid kubelet root volume encryption enabled value") + } + + effectiveVolumes.KubeletRoot.Encryption = &eks.NodePoolVolumeEncryption{ + Enabled: isVolumeEncryptionEnabled, + EncryptionKeyARN: parameters.KubeletRootVolumeEncryptionKeyARN, } - effectiveUseInstanceStore = &useInstanceStore + } + + if effectiveVolumes.KubeletRoot.Size == 0 { + effectiveVolumes.KubeletRoot.Size = parameters.KubeletRootVolumeSize + } + + if effectiveSecurityGroups == nil && + parameters.CustomNodeSecurityGroups != "" { + effectiveSecurityGroups = strings.Split(parameters.CustomNodeSecurityGroups, ",") + sort.Strings(effectiveSecurityGroups) } } var volumeSize int - if input.NodeVolumeSize > 0 { + if input.NodeVolumes != nil && input.NodeVolumes.InstanceRoot != nil && input.NodeVolumes.InstanceRoot.Size > 0 { var amiSize int { activityInput := eksWorkflow.GetAMISizeActivityInput{ @@ -230,7 +255,7 @@ func (w UpdateNodePoolWorkflow) Execute(ctx workflow.Context, input UpdateNodePo { activityInput := eksWorkflow.SelectVolumeSizeActivityInput{ AMISize: amiSize, - OptionalVolumeSize: input.NodeVolumeSize, + OptionalVolumeSize: input.NodeVolumes.InstanceRoot.Size, } var activityOutput eksWorkflow.SelectVolumeSizeActivityOutput processActivity := process.StartActivity(ctx, eksWorkflow.SelectVolumeSizeActivityName) @@ -241,19 +266,16 @@ func (w UpdateNodePoolWorkflow) Execute(ctx workflow.Context, input UpdateNodePo } volumeSize = activityOutput.VolumeSize - effectiveVolumeSize = volumeSize + effectiveVolumes.InstanceRoot.Size = volumeSize } } - // todo x var nodePoolVersion string { activityInput := eksWorkflow.CalculateNodePoolVersionActivityInput{ - Image: effectiveImage, - //VolumeEncryption: effectiveVolumeEncryption, - //VolumeSize: effectiveVolumeSize, + Image: effectiveImage, + Volumes: *effectiveVolumes, CustomSecurityGroups: effectiveSecurityGroups, - // UseInstanceStore: effectiveUseInstanceStore, } var output eksWorkflow.CalculateNodePoolVersionActivityOutput @@ -277,12 +299,9 @@ func (w UpdateNodePoolWorkflow) Execute(ctx workflow.Context, input UpdateNodePo StackName: input.StackName, NodePoolName: input.NodePoolName, NodePoolVersion: nodePoolVersion, - NodeVolumeEncryption: input.NodeVolumeEncryption, - NodeVolumeSize: volumeSize, - NodeVolumeType: input.NodeVolumeType, + NodeVolumes: effectiveVolumes, NodeImage: input.NodeImage, SecurityGroups: input.SecurityGroups, - UseInstanceStore: input.UseInstanceStore, MaxBatchSize: input.Options.MaxBatchSize, MinInstancesInService: input.Options.MaxSurge, ClusterTags: input.ClusterTags, diff --git a/internal/cluster/distribution/eks/service.go b/internal/cluster/distribution/eks/service.go index 541a27c3fe..60c92cedea 100644 --- a/internal/cluster/distribution/eks/service.go +++ b/internal/cluster/distribution/eks/service.go @@ -66,13 +66,19 @@ type ClusterUpdate struct { // A node pool update contains a partial representation of the node pool resource, // updating only the changed values. type NodePoolUpdate struct { + Volumes *NodePoolVolumes `mapstructure:"volumes,omitempty"` + + // deprecated, property replaced with Volumes.InstanceRoot.Encryption VolumeEncryption *NodePoolVolumeEncryption `mapstructure:"volumeEncryption,omitempty"` - VolumeSize int `mapstructure:"volumeSize"` - VolumeType string `mapstructure:"volumeType"` + // deprecated, property replaced with Volumes.InstanceRoot.Size + VolumeSize int `mapstructure:"volumeSize"` + // deprecated, property replaced with Volumes.InstanceRoot.Type + VolumeType string `mapstructure:"volumeType"` + // deprecated, property replaced with Volumes.KubeletRoot.Type="instance-storage" + UseInstanceStore *bool `mapstructure:"useInstanceStore,omitempty"` - Image string `mapstructure:"image"` - SecurityGroups []string `mapstructure:"securityGroups"` - UseInstanceStore *bool `mapstructure:"useInstanceStore,omitempty"` + Image string `mapstructure:"image"` + SecurityGroups []string `mapstructure:"securityGroups"` Options NodePoolUpdateOptions `mapstructure:"options"` } @@ -102,41 +108,56 @@ type NodePoolUpdateDrainOptions struct { // NodePool encapsulates information about a cluster node pool. type NodePool struct { - Name string `mapstructure:"name"` - Labels map[string]string `mapstructure:"labels"` - Size int `mapstructure:"size"` - Autoscaling Autoscaling `mapstructure:"autoscaling"` + Name string `mapstructure:"name"` + Labels map[string]string `mapstructure:"labels"` + Size int `mapstructure:"size"` + Autoscaling Autoscaling `mapstructure:"autoscaling"` + + Volumes *NodePoolVolumes `mapstructure:"volumes"` + + InstanceType string `mapstructure:"instanceType"` + Image string `mapstructure:"image"` + SpotPrice string `mapstructure:"spotPrice"` + SecurityGroups []string `mapstructure:"securityGroups,omitempty"` + SubnetID string `mapstructure:"subnetId"` + + Status NodePoolStatus `mapstructure:"status"` + StatusMessage string `mapstructure:"statusMessage"` + + // deprecated VolumeEncryption *NodePoolVolumeEncryption `mapstructure:"volumeEncryption,omitempty"` - VolumeSize int `mapstructure:"volumeSize"` - VolumeType string `mapstructure:"volumeType"` - InstanceType string `mapstructure:"instanceType"` - Image string `mapstructure:"image"` - SpotPrice string `mapstructure:"spotPrice"` - SecurityGroups []string `mapstructure:"securityGroups,omitempty"` - SubnetID string `mapstructure:"subnetId"` - UseInstanceStore bool `mapstructure:"UseInstanceStore"` - Status NodePoolStatus `mapstructure:"status"` - StatusMessage string `mapstructure:"statusMessage"` + // deprecated + VolumeSize int `mapstructure:"volumeSize"` + // deprecated + VolumeType string `mapstructure:"volumeType"` + // deprecated + UseInstanceStore bool `mapstructure:"UseInstanceStore"` } // NewNodePoolFromCFStack initializes a node pool object from a CloudFormation // stack. func NewNodePoolFromCFStack(name string, labels map[string]string, stack *cloudformation.Stack) (nodePool NodePool) { var nodePoolParameters struct { - ClusterAutoscalerEnabled bool `mapstructure:"ClusterAutoscalerEnabled"` - NodeAutoScalingGroupMaxSize int `mapstructure:"NodeAutoScalingGroupMaxSize"` - NodeAutoScalingGroupMinSize int `mapstructure:"NodeAutoScalingGroupMinSize"` - NodeAutoScalingInitSize int `mapstructure:"NodeAutoScalingInitSize"` - NodeImageID string `mapstructure:"NodeImageId"` - NodeInstanceType string `mapstructure:"NodeInstanceType"` - NodeSpotPrice string `mapstructure:"NodeSpotPrice"` - NodeVolumeEncryptionEnabled string `mapstructure:"NodeVolumeEncryptionEnabled,omitempty"` // Note: NodeVolumeEncryptionEnabled is only available from template version 2.1.0. - NodeVolumeEncryptionKeyARN string `mapstructure:"NodeVolumeEncryptionKeyARN,omitempty"` // Note: NodeVolumeEncryptionKeyARN is only available from template version 2.1.0. - NodeVolumeSize int `mapstructure:"NodeVolumeSize"` - NodeVolumeType string `mapstructure:"NodeVolumeType,omitempty"` // Note: NodeVolumeType is only available from template version 2.4.0. - CustomNodeSecurityGroups string `mapstructure:"CustomNodeSecurityGroups,omitempty"` // Note: CustomNodeSecurityGroups is only available from template version 2.0.0. - Subnets string `mapstructure:"Subnets"` - UseInstanceStore string `mapstructure:"UseInstanceStore,omitempty"` // Note: UseInstanceStore is only available from template version 2.2.0. + ClusterAutoscalerEnabled bool `mapstructure:"ClusterAutoscalerEnabled"` + NodeAutoScalingGroupMaxSize int `mapstructure:"NodeAutoScalingGroupMaxSize"` + NodeAutoScalingGroupMinSize int `mapstructure:"NodeAutoScalingGroupMinSize"` + NodeAutoScalingInitSize int `mapstructure:"NodeAutoScalingInitSize"` + NodeImageID string `mapstructure:"NodeImageId"` + NodeInstanceType string `mapstructure:"NodeInstanceType"` + NodeSpotPrice string `mapstructure:"NodeSpotPrice"` + NodeVolumeStorage string `mapstructure:"NodeVolumeStorage,omitempty"` // Note: NodeVolumeStorage is only available from template version 2.5.0. + NodeVolumeEncryptionEnabled string `mapstructure:"NodeVolumeEncryptionEnabled,omitempty"` // Note: NodeVolumeEncryptionEnabled is only available from template version 2.1.0. + NodeVolumeEncryptionKeyARN string `mapstructure:"NodeVolumeEncryptionKeyARN,omitempty"` // Note: NodeVolumeEncryptionKeyARN is only available from template version 2.1.0. + NodeVolumeSize int `mapstructure:"NodeVolumeSize"` + NodeVolumeType string `mapstructure:"NodeVolumeType,omitempty"` // Note: NodeVolumeType is only available from template version 2.4.0. + KubeletRootVolumeStorage string `mapstructure:"KubeletRootVolumeStorage,omitempty"` // Note: KubeletRootVolumeStorage is only available from template version 2.5.0. + KubeletRootVolumeEncryptionEnabled string `mapstructure:"KubeletRootVolumeEncryptionEnabled,omitempty"` // Note: KubeletRootVolumeEncryptionEnabled is only available from template version 2.5.0. + KubeletRootVolumeEncryptionKeyARN string `mapstructure:"KubeletRootVolumeEncryptionKeyARN,omitempty"` // Note: KubeletRootVolumeEncryptionKeyARN is only available from template version 2.5.0. + KubeletRootVolumeSize int `mapstructure:"KubeletRootVolumeSize,omitempty"` // Note: KubeletRootVolumeSize is only available from template version 2.5.0. + KubeletRootVolumeType string `mapstructure:"KubeletRootVolumeType,omitempty"` // Note: KubeletRootVolumeType is only available from template version 2.5.0. + CustomNodeSecurityGroups string `mapstructure:"CustomNodeSecurityGroups,omitempty"` // Note: CustomNodeSecurityGroups is only available from template version 2.0.0. + Subnets string `mapstructure:"Subnets"` + UseInstanceStore string `mapstructure:"UseInstanceStore,omitempty"` // Note: UseInstanceStore is only available from template version 2.2.0. } err := sdkCloudFormation.ParseStackParameters(stack.Parameters, &nodePoolParameters) @@ -153,6 +174,53 @@ func NewNodePoolFromCFStack(name string, labels map[string]string, stack *cloudf MaxSize: nodePoolParameters.NodeAutoScalingGroupMaxSize, } + if "" != nodePoolParameters.NodeVolumeStorage { + nodePool.Volumes = &NodePoolVolumes{ + InstanceRoot: &NodePoolVolume{ + Storage: nodePoolParameters.NodeVolumeStorage, + }, + } + } + + if EBS_STORAGE == nodePoolParameters.NodeVolumeStorage { + nodePool.Volumes.InstanceRoot.Size = nodePoolParameters.NodeVolumeSize + nodePool.Volumes.InstanceRoot.Type = nodePoolParameters.NodeVolumeType + + if nodePoolParameters.NodeVolumeEncryptionEnabled != "" { + nodePool.Volumes.InstanceRoot.Encryption = &NodePoolVolumeEncryption{} + nodePool.Volumes.InstanceRoot.Encryption.Enabled, err = strconv.ParseBool(nodePoolParameters.NodeVolumeEncryptionEnabled) + if err != nil { + return NewNodePoolWithNoValues(name, NodePoolStatusError, "invalid encryption information") + } + + nodePool.Volumes.InstanceRoot.Encryption.EncryptionKeyARN = nodePoolParameters.NodeVolumeEncryptionKeyARN + } + } + + switch nodePoolParameters.KubeletRootVolumeStorage { + case EBS_STORAGE: + nodePool.Volumes.KubeletRoot = &NodePoolVolume{ + Storage: nodePoolParameters.KubeletRootVolumeStorage, + } + nodePool.Volumes.KubeletRoot.Size = nodePoolParameters.KubeletRootVolumeSize + nodePool.Volumes.KubeletRoot.Type = nodePoolParameters.KubeletRootVolumeType + + if nodePoolParameters.KubeletRootVolumeEncryptionEnabled != "" { + nodePool.Volumes.KubeletRoot.Encryption = &NodePoolVolumeEncryption{} + nodePool.Volumes.KubeletRoot.Encryption.Enabled, err = strconv.ParseBool(nodePoolParameters.KubeletRootVolumeEncryptionEnabled) + if err != nil { + return NewNodePoolWithNoValues(name, NodePoolStatusError, "invalid encryption information") + } + + nodePool.Volumes.KubeletRoot.Encryption.EncryptionKeyARN = nodePoolParameters.KubeletRootVolumeEncryptionKeyARN + } + case INSTANCE_STORE_STORAGE: + nodePool.Volumes.KubeletRoot = &NodePoolVolume{ + Storage: nodePoolParameters.KubeletRootVolumeStorage, + } + } + + // deprecated if nodePoolParameters.NodeVolumeEncryptionEnabled != "" { nodePool.VolumeEncryption = &NodePoolVolumeEncryption{} nodePool.VolumeEncryption.Enabled, err = strconv.ParseBool(nodePoolParameters.NodeVolumeEncryptionEnabled) @@ -162,9 +230,9 @@ func NewNodePoolFromCFStack(name string, labels map[string]string, stack *cloudf nodePool.VolumeEncryption.EncryptionKeyARN = nodePoolParameters.NodeVolumeEncryptionKeyARN } - nodePool.VolumeSize = nodePoolParameters.NodeVolumeSize nodePool.VolumeType = nodePoolParameters.NodeVolumeType + nodePool.InstanceType = nodePoolParameters.NodeInstanceType nodePool.Image = nodePoolParameters.NodeImageID nodePool.SpotPrice = nodePoolParameters.NodeSpotPrice diff --git a/src/cluster/eks_update_cluster.go b/src/cluster/eks_update_cluster.go index 9a6501a18a..4d786bbfaf 100644 --- a/src/cluster/eks_update_cluster.go +++ b/src/cluster/eks_update_cluster.go @@ -205,8 +205,18 @@ func (w EKSUpdateClusterWorkflow) Execute(ctx workflow.Context, input EKSUpdateC var currentTemplateVersion semver.Version effectiveImage := updatedNodePool.NodeImage - effectiveVolumeEncryption := updatedNodePool.NodeVolumeEncryption - effectiveVolumeSize := updatedNodePool.NodeVolumeSize + + effectiveVolumes := updatedNodePool.Volumes + if effectiveVolumes == nil { + effectiveVolumes = &eks.NodePoolVolumes{} + } + if effectiveVolumes.InstanceRoot == nil { + effectiveVolumes.InstanceRoot = &eks.NodePoolVolume{} + } + if effectiveVolumes.KubeletRoot == nil { + effectiveVolumes.KubeletRoot = &eks.NodePoolVolume{} + } + effectiveSecurityGroups := updatedNodePool.SecurityGroups { // Note: needing CF stack for template version and possibly node pool version. getCFStackInput := eksWorkflow.GetCFStackActivityInput{ @@ -220,12 +230,17 @@ func (w EKSUpdateClusterWorkflow) Execute(ctx workflow.Context, input EKSUpdateC } var parameters struct { - CustomNodeSecurityGroups string `mapstructure:"CustomNodeSecurityGroups,omitempty"` // Note: CustomNodeSecurityGroups is only available from template version 2.0.0. - NodeImageID string `mapstructure:"NodeImageId"` - NodeVolumeEncryptionEnabled string `mapstructure:"NodeVolumeEncryptionEnabled,omitempty"` // Note: NodeVolumeEncryptionEnabled is only available from template version 2.1.0. - NodeVolumeEncryptionKeyARN string `mapstructure:"NodeVolumeEncryptionKeyARN,omitempty"` // Note: NodeVolumeEncryptionKeyARN is only available from template version 2.1.0. - NodeVolumeSize int `mapstructure:"NodeVolumeSize"` - TemplateVersion semver.Version `mapstructure:"TemplateVersion,omitempty"` // Note: TemplateVersion is only available from template version 2.0.0. + CustomNodeSecurityGroups string `mapstructure:"CustomNodeSecurityGroups,omitempty"` // Note: CustomNodeSecurityGroups is only available from template version 2.0.0. + NodeImageID string `mapstructure:"NodeImageId"` + NodeVolumeStorage string `mapstructure:"NodeVolumeStorage,omitempty"` // Note: NodeVolumeStorage is only available from template version 2.5.0. + NodeVolumeEncryptionEnabled string `mapstructure:"NodeVolumeEncryptionEnabled,omitempty"` // Note: NodeVolumeEncryptionEnabled is only available from template version 2.1.0. + NodeVolumeEncryptionKeyARN string `mapstructure:"NodeVolumeEncryptionKeyARN,omitempty"` // Note: NodeVolumeEncryptionKeyARN is only available from template version 2.1.0. + NodeVolumeSize int `mapstructure:"NodeVolumeSize"` + KubeletRootVolumeStorage string `mapstructure:"KubeletRootVolumeStorage,omitempty"` // Note: KubeletRootVolumeStorage is only available from template version 2.5.0. + KubeletRootVolumeEncryptionEnabled string `mapstructure:"KubeletRootVolumeEncryptionEnabled,omitempty"` // Note: KubeletRootVolumeEncryptionEnabled is only available from template version 2.5.0. + KubeletRootVolumeEncryptionKeyARN string `mapstructure:"KubeletRootVolumeEncryptionKeyARN,omitempty"` // Note: KubeletRootVolumeEncryptionKeyARN is only available from template version 2.5.0. + KubeletRootVolumeSize int `mapstructure:"KubeletRootVolumeSize,omitempty"` // Note: KubeletRootVolumeSize is only available from template version 2.5.0. + TemplateVersion semver.Version `mapstructure:"TemplateVersion,omitempty"` // Note: TemplateVersion is only available from template version 2.0.0. } err = sdkCloudFormation.ParseStackParameters(getCFStackOutput.Stack.Parameters, ¶meters) if err != nil { @@ -239,32 +254,57 @@ func (w EKSUpdateClusterWorkflow) Execute(ctx workflow.Context, input EKSUpdateC effectiveImage = parameters.NodeImageID } - if effectiveVolumeEncryption == nil && + if effectiveSecurityGroups == nil && + parameters.CustomNodeSecurityGroups != "" { + effectiveSecurityGroups = strings.Split(parameters.CustomNodeSecurityGroups, ",") + sort.Strings(effectiveSecurityGroups) + } + + if effectiveVolumes.InstanceRoot.Storage == "" { + effectiveVolumes.InstanceRoot.Storage = parameters.NodeVolumeStorage + } + + if effectiveVolumes.InstanceRoot.Encryption == nil && parameters.NodeVolumeEncryptionEnabled != "" { isNodeVolumeEncryptionEnabled, err := strconv.ParseBool(parameters.NodeVolumeEncryptionEnabled) if err != nil { return errors.WrapIf(err, "invalid node volume encryption enabled value") } - effectiveVolumeEncryption = &eks.NodePoolVolumeEncryption{ + effectiveVolumes.InstanceRoot.Encryption = &eks.NodePoolVolumeEncryption{ Enabled: isNodeVolumeEncryptionEnabled, EncryptionKeyARN: parameters.NodeVolumeEncryptionKeyARN, } } - if effectiveVolumeSize == 0 { - effectiveVolumeSize = parameters.NodeVolumeSize + if effectiveVolumes.InstanceRoot.Size == 0 { + effectiveVolumes.InstanceRoot.Size = parameters.NodeVolumeSize } - if effectiveSecurityGroups == nil && - parameters.CustomNodeSecurityGroups != "" { - effectiveSecurityGroups = strings.Split(parameters.CustomNodeSecurityGroups, ",") - sort.Strings(effectiveSecurityGroups) + if effectiveVolumes.KubeletRoot.Storage == "" { + effectiveVolumes.KubeletRoot.Storage = parameters.KubeletRootVolumeStorage + } + + if effectiveVolumes.KubeletRoot.Encryption == nil && + parameters.KubeletRootVolumeEncryptionEnabled != "" { + isVolumeEncryptionEnabled, err := strconv.ParseBool(parameters.KubeletRootVolumeEncryptionEnabled) + if err != nil { + return errors.WrapIf(err, "invalid kubelet root volume encryption enabled value") + } + + effectiveVolumes.KubeletRoot.Encryption = &eks.NodePoolVolumeEncryption{ + Enabled: isVolumeEncryptionEnabled, + EncryptionKeyARN: parameters.KubeletRootVolumeEncryptionKeyARN, + } + } + + if effectiveVolumes.KubeletRoot.Size == 0 { + effectiveVolumes.KubeletRoot.Size = parameters.KubeletRootVolumeSize } } var volumeSize int - if updatedNodePool.NodeVolumeSize > 0 { + if updatedNodePool.Volumes != nil && updatedNodePool.Volumes.InstanceRoot != nil && updatedNodePool.Volumes.InstanceRoot.Size > 0 { var amiSize int { activityInput := eksWorkflow.GetAMISizeActivityInput{ @@ -284,7 +324,7 @@ func (w EKSUpdateClusterWorkflow) Execute(ctx workflow.Context, input EKSUpdateC { activityInput := eksWorkflow.SelectVolumeSizeActivityInput{ AMISize: amiSize, - OptionalVolumeSize: updatedNodePool.NodeVolumeSize, + OptionalVolumeSize: updatedNodePool.Volumes.InstanceRoot.Size, } var activityOutput eksWorkflow.SelectVolumeSizeActivityOutput err = workflow.ExecuteActivity(ctx, eksWorkflow.SelectVolumeSizeActivityName, activityInput).Get(ctx, &activityOutput) @@ -294,17 +334,15 @@ func (w EKSUpdateClusterWorkflow) Execute(ctx workflow.Context, input EKSUpdateC } volumeSize = activityOutput.VolumeSize - effectiveVolumeSize = volumeSize + effectiveVolumes.InstanceRoot.Size = volumeSize } } - // todo x var nodePoolVersion string { activityInput := eksWorkflow.CalculateNodePoolVersionActivityInput{ - Image: effectiveImage, - //VolumeEncryption: effectiveVolumeEncryption, - //VolumeSize: effectiveVolumeSize, + Image: effectiveImage, + Volumes: *effectiveVolumes, CustomSecurityGroups: effectiveSecurityGroups, } @@ -342,16 +380,13 @@ func (w EKSUpdateClusterWorkflow) Execute(ctx workflow.Context, input EKSUpdateC NodeMinCount: updatedNodePool.NodeMinCount, NodeMaxCount: updatedNodePool.NodeMaxCount, Count: updatedNodePool.Count, - NodeVolumeEncryption: updatedNodePool.NodeVolumeEncryption, - NodeVolumeSize: volumeSize, - NodeVolumeType: updatedNodePool.NodeVolumeType, + NodeVolumes: effectiveVolumes, NodeImage: updatedNodePool.NodeImage, NodeInstanceType: updatedNodePool.NodeInstanceType, SecurityGroups: updatedNodePool.SecurityGroups, Labels: updatedNodePool.Labels, Tags: input.Tags, CurrentTemplateVersion: currentTemplateVersion, - UseInstanceStore: updatedNodePool.UseInstanceStore, } ctx = workflow.WithActivityOptions(ctx, aoWithHeartBeat) f := workflow.ExecuteActivity(ctx, eksWorkflow.UpdateAsgActivityName, activityInput)