Merge pull request gruntwork-io#81 from adamrbennett/delete-local-dat…

…a-flag

Add delete-local-data flag and pass to kubectl drain if present

cmd/eks.go

@@ -59,6 +59,10 @@ var (
 		Value: 15 * time.Minute,
 		Usage: "The length of time as duration (e.g 10m = 10 minutes) to wait for draining nodes before giving up, zero means infinite. Defaults to 15 minutes.",
 	}
+	deleteLocalDataFlag = cli.BoolFlag{
+		Name:  "delete-local-data",
+		Usage: "Continue even if there are pods using emptyDir (local data that will be deleted when the node is drained).",
+	}
 	waitMaxRetriesFlag = cli.IntFlag{
 		Name:  "max-retries",
 		Value: 0,
@@ -164,6 +168,7 @@ If max-retries is unspecified, this command will use a value that translates to
 					eksKubectlCAFlag,
 					eksKubectlTokenFlag,
 					drainTimeoutFlag,
+					deleteLocalDataFlag,
 					waitMaxRetriesFlag,
 					waitSleepBetweenRetriesFlag,
 				},
@@ -285,6 +290,7 @@ func rollOutDeployment(cliContext *cli.Context) error {
 		return errors.WithStackTrace(err)
 	}
 	drainTimeout := cliContext.Duration(drainTimeoutFlag.Name)
+	deleteLocalData := cliContext.Bool(deleteLocalDataFlag.Name)
 	waitMaxRetries := cliContext.Int(waitMaxRetriesFlag.Name)
 	waitSleepBetweenRetries := cliContext.Duration(waitSleepBetweenRetriesFlag.Name)
 
@@ -293,6 +299,7 @@ func rollOutDeployment(cliContext *cli.Context) error {
 		asgName,
 		kubectlOptions,
 		drainTimeout,
+		deleteLocalData,
 		waitMaxRetries,
 		waitSleepBetweenRetries,
 	)

eks/asg.go

@@ -193,14 +193,15 @@ func drainNodesInAsg(
 	kubectlOptions *kubectl.KubectlOptions,
 	asgInstanceIds []string,
 	drainTimeout time.Duration,
+	deleteLocalData bool,
 ) error {
 	instances, err := instanceDetailsFromIds(ec2Svc, asgInstanceIds)
 	if err != nil {
 		return err
 	}
 	eksKubeNodeNames := kubeNodeNamesFromInstances(instances)
 
-	return kubectl.DrainNodes(kubectlOptions, eksKubeNodeNames, drainTimeout)
+	return kubectl.DrainNodes(kubectlOptions, eksKubeNodeNames, drainTimeout, deleteLocalData)
 }
 
 // Make the call to cordon all the provided nodes in Kubernetes so that they won't be used to schedule new Pods.

eks/deploy.go

@@ -31,6 +31,7 @@ func RollOutDeployment(
 	eksAsgName string,
 	kubectlOptions *kubectl.KubectlOptions,
 	drainTimeout time.Duration,
+	deleteLocalData bool,
 	maxRetries int,
 	sleepBetweenRetries time.Duration,
 ) error {
@@ -109,7 +110,7 @@ func RollOutDeployment(
 	logger.Infof("Successfully cordoned old instances in cluster ASG %s", eksAsgName)
 
 	logger.Infof("Draining Pods on old instances in cluster ASG %s", eksAsgName)
-	err = drainNodesInAsg(ec2Svc, kubectlOptions, currentInstanceIds, drainTimeout)
+	err = drainNodesInAsg(ec2Svc, kubectlOptions, currentInstanceIds, drainTimeout, deleteLocalData)
 	if err != nil {
 		logger.Errorf("Error while draining nodes.")
 		logger.Errorf("Continue to drain nodes that failed manually, and then terminate the underlying instances to complete the rollout.")

kubectl/node.go

@@ -103,13 +103,13 @@ func filterNodesByID(nodes []corev1.Node, nodeIds []string) []corev1.Node {
 // See
 // https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/#use-kubectl-drain-to-remove-a-node-from-service
 // for more information.
-func DrainNodes(kubectlOptions *KubectlOptions, nodeIds []string, timeout time.Duration) error {
+func DrainNodes(kubectlOptions *KubectlOptions, nodeIds []string, timeout time.Duration, deleteLocalData bool) error {
 	// Concurrently trigger drain events for all requested nodes.
 	var wg sync.WaitGroup                      // So that we can wait for all the drain calls
 	errChannel := make(chan NodeDrainError, 1) // Collect all errors from each command
 	for _, nodeID := range nodeIds {
 		wg.Add(1)
-		go drainNode(&wg, errChannel, kubectlOptions, nodeID, timeout)
+		go drainNode(&wg, errChannel, kubectlOptions, nodeID, timeout, deleteLocalData)
 	}
 	go waitForAllDrains(&wg, errChannel)
 
@@ -131,9 +131,18 @@ func drainNode(
 	kubectlOptions *KubectlOptions,
 	nodeID string,
 	timeout time.Duration,
+	deleteLocalData bool,
 ) {
 	defer wg.Done()
-	err := RunKubectl(kubectlOptions, "drain", nodeID, "--ignore-daemonsets", "--timeout", timeout.String())
+
+	args := []string{"drain", nodeID, "--ignore-daemonsets", "--timeout", timeout.String()}
+
+	if deleteLocalData {
+		args = append(args, "--delete-local-data")
+	}
+
+	err := RunKubectl(kubectlOptions, args...)
+
 	errChannel <- NodeDrainError{NodeID: nodeID, Error: err}
 }