Assume instance exists within eventual-consistency grace period

pkg/providers/v1/aws.go

@@ -400,7 +400,8 @@ type Cloud struct {
 
 	nodeInformer informercorev1.NodeInformer
 	// Extract the function out to make it easier to test
-	nodeInformerHasSynced cache.InformerSynced
+	nodeInformerHasSynced              cache.InformerSynced
+	nodeEventualConsistencyGracePeriod time.Duration
 
 	eventBroadcaster record.EventBroadcaster
 	eventRecorder    record.EventRecorder
@@ -615,14 +616,15 @@ func newAWSCloud2(cfg config.CloudConfig, awsServices Services, provider config.
 	}
 
 	awsCloud := &Cloud{
-		ec2:      ec2,
-		elb:      elb,
-		elbv2:    elbv2,
-		asg:      asg,
-		metadata: metadata,
-		kms:      kms,
-		cfg:      &cfg,
-		region:   regionName,
+		ec2:                                ec2,
+		elb:                                elb,
+		elbv2:                              elbv2,
+		asg:                                asg,
+		metadata:                           metadata,
+		kms:                                kms,
+		cfg:                                &cfg,
+		region:                             regionName,
+		nodeEventualConsistencyGracePeriod: cfg.Global.NodeEventualConsistencyGracePeriod,
 	}
 	awsCloud.instanceCache.cloud = awsCloud
 	awsCloud.zoneCache.cloud = awsCloud

pkg/providers/v1/config/config.go

@@ -2,8 +2,10 @@ package config
 
 import (
 	"fmt"
-	"github.com/aws/aws-sdk-go/aws/request"
 	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws/request"
 
 	"github.com/aws/aws-sdk-go/aws/endpoints"
 
@@ -62,6 +64,11 @@ type CloudConfig struct {
 
 		// NodeIPFamilies determines which IP addresses are added to node objects and their ordering.
 		NodeIPFamilies []string
+
+		// NodeEventualConsistencyGracePeriod is used to account for propogation delays in the EC2 API.
+		// An instance may not appear in `ec2:DescribeInstances` output for a period of time after launch.
+		// The cloud-node-lifecycle-controller must not delete the Node prematurely in this case.
+		NodeEventualConsistencyGracePeriod time.Duration
 	}
 	// [ServiceOverride "1"]
 	//  Service = s3

pkg/providers/v1/instances_v2.go

@@ -22,7 +22,9 @@ package aws
 
 import (
 	"context"
+	"fmt"
 	"strconv"
+	"time"
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -45,6 +47,12 @@ func (c *Cloud) getProviderID(ctx context.Context, node *v1.Node) (string, error
 // InstanceExists returns true if the instance for the given node exists according to the cloud provider.
 // Use the node.name or node.spec.providerID field to find the node in the cloud provider.
 func (c *Cloud) InstanceExists(ctx context.Context, node *v1.Node) (bool, error) {
+	if time.Since(node.CreationTimestamp.Time) < c.nodeEventualConsistencyGracePeriod {
+		// recently-launched EC2 instances may not appear in `ec2:DescribeInstances`
+		// we return an error to cause the cloud-node-lifecycle-controller to ignore this node
+		// until the eventual-consistency grace period elapses
+		return false, fmt.Errorf("node is within eventual-consistency grace period (%v)", c.nodeEventualConsistencyGracePeriod)
+	}
 	providerID, err := c.getProviderID(ctx, node)
 	if err != nil {
 		return false, err