Fix up AWS and Azure tests

config/dev/adopted-clusterdeployment.yaml

@@ -7,7 +7,7 @@ spec:
   template: adopted-cluster-0-0-1
   credential: adopted-cluster-cred
   config: {}
-  services:
+  serviceSpec:
     - template: kyverno-3-2-6
       name: kyverno
       namespace: kyverno

test/e2e/clusterdeployment/aws/aws.go

@@ -17,10 +17,14 @@
 package aws
 
 import (
+	"bufio"
 	"context"
+	"fmt"
+	"strings"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -52,22 +56,37 @@ func PopulateHostedTemplateVars(ctx context.Context, kc *kubeclient.KubeClient,
 	Expect(err).NotTo(HaveOccurred(), "failed to get AWS cluster subnets")
 	Expect(found).To(BeTrue(), "AWS cluster has no subnets")
 
-	subnet, ok := subnets[0].(map[string]any)
-	Expect(ok).To(BeTrue(), "failed to cast subnet to map")
-
-	subnetID, ok := subnet["resourceID"].(string)
-	Expect(ok).To(BeTrue(), "failed to cast subnet ID to string")
-
-	subnetAZ, ok := subnet["availabilityZone"].(string)
-	Expect(ok).To(BeTrue(), "failed to cast subnet availability zone to string")
+	type awsSubnetMaps []map[string]any
+	subnetMaps := make(awsSubnetMaps, len(subnets))
+	for i, s := range subnets {
+		subnet, ok := s.(map[string]any)
+		Expect(ok).To(BeTrue(), "failed to cast subnet to map")
+		subnetMaps[i] = map[string]any{
+			"isPublic":         subnet["isPublic"],
+			"availabilityZone": subnet["availabilityZone"],
+			"id":               subnet["resourceID"],
+			"routeTableId":     subnet["routeTableId"],
+			"zoneType":         "availability-zone",
+		}
 
+		if natGatewayID, exists := subnet["natGatewayId"]; exists && natGatewayID != "" {
+			subnetMaps[i]["natGatewayId"] = natGatewayID
+		}
+	}
+	var subnetsFormatted string
+	encodedYaml, err := yaml.Marshal(subnetMaps)
+	Expect(err).NotTo(HaveOccurred(), "failed to get marshall subnet maps")
+	scanner := bufio.NewScanner(strings.NewReader(string(encodedYaml)))
+	for scanner.Scan() {
+		subnetsFormatted += fmt.Sprintf("    %s\n", scanner.Text())
+	}
+	GinkgoT().Setenv(clusterdeployment.EnvVarAWSSubnets, subnetsFormatted)
 	securityGroupID, found, err := unstructured.NestedString(
 		awsCluster.Object, "status", "networkStatus", "securityGroups", "node", "id")
 	Expect(err).NotTo(HaveOccurred(), "failed to get AWS cluster security group ID")
 	Expect(found).To(BeTrue(), "AWS cluster has no security group ID")
 
 	GinkgoT().Setenv(clusterdeployment.EnvVarAWSVPCID, vpcID)
-	GinkgoT().Setenv(clusterdeployment.EnvVarAWSSubnetID, subnetID)
-	GinkgoT().Setenv(clusterdeployment.EnvVarAWSSubnetAvailabilityZone, subnetAZ)
 	GinkgoT().Setenv(clusterdeployment.EnvVarAWSSecurityGroupID, securityGroupID)
+	GinkgoT().Setenv(clusterdeployment.EnvVarManagementClusterName, clusterName)
 }

test/e2e/clusterdeployment/clusterdeployment.go

@@ -127,8 +127,7 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured {
 		// since we populate the vars from standalone prior to this step.
 		ValidateDeploymentVars([]string{
 			EnvVarAWSVPCID,
-			EnvVarAWSSubnetID,
-			EnvVarAWSSubnetAvailabilityZone,
+			EnvVarAWSSubnets,
 			EnvVarAWSSecurityGroupID,
 		})
 		clusterDeploymentTemplateBytes = awsHostedCPClusterDeploymentTemplateBytes

test/e2e/clusterdeployment/constants.go

@@ -22,18 +22,17 @@ const (
 	EnvVarNamespace             = "NAMESPACE"
 	// EnvVarNoCleanup disables After* cleanup in provider specs to allow for
 	// debugging of test failures.
-	EnvVarNoCleanup = "NO_CLEANUP"
-
+	EnvVarNoCleanup             = "NO_CLEANUP"
+	EnvVarManagementClusterName = "MANAGEMENT_CLUSTER_NAME"
 	// AWS
-	EnvVarAWSAccessKeyID            = "AWS_ACCESS_KEY_ID"
-	EnvVarAWSSecretAccessKey        = "AWS_SECRET_ACCESS_KEY"
-	EnvVarAWSVPCID                  = "AWS_VPC_ID"
-	EnvVarAWSSubnetID               = "AWS_SUBNET_ID"
-	EnvVarAWSSubnetAvailabilityZone = "AWS_SUBNET_AVAILABILITY_ZONE"
-	EnvVarAWSInstanceType           = "AWS_INSTANCE_TYPE"
-	EnvVarAWSSecurityGroupID        = "AWS_SG_ID"
-	EnvVarAWSClusterIdentity        = "AWS_CLUSTER_IDENTITY"
-	EnvVarPublicIP                  = "AWS_PUBLIC_IP"
+	EnvVarAWSAccessKeyID     = "AWS_ACCESS_KEY_ID"
+	EnvVarAWSSecretAccessKey = "AWS_SECRET_ACCESS_KEY"
+	EnvVarAWSVPCID           = "AWS_VPC_ID"
+	EnvVarAWSSubnets         = "AWS_SUBNETS"
+	EnvVarAWSInstanceType    = "AWS_INSTANCE_TYPE"
+	EnvVarAWSSecurityGroupID = "AWS_SG_ID"
+	EnvVarAWSClusterIdentity = "AWS_CLUSTER_IDENTITY"
+	EnvVarPublicIP           = "AWS_PUBLIC_IP"
 
 	// VSphere
 	EnvVarVSphereUser            = "VSPHERE_USER"

test/e2e/clusterdeployment/providervalidator.go

@@ -65,9 +65,8 @@ func NewProviderValidator(template Template, clusterName string, action Validati
 		case TemplateAWSStandaloneCP, TemplateAWSHostedCP:
 			resourcesToValidate["ccm"] = validateCCM
 			resourceOrder = append(resourceOrder, "ccm")
-		case TemplateAzureStandaloneCP, TemplateVSphereStandaloneCP:
+		case TemplateAzureStandaloneCP, TemplateAzureHostedCP, TemplateVSphereStandaloneCP:
 			delete(resourcesToValidate, "csi-driver")
-
 		case TemplateAdoptedCluster:
 			resourcesToValidate = map[string]resourceValidationFunc{
 				"sveltoscluster": validateSveltosCluster,

test/e2e/clusterdeployment/resources/adopted-cluster.yaml.tpl

@@ -7,7 +7,7 @@ spec:
   template: adopted-cluster-0-0-1
   credential: ${ADOPTED_CREDENTIAL}
   config: {}
-  services:
+  serviceSpec:
     - template: kyverno-3-2-6
       name: kyverno
       namespace: kyverno

test/e2e/clusterdeployment/resources/aws-hosted-cp.yaml.tpl

@@ -12,8 +12,11 @@ spec:
     vpcID: ${AWS_VPC_ID}
     region: ${AWS_REGION}
     subnets:
-      - id: ${AWS_SUBNET_ID}
-        availabilityZone: ${AWS_SUBNET_AVAILABILITY_ZONE}
+${AWS_SUBNETS}
     instanceType: ${AWS_INSTANCE_TYPE:=t3.medium}
     securityGroupIDs:
       - ${AWS_SG_ID}
+    managementClusterName: ${MANAGEMENT_CLUSTER_NAME}
+    controlPlane:
+       rootVolumeSize: 30
+    rootVolumeSize: 30

test/e2e/clusterdeployment/resources/aws-standalone-cp.yaml.tpl

@@ -15,5 +15,7 @@ spec:
     workersNumber: ${WORKERS_NUMBER:=1}
     controlPlane:
       instanceType: ${AWS_INSTANCE_TYPE:=t3.small}
+      rootVolumeSize: 30
     worker:
       instanceType: ${AWS_INSTANCE_TYPE:=t3.small}
+      rootVolumeSize: 30

test/e2e/provider_adopted_test.go

@@ -114,13 +114,16 @@ var _ = Describe("Adopted Cluster Templates", Label("provider:cloud", "provider:
 			return deploymentValidator.Validate(context.Background(), kc)
 		}).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
 
+		// delete the adopted cluster
 		err := adoptedDeleteFunc()
 		Expect(err).NotTo(HaveOccurred())
+		adoptedDeleteFunc = nil
 
+		// finally delete the aws standalone cluster
 		err = clusterDeleteFunc()
 		Expect(err).NotTo(HaveOccurred())
+		clusterDeleteFunc = nil
 
-		// finally delete the aws standalone clsuter
 		deletionValidator := clusterdeployment.NewProviderValidator(
 			clusterdeployment.TemplateAWSStandaloneCP,
 			clusterName,

test/e2e/provider_aws_test.go

@@ -125,6 +125,15 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order
 			return nil
 		}).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
 
+		Eventually(func() error {
+			err = clusterdeployment.ValidateClusterTemplates(context.Background(), standaloneClient)
+			if err != nil {
+				_, _ = fmt.Fprintf(GinkgoWriter, "cluster template validation failed: %v\n", err)
+				return err
+			}
+			return nil
+		}).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
+
 		// Ensure AWS credentials are set in the standalone cluster.
 		standaloneCi := clusteridentity.New(standaloneClient, clusterdeployment.ProviderAWS)
 		standaloneCi.WaitForValidCredential(standaloneClient)
@@ -158,6 +167,7 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order
 		templateBy(clusterdeployment.TemplateAWSHostedCP, "deleting the clusterdeployment")
 		err = hostedDeleteFunc()
 		Expect(err).NotTo(HaveOccurred())
+		hostedDeleteFunc = nil
 
 		deletionValidator := clusterdeployment.NewProviderValidator(
 			clusterdeployment.TemplateAWSHostedCP,
@@ -170,22 +180,20 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order
 		// Now delete the standalone clusterdeployment and verify it is
 		// removed, it is deleted last since it is the basis for the hosted
 		// cluster.
-		/*
-			FIXME(#339): This is currently disabled as the deletion of the
-			standalone cluster is failing due to outstanding issues.
-			templateBy(clusterdeployment.TemplateAWSStandaloneCP, "deleting the clusterdeployment")
-			err = standaloneDeleteFunc()
-			Expect(err).NotTo(HaveOccurred())
-
-			deletionValidator = clusterdeployment.NewProviderValidator(
-				clusterdeployment.TemplateAWSStandaloneCP,
-				clusterName,
-				clusterdeployment.ValidationActionDelete,
-			)
-			Eventually(func() error {
-				return deletionValidator.Validate(context.Background(), kc)
-			}).WithTimeout(10 * time.Minute).WithPolling(10 *
-				time.Second).Should(Succeed())
-		*/
+
+		templateBy(clusterdeployment.TemplateAWSStandaloneCP, "deleting the clusterdeployment")
+		err = standaloneDeleteFunc()
+		Expect(err).NotTo(HaveOccurred())
+
+		standaloneDeleteFunc = nil
+		deletionValidator = clusterdeployment.NewProviderValidator(
+			clusterdeployment.TemplateAWSStandaloneCP,
+			clusterName,
+			clusterdeployment.ValidationActionDelete,
+		)
+		Eventually(func() error {
+			return deletionValidator.Validate(context.Background(), kc)
+		}).WithTimeout(10 * time.Minute).WithPolling(10 *
+			time.Second).Should(Succeed())
 	})
 })

test/e2e/provider_azure_test.go

@@ -83,7 +83,7 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or
 		sd := clusterdeployment.GetUnstructured(clusterdeployment.TemplateAzureStandaloneCP)
 		sdName = sd.GetName()
 
-		standaloneDeleteFunc := kc.CreateClusterDeployment(context.Background(), sd)
+		standaloneDeleteFunc = kc.CreateClusterDeployment(context.Background(), sd)
 
 		// verify the standalone cluster is deployed correctly
 		deploymentValidator := clusterdeployment.NewProviderValidator(
@@ -124,6 +124,15 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or
 			return nil
 		}).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
 
+		Eventually(func() error {
+			err = clusterdeployment.ValidateClusterTemplates(context.Background(), standaloneClient)
+			if err != nil {
+				_, _ = fmt.Fprintf(GinkgoWriter, "cluster template validation failed: %v\n", err)
+				return err
+			}
+			return nil
+		}).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
+
 		By("Create azure credential secret")
 		standaloneCi := clusteridentity.New(standaloneClient, clusterdeployment.ProviderAzure)
 		standaloneCi.WaitForValidCredential(standaloneClient)
@@ -151,10 +160,7 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or
 		By("verify the deployment deletes successfully")
 		err = hostedDeleteFunc()
 		Expect(err).NotTo(HaveOccurred())
-
-		err = standaloneDeleteFunc()
-		Expect(err).NotTo(HaveOccurred())
-
+		hostedDeleteFunc = nil
 		deploymentValidator = clusterdeployment.NewProviderValidator(
 			clusterdeployment.TemplateAzureHostedCP,
 			hdName,
@@ -165,6 +171,10 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or
 			return deploymentValidator.Validate(context.Background(), standaloneClient)
 		}).WithTimeout(10 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
 
+		err = standaloneDeleteFunc()
+		Expect(err).NotTo(HaveOccurred())
+		standaloneDeleteFunc = nil
+
 		deploymentValidator = clusterdeployment.NewProviderValidator(
 			clusterdeployment.TemplateAzureStandaloneCP,
 			hdName,