From 88ed748ee633d94cecebfd857914baa055febeb3 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Thu, 9 Jan 2025 09:20:31 -0800 Subject: [PATCH] Fix up AWS and Azure tests --- config/dev/adopted-clusterdeployment.yaml | 2 +- test/e2e/clusterdeployment/aws/aws.go | 39 ++++++++++++----- .../clusterdeployment/clusterdeployment.go | 3 +- test/e2e/clusterdeployment/constants.go | 21 +++++----- .../clusterdeployment/providervalidator.go | 3 +- .../resources/adopted-cluster.yaml.tpl | 2 +- .../resources/aws-hosted-cp.yaml.tpl | 7 +++- .../resources/aws-standalone-cp.yaml.tpl | 2 + test/e2e/provider_adopted_test.go | 5 ++- test/e2e/provider_aws_test.go | 42 +++++++++++-------- test/e2e/provider_azure_test.go | 20 ++++++--- 11 files changed, 94 insertions(+), 52 deletions(-) diff --git a/config/dev/adopted-clusterdeployment.yaml b/config/dev/adopted-clusterdeployment.yaml index c20e819df..b84bb6641 100644 --- a/config/dev/adopted-clusterdeployment.yaml +++ b/config/dev/adopted-clusterdeployment.yaml @@ -7,7 +7,7 @@ spec: template: adopted-cluster-0-0-1 credential: adopted-cluster-cred config: {} - services: + serviceSpec: - template: kyverno-3-2-6 name: kyverno namespace: kyverno diff --git a/test/e2e/clusterdeployment/aws/aws.go b/test/e2e/clusterdeployment/aws/aws.go index 61a2a0a21..79dd76496 100644 --- a/test/e2e/clusterdeployment/aws/aws.go +++ b/test/e2e/clusterdeployment/aws/aws.go @@ -17,10 +17,14 @@ package aws import ( + "bufio" "context" + "fmt" + "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -52,22 +56,37 @@ func PopulateHostedTemplateVars(ctx context.Context, kc *kubeclient.KubeClient, Expect(err).NotTo(HaveOccurred(), "failed to get AWS cluster subnets") Expect(found).To(BeTrue(), "AWS cluster has no subnets") - subnet, ok := subnets[0].(map[string]any) - Expect(ok).To(BeTrue(), "failed to cast subnet to map") - - subnetID, ok := subnet["resourceID"].(string) - Expect(ok).To(BeTrue(), "failed to cast subnet ID to string") - - subnetAZ, ok := subnet["availabilityZone"].(string) - Expect(ok).To(BeTrue(), "failed to cast subnet availability zone to string") + type awsSubnetMaps []map[string]any + subnetMaps := make(awsSubnetMaps, len(subnets)) + for i, s := range subnets { + subnet, ok := s.(map[string]any) + Expect(ok).To(BeTrue(), "failed to cast subnet to map") + subnetMaps[i] = map[string]any{ + "isPublic": subnet["isPublic"], + "availabilityZone": subnet["availabilityZone"], + "id": subnet["resourceID"], + "routeTableId": subnet["routeTableId"], + "zoneType": "availability-zone", + } + if natGatewayID, exists := subnet["natGatewayId"]; exists && natGatewayID != "" { + subnetMaps[i]["natGatewayId"] = natGatewayID + } + } + var subnetsFormatted string + encodedYaml, err := yaml.Marshal(subnetMaps) + Expect(err).NotTo(HaveOccurred(), "failed to get marshall subnet maps") + scanner := bufio.NewScanner(strings.NewReader(string(encodedYaml))) + for scanner.Scan() { + subnetsFormatted += fmt.Sprintf(" %s\n", scanner.Text()) + } + GinkgoT().Setenv(clusterdeployment.EnvVarAWSSubnets, subnetsFormatted) securityGroupID, found, err := unstructured.NestedString( awsCluster.Object, "status", "networkStatus", "securityGroups", "node", "id") Expect(err).NotTo(HaveOccurred(), "failed to get AWS cluster security group ID") Expect(found).To(BeTrue(), "AWS cluster has no security group ID") GinkgoT().Setenv(clusterdeployment.EnvVarAWSVPCID, vpcID) - GinkgoT().Setenv(clusterdeployment.EnvVarAWSSubnetID, subnetID) - GinkgoT().Setenv(clusterdeployment.EnvVarAWSSubnetAvailabilityZone, subnetAZ) GinkgoT().Setenv(clusterdeployment.EnvVarAWSSecurityGroupID, securityGroupID) + GinkgoT().Setenv(clusterdeployment.EnvVarManagementClusterName, clusterName) } diff --git a/test/e2e/clusterdeployment/clusterdeployment.go b/test/e2e/clusterdeployment/clusterdeployment.go index f25ef91f3..1179c37b9 100644 --- a/test/e2e/clusterdeployment/clusterdeployment.go +++ b/test/e2e/clusterdeployment/clusterdeployment.go @@ -127,8 +127,7 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured { // since we populate the vars from standalone prior to this step. ValidateDeploymentVars([]string{ EnvVarAWSVPCID, - EnvVarAWSSubnetID, - EnvVarAWSSubnetAvailabilityZone, + EnvVarAWSSubnets, EnvVarAWSSecurityGroupID, }) clusterDeploymentTemplateBytes = awsHostedCPClusterDeploymentTemplateBytes diff --git a/test/e2e/clusterdeployment/constants.go b/test/e2e/clusterdeployment/constants.go index 2a356fdd7..17c196301 100644 --- a/test/e2e/clusterdeployment/constants.go +++ b/test/e2e/clusterdeployment/constants.go @@ -22,18 +22,17 @@ const ( EnvVarNamespace = "NAMESPACE" // EnvVarNoCleanup disables After* cleanup in provider specs to allow for // debugging of test failures. - EnvVarNoCleanup = "NO_CLEANUP" - + EnvVarNoCleanup = "NO_CLEANUP" + EnvVarManagementClusterName = "MANAGEMENT_CLUSTER_NAME" // AWS - EnvVarAWSAccessKeyID = "AWS_ACCESS_KEY_ID" - EnvVarAWSSecretAccessKey = "AWS_SECRET_ACCESS_KEY" - EnvVarAWSVPCID = "AWS_VPC_ID" - EnvVarAWSSubnetID = "AWS_SUBNET_ID" - EnvVarAWSSubnetAvailabilityZone = "AWS_SUBNET_AVAILABILITY_ZONE" - EnvVarAWSInstanceType = "AWS_INSTANCE_TYPE" - EnvVarAWSSecurityGroupID = "AWS_SG_ID" - EnvVarAWSClusterIdentity = "AWS_CLUSTER_IDENTITY" - EnvVarPublicIP = "AWS_PUBLIC_IP" + EnvVarAWSAccessKeyID = "AWS_ACCESS_KEY_ID" + EnvVarAWSSecretAccessKey = "AWS_SECRET_ACCESS_KEY" + EnvVarAWSVPCID = "AWS_VPC_ID" + EnvVarAWSSubnets = "AWS_SUBNETS" + EnvVarAWSInstanceType = "AWS_INSTANCE_TYPE" + EnvVarAWSSecurityGroupID = "AWS_SG_ID" + EnvVarAWSClusterIdentity = "AWS_CLUSTER_IDENTITY" + EnvVarPublicIP = "AWS_PUBLIC_IP" // VSphere EnvVarVSphereUser = "VSPHERE_USER" diff --git a/test/e2e/clusterdeployment/providervalidator.go b/test/e2e/clusterdeployment/providervalidator.go index 80d1cdaca..0e45f4650 100644 --- a/test/e2e/clusterdeployment/providervalidator.go +++ b/test/e2e/clusterdeployment/providervalidator.go @@ -65,9 +65,8 @@ func NewProviderValidator(template Template, clusterName string, action Validati case TemplateAWSStandaloneCP, TemplateAWSHostedCP: resourcesToValidate["ccm"] = validateCCM resourceOrder = append(resourceOrder, "ccm") - case TemplateAzureStandaloneCP, TemplateVSphereStandaloneCP: + case TemplateAzureStandaloneCP, TemplateAzureHostedCP, TemplateVSphereStandaloneCP: delete(resourcesToValidate, "csi-driver") - case TemplateAdoptedCluster: resourcesToValidate = map[string]resourceValidationFunc{ "sveltoscluster": validateSveltosCluster, diff --git a/test/e2e/clusterdeployment/resources/adopted-cluster.yaml.tpl b/test/e2e/clusterdeployment/resources/adopted-cluster.yaml.tpl index 180926bae..26e0af8e1 100644 --- a/test/e2e/clusterdeployment/resources/adopted-cluster.yaml.tpl +++ b/test/e2e/clusterdeployment/resources/adopted-cluster.yaml.tpl @@ -7,7 +7,7 @@ spec: template: adopted-cluster-0-0-1 credential: ${ADOPTED_CREDENTIAL} config: {} - services: + serviceSpec: - template: kyverno-3-2-6 name: kyverno namespace: kyverno diff --git a/test/e2e/clusterdeployment/resources/aws-hosted-cp.yaml.tpl b/test/e2e/clusterdeployment/resources/aws-hosted-cp.yaml.tpl index 62fd8685e..3df4d2f0d 100644 --- a/test/e2e/clusterdeployment/resources/aws-hosted-cp.yaml.tpl +++ b/test/e2e/clusterdeployment/resources/aws-hosted-cp.yaml.tpl @@ -12,8 +12,11 @@ spec: vpcID: ${AWS_VPC_ID} region: ${AWS_REGION} subnets: - - id: ${AWS_SUBNET_ID} - availabilityZone: ${AWS_SUBNET_AVAILABILITY_ZONE} +${AWS_SUBNETS} instanceType: ${AWS_INSTANCE_TYPE:=t3.medium} securityGroupIDs: - ${AWS_SG_ID} + managementClusterName: ${MANAGEMENT_CLUSTER_NAME} + controlPlane: + rootVolumeSize: 30 + rootVolumeSize: 30 \ No newline at end of file diff --git a/test/e2e/clusterdeployment/resources/aws-standalone-cp.yaml.tpl b/test/e2e/clusterdeployment/resources/aws-standalone-cp.yaml.tpl index 3a8252d9d..52187f079 100644 --- a/test/e2e/clusterdeployment/resources/aws-standalone-cp.yaml.tpl +++ b/test/e2e/clusterdeployment/resources/aws-standalone-cp.yaml.tpl @@ -15,5 +15,7 @@ spec: workersNumber: ${WORKERS_NUMBER:=1} controlPlane: instanceType: ${AWS_INSTANCE_TYPE:=t3.small} + rootVolumeSize: 30 worker: instanceType: ${AWS_INSTANCE_TYPE:=t3.small} + rootVolumeSize: 30 \ No newline at end of file diff --git a/test/e2e/provider_adopted_test.go b/test/e2e/provider_adopted_test.go index 4ed9a6647..666595ea9 100644 --- a/test/e2e/provider_adopted_test.go +++ b/test/e2e/provider_adopted_test.go @@ -114,13 +114,16 @@ var _ = Describe("Adopted Cluster Templates", Label("provider:cloud", "provider: return deploymentValidator.Validate(context.Background(), kc) }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + // delete the adopted cluster err := adoptedDeleteFunc() Expect(err).NotTo(HaveOccurred()) + adoptedDeleteFunc = nil + // finally delete the aws standalone cluster err = clusterDeleteFunc() Expect(err).NotTo(HaveOccurred()) + clusterDeleteFunc = nil - // finally delete the aws standalone clsuter deletionValidator := clusterdeployment.NewProviderValidator( clusterdeployment.TemplateAWSStandaloneCP, clusterName, diff --git a/test/e2e/provider_aws_test.go b/test/e2e/provider_aws_test.go index 7bc7f1fd1..589d6ac1d 100644 --- a/test/e2e/provider_aws_test.go +++ b/test/e2e/provider_aws_test.go @@ -125,6 +125,15 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order return nil }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + Eventually(func() error { + err = clusterdeployment.ValidateClusterTemplates(context.Background(), standaloneClient) + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "cluster template validation failed: %v\n", err) + return err + } + return nil + }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + // Ensure AWS credentials are set in the standalone cluster. standaloneCi := clusteridentity.New(standaloneClient, clusterdeployment.ProviderAWS) standaloneCi.WaitForValidCredential(standaloneClient) @@ -158,6 +167,7 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order templateBy(clusterdeployment.TemplateAWSHostedCP, "deleting the clusterdeployment") err = hostedDeleteFunc() Expect(err).NotTo(HaveOccurred()) + hostedDeleteFunc = nil deletionValidator := clusterdeployment.NewProviderValidator( clusterdeployment.TemplateAWSHostedCP, @@ -170,22 +180,20 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order // Now delete the standalone clusterdeployment and verify it is // removed, it is deleted last since it is the basis for the hosted // cluster. - /* - FIXME(#339): This is currently disabled as the deletion of the - standalone cluster is failing due to outstanding issues. - templateBy(clusterdeployment.TemplateAWSStandaloneCP, "deleting the clusterdeployment") - err = standaloneDeleteFunc() - Expect(err).NotTo(HaveOccurred()) - - deletionValidator = clusterdeployment.NewProviderValidator( - clusterdeployment.TemplateAWSStandaloneCP, - clusterName, - clusterdeployment.ValidationActionDelete, - ) - Eventually(func() error { - return deletionValidator.Validate(context.Background(), kc) - }).WithTimeout(10 * time.Minute).WithPolling(10 * - time.Second).Should(Succeed()) - */ + + templateBy(clusterdeployment.TemplateAWSStandaloneCP, "deleting the clusterdeployment") + err = standaloneDeleteFunc() + Expect(err).NotTo(HaveOccurred()) + + standaloneDeleteFunc = nil + deletionValidator = clusterdeployment.NewProviderValidator( + clusterdeployment.TemplateAWSStandaloneCP, + clusterName, + clusterdeployment.ValidationActionDelete, + ) + Eventually(func() error { + return deletionValidator.Validate(context.Background(), kc) + }).WithTimeout(10 * time.Minute).WithPolling(10 * + time.Second).Should(Succeed()) }) }) diff --git a/test/e2e/provider_azure_test.go b/test/e2e/provider_azure_test.go index 5ce409deb..a4f39ac2f 100644 --- a/test/e2e/provider_azure_test.go +++ b/test/e2e/provider_azure_test.go @@ -83,7 +83,7 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or sd := clusterdeployment.GetUnstructured(clusterdeployment.TemplateAzureStandaloneCP) sdName = sd.GetName() - standaloneDeleteFunc := kc.CreateClusterDeployment(context.Background(), sd) + standaloneDeleteFunc = kc.CreateClusterDeployment(context.Background(), sd) // verify the standalone cluster is deployed correctly deploymentValidator := clusterdeployment.NewProviderValidator( @@ -124,6 +124,15 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or return nil }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + Eventually(func() error { + err = clusterdeployment.ValidateClusterTemplates(context.Background(), standaloneClient) + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "cluster template validation failed: %v\n", err) + return err + } + return nil + }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + By("Create azure credential secret") standaloneCi := clusteridentity.New(standaloneClient, clusterdeployment.ProviderAzure) standaloneCi.WaitForValidCredential(standaloneClient) @@ -151,10 +160,7 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or By("verify the deployment deletes successfully") err = hostedDeleteFunc() Expect(err).NotTo(HaveOccurred()) - - err = standaloneDeleteFunc() - Expect(err).NotTo(HaveOccurred()) - + hostedDeleteFunc = nil deploymentValidator = clusterdeployment.NewProviderValidator( clusterdeployment.TemplateAzureHostedCP, hdName, @@ -165,6 +171,10 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or return deploymentValidator.Validate(context.Background(), standaloneClient) }).WithTimeout(10 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + err = standaloneDeleteFunc() + Expect(err).NotTo(HaveOccurred()) + standaloneDeleteFunc = nil + deploymentValidator = clusterdeployment.NewProviderValidator( clusterdeployment.TemplateAzureStandaloneCP, hdName,