From 301aef0e1ae3e2d041b0e8dc8dcb9b743f385b80 Mon Sep 17 00:00:00 2001 From: Ekaterina Kazakova Date: Mon, 23 Dec 2024 19:48:24 +0400 Subject: [PATCH] Add EKS e2e testing --- .../clusterdeployment/clusterdeployment.go | 6 ++ .../clusterdeployment/providervalidator.go | 20 +++++- .../resources/aws-eks.yaml.tpl | 13 ++++ .../e2e/clusterdeployment/validate_deleted.go | 40 ++++++------ .../clusterdeployment/validate_deployed.go | 61 ++++++++++++------- test/e2e/kubeclient/kubeclient.go | 12 ++++ test/e2e/provider_aws_test.go | 33 ++++++++++ 7 files changed, 142 insertions(+), 43 deletions(-) create mode 100644 test/e2e/clusterdeployment/resources/aws-eks.yaml.tpl diff --git a/test/e2e/clusterdeployment/clusterdeployment.go b/test/e2e/clusterdeployment/clusterdeployment.go index 4492f4dc0..0d2417356 100644 --- a/test/e2e/clusterdeployment/clusterdeployment.go +++ b/test/e2e/clusterdeployment/clusterdeployment.go @@ -46,6 +46,7 @@ type Template string const ( TemplateAWSStandaloneCP Template = "aws-standalone-cp" TemplateAWSHostedCP Template = "aws-hosted-cp" + TemplateAWSEKS Template = "aws-eks" TemplateAzureHostedCP Template = "azure-hosted-cp" TemplateAzureStandaloneCP Template = "azure-standalone-cp" TemplateVSphereStandaloneCP Template = "vsphere-standalone-cp" @@ -58,6 +59,9 @@ var awsStandaloneCPClusterDeploymentTemplateBytes []byte //go:embed resources/aws-hosted-cp.yaml.tpl var awsHostedCPClusterDeploymentTemplateBytes []byte +//go:embed resources/aws-eks.yaml.tpl +var awsEksClusterDeploymentTemplateBytes []byte + //go:embed resources/azure-standalone-cp.yaml.tpl var azureStandaloneCPClusterDeploymentTemplateBytes []byte @@ -126,6 +130,8 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured { EnvVarAWSSecurityGroupID, }) clusterDeploymentTemplateBytes = awsHostedCPClusterDeploymentTemplateBytes + case TemplateAWSEKS: + clusterDeploymentTemplateBytes = awsEksClusterDeploymentTemplateBytes case TemplateVSphereStandaloneCP: clusterDeploymentTemplateBytes = vsphereStandaloneCPClusterDeploymentTemplateBytes case TemplateVSphereHostedCP: diff --git a/test/e2e/clusterdeployment/providervalidator.go b/test/e2e/clusterdeployment/providervalidator.go index 7ca81d1d2..552ac4237 100644 --- a/test/e2e/clusterdeployment/providervalidator.go +++ b/test/e2e/clusterdeployment/providervalidator.go @@ -65,6 +65,15 @@ func NewProviderValidator(template Template, clusterName string, action Validati case TemplateAWSStandaloneCP, TemplateAWSHostedCP: resourcesToValidate["ccm"] = validateCCM resourceOrder = append(resourceOrder, "ccm") + case TemplateAWSEKS: + resourcesToValidate = map[string]resourceValidationFunc{ + "clusters": validateCluster, + "machines": validateMachines, + "aws-managed-control-planes": validateAWSManagedControlPlanes, + "csi-driver": validateCSIDriver, + "ccm": validateCCM, + } + resourceOrder = []string{"clusters", "machines", "aws-managed-control-planes", "csi-driver", "ccm"} case TemplateAzureStandaloneCP, TemplateVSphereStandaloneCP: delete(resourcesToValidate, "csi-driver") } @@ -72,9 +81,16 @@ func NewProviderValidator(template Template, clusterName string, action Validati resourcesToValidate = map[string]resourceValidationFunc{ "clusters": validateClusterDeleted, "machinedeployments": validateMachineDeploymentsDeleted, - "control-planes": validateK0sControlPlanesDeleted, } - resourceOrder = []string{"clusters", "machinedeployments", "control-planes"} + resourceOrder = []string{"clusters", "machinedeployments"} + switch template { + case TemplateAWSEKS: + resourcesToValidate["aws-managed-control-planes"] = validateAWSManagedControlPlanesDeleted + resourceOrder = append(resourceOrder, "aws-managed-control-planes") + default: + resourcesToValidate["control-planes"] = validateK0sControlPlanesDeleted + resourceOrder = append(resourceOrder, "control-planes") + } } return &ProviderValidator{ diff --git a/test/e2e/clusterdeployment/resources/aws-eks.yaml.tpl b/test/e2e/clusterdeployment/resources/aws-eks.yaml.tpl new file mode 100644 index 000000000..9e54499dc --- /dev/null +++ b/test/e2e/clusterdeployment/resources/aws-eks.yaml.tpl @@ -0,0 +1,13 @@ +apiVersion: hmc.mirantis.com/v1alpha1 +kind: ClusterDeployment +metadata: + name: ${CLUSTER_DEPLOYMENT_NAME} +spec: + template: aws-eks-0-0-3 + credential: ${AWS_CLUSTER_IDENTITY}-cred + config: + region: ${AWS_REGION} + workersNumber: ${WORKERS_NUMBER:=1} + publicIP: ${AWS_PUBLIC_IP:=true} + worker: + instanceType: ${AWS_INSTANCE_TYPE:=t3.small} diff --git a/test/e2e/clusterdeployment/validate_deleted.go b/test/e2e/clusterdeployment/validate_deleted.go index 1789ec879..55c1fa97d 100644 --- a/test/e2e/clusterdeployment/validate_deleted.go +++ b/test/e2e/clusterdeployment/validate_deleted.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "strings" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -69,17 +70,7 @@ func validateMachineDeploymentsDeleted(ctx context.Context, kc *kubeclient.KubeC if err != nil && !apierrors.IsNotFound(err) { return err } - - var mdNames []string - if len(machineDeployments) > 0 { - for _, md := range machineDeployments { - mdNames = append(mdNames, md.GetName()) - - return fmt.Errorf("machine deployments still exist: %s", mdNames) - } - } - - return nil + return validateObjectsRemoved("MachineDeployments", machineDeployments) } // validateK0sControlPlanesDeleted validates that all k0scontrolplanes have @@ -89,15 +80,26 @@ func validateK0sControlPlanesDeleted(ctx context.Context, kc *kubeclient.KubeCli if err != nil && !apierrors.IsNotFound(err) { return err } + return validateObjectsRemoved("K0sControlPlanes", controlPlanes) +} - var cpNames []string - if len(controlPlanes) > 0 { - for _, cp := range controlPlanes { - cpNames = append(cpNames, cp.GetName()) - - return fmt.Errorf("k0s control planes still exist: %s", cpNames) - } +// validateAWSManagedControlPlanesDeleted validates that all AWSManagedControlPlanes have +// been deleted. +func validateAWSManagedControlPlanesDeleted(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error { + controlPlanes, err := kc.ListAWSManagedControlPlanes(ctx, clusterName) + if err != nil && !apierrors.IsNotFound(err) { + return err } + return validateObjectsRemoved("AWSManagedControlPlane", controlPlanes) +} - return nil +func validateObjectsRemoved(kind string, objs []unstructured.Unstructured) error { + if len(objs) == 0 { + return nil + } + names := make([]string, len(objs)) + for _, cp := range objs { + names = append(names, cp.GetName()) + } + return fmt.Errorf("one or more %s still exist: %s", kind, strings.Join(names, ", ")) } diff --git a/test/e2e/clusterdeployment/validate_deployed.go b/test/e2e/clusterdeployment/validate_deployed.go index 7f750ecc1..6c6ad0f2f 100644 --- a/test/e2e/clusterdeployment/validate_deployed.go +++ b/test/e2e/clusterdeployment/validate_deployed.go @@ -16,6 +16,7 @@ package clusterdeployment import ( "context" + "errors" "fmt" "strings" @@ -27,7 +28,6 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/intstr" - "github.com/Mirantis/hmc/internal/utils/status" "github.com/Mirantis/hmc/test/e2e/kubeclient" "github.com/Mirantis/hmc/test/utils" ) @@ -104,38 +104,55 @@ func validateK0sControlPlanes(ctx context.Context, kc *kubeclient.KubeClient, cl return err } + var errs error for _, controlPlane := range controlPlanes { if err := utils.ValidateObjectNamePrefix(&controlPlane, clusterName); err != nil { - Fail(err.Error()) + errs = errors.Join(errs, err) + continue } - objKind, objName := status.ObjKindName(&controlPlane) - // k0s does not use the metav1.Condition type for status.conditions, // instead it uses a custom type so we can't use // ValidateConditionsTrue here, instead we'll check for "ready: true". - objStatus, found, err := unstructured.NestedFieldCopy(controlPlane.Object, "status") - if !found { - return fmt.Errorf("no status found for %s: %s", objKind, objName) - } - if err != nil { - return fmt.Errorf("failed to get status conditions for %s: %s: %w", objKind, objName, err) - } - - st, ok := objStatus.(map[string]any) - if !ok { - return fmt.Errorf("expected K0sControlPlane condition to be type map[string]any, got: %T", objStatus) - } + errs = errors.Join(errs, validateReadyStatus(controlPlane)) + } - if _, ok := st["ready"]; !ok { - return fmt.Errorf("%s %s has no 'ready' status", objKind, objName) - } + return errs +} - if v, ok := st["ready"].(bool); !ok || !v { - return fmt.Errorf("%s %s is not ready, status: %+v", objKind, objName, st) - } +func validateAWSManagedControlPlanes(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error { + controlPlanes, err := kc.ListAWSManagedControlPlanes(ctx, clusterName) + if err != nil { + return err + } + var errs error + for _, controlPlane := range controlPlanes { + errs = errors.Join(errs, validateReadyStatus(controlPlane)) } + return errs +} +// validateReadyStatus validates if the provided object has ready status +func validateReadyStatus(obj unstructured.Unstructured) error { + name := obj.GetName() + kind := obj.GetKind() + objStatus, found, err := unstructured.NestedFieldCopy(obj.Object, "status") + if !found { + return fmt.Errorf("no status found for %s: %s", kind, name) + } + if err != nil { + return fmt.Errorf("failed to get status conditions for %s: %s: %w", kind, name, err) + } + st, ok := objStatus.(map[string]any) + if !ok { + return fmt.Errorf("expected %s condition to be type map[string]any, got: %T", kind, objStatus) + } + if _, ok := st["ready"]; !ok { + return fmt.Errorf("%s %s has no 'ready' status", kind, name) + } + if v, ok := st["ready"].(bool); !ok || !v { + return fmt.Errorf("%s %s is not ready, status: %+v", kind, name, st) + } return nil } diff --git a/test/e2e/kubeclient/kubeclient.go b/test/e2e/kubeclient/kubeclient.go index 45e8fdb97..e5ad2b200 100644 --- a/test/e2e/kubeclient/kubeclient.go +++ b/test/e2e/kubeclient/kubeclient.go @@ -279,3 +279,15 @@ func (kc *KubeClient) ListK0sControlPlanes( Resource: "k0scontrolplanes", }, clusterName) } + +func (kc *KubeClient) ListAWSManagedControlPlanes( + ctx context.Context, clusterName string, +) ([]unstructured.Unstructured, error) { + GinkgoHelper() + + return kc.listResource(ctx, schema.GroupVersionResource{ + Group: "controlplane.cluster.x-k8s.io", + Version: "v1beta2", + Resource: "awsmanagedcontrolplanes", + }, clusterName) +} diff --git a/test/e2e/provider_aws_test.go b/test/e2e/provider_aws_test.go index 276689032..a8c7588be 100644 --- a/test/e2e/provider_aws_test.go +++ b/test/e2e/provider_aws_test.go @@ -186,4 +186,37 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order time.Second).Should(Succeed()) */ }) + + It("should work with an AWS provider (EKS)", func() { + templateBy(clusterdeployment.TemplateAWSEKS, "creating a ClusterDeployment") + cd := clusterdeployment.GetUnstructured(clusterdeployment.TemplateAWSEKS) + clusterName = cd.GetName() + + eksDeleteFunc := kc.CreateClusterDeployment(context.Background(), cd) + + templateBy(clusterdeployment.TemplateAWSEKS, "waiting for infrastructure to deploy successfully") + deploymentValidator := clusterdeployment.NewProviderValidator( + clusterdeployment.TemplateAWSEKS, + clusterName, + clusterdeployment.ValidationActionDeploy, + ) + + Eventually(func() error { + return deploymentValidator.Validate(context.Background(), kc) + }, 30*time.Minute, 10*time.Second).Should(Succeed()) + + if !noCleanup() { + templateBy(clusterdeployment.TemplateAWSEKS, "deleting the ClusterDeployment") + Expect(eksDeleteFunc()).NotTo(HaveOccurred()) + + deletionValidator := clusterdeployment.NewProviderValidator( + clusterdeployment.TemplateAWSEKS, + clusterName, + clusterdeployment.ValidationActionDelete, + ) + Eventually(func() error { + return deletionValidator.Validate(context.Background(), kc) + }, 20*time.Minute, 10*time.Second).Should(Succeed()) + } + }) })