From 25f54c2062b0134a01274b0a3d7019a1f0029cd2 Mon Sep 17 00:00:00 2001 From: Slava Lysunkin Date: Tue, 12 Nov 2024 15:25:34 -0600 Subject: [PATCH] E2E tests [DO NOT MERGE] --- .github/workflows/build_test.yml | 2 ++ .golangci.yml | 3 +- .../hmc/templates/rbac/controller/roles.yaml | 5 +++ test/e2e/e2e_suite_test.go | 6 ++-- test/e2e/kubeclient/kubeclient.go | 12 +++++++ test/e2e/managedcluster/managedcluster.go | 6 ++++ test/e2e/managedcluster/providervalidator.go | 10 +++++- .../resources/aws-eks-cp.yaml.tpl | 16 +++++++++ test/e2e/managedcluster/validate_deleted.go | 18 ++++++++++ test/e2e/managedcluster/validate_deployed.go | 28 ++++++++++++--- test/e2e/provider_aws_test.go | 36 +++++++++++++++++++ test/utils/utils.go | 27 ++++++++++++-- 12 files changed, 158 insertions(+), 11 deletions(-) create mode 100644 test/e2e/managedcluster/resources/aws-eks-cp.yaml.tpl diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 6f1844c7a..10f8cce6e 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -93,6 +93,7 @@ jobs: make helm-push controller-e2etest: + permissions: write-all name: E2E Controller runs-on: ubuntu-latest needs: build @@ -128,6 +129,7 @@ jobs: test/e2e/*.log provider-cloud-e2etest: + permissions: write-all name: E2E Cloud Providers runs-on: ubuntu-latest if: ${{ contains( github.event.pull_request.labels.*.name, 'test e2e') }} diff --git a/.golangci.yml b/.golangci.yml index ee80ea33b..57ad67fa4 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -273,8 +273,7 @@ linters-settings: disabled: true - name: max-control-nesting - name: max-public-structs - exclude: ["TEST"] - arguments: [5] + disabled: true # the api/* pkgs have lots of structs - name: modifies-parameter - name: modifies-value-receiver - name: nested-structs diff --git a/templates/provider/hmc/templates/rbac/controller/roles.yaml b/templates/provider/hmc/templates/rbac/controller/roles.yaml index 69206ee6b..f6d7c736d 100644 --- a/templates/provider/hmc/templates/rbac/controller/roles.yaml +++ b/templates/provider/hmc/templates/rbac/controller/roles.yaml @@ -186,6 +186,11 @@ rules: - clusterprofiles - clustersummaries verbs: {{ include "rbac.editorVerbs" . | nindent 4 }} +- apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - awsmanagedcontrolplanes + verbs: {{ include "rbac.viewerVerbs" . | nindent 4 }} - apiGroups: - hmc.mirantis.com resources: diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index e76a4c245..48aab8d11 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -217,8 +217,10 @@ func collectLogArtifacts(kc *kubeclient.KubeClient, clusterName string, provider "describe", "cluster", clusterName, "--namespace", internalutils.DefaultSystemNamespace, "--show-conditions=all") output, err := utils.Run(cmd) if err != nil { - utils.WarnError(fmt.Errorf("failed to get clusterctl log: %w", err)) - return + if !strings.Contains(err.Error(), "unable to verify clusterctl version") { + utils.WarnError(fmt.Errorf("failed to get clusterctl log: %w", err)) + return + } } err = os.WriteFile(filepath.Join("test/e2e", host+"-"+"clusterctl.log"), output, 0o644) if err != nil { diff --git a/test/e2e/kubeclient/kubeclient.go b/test/e2e/kubeclient/kubeclient.go index e3801e4e0..d5e328aaa 100644 --- a/test/e2e/kubeclient/kubeclient.go +++ b/test/e2e/kubeclient/kubeclient.go @@ -279,3 +279,15 @@ func (kc *KubeClient) ListK0sControlPlanes( Resource: "k0scontrolplanes", }, clusterName) } + +func (kc *KubeClient) ListAWSManagedControlPlanes( + ctx context.Context, clusterName string, +) ([]unstructured.Unstructured, error) { + GinkgoHelper() + + return kc.listResource(ctx, schema.GroupVersionResource{ + Group: "controlplane.cluster.x-k8s.io", + Version: "v1beta2", + Resource: "awsmanagedcontrolplanes", + }, clusterName) +} diff --git a/test/e2e/managedcluster/managedcluster.go b/test/e2e/managedcluster/managedcluster.go index 72efa96f8..e8be90e15 100644 --- a/test/e2e/managedcluster/managedcluster.go +++ b/test/e2e/managedcluster/managedcluster.go @@ -50,6 +50,7 @@ const ( TemplateAzureStandaloneCP Template = "azure-standalone-cp" TemplateVSphereStandaloneCP Template = "vsphere-standalone-cp" TemplateVSphereHostedCP Template = "vsphere-hosted-cp" + TemplateEKSCP Template = "aws-eks-cp" ) //go:embed resources/aws-standalone-cp.yaml.tpl @@ -70,6 +71,9 @@ var vsphereStandaloneCPManagedClusterTemplateBytes []byte //go:embed resources/vsphere-hosted-cp.yaml.tpl var vsphereHostedCPManagedClusterTemplateBytes []byte +//go:embed resources/aws-eks-cp.yaml.tpl +var eksCPManagedClusterTemplateBytes []byte + func FilterAllProviders() []string { return []string{ utils.HMCControllerLabel, @@ -134,6 +138,8 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured { managedClusterTemplateBytes = azureHostedCPManagedClusterTemplateBytes case TemplateAzureStandaloneCP: managedClusterTemplateBytes = azureStandaloneCPManagedClusterTemplateBytes + case TemplateEKSCP: + managedClusterTemplateBytes = eksCPManagedClusterTemplateBytes default: Fail(fmt.Sprintf("Unsupported template: %s", templateName)) } diff --git a/test/e2e/managedcluster/providervalidator.go b/test/e2e/managedcluster/providervalidator.go index 4df0fa84b..3ccb5a074 100644 --- a/test/e2e/managedcluster/providervalidator.go +++ b/test/e2e/managedcluster/providervalidator.go @@ -65,14 +65,22 @@ func NewProviderValidator(template Template, clusterName string, action Validati case TemplateAWSStandaloneCP, TemplateAWSHostedCP: resourcesToValidate["ccm"] = validateCCM resourceOrder = append(resourceOrder, "ccm") + case TemplateEKSCP: + resourcesToValidate["control-planes"] = validateAWSManagedControlPlanes + delete(resourcesToValidate, "csi-driver") case TemplateAzureStandaloneCP, TemplateVSphereStandaloneCP: delete(resourcesToValidate, "csi-driver") } } else { + validateCPDeletedFunc := validateK0sControlPlanesDeleted + if template == TemplateEKSCP { + validateCPDeletedFunc = validateAWSManagedControlPlanesDeleted + } + resourcesToValidate = map[string]resourceValidationFunc{ "clusters": validateClusterDeleted, "machinedeployments": validateMachineDeploymentsDeleted, - "control-planes": validateK0sControlPlanesDeleted, + "control-planes": validateCPDeletedFunc, } resourceOrder = []string{"clusters", "machinedeployments", "control-planes"} } diff --git a/test/e2e/managedcluster/resources/aws-eks-cp.yaml.tpl b/test/e2e/managedcluster/resources/aws-eks-cp.yaml.tpl new file mode 100644 index 000000000..1b0cc50a6 --- /dev/null +++ b/test/e2e/managedcluster/resources/aws-eks-cp.yaml.tpl @@ -0,0 +1,16 @@ +apiVersion: hmc.mirantis.com/v1alpha1 +kind: ManagedCluster +metadata: + name: ${MANAGED_CLUSTER_NAME}-eks +spec: + template: aws-eks-0-0-2 + credential: ${AWS_CLUSTER_IDENTITY}-cred + config: + region: ${AWS_REGION} + workersNumber: ${WORKERS_NUMBER:=1} + clusterIdentity: + name: ${AWS_CLUSTER_IDENTITY}-cred + namespace: ${NAMESPACE} + publicIP: ${AWS_PUBLIC_IP:=true} + worker: + instanceType: ${AWS_INSTANCE_TYPE:=t3.small} diff --git a/test/e2e/managedcluster/validate_deleted.go b/test/e2e/managedcluster/validate_deleted.go index e09d4c254..4272f9fe5 100644 --- a/test/e2e/managedcluster/validate_deleted.go +++ b/test/e2e/managedcluster/validate_deleted.go @@ -101,3 +101,21 @@ func validateK0sControlPlanesDeleted(ctx context.Context, kc *kubeclient.KubeCli return nil } + +func validateAWSManagedControlPlanesDeleted(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error { + controlPlanes, err := kc.ListAWSManagedControlPlanes(ctx, clusterName) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + + var cpNames []string + if len(controlPlanes) > 0 { + for _, cp := range controlPlanes { + cpNames = append(cpNames, cp.GetName()) + + return fmt.Errorf("AWS Managed control planes still exist: %s", cpNames) + } + } + + return nil +} diff --git a/test/e2e/managedcluster/validate_deployed.go b/test/e2e/managedcluster/validate_deployed.go index bae823f75..7a600ff3f 100644 --- a/test/e2e/managedcluster/validate_deployed.go +++ b/test/e2e/managedcluster/validate_deployed.go @@ -55,7 +55,7 @@ func validateCluster(ctx context.Context, kc *kubeclient.KubeClient, clusterName Fail(err.Error()) } - return utils.ValidateConditionsTrue(cluster) + return utils.NewConditionsValidator().IfTrue(cluster) } func validateMachines(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error { @@ -79,7 +79,7 @@ func validateMachines(ctx context.Context, kc *kubeclient.KubeClient, clusterNam Fail(err.Error()) } - if err := utils.ValidateConditionsTrue(&md); err != nil { + if err := utils.NewConditionsValidator().IfTrue(&md); err != nil { return err } } @@ -90,7 +90,7 @@ func validateMachines(ctx context.Context, kc *kubeclient.KubeClient, clusterNam Fail(err.Error()) } - if err := utils.ValidateConditionsTrue(&machine); err != nil { + if err := utils.NewConditionsValidator().IfTrue(&machine); err != nil { return err } } @@ -113,7 +113,7 @@ func validateK0sControlPlanes(ctx context.Context, kc *kubeclient.KubeClient, cl // k0s does not use the metav1.Condition type for status.conditions, // instead it uses a custom type so we can't use - // ValidateConditionsTrue here, instead we'll check for "ready: true". + // ordinary conditions validation here, instead we'll check for "ready: true". objStatus, found, err := unstructured.NestedFieldCopy(controlPlane.Object, "status") if !found { return fmt.Errorf("no status found for %s: %s", objKind, objName) @@ -139,6 +139,26 @@ func validateK0sControlPlanes(ctx context.Context, kc *kubeclient.KubeClient, cl return nil } +func validateAWSManagedControlPlanes(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error { + controlPlanes, err := kc.ListAWSManagedControlPlanes(ctx, clusterName) + if err != nil { + return err + } + + for _, controlPlane := range controlPlanes { + if err := utils.ValidateObjectNamePrefix(&controlPlane, clusterName); err != nil { + Fail(err.Error()) + } + + // EKSControlPlaneCreating condition very often has READY=False, SEVERITY=Info and REASON=created (this is fine). + if err := utils.NewConditionsValidator(utils.WithExcluded([]string{"EKSControlPlaneCreating"})).IfTrue(&controlPlane); err != nil { + return err + } + } + + return nil +} + // validateCSIDriver validates that the provider CSI driver is functioning // by creating a PVC and verifying it enters "Bound" status. func validateCSIDriver(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error { diff --git a/test/e2e/provider_aws_test.go b/test/e2e/provider_aws_test.go index 6614698b4..fa39a85b3 100644 --- a/test/e2e/provider_aws_test.go +++ b/test/e2e/provider_aws_test.go @@ -186,4 +186,40 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order time.Second).Should(Succeed()) */ }) + + It("should work with an EKS provider", func() { + // Deploy a standalone cluster and verify it is running/ready. + GinkgoT().Setenv(managedcluster.EnvVarAWSInstanceType, "t3.small") + + templateBy(managedcluster.TemplateEKSCP, "creating a ManagedCluster for EKS") + sd := managedcluster.GetUnstructured(managedcluster.TemplateEKSCP) + clusterName = sd.GetName() + + standaloneDeleteFunc = kc.CreateManagedCluster(context.Background(), sd) + + templateBy(managedcluster.TemplateEKSCP, "waiting for infrastructure to deploy successfully") + deploymentValidator := managedcluster.NewProviderValidator( + managedcluster.TemplateEKSCP, + clusterName, + managedcluster.ValidationActionDeploy, + ) + + Eventually(func() error { + return deploymentValidator.Validate(context.Background(), kc) + }).WithTimeout(60 * time.Minute).WithPolling(30 * time.Second).Should(Succeed()) + + // --- clean up --- + templateBy(managedcluster.TemplateAWSStandaloneCP, "deleting the ManagedCluster for EKS") + Expect(standaloneDeleteFunc()).NotTo(HaveOccurred()) + + deletionValidator := managedcluster.NewProviderValidator( + managedcluster.TemplateAWSStandaloneCP, + clusterName, + managedcluster.ValidationActionDelete, + ) + Eventually(func() error { + return deletionValidator.Validate(context.Background(), kc) + }).WithTimeout(15 * time.Minute).WithPolling(10 * + time.Second).Should(Succeed()) + }) }) diff --git a/test/utils/utils.go b/test/utils/utils.go index 56ce5ee33..792125589 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -24,6 +24,7 @@ import ( . "github.com/onsi/ginkgo/v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/utils/strings/slices" "github.com/Mirantis/hmc/internal/utils/status" ) @@ -111,10 +112,28 @@ func GetProjectDir() (string, error) { return wd, nil } -// ValidateConditionsTrue iterates over the conditions of the given +type ConditionsValidator struct { + excludedConditions []string +} + +func NewConditionsValidator(options ...func(*ConditionsValidator)) *ConditionsValidator { + cv := &ConditionsValidator{} + for _, o := range options { + o(cv) + } + return cv +} + +func WithExcluded(excludedConditions []string) func(*ConditionsValidator) { + return func(cv *ConditionsValidator) { + cv.excludedConditions = excludedConditions + } +} + +// IfTrue iterates over the conditions of the given // unstructured object and returns an error if any of the conditions are not // true. Conditions are expected to be of type metav1.Condition. -func ValidateConditionsTrue(unstrObj *unstructured.Unstructured) error { +func (cv *ConditionsValidator) IfTrue(unstrObj *unstructured.Unstructured) error { objKind, objName := status.ObjKindName(unstrObj) conditions, err := status.ConditionsFromUnstructured(unstrObj) @@ -129,6 +148,10 @@ func ValidateConditionsTrue(unstrObj *unstructured.Unstructured) error { continue } + if slices.Contains(cv.excludedConditions, c.Type) { + continue + } + errs = errors.Join(errors.New(ConvertConditionsToString(c)), errs) }