diff --git a/Makefile b/Makefile index fed6bd818..551e70eb5 100644 --- a/Makefile +++ b/Makefile @@ -107,7 +107,7 @@ tidy: .PHONY: test test: generate-all fmt vet envtest tidy external-crd ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e/scenarios) -coverprofile cover.out # E2E_CONFIG_B64 contains the configuration for e2e testing. E2E_CONFIG_B64 ?= "" @@ -120,7 +120,7 @@ test-e2e: cli-install ginkgo_label_flag="-ginkgo.label-filter=$$GINKGO_LABEL_FILTER"; \ fi; \ KIND_CLUSTER_NAME="hmc-test" KIND_VERSION=$(KIND_VERSION) E2E_CONFIG_B64=$(E2E_CONFIG_B64) \ - go test ./test/e2e/ -v -ginkgo.v -ginkgo.timeout=3h -timeout=3h $$ginkgo_label_flag + go test ./test/e2e/scenarios/ -v -ginkgo.v -ginkgo.timeout=3h -timeout=3h $$ginkgo_label_flag .PHONY: lint lint: golangci-lint ## Run golangci-lint linter & yamllint diff --git a/test/e2e/config/config.go b/test/e2e/config/config.go index fd3065f4c..9770a2c5b 100644 --- a/test/e2e/config/config.go +++ b/test/e2e/config/config.go @@ -22,6 +22,9 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "gopkg.in/yaml.v3" + + hmc "github.com/Mirantis/hmc/api/v1alpha1" + "github.com/Mirantis/hmc/test/e2e/templates" ) type TestingProvider string @@ -72,6 +75,27 @@ func Parse() error { return nil } +func (c *ClusterTestingConfig) SetDefaults(clusterTemplates map[string][]hmc.AvailableUpgrade, templateType templates.Type) error { + var err error + if !c.Upgrade { + if c.Template == "" { + c.Template, err = templates.FindTemplate(clusterTemplates, templateType) + if err != nil { + return err + } + } + return templates.ValidateTemplate(clusterTemplates, c.Template) + } + if c.Template != "" && c.UpgradeTemplate != "" { + return templates.ValidateUpgradeSequence(clusterTemplates, c.Template, c.UpgradeTemplate) + } + c.Template, c.UpgradeTemplate, err = templates.FindTemplatesToUpgrade(clusterTemplates, templateType, c.Template) + if err != nil { + return err + } + return nil +} + func (c *ProviderTestingConfig) String() string { prettyConfig, err := yaml.Marshal(c) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/managedcluster/constants.go b/test/e2e/managedcluster/constants.go index 4f18a7832..d6809f071 100644 --- a/test/e2e/managedcluster/constants.go +++ b/test/e2e/managedcluster/constants.go @@ -16,10 +16,11 @@ package managedcluster const ( // Common - EnvVarManagedClusterName = "MANAGED_CLUSTER_NAME" - EnvVarControlPlaneNumber = "CONTROL_PLANE_NUMBER" - EnvVarWorkerNumber = "WORKER_NUMBER" - EnvVarNamespace = "NAMESPACE" + EnvVarManagedClusterName = "MANAGED_CLUSTER_NAME" + EnvVarManagedClusterTemplate = "MANAGED_CLUSTER_TEMPLATE" + EnvVarControlPlaneNumber = "CONTROL_PLANE_NUMBER" + EnvVarWorkerNumber = "WORKER_NUMBER" + EnvVarNamespace = "NAMESPACE" // EnvVarNoCleanup disables After* cleanup in provider specs to allow for // debugging of test failures. EnvVarNoCleanup = "NO_CLEANUP" diff --git a/test/e2e/managedcluster/managedcluster.go b/test/e2e/managedcluster/managedcluster.go index 8600140da..35e5f2e15 100644 --- a/test/e2e/managedcluster/managedcluster.go +++ b/test/e2e/managedcluster/managedcluster.go @@ -27,6 +27,7 @@ import ( "gopkg.in/yaml.v3" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "github.com/Mirantis/hmc/test/e2e/templates" "github.com/Mirantis/hmc/test/utils" ) @@ -43,17 +44,6 @@ const ( Namespace = "default" ) -type Template string - -const ( - TemplateAWSStandaloneCP Template = "aws-standalone-cp" - TemplateAWSHostedCP Template = "aws-hosted-cp" - TemplateAzureHostedCP Template = "azure-hosted-cp" - TemplateAzureStandaloneCP Template = "azure-standalone-cp" - TemplateVSphereStandaloneCP Template = "vsphere-standalone-cp" - TemplateVSphereHostedCP Template = "vsphere-hosted-cp" -) - //go:embed resources/aws-standalone-cp.yaml.tpl var awsStandaloneCPManagedClusterTemplateBytes []byte @@ -86,7 +76,7 @@ func GetProviderLabel(provider ProviderType) string { return fmt.Sprintf("%s=%s", providerLabel, provider) } -func setClusterName(templateName Template) { +func setClusterName(templateType templates.Type) { var generatedName string mcName := os.Getenv(EnvVarManagedClusterName) @@ -94,30 +84,35 @@ func setClusterName(templateName Template) { mcName = "e2e-test-" + uuid.New().String()[:8] } - providerName := strings.Split(string(templateName), "-")[0] + providerName := strings.Split(string(templateType), "-")[0] // Append the provider name to the cluster name to ensure uniqueness between // different deployed ManagedClusters. generatedName = fmt.Sprintf("%s-%s", mcName, providerName) - if strings.Contains(string(templateName), "hosted") { + if strings.Contains(string(templateType), "hosted") { generatedName = fmt.Sprintf("%s-%s", mcName, "hosted") } GinkgoT().Setenv(EnvVarManagedClusterName, generatedName) } +func setTemplate(templateName string) { + GinkgoT().Setenv(EnvVarManagedClusterTemplate, templateName) +} + // GetUnstructured returns an unstructured ManagedCluster object based on the // provider and template. -func GetUnstructured(templateName Template) *unstructured.Unstructured { +func GetUnstructured(templateType templates.Type, templateName string) *unstructured.Unstructured { GinkgoHelper() - setClusterName(templateName) + setClusterName(templateType) + setTemplate(templateName) var managedClusterTemplateBytes []byte - switch templateName { - case TemplateAWSStandaloneCP: + switch templateType { + case templates.TemplateAWSStandaloneCP: managedClusterTemplateBytes = awsStandaloneCPManagedClusterTemplateBytes - case TemplateAWSHostedCP: + case templates.TemplateAWSHostedCP: // Validate environment vars that do not have defaults are populated. // We perform this validation here instead of within a Before block // since we populate the vars from standalone prior to this step. @@ -128,16 +123,16 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured { EnvVarAWSSecurityGroupID, }) managedClusterTemplateBytes = awsHostedCPManagedClusterTemplateBytes - case TemplateVSphereStandaloneCP: + case templates.TemplateVSphereStandaloneCP: managedClusterTemplateBytes = vsphereStandaloneCPManagedClusterTemplateBytes - case TemplateVSphereHostedCP: + case templates.TemplateVSphereHostedCP: managedClusterTemplateBytes = vsphereHostedCPManagedClusterTemplateBytes - case TemplateAzureHostedCP: + case templates.TemplateAzureHostedCP: managedClusterTemplateBytes = azureHostedCPManagedClusterTemplateBytes - case TemplateAzureStandaloneCP: + case templates.TemplateAzureStandaloneCP: managedClusterTemplateBytes = azureStandaloneCPManagedClusterTemplateBytes default: - Fail(fmt.Sprintf("Unsupported template: %s", templateName)) + Fail(fmt.Sprintf("Unsupported template type: %s", templateType)) } managedClusterConfigBytes, err := envsubst.Bytes(managedClusterTemplateBytes) diff --git a/test/e2e/managedcluster/providervalidator.go b/test/e2e/managedcluster/providervalidator.go index 3e658087d..0af08a534 100644 --- a/test/e2e/managedcluster/providervalidator.go +++ b/test/e2e/managedcluster/providervalidator.go @@ -21,14 +21,15 @@ import ( . "github.com/onsi/ginkgo/v2" "github.com/Mirantis/hmc/test/e2e/kubeclient" + "github.com/Mirantis/hmc/test/e2e/templates" ) // ProviderValidator is a struct that contains the necessary information to // validate a provider's resources. Some providers do not support all of the // resources that can potentially be validated. type ProviderValidator struct { - // Template is the name of the template being validated. - template Template + // Template is the type of the template being validated. + template templates.Type // Namespace is the namespace of the cluster to validate. namespace string // ClusterName is the name of the cluster to validate. @@ -48,7 +49,7 @@ const ( ValidationActionDelete ValidationAction = "delete" ) -func NewProviderValidator(template Template, namespace, clusterName string, action ValidationAction) *ProviderValidator { +func NewProviderValidator(templateType templates.Type, namespace, clusterName string, action ValidationAction) *ProviderValidator { var ( resourcesToValidate map[string]resourceValidationFunc resourceOrder []string @@ -63,11 +64,11 @@ func NewProviderValidator(template Template, namespace, clusterName string, acti } resourceOrder = []string{"clusters", "machines", "control-planes", "csi-driver"} - switch template { - case TemplateAWSStandaloneCP, TemplateAWSHostedCP: + switch templateType { + case templates.TemplateAWSStandaloneCP, templates.TemplateAWSHostedCP: resourcesToValidate["ccm"] = validateCCM resourceOrder = append(resourceOrder, "ccm") - case TemplateAzureStandaloneCP, TemplateVSphereStandaloneCP: + case templates.TemplateAzureStandaloneCP, templates.TemplateVSphereStandaloneCP: delete(resourcesToValidate, "csi-driver") } } else { @@ -80,7 +81,7 @@ func NewProviderValidator(template Template, namespace, clusterName string, acti } return &ProviderValidator{ - template: template, + template: templateType, namespace: namespace, clusterName: clusterName, resourcesToValidate: resourcesToValidate, diff --git a/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl b/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl index b8758c9ab..b8108cce9 100644 --- a/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl @@ -3,7 +3,7 @@ kind: ManagedCluster metadata: name: ${MANAGED_CLUSTER_NAME} spec: - template: aws-hosted-cp-0-0-2 + template: ${MANAGED_CLUSTER_TEMPLATE} credential: ${AWS_CLUSTER_IDENTITY}-cred config: clusterIdentity: diff --git a/test/e2e/managedcluster/resources/aws-standalone-cp.yaml.tpl b/test/e2e/managedcluster/resources/aws-standalone-cp.yaml.tpl index 8d2ceab31..ed09f8080 100644 --- a/test/e2e/managedcluster/resources/aws-standalone-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/aws-standalone-cp.yaml.tpl @@ -3,7 +3,7 @@ kind: ManagedCluster metadata: name: ${MANAGED_CLUSTER_NAME} spec: - template: aws-standalone-cp-0-0-2 + template: ${MANAGED_CLUSTER_TEMPLATE} credential: ${AWS_CLUSTER_IDENTITY}-cred config: clusterIdentity: diff --git a/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl b/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl index 7cbe61b71..5474ea82b 100644 --- a/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl @@ -4,7 +4,7 @@ metadata: name: ${MANAGED_CLUSTER_NAME} namespace: ${NAMESPACE} spec: - template: azure-hosted-cp-0-0-2 + template: ${MANAGED_CLUSTER_TEMPLATE} credential: ${AZURE_CLUSTER_IDENTITY}-cred config: location: "${AZURE_REGION}" diff --git a/test/e2e/managedcluster/resources/azure-standalone-cp.yaml.tpl b/test/e2e/managedcluster/resources/azure-standalone-cp.yaml.tpl index 7f1083a83..a68891ce0 100644 --- a/test/e2e/managedcluster/resources/azure-standalone-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/azure-standalone-cp.yaml.tpl @@ -4,7 +4,7 @@ metadata: name: ${MANAGED_CLUSTER_NAME} namespace: ${NAMESPACE} spec: - template: azure-standalone-cp-0-0-2 + template: ${MANAGED_CLUSTER_TEMPLATE} credential: ${AZURE_CLUSTER_IDENTITY}-cred config: controlPlaneNumber: 1 diff --git a/test/e2e/managedcluster/resources/vsphere-hosted-cp.yaml.tpl b/test/e2e/managedcluster/resources/vsphere-hosted-cp.yaml.tpl index 03a201cce..42b5efc7c 100644 --- a/test/e2e/managedcluster/resources/vsphere-hosted-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/vsphere-hosted-cp.yaml.tpl @@ -3,7 +3,7 @@ kind: ManagedCluster metadata: name: ${MANAGED_CLUSTER_NAME} spec: - template: vsphere-hosted-cp-0-0-2 + template: ${MANAGED_CLUSTER_TEMPLATE} credential: ${VSPHERE_CLUSTER_IDENTITY}-cred config: controlPlaneNumber: ${CONTROL_PLANE_NUMBER:=1} diff --git a/test/e2e/managedcluster/resources/vsphere-standalone-cp.yaml.tpl b/test/e2e/managedcluster/resources/vsphere-standalone-cp.yaml.tpl index 47c7dc538..95a77c9e1 100644 --- a/test/e2e/managedcluster/resources/vsphere-standalone-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/vsphere-standalone-cp.yaml.tpl @@ -3,7 +3,7 @@ kind: ManagedCluster metadata: name: ${MANAGED_CLUSTER_NAME} spec: - template: vsphere-standalone-cp-0-0-2 + template: ${MANAGED_CLUSTER_TEMPLATE} credential: ${VSPHERE_CLUSTER_IDENTITY}-cred config: controlPlaneNumber: ${CONTROL_PLANE_NUMBER:=1} diff --git a/test/e2e/controller_test.go b/test/e2e/scenarios/controller_test.go similarity index 98% rename from test/e2e/controller_test.go rename to test/e2e/scenarios/controller_test.go index 7b3897a0b..05c18daf9 100644 --- a/test/e2e/controller_test.go +++ b/test/e2e/scenarios/controller_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package e2e +package scenarios import ( . "github.com/onsi/ginkgo/v2" diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/scenarios/e2e_suite_test.go similarity index 97% rename from test/e2e/e2e_suite_test.go rename to test/e2e/scenarios/e2e_suite_test.go index f3a13f6d2..a7364aed0 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/scenarios/e2e_suite_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package e2e +package scenarios import ( "bufio" @@ -34,6 +34,7 @@ import ( hmc "github.com/Mirantis/hmc/api/v1alpha1" internalutils "github.com/Mirantis/hmc/internal/utils" + "github.com/Mirantis/hmc/test/e2e/config" "github.com/Mirantis/hmc/test/e2e/kubeclient" "github.com/Mirantis/hmc/test/e2e/managedcluster" "github.com/Mirantis/hmc/test/e2e/templates" @@ -50,13 +51,16 @@ func TestE2E(t *testing.T) { var clusterTemplates map[string][]hmc.AvailableUpgrade var _ = BeforeSuite(func() { + err := config.Parse() + Expect(err).NotTo(HaveOccurred()) + GinkgoT().Setenv(managedcluster.EnvVarNamespace, internalutils.DefaultSystemNamespace) ctx := context.Background() By("building and deploying the controller-manager") cmd := exec.Command("make", "kind-deploy") - _, err := utils.Run(cmd) + _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) cmd = exec.Command("make", "test-apply") _, err = utils.Run(cmd) @@ -74,7 +78,7 @@ var _ = BeforeSuite(func() { }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) By(fmt.Sprintf("applying access rules for ClusterTemplates in %s namespace", managedcluster.Namespace)) - clusterTemplates = templates.ApplyClusterTemplateAccessRules(ctx, kc.CrClient) + clusterTemplates = templates.ApplyClusterTemplateAccessRules(ctx, kc.CrClient, managedcluster.Namespace) }) var _ = AfterSuite(func() { @@ -153,7 +157,7 @@ func validateController(kc *kubeclient.KubeClient, labelSelector, name string) e // templateBy wraps a Ginkgo By with a block describing the template being // tested. -func templateBy(t managedcluster.Template, description string) { +func templateBy(t templates.Type, description string) { GinkgoHelper() By(fmt.Sprintf("[%s] %s", t, description)) } diff --git a/test/e2e/provider_aws_test.go b/test/e2e/scenarios/provider_aws_test.go similarity index 78% rename from test/e2e/provider_aws_test.go rename to test/e2e/scenarios/provider_aws_test.go index 230c89493..c454ffea1 100644 --- a/test/e2e/provider_aws_test.go +++ b/test/e2e/scenarios/provider_aws_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package e2e +package scenarios import ( "context" @@ -25,6 +25,7 @@ import ( . "github.com/onsi/gomega" internalutils "github.com/Mirantis/hmc/internal/utils" + "github.com/Mirantis/hmc/test/e2e/config" "github.com/Mirantis/hmc/test/e2e/kubeclient" "github.com/Mirantis/hmc/test/e2e/managedcluster" "github.com/Mirantis/hmc/test/e2e/managedcluster/aws" @@ -43,9 +44,23 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order hostedDeleteFunc func() error kubecfgDeleteFunc func() error clusterName string + + testingConfig config.ProviderTestingConfig ) BeforeAll(func() { + By("get testing configuration") + testingConfig = config.Config[config.TestingProviderAWS] + + By("set defaults and validate testing configuration") + err := testingConfig.Standalone.SetDefaults(clusterTemplates, templates.TemplateAWSStandaloneCP) + Expect(err).NotTo(HaveOccurred()) + + err = testingConfig.Hosted.SetDefaults(clusterTemplates, templates.TemplateAWSHostedCP) + Expect(err).NotTo(HaveOccurred()) + + _, _ = fmt.Fprintf(GinkgoWriter, "Final AWS testing configuration:\n%s\n", testingConfig.String()) + By("providing cluster identity") kc = kubeclient.NewFromLocal(internalutils.DefaultSystemNamespace) ci := clusteridentity.New(kc, managedcluster.ProviderAWS, managedcluster.Namespace) @@ -81,15 +96,16 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order // hosting the hosted cluster. GinkgoT().Setenv(managedcluster.EnvVarAWSInstanceType, "t3.xlarge") - templateBy(managedcluster.TemplateAWSStandaloneCP, "creating a ManagedCluster") - sd := managedcluster.GetUnstructured(managedcluster.TemplateAWSStandaloneCP) + templateBy(templates.TemplateAWSStandaloneCP, fmt.Sprintf("creating a ManagedCluster with %s template", testingConfig.Standalone.Template)) + sd := managedcluster.GetUnstructured(templates.TemplateAWSStandaloneCP, testingConfig.Standalone.Template) + clusterName = sd.GetName() standaloneDeleteFunc = kc.CreateManagedCluster(context.Background(), sd, managedcluster.Namespace) - templateBy(managedcluster.TemplateAWSStandaloneCP, "waiting for infrastructure to deploy successfully") + templateBy(templates.TemplateAWSStandaloneCP, "waiting for infrastructure to deploy successfully") deploymentValidator := managedcluster.NewProviderValidator( - managedcluster.TemplateAWSStandaloneCP, + templates.TemplateAWSStandaloneCP, managedcluster.Namespace, clusterName, managedcluster.ValidationActionDeploy, @@ -99,7 +115,7 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order return deploymentValidator.Validate(context.Background(), kc) }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) - templateBy(managedcluster.TemplateAWSHostedCP, "installing controller and templates on standalone cluster") + templateBy(templates.TemplateAWSHostedCP, "installing controller and templates on standalone cluster") // Download the KUBECONFIG for the standalone cluster and load it // so we can call Make targets against this cluster. @@ -115,21 +131,21 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order Expect(err).NotTo(HaveOccurred()) Expect(os.Unsetenv("KUBECONFIG")).To(Succeed()) - templateBy(managedcluster.TemplateAWSHostedCP, "validating that the controller is ready") + templateBy(templates.TemplateAWSHostedCP, "validating that the controller is ready") standaloneClient = kc.NewFromCluster(context.Background(), managedcluster.Namespace, clusterName) Eventually(func() error { err := verifyControllersUp(standaloneClient) if err != nil { _, _ = fmt.Fprintf( GinkgoWriter, "[%s] controller validation failed: %v\n", - string(managedcluster.TemplateAWSHostedCP), err) + string(templates.TemplateAWSHostedCP), err) return err } return nil }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) By(fmt.Sprintf("applying access rules for ClusterTemplates in %s namespace", managedcluster.Namespace)) - templates.ApplyClusterTemplateAccessRules(ctx, standaloneClient.CrClient) + templates.ApplyClusterTemplateAccessRules(ctx, standaloneClient.CrClient, managedcluster.Namespace) // Ensure AWS credentials are set in the standalone cluster. clusteridentity.New(standaloneClient, managedcluster.ProviderAWS, managedcluster.Namespace) @@ -138,20 +154,20 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order // cluster. aws.PopulateHostedTemplateVars(context.Background(), kc, clusterName) - templateBy(managedcluster.TemplateAWSHostedCP, "creating a ManagedCluster") - hd := managedcluster.GetUnstructured(managedcluster.TemplateAWSHostedCP) + templateBy(templates.TemplateAWSHostedCP, fmt.Sprintf("creating a ManagedCluster with %s template", testingConfig.Hosted.Template)) + hd := managedcluster.GetUnstructured(templates.TemplateAWSHostedCP, testingConfig.Hosted.Template) hdName := hd.GetName() // Deploy the hosted cluster on top of the standalone cluster. hostedDeleteFunc = standaloneClient.CreateManagedCluster(context.Background(), hd, managedcluster.Namespace) - templateBy(managedcluster.TemplateAWSHostedCP, "Patching AWSCluster to ready") + templateBy(templates.TemplateAWSHostedCP, "Patching AWSCluster to ready") managedcluster.PatchHostedClusterReady(standaloneClient, managedcluster.ProviderAWS, managedcluster.Namespace, hdName) // Verify the hosted cluster is running/ready. - templateBy(managedcluster.TemplateAWSHostedCP, "waiting for infrastructure to deploy successfully") + templateBy(templates.TemplateAWSHostedCP, "waiting for infrastructure to deploy successfully") deploymentValidator = managedcluster.NewProviderValidator( - managedcluster.TemplateAWSHostedCP, + templates.TemplateAWSHostedCP, managedcluster.Namespace, hdName, managedcluster.ValidationActionDeploy, @@ -161,12 +177,12 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) // Delete the hosted ManagedCluster and verify it is removed. - templateBy(managedcluster.TemplateAWSHostedCP, "deleting the ManagedCluster") + templateBy(templates.TemplateAWSHostedCP, "deleting the ManagedCluster") err = hostedDeleteFunc() Expect(err).NotTo(HaveOccurred()) deletionValidator := managedcluster.NewProviderValidator( - managedcluster.TemplateAWSHostedCP, + templates.TemplateAWSHostedCP, managedcluster.Namespace, hdName, managedcluster.ValidationActionDelete, diff --git a/test/e2e/provider_azure_test.go b/test/e2e/scenarios/provider_azure_test.go similarity index 78% rename from test/e2e/provider_azure_test.go rename to test/e2e/scenarios/provider_azure_test.go index 2e7d0b52e..b7b0a5a46 100644 --- a/test/e2e/provider_azure_test.go +++ b/test/e2e/scenarios/provider_azure_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package e2e +package scenarios import ( "context" @@ -25,6 +25,7 @@ import ( . "github.com/onsi/gomega" internalutils "github.com/Mirantis/hmc/internal/utils" + "github.com/Mirantis/hmc/test/e2e/config" "github.com/Mirantis/hmc/test/e2e/kubeclient" "github.com/Mirantis/hmc/test/e2e/managedcluster" "github.com/Mirantis/hmc/test/e2e/managedcluster/azure" @@ -44,9 +45,23 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or kubecfgDeleteFunc func() error hostedKubecfgDeleteFunc func() error sdName string + + testingConfig config.ProviderTestingConfig ) BeforeAll(func() { + By("get testing configuration") + testingConfig = config.Config[config.TestingProviderAzure] + + By("set defaults and validate testing configuration") + err := testingConfig.Standalone.SetDefaults(clusterTemplates, templates.TemplateAzureStandaloneCP) + Expect(err).NotTo(HaveOccurred()) + + err = testingConfig.Hosted.SetDefaults(clusterTemplates, templates.TemplateAzureHostedCP) + Expect(err).NotTo(HaveOccurred()) + + _, _ = fmt.Fprintf(GinkgoWriter, "Final Azure testing configuration:\n%s\n", testingConfig.String()) + By("ensuring Azure credentials are set") kc = kubeclient.NewFromLocal(internalutils.DefaultSystemNamespace) ci := clusteridentity.New(kc, managedcluster.ProviderAzure, managedcluster.Namespace) @@ -81,21 +96,21 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or }) It("should work with an Azure provider", func() { - templateBy(managedcluster.TemplateAzureStandaloneCP, "creating a ManagedCluster") - sd := managedcluster.GetUnstructured(managedcluster.TemplateAzureStandaloneCP) + templateBy(templates.TemplateAzureStandaloneCP, fmt.Sprintf("creating a ManagedCluster with %s template", testingConfig.Standalone.Template)) + sd := managedcluster.GetUnstructured(templates.TemplateAzureStandaloneCP, testingConfig.Standalone.Template) sdName = sd.GetName() standaloneDeleteFunc := kc.CreateManagedCluster(context.Background(), sd, managedcluster.Namespace) // verify the standalone cluster is deployed correctly deploymentValidator := managedcluster.NewProviderValidator( - managedcluster.TemplateAzureStandaloneCP, + templates.TemplateAzureStandaloneCP, managedcluster.Namespace, sdName, managedcluster.ValidationActionDeploy, ) - templateBy(managedcluster.TemplateAzureStandaloneCP, "waiting for infrastructure provider to deploy successfully") + templateBy(templates.TemplateAzureStandaloneCP, "waiting for infrastructure provider to deploy successfully") Eventually(func() error { return deploymentValidator.Validate(context.Background(), kc) }).WithTimeout(90 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) @@ -103,7 +118,7 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or // setup environment variables for deploying the hosted template (subnet name, etc) azure.SetAzureEnvironmentVariables(sdName, kc) - hd := managedcluster.GetUnstructured(managedcluster.TemplateAzureHostedCP) + hd := managedcluster.GetUnstructured(templates.TemplateAzureHostedCP, testingConfig.Hosted.Template) hdName := hd.GetName() var kubeCfgPath string @@ -128,7 +143,7 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) By(fmt.Sprintf("applying access rules for ClusterTemplates in %s namespace", managedcluster.Namespace)) - templates.ApplyClusterTemplateAccessRules(ctx, standaloneClient.CrClient) + templates.ApplyClusterTemplateAccessRules(ctx, standaloneClient.CrClient, managedcluster.Namespace) By("Create azure credential secret") clusteridentity.New(standaloneClient, managedcluster.ProviderAzure, managedcluster.Namespace) @@ -136,15 +151,15 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or By("Create default storage class for azure-disk CSI driver") azure.CreateDefaultStorageClass(standaloneClient) - templateBy(managedcluster.TemplateAzureHostedCP, "creating a ManagedCluster") + templateBy(templates.TemplateAzureHostedCP, fmt.Sprintf("creating a ManagedCluster with %s template", testingConfig.Hosted.Template)) hostedDeleteFunc = standaloneClient.CreateManagedCluster(context.Background(), hd, managedcluster.Namespace) - templateBy(managedcluster.TemplateAzureHostedCP, "Patching AzureCluster to ready") + templateBy(templates.TemplateAzureHostedCP, "Patching AzureCluster to ready") managedcluster.PatchHostedClusterReady(standaloneClient, managedcluster.ProviderAzure, managedcluster.Namespace, hdName) - templateBy(managedcluster.TemplateAzureHostedCP, "waiting for infrastructure to deploy successfully") + templateBy(templates.TemplateAzureHostedCP, "waiting for infrastructure to deploy successfully") deploymentValidator = managedcluster.NewProviderValidator( - managedcluster.TemplateAzureHostedCP, + templates.TemplateAzureHostedCP, managedcluster.Namespace, hdName, managedcluster.ValidationActionDeploy, @@ -162,7 +177,7 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or Expect(err).NotTo(HaveOccurred()) deploymentValidator = managedcluster.NewProviderValidator( - managedcluster.TemplateAzureHostedCP, + templates.TemplateAzureHostedCP, managedcluster.Namespace, hdName, managedcluster.ValidationActionDelete, @@ -173,7 +188,7 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or }).WithTimeout(10 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) deploymentValidator = managedcluster.NewProviderValidator( - managedcluster.TemplateAzureStandaloneCP, + templates.TemplateAzureStandaloneCP, managedcluster.Namespace, hdName, managedcluster.ValidationActionDelete, diff --git a/test/e2e/provider_vsphere_test.go b/test/e2e/scenarios/provider_vsphere_test.go similarity index 77% rename from test/e2e/provider_vsphere_test.go rename to test/e2e/scenarios/provider_vsphere_test.go index 7cec432a4..acbb6ab2e 100644 --- a/test/e2e/provider_vsphere_test.go +++ b/test/e2e/scenarios/provider_vsphere_test.go @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package e2e +package scenarios import ( "context" + "fmt" "os" "time" @@ -23,10 +24,12 @@ import ( . "github.com/onsi/gomega" internalutils "github.com/Mirantis/hmc/internal/utils" + "github.com/Mirantis/hmc/test/e2e/config" "github.com/Mirantis/hmc/test/e2e/kubeclient" "github.com/Mirantis/hmc/test/e2e/managedcluster" "github.com/Mirantis/hmc/test/e2e/managedcluster/clusteridentity" "github.com/Mirantis/hmc/test/e2e/managedcluster/vsphere" + "github.com/Mirantis/hmc/test/e2e/templates" ) var _ = Context("vSphere Templates", Label("provider:onprem", "provider:vsphere"), Ordered, func() { @@ -35,9 +38,23 @@ var _ = Context("vSphere Templates", Label("provider:onprem", "provider:vsphere" deleteFunc func() error clusterName string err error + + testingConfig config.ProviderTestingConfig ) BeforeAll(func() { + By("get testing configuration") + testingConfig = config.Config[config.TestingProviderVsphere] + + By("set defaults and validate testing configuration") + err := testingConfig.Standalone.SetDefaults(clusterTemplates, templates.TemplateVSphereStandaloneCP) + Expect(err).NotTo(HaveOccurred()) + + err = testingConfig.Hosted.SetDefaults(clusterTemplates, templates.TemplateVSphereHostedCP) + Expect(err).NotTo(HaveOccurred()) + + _, _ = fmt.Fprintf(GinkgoWriter, "Final Vsphere testing configuration:\n%s\n", testingConfig.String()) + By("ensuring that env vars are set correctly") vsphere.CheckEnv() By("creating kube client") @@ -65,7 +82,7 @@ var _ = Context("vSphere Templates", Label("provider:onprem", "provider:vsphere" // fails to do so. if deleteFunc != nil && !noCleanup() { deletionValidator := managedcluster.NewProviderValidator( - managedcluster.TemplateVSphereStandaloneCP, + templates.TemplateVSphereStandaloneCP, managedcluster.Namespace, clusterName, managedcluster.ValidationActionDelete, @@ -80,15 +97,15 @@ var _ = Context("vSphere Templates", Label("provider:onprem", "provider:vsphere" }) It("should deploy standalone managed cluster", func() { - By("creating a managed cluster") - d := managedcluster.GetUnstructured(managedcluster.TemplateVSphereStandaloneCP) + By(fmt.Sprintf("creating a managed cluster with %s template", testingConfig.Standalone.Template)) + d := managedcluster.GetUnstructured(templates.TemplateVSphereStandaloneCP, testingConfig.Standalone.Template) clusterName = d.GetName() deleteFunc = kc.CreateManagedCluster(context.Background(), d, managedcluster.Namespace) By("waiting for infrastructure providers to deploy successfully") deploymentValidator := managedcluster.NewProviderValidator( - managedcluster.TemplateVSphereStandaloneCP, + templates.TemplateVSphereStandaloneCP, managedcluster.Namespace, clusterName, managedcluster.ValidationActionDeploy, diff --git a/test/e2e/templates/templates.go b/test/e2e/templates/templates.go index 586120874..45f6b66f3 100644 --- a/test/e2e/templates/templates.go +++ b/test/e2e/templates/templates.go @@ -17,6 +17,9 @@ package templates import ( "context" "fmt" + "slices" + "sort" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -28,10 +31,20 @@ import ( hmc "github.com/Mirantis/hmc/api/v1alpha1" internalutils "github.com/Mirantis/hmc/internal/utils" - "github.com/Mirantis/hmc/test/e2e/managedcluster" ) -func ApplyClusterTemplateAccessRules(ctx context.Context, client crclient.Client) map[string][]hmc.AvailableUpgrade { +type Type string + +const ( + TemplateAWSStandaloneCP Type = "aws-standalone-cp" + TemplateAWSHostedCP Type = "aws-hosted-cp" + TemplateAzureHostedCP Type = "azure-hosted-cp" + TemplateAzureStandaloneCP Type = "azure-standalone-cp" + TemplateVSphereStandaloneCP Type = "vsphere-standalone-cp" + TemplateVSphereHostedCP Type = "vsphere-hosted-cp" +) + +func ApplyClusterTemplateAccessRules(ctx context.Context, client crclient.Client, namespace string) map[string][]hmc.AvailableUpgrade { ctChains := &metav1.PartialObjectMetadataList{} gvk := hmc.GroupVersion.WithKind(hmc.ClusterTemplateChainKind) ctChains.SetGroupVersionKind(gvk) @@ -53,7 +66,7 @@ func ApplyClusterTemplateAccessRules(ctx context.Context, client crclient.Client accessRules := []hmc.AccessRule{ { TargetNamespaces: hmc.TargetNamespaces{ - List: []string{managedcluster.Namespace}, + List: []string{namespace}, }, ClusterTemplateChains: chainNames, }, @@ -68,7 +81,7 @@ func ApplyClusterTemplateAccessRules(ctx context.Context, client crclient.Client clusterTemplateChains := make([]*hmc.ClusterTemplateChain, 0, len(chainNames)) Eventually(func() error { var err error - clusterTemplateChains, err = checkClusterTemplateChains(ctx, client, managedcluster.Namespace, chainNames) + clusterTemplateChains, err = checkClusterTemplateChains(ctx, client, namespace, chainNames) if err != nil { _, _ = fmt.Fprintf(GinkgoWriter, "Not all ClusterTemplateChains were created in the target namespace: %v\n", err) } @@ -77,7 +90,7 @@ func ApplyClusterTemplateAccessRules(ctx context.Context, client crclient.Client clusterTemplates := getClusterTemplates(clusterTemplateChains) Eventually(func() error { - err := checkClusterTemplates(ctx, client, managedcluster.Namespace, clusterTemplates) + err := checkClusterTemplates(ctx, client, namespace, clusterTemplates) if err != nil { _, _ = fmt.Fprintf(GinkgoWriter, "Not all ClusterTemplates were created in the target namespace: %v\n", err) } @@ -114,8 +127,89 @@ func checkClusterTemplates(ctx context.Context, client crclient.Client, namespac gvk := hmc.GroupVersion.WithKind(hmc.ClusterTemplateKind) template.SetGroupVersionKind(gvk) if err := client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: templateName}, template); err != nil { - return fmt.Errorf("failed to get ClusterTemplate %s/%s: %w", namespace, template.Name, err) + return fmt.Errorf("failed to get ClusterTemplate %s/%s: %w", namespace, templateName, err) } } return nil } + +func FindTemplate(clusterTemplates map[string][]hmc.AvailableUpgrade, templateType Type) (string, error) { + templates := filterByType(clusterTemplates, templateType) + if len(templates) == 0 { + return "", fmt.Errorf("no Template of the %s type is supported", templateType) + } + return templates[0], nil +} + +func FindTemplatesToUpgrade( + clusterTemplates map[string][]hmc.AvailableUpgrade, + templateType Type, + sourceTemplate string, +) (template, upgradeTemplate string, err error) { + templates := filterByType(clusterTemplates, templateType) + if len(templates) == 0 { + return "", "", fmt.Errorf("no Template of the %s type is supported", templateType) + } + if sourceTemplate != "" { + // Template should be in the list of supported + if !slices.Contains(templates, sourceTemplate) { + return "", "", fmt.Errorf("invalid templates configuration. Template %s is not in the list of supported templates", sourceTemplate) + } + // Template should have available upgrades + availableUpgrades := clusterTemplates[sourceTemplate] + if len(availableUpgrades) == 0 { + return "", "", fmt.Errorf("invalid templates configuration. No upgrades are available from the Template %s", sourceTemplate) + } + // Find latest available template for the upgrade + sort.Slice(availableUpgrades, func(i, j int) bool { + return availableUpgrades[i].Name < availableUpgrades[j].Name + }) + return sourceTemplate, availableUpgrades[len(availableUpgrades)-1].Name, nil + } + + // find template with available upgrades + for _, templateName := range templates { + template = templateName + for _, au := range clusterTemplates[template] { + if upgradeTemplate < au.Name { + upgradeTemplate = au.Name + } + } + if template != "" && upgradeTemplate != "" { + return template, upgradeTemplate, nil + } + } + if template == "" || upgradeTemplate == "" { + return "", "", fmt.Errorf("invalid templates configuration. No %s templates are available for the upgrade", templateType) + } + return template, upgradeTemplate, nil +} + +func ValidateTemplate(clusterTemplates map[string][]hmc.AvailableUpgrade, template string) error { + if _, ok := clusterTemplates[template]; ok { + return nil + } + return fmt.Errorf("template %s is not in the list of supported templates", template) +} + +func ValidateUpgradeSequence(clusterTemplates map[string][]hmc.AvailableUpgrade, source, target string) error { + availableUpgrades := clusterTemplates[source] + if _, ok := clusterTemplates[source]; ok && + slices.Contains(availableUpgrades, hmc.AvailableUpgrade{Name: target}) { + return nil + } + return fmt.Errorf("upgrade sequence %s -> %s is not supported", source, target) +} + +func filterByType(clusterTemplates map[string][]hmc.AvailableUpgrade, templateType Type) []string { + var templates []string + for template := range clusterTemplates { + if strings.HasPrefix(template, string(templateType)) { + templates = append(templates, template) + } + } + sort.Slice(templates, func(i, j int) bool { + return templates[i] > templates[j] + }) + return templates +} diff --git a/test/e2e/templates/templates_test.go b/test/e2e/templates/templates_test.go new file mode 100644 index 000000000..06e4c9688 --- /dev/null +++ b/test/e2e/templates/templates_test.go @@ -0,0 +1,131 @@ +// Copyright 2024 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package templates + +import ( + "errors" + "testing" + + . "github.com/onsi/gomega" + + hmc "github.com/Mirantis/hmc/api/v1alpha1" +) + +func Test_Find(t *testing.T) { + templates := map[string][]hmc.AvailableUpgrade{ + "aws-standalone-cp-0-0-1": { + {Name: "aws-standalone-cp-0-0-3"}, + {Name: "aws-standalone-cp-0-0-2"}, + }, + "aws-standalone-cp-0-0-3": {}, + "aws-standalone-cp-0-0-2": {}, + "aws-hosted-cp-0-0-1": { + {Name: "aws-hosted-cp-0-0-4"}, + {Name: "aws-hosted-cp-0-0-2"}, + }, + "aws-hosted-cp-0-0-4": {}, + "aws-hosted-cp-0-0-2": {}, + "azure-standalone-cp-0-0-1": { + {Name: "azure-standalone-cp-0-0-2"}, + }, + "azure-hosted-cp-0-0-1": { + {Name: "azure-hosted-cp-0-0-2"}, + }, + "vsphere-standalone-cp-0-0-1": { + {Name: "vsphere-standalone-cp-0-0-2"}, + }, + "vsphere-hosted-cp-0-0-1": {}, + } + for _, tt := range []struct { + title string + upgrade bool + sourceTemplate string + clusterTemplates map[string][]hmc.AvailableUpgrade + templateType Type + expectedTemplate string + expectedUpgradeTemplate string + expectedErr error + }{ + { + title: "no templates of the provided type supported", + templateType: "aws-unsupported-cp", + expectedErr: errors.New("no Template of the aws-unsupported-cp type is supported"), + }, + { + title: "should find latest template for aws-hosted-cp", + templateType: TemplateAWSHostedCP, + expectedTemplate: "aws-hosted-cp-0-0-4", + }, + { + title: "upgrade: no upgrades are available for this type of templates", + upgrade: true, + templateType: TemplateVSphereHostedCP, + expectedErr: errors.New("invalid templates configuration. No vsphere-hosted-cp templates are available for the upgrade"), + }, + { + title: "upgrade: source template provided but it's not supported", + upgrade: true, + sourceTemplate: "aws-standalone-cp-0-0-1-1", + templateType: TemplateAWSStandaloneCP, + expectedErr: errors.New("invalid templates configuration. Template aws-standalone-cp-0-0-1-1 is not in the list of supported templates"), + }, + { + title: "upgrade: source template provided but no upgrades are available", + upgrade: true, + sourceTemplate: "aws-standalone-cp-0-0-3", + templateType: TemplateAWSStandaloneCP, + expectedErr: errors.New("invalid templates configuration. No upgrades are available from the Template aws-standalone-cp-0-0-3"), + }, + { + title: "upgrade: source template provided and the upgrade template was found", + upgrade: true, + sourceTemplate: "aws-standalone-cp-0-0-1", + templateType: TemplateAWSStandaloneCP, + expectedTemplate: "aws-standalone-cp-0-0-1", + expectedUpgradeTemplate: "aws-standalone-cp-0-0-3", + }, + { + title: "upgrade: no templates are available for the upgrade for this type of templates", + upgrade: true, + templateType: TemplateVSphereHostedCP, + expectedErr: errors.New("invalid templates configuration. No vsphere-hosted-cp templates are available for the upgrade"), + }, + { + title: "upgrade: should find latest template with available upgrades", + upgrade: true, + templateType: TemplateAWSHostedCP, + expectedTemplate: "aws-hosted-cp-0-0-1", + expectedUpgradeTemplate: "aws-hosted-cp-0-0-4", + }, + } { + t.Run(tt.title, func(t *testing.T) { + g := NewWithT(t) + var template, upgradeTemplate string + var err error + if tt.upgrade { + template, upgradeTemplate, err = FindTemplatesToUpgrade(templates, tt.templateType, tt.sourceTemplate) + } else { + template, err = FindTemplate(templates, tt.templateType) + } + if tt.expectedErr != nil { + g.Expect(err).To(MatchError(tt.expectedErr)) + } else { + g.Expect(err).To(Succeed()) + } + g.Expect(template).To(Equal(tt.expectedTemplate)) + g.Expect(upgradeTemplate).To(Equal(tt.expectedUpgradeTemplate)) + }) + } +} diff --git a/test/objects/templatechain/templatechain.go b/test/objects/templatechain/templatechain.go index ac3296008..187820732 100644 --- a/test/objects/templatechain/templatechain.go +++ b/test/objects/templatechain/templatechain.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package templatemanagement +package templatechain import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"