Implement Azure e2e template tests

.github/workflows/test.yml

@@ -25,6 +25,10 @@ jobs:
       AWS_REGION: us-west-2
       AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_SECRET_ACCESS_KEY }}
+      AZURE_SUBSCRIPTION_ID: ${{ secrets.CI_AZURE_SUBSCRIPTION_ID }}
+      AZURE_TENANT_ID: ${{ secrets.CI_AZURE_TENANT_ID }}
+      AZURE_CLIENT_ID: ${{ secrets.CI_AZURE_CLIENT_ID }}
+      AZURE_CLIENT_SECRET: ${{ secrets.CI_AZURE_CLIENT_SECRET }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

Makefile

@@ -106,7 +106,7 @@ test: generate-all fmt vet envtest tidy external-crd ## Run tests.
 # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
 .PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up.
 test-e2e: cli-install
-	KIND_CLUSTER_NAME="hmc-test" KIND_VERSION=$(KIND_VERSION) go test ./test/e2e/ -v -ginkgo.v -timeout=2h
+	KIND_CLUSTER_NAME="hmc-test" KIND_VERSION=$(KIND_VERSION) go test ./test/e2e/ -v -ginkgo.v -timeout=3h
 
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter & yamllint

test/e2e/e2e_test.go

@@ -143,15 +143,15 @@ var _ = Describe("controller", Ordered, func() {
 				}
 
 				By("creating a Deployment")
-				d := managedcluster.GetUnstructured(managedcluster.ProviderAWS, template)
+				d := managedcluster.GetUnstructured(managedcluster.ProviderAWS, template, namespace)
 				clusterName = d.GetName()
 
 				deleteFunc, err = kc.CreateManagedCluster(context.Background(), d)
 				Expect(err).NotTo(HaveOccurred())
 
 				By("waiting for infrastructure providers to deploy successfully")
 				Eventually(func() error {
-					return managedcluster.VerifyProviderDeployed(context.Background(), kc, clusterName)
+					return managedcluster.VerifyProviderDeployed(context.Background(), kc, clusterName, managedcluster.ProviderAWS)
 				}).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
 
 				By("verify the deployment deletes successfully")
@@ -163,6 +163,68 @@ var _ = Describe("controller", Ordered, func() {
 			})
 		}
 	})
+
+	Context("Azure Templates", func() {
+		var (
+			kc          *kubeclient.KubeClient
+			deleteFunc  func() error
+			clusterName string
+			err         error
+		)
+
+		BeforeAll(func() {
+			By("ensuring Azure credentials are set")
+			kc, err = kubeclient.NewFromLocal(namespace)
+			ExpectWithOffset(2, err).NotTo(HaveOccurred())
+
+			//time.Sleep(60 * time.Second)
+			ExpectWithOffset(2, kc.CreateAzureCredentialsKubeSecret(context.Background())).To(Succeed())
+		})
+
+		AfterEach(func() {
+			// If we failed collect logs from each of the affiliated controllers
+			// as well as the output of clusterctl to store as artifacts.
+			if CurrentSpecReport().Failed() {
+				By("collecting failure logs from controllers")
+				collectLogArtifacts(kc, clusterName, managedcluster.ProviderAWS, managedcluster.ProviderCAPI)
+			}
+
+			// Delete the deployments if they were created.
+			if deleteFunc != nil {
+				By("deleting the deployment")
+				err = deleteFunc()
+				Expect(err).NotTo(HaveOccurred())
+			}
+
+		})
+
+		for _, template := range []managedcluster.Template{
+			managedcluster.TemplateAzureStandaloneCP,
+			managedcluster.TemplateAzureHostedCP,
+		} {
+			It(fmt.Sprintf("should work with an Azure provider and %s template", template), func() {
+				By("creating a Deployment")
+				d := managedcluster.GetUnstructured(managedcluster.ProviderAzure, template, namespace)
+				clusterName = d.GetName()
+
+				deleteFunc, err = kc.CreateManagedCluster(context.Background(), d)
+				Expect(err).NotTo(HaveOccurred())
+
+				By("waiting for infrastructure providers to deploy successfully")
+				Eventually(func() error {
+					return managedcluster.VerifyProviderDeployed(context.Background(), kc, clusterName, managedcluster.ProviderAzure)
+				}).WithTimeout(90 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
+
+				By("verify the deployment deletes successfully")
+				err = deleteFunc()
+				Expect(err).NotTo(HaveOccurred())
+				Eventually(func() error {
+					return managedcluster.VerifyProviderDeleted(context.Background(), kc, clusterName)
+				}).WithTimeout(10 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
+			})
+		}
+	})
+
 })
 
 func verifyControllerUp(kc *kubeclient.KubeClient, labelSelector string, name string) error {

test/kubeclient/kubeclient.go

@@ -15,23 +15,31 @@
 package kubeclient
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
 
 	"github.com/Mirantis/hmc/test/utils"
+	"github.com/a8m/envsubst"
 	. "github.com/onsi/ginkgo/v2"
 	corev1 "k8s.io/api/core/v1"
 	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer/yaml"
+	yamlutil "k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/restmapper"
 	"k8s.io/client-go/tools/clientcmd"
 )
 
@@ -123,6 +131,75 @@ func new(configBytes []byte, namespace string) (*KubeClient, error) {
 	}, nil
 }
 
+func (kc *KubeClient) CreateAzureCredentialsKubeSecret(ctx context.Context) error {
+	serializer := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme)
+	yamlFile, err := os.ReadFile("./config/dev/azure-credentials.yaml")
+
+	if err != nil {
+		return fmt.Errorf("failed to read azure credential file: %w", err)
+	}
+
+	yamlFile, err = envsubst.Bytes(yamlFile)
+	if err != nil {
+		return fmt.Errorf("failed to process azure credential file: %w", err)
+	}
+
+	c := discovery.NewDiscoveryClientForConfigOrDie(kc.Config)
+	groupResources, err := restmapper.GetAPIGroupResources(c)
+	if err != nil {
+		return fmt.Errorf("failed to fetch group resources: %w", err)
+	}
+
+	yamlReader := yamlutil.NewYAMLReader(bufio.NewReader(bytes.NewReader(yamlFile)))
+	for {
+		yamlDoc, err := yamlReader.Read()
+
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return fmt.Errorf("failed to process azure credential file: %w", err)
+
+		}
+
+		credentialResource := &unstructured.Unstructured{}
+		_, _, err = serializer.Decode(yamlDoc, nil, credentialResource)
+		if err != nil {
+			return fmt.Errorf("failed to deserialize azure credential object: %w", err)
+		}
+
+		mapper := restmapper.NewDiscoveryRESTMapper(groupResources)
+		mapping, err := mapper.RESTMapping(credentialResource.GroupVersionKind().GroupKind())
+
+		if err != nil {
+			return fmt.Errorf("failed to create rest mapper: %w", err)
+		}
+
+		dc, err := kc.GetDynamicClient(schema.GroupVersionResource{
+			Group:    credentialResource.GroupVersionKind().Group,
+			Version:  credentialResource.GroupVersionKind().Version,
+			Resource: mapping.Resource.Resource,
+		})
+
+		if err != nil {
+			return fmt.Errorf("failed to create dynamic client: %w", err)
+		}
+
+		exists, err := dc.Get(ctx, credentialResource.GetName(), metav1.GetOptions{})
+		if err != nil && !apierrors.IsNotFound(err) {
+			return fmt.Errorf("failed to check for existing credential: %w", err)
+		}
+
+		if exists == nil {
+			if _, err = dc.Create(ctx, credentialResource, metav1.CreateOptions{}); err != nil {
+				return fmt.Errorf("failed to create azure credentials: %w", err)
+			}
+		}
+	}
+
+	return nil
+}
+
 // CreateAWSCredentialsKubeSecret uses clusterawsadm to encode existing AWS
 // credentials and create a secret in the given namespace if one does not
 // already exist.

test/managedcluster/managedcluster.go

@@ -40,8 +40,10 @@ const (
 type Template string
 
 const (
-	TemplateAWSStandaloneCP Template = "aws-standalone-cp"
-	TemplateAWSHostedCP     Template = "aws-hosted-cp"
+	TemplateAWSStandaloneCP   Template = "aws-standalone-cp"
+	TemplateAWSHostedCP       Template = "aws-hosted-cp"
+	TemplateAzureHostedCP     Template = "azure-hosted-cp"
+	TemplateAzureStandaloneCP Template = "azure-standalone-cp"
 )
 
 //go:embed resources/aws-standalone-cp.yaml.tpl
@@ -50,16 +52,22 @@ var awsStandaloneCPManagedClusterTemplateBytes []byte
 //go:embed resources/aws-hosted-cp.yaml.tpl
 var awsHostedCPManagedClusterTemplateBytes []byte
 
+//go:embed resources/azure-standalone-cp.yaml.tpl
+var azureStandaloneCPManagedClusterTemplateBytes []byte
+
+//go:embed resources/azure-hosted-cp.yaml.tpl
+var azureHostedCPManagedClusterTemplateBytes []byte
+
 func GetProviderLabel(provider ProviderType) string {
 	return fmt.Sprintf("%s=%s", providerLabel, provider)
 }
 
 // GetUnstructured returns an unstructured ManagedCluster object based on the
 // provider and template.
-func GetUnstructured(provider ProviderType, templateName Template) *unstructured.Unstructured {
+func GetUnstructured(provider ProviderType, templateName Template, namespace string) *unstructured.Unstructured {
 	GinkgoHelper()
 
-	generatedName := uuid.New().String()[:8] + "-e2e-test"
+	generatedName := "e2etest-" + uuid.New().String()[:8]
 	_, _ = fmt.Fprintf(GinkgoWriter, "Generated cluster name: %q\n", generatedName)
 
 	switch provider {
@@ -84,6 +92,30 @@ func GetUnstructured(provider ProviderType, templateName Template) *unstructured
 		err = yaml.Unmarshal(managedClusterConfigBytes, &managedClusterConfig)
 		Expect(err).NotTo(HaveOccurred(), "failed to unmarshal deployment config")
 
+		return &unstructured.Unstructured{Object: managedClusterConfig}
+
+	case ProviderAzure:
+		Expect(os.Setenv("MANAGED_CLUSTER_NAME", generatedName)).NotTo(HaveOccurred())
+		Expect(os.Setenv("NAMESPACE", namespace)).NotTo(HaveOccurred())
+
+		var managedClusterTemplateBytes []byte
+		switch templateName {
+		case TemplateAzureHostedCP:
+			managedClusterTemplateBytes = azureHostedCPManagedClusterTemplateBytes
+		case TemplateAzureStandaloneCP:
+			managedClusterTemplateBytes = azureStandaloneCPManagedClusterTemplateBytes
+		default:
+			Fail(fmt.Sprintf("unsupported Azure template: %s", templateName))
+		}
+
+		managedClusterConfigBytes, err := envsubst.Bytes(managedClusterTemplateBytes)
+		Expect(err).NotTo(HaveOccurred(), "failed to substitute environment variables")
+
+		var managedClusterConfig map[string]interface{}
+
+		err = yaml.Unmarshal(managedClusterConfigBytes, &managedClusterConfig)
+		Expect(err).NotTo(HaveOccurred(), "failed to unmarshal deployment config")
+
 		return &unstructured.Unstructured{Object: managedClusterConfig}
 	default:
 		Fail(fmt.Sprintf("unsupported provider: %s", provider))

test/managedcluster/resources/azure-hosted-cp.yaml.tpl

@@ -0,0 +1,22 @@
+apiVersion: hmc.mirantis.com/v1alpha1
+kind: ManagedCluster
+metadata:
+  name: ${MANAGED_CLUSTER_NAME}
+  namespace: ${NAMESPACE}
+spec:
+  template: azure-hosted-cp
+  config:
+    controlPlaneNumber: 1
+    workersNumber: 1
+    location: "westus"
+    subscriptionID: "${AZURE_SUBSCRIPTION_ID}"
+    controlPlane:
+      vmSize: Standard_A4_v2
+    worker:
+      vmSize: Standard_A4_v2
+    clusterIdentity:
+      name: azure-cluster-identity
+      namespace: ${NAMESPACE}
+    tenantID: "${AZURE_TENANT_ID}"
+    clientID: "${AZURE_CLIENT_ID}"
+    clientSecret: "${AZURE_CLIENT_SECRET}"

test/managedcluster/resources/azure-standalone-cp.yaml.tpl

@@ -0,0 +1,22 @@
+apiVersion: hmc.mirantis.com/v1alpha1
+kind: ManagedCluster
+metadata:
+  name: ${MANAGED_CLUSTER_NAME}
+  namespace: ${NAMESPACE}
+spec:
+  template: azure-standalone-cp
+  config:
+    controlPlaneNumber: 1
+    workersNumber: 1
+    location: "westus"
+    subscriptionID: "${AZURE_SUBSCRIPTION_ID}"
+    controlPlane:
+      vmSize: Standard_A4_v2
+    worker:
+      vmSize: Standard_A4_v2
+    clusterIdentity:
+      name: azure-cluster-identity
+      namespace: ${NAMESPACE}
+    tenantID: "${AZURE_TENANT_ID}"
+    clientID: "${AZURE_CLIENT_ID}"
+    clientSecret: "${AZURE_CLIENT_SECRET}"

test/managedcluster/validate_deleted.go

@@ -18,6 +18,8 @@ import (
 	"errors"
 	"fmt"
 
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+
 	"github.com/Mirantis/hmc/test/kubeclient"
 	"github.com/Mirantis/hmc/test/utils"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -41,7 +43,7 @@ func VerifyProviderDeleted(ctx context.Context, kc *kubeclient.KubeClient, clust
 func validateClusterDeleted(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error {
 	// Validate that the Cluster resource has been deleted
 	cluster, err := kc.GetCluster(ctx, clusterName)
-	if err != nil {
+	if err != nil && !apierrors.IsNotFound(err) {
 		return err
 	}
 

test/managedcluster/validate_deployed.go

@@ -45,9 +45,13 @@ var resourceValidators = map[string]resourceValidationFunc{
 // VerifyProviderDeployed is a provider-agnostic verification that checks
 // to ensure generic resources managed by the provider have been deleted.
 // It is intended to be used in conjunction with an Eventually block.
-func VerifyProviderDeployed(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error {
-	return verifyProviderAction(ctx, kc, clusterName, resourceValidators,
-		[]string{"clusters", "machines", "control-planes", "csi-driver", "ccm"})
+func VerifyProviderDeployed(ctx context.Context, kc *kubeclient.KubeClient, clusterName string,
+	providerType ProviderType) error {
+	resources := []string{"clusters", "machines", "control-planes", "ccm"}
+	if providerType != ProviderAzure {
+		resources = append(resources, "csi-driver")
+	}
+	return verifyProviderAction(ctx, kc, clusterName, resourceValidators, resources)
 }
 
 // verifyProviderAction is a provider-agnostic verification that checks for
@@ -246,7 +250,7 @@ func validateCSIDriver(ctx context.Context, kc *kubeclient.KubeClient, clusterNa
 		return fmt.Errorf("failed to get test PVC: %w", err)
 	}
 
-	if !strings.Contains(*pvc.Spec.StorageClassName, "csi") {
+	if pvc.Spec.StorageClassName != nil && !strings.Contains(*pvc.Spec.StorageClassName, "csi") {
 		Fail(fmt.Sprintf("%s PersistentVolumeClaim does not have a CSI driver storageClass", pvcName))
 	}
 
@@ -301,7 +305,7 @@ func validateCCM(ctx context.Context, kc *kubeclient.KubeClient, clusterName str
 	}
 
 	for _, i := range service.Status.LoadBalancer.Ingress {
-		if i.Hostname != "" {
+		if i.Hostname != "" || i.IP != "" {
 			return nil
 		}
 	}