diff --git a/go.mod b/go.mod index f7c6098..18ccbeb 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.23.0 require ( github.com/onsi/ginkgo/v2 v2.21.0 github.com/onsi/gomega v1.35.1 + k8s.io/api v0.31.2 k8s.io/apiextensions-apiserver v0.31.2 k8s.io/apimachinery v0.31.2 k8s.io/client-go v0.31.2 @@ -35,7 +36,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace // indirect github.com/x448/float16 v0.8.4 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/net v0.30.0 // indirect @@ -49,7 +50,6 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.31.2 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a // indirect k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect diff --git a/go.sum b/go.sum index 7d0df2c..98e5e26 100644 --- a/go.sum +++ b/go.sum @@ -69,8 +69,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace h1:9PNP1jnUjRhfmGMlkXHjYPishpcw4jpSt/V/xYY3FMA= +github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= diff --git a/test/configuration/azure-config.json b/test/configuration/azure-config.json deleted file mode 100644 index 8bc2753..0000000 --- a/test/configuration/azure-config.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "tenantId": "abc", - "subscriptionId": "123", - "resourceGroup": "test", - "aadClientId": "clientid", - "aadClientSecret": "clientpassword" - } \ No newline at end of file diff --git a/test/configuration/external-dns.yaml b/test/configuration/external-dns.yaml deleted file mode 100644 index 643155a..0000000 --- a/test/configuration/external-dns.yaml +++ /dev/null @@ -1,86 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: external-dns - namespace: hypershift -spec: - progressDeadlineSeconds: 600 - replicas: 1 - revisionHistoryLimit: 10 - selector: - matchLabels: - name: external-dns - strategy: - rollingUpdate: - maxSurge: 25% - maxUnavailable: 25% - type: RollingUpdate - template: - metadata: - creationTimestamp: null - labels: - app: external-dns - hypershift.openshift.io/operator-component: external-dns - name: external-dns - spec: - containers: - - args: - - --source=service - - --source=openshift-route - - --domain-filter=myzone - - --provider=azure - - --registry=txt - - --txt-suffix=-external-dns - - --txt-owner-id=5461617c-6757-49cd-b5ba-deda35d941f5 - - --label-filter=hypershift.openshift.io/route-visibility!=private - - --interval=1m - - --txt-cache-interval=1h - - --azure-config-file=/etc/provider/credentials - command: - - /external-dns - image: registry.redhat.io/edo/external-dns-rhel8@sha256:638fb6b5fc348f5cf52b9800d3d8e9f5315078fc9b1e57e800cb0a4a50f1b4b9 - imagePullPolicy: IfNotPresent - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: 7979 - scheme: HTTP - initialDelaySeconds: 60 - periodSeconds: 60 - successThreshold: 1 - timeoutSeconds: 5 - name: external-dns - ports: - - containerPort: 7979 - name: metrics - protocol: TCP - resources: - requests: - cpu: 5m - memory: 20Mi - securityContext: - privileged: false - readOnlyRootFilesystem: true - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /etc/provider - name: credentials - dnsPolicy: ClusterFirst - imagePullSecrets: - - name: pull-secret - - name: open-cluster-management-image-pull-credentials - priorityClassName: hypershift-operator - restartPolicy: Always - schedulerName: default-scheduler - securityContext: {} - serviceAccount: external-dns - serviceAccountName: external-dns - terminationGracePeriodSeconds: 30 - volumes: - - name: credentials - secret: - defaultMode: 420 - secretName: hypershift-operator-external-dns-credentials - diff --git a/test/configuration/hypershift-configmap.yaml b/test/configuration/hypershift-configmap.yaml deleted file mode 100644 index c8ad983..0000000 --- a/test/configuration/hypershift-configmap.yaml +++ /dev/null @@ -1,8 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: hypershift-operator-install-flags - namespace: local-cluster -data: - installFlagsToAdd: "--enable-conversion-webhook=false --managed-service ARO-HCP" - installFlagsToRemove: "--enable-uwm-telemetry-remote-write --platform-monitoring --enable-defaulting-webhook --enable-validating-webhook" diff --git a/test/e2e/common.go b/test/e2e/common.go index d060696..671f40b 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -2,6 +2,7 @@ package e2e import ( "fmt" + "k8s.io/apimachinery/pkg/runtime/schema" apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/meta" @@ -16,6 +17,31 @@ import ( workv1client "open-cluster-management.io/api/client/work/clientset/versioned" ) +var ( + hubKubeConfig string +) + +var ( + HubClients *Clients + HostedClusterName string +) + +const ( + LocalClusterName = "local-cluster" + ConfigPolicyAddonName = "config-policy-controller" + GovernancePolicyFrameworkAddonName = "governance-policy-framework" + HypershiftAddonName = "hypershift-addon" + WorkManagerAddonName = "work-manager" + MCEName = "multiclusterengine" +) + +var ( + MCEGVR = schema.GroupVersionResource{Group: "multicluster.openshift.io", Version: "v1", Resource: "multiclusterengines"} + ClusterInfoGVR = schema.GroupVersionResource{Group: "internal.open-cluster-management.io", Version: "v1beta1", Resource: "managedclusterinfos"} + HostedClusterGVR = schema.GroupVersionResource{Group: "hypershift.openshift.io", Version: "v1beta1", Resource: "hostedclusters"} + KlusterletAddonConfigGVR = schema.GroupVersionResource{Group: "agent.open-cluster-management.io", Version: "v1", Resource: "klusterletaddonconfigs"} +) + type Clients struct { KubeClient kubernetes.Interface APIExtensionsClient apiextensionsclient.Interface diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index b16ba17..16bf1b7 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -8,30 +8,9 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/tools/clientcmd" ) -var ( - hubKubeConfig string - HubClients *Clients - HostedClusterName string -) - -const ( - LocalClusterName = "local-cluster" - ConfigPolicyAddonName = "config-policy-controller" - GovernancePolicyFrameworkAddonName = "governance-policy-framework" - HypershiftAddonName = "hypershift-addon" - WorkManagerAddonName = "work-manager" - mceName = "multiclusterengine" -) - -var ( - mceGVR = schema.GroupVersionResource{Group: "multicluster.openshift.io", Version: "v1", Resource: "multiclusterengines"} - clusterInfoGVR = schema.GroupVersionResource{Group: "internal.open-cluster-management.io", Version: "v1beta1", Resource: "managedclusterinfos"} -) - // - KUBECONFIG is the location of the kubeconfig file to use // - MANAGED_CLUSTER_NAME is the name of managed cluster func TestE2E(tt *testing.T) { diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index f84d749..44bb0b5 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -3,6 +3,8 @@ package e2e import ( "context" "fmt" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -57,15 +59,12 @@ func ApplyResource(gvr schema.GroupVersionResource, obj *unstructured.Unstructur namespace := obj.GetNamespace() name := obj.GetName() - _, err := HubClients.DynamicClient.Resource(gvr).Namespace(namespace).Get(context.TODO(), name, metav1.GetOptions{}) - if err != nil && errors.IsNotFound(err) { + oldObj, err := HubClients.DynamicClient.Resource(gvr).Namespace(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if errors.IsNotFound(err) { return HubClients.DynamicClient.Resource(gvr).Namespace(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) } - if err == nil { - return nil, nil - } - return nil, err + return oldObj, err } func DeleteResource(gvr schema.GroupVersionResource, namespace, name string) error { @@ -97,7 +96,7 @@ func LoadResourceFromJSON(json string) (*unstructured.Unstructured, error) { } func CheckManagedClusterInfo(clusterName string) error { - clusterInfo, err := GetResource(clusterInfoGVR, clusterName, clusterName) + clusterInfo, err := GetResource(ClusterInfoGVR, clusterName, clusterName) if err != nil { return err } @@ -112,7 +111,7 @@ func CheckManagedClusterInfo(clusterName string) error { } func CheckMCE() error { - mce, err := GetResource(mceGVR, "", mceName) + mce, err := GetResource(MCEGVR, "", MCEName) if err != nil { return err } @@ -128,3 +127,282 @@ func CheckMCE() error { } return nil } + +func ApplyHostedManagedCluster(name string) (*clusterv1.ManagedCluster, error) { + cluster, err := HubClients.ClusterClient.ClusterV1().ManagedClusters().Get(context.TODO(), name, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return HubClients.ClusterClient.ClusterV1().ManagedClusters().Create( + context.TODO(), + &clusterv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: map[string]string{ + "addon.open-cluster-management.io/enable-hosted-mode-addons": "true", + "import.open-cluster-management.io/hosting-cluster-name": "local-cluster", + "import.open-cluster-management.io/klusterlet-deploy-mode": "Hosted", + }, + }, + Spec: clusterv1.ManagedClusterSpec{ + HubAcceptsClient: true, + }, + }, + metav1.CreateOptions{}, + ) + } + + return cluster, err +} + +func ApplyNamespace(ns string) error { + _, err := HubClients.KubeClient.CoreV1().Namespaces().Get(context.TODO(), ns, metav1.GetOptions{}) + if errors.IsNotFound(err) { + ns := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: ns, + }, + } + _, err := HubClients.KubeClient.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) + return err + } + return err +} + +var ( + hostedClusterNamespace = "hosted-clusters" + hostedClusterServiceName = "kube-apiserver" +) + +func ApplyHostedService(ns string) error { + _, err := HubClients.KubeClient.CoreV1().Services(ns).Get(context.TODO(), hostedClusterServiceName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + apiService := &v1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: hostedClusterServiceName, + Namespace: ns, + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Name: "https", + Port: 443, + Protocol: "TCP", + TargetPort: intstr.IntOrString{ + IntVal: 6443, + }, + }, + }, + }, + } + _, err := HubClients.KubeClient.CoreV1().Services(ns).Create(context.TODO(), apiService, metav1.CreateOptions{}) + return err + } + return err +} + +func ApplyHostedClusterKubeConfigSecret(clusterName string) error { + secretName := clusterName + "-admin-kubeconfig" + _, err := HubClients.KubeClient.CoreV1().Secrets(hostedClusterNamespace).Get(context.TODO(), secretName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + secret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: hostedClusterNamespace, + }, + Data: map[string][]byte{ + "kubeconfig": []byte(`apiVersion: v1 +clusters: +- cluster: + server: https://kube-apiserver.ocm-dev-1sv4l4ldnr6rd8ni12ndo4vtiq2gd7a4-sbarouti267.svc.cluster.local:7443 + name: cluster +contexts: +- context: + cluster: cluster + namespace: default + user: admin + name: admin +current-context: admin +kind: Config`), + }, + Type: v1.SecretTypeOpaque, + } + _, err := HubClients.KubeClient.CoreV1().Secrets(hostedClusterNamespace).Create(context.TODO(), secret, metav1.CreateOptions{}) + return err + } + + return err +} + +func ApplyHostedClusterResources(hostedClusterName string) (*unstructured.Unstructured, error) { + err := ApplyNamespace(hostedClusterNamespace) + if err != nil { + return nil, fmt.Errorf("failed to create ns %s. %v", hostedClusterNamespace, err) + } + + err = ApplyHostedClusterKubeConfigSecret(hostedClusterName) + if err != nil { + return nil, fmt.Errorf("failed to create kubeconfig secret %s. %v", hostedClusterName, err) + } + + serviceNamespace := hostedClusterNamespace + "-" + hostedClusterName + err = ApplyNamespace(serviceNamespace) + if err != nil { + return nil, fmt.Errorf("failed to create ns %s. %v", serviceNamespace, err) + } + err = ApplyHostedService(serviceNamespace) + if err != nil { + return nil, fmt.Errorf("failed to create service in ns %s. %v", serviceNamespace, err) + } + + hostedCluster, err := LoadResourceFromJSON(HostedClusterTemplate) + if err != nil { + return nil, fmt.Errorf("failed to load hostedCluster. %v", err) + } + + err = unstructured.SetNestedField(hostedCluster.Object, hostedClusterName, "metadata", "name") + if err != nil { + return nil, err + } + err = unstructured.SetNestedField(hostedCluster.Object, hostedClusterNamespace, "metadata", "namespace") + if err != nil { + return nil, err + } + annotations := map[string]string{ + "cluster.open-cluster-management.io/managedcluster-name": hostedClusterName, + } + err = unstructured.SetNestedStringMap(hostedCluster.Object, annotations, "metadata", "annotations") + if err != nil { + return nil, err + } + + hostedCluster, err = ApplyResource(HostedClusterGVR, hostedCluster) + if err != nil { + return nil, fmt.Errorf("failed to create hostedCluster. %v", err) + } + + conditions := []interface{}{ + map[string]interface{}{ + "lastTransitionTime": "2024-12-10T16:22:32Z", + "message": "AsExpected", + "reason": "AsExpected", + "status": "True", + "type": "Available", + }, + } + + err = unstructured.SetNestedSlice(hostedCluster.Object, conditions, "status", "conditions") + if err != nil { + return nil, err + } + return HubClients.DynamicClient.Resource(HostedClusterGVR).Namespace(hostedClusterNamespace). + UpdateStatus(context.TODO(), hostedCluster, metav1.UpdateOptions{}) + +} + +const HostedClusterTemplate = `{ + "apiVersion": "hypershift.openshift.io/v1beta1", + "kind": "HostedCluster", + "metadata": { + "name": "spoke", + "namespace": "hosted-spoke" + }, + "spec": { + "clusterID": "89693e2e-1198-4710-a254-c8277db50779", + "controllerAvailabilityPolicy": "HighlyAvailable", + "etcd": { + "managementType": "Managed" + }, + "fips": false, + "infraID": "spoke-abc", + "infrastructureAvailabilityPolicy": "SingleReplica", + "issuerURL": "https://kubernetes.default.svc", + "networking": { + "clusterNetwork": [ + { + "cidr": "10.132.0.0/14" + } + ], + "networkType": "OpenShiftSDN", + "serviceNetwork": [ + { + "cidr": "172.31.0.0/16" + } + ] + }, + "olmCatalogPlacement": "management", + "platform": { + "type": "Azure" + }, + "pullSecret": { + "name": "hosted-ipv4-pull-secret" + }, + "release": { + "image": "registry.hypershiftbm.lab:5000/openshift/release-images:4.14.0-0.nightly-2023-08-29-102237" + }, + "services": [ + { + "service": "APIServer", + "servicePublishingStrategy": { + "route": { + "hostname": "api.hosted-ipv4.hypershiftbm.lab" + }, + "type": "Route" + } + }, + { + "service": "OIDC", + "servicePublishingStrategy": { + "route": { + "hostname": "api.hosted-ipv4.hypershiftbm.lab" + }, + "type": "Route" + } + }, + { + "service": "OAuthServer", + "servicePublishingStrategy": { + "route": { + "hostname": "api.hosted-ipv4.hypershiftbm.lab" + }, + "type": "Route" + } + }, + { + "service": "Konnectivity", + "servicePublishingStrategy": { + "route": { + "hostname": "api.hosted-ipv4.hypershiftbm.lab" + }, + "type": "Route" + } + }, + { + "service": "Ignition", + "servicePublishingStrategy": { + "route": { + "hostname": "api.hosted-ipv4.hypershiftbm.lab" + }, + "type": "Route" + } + } + ], + "sshKey": { + "name": "sshkey-cluster-hosted-ipv4" + } + }, + "status": { + "conditions": [ + { + "lastTransitionTime": "2024-12-10T16:22:32Z", + "message": "AsExpected", + "reason": "AsExpected", + "status": "True", + "type": "Available" + } + ] + } +}` diff --git a/test/e2e/import_cluster_test.go b/test/e2e/import_cluster_test.go new file mode 100644 index 0000000..2285211 --- /dev/null +++ b/test/e2e/import_cluster_test.go @@ -0,0 +1,115 @@ +package e2e + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/rand" +) + +var _ = Describe("Check the status of hosted cluster.", Label("hosted cluster"), func() { + hostedClusterName := fmt.Sprintf("e2e-cluster-%s", rand.String(6)) + BeforeEach(func() { + By("import a hosted managedCluster") + _, err := ApplyHostedManagedCluster(hostedClusterName) + Expect(err).NotTo(HaveOccurred()) + + _, err = ApplyHostedClusterResources(hostedClusterName) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + By("clean up the resources of the cluster") + Eventually(func() error { + _, err := HubClients.KubeClient.CoreV1().Namespaces().Get(context.TODO(), hostedClusterNamespace, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return nil + } + + err = HubClients.KubeClient.CoreV1().Namespaces().Delete(context.TODO(), hostedClusterNamespace, metav1.DeleteOptions{}) + if errors.IsNotFound(err) { + return nil + } + return fmt.Errorf("wait the ns is deleted. %v", err) + }).Should(Succeed()) + + Eventually(func() error { + serviceNamespace := hostedClusterNamespace + "-" + hostedClusterName + _, err := HubClients.KubeClient.CoreV1().Namespaces().Get(context.TODO(), serviceNamespace, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return nil + } + + err = HubClients.KubeClient.CoreV1().Namespaces().Delete(context.TODO(), serviceNamespace, metav1.DeleteOptions{}) + if errors.IsNotFound(err) { + return nil + } + + return fmt.Errorf("wait ns is deleted. %v", err) + }).Should(Succeed()) + + Eventually(func() error { + _, err := HubClients.ClusterClient.ClusterV1().ManagedClusters().Get(context.TODO(), hostedClusterName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return nil + } + + return HubClients.ClusterClient.ClusterV1().ManagedClusters().Delete(context.TODO(), hostedClusterName, metav1.DeleteOptions{}) + }).Should(Succeed()) + }) + + It("Check the resources of imported hosted managedCluster are created", func() { + By("external-managed-kubeconfig should be created") + Eventually(func() error { + klusteletNs := "klusterlet-" + hostedClusterName + _, err := HubClients.KubeClient.CoreV1().Secrets(klusteletNs).Get(context.TODO(), "external-managed-kubeconfig", metav1.GetOptions{}) + return err + }).Should(Succeed()) + + By("klusterletAddonConfig should be created") + Eventually(func() error { + kac, err := GetResource(KlusterletAddonConfigGVR, hostedClusterName, hostedClusterName) + if err != nil { + return err + } + policyEnabled, _, err := unstructured.NestedBool(kac.Object, "spec", "policyController", "enabled") + if err != nil { + return err + } + if !policyEnabled { + return fmt.Errorf("policy in kac should be enabled") + } + return nil + + }).Should(Succeed()) + + By("addons should be created") + Eventually(func() error { + addons, err := HubClients.AddonClient.AddonV1alpha1().ManagedClusterAddOns(hostedClusterName). + List(context.Background(), metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed list addons: %v", err) + } + + if len(addons.Items) != 3 { + return fmt.Errorf("expect 3 addons but got %v", len(addons.Items)) + } + + for _, addon := range addons.Items { + switch addon.GetName() { + case "work-manager", "config-policy-controller", "governance-policy-framework": + default: + return fmt.Errorf("the addon %s is not supported", addon.GetName()) + } + } + + return nil + }).Should(Succeed()) + }) + +})