diff --git a/Makefile b/Makefile index d2ade19a8..cb30512c2 100644 --- a/Makefile +++ b/Makefile @@ -98,6 +98,7 @@ kind-env: @echo "Setting up KinD cluster" @./scripts/bootstrap-kind-env.sh @echo "Cluster has been created" + kind export kubeconfig --name=managed kind export kubeconfig --name=hub kubectl label node hub-control-plane node-role.kubernetes.io/master='' @@ -108,6 +109,8 @@ mco-kind-env: kind-env @echo "Local environment has been set up" @echo "Installing MCO" @kind get kubeconfig --name hub > /tmp/hub.yaml + @kind get kubeconfig --name managed > /tmp/managed.yaml + @kind get kubeconfig --name hub --internal > ./.hub-kubeconfig KUBECONFIG=/tmp/hub.yaml IS_KIND_ENV=true KUSTOMIZE_VERSION=${KUSTOMIZE_VERSION} ./cicd-scripts/setup-e2e-tests.sh diff --git a/cicd-scripts/run-e2e-tests.sh b/cicd-scripts/run-e2e-tests.sh index 9c2744655..d9bb22e7c 100755 --- a/cicd-scripts/run-e2e-tests.sh +++ b/cicd-scripts/run-e2e-tests.sh @@ -39,7 +39,7 @@ else fi kubecontext=$(kubectl config current-context) -cluster_name="local-cluster" +hub_cluster_name="local-cluster" if [[ -n ${IS_KIND_ENV} ]]; then clusterServerURL="https://127.0.0.1:32806" @@ -67,7 +67,7 @@ if [[ -n ${IS_KIND_ENV} ]]; then printf "\n grafanaHost: grafana-test" >>${OPTIONSFILE} fi printf "\n clusters:" >>${OPTIONSFILE} -printf "\n - name: ${cluster_name}" >>${OPTIONSFILE} +printf "\n - name: ${hub_cluster_name}" >>${OPTIONSFILE} if [[ -n ${IS_KIND_ENV} ]]; then printf "\n clusterServerURL: ${clusterServerURL}" >>${OPTIONSFILE} fi @@ -75,6 +75,18 @@ printf "\n baseDomain: ${base_domain}" >>${OPTIONSFILE} printf "\n kubeconfig: ${kubeconfig_hub_path}" >>${OPTIONSFILE} printf "\n kubecontext: ${kubecontext}" >>${OPTIONSFILE} +kubeconfig_managed_path="${SHARED_DIR}/managed-1.kc" +if [[ -z ${IS_KIND_ENV} && -f ${kubeconfig_managed_path} ]]; then + managed_cluster_name="managed-cluster-1" + kubecontext_managed=$(kubectl --kubeconfig="${kubeconfig_managed_path}" config current-context) + app_domain_managed=$(kubectl -n openshift-ingress-operator --kubeconfig="${kubeconfig_managed_path}" get ingresscontrollers default -ojsonpath='{.status.domain}') + base_domain_managed="${app_domain_managed#apps.}" + printf "\n - name: ${managed_cluster_name}" >>${OPTIONSFILE} + printf "\n baseDomain: ${base_domain_managed}" >>${OPTIONSFILE} + printf "\n kubeconfig: ${kubeconfig_managed_path}" >>${OPTIONSFILE} + printf "\n kubecontext: ${kubecontext_managed}" >>${OPTIONSFILE} +fi + if command -v ginkgo &>/dev/null; then GINKGO_CMD=ginkgo else diff --git a/cicd-scripts/setup-e2e-tests.sh b/cicd-scripts/setup-e2e-tests.sh index 028a7a2df..f53a163a3 100755 --- a/cicd-scripts/setup-e2e-tests.sh +++ b/cicd-scripts/setup-e2e-tests.sh @@ -22,7 +22,7 @@ AGENT_NS="open-cluster-management-agent" HUB_NS="open-cluster-management-hub" OBSERVABILITY_NS="open-cluster-management-observability" IMAGE_REPO="quay.io/stolostron" -export MANAGED_CLUSTER="local-cluster" # registration-operator needs this +#export MANAGED_CLUSTER="local-cluster" # registration-operator needs this SED_COMMAND=${SED}' -i-e -e' @@ -43,32 +43,40 @@ deploy_hub_spoke_core() { REGISTRATION_LATEST_SNAPSHOT='2.4.9-SNAPSHOT-2022-11-17-20-19-31' make cluster-ip IMAGE_REGISTRY=quay.io/stolostron IMAGE_TAG=${REGISTRATION_LATEST_SNAPSHOT} WORK_TAG=${REGISTRATION_LATEST_SNAPSHOT} REGISTRATION_TAG=${REGISTRATION_LATEST_SNAPSHOT} PLACEMENT_TAG=${REGISTRATION_LATEST_SNAPSHOT} make deploy IMAGE_REGISTRY=quay.io/stolostron IMAGE_TAG=${REGISTRATION_LATEST_SNAPSHOT} WORK_TAG=${REGISTRATION_LATEST_SNAPSHOT} REGISTRATION_TAG=${REGISTRATION_LATEST_SNAPSHOT} PLACEMENT_TAG=${REGISTRATION_LATEST_SNAPSHOT} - # wait until hub and spoke are ready wait_for_deployment_ready 10 60s ${HUB_NS} cluster-manager-registration-controller cluster-manager-registration-webhook cluster-manager-work-webhook wait_for_deployment_ready 10 60s ${AGENT_NS} klusterlet-registration-agent klusterlet-work-agent + } # approve the CSR for cluster join request approve_csr_joinrequest() { echo "wait for CSR for cluster join reqest is created..." - for i in {1..60}; do - # TODO(morvencao): remove the hard-coded cluster label - csrs=$(kubectl get csr -lopen-cluster-management.io/cluster-name=${MANAGED_CLUSTER}) - if [[ -n ${csrs} ]]; then - csrnames=$(kubectl get csr -lopen-cluster-management.io/cluster-name=${MANAGED_CLUSTER} -o jsonpath={.items..metadata.name}) - for csrname in ${csrnames}; do - echo "approve CSR: ${csrname}" - kubectl certificate approve ${csrname} - done - break - fi - if [[ ${i} -eq 60 ]]; then - echo "timeout wait for CSR is created." - exit 1 - fi - echo "retrying in 10s..." - sleep 10 + managed_clusters=("local-cluster" "managed-cluster-1") + + KUBECONFIG=/tmp/hub.yaml IS_KIND_ENV=true + #kubectl config use-context kind-hub + for MANAGED_CLUSTER in "${managed_clusters[@]}"; do + echo "Processing CSRs for ${MANAGED_CLUSTER}..." + for i in {1..60}; do + # TODO(morvencao): remove the hard-coded cluster label + # for loop for the case that multiple clusters are created + csrs=$(kubectl get csr -lopen-cluster-management.io/cluster-name=${MANAGED_CLUSTER}) + if [[ -n ${csrs} ]]; then + csrnames=$(kubectl get csr -lopen-cluster-management.io/cluster-name=${MANAGED_CLUSTER} -o jsonpath={.items..metadata.name}) + for csrname in ${csrnames}; do + echo "approve CSR: ${csrname}" + kubectl certificate approve ${csrname} + done + break + fi + if [[ ${i} -eq 60 ]]; then + echo "timeout wait for CSR is created." + exit 1 + fi + echo "retrying in 10s..." + sleep 10 + done done for i in {1..20}; do @@ -130,6 +138,7 @@ EOF # deploy the MCO operator via the kustomize resources deploy_mco_operator() { + kubectl config use-context kind-hub if [[ -n ${MULTICLUSTER_OBSERVABILITY_OPERATOR_IMAGE_REF} ]]; then cd ${ROOTDIR}/operators/multiclusterobservability/config/manager && kustomize edit set image quay.io/stolostron/multicluster-observability-operator=${MULTICLUSTER_OBSERVABILITY_OPERATOR_IMAGE_REF} else @@ -216,10 +225,59 @@ wait_for_deployment_ready() { done } +deploy_managed_cluster() { + echo "Setting Kubernetes context to the managed cluster..." + + KUBECONFIG=/tmp/managed.yaml IS_KIND_ENV=true + kubectl config use-context kind-managed + export MANAGED_CLUSTER="managed-cluster-1" + + cd ${ROOTDIR} + # we are pinned here so no need to re-fetch if we have the project locally. + if [[ ! -d "registration-operator" ]]; then + git clone --depth 1 -b release-2.4 https://github.com/stolostron/registration-operator.git + fi + cd registration-operator + REGISTRATION_LATEST_SNAPSHOT='2.4.9-SNAPSHOT-2022-11-17-20-19-31' + ${SED_COMMAND} "s~clusterName: cluster1$~clusterName: ${MANAGED_CLUSTER}~g" deploy/klusterlet/config/samples/operator_open-cluster-management_klusterlets.cr.yaml + make deploy-spoke IMAGE_REGISTRY=quay.io/stolostron IMAGE_TAG=${REGISTRATION_LATEST_SNAPSHOT} WORK_TAG=${REGISTRATION_LATEST_SNAPSHOT} REGISTRATION_TAG=${REGISTRATION_LATEST_SNAPSHOT} PLACEMENT_TAG=${REGISTRATION_LATEST_SNAPSHOT} + wait_for_deployment_ready 10 60s ${AGENT_NS} klusterlet-registration-agent klusterlet-work-agent +} + +deploy_hub_and_managed_cluster() { + cd $(dirname ${BASH_SOURCE}) + + set -e + + hub=${CLUSTER1:-hub} + hub_name="local-cluster" + c1=${CLUSTER1:-managed} + + hubctx="kind-${hub}" + c1ctx="kind-${c1}" + + echo "Initialize the ocm hub cluster\n" # ./.hub-kubeconfig is default value of HUB_KUBECONFIG + clusteradm init --wait --context ${hubctx} + joincmd=$(clusteradm get token --context ${hubctx} | grep clusteradm) + + echo "Join hub to hub\n" + $(echo ${joincmd} --force-internal-endpoint-lookup --wait --context ${hubctx} | sed "s//$hub_name/g") + KUBECONFIG=/tmp/managed.yaml IS_KIND_ENV=true + echo "Join cluster1 to hub\n" + $(echo ${joincmd} --force-internal-endpoint-lookup --wait --context ${c1ctx} | sed "s//$c1/g") + + echo "Accept join of hub,cluster1" + KUBECONFIG=/tmp/hub.yaml IS_KIND_ENV=true + clusteradm accept --context ${hubctx} --clusters ${c1},${hub_name} --skip-approve-check + + kubectl get managedclusters --all-namespaces --context ${hubctx} +} # function execute is the main routine to do the actual work execute() { - deploy_hub_spoke_core - approve_csr_joinrequest + # deploy_hub_spoke_core + # approve_csr_joinrequest + # deploy_managed_cluster + deploy_hub_and_managed_cluster deploy_mco_operator deploy_grafana_test echo "OCM and MCO are installed successfully..." diff --git a/scripts/bootstrap-kind-env.sh b/scripts/bootstrap-kind-env.sh index 206ce9883..e92607264 100755 --- a/scripts/bootstrap-kind-env.sh +++ b/scripts/bootstrap-kind-env.sh @@ -39,12 +39,22 @@ deploy_openshift_router() { kubectl apply -f ${WORKDIR}/router/ } +create_kind_cluster_managed() { + echo "Coleen Delete the KinD cluster if exists" + kind delete cluster --name $1 || true + rm -rf $HOME/.kube/kind-config-$1 + + echo "Start KinD cluster with the default cluster name - $1" + kind create cluster --kubeconfig $HOME/.kube/kind-config-$1 --name $1 --config ${WORKDIR}/kind/kind-$1.config.yaml +} + run() { create_kind_cluster hub deploy_crds deploy_templates deploy_service_ca_operator deploy_openshift_router + create_kind_cluster_managed managed } run diff --git a/tests/pkg/tests/observability_addon_test.go b/tests/pkg/tests/observability_addon_test.go index f9ce37591..3318c6541 100644 --- a/tests/pkg/tests/observability_addon_test.go +++ b/tests/pkg/tests/observability_addon_test.go @@ -19,15 +19,6 @@ import ( var _ = Describe("Observability:", func() { BeforeEach(func() { - hubClient = utils.NewKubeClient( - testOptions.HubCluster.ClusterServerURL, - testOptions.KubeConfig, - testOptions.HubCluster.KubeContext) - - dynClient = utils.NewKubeClientDynamic( - testOptions.HubCluster.ClusterServerURL, - testOptions.KubeConfig, - testOptions.HubCluster.KubeContext) if utils.GetManagedClusterName(testOptions) == hubManagedClusterName { Skip("Skip the case for local-cluster since no observability addon") } diff --git a/tests/pkg/tests/observability_alert_test.go b/tests/pkg/tests/observability_alert_test.go index 0d21d1aa1..95a3ab6ed 100644 --- a/tests/pkg/tests/observability_alert_test.go +++ b/tests/pkg/tests/observability_alert_test.go @@ -39,11 +39,6 @@ var _ = Describe("Observability:", func() { testOptions.HubCluster.ClusterServerURL, testOptions.KubeConfig, testOptions.HubCluster.KubeContext) - - dynClient = utils.NewKubeClientDynamic( - testOptions.HubCluster.ClusterServerURL, - testOptions.KubeConfig, - testOptions.HubCluster.KubeContext) }) statefulsetLabels := [...]string{ ALERTMANAGER_LABEL, diff --git a/tests/pkg/tests/observability_certrenew_test.go b/tests/pkg/tests/observability_certrenew_test.go index cdebc9b8c..5a5d45dd2 100644 --- a/tests/pkg/tests/observability_certrenew_test.go +++ b/tests/pkg/tests/observability_certrenew_test.go @@ -15,19 +15,9 @@ import ( "github.com/stolostron/multicluster-observability-operator/tests/pkg/utils" ) -var _ = Describe("Observability:", func() { +func runCertRenewTests(clusterConfig utils.Cluster) { BeforeEach(func() { - hubClient = utils.NewKubeClient( - testOptions.HubCluster.ClusterServerURL, - testOptions.KubeConfig, - testOptions.HubCluster.KubeContext) - - dynClient = utils.NewKubeClientDynamic( - testOptions.HubCluster.ClusterServerURL, - testOptions.KubeConfig, - testOptions.HubCluster.KubeContext) - clusterName := utils.GetManagedClusterName(testOptions) - if clusterName == hubManagedClusterName { + if clusterConfig.Name == hubManagedClusterName { namespace = hubMetricsCollectorNamespace isHub = false } @@ -165,7 +155,7 @@ var _ = Describe("Observability:", func() { // debug code to check label "cert/time-restarted" deployment, err := utils.GetDeployment( - testOptions, + clusterConfig, isHub, "metrics-collector-deployment", namespace, @@ -189,4 +179,10 @@ var _ = Describe("Observability:", func() { namespace = MCO_ADDON_NAMESPACE isHub = false }) +} + +var _ = Describe("Observability:", func() { + for _, clusterConfig := range testOptions.ManagedClusters { + runCertRenewTests(clusterConfig) + } }) diff --git a/tests/pkg/tests/observability_dashboard_test.go b/tests/pkg/tests/observability_dashboard_test.go index a2ab14765..33e8d801a 100644 --- a/tests/pkg/tests/observability_dashboard_test.go +++ b/tests/pkg/tests/observability_dashboard_test.go @@ -19,18 +19,6 @@ const ( ) var _ = Describe("Observability:", func() { - BeforeEach(func() { - hubClient = utils.NewKubeClient( - testOptions.HubCluster.ClusterServerURL, - testOptions.KubeConfig, - testOptions.HubCluster.KubeContext) - - dynClient = utils.NewKubeClientDynamic( - testOptions.HubCluster.ClusterServerURL, - testOptions.KubeConfig, - testOptions.HubCluster.KubeContext) - }) - It("[P2][Sev2][observability][Stable] Should have custom dashboard which defined in configmap (dashboard/g0)", func() { By("Creating custom dashboard configmap") yamlB, _ := kustomize.Render( @@ -71,7 +59,7 @@ var _ = Describe("Observability:", func() { It("[P2][Sev2][observability][Stable] Should have no custom dashboard in grafana after related configmap removed (dashboard/g0)", func() { By("Deleting custom dashboard configmap") - err = utils.DeleteConfigMap(testOptions, true, dashboardName, MCO_NAMESPACE) + err = utils.DeleteConfigMap(testOptions.ManagedClusters[0], true, dashboardName, MCO_NAMESPACE) Expect(err).ToNot(HaveOccurred()) Eventually(func() bool { _, result := utils.ContainDashboard(testOptions, updateDashboardTitle) diff --git a/tests/pkg/tests/observability_endpoint_preserve_test.go b/tests/pkg/tests/observability_endpoint_preserve_test.go index c79d31332..30e206271 100644 --- a/tests/pkg/tests/observability_endpoint_preserve_test.go +++ b/tests/pkg/tests/observability_endpoint_preserve_test.go @@ -5,6 +5,7 @@ package tests import ( + "fmt" "os" . "github.com/onsi/ginkgo" @@ -17,19 +18,9 @@ import ( "github.com/stolostron/multicluster-observability-operator/tests/pkg/utils" ) -var _ = Describe("Observability:", func() { +func runMetricsCollectorTests(clusterConfig utils.Cluster) { BeforeEach(func() { - hubClient = utils.NewKubeClient( - testOptions.HubCluster.ClusterServerURL, - testOptions.KubeConfig, - testOptions.HubCluster.KubeContext) - - dynClient = utils.NewKubeClientDynamic( - testOptions.HubCluster.ClusterServerURL, - testOptions.KubeConfig, - testOptions.HubCluster.KubeContext) - clusterName := utils.GetManagedClusterName(testOptions) - if clusterName == hubManagedClusterName { + if clusterConfig.Name == hubManagedClusterName { namespace = hubMetricsCollectorNamespace isHub = false } @@ -42,46 +33,49 @@ var _ = Describe("Observability:", func() { err error dep *appv1.Deployment ) - Eventually(func() error { - dep, err = utils.GetDeployment( - testOptions, - isHub, - "metrics-collector-deployment", - namespace, - ) - return err - }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*1).Should(Succeed()) - - Eventually(func() error { - err = utils.DeleteDeployment( - testOptions, - isHub, - "metrics-collector-deployment", - namespace, - ) - return err - }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*1).Should(Succeed()) - - Eventually(func() bool { - newDep, err = utils.GetDeployment( - testOptions, - isHub, - "metrics-collector-deployment", - namespace, - ) - if err == nil { - if dep.ObjectMeta.ResourceVersion != newDep.ObjectMeta.ResourceVersion { - return true + By(fmt.Sprintf("Handling cluster: %s", clusterConfig.Name), func() { + Eventually(func() error { + dep, err = utils.GetDeployment( + clusterConfig, + isHub, + "metrics-collector-deployment", + namespace, + ) + return err + }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*1).Should(Succeed()) + + Eventually(func() error { + err = utils.DeleteDeployment( + testOptions, + isHub, + "metrics-collector-deployment", + namespace, + ) + return err + }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*1).Should(Succeed()) + + Eventually(func() bool { + newDep, err = utils.GetDeployment( + clusterConfig, + isHub, + "metrics-collector-deployment", + namespace, + ) + if err == nil { + if dep.ObjectMeta.ResourceVersion != newDep.ObjectMeta.ResourceVersion { + return true + } } - } - return false - }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*1).Should(BeTrue()) + return false + }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*1).Should(BeTrue()) + + }) }) It("[Stable] Updating metrics-collector deployment", func() { updateSaName := "test-serviceaccount" Eventually(func() error { newDep, err = utils.GetDeployment( - testOptions, + clusterConfig, isHub, "metrics-collector-deployment", namespace, @@ -91,7 +85,7 @@ var _ = Describe("Observability:", func() { } newDep.Spec.Template.Spec.ServiceAccountName = updateSaName newDep, err = utils.UpdateDeployment( - testOptions, + clusterConfig, isHub, "metrics-collector-deployment", namespace, @@ -102,7 +96,7 @@ var _ = Describe("Observability:", func() { Eventually(func() bool { revertDep, err := utils.GetDeployment( - testOptions, + clusterConfig, isHub, "metrics-collector-deployment", namespace, @@ -167,7 +161,7 @@ var _ = Describe("Observability:", func() { ) Eventually(func() error { err, cm = utils.GetConfigMap( - testOptions, + clusterConfig, isHub, "metrics-collector-serving-certs-ca-bundle", namespace, @@ -176,7 +170,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*1).Should(Succeed()) Eventually(func() error { err = utils.DeleteConfigMap( - testOptions, + clusterConfig, isHub, "metrics-collector-serving-certs-ca-bundle", namespace, @@ -186,7 +180,7 @@ var _ = Describe("Observability:", func() { newCm := &v1.ConfigMap{} Eventually(func() bool { err, newCm = utils.GetConfigMap( - testOptions, + clusterConfig, isHub, "metrics-collector-serving-certs-ca-bundle", namespace, @@ -213,4 +207,10 @@ var _ = Describe("Observability:", func() { isHub = false }) +} + +var _ = Describe("Observability:", func() { + for _, clusterConfig := range testOptions.ManagedClusters { + runMetricsCollectorTests(clusterConfig) + } }) diff --git a/tests/pkg/tests/observability_export_test.go b/tests/pkg/tests/observability_export_test.go index 508ca1159..48b077de4 100644 --- a/tests/pkg/tests/observability_export_test.go +++ b/tests/pkg/tests/observability_export_test.go @@ -16,18 +16,6 @@ import ( ) var _ = Describe("Observability:", func() { - BeforeEach(func() { - hubClient = utils.NewKubeClient( - testOptions.HubCluster.ClusterServerURL, - testOptions.KubeConfig, - testOptions.HubCluster.KubeContext) - - dynClient = utils.NewKubeClientDynamic( - testOptions.HubCluster.ClusterServerURL, - testOptions.KubeConfig, - testOptions.HubCluster.KubeContext) - }) - JustBeforeEach(func() { Eventually(func() error { clusters, clusterError = utils.ListManagedClusters(testOptions) diff --git a/tests/pkg/tests/observability_grafana_test.go b/tests/pkg/tests/observability_grafana_test.go index 908351401..ae2186c31 100644 --- a/tests/pkg/tests/observability_grafana_test.go +++ b/tests/pkg/tests/observability_grafana_test.go @@ -14,18 +14,6 @@ import ( ) var _ = Describe("Observability:", func() { - BeforeEach(func() { - hubClient = utils.NewKubeClient( - testOptions.HubCluster.ClusterServerURL, - testOptions.KubeConfig, - testOptions.HubCluster.KubeContext) - - dynClient = utils.NewKubeClientDynamic( - testOptions.HubCluster.ClusterServerURL, - testOptions.KubeConfig, - testOptions.HubCluster.KubeContext) - }) - It("@BVT - [P1][Sev1][observability][Stable] Should have metric data in grafana console (grafana/g0)", func() { Eventually(func() error { clusters, err := utils.ListManagedClusters(testOptions) diff --git a/tests/pkg/tests/observability_manifestwork_test.go b/tests/pkg/tests/observability_manifestwork_test.go index 193abd615..dbab33fd9 100644 --- a/tests/pkg/tests/observability_manifestwork_test.go +++ b/tests/pkg/tests/observability_manifestwork_test.go @@ -18,15 +18,6 @@ import ( var _ = Describe("Observability:", func() { BeforeEach(func() { - hubClient = utils.NewKubeClient( - testOptions.HubCluster.ClusterServerURL, - testOptions.KubeConfig, - testOptions.HubCluster.KubeContext) - - dynClient = utils.NewKubeClientDynamic( - testOptions.HubCluster.ClusterServerURL, - testOptions.KubeConfig, - testOptions.HubCluster.KubeContext) if utils.GetManagedClusterName(testOptions) == hubManagedClusterName { Skip("Skip the case for local-cluster since no observability addon") } diff --git a/tests/pkg/tests/observability_metrics_test.go b/tests/pkg/tests/observability_metrics_test.go index 03f1b59e2..c1de483c5 100644 --- a/tests/pkg/tests/observability_metrics_test.go +++ b/tests/pkg/tests/observability_metrics_test.go @@ -31,11 +31,6 @@ var _ = Describe("Observability:", func() { testOptions.HubCluster.ClusterServerURL, testOptions.KubeConfig, testOptions.HubCluster.KubeContext) - - dynClient = utils.NewKubeClientDynamic( - testOptions.HubCluster.ClusterServerURL, - testOptions.KubeConfig, - testOptions.HubCluster.KubeContext) }) JustBeforeEach(func() { diff --git a/tests/pkg/tests/observability_observatorium_preserve_test.go b/tests/pkg/tests/observability_observatorium_preserve_test.go index b1bbd2aab..b099e4ecd 100644 --- a/tests/pkg/tests/observability_observatorium_preserve_test.go +++ b/tests/pkg/tests/observability_observatorium_preserve_test.go @@ -18,11 +18,6 @@ import ( var _ = Describe("Observability:", func() { BeforeEach(func() { - hubClient = utils.NewKubeClient( - testOptions.HubCluster.ClusterServerURL, - testOptions.KubeConfig, - testOptions.HubCluster.KubeContext) - dynClient = utils.NewKubeClientDynamic( testOptions.HubCluster.ClusterServerURL, testOptions.KubeConfig, diff --git a/tests/pkg/utils/client.go b/tests/pkg/utils/client.go index cb606ee11..3ea7e6bd5 100644 --- a/tests/pkg/utils/client.go +++ b/tests/pkg/utils/client.go @@ -23,6 +23,13 @@ func getKubeClient(opt TestOptions, isHub bool) kubernetes.Interface { return clientKube } +func getKubeClientForCluster(clusterConfig Cluster, isHub bool) kubernetes.Interface { + return NewKubeClient( + clusterConfig.ClusterServerURL, + clusterConfig.KubeConfig, + clusterConfig.KubeContext) +} + func GetKubeClientDynamic(opt TestOptions, isHub bool) dynamic.Interface { url := opt.HubCluster.ClusterServerURL kubeConfig := opt.KubeConfig diff --git a/tests/pkg/utils/mco_configmaps.go b/tests/pkg/utils/mco_configmaps.go index 5ec376848..ac48179b4 100644 --- a/tests/pkg/utils/mco_configmaps.go +++ b/tests/pkg/utils/mco_configmaps.go @@ -12,9 +12,9 @@ import ( "k8s.io/klog" ) -func GetConfigMap(opt TestOptions, isHub bool, name string, +func GetConfigMap(clusterConfig Cluster, isHub bool, name string, namespace string) (error, *corev1.ConfigMap) { - clientKube := getKubeClient(opt, isHub) + clientKube := getKubeClientForCluster(clusterConfig, isHub) cm, err := clientKube.CoreV1().ConfigMaps(namespace).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { klog.Errorf("Failed to get configmap %s in namespace %s due to %v", name, namespace, err) @@ -22,8 +22,8 @@ func GetConfigMap(opt TestOptions, isHub bool, name string, return err, cm } -func DeleteConfigMap(opt TestOptions, isHub bool, name string, namespace string) error { - clientKube := getKubeClient(opt, isHub) +func DeleteConfigMap(clusterConfig Cluster, isHub bool, name string, namespace string) error { + clientKube := getKubeClientForCluster(clusterConfig, isHub) err := clientKube.CoreV1().ConfigMaps(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) if err != nil { klog.Errorf("Failed to delete configmap %s in namespace %s due to %v", name, namespace, err) diff --git a/tests/pkg/utils/mco_deployments.go b/tests/pkg/utils/mco_deployments.go index fa2841d83..f3856e448 100644 --- a/tests/pkg/utils/mco_deployments.go +++ b/tests/pkg/utils/mco_deployments.go @@ -6,16 +6,15 @@ package utils import ( "context" - "errors" appv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog" ) -func GetDeployment(opt TestOptions, isHub bool, name string, +func GetDeployment(clusterConfig Cluster, isHub bool, name string, namespace string) (*appv1.Deployment, error) { - clientKube := getKubeClient(opt, isHub) + clientKube := getKubeClientForCluster(clusterConfig, isHub) klog.V(1).Infof("Get deployment <%v> in namespace <%v>, isHub: <%v>", name, namespace, isHub) dep, err := clientKube.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { @@ -51,46 +50,15 @@ func DeleteDeployment(opt TestOptions, isHub bool, name string, namespace string } func UpdateDeployment( - opt TestOptions, + clusterConfig Cluster, isHub bool, name string, namespace string, dep *appv1.Deployment) (*appv1.Deployment, error) { - clientKube := getKubeClient(opt, isHub) + clientKube := getKubeClientForCluster(clusterConfig, isHub) updateDep, err := clientKube.AppsV1().Deployments(namespace).Update(context.TODO(), dep, metav1.UpdateOptions{}) if err != nil { klog.Errorf("Failed to update deployment %s in namespace %s due to %v", name, namespace, err) } return updateDep, err } - -func UpdateDeploymentReplicas( - opt TestOptions, - deployName, crProperty string, - desiredReplicas, expectedReplicas int32, -) error { - clientDynamic := GetKubeClientDynamic(opt, true) - deploy, err := GetDeployment(opt, true, deployName, MCO_NAMESPACE) - if err != nil { - return err - } - deploy.Spec.Replicas = &desiredReplicas - _, err = UpdateDeployment(opt, true, deployName, MCO_NAMESPACE, deploy) - if err != nil { - return err - } - - obs, err := clientDynamic.Resource(NewMCOMObservatoriumGVR()). - Namespace(MCO_NAMESPACE). - Get(context.TODO(), MCO_CR_NAME, metav1.GetOptions{}) - if err != nil { - return err - } - thanos := obs.Object["spec"].(map[string]interface{})["thanos"] - currentReplicas := thanos.(map[string]interface{})[crProperty].(map[string]interface{})["replicas"].(int64) - if int(currentReplicas) != int(expectedReplicas) { - klog.Errorf("Failed to update deployment %s replicas to %v", deployName, expectedReplicas) - return errors.New("the replicas was not updated successfully") - } - return nil -} diff --git a/tests/pkg/utils/mco_managedcluster.go b/tests/pkg/utils/mco_managedcluster.go index a92b4e205..763c85405 100644 --- a/tests/pkg/utils/mco_managedcluster.go +++ b/tests/pkg/utils/mco_managedcluster.go @@ -51,8 +51,10 @@ func ListManagedClusters(opt TestOptions) ([]string, error) { for _, obj := range objs.Items { metadata := obj.Object["metadata"].(map[string]interface{}) name := metadata["name"].(string) - labels := metadata["labels"].(map[string]interface{}) - if labels != nil { + if name == "local-cluster" { + continue + } + if labels, ok := metadata["labels"].(map[string]interface{}); ok { obsControllerStr := "" if obsController, ok := labels["feature.open-cluster-management.io/addon-observability-controller"]; ok { obsControllerStr = obsController.(string) diff --git a/tests/pkg/utils/mco_oba.go b/tests/pkg/utils/mco_oba.go index 43dfbdae1..9a2e92d3c 100644 --- a/tests/pkg/utils/mco_oba.go +++ b/tests/pkg/utils/mco_oba.go @@ -54,25 +54,6 @@ func CheckOBADeleted(opt TestOptions, namespace string) error { return nil } -func CheckManagedClusterAddonsStatus(opt TestOptions, namespace, status string) error { - dynClient := NewKubeClientDynamic( - opt.HubCluster.ClusterServerURL, - opt.KubeConfig, - opt.HubCluster.KubeContext) - - mca, err := dynClient.Resource(NewMCOManagedClusterAddonsGVR()). - Namespace(namespace). - Get(context.TODO(), "observability-controller", metav1.GetOptions{}) - if err != nil { - return err - } - if mca.Object["status"] != nil && strings.Contains(fmt.Sprint(mca.Object["status"]), status) { - return nil - } else { - return fmt.Errorf("observability-controller is disabled for managed cluster %s", namespace) - } -} - func CheckAllOBAsEnabled(opt TestOptions) error { clusters, err := ListManagedClusters(opt) if err != nil { @@ -95,29 +76,6 @@ func CheckAllOBAsEnabled(opt TestOptions) error { return nil } -func CheckAllOBADisabled(opt TestOptions) error { - clusters, err := ListManagedClusters(opt) - if err != nil { - return err - } - for _, cluster := range clusters { - // skip the check for local-cluster - if cluster == "local-cluster" { - klog.V(1).Infof("Skip OBA status for managedcluster: %v", cluster) - continue - } - err = CheckOBAStatus(opt, cluster, ManagedClusterAddOnDisabledMessage) - if err != nil { - return err - } - err = CheckManagedClusterAddonsStatus(opt, cluster, ManagedClusterAddOnDisabledMessage) - if err != nil { - return err - } - } - return nil -} - func CheckAllOBAsDeleted(opt TestOptions) error { clusters, err := ListManagedClusters(opt) if err != nil { diff --git a/tests/run-in-kind/kind/kind-managed.config.yaml b/tests/run-in-kind/kind/kind-managed.config.yaml new file mode 100644 index 000000000..9ad36aac7 --- /dev/null +++ b/tests/run-in-kind/kind/kind-managed.config.yaml @@ -0,0 +1,17 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 80 + hostPort: 83 + listenAddress: "0.0.0.0" + - containerPort: 444 + hostPort: 445 + listenAddress: "0.0.0.0" + - containerPort: 6444 + hostPort: 32808 + listenAddress: "0.0.0.0" + - containerPort: 31002 + hostPort: 31003 + listenAddress: "127.0.0.1" diff --git a/tests/run-in-kind/run-e2e-in-kind.sh b/tests/run-in-kind/run-e2e-in-kind.sh index 49d36ec29..4838b46a3 100755 --- a/tests/run-in-kind/run-e2e-in-kind.sh +++ b/tests/run-in-kind/run-e2e-in-kind.sh @@ -14,7 +14,7 @@ export IS_KIND_ENV=true source ${WORKDIR}/env.sh create_kind_cluster() { - echo "Delete the KinD cluster if exists" + echo "Delete the KinD cluster if exists coleen" kind delete cluster --name $1 || true rm -rf $HOME/.kube/kind-config-$1 @@ -51,6 +51,7 @@ run_e2e_test() { run() { create_kind_cluster hub + create_kind_cluster_managed managed deploy_crds deploy_templates deploy_service_ca_operator