diff --git a/test/e2e/clusterclass_rollout.go b/test/e2e/clusterclass_rollout.go index e9c1c3238429..d546fc9579fa 100644 --- a/test/e2e/clusterclass_rollout.go +++ b/test/e2e/clusterclass_rollout.go @@ -415,7 +415,7 @@ func assertControlPlane(g Gomega, clusterClassObjects clusterClassObjects, clust clusterv1.TemplateClonedFromNameAnnotation: clusterClass.Spec.ControlPlane.MachineInfrastructure.Ref.Name, }, clusterClassObjects.ControlPlaneInfrastructureMachineTemplate.GetAnnotations(), - ).without(g, corev1.LastAppliedConfigAnnotation), + ), )) // ControlPlane InfrastructureMachineTemplate.spec.template.metadata @@ -587,7 +587,7 @@ func assertMachineDeployments(g Gomega, clusterClassObjects clusterClassObjects, clusterv1.TemplateClonedFromNameAnnotation: mdClass.Template.Infrastructure.Ref.Name, }, ccInfrastructureMachineTemplate.GetAnnotations(), - ).without(g, corev1.LastAppliedConfigAnnotation), + ), )) // MachineDeployment InfrastructureMachineTemplate.spec.template.metadata g.Expect(infrastructureMachineTemplateTemplateMetadata.Labels).To(BeEquivalentTo( @@ -619,7 +619,7 @@ func assertMachineDeployments(g Gomega, clusterClassObjects clusterClassObjects, clusterv1.TemplateClonedFromNameAnnotation: mdClass.Template.Bootstrap.Ref.Name, }, ccBootstrapConfigTemplate.GetAnnotations(), - ).without(g, corev1.LastAppliedConfigAnnotation), + ), )) // MachineDeployment BootstrapConfigTemplate.spec.template.metadata g.Expect(bootstrapConfigTemplateTemplateMetadata.Labels).To(BeEquivalentTo( diff --git a/test/e2e/clusterctl_upgrade.go b/test/e2e/clusterctl_upgrade.go index af4f8475d821..3ec5e5d96dd6 100644 --- a/test/e2e/clusterctl_upgrade.go +++ b/test/e2e/clusterctl_upgrade.go @@ -411,7 +411,7 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg Expect(workloadClusterTemplate).ToNot(BeNil(), "Failed to get the cluster template") log.Logf("Applying the cluster template yaml to the cluster") - Expect(managementClusterProxy.Apply(ctx, workloadClusterTemplate)).To(Succeed()) + Expect(managementClusterProxy.CreateOrUpdate(ctx, workloadClusterTemplate)).To(Succeed()) if input.PreWaitForCluster != nil { By("Running PreWaitForCluster steps against the management cluster") diff --git a/test/e2e/kcp_adoption.go b/test/e2e/kcp_adoption.go index 44b9fe1e2a07..0c60fa2451cb 100644 --- a/test/e2e/kcp_adoption.go +++ b/test/e2e/kcp_adoption.go @@ -138,7 +138,7 @@ func KCPAdoptionSpec(ctx context.Context, inputGetter func() KCPAdoptionSpecInpu Expect(workloadClusterTemplate).ToNot(BeNil(), "Failed to get the cluster template") By("Applying the cluster template yaml to the cluster with the 'initial' selector") - Expect(input.BootstrapClusterProxy.Apply(ctx, workloadClusterTemplate, "--selector", "kcp-adoption.step1")).ShouldNot(HaveOccurred()) + Expect(input.BootstrapClusterProxy.CreateOrUpdate(ctx, workloadClusterTemplate, "kcp-adoption.step1")).ShouldNot(HaveOccurred()) cluster = framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{ Getter: client, @@ -159,7 +159,7 @@ func KCPAdoptionSpec(ctx context.Context, inputGetter func() KCPAdoptionSpecInpu }, WaitForControlPlaneIntervals...) By("Applying the cluster template yaml to the cluster with the 'kcp' selector") - Expect(input.BootstrapClusterProxy.Apply(ctx, workloadClusterTemplate, "--selector", "kcp-adoption.step2")).ShouldNot(HaveOccurred()) + Expect(input.BootstrapClusterProxy.CreateOrUpdate(ctx, workloadClusterTemplate, "kcp-adoption.step2")).ShouldNot(HaveOccurred()) var controlPlane *controlplanev1.KubeadmControlPlane Eventually(func() *controlplanev1.KubeadmControlPlane { diff --git a/test/e2e/kcp_remediations.go b/test/e2e/kcp_remediations.go index 0f93c58c367f..982cf0e777f1 100644 --- a/test/e2e/kcp_remediations.go +++ b/test/e2e/kcp_remediations.go @@ -489,7 +489,7 @@ func createWorkloadClusterAndWait(ctx context.Context, input createWorkloadClust Expect(workloadClusterTemplate).ToNot(BeNil(), "Failed to get the cluster template") Eventually(func() error { - return input.Proxy.Apply(ctx, workloadClusterTemplate) + return input.Proxy.CreateOrUpdate(ctx, workloadClusterTemplate) }, 10*time.Second).Should(Succeed(), "Failed to apply the cluster template") log.Logf("Waiting for the cluster infrastructure to be provisioned") diff --git a/test/e2e/scale.go b/test/e2e/scale.go index 7e0d3b9d6983..dc5b1f1eeba5 100644 --- a/test/e2e/scale.go +++ b/test/e2e/scale.go @@ -290,7 +290,7 @@ func scaleSpec(ctx context.Context, inputGetter func() scaleSpecInput) { clusterClassYAML := bytes.Replace(baseClusterClassYAML, []byte(scaleClusterNamespacePlaceholder), []byte(namespace.Name), -1) log.Logf("Apply ClusterClass") Eventually(func() error { - return input.BootstrapClusterProxy.Apply(ctx, clusterClassYAML) + return input.BootstrapClusterProxy.CreateOrUpdate(ctx, clusterClassYAML) }, 1*time.Minute).Should(Succeed()) } else { log.Logf("ClusterClass already exists. Skipping creation.") @@ -551,7 +551,7 @@ func getClusterCreateAndWaitFn(input clusterctl.ApplyCustomClusterTemplateAndWai WaitForClusterIntervals: input.WaitForClusterIntervals, WaitForControlPlaneIntervals: input.WaitForControlPlaneIntervals, WaitForMachineDeployments: input.WaitForMachineDeployments, - Args: input.Args, + LabelSelectors: input.LabelSelectors, PreWaitForCluster: input.PreWaitForCluster, PostMachinesProvisioned: input.PostMachinesProvisioned, ControlPlaneWaiters: input.ControlPlaneWaiters, @@ -563,7 +563,7 @@ func getClusterCreateFn(clusterProxy framework.ClusterProxy) clusterCreator { return func(ctx context.Context, namespace, clusterName string, clusterTemplateYAML []byte) { log.Logf("Applying the cluster template yaml of cluster %s", klog.KRef(namespace, clusterName)) Eventually(func() error { - return clusterProxy.Apply(ctx, clusterTemplateYAML) + return clusterProxy.CreateOrUpdate(ctx, clusterTemplateYAML) }, 1*time.Minute).Should(Succeed(), "Failed to apply the cluster template of cluster %s", klog.KRef(namespace, clusterName)) } } @@ -616,7 +616,7 @@ func createClusterWorker(ctx context.Context, clusterProxy framework.ClusterProx log.Logf("Apply ClusterClass in namespace %", namespaceName) clusterClassYAML := bytes.Replace(baseClusterClassYAML, []byte(scaleClusterNamespacePlaceholder), []byte(namespaceName), -1) Eventually(func() error { - return clusterProxy.Apply(ctx, clusterClassYAML) + return clusterProxy.CreateOrUpdate(ctx, clusterClassYAML) }, 1*time.Minute).Should(Succeed()) } diff --git a/test/framework/autoscaler_helpers.go b/test/framework/autoscaler_helpers.go index c1f3f5097080..13a5a33ee153 100644 --- a/test/framework/autoscaler_helpers.go +++ b/test/framework/autoscaler_helpers.go @@ -107,7 +107,7 @@ func ApplyAutoscalerToWorkloadCluster(ctx context.Context, input ApplyAutoscaler }, }) Expect(err).ToNot(HaveOccurred(), "failed to parse %s", workloadYamlTemplate) - Expect(input.WorkloadClusterProxy.Apply(ctx, workloadYaml)).To(Succeed(), "failed to apply %s", workloadYamlTemplate) + Expect(input.WorkloadClusterProxy.CreateOrUpdate(ctx, workloadYaml)).To(Succeed(), "failed to apply %s", workloadYamlTemplate) By("Wait for the autoscaler deployment and collect logs") deployment := &appsv1.Deployment{ diff --git a/test/framework/cluster_proxy.go b/test/framework/cluster_proxy.go index f1b58b425fe4..0bdf519f2a75 100644 --- a/test/framework/cluster_proxy.go +++ b/test/framework/cluster_proxy.go @@ -22,17 +22,21 @@ import ( "fmt" "net/url" "os" - "os/exec" "path" goruntime "runtime" + "strings" "sync" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - pkgerrors "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -44,9 +48,9 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" - testexec "sigs.k8s.io/cluster-api/test/framework/exec" "sigs.k8s.io/cluster-api/test/framework/internal/log" "sigs.k8s.io/cluster-api/test/infrastructure/container" + "sigs.k8s.io/cluster-api/util/yaml" ) const ( @@ -89,6 +93,9 @@ type ClusterProxy interface { // GetLogCollector returns the machine log collector for the Kubernetes cluster. GetLogCollector() ClusterLogCollector + // Creates objects using the clusterProxy client + CreateOrUpdate(ctx context.Context, resources []byte, selector ...string) error + // Apply to apply YAML to the Kubernetes cluster, `kubectl apply`. Apply(ctx context.Context, resources []byte, args ...string) error @@ -247,20 +254,60 @@ func (p *clusterProxy) GetCache(ctx context.Context) cache.Cache { return p.cache } -// Apply wraps `kubectl apply ...` and prints the output so we can see what gets applied to the cluster. -func (p *clusterProxy) Apply(ctx context.Context, resources []byte, args ...string) error { - Expect(ctx).NotTo(BeNil(), "ctx is required for Apply") - Expect(resources).NotTo(BeNil(), "resources is required for Apply") - - if err := testexec.KubectlApply(ctx, p.kubeconfigPath, resources, args...); err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - return pkgerrors.New(fmt.Sprintf("%s: stderr: %s", err.Error(), exitErr.Stderr)) +// Creates or updates objects using the clusterProxy client. +func (p *clusterProxy) CreateOrUpdate(ctx context.Context, resources []byte, selector ...string) error { + Expect(ctx).NotTo(BeNil(), "ctx is required for Create") + Expect(resources).NotTo(BeNil(), "resources is required for Create") + labelSelector := labels.Everything() + // Construct label selector from parameters if provided + if selector != nil { + if len(selector) > 1 { + return fmt.Errorf("%s", fmt.Sprintf("only one selector is supported, got %d", len(selector))) + } + ls, err := metav1.ParseToLabelSelector(selector[0]) + if err != nil { + return fmt.Errorf("could not parse selector: %+w", err) + } + labelSelector, err = metav1.LabelSelectorAsSelector(ls) + if err != nil { + return fmt.Errorf("could not convert metav1.LabelSelector to labels.Selector: %+w", err) } + } + var retErrs []error + objs, err := yaml.ToUnstructured(resources) + if err != nil { return err } - return nil + existingObject := &unstructured.Unstructured{} + for _, o := range objs { + o := o + objectKey := types.NamespacedName{ + Name: o.GetName(), + Namespace: o.GetNamespace(), + } + existingObject.SetAPIVersion(o.GetAPIVersion()) + existingObject.SetKind(o.GetKind()) + labels := labels.Set(o.GetLabels()) + if labelSelector.Matches(labels) { + if err := p.GetClient().Get(ctx, objectKey, existingObject); err != nil { + // Expected error -- if the object does not exist, create it + if strings.Contains(err.Error(), "not found") { + if err := p.GetClient().Create(ctx, &o); err != nil { + retErrs = append(retErrs, err) + } + } else { + retErrs = append(retErrs, err) + } + } else { + o.SetResourceVersion(existingObject.GetResourceVersion()) + if err := p.GetClient().Update(ctx, &o); err != nil { + retErrs = append(retErrs, err) + } + } + } + } + return kerrors.NewAggregate(retErrs) } func (p *clusterProxy) GetRESTConfig() *rest.Config { diff --git a/test/framework/clusterctl/clusterctl_helpers.go b/test/framework/clusterctl/clusterctl_helpers.go index 38788af2aab4..d34436192f27 100644 --- a/test/framework/clusterctl/clusterctl_helpers.go +++ b/test/framework/clusterctl/clusterctl_helpers.go @@ -358,7 +358,7 @@ func ApplyClusterTemplateAndWait(ctx context.Context, input ApplyClusterTemplate WaitForControlPlaneIntervals: input.WaitForControlPlaneIntervals, WaitForMachineDeployments: input.WaitForMachineDeployments, WaitForMachinePools: input.WaitForMachinePools, - Args: input.Args, + LabelSelectors: input.Args, PreWaitForCluster: input.PreWaitForCluster, PostMachinesProvisioned: input.PostMachinesProvisioned, ControlPlaneWaiters: input.ControlPlaneWaiters, @@ -377,7 +377,7 @@ type ApplyCustomClusterTemplateAndWaitInput struct { WaitForControlPlaneIntervals []interface{} WaitForMachineDeployments []interface{} WaitForMachinePools []interface{} - Args []string // extra args to be used during `kubectl apply` + LabelSelectors []string // label selector to be used during creation or update of objects PreWaitForCluster func() PostMachinesProvisioned func() ControlPlaneWaiters @@ -412,7 +412,7 @@ func ApplyCustomClusterTemplateAndWait(ctx context.Context, input ApplyCustomClu log.Logf("Applying the cluster template yaml of cluster %s", klog.KRef(input.Namespace, input.ClusterName)) Eventually(func() error { - return input.ClusterProxy.Apply(ctx, input.CustomTemplateYAML, input.Args...) + return input.ClusterProxy.CreateOrUpdate(ctx, input.CustomTemplateYAML, input.LabelSelectors...) }, 1*time.Minute).Should(Succeed(), "Failed to apply the cluster template") // Once we applied the cluster template we can run PreWaitForCluster. @@ -448,7 +448,7 @@ func ApplyCustomClusterTemplateAndWait(ctx context.Context, input ApplyCustomClu cniYaml, err := os.ReadFile(input.CNIManifestPath) Expect(err).ShouldNot(HaveOccurred()) - Expect(workloadCluster.Apply(ctx, cniYaml)).ShouldNot(HaveOccurred()) + Expect(workloadCluster.CreateOrUpdate(ctx, cniYaml)).ShouldNot(HaveOccurred()) } log.Logf("Waiting for control plane of cluster %s to be ready", klog.KRef(input.Namespace, input.ClusterName))