From 9955bae30c75229bf84e1c111bb4d5d33653ebd6 Mon Sep 17 00:00:00 2001 From: Justin Kulikauskas <44813129+JustinKuli@users.noreply.github.com> Date: Mon, 20 May 2024 05:13:14 +0000 Subject: [PATCH] WIP --- Makefile | 4 + api/v1beta1/policycore_types.go | 24 +++ pkg/compliance/k8sEventEmitter.go | 111 ++++++++++++ pkg/testutils/courtesies.go | 36 ++++ pkg/testutils/courtesies_test.go | 155 ++++++++++++++++ pkg/testutils/toolkit.go | 114 +++++++++++- .../api/v1beta1/fakepolicy_types.go | 35 +++- ...en-cluster-management.io_fakepolicies.yaml | 8 +- test/fakepolicy/config/deploy.yaml | 8 +- .../controllers/fakepolicy_controller.go | 61 ++++++- .../test/compliance/complianceEvent_test.go | 169 ++++++++++++++++++ test/fakepolicy/test/compliance/suite_test.go | 84 +++++++++ 12 files changed, 791 insertions(+), 18 deletions(-) create mode 100644 pkg/compliance/k8sEventEmitter.go create mode 100644 pkg/testutils/courtesies_test.go create mode 100644 test/fakepolicy/test/compliance/complianceEvent_test.go create mode 100644 test/fakepolicy/test/compliance/suite_test.go diff --git a/Makefile b/Makefile index 2b23318..1115764 100644 --- a/Makefile +++ b/Makefile @@ -117,6 +117,10 @@ test-basicsuite: manifests generate $(GINKGO) $(ENVTEST) ## Run just the basic s KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" $(GINKGO) \ --coverpkg=./... --covermode=count --coverprofile=cover-basic.out ./test/fakepolicy/test/basic +test-compsuite: manifests generate $(GINKGO) $(ENVTEST) ## Run just the basic suite of tests + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" $(GINKGO) \ + --coverpkg=./... --covermode=count --coverprofile=cover-comp.out ./test/fakepolicy/test/compliance + .PHONY: fuzz-test fuzz-test: go test ./api/v1beta1 -fuzz=FuzzMatchesExcludeAll -fuzztime=20s diff --git a/api/v1beta1/policycore_types.go b/api/v1beta1/policycore_types.go index 2acde7a..3dea60f 100644 --- a/api/v1beta1/policycore_types.go +++ b/api/v1beta1/policycore_types.go @@ -193,3 +193,27 @@ type PolicyCore struct { Spec PolicyCoreSpec `json:"spec,omitempty"` Status PolicyCoreStatus `json:"status,omitempty"` } + +//+kubebuilder:object:generate=false + +// PolicyLike is an interface that policies should implement so that they can be worked with +// generally, without worrying about the specific kind of policy. +type PolicyLike interface { + client.Object + + // The ComplianceState (Compliant/NonCompliant) of the specific policy. + ComplianceState() ComplianceState + + // A human-readable string describing the current state of the policy, and why it is either + // Compliant or NonCompliant. + ComplianceMessage() string + + // The "parent" object on this cluster for the specific policy. Generally a Policy, in the API + // GroupVersion `policy.open-cluster-management.io/v1`. For namespaced kinds of policies, this + // will usually be the owner of the policy. For cluster-scoped policies, this must be stored + // some other way. + Parent() metav1.OwnerReference + + // The namespace of the "parent" object. + ParentNamespace() string +} diff --git a/pkg/compliance/k8sEventEmitter.go b/pkg/compliance/k8sEventEmitter.go new file mode 100644 index 0000000..a74bbb9 --- /dev/null +++ b/pkg/compliance/k8sEventEmitter.go @@ -0,0 +1,111 @@ +// Copyright Contributors to the Open Cluster Management project + +package compliance + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + nucleusv1beta1 "open-cluster-management.io/governance-policy-nucleus/api/v1beta1" +) + +// Can't be an interface because then Emit couldn't be a method +type K8sEmitter struct { + Client client.Client // TODO: required + Source corev1.EventSource // TODO: optional? + + // TODO: debatable for inclusion; allows tweaks of the event like adding/removing labels + // if not included, we could skip creating the event, just build it for the user to send + // but if it is included, we can build bigger things on top of this, and still have extensibility + Mutators []func(corev1.Event) (corev1.Event, error) +} + +// TODO: maybe this would be a good interface for multiple emitters? +func (e K8sEmitter) Emit(ctx context.Context, pl nucleusv1beta1.PolicyLike) error { + _, err := e.EmitEvent(ctx, pl) + + return err +} + +// TODO: doc +func (e K8sEmitter) EmitEvent(ctx context.Context, pl nucleusv1beta1.PolicyLike) (*corev1.Event, error) { + plGVK := pl.GetObjectKind().GroupVersionKind() + time := time.Now() + + // This event name matches the convention of recorders from client-go + name := fmt.Sprintf("%v.%x", pl.Parent().Name, time.UnixNano()) + + // The reason must match a pattern looked for by the policy framework + var reason string + if ns := pl.GetNamespace(); ns != "" { + reason = "policy: " + ns + "/" + pl.GetName() + } else { + reason = "policy: " + pl.GetName() + } + + // The message must begin with the compliance, then should go into a descriptive message + message := string(pl.ComplianceState()) + "; " + pl.ComplianceMessage() + + evType := "Normal" + if pl.ComplianceState() != nucleusv1beta1.Compliant { + evType = "Warning" + } + + event := corev1.Event{ + TypeMeta: metav1.TypeMeta{ + Kind: "Event", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: pl.ParentNamespace(), + Labels: pl.GetLabels(), + Annotations: pl.GetAnnotations(), + }, + InvolvedObject: corev1.ObjectReference{ + Kind: pl.Parent().Kind, + Namespace: pl.ParentNamespace(), + Name: pl.Parent().Name, + UID: pl.Parent().UID, + APIVersion: pl.Parent().APIVersion, + }, + Reason: reason, + Message: message, + Source: e.Source, + FirstTimestamp: metav1.NewTime(time), + LastTimestamp: metav1.NewTime(time), + Count: 1, + Type: evType, + EventTime: metav1.NewMicroTime(time), // does this break anything? + Series: nil, + Action: "ComplianceStateUpdate", + Related: &corev1.ObjectReference{ + Kind: plGVK.Kind, + Namespace: pl.GetNamespace(), + Name: pl.GetName(), + UID: pl.GetUID(), + APIVersion: plGVK.GroupVersion().String(), + ResourceVersion: pl.GetResourceVersion(), + }, + ReportingController: e.Source.Component, + ReportingInstance: e.Source.Host, + } + + for _, mutatorFunc := range e.Mutators { + var err error + + event, err = mutatorFunc(event) + if err != nil { + return nil, err + } + } + + err := e.Client.Create(ctx, &event) + + return &event, err +} diff --git a/pkg/testutils/courtesies.go b/pkg/testutils/courtesies.go index 797ebc1..6f11fb9 100644 --- a/pkg/testutils/courtesies.go +++ b/pkg/testutils/courtesies.go @@ -3,13 +3,49 @@ package testutils import ( + "regexp" + "time" + + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) +// ObjNN returns a NamespacedName for the given Object. func ObjNN(obj client.Object) types.NamespacedName { return types.NamespacedName{ Namespace: obj.GetNamespace(), Name: obj.GetName(), } } + +// EventFilter filters the given events. Any of the filter parameters can be passed an empty +// value to ignore that field when filtering. The msg parameter will be compiled into a regex if +// possible. The since parameter checks against the event's EventTime - but if the event does not +// specify an EventTime, it will not be filtered out. +func EventFilter(evs []corev1.Event, evType, msg string, since time.Time) []corev1.Event { + msgRegex, err := regexp.Compile(msg) + if err != nil { + msgRegex = regexp.MustCompile(regexp.QuoteMeta(msg)) + } + + ans := make([]corev1.Event, 0) + + for _, ev := range evs { + if evType != "" && ev.Type != evType { + continue + } + + if !msgRegex.MatchString(ev.Message) { + continue + } + + if !ev.EventTime.IsZero() && since.After(ev.EventTime.Time) { + continue + } + + ans = append(ans, ev) + } + + return ans +} diff --git a/pkg/testutils/courtesies_test.go b/pkg/testutils/courtesies_test.go new file mode 100644 index 0000000..49d3dc3 --- /dev/null +++ b/pkg/testutils/courtesies_test.go @@ -0,0 +1,155 @@ +// Copyright Contributors to the Open Cluster Management project + +package testutils + +import ( + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestObjNN(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + inpObj client.Object + wantName string + wantNS string + }{ + "namespaced unstructured": { + inpObj: &unstructured.Unstructured{Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "foo", + "namespace": "world", + }, + }}, + wantName: "foo", + wantNS: "world", + }, + "cluster-scoped unstructured": { + inpObj: &unstructured.Unstructured{Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "bar", + }, + }}, + wantName: "bar", + wantNS: "", + }, + "(namespaced) configmap": { + inpObj: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "my-cm", + Namespace: "kube-one", + }}, + wantName: "my-cm", + wantNS: "kube-one", + }, + "(cluster-scoped) node": { + inpObj: &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "unit-tests-only", + }}, + wantName: "unit-tests-only", + wantNS: "", + }, + } + + for name, tcase := range tests { + got := ObjNN(tcase.inpObj) + + if got.Name != tcase.wantName { + t.Errorf("Wanted name '%v', got '%v' in test '%v'", tcase.wantName, got.Name, name) + } + + if got.Namespace != tcase.wantNS { + t.Errorf("Wanted namespace '%v', got '%v' in test '%v'", tcase.wantNS, got.Namespace, name) + } + } +} + +func TestEventFilter(t *testing.T) { + t.Parallel() + + now := time.Now() + old := now.Add(-time.Minute) + veryOld := now.Add(-time.Hour) + + sampleEvents := []corev1.Event{{ + Message: "hello", + Type: "Normal", + EventTime: metav1.NewMicroTime(veryOld), + }, { + Message: "goodbye", + Type: "Warning", + EventTime: metav1.NewMicroTime(old), + }, { + Message: "carpe diem [", + Type: "Normal", + EventTime: metav1.NewMicroTime(now), + }, { + Message: "what time is it?", + Type: "Warning", + }} + + tests := map[string]struct { + inpType string + inpMsg string + inpSince time.Time + wantIdxs []int + }{ + "#NoFilter": { + inpType: "", + inpMsg: "", + inpSince: time.Time{}, + wantIdxs: []int{0, 1, 2, 3}, + }, + "recent events, plus the one with no time specified": { + inpType: "", + inpMsg: "", + inpSince: now.Add(-5 * time.Minute), + wantIdxs: []int{1, 2, 3}, + }, + "only warnings": { + inpType: "Warning", + inpMsg: "", + inpSince: time.Time{}, + wantIdxs: []int{1, 3}, + }, + "basic regex for a space": { + inpType: "", + inpMsg: ".* .*", + inpSince: time.Time{}, + wantIdxs: []int{2, 3}, + }, + "just a space": { + inpType: "", + inpMsg: " ", + inpSince: time.Time{}, + wantIdxs: []int{2, 3}, + }, + "invalid inescaped regex": { + inpType: "", + inpMsg: "[", + inpSince: time.Time{}, + wantIdxs: []int{2}, + }, + } + + for name, tcase := range tests { + got := EventFilter(sampleEvents, tcase.inpType, tcase.inpMsg, tcase.inpSince) + + if len(got) != len(tcase.wantIdxs) { + t.Fatalf("Expected %v events to be returned, got %v in test %v: got events: %v", + len(tcase.wantIdxs), len(got), name, got) + } + + for i, wantIdx := range tcase.wantIdxs { + if sampleEvents[wantIdx].String() != got[i].String() { + t.Errorf("Mismatch on item #%v in test %v. Expected '%v' got '%v'", + i, name, sampleEvents[wantIdx], got[i]) + } + } + } +} diff --git a/pkg/testutils/toolkit.go b/pkg/testutils/toolkit.go index f8ded15..e79e13e 100644 --- a/pkg/testutils/toolkit.go +++ b/pkg/testutils/toolkit.go @@ -5,14 +5,19 @@ package testutils import ( "context" "fmt" + "regexp" + "sort" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" gomegaTypes "github.com/onsi/gomega/types" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) +// TODO: doc type Toolkit struct { client.Client EventuallyPoll string @@ -22,6 +27,8 @@ type Toolkit struct { BackgroundCtx context.Context //nolint:containedctx } +// NewToolkit returns a toolkit using the given Client, with some basic defaults. +// This should be the preferred way to get a Toolkit instance, to avoid unset fields. func NewToolkit(client client.Client) Toolkit { return Toolkit{ Client: client, @@ -36,11 +43,8 @@ func NewToolkit(client client.Client) Toolkit { // cleanlyCreate creates the given object, and registers a callback to delete the object which // Ginkgo will call at the appropriate time. The error from the `Create` call is returned (so it // can be checked) and the `Delete` callback handles 'NotFound' errors as a success. -func (tk Toolkit) CleanlyCreate(ctx context.Context, obj client.Object) error { - // Save and then re-set the GVK because the API call removes it - savedGVK := obj.GetObjectKind().GroupVersionKind() +func (tk Toolkit) CleanlyCreate(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { createErr := tk.Create(ctx, obj) - obj.GetObjectKind().SetGroupVersionKind(savedGVK) if createErr == nil { ginkgo.DeferCleanup(func() { @@ -59,6 +63,83 @@ func (tk Toolkit) CleanlyCreate(ctx context.Context, obj client.Object) error { return createErr } +// Create uses the toolkit's client to save the object in the Kubernetes cluster. +// The only change in behavior is that it saves and restores the object's type +// information, which is for some reason stripped during the API call. +func (tk Toolkit) Create( + ctx context.Context, obj client.Object, opts ...client.CreateOption, +) error { + savedGVK := obj.GetObjectKind().GroupVersionKind() + err := tk.Client.Create(ctx, obj, opts...) + obj.GetObjectKind().SetGroupVersionKind(savedGVK) + + return err +} + +// Patch uses the toolkit's client to patch the object in the Kubernetes cluster. +// The only change in behavior is that it saves and restores the object's type +// information, which is for some reason stripped during the API call. +func (tk Toolkit) Patch( + ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption, +) error { + savedGVK := obj.GetObjectKind().GroupVersionKind() + err := tk.Client.Patch(ctx, obj, patch, opts...) + obj.GetObjectKind().SetGroupVersionKind(savedGVK) + + return err +} + +// Update uses the toolkit's client to update the object in the Kubernetes cluster. +// The only change in behavior is that it saves and restores the object's type +// information, which is for some reason stripped during the API call. +func (tk Toolkit) Update( + ctx context.Context, obj client.Object, opts ...client.UpdateOption, +) error { + savedGVK := obj.GetObjectKind().GroupVersionKind() + err := tk.Client.Update(ctx, obj, opts...) + obj.GetObjectKind().SetGroupVersionKind(savedGVK) + + return err +} + +// This regular expression is copied from +// https://github.com/open-cluster-management-io/governance-policy-framework-addon/blob/v0.13.0/controllers/statussync/policy_status_sync.go#L220 +var compEventRegex = regexp.MustCompile(`(?i)^policy:\s*(?:([a-z0-9.-]+)\s*\/)?(.+)`) + +// GetComplianceEvents queries the cluster and returns a sorted list of the kubernetes compliance +// events for the given policy. +func (tk Toolkit) GetComplianceEvents( + ctx context.Context, ns string, parentUID types.UID, templateName string, +) ([]corev1.Event, error) { + list := &corev1.EventList{} + + err := tk.List(ctx, list, client.InNamespace(ns)) + if err != nil { + return nil, err + } + + events := make([]corev1.Event, 0) + + for _, event := range list.Items { + event := event + + if event.InvolvedObject.UID != parentUID { + continue + } + + submatch := compEventRegex.FindStringSubmatch(event.Reason) + if len(submatch) >= 3 && submatch[2] == templateName { + events = append(events, event) + } + } + + sort.SliceStable(events, func(i, j int) bool { + return events[i].Name < events[j].Name + }) + + return events, nil +} + // EC runs assertions on asynchronous behavior, both *E*ventually and *C*onsistently, // using the polling and timeout settings of the toolkit. Its usage should feel familiar // to gomega users, simply skip the `.Should(...)` call and put your matcher as the second @@ -66,11 +147,32 @@ func (tk Toolkit) CleanlyCreate(ctx context.Context, obj client.Object) error { func (tk Toolkit) EC( actualOrCtx interface{}, matcher gomegaTypes.GomegaMatcher, optionalDescription ...interface{}, ) bool { + ginkgo.GinkgoHelper() + + eventuallyDescription := []interface{}{"Failed in Eventually"} + eventuallyDescription = append(eventuallyDescription, optionalDescription...) + gomega.Eventually( actualOrCtx, tk.EventuallyTimeout, tk.EventuallyPoll, - ).Should(matcher, optionalDescription...) + ).Should(matcher, eventuallyDescription...) + + consistentlyDescription := []interface{}{"Failed in Consistently"} + consistentlyDescription = append(consistentlyDescription, optionalDescription...) return gomega.Consistently( actualOrCtx, tk.ConsistentallyTimeout, tk.ConsistentlyPoll, - ).Should(matcher, optionalDescription...) + ).Should(matcher, consistentlyDescription...) +} + +// TODO: doc this, and enable/configure revive to prevent undoc'd exports +func RegisterDebugMessage() *string { + var debugMsg string + + ginkgo.DeferCleanup(func() { + if ginkgo.CurrentSpecReport().Failed() { + ginkgo.GinkgoWriter.Println(debugMsg) + } + }) + + return &debugMsg } diff --git a/test/fakepolicy/api/v1beta1/fakepolicy_types.go b/test/fakepolicy/api/v1beta1/fakepolicy_types.go index 32ea63e..2475d7f 100644 --- a/test/fakepolicy/api/v1beta1/fakepolicy_types.go +++ b/test/fakepolicy/api/v1beta1/fakepolicy_types.go @@ -12,11 +12,14 @@ import ( type FakePolicySpec struct { nucleusv1beta1.PolicyCoreSpec `json:",inline"` - // targetConfigMaps defines the ConfigMaps which should be examined by this policy + // TargetConfigMaps defines the ConfigMaps which should be examined by this policy TargetConfigMaps nucleusv1beta1.Target `json:"targetConfigMaps,omitempty"` - // targetUsingReflection defines whether to use reflection to find the ConfigMaps + // TargetUsingReflection defines whether to use reflection to find the ConfigMaps TargetUsingReflection bool `json:"targetUsingReflection,omitempty"` + + // DesiredConfigMapName - if this name is found, the policy will be compliant + DesiredConfigMapName string `json:"desiredConfigMapName,omitempty"` } //+kubebuilder:validation:Optional @@ -41,6 +44,34 @@ type FakePolicy struct { Status FakePolicyStatus `json:"status,omitempty"` } +// ensure FakePolicy implements PolicyLike +var _ nucleusv1beta1.PolicyLike = (*FakePolicy)(nil) + +func (f FakePolicy) ComplianceState() nucleusv1beta1.ComplianceState { + return f.Status.ComplianceState +} + +func (f FakePolicy) ComplianceMessage() string { + idx, compCond := f.Status.GetCondition("Compliant") + if idx == -1 { + return "" + } + + return compCond.Message +} + +func (f FakePolicy) Parent() metav1.OwnerReference { + if len(f.OwnerReferences) == 0 { + return metav1.OwnerReference{} + } + + return f.OwnerReferences[0] +} + +func (f FakePolicy) ParentNamespace() string { + return f.Namespace +} + //+kubebuilder:object:root=true // FakePolicyList contains a list of FakePolicy diff --git a/test/fakepolicy/config/crd/bases/policy.open-cluster-management.io_fakepolicies.yaml b/test/fakepolicy/config/crd/bases/policy.open-cluster-management.io_fakepolicies.yaml index 104214a..2d08ff7 100644 --- a/test/fakepolicy/config/crd/bases/policy.open-cluster-management.io_fakepolicies.yaml +++ b/test/fakepolicy/config/crd/bases/policy.open-cluster-management.io_fakepolicies.yaml @@ -39,6 +39,10 @@ spec: spec: description: FakePolicySpec defines the desired state of FakePolicy properties: + desiredConfigMapName: + description: DesiredConfigMapName - if this name is found, the policy + will be compliant + type: string namespaceSelector: description: |- NamespaceSelector indicates which namespaces on the cluster this policy @@ -130,7 +134,7 @@ spec: - Critical type: string targetConfigMaps: - description: targetConfigMaps defines the ConfigMaps which should + description: TargetConfigMaps defines the ConfigMaps which should be examined by this policy properties: exclude: @@ -196,7 +200,7 @@ spec: type: object x-kubernetes-map-type: atomic targetUsingReflection: - description: targetUsingReflection defines whether to use reflection + description: TargetUsingReflection defines whether to use reflection to find the ConfigMaps type: boolean type: object diff --git a/test/fakepolicy/config/deploy.yaml b/test/fakepolicy/config/deploy.yaml index 90a6203..3531df1 100644 --- a/test/fakepolicy/config/deploy.yaml +++ b/test/fakepolicy/config/deploy.yaml @@ -48,6 +48,10 @@ spec: spec: description: FakePolicySpec defines the desired state of FakePolicy properties: + desiredConfigMapName: + description: DesiredConfigMapName - if this name is found, the policy + will be compliant + type: string namespaceSelector: description: |- NamespaceSelector indicates which namespaces on the cluster this policy @@ -139,7 +143,7 @@ spec: - Critical type: string targetConfigMaps: - description: targetConfigMaps defines the ConfigMaps which should + description: TargetConfigMaps defines the ConfigMaps which should be examined by this policy properties: exclude: @@ -205,7 +209,7 @@ spec: type: object x-kubernetes-map-type: atomic targetUsingReflection: - description: targetUsingReflection defines whether to use reflection + description: TargetUsingReflection defines whether to use reflection to find the ConfigMaps type: boolean type: object diff --git a/test/fakepolicy/controllers/fakepolicy_controller.go b/test/fakepolicy/controllers/fakepolicy_controller.go index ab0999f..cfc06be 100644 --- a/test/fakepolicy/controllers/fakepolicy_controller.go +++ b/test/fakepolicy/controllers/fakepolicy_controller.go @@ -19,10 +19,13 @@ import ( nucleusv1alpha1 "open-cluster-management.io/governance-policy-nucleus/api/v1alpha1" nucleusv1beta1 "open-cluster-management.io/governance-policy-nucleus/api/v1beta1" + "open-cluster-management.io/governance-policy-nucleus/pkg/compliance" fakev1beta1 "open-cluster-management.io/governance-policy-nucleus/test/fakepolicy/api/v1beta1" ) -// FakePolicyReconciler reconciles a FakePolicy object +// FakePolicyReconciler reconciles a FakePolicy object. +// NOTE: it does not watch anything other than FakePolcies, so it will not react to other changes +// in the cluster - update something on the policy to make it re-reconcile. type FakePolicyReconciler struct { client.Client Scheme *runtime.Scheme @@ -42,11 +45,13 @@ type FakePolicyReconciler struct { func (r *FakePolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := log.FromContext(ctx) + log.Info("Starting a reconcile") policy := &fakev1beta1.FakePolicy{} if err := r.Get(ctx, req.NamespacedName, policy); err != nil { if errors.IsNotFound(err) { - // Request object not found, probably deleted + log.Info("Request object not found, probably deleted") + return ctrl.Result{}, nil } @@ -55,7 +60,33 @@ func (r *FakePolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - r.doSelections(ctx, policy) + cmFound := r.doSelections(ctx, policy) + + policy.Status.SelectionComplete = true + + complianceCondition := metav1.Condition{ + Type: "Compliant", + Status: metav1.ConditionTrue, + Reason: "Found", + Message: "the desired configmap was found", + } + + policy.Status.ComplianceState = nucleusv1beta1.Compliant + + if !cmFound { + complianceCondition.Status = metav1.ConditionFalse + complianceCondition.Reason = "NotFound" + complianceCondition.Message = "the desired configmap was missing" + policy.Status.ComplianceState = nucleusv1beta1.NonCompliant + } + + changed := policy.Status.UpdateCondition(complianceCondition) + + if !changed { + log.Info("No change; no compliance event to emit") + + return ctrl.Result{}, nil + } if err := r.Status().Update(ctx, policy); err != nil { log.Error(err, "Failed to update status") @@ -63,10 +94,24 @@ func (r *FakePolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - return ctrl.Result{}, nil + emitter := compliance.K8sEmitter{ + Client: r.Client, + Source: corev1.EventSource{ + Component: "hello", + Host: "world", + }, + } + + ev, err := emitter.EmitEvent(ctx, policy) + + log.Info("Event emitted", "eventName", ev.Name) + + return ctrl.Result{}, err } -func (r *FakePolicyReconciler) doSelections(ctx context.Context, policy *fakev1beta1.FakePolicy) { +func (r *FakePolicyReconciler) doSelections( + ctx context.Context, policy *fakev1beta1.FakePolicy, +) (configMapFound bool) { log := log.FromContext(ctx) nsCond := metav1.Condition{ @@ -150,6 +195,10 @@ func (r *FakePolicyReconciler) doSelections(ctx context.Context, policy *fakev1b clientCMs := make([]string, len(clientMatchedCMs)) for i, cm := range dynamicMatchedCMs { clientCMs[i] = cm.GetNamespace() + "/" + cm.GetName() + + if cm.GetName() == policy.Spec.DesiredConfigMapName { + configMapFound = true + } } slices.Sort(clientCMs) @@ -159,7 +208,7 @@ func (r *FakePolicyReconciler) doSelections(ctx context.Context, policy *fakev1b policy.Status.UpdateCondition(clientCond) - policy.Status.SelectionComplete = true + return configMapFound } type configMapResList struct { diff --git a/test/fakepolicy/test/compliance/complianceEvent_test.go b/test/fakepolicy/test/compliance/complianceEvent_test.go new file mode 100644 index 0000000..41704d5 --- /dev/null +++ b/test/fakepolicy/test/compliance/complianceEvent_test.go @@ -0,0 +1,169 @@ +// Copyright Contributors to the Open Cluster Management project + +package compliance + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "open-cluster-management.io/governance-policy-nucleus/pkg/testutils" + fakev1beta1 "open-cluster-management.io/governance-policy-nucleus/test/fakepolicy/api/v1beta1" + . "open-cluster-management.io/governance-policy-nucleus/test/fakepolicy/test/utils" +) + +var _ = Describe("Classic Compliance Events", Ordered, func() { + const testNS string = "classic-comp-test" + + var ( + parent *corev1.ConfigMap + policy *fakev1beta1.FakePolicy + ) + + BeforeAll(func(ctx SpecContext) { + ns := &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: testNS, + }, + } + Expect(tk.CleanlyCreate(ctx, ns)).To(Succeed()) + + parent = &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "parent", + Namespace: testNS, + }, + } + Expect(tk.CleanlyCreate(ctx, parent)).To(Succeed()) + + sample := SampleFakePolicy() + policy = &sample + policy.Name += "-classic-comp-test" + policy.Namespace = testNS + policy.Spec.DesiredConfigMapName = "hello-world" + policy.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: parent.APIVersion, + Kind: parent.Kind, + Name: parent.Name, + UID: parent.UID, + }} + Expect(tk.CleanlyCreate(ctx, policy)).To(Succeed()) + }) + + It("Should start NonCompliant", func(ctx SpecContext) { + tk.EC(func(g Gomega) string { + g.Expect(tk.Get(ctx, testutils.ObjNN(policy), policy)).To(Succeed()) + + return string(policy.Status.ComplianceState) + }, Equal("NonCompliant")) + }) + + It("Should emit one NonCompliant event", func(ctx SpecContext) { + tk.EC(func(g Gomega) []string { + evs, err := tk.GetComplianceEvents(ctx, testNS, parent.UID, policy.Name) + g.Expect(err).NotTo(HaveOccurred()) + + evs = testutils.EventFilter(evs, "Warning", "NonCompliant", time.Time{}) + + names := make([]string, 0, len(evs)) + for _, ev := range evs { + names = append(names, ev.Name) + } + + return names + }, HaveLen(1)) + }) + + It("Should emit one Compliant event after the configmap is created", func(ctx SpecContext) { + Expect(tk.CleanlyCreate(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "hello-world", + Namespace: "default", + Labels: map[string]string{"sample": ""}, + }})).To(Succeed()) + + // Refresh the test's copy of the policy + Expect(tk.Get(ctx, testutils.ObjNN(policy), policy)).To(Succeed()) + + By("Setting an annotation on the policy to trigger a re-reconcile") + policy.SetAnnotations(map[string]string{ + "classic-comp-test-1": "1", + }) + Expect(tk.Update(ctx, policy)).To(Succeed()) + + tk.EC(func(g Gomega) []string { + evs, err := tk.GetComplianceEvents(ctx, testNS, parent.UID, policy.Name) + g.Expect(err).NotTo(HaveOccurred()) + + evs = testutils.EventFilter(evs, "Normal", "^Compliant", time.Time{}) + + names := make([]string, 0, len(evs)) + for _, ev := range evs { + names = append(names, ev.Name) + } + + return names + }, HaveLen(1)) + }) + + It("Should emit a NonCompliant event after the configmap is deleted", func(ctx SpecContext) { + By("Ensuring that the configmap is gone") + Eventually(func() string { + cm := corev1.ConfigMap{} + _ = tk.Get(ctx, types.NamespacedName{Name: "hello-world", Namespace: "default"}, &cm) + + return cm.Name + }, "1s", "50ms").Should(BeEmpty()) + + // Refresh the test's copy of the policy + Expect(tk.Get(ctx, testutils.ObjNN(policy), policy)).To(Succeed()) + + By("Patching an annotation on the policy to trigger a re-reconcile") + patch := `[{"op":"replace","path":"/metadata/annotations/classic-comp-test-1","value":"2"}]` + err := tk.Patch(ctx, policy, client.RawPatch(types.JSONPatchType, []byte(patch))) + Expect(err).NotTo(HaveOccurred()) + + // This is just an example usage of this function, not an actual case where it was necessary. + // It's _obvious_ that the test must use a filter for 2 seconds ago; if it filtered to only + // 1 second ago, then since the Consistently runs for a whole second, it would always + // get an empty list near the end, and fail. + debugMsg := testutils.RegisterDebugMessage() + + tk.EC(func(g Gomega) []string { + evs, err := tk.GetComplianceEvents(ctx, testNS, parent.UID, policy.Name) + g.Expect(err).NotTo(HaveOccurred()) + + *debugMsg = "unfiltered events: " + for _, ev := range evs { + *debugMsg += fmt.Sprintf("(%v: %v), ", ev.Name, ev.Message) + } + + evs = testutils.EventFilter(evs, "Warning", "NonCompliant", + time.Now().Add(-2*time.Second)) + + names := make([]string, 0, len(evs)) + for _, ev := range evs { + names = append(names, ev.Name) + } + + return names + }, HaveLen(1), "intest") + // TODO: open a bug with ginkgo: this does not format correctly + // see https://github.com/onsi/gomega/blob/v1.33.0/internal/async_assertion.go#L163 + }) +}) + +// TODO: add test / implementation for event mutators diff --git a/test/fakepolicy/test/compliance/suite_test.go b/test/fakepolicy/test/compliance/suite_test.go new file mode 100644 index 0000000..1d153f5 --- /dev/null +++ b/test/fakepolicy/test/compliance/suite_test.go @@ -0,0 +1,84 @@ +// Copyright Contributors to the Open Cluster Management project + +package compliance + +import ( + "context" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/format" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "open-cluster-management.io/governance-policy-nucleus/pkg/testutils" + "open-cluster-management.io/governance-policy-nucleus/test/fakepolicy" + fakev1beta1 "open-cluster-management.io/governance-policy-nucleus/test/fakepolicy/api/v1beta1" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + cfg *rest.Config + k8sClient client.Client + testEnv *envtest.Environment + ctx context.Context + cancel context.CancelFunc + tk testutils.Toolkit +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Compliance Details Suite") +} + +var _ = BeforeSuite(func() { + format.TruncatedDiff = false + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + err = fakev1beta1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + tk = testutils.NewToolkit(k8sClient) + tk.BackgroundCtx = ctx + + go func() { + defer GinkgoRecover() + Expect(fakepolicy.Run(ctx, cfg)).To(Succeed()) + }() +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +})