Skip to content

Commit

Permalink
Authorino CR reconcile moved to state of the world reconciler. (#865)
Browse files Browse the repository at this point in the history
* Authorino CR reconcile moved to state of the world reconciler.
# Conflicts:
#	api/v1beta1/topology.go
#	controllers/state_of_the_world.go
* PR comments updates
* Fix integration test.
* Fix integration test attempt 2.
* Refactor
* Change logic on finding initial root kuadrant
* Improve logging statements
* Kuadrant only expects one kuadrant CR on the cluster. Recent changes enforces the creation of only one authorino. This enforcement was causing the tests to fail as it was creating a second kuadrant CR that would never go to a ready state.
* Improve the getting of the kuadrant CR.
* Get the oldest kuadrant CR and use that as the only CR.
* Refactor structure
* PR comment changes
* Rebase from main for changes in the policy machinery

---------

Signed-off-by: Jim Fitzpatrick <[email protected]>
  • Loading branch information
Boomatang authored Oct 4, 2024
1 parent a344637 commit b3290b5
Show file tree
Hide file tree
Showing 6 changed files with 369 additions and 60 deletions.
17 changes: 17 additions & 0 deletions api/v1beta1/topology.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package v1beta1

import (
authorinov1beta1 "github.com/kuadrant/authorino-operator/api/v1beta1"
limitadorv1alpha1 "github.com/kuadrant/limitador-operator/api/v1alpha1"
"github.com/kuadrant/policy-machinery/controller"
"github.com/kuadrant/policy-machinery/machinery"
Expand All @@ -10,9 +11,11 @@ import (
)

var (
AuthorinoKind = schema.GroupKind{Group: authorinov1beta1.GroupVersion.Group, Kind: "Authorino"}
KuadrantKind = schema.GroupKind{Group: GroupVersion.Group, Kind: "Kuadrant"}
LimitadorKind = schema.GroupKind{Group: limitadorv1alpha1.GroupVersion.Group, Kind: "Limitador"}

AuthorinoResource = authorinov1beta1.GroupVersion.WithResource("authorinos")
KuadrantResource = GroupVersion.WithResource("kuadrants")
LimitadorResource = limitadorv1alpha1.GroupVersion.WithResource("limitadors")
)
Expand Down Expand Up @@ -52,3 +55,17 @@ func LinkKuadrantToLimitador(objs controller.Store) machinery.LinkFunc {
},
}
}

func LinkKuadrantToAuthorino(objs controller.Store) machinery.LinkFunc {
kuadrants := lo.Map(objs.FilterByGroupKind(KuadrantKind), controller.ObjectAs[machinery.Object])

return machinery.LinkFunc{
From: KuadrantKind,
To: AuthorinoKind,
Func: func(child machinery.Object) []machinery.Object {
return lo.Filter(kuadrants, func(kuadrant machinery.Object, _ int) bool {
return kuadrant.GetNamespace() == child.GetNamespace() && child.GetName() == "authorino"
})
},
}
}
121 changes: 121 additions & 0 deletions controllers/authorino_task.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
package controllers

import (
"context"
"strings"
"sync"

v1beta2 "github.com/kuadrant/authorino-operator/api/v1beta1"
"github.com/kuadrant/policy-machinery/controller"
"github.com/kuadrant/policy-machinery/machinery"
"github.com/samber/lo"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/dynamic"
"k8s.io/utils/ptr"

"github.com/kuadrant/kuadrant-operator/api/v1beta1"
)

type AuthorinoCrReconciler struct {
Client *dynamic.DynamicClient
}

func NewAuthorinoCrReconciler(client *dynamic.DynamicClient) *AuthorinoCrReconciler {
return &AuthorinoCrReconciler{Client: client}
}

func (r *AuthorinoCrReconciler) Subscription() *controller.Subscription {
return &controller.Subscription{
ReconcileFunc: r.Reconcile,
Events: []controller.ResourceEventMatcher{
{Kind: ptr.To(v1beta1.KuadrantKind), EventType: ptr.To(controller.CreateEvent)},
{Kind: ptr.To(v1beta1.AuthorinoKind), EventType: ptr.To(controller.DeleteEvent)},
},
}
}

func (r *AuthorinoCrReconciler) Reconcile(ctx context.Context, _ []controller.ResourceEvent, topology *machinery.Topology, _ error, _ *sync.Map) error {
logger := controller.LoggerFromContext(ctx).WithName("AuthorinoCrReconciler")
logger.Info("reconciling authorino resource", "status", "started")
defer logger.Info("reconciling authorino resource", "status", "completed")

kobjs := lo.FilterMap(topology.Objects().Roots(), func(item machinery.Object, _ int) (*v1beta1.Kuadrant, bool) {
if item.GroupVersionKind().Kind == v1beta1.KuadrantKind.Kind {
return item.(*v1beta1.Kuadrant), true
}
return nil, false
})

kobj, err := GetOldestKuadrant(kobjs)
if err != nil {
if strings.Contains(err.Error(), "empty list passed") {
logger.Info("kuadrant resource not found, ignoring", "status", "skipping")
return err
}
logger.Error(err, "cannot find Kuadrant resource", "status", "error")
return err
}

aobjs := lo.FilterMap(topology.Objects().Objects().Children(kobj), func(item machinery.Object, _ int) (machinery.Object, bool) {
if item.GroupVersionKind().Kind == v1beta1.AuthorinoKind.Kind {
return item, true
}
return nil, false
})

if len(aobjs) > 0 {
logger.Info("authorino resource already exists, no need to create", "status", "skipping")
return nil
}

authorino := &v1beta2.Authorino{
TypeMeta: metav1.TypeMeta{
Kind: "Authorino",
APIVersion: "operator.authorino.kuadrant.io/v1beta1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "authorino",
Namespace: kobj.Namespace,
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: kobj.GroupVersionKind().GroupVersion().String(),
Kind: kobj.GroupVersionKind().Kind,
Name: kobj.Name,
UID: kobj.UID,
BlockOwnerDeletion: ptr.To(true),
Controller: ptr.To(true),
},
},
},
Spec: v1beta2.AuthorinoSpec{
ClusterWide: true,
SupersedingHostSubsets: true,
Listener: v1beta2.Listener{
Tls: v1beta2.Tls{
Enabled: ptr.To(false),
},
},
OIDCServer: v1beta2.OIDCServer{
Tls: v1beta2.Tls{
Enabled: ptr.To(false),
},
},
},
}

unstructuredAuthorino, err := controller.Destruct(authorino)
if err != nil {
logger.Error(err, "failed to destruct authorino", "status", "error")
}
logger.Info("creating authorino resource", "status", "processing")
_, err = r.Client.Resource(v1beta1.AuthorinoResource).Namespace(authorino.Namespace).Create(ctx, unstructuredAuthorino, metav1.CreateOptions{})
if err != nil {
if errors.IsAlreadyExists(err) {
logger.Info("already created authorino resource", "status", "acceptable")
} else {
logger.Error(err, "failed to create authorino resource", "status", "error")
}
}
return nil
}
43 changes: 1 addition & 42 deletions controllers/kuadrant_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"encoding/json"

"github.com/go-logr/logr"
authorinov1beta1 "github.com/kuadrant/authorino-operator/api/v1beta1"
limitadorv1alpha1 "github.com/kuadrant/limitador-operator/api/v1alpha1"
iopv1alpha1 "istio.io/istio/operator/pkg/apis/istio/v1alpha1"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -399,11 +398,7 @@ func (r *KuadrantReconciler) reconcileSpec(ctx context.Context, kObj *kuadrantv1
return err
}

if err := r.reconcileLimitador(ctx, kObj); err != nil {
return err
}

return r.reconcileAuthorino(ctx, kObj)
return r.reconcileLimitador(ctx, kObj)
}

func (r *KuadrantReconciler) reconcileLimitador(ctx context.Context, kObj *kuadrantv1beta1.Kuadrant) error {
Expand All @@ -427,41 +422,6 @@ func (r *KuadrantReconciler) reconcileLimitador(ctx context.Context, kObj *kuadr
return r.ReconcileResource(ctx, &limitadorv1alpha1.Limitador{}, limitador, reconcilers.CreateOnlyMutator)
}

func (r *KuadrantReconciler) reconcileAuthorino(ctx context.Context, kObj *kuadrantv1beta1.Kuadrant) error {
tmpFalse := false
authorino := &authorinov1beta1.Authorino{
TypeMeta: metav1.TypeMeta{
Kind: "Authorino",
APIVersion: "operator.authorino.kuadrant.io/v1beta1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "authorino",
Namespace: kObj.Namespace,
},
Spec: authorinov1beta1.AuthorinoSpec{
ClusterWide: true,
SupersedingHostSubsets: true,
Listener: authorinov1beta1.Listener{
Tls: authorinov1beta1.Tls{
Enabled: &tmpFalse,
},
},
OIDCServer: authorinov1beta1.OIDCServer{
Tls: authorinov1beta1.Tls{
Enabled: &tmpFalse,
},
},
},
}

err := r.SetOwnerReference(kObj, authorino)
if err != nil {
return err
}

return r.ReconcileResource(ctx, &authorinov1beta1.Authorino{}, authorino, reconcilers.CreateOnlyMutator)
}

// SetupWithManager sets up the controller with the Manager.
func (r *KuadrantReconciler) SetupWithManager(mgr ctrl.Manager) error {
ok, err := kuadrantgatewayapi.IsGatewayAPIInstalled(mgr.GetRESTMapper())
Expand All @@ -476,6 +436,5 @@ func (r *KuadrantReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&kuadrantv1beta1.Kuadrant{}).
Owns(&limitadorv1alpha1.Limitador{}).
Owns(&authorinov1beta1.Authorino{}).
Complete(r)
}
52 changes: 47 additions & 5 deletions controllers/state_of_the_world.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,15 @@ import (
egv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1"
"github.com/go-logr/logr"
"github.com/google/go-cmp/cmp"
authorinov1beta1 "github.com/kuadrant/authorino-operator/api/v1beta1"
limitadorv1alpha1 "github.com/kuadrant/limitador-operator/api/v1alpha1"
"github.com/kuadrant/policy-machinery/controller"
"github.com/kuadrant/policy-machinery/machinery"
istioclientgoextensionv1alpha1 "istio.io/client-go/pkg/apis/extensions/v1alpha1"
istioclientnetworkingv1alpha3 "istio.io/client-go/pkg/apis/networking/v1alpha3"
istioclientgosecurityv1beta1 "istio.io/client-go/pkg/apis/security/v1beta1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic"
Expand All @@ -35,7 +37,7 @@ import (

var (
ConfigMapGroupKind = schema.GroupKind{Group: corev1.GroupName, Kind: "ConfigMap"}
operatorNamespace = env.GetString("OPERATOR_NAMESPACE", "")
operatorNamespace = env.GetString("OPERATOR_NAMESPACE", "kuadrant-system")
)

//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gatewayclasses,verbs=list;watch
Expand All @@ -52,6 +54,7 @@ func NewPolicyMachineryController(manager ctrlruntime.Manager, client *dynamic.D
controller.WithRunnable("ratelimitpolicy watcher", controller.Watch(&kuadrantv1beta2.RateLimitPolicy{}, kuadrantv1beta2.RateLimitPoliciesResource, metav1.NamespaceAll)),
controller.WithRunnable("topology configmap watcher", controller.Watch(&corev1.ConfigMap{}, controller.ConfigMapsResource, operatorNamespace, controller.FilterResourcesByLabel[*corev1.ConfigMap](fmt.Sprintf("%s=true", kuadrant.TopologyLabel)))),
controller.WithRunnable("limitador watcher", controller.Watch(&limitadorv1alpha1.Limitador{}, kuadrantv1beta1.LimitadorResource, metav1.NamespaceAll)),
controller.WithRunnable("authorino watcher", controller.Watch(&authorinov1beta1.Authorino{}, kuadrantv1beta1.AuthorinoResource, metav1.NamespaceAll)),
controller.WithPolicyKinds(
kuadrantv1alpha1.DNSPolicyKind,
kuadrantv1alpha1.TLSPolicyKind,
Expand All @@ -62,10 +65,12 @@ func NewPolicyMachineryController(manager ctrlruntime.Manager, client *dynamic.D
kuadrantv1beta1.KuadrantKind,
ConfigMapGroupKind,
kuadrantv1beta1.LimitadorKind,
kuadrantv1beta1.AuthorinoKind,
),
controller.WithObjectLinks(
kuadrantv1beta1.LinkKuadrantToGatewayClasses,
kuadrantv1beta1.LinkKuadrantToLimitador,
kuadrantv1beta1.LinkKuadrantToAuthorino,
),
controller.WithReconcile(buildReconciler(client)),
}
Expand Down Expand Up @@ -140,9 +145,14 @@ func NewPolicyMachineryController(manager ctrlruntime.Manager, client *dynamic.D

func buildReconciler(client *dynamic.DynamicClient) controller.ReconcileFunc {
reconciler := &controller.Workflow{
Precondition: NewEventLogger().Log,
Precondition: (&controller.Workflow{
Precondition: NewEventLogger().Log,
Tasks: []controller.ReconcileFunc{
NewTopologyFileReconciler(client, operatorNamespace).Reconcile,
},
}).Run,
Tasks: []controller.ReconcileFunc{
NewTopologyFileReconciler(client, operatorNamespace).Reconcile,
NewAuthorinoCrReconciler(client).Subscription().Reconcile,
},
}
return reconciler.Run
Expand Down Expand Up @@ -184,8 +194,13 @@ func (r *TopologyFileReconciler) Reconcile(ctx context.Context, _ []controller.R
})

if len(existingTopologyConfigMaps) == 0 {
_, err := r.Client.Resource(controller.ConfigMapsResource).Namespace(cm.Namespace).Create(ctx, unstructuredCM, metav1.CreateOptions{})
_, err = r.Client.Resource(controller.ConfigMapsResource).Namespace(cm.Namespace).Create(ctx, unstructuredCM, metav1.CreateOptions{})
if err != nil {
if errors.IsAlreadyExists(err) {
// This error can happen when the operator is starting, and the create event for the topology has not being processed.
logger.Info("already created topology configmap, must not be in topology yet")
return err
}
logger.Error(err, "failed to write topology configmap")
}
return err
Expand All @@ -198,7 +213,7 @@ func (r *TopologyFileReconciler) Reconcile(ctx context.Context, _ []controller.R
cmTopology := existingTopologyConfigMap.Object.(*corev1.ConfigMap)

if d, found := cmTopology.Data["topology"]; !found || strings.Compare(d, cm.Data["topology"]) != 0 {
_, err := r.Client.Resource(controller.ConfigMapsResource).Namespace(cm.Namespace).Update(ctx, unstructuredCM, metav1.UpdateOptions{})
_, err = r.Client.Resource(controller.ConfigMapsResource).Namespace(cm.Namespace).Update(ctx, unstructuredCM, metav1.UpdateOptions{})
if err != nil {
logger.Error(err, "failed to update topology configmap")
}
Expand Down Expand Up @@ -239,3 +254,30 @@ func (e *EventLogger) Log(ctx context.Context, resourceEvents []controller.Resou

return nil
}

// GetOldestKuadrant returns the oldest kuadrant resource from a list of kuadrant resources that is not marked for deletion.
func GetOldestKuadrant(kuadrants []*kuadrantv1beta1.Kuadrant) (*kuadrantv1beta1.Kuadrant, error) {
if len(kuadrants) == 1 {
return kuadrants[0], nil
}
if len(kuadrants) == 0 {
return nil, fmt.Errorf("empty list passed")
}
oldest := kuadrants[0]
for _, k := range kuadrants[1:] {
if k == nil || k.DeletionTimestamp != nil {
continue
}
if oldest == nil {
oldest = k
continue
}
if k.CreationTimestamp.Before(&oldest.CreationTimestamp) {
oldest = k
}
}
if oldest == nil {
return nil, fmt.Errorf("only nil pointers in list")
}
return oldest, nil
}
Loading

0 comments on commit b3290b5

Please sign in to comment.