From e8d1bb5575b3ced70c34017a86d02efb16da949a Mon Sep 17 00:00:00 2001 From: Eguzki Astiz Lezaun Date: Thu, 24 Mar 2022 11:06:17 +0100 Subject: [PATCH] Openshift routes support (#118) * Add openshift route api initial route controller impl * openshift route controller * openshift route controller based on sidecar * openshift route doc fix * remove code commented out * remove unused code * comment method as not thread-safe --- README.md | 1 + config/deploy/manifests.yaml | 34 + config/rbac/role.yaml | 33 + controllers/apim/httproute_controller.go | 10 +- controllers/apim/ratelimitpolicy_mapper.go | 28 +- controllers/apim/utils.go | 5 - controllers/apim/virtualservice_controller.go | 5 +- controllers/ocp_route_controller.go | 612 ++++++++++++++++++ doc/openshift-routes.md | 232 +++++++ go.mod | 7 +- go.sum | 183 +----- main.go | 25 +- pkg/common/common.go | 19 + pkg/common/k8s_utils.go | 27 + pkg/istio/utils.go | 31 + pkg/mappers/rlp_to_route.go | 54 ++ 16 files changed, 1104 insertions(+), 202 deletions(-) create mode 100644 controllers/ocp_route_controller.go create mode 100644 doc/openshift-routes.md create mode 100644 pkg/istio/utils.go create mode 100644 pkg/mappers/rlp_to_route.go diff --git a/README.md b/README.md index 750853c..450abb1 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ * [Overview](#overview) * [CustomResourceDefinitions](#customresourcedefinitions) * [Getting started](#getting-started) +* [Openshift Routes](/doc/openshift-routes.md) * [Contributing](#contributing) * [Licensing](#licensing) diff --git a/config/deploy/manifests.yaml b/config/deploy/manifests.yaml index 52abcbc..767430f 100644 --- a/config/deploy/manifests.yaml +++ b/config/deploy/manifests.yaml @@ -333,6 +333,40 @@ rules: - patch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + app: kuadrant + name: kuadrant-manager-role + namespace: kuadrant-system +rules: +- apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - route.openshift.io + resources: + - routes/custom-host + verbs: + - create +- apiGroups: + - route.openshift.io + resources: + - routes/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: creationTimestamp: null diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index ef37d52..8ce2d29 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -96,3 +96,36 @@ rules: - patch - update - watch + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + name: manager-role + namespace: placeholder +rules: +- apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - route.openshift.io + resources: + - routes/custom-host + verbs: + - create +- apiGroups: + - route.openshift.io + resources: + - routes/status + verbs: + - get diff --git a/controllers/apim/httproute_controller.go b/controllers/apim/httproute_controller.go index 0a558b9..209454c 100644 --- a/controllers/apim/httproute_controller.go +++ b/controllers/apim/httproute_controller.go @@ -5,8 +5,6 @@ import ( "fmt" "github.com/go-logr/logr" - "github.com/kuadrant/kuadrant-controller/pkg/log" - "github.com/kuadrant/kuadrant-controller/pkg/reconcilers" securityv1beta1 "istio.io/api/security/v1beta1" istiosecurityv1beta1 "istio.io/client-go/pkg/apis/security/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -16,6 +14,10 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" gatewayapi_v1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + "github.com/kuadrant/kuadrant-controller/pkg/log" + "github.com/kuadrant/kuadrant-controller/pkg/mappers" + "github.com/kuadrant/kuadrant-controller/pkg/reconcilers" ) const HTTPRouteNamePrefix = "hr" @@ -43,7 +45,7 @@ func (r *HTTPRouteReconciler) Reconcile(eventCtx context.Context, req ctrl.Reque // TODO(rahulanand16nov): handle HTTPRoute deletion for AuthPolicy // check if this httproute has to be protected or not. - _, present := httproute.GetAnnotations()[KuadrantAuthProviderAnnotation] + _, present := httproute.GetAnnotations()[mappers.KuadrantAuthProviderAnnotation] if !present { for _, parentRef := range httproute.Spec.ParentRefs { gwNamespace := httproute.Namespace // consider gateway local if namespace is not given @@ -88,7 +90,7 @@ func (r *HTTPRouteReconciler) reconcileAuthPolicy(ctx context.Context, logger lo logger.Info("Reconciling AuthorizationPolicy") // annotation presence is already checked. - providerName := hr.GetAnnotations()[KuadrantAuthProviderAnnotation] + providerName := hr.GetAnnotations()[mappers.KuadrantAuthProviderAnnotation] // pre-convert hostnames to string slice hosts := []string{} diff --git a/controllers/apim/ratelimitpolicy_mapper.go b/controllers/apim/ratelimitpolicy_mapper.go index 2bbb181..fafb1bd 100644 --- a/controllers/apim/ratelimitpolicy_mapper.go +++ b/controllers/apim/ratelimitpolicy_mapper.go @@ -4,12 +4,14 @@ import ( "context" "github.com/go-logr/logr" - apimv1alpha1 "github.com/kuadrant/kuadrant-controller/apis/apim/v1alpha1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" + + apimv1alpha1 "github.com/kuadrant/kuadrant-controller/apis/apim/v1alpha1" + "github.com/kuadrant/kuadrant-controller/pkg/mappers" ) const ( @@ -26,7 +28,7 @@ const ( func routingPredicate(m *rateLimitPolicyMapper) predicate.Predicate { return predicate.Funcs{ CreateFunc: func(e event.CreateEvent) bool { - if _, toRateLimit := e.Object.GetAnnotations()[KuadrantRateLimitPolicyAnnotation]; toRateLimit { + if _, toRateLimit := e.Object.GetAnnotations()[mappers.KuadrantRateLimitPolicyAnnotation]; toRateLimit { if err := m.SignalCreate(e.Object); err != nil { m.Logger.Error(err, "failed to signal create event to referenced RateLimitPolicy") // lets still try for auth annotation @@ -35,32 +37,32 @@ func routingPredicate(m *rateLimitPolicyMapper) predicate.Predicate { // only create reconcile request for routing objects' controllers when auth // annotation is present. - _, toProtect := e.Object.GetAnnotations()[KuadrantAuthProviderAnnotation] + _, toProtect := e.Object.GetAnnotations()[mappers.KuadrantAuthProviderAnnotation] return toProtect }, UpdateFunc: func(e event.UpdateEvent) bool { - _, toRateLimitOld := e.ObjectOld.GetAnnotations()[KuadrantRateLimitPolicyAnnotation] - _, toRateLimitNew := e.ObjectNew.GetAnnotations()[KuadrantRateLimitPolicyAnnotation] + _, toRateLimitOld := e.ObjectOld.GetAnnotations()[mappers.KuadrantRateLimitPolicyAnnotation] + _, toRateLimitNew := e.ObjectNew.GetAnnotations()[mappers.KuadrantRateLimitPolicyAnnotation] if toRateLimitNew || toRateLimitOld { if err := m.SignalUpdate(e.ObjectOld, e.ObjectNew); err != nil { m.Logger.Error(err, "failed to signal update event to referenced RateLimitPolicy") } } - _, toProtectOld := e.ObjectOld.GetAnnotations()[KuadrantAuthProviderAnnotation] - _, toProtectNew := e.ObjectNew.GetAnnotations()[KuadrantAuthProviderAnnotation] + _, toProtectOld := e.ObjectOld.GetAnnotations()[mappers.KuadrantAuthProviderAnnotation] + _, toProtectNew := e.ObjectNew.GetAnnotations()[mappers.KuadrantAuthProviderAnnotation] return toProtectOld || toProtectNew }, DeleteFunc: func(e event.DeleteEvent) bool { // If the object had the Kuadrant label, we need to handle its deletion - _, toRateLimit := e.Object.GetAnnotations()[KuadrantRateLimitPolicyAnnotation] + _, toRateLimit := e.Object.GetAnnotations()[mappers.KuadrantRateLimitPolicyAnnotation] if toRateLimit { if err := m.SignalDelete(e.Object); err != nil { m.Logger.Error(err, "failed to signal delete event to referenced RateLimitPolicy") } } - _, toProtect := e.Object.GetAnnotations()[KuadrantAuthProviderAnnotation] + _, toProtect := e.Object.GetAnnotations()[mappers.KuadrantAuthProviderAnnotation] return toProtect }, } @@ -78,7 +80,7 @@ func (m *rateLimitPolicyMapper) SignalCreate(obj client.Object) error { if obj.GetObjectKind().GroupVersionKind().Kind == "HTTPRoute" { addAnnotation = KuadrantAddHRAnnotation } - rlpName := obj.GetAnnotations()[KuadrantRateLimitPolicyAnnotation] + rlpName := obj.GetAnnotations()[mappers.KuadrantRateLimitPolicyAnnotation] m.Logger.Info("Signaling create event to RateLimitPolicy", "RateLimitPolicy", rlpName) rlpKey := types.NamespacedName{ Name: rlpName, @@ -105,7 +107,7 @@ func (m *rateLimitPolicyMapper) SignalDelete(obj client.Object) error { if obj.GetObjectKind().GroupVersionKind().Kind == "HTTPRoute" { deleteAnnotation = KuadrantAddHRAnnotation } - rlpName := obj.GetAnnotations()[KuadrantRateLimitPolicyAnnotation] + rlpName := obj.GetAnnotations()[mappers.KuadrantRateLimitPolicyAnnotation] m.Logger.Info("Signaling delete event to RateLimitPolicy", "RateLimitPolicy", rlpName) rlpKey := types.NamespacedName{ Name: rlpName, @@ -130,8 +132,8 @@ func (m *rateLimitPolicyMapper) SignalDelete(obj client.Object) error { // SignalUpdate is used when either old or new object had/has the ratelimit annotaiton func (m *rateLimitPolicyMapper) SignalUpdate(oldObj, newObj client.Object) error { m.Logger.Info("Signaling update event to RateLimitPolicy") - oldRlpName, toRateLimitOld := oldObj.GetAnnotations()[KuadrantRateLimitPolicyAnnotation] - newRlpName, toRateLimitNew := newObj.GetAnnotations()[KuadrantRateLimitPolicyAnnotation] + oldRlpName, toRateLimitOld := oldObj.GetAnnotations()[mappers.KuadrantRateLimitPolicyAnnotation] + newRlpName, toRateLimitNew := newObj.GetAnnotations()[mappers.KuadrantRateLimitPolicyAnnotation] // case when rlp name is added (same as create event) if !toRateLimitOld && toRateLimitNew { diff --git a/controllers/apim/utils.go b/controllers/apim/utils.go index 54556f2..07f3d92 100644 --- a/controllers/apim/utils.go +++ b/controllers/apim/utils.go @@ -11,11 +11,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const ( - KuadrantAuthProviderAnnotation = "kuadrant.io/auth-provider" - KuadrantRateLimitPolicyAnnotation = "kuadrant.io/ratelimitpolicy" -) - // gatewayLabels fetches labels of an Istio gateway identified using the given ObjectKey. func gatewayLabels(ctx context.Context, client client.Client, gwKey client.ObjectKey) map[string]string { gateway := &istio.Gateway{} diff --git a/controllers/apim/virtualservice_controller.go b/controllers/apim/virtualservice_controller.go index 49ff835..c37c34b 100644 --- a/controllers/apim/virtualservice_controller.go +++ b/controllers/apim/virtualservice_controller.go @@ -19,6 +19,7 @@ import ( "github.com/go-logr/logr" "github.com/kuadrant/kuadrant-controller/pkg/common" "github.com/kuadrant/kuadrant-controller/pkg/log" + "github.com/kuadrant/kuadrant-controller/pkg/mappers" "github.com/kuadrant/kuadrant-controller/pkg/reconcilers" ) @@ -48,7 +49,7 @@ func (r *VirtualServiceReconciler) Reconcile(eventCtx context.Context, req ctrl. // TODO(rahulanand16nov): handle VirtualService deletion for AuthPolicy // check if this virtualservice is to be protected or not. - _, present := virtualService.GetAnnotations()[KuadrantAuthProviderAnnotation] + _, present := virtualService.GetAnnotations()[mappers.KuadrantAuthProviderAnnotation] if !present { for _, gateway := range virtualService.Spec.Gateways { gwKey := common.NamespacedNameToObjectKey(gateway, virtualService.Namespace) @@ -78,7 +79,7 @@ func (r *VirtualServiceReconciler) reconcileAuthPolicy(ctx context.Context, logg logger.Info("Reconciling AuthorizationPolicy") // annotation presence is already checked. - providerName := vs.GetAnnotations()[KuadrantAuthProviderAnnotation] + providerName := vs.GetAnnotations()[mappers.KuadrantAuthProviderAnnotation] // TODO(rahulanand16nov): update following to match HTTPRoute controller // fill out the rules diff --git a/controllers/ocp_route_controller.go b/controllers/ocp_route_controller.go new file mode 100644 index 0000000..205087c --- /dev/null +++ b/controllers/ocp_route_controller.go @@ -0,0 +1,612 @@ +package controllers + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strconv" + + "github.com/go-logr/logr" + "github.com/gogo/protobuf/types" + routev1 "github.com/openshift/api/route/v1" + istioapinetworkingv1alpha3 "istio.io/api/networking/v1alpha3" + securityv1beta1 "istio.io/api/security/v1beta1" + istionetworkingv1alpha3 "istio.io/client-go/pkg/apis/networking/v1alpha3" + istiosecurityv1beta1 "istio.io/client-go/pkg/apis/security/v1beta1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/source" + + apimv1alpha1 "github.com/kuadrant/kuadrant-controller/apis/apim/v1alpha1" + "github.com/kuadrant/kuadrant-controller/pkg/common" + kuadrantistio "github.com/kuadrant/kuadrant-controller/pkg/istio" + "github.com/kuadrant/kuadrant-controller/pkg/log" + "github.com/kuadrant/kuadrant-controller/pkg/mappers" + "github.com/kuadrant/kuadrant-controller/pkg/reconcilers" +) + +const ( + routeFinalizerName = "kuadrant.io/route" +) + +// +kubebuilder:rbac:groups=route.openshift.io,namespace=placeholder,resources=routes,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=route.openshift.io,namespace=placeholder,resources=routes/custom-host,verbs=create +// +kubebuilder:rbac:groups=route.openshift.io,namespace=placeholder,resources=routes/status,verbs=get + +// RouteReconciler reconciles Openshift Route object +type RouteReconciler struct { + *reconcilers.BaseReconciler + Scheme *runtime.Scheme +} + +func (r *RouteReconciler) Reconcile(eventCtx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := r.Logger().WithValues("Route", req.NamespacedName) + ctx := logr.NewContext(eventCtx, logger) + + route := &routev1.Route{} + if err := r.Client().Get(ctx, req.NamespacedName, route); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + logger.Error(err, "failed to get Route") + return ctrl.Result{}, err + } + + if logger.V(1).Enabled() { + jsonData, err := json.MarshalIndent(route, "", " ") + if err != nil { + return ctrl.Result{}, err + } + logger.V(1).Info(string(jsonData)) + } + + // Route has been marked for deletion + if route.GetDeletionTimestamp() != nil && controllerutil.ContainsFinalizer(route, routeFinalizerName) { + err := r.deleteAuthPolicy(ctx, route) + if err != nil { + return ctrl.Result{}, err + } + + err = r.deleteRateLimitFilterEnvoyFilter(ctx, route) + if err != nil { + return ctrl.Result{}, err + } + + err = r.deleteRateLimitDescriptorsEnvoyFilter(ctx, route) + if err != nil { + return ctrl.Result{}, err + } + + //Remove finalizer and update the object. + controllerutil.RemoveFinalizer(route, routeFinalizerName) + err = r.UpdateResource(ctx, route) + logger.Info("Removing finalizer", "error", err) + return ctrl.Result{Requeue: true}, err + } + + // Ignore deleted resources, this can happen when foregroundDeletion is enabled + // https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/#foreground-cascading-deletion + if route.GetDeletionTimestamp() != nil { + return ctrl.Result{}, nil + } + + if !controllerutil.ContainsFinalizer(route, routeFinalizerName) { + controllerutil.AddFinalizer(route, routeFinalizerName) + err := r.UpdateResource(ctx, route) + logger.Info("Adding finalizer", "error", err) + return ctrl.Result{Requeue: true}, err + } + + routeLabels := route.GetLabels() + if kuadrantEnabled, ok := routeLabels[mappers.KuadrantManagedLabel]; !ok || kuadrantEnabled != "true" { + // this route used to be kuadrant protected, not anymore + + err := r.deleteAuthPolicy(ctx, route) + if err != nil { + return ctrl.Result{}, err + } + + err = r.deleteRateLimitFilterEnvoyFilter(ctx, route) + if err != nil { + return ctrl.Result{}, err + } + + err = r.deleteRateLimitDescriptorsEnvoyFilter(ctx, route) + if err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil + } + + desiredAuthPolicy := r.desiredAuthorizationPolicy(route) + err := r.ReconcileResource(ctx, &istiosecurityv1beta1.AuthorizationPolicy{}, desiredAuthPolicy, basicAuthPolicyMutator) + if err != nil { + return ctrl.Result{}, err + } + + // Limitador rate limits are managed by the ratelimitpolicy controller + + desiredRLFilterEF, err := r.desiredRateLimitFilterEnvoyFilter(route) + if err != nil { + return ctrl.Result{}, err + } + + err = r.ReconcileResource(ctx, &istionetworkingv1alpha3.EnvoyFilter{}, desiredRLFilterEF, basicEnvoyFilterMutator) + if err != nil { + return ctrl.Result{}, err + } + + desiredDescriptorsEF, err := r.desiredDescriptorsEnvoyFilter(ctx, route) + if err != nil { + return ctrl.Result{}, err + } + + err = r.ReconcileResource(ctx, &istionetworkingv1alpha3.EnvoyFilter{}, desiredDescriptorsEF, basicEnvoyFilterMutator) + if err != nil { + return ctrl.Result{}, err + } + + logger.Info("successfully reconciled") + return ctrl.Result{}, nil +} + +func kuadrantRoutePredicate() predicate.Predicate { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + // Lets filter for only Routes that have the kuadrant label and are enabled. + if val, ok := e.Object.GetLabels()[mappers.KuadrantManagedLabel]; ok { + enabled, _ := strconv.ParseBool(val) + return enabled + } + return false + }, + UpdateFunc: func(e event.UpdateEvent) bool { + // In case the update object had the kuadrant label set to true, we need to reconcile it. + if val, ok := e.ObjectOld.GetLabels()[mappers.KuadrantManagedLabel]; ok { + enabled, _ := strconv.ParseBool(val) + return enabled + } + // In case that route gets update by adding the label, and set to true, we need to reconcile it. + if val, ok := e.ObjectNew.GetLabels()[mappers.KuadrantManagedLabel]; ok { + enabled, _ := strconv.ParseBool(val) + return enabled + } + + return false + }, + DeleteFunc: func(e event.DeleteEvent) bool { + // If the object had the Kuadrant label, we need to handle its deletion + _, ok := e.Object.GetLabels()[mappers.KuadrantManagedLabel] + return ok + }, + } +} + +func desiredRateLimitFilterEnvoyFilterName() string { + return "kuadrant-ratelimit-http-filter" +} + +func desiredRateLimitDescriptorsEnvoyFilterName(route *routev1.Route) string { + return fmt.Sprintf("route-%s", route.Name) +} + +func (r *RouteReconciler) desiredAuthorizationPolicy(route *routev1.Route) *istiosecurityv1beta1.AuthorizationPolicy { + authPolicy := &istiosecurityv1beta1.AuthorizationPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: "AuthorizationPolicy", + APIVersion: "security.istio.io/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: route.Name, + Namespace: route.Namespace, + }, + } + + annotations := route.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + + providerName, ok := annotations[mappers.KuadrantAuthProviderAnnotation] + + if !ok { + common.TagObjectToDelete(authPolicy) + return authPolicy + } + + authPolicy.Spec = securityv1beta1.AuthorizationPolicy{ + Rules: []*securityv1beta1.Rule{ + { + To: []*securityv1beta1.Rule_To{ + { + Operation: &securityv1beta1.Operation{ + Hosts: []string{route.Spec.Host}, + }, + }, + }, + }, + }, + Action: securityv1beta1.AuthorizationPolicy_CUSTOM, + ActionDetail: &securityv1beta1.AuthorizationPolicy_Provider{ + Provider: &securityv1beta1.AuthorizationPolicy_ExtensionProvider{ + Name: providerName, + }, + }, + } + + return authPolicy +} + +func (r *RouteReconciler) deleteAuthPolicy(ctx context.Context, route *routev1.Route) error { + authPolicy := &istiosecurityv1beta1.AuthorizationPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: "AuthorizationPolicy", + APIVersion: "security.istio.io/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: route.Name, + Namespace: route.Namespace, + }, + } + + if err := r.DeleteResource(ctx, authPolicy); client.IgnoreNotFound(err) != nil { + return err + } + + return nil +} + +func (r *RouteReconciler) deleteRateLimitFilterEnvoyFilter(ctx context.Context, route *routev1.Route) error { + ef := &istionetworkingv1alpha3.EnvoyFilter{ + TypeMeta: metav1.TypeMeta{ + Kind: "EnvoyFilter", + APIVersion: "networking.istio.io/v1alpha3", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: desiredRateLimitFilterEnvoyFilterName(), + Namespace: route.Namespace, + }, + } + + if err := r.DeleteResource(ctx, ef); client.IgnoreNotFound(err) != nil { + return err + } + + return nil +} + +func (r *RouteReconciler) deleteRateLimitDescriptorsEnvoyFilter(ctx context.Context, route *routev1.Route) error { + ef := &istionetworkingv1alpha3.EnvoyFilter{ + TypeMeta: metav1.TypeMeta{ + Kind: "EnvoyFilter", + APIVersion: "networking.istio.io/v1alpha3", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: desiredRateLimitDescriptorsEnvoyFilterName(route), + Namespace: route.Namespace, + }, + } + + if err := r.DeleteResource(ctx, ef); client.IgnoreNotFound(err) != nil { + return err + } + + return nil +} + +func basicAuthPolicyMutator(existingObj, desiredObj client.Object) (bool, error) { + existing, ok := existingObj.(*istiosecurityv1beta1.AuthorizationPolicy) + if !ok { + return false, fmt.Errorf("%T is not a *istiosecurityv1beta1.AuthorizationPolicy", existingObj) + } + desired, ok := desiredObj.(*istiosecurityv1beta1.AuthorizationPolicy) + if !ok { + return false, fmt.Errorf("%T is not a *istiosecurityv1beta1.AuthorizationPolicy", desiredObj) + } + + updated := false + if !reflect.DeepEqual(existing.Spec, desired.Spec) { + existing.Spec = desired.Spec + updated = true + } + + tmpAnnotations := existing.GetAnnotations() + tmpUpdated := common.MergeMapStringString(&tmpAnnotations, desired.GetAnnotations()) + if tmpUpdated { + existing.SetAnnotations(tmpAnnotations) + updated = true + } + + tmpLabels := existing.GetLabels() + tmpUpdated = common.MergeMapStringString(&tmpLabels, desired.GetLabels()) + if tmpUpdated { + existing.SetLabels(tmpLabels) + updated = true + } + + return updated, nil +} + +func basicEnvoyFilterMutator(existingObj, desiredObj client.Object) (bool, error) { + existing, ok := existingObj.(*istionetworkingv1alpha3.EnvoyFilter) + if !ok { + return false, fmt.Errorf("%T is not a *istionetworkingv1alpha3.EnvoyFilter", existingObj) + } + desired, ok := desiredObj.(*istionetworkingv1alpha3.EnvoyFilter) + if !ok { + return false, fmt.Errorf("%T is not a *istionetworkingv1alpha3.EnvoyFilter", desiredObj) + } + + updated := false + if !reflect.DeepEqual(existing.Spec, desired.Spec) { + existing.Spec = desired.Spec + updated = true + } + + tmpAnnotations := existing.GetAnnotations() + tmpUpdated := common.MergeMapStringString(&tmpAnnotations, desired.GetAnnotations()) + if tmpUpdated { + existing.SetAnnotations(tmpAnnotations) + updated = true + } + + tmpLabels := existing.GetLabels() + tmpUpdated = common.MergeMapStringString(&tmpLabels, desired.GetLabels()) + if tmpUpdated { + existing.SetLabels(tmpLabels) + updated = true + } + + return updated, nil +} + +func (r *RouteReconciler) desiredRateLimitFilterEnvoyFilter(route *routev1.Route) (*istionetworkingv1alpha3.EnvoyFilter, error) { + ef := &istionetworkingv1alpha3.EnvoyFilter{ + TypeMeta: metav1.TypeMeta{ + Kind: "EnvoyFilter", + APIVersion: "networking.istio.io/v1alpha3", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: desiredRateLimitFilterEnvoyFilterName(), + Namespace: route.Namespace, + }, + } + + annotations := route.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + + _, ok := annotations[mappers.KuadrantRateLimitPolicyAnnotation] + + if !ok { + // TODO(eastizle): implement references and remove when all references are gone + common.TagObjectToDelete(ef) + return ef, nil + } + + patchUnstructured := map[string]interface{}{ + "operation": "INSERT_FIRST", // preauth should be the first filter in the chain + "value": map[string]interface{}{ + "name": "envoy.filters.http.preauth.ratelimit", + "typed_config": map[string]interface{}{ + "@type": "type.googleapis.com/envoy.extensions.filters.http.ratelimit.v3.RateLimit", + "domain": "preauth", + "stage": kuadrantistio.PreAuthStage, + "failure_mode_deny": true, + // If not specified, returns success immediately (can be useful for us) + "rate_limit_service": map[string]interface{}{ + "transport_api_version": "V3", + "grpc_service": map[string]interface{}{ + "timeout": "3s", + "envoy_grpc": map[string]string{ + "cluster_name": kuadrantistio.PatchedLimitadorClusterName, + }, + }, + }, + }, + }, + } + + patchRaw, _ := json.Marshal(patchUnstructured) + prePatch := istioapinetworkingv1alpha3.EnvoyFilter_Patch{} + if err := prePatch.UnmarshalJSON(patchRaw); err != nil { + return nil, err + } + + postPatch := prePatch.DeepCopy() + postPatch.Value.Fields["name"] = &types.Value{ + Kind: &types.Value_StringValue{ + StringValue: "envoy.filters.http.postauth.ratelimit", + }, + } + + // update domain for postauth filter + postPatch.Value.Fields["typed_config"].GetStructValue().Fields["domain"] = &types.Value{ + Kind: &types.Value_StringValue{ + StringValue: "postauth", + }, + } + // update stage for postauth filter + postPatch.Value.Fields["typed_config"].GetStructValue().Fields["stage"] = &types.Value{ + Kind: &types.Value_NumberValue{ + NumberValue: float64(kuadrantistio.PostAuthStage), + }, + } + // update operation for postauth filter + postPatch.Operation = istioapinetworkingv1alpha3.EnvoyFilter_Patch_INSERT_BEFORE + + preAuthFilterPatch := &istioapinetworkingv1alpha3.EnvoyFilter_EnvoyConfigObjectPatch{ + ApplyTo: istioapinetworkingv1alpha3.EnvoyFilter_HTTP_FILTER, + Match: &istioapinetworkingv1alpha3.EnvoyFilter_EnvoyConfigObjectMatch{ + Context: istioapinetworkingv1alpha3.EnvoyFilter_SIDECAR_INBOUND, + ObjectTypes: &istioapinetworkingv1alpha3.EnvoyFilter_EnvoyConfigObjectMatch_Listener{ + Listener: &istioapinetworkingv1alpha3.EnvoyFilter_ListenerMatch{ + FilterChain: &istioapinetworkingv1alpha3.EnvoyFilter_ListenerMatch_FilterChainMatch{ + Filter: &istioapinetworkingv1alpha3.EnvoyFilter_ListenerMatch_FilterMatch{ + Name: "envoy.filters.network.http_connection_manager", + }, + }, + }, + }, + }, + Patch: &prePatch, + } + + postAuthFilterPatch := preAuthFilterPatch.DeepCopy() + postAuthFilterPatch.Patch = postPatch + + // postauth filter should be injected just before the router filter + postAuthFilterPatch.Match.ObjectTypes = &istioapinetworkingv1alpha3.EnvoyFilter_EnvoyConfigObjectMatch_Listener{ + Listener: &istioapinetworkingv1alpha3.EnvoyFilter_ListenerMatch{ + FilterChain: &istioapinetworkingv1alpha3.EnvoyFilter_ListenerMatch_FilterChainMatch{ + Filter: &istioapinetworkingv1alpha3.EnvoyFilter_ListenerMatch_FilterMatch{ + Name: "envoy.filters.network.http_connection_manager", + SubFilter: &istioapinetworkingv1alpha3.EnvoyFilter_ListenerMatch_SubFilterMatch{ + Name: "envoy.filters.http.router", + }, + }, + }, + }, + } + + // Eventually, this should be dropped since it's a temp-fix for Kuadrant/limitador#53 + clusterPatch := kuadrantistio.LimitadorClusterEnvoyPatch() + + ef.Spec = istioapinetworkingv1alpha3.EnvoyFilter{ + ConfigPatches: []*istioapinetworkingv1alpha3.EnvoyFilter_EnvoyConfigObjectPatch{ + preAuthFilterPatch, + postAuthFilterPatch, + clusterPatch, + }, + } + + return ef, nil +} + +func (r *RouteReconciler) desiredDescriptorsEnvoyFilter(ctx context.Context, route *routev1.Route) (*istionetworkingv1alpha3.EnvoyFilter, error) { + ef := &istionetworkingv1alpha3.EnvoyFilter{ + TypeMeta: metav1.TypeMeta{ + Kind: "EnvoyFilter", + APIVersion: "networking.istio.io/v1alpha3", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: desiredRateLimitDescriptorsEnvoyFilterName(route), + Namespace: route.Namespace, + }, + } + + annotations := route.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + + rlpName, ok := annotations[mappers.KuadrantRateLimitPolicyAnnotation] + + if !ok { + common.TagObjectToDelete(ef) + return ef, nil + } + + rlpKey := client.ObjectKey{Name: rlpName, Namespace: route.Namespace} + rlp := &apimv1alpha1.RateLimitPolicy{} + if err := r.Client().Get(ctx, rlpKey, rlp); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + + patches := make([]*istioapinetworkingv1alpha3.EnvoyFilter_EnvoyConfigObjectPatch, 0) + + // A patch example would look like this: + // + //- applyTo: VIRTUAL_HOST + // match: + // context: SIDECAR_INBOUND + // routeConfiguration: + // vhost: + // name: inbound|http|80 + // patch: + // operation: MERGE + // value: + // rate_limits: + // - actions: + // - generic_key: + // descriptor_key: vhaction + // descriptor_value: "yes" + // stage: 0 + patchUnstructured := map[string]interface{}{ + "operation": "MERGE", + "value": map[string]interface{}{ + "rate_limits": kuadrantistio.EnvoyFilterRatelimitsUnstructured(rlp.Spec.RateLimits), + "typed_per_filter_config": map[string]interface{}{ + // Note the following name is different from what we have given to our pre/post-auth + // ratelimit filters. It's because you refer to the type of filter and not the name field + // of the filter. This infers it's configured for both filters in our case. + "envoy.filters.http.ratelimit": map[string]interface{}{ + "@type": "type.googleapis.com/envoy.extensions.filters.http.ratelimit.v3.RateLimitPerRoute", + "vh_rate_limits": "INCLUDE", + }, + }, + }, + } + + patchRaw, _ := json.Marshal(patchUnstructured) + patch := &istioapinetworkingv1alpha3.EnvoyFilter_Patch{} + if err := patch.UnmarshalJSON(patchRaw); err != nil { + //TODO(eguzki): handle error + panic(err) + } + + vhPatch := &istioapinetworkingv1alpha3.EnvoyFilter_EnvoyConfigObjectPatch{ + ApplyTo: istioapinetworkingv1alpha3.EnvoyFilter_VIRTUAL_HOST, + Match: &istioapinetworkingv1alpha3.EnvoyFilter_EnvoyConfigObjectMatch{ + Context: istioapinetworkingv1alpha3.EnvoyFilter_SIDECAR_INBOUND, + ObjectTypes: &istioapinetworkingv1alpha3.EnvoyFilter_EnvoyConfigObjectMatch_RouteConfiguration{ + RouteConfiguration: &istioapinetworkingv1alpha3.EnvoyFilter_RouteConfigurationMatch{ + Vhost: &istioapinetworkingv1alpha3.EnvoyFilter_RouteConfigurationMatch_VirtualHostMatch{ + // TODO(eastizle): hardcoded. If the API k8s service does not expose API in port 80, it will not work + Name: "inbound|http|80", + }, + }, + }, + }, + Patch: patch, + } + + patches = append(patches, vhPatch) + + ef.Spec = istioapinetworkingv1alpha3.EnvoyFilter{ + ConfigPatches: patches, + } + + return ef, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *RouteReconciler) SetupWithManager(mgr ctrl.Manager) error { + rateLimitPolicyToRouteEventMapper := &mappers.RateLimitPolicyToRouteEventMapper{ + K8sClient: r.Client(), + Logger: r.Logger().WithName("rateLimitPolicyToRouteHandler"), + } + + return ctrl.NewControllerManagedBy(mgr). + For(&routev1.Route{}, builder.WithPredicates(kuadrantRoutePredicate())). + Watches( + &source.Kind{Type: &apimv1alpha1.RateLimitPolicy{}}, + handler.EnqueueRequestsFromMapFunc(rateLimitPolicyToRouteEventMapper.Map), + ). + WithLogger(log.Log). // use base logger, the manager will add prefixes for watched sources + Complete(r) +} diff --git a/doc/openshift-routes.md b/doc/openshift-routes.md new file mode 100644 index 0000000..962af6c --- /dev/null +++ b/doc/openshift-routes.md @@ -0,0 +1,232 @@ +This guide shows how to apply API protection (authZ and rate limiting) on Openshift Routes. +The protected API service must be part of the Istio's mesh injecting a sidecar in the service's pod. + +### Installing Istio in openshift + +[Official doc: Istio on Openshift](https://istio.io/latest/docs/setup/platform-setup/openshift/) + +``` +oc adm policy add-scc-to-group anyuid system:serviceaccounts:istio-system +``` + +``` +istioctl install --set profile=openshift +``` + +### Install kuadrant + +Patch istio with Authorino as ext authZ +``` +kubectl edit configmap istio -n istio-system +``` + +In the editor, add the extension provider definitions +```yaml +data: + mesh: |- + # Add the following content to define the external authorizers. + extensionProviders: + - name: "kuadrant-authorization" + envoyExtAuthzGrpc: + service: "authorino-authorino-authorization.kuadrant-system.svc.cluster.local" + port: "50051" +``` + +Restart Istiod to allow the change to take effect with the following command: +``` +kubectl rollout restart deployment/istiod -n istio-system +``` + +Install kuadrant components + +``` +export KUADRANT_NAMESPACE="kuadrant-system" +oc new-project "${KUADRANT_NAMESPACE}" + +# Authorino +kubectl apply -f utils/local-deployment/authorino-operator.yaml +kubectl apply -n "${KUADRANT_NAMESPACE}" -f utils/local-deployment/authorino.yaml + +# Limitador +kubectl apply -n "${KUADRANT_NAMESPACE}" -f utils/local-deployment/limitador-operator.yaml +kubectl apply -n "${KUADRANT_NAMESPACE}" -f utils/local-deployment/limitador.yaml + +kubectl -n "${KUADRANT_NAMESPACE}" wait --timeout=300s --for=condition=Available deployments --all + +# Run locally the kuadrant controller +make run +``` + +### User Guide + +#### Deploy Toystore app + +``` +oc new-project myns + +oc adm policy add-scc-to-group anyuid system:serviceaccounts:myns + +oc apply -n myns -f - <