From 60e15b73e8ee1c2f65bd96c0dc57739010539ada Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Thu, 14 Nov 2024 11:22:17 +0100 Subject: [PATCH 1/4] Standarize gateway API asset deployment Signed-off-by: Raul Sevilla --- cmd/ingress-perf.go | 10 +++-- pkg/runner/runner.go | 52 ++++++++++++++++++------- pkg/runner/types.go | 93 +++++++++++++++++--------------------------- 3 files changed, 81 insertions(+), 74 deletions(-) diff --git a/cmd/ingress-perf.go b/cmd/ingress-perf.go index 6f27b79..6fca21a 100644 --- a/cmd/ingress-perf.go +++ b/cmd/ingress-perf.go @@ -43,7 +43,7 @@ var versionCmd = &cobra.Command{ } func run() *cobra.Command { - var cfg, uuid, esServer, esIndex, logLevel, outputDir, igNamespace string + var cfg, uuid, esServer, esIndex, logLevel, outputDir, igNamespace, gatewayLB, gwClassController string var cleanup, podMetrics, serviceMesh, gatewayAPI bool cmd := &cobra.Command{ Use: "run", @@ -67,7 +67,7 @@ func run() *cobra.Command { uuid, cleanup, runner.WithIndexer(esServer, esIndex, outputDir, podMetrics), runner.WithServiceMesh(serviceMesh, igNamespace), - runner.WithGatewayAPI(gatewayAPI), + runner.WithGatewayAPI(gatewayAPI, gatewayLB, gwClassController), ) return r.Start() }, @@ -80,9 +80,11 @@ func run() *cobra.Command { cmd.Flags().BoolVar(&cleanup, "cleanup", true, "Cleanup benchmark assets") cmd.Flags().BoolVar(&podMetrics, "pod-metrics", false, "Index per pod metrics") cmd.Flags().StringVar(&logLevel, "loglevel", "info", "Log level. Allowed levels are error, info and debug") - cmd.Flags().StringVar(&igNamespace, "gw-ns", "istio-system", "Ingress gateway namespace") cmd.Flags().BoolVar(&serviceMesh, "service-mesh", false, "Enable service mesh mode") - cmd.Flags().BoolVar(&gatewayAPI, "gateway-api", false, "Enable gateway api mode") + cmd.Flags().StringVar(&igNamespace, "gw-ns", "istio-system", "Ingress gateway namespace") + cmd.Flags().BoolVar(&gatewayAPI, "gw-api", false, "Enable gateway API mode") + cmd.Flags().StringVar(&gatewayLB, "gw-lb", "gateway", "Name of the load balancer service of the gateway pods") + cmd.Flags().StringVar(&gwClassController, "gw-class", "openshift.io/gateway-controller", "Gateway class controller name") cmd.MarkFlagRequired("cfg") return cmd } diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index ded3cac..d2c0848 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -32,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/utils/ptr" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" @@ -43,7 +44,7 @@ import ( openshiftrouteclientset "github.com/openshift/client-go/route/clientset/versioned" istioclient "istio.io/client-go/pkg/clientset/versioned" "k8s.io/client-go/tools/clientcmd" - gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" gatewayApiClientset "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" ) @@ -104,9 +105,11 @@ func WithServiceMesh(enable bool, igNamespace string) OptsFunctions { } } -func WithGatewayAPI(enable bool) OptsFunctions { +func WithGatewayAPI(enable bool, gwLb, gwClassController string) OptsFunctions { return func(r *Runner) { r.gatewayAPI = enable + r.gwLb = gwLb + r.gwClassController = gwClassController } } @@ -309,23 +312,27 @@ func (r *Runner) deployAssets() error { if err != nil { return err } - listenerHostName = gatewayv1beta1.Hostname("*.gwapi." + ingressDomain) - httproutes.Spec.Hostnames = append(httproutes.Spec.Hostnames, gatewayv1beta1.Hostname("nginx.gwapi."+ingressDomain)) - log.Debugf("Creating GatewayClass...") - _, err = hrClientSet.GatewayV1beta1().GatewayClasses().Create(context.TODO(), gatewayClass, metav1.CreateOptions{}) - if err != nil && !errors.IsAlreadyExists(err) { + listenerHostName = gwv1.Hostname("*.gwapi." + ingressDomain) + httproutes.Spec.Hostnames = append(httproutes.Spec.Hostnames, gwv1.Hostname("nginx.gwapi."+ingressDomain)) + svc, err := clientSet.CoreV1().Services(r.igNamespace).Get(context.TODO(), r.gwLb, metav1.GetOptions{}) + if err != nil { return err } - time.Sleep(5 * time.Second) // wait for ServiceMeshControlPlane to be ready - log.Debugf("Creating Gateway...") - _, err = hrClientSet.GatewayV1beta1().Gateways(string(gatewayNamespace)).Create(context.TODO(), &gateway, metav1.CreateOptions{}) + gateway.Spec.Addresses = append(gateway.Spec.Addresses, gwv1.GatewayAddress{ + Type: ptr.To(gwv1.HostnameAddressType), + Value: svc.Name, + }) + gateway.Spec.GatewayClassName = gwv1.ObjectName(r.gwClassController) + log.Debugf("Creating Gateway bound to gateway class %s", r.gwClassController) + _, err = hrClientSet.GatewayV1().Gateways(r.igNamespace).Create(context.TODO(), &gateway, metav1.CreateOptions{}) if err != nil && !errors.IsAlreadyExists(err) { return err } - log.Debugf("Waiting 4 minutes for Gateway to be ready...") - time.Sleep(4 * time.Minute) + if err := waitForGateway(r.igNamespace, gateway.Name, 4*time.Minute); err != nil { + return err + } log.Debugf("Creating HTTPRoute...") - _, err := hrClientSet.GatewayV1beta1().HTTPRoutes(routesNamespace).Create(context.TODO(), &httproutes, metav1.CreateOptions{}) + _, err = hrClientSet.GatewayV1().HTTPRoutes(r.igNamespace).Create(context.TODO(), &httproutes, metav1.CreateOptions{}) if err != nil && !errors.IsAlreadyExists(err) { return err } @@ -389,3 +396,22 @@ func waitForDeployment(ns, deployment string, maxWaitTimeout time.Duration) erro } return err } + +func waitForGateway(ns, gateway string, maxWaitTimeout time.Duration) error { + var err error + var gw *gwv1.Gateway + log.Infof("Waiting for gateway %s in ns %s to be programmed", gateway, ns) + err = wait.PollUntilContextTimeout(context.TODO(), time.Second, maxWaitTimeout, true, func(ctx context.Context) (bool, error) { + gw, err = hrClientSet.GatewayV1().Gateways(ns).Get(context.TODO(), gateway, metav1.GetOptions{}) + if err != nil { + return false, err + } + for _, cond := range gw.Status.Conditions { + if cond.Reason == string(gwv1.GatewayConditionProgrammed) && cond.Status == metav1.ConditionTrue { + return true, nil + } + } + return false, nil + }) + return err +} diff --git a/pkg/runner/types.go b/pkg/runner/types.go index 658b948..67a41d5 100644 --- a/pkg/runner/types.go +++ b/pkg/runner/types.go @@ -27,7 +27,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" - gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" ) const ( @@ -38,24 +38,24 @@ const ( ) type Runner struct { - uuid string - indexer *indexers.Indexer - podMetrics bool - cleanup bool - serviceMesh bool - gatewayAPI bool - igNamespace string + uuid string + indexer *indexers.Indexer + podMetrics bool + cleanup bool + serviceMesh bool + gatewayAPI bool + gwLb string + igNamespace string + gwClassController string } type OptsFunctions func(r *Runner) var routesNamespace = benchmarkNs.Name -var gatewayClassName = "openshift-default" -var gatewayNamespace gatewayv1beta1.Namespace = "openshift-ingress" -var portNumber gatewayv1beta1.PortNumber = 8080 -var tlsType gatewayv1beta1.TLSModeType = "Terminate" -var fromNamespaces gatewayv1beta1.FromNamespaces = "All" -var listenerHostName gatewayv1beta1.Hostname +var portNumber gwv1.PortNumber = 8080 +var tlsType gwv1.TLSModeType = "Terminate" +var fromNamespaces gwv1.FromNamespaces = "All" +var listenerHostName gwv1.Hostname var ingressDomain string var benchmarkNs = corev1.Namespace{ @@ -343,39 +343,19 @@ var virtualService = v1networking.VirtualService{ }, } -var gatewayClass = &gatewayv1beta1.GatewayClass{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "gateway.networking.k8s.io/v1beta1", - Kind: "GatewayClass", - }, +var gateway = gwv1.Gateway{ ObjectMeta: metav1.ObjectMeta{ - Name: gatewayClassName, - }, - Spec: gatewayv1beta1.GatewayClassSpec{ - ControllerName: "openshift.io/gateway-controller", - }, -} - -var gateway = gatewayv1beta1.Gateway{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "gateway.networking.k8s.io/v1beta1", - Kind: "Gateway", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "gateway", - Namespace: string(gatewayNamespace), + Name: "gateway", }, - Spec: gatewayv1beta1.GatewaySpec{ - GatewayClassName: gatewayv1beta1.ObjectName(gatewayClassName), - Listeners: []gatewayv1beta1.Listener{ - + Spec: gwv1.GatewaySpec{ + Listeners: []gwv1.Listener{ { Name: "http", Port: 80, - Protocol: gatewayv1beta1.ProtocolType("HTTP"), + Protocol: gwv1.ProtocolType("HTTP"), Hostname: &listenerHostName, - AllowedRoutes: &gatewayv1beta1.AllowedRoutes{ - Namespaces: &gatewayv1beta1.RouteNamespaces{ + AllowedRoutes: &gwv1.AllowedRoutes{ + Namespaces: &gwv1.RouteNamespaces{ From: &fromNamespaces, }, }, @@ -383,16 +363,16 @@ var gateway = gatewayv1beta1.Gateway{ { Name: "https", Port: 443, - Protocol: gatewayv1beta1.ProtocolType("HTTPS"), + Protocol: gwv1.ProtocolType("HTTPS"), Hostname: &listenerHostName, - AllowedRoutes: &gatewayv1beta1.AllowedRoutes{ - Namespaces: &gatewayv1beta1.RouteNamespaces{ + AllowedRoutes: &gwv1.AllowedRoutes{ + Namespaces: &gwv1.RouteNamespaces{ From: &fromNamespaces, }, }, - TLS: &gatewayv1beta1.GatewayTLSConfig{ + TLS: &gwv1.GatewayTLSConfig{ Mode: &tlsType, - CertificateRefs: []gatewayv1beta1.SecretObjectReference{ + CertificateRefs: []gwv1.SecretObjectReference{ { Name: "router-certs-default", }, @@ -403,26 +383,25 @@ var gateway = gatewayv1beta1.Gateway{ }, } -var httproutes = gatewayv1beta1.HTTPRoute{ +var httproutes = gwv1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: service.Name, }, - Spec: gatewayv1beta1.HTTPRouteSpec{ - CommonRouteSpec: gatewayv1beta1.CommonRouteSpec{ - ParentRefs: []gatewayv1beta1.ParentReference{ + Spec: gwv1.HTTPRouteSpec{ + CommonRouteSpec: gwv1.CommonRouteSpec{ + ParentRefs: []gwv1.ParentReference{ { - Namespace: &gatewayNamespace, - Name: gatewayv1beta1.ObjectName("gateway"), + Name: gwv1.ObjectName("gateway"), }, }, }, - Rules: []gatewayv1beta1.HTTPRouteRule{ + Rules: []gwv1.HTTPRouteRule{ { - BackendRefs: []gatewayv1beta1.HTTPBackendRef{ + BackendRefs: []gwv1.HTTPBackendRef{ { - BackendRef: gatewayv1beta1.BackendRef{ - BackendObjectReference: gatewayv1beta1.BackendObjectReference{ - Name: gatewayv1beta1.ObjectName(service.Name), + BackendRef: gwv1.BackendRef{ + BackendObjectReference: gwv1.BackendObjectReference{ + Name: gwv1.ObjectName(service.Name), Port: &portNumber, }, }, From d021019c751e3c11fffecb5c3e4d926446a101f5 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Thu, 14 Nov 2024 22:18:10 +0100 Subject: [PATCH 2/4] Deploy HTTP routes in ingress-perf namespace Signed-off-by: Raul Sevilla --- pkg/runner/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index d2c0848..258dd7d 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -332,7 +332,7 @@ func (r *Runner) deployAssets() error { return err } log.Debugf("Creating HTTPRoute...") - _, err = hrClientSet.GatewayV1().HTTPRoutes(r.igNamespace).Create(context.TODO(), &httproutes, metav1.CreateOptions{}) + _, err = hrClientSet.GatewayV1().HTTPRoutes(routesNamespace).Create(context.TODO(), &httproutes, metav1.CreateOptions{}) if err != nil && !errors.IsAlreadyExists(err) { return err } From 5d9e99138d59c72e5f5668706f9c427a328e0879 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Thu, 14 Nov 2024 23:53:38 +0100 Subject: [PATCH 3/4] Prometheus expressions Signed-off-by: Raul Sevilla --- cmd/ingress-perf.go | 2 +- pkg/runner/runner.go | 32 +++++++++++++++++++++----------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/cmd/ingress-perf.go b/cmd/ingress-perf.go index 6fca21a..d22bea1 100644 --- a/cmd/ingress-perf.go +++ b/cmd/ingress-perf.go @@ -67,7 +67,7 @@ func run() *cobra.Command { uuid, cleanup, runner.WithIndexer(esServer, esIndex, outputDir, podMetrics), runner.WithServiceMesh(serviceMesh, igNamespace), - runner.WithGatewayAPI(gatewayAPI, gatewayLB, gwClassController), + runner.WithGatewayAPI(gatewayAPI, igNamespace, gatewayLB, gwClassController), ) return r.Start() }, diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 258dd7d..5efec78 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -96,20 +96,29 @@ func WithIndexer(esServer, esIndex, resultsDir string, podMetrics bool) OptsFunc func WithServiceMesh(enable bool, igNamespace string) OptsFunctions { return func(r *Runner) { - r.serviceMesh = enable - r.igNamespace = igNamespace - config.PrometheusQueries["avg_cpu_usage_ingress_gateway_pods"] = - fmt.Sprintf("avg(avg_over_time(sum(irate(container_cpu_usage_seconds_total{name!='', namespace='%s', pod=~'istio-ingressgateway.+'}[2m])) by (pod)[ELAPSED:]))", igNamespace) - config.PrometheusQueries["avg_memory_usage_ingress_gateway_pods_bytes"] = - fmt.Sprintf("avg(avg_over_time(sum(container_memory_working_set_bytes{name!='', namespace='%s', pod=~'istio-ingressgateway.+'}) by (pod)[ELAPSED:]))", igNamespace) + if enable { + r.serviceMesh = enable + r.igNamespace = igNamespace + config.PrometheusQueries["avg_cpu_usage_ingress_gateway_pods"] = + fmt.Sprintf("avg(avg_over_time(sum(irate(container_cpu_usage_seconds_total{name!='', namespace='%s', pod=~'istio-ingressgateway.+'}[2m])) by (pod)[ELAPSED:]))", igNamespace) + config.PrometheusQueries["avg_memory_usage_ingress_gateway_pods_bytes"] = + fmt.Sprintf("avg(avg_over_time(sum(container_memory_working_set_bytes{name!='', namespace='%s', pod=~'istio-ingressgateway.+'}) by (pod)[ELAPSED:]))", igNamespace) + } } } -func WithGatewayAPI(enable bool, gwLb, gwClassController string) OptsFunctions { +func WithGatewayAPI(enable bool, igNamespace, gwLb, gwClassController string) OptsFunctions { return func(r *Runner) { - r.gatewayAPI = enable - r.gwLb = gwLb - r.gwClassController = gwClassController + if enable { + r.gatewayAPI = enable + r.gwLb = gwLb + r.gwClassController = gwClassController + r.igNamespace = igNamespace + config.PrometheusQueries["avg_cpu_usage_ingress_gateway_pods"] = + fmt.Sprintf("avg(avg_over_time(sum(irate(container_cpu_usage_seconds_total{name!='', namespace='%s', container='istio-proxy'}[2m])) by (pod)[ELAPSED:]))", igNamespace) + config.PrometheusQueries["avg_memory_usage_ingress_gateway_pods_bytes"] = + fmt.Sprintf("avg(avg_over_time(sum(container_memory_working_set_bytes{name!='', namespace='%s', container='istio-proxy'}) by (pod)[ELAPSED:]))", igNamespace) + } } } @@ -195,7 +204,7 @@ func (r *Runner) Start() error { // When not using local indexer, empty the documents array when all documents after indexing them if _, ok := (*r.indexer).(*indexers.Local); !ok { if indexDocuments(*r.indexer, benchmarkResultDocuments, indexers.IndexingOpts{}) != nil { - log.Errorf("Indexing error: %v", err.Error()) + log.Errorf("Indexing error: %v", err) } benchmarkResultDocuments = []interface{}{} } @@ -332,6 +341,7 @@ func (r *Runner) deployAssets() error { return err } log.Debugf("Creating HTTPRoute...") + httproutes.Spec.ParentRefs[0].Namespace = (*gwv1.Namespace)(&r.igNamespace) _, err = hrClientSet.GatewayV1().HTTPRoutes(routesNamespace).Create(context.TODO(), &httproutes, metav1.CreateOptions{}) if err != nil && !errors.IsAlreadyExists(err) { return err From d768c733811a14a9c82c9a9875dc24a3763f67e9 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Tue, 3 Dec 2024 10:13:30 +0100 Subject: [PATCH 4/4] Fix linting Signed-off-by: Raul Sevilla --- pkg/runner/runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 5efec78..7ab0041 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -410,8 +410,8 @@ func waitForDeployment(ns, deployment string, maxWaitTimeout time.Duration) erro func waitForGateway(ns, gateway string, maxWaitTimeout time.Duration) error { var err error var gw *gwv1.Gateway - log.Infof("Waiting for gateway %s in ns %s to be programmed", gateway, ns) - err = wait.PollUntilContextTimeout(context.TODO(), time.Second, maxWaitTimeout, true, func(ctx context.Context) (bool, error) { + log.Infof("Waiting for gateway/%s in ns %s to be programmed", gateway, ns) + err = wait.PollUntilContextTimeout(context.TODO(), time.Second, maxWaitTimeout, true, func(_ context.Context) (bool, error) { gw, err = hrClientSet.GatewayV1().Gateways(ns).Get(context.TODO(), gateway, metav1.GetOptions{}) if err != nil { return false, err