Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Standarize gateway API asset deployment #74

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Standarize gateway API asset deployment #74

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
60e15b7
Standarize gateway API asset deployment
rsevilla87 Nov 14, 2024
d021019
Deploy HTTP routes in ingress-perf namespace
rsevilla87 Nov 14, 2024
5d9e991
Prometheus expressions
rsevilla87 Nov 14, 2024
d768c73
Fix linting
rsevilla87 Dec 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions cmd/ingress-perf.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ var versionCmd = &cobra.Command{
}

func run() *cobra.Command {
var cfg, uuid, esServer, esIndex, logLevel, outputDir, igNamespace string
var cfg, uuid, esServer, esIndex, logLevel, outputDir, igNamespace, gatewayLB, gwClassController string
var cleanup, podMetrics, serviceMesh, gatewayAPI bool
cmd := &cobra.Command{
Use: "run",
Expand All @@ -67,7 +67,7 @@ func run() *cobra.Command {
uuid, cleanup,
runner.WithIndexer(esServer, esIndex, outputDir, podMetrics),
runner.WithServiceMesh(serviceMesh, igNamespace),
runner.WithGatewayAPI(gatewayAPI),
runner.WithGatewayAPI(gatewayAPI, igNamespace, gatewayLB, gwClassController),
)
return r.Start()
},
Expand All @@ -80,9 +80,11 @@ func run() *cobra.Command {
cmd.Flags().BoolVar(&cleanup, "cleanup", true, "Cleanup benchmark assets")
cmd.Flags().BoolVar(&podMetrics, "pod-metrics", false, "Index per pod metrics")
cmd.Flags().StringVar(&logLevel, "loglevel", "info", "Log level. Allowed levels are error, info and debug")
cmd.Flags().StringVar(&igNamespace, "gw-ns", "istio-system", "Ingress gateway namespace")
cmd.Flags().BoolVar(&serviceMesh, "service-mesh", false, "Enable service mesh mode")
cmd.Flags().BoolVar(&gatewayAPI, "gateway-api", false, "Enable gateway api mode")
cmd.Flags().StringVar(&igNamespace, "gw-ns", "istio-system", "Ingress gateway namespace")
cmd.Flags().BoolVar(&gatewayAPI, "gw-api", false, "Enable gateway API mode")
cmd.Flags().StringVar(&gatewayLB, "gw-lb", "gateway", "Name of the load balancer service of the gateway pods")
cmd.Flags().StringVar(&gwClassController, "gw-class", "openshift.io/gateway-controller", "Gateway class controller name")
cmd.MarkFlagRequired("cfg")
return cmd
}
Expand Down
78 changes: 57 additions & 21 deletions pkg/runner/runner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/utils/ptr"

"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/wait"
Expand All @@ -43,7 +44,7 @@ import (
openshiftrouteclientset "github.com/openshift/client-go/route/clientset/versioned"
istioclient "istio.io/client-go/pkg/clientset/versioned"
"k8s.io/client-go/tools/clientcmd"
gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
gwv1 "sigs.k8s.io/gateway-api/apis/v1"
gatewayApiClientset "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned"
)

Expand Down Expand Up @@ -95,18 +96,29 @@ func WithIndexer(esServer, esIndex, resultsDir string, podMetrics bool) OptsFunc

func WithServiceMesh(enable bool, igNamespace string) OptsFunctions {
return func(r *Runner) {
r.serviceMesh = enable
r.igNamespace = igNamespace
config.PrometheusQueries["avg_cpu_usage_ingress_gateway_pods"] =
fmt.Sprintf("avg(avg_over_time(sum(irate(container_cpu_usage_seconds_total{name!='', namespace='%s', pod=~'istio-ingressgateway.+'}[2m])) by (pod)[ELAPSED:]))", igNamespace)
config.PrometheusQueries["avg_memory_usage_ingress_gateway_pods_bytes"] =
fmt.Sprintf("avg(avg_over_time(sum(container_memory_working_set_bytes{name!='', namespace='%s', pod=~'istio-ingressgateway.+'}) by (pod)[ELAPSED:]))", igNamespace)
if enable {
r.serviceMesh = enable
r.igNamespace = igNamespace
config.PrometheusQueries["avg_cpu_usage_ingress_gateway_pods"] =
fmt.Sprintf("avg(avg_over_time(sum(irate(container_cpu_usage_seconds_total{name!='', namespace='%s', pod=~'istio-ingressgateway.+'}[2m])) by (pod)[ELAPSED:]))", igNamespace)
config.PrometheusQueries["avg_memory_usage_ingress_gateway_pods_bytes"] =
fmt.Sprintf("avg(avg_over_time(sum(container_memory_working_set_bytes{name!='', namespace='%s', pod=~'istio-ingressgateway.+'}) by (pod)[ELAPSED:]))", igNamespace)
}
}
}

func WithGatewayAPI(enable bool) OptsFunctions {
func WithGatewayAPI(enable bool, igNamespace, gwLb, gwClassController string) OptsFunctions {
return func(r *Runner) {
r.gatewayAPI = enable
if enable {
r.gatewayAPI = enable
r.gwLb = gwLb
r.gwClassController = gwClassController
r.igNamespace = igNamespace
config.PrometheusQueries["avg_cpu_usage_ingress_gateway_pods"] =
fmt.Sprintf("avg(avg_over_time(sum(irate(container_cpu_usage_seconds_total{name!='', namespace='%s', container='istio-proxy'}[2m])) by (pod)[ELAPSED:]))", igNamespace)
config.PrometheusQueries["avg_memory_usage_ingress_gateway_pods_bytes"] =
fmt.Sprintf("avg(avg_over_time(sum(container_memory_working_set_bytes{name!='', namespace='%s', container='istio-proxy'}) by (pod)[ELAPSED:]))", igNamespace)
}
}
}

Expand Down Expand Up @@ -192,7 +204,7 @@ func (r *Runner) Start() error {
// When not using local indexer, empty the documents array when all documents after indexing them
if _, ok := (*r.indexer).(*indexers.Local); !ok {
if indexDocuments(*r.indexer, benchmarkResultDocuments, indexers.IndexingOpts{}) != nil {
log.Errorf("Indexing error: %v", err.Error())
log.Errorf("Indexing error: %v", err)
}
benchmarkResultDocuments = []interface{}{}
}
Expand Down Expand Up @@ -309,23 +321,28 @@ func (r *Runner) deployAssets() error {
if err != nil {
return err
}
listenerHostName = gatewayv1beta1.Hostname("*.gwapi." + ingressDomain)
httproutes.Spec.Hostnames = append(httproutes.Spec.Hostnames, gatewayv1beta1.Hostname("nginx.gwapi."+ingressDomain))
log.Debugf("Creating GatewayClass...")
_, err = hrClientSet.GatewayV1beta1().GatewayClasses().Create(context.TODO(), gatewayClass, metav1.CreateOptions{})
if err != nil && !errors.IsAlreadyExists(err) {
listenerHostName = gwv1.Hostname("*.gwapi." + ingressDomain)
httproutes.Spec.Hostnames = append(httproutes.Spec.Hostnames, gwv1.Hostname("nginx.gwapi."+ingressDomain))
svc, err := clientSet.CoreV1().Services(r.igNamespace).Get(context.TODO(), r.gwLb, metav1.GetOptions{})
if err != nil {
return err
}
time.Sleep(5 * time.Second) // wait for ServiceMeshControlPlane to be ready
log.Debugf("Creating Gateway...")
_, err = hrClientSet.GatewayV1beta1().Gateways(string(gatewayNamespace)).Create(context.TODO(), &gateway, metav1.CreateOptions{})
gateway.Spec.Addresses = append(gateway.Spec.Addresses, gwv1.GatewayAddress{
Type: ptr.To(gwv1.HostnameAddressType),
Value: svc.Name,
})
gateway.Spec.GatewayClassName = gwv1.ObjectName(r.gwClassController)
log.Debugf("Creating Gateway bound to gateway class %s", r.gwClassController)
_, err = hrClientSet.GatewayV1().Gateways(r.igNamespace).Create(context.TODO(), &gateway, metav1.CreateOptions{})
if err != nil && !errors.IsAlreadyExists(err) {
return err
}
log.Debugf("Waiting 4 minutes for Gateway to be ready...")
time.Sleep(4 * time.Minute)
if err := waitForGateway(r.igNamespace, gateway.Name, 4*time.Minute); err != nil {
return err
}
log.Debugf("Creating HTTPRoute...")
_, err := hrClientSet.GatewayV1beta1().HTTPRoutes(routesNamespace).Create(context.TODO(), &httproutes, metav1.CreateOptions{})
httproutes.Spec.ParentRefs[0].Namespace = (*gwv1.Namespace)(&r.igNamespace)
_, err = hrClientSet.GatewayV1().HTTPRoutes(routesNamespace).Create(context.TODO(), &httproutes, metav1.CreateOptions{})
if err != nil && !errors.IsAlreadyExists(err) {
return err
}
Expand Down Expand Up @@ -389,3 +406,22 @@ func waitForDeployment(ns, deployment string, maxWaitTimeout time.Duration) erro
}
return err
}

func waitForGateway(ns, gateway string, maxWaitTimeout time.Duration) error {
var err error
var gw *gwv1.Gateway
log.Infof("Waiting for gateway/%s in ns %s to be programmed", gateway, ns)
err = wait.PollUntilContextTimeout(context.TODO(), time.Second, maxWaitTimeout, true, func(_ context.Context) (bool, error) {
gw, err = hrClientSet.GatewayV1().Gateways(ns).Get(context.TODO(), gateway, metav1.GetOptions{})
if err != nil {
return false, err
}
for _, cond := range gw.Status.Conditions {
if cond.Reason == string(gwv1.GatewayConditionProgrammed) && cond.Status == metav1.ConditionTrue {
return true, nil
}
}
return false, nil
})
return err
}
93 changes: 36 additions & 57 deletions pkg/runner/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/utils/ptr"
gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
gwv1 "sigs.k8s.io/gateway-api/apis/v1"
)

const (
Expand All @@ -38,24 +38,24 @@ const (
)

type Runner struct {
uuid string
indexer *indexers.Indexer
podMetrics bool
cleanup bool
serviceMesh bool
gatewayAPI bool
igNamespace string
uuid string
indexer *indexers.Indexer
podMetrics bool
cleanup bool
serviceMesh bool
gatewayAPI bool
gwLb string
igNamespace string
gwClassController string
}

type OptsFunctions func(r *Runner)

var routesNamespace = benchmarkNs.Name
var gatewayClassName = "openshift-default"
var gatewayNamespace gatewayv1beta1.Namespace = "openshift-ingress"
var portNumber gatewayv1beta1.PortNumber = 8080
var tlsType gatewayv1beta1.TLSModeType = "Terminate"
var fromNamespaces gatewayv1beta1.FromNamespaces = "All"
var listenerHostName gatewayv1beta1.Hostname
var portNumber gwv1.PortNumber = 8080
var tlsType gwv1.TLSModeType = "Terminate"
var fromNamespaces gwv1.FromNamespaces = "All"
var listenerHostName gwv1.Hostname
var ingressDomain string

var benchmarkNs = corev1.Namespace{
Expand Down Expand Up @@ -343,56 +343,36 @@ var virtualService = v1networking.VirtualService{
},
}

var gatewayClass = &gatewayv1beta1.GatewayClass{
TypeMeta: metav1.TypeMeta{
APIVersion: "gateway.networking.k8s.io/v1beta1",
Kind: "GatewayClass",
},
var gateway = gwv1.Gateway{
ObjectMeta: metav1.ObjectMeta{
Name: gatewayClassName,
},
Spec: gatewayv1beta1.GatewayClassSpec{
ControllerName: "openshift.io/gateway-controller",
},
}

var gateway = gatewayv1beta1.Gateway{
TypeMeta: metav1.TypeMeta{
APIVersion: "gateway.networking.k8s.io/v1beta1",
Kind: "Gateway",
},
ObjectMeta: metav1.ObjectMeta{
Name: "gateway",
Namespace: string(gatewayNamespace),
Name: "gateway",
},
Spec: gatewayv1beta1.GatewaySpec{
GatewayClassName: gatewayv1beta1.ObjectName(gatewayClassName),
Listeners: []gatewayv1beta1.Listener{

Spec: gwv1.GatewaySpec{
Listeners: []gwv1.Listener{
{
Name: "http",
Port: 80,
Protocol: gatewayv1beta1.ProtocolType("HTTP"),
Protocol: gwv1.ProtocolType("HTTP"),
Hostname: &listenerHostName,
AllowedRoutes: &gatewayv1beta1.AllowedRoutes{
Namespaces: &gatewayv1beta1.RouteNamespaces{
AllowedRoutes: &gwv1.AllowedRoutes{
Namespaces: &gwv1.RouteNamespaces{
From: &fromNamespaces,
},
},
},
{
Name: "https",
Port: 443,
Protocol: gatewayv1beta1.ProtocolType("HTTPS"),
Protocol: gwv1.ProtocolType("HTTPS"),
Hostname: &listenerHostName,
AllowedRoutes: &gatewayv1beta1.AllowedRoutes{
Namespaces: &gatewayv1beta1.RouteNamespaces{
AllowedRoutes: &gwv1.AllowedRoutes{
Namespaces: &gwv1.RouteNamespaces{
From: &fromNamespaces,
},
},
TLS: &gatewayv1beta1.GatewayTLSConfig{
TLS: &gwv1.GatewayTLSConfig{
Mode: &tlsType,
CertificateRefs: []gatewayv1beta1.SecretObjectReference{
CertificateRefs: []gwv1.SecretObjectReference{
{
Name: "router-certs-default",
},
Expand All @@ -403,26 +383,25 @@ var gateway = gatewayv1beta1.Gateway{
},
}

var httproutes = gatewayv1beta1.HTTPRoute{
var httproutes = gwv1.HTTPRoute{
ObjectMeta: metav1.ObjectMeta{
Name: service.Name,
},
Spec: gatewayv1beta1.HTTPRouteSpec{
CommonRouteSpec: gatewayv1beta1.CommonRouteSpec{
ParentRefs: []gatewayv1beta1.ParentReference{
Spec: gwv1.HTTPRouteSpec{
CommonRouteSpec: gwv1.CommonRouteSpec{
ParentRefs: []gwv1.ParentReference{
{
Namespace: &gatewayNamespace,
Name: gatewayv1beta1.ObjectName("gateway"),
Name: gwv1.ObjectName("gateway"),
},
},
},
Rules: []gatewayv1beta1.HTTPRouteRule{
Rules: []gwv1.HTTPRouteRule{
{
BackendRefs: []gatewayv1beta1.HTTPBackendRef{
BackendRefs: []gwv1.HTTPBackendRef{
{
BackendRef: gatewayv1beta1.BackendRef{
BackendObjectReference: gatewayv1beta1.BackendObjectReference{
Name: gatewayv1beta1.ObjectName(service.Name),
BackendRef: gwv1.BackendRef{
BackendObjectReference: gwv1.BackendObjectReference{
Name: gwv1.ObjectName(service.Name),
Port: &portNumber,
},
},
Expand Down
Loading