From 45ae657cd007301884ea8b51140d4c7b9192a24d Mon Sep 17 00:00:00 2001 From: Stephane Janel Date: Thu, 9 Feb 2023 09:43:57 +0100 Subject: [PATCH] Allow rolling update maxSurge maxUnavailable to be configurable --- README.md | 33 ++++++++++++++++++++------------- controllers/deployment.go | 18 ++++++++++++++++-- 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index db1c21a..4bca890 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # egress-operator + An operator to produce egress gateway pods and control access to them with network policies, and a coredns plugin to route egress traffic to these pods. The idea is that instead of authorizing egress traffic with protocol inspection, @@ -6,10 +7,11 @@ you instead create a internal clusterIP for every external service you use, lock it down to only a few pods via a network policy, and then set up your dns server to resolve the external service to that clusterIP. -Built with kubebuilder: https://book.kubebuilder.io/ +Built with kubebuilder: The operator accepts ExternalService objects, which aren't namespaced, which define a dns name and ports for an external service. In the `egress-operator-system` namespace, it creates: + - An envoy configmap for a TCP/UDP proxy to that service (UDP not working until the next envoy release that enables it) - A deployment for some envoy pods with that config - A horizontal pod autoscaler to keep the deployment correctly sized @@ -31,6 +33,7 @@ In the `egress-operator-system` namespace, it creates: ```bash make run ``` + This creates an ExternalService object to see the controller-manager creating managed resources in the remote cluster. ### Setting up CoreDNS plugin @@ -38,17 +41,20 @@ This creates an ExternalService object to see the controller-manager creating ma The CoreDNS plugin rewrites responses for external service hostnames managed by egress-operator. Build a CoreDNS image which contains the plugin: + ```bash cd coredns-plugin make docker-build docker-push IMG=yourrepo/egress-operator-coredns:latest ``` You'll need to swap out the image of your coredns kubedns Deployment for `yourrepo/egress-operator-coredns:latest`: + ```bash kubectl edit deploy coredns -n kube-system # Your Deployment name may vary ``` And edit the coredns Corefile in ConfigMap to put in `egressoperator egress-operator-system cluster.local`: + ```bash kubectl edit configmap coredns-config -n kube-system # Your ConfigMap name may vary ``` @@ -202,15 +208,16 @@ spec: egress.monzo.com/gateway: egress-gateway-name ``` -| Variable name | Default | Description | -|------------------------------------|-------------------------------------------|----------------------------------------------------| -| ENVOY_IMAGE | `envoyproxy/envoy-alpine:v1.16.5` | Name of the Envoy Proxy image to use | -| TAINT_TOLERATION_KEY | Empty, no tolerations applied | Toleration key to apply to gateway pods | -| TAINT_TOLERATION_VALUE | Empty, no tolerations applied | Toleration value to apply to gateway pods | -| NODE_SELECTOR_KEY | Empty, no node selector added | Node selector label key to apply to gateway pods | -| NODE_SELECTOR_VALUE | Empty, no node selector added | Node selector label value to apply to gateway pods | -| POD_TOPOLOGY_ZONE_MAX_SKEW_KEY | `topology.kubernetes.io/zone` | Topology key for the zone constraint | -| POD_TOPOLOGY_ZONE_MAX_SKEW | Empty, won't inject a zone constraint | Value of maxSkew for the zone constraint | -| POD_TOPOLOGY_HOSTNAME_MAX_SKEW_KEY | `kubernetes.io/hostname` | Topology key for the hostname constraint | -| POD_TOPOLOGY_HOSTNAME_MAX_SKEW | Empty, won't inject a hostname constraint | Value of maxSkew for the hostname constraint | - +| Variable name | Default | Description | +| ---------------------------------- | ----------------------------------------- | ------------------------------------------------------- | +| ENVOY_IMAGE | `envoyproxy/envoy-alpine:v1.16.5` | Name of the Envoy Proxy image to use | +| TAINT_TOLERATION_KEY | Empty, no tolerations applied | Toleration key to apply to gateway pods | +| TAINT_TOLERATION_VALUE | Empty, no tolerations applied | Toleration value to apply to gateway pods | +| NODE_SELECTOR_KEY | Empty, no node selector added | Node selector label key to apply to gateway pods | +| NODE_SELECTOR_VALUE | Empty, no node selector added | Node selector label value to apply to gateway pods | +| POD_TOPOLOGY_ZONE_MAX_SKEW_KEY | `topology.kubernetes.io/zone` | Topology key for the zone constraint | +| POD_TOPOLOGY_ZONE_MAX_SKEW | Empty, won't inject a zone constraint | Value of maxSkew for the zone constraint | +| POD_TOPOLOGY_HOSTNAME_MAX_SKEW_KEY | `kubernetes.io/hostname` | Topology key for the hostname constraint | +| POD_TOPOLOGY_HOSTNAME_MAX_SKEW | Empty, won't inject a hostname constraint | Value of maxSkew for the hostname constraint | +| ROLLING_UPDATE_MAX_UNAVAILABLE | 25% | Rolling Update max unavailable to apply to gateway pods | +| ROLLING_UPDATE_MAX_SURGE | 25% | Rolling Update max surge to apply to gateway pods | diff --git a/controllers/deployment.go b/controllers/deployment.go index 59db3dd..96cef15 100644 --- a/controllers/deployment.go +++ b/controllers/deployment.go @@ -134,6 +134,9 @@ func deployment(es *egressv1.ExternalService, configHash string) *appsv1.Deploym } } + maxUnavailableStr := lookupEnvOr("ROLLING_UPDATE_MAX_UNAVAILABLE", "25%") + maxSurgeStr := lookupEnvOr("ROLLING_UPDATE_MAX_SURGE", "25%") + var resources corev1.ResourceRequirements if es.Spec.Resources != nil { resources = *es.Spec.Resources @@ -150,6 +153,9 @@ func deployment(es *egressv1.ExternalService, configHash string) *appsv1.Deploym } } + maxUnavailable := intstr.FromString(maxUnavailableStr) + maxSurge := intstr.FromString(maxSurgeStr) + return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: es.Name, @@ -163,8 +169,8 @@ func deployment(es *egressv1.ExternalService, configHash string) *appsv1.Deploym Strategy: appsv1.DeploymentStrategy{ Type: appsv1.RollingUpdateDeploymentStrategyType, RollingUpdate: &appsv1.RollingUpdateDeployment{ - MaxUnavailable: intstr.ValueOrDefault(nil, intstr.FromString("25%")), - MaxSurge: intstr.ValueOrDefault(nil, intstr.FromString("25%")), + MaxUnavailable: &maxUnavailable, + MaxSurge: &maxSurge, }, }, Selector: labelSelector, @@ -245,3 +251,11 @@ func deployment(es *egressv1.ExternalService, configHash string) *appsv1.Deploym }, } } + +func lookupEnvOr(envKey, envDefaultValue string) string { + valueStr, isSet := os.LookupEnv(envKey) + if !isSet || len(valueStr) == 0 { + return envDefaultValue + } + return valueStr +}