From e1fd576312ce542d4761a01f178c95f0c398e7a1 Mon Sep 17 00:00:00 2001 From: Pubudu Gunatilaka Date: Sun, 7 Apr 2024 16:14:26 +0530 Subject: [PATCH 1/4] Add affinity support for APK --- helm-charts/README.md | 10 ++- helm-charts/templates/_helpers.tpl | 20 +++++ .../config-deployer/config-ds-deployment.yaml | 1 + .../data-plane/gateway-api/gateway-api.yaml | 5 +- .../adapter/adapter-deployment.yaml | 1 + .../common-controller-deployment.yaml | 1 + .../gateway-runtime-deployment.yaml | 1 + .../ratelimiter/ratelimiter-deployment.yaml | 1 + helm-charts/values.yaml | 5 +- helm-charts/values.yaml.template | 73 ++++++++++++++++++- 10 files changed, 113 insertions(+), 5 deletions(-) diff --git a/helm-charts/README.md b/helm-charts/README.md index 8b4e8c54b..b70ed77fe 100644 --- a/helm-charts/README.md +++ b/helm-charts/README.md @@ -1,6 +1,6 @@ # apk-helm -![Version: 1.1.0-beta](https://img.shields.io/badge/Version-1.1.0--beta-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) +![Version: 1.1.0-rc](https://img.shields.io/badge/Version-1.1.0--rc-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) A Helm chart for APK components @@ -73,6 +73,7 @@ A Helm chart for APK components | wso2.apk.dp.partitionServer.tls.secretName | string | `"managetment-server-cert"` | TLS secret name for Partition Server Public Certificate. | | wso2.apk.dp.partitionServer.tls.fileName | string | `"certificate.crt"` | TLS certificate file name. | | wso2.apk.dp.configdeployer.enabled | bool | `true` | | +| wso2.apk.dp.configdeployer.deployment.affinity | object | `{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app.kubernetes.io/app","operator":"In","values":["config-ds"]}]}}}]}}` | Configure Affinity for the deployment. | | wso2.apk.dp.configdeployer.deployment.resources.requests.memory | string | `"128Mi"` | CPU request for the container | | wso2.apk.dp.configdeployer.deployment.resources.requests.cpu | string | `"100m"` | Memory request for the container | | wso2.apk.dp.configdeployer.deployment.resources.limits.memory | string | `"1028Mi"` | CPU limit for the container | @@ -108,6 +109,7 @@ A Helm chart for APK components | wso2.apk.dp.adapter.deployment.imagePullPolicy | string | `"Always"` | Image pull policy | | wso2.apk.dp.adapter.deployment.image | string | `"wso2/apk-adapter:1.1.0-beta"` | Image | | wso2.apk.dp.adapter.deployment.security.sslHostname | string | `"adapter"` | Enable security for adapter. | +| wso2.apk.dp.adapter.deployment.affinity | object | `{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app.kubernetes.io/app","operator":"In","values":["adapter"]}]}}}]}}` | Configure Affinity for the deployment. | | wso2.apk.dp.adapter.configs.apiNamespaces | string | `nil` | Optionally configure namespaces to watch for apis. | | wso2.apk.dp.adapter.configs.tls.secretName | string | `""` | TLS secret name for adapter public certificate. | | wso2.apk.dp.adapter.configs.tls.certKeyFilename | string | `""` | TLS certificate file name. | @@ -131,6 +133,7 @@ A Helm chart for APK components | wso2.apk.dp.commonController.deployment.image | string | `"wso2/apk-common-controller:1.1.0-beta"` | Image | | wso2.apk.dp.commonController.deployment.security.sslHostname | string | `"commoncontroller"` | hostname for the common controller | | wso2.apk.dp.commonController.deployment.configs.apiNamespaces | list | `["apk-v12"]` | Optionally configure namespaces to watch for apis,ratelimitpolicies,etc. | +| wso2.apk.dp.commonController.deployment.affinity | object | `{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app.kubernetes.io/app","operator":"In","values":["common-controller"]}]}}}]}}` | Configure Affinity for the deployment. | | wso2.apk.dp.commonController.deployment.redis.host | string | `"redis-master"` | Redis host | | wso2.apk.dp.commonController.deployment.redis.port | string | `"6379"` | Redis port | | wso2.apk.dp.commonController.deployment.redis.username | string | `"default"` | Redis user name | @@ -174,8 +177,10 @@ A Helm chart for APK components | wso2.apk.dp.ratelimiter.deployment.configs.tls.certKeyFilename | string | `""` | TLS certificate file name. | | wso2.apk.dp.ratelimiter.deployment.configs.tls.certFilename | string | `""` | TLS certificate file name. | | wso2.apk.dp.ratelimiter.deployment.configs.tls.certCAFilename | string | `""` | TLS CA certificate file name. | +| wso2.apk.dp.ratelimiter.deployment.affinity | object | `{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app.kubernetes.io/app","operator":"In","values":["rate-limiter"]}]}}}]}}` | Configure Affinity for the deployment. | | wso2.apk.dp.gatewayRuntime.service.annotations | string | `nil` | Gateway service related annotations. | | wso2.apk.dp.gatewayRuntime.deployment.replicas | int | `1` | Number of replicas | +| wso2.apk.dp.gatewayRuntime.deployment.affinity | object | `{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app.kubernetes.io/app","operator":"In","values":["gateway-runtime"]}]}}}]}}` | Configure Affinity for the deployment. | | wso2.apk.dp.gatewayRuntime.deployment.router.resources.requests.memory | string | `"128Mi"` | CPU request for the container | | wso2.apk.dp.gatewayRuntime.deployment.router.resources.requests.cpu | string | `"100m"` | Memory request for the container | | wso2.apk.dp.gatewayRuntime.deployment.router.resources.limits.memory | string | `"1028Mi"` | CPU limit for the container | @@ -322,6 +327,9 @@ A Helm chart for APK components | gatewaySystem.enableClusterRoleCreation | bool | `true` | | | gatewaySystem.serviceAccountName | string | `"gateway-api-admission"` | | | gatewaySystem.applyGatewayWehbhookJobs | bool | `true` | | +| gatewaySystem.deployment.image | string | `"registry.k8s.io/gateway-api/admission-server:v1.0.0"` | | +| gatewaySystem.deployment.imagePullPolicy | string | `"Always"` | | +| gatewaySystem.deployment.affinity | object | `{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app.kubernetes.io/app","operator":"In","values":["gateway-api-ad-server"]}]}}}]}}` | Configure Affinity for the deployment. | | certmanager.enabled | bool | `true` | Enable certificate manager to generate certificates | | certmanager.enableClusterIssuer | bool | `true` | Enable cluster issuer to generate certificates | | certmanager.enableRootCa | bool | `true` | Enable root CA to generate certificates | diff --git a/helm-charts/templates/_helpers.tpl b/helm-charts/templates/_helpers.tpl index 98e3434ac..5c6c1ed34 100644 --- a/helm-charts/templates/_helpers.tpl +++ b/helm-charts/templates/_helpers.tpl @@ -25,6 +25,26 @@ app.kubernetes.io/release: {{ .root.Release.Name }} {{- end }} +{{- define "apk-helm.deployment.affinity" -}} +{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }} +{{- if (not .value) }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/app + operator: In + values: + - {{ .app }} + topologyKey: "topology.kubernetes.io/zone" + weight: 100 +{{- else if contains "{{" (toJson .value) }} + {{- tpl $value .context }} +{{- else }} + {{- $value }} +{{- end }} +{{- end -}} {{- define "apk-helm.deployment.readinessProbe.http" -}} readinessProbe: diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml index 8108175a3..df18a9505 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml @@ -34,6 +34,7 @@ spec: annotations: checksum/config: {{ include (print $.Template.BasePath "/data-plane/config-deployer/config-ds-configmap.yaml") . | sha256sum }} spec: + affinity: {{- include "apk-helm.deployment.affinity" ( dict "value" .Values.wso2.apk.dp.configdeployer.deployment.affinity "app" "config-ds" "context" $) | nindent 8 }} serviceAccountName: {{ .Values.wso2.apk.auth.serviceAccountName }} containers: - name: config-ds diff --git a/helm-charts/templates/data-plane/gateway-api/gateway-api.yaml b/helm-charts/templates/data-plane/gateway-api/gateway-api.yaml index 66b2f0dba..702b70739 100644 --- a/helm-charts/templates/data-plane/gateway-api/gateway-api.yaml +++ b/helm-charts/templates/data-plane/gateway-api/gateway-api.yaml @@ -75,10 +75,11 @@ spec: labels: name: gateway-api-admission-server spec: + affinity: {{- include "apk-helm.deployment.affinity" ( dict "value" .Values.gatewaySystem.deployment.affinity "app" "gateway-api-ad-server" "context" $) | nindent 8 }} containers: - name: webhook - image: registry.k8s.io/gateway-api/admission-server:v1.0.0 - imagePullPolicy: Always + image: {{ .Values.gatewaySystem.deployment.image }} + imagePullPolicy: {{ .Values.gatewaySystem.deployment.imagePullPolicy }} args: - -logtostderr - --tlsCertFile=/etc/certs/cert diff --git a/helm-charts/templates/data-plane/gateway-components/adapter/adapter-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/adapter/adapter-deployment.yaml index 93eb9e18b..4605a1164 100644 --- a/helm-charts/templates/data-plane/gateway-components/adapter/adapter-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/adapter/adapter-deployment.yaml @@ -34,6 +34,7 @@ spec: annotations: checksum/config: {{ include (print $.Template.BasePath "/data-plane/gateway-components/log-conf.yaml") . | sha256sum }} spec: + affinity: {{- include "apk-helm.deployment.affinity" ( dict "value" .Values.wso2.apk.dp.adapter.deployment.affinity "app" "adapter" "context" $) | nindent 8 }} serviceAccountName: {{ .Values.wso2.apk.auth.serviceAccountName }} containers: - name: adapter diff --git a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml index 6e0ef38f0..7ca69b18e 100644 --- a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml @@ -34,6 +34,7 @@ spec: annotations: checksum/config: {{ include (print $.Template.BasePath "/data-plane/gateway-components/common-log-conf.yaml") . | sha256sum }} spec: + affinity: {{- include "apk-helm.deployment.affinity" ( dict "value" .Values.wso2.apk.dp.commonController.deployment.affinity "app" "common-controller" "context" $) | nindent 8 }} serviceAccountName: {{ .Values.wso2.apk.auth.serviceAccountName }} containers: - name: commoncontroller diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml index b0d8ed0b2..6f68c8dde 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml @@ -34,6 +34,7 @@ spec: annotations: checksum/config: {{ include (print $.Template.BasePath "/data-plane/gateway-components/log-conf.yaml") . | sha256sum }} spec: + affinity: {{- include "apk-helm.deployment.affinity" ( dict "value" .Values.wso2.apk.dp.gatewayRuntime.deployment.affinity "app" "gateway-runtime" "context" $) | nindent 8 }} automountServiceAccountToken: false containers: - name: enforcer diff --git a/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml b/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml index 75df7a35d..14e1df0fd 100644 --- a/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml +++ b/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml @@ -32,6 +32,7 @@ spec: labels: {{ include "apk-helm.pod.selectorLabels" (dict "root" . "app" "ratelimiter" ) | indent 8}} spec: + affinity: {{- include "apk-helm.deployment.affinity" ( dict "value" .Values.wso2.apk.dp.ratelimiter.deployment.affinity "app" "rate-limiter" "context" $) | nindent 8 }} automountServiceAccountToken: false serviceAccountName: {{ .Values.wso2.apk.auth.serviceAccountName }} containers: diff --git a/helm-charts/values.yaml b/helm-charts/values.yaml index 7cf0c5bcb..7abe757af 100644 --- a/helm-charts/values.yaml +++ b/helm-charts/values.yaml @@ -344,7 +344,10 @@ gatewaySystem: enableClusterRoleCreation: true serviceAccountName: gateway-api-admission applyGatewayWehbhookJobs: true - + deployment: + image: registry.k8s.io/gateway-api/admission-server:v1.0.0 + imagePullPolicy: Always + certmanager: enabled: true enableClusterIssuer: true diff --git a/helm-charts/values.yaml.template b/helm-charts/values.yaml.template index dcdf05928..a44313d73 100644 --- a/helm-charts/values.yaml.template +++ b/helm-charts/values.yaml.template @@ -152,6 +152,17 @@ wso2: configdeployer: enabled: true deployment: + # -- Configure Affinity for the deployment. + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/app + operator: In + values: + - config-ds resources: requests: # -- CPU request for the container @@ -243,6 +254,20 @@ wso2: security: # -- Enable security for adapter. sslHostname: "adapter" + # -- Configure Affinity for the deployment. + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/app + operator: In + values: + - adapter + # - "{{ .Template.Name }}" + # - "{{ .Values.somevalue }}" + configs: # -- Optionally configure namespaces to watch for apis. apiNamespaces: @@ -303,6 +328,17 @@ wso2: # -- Optionally configure namespaces to watch for apis,ratelimitpolicies,etc. apiNamespaces: - "apk-v12" + # -- Configure Affinity for the deployment. + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/app + operator: In + values: + - common-controller redis: # -- Redis host host: "redis-master" @@ -394,6 +430,17 @@ wso2: certFilename: "" # -- TLS CA certificate file name. certCAFilename: "" + # -- Configure Affinity for the deployment. + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/app + operator: In + values: + - rate-limiter gatewayRuntime: service: # -- Gateway service related annotations. @@ -401,6 +448,17 @@ wso2: deployment: # -- Number of replicas replicas: 1 + # -- Configure Affinity for the deployment. + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/app + operator: In + values: + - gateway-runtime router: resources: requests: @@ -774,7 +832,20 @@ gatewaySystem: enableClusterRoleCreation: true serviceAccountName: gateway-api-admission applyGatewayWehbhookJobs: true - + deployment: + image: registry.k8s.io/gateway-api/admission-server:v1.0.0 + imagePullPolicy: Always + # -- Configure Affinity for the deployment. + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/app + operator: In + values: + - gateway-api-ad-server certmanager: # -- Enable certificate manager to generate certificates enabled: true From c5e99a2e676f250fc1bda1f50aef0322ad2274a4 Mon Sep 17 00:00:00 2001 From: Pubudu Gunatilaka Date: Sun, 7 Apr 2024 17:28:27 +0530 Subject: [PATCH 2/4] Add node selector support for the APK --- helm-charts/README.md | 6 ++++ helm-charts/templates/_helpers.tpl | 34 ++++++++++++------- .../config-deployer/config-ds-deployment.yaml | 3 ++ .../data-plane/gateway-api/gateway-api.yaml | 3 ++ .../adapter/adapter-deployment.yaml | 3 ++ .../common-controller-deployment.yaml | 3 ++ .../gateway-runtime-deployment.yaml | 3 ++ .../ratelimiter/ratelimiter-deployment.yaml | 3 ++ helm-charts/values.yaml.template | 13 ++++++- 9 files changed, 57 insertions(+), 14 deletions(-) diff --git a/helm-charts/README.md b/helm-charts/README.md index b70ed77fe..26a82cf07 100644 --- a/helm-charts/README.md +++ b/helm-charts/README.md @@ -74,6 +74,7 @@ A Helm chart for APK components | wso2.apk.dp.partitionServer.tls.fileName | string | `"certificate.crt"` | TLS certificate file name. | | wso2.apk.dp.configdeployer.enabled | bool | `true` | | | wso2.apk.dp.configdeployer.deployment.affinity | object | `{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app.kubernetes.io/app","operator":"In","values":["config-ds"]}]}}}]}}` | Configure Affinity for the deployment. | +| wso2.apk.dp.configdeployer.deployment.nodeSelector | object | `{}` | Configure Node Selector for the deployment. | | wso2.apk.dp.configdeployer.deployment.resources.requests.memory | string | `"128Mi"` | CPU request for the container | | wso2.apk.dp.configdeployer.deployment.resources.requests.cpu | string | `"100m"` | Memory request for the container | | wso2.apk.dp.configdeployer.deployment.resources.limits.memory | string | `"1028Mi"` | CPU limit for the container | @@ -110,6 +111,7 @@ A Helm chart for APK components | wso2.apk.dp.adapter.deployment.image | string | `"wso2/apk-adapter:1.1.0-beta"` | Image | | wso2.apk.dp.adapter.deployment.security.sslHostname | string | `"adapter"` | Enable security for adapter. | | wso2.apk.dp.adapter.deployment.affinity | object | `{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app.kubernetes.io/app","operator":"In","values":["adapter"]}]}}}]}}` | Configure Affinity for the deployment. | +| wso2.apk.dp.adapter.deployment.nodeSelector | object | `{}` | Configure Node Selector for the deployment. | | wso2.apk.dp.adapter.configs.apiNamespaces | string | `nil` | Optionally configure namespaces to watch for apis. | | wso2.apk.dp.adapter.configs.tls.secretName | string | `""` | TLS secret name for adapter public certificate. | | wso2.apk.dp.adapter.configs.tls.certKeyFilename | string | `""` | TLS certificate file name. | @@ -134,6 +136,7 @@ A Helm chart for APK components | wso2.apk.dp.commonController.deployment.security.sslHostname | string | `"commoncontroller"` | hostname for the common controller | | wso2.apk.dp.commonController.deployment.configs.apiNamespaces | list | `["apk-v12"]` | Optionally configure namespaces to watch for apis,ratelimitpolicies,etc. | | wso2.apk.dp.commonController.deployment.affinity | object | `{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app.kubernetes.io/app","operator":"In","values":["common-controller"]}]}}}]}}` | Configure Affinity for the deployment. | +| wso2.apk.dp.commonController.deployment.nodeSelector | object | `{}` | Configure Node Selector for the deployment. | | wso2.apk.dp.commonController.deployment.redis.host | string | `"redis-master"` | Redis host | | wso2.apk.dp.commonController.deployment.redis.port | string | `"6379"` | Redis port | | wso2.apk.dp.commonController.deployment.redis.username | string | `"default"` | Redis user name | @@ -178,8 +181,10 @@ A Helm chart for APK components | wso2.apk.dp.ratelimiter.deployment.configs.tls.certFilename | string | `""` | TLS certificate file name. | | wso2.apk.dp.ratelimiter.deployment.configs.tls.certCAFilename | string | `""` | TLS CA certificate file name. | | wso2.apk.dp.ratelimiter.deployment.affinity | object | `{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app.kubernetes.io/app","operator":"In","values":["rate-limiter"]}]}}}]}}` | Configure Affinity for the deployment. | +| wso2.apk.dp.ratelimiter.deployment.nodeSelector | object | `{}` | Configure Node Selector for the deployment. | | wso2.apk.dp.gatewayRuntime.service.annotations | string | `nil` | Gateway service related annotations. | | wso2.apk.dp.gatewayRuntime.deployment.replicas | int | `1` | Number of replicas | +| wso2.apk.dp.gatewayRuntime.deployment.nodeSelector | object | `{}` | Configure Node Selector for the deployment. | | wso2.apk.dp.gatewayRuntime.deployment.affinity | object | `{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app.kubernetes.io/app","operator":"In","values":["gateway-runtime"]}]}}}]}}` | Configure Affinity for the deployment. | | wso2.apk.dp.gatewayRuntime.deployment.router.resources.requests.memory | string | `"128Mi"` | CPU request for the container | | wso2.apk.dp.gatewayRuntime.deployment.router.resources.requests.cpu | string | `"100m"` | Memory request for the container | @@ -330,6 +335,7 @@ A Helm chart for APK components | gatewaySystem.deployment.image | string | `"registry.k8s.io/gateway-api/admission-server:v1.0.0"` | | | gatewaySystem.deployment.imagePullPolicy | string | `"Always"` | | | gatewaySystem.deployment.affinity | object | `{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app.kubernetes.io/app","operator":"In","values":["gateway-api-ad-server"]}]}}}]}}` | Configure Affinity for the deployment. | +| gatewaySystem.deployment.nodeSelector | object | `{}` | Configure Node Selector for the deployment. | | certmanager.enabled | bool | `true` | Enable certificate manager to generate certificates | | certmanager.enableClusterIssuer | bool | `true` | Enable cluster issuer to generate certificates | | certmanager.enableRootCa | bool | `true` | Enable root CA to generate certificates | diff --git a/helm-charts/templates/_helpers.tpl b/helm-charts/templates/_helpers.tpl index 5c6c1ed34..0fdabc832 100644 --- a/helm-charts/templates/_helpers.tpl +++ b/helm-charts/templates/_helpers.tpl @@ -24,21 +24,20 @@ app.kubernetes.io/app: {{ .app }} app.kubernetes.io/release: {{ .root.Release.Name }} {{- end }} - {{- define "apk-helm.deployment.affinity" -}} {{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }} -{{- if (not .value) }} - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app.kubernetes.io/app - operator: In - values: - - {{ .app }} - topologyKey: "topology.kubernetes.io/zone" - weight: 100 +{{- if (not .value) -}} +podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/app + operator: In + values: + - {{ .app }} + topologyKey: "topology.kubernetes.io/zone" + weight: 100 {{- else if contains "{{" (toJson .value) }} {{- tpl $value .context }} {{- else }} @@ -46,6 +45,15 @@ app.kubernetes.io/release: {{ .root.Release.Name }} {{- end }} {{- end -}} +{{- define "apk-helm.deployment.nodeSelector" -}} +{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }} +{{- if contains "{{" (toJson .value) }} + {{- tpl $value .context }} +{{- else }} + {{- $value }} +{{- end }} +{{- end -}} + {{- define "apk-helm.deployment.readinessProbe.http" -}} readinessProbe: httpGet: diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml index df18a9505..5b29d932b 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml @@ -35,6 +35,9 @@ spec: checksum/config: {{ include (print $.Template.BasePath "/data-plane/config-deployer/config-ds-configmap.yaml") . | sha256sum }} spec: affinity: {{- include "apk-helm.deployment.affinity" ( dict "value" .Values.wso2.apk.dp.configdeployer.deployment.affinity "app" "config-ds" "context" $) | nindent 8 }} + {{- if .Values.wso2.apk.dp.configdeployer.deployment.nodeSelector }} + nodeSelector: {{- include "apk-helm.deployment.nodeSelector" ( dict "value" .Values.wso2.apk.dp.configdeployer.deployment.nodeSelector "context" $) | nindent 8 }} + {{- end }} serviceAccountName: {{ .Values.wso2.apk.auth.serviceAccountName }} containers: - name: config-ds diff --git a/helm-charts/templates/data-plane/gateway-api/gateway-api.yaml b/helm-charts/templates/data-plane/gateway-api/gateway-api.yaml index 702b70739..cb27af35e 100644 --- a/helm-charts/templates/data-plane/gateway-api/gateway-api.yaml +++ b/helm-charts/templates/data-plane/gateway-api/gateway-api.yaml @@ -76,6 +76,9 @@ spec: name: gateway-api-admission-server spec: affinity: {{- include "apk-helm.deployment.affinity" ( dict "value" .Values.gatewaySystem.deployment.affinity "app" "gateway-api-ad-server" "context" $) | nindent 8 }} + {{- if .Values.gatewaySystem.deployment.nodeSelector }} + nodeSelector: {{- include "apk-helm.deployment.nodeSelector" ( dict "value" .Values.gatewaySystem.deployment.nodeSelector "context" $) | nindent 8 }} + {{- end }} containers: - name: webhook image: {{ .Values.gatewaySystem.deployment.image }} diff --git a/helm-charts/templates/data-plane/gateway-components/adapter/adapter-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/adapter/adapter-deployment.yaml index 4605a1164..9167c587d 100644 --- a/helm-charts/templates/data-plane/gateway-components/adapter/adapter-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/adapter/adapter-deployment.yaml @@ -35,6 +35,9 @@ spec: checksum/config: {{ include (print $.Template.BasePath "/data-plane/gateway-components/log-conf.yaml") . | sha256sum }} spec: affinity: {{- include "apk-helm.deployment.affinity" ( dict "value" .Values.wso2.apk.dp.adapter.deployment.affinity "app" "adapter" "context" $) | nindent 8 }} + {{- if .Values.wso2.apk.dp.adapter.deployment.nodeSelector }} + nodeSelector: {{- include "apk-helm.deployment.nodeSelector" ( dict "value" .Values.wso2.apk.dp.adapter.deployment.nodeSelector "context" $) | nindent 8 }} + {{- end }} serviceAccountName: {{ .Values.wso2.apk.auth.serviceAccountName }} containers: - name: adapter diff --git a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml index 7ca69b18e..c067f14dc 100644 --- a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml @@ -35,6 +35,9 @@ spec: checksum/config: {{ include (print $.Template.BasePath "/data-plane/gateway-components/common-log-conf.yaml") . | sha256sum }} spec: affinity: {{- include "apk-helm.deployment.affinity" ( dict "value" .Values.wso2.apk.dp.commonController.deployment.affinity "app" "common-controller" "context" $) | nindent 8 }} + {{- if .Values.wso2.apk.dp.commonController.deployment.nodeSelector }} + nodeSelector: {{- include "apk-helm.deployment.nodeSelector" ( dict "value" .Values.wso2.apk.dp.commonController.deployment.nodeSelector "context" $) | nindent 8 }} + {{- end }} serviceAccountName: {{ .Values.wso2.apk.auth.serviceAccountName }} containers: - name: commoncontroller diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml index 6f68c8dde..2e24eb314 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml @@ -35,6 +35,9 @@ spec: checksum/config: {{ include (print $.Template.BasePath "/data-plane/gateway-components/log-conf.yaml") . | sha256sum }} spec: affinity: {{- include "apk-helm.deployment.affinity" ( dict "value" .Values.wso2.apk.dp.gatewayRuntime.deployment.affinity "app" "gateway-runtime" "context" $) | nindent 8 }} + {{- if .Values.wso2.apk.dp.gatewayRuntime.deployment.nodeSelector }} + nodeSelector: {{- include "apk-helm.deployment.nodeSelector" ( dict "value" .Values.wso2.apk.dp.gatewayRuntime.deployment.nodeSelector "context" $) | nindent 8 }} + {{- end }} automountServiceAccountToken: false containers: - name: enforcer diff --git a/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml b/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml index 14e1df0fd..cb2faea60 100644 --- a/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml +++ b/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml @@ -33,6 +33,9 @@ spec: {{ include "apk-helm.pod.selectorLabels" (dict "root" . "app" "ratelimiter" ) | indent 8}} spec: affinity: {{- include "apk-helm.deployment.affinity" ( dict "value" .Values.wso2.apk.dp.ratelimiter.deployment.affinity "app" "rate-limiter" "context" $) | nindent 8 }} + {{- if .Values.wso2.apk.dp.ratelimiter.deployment.nodeSelector }} + nodeSelector: {{- include "apk-helm.deployment.nodeSelector" ( dict "value" .Values.wso2.apk.dp.ratelimiter.deployment.nodeSelector "context" $) | nindent 8 }} + {{- end }} automountServiceAccountToken: false serviceAccountName: {{ .Values.wso2.apk.auth.serviceAccountName }} containers: diff --git a/helm-charts/values.yaml.template b/helm-charts/values.yaml.template index a44313d73..8dd0d2866 100644 --- a/helm-charts/values.yaml.template +++ b/helm-charts/values.yaml.template @@ -163,6 +163,8 @@ wso2: operator: In values: - config-ds + # -- Configure Node Selector for the deployment. + nodeSelector: {} resources: requests: # -- CPU request for the container @@ -267,7 +269,8 @@ wso2: - adapter # - "{{ .Template.Name }}" # - "{{ .Values.somevalue }}" - + # -- Configure Node Selector for the deployment. + nodeSelector: {} configs: # -- Optionally configure namespaces to watch for apis. apiNamespaces: @@ -339,6 +342,8 @@ wso2: operator: In values: - common-controller + # -- Configure Node Selector for the deployment. + nodeSelector: {} redis: # -- Redis host host: "redis-master" @@ -441,6 +446,8 @@ wso2: operator: In values: - rate-limiter + # -- Configure Node Selector for the deployment. + nodeSelector: {} gatewayRuntime: service: # -- Gateway service related annotations. @@ -448,6 +455,8 @@ wso2: deployment: # -- Number of replicas replicas: 1 + # -- Configure Node Selector for the deployment. + nodeSelector: {} # -- Configure Affinity for the deployment. affinity: podAntiAffinity: @@ -846,6 +855,8 @@ gatewaySystem: operator: In values: - gateway-api-ad-server + # -- Configure Node Selector for the deployment. + nodeSelector: {} certmanager: # -- Enable certificate manager to generate certificates enabled: true From 546fe73b8e1b05a1fcc34725515b4bb6d0e43f34 Mon Sep 17 00:00:00 2001 From: Pubudu Gunatilaka Date: Sun, 7 Apr 2024 17:48:08 +0530 Subject: [PATCH 3/4] Make gateway.service.type configurable other than LB type --- helm-charts/README.md | 1 + .../gateway-components/gateway-runtime/gateway-service.yaml | 2 +- helm-charts/values.yaml | 1 + helm-charts/values.yaml.template | 2 ++ 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/helm-charts/README.md b/helm-charts/README.md index 26a82cf07..9cab9356c 100644 --- a/helm-charts/README.md +++ b/helm-charts/README.md @@ -60,6 +60,7 @@ A Helm chart for APK components | wso2.apk.dp.gateway.autoscaling.maxReplicas | int | `2` | Maximum number of replicas for Gateway | | wso2.apk.dp.gateway.autoscaling.targetMemory | int | `80` | Target memory utilization percentage for Gateway | | wso2.apk.dp.gateway.autoscaling.targetCPU | int | `80` | Target CPU utilization percentage for Gateway | +| wso2.apk.dp.gateway.service.type | string | `"LoadBalancer"` | | | wso2.apk.dp.redis.type | string | `"single"` | Redis type | | wso2.apk.dp.redis.url | string | `"redis-master:6379"` | Redis URL | | wso2.apk.dp.redis.tls | bool | `false` | TLS enabled | diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-service.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-service.yaml index 1a1ef3f93..0173aed7a 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-service.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-service.yaml @@ -25,7 +25,7 @@ metadata: {{ toYaml .Values.wso2.apk.dp.gatewayRuntime.service.annotations | indent 4 }} {{ end }} spec: - type: LoadBalancer + type: {{ .Values.wso2.apk.dp.gateway.service.type | default "LoadBalancer" }} # label keys and values that must match in order to receive traffic for this service selector: {{ include "apk-helm.pod.selectorLabels" (dict "root" . "app" "gateway" ) | indent 4}} diff --git a/helm-charts/values.yaml b/helm-charts/values.yaml index 7abe757af..0d60d7ff6 100644 --- a/helm-charts/values.yaml +++ b/helm-charts/values.yaml @@ -59,6 +59,7 @@ wso2: gateway: listener: hostname: "gw.wso2.com" + service: {} # secretName: "idp-tls" # partitionServer: # enabled: false diff --git a/helm-charts/values.yaml.template b/helm-charts/values.yaml.template index 8dd0d2866..70b7c094b 100644 --- a/helm-charts/values.yaml.template +++ b/helm-charts/values.yaml.template @@ -121,6 +121,8 @@ wso2: targetMemory: 80 # -- Target CPU utilization percentage for Gateway targetCPU: 80 + service: + type: "LoadBalancer" redis: # -- Redis type type: "single" From 3d564be1c36f74b3e9a7463111b9511b2287ed2b Mon Sep 17 00:00:00 2001 From: Pubudu Gunatilaka Date: Sun, 7 Apr 2024 17:52:33 +0530 Subject: [PATCH 4/4] Update helm docs --- helm-charts/README.md | 2 +- helm-charts/values.yaml.template | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/helm-charts/README.md b/helm-charts/README.md index 9cab9356c..2e7b887ce 100644 --- a/helm-charts/README.md +++ b/helm-charts/README.md @@ -60,7 +60,7 @@ A Helm chart for APK components | wso2.apk.dp.gateway.autoscaling.maxReplicas | int | `2` | Maximum number of replicas for Gateway | | wso2.apk.dp.gateway.autoscaling.targetMemory | int | `80` | Target memory utilization percentage for Gateway | | wso2.apk.dp.gateway.autoscaling.targetCPU | int | `80` | Target CPU utilization percentage for Gateway | -| wso2.apk.dp.gateway.service.type | string | `"LoadBalancer"` | | +| wso2.apk.dp.gateway.service | object | `{"type":"LoadBalancer"}` | Kubernetes service type for Gateway | | wso2.apk.dp.redis.type | string | `"single"` | Redis type | | wso2.apk.dp.redis.url | string | `"redis-master:6379"` | Redis URL | | wso2.apk.dp.redis.tls | bool | `false` | TLS enabled | diff --git a/helm-charts/values.yaml.template b/helm-charts/values.yaml.template index 70b7c094b..a782a1658 100644 --- a/helm-charts/values.yaml.template +++ b/helm-charts/values.yaml.template @@ -121,6 +121,7 @@ wso2: targetMemory: 80 # -- Target CPU utilization percentage for Gateway targetCPU: 80 + # -- Kubernetes service type for Gateway service: type: "LoadBalancer" redis: