From 87d3128f714f4fdc6bba867af73520d937e60a93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20B=C3=A4hler?= Date: Tue, 20 Feb 2024 13:17:59 +0100 Subject: [PATCH] feat: add scheduling values (#396) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(makefile): improve helm-testing makes Signed-off-by: Oliver Bähler * feat(helm): add scheduling values Signed-off-by: Oliver Bähler --------- Signed-off-by: Oliver Bähler --- Makefile | 13 +- charts/capsule-proxy/README.md | 8 + charts/capsule-proxy/ci/ds-values.yaml | 36 +++ .../capsule-proxy/templates/certgen-job.yaml | 21 +- .../capsule-proxy/templates/certmanager.yaml | 2 + charts/capsule-proxy/values.yaml | 239 +++++++++--------- 6 files changed, 198 insertions(+), 121 deletions(-) diff --git a/Makefile b/Makefile index 8dc471ed..16e6b57a 100644 --- a/Makefile +++ b/Makefile @@ -103,17 +103,18 @@ helm-docs: docker helm-lint: docker @docker run -v "$(SRC_ROOT):/workdir" --entrypoint /bin/sh quay.io/helmpack/chart-testing:v3.3.1 -c "cd /workdir; ct lint --config .github/configs/ct.yaml --lint-conf .github/configs/lintconf.yaml --all --debug" -helm-test: helm-controller-version kind ct ko-build-all - @kind create cluster --wait=60s --name capsule-charts - @kind load docker-image --name capsule-charts $(CAPSULE_PROXY_IMG):$(VERSION) - @kubectl create ns capsule-system - @make helm-install +helm-test: helm-controller-version kind ct ko-build-all helm-create helm-install helm-destroy helm-install: @kubectl apply --server-side=true -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.2/cert-manager.yaml @make install-capsule @kubectl apply --server-side=true -f https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.58.0/bundle.yaml - @ct install --config $(SRC_ROOT)/.github/configs/ct.yaml --namespace=capsule-system --all --debug + @$(CT) install --config $(SRC_ROOT)/.github/configs/ct.yaml --namespace=capsule-system --all --debug + +helm-create: + @kind create cluster --wait=60s --name capsule-charts + @kind load docker-image --name capsule-charts $(CAPSULE_PROXY_IMG):$(VERSION) + @kubectl create ns capsule-system helm-destroy: @kind delete cluster --name capsule-charts diff --git a/charts/capsule-proxy/README.md b/charts/capsule-proxy/README.md index b52a71d9..127d827b 100644 --- a/charts/capsule-proxy/README.md +++ b/charts/capsule-proxy/README.md @@ -62,6 +62,7 @@ If you only need to make minor customizations, you can specify them on the comma |-----|------|---------|-------------| | affinity | object | `{}` | Set affinity rules for the capsule-proxy pod. | | certManager.certificate.dnsNames | list | `[]` | Additional DNS Names to include in certificate | +| certManager.certificate.includeInternalServiceNames | bool | `true` | Include internal service names in certificate (disable if you create a public cert) | | certManager.certificate.ipAddresses | list | `[]` | Additional IP Addresses to include in certificate | | certManager.certificate.uris | list | `[]` | Additional URIs to include in certificate | | certManager.externalCA.enabled | bool | `false` | Set if want cert manager to sign certificates with an external CA | @@ -71,18 +72,25 @@ If you only need to make minor customizations, you can specify them on the comma | certManager.issuer.name | string | `""` | Set the name of the ClusterIssuer if issuer kind is ClusterIssuer and if cert manager will generate CA signed SSL certificates | | daemonset.hostNetwork | bool | `false` | Use the host network namespace for capsule-proxy pod. | | daemonset.hostPort | bool | `false` | Binding the capsule-proxy listening port to the host port. | +| hostNetwork | bool | `false` | When deployed as DaemonSet use | | image.pullPolicy | string | `"IfNotPresent"` | Set the image pull policy. | | image.registry | string | `"ghcr.io"` | Set the image registry for capsule-proxy | | image.repository | string | `"projectcapsule/capsule-proxy"` | Set the image repository for capsule-proxy. | | image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | | imagePullSecrets | list | `[]` | Configuration for `imagePullSecrets` so that you can use a private images registry. | +| jobs.affinity | object | `{}` | Set affinity rules | | jobs.annotations | object | `{}` | Annotations to add to the certgen job. | | jobs.certs.pullPolicy | string | `"IfNotPresent"` | Set the image pull policy of the post install certgen job | | jobs.certs.registry | string | `"docker.io"` | Set the image repository of the post install certgen job | | jobs.certs.repository | string | `"jettech/kube-webhook-certgen"` | Set the image repository of the post install certgen job | | jobs.certs.tag | string | `"v1.3.0"` | Set the image tag of the post install certgen job | +| jobs.nodeSelector | object | `{}` | Set the node selector | | jobs.podSecurityContext | object | `{"seccompProfile":{"type":"RuntimeDefault"}}` | Security context for the job pods. | +| jobs.priorityClassName | string | `""` | Set a pod priorityClassName | +| jobs.restartPolicy | string | `"Never"` | Set the restartPolicy | | jobs.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsGroup":1002,"runAsNonRoot":true,"runAsUser":1002}` | Security context for the job containers. | +| jobs.tolerations | list | `[]` | Set list of tolerations | +| jobs.topologySpreadConstraints | list | `[]` | Set Topology Spread Constraints | | jobs.ttlSecondsAfterFinished | int | `60` | Sets the ttl in seconds after a finished certgen job is deleted. Set to -1 to never delete. | | kind | string | `"Deployment"` | Set the deployment mode of the capsule-proxy as `Deployment` or `DaemonSet`. | | nodeSelector | object | `{}` | Set the node selector for the capsule-proxy pod. | diff --git a/charts/capsule-proxy/ci/ds-values.yaml b/charts/capsule-proxy/ci/ds-values.yaml index daecd9df..c3c32c70 100644 --- a/charts/capsule-proxy/ci/ds-values.yaml +++ b/charts/capsule-proxy/ci/ds-values.yaml @@ -45,6 +45,16 @@ tolerations: operator: Exists - effect: NoSchedule key: node-role.kubernetes.io/master +affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: another-node-label-key + operator: In + values: + - another-node-label-value # Ingress ingress: enabled: true @@ -59,3 +69,29 @@ ingress: # ServiceMonitor serviceMonitor: enabled: true +jobs: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: another-node-label-key + operator: In + values: + - another-node-label-value + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: capsule-proxy + priorityClassName: "system-node-critical" + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master diff --git a/charts/capsule-proxy/templates/certgen-job.yaml b/charts/capsule-proxy/templates/certgen-job.yaml index 46207e7d..0597f53d 100644 --- a/charts/capsule-proxy/templates/certgen-job.yaml +++ b/charts/capsule-proxy/templates/certgen-job.yaml @@ -19,11 +19,30 @@ spec: labels: {{- include "capsule-proxy.selectorLabels" . | nindent 8 }} spec: - restartPolicy: Never + restartPolicy: {{ $.Values.jobs.restartPolicy }} {{- with $.Values.jobs.podSecurityContext }} securityContext: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.jobs.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.jobs.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.jobs.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.jobs.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.jobs.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} containers: - name: post-install-job image: {{ include "capsule.jobs.certsFullyQualifiedDockerImage" $ }} diff --git a/charts/capsule-proxy/templates/certmanager.yaml b/charts/capsule-proxy/templates/certmanager.yaml index 22e60a7d..b34c1813 100644 --- a/charts/capsule-proxy/templates/certmanager.yaml +++ b/charts/capsule-proxy/templates/certmanager.yaml @@ -48,8 +48,10 @@ spec: {{- range $dns := .Values.certManager.certificate.dnsNames }} - {{ $dns }} {{- end }} + {{- if $.Values.certManager.certificate.includeInternalServiceNames }} - {{ include "capsule-proxy.fullname" . }} - {{ include "capsule-proxy.fullname" . }}.{{ .Release.Namespace }}.svc + {{- end }} {{- with .Values.certManager.certificate.ipAddresses }} ipAddresses: {{- range $ip := . }} diff --git a/charts/capsule-proxy/values.yaml b/charts/capsule-proxy/values.yaml index 10cccffa..759e3cee 100644 --- a/charts/capsule-proxy/values.yaml +++ b/charts/capsule-proxy/values.yaml @@ -2,8 +2,6 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. -# -- Set the deployment mode of the capsule-proxy as `Deployment` or `DaemonSet`. -kind: Deployment image: # -- Set the image registry for capsule-proxy registry: ghcr.io @@ -13,41 +11,93 @@ image: pullPolicy: IfNotPresent # -- Overrides the image tag whose default is the chart appVersion. tag: "" -jobs: - certs: - # -- Set the image repository of the post install certgen job - registry: docker.io - # -- Set the image repository of the post install certgen job - repository: jettech/kube-webhook-certgen - # -- Set the image pull policy of the post install certgen job - pullPolicy: IfNotPresent - # -- Set the image tag of the post install certgen job - tag: "v1.3.0" - # -- Annotations to add to the certgen job. - annotations: {} +# -- Configuration for `imagePullSecrets` so that you can use a private images registry. +imagePullSecrets: [] - # -- Sets the ttl in seconds after a finished certgen job is deleted. Set to -1 to never delete. - ttlSecondsAfterFinished: 60 +# -- Set the deployment mode of the capsule-proxy as `Deployment` or `DaemonSet`. +kind: Deployment - # -- Security context for the job pods. - podSecurityContext: - seccompProfile: - type: "RuntimeDefault" +# -- When deployed as DaemonSet use +hostNetwork: false +daemonset: + # -- Use the host network namespace for capsule-proxy pod. + hostNetwork: false + # -- Binding the capsule-proxy listening port to the host port. + hostPort: false - # -- Security context for the job containers. - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1002 - runAsNonRoot: true - runAsUser: 1002 -# -- Configuration for `imagePullSecrets` so that you can use a private images registry. -imagePullSecrets: [] +# -- Set the replica count for capsule-proxy pod. +replicaCount: 1 + +# The following annotations guarantee scheduling for critical add-on pods +# podAnnotations: +# scheduler.alpha.kubernetes.io/critical-pod: '' +# -- Annotations to add to the capsule-proxy pod. +podAnnotations: {} + +# -- Labels to add to the capsule-proxy pod. +podLabels: {} + +# Enable node selector and tolerations when deploying on the controlplane +# nodeSelector: +# node-role.kubernetes.io/master: "" +# tolerations: +# - key: CriticalAddonsOnly +# operator: Exists +# - effect: NoSchedule +# key: node-role.kubernetes.io/master +# -- Set the node selector for the capsule-proxy pod. +nodeSelector: {} +# -- Set list of tolerations for the capsule-proxy pod. +tolerations: [] +# -- Set affinity rules for the capsule-proxy pod. +affinity: {} +# -- Set the restartPolicy for the capsule-proxy pod. +restartPolicy: Always + +# -- Topology Spread Constraints for the capsule-proxy pod. +topologySpreadConstraints: [] +# Ensure Replicas are spreaded across different Nodes: +# - maxSkew: 1 +# topologyKey: kubernetes.io/hostname +# whenUnsatisfiable: ScheduleAnyway +# labelSelector: +# matchLabels: +# app.kubernetes.io/name: capsule-proxy + +# Set a pod priorityClassName +# priorityClassName: system-node-critical +# -- Specifies PriorityClass of the capsule-proxy pod. +priorityClassName: "" + +# -- Security context for the capsule-proxy pod. +podSecurityContext: + seccompProfile: + type: "RuntimeDefault" + +# -- Security context for the capsule-proxy container. +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 1002 + runAsNonRoot: true + runAsUser: 1002 + +resources: + limits: + # -- Set the CPU requests assigned to the controller. + cpu: 200m + # -- Set the memory requests assigned to the controller. + memory: 128Mi + requests: + # -- Set the CPU limits assigned to the controller. + cpu: 200m + # -- Set the memory limits assigned to the controller. + memory: 128Mi # Controller Options options: @@ -86,6 +136,47 @@ options: # -- Burst to use for interacting with kubernetes API Server. clientConnectionBurst: 30 +jobs: + certs: + # -- Set the image repository of the post install certgen job + registry: docker.io + # -- Set the image repository of the post install certgen job + repository: jettech/kube-webhook-certgen + # -- Set the image pull policy of the post install certgen job + pullPolicy: IfNotPresent + # -- Set the image tag of the post install certgen job + tag: "v1.3.0" + # -- Annotations to add to the certgen job. + annotations: {} + # -- Set the restartPolicy + restartPolicy: Never + # -- Sets the ttl in seconds after a finished certgen job is deleted. Set to -1 to never delete. + ttlSecondsAfterFinished: 60 + # -- Security context for the job pods. + podSecurityContext: + seccompProfile: + type: "RuntimeDefault" + # -- Security context for the job containers. + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 1002 + runAsNonRoot: true + runAsUser: 1002 + # -- Set the node selector + nodeSelector: {} + # -- Set list of tolerations + tolerations: [] + # -- Set affinity rules + affinity: {} + # -- Set Topology Spread Constraints + topologySpreadConstraints: [] + # -- Set a pod priorityClassName + priorityClassName: "" + certManager: # -- Set if the cert manager will generate SSL certificates (self-signed or CA-signed) generateCertificates: false @@ -100,6 +191,8 @@ certManager: # -- Set the name of the ClusterIssuer if issuer kind is ClusterIssuer and if cert manager will generate CA signed SSL certificates name: "" # Name of the ClusterIssuer certificate: + # -- Include internal service names in certificate (disable if you create a public cert) + includeInternalServiceNames: true # -- Additional DNS Names to include in certificate dnsNames: [] # -- Additional IP Addresses to include in certificate @@ -107,7 +200,6 @@ certManager: # -- Additional URIs to include in certificate uris: [] - # ServiceAccount serviceAccount: # -- Specifies whether a service account should be created. @@ -118,62 +210,6 @@ serviceAccount: # @default -- `capsule-proxy`` name: "" -# -- Set the replica count for capsule-proxy pod. -replicaCount: 1 - -# The following annotations guarantee scheduling for critical add-on pods -# podAnnotations: -# scheduler.alpha.kubernetes.io/critical-pod: '' -# -- Annotations to add to the capsule-proxy pod. -podAnnotations: {} - -# -- Labels to add to the capsule-proxy pod. -podLabels: {} - -# -- Topology Spread Constraints for the capsule-proxy pod. -topologySpreadConstraints: [] -# Ensure Replicas are spreaded across different Nodes: -# - maxSkew: 1 -# topologyKey: kubernetes.io/hostname -# whenUnsatisfiable: ScheduleAnyway -# labelSelector: -# matchLabels: -# app.kubernetes.io/name: capsule-proxy - -# Set a pod priorityClassName -# priorityClassName: system-node-critical -# -- Specifies PriorityClass of the capsule-proxy pod. -priorityClassName: "" - -# -- Security context for the capsule-proxy pod. -podSecurityContext: - seccompProfile: - type: "RuntimeDefault" - -# -- Security context for the capsule-proxy container. -securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1002 - runAsNonRoot: true - runAsUser: 1002 - - -resources: - limits: - # -- Set the CPU requests assigned to the controller. - cpu: 200m - # -- Set the memory requests assigned to the controller. - memory: 128Mi - requests: - # -- Set the CPU limits assigned to the controller. - cpu: 200m - # -- Set the memory limits assigned to the controller. - memory: 128Mi - # HorizontalPodAutoscaler autoscaling: # -- Specifies whether an hpa for capsule-proxy should be created. @@ -185,31 +221,6 @@ autoscaling: # -- Set the targetCPUUtilizationPercentage for capsule-proxy hpa. targetCPUUtilizationPercentage: 80 -# Enable node selector and tolerations when deploying on the controlplane -# nodeSelector: -# node-role.kubernetes.io/master: "" -# tolerations: -# - key: CriticalAddonsOnly -# operator: Exists -# - effect: NoSchedule -# key: node-role.kubernetes.io/master -# -- Set the node selector for the capsule-proxy pod. -nodeSelector: {} -# -- Set list of tolerations for the capsule-proxy pod. -tolerations: [] -# -- Set affinity rules for the capsule-proxy pod. -affinity: {} -# -- Set the restartPolicy for the capsule-proxy pod. -restartPolicy: Always - -# When deployed as DaemonSet use -# hostNetwork: true -daemonset: - # -- Use the host network namespace for capsule-proxy pod. - hostNetwork: false - # -- Binding the capsule-proxy listening port to the host port. - hostPort: false - # Service service: # -- Specifies the service type should be created (`ClusterIP`, `NodePort`or `LoadBalancer`)