From c830b3b4fdc5acb6a509b44d7a7abe1fdd39437e Mon Sep 17 00:00:00 2001 From: Stefan Nica Date: Thu, 21 Nov 2024 21:38:28 +0100 Subject: [PATCH] Helm chart improvements for robustness and high availability --- .../deploy/helm/templates/_environment.tpl | 117 ++++++++++++++++++ .../deploy/helm/templates/server-db-job.yaml | 17 +-- .../helm/templates/server-deployment.yaml | 20 ++- .../deploy/helm/templates/server-secret.yaml | 19 +-- 4 files changed, 138 insertions(+), 35 deletions(-) diff --git a/src/zenml/zen_server/deploy/helm/templates/_environment.tpl b/src/zenml/zen_server/deploy/helm/templates/_environment.tpl index a1c0ac5b5bf..2f64d5881c3 100644 --- a/src/zenml/zen_server/deploy/helm/templates/_environment.tpl +++ b/src/zenml/zen_server/deploy/helm/templates/_environment.tpl @@ -3,6 +3,123 @@ Helpers for environment variables configured in ZenML deployments and secrets st */}} +{{/* +ZenML store configuration options (non-secret values). + +This template constructs a dictionary that is similar to the python values that +can be configured in the zenml.zen_store.sql_zen_store.SqlZenStoreConfiguration +class. Only non-secret values are included in this dictionary. + +The dictionary is then converted into deployment environment variables by other +templates and inserted where it is needed. + +The input is taken from a .ZenML dict that is passed to the template and +contains the values configured in the values.yaml file for the ZenML server. + +Args: + .ZenML: A dictionary with the ZenML configuration values configured for the + ZenML server. +Returns: + A dictionary with the non-secret values configured for the ZenML store. +*/}} +{{- define "zenml.storeConfigurationAttrs" -}} +{{- if .ZenML.database.url }} +type: sql +ssl_verify_server_cert: {{ .ZenML.database.sslVerifyServerCert | default "false" | quote }} +{{- if .ZenML.database.backupStrategy }} +backup_strategy: {{ .ZenML.database.backupStrategy | quote }} +{{- if eq .ZenML.database.backupStrategy "database" }} +backup_database: {{ .ZenML.database.backupDatabase | quote }} +{{- else if eq .ZenML.database.backupStrategy "dump-file" }} +backup_directory: "/backups" +{{- end }} +{{- end }} +{{- if .ZenML.database.poolSize }} +pool_size: {{ .ZenML.database.poolSize | quote }} +{{- end }} +{{- if .ZenML.database.maxOverflow }} +max_overflow: {{ .ZenML.database.maxOverflow | quote }} +{{- end }} +{{- end }} +{{- end }} + + +{{/* +ZenML store configuration options (secret values). + +This template constructs a dictionary that is similar to the python values that +can be configured in the zenml.zen_store.sql_zen_store.SqlZenStoreConfiguration +class. Only secret values are included in this dictionary. + +The dictionary is then converted into deployment environment variables by other +templates and inserted where it is needed. + +The input is taken from a .ZenML dict that is passed to the template and +contains the values configured in the values.yaml file for the ZenML server. + +Args: + .ZenML: A dictionary with the ZenML configuration values configured for the + ZenML server. +Returns: + A dictionary with the secret values configured for the ZenML store. +*/}} +{{- define "zenml.storeSecretConfigurationAttrs" -}} +{{- if .ZenML.database.url }} +url: {{ .ZenML.database.url | quote }} +{{- if .ZenML.database.sslCa }} +ssl_ca: {{ .Files.Get .ZenML.database.sslCa }} +{{- end }} +{{- if .ZenML.database.sslCert }} +ssl_cert: {{ .Files.Get .ZenML.database.sslCert }} +{{- end }} +{{- if .ZenML.database.sslKey }} +ssl_key: {{ .Files.Get .ZenML.database.sslKey }} +{{- end }} +{{- end }} +{{- end }} + + +{{/* +Store configuration environment variables (non-secret values). + +Passes the .Values.zenml dict as input to the `zenml.storeConfigurationAttrs` +template and converts the output into a dictionary of environment variables that +need to be configured for the store. + +Args: + .Values: The values.yaml file for the ZenML deployment. +Returns: + A dictionary with the non-secret environment variables that are configured for + the store (i.e. keys starting with `ZENML_STORE_`). +*/}} +{{- define "zenml.storeEnvVariables" -}} +{{ $zenml := dict "ZenML" .Values.zenml }} +{{- range $k, $v := include "zenml.storeConfigurationAttrs" $zenml | fromYaml }} +ZENML_STORE_{{ $k | upper }}: {{ $v | quote }} +{{- end }} +{{- end }} + + +{{/* +Store configuration environment variables (secret values). + +Passes the .Values.zenml dict as input to the `zenml.storeSecretConfigurationAttrs` +template and converts the output into a dictionary of environment variables that +need to be configured for the store. + +Args: + .Values: The values.yaml file for the ZenML deployment. +Returns: + A dictionary with the secret environment variables that are configured for + the store (i.e. keys starting with `ZENML_STORE_`). +*/}} +{{- define "zenml.storeSecretEnvVariables" -}} +{{ $zenml := dict "ZenML" .Values.zenml }} +{{- range $k, $v := include "zenml.storeSecretConfigurationAttrs" $zenml | fromYaml }} +ZENML_STORE_{{ $k | upper }}: {{ $v | quote }} +{{- end }} +{{- end }} + {{/* ZenML server configuration options (non-secret values). diff --git a/src/zenml/zen_server/deploy/helm/templates/server-db-job.yaml b/src/zenml/zen_server/deploy/helm/templates/server-db-job.yaml index ba1a94ef56d..0e6614b0032 100644 --- a/src/zenml/zen_server/deploy/helm/templates/server-db-job.yaml +++ b/src/zenml/zen_server/deploy/helm/templates/server-db-job.yaml @@ -79,20 +79,9 @@ spec: {{- end }} - name: ZENML_DEFAULT_PROJECT_NAME value: {{ .Values.zenml.defaultProject | quote }} - - name: ZENML_STORE_TYPE - value: sql - - name: ZENML_STORE_SSL_VERIFY_SERVER_CERT - value: {{ .Values.zenml.database.sslVerifyServerCert | default "false" | quote }} - {{- if .Values.zenml.database.backupStrategy }} - - name: ZENML_STORE_BACKUP_STRATEGY - value: {{ .Values.zenml.database.backupStrategy | quote }} - {{- if eq .Values.zenml.database.backupStrategy "database" }} - - name: ZENML_STORE_BACKUP_DATABASE - value: {{ .Values.zenml.database.backupDatabase | quote }} - {{- else if eq .Values.zenml.database.backupStrategy "dump-file" }} - - name: ZENML_STORE_BACKUP_DIRECTORY - value: /backups - {{- end }} + {{- range $k, $v := include "zenml.storeEnvVariables" . | fromYaml }} + - name: {{ $k }} + value: {{ $v | quote }} {{- end }} {{- range $k, $v := include "zenml.serverEnvVariables" . | fromYaml }} - name: {{ $k }} diff --git a/src/zenml/zen_server/deploy/helm/templates/server-deployment.yaml b/src/zenml/zen_server/deploy/helm/templates/server-deployment.yaml index 38923bd04f6..c74e3447bb9 100644 --- a/src/zenml/zen_server/deploy/helm/templates/server-deployment.yaml +++ b/src/zenml/zen_server/deploy/helm/templates/server-deployment.yaml @@ -63,14 +63,14 @@ spec: value: "True" {{- end }} {{- if .Values.zenml.database.url }} - - name: ZENML_STORE_TYPE - value: sql - name: DISABLE_DATABASE_MIGRATION value: "True" - - name: ZENML_STORE_SSL_VERIFY_SERVER_CERT - value: {{ .Values.zenml.database.sslVerifyServerCert | default "false" | quote }} {{- end }} + {{- range $k, $v := include "zenml.storeEnvVariables" . | fromYaml }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} {{- range $k, $v := include "zenml.serverEnvVariables" . | fromYaml }} - name: {{ $k }} value: {{ $v | quote }} @@ -104,6 +104,18 @@ spec: httpGet: path: /health port: http + lifecycle: + preStop: + exec: + # Give the process 15 more seconds before the SIGTERM signal is + # sent. This allows the endpoint removal to reach the ingress + # controller in time and for traffic to be routed away from the + # pod before it is shut down. This eliminates the number of 502 + # errors returned to the user. + # + # See https://learnk8s.io/graceful-shutdown for more information. + # + command: ["sleep", "15"] resources: {{- toYaml .Values.resources | nindent 12 }} {{- with .Values.nodeSelector }} diff --git a/src/zenml/zen_server/deploy/helm/templates/server-secret.yaml b/src/zenml/zen_server/deploy/helm/templates/server-secret.yaml index ae09343b0e3..9ebf401b516 100644 --- a/src/zenml/zen_server/deploy/helm/templates/server-secret.yaml +++ b/src/zenml/zen_server/deploy/helm/templates/server-secret.yaml @@ -10,23 +10,8 @@ data: {{- else }} ZENML_SERVER_JWT_SECRET_KEY: {{ $prevServerSecret.data.ZENML_SERVER_JWT_SECRET_KEY | default (randAlphaNum 32 | b64enc | quote) }} {{- end }} - {{- if .Values.zenml.database.url }} - ZENML_STORE_URL: {{ .Values.zenml.database.url | b64enc | quote }} - {{- if .Values.zenml.database.sslCa }} - ZENML_STORE_SSL_CA: {{ .Files.Get .Values.zenml.database.sslCa | b64enc }} - {{- end }} - {{- if .Values.zenml.database.sslCert }} - ZENML_STORE_SSL_CERT: {{ .Files.Get .Values.zenml.database.sslCert | b64enc }} - {{- end }} - {{- if .Values.zenml.database.sslKey }} - ZENML_STORE_SSL_KEY: {{ .Files.Get .Values.zenml.database.sslKey | b64enc }} - {{- end }} - {{- if .Values.zenml.database.poolSize }} - ZENML_STORE_POOL_SIZE: {{ .Values.zenml.database.poolSize | b64enc | quote }} - {{- end }} - {{- if .Values.zenml.database.maxOverflow }} - ZENML_STORE_MAX_OVERFLOW: {{ .Values.zenml.database.maxOverflow | b64enc | quote }} - {{- end }} + {{- range $k, $v := include "zenml.storeSecretEnvVariables" . | fromYaml}} + {{ $k }}: {{ $v | b64enc | quote }} {{- end }} {{- range $k, $v := include "zenml.secretsStoreSecretEnvVariables" . | fromYaml}} {{ $k }}: {{ $v | b64enc | quote }}