Upgrade Jitsu to v2.8.2 (#50)

* fix!: set default clickhouse database to newjitsu_metrics * feat: bump appversion/jitsu to v2.8.2 * feat: set sidecar image for syncctl Fixes #47 * refactor: remove syncctl connection to bulker * refactor: default sidecar image tag to the tag of syncctl before appversion * feat: add clickhouse configuration to migration * feat: limit history for trim job * refactor: only deploy a single replica of kafka * refactor: only deploy a single shard and replica of clickhouse and disable zookeeper * chore(deps): update subcharts * fix: update kafka values * fix: set history limits for event log cronjob in cronjob spec * fix: downgrade postgres chart * docs: bring documentation up to date with current features * docs: describe potentially breaking changes for new release * docs: update version number for upgrade notes * chore: add license * chore: wrap lines * docs: clarify upgrade notes * docs: direct users to github releases when looking for the chart version
stafftastic · Nov 1, 2024 · 9ffa6ae · 9ffa6ae
1 parent c33d730
commit 9ffa6ae
Show file tree

Hide file tree

Showing 10 changed files with 140 additions and 70 deletions.
diff --git a/Chart.lock b/Chart.lock
@@ -1,18 +1,18 @@
 dependencies:
 - name: postgresql
   repository: oci://registry-1.docker.io/bitnamicharts
-  version: 14.3.3
+  version: 15.5.38
 - name: redis
   repository: oci://registry-1.docker.io/bitnamicharts
-  version: 18.19.4
+  version: 20.2.0
 - name: kafka
   repository: oci://registry-1.docker.io/bitnamicharts
-  version: 26.11.4
+  version: 30.1.5
 - name: mongodb
   repository: oci://registry-1.docker.io/bitnamicharts
-  version: 14.13.0
+  version: 16.0.3
 - name: clickhouse
   repository: oci://registry-1.docker.io/bitnamicharts
-  version: 6.0.2
-digest: sha256:849a9bf101f0f355ef5462e3523a9b8aa848a5e80b9d24897cc75fb61407d6b2
-generated: "2024-04-11T08:04:50.759931548+02:00"
+  version: 6.2.27
+digest: sha256:edeea0d4376d049530cd041224b2ceee14a60ae0051b5d5ebf5a23e98d2de452
+generated: "2024-10-11T12:17:19.12668398+02:00"
diff --git a/Chart.yaml b/Chart.yaml
@@ -2,28 +2,29 @@ apiVersion: v2
 name: jitsu
 type: application
 
-# Chart version is set automatically as part of the release process
+# Chart version is set automatically as part of the release process.
+# See releases on GitHub to find the latest version.
 version: 0.0.0
-appVersion: 2.7.0
+appVersion: 2.8.2
 
 dependencies:
   - name: postgresql
-    version: ^14.0.1
+    version: ^15.5.38
     repository: oci://registry-1.docker.io/bitnamicharts
     condition: postgresql.enabled
   - name: redis
-    version: ^18.12.1
+    version: ^20.2.0
     repository: oci://registry-1.docker.io/bitnamicharts
     condition: redis.enabled
   - name: kafka
-    version: ^26.8.4
+    version: ^30.1.5
     repository: oci://registry-1.docker.io/bitnamicharts
     condition: kafka.enabled
   - name: mongodb
-    version: ^14.8.2
+    version: ^16.0.3
     repository: oci://registry-1.docker.io/bitnamicharts
     condition: mongodb.enabled
   - name: clickhouse
-    version: ^6.0.2
+    version: ^6.2.27
     repository: oci://registry-1.docker.io/bitnamicharts
     condition: clickhouse.enabled
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,20 @@
+Copyright 2024 stafftastic GmbH
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the
+“Software”), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/README.md b/README.md
@@ -15,7 +15,8 @@ EOF
 ```
 
 For a production deployment it is recommended to read through `values.yaml` and make conscious
-decisions in order to ensure the deployment is secure, reliable and scalable.
+decisions in order to ensure the deployment is secure, reliable and scalable. Dependencies are
+minimally configured and do not provide high-availability out of the box.
 
 ## Basic Configuration
 `values.yaml`:
@@ -26,7 +27,7 @@ postgresql:
 mongodb:
   auth:
     passwords: ["changeMe"]
-redis:
+clickhouse:
   auth:
     password: "changeMe"
 
@@ -54,29 +55,32 @@ This chart deploys the following dependencies by default in order to provide an
 experience, however for production it is recommended you deploy these separately:
 
 * Postgres
-* Redis
 * Kafka
 * MongoDB
+* Clickhouse
 
 In order to use your own instances of these, disable them in with their respective options:
 ```yaml
 postgresql:
   enabled: false
-redis:
-  enabled: false
 kafka:
   enabled: false
 mongodb:
   enabled: false
+clickhouse:
+  enabled: false
 ```
 
 Then supply the connection details in the `config` section (or specifically per service):
 ```yaml
 config:
   databaseUrl: "postgres://..."
-  redisUrl: "redis://..."
   kafkaBootstrapServers: "kafka:9092,..."
   mongodbUrl: "mongodb://..."
+  clickhouseHttpHost: "...:8123"
+  clickhouseTcpHost: "...:9000"
+  clickhouseUsername: "..."
+  clickhousePassword: "..."
 ```
 
 ## Configuration Options
@@ -156,20 +160,21 @@ necessary if you're using the Kafka subchart.
 
 ## Upgrading
 It's not necessary to go through all intermediate versions when upgrading, however if upgrading to a
-version greater or equal to one mentioned below, additional steps may be required.
+version greater or equal to one mentioned below, additional steps may be required. This generally
+only encompasses breaking changes within the chart itself. New versions of Jitsu may still introduce
+breaking changes not covered here.
 
-### v1.1.0
-The Rotor is now also protected with an auth token when using the token generator (enabled by
-default). This means that if you have an old token secret you will either need to add values for the
-Rotor to the secret or delete the secret and let the token generator create a new one upon
-deployment. Deleting the secret will also use the uniform format across services introduced in Jitsu
-v2.4.5.
+### v2.0.0
+This release sets the default ClickHouse database to `newjitsu_metrics` as some components did not
+behave correctly with the old default (`default`). If you have data you wish to keep in the old
+`default` database, you may need to manually migrate this.
 
-### v1.4.0
-This release disables the Redis deployment by default as it is no longer required by Jitsu v2.5.0.
-If you have functions persistent storage or identity stitching data you wish to keep, set
-`redis.enabled` to `true` to enable "double read" mode as outlined in the [release notes for Jitsu
-v2.5.0](https://github.com/jitsucom/jitsu/releases/tag/jitsu2-v2.5.0).
+Kafka is now configured with only a single controller replica by default. It is recommended to
+explicitly set this to 3 replicas if you are upgrading and using the default configuration.
+
+ClickHouse is now configured with only a single shard and single replica by default. It is
+recommended to explicitly set this to 2 shards and 3 replicas if you are upgrading and using the
+default configuration.
 
 ### v1.6.0
 This release splits the `config.clickhouseHost` and `config.clickhouseHostFrom` parameters up into
@@ -182,3 +187,16 @@ ClickHouse is now also set up to use Zookeeper instead of ClickHouse Keeper as i
 broken in Bitnami's Helm chart for ClickHouse: https://github.com/bitnami/charts/issues/15935. If
 you had a working configuration using ClickHouse Keeper, you will need to explicitly enable it and
 disable Zookeeper to avoid switching over to a fresh Zookeeper deployment.
+
+### v1.4.0
+This release disables the Redis deployment by default as it is no longer required by Jitsu v2.5.0.
+If you have functions persistent storage or identity stitching data you wish to keep, set
+`redis.enabled` to `true` to enable "double read" mode as outlined in the [release notes for Jitsu
+v2.5.0](https://github.com/jitsucom/jitsu/releases/tag/jitsu2-v2.5.0).
+
+### v1.1.0
+The Rotor is now also protected with an auth token when using the token generator (enabled by
+default). This means that if you have an old token secret you will either need to add values for the
+Rotor to the secret or delete the secret and let the token generator create a new one upon
+deployment. Deleting the secret will also use the uniform format across services introduced in Jitsu
+v2.4.5.
diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
@@ -143,7 +143,7 @@ Create the name of the service account to use
 
 {{- define "jitsu.clickhouseDatabase" -}}
 {{- if and (not .Values.config.clickhouseDatabase) .Values.clickhouse.enabled -}}
-default
+newjitsu_metrics
 {{- else -}}
 {{ .Values.config.clickhouseDatabase }}
 {{- end }}

diff --git a/templates/event-log-trim/cronjob.yaml b/templates/event-log-trim/cronjob.yaml
@@ -11,6 +11,8 @@ metadata:
   {{- end }}
 spec:
   schedule: {{ .Values.eventLogTrim.schedule | quote }}
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
   jobTemplate:
     spec:
       backoffLimit: 1

diff --git a/templates/migration/job.yaml b/templates/migration/job.yaml
@@ -64,13 +64,55 @@ spec:
           {{- end }}
           env:
             {{- if and .Values.config.enabled .Values.migration.config.enabled }}
+            {{- with .Values.migration.config }}
             {{- if or .databaseUrlFrom $.Values.config.databaseUrlFrom }}
             - name: DATABASE_URL
               valueFrom:
                 {{- toYaml (.databaseUrlFrom | default $.Values.config.databaseUrlFrom) | nindent 16 }}
             {{- else }}
             - name: DATABASE_URL
-              value: {{ .Values.migration.config.databaseUrl | default (include "jitsu.databaseUrl" .) | quote }}
+              value: {{ .databaseUrl | default (include "jitsu.databaseUrl" $) | quote }}
+            {{- end }}
+            {{- if or .clickhouseHostFrom $.Values.config.clickhouseHttpHostFrom }}
+            - name: CLICKHOUSE_HOST
+              valueFrom:
+                {{- toYaml (.clickhouseHostFrom | default $.Values.config.clickhouseHttpHostFrom) | nindent 4 }}
+            {{- else }}
+            - name: CLICKHOUSE_HOST
+              value: {{ .clickhouseHost | default (include "jitsu.clickhouseHttpHost" $) | quote }}
+            {{- end }}
+
+            {{- if or .clickhouseDatabaseFrom $.Values.config.clickhouseDatabaseFrom }}
+            - name: CLICKHOUSE_DATABASE
+              valueFrom:
+                {{- toYaml (.clickhouseDatabaseFrom | default $.Values.config.clickhouseDatabaseFrom) | nindent 4 }}
+            {{- else }}
+            - name: CLICKHOUSE_DATABASE
+              value: {{ .clickhouseDatabase | default (include "jitsu.clickhouseDatabase" $) | quote }}
+            {{- end }}
+
+            {{- if or .clickhouseUsernameFrom $.Values.config.clickhouseUsernameFrom }}
+            - name: CLICKHOUSE_USERNAME
+              valueFrom:
+                {{- toYaml (.clickhouseUsernameFrom | default $.Values.config.clickhouseUsernameFrom) | nindent 4 }}
+            {{- else }}
+            - name: CLICKHOUSE_USERNAME
+              value: {{ .clickhouseUsername | default (include "jitsu.clickhouseUsername" $) | quote }}
+            {{- end }}
+
+            {{- if or .clickhousePasswordFrom $.Values.config.clickhousePasswordFrom }}
+            - name: CLICKHOUSE_PASSWORD
+              valueFrom:
+                {{- toYaml (.clickhousePasswordFrom | default $.Values.config.clickhousePasswordFrom) | nindent 4 }}
+            {{- else }}
+            - name: CLICKHOUSE_PASSWORD
+              value: {{ .clickhousePassword | default (include "jitsu.clickhousePassword" $) | quote }}
+            {{- end }}
+
+            {{- with (.clickhouseSsl | default $.Values.config.clickhouseSsl) }}
+            - name: CLICKHOUSE_SSL
+              value: {{ . | quote }}
+            {{- end }}
             {{- end }}
             {{- end }}
             {{- with .Values.migration.env }}

diff --git a/templates/syncctl/_helpers.tpl b/templates/syncctl/_helpers.tpl
@@ -82,32 +82,8 @@ app.kubernetes.io/component: syncctl
 {{- end }}
 {{- end }}
 
-{{- if and (not .bulkerUrl) (not $.Values.config.bulkerUrl) $.Values.bulker.enabled }}
-- name: SYNCCTL_BULKER_URL
-  value: {{ printf "http://%s-bulker:%d" (include "jitsu.fullname" $) (int $.Values.bulker.service.port) | quote }}
-{{- end }}
-{{- with (.bulkerUrl | default $.Values.config.bulkerUrl) }}
-- name: SYNCCTL_BULKER_URL
-  value: {{ . | quote }}
-{{- end }}
-
-{{- if .bulkerAuthTokenFrom }}
-- name: SYNCCTL_BULKER_AUTH_TOKEN
-  valueFrom:
-    {{- toYaml .bulkerAuthTokenFrom | nindent 4 }}
-{{- else }}
-{{- if and (not .bulkerAuthToken ) $.Values.bulker.enabled $.Values.tokenGenerator.enabled }}
-- name: SYNCCTL_BULKER_AUTH_TOKEN
-  valueFrom:
-    secretKeyRef:
-      name: {{ include "jitsu.fullname" $ }}-tokens
-      key: bulkerAuthToken
-{{- end }}
-{{- with .bulkerAuthToken }}
-- name: SYNCCTL_BULKER_AUTH_TOKEN
-  value: {{ . | quote }}
-{{- end }}
-{{- end }}
+- name: SYNCCTL_SIDECAR_IMAGE
+  value: {{ .sidecarImage | default (printf "jitsucom/sidecar:%s" ($.Values.syncctl.image.tag | default $.Chart.AppVersion)) | quote }}
 
 {{- if .kubernetesClientConfigFrom }}
 - name: SYNCCTL_KUBERNETES_CLIENT_CONFIG

diff --git a/templates/syncctl/proxy-services.yaml b/templates/syncctl/proxy-services.yaml
@@ -5,7 +5,6 @@
 }}
 {{- range list
   (printf "%s-postgresql" .Release.Name)
-  (printf "%s-bulker" (include "jitsu.fullname" .))
 }}
 ---
 apiVersion: v1

diff --git a/values.yaml b/values.yaml
@@ -750,14 +750,8 @@ syncctl:
     sidecarDatabaseUrl: ""
     sidecarDatabaseUrlFrom: {}
 
-    # Leave empty to configure automatically.
-    # https://docs.jitsu.com/self-hosting/configuration#syncctl_bulker_url
-    bulkerUrl: ""
-
-    # Leave empty to configure automatically.
-    # https://docs.jitsu.com/self-hosting/configuration#syncctl_bulker_auth_key
-    bulkerAuthToken: ""
-    bulkerAuthTokenFrom: {}
+    # Leave empty to default to jitsucom/sidecar:{{ .Values.syncctl.image.tag | default .Chart.AppVersion }}
+    sidecarImage: ""
 
     # https://docs.jitsu.com/self-hosting/configuration#syncctl_kubernetes_client_config
     kubernetesClientConfig: ""
@@ -825,6 +819,19 @@ migration:
     # https://docs.jitsu.com/self-hosting/configuration#database_url
     databaseUrl: ""
     databaseUrlFrom: {}
+
+    # Leave empty to use global configuration
+    clickhouseHttpHost: ""
+    clickhouseHttpHostFrom: {}
+    clickhouseTcpHost: ""
+    clickhouseTcpHostFrom: {}
+    clickhouseDatabase: ""
+    clickhouseDatabaseFrom: {}
+    clickhouseUsername: ""
+    clickhouseUsernameFrom: {}
+    clickhousePassword: ""
+    clickhousePasswordFrom: {}
+    clickhouseSsl: ""
   image:
     repository: jitsucom/console
     pullPolicy: IfNotPresent
@@ -939,21 +946,26 @@ mongodb:
 
 kafka:
   enabled: true
+  controller:
+    replicaCount: 1
   listeners:
     interbroker:
       protocol: PLAINTEXT
     controller:
       protocol: PLAINTEXT
     client:
       protocol: PLAINTEXT
+    external:
       external: PLAINTEXT
 
 clickhouse:
   enabled: true
+  shards: 1
+  replicaCount: 1
   auth:
     username: "jitsu"
     password: "jitsu"
   keeper:
     enabled: false
   zookeeper:
-    enabled: true
+    enabled: false