diff --git a/Chart.lock b/Chart.lock index f95f834..fdafebf 100644 --- a/Chart.lock +++ b/Chart.lock @@ -1,15 +1,18 @@ dependencies: - name: postgresql repository: oci://registry-1.docker.io/bitnamicharts - version: 14.0.1 + version: 14.3.3 - name: redis repository: oci://registry-1.docker.io/bitnamicharts - version: 18.12.1 + version: 18.19.4 - name: kafka repository: oci://registry-1.docker.io/bitnamicharts - version: 26.8.4 + version: 26.11.4 - name: mongodb repository: oci://registry-1.docker.io/bitnamicharts - version: 14.8.2 -digest: sha256:8a5ee2a84944c85ca2eec100296242a997a6f83f5840386fe0b0543781db7661 -generated: "2024-02-05T00:49:32.33554551+01:00" + version: 14.13.0 +- name: clickhouse + repository: oci://registry-1.docker.io/bitnamicharts + version: 6.0.2 +digest: sha256:849a9bf101f0f355ef5462e3523a9b8aa848a5e80b9d24897cc75fb61407d6b2 +generated: "2024-04-11T08:04:50.759931548+02:00" diff --git a/Chart.yaml b/Chart.yaml index 7af9f19..63cfa41 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -4,7 +4,7 @@ type: application # Chart version is set automatically as part of the release process version: 0.0.0 -appVersion: 2.4.6 +appVersion: 2.5.0 dependencies: - name: postgresql @@ -23,3 +23,7 @@ dependencies: version: ^14.8.2 repository: oci://registry-1.docker.io/bitnamicharts condition: mongodb.enabled + - name: clickhouse + version: ^6.0.2 + repository: oci://registry-1.docker.io/bitnamicharts + condition: clickhouse.enabled diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl index 3320823..a6cfc79 100644 --- a/templates/_helpers.tpl +++ b/templates/_helpers.tpl @@ -117,3 +117,35 @@ Create the name of the service account to use {{ .Values.config.mongodbUrl }} {{- end }} {{- end }} + +{{- define "jitsu.clickhouseHost" -}} +{{- if and (not .Values.config.clickhouseHost) .Values.clickhouse.enabled }} +{{ .Release.Name }}-clickhouse:8123 +{{- else }} +{{ .Values.config.clickhouseHost }} +{{- end }} +{{- end }} + +{{- define "jitsu.clickhouseDatabase" -}} +{{- if and (not .Values.config.clickhouseDatabase) .Values.clickhouse.enabled }} +default +{{- else }} +{{ .Values.config.clickhouseDatabase }} +{{- end }} +{{- end }} + +{{- define "jitsu.clickhouseUsername" -}} +{{- if and (not .Values.config.clickhouseUsername) .Values.clickhouse.enabled }} +jitsu +{{- else }} +{{ .Values.config.clickhouseUsername }} +{{- end }} +{{- end }} + +{{- define "jitsu.clickhousePassword" -}} +{{- if and (not .Values.config.clickhousePassword) .Values.clickhouse.enabled }} +jitsu +{{- else }} +{{ .Values.config.clickhousePassword }} +{{- end }} +{{- end }} diff --git a/templates/bulker/_helpers.tpl b/templates/bulker/_helpers.tpl index 0b1f6eb..50be5c0 100644 --- a/templates/bulker/_helpers.tpl +++ b/templates/bulker/_helpers.tpl @@ -10,7 +10,6 @@ app.kubernetes.io/component: bulker - name: BULKER_INSTANCE_ID value: {{ .instanceId | quote }} - {{- if or .redisUrlFrom $.Values.config.redisUrlFrom }} - name: BULKER_REDIS_URL valueFrom: @@ -168,6 +167,47 @@ app.kubernetes.io/component: bulker {{- end }} {{- end }} +{{- if or .clickhouseHostFrom $.Values.config.clickhouseHostFrom }} +- name: BULKER_CLICKHOUSE_HOST + valueFrom: + {{- toYaml (.clickhouseHostFrom | default $.Values.config.clickhouseHostFrom) | nindent 4 }} +{{- else }} +- name: BULKER_CLICKHOUSE_HOST + value: {{ .clickhouseHost | default (include "jitsu.clickhouseHost" $) | quote }} +{{- end }} + +{{- if or .clickhouseDatabaseFrom $.Values.config.clickhouseDatabaseFrom }} +- name: BULKER_CLICKHOUSE_DATABASE + valueFrom: + {{- toYaml (.clickhouseDatabaseFrom | default $.Values.config.clickhouseDatabaseFrom) | nindent 4 }} +{{- else }} +- name: BULKER_CLICKHOUSE_DATABASE + value: {{ .clickhouseDatabase | default (include "jitsu.clickhouseDatabase" $) | quote }} +{{- end }} + +{{- if or .clickhouseUsernameFrom $.Values.config.clickhouseUsernameFrom }} +- name: BULKER_CLICKHOUSE_USERNAME + valueFrom: + {{- toYaml (.clickhouseUsernameFrom | default $.Values.config.clickhouseUsernameFrom) | nindent 4 }} +{{- else }} +- name: BULKER_CLICKHOUSE_USERNAME + value: {{ .clickhouseUsername | default (include "jitsu.clickhouseUsername" $) | quote }} +{{- end }} + +{{- if or .clickhousePasswordFrom $.Values.config.clickhousePasswordFrom }} +- name: BULKER_CLICKHOUSE_PASSWORD + valueFrom: + {{- toYaml (.clickhousePasswordFrom | default $.Values.config.clickhousePasswordFrom) | nindent 4 }} +{{- else }} +- name: BULKER_CLICKHOUSE_PASSWORD + value: {{ .clickhousePassword | default (include "jitsu.clickhousePassword" $) | quote }} +{{- end }} + +{{- with (.clickhouseSsl | default $.Values.config.clickhouseSsl) }} +- name: BULKER_CLICKHOUSE_SSL + value: {{ . | quote }} +{{- end }} + {{- with .batchRunnerDefaultPeriodSec }} - name: BULKER_BATCH_RUNNER_DEFAULT_PERIOD_SEC value: {{ . | quote }} diff --git a/templates/bulker/deployment.yaml b/templates/bulker/deployment.yaml index 75356b4..7a09bbf 100644 --- a/templates/bulker/deployment.yaml +++ b/templates/bulker/deployment.yaml @@ -46,6 +46,15 @@ spec: image: ghcr.io/groundnuty/k8s-wait-for:v2.0 args: ["job-wr", "{{ include "jitsu.fullname" . }}-token-generator-{{ sha1sum (toJson .Values) | substr 0 8 }}"] {{- end }} + {{- if .Values.clickhouse.enabled }} + - name: wait-for-clickhouse + {{- with (.Values.bulker.securityContext | default .Values.global.securityContext) }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: ghcr.io/groundnuty/k8s-wait-for:v2.0 + args: ["service", "{{ .Release.Name }}-clickhouse"] + {{- end }} {{- if .Values.redis.enabled }} - name: wait-for-redis {{- with (.Values.bulker.securityContext | default .Values.global.securityContext) }} diff --git a/templates/console/_helpers.tpl b/templates/console/_helpers.tpl index 9656e19..91e2c88 100644 --- a/templates/console/_helpers.tpl +++ b/templates/console/_helpers.tpl @@ -25,6 +25,47 @@ app.kubernetes.io/component: console value: {{ .databaseUrl | default (include "jitsu.databaseUrl" $) | quote }} {{- end }} +{{- if or .clickhouseHostFrom $.Values.config.clickhouseHostFrom }} +- name: CLICKHOUSE_HOST + valueFrom: + {{- toYaml (.clickhouseHostFrom | default $.Values.config.clickhouseHostFrom) | nindent 4 }} +{{- else }} +- name: CLICKHOUSE_HOST + value: {{ .clickhouseHost | default (include "jitsu.clickhouseHost" $) | quote }} +{{- end }} + +{{- if or .clickhouseDatabaseFrom $.Values.config.clickhouseDatabaseFrom }} +- name: CLICKHOUSE_DATABASE + valueFrom: + {{- toYaml (.clickhouseDatabaseFrom | default $.Values.config.clickhouseDatabaseFrom) | nindent 4 }} +{{- else }} +- name: CLICKHOUSE_DATABASE + value: {{ .clickhouseDatabase | default (include "jitsu.clickhouseDatabase" $) | quote }} +{{- end }} + +{{- if or .clickhouseUsernameFrom $.Values.config.clickhouseUsernameFrom }} +- name: CLICKHOUSE_USERNAME + valueFrom: + {{- toYaml (.clickhouseUsernameFrom | default $.Values.config.clickhouseUsernameFrom) | nindent 4 }} +{{- else }} +- name: CLICKHOUSE_USERNAME + value: {{ .clickhouseUsername | default (include "jitsu.clickhouseUsername" $) | quote }} +{{- end }} + +{{- if or .clickhousePasswordFrom $.Values.config.clickhousePasswordFrom }} +- name: CLICKHOUSE_PASSWORD + valueFrom: + {{- toYaml (.clickhousePasswordFrom | default $.Values.config.clickhousePasswordFrom) | nindent 4 }} +{{- else }} +- name: CLICKHOUSE_PASSWORD + value: {{ .clickhousePassword | default (include "jitsu.clickhousePassword" $) | quote }} +{{- end }} + +{{- with (.clickhouseSsl | default $.Values.config.clickhouseSsl) }} +- name: CLICKHOUSE_SSL + value: {{ . | quote }} +{{- end }} + {{- if and (not .bulkerUrl) (not $.Values.config.bulkerUrl) $.Values.bulker.enabled }} - name: BULKER_URL value: {{ printf "http://%s-bulker:%d" (include "jitsu.fullname" $) (int $.Values.bulker.service.port) | quote }} diff --git a/templates/console/deployment.yaml b/templates/console/deployment.yaml index 8839c6b..dd198da 100644 --- a/templates/console/deployment.yaml +++ b/templates/console/deployment.yaml @@ -46,6 +46,15 @@ spec: image: ghcr.io/groundnuty/k8s-wait-for:v2.0 args: ["job-wr", "{{ include "jitsu.fullname" . }}-token-generator-{{ sha1sum (toJson .Values) | substr 0 8 }}"] {{- end }} + {{- if .Values.clickhouse.enabled }} + - name: wait-for-clickhouse + {{- with (.Values.console.securityContext | default .Values.global.securityContext) }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: ghcr.io/groundnuty/k8s-wait-for:v2.0 + args: ["service", "{{ .Release.Name }}-clickhouse"] + {{- end }} {{- if .Values.migration.enabled }} - name: wait-for-migration {{- with (.Values.console.securityContext | default .Values.global.securityContext) }} diff --git a/templates/ingest/_helpers.tpl b/templates/ingest/_helpers.tpl index c5c98b6..6578bbb 100644 --- a/templates/ingest/_helpers.tpl +++ b/templates/ingest/_helpers.tpl @@ -19,6 +19,47 @@ app.kubernetes.io/component: ingest value: {{ .redisUrl | default (include "jitsu.redisUrl" $) | quote }} {{- end }} +{{- if or .clickhouseHostFrom $.Values.config.clickhouseHostFrom }} +- name: INGEST_CLICKHOUSE_HOST + valueFrom: + {{- toYaml (.clickhouseHostFrom | default $.Values.config.clickhouseHostFrom) | nindent 4 }} +{{- else }} +- name: INGEST_CLICKHOUSE_HOST + value: {{ .clickhouseHost | default (include "jitsu.clickhouseHost" $) | quote }} +{{- end }} + +{{- if or .clickhouseDatabaseFrom $.Values.config.clickhouseDatabaseFrom }} +- name: INGEST_CLICKHOUSE_DATABASE + valueFrom: + {{- toYaml (.clickhouseDatabaseFrom | default $.Values.config.clickhouseDatabaseFrom) | nindent 4 }} +{{- else }} +- name: INGEST_CLICKHOUSE_DATABASE + value: {{ .clickhouseDatabase | default (include "jitsu.clickhouseDatabase" $) | quote }} +{{- end }} + +{{- if or .clickhouseUsernameFrom $.Values.config.clickhouseUsernameFrom }} +- name: INGEST_CLICKHOUSE_USERNAME + valueFrom: + {{- toYaml (.clickhouseUsernameFrom | default $.Values.config.clickhouseUsernameFrom) | nindent 4 }} +{{- else }} +- name: INGEST_CLICKHOUSE_USERNAME + value: {{ .clickhouseUsername | default (include "jitsu.clickhouseUsername" $) | quote }} +{{- end }} + +{{- if or .clickhousePasswordFrom $.Values.config.clickhousePasswordFrom }} +- name: INGEST_CLICKHOUSE_PASSWORD + valueFrom: + {{- toYaml (.clickhousePasswordFrom | default $.Values.config.clickhousePasswordFrom) | nindent 4 }} +{{- else }} +- name: INGEST_CLICKHOUSE_PASSWORD + value: {{ .clickhousePassword | default (include "jitsu.clickhousePassword" $) | quote }} +{{- end }} + +{{- with (.clickhouseSsl | default $.Values.config.clickhouseSsl) }} +- name: INGEST_CLICKHOUSE_SSL + value: {{ . | quote }} +{{- end }} + {{- if .authTokensFrom }} - name: INGEST_AUTH_TOKENS valueFrom: diff --git a/templates/ingest/deployment.yaml b/templates/ingest/deployment.yaml index 9c31b97..30ab006 100644 --- a/templates/ingest/deployment.yaml +++ b/templates/ingest/deployment.yaml @@ -42,6 +42,15 @@ spec: image: ghcr.io/groundnuty/k8s-wait-for:v2.0 args: ["job-wr", "{{ include "jitsu.fullname" . }}-token-generator-{{ sha1sum (toJson .Values) | substr 0 8 }}"] {{- end }} + {{- if .Values.clickhouse.enabled }} + - name: wait-for-clickhouse + {{- with (.Values.ingest.securityContext | default .Values.global.securityContext) }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: ghcr.io/groundnuty/k8s-wait-for:v2.0 + args: ["service", "{{ .Release.Name }}-clickhouse"] + {{- end }} {{- if .Values.redis.enabled }} - name: wait-for-redis {{- with (.Values.ingest.securityContext | default .Values.global.securityContext) }} diff --git a/templates/rotor/_helpers.tpl b/templates/rotor/_helpers.tpl index f3214aa..d094b06 100644 --- a/templates/rotor/_helpers.tpl +++ b/templates/rotor/_helpers.tpl @@ -25,6 +25,47 @@ app.kubernetes.io/component: rotor value: {{ .mongodbUrl | default (include "jitsu.mongodbUrl" $) | quote }} {{- end }} +{{- if or .clickhouseHostFrom $.Values.config.clickhouseHostFrom }} +- name: CLICKHOUSE_HOST + valueFrom: + {{- toYaml (.clickhouseHostFrom | default $.Values.config.clickhouseHostFrom) | nindent 4 }} +{{- else }} +- name: CLICKHOUSE_HOST + value: {{ .clickhouseHost | default (include "jitsu.clickhouseHost" $) | quote }} +{{- end }} + +{{- if or .clickhouseDatabaseFrom $.Values.config.clickhouseDatabaseFrom }} +- name: CLICKHOUSE_DATABASE + valueFrom: + {{- toYaml (.clickhouseDatabaseFrom | default $.Values.config.clickhouseDatabaseFrom) | nindent 4 }} +{{- else }} +- name: CLICKHOUSE_DATABASE + value: {{ .clickhouseDatabase | default (include "jitsu.clickhouseDatabase" $) | quote }} +{{- end }} + +{{- if or .clickhouseUsernameFrom $.Values.config.clickhouseUsernameFrom }} +- name: CLICKHOUSE_USERNAME + valueFrom: + {{- toYaml (.clickhouseUsernameFrom | default $.Values.config.clickhouseUsernameFrom) | nindent 4 }} +{{- else }} +- name: CLICKHOUSE_USERNAME + value: {{ .clickhouseUsername | default (include "jitsu.clickhouseUsername" $) | quote }} +{{- end }} + +{{- if or .clickhousePasswordFrom $.Values.config.clickhousePasswordFrom }} +- name: CLICKHOUSE_PASSWORD + valueFrom: + {{- toYaml (.clickhousePasswordFrom | default $.Values.config.clickhousePasswordFrom) | nindent 4 }} +{{- else }} +- name: CLICKHOUSE_PASSWORD + value: {{ .clickhousePassword | default (include "jitsu.clickhousePassword" $) | quote }} +{{- end }} + +{{- with (.clickhouseSsl | default $.Values.config.clickhouseSsl) }} +- name: CLICKHOUSE_SSL + value: {{ . | quote }} +{{- end }} + {{- if and (not .repositoryBaseUrl) $.Values.console.enabled $.Values.tokenGenerator.enabled }} - name: REPOSITORY_BASE_URL value: {{ printf "http://%s-console:%d/api/admin/export" diff --git a/templates/rotor/deployment.yaml b/templates/rotor/deployment.yaml index 6eaff48..e17a47a 100644 --- a/templates/rotor/deployment.yaml +++ b/templates/rotor/deployment.yaml @@ -46,6 +46,15 @@ spec: image: ghcr.io/groundnuty/k8s-wait-for:v2.0 args: ["job-wr", "{{ include "jitsu.fullname" . }}-token-generator-{{ sha1sum (toJson .Values) | substr 0 8 }}"] {{- end }} + {{- if .Values.clickhouse.enabled }} + - name: wait-for-clickhouse + {{- with (.Values.rotor.securityContext | default .Values.global.securityContext) }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: ghcr.io/groundnuty/k8s-wait-for:v2.0 + args: ["service", "{{ .Release.Name }}-clickhouse"] + {{- end }} {{- if .Values.redis.enabled }} - name: wait-for-redis {{- with (.Values.rotor.securityContext | default .Values.global.securityContext) }} diff --git a/values.yaml b/values.yaml index ac93994..6ea2358 100644 --- a/values.yaml +++ b/values.yaml @@ -30,6 +30,18 @@ config: mongodbUrl: "" mongodbUrlFrom: {} + # Global Clickhouse configuration. + # Will be configured automatically if left empty and using the Clickhouse subchart. + clickhouseHost: "" + clickhouseHostFrom: {} + clickhouseDatabase: "" + clickhouseDatabaseFrom: {} + clickhouseUsername: "" + clickhouseUsernameFrom: {} + clickhousePassword: "" + clickhousePasswordFrom: {} + clickhouseSsl: "" + # Global hash secret used by console, bulker and ingest. # Configured automatically when tokenGenerator.enabled is true. globalHashSecret: "" @@ -194,12 +206,17 @@ console: # https://docs.jitsu.com/self-hosting/configuration#clickhouse_metrics_schema clickhouseMetricsSchema: "" clickhouseMetricsSchemaFrom: {} - clickhouseUrl: "" - clickhouseUrlFrom: {} + + # Leave empty to configure automatically when using Clickhouse subchart. + clickhouseHost: "" + clickhouseHostFrom: {} + clickhouseDatabase: "" + clickhouseDatabaseFrom: {} clickhouseUsername: "" clickhouseUsernameFrom: {} clickhousePassword: "" clickhousePasswordFrom: {} + clickhouseSsl: "" # Log format. One of "text", "json". # https://docs.jitsu.com/self-hosting/configuration#log_format-1 @@ -311,6 +328,17 @@ bulker: kafkaSasl: "" kafkaSaslFrom: {} + # Leave empty to configure automatically when using Clickhouse subchart. + clickhouseHost: "" + clickhouseHostFrom: {} + clickhouseDatabase: "" + clickhouseDatabaseFrom: {} + clickhouseUsername: "" + clickhouseUsernameFrom: {} + clickhousePassword: "" + clickhousePasswordFrom: {} + clickhouseSsl: "" + # https://github.com/jitsucom/bulker/blob/main/.docs/server-config.md#bulker_batch_runner_default_period_sec batchRunnerDefaultPeriodSec: "" @@ -415,6 +443,17 @@ rotor: redisUrl: "" redisUrlFrom: {} + # Leave empty to configure automatically when using Clickhouse subchart. + clickhouseHost: "" + clickhouseHostFrom: {} + clickhouseDatabase: "" + clickhouseDatabaseFrom: {} + clickhouseUsername: "" + clickhouseUsernameFrom: {} + clickhousePassword: "" + clickhousePasswordFrom: {} + clickhouseSsl: "false" + # Leave empty to configure automatically. # https://docs.jitsu.com/self-hosting/configuration#bulker_url bulkerUrl: "" @@ -545,6 +584,17 @@ ingest: kafkaSasl: "" kafkaSaslFrom: {} + # Leave empty to configure automatically when using Clickhouse subchart. + clickhouseHost: "" + clickhouseHostFrom: {} + clickhouseDatabase: "" + clickhouseDatabaseFrom: {} + clickhouseUsername: "" + clickhouseUsernameFrom: {} + clickhousePassword: "" + clickhousePasswordFrom: {} + clickhouseSsl: "false" + # Leave empty to configure automatically. # https://docs.jitsu.com/self-hosting/configuration#ingest_rotor_url rotorUrl: "" @@ -774,3 +824,13 @@ kafka: client: protocol: PLAINTEXT external: PLAINTEXT + +clickhouse: + enabled: true + auth: + username: "jitsu" + password: "jitsu" + keeper: + enabled: true + zookeeper: + enabled: false