diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..8cd1beb --- /dev/null +++ b/.editorconfig @@ -0,0 +1,3 @@ +[*] +indent_style=space +indent_size=2 diff --git a/Chart.yaml b/Chart.yaml index 04df389..38b28ec 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -4,7 +4,7 @@ type: application # Chart version is set automatically as part of the release process version: 0.0.0 -appVersion: 2.4.1 +appVersion: 2.4.3 dependencies: - name: postgresql diff --git a/README.md b/README.md index da31f77..5a3fb60 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,148 @@ # Jitsu Helm Chart -:warning: **This chart is under development and may receive breaking changes at any time.** + +## TL;DR +```bash +helm install jitsu oci://registry-1.docker.io/stafftasticcharts/jitsu -f-<=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.console.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.console.ingress.annotations "kubernetes.io/ingress.class" .Values.console.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} kind: Ingress metadata: - name: {{ $fullName }}-console + name: {{ include "jitsu.fullname" . }}-console labels: {{- include "jitsu.labels" . | nindent 4 }} {{- with .Values.console.ingress.annotations }} @@ -23,9 +10,7 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if and .Values.console.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} ingressClassName: {{ .Values.console.ingress.className }} - {{- end }} {{- if .Values.console.ingress.tls }} tls: {{- range .Values.console.ingress.tls }} @@ -43,19 +28,12 @@ spec: paths: {{- range .paths }} - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} pathType: {{ .pathType }} - {{- end }} backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} service: - name: {{ $fullName }}-console + name: {{ include "jitsu.fullname" $ }}-console port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }}-console - servicePort: {{ $svcPort }} - {{- end }} + number: {{ $.Values.console.service.port }} {{- end }} {{- end }} {{- end }} diff --git a/templates/ingest/_helpers.tpl b/templates/ingest/_helpers.tpl index 57058a7..ffb75b4 100644 --- a/templates/ingest/_helpers.tpl +++ b/templates/ingest/_helpers.tpl @@ -4,3 +4,172 @@ Console selector labels {{- define "jitsu.ingest.selectorLabels" -}} app.kubernetes.io/component: ingest {{- end }} + +{{- define "jitsu.ingest.env" -}} +{{- with .Values.ingest.config }} +- name: INGEST_DATA_DOMAIN + value: {{ .dataDomain | default ($.Values.ingress.enabled | ternary $.Values.ingress.host "") | quote }} + +{{- if or .redisUrlFrom $.Values.config.redisUrlFrom }} +- name: INGEST_REDIS_URL + valueFrom: + {{- toYaml (.redisUrlFrom | default $.Values.config.redisUrlFrom) | nindent 4 }} +{{- else }} +- name: INGEST_REDIS_URL + value: {{ .redisUrl | default (include "jitsu.redisUrl" $) | quote }} +{{- end }} + +{{- if .authTokensFrom }} +- name: INGEST_AUTH_TOKENS + valueFrom: + {{- toYaml .authTokensFrom | nindent 4 }} +{{- else }} +{{- if and (not .authTokens) $.Values.tokenGenerator.enabled }} +- name: INGEST_AUTH_TOKENS + valueFrom: + secretKeyRef: + name: {{ include "jitsu.fullname" $ }}-tokens + key: ingestAuthTokens +{{- end }} +{{- with .authTokens }} +- name: INGEST_AUTH_TOKENS + value: {{ . | quote }} +{{- end }} +{{- end }} + +{{- if .tokenSecretFrom }} +- name: INGEST_TOKEN_SECRET + valueFrom: + {{- toYaml .tokenSecretFrom | nindent 4 }} +{{- else }} +{{- if and (not .tokenSecret) $.Values.tokenGenerator.enabled }} +- name: INGEST_TOKEN_SECRET + valueFrom: + secretKeyRef: + name: {{ include "jitsu.fullname" $ }}-tokens + key: ingestTokenSecret +{{- end }} +{{- with .tokenSecret }} +- name: INGEST_TOKEN_SECRET + value: {{ . | quote }} +{{- end }} +{{- end }} + +{{- if .rawAuthTokensFrom }} +- name: INGEST_RAW_AUTH_TOKENS + valueFrom: + {{- toYaml .rawAuthTokensFrom | nindent 4 }} +{{- else }} +{{- with .rawAuthTokens }} +- name: INGEST_RAW_AUTH_TOKENS + value: {{ . | quote }} +{{- end }} +{{- end }} + +{{- if or .globalHashSecretFrom $.Values.config.globalHashSecretFrom }} +- name: INGEST_GLOBAL_HASH_SECRET + valueFrom: + {{- toYaml (.globalHashSecretFrom | default $.Values.config.globalHashSecretFrom) | nindent 4 }} +{{- else }} +{{- if and (not .globalHashSecret) (not $.Values.config.globalHashSecret) $.Values.tokenGenerator.enabled }} +- name: INGEST_GLOBAL_HASH_SECRET + valueFrom: + secretKeyRef: + name: {{ include "jitsu.fullname" $ }}-tokens + key: globalHashSecret +{{- end }} +{{- with (.globalHashSecret | default $.Values.config.globalHashSecret) }} +- name: GLOBAL_HASH_SECRET + value: {{ . | quote }} +{{- end }} +{{- end }} + +{{- if and (not .repositoryUrl) $.Values.console.enabled $.Values.tokenGenerator.enabled }} +- name: INGEST_REPOSITORY_URL + value: {{ printf "http://%s-console:%d/api/admin/export/streams-with-destinations" + (include "jitsu.fullname" $) + (int $.Values.console.service.port) + | quote }} +{{- end }} +{{- with .repositoryUrl }} +- name: INGEST_REPOSITORY_URL + value: {{ . | quote }} +{{- end }} + +{{- if .repositoryAuthTokenFrom }} +- name: INGEST_REPOSITORY_AUTH_TOKEN + valueFrom: + {{- toYaml .repositoryAuthTokenFrom | nindent 4 }} +{{- else }} +{{- if and (not .repositoryAuthToken) $.Values.console.enabled $.Values.tokenGenerator.enabled }} +- name: INGEST_REPOSITORY_AUTH_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "jitsu.fullname" $ }}-tokens + key: consoleAuthToken +{{- end }} +{{- with .repositoryAuthToken }} +- name: INGEST_REPOSITORY_AUTH_TOKEN + value: {{ . | quote }} +{{- end }} +{{- end }} + +{{- with .repositoryRefreshPeriodSec }} +- name: INGEST_REPOSITORY_REFRESH_PERIOD_SEC + value: {{ . | quote }} +{{- end }} + +{{- if and (not .kafkaBootstrapServers) (not $.Values.config.kafkaBootstrapServers) $.Values.kafka.enabled }} +- name: INGEST_KAFKA_BOOTSTRAP_SERVERS + value: "{{ $.Release.Name }}-kafka:9092" +{{- end }} +{{- with (.kafkaBootstrapServers | default $.Values.config.kafkaBootstrapServers) }} +- name: INGEST_KAFKA_BOOTSTRAP_SERVERS + value: {{ . | quote }} +{{- end }} + +{{- with (.kafkaSsl | default $.Values.config.kafkaSsl) }} +- name: INGEST_KAFKA_SSL + value: {{ . | quote }} +{{- end }} + +{{- with (.kafkaSslSkipVerify | default $.Values.config.kafkaSslSkipVerify) }} +- name: INGEST_KAFKA_SSL_SKIP_VERIFY + value: {{ . | quote }} +{{- end }} + +{{- if or .kafkaSaslFrom $.Values.config.kafkaSaslFrom }} +- name: INGEST_KAFKA_SASL + valueFrom: + {{- toYaml (.kafkaSaslFrom | default $.Values.config.kafkaSaslFrom) | nindent 4 }} +{{- else }} +{{- with (.kafkaSasl | default $.Values.config.kafkaSasl) }} +- name: INGEST_KAFKA_SASL + {{- if kindIs "string" . }} + value: {{ . | quote }} + {{- else }} + value: {{ toJson . | quote }} + {{- end }} +{{- end }} +{{- end }} + +{{- if and (not .rotorUrl) (not $.Values.config.rotorUrl) $.Values.rotor.enabled }} +- name: INGEST_ROTOR_URL + value: {{ printf "http://%s-rotor:%d" (include "jitsu.fullname" $) (int $.Values.rotor.service.port) | quote }} +{{- end }} +{{- with (.rotorUrl | default $.Values.config.rotorUrl) }} +- name: INGEST_ROTOR_URL + value: {{ . | quote }} +{{- end }} + +{{- with .eventsLogMaxSize }} +- name: INGEST_EVENTS_LOG_MAX_SIZE + value: {{ . | quote }} +{{- end }} + +{{- with .logFormat }} +- name: INGEST_LOG_FORMAT + value: {{ . | quote }} +{{- end }} +{{- end }} +{{- end }} diff --git a/templates/ingest/deployment.yaml b/templates/ingest/deployment.yaml index d43ad73..0077a08 100644 --- a/templates/ingest/deployment.yaml +++ b/templates/ingest/deployment.yaml @@ -34,6 +34,32 @@ spec: serviceAccountName: {{ include "jitsu.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + initContainers: + {{- if .Values.tokenGenerator.enabled }} + - name: wait-for-tokens + image: ghcr.io/groundnuty/k8s-wait-for:v2.0 + args: ["job-wr", "{{ include "jitsu.fullname" . }}-token-generator-{{ sha1sum (toJson .Values) | substr 0 8 }}"] + {{- end }} + {{- if .Values.redis.enabled }} + - name: wait-for-redis + image: ghcr.io/groundnuty/k8s-wait-for:v2.0 + args: ["service", "{{ .Release.Name }}-redis-master"] + {{- end }} + {{- if .Values.kafka.enabled }} + - name: wait-for-kafka + image: ghcr.io/groundnuty/k8s-wait-for:v2.0 + args: ["service", "{{ .Release.Name }}-kafka"] + {{- end }} + {{- if .Values.console.enabled }} + - name: wait-for-console + image: ghcr.io/groundnuty/k8s-wait-for:v2.0 + args: ["service", "{{ include "jitsu.fullname" . }}-console"] + {{- end }} + {{- if .Values.rotor.enabled }} + - name: wait-for-rotor + image: ghcr.io/groundnuty/k8s-wait-for:v2.0 + args: ["service", "{{ include "jitsu.fullname" . }}-rotor"] + {{- end }} containers: - name: {{ .Chart.Name }} securityContext: @@ -54,11 +80,16 @@ spec: port: http path: /health {{- end }} + {{- if .Values.ingest.envFrom }} + envFrom: + {{- toYaml .Values.ingest.envFrom | nindent 12 }} + {{- end }} env: - {{- $env := merge (deepCopy .Values.ingest.environment) .Values.jitsu.environment -}} - {{- range $k, $v := $env }} - - name: {{ $k | quote }} - {{- toYaml $v | nindent 16 }} + {{- if and .Values.config.enabled .Values.ingest.config.enabled }} + {{- include "jitsu.ingest.env" . | nindent 12 }} + {{- end }} + {{- with .Values.ingest.env }} + {{- toYaml . | nindent 12 }} {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} diff --git a/templates/ingest/ingress.yaml b/templates/ingest/ingress.yaml index 70cfd68..02d095e 100644 --- a/templates/ingest/ingress.yaml +++ b/templates/ingest/ingress.yaml @@ -1,21 +1,8 @@ {{- if .Values.ingest.ingress.enabled -}} -{{- $fullName := include "jitsu.fullname" . -}} -{{- $svcPort := .Values.ingest.service.port -}} -{{- if and .Values.ingest.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingest.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingest.ingress.annotations "kubernetes.io/ingress.class" .Values.ingest.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} kind: Ingress metadata: - name: {{ $fullName }}-ingest + name: {{ include "jitsu.fullname" . }}-ingest labels: {{- include "jitsu.labels" . | nindent 4 }} {{- with .Values.ingest.ingress.annotations }} @@ -23,9 +10,7 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if and .Values.ingest.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} ingressClassName: {{ .Values.ingest.ingress.className }} - {{- end }} {{- if .Values.ingest.ingress.tls }} tls: {{- range .Values.ingest.ingress.tls }} @@ -43,19 +28,12 @@ spec: paths: {{- range .paths }} - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} pathType: {{ .pathType }} - {{- end }} backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} service: - name: {{ $fullName }}-ingest + name: {{ include "jitsu.fullname" $ }}-ingest port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }}-ingest - servicePort: {{ $svcPort }} - {{- end }} + number: {{ $.Values.ingest.service.port }} {{- end }} {{- end }} {{- end }} diff --git a/templates/ingress.yaml b/templates/ingress.yaml new file mode 100644 index 0000000..509a8b9 --- /dev/null +++ b/templates/ingress.yaml @@ -0,0 +1,49 @@ +{{- if and .Values.ingress.enabled (or .Values.console.enabled .Values.ingest.enabled) -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "jitsu.fullname" . }} + labels: + {{- include "jitsu.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingressClassName: {{ .Values.ingress.className }} + {{- if .Values.ingress.tls }} + tls: + - hosts: + - {{ .Values.ingress.host | quote }} + secretName: {{ include "jitsu.fullname" . }}-tls + {{- end }} + rules: + - host: {{ .Values.ingress.host | quote }} + http: + paths: + {{- if .Values.console.enabled }} + - path: / + pathType: Prefix + backend: + service: + name: {{ include "jitsu.fullname" . }}-console + port: + number: {{ .Values.console.service.port }} + {{- end }} + {{- if .Values.ingest.enabled }} + - path: /api/s/ + pathType: Prefix + backend: + service: + name: {{ include "jitsu.fullname" . }}-ingest + port: + number: {{ .Values.ingest.service.port }} + - path: /v1/batch + pathType: Prefix + backend: + service: + name: {{ include "jitsu.fullname" . }}-ingest + port: + number: {{ .Values.ingest.service.port }} + {{- end }} +{{- end }} diff --git a/templates/migration/job.yaml b/templates/migration/job.yaml index 15f0157..f4379ee 100644 --- a/templates/migration/job.yaml +++ b/templates/migration/job.yaml @@ -22,16 +22,39 @@ spec: {{- end }} restartPolicy: "Never" serviceAccountName: {{ include "jitsu.serviceAccountName" . }} + initContainers: + {{- if .Values.tokenGenerator.enabled }} + - name: wait-for-tokens + image: ghcr.io/groundnuty/k8s-wait-for:v2.0 + args: ["job-wr", "{{ include "jitsu.fullname" . }}-token-generator-{{ sha1sum (toJson .Values) | substr 0 8 }}"] + {{- end }} + {{- if .Values.postgresql.enabled }} + - name: wait-for-postgresql + image: ghcr.io/groundnuty/k8s-wait-for:v2.0 + args: ["service", "{{ .Release.Name }}-postgresql"] + {{- end }} containers: - - name: {{ .Chart.Name }} - image: "{{ .Values.migration.image.repository }}:{{ .Values.migration.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.migration.image.pullPolicy }} - command: [prisma] - args: [db, push, --skip-generate, --schema, /app/schema.prisma] - env: - {{- $env := merge (deepCopy .Values.migration.environment) .Values.jitsu.environment -}} - {{- range $k, $v := $env }} - - name: {{ $k | quote }} - {{- toYaml $v | nindent 14 }} + - name: {{ .Chart.Name }} + image: "{{ .Values.migration.image.repository }}:{{ .Values.migration.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.migration.image.pullPolicy }} + command: [prisma] + args: [db, push, --skip-generate, --schema, /app/schema.prisma] + {{- if .Values.migration.envFrom }} + envFrom: + {{- toYaml .Values.migration.envFrom | nindent 12 }} {{- end }} + env: + {{- if and .Values.config.enabled .Values.migration.config.enabled }} + {{- if or .databaseUrlFrom $.Values.config.databaseUrlFrom }} + - name: DATABASE_URL + valueFrom: + {{- toYaml (.databaseUrlFrom | default $.Values.config.databaseUrlFrom) | nindent 4 }} + {{- else }} + - name: DATABASE_URL + value: {{ .Values.migration.config.databaseUrl | default (include "jitsu.databaseUrl" .) | quote }} + {{- end }} + {{- end }} + {{- with .Values.migration.env }} + {{- toYaml . | nindent 12 }} + {{- end }} {{- end }} diff --git a/templates/role.yaml b/templates/role.yaml new file mode 100644 index 0000000..33c7471 --- /dev/null +++ b/templates/role.yaml @@ -0,0 +1,18 @@ +{{- if .Values.serviceAccount.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "jitsu.fullname" . }}-k8s-wait-for + labels: + {{- include "jitsu.labels" . | nindent 4 }} +rules: + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +{{- end }} diff --git a/templates/rolebinding.yaml b/templates/rolebinding.yaml new file mode 100644 index 0000000..f446314 --- /dev/null +++ b/templates/rolebinding.yaml @@ -0,0 +1,23 @@ +{{- if or .Values.serviceAccount.rbac.create .Values.syncctl.serviceAccount.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "jitsu.fullname" . }}-k8s-wait-for + labels: + {{- include "jitsu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "jitsu.fullname" . }}-k8s-wait-for +subjects: + {{- if .Values.serviceAccount.rbac.create }} + - kind: ServiceAccount + name: {{ include "jitsu.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + {{- end }} + {{- if .Values.syncctl.serviceAccount.rbac.create }} + - kind: ServiceAccount + name: {{ include "jitsu.syncctl.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + {{- end }} +{{- end }} diff --git a/templates/rotor/_helpers.tpl b/templates/rotor/_helpers.tpl index f1c10c5..cc773db 100644 --- a/templates/rotor/_helpers.tpl +++ b/templates/rotor/_helpers.tpl @@ -4,3 +4,154 @@ Rotor selector labels {{- define "jitsu.rotor.selectorLabels" -}} app.kubernetes.io/component: rotor {{- end }} + +{{- define "jitsu.rotor.env" -}} +{{- with .Values.rotor.config }} +{{- if or .redisUrlFrom $.Values.config.redisUrlFrom }} +- name: REDIS_URL + valueFrom: + {{- toYaml (.redisUrlFrom | default $.Values.config.redisUrlFrom) | nindent 4 }} +{{- else }} +- name: REDIS_URL + value: {{ .redisUrl | default (include "jitsu.redisUrl" $) | quote }} +{{- end }} + +{{- if or .mongodbUrlFrom $.Values.config.mongodbUrlFrom }} +- name: MONGODB_URL + valueFrom: + {{- toYaml (.mongodbUrlFrom | default $.Values.config.mongodbUrlFrom) | nindent 4 }} +{{- else }} +- name: MONGODB_URL + value: {{ .mongodbUrl | default (include "jitsu.mongodbUrl" $) | quote }} +{{- end }} + +{{- if and (not .repositoryBaseUrl) $.Values.console.enabled $.Values.tokenGenerator.enabled }} +- name: REPOSITORY_BASE_URL + value: {{ printf "http://%s-console:%d/api/admin/export" + (include "jitsu.fullname" $) + (int $.Values.console.service.port) + | quote }} +{{- end }} +{{- with .repositoryBaseUrl }} +- name: REPOSITORY_BASE_URL + value: {{ . | quote }} +{{- end }} + +{{- if .repositoryAuthTokenFrom }} +- name: REPOSITORY_AUTH_TOKEN + valueFrom: + {{- toYaml .repositoryAuthTokenFrom | nindent 4 }} +{{- else }} +{{- if and (not .repositoryAuthToken) $.Values.console.enabled $.Values.tokenGenerator.enabled }} +- name: REPOSITORY_AUTH_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "jitsu.fullname" $ }}-tokens + key: consoleAuthToken +{{- end }} +{{- with .repositoryAuthToken }} +- name: REPOSITORY_AUTH_TOKEN + value: {{ . | quote }} +{{- end }} +{{- end }} + +{{- with .repositoryRefreshPeriodSec }} +- name: REPOSITORY_REFRESH_PERIOD_SEC + value: {{ . | quote }} +{{- end }} + +{{- if and (not .bulkerUrl) (not $.Values.config.bulkerUrl) $.Values.bulker.enabled }} +- name: BULKER_URL + value: {{ printf "http://%s-bulker:%d" + (include "jitsu.fullname" $) + (int $.Values.bulker.service.port) + | quote }} +{{- end }} +{{- with (.bulkerUrl | default $.Values.config.bulkerUrl) }} +- name: BULKER_URL + value: {{ . | quote }} +{{- end }} + +{{- if .bulkerAuthKeyFrom }} +- name: BULKER_AUTH_KEY + valueFrom: + {{- toYaml .bulkerAuthKeyFrom | nindent 4 }} +{{- else }} +{{- if and (not .bulkerAuthKey) $.Values.bulker.enabled $.Values.tokenGenerator.enabled }} +- name: BULKER_AUTH_KEY + valueFrom: + secretKeyRef: + name: {{ include "jitsu.fullname" $ }}-tokens + key: bulkerAuthToken +{{- end }} +{{- with .bulkerAuthKey }} +- name: BULKER_AUTH_KEY + value: {{ . | quote }} +{{- end }} +{{- end }} + +{{- if and (not .kafkaBootstrapServers) (not $.Values.config.kafkaBootstrapServers) $.Values.kafka.enabled }} +- name: KAFKA_BOOTSTRAP_SERVERS + value: {{ printf "%s-kafka:9092" $.Release.Name | quote }} +{{- end }} +{{- with (.kafkaBootstrapServers | default $.Values.config.kafkaBootstrapServers) }} +- name: KAFKA_BOOTSTRAP_SERVERS + value: {{ . | quote }} +{{- end }} + +{{- with (.kafkaSsl | default $.Values.config.kafkaSsl) }} +- name: KAFKA_SSL + value: {{ . | quote }} +{{- end }} + +{{- if or .kafkaSaslFrom $.Values.config.kafkaSaslFrom }} +- name: KAFKA_SASL + valueFrom: + {{- toYaml (.kafkaSaslFrom | default $.Values.config.kafkaSaslFrom) | nindent 4 }} +{{- else }} +{{- with (.kafkaSasl | default $.Values.config.kafkaSasl) }} +- name: KAFKA_SASL + {{- if kindIs "string" . }} + value: {{ . | quote }} + {{- else }} + value: {{ toJson . | quote }} + {{- end }} +{{- end }} +{{- end }} + +{{- with .metricsDestinationId }} +- name: METRICS_DESTINATION_ID + value: {{ . | quote }} +{{- end }} + +{{- with .eventsLogMaxSize }} +- name: EVENTS_LOG_MAX_SIZE + value: {{ . | quote }} +{{- end }} + +{{- with .concurrency }} +- name: CONCURRENCY + value: {{ . | quote }} +{{- end }} + +{{- with .messagesRetryCount }} +- name: MESSAGES_RETRY_COUNT + value: {{ . | quote }} +{{- end }} + +{{- with .messagesRetryBackoffBase }} +- name: MESSAGES_RETRY_BACKOFF_BASE + value: {{ . | quote }} +{{- end }} + +{{- with .messagesRetryBackoffMaxDelay }} +- name: MESSAGES_RETRY_BACKOFF_MAX_DELAY + value: {{ . | quote }} +{{- end }} + +{{- with .logFormat }} +- name: LOG_FORMAT + value: {{ . | quote }} +{{- end }} +{{- end }} +{{- end }} diff --git a/templates/rotor/deployment.yaml b/templates/rotor/deployment.yaml index 3b6b189..f837f28 100644 --- a/templates/rotor/deployment.yaml +++ b/templates/rotor/deployment.yaml @@ -34,6 +34,37 @@ spec: serviceAccountName: {{ include "jitsu.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + initContainers: + {{- if .Values.tokenGenerator.enabled }} + - name: wait-for-tokens + image: ghcr.io/groundnuty/k8s-wait-for:v2.0 + args: ["job-wr", "{{ include "jitsu.fullname" . }}-token-generator-{{ sha1sum (toJson .Values) | substr 0 8 }}"] + {{- end }} + {{- if .Values.redis.enabled }} + - name: wait-for-redis + image: ghcr.io/groundnuty/k8s-wait-for:v2.0 + args: ["service", "{{ include "jitsu.fullname" . }}-redis-master"] + {{- end }} + {{- if .Values.kafka.enabled }} + - name: wait-for-kafka + image: ghcr.io/groundnuty/k8s-wait-for:v2.0 + args: ["service", "{{ include "jitsu.fullname" . }}-kafka"] + {{- end }} + {{- if .Values.mongodb.enabled }} + - name: wait-for-mongodb + image: ghcr.io/groundnuty/k8s-wait-for:v2.0 + args: ["service", "{{ .Release.Name }}-mongodb"] + {{- end }} + {{- if .Values.console.enabled }} + - name: wait-for-console + image: ghcr.io/groundnuty/k8s-wait-for:v2.0 + args: ["service", "{{ include "jitsu.fullname" . }}-console"] + {{- end }} + {{- if .Values.bulker.enabled }} + - name: wait-for-bulker + image: ghcr.io/groundnuty/k8s-wait-for:v2.0 + args: ["service", "{{ include "jitsu.fullname" . }}-bulker"] + {{- end }} containers: - name: {{ .Chart.Name }} securityContext: @@ -54,11 +85,16 @@ spec: port: http path: /health {{- end }} + {{- if .Values.rotor.envFrom }} + envFrom: + {{- toYaml .Values.rotor.envFrom | nindent 12 }} + {{- end }} env: - {{- $env := merge (deepCopy .Values.rotor.environment) .Values.jitsu.environment -}} - {{- range $k, $v := $env }} - - name: {{ $k | quote }} - {{- toYaml $v | nindent 16 }} + {{- if and .Values.config.enabled .Values.rotor.config.enabled }} + {{- include "jitsu.rotor.env" . | nindent 12 }} + {{- end }} + {{- with .Values.rotor.env }} + {{- toYaml .Values.rotor.env | nindent 12 }} {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} diff --git a/templates/syncctl/_helpers.tpl b/templates/syncctl/_helpers.tpl index 65ee7a6..47b2f40 100644 --- a/templates/syncctl/_helpers.tpl +++ b/templates/syncctl/_helpers.tpl @@ -4,3 +4,160 @@ Rotor selector labels {{- define "jitsu.syncctl.selectorLabels" -}} app.kubernetes.io/component: syncctl {{- end }} + +{{- define "jitsu.syncctl.serviceAccountName" -}} +{{- if .Values.syncctl.serviceAccount.create }} +{{- default (printf "%s-syncctl" (include "jitsu.fullname" .)) .Values.syncctl.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.syncctl.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "jitsu.syncctl.env" -}} +{{- with .Values.syncctl.config -}} +{{- if or .databaseUrlFrom $.Values.config.databaseUrlFrom }} +- name: SYNCCTL_DATABASE_URL + valueFrom: + {{- toYaml (.databaseUrlFrom | default $.Values.config.databaseUrlFrom) | nindent 4 }} +{{- else }} +- name: SYNCCTL_DATABASE_URL + value: {{ .databaseUrl | default (include "jitsu.databaseUrl" $ | replace "schema=" "search_path=") | quote }} +{{- end }} + +{{- if .authTokensFrom }} +- name: SYNCCTL_AUTH_TOKENS + valueFrom: + {{- toYaml .authTokensFrom | nindent 4 }} +{{- else }} +{{- if and (not .authTokens) $.Values.tokenGenerator.enabled }} +- name: SYNCCTL_AUTH_TOKENS + valueFrom: + secretKeyRef: + name: {{ include "jitsu.fullname" $ }}-tokens + key: syncctlAuthTokens +{{- end }} +{{- with .authTokens }} +- name: SYNCCTL_AUTH_TOKENS + value: {{ . | quote }} +{{- end }} +{{- end }} + +{{- if .tokenSecretFrom }} +- name: SYNCCTL_TOKEN_SECRET + valueFrom: + {{- toYaml .tokenSecretFrom | nindent 4 }} +{{- else }} +{{- if and (not .tokenSecret) $.Values.tokenGenerator.enabled }} +- name: SYNCCTL_TOKEN_SECRET + valueFrom: + secretKeyRef: + name: {{ include "jitsu.fullname" $ }}-tokens + key: syncctlTokenSecret +{{- end }} +{{- with .tokenSecret }} +- name: SYNCCTL_TOKEN_SECRET + value: {{ . | quote }} +{{- end }} +{{- end }} + +{{- if .rawAuthTokensFrom }} +- name: SYNCCTL_RAW_AUTH_TOKENS + valueFrom: + {{- toYaml .rawAuthTokensFrom | nindent 4 }} +{{- else }} +{{- with .rawAuthTokens }} +- name: SYNCCTL_RAW_AUTH_TOKENS + value: {{ . | quote }} +{{- end }} +{{- end }} + +{{- if .sidecarDatabaseUrlFrom }} +- name: SYNCCTL_SIDECAR_DATABASE_URL + valueFrom: + {{- toYaml .sidecarDatabaseUrlFrom | nindent 4 }} +{{- else }} +{{- with .sidecarDatabaseUrl }} +- name: SYNCCTL_SIDECAR_DATABASE_URL + value: {{ . | quote }} +{{- end }} +{{- end }} + +{{- if and (not .bulkerUrl) (not $.Values.config.bulkerUrl) $.Values.bulker.enabled }} +- name: SYNCCTL_BULKER_URL + value: {{ printf "http://%s-bulker:%d" (include "jitsu.fullname" $) (int $.Values.bulker.service.port) | quote }} +{{- end }} +{{- with (.bulkerUrl | default $.Values.config.bulkerUrl) }} +- name: SYNCCTL_BULKER_URL + value: {{ . | quote }} +{{- end }} + +{{- if .bulkerAuthTokenFrom }} +- name: SYNCCTL_BULKER_AUTH_TOKEN + valueFrom: + {{- toYaml .bulkerAuthTokenFrom | nindent 4 }} +{{- else }} +{{- if and (not .bulkerAuthToken ) $.Values.bulker.enabled $.Values.tokenGenerator.enabled }} +- name: SYNCCTL_BULKER_AUTH_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "jitsu.fullname" $ }}-tokens + key: bulkerAuthToken +{{- end }} +{{- with .bulkerAuthToken }} +- name: SYNCCTL_BULKER_AUTH_TOKEN + value: {{ . | quote }} +{{- end }} +{{- end }} + +{{- if .kubernetesClientConfigFrom }} +- name: SYNCCTL_KUBERNETES_CLIENT_CONFIG + valueFrom: + {{- toYaml .kubernetesClientConfigFrom | nindent 4 }} +{{- else }} +{{- with .kubernetesClientConfig }} +- name: SYNCCTL_KUBERNETES_CLIENT_CONFIG + {{- if kindIs "string" . }} + value: {{ . | quote }} + {{- else }} + value: {{ toYaml . | quote }} + {{- end }} +{{- end }} +{{- end }} + +{{- if .kubernetesContextFrom }} +- name: SYNCCTL_KUBERNETES_CONTEXT + valueFrom: + {{- toYaml .kubernetesContextFrom | nindent 4 }} +{{- else }} +{{- with .kubernetesContext }} +- name: SYNCCTL_KUBERNETES_CONTEXT + value: {{ . | quote }} +{{- end }} +{{- end }} + +{{- if .kubernetesNamespaceFrom }} +- name: SYNCCTL_KUBERNETES_NAMESPACE + valueFrom: + {{- toYaml .kubernetesNamespaceFrom | nindent 4 }} +{{- else }} +{{- if not .kubernetesNamespace }} +- name: SYNCCTL_KUBERNETES_NAMESPACE + value: "{{ $.Release.Namespace }}" +{{- end }} +{{- with .kubernetesNamespace }} +- name: SYNCCTL_KUBERNETES_NAMESPACE + value: {{ . | quote }} +{{- end }} +{{- end }} + +{{- with .taskTimeoutHours }} +- name: SYNCCTL_TASK_TIMEOUT_HOURS + value: {{ . | quote }} +{{- end }} + +{{- with .logFormat }} +- name: SYNCCTL_LOG_FORMAT + value: {{ . | quote }} +{{- end }} +{{- end }} +{{- end }} diff --git a/templates/syncctl/deployment.yaml b/templates/syncctl/deployment.yaml index 7e265ec..29929d0 100644 --- a/templates/syncctl/deployment.yaml +++ b/templates/syncctl/deployment.yaml @@ -31,9 +31,20 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ include "jitsu.serviceAccountName" . }} + serviceAccountName: {{ include "jitsu.syncctl.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + initContainers: + {{- if .Values.tokenGenerator.enabled }} + - name: wait-for-tokens + image: ghcr.io/groundnuty/k8s-wait-for:v2.0 + args: ["job-wr", "{{ include "jitsu.fullname" . }}-token-generator-{{ sha1sum (toJson .Values) | substr 0 8 }}"] + {{- end }} + {{- if .Values.migration.enabled }} + - name: wait-for-migration + image: ghcr.io/groundnuty/k8s-wait-for:v2.0 + args: ["job-wr", "{{ include "jitsu.fullname" . }}-migration-{{ sha1sum (toJson .Values) | substr 0 8 }}"] + {{- end }} containers: - name: {{ .Chart.Name }} securityContext: @@ -54,11 +65,16 @@ spec: port: http path: /health {{- end }} + {{- if .Values.syncctl.envFrom }} + envFrom: + {{- toYaml .Values.syncctl.envFrom | nindent 12 }} + {{- end }} env: - {{- $env := merge (deepCopy .Values.syncctl.environment) .Values.jitsu.environment -}} - {{- range $k, $v := $env }} - - name: {{ $k | quote }} - {{- toYaml $v | nindent 16 }} + {{- if and .Values.config.enabled .Values.syncctl.config.enabled }} + {{- include "jitsu.syncctl.env" . | nindent 12 }} + {{- end }} + {{- with .Values.syncctl.env }} + {{- toYaml . | nindent 12 }} {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} diff --git a/templates/syncctl/namespace.yaml b/templates/syncctl/namespace.yaml new file mode 100644 index 0000000..e84bc6a --- /dev/null +++ b/templates/syncctl/namespace.yaml @@ -0,0 +1,11 @@ +{{- if and + .Values.syncctl.enabled + .Values.syncctl.createNamespace + (not (eq (.Values.syncctl.config.kubernetesNamespace | default .Release.Namespace) .Release.Namespace)) + (not .Values.syncctl.config.kubernetesClientConfig) +}} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.syncctl.config.kubernetesNamespace }} +{{- end }} diff --git a/templates/syncctl/proxy-services.yaml b/templates/syncctl/proxy-services.yaml new file mode 100644 index 0000000..5be9453 --- /dev/null +++ b/templates/syncctl/proxy-services.yaml @@ -0,0 +1,20 @@ +{{- if and + .Values.syncctl.enabled + (not (eq (.Values.syncctl.config.kubernetesNamespace | default .Release.Namespace) .Release.Namespace)) + (not .Values.syncctl.config.kubernetesClientConfig) +}} +{{- range list + (printf "%s-postgresql" .Release.Name) + (printf "%s-bulker" (include "jitsu.fullname" .)) +}} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ . | quote }} + namespace: {{ $.Values.syncctl.config.kubernetesNamespace }} +spec: + type: ExternalName + externalName: {{ printf "%s.%s.svc.%s" . $.Release.Namespace ($.Values.syncctl.clusterDomain | default "cluster.local") | quote }} +{{- end }} +{{- end }} diff --git a/templates/syncctl/role.yaml b/templates/syncctl/role.yaml new file mode 100644 index 0000000..709ad47 --- /dev/null +++ b/templates/syncctl/role.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.syncctl.enabled .Values.syncctl.serviceAccount.rbac.create (not .Values.syncctl.config.kubernetesClientConfig) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "jitsu.fullname" . }}-syncctl + labels: + {{- include "jitsu.labels" . | nindent 4 }} + namespace: {{ .Values.syncctl.config.kubernetesNamespace | default .Release.Namespace }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create","delete"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["delete"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get","list","watch","create","delete"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] +{{- end }} diff --git a/templates/syncctl/rolebinding.yaml b/templates/syncctl/rolebinding.yaml new file mode 100644 index 0000000..bb865e6 --- /dev/null +++ b/templates/syncctl/rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.syncctl.enabled .Values.syncctl.serviceAccount.rbac.create (not .Values.syncctl.config.kubernetesClientConfig) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "jitsu.fullname" . }}-syncctl + labels: + {{- include "jitsu.labels" . | nindent 4 }} + namespace: {{ .Values.syncctl.config.kubernetesNamespace | default .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "jitsu.fullname" . }}-syncctl +subjects: + - kind: ServiceAccount + name: {{ include "jitsu.syncctl.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/templates/syncctl/serviceaccount.yaml b/templates/syncctl/serviceaccount.yaml new file mode 100644 index 0000000..da186ef --- /dev/null +++ b/templates/syncctl/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.syncctl.serviceAccount.create .Values.syncctl.serviceAccount.rbac.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "jitsu.syncctl.serviceAccountName" . }} + labels: + {{- include "jitsu.labels" . | nindent 4 }} + {{- with .Values.syncctl.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/templates/token-generator/_helpers.tpl b/templates/token-generator/_helpers.tpl new file mode 100644 index 0000000..e8bfa40 --- /dev/null +++ b/templates/token-generator/_helpers.tpl @@ -0,0 +1,7 @@ +{{- define "jitsu.tokenGenerator.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (printf "%s-token-generator" (include "jitsu.fullname" .)) .Values.tokenGenerator.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.tokenGenerator.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/templates/token-generator/job.yaml b/templates/token-generator/job.yaml new file mode 100644 index 0000000..2bf6a0a --- /dev/null +++ b/templates/token-generator/job.yaml @@ -0,0 +1,94 @@ +{{- if .Values.tokenGenerator.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "jitsu.fullname" . }}-token-generator-{{ sha1sum (toJson .Values) | substr 0 8 }} + labels: + {{- include "jitsu.labels" . | nindent 4 }} +spec: + backoffLimit: 1 + template: + metadata: + labels: + {{- include "jitsu.selectorLabels" . | nindent 8 }} + spec: + restartPolicy: Never + serviceAccountName: {{ include "jitsu.tokenGenerator.serviceAccountName" . }} + containers: + - name: token-generator + image: "alpine/k8s:{{ .Capabilities.KubeVersion.Version | trimPrefix "v"}}" + env: + - name: FULLNAME + value: {{ include "jitsu.fullname" . }} + - name: SECRET_LENGTH + value: {{ .Values.tokenGenerator.secretLength | default "32" | quote }} + - name: TOKEN_LENGTH + value: {{ .Values.tokenGenerator.tokenLength | default "32" | quote }} + - name: SALT_LENGTH + value: {{ .Values.tokenGenerator.saltLength | default "32" | quote }} + command: ["sh", "-c"] + args: + - | + if kubectl get secrets "$FULLNAME-tokens" > /dev/null 2>&1; then + echo "Secret $FULLNAME-tokens already exists, skipping generation" + exit 0 + fi + set -e + random_string() { + length="$1" + cat /dev/urandom | tr -dc 'a-zA-Z0-9_-' | head -c "$length" + } + hex_hash_token() { + token="$1" + salt="$2" + secret="$3" + echo -n "$token$salt$secret" | sha512sum | awk '{print $1}' | tr -d '\n' + } + base64_hash_token() { + hex_hash_token "$@" | xxd -r -p | base64 -w 0 | tr -d '=' + } + globalHashSecret=$(random_string $SECRET_LENGTH) + ingestTokenSecret=$(random_string $SECRET_LENGTH) + bulkerTokenSecret=$(random_string $SECRET_LENGTH) + syncctlTokenSecret=$(random_string $SECRET_LENGTH) + + consoleAuthToken=$(random_string $TOKEN_LENGTH) + ingestAuthToken=$(random_string $TOKEN_LENGTH) + bulkerAuthToken=$(random_string $TOKEN_LENGTH) + syncctlAuthToken=$(random_string $TOKEN_LENGTH) + + consoleAuthTokenSalt=$(random_string $SALT_LENGTH) + ingestAuthTokenSalt=$(random_string $SALT_LENGTH) + bulkerAuthTokenSalt=$(random_string $SALT_LENGTH) + syncctlAuthTokenSalt=$(random_string $SALT_LENGTH) + + ingestAuthTokens="$ingestAuthTokenSalt.$(base64_hash_token $ingestAuthToken $ingestAuthTokenSalt $ingestTokenSecret)" + bulkerAuthTokens="$bulkerAuthTokenSalt.$(base64_hash_token $bulkerAuthToken $bulkerAuthTokenSalt $bulkerTokenSecret)" + syncctlAuthTokens="$syncctlAuthTokenSalt.$(base64_hash_token $syncctlAuthToken $syncctlAuthTokenSalt $syncctlTokenSecret)" + + consoleAuthTokens="$consoleAuthTokenSalt.$(hex_hash_token $consoleAuthToken $consoleAuthTokenSalt $globalHashSecret)" + + kubectl apply -f- <