Skip to content

Commit

Permalink
refactor: add logging and monitoring configurations (#171)
Browse files Browse the repository at this point in the history 
* refactor: add logging and monitoring configurations

* refactor: add example of standalone
  • Loading branch information
@zyy17
zyy17 authored Oct 17, 2024
1 parent c6ff85f commit a3f1fcb
Show file tree
Hide file tree
Showing 4 changed files with 303 additions and 11 deletions.
2 changes: 1 addition & 1 deletion charts/greptimedb-cluster/Chart.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ apiVersion: v2
name: greptimedb-cluster
description: A Helm chart for deploying GreptimeDB cluster in Kubernetes.
type: application
version: 0.2.16
version: 0.2.17
appVersion: 0.9.4
home: https://github.com/GreptimeTeam/greptimedb
sources:
Expand Down
34 changes: 27 additions & 7 deletions charts/greptimedb-cluster/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

A Helm chart for deploying GreptimeDB cluster in Kubernetes.

![Version: 0.2.16](https://img.shields.io/badge/Version-0.2.16-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.9.4](https://img.shields.io/badge/AppVersion-0.9.4-informational?style=flat-square)
![Version: 0.2.17](https://img.shields.io/badge/Version-0.2.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.9.4](https://img.shields.io/badge/AppVersion-0.9.4-informational?style=flat-square)

## Source Code

Expand Down Expand Up @@ -92,9 +92,10 @@ helm uninstall mycluster -n default
| base.podTemplate.nodeSelector | object | `{}` | The pod node selector |
| base.podTemplate.serviceAccountName | string | `""` | The global service account |
| base.podTemplate.tolerations | list | `[]` | The pod tolerations |
| datanode | object | `{"configData":"","configFile":"","podTemplate":{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","readinessProbe":{},"resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]},"replicas":3,"storage":{"dataHome":"/data/greptimedb","mountPath":"/data/greptimedb","storageClassName":null,"storageRetainPolicy":"Retain","storageSize":"10Gi","walDir":"/data/greptimedb/wal"}}` | Datanode configure |
| datanode | object | `{"configData":"","configFile":"","logging":{},"podTemplate":{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","readinessProbe":{},"resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]},"replicas":1,"storage":{"dataHome":"/data/greptimedb","mountPath":"/data/greptimedb","storageClassName":null,"storageRetainPolicy":"Retain","storageSize":"10Gi","walDir":"/data/greptimedb/wal"}}` | Datanode configure |
| datanode.configData | string | `""` | Extra raw toml config data of datanode. Skip if the `configFile` is used. |
| datanode.configFile | string | `""` | Extra toml file of datanode. |
| datanode.logging | object | `{}` | Logging configuration for datanode, if not set, it will use the global logging configuration. |
| datanode.podTemplate | object | `{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","readinessProbe":{},"resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]}` | The pod template for datanode |
| datanode.podTemplate.affinity | object | `{}` | The pod affinity |
| datanode.podTemplate.annotations | object | `{}` | The annotations to be created to the pod. |
Expand All @@ -113,7 +114,7 @@ helm uninstall mycluster -n default
| datanode.podTemplate.serviceAccount.create | bool | `false` | Create a service account |
| datanode.podTemplate.tolerations | list | `[]` | The pod tolerations |
| datanode.podTemplate.volumes | list | `[]` | The pod volumes |
| datanode.replicas | int | `3` | Datanode replicas |
| datanode.replicas | int | `1` | Datanode replicas |
| datanode.storage.dataHome | string | `"/data/greptimedb"` | The dataHome directory, default is "/data/greptimedb/" |
| datanode.storage.mountPath | string | `"/data/greptimedb"` | The data directory of the storage, default is "/data/greptimedb" |
| datanode.storage.storageClassName | string | `nil` | Storage class for datanode persistent volume |
Expand All @@ -123,10 +124,11 @@ helm uninstall mycluster -n default
| debugPod.enabled | bool | `false` | Enable debug pod |
| debugPod.image | object | `{"registry":"docker.io","repository":"greptime/greptime-tool","tag":"20240905-67eaa147"}` | The debug pod image |
| debugPod.resources | object | `{"limits":{"cpu":"200m","memory":"256Mi"},"requests":{"cpu":"50m","memory":"64Mi"}}` | The debug pod resource |
| flownode | object | `{"configData":"","configFile":"","enabled":false,"podTemplate":{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","readinessProbe":{},"resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]},"replicas":1}` | Flownode configure. **It's NOT READY YET** |
| flownode | object | `{"configData":"","configFile":"","enabled":false,"logging":{},"podTemplate":{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","readinessProbe":{},"resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]},"replicas":1}` | Flownode configure. **It's NOT READY YET** |
| flownode.configData | string | `""` | Extra raw toml config data of flownode. Skip if the `configFile` is used. |
| flownode.configFile | string | `""` | Extra toml file of flownode. |
| flownode.enabled | bool | `false` | Enable flownode |
| flownode.logging | object | `{}` | Logging configuration for flownode, if not set, it will use the global logging configuration. |
| flownode.podTemplate | object | `{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","readinessProbe":{},"resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]}` | The pod template for frontend |
| flownode.podTemplate.affinity | object | `{}` | The pod affinity |
| flownode.podTemplate.annotations | object | `{}` | The annotations to be created to the pod. |
Expand All @@ -146,9 +148,10 @@ helm uninstall mycluster -n default
| flownode.podTemplate.tolerations | list | `[]` | The pod tolerations |
| flownode.podTemplate.volumes | list | `[]` | The pod volumes |
| flownode.replicas | int | `1` | Flownode replicas |
| frontend | object | `{"configData":"","configFile":"","podTemplate":{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","readinessProbe":{},"resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]},"replicas":1,"service":{},"tls":{}}` | Frontend configure |
| frontend | object | `{"configData":"","configFile":"","logging":{},"podTemplate":{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","readinessProbe":{},"resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]},"replicas":1,"service":{},"tls":{}}` | Frontend configure |
| frontend.configData | string | `""` | Extra raw toml config data of frontend. Skip if the `configFile` is used. |
| frontend.configFile | string | `""` | Extra toml file of frontend. |
| frontend.logging | object | `{}` | Logging configuration for frontend, if not set, it will use the global logging configuration. |
| frontend.podTemplate | object | `{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","readinessProbe":{},"resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]}` | The pod template for frontend |
| frontend.podTemplate.affinity | object | `{}` | The pod affinity |
| frontend.podTemplate.annotations | object | `{}` | The annotations to be created to the pod. |
Expand Down Expand Up @@ -178,12 +181,19 @@ helm uninstall mycluster -n default
| image.tag | string | `"v0.9.4"` | The image tag |
| initializer.registry | string | `"docker.io"` | Initializer image registry |
| initializer.repository | string | `"greptime/greptimedb-initializer"` | Initializer image repository |
| initializer.tag | string | `"v0.1.0"` | Initializer image tag |
| meta | object | `{"configData":"","configFile":"","enableRegionFailover":false,"etcdEndpoints":"etcd.etcd-cluster.svc.cluster.local:2379","podTemplate":{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"readinessProbe":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]},"replicas":1,"storeKeyPrefix":""}` | Meta configure |
| initializer.tag | string | `"v0.1.1"` | Initializer image tag |
| logging | object | `{"format":"text","level":"info","logsDir":"/data/greptimedb/logs","onlyLogToStdout":false,"persistentWithData":false}` | Global logging configuration |
| logging.format | string | `"text"` | The log format for greptimedb, only support "json" and "text" |
| logging.level | string | `"info"` | The log level for greptimedb, only support "debug", "info", "warn", "debug" |
| logging.logsDir | string | `"/data/greptimedb/logs"` | The logs directory for greptimedb |
| logging.onlyLogToStdout | bool | `false` | Whether to log to stdout only |
| logging.persistentWithData | bool | `false` | indicates whether to persist the log with the datanode data storage. It **ONLY** works for the datanode component. |
| meta | object | `{"configData":"","configFile":"","enableRegionFailover":false,"etcdEndpoints":"etcd.etcd-cluster.svc.cluster.local:2379","logging":{},"podTemplate":{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"readinessProbe":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]},"replicas":1,"storeKeyPrefix":""}` | Meta configure |
| meta.configData | string | `""` | Extra raw toml config data of meta. Skip if the `configFile` is used. |
| meta.configFile | string | `""` | Extra toml file of meta. |
| meta.enableRegionFailover | bool | `false` | Whether to enable region failover |
| meta.etcdEndpoints | string | `"etcd.etcd-cluster.svc.cluster.local:2379"` | Meta etcd endpoints |
| meta.logging | object | `{}` | Logging configuration for meta, if not set, it will use the global logging configuration. |
| meta.podTemplate | object | `{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"readinessProbe":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]}` | The pod template for meta |
| meta.podTemplate.affinity | object | `{}` | The pod affinity |
| meta.podTemplate.annotations | object | `{}` | The annotations to be created to the pod. |
Expand All @@ -204,6 +214,16 @@ helm uninstall mycluster -n default
| meta.podTemplate.volumes | list | `[]` | The pod volumes |
| meta.replicas | int | `1` | Meta replicas |
| meta.storeKeyPrefix | string | `""` | Meta will store data with this key prefix |
| monitoring | object | `{"enabled":true,"logsCollection":{"pipeline":{"data":""}},"standalone":{},"vector":{"registry":"docker.io","repository":"timberio/vector","resource":{"limits":{"cpu":"50m","memory":"64Mi"},"requests":{"cpu":"50m","memory":"64Mi"}},"tag":"nightly-alpine"}}` | The monitoring bootstrap configuration |
| monitoring.enabled | bool | `true` | Enable monitoring |
| monitoring.logsCollection | object | `{"pipeline":{"data":""}}` | Configure the logs collection |
| monitoring.logsCollection.pipeline | object | `{"data":""}` | The greptimedb pipeline for logs collection |
| monitoring.standalone | object | `{}` | Configure the standalone instance for storing monitoring data |
| monitoring.vector | object | `{"registry":"docker.io","repository":"timberio/vector","resource":{"limits":{"cpu":"50m","memory":"64Mi"},"requests":{"cpu":"50m","memory":"64Mi"}},"tag":"nightly-alpine"}` | Configure vector for logs and metrics collection. |
| monitoring.vector.registry | string | `"docker.io"` | vector image registry |
| monitoring.vector.repository | string | `"timberio/vector"` | vector image repository |
| monitoring.vector.resource | object | `{"limits":{"cpu":"50m","memory":"64Mi"},"requests":{"cpu":"50m","memory":"64Mi"}}` | vector resource |
| monitoring.vector.tag | string | `"nightly-alpine"` | vector image tag |
| mysqlServicePort | int | `4002` | GreptimeDB mysql service port |
| objectStorage | object | `{"gcs":{},"oss":{},"s3":{}}` | Configure to object storage |
| postgresServicePort | int | `4003` | GreptimeDB postgres service port |
Expand Down
111 changes: 110 additions & 1 deletion charts/greptimedb-cluster/templates/cluster.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,24 @@ spec:
secretName: {{ .Release.Name }}-users-auth
{{- end }}
{{- end }}
{{- if .Values.frontend.logging }}
logging:
{{- if .Values.frontend.logging.level }}
level: {{ .Values.frontend.logging.level }}
{{- end }}
{{- if .Values.frontend.logging.format }}
format: {{ .Values.frontend.logging.format }}
{{- end }}
{{- if .Values.frontend.logging.logsDir }}
logsDir: {{ .Values.frontend.logging.logsDir }}
{{- end }}
{{- if .Values.frontend.logging.onlyLogToStdout }}
onlyLogToStdout: {{ .Values.frontend.logging.onlyLogToStdout }}
{{- end }}
{{- if .Values.frontend.logging.persistentWithData }}
persistentWithData: {{ .Values.frontend.logging.persistentWithData }}
{{- end }}
{{- end }}
meta:
replicas: {{ .Values.meta.replicas }}
{{- if or .Values.meta.configFile .Values.meta.configData }}
Expand All @@ -149,6 +167,24 @@ spec:
{{- if .Values.meta.enableRegionFailover }}
enableRegionFailover: {{ .Values.meta.enableRegionFailover }}
{{- end }}
{{- if .Values.meta.logging }}
logging:
{{- if .Values.meta.logging.level }}
level: {{ .Values.meta.logging.level }}
{{- end }}
{{- if .Values.meta.logging.format }}
format: {{ .Values.meta.logging.format }}
{{- end }}
{{- if .Values.meta.logging.logsDir }}
logsDir: {{ .Values.meta.logging.logsDir }}
{{- end }}
{{- if .Values.meta.logging.onlyLogToStdout }}
onlyLogToStdout: {{ .Values.meta.logging.onlyLogToStdout }}
{{- end }}
{{- if .Values.meta.logging.persistentWithData }}
persistentWithData: {{ .Values.meta.logging.persistentWithData }}
{{- end }}
{{- end }}
template:
main:
{{- if .Values.meta.podTemplate.main.image }}
Expand Down Expand Up @@ -247,6 +283,24 @@ spec:
{{- if .Values.datanode.podTemplate.volumes}}
volumes: {{ .Values.datanode.podTemplate.volumes | toYaml | nindent 8 }}
{{- end }}
{{- if .Values.datanode.logging }}
logging:
{{- if .Values.datanode.logging.level }}
level: {{ .Values.datanode.logging.level }}
{{- end }}
{{- if .Values.datanode.logging.format }}
format: {{ .Values.datanode.logging.format }}
{{- end }}
{{- if .Values.datanode.logging.logsDir }}
logsDir: {{ .Values.datanode.logging.logsDir }}
{{- end }}
{{- if .Values.datanode.logging.onlyLogToStdout }}
onlyLogToStdout: {{ .Values.datanode.logging.onlyLogToStdout }}
{{- end }}
{{- if .Values.datanode.logging.persistentWithData }}
persistentWithData: {{ .Values.datanode.logging.persistentWithData }}
{{- end }}
{{- end }}
storage:
dataHome: {{ .Values.datanode.storage.dataHome }}
fs:
Expand Down Expand Up @@ -306,9 +360,27 @@ spec:
{{- if .Values.flownode.podTemplate.nodeSelector }}
nodeSelector: {{ .Values.flownode.podTemplate.nodeSelector | toYaml | nindent 8 }}
{{- end }}
{{- if .Values.flownode.podTemplate.volumes}}
{{- if .Values.flownode.podTemplate.volumes }}
volumes: {{ .Values.flownode.podTemplate.volumes | toYaml | nindent 8 }}
{{- end }}
{{- if .Values.flownode.logging }}
logging:
{{- if .Values.flownode.logging.level }}
level: {{ .Values.flownode.logging.level }}
{{- end }}
{{- if .Values.flownode.logging.format }}
format: {{ .Values.flownode.logging.format }}
{{- end }}
{{- if .Values.flownode.logging.logsDir }}
logsDir: {{ .Values.flownode.logging.logsDir }}
{{- end }}
{{- if .Values.flownode.logging.onlyLogToStdout }}
onlyLogToStdout: {{ .Values.flownode.logging.onlyLogToStdout }}
{{- end }}
{{- if .Values.flownode.logging.persistentWithData }}
persistentWithData: {{ .Values.flownode.logging.persistentWithData }}
{{- end }}
{{- end }}
{{- end }}
{{- if (and .Values.prometheusMonitor (eq .Values.prometheusMonitor.enabled true ))}}
prometheusMonitor: {{- toYaml .Values.prometheusMonitor | nindent 4 }}
Expand Down Expand Up @@ -362,3 +434,40 @@ spec:
kafka: {{- toYaml .Values.remoteWal.kafka | nindent 6 }}
{{- end }}
{{- end }}
{{- if .Values.monitoring.enabled }}
monitoring:
enabled: {{ .Values.monitoring.enabled }}
{{- if .Values.monitoring.standalone }}
standalone: {{ .Values.monitoring.standalone | toYaml | nindent 6 }}
{{- end }}
{{- if .Values.monitoring.logsCollection.pipeline.data }}
logsCollection:
pipeline:
data: {{ .Values.monitoring.logsCollection.pipeline.data }}
{{- end }}
{{- if .Values.monitoring.vector }}
vector:
image: '{{ .Values.monitoring.vector.registry }}/{{ .Values.monitoring.vector.repository }}:{{ .Values.monitoring.vector.tag }}'
{{- if .Values.monitoring.vector.resource }}
resource: {{- toYaml .Values.monitoring.vector.resource | nindent 8 }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.logging}}
logging:
{{- if .Values.logging.level }}
level: {{ .Values.logging.level }}
{{- end }}
{{- if .Values.logging.format }}
format: {{ .Values.logging.format }}
{{- end }}
{{- if .Values.logging.logsDir }}
logsDir: {{ .Values.logging.logsDir }}
{{- end }}
{{- if .Values.logging.onlyLogToStdout }}
onlyLogToStdout: {{ .Values.logging.onlyLogToStdout }}
{{- end }}
{{- if .Values.logging.persistentWithData }}
persistentWithData: {{ .Values.logging.persistentWithData }}
{{- end }}
{{- end }}
Loading

0 comments on commit a3f1fcb

Please sign in to comment.