refactor(greptimedb-standalone): refine object storage configuration (#…

…111)

* refactor: object storage configure

* chore: revert greptimedb-cluster change

* chore: revert greptimedb-cluster change

charts/greptimedb-standalone/Chart.yaml

@@ -2,5 +2,18 @@ apiVersion: v2
 name: greptimedb-standalone
 description: A Helm chart for deploying standalone greptimedb
 type: application
-version: 0.1.9
+version: 0.1.10
 appVersion: 0.6.0
+home: https://github.com/GreptimeTeam/greptimedb
+sources:
+  - https://github.com/GreptimeTeam/greptimedb
+keywords:
+  - database
+  - greptimedb
+maintainers:
+  - name: daviderli614
+    email: [email protected]
+    url: https://github.com/daviderli614
+  - name: zyy17
+    email: [email protected]
+    url: https://github.com/zyy17

charts/greptimedb-standalone/README.md

@@ -2,7 +2,7 @@
 
 A Helm chart for deploying standalone greptimedb
 
-![Version: 0.1.9](https://img.shields.io/badge/Version-0.1.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.0](https://img.shields.io/badge/AppVersion-0.6.0-informational?style=flat-square)
+![Version: 0.1.10](https://img.shields.io/badge/Version-0.1.10-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.0](https://img.shields.io/badge/AppVersion-0.6.0-informational?style=flat-square)
 
 ## Source Code
 - https://github.com/GreptimeTeam/greptimedb
@@ -18,6 +18,18 @@ helm repo update
 helm upgrade --install greptimedb-standalone greptime/greptimedb-standalone -n default
 ```
 
+**Use AWS S3 as backend storage**
+```console
+helm upgrade --install greptimedb-standalone greptime/greptimedb-standalone \
+  --set objectStorage.credentials.accessKeyId="your-access-key-id" \
+  --set objectStorage.credentials.secretAccessKey="your-secret-access-key" \
+  --set objectStorage.s3.bucket="your-bucket-name" \
+  --set objectStorage.s3.region="region-of-bucket" \
+  --set objectStorage.s3.endpoint="s3-endpoint" \
+  --set objectStorage.s3.root="root-directory-of-data" \
+  -n default
+```
+
 ## Connection
 
 ```console
@@ -45,7 +57,6 @@ helm uninstall greptimedb-standalone -n default
 | configToml | string | `"mode = 'standalone'\n"` | The extra configuration for greptimedb |
 | dataHome | string | `"/data/greptimedb/"` | Storage root directory |
 | env | object | `{"GREPTIMEDB_STANDALONE__HTTP__ADDR":"0.0.0.0:4000"}` | Environment variables |
-| envFrom | object | `{}` | Maps all the keys on a configmap or secret as environment variables |
 | fullnameOverride | string | `""` | Provide a name to substitute for the full names of resources |
 | grpcServicePort | int | `4001` | GreptimeDB grpc service port |
 | httpServicePort | int | `4000` | GreptimeDB http service port |
@@ -61,6 +72,7 @@ helm uninstall greptimedb-standalone -n default
 | mysqlServicePort | int | `4002` | GreptimeDB mysql service port |
 | nameOverride | string | `""` | Overrides the chart's name |
 | nodeSelector | object | `{}` | NodeSelector to apply pod |
+| objectStorage | object | `{"oss":{},"s3":{}}` | Configure to object storage |
 | opentsdbServicePort | int | `4242` | GreptimeDB opentsdb service port |
 | persistence.enableStatefulSetAutoDeletePVC | bool | `false` | Enable StatefulSetAutoDeletePVC feature |
 | persistence.enabled | bool | `true` | Enable persistent disk |

charts/greptimedb-standalone/README.md.gotmpl

@@ -17,6 +17,18 @@ helm repo update
 helm upgrade --install greptimedb-standalone greptime/greptimedb-standalone -n default
 ```
 
+**Use AWS S3 as backend storage**
+```console
+helm upgrade --install greptimedb-standalone greptime/greptimedb-standalone \
+  --set objectStorage.credentials.accessKeyId="your-access-key-id" \
+  --set objectStorage.credentials.secretAccessKey="your-secret-access-key" \
+  --set objectStorage.s3.bucket="your-bucket-name" \
+  --set objectStorage.s3.region="region-of-bucket" \
+  --set objectStorage.s3.endpoint="s3-endpoint" \
+  --set objectStorage.s3.root="root-directory-of-data" \
+  -n default
+```
+
 ## Connection
 
 ```console

charts/greptimedb-standalone/templates/configmap.yaml

@@ -4,6 +4,8 @@ kind: ConfigMap
 metadata:
   name: {{ .Release.Name }}-config
   namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "greptimedb-standalone.labels" . | nindent 4 }}
 data:
   config.toml: |
 {{ .Values.configToml | indent 4 }}

charts/greptimedb-standalone/templates/secret.yaml

@@ -0,0 +1,28 @@
+{{- if .Values.objectStorage }}
+{{- if .Values.objectStorage.credentials }}
+apiVersion: v1
+metadata:
+  name: {{ .Release.Name }}-secret
+  namespace: {{ .Release.Namespace }}
+kind: Secret
+type: Opaque
+stringData:
+  {{- if .Values.objectStorage.s3}}
+  GREPTIMEDB_STANDALONE__STORAGE__TYPE: "S3"
+  GREPTIMEDB_STANDALONE__STORAGE__ACCESS_KEY_ID: {{ .Values.objectStorage.credentials.accessKeyId }}
+  GREPTIMEDB_STANDALONE__STORAGE__SECRET_ACCESS_KEY: {{ .Values.objectStorage.credentials.secretAccessKey }}
+  GREPTIMEDB_STANDALONE__STORAGE__BUCKET: {{ .Values.objectStorage.s3.bucket}}
+  GREPTIMEDB_STANDALONE__STORAGE__ROOT: {{ .Values.objectStorage.s3.root }}
+  GREPTIMEDB_STANDALONE__STORAGE__REGION: {{ .Values.objectStorage.s3.region }}
+  GREPTIMEDB_STANDALONE__STORAGE__ENDPOINT: {{ .Values.objectStorage.s3.endpoint }}
+  {{ else if .Values.objectStorage.oss }}
+  GREPTIMEDB_STANDALONE__STORAGE__TYPE: "Oss"
+  GREPTIMEDB_STANDALONE__STORAGE__ACCESS_KEY_ID: {{ .Values.objectStorage.credentials.accessKeyId }}
+  GREPTIMEDB_STANDALONE__STORAGE__ACCESS_KEY_SECRET: {{ .Values.objectStorage.credentials.secretAccessKey }}
+  GREPTIMEDB_STANDALONE__STORAGE__BUCKET: {{ .Values.objectStorage.oss.bucket}}
+  GREPTIMEDB_STANDALONE__STORAGE__ROOT: {{ .Values.objectStorage.oss.root }}
+  GREPTIMEDB_STANDALONE__STORAGE__REGION: {{ .Values.objectStorage.oss.region }}
+  GREPTIMEDB_STANDALONE__STORAGE__ENDPOINT: {{ .Values.objectStorage.oss.endpoint }}
+  {{- end}}
+{{- end }}
+{{- end }}

charts/greptimedb-standalone/templates/statefulset.yaml

@@ -96,9 +96,12 @@ spec:
               value: {{ $val | quote }}
             {{- end }}
           {{- end }}
-          {{- with .Values.envFrom }}
+          {{- if .Values.objectStorage }}
+          {{- if .Values.objectStorage.credentials }}
           envFrom:
-            {{- toYaml . | nindent 12 }}
+            - secretRef:
+                name: {{ .Release.Name }}-secret
+          {{- end }}
           {{- end }}
           {{- with .Values.securityContext }}
           securityContext:

charts/greptimedb-standalone/values.yaml

@@ -39,54 +39,37 @@ configToml: |
 # -- Storage root directory
 dataHome: "/data/greptimedb/"
 
+# -- Configure to object storage
+objectStorage:
+#  credentials:
+#    accessKeyId: "you-should-set-the-access-key-id-here"
+#    secretAccessKey: "you-should-set-the-secret-access-key-here"
+
+  # configure to use s3 storage
+  s3: {}
+  #  bucket: "bucket-name"
+  #  region: "us-west-2"
+
+  #  # The data directory in S3 will be: 's3://<bucket>/<root>/data/...'.
+  #  root: "greptimedb-standalone"
+  #  endpoint: ""  # See more detail: https://docs.aws.amazon.com/general/latest/gr/s3.html
+
+  # configure to use oss storage
+  oss: {}
+  #  bucket: "bucket-name"
+  #  region: "cn-hangzhou"
+
+  #  # The data directory in OSS will be: 'oss://<bucket>/<root>/data/...'.
+  #  root: "greptimedb-standalone"
+  #  endpoint: "oss-cn-hangzhou.aliyuncs.com"
+
 # -- Environment variables
 env:
   GREPTIMEDB_STANDALONE__HTTP__ADDR: "0.0.0.0:4000"
 #  GREPTIMEDB_STANDALONE__GRPC__ADDR: "0.0.0.0:4001"
 #  GREPTIMEDB_STANDALONE__MYSQL__ADDR: "0.0.0.0:4002"
 #  GREPTIMEDB_STANDALONE__POSTGRES__ADDR: "0.0.0.0:4003"
 #  GREPTIMEDB_STANDALONE__OPENTSDB__ADDR: "0.0.0.0:4242"
-#  GREPTIMEDB_STANDALONE__HTTP__TIMEOUT: "120s"
-
-#  GREPTIMEDB_STANDALONE__WAL__FILE_SIZE: "128MB"
-#  GREPTIMEDB_STANDALONE__WAL__PURGE_THRESHOLD: "2GB"
-#  GREPTIMEDB_STANDALONE__WAL__DIR: "/data/greptimedb/wal"
-
-#  # configure to use local storage.
-#  GREPTIMEDB_STANDALONE__STORAGE__TYPE: "File"
-
-#  # configure to use aws s3 storage.
-#  GREPTIMEDB_STANDALONE__STORAGE__TYPE: "S3"
-#  GREPTIMEDB_STANDALONE__STORAGE__BUCKET: "aws_s3_name"
-#  GREPTIMEDB_STANDALONE__STORAGE__ROOT: "/data"
-#  GREPTIMEDB_STANDALONE__STORAGE__REGION: "aws_s3_region"
-#  GREPTIMEDB_STANDALONE__STORAGE__ACCESS_KEY_ID: "aws_access_key_id"
-#  GREPTIMEDB_STANDALONE__STORAGE__SECRET_ACCESS_KEY: "aws_secret_access_key"
-#  GREPTIMEDB_STANDALONE__STORAGE__CACHE_PATH: "/data/greptimedb/s3cache"
-
-#  # configure to use alicloud oss storage.
-#  GREPTIMEDB_STANDALONE__STORAGE__TYPE: "Oss"
-#  GREPTIMEDB_STANDALONE__STORAGE__BUCKET: "alicloud_oss_name"
-#  GREPTIMEDB_STANDALONE__STORAGE__ROOT: "/data"
-#  GREPTIMEDB_STANDALONE__STORAGE__REGION: "alicloud_oss_region"
-#  GREPTIMEDB_STANDALONE__STORAGE__ACCESS_KEY_ID: "alicloud_access_key_id"
-#  GREPTIMEDB_STANDALONE__STORAGE__ACCESS_KEY_SECRET: "alicloud_access_key_secret"
-#  GREPTIMEDB_STANDALONE__STORAGE__ENDPOINT: "alicloud_oss_endpoint" # For example: oss-cn-hangzhou.aliyuncs.com, more detail see https://www.alibabacloud.com/help/en/oss/user-guide/regions-and-endpoints
-#  GREPTIMEDB_STANDALONE__STORAGE__CACHE_PATH: "/data/greptimedb/osscache"
-
-#  GREPTIMEDB_STANDALONE__STORAGE__GLOBAL_TTL: "1d"
-#  GREPTIMEDB_STANDALONE__STORAGE__FLUSH__GLOBAL_WRITE_BUFFER_SIZE: "512MB"
-#  GREPTIMEDB_STANDALONE__STORAGE__FLUSH__REGION_WRITE_BUFFER_SIZE: "256MB"
-#  GREPTIMEDB_STANDALONE__STORAGE__FLUSH__AUTO_FLUSH_INTERVAL: "5m"
-#  GREPTIMEDB_STANDALONE__STORAGE__FLUSH__MAX_FLUSH_TASKS: "2"
-#  GREPTIMEDB_STANDALONE__STORAGE__COMPACTION__MAX_INFLIGHT_TASKS: "2"
-#  GREPTIMEDB_DATANODE__STORAGE__MANIFEST__CHECKPOINT_MARGIN: "100"
-#  GREPTIMEDB_DATANODE__STORAGE__MANIFEST__GC_DURATION: "5m"
-
-# -- Maps all the keys on a configmap or secret as environment variables
-envFrom: {}
-#  - secretRef:
-#      name: config
 
 # -- Extra pod annotations to add
 podAnnotations: {}