Merge pull request #65 from graphistry/dev/improve-postgres-conf

Improve Postgres cluster configuration
graphistry · Dec 31, 2024 · 00ea733 · 00ea733
2 parents 7ba2ace + 9d10247
commit 00ea733
Show file tree

Hide file tree

Showing 5 changed files with 89 additions and 52 deletions.
diff --git a/charts/postgres-cluster/templates/postgres-cluster.yaml b/charts/postgres-cluster/templates/postgres-cluster.yaml
@@ -56,7 +56,7 @@ spec:
       {{- end }}
         resources:
           requests:
-            storage: 4Gi
+            storage: {{ toYaml .Values.postgresCluster.instance1.dataVolumeClaimSpec.resources.requests.storage | quote }}
 {{ if .Values.global.postgresVolumeLabel }}
         selector:
           matchLabels:
@@ -66,61 +66,19 @@ spec:
     dynamicConfiguration:
       postgresql:
         pg_hba:
-          - "host all all 0.0.0.0/0 trust" # this line enabled logical replication with programmatic access
-          - "host all postgres 127.0.0.1/32 md5"
-
+{{ toYaml .Values.postgresCluster.patroni.dynamicConfiguration.postgresql.pg_hba | indent 10 }}
+
   backups:
     pgbackrest:
-      repoHost:
-        resources:
-          requests:
-            cpu: 100m
-            memory: 128Mi
-          limits:
-            cpu: 100m
-            memory: 128Mi
-      sidecars:
-        pgbackrest:
-          resources:
-            requests:
-              cpu: 100m
-              memory: 128Mi
-            limits:
-              cpu: 100m
-              memory: 128Mi
-        pgbackrestConfig:
-          resources:
-            requests:
-              cpu: 100m
-              memory: 128Mi
-            limits:
-              cpu: 100m
-              memory: 128Mi
-      jobs:
-        resources:
-          requests:
-            cpu: 100m
-            memory: 128Mi
-          limits:
-            cpu: 100m
-            memory: 128Mi
-      restore:
-        repoName: repo1
-        enabled: false
-        resources:
-          requests:
-            cpu: 100m
-            memory: 128Mi
-          limits:
-            cpu: 100m
-            memory: 128Mi
       {{- if eq .Values.global.containerregistry.name "docker.io/graphistry" }}
       image: registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.40-1 #crunchydata
       {{ else }}
       image: {{.Values.global.containerregistry.name}}/crunchy-pgbackrest:ubi8-2.40-1
-      {{- end }}    
+      {{- end }}
+{{ toYaml .Values.postgresCluster.backups.pgbackrest | indent 6 }}
       repos:
         - name: repo1
+{{ toYaml .Values.postgresCluster.backups.pgbackrest.repos.repo1 | indent 10 }}
           volume:
             volumeClaimSpec:
             {{- if eq .Values.global.multiNode true  }}
@@ -134,7 +92,7 @@ spec:
             {{- end }}
               resources:
                 requests:
-                  storage: 4Gi     
+                  storage: {{ toYaml .Values.postgresCluster.backups.pgbackrest.repos.repo1.volume.volumeClaimSpec.resources.requests.storage | quote }}
   imagePullPolicy: {{ .Values.global.imagePullPolicy | quote }}
   {{- with .Values.global.imagePullSecrets }}
   imagePullSecrets:

diff --git a/charts/postgres-cluster/values.yaml b/charts/postgres-cluster/values.yaml
@@ -38,3 +38,80 @@ global:
     #   cpu: 100m
     #   memory: 128Mi
 
+postgresCluster:
+  instance1:
+    dataVolumeClaimSpec:
+      resources:
+        requests:
+          storage: 10Gi
+
+  patroni:
+    dynamicConfiguration:
+      postgresql:
+        pg_hba:
+          - "host all all 0.0.0.0/0 trust" # This line enable logical replication with programmatic access
+          - "host all postgres 127.0.0.1/32 md5"
+
+  backups:
+    pgbackrest: # Check for more config options https://pgbackrest.org/configuration.html
+      global:
+        repo1-bundle-size: "1G"  # Set a 1GB maximum size for each 'backup bundle'
+        repo1-bundle-limit: "15"    # Limit to 15 'backup bundles'
+        repo1-retention-full: "5"  # Keeps 5 'full backups'
+        repo1-retention-archive: "10"  # Retain the last 10 archive logs
+        repo1-retention-diff: "7"  # Retain 7 'differential backups'
+      repoHost:
+        resources:
+          requests:
+            cpu: 200m
+            memory: 256Mi
+          limits:
+            cpu: 200m
+            memory: 256Mi
+      sidecars:
+        pgbackrest:
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: 100m
+              memory: 128Mi
+        pgbackrestConfig:
+          resources:
+            requests:
+              cpu: 200m
+              memory: 128Mi
+            limits:
+              cpu: 200m
+              memory: 128Mi
+      jobs:
+        resources:
+          requests:
+            cpu: 200m
+            memory: 128Mi
+          limits:
+            cpu: 200m
+            memory: 128Mi
+      restore:
+        repoName: repo1
+        enabled: false
+        resources:
+          requests:
+            cpu: 100m
+            memory: 128Mi
+          limits:
+            cpu: 100m
+            memory: 128Mi
+
+      repos:
+        repo1:
+          schedule:
+            full: "0 1 * * 0"  # Weekly full backup every Sunday at 1 AM
+            differential: "0 3 * * *"  # Daily differential backup
+            incremental: "*/30 * * * *"  # Every 30 minutes for incremental backups
+          volume:
+            volumeClaimSpec:
+              resources:
+                requests:
+                  storage: 50Gi
diff --git a/charts/values-overrides/examples/gke/README.md b/charts/values-overrides/examples/gke/README.md
@@ -1,4 +1,4 @@
-# Deploy a Graphistry k8s cluster using GKE
+# Deploy a Graphistry K8s cluster using GKE
 This guide provides step-by-step instructions for deploying Graphistry on Google Kubernetes Engine (GKE).  The steps are based on the official documentation of [Graphistry Helm Charts](https://github.com/graphistry/graphistry-helm) and the [NVIDIA GPU Operator with Google GKE](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/google-gke.html).
 
 ## Prerequisites
@@ -375,7 +375,7 @@ cuda:
   version: "11.8" #cuda version
 
 global:  ## global settings for all charts
-  tag: v2.41.15
+  tag: v2.41.10
 ```
 
 Print more values:

diff --git a/charts/values-overrides/examples/gke/gke_values.yaml b/charts/values-overrides/examples/gke/gke_values.yaml
@@ -213,7 +213,7 @@ cuda:
   version: "11.8" #cuda version
 
 global:  ## global settings for all charts
-  tag: v2.41.15
+  tag: v2.41.10
   logs: #change log levels
       LogLevel: DEBUG #log level for the application
       GraphistryLogLevel: DEBUG #log level for graphistry

diff --git a/docs/source/values-override.rst b/docs/source/values-override.rst
@@ -80,6 +80,8 @@ Here is an example of a values.yaml for all of the individual services taken fro
             imagePullSecrets: 
               - name: docker-secret-prod
 
+If you're deploying Graphistry on Google Kubernetes Engine (GKE), you can override values for GKE-specific configurations.  For detailed examples and guidance, refer to the `Deploy a Graphistry K8s cluster using GKE <https://github.com/graphistry/graphistry-helm/tree/main/charts/values-overrides/examples/gke>`_.
+
 Mandatory Values
 ----------------