add cloudnative-pg

prehor · May 17, 2024 · 50f8850 · 50f8850
1 parent 0e26fef
commit 50f8850
Show file tree

Hide file tree

Showing 21 changed files with 446 additions and 1 deletion.
diff --git a/.github/renovate/versioning.json5 b/.github/renovate/versioning.json5
@@ -6,6 +6,13 @@
       "matchDatasources": ["github-releases"],
       "versioning": "regex:^v(?<major>\\d+)\\.(?<minor>\\d+)\\.(?<patch>\\d+)(?<compatibility>\\+k.s)\\.?(?<build>\\d+)$",
       "matchPackagePatterns": ["k3s"]
+    },
+    {
+      "description": ["Use custom versioning for TimescaleDB"],
+      "matchDatasources": ["docker"],
+      // https://docs.renovatebot.com/modules/versioning/#regular-expression-versioning
+      "versioning": "regex:^(?<major>\\d+)(-(?<minor>\\d+)\\.(?<patch>\\d+)(-(?<build>\\d+))?)?$",
+      "matchPackagePatterns": ["timescaledb"]
     }
   ]
 }
diff --git a/kubernetes/main/apps/database/cloudnative-pg/app/helmrelease.yaml b/kubernetes/main/apps/database/cloudnative-pg/app/helmrelease.yaml
@@ -0,0 +1,34 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: cloudnative-pg
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: cloudnative-pg
+      version: 0.21.2
+      sourceRef:
+        kind: HelmRepository
+        name: cloudnative-pg
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      strategy: rollback
+      retries: 3
+  dependsOn:
+    - name: openebs
+      namespace: storage
+  values:
+    crds:
+      create: true
+    monitoring:
+      podMonitorEnabled: false
+      # grafanaDashboard:
+      #   create: true
diff --git a/kubernetes/main/apps/database/cloudnative-pg/app/kustomization.yaml b/kubernetes/main/apps/database/cloudnative-pg/app/kustomization.yaml
@@ -0,0 +1,10 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml
+  - ./role.yaml
+  - ./rolebinding.yaml
+  - ./secret.sops.yaml
+
+
diff --git a/kubernetes/main/apps/database/cloudnative-pg/app/role.yaml b/kubernetes/main/apps/database/cloudnative-pg/app/role.yaml
@@ -0,0 +1,21 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: postgres-secrets-manager
+rules:
+  - apiGroups: [""]
+    resources:
+      - secrets
+    resourceNames:
+      - postgres-secret
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - selfsubjectrulesreviews
+    verbs:
+      - create
diff --git a/kubernetes/main/apps/database/cloudnative-pg/app/rolebinding.yaml b/kubernetes/main/apps/database/cloudnative-pg/app/rolebinding.yaml
@@ -0,0 +1,13 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: postgres-secrets-manager-binding
+subjects:
+  - kind: ServiceAccount
+    name: postgres-secrets-manager
+    namespace: security
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: postgres-secrets-manager
diff --git a/kubernetes/main/apps/database/cloudnative-pg/app/secret.sops.yaml b/kubernetes/main/apps/database/cloudnative-pg/app/secret.sops.yaml
@@ -0,0 +1,61 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: cloudnative-pg-secret
+type: kubernetes.io/basic-auth
+stringData:
+    username: ENC[AES256_GCM,data:kwuEGDkR85g=,iv:9ShWFqwYrbqLQtTYOGbfACdC4qJOkuS99kJqDvgDPpQ=,tag:bpa6Q27d1DLQILess4eZAw==,type:str]
+    password: ENC[AES256_GCM,data:ln9wP53YPAKl3QehDnhKpeM=,iv:qa3iqMtvf9ONTY+yqnmwxRrB/hvqq0h2EZG5duWcZ5A=,tag:dpbuMt1u0HWspzVbSO7L3g==,type:str]
+    MINIO_ACCESS_KEY: ENC[AES256_GCM,data:VdeOLuCKGDZaOk5W1uWhrcWuL2Y=,iv:TH99+jRIIcKjTLJskGmVc1n9USTqtni67UAMb8yXtPA=,tag:o6POe/jIzyYZt72NIYEHzQ==,type:str]
+    MINIO_SECRET_KEY: ENC[AES256_GCM,data:Jzls9LA5BZifOvlQonAd1NIHiBhOr8r9/gHEaP79IcS4q45rbhIjLA==,iv:QF4yaWKCQMLI8vFZ9no89o0/g020X+I8MWkdtN+rpxI=,tag:tF4mhbx93OAHrNq4CO4AsQ==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1ve9kzacrwq7l9l0emvs326uk6t576d75r596e083r2tq6xu28qcsacy3s7
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4a2pOK1plN0FLMHk2MXRo
+            OUpPWTVKbDVqVlZxOXJodzMwNmNRV2x1OGhJCnFOaWVUdnRpQjNPb09rUkIyUXpK
+            L29LU3B0bjR0VzBXSGNmSElNK2QyNWcKLS0tIERWNml6d1lJNzVXYWhteUFucHhL
+            SG1pSlhtOWZBU0hnMktiZWFyZ1B3b0UKX32ll3xgJig7u0zDqEuK9D+sGzyYqKWK
+            3zIGbgw2qvrflTlMLsygBV4ARdu9AMDptpTR+v7vvFnf89S/XNwouQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-05-17T19:21:56Z"
+    mac: ENC[AES256_GCM,data:QLsg/yuY5kdCBT8sfCJUrOJMpoIVmr7PeXKXbGoUjpnAt/jq9fJ2g0lGgvTOaos3hhyqG5ODrzzwJw4qBzaZzRsYx3xucYjCsBjSUDQ0tsBiNHEEPgAyJ5Wzxzo5bqdPK//knxc9pgAft1BH3BuaVJ0YM5Jg1TVbOdEsadLAK5U=,iv:zk4cG7A1eVVsDLxXN/MeMUajFfVEjB7ivE1zkl8qh4s=,tag:Z0UyQEwBuPtSE4B2C27Ncw==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.8.1
+---
+apiVersion: v1
+kind: Secret
+metadata:
+    name: postgres-secret
+type: kubernetes.io/basic-auth
+stringData:
+    username: ENC[AES256_GCM,data:S5ZdSUTkietBmDoxd9FYivPDLw==,iv:jX+r0kA6P6lhx1fa3RsGEUgu374KWvsi65J0KAlStzg=,tag:CU0rHTB14FyGBfPOuLjUPA==,type:str]
+    password: ENC[AES256_GCM,data:0cOclwIjTueDLa50Gm87CAtIV7kkB75UGCpj1fGc/+IbZrQ29DSVHg==,iv:YbmWDjCr4WBM8fXlSa0Qq4W7QeL+/BKzBX44Ac1GDCk=,tag:aQppQAXK5hJVXpBHn7m/XQ==,type:str]
+    host: ENC[AES256_GCM,data:F5a4af0LP/ZjpJgiaMxq1vBin4ogKha4HXGFtnuBxF+L71/A7c0FUQ==,iv:mpUadwe1cluPTtiro4bHPmGawDi9VCIRzdF6+/TX2MI=,tag:EZoy6byTtjXgRIGouqn/Gg==,type:str]
+    port: ENC[AES256_GCM,data:d2h/Jg==,iv:YUvt0sD+NACV/J568s4IW4HQIRquwI+pttg84Mic1vg=,tag:tzjf4hIRbbYWM0duPrpK8w==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1ve9kzacrwq7l9l0emvs326uk6t576d75r596e083r2tq6xu28qcsacy3s7
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4a2pOK1plN0FLMHk2MXRo
+            OUpPWTVKbDVqVlZxOXJodzMwNmNRV2x1OGhJCnFOaWVUdnRpQjNPb09rUkIyUXpK
+            L29LU3B0bjR0VzBXSGNmSElNK2QyNWcKLS0tIERWNml6d1lJNzVXYWhteUFucHhL
+            SG1pSlhtOWZBU0hnMktiZWFyZ1B3b0UKX32ll3xgJig7u0zDqEuK9D+sGzyYqKWK
+            3zIGbgw2qvrflTlMLsygBV4ARdu9AMDptpTR+v7vvFnf89S/XNwouQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-05-17T19:21:56Z"
+    mac: ENC[AES256_GCM,data:QLsg/yuY5kdCBT8sfCJUrOJMpoIVmr7PeXKXbGoUjpnAt/jq9fJ2g0lGgvTOaos3hhyqG5ODrzzwJw4qBzaZzRsYx3xucYjCsBjSUDQ0tsBiNHEEPgAyJ5Wzxzo5bqdPK//knxc9pgAft1BH3BuaVJ0YM5Jg1TVbOdEsadLAK5U=,iv:zk4cG7A1eVVsDLxXN/MeMUajFfVEjB7ivE1zkl8qh4s=,tag:Z0UyQEwBuPtSE4B2C27Ncw==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.8.1
diff --git a/kubernetes/main/apps/database/cloudnative-pg/cluster/cluster16.yaml b/kubernetes/main/apps/database/cloudnative-pg/cluster/cluster16.yaml
@@ -0,0 +1,100 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/postgresql.cnpg.io/cluster_v1.json
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: postgres16
+spec:
+  instances: 3
+  imageCatalogRef:
+    apiGroup: postgresql.cnpg.io
+    kind: ImageCatalog
+    name: timescaledb
+    major: 16
+  primaryUpdateStrategy: unsupervised
+  storage:
+    size: 10Gi
+    storageClass: local-database
+  superuserSecret:
+    name: postgres-secret
+  enableSuperuserAccess: true
+  postgresql:
+    parameters:
+      max_connections: "400"
+      shared_buffers: "256MB"
+      # ZFS tuning
+      # https://openzfs.github.io/openzfs-docs/Performance%20and%20Tuning/Workload%20Tuning.html#postgresql
+      # https://vadosware.io/post/everything-ive-seen-on-optimizing-postgres-on-zfs-on-linux/
+      # https://kubeblocks.io/blog/A-testing-report-for-optimizing-PG-performance-on-Kubernetes
+      # https://github.com/cloudnative-pg/cloudnative-pg/discussions/3058
+      # full_page_writes: "off"
+      recovery_prefetch: "try"
+      wal_init_zero: "off"
+      wal_recycle: "off"
+      wal_segment_size: "1024"
+    shared_preload_libraries:
+      - timescaledb
+  nodeMaintenanceWindow:
+    inProgress: false
+    reusePVC: true
+  resources:
+    # requests:
+    #   cpu: 500m
+    limits:
+      memory: 4Gi
+  monitoring:
+    enablePodMonitor: true
+    # https://github.com/cloudnative-pg/cloudnative-pg/issues/2501
+    podMonitorMetricRelabelings:
+      - { sourceLabels: ["cluster"], targetLabel: cnpg_cluster, action: replace }
+      - { regex: cluster, action: labeldrop }
+  backup:
+    retentionPolicy: 30d
+    barmanObjectStore: &barmanObjectStore
+      data:
+        compression: bzip2
+      wal:
+        compression: bzip2
+        maxParallel: 4
+      destinationPath: s3://home-ops-postgresql
+      endpointURL: https://s3.${STORAGE_DOMAIN}
+      # Note: serverName version needs to be inclemented
+      # when recovering from an existing cnpg cluster
+      serverName: &currentCluster postgres16-v0
+      s3Credentials:
+        accessKeyId:
+          name: cloudnative-pg-secret
+          key: MINIO_ACCESS_KEY
+        secretAccessKey:
+          name: cloudnative-pg-secret
+          key: MINIO_SECRET_KEY
+  # Note: previousCluster needs to be set to the name of the previous
+  # cluster when recovering from an existing cnpg cluster
+  bootstrap:
+    initdb:
+      # User for onedr0p/postgres-init database inital creation
+      owner: postgres-db-manager
+      # https://cloudnative-pg.io/documentation/1.19/cloudnative-pg.v1/#postgresql-cnpg-io-v1-BootstrapInitDB
+      postInitApplicationSQL:
+        # Assign Postgres DB Manager permissions
+        # https://www.atlassian.com/data/admin/how-to-change-a-user-to-superuser-in-postgresql
+        - ALTER USER postgres-db-manager WITH CREATEDB;
+        - ALTER USER postgres-db-manager WITH CREATEROLE;
+      # Enable PostGIS and TimescaleDB extensions
+      postInitTemplateSQL:
+        - CREATE EXTENSION IF NOT EXISTS postgis;
+        - CREATE EXTENSION IF NOT EXISTS postgis_topology;
+        - CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
+        - CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder;
+        - CREATE EXTENSION IF NOT EXISTS timescaledb;
+        - CREATE EXTENSION IF NOT EXISTS timescaledb_toolkit;
+      secret:
+        name: postgres-secret
+  #   recovery:
+  #     source: &previousCluster postgres16-v0
+  # # Note: externalClusters is needed when recovering from an existing cnpg cluster
+  # externalClusters:
+  #   - name: *previousCluster
+  #     barmanObjectStore:
+  #       <<: *barmanObjectStore
+  #       serverName: *previousCluster
diff --git a/kubernetes/main/apps/database/cloudnative-pg/cluster/imagecatalog.yaml b/kubernetes/main/apps/database/cloudnative-pg/cluster/imagecatalog.yaml
@@ -0,0 +1,10 @@
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ImageCatalog
+metadata:
+  name: timescaledb
+spec:
+  images:
+    - major: 16
+      # renovate: datasource=docker
+      image: ghcr.io/imusmanmalik/timescaledb-postgis:16-3.4-42
diff --git a/kubernetes/main/apps/database/cloudnative-pg/cluster/kustomization.yaml b/kubernetes/main/apps/database/cloudnative-pg/cluster/kustomization.yaml
@@ -0,0 +1,9 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./cluster16.yaml
+  - ./imagecatalog.yaml
+  - ./prometheusrule.yaml
+  - ./scheduledbackup.yaml
diff --git a/kubernetes/main/apps/database/cloudnative-pg/cluster/prometheusrule.yaml b/kubernetes/main/apps/database/cloudnative-pg/cluster/prometheusrule.yaml
@@ -0,0 +1,67 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/monitoring.coreos.com/prometheusrule_v1.json
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: cloudnative-pg-rules
+  labels:
+    prometheus: k8s
+    role: alert-rules
+spec:
+  groups:
+    - name: cloudnative-pg.rules
+      rules:
+        - alert: LongRunningTransaction
+          annotations:
+            description: Pod {{ $labels.pod }} is taking more than 5 minutes (300 seconds) for a query.
+            summary: A query is taking longer than 5 minutes.
+          expr: |-
+            cnpg_backends_max_tx_duration_seconds > 300
+          for: 1m
+          labels:
+            severity: warning
+        - alert: BackendsWaiting
+          annotations:
+            description: Pod {{ $labels.pod }} has been waiting for longer than 5 minutes
+            summary: If a backend is waiting for longer than 5 minutes
+          expr: |-
+            cnpg_backends_waiting_total > 300
+          for: 1m
+          labels:
+            severity: warning
+        - alert: PGDatabase
+          annotations:
+            description: Over 150,000,000 transactions from frozen xid on pod {{ $labels.pod }}
+            summary: Number of transactions from the frozen XID to the current one
+          expr: |-
+            cnpg_pg_database_xid_age > 150000000
+          for: 1m
+          labels:
+            severity: warning
+        - alert: PGReplication
+          annotations:
+            description: Standby is lagging behind by over 300 seconds (5 minutes)
+            summary: The standby is lagging behind the primary
+          expr: |-
+            cnpg_pg_replication_lag > 300
+          for: 1m
+          labels:
+            severity: warning
+        - alert: LastFailedArchiveTime
+          annotations:
+            description: Archiving failed for {{ $labels.pod }}
+            summary: Checks the last time archiving failed. Will be < 0 when it has not failed.
+          expr: |-
+            (cnpg_pg_stat_archiver_last_failed_time - cnpg_pg_stat_archiver_last_archived_time) > 1
+          for: 1m
+          labels:
+            severity: warning
+        - alert: DatabaseDeadlockConflicts
+          annotations:
+            description: There are over 10 deadlock conflicts in {{ $labels.pod }}
+            summary: Checks the number of database conflicts
+          expr: |-
+            cnpg_pg_stat_database_deadlocks > 10
+          for: 1m
+          labels:
+            severity: warning
diff --git a/kubernetes/main/apps/database/cloudnative-pg/cluster/scheduledbackup.yaml b/kubernetes/main/apps/database/cloudnative-pg/cluster/scheduledbackup.yaml
@@ -0,0 +1,12 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/postgresql.cnpg.io/scheduledbackup_v1.json
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: postgres
+spec:
+  schedule: "@daily"
+  immediate: true
+  backupOwnerReference: self
+  cluster:
+    name: postgres16