feat: add volsync

prehor · May 9, 2024 · fa8bfbf · fa8bfbf
1 parent 26e7270
commit fa8bfbf
Show file tree

Hide file tree

Showing 24 changed files with 306 additions and 8 deletions.
diff --git a/kubernetes/main/apps/security/external-secrets/stores/kustomization.yaml b/kubernetes/main/apps/security/external-secrets/stores/kustomization.yaml
@@ -2,4 +2,5 @@
 # yaml-language-server: $schema=https://json.schemastore.org/kustomization
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-resources: []
+resources:
+  - ./storage-secrets
diff --git a/...rnetes/main/apps/security/external-secrets/stores/storage-secrets/clustersecretstore.yaml b/...rnetes/main/apps/security/external-secrets/stores/storage-secrets/clustersecretstore.yaml
@@ -0,0 +1,19 @@
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ClusterSecretStore
+metadata:
+  name: storage-secrets-store
+spec:
+  provider:
+    kubernetes:
+      remoteNamespace: storage
+      auth:
+        serviceAccount:
+          name: storage-secrets-manager
+          namespace: security
+      server:
+        caProvider:
+          type: ConfigMap
+          name: kube-root-ca.crt
+          namespace: security
+          key: ca.crt
diff --git a/kubernetes/main/apps/security/external-secrets/stores/storage-secrets/kustomization.yaml b/kubernetes/main/apps/security/external-secrets/stores/storage-secrets/kustomization.yaml
@@ -0,0 +1,7 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./clustersecretstore.yaml
+  - ./serviceaccount.yaml
diff --git a/kubernetes/main/apps/security/external-secrets/stores/storage-secrets/serviceaccount.yaml b/kubernetes/main/apps/security/external-secrets/stores/storage-secrets/serviceaccount.yaml
@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: storage-secrets-manager
diff --git a/kubernetes/main/apps/storage/kustomization.yaml b/kubernetes/main/apps/storage/kustomization.yaml
@@ -3,7 +3,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ./namespace.yaml
-  - ./snapshot-controller/ks.yaml
   - ./minio/ks.yaml
   - ./openebs/ks.yaml
   - ./rook-ceph/ks.yaml
+  - ./snapshot-controller/ks.yaml
+  - ./storage-secrets/ks.yaml
+  - ./volsync/ks.yaml
diff --git a/kubernetes/main/apps/storage/openebs/app/local-database-sc.yaml b/kubernetes/main/apps/storage/openebs/app/local-database-sc.yaml
@@ -11,6 +11,6 @@ parameters:
   compresion: lz4
   dedup: off
   thinprovision: yes
-  shared: no
+  shared: yes
 provisioner: zfs.csi.openebs.io
 volumeBindingMode: WaitForFirstConsumer
diff --git a/kubernetes/main/apps/storage/storage-secrets/ks.yaml b/kubernetes/main/apps/storage/storage-secrets/ks.yaml
@@ -0,0 +1,21 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app storage-secrets
+  namespace: flux-system
+spec:
+  targetNamespace: storage
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: "./kubernetes/main/apps/storage/storage-secrets/rbac"
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-ops
+  wait: true
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
diff --git a/kubernetes/main/apps/storage/storage-secrets/rbac/kustomization.yaml b/kubernetes/main/apps/storage/storage-secrets/rbac/kustomization.yaml
@@ -0,0 +1,7 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./role.yaml
+  - ./rolebinding.yaml
diff --git a/kubernetes/main/apps/storage/storage-secrets/rbac/role.yaml b/kubernetes/main/apps/storage/storage-secrets/rbac/role.yaml
@@ -0,0 +1,22 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: storage-secrets-manager
+rules:
+  - apiGroups: [""]
+    resources:
+      - secrets
+    resourceNames:
+      - minio-secret
+      - volsync-secret
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - selfsubjectrulesreviews
+    verbs:
+      - create
diff --git a/kubernetes/main/apps/storage/storage-secrets/rbac/rolebinding.yaml b/kubernetes/main/apps/storage/storage-secrets/rbac/rolebinding.yaml
@@ -0,0 +1,13 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: storage-secrets-manager-binding
+subjects:
+  - kind: ServiceAccount
+    name: storage-secrets-manager
+    namespace: security
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: storage-secrets-manager
diff --git a/kubernetes/main/apps/storage/volsync/app/helmrelease.yaml b/kubernetes/main/apps/storage/volsync/app/helmrelease.yaml
@@ -0,0 +1,31 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: volsync
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: volsync
+      version: 0.9.1
+      sourceRef:
+        kind: HelmRepository
+        name: backube
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      strategy: rollback
+      retries: 3
+  dependsOn:
+    - name: openebs
+    - name: rook-ceph-cluster
+    - name: snapshot-controller
+  values:
+    metrics:
+      disableAuth: true
diff --git a/kubernetes/main/apps/storage/volsync/app/kustomization.yaml b/kubernetes/main/apps/storage/volsync/app/kustomization.yaml
@@ -0,0 +1,7 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml
+  - ./secret.sops.yaml
diff --git a/kubernetes/main/apps/storage/volsync/app/secret.sops.yaml b/kubernetes/main/apps/storage/volsync/app/secret.sops.yaml
@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: volsync-secret
+stringData:
+    RESTIC_PASSWORD: ENC[AES256_GCM,data:OutPX3ZJDNcW4ZnSxeUj/47RBKf1bbkge/52H3xfTy3XO7n4BObuEHOqfEjITnxt0dpDM588a0bTclmB8MoDJQ==,iv:8RSryIOUNsLj6c9JlAYPyWSia87ojwqe12/CRUxXlas=,tag:YoidVddlHlXl4q7MOP9Ljw==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1ve9kzacrwq7l9l0emvs326uk6t576d75r596e083r2tq6xu28qcsacy3s7
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVK1FDUE5ydDM2SzA0ZWFr
+            ZElOb1JXUjA3Z0NIYW1wZnJvZjdXNC8zV0Y0CjRKbHpvVmtKRVIvRmF4V1dGY3ly
+            SWpuc0RhcEdqcHpKcTZiNko5VG1LS2cKLS0tIFVvOERtQVF4ZFVTMk1MZ3A1QnFT
+            RFN6U2lQU2Q3M0wwYmNuc0pLN1IzczQKwRAph52iWkz4M/qYBepyfe0OKZV9Qyje
+            LglPEeuTwBc+sybPVtiXIcFzPfIcI03yPsvdFbTbn5eRJWBOA7t3JQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-05-08T23:38:24Z"
+    mac: ENC[AES256_GCM,data:+nKFQ0QKPg0MfztqzaR7TDqQDljTxhwZmu9MxaJHdMUbNgdaZEd6XQYdX/8HpsOPAk3rG4kkHWIV6BNg0AcNvMygnD0l60yavql697qyy0fZ+YnHdO2QrXKFRgAd9ZExQcmOoOu97vYMIAqK2TgStHe74ojxrf7H/7cbd8Uhcoc=,iv:bz0gnOfd02onDxJfIXL+MA9nTtL62p9Uy51BLIWqMtQ=,tag:k1Ny2QJRfwzTFXVlz5SnHA==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.8.1
diff --git a/kubernetes/main/apps/storage/volsync/ks.yaml b/kubernetes/main/apps/storage/volsync/ks.yaml
@@ -0,0 +1,21 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app volsync
+  namespace: flux-system
+spec:
+  targetNamespace: storage
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/main/apps/storage/volsync/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-ops
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
diff --git a/kubernetes/main/flux/repositories/helm/backube.yaml b/kubernetes/main/flux/repositories/helm/backube.yaml
@@ -0,0 +1,10 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/source.toolkit.fluxcd.io/helmrepository_v1beta2.json
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: backube
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://backube.github.io/helm-charts/
diff --git a/kubernetes/main/flux/repositories/helm/kustomization.yaml b/kubernetes/main/flux/repositories/helm/kustomization.yaml
@@ -2,6 +2,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - ./backube.yaml
   - ./bjw-s.yaml
   - ./cilium.yaml
   - ./descheduler.yaml

diff --git a/kubernetes/main/templates/volsync/kustomization.yaml b/kubernetes/main/templates/volsync/kustomization.yaml
@@ -0,0 +1,7 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./minio
+  - ./pvc.yaml
diff --git a/kubernetes/main/templates/volsync/minio/externalsecret.yaml b/kubernetes/main/templates/volsync/minio/externalsecret.yaml
@@ -0,0 +1,25 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/external-secrets.io/externalsecret_v1beta1.json
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: "${APP}-volsync-minio"
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: storage-secrets-store
+  dataFrom:
+    - extract:
+        key: minio-secret
+    - extract:
+        key: volsync-secret
+  target:
+    name: "${APP}-volsync-minio"
+    creationPolicy: Owner
+    template:
+      engineVersion: v2
+      data:
+        RESTIC_REPOSITORY: "s3:s3.${STORAGE_DOMAIN}/${SECRET_CLUSTER_NAME}-volsync/${APP}"
+        RESTIC_PASSWORD: "{{ .RESTIC_PASSWORD }}"
+        AWS_ACCESS_KEY_ID: "{{ .MINIO_ROOT_USER }}"
+        AWS_SECRET_ACCESS_KEY: "{{ .MINIO_ROOT_PASSWORD }}"
diff --git a/kubernetes/main/templates/volsync/minio/kustomization.yaml b/kubernetes/main/templates/volsync/minio/kustomization.yaml
@@ -0,0 +1,8 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./externalsecret.yaml
+  - ./replicationdestination.yaml
+  - ./replicationsource.yaml
diff --git a/kubernetes/main/templates/volsync/minio/replicationdestination.yaml b/kubernetes/main/templates/volsync/minio/replicationdestination.yaml
@@ -0,0 +1,24 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/volsync.backube/replicationdestination_v1alpha1.json
+apiVersion: volsync.backube/v1alpha1
+kind: ReplicationDestination
+metadata:
+  name: "${APP}-bootstrap"
+spec:
+  trigger:
+    manual: restore-once
+  restic:
+    copyMethod: Snapshot
+    repository: "${APP}-volsync-minio"
+    cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-local-generic}"
+    cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"]
+    cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-1Gi}"
+    storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}"
+    volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-csi-ceph-blockpool}"
+    moverSecurityContext:
+      runAsUser: ${APP_UID:-568}
+      runAsGroup: ${APP_GID:-568}
+      fsGroup: ${APP_GID:-568}
+    accessModes:
+      - "${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"
+    capacity: "${VOLSYNC_CAPACITY:-1Gi}"
diff --git a/kubernetes/main/templates/volsync/minio/replicationsource.yaml b/kubernetes/main/templates/volsync/minio/replicationsource.yaml
@@ -0,0 +1,27 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/volsync.backube/replicationsource_v1alpha1.json
+apiVersion: volsync.backube/v1alpha1
+kind: ReplicationSource
+metadata:
+  name: "${APP}-backup-local"
+spec:
+  sourcePVC: "${APP}"
+  trigger:
+    schedule: "0 * * * *"
+  restic:
+    copyMethod: Snapshot
+    repository: ${APP}-volsync-minio
+    cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-local-generic}"
+    cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"]
+    cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-1Gi}"
+    storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}"
+    volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-csi-ceph-blockpool}"
+    moverSecurityContext:
+      runAsUser: ${APP_UID:-568}
+      runAsGroup: ${APP_GID:-568}
+      fsGroup: ${APP_GID:-568}
+    pruneIntervalDays: 7
+    retain:
+      hourly: 24
+      daily: 7
+      weekly: 5
diff --git a/kubernetes/main/templates/volsync/pvc.yaml b/kubernetes/main/templates/volsync/pvc.yaml
@@ -0,0 +1,16 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: "${APP}"
+spec:
+  accessModes:
+    - "${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"
+  dataSourceRef:
+    kind: ReplicationDestination
+    apiGroup: volsync.backube
+    name: "${APP}-bootstrap"
+  resources:
+    requests:
+      storage: "${VOLSYNC_CAPACITY:-1Gi}"
+  storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}"
diff --git a/kubernetes/storage/apps/storage/openebs/app/local-database-sc.yaml b/kubernetes/storage/apps/storage/openebs/app/local-database-sc.yaml
@@ -11,6 +11,6 @@ parameters:
   compresion: lz4
   dedup: off
   thinprovision: yes
-  shared: no
+  shared: yes
 provisioner: zfs.csi.openebs.io
 volumeBindingMode: WaitForFirstConsumer
diff --git a/kubernetes/storage/flux/vars/cluster-secrets.sops.yaml b/kubernetes/storage/flux/vars/cluster-secrets.sops.yaml
@@ -8,8 +8,6 @@ stringData:
     SECRET_DOMAIN: ENC[AES256_GCM,data:OZEIzVX+AozGHM/X3w==,iv:EieC8Xo6RV59JxCrisie4HkD+Z8KvfOGReiHYuPozMw=,tag:ToxxSaMMVrXgkSpMrUOzGw==,type:str]
     SECRET_ACME_EMAIL: ENC[AES256_GCM,data:dx7jXat1y9g=,iv:8YgKWAJoynF30NXjD3r7dnUs5upHoeIhvlJ6yJnAXHU=,tag:NVORv/PEjhf6ViwxKvPRTQ==,type:str]
     SECRET_CLOUDFLARE_TUNNEL_ID: ENC[AES256_GCM,data:ypCuh2C+0R5rFRsIdiFxi8giFMukj8nKbCKJCAJ2HuGwOXbS,iv:yyEdTwS9HfRJvfXDpq41GG1b3wcPeGjw72srphMEp/w=,tag:amnRWPKfv/qYCwJgOPCxpw==,type:str]
-    STORAGE_CLUSTER_NAME: ENC[AES256_GCM,data:KVWcogpDMR9f2zlN,iv:XhrcT00eLANes7Tv23reeW4+Wp2sBqG9/Bt89k36bqQ=,tag:3xT/oWPT70kErPYvuLLguw==,type:str]
-    STORAGE_DOMAIN: ENC[AES256_GCM,data:plRvWzy/tzeU7+BnKw==,iv:DZpsGeT0+5puoJcPkhqo73UBSFKiDm2sMmOxbX9Ci3A=,tag:jIMQ2eW8XyXupNEnwFY/sw==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -25,8 +23,8 @@ sops:
             a3B3eURqMEhYOXZzZDBRSUVMU2NMcjgK8mSJC86ODpP1kwv4+/gqvr/vAV3WFhRY
             m8vGz0qw50HrYQStXoLE++x2CQnGm3Fi4DvUmtD3GPoFCYzjL49LNQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-05-08T16:51:42Z"
-    mac: ENC[AES256_GCM,data:MZn4BWY1zZTVFxlJcW8PgUs4t9E1LarHltZ5AdTRT9RkNcM77pdLItSD6ZeGqRFDk+JxwZgYbXVRe+tK9XDlgCzmfXN6ErGlBNZ2yGaR37GcEM0T+EU1042eON9/Wx3kdqKd8U9Yxn36ao40CfYIgyF5M85LVLjdych56xclJwI=,iv:cQGyk3nX5ou46t47/zf7ZUZq0SzamlgZsz1adMHWHz0=,tag:aa0KhV23061PL0s91hPrwQ==,type:str]
+    lastmodified: "2024-05-08T21:16:32Z"
+    mac: ENC[AES256_GCM,data:U86PLNzQeh/M3VUmxSCUQ2JgQoX4eGHTp4TFK+wTIfzmfc6g7ph6VvXQeXYI7kcM0RtQxzxfjvMriy2mRpPXBF8vcuuE/G44DiynmVfy+myMwPs+gVPZGRPDM1XheYnB3VAJQY4rVvfA+ZeoPN+U0QhiAHkc3x+N0jMs3WBWsMM=,iv:Sf1QEuDOeowuTRaA0gmmHcXrueFaLMLMcTSFd+pa4vo=,tag:W8lZkxC4bMBMkkZBOKC4IA==,type:str]
     pgp: []
     encrypted_regex: ^(data|stringData)$
     version: 3.8.1