diff --git a/.taskfiles/Volsync/Taskfile.yaml b/.taskfiles/Volsync/Taskfile.yaml new file mode 100644 index 00000000..0fb1d9f1 --- /dev/null +++ b/.taskfiles/Volsync/Taskfile.yaml @@ -0,0 +1,234 @@ +--- +# yaml-language-server: $schema=https://taskfile.dev/schema.json +version: "3" + +# This taskfile is used to manage certain VolSync tasks for a given application, limitations are described below. +# 1. Fluxtomization, HelmRelease, PVC, ReplicationSource all have the same name (e.g. plex) +# 2. ReplicationSource and ReplicationDestination are a Restic repository +# 3. Applications are deployed as either a Kubernetes Deployment or StatefulSet +# 4. Each application only has one PVC that is being replicated + +x-env: &env + app: "{{.app}}" + claim: "{{.claim}}" + cluster: "{{.cluster}}" + controller: "{{.controller}}" + dst: "{{.dst}}" + job: "{{.job}}" + ns: "{{.ns}}" + pgid: "{{.pgid}}" + previous: "{{.previous}}" + puid: "{{.puid}}" + repository: "{{.repository}}" + +vars: + VOLSYNC_SCRIPTS_DIR: "{{.ROOT_DIR}}/.taskfiles/VolSync/scripts" + VOLSYNC_TEMPLATES_DIR: "{{.ROOT_DIR}}/.taskfiles/VolSync/templates" + +tasks: + + state-*: + desc: Suspend or Resume Volsync + summary: | + Args: + cluster: Cluster to run command against (default:main) + state: resume or suspend (required) + cmds: + - flux --context {{.cluster}} {{.state}} kustomization volsync + - flux --context {{.cluster}} -n {{.ns}} {{.state}} helmrelease volsync + - kubectl --context {{.cluster}} -n {{.ns}} scale deployment volsync --replicas {{if eq "suspend" .state}}0{{else}}1{{end}} + env: *env + vars: + cluster: '{{.cluster | default "main"}}' + ns: '{{.ns | default "storage"}}' + state: '{{index .MATCH 0}}' + + list: + desc: List snapshots for an application + summary: | + Args: + cluster: Cluster to run command against (default:main) + ns: Namespace the PVC is in (required) + app: Application to list snapshots for (required) + cmds: + - envsubst < <(cat {{.VOLSYNC_TEMPLATES_DIR}}/list.tmpl.yaml) | kubectl --context {{.cluster}} apply -f - + - bash {{.VOLSYNC_SCRIPTS_DIR}}/wait-for-job.sh {{.job}} {{.ns}} {{.cluster}} + - kubectl --context {{.cluster}} -n {{.ns}} wait job/{{.job}} --for condition=complete --timeout=1m + - kubectl --context {{.cluster}} -n {{.ns}} logs job/{{.job}} --container minio + - kubectl --context {{.cluster}} -n {{.ns}} logs job/{{.job}} --container backblaze + - kubectl --context {{.cluster}} -n {{.ns}} delete job {{.job}} + env: *env + requires: + vars: ["ns","app"] + vars: + cluster: '{{.cluster | default "main"}}' + job: volsync-list-{{.app}} + preconditions: + - test -f {{.VOLSYNC_SCRIPTS_DIR}}/wait-for-job.sh + - test -f {{.VOLSYNC_TEMPLATES_DIR}}/list.tmpl.yaml + # silent: true + + unlock: + desc: Unlock a Restic repositories for an application + summary: | + Args: + cluster: Cluster to run command against (default:main) + ns: Namespace the PVC is in (required) + app: Application to unlock (required) + cmds: + - envsubst < <(cat {{.VOLSYNC_TEMPLATES_DIR}}/unlock.tmpl.yaml) | kubectl --context {{.cluster}} apply -f - + - bash {{.VOLSYNC_SCRIPTS_DIR}}/wait-for-job.sh {{.job}} {{.ns}} {{.cluster}} + - kubectl --context {{.cluster}} -n {{.ns}} wait job/{{.job}} --for condition=complete --timeout=1m + - kubectl --context {{.cluster}} -n {{.ns}} logs job/{{.job}} --container minio + - kubectl --context {{.cluster}} -n {{.ns}} logs job/{{.job}} --container backblaze + - kubectl --context {{.cluster}} -n {{.ns}} delete job {{.job}} + env: *env + requires: + vars: ["ns","app"] + vars: + cluster: '{{.cluster | default "main"}}' + job: volsync-unlock-{{.app}} + preconditions: + - test -f {{.VOLSYNC_SCRIPTS_DIR}}/wait-for-job.sh + - test -f {{.VOLSYNC_TEMPLATES_DIR}}/unlock.tmpl.yaml + silent: true + + # To run backup jobs in parallel for all replicationsources: + # - kubectl get replicationsources --all-namespaces --no-headers | awk '{print $2, $1}' | xargs --max-procs=4 -l bash -c 'task volsync:snapshot app=$0 ns=$1' + backup: + desc: Snapshot a PVC for an application + summary: | + Args: + cluster: Cluster to run command against (default:main) + ns: Namespace the PVC is in (required) + app: Application to snapshot (required) + repository: Restic application from this repository (default: minio) + cmds: + - kubectl --context {{.cluster}} -n {{.ns}} patch replicationsources {{.app}}-{{.repository}} --type merge -p '{"spec":{"trigger":{"manual":"{{.now}}"}}}' + - bash {{.VOLSYNC_SCRIPTS_DIR}}/wait-for-job.sh {{.job}} {{.ns}} {{.cluster}} + - kubectl --context {{.cluster}} -n {{.ns}} wait job/{{.job}} --for condition=complete --timeout=120m + env: *env + requires: + vars: ["ns","app"] + vars: + cluster: '{{.cluster | default "main"}}' + repository: '{{.repository | default "minio"}}' + now: '{{now | date "150405"}}' + job: volsync-src-{{.app}}-{{.repository}} + controller: + sh: true && {{.VOLSYNC_SCRIPTS_DIR}}/which-controller.sh {{.app}} {{.ns}} {{.cluster}} + preconditions: + - test -f {{.VOLSYNC_SCRIPTS_DIR}}/which-controller.sh + - test -f {{.VOLSYNC_SCRIPTS_DIR}}/wait-for-job.sh + - kubectl --context {{.cluster}} -n {{.ns}} get replicationsources {{.app}}-{{.repository}} + + # To run restore jobs in parallel for all replicationdestinations: + # - kubectl get replicationsources --all-namespaces --no-headers | awk '{print $2, $1}' | xargs --max-procs=4 -l bash -c 'task volsync:restore app=$0 ns=$1' + restore: + desc: Restore a PVC for an application + summary: | + Args: + cluster: Cluster to run command against (default:main) + ns: Namespace the PVC is in (required) + app: Application to restore (required) + repository: Restic application from this repository (default: minio) + previous: Previous number of snapshots to restore (default: 2) + cmds: + - { task: .suspend, vars: *env } + - { task: .wipe, vars: *env } + - { task: .restore, vars: *env } + - { task: .resume, vars: *env } + env: *env + requires: + vars: ["ns","app"] + vars: + cluster: '{{.cluster | default "main"}}' + repository: '{{.repository | default "minio"}}' + previous: '{{.previous | default 2}}' + controller: + sh: "{{.VOLSYNC_SCRIPTS_DIR}}/which-controller.sh {{.app}} {{.ns}}" + claim: + sh: kubectl --context {{.cluster}} -n {{.ns}} get replicationsources/{{.app}}-{{.repository}} -o jsonpath="{.spec.sourcePVC}" + puid: + sh: kubectl --context {{.cluster}} -n {{.ns}} get replicationsources/{{.app}}-{{.repository}} -o jsonpath="{.spec.restic.moverSecurityContext.runAsUser}" + pgid: + sh: kubectl --context {{.cluster}} -n {{.ns}} get replicationsources/{{.app}}-{{.repository}} -o jsonpath="{.spec.restic.moverSecurityContext.runAsGroup}" + preconditions: + - test -f {{.VOLSYNC_SCRIPTS_DIR}}/which-controller.sh + - test -f {{.VOLSYNC_SCRIPTS_DIR}}/wait-for-job.sh + - test -f {{.VOLSYNC_TEMPLATES_DIR}}/replicationdestination.tmpl.yaml + - test -f {{.VOLSYNC_TEMPLATES_DIR}}/wipe.tmpl.yaml + + clean-bootstrap: + desc: Delete volume populator PVCs in all namespaces + summary: | + Args: + cluster: Cluster to run command against (default:main) + cmds: + - for: { var: dest } + cmd: | + {{- $items := (split "/" .ITEM) }} + kubectl --context {{.cluster}} delete pvc -n {{ $items._0 }} {{ $items._1 }} + - for: { var: cache } + cmd: | + {{- $items := (split "/" .ITEM) }} + kubectl --context {{.cluster}} delete pvc -n {{ $items._0 }} {{ $items._1 }} + - for: { var: snaps } + cmd: | + {{- $items := (split "/" .ITEM) }} + kubectl --context {{.cluster}} delete volumesnapshot -n {{ $items._0 }} {{ $items._1 }} + env: *env + vars: + cluster: '{{.cluster | default "main"}}' + dest: + sh: kubectl --context {{.cluster}} get pvc --all-namespaces --no-headers | grep "volsync-.*-bootstrap-dest" | awk '{print $1 "/" $2}' + cache: + sh: kubectl --context {{.cluster}} get pvc --all-namespaces --no-headers | grep "volsync-.*-bootstrap-cache" | awk '{print $1 "/" $2}' + snaps: + sh: kubectl --context {{.cluster}} get volumesnapshot --all-namespaces --no-headers | grep "volsync-.*-bootstrap-dest" | awk '{print $1 "/" $2}' + + # Suspend the Flux ks and hr + .suspend: + internal: true + cmds: + - flux --context {{.cluster}} -n flux-system suspend kustomization {{.app}} + - flux --context {{.cluster}} -n {{.ns}} suspend helmrelease {{.app}} + - kubectl --context {{.cluster}} -n {{.ns}} scale {{.controller}} --replicas 0 + - kubectl --context {{.cluster}} -n {{.ns}} wait pod --for delete --selector="app.kubernetes.io/name={{.app}}" --timeout=2m + env: *env + + # Wipe the PVC of all data + .wipe: + internal: true + cmds: + - envsubst < <(cat {{.VOLSYNC_TEMPLATES_DIR}}/wipe.tmpl.yaml) | kubectl --context {{.cluster}} apply -f - + - bash {{.VOLSYNC_SCRIPTS_DIR}}/wait-for-job.sh {{.job}} {{.ns}} {{.cluster}} + - kubectl --context {{.cluster}} -n {{.ns}} wait job/{{.job}} --for condition=complete --timeout=120m + - kubectl --context {{.cluster}} -n {{.ns}} logs job/{{.job}} --container main + - kubectl --context {{.cluster}} -n {{.ns}} delete job {{.job}} + env: *env + vars: + job: volsync-wipe-{{.app}} + + # Create VolSync replicationdestination CR to restore data + .restore: + internal: true + cmds: + - envsubst < <(cat {{.VOLSYNC_TEMPLATES_DIR}}/replicationdestination.tmpl.yaml) | kubectl --context {{.cluster}} apply -f - + - bash {{.VOLSYNC_SCRIPTS_DIR}}/wait-for-job.sh {{.job}} {{.ns}} {{.cluster}} + - kubectl --context {{.cluster}} -n {{.ns}} wait job/{{.job}} --for condition=complete --timeout=120m + - kubectl --context {{.cluster}} -n {{.ns}} delete replicationdestination {{.dst}} + env: *env + vars: + dst: "{{.app}}-restore-{{.repository}}" + # job: volsync-dst-{{.app}}-restore-{{.repository}} + job: volsync-dst-{{.dst}} + + # Resume Flux ks and hr + .resume: + internal: true + cmds: + - flux --context {{.cluster}} -n {{.ns}} resume helmrelease {{.app}} + - flux --context {{.cluster}} -n flux-system resume kustomization {{.app}} + - flux --context {{.cluster}} -n {{.ns}} reconcile helmrelease {{.app}} --force + env: *env diff --git a/.taskfiles/Volsync/scripts/wait-for-job.sh b/.taskfiles/Volsync/scripts/wait-for-job.sh new file mode 100755 index 00000000..2e48889f --- /dev/null +++ b/.taskfiles/Volsync/scripts/wait-for-job.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +JOB=$1 +NAMESPACE="${2:-default}" +CLUSTER="${3:-main}" + +[[ -z "${JOB}" ]] && echo "Job name not specified" && exit 1 +while true; do + STATUS="$(kubectl --context "${CLUSTER}" -n "${NAMESPACE}" get pod -l job-name="${JOB}" -o jsonpath='{.items[*].status.phase}')" + if [ -n "${STATUS}" ]; then + break + fi + sleep 1 +done diff --git a/.taskfiles/Volsync/scripts/which-controller.sh b/.taskfiles/Volsync/scripts/which-controller.sh new file mode 100755 index 00000000..15f1e9e0 --- /dev/null +++ b/.taskfiles/Volsync/scripts/which-controller.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +APP=$1 +NAMESPACE="${2:-default}" +CLUSTER="${3:-main}" + +is_deployment() { + kubectl --context "${CLUSTER}" -n "${NAMESPACE}" get deployment "${APP}" >/dev/null 2>&1 +} + +is_statefulset() { + kubectl --context "${CLUSTER}" -n "${NAMESPACE}" get statefulset "${APP}" >/dev/null 2>&1 +} + +if is_deployment; then + echo "deployment.apps/${APP}" +elif is_statefulset; then + echo "statefulset.apps/${APP}" +else + echo "No deployment or statefulset found for ${APP}" + exit 1 +fi diff --git a/.taskfiles/Volsync/templates/list.tmpl.yaml b/.taskfiles/Volsync/templates/list.tmpl.yaml new file mode 100644 index 00000000..c2a892c9 --- /dev/null +++ b/.taskfiles/Volsync/templates/list.tmpl.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ${job} + namespace: ${ns} +spec: + ttlSecondsAfterFinished: 3600 + template: + spec: + automountServiceAccountToken: false + restartPolicy: OnFailure + containers: + - name: minio + image: docker.io/restic/restic:0.16.4 + args: ["snapshots"] + envFrom: + - secretRef: + name: ${app}-volsync-minio + resources: {} + - name: backblaze + image: docker.io/restic/restic:0.16.4 + args: ["snapshots"] + envFrom: + - secretRef: + name: ${app}-volsync-backblaze + resources: {} diff --git a/.taskfiles/Volsync/templates/replicationdestination.tmpl.yaml b/.taskfiles/Volsync/templates/replicationdestination.tmpl.yaml new file mode 100644 index 00000000..88303628 --- /dev/null +++ b/.taskfiles/Volsync/templates/replicationdestination.tmpl.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: volsync.backube/v1alpha1 +kind: ReplicationDestination +metadata: + name: ${dst} + namespace: ${ns} +spec: + trigger: + manual: restore-once + restic: + repository: ${app}-volsync-${repository} + destinationPVC: ${claim} + copyMethod: Direct + storageClassName: ceph-block + # storageClassName: ceph-filesystem + # accessModes: ["ReadWriteMany"] + # IMPORTANT NOTE: + # Set to the last X number of snapshots to restore from + previous: ${previous} + # OR; + # IMPORTANT NOTE: + # On bootstrap set `restoreAsOf` to the time the old cluster was destroyed. + # This will essentially prevent volsync from trying to restore a backup + # from a application that started with default data in the PVC. + # Do not restore snapshots made after the following RFC3339 Timestamp. + # date --rfc-3339=seconds (--utc) + # restoreAsOf: "2022-12-10T16:00:00-05:00" + moverSecurityContext: + runAsUser: ${puid} + runAsGroup: ${pgid} + fsGroup: ${pgid} diff --git a/.taskfiles/Volsync/templates/unlock.tmpl.yaml b/.taskfiles/Volsync/templates/unlock.tmpl.yaml new file mode 100644 index 00000000..d9d3da00 --- /dev/null +++ b/.taskfiles/Volsync/templates/unlock.tmpl.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ${job} + namespace: ${ns} +spec: + ttlSecondsAfterFinished: 3600 + template: + spec: + automountServiceAccountToken: false + restartPolicy: OnFailure + containers: + - name: minio + image: docker.io/restic/restic:0.16.4 + args: ["unlock", "--remove-all"] + envFrom: + - secretRef: + name: ${app}-volsync-minio + resources: {} + - name: backblaze + image: docker.io/restic/restic:0.16.4 + args: ["unlock", "--remove-all"] + envFrom: + - secretRef: + name: ${app}-volsync-backblaze + resources: {} diff --git a/.taskfiles/Volsync/templates/wipe.tmpl.yaml b/.taskfiles/Volsync/templates/wipe.tmpl.yaml new file mode 100644 index 00000000..ffc1cc75 --- /dev/null +++ b/.taskfiles/Volsync/templates/wipe.tmpl.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ${job} + namespace: ${ns} +spec: + ttlSecondsAfterFinished: 3600 + template: + spec: + automountServiceAccountToken: false + restartPolicy: OnFailure + containers: + - name: main + image: docker.io/library/alpine:latest + command: ["/bin/sh", "-c", "cd /config; find . -delete"] + volumeMounts: + - name: config + mountPath: /config + securityContext: + privileged: true + resources: {} + volumes: + - name: config + persistentVolumeClaim: + claimName: ${claim} diff --git a/Taskfile.yaml b/Taskfile.yaml index 91b369ae..763b2643 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -33,6 +33,7 @@ includes: taskfile: .taskfiles/Repository/Taskfile.yaml talos: .taskfiles/Talos/Taskfile.yaml sops: .taskfiles/Sops/Taskfile.yaml + volsync: .taskfiles/Volsync/Taskfile.yaml workstation: .taskfiles/Workstation/Taskfile.yaml user: taskfile: .taskfiles/User diff --git a/kubernetes/main/apps/security/external-secrets/stores/kustomization.yaml b/kubernetes/main/apps/security/external-secrets/stores/kustomization.yaml index 8fb7c142..f76ee58a 100644 --- a/kubernetes/main/apps/security/external-secrets/stores/kustomization.yaml +++ b/kubernetes/main/apps/security/external-secrets/stores/kustomization.yaml @@ -2,4 +2,5 @@ # yaml-language-server: $schema=https://json.schemastore.org/kustomization apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -resources: [] +resources: + - ./volsync-secrets diff --git a/kubernetes/main/apps/security/external-secrets/stores/volsync-secrets/clustersecretstore.yaml b/kubernetes/main/apps/security/external-secrets/stores/volsync-secrets/clustersecretstore.yaml new file mode 100644 index 00000000..4a97ce0c --- /dev/null +++ b/kubernetes/main/apps/security/external-secrets/stores/volsync-secrets/clustersecretstore.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: external-secrets.io/v1beta1 +kind: ClusterSecretStore +metadata: + name: volsync-secrets-store +spec: + provider: + kubernetes: + remoteNamespace: storage + auth: + serviceAccount: + name: volsync-secrets-manager + namespace: security + server: + caProvider: + type: ConfigMap + name: kube-root-ca.crt + namespace: security + key: ca.crt diff --git a/kubernetes/main/apps/security/external-secrets/stores/volsync-secrets/kustomization.yaml b/kubernetes/main/apps/security/external-secrets/stores/volsync-secrets/kustomization.yaml new file mode 100644 index 00000000..c2a0b8f2 --- /dev/null +++ b/kubernetes/main/apps/security/external-secrets/stores/volsync-secrets/kustomization.yaml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./clustersecretstore.yaml + - ./serviceaccount.yaml diff --git a/kubernetes/main/apps/security/external-secrets/stores/volsync-secrets/serviceaccount.yaml b/kubernetes/main/apps/security/external-secrets/stores/volsync-secrets/serviceaccount.yaml new file mode 100644 index 00000000..d1e0eb08 --- /dev/null +++ b/kubernetes/main/apps/security/external-secrets/stores/volsync-secrets/serviceaccount.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: volsync-secrets-manager diff --git a/kubernetes/main/apps/storage/kustomization.yaml b/kubernetes/main/apps/storage/kustomization.yaml index 20304be9..fed6927b 100644 --- a/kubernetes/main/apps/storage/kustomization.yaml +++ b/kubernetes/main/apps/storage/kustomization.yaml @@ -3,7 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ./namespace.yaml - - ./snapshot-controller/ks.yaml - - ./minio/ks.yaml - ./openebs/ks.yaml - ./rook-ceph/ks.yaml + - ./snapshot-controller/ks.yaml + - ./volsync/ks.yaml diff --git a/kubernetes/main/apps/storage/openebs/app/local-database-sc.yaml b/kubernetes/main/apps/storage/openebs/app/local-database-sc.yaml index 8493792f..1e7542b9 100644 --- a/kubernetes/main/apps/storage/openebs/app/local-database-sc.yaml +++ b/kubernetes/main/apps/storage/openebs/app/local-database-sc.yaml @@ -11,6 +11,6 @@ parameters: compresion: lz4 dedup: off thinprovision: yes - shared: no + shared: yes provisioner: zfs.csi.openebs.io volumeBindingMode: WaitForFirstConsumer diff --git a/kubernetes/main/apps/storage/volsync/app/helmrelease.yaml b/kubernetes/main/apps/storage/volsync/app/helmrelease.yaml new file mode 100644 index 00000000..e1acac3a --- /dev/null +++ b/kubernetes/main/apps/storage/volsync/app/helmrelease.yaml @@ -0,0 +1,31 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json +apiVersion: helm.toolkit.fluxcd.io/v2beta2 +kind: HelmRelease +metadata: + name: volsync +spec: + interval: 30m + chart: + spec: + chart: volsync + version: 0.9.1 + sourceRef: + kind: HelmRepository + name: backube + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + strategy: rollback + retries: 3 + dependsOn: + - name: openebs + - name: rook-ceph-cluster + - name: snapshot-controller + values: + metrics: + disableAuth: true diff --git a/kubernetes/main/apps/storage/volsync/app/kustomization.yaml b/kubernetes/main/apps/storage/volsync/app/kustomization.yaml new file mode 100644 index 00000000..c11fd893 --- /dev/null +++ b/kubernetes/main/apps/storage/volsync/app/kustomization.yaml @@ -0,0 +1,9 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml + - ./role.yaml + - ./rolebinding.yaml + - ./secret.sops.yaml diff --git a/kubernetes/main/apps/storage/volsync/app/role.yaml b/kubernetes/main/apps/storage/volsync/app/role.yaml new file mode 100644 index 00000000..2c850cc5 --- /dev/null +++ b/kubernetes/main/apps/storage/volsync/app/role.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: volsync-secrets-manager +rules: + - apiGroups: [""] + resources: + - secrets + resourceNames: + - volsync-secret + verbs: + - get + - list + - watch + - apiGroups: + - authorization.k8s.io + resources: + - selfsubjectrulesreviews + verbs: + - create diff --git a/kubernetes/main/apps/storage/volsync/app/rolebinding.yaml b/kubernetes/main/apps/storage/volsync/app/rolebinding.yaml new file mode 100644 index 00000000..7e6faea1 --- /dev/null +++ b/kubernetes/main/apps/storage/volsync/app/rolebinding.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: volsync-secrets-manager-binding +subjects: + - kind: ServiceAccount + name: volsync-secrets-manager + namespace: security +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: volsync-secrets-manager diff --git a/kubernetes/main/apps/storage/volsync/app/secret.sops.yaml b/kubernetes/main/apps/storage/volsync/app/secret.sops.yaml new file mode 100644 index 00000000..9450796c --- /dev/null +++ b/kubernetes/main/apps/storage/volsync/app/secret.sops.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Secret +metadata: + name: volsync-secret +stringData: + BACKBLAZE_ACCESS_KEY: ENC[AES256_GCM,data:BYVsKQRGLPzQNXmKuFqE/zxpDodXqklnNw==,iv:IQqQprf8SU4tVl3itLHVF5RzIdEyOagy7k4PhzH1EBk=,tag:N/uMXGppD3nNhkioLGemLQ==,type:str] + BACKBLAZE_SECRET_KEY: ENC[AES256_GCM,data:ggPtXu2J2WAA0cEsFOadauKVMZf72qiW9KRwzOVsrg==,iv:omm4AvVPsSiX0ZgSwuwVHkLZ7vfkhlJFbZNv5ir8kBM=,tag:JILjUcJ4HELIg1xwq34M3Q==,type:str] + BACKBLAZE_REPOSITORY: ENC[AES256_GCM,data:U+RYL4dMwkHhQE5Iu41SrRbxkjFu3FYfEteAz4ZqPjn7D44a4FmSXEYb6xddWRQrHIStx4BOHsWyUcc8htlupOT4yg==,iv:1zG7sPXpUeT+m1nfC2q1U/k1pv2JTP2mXfTqLlqGUxg=,tag:jFsrVMGpGc+0hGXr7R1N4A==,type:str] + MINIO_ACCESS_KEY: ENC[AES256_GCM,data:HuvbaVh8m5Qb/hy1Sl5aYAtquP0=,iv:KBU1aP8duts53ylM9VRrtunQesblHHO/NPY+bh4asDU=,tag:cRSG3uiIoDRb34+0Q6hRJg==,type:str] + MINIO_SECRET_KEY: ENC[AES256_GCM,data:tvB7rgl/lFOSt7q6gOCEWRCzZlyCTqz15DlyRRUQtFwcBJfILPF+eA==,iv:1U+2zV21DVEZxBz9B/n0GjxVVUPY1fuwHpVtIea3Xvo=,tag:GdVRxmbpoTwhF1dcFHPKqw==,type:str] + MINIO_REPOSITORY: ENC[AES256_GCM,data:7P7iUH06dmH8tGDfdiH0Fa2VI+6ZRcH24chSiz9nM2u+fxcm1qT3ENgnxtcK/nn4H8A09Y/o,iv:zBu6Kj7H/JH/72L7nSYUwF6xv8uVb05QoU/XyemLGEg=,tag:7rdQq0Zv0Bt0DP+jjmmjjA==,type:str] + RESTIC_PASSWORD: ENC[AES256_GCM,data:OutPX3ZJDNcW4ZnSxeUj/47RBKf1bbkge/52H3xfTy3XO7n4BObuEHOqfEjITnxt0dpDM588a0bTclmB8MoDJQ==,iv:8RSryIOUNsLj6c9JlAYPyWSia87ojwqe12/CRUxXlas=,tag:YoidVddlHlXl4q7MOP9Ljw==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1ve9kzacrwq7l9l0emvs326uk6t576d75r596e083r2tq6xu28qcsacy3s7 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVK1FDUE5ydDM2SzA0ZWFr + ZElOb1JXUjA3Z0NIYW1wZnJvZjdXNC8zV0Y0CjRKbHpvVmtKRVIvRmF4V1dGY3ly + SWpuc0RhcEdqcHpKcTZiNko5VG1LS2cKLS0tIFVvOERtQVF4ZFVTMk1MZ3A1QnFT + RFN6U2lQU2Q3M0wwYmNuc0pLN1IzczQKwRAph52iWkz4M/qYBepyfe0OKZV9Qyje + LglPEeuTwBc+sybPVtiXIcFzPfIcI03yPsvdFbTbn5eRJWBOA7t3JQ== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2024-05-12T18:47:06Z" + mac: ENC[AES256_GCM,data:D39SZqxPKAdi+2nPeDiC7/hdugYxTOMsB1ZnQ0Ezce2T6IhmylsOfzmrj+wRo7JJkNp4uOV6dPpFBOuntQuGVducByCrtMjLbNs+Y60dEODxSVGGziUUtLmrvjmvuv8jKYnIrHsHUvt0ZOQbjXBpX279jSi49La7mAXMOliRssQ=,iv:Ap918qkqmUAVotWCdnzqfStI6aRQfEBjK5g8QvGxNR8=,tag:U7V5up7tv+rWbUY5TiiA0g==,type:str] + pgp: [] + encrypted_regex: ^(data|stringData)$ + version: 3.8.1 diff --git a/kubernetes/main/apps/storage/volsync/ks.yaml b/kubernetes/main/apps/storage/volsync/ks.yaml new file mode 100644 index 00000000..9d04d664 --- /dev/null +++ b/kubernetes/main/apps/storage/volsync/ks.yaml @@ -0,0 +1,21 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app volsync + namespace: flux-system +spec: + targetNamespace: storage + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/main/apps/storage/volsync/app + prune: true + sourceRef: + kind: GitRepository + name: home-ops + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/kubernetes/main/flux/repositories/helm/backube.yaml b/kubernetes/main/flux/repositories/helm/backube.yaml new file mode 100644 index 00000000..7490095c --- /dev/null +++ b/kubernetes/main/flux/repositories/helm/backube.yaml @@ -0,0 +1,10 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/source.toolkit.fluxcd.io/helmrepository_v1beta2.json +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: backube + namespace: flux-system +spec: + interval: 1h + url: https://backube.github.io/helm-charts/ diff --git a/kubernetes/main/flux/repositories/helm/kustomization.yaml b/kubernetes/main/flux/repositories/helm/kustomization.yaml index f250c4ff..042998cf 100644 --- a/kubernetes/main/flux/repositories/helm/kustomization.yaml +++ b/kubernetes/main/flux/repositories/helm/kustomization.yaml @@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - ./backube.yaml - ./bjw-s.yaml - ./cilium.yaml - ./descheduler.yaml diff --git a/kubernetes/main/templates/volsync/backblaze/externalsecret.yaml b/kubernetes/main/templates/volsync/backblaze/externalsecret.yaml new file mode 100644 index 00000000..881acdfa --- /dev/null +++ b/kubernetes/main/templates/volsync/backblaze/externalsecret.yaml @@ -0,0 +1,23 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/external-secrets.io/externalsecret_v1beta1.json +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: "${APP}-volsync-backblaze" +spec: + secretStoreRef: + kind: ClusterSecretStore + name: volsync-secrets-store + dataFrom: + - extract: + key: volsync-secret + target: + name: "${APP}-volsync-backblaze" + creationPolicy: Owner + template: + engineVersion: v2 + data: + AWS_ACCESS_KEY_ID: "{{ .BACKBLAZE_ACCESS_KEY }}" + AWS_SECRET_ACCESS_KEY: "{{ .BACKBLAZE_SECRET_KEY }}" + RESTIC_REPOSITORY: "{{ .BACKBLAZE_REPOSITORY }}/${APP}" + RESTIC_PASSWORD: "{{ .RESTIC_PASSWORD }}" diff --git a/kubernetes/main/templates/volsync/backblaze/kustomization.yaml b/kubernetes/main/templates/volsync/backblaze/kustomization.yaml new file mode 100644 index 00000000..04467fd1 --- /dev/null +++ b/kubernetes/main/templates/volsync/backblaze/kustomization.yaml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./externalsecret.yaml + - ./replicationsource.yaml diff --git a/kubernetes/main/templates/volsync/backblaze/replicationsource.yaml b/kubernetes/main/templates/volsync/backblaze/replicationsource.yaml new file mode 100644 index 00000000..6385ae55 --- /dev/null +++ b/kubernetes/main/templates/volsync/backblaze/replicationsource.yaml @@ -0,0 +1,25 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/volsync.backube/replicationsource_v1alpha1.json +apiVersion: volsync.backube/v1alpha1 +kind: ReplicationSource +metadata: + name: "${APP}-backblaze" +spec: + sourcePVC: "${APP}" + trigger: + schedule: "0 0 * * *" + restic: + copyMethod: Snapshot + repository: ${APP}-volsync-backblaze + cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-local-generic}" + cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"] + cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-1Gi}" + storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}" + volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-csi-ceph-blockpool}" + moverSecurityContext: + runAsUser: ${APP_UID:-568} + runAsGroup: ${APP_GID:-568} + fsGroup: ${APP_GID:-568} + pruneIntervalDays: 7 + retain: + daily: 7 diff --git a/kubernetes/main/templates/volsync/kustomization.yaml b/kubernetes/main/templates/volsync/kustomization.yaml new file mode 100644 index 00000000..0b260f90 --- /dev/null +++ b/kubernetes/main/templates/volsync/kustomization.yaml @@ -0,0 +1,8 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./backblaze + - ./minio + - ./pvc.yaml diff --git a/kubernetes/main/templates/volsync/minio/externalsecret.yaml b/kubernetes/main/templates/volsync/minio/externalsecret.yaml new file mode 100644 index 00000000..775514ba --- /dev/null +++ b/kubernetes/main/templates/volsync/minio/externalsecret.yaml @@ -0,0 +1,23 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/external-secrets.io/externalsecret_v1beta1.json +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: "${APP}-volsync-minio" +spec: + secretStoreRef: + kind: ClusterSecretStore + name: volsync-secrets-store + dataFrom: + - extract: + key: volsync-secret + target: + name: "${APP}-volsync-minio" + creationPolicy: Owner + template: + engineVersion: v2 + data: + AWS_ACCESS_KEY_ID: "{{ .MINIO_ACCESS_KEY }}" + AWS_SECRET_ACCESS_KEY: "{{ .MINIO_SECRET_KEY }}" + RESTIC_REPOSITORY: "{{ .MINIO_REPOSITORY }}/${APP}" + RESTIC_PASSWORD: "{{ .RESTIC_PASSWORD }}" diff --git a/kubernetes/main/templates/volsync/minio/kustomization.yaml b/kubernetes/main/templates/volsync/minio/kustomization.yaml new file mode 100644 index 00000000..3d2cb18d --- /dev/null +++ b/kubernetes/main/templates/volsync/minio/kustomization.yaml @@ -0,0 +1,8 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./externalsecret.yaml + - ./replicationdestination.yaml + - ./replicationsource.yaml diff --git a/kubernetes/main/templates/volsync/minio/replicationdestination.yaml b/kubernetes/main/templates/volsync/minio/replicationdestination.yaml new file mode 100644 index 00000000..dfb78f56 --- /dev/null +++ b/kubernetes/main/templates/volsync/minio/replicationdestination.yaml @@ -0,0 +1,24 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/volsync.backube/replicationdestination_v1alpha1.json +apiVersion: volsync.backube/v1alpha1 +kind: ReplicationDestination +metadata: + name: "${APP}-bootstrap" +spec: + trigger: + manual: restore-once + restic: + copyMethod: Snapshot + repository: "${APP}-volsync-minio" + cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-local-generic}" + cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"] + cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-1Gi}" + storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}" + volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-csi-ceph-blockpool}" + moverSecurityContext: + runAsUser: ${APP_UID:-568} + runAsGroup: ${APP_GID:-568} + fsGroup: ${APP_GID:-568} + accessModes: + - "${VOLSYNC_ACCESSMODES:-ReadWriteOnce}" + capacity: "${VOLSYNC_CAPACITY:-1Gi}" diff --git a/kubernetes/main/templates/volsync/minio/replicationsource.yaml b/kubernetes/main/templates/volsync/minio/replicationsource.yaml new file mode 100644 index 00000000..6b235f37 --- /dev/null +++ b/kubernetes/main/templates/volsync/minio/replicationsource.yaml @@ -0,0 +1,27 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/volsync.backube/replicationsource_v1alpha1.json +apiVersion: volsync.backube/v1alpha1 +kind: ReplicationSource +metadata: + name: "${APP}-minio" +spec: + sourcePVC: "${APP}" + trigger: + schedule: "0 * * * *" + restic: + copyMethod: Snapshot + repository: ${APP}-volsync-minio + cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-local-generic}" + cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"] + cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-1Gi}" + storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}" + volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-csi-ceph-blockpool}" + moverSecurityContext: + runAsUser: ${APP_UID:-568} + runAsGroup: ${APP_GID:-568} + fsGroup: ${APP_GID:-568} + pruneIntervalDays: 7 + retain: + hourly: 24 + daily: 7 + weekly: 5 diff --git a/kubernetes/main/templates/volsync/pvc.yaml b/kubernetes/main/templates/volsync/pvc.yaml new file mode 100644 index 00000000..d611ca72 --- /dev/null +++ b/kubernetes/main/templates/volsync/pvc.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: "${APP}" +spec: + accessModes: + - "${VOLSYNC_ACCESSMODES:-ReadWriteOnce}" + dataSourceRef: + kind: ReplicationDestination + apiGroup: volsync.backube + name: "${APP}-bootstrap" + resources: + requests: + storage: "${VOLSYNC_CAPACITY:-1Gi}" + storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}" diff --git a/kubernetes/storage/apps/storage/openebs/app/local-database-sc.yaml b/kubernetes/storage/apps/storage/openebs/app/local-database-sc.yaml index 8493792f..1e7542b9 100644 --- a/kubernetes/storage/apps/storage/openebs/app/local-database-sc.yaml +++ b/kubernetes/storage/apps/storage/openebs/app/local-database-sc.yaml @@ -11,6 +11,6 @@ parameters: compresion: lz4 dedup: off thinprovision: yes - shared: no + shared: yes provisioner: zfs.csi.openebs.io volumeBindingMode: WaitForFirstConsumer diff --git a/kubernetes/storage/flux/vars/cluster-secrets.sops.yaml b/kubernetes/storage/flux/vars/cluster-secrets.sops.yaml index c8c1a5be..748b9f55 100644 --- a/kubernetes/storage/flux/vars/cluster-secrets.sops.yaml +++ b/kubernetes/storage/flux/vars/cluster-secrets.sops.yaml @@ -8,8 +8,6 @@ stringData: SECRET_DOMAIN: ENC[AES256_GCM,data:OZEIzVX+AozGHM/X3w==,iv:EieC8Xo6RV59JxCrisie4HkD+Z8KvfOGReiHYuPozMw=,tag:ToxxSaMMVrXgkSpMrUOzGw==,type:str] SECRET_ACME_EMAIL: ENC[AES256_GCM,data:dx7jXat1y9g=,iv:8YgKWAJoynF30NXjD3r7dnUs5upHoeIhvlJ6yJnAXHU=,tag:NVORv/PEjhf6ViwxKvPRTQ==,type:str] SECRET_CLOUDFLARE_TUNNEL_ID: ENC[AES256_GCM,data:ypCuh2C+0R5rFRsIdiFxi8giFMukj8nKbCKJCAJ2HuGwOXbS,iv:yyEdTwS9HfRJvfXDpq41GG1b3wcPeGjw72srphMEp/w=,tag:amnRWPKfv/qYCwJgOPCxpw==,type:str] - STORAGE_CLUSTER_NAME: ENC[AES256_GCM,data:KVWcogpDMR9f2zlN,iv:XhrcT00eLANes7Tv23reeW4+Wp2sBqG9/Bt89k36bqQ=,tag:3xT/oWPT70kErPYvuLLguw==,type:str] - STORAGE_DOMAIN: ENC[AES256_GCM,data:plRvWzy/tzeU7+BnKw==,iv:DZpsGeT0+5puoJcPkhqo73UBSFKiDm2sMmOxbX9Ci3A=,tag:jIMQ2eW8XyXupNEnwFY/sw==,type:str] sops: kms: [] gcp_kms: [] @@ -25,8 +23,8 @@ sops: a3B3eURqMEhYOXZzZDBRSUVMU2NMcjgK8mSJC86ODpP1kwv4+/gqvr/vAV3WFhRY m8vGz0qw50HrYQStXoLE++x2CQnGm3Fi4DvUmtD3GPoFCYzjL49LNQ== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-05-08T16:51:42Z" - mac: ENC[AES256_GCM,data:MZn4BWY1zZTVFxlJcW8PgUs4t9E1LarHltZ5AdTRT9RkNcM77pdLItSD6ZeGqRFDk+JxwZgYbXVRe+tK9XDlgCzmfXN6ErGlBNZ2yGaR37GcEM0T+EU1042eON9/Wx3kdqKd8U9Yxn36ao40CfYIgyF5M85LVLjdych56xclJwI=,iv:cQGyk3nX5ou46t47/zf7ZUZq0SzamlgZsz1adMHWHz0=,tag:aa0KhV23061PL0s91hPrwQ==,type:str] + lastmodified: "2024-05-08T21:16:32Z" + mac: ENC[AES256_GCM,data:U86PLNzQeh/M3VUmxSCUQ2JgQoX4eGHTp4TFK+wTIfzmfc6g7ph6VvXQeXYI7kcM0RtQxzxfjvMriy2mRpPXBF8vcuuE/G44DiynmVfy+myMwPs+gVPZGRPDM1XheYnB3VAJQY4rVvfA+ZeoPN+U0QhiAHkc3x+N0jMs3WBWsMM=,iv:Sf1QEuDOeowuTRaA0gmmHcXrueFaLMLMcTSFd+pa4vo=,tag:W8lZkxC4bMBMkkZBOKC4IA==,type:str] pgp: [] encrypted_regex: ^(data|stringData)$ version: 3.8.1