Merge pull request #257 from barrucadu/restic

Switch from duplicity to restic for backups

docs/src/runbooks/set-up-a-new-host.md

@@ -153,18 +153,16 @@ nix run .#secrets
 Then enable backups in the host configuration:
 
 ```nix
-nixfiles.backups.enable = true;
-nixfiles.backups.environmentFile = config.sops.secrets."nixfiles/backups/env".path;
-sops.secrets."nixfiles/backups/env" = { };
+nixfiles.restic-backups.enable = true;
+nixfiles.restic-backups.environmentFile = config.sops.secrets."nixfiles/restic-backups/env".path;
+sops.secrets."nixfiles/restic-backups/env" = { };
 ```
 
 Most services define their own backup scripts.  For any other needs, write a
-custom script:
+custom backup job:
 
 ```nix
-nixfiles.backups.scripts.<name> = ''
-  <script which copies files to backup to the current working directory>
-'';
+nixfiles.restic-backups.backups.<name> = { ... };
 ```
 
 

flake.nix

@@ -70,7 +70,7 @@
         in
         {
           backups = mkApp "backups" ''
-            PATH=${with pkgs; lib.makeBinPath [ duplicity sops nettools ]}
+            PATH=${with pkgs; lib.makeBinPath [ restic sops nettools ]}
 
             ${pkgs.lib.fileContents ./scripts/backups.sh}
           '';
@@ -111,9 +111,9 @@
                   ./shared/bookdb/options.nix
                   ./shared/minecraft/options.nix
                   ./shared/erase-your-darlings/options.nix
-                  ./shared/backups/options.nix
                   ./shared/foundryvtt/options.nix
                   ./shared/finder/options.nix
+                  ./shared/restic-backups/options.nix
                 ];
               };
               optionsDoc = pkgs.nixosOptionsDoc {

hosts/carcosa/configuration.nix

@@ -64,18 +64,33 @@ in
   ## Backups
   ###############################################################################
 
-  nixfiles.backups.enable = true;
-  nixfiles.backups.environmentFile = config.sops.secrets."nixfiles/backups/env".path;
-  nixfiles.backups.scripts.syncthing = "cp -a /home/barrucadu/s .";
-  # TODO: this will break when I have >100 github repos
-  nixfiles.backups.scripts.git = ''
-    curl -u "barrucadu:''${GITHUB_TOKEN}" 'https://api.github.com/user/repos?type=owner&per_page=100' 2>/dev/null | \
-      jq -r '.[].ssh_url' | \
-      while read url; do
-        git clone --bare "$url"
-      done
-  '';
-  sops.secrets."nixfiles/backups/env" = { };
+  nixfiles.restic-backups.enable = true;
+  nixfiles.restic-backups.environmentFile = config.sops.secrets."nixfiles/restic-backups/env".path;
+  nixfiles.restic-backups.checkRepositoryAt = "Wed, 12:00";
+  nixfiles.restic-backups.backups.github = {
+    # TODO: this will break when I have >100 github repos
+    # TODO: use a backup-specific SSH key?
+    prepareCommand = ''
+      ${pkgs.coreutils}/bin/mkdir repositories
+      cd repositories
+
+      ${pkgs.curl}/bin/curl -u "barrucadu:''${GITHUB_TOKEN}" 'https://api.github.com/user/repos?type=owner&per_page=100' 2>/dev/null | \
+        ${pkgs.jq}/bin/jq -r '.[].ssh_url' | \
+        while read url; do
+          env GIT_SSH_COMMAND="${pkgs.openssh}/bin/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i /home/barrucadu/.ssh/id_ed25519" \
+            ${pkgs.git}/bin/git clone --bare "$url"
+        done
+    '';
+    paths = [
+      "repositories"
+    ];
+  };
+  nixfiles.restic-backups.backups.syncthing = {
+    paths = [
+      "/home/barrucadu/s"
+    ];
+  };
+  sops.secrets."nixfiles/restic-backups/env" = { };
 
 
   ###############################################################################

hosts/carcosa/secrets.yaml

@@ -1,14 +1,14 @@
 users:
     barrucadu: ENC[AES256_GCM,data:5mAoxmMEn4zOhwDHS4cal7b0fPZb7SshS3QmfrBgLKQMJfExiRdnWvwnqTvLY/ObRTYqeWrcik+GVyeFF4gkMm8FxcR7U5ExIXBbtg5uDQgcvQYNlWEfJWEjVYrfKCoIxCAhUDf+2oNr,iv:YEpqdGz/DNdjxNmIrVgahEoH7BNYnqBEHV6L7UrV2aI=,tag:4teSWBW+SJ06KCoTMwjN6g==,type:str]
 nixfiles:
-    backups:
-        env: ENC[AES256_GCM,data:U3PwGmQkEMHEYE89g5LlzVAjftG4b7WIvvfNDOw9OUpKjmZX23SlfvDYwZhAABexAQyG1Izi+wdBnnLbD/as7sTgGhQk7K+mK2JcVnekCx8tBggvDrsu+rJqegFYj/fmGrVI4DfcD8d6V7/sc5/oIiERM3xp78ZaNgwUC7+FWqljcNcEGD36+GXeSXR+4xjDclC9iZKYYOM1Ef4FHj21YrUA3FYXEMmvQO87PLYQ745+RxVgOKy62hcjSpgf201i55WCBWVpS4GrY6DI2Vg/QoEp303zIymhTsxu+VS7hHGctwZY/9jEzQn6TnDI3SEsNxRPaorSLvYgHXnYWiNjeYndUJbb9A/9+Q3EHgRXkkKsKyjobvV+AIU5O4wv9wRGF61GjUjd3f9AmYa/1M/QZWsJOjoYnuG5925M3V3lK1Df6A7lhNEbX8kPtp+zXsr1ubR1Vtg5DbSKdLSjBm+TAOKq0VmwZ7FN+fvAX8CZV7NjurlZ7Q7Os75aXlDAxyMYCsiE5mKcr0vmEmXAKEyVicjDcNeVVOBsv5uaeHiTPmi2Eiq7itVtgV5PLYQ3QoISJwumSJ9i3IaI9P+fTLEzAeFC9ykR8uWpV7a3wXqND0uQ2FNbgVeyuZ492adaVH3w3eHWSHfB0xQCpeUD4SuyC71Cmn+4NoU3uVO9L1w7R5Nl2rE0DbcxDbLXTHwmJegIaHHN4cnxCZBU8Ec7moQMLAZ24NydTF2U5fBjYBCz3itoVfwx5gEuiSnaPVxZ81zZ2fZSB5gcKtFRjAYnZKfwkBcnHpxI6FWkHHvt/fgPkjhiNKkjVEzHicD5g7HLwalwNkUy+hBOyBPhNrOlX3cUvjdgoROCrs1xuWWR2RdZYnE1HzDDLVja8TmjEOIqzRW+wABws3vGQK/3RCPZ1aQmb5c7LF3vTJPeyZDoRXghs7Xy6SCpEJRnnMqWaPkKh3yV6BWda8dn9kFv4HuOZYr1zo2i/2Efja8p3uHpbcfMrQLfbs9u72eM0efB92LsVxYiDMfZdQUdEdo=,iv:RSK2SK73RM//TUw2PNRF9UXmNCE+RkEWdrPGdMGCkw8=,tag:j6R8qE0+pO7CxTxcVmRs5Q==,type:str]
     concourse:
         env: ENC[AES256_GCM,data:PmopaM/Gn8sHW+rOSQM2scZ+KBjlhsgiAGO0nLtcflZE6gBq81lxnK2Sq/w9YUjfD2OdTLAMGPfSyYv2oZNQxBeXYbr8UY+FeY6qorFT9c0H7pwR0vYqmhlKc+3X4EQuqm+QzBT7SXUCdoSKlM6Q0nS0pvQUU67iyTk6NhPCagVRub4vq0oYMCfeiGWCHKqILzd4JUi/Ijk+W2BTlLkT3YQZPZugz7cZcZybUS60hkCRAE91Up9iHmcDIPwvi8gwxkssnby2mzK9u6wEtUtSSh+628KkQsXHoN4XSUuhiwJDO/3513c4IEQuZIV72v5t6sCRkI3UkuQ4GH8yrBM4XP0FGADAOVQObwO+a6xry8y9L2U=,iv:YQ10+rk4rQtZlRvG7cr2SiTMRFI8YzH2rcO+c746mkw=,tag:urQeoA15qNWiDDHmf+81Kw==,type:str]
     firewall:
         ip_blocklist: ENC[AES256_GCM,data:WgDvhHX/CTbbBRVJVr5v4nLQP5sxI8A5KomrSoXble7eUlAZeXqnhw4UT/EugIKaudZzVWovHe30G1U9W86X1/dgAj1S8hD0mA0TH6Qul3RfmQdqnZIdJXfDBLH1mF4OXaox6A==,iv:PVKlgOErmZGAC2tf5mW7d3S7vvMRclrpr+aDd/xJ1WI=,tag:EeXzbX9BELSCeBcxifbHPQ==,type:str]
     pleroma:
         exc: ENC[AES256_GCM,data:ZyA/hBlQs8S56oy7J1f9u36nYBFwWH4Ehq5VbjYDAxcThgP5x09qqt0G8URuLSbBZYuWOOrCjSF1gkt9oLKBPYw2bKxf0yg69GL8xkBtZuJU9NNLXcVswG8HZFWG6Rwa6F65GIHQWdZLwkA8ef5RcYD77FFV3YG4qCIxyH5efyVd0xTO7ukuapGC23mRPd1auUNHlTt0/mac8a+wjBmG175j2MFnzDj7e4Z2ynqgSVrkM/DKm4/xiED5J1vNpPQAkNWouzELo70x7XsLk8/WIVXtsLI24OEo+BE+9/pf9T+RObftUDMnSejhLFadKTqa7QHh6RJgV1lkJL+ZvDm5XY962BzJrMw/3N+h30qD+QCmkuyLqIbTAP7nFeJ7iumU1Bgv6ojLPtrxLX4TE/lP9Mv9JG1/w+4oMAeSv5t/uFw4pI4P07c5cGRwE9VHIT57roJ6beKRD/3p7FJxeEFOu5/O9EoxHTCjXWXMrdEM6bYfMMQZYBM7p8Ve0bE=,iv:4njRxb8LiKdW5YgQEP2Esvh8/yKHpiCxTiL/n6b+c5M=,tag:lPqLddPhY0r0rnuxtwvwIg==,type:str]
+    restic-backups:
+        env: ENC[AES256_GCM,data:Uls7HrNxFyoMZf3u4zNSk/F3XIYE1mQotcCvnNggvbOqA9RVK9FQpIfqJms8WtKZl6ol15rHW+au73QldJ9TE2Km7EJmCSa87AmRQ1Azt0lrJtn+v9l9rFqnu82WSD2awv8geB82OPQ0/w2x9W4qHPXw3teYx9p9Kiz/C1NVw0Z9TlsChKHsVpR6FVdyB8FY8zZG0Z1iuzDHMr6q9QsI8pMn5DkCruYhcg8R6GR73naWQb0nHxx3oFfNR3KtPHQn8diJZsv0+Hh2IWbHyPRlJOs+WYJtVc7LbqjB1jPrTozqteijclVSHAoB1RTQ7gVe9TiYdQIhOJswo89xH4l7U/D9wYH7tGRlrrKz8F8iUQR02G7gADftmwI9kekhCjGDgSudYrRJ//UCJKTn6bSsEpW0Eg9knozFCUOhc5RREmX7JyKA,iv:BqaFmxO+HRZzQchypReabxer0XwHW9KSsA1lfhWmdIM=,tag:3rJjQm+kLRMscygvkLHTcw==,type:str]
     umami:
         env: ENC[AES256_GCM,data:FDH0WI/eUDeuylLoL0xNgLawV6ROVTyK3iW25a7z7wesEBPrH1ffs/mQlBiCVG4a0KsC6QCcsCyJxXA2kFrFNm2ktVDouJd4ECuN,iv:SYzk1s24nFgiKswkXiTLSPIBW6v0JoeN6J5o89zzZuQ=,tag:LAwSKw+9GaqHWo9A9gQjsw==,type:str]
 services:
@@ -45,8 +45,8 @@ sops:
             dTQwclBRU3JmQ2hBOFVlaVIxZHN4cVEKvVR1bSH4yo2Td8YGNI2hmflT0M3hcYu8
             qyYoWd5MFP9VBYKxrF8W2naQSY6Ax4IiVuNbKDDLFooTC93V+oxNDA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2023-04-01T13:59:46Z"
-    mac: ENC[AES256_GCM,data:TmeJV5P+bAQ3IfnIWrEGlCQ1VuBv6dDTxT5l6v++cvTSAY+ed2kE0nfc+D159EvX2KkcUSPcApZyQPCk4EuREQwdd6ugb1SAowZSe3I62uZRegZhdAEVfOip1e9zFXyPSNFLoCo/q7xaBI8anLxpEjIzLhyn3WyR/ecs6BVS67w=,iv:2jNyjkXTDcc5qHXFjHODKZNOmoj8K3BJbhUjZwjOKVg=,tag:t+66U+bgYFY0H+xFXGbaJg==,type:str]
+    lastmodified: "2024-01-26T23:48:26Z"
+    mac: ENC[AES256_GCM,data:Ay1JpYoj+XRBPoPwuy5QJ8jmbxfzyBKe0p1F/Twp/F4nMstPv65+W7D+KLOJg66ki2/3j2nuhJnhsypmG20bAHA+Yr3nRPwO/0NHNQCuTxLmjFI4TjTlqHfofUzjAajXHSThm2bCz4nuqSpI+UVL9VOTJl1OR05jhYclozrZacw=,iv:V3vVrISCkFaLP/67IAd1VLVNpHkzZwtni6ace5kTWIk=,tag:LUCeYK2M85u/aUl3cdBNpQ==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
-    version: 3.7.3
+    version: 3.8.1

hosts/nyarlathotep/configuration.nix

@@ -63,13 +63,26 @@ in
   ## Backups
   ###############################################################################
 
-  nixfiles.backups.enable = true;
-  nixfiles.backups.environmentFile = config.sops.secrets."nixfiles/backups/env".path;
-  nixfiles.backups.pythonScripts = {
-    share = fileContents ./jobs/backup-share.py;
-    youtube = fileContents ./jobs/backup-youtube.py;
+  nixfiles.restic-backups.enable = true;
+  nixfiles.restic-backups.environmentFile = config.sops.secrets."nixfiles/restic-backups/env".path;
+  nixfiles.restic-backups.backups.torrents = {
+    prepareCommand = ''
+      ${pkgs.python3}/bin/python3 ${./jobs/restic-prepare--hardlink-torrent-files.py} > hardlink-torrent-files.sh
+    '';
+    paths = [
+      "hardlink-torrent-files.sh"
+      "/mnt/nas/torrents/watch"
+    ];
   };
-  sops.secrets."nixfiles/backups/env" = { };
+  nixfiles.restic-backups.backups.youtube = {
+    prepareCommand = ''
+      ${pkgs.python3}/bin/python3 ${./jobs/restic-prepare--fetch-youtube.py} > fetch-youtube.sh
+    '';
+    paths = [
+      "fetch-youtube.sh"
+    ];
+  };
+  sops.secrets."nixfiles/restic-backups/env" = { };
 
 
   ###############################################################################

hosts/nyarlathotep/jobs/restic-prepare--fetch-youtube.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+"""Youtube "backup" script - generates a script to download videos.
+"""
+
+import os
+import shlex
+
+SOURCE_DIR = "/mnt/nas/misc/youtube"
+VIDEO_URL = "https://www.youtube.com/watch?v="
+
+print("#!/bin/sh")
+print("")
+
+for dirpath, dirnames, filenames in os.walk(SOURCE_DIR, topdown=True):
+    for dirname in dirnames:
+        print(f"mkdir {shlex.quote(os.path.join(dirpath, dirname))}")
+    for filename in filenames:
+        # filenames are of the form "title [id].ext"
+        name_pattern = filename.split("[")[-2] + "[%(id)s].%(ext)s"
+        url = VIDEO_URL + filename.split("[")[-1].split("]")[0]
+        print(
+            f"yt-dlp -P {shlex.quote(dirpath)} -o {shlex.quote(name_pattern)} {shlex.quote(url)}",
+        )