Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor: move bookdb & bookmarks sync scripts into modules #311

Merged
merged 2 commits into from
Dec 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 6 additions & 43 deletions hosts/carcosa/configuration.nix
Original file line number Diff line number Diff line change
Expand Up @@ -420,50 +420,13 @@ in
## Nyarlathotep Sync
###############################################################################

users.extraUsers.nyarlathotep-remote-sync = {
home = "/var/lib/nyarlathotep-remote-sync";
createHome = true;
isSystemUser = true;
openssh.authorizedKeys.keys =
[ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIChVw9DPLafA3lCLCI4Df9rYuxedFQTXAwDOOHUfZ0Ac remote-sync@nyarlathotep" ];
shell = pkgs.bashInteractive;
group = "nogroup";
packages =
let
bookdb-receive-covers = ''
if [[ ! -d ~/bookdb-covers ]]; then
echo "bookdb-covers does not exist"
exit 1
fi

/run/wrappers/bin/sudo ${pkgs.rsync}/bin/rsync -a --delete ~/bookdb-covers/ ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR} || exit 1
/run/wrappers/bin/sudo ${pkgs.coreutils}/bin/chown -R ${config.users.users.bookdb.name}.nogroup ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR} || exit 1
'';
bookdb-receive-elasticsearch = ''
env ES_HOST=${config.systemd.services.bookdb.environment.ES_HOST} \
${pkgs.nixfiles.bookdb}/bin/bookdb_ctl import-index --drop-existing
'';
bookmarks-receive-elasticsearch = ''
env ES_HOST=${config.systemd.services.bookmarks.environment.ES_HOST} \
${pkgs.nixfiles.bookmarks}/bin/bookmarks_ctl import-index --drop-existing
'';
in
[
(pkgs.writeShellScriptBin "bookdb-receive-covers" bookdb-receive-covers)
(pkgs.writeShellScriptBin "bookdb-receive-elasticsearch" bookdb-receive-elasticsearch)
(pkgs.writeShellScriptBin "bookmarks-receive-elasticsearch" bookmarks-receive-elasticsearch)
];
};
nixfiles.bookdb.remoteSync.receive.enable = true;
nixfiles.bookdb.remoteSync.receive.authorizedKeys =
[ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIChVw9DPLafA3lCLCI4Df9rYuxedFQTXAwDOOHUfZ0Ac remote-sync@nyarlathotep" ];

security.sudo.extraRules = [
{
users = [ config.users.extraUsers.nyarlathotep-remote-sync.name ];
commands = [
{ command = "${pkgs.rsync}/bin/rsync -a --delete ${config.users.extraUsers.nyarlathotep-remote-sync.home}/bookdb-covers/ ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR}"; options = [ "NOPASSWD" ]; }
{ command = "${pkgs.coreutils}/bin/chown -R ${config.users.users.bookdb.name}.nogroup ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR}"; options = [ "NOPASSWD" ]; }
];
}
];
nixfiles.bookmarks.remoteSync.receive.enable = true;
nixfiles.bookmarks.remoteSync.receive.authorizedKeys =
[ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIChVw9DPLafA3lCLCI4Df9rYuxedFQTXAwDOOHUfZ0Ac remote-sync@nyarlathotep" ];

###############################################################################
## Miscellaneous
Expand Down
88 changes: 13 additions & 75 deletions hosts/nyarlathotep/configuration.nix
Original file line number Diff line number Diff line change
Expand Up @@ -491,85 +491,23 @@ in
# Remote Sync
###############################################################################

users.extraUsers.remote-sync = {
home = "/var/lib/remote-sync";
createHome = true;
isSystemUser = true;
shell = pkgs.bashInteractive;
group = "nogroup";
};
nixfiles.bookdb.remoteSync.send.enable = true;
nixfiles.bookdb.remoteSync.send.sshKeyFile = config.sops.secrets."users/bookdb_remote_sync/ssh_private_key".path;
nixfiles.bookdb.remoteSync.send.targets = [ "carcosa.barrucadu.co.uk" ];

systemd.services.bookdb-sync = {
description = "Upload bookdb data to carcosa";
startAt = "*:15";
path = with pkgs; [ openssh rsync ];
serviceConfig = {
ExecStart = pkgs.writeShellScript "bookdb-sync" ''
set -ex

/run/wrappers/bin/sudo ${pkgs.coreutils}/bin/cp -r ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR}/ ~/bookdb-covers
trap "/run/wrappers/bin/sudo ${pkgs.coreutils}/bin/rm -rf ~/bookdb-covers" EXIT
rsync -az\
-e "ssh -i $SSH_KEY_FILE -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" \
~/bookdb-covers/ \
[email protected]:~/bookdb-covers/
ssh -i "$SSH_KEY_FILE" \
-o UserKnownHostsFile=/dev/null \
-o StrictHostKeyChecking=no \
[email protected] \
bookdb-receive-covers

env "ES_HOST=$ES_HOST" \
${pkgs.nixfiles.bookdb}/bin/bookdb_ctl export-index | \
ssh -i "$SSH_KEY_FILE" \
-o UserKnownHostsFile=/dev/null \
-o StrictHostKeyChecking=no \
[email protected] \
bookdb-receive-elasticsearch
'';
User = config.users.extraUsers.remote-sync.name;
};
environment = {
ES_HOST = config.systemd.services.bookdb.environment.ES_HOST;
SSH_KEY_FILE = config.sops.secrets."users/remote_sync/ssh_private_key".path;
};
sops.secrets."users/bookdb_remote_sync/ssh_private_key" = {
owner = config.users.extraUsers.bookdb-remote-sync-send.name;
key = "users/remote_sync/ssh_private_key";
};

systemd.services.bookmarks-sync = {
description = "Upload bookmarks data to carcosa";
startAt = "*:15";
path = with pkgs; [ openssh ];
serviceConfig = {
ExecStart = pkgs.writeShellScript "bookmarks-sync" ''
set -ex

env "ES_HOST=$ES_HOST" \
${pkgs.nixfiles.bookmarks}/bin/bookmarks_ctl export-index | \
ssh -i "$SSH_KEY_FILE" \
-o UserKnownHostsFile=/dev/null \
-o StrictHostKeyChecking=no \
[email protected] \
bookmarks-receive-elasticsearch
'';
User = config.users.extraUsers.remote-sync.name;
};
environment = {
ES_HOST = config.systemd.services.bookmarks.environment.ES_HOST;
SSH_KEY_FILE = config.sops.secrets."users/remote_sync/ssh_private_key".path;
};
};
nixfiles.bookmarks.remoteSync.send.enable = true;
nixfiles.bookmarks.remoteSync.send.sshKeyFile = config.sops.secrets."users/bookmarks_remote_sync/ssh_private_key".path;
nixfiles.bookmarks.remoteSync.send.targets = [ "carcosa.barrucadu.co.uk" ];

security.sudo.extraRules = [
{
users = [ config.users.extraUsers.remote-sync.name ];
commands = [
{ command = "${pkgs.coreutils}/bin/cp -r ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR}/ ${config.users.extraUsers.remote-sync.home}/bookdb-covers"; options = [ "NOPASSWD" ]; }
{ command = "${pkgs.coreutils}/bin/rm -rf ${config.users.extraUsers.remote-sync.home}/bookdb-covers"; options = [ "NOPASSWD" ]; }
];
}
];

sops.secrets."users/remote_sync/ssh_private_key".owner = config.users.extraUsers.remote-sync.name;
sops.secrets."users/bookmarks_remote_sync/ssh_private_key" = {
owner = config.users.extraUsers.bookmarks-remote-sync-send.name;
key = "users/remote_sync/ssh_private_key";
};

###############################################################################
# RSS-to-Mastodon
Expand Down
2 changes: 2 additions & 0 deletions shared/bookdb/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ in
imports = [
./erase-your-darlings.nix
./options.nix
./remote-sync-receive.nix
./remote-sync-send.nix
];

config = mkIf cfg.enable {
Expand Down
42 changes: 42 additions & 0 deletions shared/bookdb/options.nix
Original file line number Diff line number Diff line change
Expand Up @@ -71,5 +71,47 @@ with lib;
Format of the log messages.
'';
};

remoteSync = {
receive = {
enable = mkOption {
type = types.bool;
default = false;
description = mdDoc ''
Enable receiving push-based remote sync from other hosts.
'';
};
authorizedKeys = mkOption {
type = types.listOf types.str;
default = [ ];
description = mdDoc ''
SSH public keys to allow pushes from.
'';
};
};

send = {
enable = mkOption {
type = types.bool;
default = false;
description = mdDoc ''
Enable periodically pushing local state to other hosts.
'';
};
sshKeyFile = mkOption {
type = types.str;
description = mdDoc ''
Path to SSH private key.
'';
};
targets = mkOption {
type = types.listOf types.str;
default = [ ];
description = mdDoc ''
Hosts to push to.
'';
};
};
};
};
}
49 changes: 49 additions & 0 deletions shared/bookdb/remote-sync-receive.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
# See remote-sync-send.nix
{ config, lib, pkgs, ... }:

with lib;
let
cfg = config.nixfiles.bookdb.remoteSync.receive;
in
{
config = mkIf cfg.enable {
users.extraUsers.bookdb-remote-sync-receive = {
home = "/var/lib/bookdb-remote-sync-receive";
createHome = true;
isSystemUser = true;
openssh.authorizedKeys.keys = cfg.authorizedKeys;
shell = pkgs.bashInteractive;
group = "nogroup";
packages =
let
receive-covers = ''
if [[ ! -d ~/bookdb-covers ]]; then
echo "bookdb-covers does not exist"
exit 1
fi

/run/wrappers/bin/sudo ${pkgs.rsync}/bin/rsync -a --delete ~/bookdb-covers/ ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR} || exit 1
/run/wrappers/bin/sudo ${pkgs.coreutils}/bin/chown -R ${config.users.users.bookdb.name}.nogroup ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR} || exit 1
'';
receive-elasticsearch = ''
env ES_HOST=${config.systemd.services.bookdb.environment.ES_HOST} \
${pkgs.nixfiles.bookdb}/bin/bookdb_ctl import-index --drop-existing
'';
in
[
(pkgs.writeShellScriptBin "receive-covers" receive-covers)
(pkgs.writeShellScriptBin "receive-elasticsearch" receive-elasticsearch)
];
};

security.sudo.extraRules = [
{
users = [ config.users.extraUsers.bookdb-remote-sync-receive.name ];
commands = [
{ command = "${pkgs.rsync}/bin/rsync -a --delete ${config.users.extraUsers.bookdb-remote-sync-receive.home}/bookdb-covers/ ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR}"; options = [ "NOPASSWD" ]; }
{ command = "${pkgs.coreutils}/bin/chown -R ${config.users.users.bookdb.name}.nogroup ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR}"; options = [ "NOPASSWD" ]; }
];
}
];
};
}
69 changes: 69 additions & 0 deletions shared/bookdb/remote-sync-send.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
# See remote-sync-receive.nix
{ config, lib, pkgs, ... }:

with lib;
let
cfg = config.nixfiles.bookdb.remoteSync.send;

toService = target: {
name = "bookdb-sync-${target}";
value = {
description = "Upload bookdb data to ${target}";
startAt = "*:15";
path = with pkgs; [ openssh rsync ];
serviceConfig = {
ExecStart = pkgs.writeShellScript "bookdb-sync" ''
set -ex

/run/wrappers/bin/sudo ${pkgs.coreutils}/bin/cp -r ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR}/ ~/bookdb-covers
trap "/run/wrappers/bin/sudo ${pkgs.coreutils}/bin/rm -rf ~/bookdb-covers" EXIT
rsync -az\
-e "ssh -i $SSH_KEY_FILE -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" \
~/bookdb-covers/ \
bookdb-remote-sync-receive@${target}:~/bookdb-covers/
ssh -i "$SSH_KEY_FILE" \
-o UserKnownHostsFile=/dev/null \
-o StrictHostKeyChecking=no \
bookdb-remote-sync-receive@${target} \
receive-covers

env "ES_HOST=$ES_HOST" \
${pkgs.nixfiles.bookdb}/bin/bookdb_ctl export-index | \
ssh -i "$SSH_KEY_FILE" \
-o UserKnownHostsFile=/dev/null \
-o StrictHostKeyChecking=no \
bookdb-remote-sync-receive@${target} \
receive-elasticsearch
'';
User = config.users.extraUsers.bookdb-remote-sync-send.name;
};
environment = {
ES_HOST = config.systemd.services.bookdb.environment.ES_HOST;
SSH_KEY_FILE = cfg.sshKeyFile;
};
};
};
in
{
config = mkIf cfg.enable {
users.extraUsers.bookdb-remote-sync-send = {
home = "/var/lib/bookdb-remote-sync-send";
createHome = true;
isSystemUser = true;
shell = pkgs.bashInteractive;
group = "nogroup";
};

systemd.services = listToAttrs (map toService cfg.targets);

security.sudo.extraRules = [
{
users = [ config.users.extraUsers.bookdb-remote-sync-send.name ];
commands = [
{ command = "${pkgs.coreutils}/bin/cp -r ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR}/ ${config.users.extraUsers.bookdb-remote-sync-send.home}/bookdb-covers"; options = [ "NOPASSWD" ]; }
{ command = "${pkgs.coreutils}/bin/rm -rf ${config.users.extraUsers.bookdb-remote-sync-send.home}/bookdb-covers"; options = [ "NOPASSWD" ]; }
];
}
];
};
}
2 changes: 2 additions & 0 deletions shared/bookmarks/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ in
{
imports = [
./options.nix
./remote-sync-receive.nix
./remote-sync-send.nix
];

config = mkIf cfg.enable {
Expand Down
42 changes: 42 additions & 0 deletions shared/bookmarks/options.nix
Original file line number Diff line number Diff line change
Expand Up @@ -60,5 +60,47 @@ with lib;
Format of the log messages.
'';
};

remoteSync = {
receive = {
enable = mkOption {
type = types.bool;
default = false;
description = mdDoc ''
Enable receiving push-based remote sync from other hosts.
'';
};
authorizedKeys = mkOption {
type = types.listOf types.str;
default = [ ];
description = mdDoc ''
SSH public keys to allow pushes from.
'';
};
};

send = {
enable = mkOption {
type = types.bool;
default = false;
description = mdDoc ''
Enable periodically pushing local state to other hosts.
'';
};
sshKeyFile = mkOption {
type = types.str;
description = mdDoc ''
Path to SSH private key.
'';
};
targets = mkOption {
type = types.listOf types.str;
default = [ ];
description = mdDoc ''
Hosts to push to.
'';
};
};
};
};
}
Loading
Loading