overhaul of backup system

README.md

@@ -13,6 +13,8 @@
 - `system/`
   System-wide configurations
 
+All modules are annotated with comments where appropriate.
+
 ## External Dependencies
 
 - [Nixpkgs/NixOS](https://github.com/nixos/nixpkgs), forms the basis of the NixOS system

helpers/services.nix

@@ -1,9 +1,12 @@
+# Helper module to configure multiple services with onion and clearnet access, as well as optional backups.
+# Each service can specify its own URL, onion settings, data directory, and backup options.
 {
   config,
   lib,
   ...
 }: let
   cfg = config.distrust.services;
+  backup_cfg = config.distrust.backups;
 in {
   options = {
     distrust = {
@@ -36,6 +39,29 @@ in {
                 };
                 default = {};
               };
+              backup = lib.mkOption {
+                description = "Backup configuration";
+                type = lib.types.submodule {
+                  options = {
+                    enable = lib.mkOption {
+                      description = "Enable backups for this service";
+                      type = lib.types.bool;
+                      default = false;
+                    };
+                    paths = lib.mkOption {
+                      description = "Paths to back up";
+                      type = lib.types.listOf lib.types.str;
+                      default = [];
+                    };
+                    database = lib.mkOption {
+                      description = "Database to back up";
+                      type = lib.types.nullOr lib.types.str;
+                      default = null;
+                    };
+                  };
+                };
+                default = {};
+              };
               virtualHostConfig = lib.mkOption {
                 description = "Caddy virtual host config";
                 type = lib.types.str;
@@ -45,6 +71,25 @@ in {
         );
         default = {};
       };
+      backups = lib.mkOption {
+        description = "Global backup configuration";
+        type = lib.types.submodule {
+          options = {
+            borgRepository = lib.mkOption {
+              description = "Borg backup repository URL";
+              type = lib.types.str;
+            };
+            borgSSHKey = lib.mkOption {
+              description = "Path to SSH key for Borg backup repository";
+              type = lib.types.path;
+            };
+            borgPassCommand = lib.mkOption {
+              description = "Command to retrieve Borg repository passphrase";
+              type = lib.types.str;
+            };
+          };
+        };
+      };
     };
   };
 
@@ -78,6 +123,54 @@ in {
             extraConfig = extraCfg;
           };
         }) {} (builtins.attrNames cfg);
+
+      services.borgbackup.jobs =
+        builtins.foldl'
+        (acc: key: let
+          site = cfg.${key};
+          dump = site.backup.database;
+          paths = lib.mkIf (dump != null) ["/var/backup/${key}.sql"] ++ site.backup.paths;
+          preHook = lib.mkIf (dump != null) ''
+            mkdir -p /var/backup
+            ${pkgs.sudo}/bin/sudo -u postgres pg_dump ${dump} > /var/backup/postgres/${key}.sql
+          '';
+          postHook = lib.mkIf (dump != null) ''
+            rm -f /var/backup/postgres/${key}.sql
+          '';
+        in
+          acc
+          // lib.mkIf site.backup.enable {
+            "${key}" = {
+              repo = backup_cfg.borgRepository + "/./${key}";
+              environment = {
+                BORG_RSH = "ssh -i ${backup_cfg.borgSSHKey} -o 'StrictHostKeyChecking=no'";
+              };
+              inherit paths;
+              encryption = {
+                mode = "keyfile";
+                passCommand = backup_cfg.borgPassCommand;
+              };
+              compression = "auto,lzma";
+              startAt = "daily";
+              prune.keep = {
+                daily = 7;
+                weekly = 4;
+                monthly = -1;
+              };
+              inherit preHook;
+              inherit postHook;
+            };
+          })
+        {}
+        (builtins.attrNames cfg);
+
+      systemd.tmpfiles.settings = {
+        "99-borgdatabasebackups"."/var/backup/postgres".d = {
+          user = "root";
+          group = "root";
+          mode = "0755";
+        };
+      };
     };
   };
 }

homepage/default.nix

@@ -1,3 +1,4 @@
+# This derivation generates static HTML files for the Distrust homepage
 {pkgs ? import <nixpkgs> {}}:
 pkgs.stdenv.mkDerivation {
   name = "distrust-network-static-webroot";

secrets/secrets.nix

@@ -1,19 +1,25 @@
 let
+  # Only myself
   user = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHxah5pnxmk+P7HtwRsryDoAHZsDs5RcGP9IPCNg1KFe cardno:16_179_196";
   users = [user];
 
+  # Current host
   system = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMKxw1fDsIUUh3vWCD90LDgDMAG/NSVRg7QamUbknz5A root@distrust";
   systems = [system];
 
   all = users ++ systems;
 in {
+  # Multi-service secrets
+  "borg_ed25519".publicKeys = all;
+  "borg_pass".publicKeys = all;
+
+  # Service-specific secrets
   "bind_pw".publicKeys = all;
   "nextcloud-admin-pass".publicKeys = all;
   "prosody.env".publicKeys = all;
   "vaultwarden.env".publicKeys = all;
-  "borg_ed25519".publicKeys = all;
-  "borg_pass".publicKeys = all;
 
+  # Hidden service secret keys
   "hidden_service/akkoma".publicKeys = all;
   "hidden_service/forgejo".publicKeys = all;
   "hidden_service/lldap".publicKeys = all;

services/akkoma.nix

@@ -40,37 +40,6 @@ in {
         };
       };
     };
-    borgbackup.jobs."akkoma" = {
-      repo = "ssh://u506783@u506783.your-storagebox.de:23/./akkoma";
-      environment = {
-        BORG_RSH = "ssh -i ${config.age.secrets."borg_ed25519".path} -o 'StrictHostKeyChecking=no'";
-      };
-      paths = [
-        "/var/lib/akkoma"
-        "/var/backup/postgres/akkoma.sql"
-      ];
-      encryption = {
-        mode = "keyfile";
-        passCommand = "cat ${config.age.secrets."borg_pass".path}";
-      };
-      compression = "auto,lzma";
-      startAt = "daily";
-      prune.keep = {
-        daily = 7;
-        weekly = 4;
-        monthly = -1;
-      };
-      readWritePaths = [
-        "/var/backup/postgres"
-      ];
-      preHook = ''
-        mkdir -p /var/backup/postgres
-        ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump akkoma > /var/backup/postgres/akkoma.sql
-      '';
-      postHook = ''
-        rm -f /var/backup/postgres/akkoma.sql
-      '';
-    };
   };
 
   distrust.services."akkoma" = {
@@ -82,5 +51,12 @@ in {
     virtualHostConfig = ''
       reverse_proxy localhost:${toString fediPort}
     '';
+    backup = {
+      enable = true;
+      paths = [
+        "/var/lib/akkoma"
+      ];
+      database = "akkoma";
+    };
   };
 }

services/borg.nix

@@ -2,11 +2,9 @@
   age.secrets."borg_ed25519".file = ../secrets/borg_ed25519;
   age.secrets."borg_pass".file = ../secrets/borg_pass;
 
-  systemd.tmpfiles.settings = {
+  distrust.backups = {
-    "99-borgdatabasebackups"."/var/backup/postgres".d = {
+    borgRepository = "";
-      user = "root";
+    borgSSHKey = config.age.secrets."borg_ed25519".path;
-      group = "root";
+    borgPassCommand = "cat ${config.age.secrets."borg_pass".path}";
-      mode = "0755";
-    };
   };
 }

services/default.nix

@@ -4,12 +4,12 @@
     ./borg.nix
     ./caddy.nix
 
-    # Non-Stateful
+    # Non-stateful services
     ./ipfs.nix
     ./site.nix
     ./tor.nix
 
-    # Stateful
+    # Stateful services (backed up by borg)
     ./akkoma.nix
     ./forgejo.nix
     ./lldap.nix

services/forgejo.nix

@@ -5,8 +5,7 @@ in {
     file = ../secrets/hidden_service/forgejo;
   };
 
-  services = {
+  services.forgejo = {
-    forgejo = {
     enable = true;
     lfs.enable = false;
     settings.server = {
@@ -16,27 +15,6 @@ in {
       SSH_PORT = builtins.head config.services.openssh.ports;
     };
   };
-    borgbackup.jobs."forgejo" = {
-      repo = "ssh://u506783@u506783.your-storagebox.de:23/./forgejo";
-      environment = {
-        BORG_RSH = "ssh -i ${config.age.secrets."borg_ed25519".path} -o 'StrictHostKeyChecking=no'";
-      };
-      paths = [
-        "/var/lib/forgejo"
-      ];
-      encryption = {
-        mode = "keyfile";
-        passCommand = "cat ${config.age.secrets."borg_pass".path}";
-      };
-      compression = "auto,lzma";
-      startAt = "daily";
-      prune.keep = {
-        daily = 7;
-        weekly = 4;
-        monthly = -1;
-      };
-    };
-  };
 
   distrust.services."forgejo" = {
     url = "https://git.distrust.network";
@@ -47,5 +25,11 @@ in {
     virtualHostConfig = ''
       reverse_proxy localhost:${toString forgejoPort}
     '';
+    backup = {
+      enable = true;
+      paths = [
+        "/var/lib/forgejo"
+      ];
+    };
   };
 }

services/ipfs.nix

@@ -1,7 +1,6 @@
 {
   services = {
     kubo.enable = true;
-
     tor.relay.onionServices."site".map = [
       4001
       8080

services/lldap.nix

@@ -3,8 +3,7 @@
 in {
   age.secrets."hidden_service/lldap".file = ../secrets/hidden_service/lldap;
 
-  services = {
+  services.lldap = {
-    lldap = {
     enable = true;
     settings = {
       http_url = "https://login.distrust.network";
@@ -15,27 +14,6 @@ in {
       ldap_user_pass = "VERY_SECURE";
     };
   };
-    borgbackup.jobs."lldap" = {
-      repo = "ssh://u506783@u506783.your-storagebox.de:23/./lldap";
-      environment = {
-        BORG_RSH = "ssh -i ${config.age.secrets."borg_ed25519".path} -o 'StrictHostKeyChecking=no'";
-      };
-      paths = [
-        "/var/lib/lldap"
-      ];
-      encryption = {
-        mode = "keyfile";
-        passCommand = "cat ${config.age.secrets."borg_pass".path}";
-      };
-      compression = "auto,lzma";
-      startAt = "daily";
-      prune.keep = {
-        daily = 7;
-        weekly = 4;
-        monthly = -1;
-      };
-    };
-  };
 
   distrust.services."lldap" = {
     url = "https://login.distrust.network";
@@ -46,5 +24,11 @@ in {
     virtualHostConfig = ''
       reverse_proxy localhost:${toString lldapPort}
     '';
+    backup = {
+      enable = true;
+      paths = [
+        "/var/lib/lldap"
+      ];
+    };
   };
 }

services/mailserver.nix

@@ -17,16 +17,16 @@
   };
 
   services.borgbackup.jobs."mailserver" = {
-    repo = "ssh://u506783@u506783.your-storagebox.de:23/./mailserver";
+    repo = config.distrust.backups.borgRepository + "/./mailserver";
     environment = {
-      BORG_RSH = "ssh -i ${config.age.secrets."borg_ed25519".path} -o 'StrictHostKeyChecking=no'";
+      BORG_RSH = "ssh -i ${config.distrust.backups.borgSSHKey} -o 'StrictHostKeyChecking=no'";
     };
     paths = [
       "/var/vmail/ldap"
     ];
     encryption = {
       mode = "keyfile";
-      passCommand = "cat ${config.age.secrets."borg_pass".path}";
+      passCommand = config.distrust.backups.borgPassCommand;
     };
     compression = "auto,lzma";
     startAt = "daily";

services/nextcloud.nix

@@ -36,42 +36,12 @@ in {
       };
     };
 
+    # Force disable nginx and adjust permissions as we use caddy
     nginx.enable = lib.mkForce false;
     phpfpm.pools.nextcloud.settings = {
       "listen.owner" = "caddy";
       "listen.group" = "caddy";
     };
-    borgbackup.jobs."nextcloud" = {
-      repo = "ssh://u506783@u506783.your-storagebox.de:23/./nextcloud";
-      environment = {
-        BORG_RSH = "ssh -i ${config.age.secrets."borg_ed25519".path} -o 'StrictHostKeyChecking=no'";
-      };
-      paths = [
-        "/var/lib/nextcloud"
-        "/var/backup/postgres/nextcloud.sql"
-      ];
-      encryption = {
-        mode = "keyfile";
-        passCommand = "cat ${config.age.secrets."borg_pass".path}";
-      };
-      compression = "auto,lzma";
-      startAt = "daily";
-      prune.keep = {
-        daily = 7;
-        weekly = 4;
-        monthly = -1;
-      };
-      readWritePaths = [
-        "/var/backup/postgres"
-      ];
-      preHook = ''
-        mkdir -p /var/backup/postgres
-        ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump nextcloud > /var/backup/postgres/nextcloud.sql
-      '';
-      postHook = ''
-        rm -f /var/backup/postgres/nextcloud.sql
-      '';
-    };
   };
 
   distrust.services."nextcloud" = {
@@ -140,5 +110,12 @@ in {
 
       file_server
     '';
+    backup = {
+      enable = true;
+      paths = [
+        "/var/lib/nextcloud"
+      ];
+      database = "nextcloud";
+    };
   };
 }

services/paste.nix

@@ -3,8 +3,7 @@
 in {
   age.secrets."hidden_service/microbin".file = ../secrets/hidden_service/microbin;
 
-  services = {
+  services.microbin = {
-    microbin = {
     enable = true;
     settings = {
       MICROBIN_PORT = pastePort;
@@ -15,27 +14,6 @@ in {
       MICROBIN_PUBLIC_PATH = "https://paste.distrust.network/";
     };
   };
-    borgbackup.jobs."microbin" = {
-      repo = "ssh://u506783@u506783.your-storagebox.de:23/./microbin";
-      environment = {
-        BORG_RSH = "ssh -i ${config.age.secrets."borg_ed25519".path} -o 'StrictHostKeyChecking=no'";
-      };
-      paths = [
-        "/var/lib/microbin"
-      ];
-      encryption = {
-        mode = "keyfile";
-        passCommand = "cat ${config.age.secrets."borg_pass".path}";
-      };
-      compression = "auto,lzma";
-      startAt = "daily";
-      prune.keep = {
-        daily = 7;
-        weekly = 4;
-        monthly = -1;
-      };
-    };
-  };
 
   distrust.services."microbin" = {
     url = "https://paste.distrust.network";
@@ -46,5 +24,11 @@ in {
     virtualHostConfig = ''
       reverse_proxy localhost:${toString pastePort}
     '';
+    backup = {
+      enable = true;
+      paths = [
+        "/var/lib/microbin"
+      ];
+    };
   };
 }

services/prosody.nix

@@ -33,30 +33,34 @@
         ldap_password = os.getenv("LDAP_BIND_PASSWORD")
       '';
     };
-    caddy.virtualHosts."distrust.network".extraConfig = ''
-      handle /.well-known/* {
-        root * /var/lib/acme/
-        file_server
-      }
-    '';
 
-    caddy.virtualHosts."conference.distrust.network upload.distrust.network".extraConfig = ''
+    # Adjust caddy to serve the ACME challenges for prosody
+    caddy.virtualHosts = {
+      "distrust.network".extraConfig = ''
         handle /.well-known/* {
           root * /var/lib/acme/
           file_server
         }
       '';
+      "conference.distrust.network upload.distrust.network".extraConfig = ''
+        handle /.well-known/* {
+          root * /var/lib/acme/
+          file_server
+        }
+      '';
+    };
+
     borgbackup.jobs."prosody" = {
-      repo = "ssh://u506783@u506783.your-storagebox.de:23/./prosody";
+      repo = config.distrust.backups.borgRepository + "/./prosody";
       environment = {
-        BORG_RSH = "ssh -i ${config.age.secrets."borg_ed25519".path} -o 'StrictHostKeyChecking=no'";
+        BORG_RSH = "ssh -i ${config.distrust.backups.borgSSHKey} -o 'StrictHostKeyChecking=no'";
       };
       paths = [
         "/var/lib/prosody"
       ];
       encryption = {
         mode = "keyfile";
-        passCommand = "cat ${config.age.secrets."borg_pass".path}";
+        passCommand = config.distrust.backups.borgPassCommand;
       };
       compression = "auto,lzma";
       startAt = "daily";
@@ -88,13 +92,17 @@
     };
   };
 
-  networking.resolvconf.dnsExtensionMechanism = false;
+  networking = {
-  networking.firewall.allowedTCPPorts = [5222 5269 5281 5000];
+    # This can mess with prosody's DNS resolution, so we disable it
+    resolvconf.dnsExtensionMechanism = false;
+    firewall.allowedTCPPorts = [5222 5269 5281 5000];
+  };
 
   systemd.services = {
     caddy.serviceConfig.SupplementaryGroups = ["acme"];
     prosody.serviceConfig = {
       SupplementaryGroups = ["acme"];
+      # Slightly hacky way to inject the LDAP password into prosody without builtins.readFile exposing it system-wide
       EnvironmentFile = config.age.secrets."prosody.env".path;
     };
   };

services/site.nix

@@ -20,5 +20,6 @@ in {
       root * ${distrust-homepage.out}
       file_server
     '';
+    backup.enable = false;
   };
 }

services/tor.nix

@@ -7,6 +7,7 @@
     };
     settings = {
       Nickname = "Distrust";
+      ContactInfo = "root@distrust.network";
       ORPort = 9001;
     };
   };

services/vaultwarden.nix

@@ -46,5 +46,11 @@ in {
     virtualHostConfig = ''
       reverse_proxy localhost:${toString vaultPort}
     '';
+    backup = {
+      enable = true;
+      paths = [
+        "/var/lib/vaultwarden"
+      ];
+    };
   };
 }

system/configuration.nix

@@ -1,20 +1,37 @@
 {pkgs, ...}: let
+  # Helper script to update the system based on local copy of flake
   updateScript = pkgs.writeShellScriptBin "rebuild" ''
     nixos-rebuild switch --flake /etc/nixos#distrust
   '';
+  # Helper script to clear /var/log and systemd journal
   clearLogsScript = pkgs.writeShellScriptBin "clear_logs" ''
     ${pkgs.coreutils}/bin/rm -rf /var/log/*
     ${pkgs.systemd}/bin/journalctl --vacuum-time=0s
   '';
+  # Currently unused, calculates TOR .onion hostname based on secret key
   tor-hostname = import ../helpers/tor-hostname.nix {inherit pkgs;};
 in {
   environment.systemPackages = with pkgs; [vim btop git alejandra statix deadnix] ++ [updateScript tor-hostname];
 
+  # Necessary for flake support
   nix.settings.experimental-features = ["nix-command" "flakes"];
 
-  boot.tmp.cleanOnBoot = true;
-  zramSwap.enable = true;
   networking.hostName = "distrust";
+  zramSwap.enable = true;
+  boot.tmp.cleanOnBoot = true;
+
+  users.users = {
+    # Disables root login by setting an invalid password
+    root.hashedPassword = "!";
+    anon = {
+      isNormalUser = true;
+      extraGroups = ["wheel"];
+      hashedPassword = "$6$GAyfgaTQgaBipAbb$gF/9YBh2ucVa/9vDQvEu9DVjSbsqdvSoXwA5RX0kP7.xdCfLqXhGBLlSXHg0e4rkLLd6zI1gRTWd4TfMjnnpS/";
+      openssh.authorizedKeys.keys = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHxah5pnxmk+P7HtwRsryDoAHZsDs5RcGP9IPCNg1KFe cardno:16_179_196"];
+    };
+  };
+
+  security.sudo.wheelNeedsPassword = true;
 
   services = {
     openssh = {
@@ -30,15 +47,6 @@ in {
     };
   };
 
-  users.users.root.hashedPassword = "!";
-  users.users.anon = {
-    isNormalUser = true;
-    extraGroups = ["wheel"];
-    hashedPassword = "$6$GAyfgaTQgaBipAbb$gF/9YBh2ucVa/9vDQvEu9DVjSbsqdvSoXwA5RX0kP7.xdCfLqXhGBLlSXHg0e4rkLLd6zI1gRTWd4TfMjnnpS/";
-    openssh.authorizedKeys.keys = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHxah5pnxmk+P7HtwRsryDoAHZsDs5RcGP9IPCNg1KFe cardno:16_179_196"];
-  };
-  security.sudo.wheelNeedsPassword = true;
-
   systemd = {
     services.clear-var-log = {
       description = "Clear /var/log directory";

system/default.nix

@@ -5,6 +5,8 @@
 }: {
   imports = [
     ./configuration.nix
+
+    # Auto generated, do not edit. Replace per host
     ./hardware-configuration.nix
     ./networking.nix