Implement ZFS dataset management and snapshot configuration

- Introduced a new `datasets.nix` file to define ZFS datasets and their properties, allowing for centralized management of dataset configurations.
- Updated `default.nix` to import and enable the `zfs_manager` service, integrating it into the system configuration.
- Refactored `zfs.sh` to remove dataset creation commands, delegating dataset management to the `zfs_manager` service.
- Removed the legacy `snapshot_config.toml` file, as snapshot configurations are now handled within `datasets.nix`.
- Modified `vars.nix` to derive mountpoint paths from `datasets.nix`, ensuring consistency across the configuration.
- Created a new `zfs.nix` file to define the `zfs_manager` service and its dependencies.
- Added comprehensive tests for the `zfs_manager` functionality, covering dataset creation, property management, and error handling.

systems/jeeves/datasets.nix

@@ -0,0 +1,364 @@
+# Dataset declarations for jeeves, kept as plain data rather than inside
+# zfs.nix so that vars.nix can derive its paths from the same source.
+#
+# Datasets are nested the way zfs nests them: a pool holds datasets, which can
+# hold datasets of their own. The tree is flattened into "pool/parent/child"
+# names below, which is what zfs and services.zfs_manager work in.
+#
+# Consumed by ./zfs.nix (which feeds it to services.zfs_manager) and by
+# ./vars.nix (which resolves the mountpoints).
+let
+  # Every pool on jeeves was created with the same -O options.
+  poolDefaults = mountpoint: {
+    inherit mountpoint;
+    acltype = "posix"; # zfs reports posixacl back as posix
+    atime = "off";
+    compression = "zstd";
+    dnodesize = "auto";
+    xattr = "sa";
+  };
+
+  zfsKey = "file:///root/zfs.key";
+
+  # What a dataset gets when it is not called out below, kept identical to the
+  # "default" table so the datasets that used to fall through are unchanged.
+  standard = {
+    "15_min" = 8;
+    hourly = 24;
+  };
+
+  pools = {
+    # root_pool: retention only, its properties are not managed yet.
+    root_pool = {
+      manageProperties = false;
+      datasets = {
+        home = {
+          manageProperties = false;
+          snapshots = {
+            "15_min" = 8;
+            hourly = 24;
+            daily = 14;
+          };
+        };
+        root = {
+          manageProperties = false;
+          snapshots = standard;
+        };
+        nix = {
+          manageProperties = false;
+          snapshots."15_min" = 4;
+        };
+        var = {
+          manageProperties = false;
+          snapshots = {
+            "15_min" = 8;
+            hourly = 24;
+            daily = 30;
+            monthly = 6;
+          };
+        };
+      };
+    };
+
+    media = {
+      properties = poolDefaults "/zfs/media";
+      datasets = {
+        temp = {
+          properties = {
+            redundant_metadata = "none";
+            sync = "disabled";
+          };
+          snapshots."15_min" = 2;
+        };
+        secure = {
+          # An encryption root. zfs_manager must never create this: encryption
+          # is fixed at creation and is not declared here, so an automatic
+          # create would produce an unencrypted dataset. See scripts/zfs.sh.
+          createIfMissing = false;
+          properties.keylocation = zfsKey;
+          snapshots = { };
+          datasets = {
+            docker = {
+              properties = {
+                mountpoint = "/zfs/media/docker";
+                compression = "zstd-9";
+              };
+              snapshots = {
+                "15_min" = 3;
+                hourly = 12;
+                daily = 14;
+                monthly = 2;
+              };
+            };
+            "github-runners" = {
+              properties = {
+                mountpoint = "/zfs/media/github-runners";
+                compression = "zstd-9";
+                sync = "disabled";
+              };
+              snapshots = {
+                "15_min" = 6;
+                hourly = 2;
+                daily = 1;
+              };
+            };
+            home_assistant = {
+              properties = {
+                mountpoint = "/zfs/media/home_assistant";
+                compression = "zstd-19";
+              };
+              snapshots = standard;
+            };
+            important = {
+              properties = {
+                compression = "zstd-9";
+                copies = "2";
+              };
+              snapshots = standard;
+            };
+            notes = {
+              properties = {
+                mountpoint = "/zfs/media/notes";
+                copies = "2";
+              };
+              snapshots = {
+                "15_min" = 8;
+                hourly = 24;
+                daily = 30;
+                monthly = 12;
+              };
+            };
+            postgres = {
+              properties = {
+                mountpoint = "/zfs/media/database/postgres";
+                primarycache = "metadata";
+                recordsize = "16K";
+              };
+              snapshots = {
+                "15_min" = 8;
+                hourly = 24;
+                daily = 7;
+              };
+            };
+            "postgres-wal" = {
+              properties = {
+                compression = "lz4";
+                logbias = "latency";
+                mountpoint = "/zfs/media/database/postgres-wal";
+                primarycache = "metadata";
+                recordsize = "32K";
+                secondarycache = "none";
+                special_small_blocks = "32K";
+              };
+              snapshots = {
+                "15_min" = 4;
+                hourly = 2;
+              };
+            };
+            prometheus = {
+              properties = {
+                mountpoint = "/zfs/media/database/prometheus";
+                compression = "lz4";
+              };
+              snapshots = standard;
+            };
+            services = {
+              properties = {
+                mountpoint = "/zfs/media/services";
+                compression = "zstd-9";
+              };
+              snapshots = standard;
+            };
+            share = {
+              properties = {
+                mountpoint = "/zfs/media/share";
+                exec = "off";
+              };
+              snapshots."15_min" = 4;
+            };
+          };
+        };
+      };
+    };
+
+    storage = {
+      properties = poolDefaults "/zfs/storage";
+      datasets = {
+        nomad = {
+          properties = {
+            mountpoint = "/zfs/storage/nomad";
+            compression = "zstd-9";
+          };
+          snapshots = standard;
+        };
+        ollama = {
+          properties = {
+            compression = "zstd-19";
+            recordsize = "1M";
+            sync = "disabled";
+          };
+          snapshots."15_min" = 2;
+        };
+        secure = {
+          # An encryption root. zfs_manager must never create this: encryption
+          # is fixed at creation and is not declared here, so an automatic
+          # create would produce an unencrypted dataset. See scripts/zfs.sh.
+          createIfMissing = false;
+          properties.keylocation = zfsKey;
+          snapshots = { };
+          datasets = {
+            archive = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/archive";
+                recordsize = "1M";
+              };
+              snapshots = standard;
+            };
+            important = {
+              properties = {
+                compression = "zstd-19";
+                copies = "2";
+                mountpoint = "/zfs/storage/important";
+              };
+              snapshots = standard;
+            };
+            library = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/library";
+                recordsize = "1M";
+              };
+              snapshots = standard;
+            };
+            main = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/main";
+              };
+              snapshots = standard;
+            };
+            photos = {
+              properties = {
+                compression = "zstd-19";
+                copies = "2";
+                mountpoint = "/zfs/storage/photos";
+                recordsize = "16K";
+              };
+              snapshots = standard;
+            };
+            plex = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/plex";
+                recordsize = "1M";
+              };
+              snapshots = {
+                "15_min" = 6;
+                hourly = 2;
+                daily = 1;
+              };
+            };
+            secrets = {
+              properties = {
+                compression = "zstd-19";
+                copies = "3";
+                mountpoint = "/zfs/storage/secrets";
+              };
+              snapshots = {
+                "15_min" = 8;
+                hourly = 24;
+                daily = 30;
+                monthly = 12;
+              };
+            };
+            syncthing = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/syncthing";
+              };
+              snapshots = standard;
+            };
+            transmission = {
+              properties = {
+                compression = "zstd-9";
+                exec = "off";
+                mountpoint = "/zfs/storage/transmission";
+                recordsize = "1M";
+                sync = "disabled";
+              };
+              snapshots."15_min" = 4;
+            };
+          };
+        };
+      };
+    };
+
+    scratch = {
+      properties = poolDefaults "/zfs/scratch" // {
+        keylocation = zfsKey;
+      };
+      datasets = {
+        kafka = {
+          properties = {
+            mountpoint = "/zfs/scratch/kafka";
+            recordsize = "1M";
+          };
+          snapshots = standard;
+        };
+        kestra = {
+          properties = {
+            mountpoint = "/zfs/scratch/kestra";
+            sync = "disabled";
+          };
+          snapshots = standard;
+        };
+        transmission = {
+          properties = {
+            mountpoint = "/zfs/scratch/transmission";
+            recordsize = "16K";
+            sync = "disabled";
+          };
+          snapshots."15_min" = 2;
+        };
+        uv_cache = {
+          properties.mountpoint = "/zfs/scratch/uv_cache";
+          snapshots."15_min" = 2;
+        };
+      };
+    };
+  };
+
+  # Collapse the tree into the flat "pool/parent/child" names zfs uses. Each
+  # node keeps everything except its children.
+  flatten =
+    name: node:
+    builtins.foldl' (result: child: result // flatten "${name}/${child}" node.datasets.${child}) {
+      ${name} = builtins.removeAttrs node [ "datasets" ];
+    } (builtins.attrNames (node.datasets or { }));
+
+  datasets = builtins.foldl' (result: pool: result // flatten pool pools.${pool}) { } (
+    builtins.attrNames pools
+  );
+
+  # zfs gives a dataset with no mountpoint of its own its parent's mountpoint
+  # plus its final name component, so resolve it the same way. Pool roots all
+  # declare a mountpoint, which terminates the recursion.
+  mountpointOf =
+    name:
+    let
+      properties = datasets.${name}.properties or { };
+    in
+    if properties ? mountpoint then
+      properties.mountpoint
+    else if builtins.match ".*/.*" name != null then
+      "${mountpointOf (builtins.dirOf name)}/${builtins.baseNameOf name}"
+    else
+      throw "jeeves: ${name} has no mountpoint and no parent to inherit one from";
+
+  mountpoints = builtins.mapAttrs (name: _: mountpointOf name) datasets;
+in
+{
+  inherit datasets mountpoints;
+  defaultSnapshots = standard;
+}

systems/jeeves/default.nix

@@ -1,7 +1,4 @@
 { inputs, ... }:
-let
-  vars = import ./vars.nix;
-in
 {
   imports = [
     "${inputs.self}/users/dov"
@@ -15,6 +12,7 @@ in
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/update.nix"
     "${inputs.self}/common/optional/zerotier.nix"
+    "${inputs.self}/common/optional/zfs_manager.nix"
     ./monitoring
     ./docker
     ./services
@@ -24,6 +22,7 @@ in
     ./programs.nix
     ./runners
     ./syncthing.nix
+    ./zfs.nix
   ];
 
   services = {
@@ -31,11 +30,6 @@ in
 
     smartd.enable = true;
 
-    snapshot_manager = {
-      path = ./snapshot_config.toml;
-      EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
-    };
-
     zerotierone.joinNetworks = [ "a09acf02330d37b9" ];
   };
 

systems/jeeves/scripts/zfs.sh

@@ -1,5 +1,10 @@
 #!/bin/bash
 
+# Pool and vdev creation only. This is run by hand once per pool.
+#
+# Datasets and their properties are declared in systems/jeeves/zfs.nix and
+# reconciled by the zfs_manager service. Do not add zfs create lines here.
+
 # zpools
 
 # media
@@ -12,34 +17,10 @@ sudo zpool add storage -o ashift=12 special mirror
 sudo zpool add storage -o ashift=12 logs mirror
 
 # scratch
-sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnodesize=auto -O xattr=sa -O compression=zstd -O encryption=aes-256-gcm -O keyformat=hex -O keylocation=file:///key -m /zfs/scratch
+sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnodesize=auto -O xattr=sa -O compression=zstd -O encryption=aes-256-gcm -O keyformat=hex -O keylocation=file:///root/zfs.key -m /zfs/scratch
 
-# media datasets
-sudo zfs create media/temp -o sync=disabled -o redundant_metadata=none
+# The two encrypted parent datasets have to exist before zfs_manager can create
+# anything under them, since encryption cannot be set after creation.
+# These will be removed if/when the media and storage pools are encrypted in the future.
 sudo zfs create media/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key
-sudo zfs create media/secure/docker -o compression=zstd-9
-sudo zfs create media/secure/github-runners -o compression=zstd-9 -o sync=disabled
-sudo zfs create media/secure/notes -o copies=2
-sudo zfs create media/secure/postgres -o mountpoint=/zfs/media/database/postgres -o recordsize=16k -o primarycache=metadata
-sudo zfs create media/secure/postgres-wal -o mountpoint=/zfs/media/database/postgres-wal -o recordsize=32k -o primarycache=metadata -o special_small_blocks=32K -o compression=lz4 -o secondarycache=none -o logbias=latency
-sudo zfs create media/secure/prometheus -o mountpoint=/zfs/media/database/prometheus -o compression=lz4
-sudo zfs create media/secure/services -o compression=zstd-9
-sudo zfs create media/secure/share -o mountpoint=/zfs/media/share -o exec=off
-
-# scratch datasets
-sudo zfs create scratch/kafka -o mountpoint=/zfs/scratch/kafka -o recordsize=1M
-sudo zfs create scratch/transmission -o mountpoint=/zfs/scratch/transmission -o recordsize=16k -o sync=disabled -o redundant_metadata=none
-sudo zfs create scratch/uv_cache -o mountpoint=/zfs/scratch/uv_cache
-
-# storage datasets
-sudo zfs create storage/ollama -o recordsize=1M -o compression=zstd-19 -o sync=disabled
 sudo zfs create storage/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key
-sudo zfs create storage/secure/archive -o recordsize=1M -o compression=zstd-19
-sudo zfs create storage/secure/library -o recordsize=1M -o compression=zstd-19
-sudo zfs create storage/secure/main -o compression=zstd-19
-sudo zfs create storage/secure/photos -o recordsize=16K -o compression=zstd-19 -o copies=2
-sudo zfs create storage/secure/plex -o recordsize=1M -o compression=zstd-19
-sudo zfs create storage/secure/secrets -o compression=zstd-19 -o copies=3
-sudo zfs create storage/secure/syncthing -o compression=zstd-19
-sudo zfs create storage/secure/transmission -o recordsize=1M -o compression=zstd-9 -o exec=off -o sync=disabled
-sudo zfs create storage/secure/important -o compression=zstd-19 -o copies=2 -o mountpoint=/zfs/storage/important

systems/jeeves/snapshot_config.toml

@@ -1,129 +0,0 @@
-["default"]
-15_min = 8
-hourly = 24
-daily = 0
-monthly = 0
-
-# root_pool
-["root_pool/home"]
-15_min = 8
-hourly = 24
-daily = 14
-monthly = 0
-
-["root_pool/root"]
-15_min = 8
-hourly = 24
-daily = 0
-monthly = 0
-
-["root_pool/nix"]
-15_min = 4
-hourly = 0
-daily = 0
-monthly = 0
-
-["root_pool/var"]
-15_min = 8
-hourly = 24
-daily = 30
-monthly = 6
-# storage
-["storage/ollama"]
-15_min = 2
-hourly = 0
-daily = 0
-monthly = 0
-
-["storage/secure"]
-15_min = 0
-hourly = 0
-daily = 0
-monthly = 0
-
-["storage/secure/plex"]
-15_min = 6
-hourly = 2
-daily = 1
-monthly = 0
-
-["storage/secure/transmission"]
-15_min = 4
-hourly = 0
-daily = 0
-monthly = 0
-
-["storage/secure/secrets"]
-15_min = 8
-hourly = 24
-daily = 30
-monthly = 12
-
-# media
-["media/temp"]
-15_min = 2
-hourly = 0
-daily = 0
-monthly = 0
-
-["media/secure"]
-15_min = 0
-hourly = 0
-daily = 0
-monthly = 0
-
-["media/secure/plex"]
-15_min = 6
-hourly = 2
-daily = 1
-monthly = 0
-
-["media/secure/postgres-wal"]
-15_min = 4
-hourly = 2
-daily = 0
-monthly = 0
-
-
-["media/secure/postgres"]
-15_min = 8
-hourly = 24
-daily = 7
-monthly = 0
-
-["media/secure/share"]
-15_min = 4
-hourly = 0
-daily = 0
-monthly = 0
-
-["media/secure/github-runners"]
-15_min = 6
-hourly = 2
-daily = 1
-monthly = 0
-
-["media/secure/notes"]
-15_min = 8
-hourly = 24
-daily = 30
-monthly = 12
-
-["media/secure/docker"]
-15_min = 3
-hourly = 12
-daily = 14
-monthly = 2
-
-# scratch
-["scratch/transmission"]
-15_min = 2
-hourly = 0
-daily = 0
-monthly = 0
-
-["scratch/uv_cache"]
-15_min = 2
-hourly = 0
-daily = 0
-monthly = 0

systems/jeeves/vars.nix

@@ -1,21 +1,22 @@
+# Paths are derived from the dataset declarations in ./datasets.nix so that a
+# mountpoint only ever has to change in one place.
 let
-  zfs_media = "/zfs/media";
-  zfs_storage = "/zfs/storage";
-  zfs_scratch = "/zfs/scratch";
+  inherit (import ./datasets.nix) mountpoints;
 in
 {
-  inherit zfs_media zfs_storage zfs_scratch;
-  database = "${zfs_media}/database";
-  docker = "${zfs_media}/docker";
-  docker_configs = "${zfs_media}/docker/configs";
-  notes = "${zfs_media}/notes";
-  secrets = "${zfs_storage}/secrets";
-  services = "${zfs_media}/services";
-  share = "${zfs_media}/share";
-  syncthing = "${zfs_storage}/syncthing";
-  transmission = "${zfs_storage}/transmission";
-  ollama = "${zfs_storage}/ollama";
-  transmission_scratch = "${zfs_scratch}/transmission";
-  uv_cache = "${zfs_scratch}/uv_cache";
-  kafka = "${zfs_scratch}/kafka";
+  # Not a dataset of its own, it is the directory the postgres datasets share.
+  database = "/zfs/media/database";
+  docker = mountpoints."media/secure/docker";
+  docker_configs = "${mountpoints."media/secure/docker"}/configs";
+  home_assistant = mountpoints."media/secure/home_assistant";
+  notes = mountpoints."media/secure/notes";
+  secrets = mountpoints."storage/secure/secrets";
+  services = mountpoints."media/secure/services";
+  share = mountpoints."media/secure/share";
+  syncthing = mountpoints."storage/secure/syncthing";
+  transmission = mountpoints."storage/secure/transmission";
+  ollama = mountpoints."storage/ollama";
+  transmission_scratch = mountpoints."scratch/transmission";
+  uv_cache = mountpoints."scratch/uv_cache";
+  kafka = mountpoints."scratch/kafka";
 }

systems/jeeves/zfs.nix

@@ -0,0 +1,20 @@
+{ inputs, ... }:
+let
+  vars = import ./vars.nix;
+  jeeves_zfs = import ./datasets.nix;
+in
+{
+  services = {
+    zfs_manager = {
+      enable = true;
+      PYTHONPATH = "${inputs.self}/";
+      EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
+
+      inherit (jeeves_zfs) datasets defaultSnapshots;
+    };
+
+    # Its retention config is generated from ./datasets.nix by
+    # common/optional/zfs_manager.nix, so only the credentials are set here.
+    snapshot_manager.EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
+  };
+}