feat(zfs): manage jeeves datasets declaratively from nix

Add a zfs_manager module that reconciles the live datasets on jeeves
against a nix declaration, and generate the snapshot retention config
from that same declaration so the two can no longer drift apart.

systems/jeeves/datasets.nix declares every dataset on the media, storage
and scratch pools, nested the way zfs nests them and flattened into
pool/parent/child names. Values were transcribed from the live pools
rather than from scripts/zfs.sh, which had gone stale: acltype reads back
as posix, and media/secure/important, scratch/kestra and storage/nomad
were never recorded. root_pool datasets are declared for retention only,
their properties stay unmanaged for now.

python.tools.zfs_manager creates missing datasets and corrects drifted
properties, and never destroys anything. Undeclared properties are judged
by the zfs source field, so inherited and default values stay quiet while
locally set ones warn. Size values are normalised to bytes so that 16K and
16384 do not re-issue zfs set on every run.

vars.nix now derives its paths from the declared mountpoints instead of
repeating them, dropping three zfs_* keys that nothing referenced.

Replaces systems/jeeves/snapshot_config.toml, which listed a dataset that
does not exist and omitted thirteen that do.

systems/jeeves/datasets.nix

@@ -0,0 +1,356 @@
+# Dataset declarations for jeeves, kept as plain data rather than inside
+# zfs.nix so that vars.nix can derive its paths from the same source.
+#
+# Datasets are nested the way zfs nests them: a pool holds datasets, which can
+# hold datasets of their own. The tree is flattened into "pool/parent/child"
+# names below, which is what zfs and services.zfs_manager work in.
+#
+# Consumed by ./zfs.nix (which feeds it to services.zfs_manager) and by
+# ./vars.nix (which resolves the mountpoints).
+let
+  # Every pool on jeeves was created with the same -O options.
+  poolDefaults = mountpoint: {
+    inherit mountpoint;
+    acltype = "posix"; # zfs reports posixacl back as posix
+    atime = "off";
+    compression = "zstd";
+    dnodesize = "auto";
+    xattr = "sa";
+  };
+
+  zfsKey = "file:///root/zfs.key";
+
+  # What a dataset gets when it is not called out below, kept identical to the
+  # "default" table so the datasets that used to fall through are unchanged.
+  standard = {
+    "15_min" = 8;
+    hourly = 24;
+  };
+
+  pools = {
+    # root_pool: retention only, its properties are not managed yet.
+    root_pool = {
+      manageProperties = false;
+      datasets = {
+        home = {
+          manageProperties = false;
+          snapshots = {
+            "15_min" = 8;
+            hourly = 24;
+            daily = 14;
+          };
+        };
+        root = {
+          manageProperties = false;
+          snapshots = standard;
+        };
+        nix = {
+          manageProperties = false;
+          snapshots."15_min" = 4;
+        };
+        var = {
+          manageProperties = false;
+          snapshots = {
+            "15_min" = 8;
+            hourly = 24;
+            daily = 30;
+            monthly = 6;
+          };
+        };
+      };
+    };
+
+    media = {
+      properties = poolDefaults "/zfs/media";
+      datasets = {
+        temp = {
+          properties = {
+            redundant_metadata = "none";
+            sync = "disabled";
+          };
+          snapshots."15_min" = 2;
+        };
+        secure = {
+          properties.keylocation = zfsKey;
+          snapshots = { };
+          datasets = {
+            docker = {
+              properties = {
+                mountpoint = "/zfs/media/docker";
+                compression = "zstd-9";
+              };
+              snapshots = {
+                "15_min" = 3;
+                hourly = 12;
+                daily = 14;
+                monthly = 2;
+              };
+            };
+            "github-runners" = {
+              properties = {
+                mountpoint = "/zfs/media/github-runners";
+                compression = "zstd-9";
+                sync = "disabled";
+              };
+              snapshots = {
+                "15_min" = 6;
+                hourly = 2;
+                daily = 1;
+              };
+            };
+            home_assistant = {
+              properties = {
+                mountpoint = "/zfs/media/home_assistant";
+                compression = "zstd-19";
+              };
+              snapshots = standard;
+            };
+            important = {
+              properties = {
+                compression = "zstd-9";
+                copies = "2";
+              };
+              snapshots = standard;
+            };
+            notes = {
+              properties = {
+                mountpoint = "/zfs/media/notes";
+                copies = "2";
+              };
+              snapshots = {
+                "15_min" = 8;
+                hourly = 24;
+                daily = 30;
+                monthly = 12;
+              };
+            };
+            postgres = {
+              properties = {
+                mountpoint = "/zfs/media/database/postgres";
+                primarycache = "metadata";
+                recordsize = "16K";
+              };
+              snapshots = {
+                "15_min" = 8;
+                hourly = 24;
+                daily = 7;
+              };
+            };
+            "postgres-wal" = {
+              properties = {
+                compression = "lz4";
+                logbias = "latency";
+                mountpoint = "/zfs/media/database/postgres-wal";
+                primarycache = "metadata";
+                recordsize = "32K";
+                secondarycache = "none";
+                special_small_blocks = "32K";
+              };
+              snapshots = {
+                "15_min" = 4;
+                hourly = 2;
+              };
+            };
+            prometheus = {
+              properties = {
+                mountpoint = "/zfs/media/database/prometheus";
+                compression = "lz4";
+              };
+              snapshots = standard;
+            };
+            services = {
+              properties = {
+                mountpoint = "/zfs/media/services";
+                compression = "zstd-9";
+              };
+              snapshots = standard;
+            };
+            share = {
+              properties = {
+                mountpoint = "/zfs/media/share";
+                exec = "off";
+              };
+              snapshots."15_min" = 4;
+            };
+          };
+        };
+      };
+    };
+
+    storage = {
+      properties = poolDefaults "/zfs/storage";
+      datasets = {
+        nomad = {
+          properties = {
+            mountpoint = "/zfs/storage/nomad";
+            compression = "zstd-9";
+          };
+          snapshots = standard;
+        };
+        ollama = {
+          properties = {
+            compression = "zstd-19";
+            recordsize = "1M";
+            sync = "disabled";
+          };
+          snapshots."15_min" = 2;
+        };
+        secure = {
+          properties.keylocation = zfsKey;
+          snapshots = { };
+          datasets = {
+            archive = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/archive";
+                recordsize = "1M";
+              };
+              snapshots = standard;
+            };
+            important = {
+              properties = {
+                compression = "zstd-19";
+                copies = "2";
+                mountpoint = "/zfs/storage/important";
+              };
+              snapshots = standard;
+            };
+            library = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/library";
+                recordsize = "1M";
+              };
+              snapshots = standard;
+            };
+            main = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/main";
+              };
+              snapshots = standard;
+            };
+            photos = {
+              properties = {
+                compression = "zstd-19";
+                copies = "2";
+                mountpoint = "/zfs/storage/photos";
+                recordsize = "16K";
+              };
+              snapshots = standard;
+            };
+            plex = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/plex";
+                recordsize = "1M";
+              };
+              snapshots = {
+                "15_min" = 6;
+                hourly = 2;
+                daily = 1;
+              };
+            };
+            secrets = {
+              properties = {
+                compression = "zstd-19";
+                copies = "3";
+                mountpoint = "/zfs/storage/secrets";
+              };
+              snapshots = {
+                "15_min" = 8;
+                hourly = 24;
+                daily = 30;
+                monthly = 12;
+              };
+            };
+            syncthing = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/syncthing";
+              };
+              snapshots = standard;
+            };
+            transmission = {
+              properties = {
+                compression = "zstd-9";
+                exec = "off";
+                mountpoint = "/zfs/storage/transmission";
+                recordsize = "1M";
+                sync = "disabled";
+              };
+              snapshots."15_min" = 4;
+            };
+          };
+        };
+      };
+    };
+
+    scratch = {
+      properties = poolDefaults "/zfs/scratch" // {
+        keylocation = zfsKey;
+      };
+      datasets = {
+        kafka = {
+          properties = {
+            mountpoint = "/zfs/scratch/kafka";
+            recordsize = "1M";
+          };
+          snapshots = standard;
+        };
+        kestra = {
+          properties = {
+            mountpoint = "/zfs/scratch/kestra";
+            sync = "disabled";
+          };
+          snapshots = standard;
+        };
+        transmission = {
+          properties = {
+            mountpoint = "/zfs/scratch/transmission";
+            recordsize = "16K";
+            sync = "disabled";
+          };
+          snapshots."15_min" = 2;
+        };
+        uv_cache = {
+          properties.mountpoint = "/zfs/scratch/uv_cache";
+          snapshots."15_min" = 2;
+        };
+      };
+    };
+  };
+
+  # Collapse the tree into the flat "pool/parent/child" names zfs uses. Each
+  # node keeps everything except its children.
+  flatten =
+    name: node:
+    builtins.foldl' (result: child: result // flatten "${name}/${child}" node.datasets.${child}) {
+      ${name} = builtins.removeAttrs node [ "datasets" ];
+    } (builtins.attrNames (node.datasets or { }));
+
+  datasets = builtins.foldl' (result: pool: result // flatten pool pools.${pool}) { } (
+    builtins.attrNames pools
+  );
+
+  # zfs gives a dataset with no mountpoint of its own its parent's mountpoint
+  # plus its final name component, so resolve it the same way. Pool roots all
+  # declare a mountpoint, which terminates the recursion.
+  mountpointOf =
+    name:
+    let
+      properties = datasets.${name}.properties or { };
+    in
+    if properties ? mountpoint then
+      properties.mountpoint
+    else if builtins.match ".*/.*" name != null then
+      "${mountpointOf (builtins.dirOf name)}/${builtins.baseNameOf name}"
+    else
+      throw "jeeves: ${name} has no mountpoint and no parent to inherit one from";
+
+  mountpoints = builtins.mapAttrs (name: _: mountpointOf name) datasets;
+in
+{
+  inherit datasets mountpoints;
+  defaultSnapshots = standard;
+}

systems/jeeves/default.nix

@@ -15,6 +15,7 @@ in
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/update.nix"
     "${inputs.self}/common/optional/zerotier.nix"
+    "${inputs.self}/common/optional/zfs_manager.nix"
     ./monitoring
     ./docker
     ./services
@@ -24,6 +25,7 @@ in
     ./programs.nix
     ./runners
     ./syncthing.nix
+    ./zfs.nix
   ];
 
   services = {
@@ -31,10 +33,8 @@ in
 
     smartd.enable = true;
 
-    snapshot_manager = {
-      path = ./snapshot_config.toml;
-      EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
-    };
+    # path is generated from ./zfs.nix by common/optional/zfs_manager.nix
+    snapshot_manager.EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
 
     zerotierone.joinNetworks = [ "a09acf02330d37b9" ];
   };

systems/jeeves/scripts/zfs.sh

@@ -1,5 +1,10 @@
 #!/bin/bash
 
+# Pool and vdev creation only. This is run by hand once per pool.
+#
+# Datasets and their properties are declared in systems/jeeves/zfs.nix and
+# reconciled by the zfs_manager service. Do not add zfs create lines here.
+
 # zpools
 
 # media
@@ -12,35 +17,10 @@ sudo zpool add storage -o ashift=12 special mirror
 sudo zpool add storage -o ashift=12 logs mirror
 
 # scratch
-sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnodesize=auto -O xattr=sa -O compression=zstd -O encryption=aes-256-gcm -O keyformat=hex -O keylocation=file:///key -m /zfs/scratch
+sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnodesize=auto -O xattr=sa -O compression=zstd -O encryption=aes-256-gcm -O keyformat=hex -O keylocation=file:///root/zfs.key -m /zfs/scratch
 
-# media datasets
-sudo zfs create media/temp -o sync=disabled -o redundant_metadata=none
+# The two encrypted parent datasets have to exist before zfs_manager can create
+# anything under them, since encryption cannot be set after creation.
+# These will be removed if/when the media and storage pools are encrypted in the future.
 sudo zfs create media/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key
-sudo zfs create media/secure/docker -o compression=zstd-9
-sudo zfs create media/secure/github-runners -o compression=zstd-9 -o sync=disabled
-sudo zfs create media/secure/home_assistant -o compression=zstd-19
-sudo zfs create media/secure/notes -o copies=2
-sudo zfs create media/secure/postgres -o mountpoint=/zfs/media/database/postgres -o recordsize=16k -o primarycache=metadata
-sudo zfs create media/secure/postgres-wal -o mountpoint=/zfs/media/database/postgres-wal -o recordsize=32k -o primarycache=metadata -o special_small_blocks=32K -o compression=lz4 -o secondarycache=none -o logbias=latency
-sudo zfs create media/secure/prometheus -o mountpoint=/zfs/media/database/prometheus -o compression=lz4
-sudo zfs create media/secure/services -o compression=zstd-9
-sudo zfs create media/secure/share -o mountpoint=/zfs/media/share -o exec=off
-
-# scratch datasets
-sudo zfs create scratch/kafka -o mountpoint=/zfs/scratch/kafka -o recordsize=1M
-sudo zfs create scratch/transmission -o mountpoint=/zfs/scratch/transmission -o recordsize=16k -o sync=disabled -o redundant_metadata=none
-sudo zfs create scratch/uv_cache -o mountpoint=/zfs/scratch/uv_cache
-
-# storage datasets
-sudo zfs create storage/ollama -o recordsize=1M -o compression=zstd-19 -o sync=disabled
 sudo zfs create storage/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key
-sudo zfs create storage/secure/archive -o recordsize=1M -o compression=zstd-19
-sudo zfs create storage/secure/library -o recordsize=1M -o compression=zstd-19
-sudo zfs create storage/secure/main -o compression=zstd-19
-sudo zfs create storage/secure/photos -o recordsize=16K -o compression=zstd-19 -o copies=2
-sudo zfs create storage/secure/plex -o recordsize=1M -o compression=zstd-19
-sudo zfs create storage/secure/secrets -o compression=zstd-19 -o copies=3
-sudo zfs create storage/secure/syncthing -o compression=zstd-19
-sudo zfs create storage/secure/transmission -o recordsize=1M -o compression=zstd-9 -o exec=off -o sync=disabled
-sudo zfs create storage/secure/important -o compression=zstd-19 -o copies=2 -o mountpoint=/zfs/storage/important

systems/jeeves/snapshot_config.toml

@@ -1,129 +0,0 @@
-["default"]
-15_min = 8
-hourly = 24
-daily = 0
-monthly = 0
-
-# root_pool
-["root_pool/home"]
-15_min = 8
-hourly = 24
-daily = 14
-monthly = 0
-
-["root_pool/root"]
-15_min = 8
-hourly = 24
-daily = 0
-monthly = 0
-
-["root_pool/nix"]
-15_min = 4
-hourly = 0
-daily = 0
-monthly = 0
-
-["root_pool/var"]
-15_min = 8
-hourly = 24
-daily = 30
-monthly = 6
-# storage
-["storage/ollama"]
-15_min = 2
-hourly = 0
-daily = 0
-monthly = 0
-
-["storage/secure"]
-15_min = 0
-hourly = 0
-daily = 0
-monthly = 0
-
-["storage/secure/plex"]
-15_min = 6
-hourly = 2
-daily = 1
-monthly = 0
-
-["storage/secure/transmission"]
-15_min = 4
-hourly = 0
-daily = 0
-monthly = 0
-
-["storage/secure/secrets"]
-15_min = 8
-hourly = 24
-daily = 30
-monthly = 12
-
-# media
-["media/temp"]
-15_min = 2
-hourly = 0
-daily = 0
-monthly = 0
-
-["media/secure"]
-15_min = 0
-hourly = 0
-daily = 0
-monthly = 0
-
-["media/secure/plex"]
-15_min = 6
-hourly = 2
-daily = 1
-monthly = 0
-
-["media/secure/postgres-wal"]
-15_min = 4
-hourly = 2
-daily = 0
-monthly = 0
-
-
-["media/secure/postgres"]
-15_min = 8
-hourly = 24
-daily = 7
-monthly = 0
-
-["media/secure/share"]
-15_min = 4
-hourly = 0
-daily = 0
-monthly = 0
-
-["media/secure/github-runners"]
-15_min = 6
-hourly = 2
-daily = 1
-monthly = 0
-
-["media/secure/notes"]
-15_min = 8
-hourly = 24
-daily = 30
-monthly = 12
-
-["media/secure/docker"]
-15_min = 3
-hourly = 12
-daily = 14
-monthly = 2
-
-# scratch
-["scratch/transmission"]
-15_min = 2
-hourly = 0
-daily = 0
-monthly = 0
-
-["scratch/uv_cache"]
-15_min = 2
-hourly = 0
-daily = 0
-monthly = 0

systems/jeeves/vars.nix

@@ -1,22 +1,22 @@
+# Paths are derived from the dataset declarations in ./datasets.nix so that a
+# mountpoint only ever has to change in one place.
 let
-  zfs_media = "/zfs/media";
-  zfs_storage = "/zfs/storage";
-  zfs_scratch = "/zfs/scratch";
+  inherit (import ./datasets.nix) mountpoints;
 in
 {
-  inherit zfs_media zfs_storage zfs_scratch;
-  database = "${zfs_media}/database";
-  docker = "${zfs_media}/docker";
-  docker_configs = "${zfs_media}/docker/configs";
-  home_assistant = "${zfs_media}/home_assistant";
-  notes = "${zfs_media}/notes";
-  secrets = "${zfs_storage}/secrets";
-  services = "${zfs_media}/services";
-  share = "${zfs_media}/share";
-  syncthing = "${zfs_storage}/syncthing";
-  transmission = "${zfs_storage}/transmission";
-  ollama = "${zfs_storage}/ollama";
-  transmission_scratch = "${zfs_scratch}/transmission";
-  uv_cache = "${zfs_scratch}/uv_cache";
-  kafka = "${zfs_scratch}/kafka";
+  # Not a dataset of its own, it is the directory the postgres datasets share.
+  database = "/zfs/media/database";
+  docker = mountpoints."media/secure/docker";
+  docker_configs = "${mountpoints."media/secure/docker"}/configs";
+  home_assistant = mountpoints."media/secure/home_assistant";
+  notes = mountpoints."media/secure/notes";
+  secrets = mountpoints."storage/secure/secrets";
+  services = mountpoints."media/secure/services";
+  share = mountpoints."media/secure/share";
+  syncthing = mountpoints."storage/secure/syncthing";
+  transmission = mountpoints."storage/secure/transmission";
+  ollama = mountpoints."storage/ollama";
+  transmission_scratch = mountpoints."scratch/transmission";
+  uv_cache = mountpoints."scratch/uv_cache";
+  kafka = mountpoints."scratch/kafka";
 }

systems/jeeves/zfs.nix

@@ -0,0 +1,14 @@
+{ inputs, ... }:
+let
+  vars = import ./vars.nix;
+  jeeves_zfs = import ./datasets.nix;
+in
+{
+  services.zfs_manager = {
+    enable = true;
+    PYTHONPATH = "${inputs.self}/";
+    EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
+
+    inherit (jeeves_zfs) datasets defaultSnapshots;
+  };
+}