common/optional/zfs_manager.nix

@@ -0,0 +1,168 @@
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  cfg = config.services.zfs_manager;
+
+  snapshotOptions = {
+    options = {
+      "15_min" = lib.mkOption {
+        type = lib.types.int;
+        default = 0;
+        description = "How many 15 minute snapshots to keep.";
+      };
+      hourly = lib.mkOption {
+        type = lib.types.int;
+        default = 0;
+        description = "How many hourly snapshots to keep.";
+      };
+      daily = lib.mkOption {
+        type = lib.types.int;
+        default = 0;
+        description = "How many daily snapshots to keep.";
+      };
+      monthly = lib.mkOption {
+        type = lib.types.int;
+        default = 0;
+        description = "How many monthly snapshots to keep.";
+      };
+    };
+  };
+
+  datasetOptions = {
+    options = {
+      manageProperties = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+        description = ''
+          Whether zfs_manager owns this dataset's properties. When false the
+          dataset only contributes its snapshot retention, which is how
+          root_pool datasets are declared.
+        '';
+      };
+      properties = lib.mkOption {
+        type = lib.types.attrsOf lib.types.str;
+        default = { };
+        description = ''
+          The zfs properties this dataset should have. Values are compared
+          against the live dataset and corrected when they differ.
+        '';
+      };
+      snapshots = lib.mkOption {
+        type = lib.types.submodule snapshotOptions;
+        default = { };
+        description = "Snapshot retention for this dataset.";
+      };
+    };
+  };
+
+  # snapshot_manager.py only ever walks datasets below a pool root, so pool
+  # roots are left out of the retention table. It also indexes the table
+  # directly, which is why every entry carries all four keys.
+  snapshotTable = lib.mapAttrs (_: dataset: dataset.snapshots) (
+    lib.filterAttrs (name: _: lib.hasInfix "/" name) cfg.datasets
+  );
+
+  snapshotConfig = (pkgs.formats.toml { }).generate "snapshot_config.toml" (
+    snapshotTable // { default = cfg.defaultSnapshots; }
+  );
+
+  # Every declared dataset is emitted, including the ones whose properties are
+  # not managed, so the tool can tell "deliberately hands off" apart from
+  # "nobody has written this down yet".
+  datasetConfig = (pkgs.formats.json { }).generate "zfs_datasets.json" {
+    datasets = lib.mapAttrs (_: dataset: {
+      inherit (dataset) manageProperties properties;
+    }) cfg.datasets;
+  };
+in
+{
+  options = {
+    services.zfs_manager = {
+      enable = lib.mkEnableOption "declarative ZFS dataset management";
+      datasets = lib.mkOption {
+        type = lib.types.attrsOf (lib.types.submodule datasetOptions);
+        default = { };
+        example = lib.literalExpression ''
+          {
+            "media/temp".properties = {
+              sync = "disabled";
+              redundant_metadata = "none";
+            };
+          }
+        '';
+        description = ''
+          The datasets to manage, keyed by full dataset name. Missing datasets
+          are created and drifted properties are corrected. Nothing is ever
+          destroyed, and datasets that are not declared are left alone.
+
+          A name without a "/" is a pool root filesystem. Its properties are
+          managed but it is never created, pool creation stays manual.
+        '';
+      };
+      defaultSnapshots = lib.mkOption {
+        type = lib.types.submodule snapshotOptions;
+        default = { };
+        description = ''
+          Retention for any dataset that is not declared above, emitted as the
+          "default" table of the snapshot config.
+        '';
+      };
+      dryRun = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Log every change that would be made without touching zfs. Use this to
+          validate a new or heavily edited declaration before applying it.
+        '';
+      };
+      PYTHONPATH = lib.mkOption {
+        type = lib.types.str;
+        description = ''
+          the PYTHONPATH to use for the zfs_manager service.
+        '';
+      };
+      EnvironmentFile = lib.mkOption {
+        type = lib.types.nullOr (lib.types.coercedTo lib.types.path toString lib.types.str);
+        default = null;
+        description = ''
+          Single environment file for the service (e.g. /etc/zfs-manager/env).
+          Use a leading "-" to ignore if missing (systemd feature).
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.snapshot_manager.path = snapshotConfig;
+
+    systemd = {
+      services.zfs_manager = {
+        description = "ZFS Dataset Manager";
+        requires = [ "zfs-import.target" ];
+        after = [
+          "zfs-import.target"
+          "zfs-mount.service"
+        ];
+        wantedBy = [ "multi-user.target" ];
+        path = [ pkgs.zfs ];
+        # Re-run on nixos-rebuild switch whenever the declaration changes.
+        restartTriggers = [ datasetConfig ];
+        environment = {
+          PYTHONPATH = cfg.PYTHONPATH;
+        };
+        serviceConfig = {
+          Type = "oneshot";
+          RemainAfterExit = true;
+          ExecStart = "${pkgs.my_python}/bin/python -m python.tools.zfs_manager ${lib.escapeShellArg datasetConfig}${lib.optionalString cfg.dryRun " --dry-run"}";
+        }
+        // lib.optionalAttrs (cfg.EnvironmentFile != null) {
+          EnvironmentFile = cfg.EnvironmentFile;
+        };
+      };
+    };
+  };
+}

python/tools/zfs_manager.py

@@ -0,0 +1,267 @@
+"""zfs_manager.
+
+Reconciles the live zfs datasets against a declaration generated by
+common/optional/zfs_manager.nix. Datasets are created and properties are
+corrected, but nothing is ever destroyed or renamed.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sys
+from pathlib import Path  # noqa: TC003 This is required for the typer CLI
+
+import typer
+
+from python.common import configure_logger
+from python.signal_alert import signal_alert
+from python.zfs import create_dataset, get_properties, list_dataset_names, set_property
+
+logger = logging.getLogger(__name__)
+
+# Properties that can only be chosen at creation time. Attempting to zfs set
+# these fails on every run, so a mismatch is reported instead of retried.
+CREATE_ONLY_PROPERTIES = frozenset(
+    {
+        "casesensitivity",
+        "encryption",
+        "keyformat",
+        "normalization",
+        "utf8only",
+        "volblocksize",
+    },
+)
+
+# Properties whose values zfs reports in bytes but which are conventionally
+# declared with a size suffix, so "16k" and "16384" mean the same thing.
+SIZE_PROPERTIES = frozenset(
+    {
+        "quota",
+        "recordsize",
+        "refquota",
+        "refreservation",
+        "reservation",
+        "special_small_blocks",
+        "volblocksize",
+        "volsize",
+    },
+)
+
+SIZE_SUFFIXES = {"b": 1, "k": 1024, "m": 1024**2, "g": 1024**3, "t": 1024**4, "p": 1024**5}
+
+# Sources that mean the value was deliberately put on this dataset rather than
+# inherited from a parent or left at the zfs default.
+LOCAL_SOURCES = ("local", "received")
+
+
+def main(config_file: Path, *, dry_run: bool = False) -> None:
+    """Main.
+
+    Args:
+        config_file (Path): The path to the generated dataset declaration.
+        dry_run (bool): Log the changes that would be made without making them.
+    """
+    configure_logger(level="DEBUG")
+    logger.info(f"Starting zfs_manager {dry_run=}")
+
+    try:
+        declared = json.loads(config_file.read_text())["datasets"]
+        existing = set(list_dataset_names())
+        unusable: set[str] = set()
+
+        # Parents before children so a newly created parent exists by the time
+        # its children are reconciled.
+        for name in sorted(declared, key=lambda name: (name.count("/"), name)):
+            if has_unusable_parent(name, unusable):
+                logger.error(f"skipping {name}, its parent could not be created")
+                continue
+
+            # Declared purely to record retention, its properties belong to
+            # whoever set them.
+            if not declared[name].get("manageProperties", True):
+                logger.debug(f"{name} is declared but its properties are not managed")
+                continue
+
+            properties = declared[name]["properties"]
+            if name in existing:
+                reconcile_dataset(name, properties, dry_run=dry_run)
+            elif create_missing_dataset(name, properties, dry_run=dry_run):
+                existing.add(name)
+            elif not dry_run:
+                unusable.add(name)
+
+        report_undeclared_datasets(existing, declared)
+    except Exception:
+        logger.exception("zfs_manager failed")
+        signal_alert("zfs_manager failed")
+        sys.exit(1)
+    else:
+        logger.info("zfs_manager completed")
+
+
+def has_unusable_parent(name: str, unusable: set[str]) -> bool:
+    """Check whether an ancestor of a dataset could not be created.
+
+    Args:
+        name (str): The name of the dataset.
+        unusable (set[str]): The datasets that do not exist and could not be created.
+
+    Returns:
+        bool: True if any ancestor is unusable.
+    """
+    parts = name.split("/")
+    return any("/".join(parts[:depth]) in unusable for depth in range(1, len(parts)))
+
+
+def create_missing_dataset(name: str, properties: dict[str, str], *, dry_run: bool) -> bool:
+    """Create a dataset that is declared but does not exist yet.
+
+    Pool root filesystems are never created, creating a pool is out of scope.
+
+    Args:
+        name (str): The name of the dataset.
+        properties (dict[str, str]): The properties to create the dataset with.
+        dry_run (bool): Log the change without making it.
+
+    Returns:
+        bool: True if the dataset now exists.
+    """
+    if "/" not in name:
+        message = f"pool {name} is declared but does not exist, zfs_manager does not create pools"
+        logger.error(message)
+        signal_alert(message)
+        return False
+
+    if dry_run:
+        logger.info(f"would create {name} with {properties}")
+        return False
+
+    logger.info(f"creating {name} with {properties}")
+    if error := create_dataset(name, properties):
+        logger.error(error)
+        signal_alert(error)
+        return False
+
+    return True
+
+
+def reconcile_dataset(name: str, properties: dict[str, str], *, dry_run: bool) -> None:
+    """Bring an existing dataset in line with its declared properties.
+
+    Args:
+        name (str): The name of the dataset.
+        properties (dict[str, str]): The declared properties.
+        dry_run (bool): Log the changes without making them.
+    """
+    current = get_properties(name)
+
+    for key, wanted in sorted(properties.items()):
+        current_value, _ = current.get(key, ("-", "-"))
+        if values_match(key, wanted, current_value):
+            continue
+
+        if key in CREATE_ONLY_PROPERTIES:
+            message = (
+                f"{name} {key} is {current_value} but {wanted} is declared, "
+                f"{key} can only be set when the dataset is created"
+            )
+            logger.error(message)
+            signal_alert(message)
+            continue
+
+        if dry_run:
+            logger.info(f"would set {key}={wanted} on {name}, currently {current_value}")
+            continue
+
+        logger.info(f"setting {key}={wanted} on {name}, was {current_value}")
+        if error := set_property(name, key, wanted):
+            logger.error(error)
+            signal_alert(error)
+
+    report_undeclared_properties(name, properties, current)
+
+
+def report_undeclared_properties(name: str, properties: dict[str, str], current: dict[str, tuple[str, str]]) -> None:
+    """Warn about properties set on the dataset but absent from the declaration.
+
+    Inherited and default values are silent, they are not drift. A locally set
+    value that nix does not know about was changed outside of this tool and
+    will be lost the next time the dataset is recreated, so it is worth saying.
+
+    Args:
+        name (str): The name of the dataset.
+        properties (dict[str, str]): The declared properties.
+        current (dict[str, tuple[str, str]]): The live properties keyed to (value, source).
+    """
+    for key, (value, source) in sorted(current.items()):
+        # User properties such as nixos:shutdown-time are written by other
+        # tools and are not something a dataset declaration should own.
+        if key in properties or ":" in key or source not in LOCAL_SOURCES:
+            continue
+
+        logger.warning(f"{name} has {key}={value} set outside of nix")
+        signal_alert(f"{name} has {key}={value} set outside of nix")
+
+
+def report_undeclared_datasets(existing: set[str], declared: dict[str, dict]) -> None:
+    """Warn about datasets that exist but are not declared.
+
+    These are left completely alone. They still get snapshots through the
+    default retention table.
+
+    Args:
+        existing (set[str]): The names of every live dataset.
+        declared (dict[str, dict]): The declaration.
+    """
+    for name in sorted(existing - set(declared)):
+        logger.warning(f"{name} exists but is not declared in nix")
+
+
+def values_match(key: str, wanted: str, current: str) -> bool:
+    """Compare a declared property value against the live one.
+
+    Args:
+        key (str): The property name.
+        wanted (str): The declared value.
+        current (str): The live value.
+
+    Returns:
+        bool: True if the two values mean the same thing.
+    """
+    if key in SIZE_PROPERTIES:
+        wanted_size = parse_size(wanted)
+        current_size = parse_size(current)
+        if wanted_size is not None and current_size is not None:
+            return wanted_size == current_size
+
+    return wanted == current
+
+
+def parse_size(value: str) -> int | None:
+    """Convert a zfs size such as 16k or 1M into bytes.
+
+    Args:
+        value (str): The size to convert.
+
+    Returns:
+        int | None: The size in bytes, or None if it is not a size.
+    """
+    value = value.strip()
+    if value.isdigit():
+        return int(value)
+
+    number, suffix = value[:-1], value[-1:].lower()
+    if suffix in SIZE_SUFFIXES and number.isdigit():
+        return int(number) * SIZE_SUFFIXES[suffix]
+
+    return None
+
+
+def cli() -> None:
+    """CLI."""
+    typer.run(main)
+
+
+if __name__ == "__main__":
+    cli()

python/zfs/__init__.py

@@ -1,11 +1,23 @@
 """init."""
 
-from python.zfs.dataset import Dataset, Snapshot, get_datasets
+from python.zfs.dataset import (
+    Dataset,
+    Snapshot,
+    create_dataset,
+    get_datasets,
+    get_properties,
+    list_dataset_names,
+    set_property,
+)
 from python.zfs.zpool import Zpool
 
 __all__ = [
     "Dataset",
     "Snapshot",
     "Zpool",
+    "create_dataset",
     "get_datasets",
+    "get_properties",
+    "list_dataset_names",
+    "set_property",
 ]

python/zfs/dataset.py

@@ -207,8 +207,83 @@ def get_datasets() -> list[Dataset]:
     """
     logger.info("Getting zfs list")
 
+    return [Dataset(dataset_name) for dataset_name in list_dataset_names() if "/" in dataset_name]
+
+
+def get_properties(name: str) -> dict[str, tuple[str, str]]:
+    """Get every property of a dataset along with where its value came from.
+
+    The source is what distinguishes a property that was deliberately set on
+    this dataset from one that is merely inherited or left at its default.
+
+    Args:
+        name (str): The name of the dataset.
+
+    Returns:
+        dict[str, tuple[str, str]]: A mapping of property name to (value, source).
+    """
+    raw_properties, return_code = bash_wrapper(f"zfs get -Hp -o property,value,source all {name}")
+    if return_code != 0:
+        error = f"Failed to get properties for {name}: {raw_properties}"
+        raise RuntimeError(error)
+
+    properties = {}
+    for line in raw_properties.strip().split("\n"):
+        if not line:
+            continue
+        prop, value, source = line.split("\t")
+        properties[prop] = (value, source)
+
+    return properties
+
+
+def create_dataset(name: str, properties: dict[str, str]) -> str | None:
+    """Create a dataset with the given properties.
+
+    Args:
+        name (str): The name of the dataset.
+        properties (dict[str, str]): The properties to create the dataset with.
+
+    Returns:
+        str | None: An error message on failure, None on success.
+    """
+    options = " ".join(f"-o {key}={value}" for key, value in sorted(properties.items()))
+    logger.debug(f"creating {name} with {options}")
+
+    msg, return_code = bash_wrapper(f"zfs create {options} {name}")
+    if return_code != 0:
+        return f"Failed to create {name}: {msg.strip()}"
+    return None
+
+
+def set_property(name: str, key: str, value: str) -> str | None:
+    """Set a single property on a dataset.
+
+    Args:
+        name (str): The name of the dataset.
+        key (str): The property to set.
+        value (str): The value to set the property to.
+
+    Returns:
+        str | None: An error message on failure, None on success.
+    """
+    logger.debug(f"setting {key}={value} on {name}")
+
+    msg, return_code = bash_wrapper(f"zfs set {key}={value} {name}")
+    if return_code != 0:
+        return f"Failed to set {key}={value} on {name}: {msg.strip()}"
+    return None
+
+
+def list_dataset_names() -> list[str]:
+    """List every zfs filesystem name, including pool root filesystems.
+
+    Unlike get_datasets this does not build Dataset objects and does not filter
+    out pool roots, which makes it usable for existence checks.
+
+    Returns:
+        list[str]: The names of every zfs filesystem.
+    """
     dataset_names, _ = bash_wrapper("zfs list -Hp -t filesystem -o name")
 
-    cleaned_datasets = dataset_names.strip().split("\n")
-
-    return [Dataset(dataset_name) for dataset_name in cleaned_datasets if "/" in dataset_name]
+    return [name for name in dataset_names.strip().split("\n") if name]

systems/jeeves/datasets.nix

@@ -0,0 +1,356 @@
+# Dataset declarations for jeeves, kept as plain data rather than inside
+# zfs.nix so that vars.nix can derive its paths from the same source.
+#
+# Datasets are nested the way zfs nests them: a pool holds datasets, which can
+# hold datasets of their own. The tree is flattened into "pool/parent/child"
+# names below, which is what zfs and services.zfs_manager work in.
+#
+# Consumed by ./zfs.nix (which feeds it to services.zfs_manager) and by
+# ./vars.nix (which resolves the mountpoints).
+let
+  # Every pool on jeeves was created with the same -O options.
+  poolDefaults = mountpoint: {
+    inherit mountpoint;
+    acltype = "posix"; # zfs reports posixacl back as posix
+    atime = "off";
+    compression = "zstd";
+    dnodesize = "auto";
+    xattr = "sa";
+  };
+
+  zfsKey = "file:///root/zfs.key";
+
+  # What a dataset gets when it is not called out below, kept identical to the
+  # "default" table so the datasets that used to fall through are unchanged.
+  standard = {
+    "15_min" = 8;
+    hourly = 24;
+  };
+
+  pools = {
+    # root_pool: retention only, its properties are not managed yet.
+    root_pool = {
+      manageProperties = false;
+      datasets = {
+        home = {
+          manageProperties = false;
+          snapshots = {
+            "15_min" = 8;
+            hourly = 24;
+            daily = 14;
+          };
+        };
+        root = {
+          manageProperties = false;
+          snapshots = standard;
+        };
+        nix = {
+          manageProperties = false;
+          snapshots."15_min" = 4;
+        };
+        var = {
+          manageProperties = false;
+          snapshots = {
+            "15_min" = 8;
+            hourly = 24;
+            daily = 30;
+            monthly = 6;
+          };
+        };
+      };
+    };
+
+    media = {
+      properties = poolDefaults "/zfs/media";
+      datasets = {
+        temp = {
+          properties = {
+            redundant_metadata = "none";
+            sync = "disabled";
+          };
+          snapshots."15_min" = 2;
+        };
+        secure = {
+          properties.keylocation = zfsKey;
+          snapshots = { };
+          datasets = {
+            docker = {
+              properties = {
+                mountpoint = "/zfs/media/docker";
+                compression = "zstd-9";
+              };
+              snapshots = {
+                "15_min" = 3;
+                hourly = 12;
+                daily = 14;
+                monthly = 2;
+              };
+            };
+            "github-runners" = {
+              properties = {
+                mountpoint = "/zfs/media/github-runners";
+                compression = "zstd-9";
+                sync = "disabled";
+              };
+              snapshots = {
+                "15_min" = 6;
+                hourly = 2;
+                daily = 1;
+              };
+            };
+            home_assistant = {
+              properties = {
+                mountpoint = "/zfs/media/home_assistant";
+                compression = "zstd-19";
+              };
+              snapshots = standard;
+            };
+            important = {
+              properties = {
+                compression = "zstd-9";
+                copies = "2";
+              };
+              snapshots = standard;
+            };
+            notes = {
+              properties = {
+                mountpoint = "/zfs/media/notes";
+                copies = "2";
+              };
+              snapshots = {
+                "15_min" = 8;
+                hourly = 24;
+                daily = 30;
+                monthly = 12;
+              };
+            };
+            postgres = {
+              properties = {
+                mountpoint = "/zfs/media/database/postgres";
+                primarycache = "metadata";
+                recordsize = "16K";
+              };
+              snapshots = {
+                "15_min" = 8;
+                hourly = 24;
+                daily = 7;
+              };
+            };
+            "postgres-wal" = {
+              properties = {
+                compression = "lz4";
+                logbias = "latency";
+                mountpoint = "/zfs/media/database/postgres-wal";
+                primarycache = "metadata";
+                recordsize = "32K";
+                secondarycache = "none";
+                special_small_blocks = "32K";
+              };
+              snapshots = {
+                "15_min" = 4;
+                hourly = 2;
+              };
+            };
+            prometheus = {
+              properties = {
+                mountpoint = "/zfs/media/database/prometheus";
+                compression = "lz4";
+              };
+              snapshots = standard;
+            };
+            services = {
+              properties = {
+                mountpoint = "/zfs/media/services";
+                compression = "zstd-9";
+              };
+              snapshots = standard;
+            };
+            share = {
+              properties = {
+                mountpoint = "/zfs/media/share";
+                exec = "off";
+              };
+              snapshots."15_min" = 4;
+            };
+          };
+        };
+      };
+    };
+
+    storage = {
+      properties = poolDefaults "/zfs/storage";
+      datasets = {
+        nomad = {
+          properties = {
+            mountpoint = "/zfs/storage/nomad";
+            compression = "zstd-9";
+          };
+          snapshots = standard;
+        };
+        ollama = {
+          properties = {
+            compression = "zstd-19";
+            recordsize = "1M";
+            sync = "disabled";
+          };
+          snapshots."15_min" = 2;
+        };
+        secure = {
+          properties.keylocation = zfsKey;
+          snapshots = { };
+          datasets = {
+            archive = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/archive";
+                recordsize = "1M";
+              };
+              snapshots = standard;
+            };
+            important = {
+              properties = {
+                compression = "zstd-19";
+                copies = "2";
+                mountpoint = "/zfs/storage/important";
+              };
+              snapshots = standard;
+            };
+            library = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/library";
+                recordsize = "1M";
+              };
+              snapshots = standard;
+            };
+            main = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/main";
+              };
+              snapshots = standard;
+            };
+            photos = {
+              properties = {
+                compression = "zstd-19";
+                copies = "2";
+                mountpoint = "/zfs/storage/photos";
+                recordsize = "16K";
+              };
+              snapshots = standard;
+            };
+            plex = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/plex";
+                recordsize = "1M";
+              };
+              snapshots = {
+                "15_min" = 6;
+                hourly = 2;
+                daily = 1;
+              };
+            };
+            secrets = {
+              properties = {
+                compression = "zstd-19";
+                copies = "3";
+                mountpoint = "/zfs/storage/secrets";
+              };
+              snapshots = {
+                "15_min" = 8;
+                hourly = 24;
+                daily = 30;
+                monthly = 12;
+              };
+            };
+            syncthing = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/syncthing";
+              };
+              snapshots = standard;
+            };
+            transmission = {
+              properties = {
+                compression = "zstd-9";
+                exec = "off";
+                mountpoint = "/zfs/storage/transmission";
+                recordsize = "1M";
+                sync = "disabled";
+              };
+              snapshots."15_min" = 4;
+            };
+          };
+        };
+      };
+    };
+
+    scratch = {
+      properties = poolDefaults "/zfs/scratch" // {
+        keylocation = zfsKey;
+      };
+      datasets = {
+        kafka = {
+          properties = {
+            mountpoint = "/zfs/scratch/kafka";
+            recordsize = "1M";
+          };
+          snapshots = standard;
+        };
+        kestra = {
+          properties = {
+            mountpoint = "/zfs/scratch/kestra";
+            sync = "disabled";
+          };
+          snapshots = standard;
+        };
+        transmission = {
+          properties = {
+            mountpoint = "/zfs/scratch/transmission";
+            recordsize = "16K";
+            sync = "disabled";
+          };
+          snapshots."15_min" = 2;
+        };
+        uv_cache = {
+          properties.mountpoint = "/zfs/scratch/uv_cache";
+          snapshots."15_min" = 2;
+        };
+      };
+    };
+  };
+
+  # Collapse the tree into the flat "pool/parent/child" names zfs uses. Each
+  # node keeps everything except its children.
+  flatten =
+    name: node:
+    builtins.foldl' (result: child: result // flatten "${name}/${child}" node.datasets.${child}) {
+      ${name} = builtins.removeAttrs node [ "datasets" ];
+    } (builtins.attrNames (node.datasets or { }));
+
+  datasets = builtins.foldl' (result: pool: result // flatten pool pools.${pool}) { } (
+    builtins.attrNames pools
+  );
+
+  # zfs gives a dataset with no mountpoint of its own its parent's mountpoint
+  # plus its final name component, so resolve it the same way. Pool roots all
+  # declare a mountpoint, which terminates the recursion.
+  mountpointOf =
+    name:
+    let
+      properties = datasets.${name}.properties or { };
+    in
+    if properties ? mountpoint then
+      properties.mountpoint
+    else if builtins.match ".*/.*" name != null then
+      "${mountpointOf (builtins.dirOf name)}/${builtins.baseNameOf name}"
+    else
+      throw "jeeves: ${name} has no mountpoint and no parent to inherit one from";
+
+  mountpoints = builtins.mapAttrs (name: _: mountpointOf name) datasets;
+in
+{
+  inherit datasets mountpoints;
+  defaultSnapshots = standard;
+}

systems/jeeves/default.nix

@@ -15,6 +15,7 @@ in
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/update.nix"
     "${inputs.self}/common/optional/zerotier.nix"
+    "${inputs.self}/common/optional/zfs_manager.nix"
     ./monitoring
     ./docker
     ./services
@@ -24,6 +25,7 @@ in
     ./programs.nix
     ./runners
     ./syncthing.nix
+    ./zfs.nix
   ];
 
   services = {
@@ -31,10 +33,8 @@ in
 
     smartd.enable = true;
 
-    snapshot_manager = {
-      path = ./snapshot_config.toml;
-      EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
-    };
+    # path is generated from ./zfs.nix by common/optional/zfs_manager.nix
+    snapshot_manager.EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
 
     zerotierone.joinNetworks = [ "a09acf02330d37b9" ];
   };

systems/jeeves/scripts/zfs.sh

@@ -1,5 +1,10 @@
 #!/bin/bash
 
+# Pool and vdev creation only. This is run by hand once per pool.
+#
+# Datasets and their properties are declared in systems/jeeves/zfs.nix and
+# reconciled by the zfs_manager service. Do not add zfs create lines here.
+
 # zpools
 
 # media
@@ -12,35 +17,10 @@ sudo zpool add storage -o ashift=12 special mirror
 sudo zpool add storage -o ashift=12 logs mirror
 
 # scratch
-sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnodesize=auto -O xattr=sa -O compression=zstd -O encryption=aes-256-gcm -O keyformat=hex -O keylocation=file:///key -m /zfs/scratch
+sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnodesize=auto -O xattr=sa -O compression=zstd -O encryption=aes-256-gcm -O keyformat=hex -O keylocation=file:///root/zfs.key -m /zfs/scratch
 
-# media datasets
-sudo zfs create media/temp -o sync=disabled -o redundant_metadata=none
+# The two encrypted parent datasets have to exist before zfs_manager can create
+# anything under them, since encryption cannot be set after creation.
+# These will be removed if/when the media and storage pools are encrypted in the future.
 sudo zfs create media/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key
-sudo zfs create media/secure/docker -o compression=zstd-9
-sudo zfs create media/secure/github-runners -o compression=zstd-9 -o sync=disabled
-sudo zfs create media/secure/home_assistant -o compression=zstd-19
-sudo zfs create media/secure/notes -o copies=2
-sudo zfs create media/secure/postgres -o mountpoint=/zfs/media/database/postgres -o recordsize=16k -o primarycache=metadata
-sudo zfs create media/secure/postgres-wal -o mountpoint=/zfs/media/database/postgres-wal -o recordsize=32k -o primarycache=metadata -o special_small_blocks=32K -o compression=lz4 -o secondarycache=none -o logbias=latency
-sudo zfs create media/secure/prometheus -o mountpoint=/zfs/media/database/prometheus -o compression=lz4
-sudo zfs create media/secure/services -o compression=zstd-9
-sudo zfs create media/secure/share -o mountpoint=/zfs/media/share -o exec=off
-
-# scratch datasets
-sudo zfs create scratch/kafka -o mountpoint=/zfs/scratch/kafka -o recordsize=1M
-sudo zfs create scratch/transmission -o mountpoint=/zfs/scratch/transmission -o recordsize=16k -o sync=disabled -o redundant_metadata=none
-sudo zfs create scratch/uv_cache -o mountpoint=/zfs/scratch/uv_cache
-
-# storage datasets
-sudo zfs create storage/ollama -o recordsize=1M -o compression=zstd-19 -o sync=disabled
 sudo zfs create storage/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key
-sudo zfs create storage/secure/archive -o recordsize=1M -o compression=zstd-19
-sudo zfs create storage/secure/library -o recordsize=1M -o compression=zstd-19
-sudo zfs create storage/secure/main -o compression=zstd-19
-sudo zfs create storage/secure/photos -o recordsize=16K -o compression=zstd-19 -o copies=2
-sudo zfs create storage/secure/plex -o recordsize=1M -o compression=zstd-19
-sudo zfs create storage/secure/secrets -o compression=zstd-19 -o copies=3
-sudo zfs create storage/secure/syncthing -o compression=zstd-19
-sudo zfs create storage/secure/transmission -o recordsize=1M -o compression=zstd-9 -o exec=off -o sync=disabled
-sudo zfs create storage/secure/important -o compression=zstd-19 -o copies=2 -o mountpoint=/zfs/storage/important

systems/jeeves/snapshot_config.toml

@@ -1,129 +0,0 @@
-["default"]
-15_min = 8
-hourly = 24
-daily = 0
-monthly = 0
-
-# root_pool
-["root_pool/home"]
-15_min = 8
-hourly = 24
-daily = 14
-monthly = 0
-
-["root_pool/root"]
-15_min = 8
-hourly = 24
-daily = 0
-monthly = 0
-
-["root_pool/nix"]
-15_min = 4
-hourly = 0
-daily = 0
-monthly = 0
-
-["root_pool/var"]
-15_min = 8
-hourly = 24
-daily = 30
-monthly = 6
-# storage
-["storage/ollama"]
-15_min = 2
-hourly = 0
-daily = 0
-monthly = 0
-
-["storage/secure"]
-15_min = 0
-hourly = 0
-daily = 0
-monthly = 0
-
-["storage/secure/plex"]
-15_min = 6
-hourly = 2
-daily = 1
-monthly = 0
-
-["storage/secure/transmission"]
-15_min = 4
-hourly = 0
-daily = 0
-monthly = 0
-
-["storage/secure/secrets"]
-15_min = 8
-hourly = 24
-daily = 30
-monthly = 12
-
-# media
-["media/temp"]
-15_min = 2
-hourly = 0
-daily = 0
-monthly = 0
-
-["media/secure"]
-15_min = 0
-hourly = 0
-daily = 0
-monthly = 0
-
-["media/secure/plex"]
-15_min = 6
-hourly = 2
-daily = 1
-monthly = 0
-
-["media/secure/postgres-wal"]
-15_min = 4
-hourly = 2
-daily = 0
-monthly = 0
-
-
-["media/secure/postgres"]
-15_min = 8
-hourly = 24
-daily = 7
-monthly = 0
-
-["media/secure/share"]
-15_min = 4
-hourly = 0
-daily = 0
-monthly = 0
-
-["media/secure/github-runners"]
-15_min = 6
-hourly = 2
-daily = 1
-monthly = 0
-
-["media/secure/notes"]
-15_min = 8
-hourly = 24
-daily = 30
-monthly = 12
-
-["media/secure/docker"]
-15_min = 3
-hourly = 12
-daily = 14
-monthly = 2
-
-# scratch
-["scratch/transmission"]
-15_min = 2
-hourly = 0
-daily = 0
-monthly = 0
-
-["scratch/uv_cache"]
-15_min = 2
-hourly = 0
-daily = 0
-monthly = 0

systems/jeeves/vars.nix

@@ -1,22 +1,22 @@
+# Paths are derived from the dataset declarations in ./datasets.nix so that a
+# mountpoint only ever has to change in one place.
 let
-  zfs_media = "/zfs/media";
-  zfs_storage = "/zfs/storage";
-  zfs_scratch = "/zfs/scratch";
+  inherit (import ./datasets.nix) mountpoints;
 in
 {
-  inherit zfs_media zfs_storage zfs_scratch;
-  database = "${zfs_media}/database";
-  docker = "${zfs_media}/docker";
-  docker_configs = "${zfs_media}/docker/configs";
-  home_assistant = "${zfs_media}/home_assistant";
-  notes = "${zfs_media}/notes";
-  secrets = "${zfs_storage}/secrets";
-  services = "${zfs_media}/services";
-  share = "${zfs_media}/share";
-  syncthing = "${zfs_storage}/syncthing";
-  transmission = "${zfs_storage}/transmission";
-  ollama = "${zfs_storage}/ollama";
-  transmission_scratch = "${zfs_scratch}/transmission";
-  uv_cache = "${zfs_scratch}/uv_cache";
-  kafka = "${zfs_scratch}/kafka";
+  # Not a dataset of its own, it is the directory the postgres datasets share.
+  database = "/zfs/media/database";
+  docker = mountpoints."media/secure/docker";
+  docker_configs = "${mountpoints."media/secure/docker"}/configs";
+  home_assistant = mountpoints."media/secure/home_assistant";
+  notes = mountpoints."media/secure/notes";
+  secrets = mountpoints."storage/secure/secrets";
+  services = mountpoints."media/secure/services";
+  share = mountpoints."media/secure/share";
+  syncthing = mountpoints."storage/secure/syncthing";
+  transmission = mountpoints."storage/secure/transmission";
+  ollama = mountpoints."storage/ollama";
+  transmission_scratch = mountpoints."scratch/transmission";
+  uv_cache = mountpoints."scratch/uv_cache";
+  kafka = mountpoints."scratch/kafka";
 }

systems/jeeves/zfs.nix

@@ -0,0 +1,14 @@
+{ inputs, ... }:
+let
+  vars = import ./vars.nix;
+  jeeves_zfs = import ./datasets.nix;
+in
+{
+  services.zfs_manager = {
+    enable = true;
+    PYTHONPATH = "${inputs.self}/";
+    EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
+
+    inherit (jeeves_zfs) datasets defaultSnapshots;
+  };
+}

tests/test_zfs_manager.py

@@ -0,0 +1,242 @@
+"""test_zfs_manager."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+
+from python.tools.zfs_manager import main, parse_size, values_match
+
+if TYPE_CHECKING:
+    from pyfakefs.fake_filesystem import FakeFilesystem
+    from pytest_mock import MockerFixture
+
+ZFS_MANAGER = "python.tools.zfs_manager"
+CONFIG_PATH = "/mock_zfs_datasets.json"
+
+
+def write_config(
+    fs: FakeFilesystem,
+    datasets: dict[str, dict[str, str]],
+    unmanaged: list[str] | None = None,
+) -> Path:
+    """Write a dataset declaration to the fake filesystem."""
+    contents = {
+        "datasets": {name: {"manageProperties": True, "properties": props} for name, props in datasets.items()}
+        | {name: {"manageProperties": False, "properties": {}} for name in unmanaged or []},
+    }
+    fs.create_file(CONFIG_PATH, contents=json.dumps(contents))
+    return Path(CONFIG_PATH)
+
+
+def patch_zfs(
+    mocker: MockerFixture,
+    existing: list[str],
+    properties: dict[str, dict[str, tuple[str, str]]] | None = None,
+) -> dict[str, object]:
+    """Patch every zfs call zfs_manager makes."""
+    return {
+        "list": mocker.patch(f"{ZFS_MANAGER}.list_dataset_names", return_value=existing),
+        "get": mocker.patch(f"{ZFS_MANAGER}.get_properties", side_effect=lambda name: (properties or {}).get(name, {})),
+        "create": mocker.patch(f"{ZFS_MANAGER}.create_dataset", return_value=None),
+        "set": mocker.patch(f"{ZFS_MANAGER}.set_property", return_value=None),
+        "alert": mocker.patch(f"{ZFS_MANAGER}.signal_alert"),
+    }
+
+
+def test_creates_missing_dataset(mocker: MockerFixture, fs: FakeFilesystem) -> None:
+    zfs = patch_zfs(mocker, existing=["media", "media/secure"])
+    config = write_config(fs, {"media/secure/new": {"compression": "zstd-9"}})
+
+    main(config)
+
+    zfs["create"].assert_called_once_with("media/secure/new", {"compression": "zstd-9"})
+    zfs["set"].assert_not_called()
+    zfs["alert"].assert_not_called()
+
+
+def test_sets_drifted_property(mocker: MockerFixture, fs: FakeFilesystem) -> None:
+    zfs = patch_zfs(
+        mocker,
+        existing=["media", "media/temp"],
+        properties={"media/temp": {"compression": ("zstd", "inherited from media")}},
+    )
+    config = write_config(fs, {"media/temp": {"compression": "zstd-9"}})
+
+    main(config)
+
+    zfs["set"].assert_called_once_with("media/temp", "compression", "zstd-9")
+    zfs["create"].assert_not_called()
+
+
+def test_no_op_when_in_sync(mocker: MockerFixture, fs: FakeFilesystem) -> None:
+    zfs = patch_zfs(
+        mocker,
+        existing=["media", "media/temp"],
+        properties={"media/temp": {"sync": ("disabled", "local")}},
+    )
+    config = write_config(fs, {"media/temp": {"sync": "disabled"}})
+
+    main(config)
+
+    zfs["set"].assert_not_called()
+    zfs["create"].assert_not_called()
+    zfs["alert"].assert_not_called()
+
+
+def test_size_property_does_not_churn(mocker: MockerFixture, fs: FakeFilesystem) -> None:
+    """zfs get -p reports recordsize in bytes, the declaration uses a suffix."""
+    zfs = patch_zfs(
+        mocker,
+        existing=["media", "media/db"],
+        properties={"media/db": {"recordsize": ("16384", "local"), "special_small_blocks": ("32768", "local")}},
+    )
+    config = write_config(fs, {"media/db": {"recordsize": "16k", "special_small_blocks": "32K"}})
+
+    main(config)
+
+    zfs["set"].assert_not_called()
+
+
+def test_create_only_property_alerts_instead_of_setting(mocker: MockerFixture, fs: FakeFilesystem) -> None:
+    zfs = patch_zfs(
+        mocker,
+        existing=["media", "media/secure"],
+        properties={"media/secure": {"encryption": ("aes-256-gcm", "local")}},
+    )
+    config = write_config(fs, {"media/secure": {"encryption": "off"}})
+
+    main(config)
+
+    zfs["set"].assert_not_called()
+    assert zfs["alert"].call_count == 1
+    assert "can only be set when the dataset is created" in zfs["alert"].call_args.args[0]
+
+
+def test_undeclared_local_property_warns(mocker: MockerFixture, fs: FakeFilesystem) -> None:
+    zfs = patch_zfs(
+        mocker,
+        existing=["media", "media/temp"],
+        properties={"media/temp": {"exec": ("off", "local")}},
+    )
+    config = write_config(fs, {"media/temp": {}})
+
+    main(config)
+
+    zfs["alert"].assert_called_once_with("media/temp has exec=off set outside of nix")
+
+
+def test_undeclared_inherited_property_is_silent(mocker: MockerFixture, fs: FakeFilesystem) -> None:
+    zfs = patch_zfs(
+        mocker,
+        existing=["media", "media/temp"],
+        properties={
+            "media/temp": {
+                "compression": ("zstd", "inherited from media"),
+                "exec": ("on", "default"),
+                "nixos:shutdown-time": ("whenever", "local"),
+            },
+        },
+    )
+    config = write_config(fs, {"media/temp": {}})
+
+    main(config)
+
+    zfs["alert"].assert_not_called()
+
+
+def test_dry_run_makes_no_changes(mocker: MockerFixture, fs: FakeFilesystem) -> None:
+    zfs = patch_zfs(
+        mocker,
+        existing=["media", "media/temp"],
+        properties={"media/temp": {"compression": ("zstd", "local")}},
+    )
+    config = write_config(fs, {"media/temp": {"compression": "zstd-9"}, "media/new": {}})
+
+    main(config, dry_run=True)
+
+    zfs["set"].assert_not_called()
+    zfs["create"].assert_not_called()
+
+
+def test_pool_root_is_never_created(mocker: MockerFixture, fs: FakeFilesystem) -> None:
+    zfs = patch_zfs(mocker, existing=[])
+    config = write_config(fs, {"media": {"atime": "off"}})
+
+    main(config)
+
+    zfs["create"].assert_not_called()
+    assert "does not create pools" in zfs["alert"].call_args.args[0]
+
+
+def test_children_skipped_when_parent_creation_fails(mocker: MockerFixture, fs: FakeFilesystem) -> None:
+    zfs = patch_zfs(mocker, existing=["media"])
+    zfs["create"].return_value = "Failed to create media/secure: key not loaded"
+    config = write_config(fs, {"media/secure": {}, "media/secure/child": {}})
+
+    main(config)
+
+    zfs["create"].assert_called_once_with("media/secure", {})
+
+
+def test_unmanaged_dataset_properties_are_untouched(mocker: MockerFixture, fs: FakeFilesystem) -> None:
+    """A snapshots-only dataset is neither reconciled nor reported as unknown."""
+    zfs = patch_zfs(
+        mocker,
+        existing=["root_pool", "root_pool/var"],
+        properties={"root_pool/var": {"compression": ("lz4", "local")}},
+    )
+    config = write_config(fs, {}, unmanaged=["root_pool", "root_pool/var"])
+
+    main(config)
+
+    zfs["get"].assert_not_called()
+    zfs["set"].assert_not_called()
+    zfs["alert"].assert_not_called()
+
+
+def test_undeclared_dataset_is_left_alone(mocker: MockerFixture, fs: FakeFilesystem) -> None:
+    zfs = patch_zfs(mocker, existing=["media", "media/undeclared"])
+    config = write_config(fs, {})
+
+    main(config)
+
+    zfs["create"].assert_not_called()
+    zfs["set"].assert_not_called()
+    zfs["alert"].assert_not_called()
+
+
+def test_main_exception(mocker: MockerFixture, fs: FakeFilesystem) -> None:
+    zfs = patch_zfs(mocker, existing=[])
+    zfs["list"].side_effect = Exception("test")
+    config = write_config(fs, {})
+
+    with pytest.raises(SystemExit) as pytest_wrapped_e:
+        main(config)
+
+    assert pytest_wrapped_e.value.code == 1
+    zfs["alert"].assert_called_once_with("zfs_manager failed")
+
+
+@pytest.mark.parametrize(
+    ("value", "expected"),
+    [
+        ("16384", 16384),
+        ("16k", 16384),
+        ("16K", 16384),
+        ("1M", 1048576),
+        ("none", None),
+        ("", None),
+    ],
+)
+def test_parse_size(value, expected) -> None:
+    assert parse_size(value) == expected
+
+
+def test_values_match_falls_back_to_string_for_unparsable_sizes() -> None:
+    assert not values_match("recordsize", "none", "16384")
+    assert values_match("recordsize", "none", "none")
+    assert not values_match("compression", "zstd", "zstd-9")