From ae18feb0fdcf15cbfff07acf6f0bf75ea42302c2 Mon Sep 17 00:00:00 2001 From: Richie Cahill Date: Mon, 27 Jul 2026 23:26:38 -0400 Subject: [PATCH] feat(zfs): manage jeeves datasets declaratively from nix Add a zfs_manager module that reconciles the live datasets on jeeves against a nix declaration, and generate the snapshot retention config from that same declaration so the two can no longer drift apart. systems/jeeves/datasets.nix declares every dataset on the media, storage and scratch pools, nested the way zfs nests them and flattened into pool/parent/child names. Values were transcribed from the live pools rather than from scripts/zfs.sh, which had gone stale: acltype reads back as posix, and media/secure/important, scratch/kestra and storage/nomad were never recorded. root_pool datasets are declared for retention only, their properties stay unmanaged for now. python.tools.zfs_manager creates missing datasets and corrects drifted properties, and never destroys anything. Undeclared properties are judged by the zfs source field, so inherited and default values stay quiet while locally set ones warn. Size values are normalised to bytes so that 16K and 16384 do not re-issue zfs set on every run. vars.nix now derives its paths from the declared mountpoints instead of repeating them, dropping three zfs_* keys that nothing referenced. Replaces systems/jeeves/snapshot_config.toml, which listed a dataset that does not exist and omitted thirteen that do. --- common/optional/zfs_manager.nix | 168 +++++++++++++ python/tools/zfs_manager.py | 267 +++++++++++++++++++++ python/zfs/__init__.py | 14 +- python/zfs/dataset.py | 81 ++++++- systems/jeeves/datasets.nix | 356 ++++++++++++++++++++++++++++ systems/jeeves/default.nix | 8 +- systems/jeeves/scripts/zfs.sh | 38 +-- systems/jeeves/snapshot_config.toml | 129 ---------- systems/jeeves/vars.nix | 36 +-- systems/jeeves/zfs.nix | 14 ++ tests/test_zfs_manager.py | 242 +++++++++++++++++++ 11 files changed, 1169 insertions(+), 184 deletions(-) create mode 100644 common/optional/zfs_manager.nix create mode 100644 python/tools/zfs_manager.py create mode 100644 systems/jeeves/datasets.nix delete mode 100644 systems/jeeves/snapshot_config.toml create mode 100644 systems/jeeves/zfs.nix create mode 100644 tests/test_zfs_manager.py diff --git a/common/optional/zfs_manager.nix b/common/optional/zfs_manager.nix new file mode 100644 index 0000000..9cf9839 --- /dev/null +++ b/common/optional/zfs_manager.nix @@ -0,0 +1,168 @@ +{ + pkgs, + lib, + config, + ... +}: +let + cfg = config.services.zfs_manager; + + snapshotOptions = { + options = { + "15_min" = lib.mkOption { + type = lib.types.int; + default = 0; + description = "How many 15 minute snapshots to keep."; + }; + hourly = lib.mkOption { + type = lib.types.int; + default = 0; + description = "How many hourly snapshots to keep."; + }; + daily = lib.mkOption { + type = lib.types.int; + default = 0; + description = "How many daily snapshots to keep."; + }; + monthly = lib.mkOption { + type = lib.types.int; + default = 0; + description = "How many monthly snapshots to keep."; + }; + }; + }; + + datasetOptions = { + options = { + manageProperties = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Whether zfs_manager owns this dataset's properties. When false the + dataset only contributes its snapshot retention, which is how + root_pool datasets are declared. + ''; + }; + properties = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = { }; + description = '' + The zfs properties this dataset should have. Values are compared + against the live dataset and corrected when they differ. + ''; + }; + snapshots = lib.mkOption { + type = lib.types.submodule snapshotOptions; + default = { }; + description = "Snapshot retention for this dataset."; + }; + }; + }; + + # snapshot_manager.py only ever walks datasets below a pool root, so pool + # roots are left out of the retention table. It also indexes the table + # directly, which is why every entry carries all four keys. + snapshotTable = lib.mapAttrs (_: dataset: dataset.snapshots) ( + lib.filterAttrs (name: _: lib.hasInfix "/" name) cfg.datasets + ); + + snapshotConfig = (pkgs.formats.toml { }).generate "snapshot_config.toml" ( + snapshotTable // { default = cfg.defaultSnapshots; } + ); + + # Every declared dataset is emitted, including the ones whose properties are + # not managed, so the tool can tell "deliberately hands off" apart from + # "nobody has written this down yet". + datasetConfig = (pkgs.formats.json { }).generate "zfs_datasets.json" { + datasets = lib.mapAttrs (_: dataset: { + inherit (dataset) manageProperties properties; + }) cfg.datasets; + }; +in +{ + options = { + services.zfs_manager = { + enable = lib.mkEnableOption "declarative ZFS dataset management"; + datasets = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule datasetOptions); + default = { }; + example = lib.literalExpression '' + { + "media/temp".properties = { + sync = "disabled"; + redundant_metadata = "none"; + }; + } + ''; + description = '' + The datasets to manage, keyed by full dataset name. Missing datasets + are created and drifted properties are corrected. Nothing is ever + destroyed, and datasets that are not declared are left alone. + + A name without a "/" is a pool root filesystem. Its properties are + managed but it is never created, pool creation stays manual. + ''; + }; + defaultSnapshots = lib.mkOption { + type = lib.types.submodule snapshotOptions; + default = { }; + description = '' + Retention for any dataset that is not declared above, emitted as the + "default" table of the snapshot config. + ''; + }; + dryRun = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Log every change that would be made without touching zfs. Use this to + validate a new or heavily edited declaration before applying it. + ''; + }; + PYTHONPATH = lib.mkOption { + type = lib.types.str; + description = '' + the PYTHONPATH to use for the zfs_manager service. + ''; + }; + EnvironmentFile = lib.mkOption { + type = lib.types.nullOr (lib.types.coercedTo lib.types.path toString lib.types.str); + default = null; + description = '' + Single environment file for the service (e.g. /etc/zfs-manager/env). + Use a leading "-" to ignore if missing (systemd feature). + ''; + }; + }; + }; + + config = lib.mkIf cfg.enable { + services.snapshot_manager.path = snapshotConfig; + + systemd = { + services.zfs_manager = { + description = "ZFS Dataset Manager"; + requires = [ "zfs-import.target" ]; + after = [ + "zfs-import.target" + "zfs-mount.service" + ]; + wantedBy = [ "multi-user.target" ]; + path = [ pkgs.zfs ]; + # Re-run on nixos-rebuild switch whenever the declaration changes. + restartTriggers = [ datasetConfig ]; + environment = { + PYTHONPATH = cfg.PYTHONPATH; + }; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = "${pkgs.my_python}/bin/python -m python.tools.zfs_manager ${lib.escapeShellArg datasetConfig}${lib.optionalString cfg.dryRun " --dry-run"}"; + } + // lib.optionalAttrs (cfg.EnvironmentFile != null) { + EnvironmentFile = cfg.EnvironmentFile; + }; + }; + }; + }; +} diff --git a/python/tools/zfs_manager.py b/python/tools/zfs_manager.py new file mode 100644 index 0000000..8c3230c --- /dev/null +++ b/python/tools/zfs_manager.py @@ -0,0 +1,267 @@ +"""zfs_manager. + +Reconciles the live zfs datasets against a declaration generated by +common/optional/zfs_manager.nix. Datasets are created and properties are +corrected, but nothing is ever destroyed or renamed. +""" + +from __future__ import annotations + +import json +import logging +import sys +from pathlib import Path # noqa: TC003 This is required for the typer CLI + +import typer + +from python.common import configure_logger +from python.signal_alert import signal_alert +from python.zfs import create_dataset, get_properties, list_dataset_names, set_property + +logger = logging.getLogger(__name__) + +# Properties that can only be chosen at creation time. Attempting to zfs set +# these fails on every run, so a mismatch is reported instead of retried. +CREATE_ONLY_PROPERTIES = frozenset( + { + "casesensitivity", + "encryption", + "keyformat", + "normalization", + "utf8only", + "volblocksize", + }, +) + +# Properties whose values zfs reports in bytes but which are conventionally +# declared with a size suffix, so "16k" and "16384" mean the same thing. +SIZE_PROPERTIES = frozenset( + { + "quota", + "recordsize", + "refquota", + "refreservation", + "reservation", + "special_small_blocks", + "volblocksize", + "volsize", + }, +) + +SIZE_SUFFIXES = {"b": 1, "k": 1024, "m": 1024**2, "g": 1024**3, "t": 1024**4, "p": 1024**5} + +# Sources that mean the value was deliberately put on this dataset rather than +# inherited from a parent or left at the zfs default. +LOCAL_SOURCES = ("local", "received") + + +def main(config_file: Path, *, dry_run: bool = False) -> None: + """Main. + + Args: + config_file (Path): The path to the generated dataset declaration. + dry_run (bool): Log the changes that would be made without making them. + """ + configure_logger(level="DEBUG") + logger.info(f"Starting zfs_manager {dry_run=}") + + try: + declared = json.loads(config_file.read_text())["datasets"] + existing = set(list_dataset_names()) + unusable: set[str] = set() + + # Parents before children so a newly created parent exists by the time + # its children are reconciled. + for name in sorted(declared, key=lambda name: (name.count("/"), name)): + if has_unusable_parent(name, unusable): + logger.error(f"skipping {name}, its parent could not be created") + continue + + # Declared purely to record retention, its properties belong to + # whoever set them. + if not declared[name].get("manageProperties", True): + logger.debug(f"{name} is declared but its properties are not managed") + continue + + properties = declared[name]["properties"] + if name in existing: + reconcile_dataset(name, properties, dry_run=dry_run) + elif create_missing_dataset(name, properties, dry_run=dry_run): + existing.add(name) + elif not dry_run: + unusable.add(name) + + report_undeclared_datasets(existing, declared) + except Exception: + logger.exception("zfs_manager failed") + signal_alert("zfs_manager failed") + sys.exit(1) + else: + logger.info("zfs_manager completed") + + +def has_unusable_parent(name: str, unusable: set[str]) -> bool: + """Check whether an ancestor of a dataset could not be created. + + Args: + name (str): The name of the dataset. + unusable (set[str]): The datasets that do not exist and could not be created. + + Returns: + bool: True if any ancestor is unusable. + """ + parts = name.split("/") + return any("/".join(parts[:depth]) in unusable for depth in range(1, len(parts))) + + +def create_missing_dataset(name: str, properties: dict[str, str], *, dry_run: bool) -> bool: + """Create a dataset that is declared but does not exist yet. + + Pool root filesystems are never created, creating a pool is out of scope. + + Args: + name (str): The name of the dataset. + properties (dict[str, str]): The properties to create the dataset with. + dry_run (bool): Log the change without making it. + + Returns: + bool: True if the dataset now exists. + """ + if "/" not in name: + message = f"pool {name} is declared but does not exist, zfs_manager does not create pools" + logger.error(message) + signal_alert(message) + return False + + if dry_run: + logger.info(f"would create {name} with {properties}") + return False + + logger.info(f"creating {name} with {properties}") + if error := create_dataset(name, properties): + logger.error(error) + signal_alert(error) + return False + + return True + + +def reconcile_dataset(name: str, properties: dict[str, str], *, dry_run: bool) -> None: + """Bring an existing dataset in line with its declared properties. + + Args: + name (str): The name of the dataset. + properties (dict[str, str]): The declared properties. + dry_run (bool): Log the changes without making them. + """ + current = get_properties(name) + + for key, wanted in sorted(properties.items()): + current_value, _ = current.get(key, ("-", "-")) + if values_match(key, wanted, current_value): + continue + + if key in CREATE_ONLY_PROPERTIES: + message = ( + f"{name} {key} is {current_value} but {wanted} is declared, " + f"{key} can only be set when the dataset is created" + ) + logger.error(message) + signal_alert(message) + continue + + if dry_run: + logger.info(f"would set {key}={wanted} on {name}, currently {current_value}") + continue + + logger.info(f"setting {key}={wanted} on {name}, was {current_value}") + if error := set_property(name, key, wanted): + logger.error(error) + signal_alert(error) + + report_undeclared_properties(name, properties, current) + + +def report_undeclared_properties(name: str, properties: dict[str, str], current: dict[str, tuple[str, str]]) -> None: + """Warn about properties set on the dataset but absent from the declaration. + + Inherited and default values are silent, they are not drift. A locally set + value that nix does not know about was changed outside of this tool and + will be lost the next time the dataset is recreated, so it is worth saying. + + Args: + name (str): The name of the dataset. + properties (dict[str, str]): The declared properties. + current (dict[str, tuple[str, str]]): The live properties keyed to (value, source). + """ + for key, (value, source) in sorted(current.items()): + # User properties such as nixos:shutdown-time are written by other + # tools and are not something a dataset declaration should own. + if key in properties or ":" in key or source not in LOCAL_SOURCES: + continue + + logger.warning(f"{name} has {key}={value} set outside of nix") + signal_alert(f"{name} has {key}={value} set outside of nix") + + +def report_undeclared_datasets(existing: set[str], declared: dict[str, dict]) -> None: + """Warn about datasets that exist but are not declared. + + These are left completely alone. They still get snapshots through the + default retention table. + + Args: + existing (set[str]): The names of every live dataset. + declared (dict[str, dict]): The declaration. + """ + for name in sorted(existing - set(declared)): + logger.warning(f"{name} exists but is not declared in nix") + + +def values_match(key: str, wanted: str, current: str) -> bool: + """Compare a declared property value against the live one. + + Args: + key (str): The property name. + wanted (str): The declared value. + current (str): The live value. + + Returns: + bool: True if the two values mean the same thing. + """ + if key in SIZE_PROPERTIES: + wanted_size = parse_size(wanted) + current_size = parse_size(current) + if wanted_size is not None and current_size is not None: + return wanted_size == current_size + + return wanted == current + + +def parse_size(value: str) -> int | None: + """Convert a zfs size such as 16k or 1M into bytes. + + Args: + value (str): The size to convert. + + Returns: + int | None: The size in bytes, or None if it is not a size. + """ + value = value.strip() + if value.isdigit(): + return int(value) + + number, suffix = value[:-1], value[-1:].lower() + if suffix in SIZE_SUFFIXES and number.isdigit(): + return int(number) * SIZE_SUFFIXES[suffix] + + return None + + +def cli() -> None: + """CLI.""" + typer.run(main) + + +if __name__ == "__main__": + cli() diff --git a/python/zfs/__init__.py b/python/zfs/__init__.py index 74d7d0b..aa416ee 100644 --- a/python/zfs/__init__.py +++ b/python/zfs/__init__.py @@ -1,11 +1,23 @@ """init.""" -from python.zfs.dataset import Dataset, Snapshot, get_datasets +from python.zfs.dataset import ( + Dataset, + Snapshot, + create_dataset, + get_datasets, + get_properties, + list_dataset_names, + set_property, +) from python.zfs.zpool import Zpool __all__ = [ "Dataset", "Snapshot", "Zpool", + "create_dataset", "get_datasets", + "get_properties", + "list_dataset_names", + "set_property", ] diff --git a/python/zfs/dataset.py b/python/zfs/dataset.py index a184c78..0eff6a6 100644 --- a/python/zfs/dataset.py +++ b/python/zfs/dataset.py @@ -207,8 +207,83 @@ def get_datasets() -> list[Dataset]: """ logger.info("Getting zfs list") + return [Dataset(dataset_name) for dataset_name in list_dataset_names() if "/" in dataset_name] + + +def get_properties(name: str) -> dict[str, tuple[str, str]]: + """Get every property of a dataset along with where its value came from. + + The source is what distinguishes a property that was deliberately set on + this dataset from one that is merely inherited or left at its default. + + Args: + name (str): The name of the dataset. + + Returns: + dict[str, tuple[str, str]]: A mapping of property name to (value, source). + """ + raw_properties, return_code = bash_wrapper(f"zfs get -Hp -o property,value,source all {name}") + if return_code != 0: + error = f"Failed to get properties for {name}: {raw_properties}" + raise RuntimeError(error) + + properties = {} + for line in raw_properties.strip().split("\n"): + if not line: + continue + prop, value, source = line.split("\t") + properties[prop] = (value, source) + + return properties + + +def create_dataset(name: str, properties: dict[str, str]) -> str | None: + """Create a dataset with the given properties. + + Args: + name (str): The name of the dataset. + properties (dict[str, str]): The properties to create the dataset with. + + Returns: + str | None: An error message on failure, None on success. + """ + options = " ".join(f"-o {key}={value}" for key, value in sorted(properties.items())) + logger.debug(f"creating {name} with {options}") + + msg, return_code = bash_wrapper(f"zfs create {options} {name}") + if return_code != 0: + return f"Failed to create {name}: {msg.strip()}" + return None + + +def set_property(name: str, key: str, value: str) -> str | None: + """Set a single property on a dataset. + + Args: + name (str): The name of the dataset. + key (str): The property to set. + value (str): The value to set the property to. + + Returns: + str | None: An error message on failure, None on success. + """ + logger.debug(f"setting {key}={value} on {name}") + + msg, return_code = bash_wrapper(f"zfs set {key}={value} {name}") + if return_code != 0: + return f"Failed to set {key}={value} on {name}: {msg.strip()}" + return None + + +def list_dataset_names() -> list[str]: + """List every zfs filesystem name, including pool root filesystems. + + Unlike get_datasets this does not build Dataset objects and does not filter + out pool roots, which makes it usable for existence checks. + + Returns: + list[str]: The names of every zfs filesystem. + """ dataset_names, _ = bash_wrapper("zfs list -Hp -t filesystem -o name") - cleaned_datasets = dataset_names.strip().split("\n") - - return [Dataset(dataset_name) for dataset_name in cleaned_datasets if "/" in dataset_name] + return [name for name in dataset_names.strip().split("\n") if name] diff --git a/systems/jeeves/datasets.nix b/systems/jeeves/datasets.nix new file mode 100644 index 0000000..0cbc5b9 --- /dev/null +++ b/systems/jeeves/datasets.nix @@ -0,0 +1,356 @@ +# Dataset declarations for jeeves, kept as plain data rather than inside +# zfs.nix so that vars.nix can derive its paths from the same source. +# +# Datasets are nested the way zfs nests them: a pool holds datasets, which can +# hold datasets of their own. The tree is flattened into "pool/parent/child" +# names below, which is what zfs and services.zfs_manager work in. +# +# Consumed by ./zfs.nix (which feeds it to services.zfs_manager) and by +# ./vars.nix (which resolves the mountpoints). +let + # Every pool on jeeves was created with the same -O options. + poolDefaults = mountpoint: { + inherit mountpoint; + acltype = "posix"; # zfs reports posixacl back as posix + atime = "off"; + compression = "zstd"; + dnodesize = "auto"; + xattr = "sa"; + }; + + zfsKey = "file:///root/zfs.key"; + + # What a dataset gets when it is not called out below, kept identical to the + # "default" table so the datasets that used to fall through are unchanged. + standard = { + "15_min" = 8; + hourly = 24; + }; + + pools = { + # root_pool: retention only, its properties are not managed yet. + root_pool = { + manageProperties = false; + datasets = { + home = { + manageProperties = false; + snapshots = { + "15_min" = 8; + hourly = 24; + daily = 14; + }; + }; + root = { + manageProperties = false; + snapshots = standard; + }; + nix = { + manageProperties = false; + snapshots."15_min" = 4; + }; + var = { + manageProperties = false; + snapshots = { + "15_min" = 8; + hourly = 24; + daily = 30; + monthly = 6; + }; + }; + }; + }; + + media = { + properties = poolDefaults "/zfs/media"; + datasets = { + temp = { + properties = { + redundant_metadata = "none"; + sync = "disabled"; + }; + snapshots."15_min" = 2; + }; + secure = { + properties.keylocation = zfsKey; + snapshots = { }; + datasets = { + docker = { + properties = { + mountpoint = "/zfs/media/docker"; + compression = "zstd-9"; + }; + snapshots = { + "15_min" = 3; + hourly = 12; + daily = 14; + monthly = 2; + }; + }; + "github-runners" = { + properties = { + mountpoint = "/zfs/media/github-runners"; + compression = "zstd-9"; + sync = "disabled"; + }; + snapshots = { + "15_min" = 6; + hourly = 2; + daily = 1; + }; + }; + home_assistant = { + properties = { + mountpoint = "/zfs/media/home_assistant"; + compression = "zstd-19"; + }; + snapshots = standard; + }; + important = { + properties = { + compression = "zstd-9"; + copies = "2"; + }; + snapshots = standard; + }; + notes = { + properties = { + mountpoint = "/zfs/media/notes"; + copies = "2"; + }; + snapshots = { + "15_min" = 8; + hourly = 24; + daily = 30; + monthly = 12; + }; + }; + postgres = { + properties = { + mountpoint = "/zfs/media/database/postgres"; + primarycache = "metadata"; + recordsize = "16K"; + }; + snapshots = { + "15_min" = 8; + hourly = 24; + daily = 7; + }; + }; + "postgres-wal" = { + properties = { + compression = "lz4"; + logbias = "latency"; + mountpoint = "/zfs/media/database/postgres-wal"; + primarycache = "metadata"; + recordsize = "32K"; + secondarycache = "none"; + special_small_blocks = "32K"; + }; + snapshots = { + "15_min" = 4; + hourly = 2; + }; + }; + prometheus = { + properties = { + mountpoint = "/zfs/media/database/prometheus"; + compression = "lz4"; + }; + snapshots = standard; + }; + services = { + properties = { + mountpoint = "/zfs/media/services"; + compression = "zstd-9"; + }; + snapshots = standard; + }; + share = { + properties = { + mountpoint = "/zfs/media/share"; + exec = "off"; + }; + snapshots."15_min" = 4; + }; + }; + }; + }; + }; + + storage = { + properties = poolDefaults "/zfs/storage"; + datasets = { + nomad = { + properties = { + mountpoint = "/zfs/storage/nomad"; + compression = "zstd-9"; + }; + snapshots = standard; + }; + ollama = { + properties = { + compression = "zstd-19"; + recordsize = "1M"; + sync = "disabled"; + }; + snapshots."15_min" = 2; + }; + secure = { + properties.keylocation = zfsKey; + snapshots = { }; + datasets = { + archive = { + properties = { + compression = "zstd-19"; + mountpoint = "/zfs/storage/archive"; + recordsize = "1M"; + }; + snapshots = standard; + }; + important = { + properties = { + compression = "zstd-19"; + copies = "2"; + mountpoint = "/zfs/storage/important"; + }; + snapshots = standard; + }; + library = { + properties = { + compression = "zstd-19"; + mountpoint = "/zfs/storage/library"; + recordsize = "1M"; + }; + snapshots = standard; + }; + main = { + properties = { + compression = "zstd-19"; + mountpoint = "/zfs/storage/main"; + }; + snapshots = standard; + }; + photos = { + properties = { + compression = "zstd-19"; + copies = "2"; + mountpoint = "/zfs/storage/photos"; + recordsize = "16K"; + }; + snapshots = standard; + }; + plex = { + properties = { + compression = "zstd-19"; + mountpoint = "/zfs/storage/plex"; + recordsize = "1M"; + }; + snapshots = { + "15_min" = 6; + hourly = 2; + daily = 1; + }; + }; + secrets = { + properties = { + compression = "zstd-19"; + copies = "3"; + mountpoint = "/zfs/storage/secrets"; + }; + snapshots = { + "15_min" = 8; + hourly = 24; + daily = 30; + monthly = 12; + }; + }; + syncthing = { + properties = { + compression = "zstd-19"; + mountpoint = "/zfs/storage/syncthing"; + }; + snapshots = standard; + }; + transmission = { + properties = { + compression = "zstd-9"; + exec = "off"; + mountpoint = "/zfs/storage/transmission"; + recordsize = "1M"; + sync = "disabled"; + }; + snapshots."15_min" = 4; + }; + }; + }; + }; + }; + + scratch = { + properties = poolDefaults "/zfs/scratch" // { + keylocation = zfsKey; + }; + datasets = { + kafka = { + properties = { + mountpoint = "/zfs/scratch/kafka"; + recordsize = "1M"; + }; + snapshots = standard; + }; + kestra = { + properties = { + mountpoint = "/zfs/scratch/kestra"; + sync = "disabled"; + }; + snapshots = standard; + }; + transmission = { + properties = { + mountpoint = "/zfs/scratch/transmission"; + recordsize = "16K"; + sync = "disabled"; + }; + snapshots."15_min" = 2; + }; + uv_cache = { + properties.mountpoint = "/zfs/scratch/uv_cache"; + snapshots."15_min" = 2; + }; + }; + }; + }; + + # Collapse the tree into the flat "pool/parent/child" names zfs uses. Each + # node keeps everything except its children. + flatten = + name: node: + builtins.foldl' (result: child: result // flatten "${name}/${child}" node.datasets.${child}) { + ${name} = builtins.removeAttrs node [ "datasets" ]; + } (builtins.attrNames (node.datasets or { })); + + datasets = builtins.foldl' (result: pool: result // flatten pool pools.${pool}) { } ( + builtins.attrNames pools + ); + + # zfs gives a dataset with no mountpoint of its own its parent's mountpoint + # plus its final name component, so resolve it the same way. Pool roots all + # declare a mountpoint, which terminates the recursion. + mountpointOf = + name: + let + properties = datasets.${name}.properties or { }; + in + if properties ? mountpoint then + properties.mountpoint + else if builtins.match ".*/.*" name != null then + "${mountpointOf (builtins.dirOf name)}/${builtins.baseNameOf name}" + else + throw "jeeves: ${name} has no mountpoint and no parent to inherit one from"; + + mountpoints = builtins.mapAttrs (name: _: mountpointOf name) datasets; +in +{ + inherit datasets mountpoints; + defaultSnapshots = standard; +} diff --git a/systems/jeeves/default.nix b/systems/jeeves/default.nix index 1d47f8d..de4150f 100644 --- a/systems/jeeves/default.nix +++ b/systems/jeeves/default.nix @@ -15,6 +15,7 @@ in "${inputs.self}/common/optional/syncthing_base.nix" "${inputs.self}/common/optional/update.nix" "${inputs.self}/common/optional/zerotier.nix" + "${inputs.self}/common/optional/zfs_manager.nix" ./monitoring ./docker ./services @@ -24,6 +25,7 @@ in ./programs.nix ./runners ./syncthing.nix + ./zfs.nix ]; services = { @@ -31,10 +33,8 @@ in smartd.enable = true; - snapshot_manager = { - path = ./snapshot_config.toml; - EnvironmentFile = "${vars.secrets}/services/snapshot_manager"; - }; + # path is generated from ./zfs.nix by common/optional/zfs_manager.nix + snapshot_manager.EnvironmentFile = "${vars.secrets}/services/snapshot_manager"; zerotierone.joinNetworks = [ "a09acf02330d37b9" ]; }; diff --git a/systems/jeeves/scripts/zfs.sh b/systems/jeeves/scripts/zfs.sh index 8dd630c..802c851 100644 --- a/systems/jeeves/scripts/zfs.sh +++ b/systems/jeeves/scripts/zfs.sh @@ -1,5 +1,10 @@ #!/bin/bash +# Pool and vdev creation only. This is run by hand once per pool. +# +# Datasets and their properties are declared in systems/jeeves/zfs.nix and +# reconciled by the zfs_manager service. Do not add zfs create lines here. + # zpools # media @@ -12,35 +17,10 @@ sudo zpool add storage -o ashift=12 special mirror sudo zpool add storage -o ashift=12 logs mirror # scratch -sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnodesize=auto -O xattr=sa -O compression=zstd -O encryption=aes-256-gcm -O keyformat=hex -O keylocation=file:///key -m /zfs/scratch +sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnodesize=auto -O xattr=sa -O compression=zstd -O encryption=aes-256-gcm -O keyformat=hex -O keylocation=file:///root/zfs.key -m /zfs/scratch -# media datasets -sudo zfs create media/temp -o sync=disabled -o redundant_metadata=none +# The two encrypted parent datasets have to exist before zfs_manager can create +# anything under them, since encryption cannot be set after creation. +# These will be removed if/when the media and storage pools are encrypted in the future. sudo zfs create media/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key -sudo zfs create media/secure/docker -o compression=zstd-9 -sudo zfs create media/secure/github-runners -o compression=zstd-9 -o sync=disabled -sudo zfs create media/secure/home_assistant -o compression=zstd-19 -sudo zfs create media/secure/notes -o copies=2 -sudo zfs create media/secure/postgres -o mountpoint=/zfs/media/database/postgres -o recordsize=16k -o primarycache=metadata -sudo zfs create media/secure/postgres-wal -o mountpoint=/zfs/media/database/postgres-wal -o recordsize=32k -o primarycache=metadata -o special_small_blocks=32K -o compression=lz4 -o secondarycache=none -o logbias=latency -sudo zfs create media/secure/prometheus -o mountpoint=/zfs/media/database/prometheus -o compression=lz4 -sudo zfs create media/secure/services -o compression=zstd-9 -sudo zfs create media/secure/share -o mountpoint=/zfs/media/share -o exec=off - -# scratch datasets -sudo zfs create scratch/kafka -o mountpoint=/zfs/scratch/kafka -o recordsize=1M -sudo zfs create scratch/transmission -o mountpoint=/zfs/scratch/transmission -o recordsize=16k -o sync=disabled -o redundant_metadata=none -sudo zfs create scratch/uv_cache -o mountpoint=/zfs/scratch/uv_cache - -# storage datasets -sudo zfs create storage/ollama -o recordsize=1M -o compression=zstd-19 -o sync=disabled sudo zfs create storage/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key -sudo zfs create storage/secure/archive -o recordsize=1M -o compression=zstd-19 -sudo zfs create storage/secure/library -o recordsize=1M -o compression=zstd-19 -sudo zfs create storage/secure/main -o compression=zstd-19 -sudo zfs create storage/secure/photos -o recordsize=16K -o compression=zstd-19 -o copies=2 -sudo zfs create storage/secure/plex -o recordsize=1M -o compression=zstd-19 -sudo zfs create storage/secure/secrets -o compression=zstd-19 -o copies=3 -sudo zfs create storage/secure/syncthing -o compression=zstd-19 -sudo zfs create storage/secure/transmission -o recordsize=1M -o compression=zstd-9 -o exec=off -o sync=disabled -sudo zfs create storage/secure/important -o compression=zstd-19 -o copies=2 -o mountpoint=/zfs/storage/important diff --git a/systems/jeeves/snapshot_config.toml b/systems/jeeves/snapshot_config.toml deleted file mode 100644 index cb9a3bf..0000000 --- a/systems/jeeves/snapshot_config.toml +++ /dev/null @@ -1,129 +0,0 @@ -["default"] -15_min = 8 -hourly = 24 -daily = 0 -monthly = 0 - -# root_pool -["root_pool/home"] -15_min = 8 -hourly = 24 -daily = 14 -monthly = 0 - -["root_pool/root"] -15_min = 8 -hourly = 24 -daily = 0 -monthly = 0 - -["root_pool/nix"] -15_min = 4 -hourly = 0 -daily = 0 -monthly = 0 - -["root_pool/var"] -15_min = 8 -hourly = 24 -daily = 30 -monthly = 6 -# storage -["storage/ollama"] -15_min = 2 -hourly = 0 -daily = 0 -monthly = 0 - -["storage/secure"] -15_min = 0 -hourly = 0 -daily = 0 -monthly = 0 - -["storage/secure/plex"] -15_min = 6 -hourly = 2 -daily = 1 -monthly = 0 - -["storage/secure/transmission"] -15_min = 4 -hourly = 0 -daily = 0 -monthly = 0 - -["storage/secure/secrets"] -15_min = 8 -hourly = 24 -daily = 30 -monthly = 12 - -# media -["media/temp"] -15_min = 2 -hourly = 0 -daily = 0 -monthly = 0 - -["media/secure"] -15_min = 0 -hourly = 0 -daily = 0 -monthly = 0 - -["media/secure/plex"] -15_min = 6 -hourly = 2 -daily = 1 -monthly = 0 - -["media/secure/postgres-wal"] -15_min = 4 -hourly = 2 -daily = 0 -monthly = 0 - - -["media/secure/postgres"] -15_min = 8 -hourly = 24 -daily = 7 -monthly = 0 - -["media/secure/share"] -15_min = 4 -hourly = 0 -daily = 0 -monthly = 0 - -["media/secure/github-runners"] -15_min = 6 -hourly = 2 -daily = 1 -monthly = 0 - -["media/secure/notes"] -15_min = 8 -hourly = 24 -daily = 30 -monthly = 12 - -["media/secure/docker"] -15_min = 3 -hourly = 12 -daily = 14 -monthly = 2 - -# scratch -["scratch/transmission"] -15_min = 2 -hourly = 0 -daily = 0 -monthly = 0 - -["scratch/uv_cache"] -15_min = 2 -hourly = 0 -daily = 0 -monthly = 0 diff --git a/systems/jeeves/vars.nix b/systems/jeeves/vars.nix index a8ffbeb..31b7f4f 100644 --- a/systems/jeeves/vars.nix +++ b/systems/jeeves/vars.nix @@ -1,22 +1,22 @@ +# Paths are derived from the dataset declarations in ./datasets.nix so that a +# mountpoint only ever has to change in one place. let - zfs_media = "/zfs/media"; - zfs_storage = "/zfs/storage"; - zfs_scratch = "/zfs/scratch"; + inherit (import ./datasets.nix) mountpoints; in { - inherit zfs_media zfs_storage zfs_scratch; - database = "${zfs_media}/database"; - docker = "${zfs_media}/docker"; - docker_configs = "${zfs_media}/docker/configs"; - home_assistant = "${zfs_media}/home_assistant"; - notes = "${zfs_media}/notes"; - secrets = "${zfs_storage}/secrets"; - services = "${zfs_media}/services"; - share = "${zfs_media}/share"; - syncthing = "${zfs_storage}/syncthing"; - transmission = "${zfs_storage}/transmission"; - ollama = "${zfs_storage}/ollama"; - transmission_scratch = "${zfs_scratch}/transmission"; - uv_cache = "${zfs_scratch}/uv_cache"; - kafka = "${zfs_scratch}/kafka"; + # Not a dataset of its own, it is the directory the postgres datasets share. + database = "/zfs/media/database"; + docker = mountpoints."media/secure/docker"; + docker_configs = "${mountpoints."media/secure/docker"}/configs"; + home_assistant = mountpoints."media/secure/home_assistant"; + notes = mountpoints."media/secure/notes"; + secrets = mountpoints."storage/secure/secrets"; + services = mountpoints."media/secure/services"; + share = mountpoints."media/secure/share"; + syncthing = mountpoints."storage/secure/syncthing"; + transmission = mountpoints."storage/secure/transmission"; + ollama = mountpoints."storage/ollama"; + transmission_scratch = mountpoints."scratch/transmission"; + uv_cache = mountpoints."scratch/uv_cache"; + kafka = mountpoints."scratch/kafka"; } diff --git a/systems/jeeves/zfs.nix b/systems/jeeves/zfs.nix new file mode 100644 index 0000000..517a55d --- /dev/null +++ b/systems/jeeves/zfs.nix @@ -0,0 +1,14 @@ +{ inputs, ... }: +let + vars = import ./vars.nix; + jeeves_zfs = import ./datasets.nix; +in +{ + services.zfs_manager = { + enable = true; + PYTHONPATH = "${inputs.self}/"; + EnvironmentFile = "${vars.secrets}/services/snapshot_manager"; + + inherit (jeeves_zfs) datasets defaultSnapshots; + }; +} diff --git a/tests/test_zfs_manager.py b/tests/test_zfs_manager.py new file mode 100644 index 0000000..c037568 --- /dev/null +++ b/tests/test_zfs_manager.py @@ -0,0 +1,242 @@ +"""test_zfs_manager.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +from python.tools.zfs_manager import main, parse_size, values_match + +if TYPE_CHECKING: + from pyfakefs.fake_filesystem import FakeFilesystem + from pytest_mock import MockerFixture + +ZFS_MANAGER = "python.tools.zfs_manager" +CONFIG_PATH = "/mock_zfs_datasets.json" + + +def write_config( + fs: FakeFilesystem, + datasets: dict[str, dict[str, str]], + unmanaged: list[str] | None = None, +) -> Path: + """Write a dataset declaration to the fake filesystem.""" + contents = { + "datasets": {name: {"manageProperties": True, "properties": props} for name, props in datasets.items()} + | {name: {"manageProperties": False, "properties": {}} for name in unmanaged or []}, + } + fs.create_file(CONFIG_PATH, contents=json.dumps(contents)) + return Path(CONFIG_PATH) + + +def patch_zfs( + mocker: MockerFixture, + existing: list[str], + properties: dict[str, dict[str, tuple[str, str]]] | None = None, +) -> dict[str, object]: + """Patch every zfs call zfs_manager makes.""" + return { + "list": mocker.patch(f"{ZFS_MANAGER}.list_dataset_names", return_value=existing), + "get": mocker.patch(f"{ZFS_MANAGER}.get_properties", side_effect=lambda name: (properties or {}).get(name, {})), + "create": mocker.patch(f"{ZFS_MANAGER}.create_dataset", return_value=None), + "set": mocker.patch(f"{ZFS_MANAGER}.set_property", return_value=None), + "alert": mocker.patch(f"{ZFS_MANAGER}.signal_alert"), + } + + +def test_creates_missing_dataset(mocker: MockerFixture, fs: FakeFilesystem) -> None: + zfs = patch_zfs(mocker, existing=["media", "media/secure"]) + config = write_config(fs, {"media/secure/new": {"compression": "zstd-9"}}) + + main(config) + + zfs["create"].assert_called_once_with("media/secure/new", {"compression": "zstd-9"}) + zfs["set"].assert_not_called() + zfs["alert"].assert_not_called() + + +def test_sets_drifted_property(mocker: MockerFixture, fs: FakeFilesystem) -> None: + zfs = patch_zfs( + mocker, + existing=["media", "media/temp"], + properties={"media/temp": {"compression": ("zstd", "inherited from media")}}, + ) + config = write_config(fs, {"media/temp": {"compression": "zstd-9"}}) + + main(config) + + zfs["set"].assert_called_once_with("media/temp", "compression", "zstd-9") + zfs["create"].assert_not_called() + + +def test_no_op_when_in_sync(mocker: MockerFixture, fs: FakeFilesystem) -> None: + zfs = patch_zfs( + mocker, + existing=["media", "media/temp"], + properties={"media/temp": {"sync": ("disabled", "local")}}, + ) + config = write_config(fs, {"media/temp": {"sync": "disabled"}}) + + main(config) + + zfs["set"].assert_not_called() + zfs["create"].assert_not_called() + zfs["alert"].assert_not_called() + + +def test_size_property_does_not_churn(mocker: MockerFixture, fs: FakeFilesystem) -> None: + """zfs get -p reports recordsize in bytes, the declaration uses a suffix.""" + zfs = patch_zfs( + mocker, + existing=["media", "media/db"], + properties={"media/db": {"recordsize": ("16384", "local"), "special_small_blocks": ("32768", "local")}}, + ) + config = write_config(fs, {"media/db": {"recordsize": "16k", "special_small_blocks": "32K"}}) + + main(config) + + zfs["set"].assert_not_called() + + +def test_create_only_property_alerts_instead_of_setting(mocker: MockerFixture, fs: FakeFilesystem) -> None: + zfs = patch_zfs( + mocker, + existing=["media", "media/secure"], + properties={"media/secure": {"encryption": ("aes-256-gcm", "local")}}, + ) + config = write_config(fs, {"media/secure": {"encryption": "off"}}) + + main(config) + + zfs["set"].assert_not_called() + assert zfs["alert"].call_count == 1 + assert "can only be set when the dataset is created" in zfs["alert"].call_args.args[0] + + +def test_undeclared_local_property_warns(mocker: MockerFixture, fs: FakeFilesystem) -> None: + zfs = patch_zfs( + mocker, + existing=["media", "media/temp"], + properties={"media/temp": {"exec": ("off", "local")}}, + ) + config = write_config(fs, {"media/temp": {}}) + + main(config) + + zfs["alert"].assert_called_once_with("media/temp has exec=off set outside of nix") + + +def test_undeclared_inherited_property_is_silent(mocker: MockerFixture, fs: FakeFilesystem) -> None: + zfs = patch_zfs( + mocker, + existing=["media", "media/temp"], + properties={ + "media/temp": { + "compression": ("zstd", "inherited from media"), + "exec": ("on", "default"), + "nixos:shutdown-time": ("whenever", "local"), + }, + }, + ) + config = write_config(fs, {"media/temp": {}}) + + main(config) + + zfs["alert"].assert_not_called() + + +def test_dry_run_makes_no_changes(mocker: MockerFixture, fs: FakeFilesystem) -> None: + zfs = patch_zfs( + mocker, + existing=["media", "media/temp"], + properties={"media/temp": {"compression": ("zstd", "local")}}, + ) + config = write_config(fs, {"media/temp": {"compression": "zstd-9"}, "media/new": {}}) + + main(config, dry_run=True) + + zfs["set"].assert_not_called() + zfs["create"].assert_not_called() + + +def test_pool_root_is_never_created(mocker: MockerFixture, fs: FakeFilesystem) -> None: + zfs = patch_zfs(mocker, existing=[]) + config = write_config(fs, {"media": {"atime": "off"}}) + + main(config) + + zfs["create"].assert_not_called() + assert "does not create pools" in zfs["alert"].call_args.args[0] + + +def test_children_skipped_when_parent_creation_fails(mocker: MockerFixture, fs: FakeFilesystem) -> None: + zfs = patch_zfs(mocker, existing=["media"]) + zfs["create"].return_value = "Failed to create media/secure: key not loaded" + config = write_config(fs, {"media/secure": {}, "media/secure/child": {}}) + + main(config) + + zfs["create"].assert_called_once_with("media/secure", {}) + + +def test_unmanaged_dataset_properties_are_untouched(mocker: MockerFixture, fs: FakeFilesystem) -> None: + """A snapshots-only dataset is neither reconciled nor reported as unknown.""" + zfs = patch_zfs( + mocker, + existing=["root_pool", "root_pool/var"], + properties={"root_pool/var": {"compression": ("lz4", "local")}}, + ) + config = write_config(fs, {}, unmanaged=["root_pool", "root_pool/var"]) + + main(config) + + zfs["get"].assert_not_called() + zfs["set"].assert_not_called() + zfs["alert"].assert_not_called() + + +def test_undeclared_dataset_is_left_alone(mocker: MockerFixture, fs: FakeFilesystem) -> None: + zfs = patch_zfs(mocker, existing=["media", "media/undeclared"]) + config = write_config(fs, {}) + + main(config) + + zfs["create"].assert_not_called() + zfs["set"].assert_not_called() + zfs["alert"].assert_not_called() + + +def test_main_exception(mocker: MockerFixture, fs: FakeFilesystem) -> None: + zfs = patch_zfs(mocker, existing=[]) + zfs["list"].side_effect = Exception("test") + config = write_config(fs, {}) + + with pytest.raises(SystemExit) as pytest_wrapped_e: + main(config) + + assert pytest_wrapped_e.value.code == 1 + zfs["alert"].assert_called_once_with("zfs_manager failed") + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("16384", 16384), + ("16k", 16384), + ("16K", 16384), + ("1M", 1048576), + ("none", None), + ("", None), + ], +) +def test_parse_size(value, expected) -> None: + assert parse_size(value) == expected + + +def test_values_match_falls_back_to_string_for_unparsable_sizes() -> None: + assert not values_match("recordsize", "none", "16384") + assert values_match("recordsize", "none", "none") + assert not values_match("compression", "zstd", "zstd-9")