chore: update flake.lock

.github/workflows/zfs-integration.yml

@@ -1,51 +0,0 @@
-name: zfs integration
-
-on:
-  workflow_dispatch:
-  pull_request:
-    branches: [main]
-    paths:
-      - ".github/workflows/zfs-integration.yml"
-      - "flake.nix"
-      - "flake.lock"
-      - "overlays/default.nix"
-      - "common/global/snapshot_manager.nix"
-      - "common/optional/zfs_manager.nix"
-      - "python/signal_alert.py"
-      - "python/tools/snapshot_manager.py"
-      - "python/tools/zfs_manager.py"
-      - "python/zfs/**"
-      - "systems/jeeves/datasets.nix"
-      - "systems/jeeves/scripts/zfs.sh"
-      - "systems/jeeves/zfs.nix"
-      - "tests/zfs_integration.py"
-      - "tests/zfs-integration.nix"
-  push:
-    branches: [main]
-    paths:
-      - ".github/workflows/zfs-integration.yml"
-      - "flake.nix"
-      - "flake.lock"
-      - "overlays/default.nix"
-      - "common/global/snapshot_manager.nix"
-      - "common/optional/zfs_manager.nix"
-      - "python/signal_alert.py"
-      - "python/tools/snapshot_manager.py"
-      - "python/tools/zfs_manager.py"
-      - "python/zfs/**"
-      - "systems/jeeves/datasets.nix"
-      - "systems/jeeves/scripts/zfs.sh"
-      - "systems/jeeves/zfs.nix"
-      - "tests/zfs_integration.py"
-      - "tests/zfs-integration.nix"
-
-jobs:
-  zfs-integration:
-    runs-on: self-hosted
-    timeout-minutes: 30
-    steps:
-      - uses: actions/checkout@v4
-      - name: Build and run ZFS integration VM
-        run: >-
-          nix build --accept-flake-config --print-build-logs
-          .#packages.x86_64-linux.zfs-integration

Cargo.toml

@@ -1,4 +0,0 @@
-[workspace]
-resolver = "2"
-
-members = ["rust/*"]

common/global/snapshot_manager.nix

@@ -22,11 +22,6 @@ in
           the PYTHONPATH to use for the snapshot_manager service.
         '';
       };
-      package = lib.mkOption {
-        type = lib.types.package;
-        default = pkgs.my_python;
-        description = "Python environment used to run snapshot_manager.";
-      };
       EnvironmentFile = lib.mkOption {
         type = lib.types.nullOr (lib.types.coercedTo lib.types.path toString lib.types.str);
         default = null;
@@ -50,7 +45,7 @@ in
         };
         serviceConfig = {
           Type = "oneshot";
-          ExecStart = "${cfg.package}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
+          ExecStart = "${pkgs.my_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
         }
         // lib.optionalAttrs (cfg.EnvironmentFile != null) {
           EnvironmentFile = cfg.EnvironmentFile;

common/optional/zfs_manager.nix

@@ -1,191 +0,0 @@
-{
-  pkgs,
-  lib,
-  config,
-  ...
-}:
-let
-  cfg = config.services.zfs_manager;
-
-  snapshotOptions = {
-    options = {
-      "15_min" = lib.mkOption {
-        type = lib.types.ints.unsigned;
-        default = 0;
-        description = "How many 15 minute snapshots to keep.";
-      };
-      hourly = lib.mkOption {
-        type = lib.types.ints.unsigned;
-        default = 0;
-        description = "How many hourly snapshots to keep.";
-      };
-      daily = lib.mkOption {
-        type = lib.types.ints.unsigned;
-        default = 0;
-        description = "How many daily snapshots to keep.";
-      };
-      monthly = lib.mkOption {
-        type = lib.types.ints.unsigned;
-        default = 0;
-        description = "How many monthly snapshots to keep.";
-      };
-    };
-  };
-
-  datasetOptions = {
-    options = {
-      manageProperties = lib.mkOption {
-        type = lib.types.bool;
-        default = true;
-        description = ''
-          Whether zfs_manager owns this dataset's properties. When false the
-          dataset only contributes its snapshot retention, which is how
-          root_pool datasets are declared.
-        '';
-      };
-      createIfMissing = lib.mkOption {
-        type = lib.types.bool;
-        default = true;
-        description = ''
-          Whether zfs_manager may create this dataset when it is absent.
-
-          Set it false for a dataset that has to be provisioned by hand, such
-          as an encryption root: encryption is fixed at creation time and
-          cannot be expressed here, so creating it automatically would silently
-          produce an unencrypted dataset where an encrypted one was intended.
-          The dataset is still property checked, and its absence is reported as
-          a failure rather than quietly fixed.
-        '';
-      };
-      properties = lib.mkOption {
-        type = lib.types.attrsOf lib.types.str;
-        default = { };
-        description = ''
-          The zfs properties this dataset should have. Values are compared
-          against the live dataset and corrected when they differ.
-        '';
-      };
-      snapshots = lib.mkOption {
-        type = lib.types.submodule snapshotOptions;
-        default = cfg.defaultSnapshots;
-        description = ''
-          Snapshot retention for this dataset. Defaults to defaultSnapshots.
-        '';
-      };
-    };
-  };
-
-  # snapshot_manager.py only ever walks datasets below a pool root, so pool
-  # roots are left out of the retention table. It also indexes the table
-  # directly, which is why every entry carries all four keys.
-  snapshotTable = lib.mapAttrs (_: dataset: dataset.snapshots) (
-    lib.filterAttrs (name: _: lib.hasInfix "/" name) cfg.datasets
-  );
-
-  snapshotConfig = (pkgs.formats.toml { }).generate "snapshot_config.toml" (
-    snapshotTable // { default = cfg.defaultSnapshots; }
-  );
-
-  # Every declared dataset is emitted, including the ones whose properties are
-  # not managed, so the tool can tell "deliberately hands off" apart from
-  # "nobody has written this down yet".
-  datasetConfig = (pkgs.formats.json { }).generate "zfs_datasets.json" {
-    datasets = lib.mapAttrs (_: dataset: {
-      inherit (dataset) manageProperties createIfMissing properties;
-    }) cfg.datasets;
-  };
-in
-{
-  options = {
-    services.zfs_manager = {
-      enable = lib.mkEnableOption "declarative ZFS dataset management";
-      datasets = lib.mkOption {
-        type = lib.types.attrsOf (lib.types.submodule datasetOptions);
-        default = { };
-        example = lib.literalExpression ''
-          {
-            "media/temp".properties = {
-              sync = "disabled";
-              redundant_metadata = "none";
-            };
-          }
-        '';
-        description = ''
-          The datasets to manage, keyed by full dataset name. Missing datasets
-          are created and drifted properties are corrected. Nothing is ever
-          destroyed, and datasets that are not declared are left alone.
-
-          A name without a "/" is a pool root filesystem. Its properties are
-          managed but it is never created, pool creation stays manual.
-        '';
-      };
-      defaultSnapshots = lib.mkOption {
-        type = lib.types.submodule snapshotOptions;
-        default = { };
-        description = ''
-          Retention for undeclared datasets and for declared datasets that do
-          not override their snapshots. Emitted as the "default" table of the
-          snapshot config.
-        '';
-      };
-      dryRun = lib.mkOption {
-        type = lib.types.bool;
-        default = false;
-        description = ''
-          Log every change that would be made without touching zfs. Use this to
-          validate a new or heavily edited declaration before applying it.
-        '';
-      };
-      PYTHONPATH = lib.mkOption {
-        type = lib.types.str;
-        description = ''
-          the PYTHONPATH to use for the zfs_manager service.
-        '';
-      };
-      package = lib.mkOption {
-        type = lib.types.package;
-        default = pkgs.my_python;
-        description = "Python environment used to run zfs_manager.";
-      };
-      EnvironmentFile = lib.mkOption {
-        type = lib.types.nullOr (lib.types.coercedTo lib.types.path toString lib.types.str);
-        default = null;
-
-        description = ''
-          Single environment file for the service (e.g. /etc/zfs-manager/env).
-          Use a leading "-" to ignore if missing (systemd feature).
-        '';
-      };
-    };
-  };
-
-  config = lib.mkIf cfg.enable {
-    services.snapshot_manager.path = snapshotConfig;
-
-    systemd = {
-      services.zfs_manager = {
-        description = "ZFS Dataset Manager";
-        requires = [ "zfs-import.target" ];
-        after = [
-          "zfs-import.target"
-          "zfs-mount.service"
-        ];
-        wantedBy = [ "multi-user.target" ];
-        path = [ pkgs.zfs ];
-        # Re-run on nixos-rebuild switch whenever the declaration changes.
-        restartTriggers = [ datasetConfig ];
-        environment = {
-          PYTHONPATH = cfg.PYTHONPATH;
-        };
-        serviceConfig = {
-          Type = "oneshot";
-          RemainAfterExit = true;
-          ExecStart = "${cfg.package}/bin/python -m python.tools.zfs_manager ${lib.escapeShellArg datasetConfig}${lib.optionalString cfg.dryRun " --dry-run"}";
-        }
-        // lib.optionalAttrs (cfg.EnvironmentFile != null) {
-          EnvironmentFile = cfg.EnvironmentFile;
-        };
-      };
-    };
-  };
-}

flake.lock

@@ -8,11 +8,11 @@
       },
       "locked": {
         "dir": "pkgs/firefox-addons",
-        "lastModified": 1787025780,
-        "narHash": "sha256-NhyLP9G4DFOn/7aYr7K/D7hWrzEGr5EgUBV+lpdmJ24=",
+        "lastModified": 1786075368,
+        "narHash": "sha256-vSiTq6wa9WiKGHOdCRlABkKksOgfrsUuRz/K6tQ9EYA=",
         "owner": "rycee",
         "repo": "nur-expressions",
-        "rev": "5ad360b6d3cb0aa1b61f9cb27fef113ca9117c37",
+        "rev": "f516bdb9bc003c9d1f041d8ad9f0a9627fe800d2",
         "type": "gitlab"
       },
       "original": {
@@ -29,11 +29,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1786999651,
-        "narHash": "sha256-MTGMFlLDTklsXhCp4r5GXB4VAVadPdalXLvUjd/K7h0=",
+        "lastModified": 1786031233,
+        "narHash": "sha256-TIDlLTLI1/pB7IqgjzcKQjpODQsZE2oII4XGG9B6KjI=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "353742587cbaf079b3caee743115d037bc51fea6",
+        "rev": "7834e82588860aaf780cec1366524456a70898d7",
         "type": "github"
       },
       "original": {
@@ -47,11 +47,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1786867632,
-        "narHash": "sha256-ez+ubZlA1RtdjCB18a6zJ9M4u8qoPDy08EcnsW5M3Xw=",
+        "lastModified": 1785232496,
+        "narHash": "sha256-65EQYIRRpTdpH8lUiB6Mvo5uBkG60aBIzAJuALfx+O0=",
         "owner": "nixos",
         "repo": "nixos-hardware",
-        "rev": "ff17823245ab9ff7bcae6acf950bd89cba82c38c",
+        "rev": "2e790b0a6be8ec2b76174ac0931b8ff11919ec98",
         "type": "github"
       },
       "original": {
@@ -76,11 +76,11 @@
     },
     "nixpkgs-master": {
       "locked": {
-        "lastModified": 1787081018,
-        "narHash": "sha256-K0uwZBtZsbBigHAMQW7YWti3gPe6a5ct5bcOw5F+Q9Y=",
+        "lastModified": 1786146526,
+        "narHash": "sha256-fNl4co87gT5b7Q/S205uV0TUS7rVdZpZBf5OODk5fW8=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "cacac5ac351a010599d9f9d106acfed25a8e4c77",
+        "rev": "593eeb4d36f60968c197fcb5f356e1e09927415f",
         "type": "github"
       },
       "original": {
@@ -108,11 +108,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1787001381,
-        "narHash": "sha256-Ue1Yo8gfHdD4TMtNewhA4tkSYeFqXThju0nCyJc3ALo=",
+        "lastModified": 1785967620,
+        "narHash": "sha256-IItrdb7Puk05RqOBWZYFC5X6Wl1sJmCfh5MWVHw5iMM=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "ec2d622de0773551768cf98f3fc50cbcc003b9c5",
+        "rev": "b7c2ada94fe99c15b0dbcf4d11fd7850b957a436",
         "type": "github"
       },
       "original": {
@@ -141,11 +141,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1786629091,
-        "narHash": "sha256-gkig4nPi1CWc4Z50GBsjE4ygSE7hMpl/TwID2an2Cck=",
+        "lastModified": 1783174389,
+        "narHash": "sha256-aCWC8ngycU7OdJrU2+Je3qf+1a2ykuBvpPhZT/9tXMc=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "a8627b21b9107c5711c96b84f32a9a4b3d45295f",
+        "rev": "f1406619a3884cd5c47992a70b8b35c9c0fcb4c9",
         "type": "github"
       },
       "original": {

flake.nix

@@ -77,7 +77,6 @@
         }
         // lib.optionalAttrs (pkgs.stdenv.hostPlatform.system == "x86_64-linux") {
           iso = self.nixosConfigurations.iso.config.system.build.isoImage;
-          zfs-integration = pkgs.testers.runNixOSTest (import ./tests/zfs-integration.nix { inherit self; });
         }
       );
       apps = forEachSystem (

pyproject.toml

@@ -65,7 +65,6 @@ lint.ignore = [
     "ISC001", # (TEMP) conflicts when used with the formatter
     "S603",   # (PERM) This is known to cause a false positive
     "S607",   # (PERM) This is becoming a consistent annoyance
-    "CPY001",  # (PERM) I don't include the license in every file
 ]
 
 [tool.ruff.lint.per-file-ignores]

python/tools/snapshot_manager.py

@@ -23,7 +23,6 @@ def main(config_file: Path) -> None:
     """Main."""
     configure_logger(level="DEBUG")
     logger.info("Starting snapshot_manager")
-    failures: list[str] = []
 
     try:
         time_stamp = get_time_stamp()
@@ -35,23 +34,16 @@ def main(config_file: Path) -> None:
                 msg = f"{dataset.name} failed to create snapshot {time_stamp}"
                 logger.error(msg)
                 signal_alert(msg)
-                failures.append(msg)
                 continue
             count_lookup = get_count_lookup(config_file, dataset.name)
             logger.info(f"using {count_lookup} for {dataset.name}")
-            failures.extend(get_snapshots_to_delete(dataset, count_lookup))
+            get_snapshots_to_delete(dataset, count_lookup)
     except Exception:
         logger.exception("snapshot_manager failed")
         signal_alert("snapshot_manager failed")
         sys.exit(1)
-
-    if failures:
-        logger.error(f"snapshot_manager completed with {len(failures)} errors")
-        for failure in failures:
-            logger.error(f"  {failure}")
-        sys.exit(1)
-
-    logger.info("snapshot_manager completed")
+    else:
+        logger.info("snapshot_manager completed")
 
 
 def get_count_lookup(config_file: Path, dataset_name: str) -> dict[str, int]:
@@ -100,29 +92,19 @@ def load_config_data(config_file: Path) -> dict[str, dict[str, int]]:
 def get_snapshots_to_delete(
     dataset: Dataset,
     count_lookup: dict[str, int],
-) -> list[str]:
+) -> None:
     """Get snapshots to delete.
 
     Args:
         dataset (Dataset): the dataset
         count_lookup (dict[str, int]): the count lookup
-
-    Returns:
-        list[str]: Snapshot deletion failures encountered while pruning.
     """
-    for retention_class in ("15_min", "hourly", "daily", "monthly"):
-        count = count_lookup.get(retention_class)
-        if not isinstance(count, int) or isinstance(count, bool) or count < 0:
-            error = f"{retention_class} retention must be a non-negative integer, got {count!r}"
-            raise ValueError(error)
-
-    failures: list[str] = []
     snapshots = dataset.get_snapshots()
 
     logger.info(f"calculating snapshots for {dataset.name} to be deleted")
     if not snapshots:
         logger.info(f"{dataset.name} has no snapshots")
-        return failures
+        return
 
     filters = (
         ("15_min", re_compile(r"auto_\d{10}(?:15|30|45)")),
@@ -147,9 +129,6 @@ def get_snapshots_to_delete(
                 error_message = f"{dataset.name}@{snapshot} failed to delete: {error}"
                 signal_alert(error_message)
                 logger.error(error_message)
-                failures.append(error_message)
-
-    return failures
 
 
 def get_time_stamp() -> str:

python/tools/zfs_manager.py

@@ -1,340 +0,0 @@
-"""zfs_manager.
-
-Reconciles the live zfs datasets against a declaration generated by
-common/optional/zfs_manager.nix. Datasets are created and properties are
-corrected, but nothing is ever destroyed or renamed.
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import sys
-from pathlib import Path  # noqa: TC003 This is required for the typer CLI
-
-import typer
-
-from python.common import configure_logger
-from python.signal_alert import signal_alert
-from python.zfs import create_dataset, get_properties, list_dataset_names, set_property
-
-logger = logging.getLogger(__name__)
-
-# Properties that can only be chosen at creation time. Attempting to zfs set
-# these fails on every run, so a mismatch is reported instead of retried.
-CREATE_ONLY_PROPERTIES = frozenset(
-    {
-        "casesensitivity",
-        "encryption",
-        "keyformat",
-        "normalization",
-        "utf8only",
-        "volblocksize",
-    },
-)
-
-# Properties whose values zfs reports in bytes but which are conventionally
-# declared with a size suffix, so "16k" and "16384" mean the same thing.
-SIZE_PROPERTIES = frozenset(
-    {
-        "quota",
-        "recordsize",
-        "refquota",
-        "refreservation",
-        "reservation",
-        "special_small_blocks",
-        "volblocksize",
-        "volsize",
-    },
-)
-
-SIZE_SUFFIXES = {"b": 1, "k": 1024, "m": 1024**2, "g": 1024**3, "t": 1024**4, "p": 1024**5}
-
-# Sources that mean the value was deliberately put on this dataset rather than
-# inherited from a parent or left at the zfs default.
-LOCAL_SOURCES = ("local", "received")
-
-
-class ReconciliationError(RuntimeError):
-    """One or more datasets could not be brought in line with the declaration."""
-
-    def __init__(self, failures: list[str]) -> None:
-        """Record the individual failures behind this run's exit code."""
-        self.failures = failures
-        super().__init__(f"ZFS reconciliation failed with {len(failures)} errors")
-
-
-def main(config_file: Path, *, dry_run: bool = False) -> None:
-    """Main.
-
-    Args:
-        config_file (Path): The path to the generated dataset declaration.
-        dry_run (bool): Log the changes that would be made without making them.
-    """
-    configure_logger(level="DEBUG")
-    logger.info(f"Starting zfs_manager {dry_run=}")
-
-    try:
-        reconcile(config_file, dry_run=dry_run)
-    except ReconciliationError as error:
-        summary = error
-    except Exception:
-        logger.exception("zfs_manager failed")
-        signal_alert("zfs_manager failed")
-        sys.exit(1)
-    else:
-        logger.info("zfs_manager completed")
-        return
-
-    # Each failure was logged and alerted as it happened. Repeating them
-    # together puts the whole picture at the end of the journal, which is what
-    # systemctl status shows. No traceback: this is an expected outcome, not a
-    # crash, and a stack trace would only bury the list.
-    logger.error(str(summary))
-    for failure in summary.failures:
-        logger.error(f"  {failure}")
-    sys.exit(1)
-
-
-def reconcile(config_file: Path, *, dry_run: bool) -> None:
-    """Bring every declared dataset in line, collecting problems as it goes.
-
-    One bad dataset must not hide the state of the others, so everything is
-    checked before anything is raised.
-
-    Args:
-        config_file (Path): The path to the generated dataset declaration.
-        dry_run (bool): Log the changes without making them.
-
-    Raises:
-        ReconciliationError: If anything could not be reconciled.
-    """
-    declared = json.loads(config_file.read_text())["datasets"]
-    existing = set(list_dataset_names())
-    unusable: set[str] = set()
-    failures: list[str] = []
-
-    # Parents before children so a newly created parent exists by the time its
-    # children are reconciled.
-    for name in sorted(declared, key=lambda name: (name.count("/"), name)):
-        entry = declared[name]
-
-        # Declared purely to record retention, its properties belong to
-        # whoever set them.
-        if not entry.get("manageProperties", True):
-            logger.debug(f"{name} is declared but its properties are not managed")
-            continue
-
-        if has_unusable_parent(name, unusable):
-            failures.append(fail(f"cannot reconcile {name}, its parent is missing"))
-            continue
-
-        if name in existing:
-            failures.extend(reconcile_dataset(name, entry["properties"], dry_run=dry_run))
-            continue
-
-        created, failure = handle_missing_dataset(name, entry, dry_run=dry_run)
-        if failure is not None:
-            failures.append(failure)
-        if created:
-            existing.add(name)
-        elif not dry_run:
-            unusable.add(name)
-
-    report_undeclared_datasets(existing, declared)
-
-    if failures:
-        raise ReconciliationError(failures)
-
-
-def fail(message: str) -> str:
-    """Log and alert a problem, and hand it back for the failure tally.
-
-    Args:
-        message (str): What went wrong.
-
-    Returns:
-        str: The same message, so the caller can collect it.
-    """
-    logger.error(message)
-    signal_alert(message)
-    return message
-
-
-def has_unusable_parent(name: str, unusable: set[str]) -> bool:
-    """Check whether an ancestor of a dataset is missing.
-
-    Args:
-        name (str): The name of the dataset.
-        unusable (set[str]): The datasets that do not exist and were not created.
-
-    Returns:
-        bool: True if any ancestor is unusable.
-    """
-    parts = name.split("/")
-    return any("/".join(parts[:depth]) in unusable for depth in range(1, len(parts)))
-
-
-def handle_missing_dataset(name: str, entry: dict, *, dry_run: bool) -> tuple[bool, str | None]:
-    """Deal with a declared dataset that is not on the system.
-
-    Pool roots are never created, and neither is anything the declaration marks
-    as provisioned outside of nix, such as an encryption root whose key
-    settings cannot be reproduced from the declaration.
-
-    Args:
-        name (str): The name of the dataset.
-        entry (dict): The declaration for this dataset.
-        dry_run (bool): Log the change without making it.
-
-    Returns:
-        tuple[bool, str | None]: Whether the dataset now exists, and a failure
-            message if there was one.
-    """
-    properties = entry["properties"]
-
-    if "/" not in name:
-        return False, fail(f"pool {name} is declared but does not exist, zfs_manager does not create pools")
-
-    if not entry.get("createIfMissing", True):
-        return False, fail(
-            f"{name} is declared but does not exist, and is marked as created outside of nix. "
-            "It has to be made by hand, see systems/jeeves/scripts/zfs.sh.",
-        )
-
-    if dry_run:
-        logger.info(f"would create {name} with {properties}")
-        return False, None
-
-    logger.info(f"creating {name} with {properties}")
-    if error := create_dataset(name, properties):
-        return False, fail(error)
-
-    return True, None
-
-
-def reconcile_dataset(name: str, properties: dict[str, str], *, dry_run: bool) -> list[str]:
-    """Bring an existing dataset in line with its declared properties.
-
-    Args:
-        name (str): The name of the dataset.
-        properties (dict[str, str]): The declared properties.
-        dry_run (bool): Log the changes without making them.
-
-    Returns:
-        list[str]: Anything that could not be put right.
-    """
-    failures: list[str] = []
-    current = get_properties(name)
-
-    for key, wanted in sorted(properties.items()):
-        current_value, _ = current.get(key, ("-", "-"))
-        if values_match(key, wanted, current_value):
-            continue
-
-        if key in CREATE_ONLY_PROPERTIES:
-            # Nothing can put this right while the dataset exists, so it is a
-            # hard failure rather than a warning that repeats unnoticed.
-            failures.append(
-                fail(
-                    f"{name} {key} is {current_value} but {wanted} is declared, "
-                    f"{key} can only be set when the dataset is created",
-                ),
-            )
-            continue
-
-        if dry_run:
-            logger.info(f"would set {key}={wanted} on {name}, currently {current_value}")
-            continue
-
-        logger.info(f"setting {key}={wanted} on {name}, was {current_value}")
-        if error := set_property(name, key, wanted):
-            failures.append(fail(error))
-
-    report_undeclared_properties(name, properties, current)
-    return failures
-
-
-def report_undeclared_properties(name: str, properties: dict[str, str], current: dict[str, tuple[str, str]]) -> None:
-    """Warn about properties set on the dataset but absent from the declaration.
-
-    Inherited and default values are silent, they are not drift. A locally set
-    value that nix does not know about was changed outside of this tool and
-    will be lost the next time the dataset is recreated, so it is worth saying.
-
-    Args:
-        name (str): The name of the dataset.
-        properties (dict[str, str]): The declared properties.
-        current (dict[str, tuple[str, str]]): The live properties keyed to (value, source).
-    """
-    for key, (value, source) in sorted(current.items()):
-        # User properties such as nixos:shutdown-time are written by other
-        # tools and are not something a dataset declaration should own.
-        if key in properties or ":" in key or source not in LOCAL_SOURCES:
-            continue
-
-        logger.warning(f"{name} has {key}={value} set outside of nix")
-        signal_alert(f"{name} has {key}={value} set outside of nix")
-
-
-def report_undeclared_datasets(existing: set[str], declared: dict[str, dict]) -> None:
-    """Warn about datasets that exist but are not declared.
-
-    These are left completely alone. They still get snapshots through the
-    default retention table.
-
-    Args:
-        existing (set[str]): The names of every live dataset.
-        declared (dict[str, dict]): The declaration.
-    """
-    for name in sorted(existing - set(declared)):
-        logger.warning(f"{name} exists but is not declared in nix")
-
-
-def values_match(key: str, wanted: str, current: str) -> bool:
-    """Compare a declared property value against the live one.
-
-    Args:
-        key (str): The property name.
-        wanted (str): The declared value.
-        current (str): The live value.
-
-    Returns:
-        bool: True if the two values mean the same thing.
-    """
-    if key in SIZE_PROPERTIES:
-        wanted_size = parse_size(wanted)
-        current_size = parse_size(current)
-        if wanted_size is not None and current_size is not None:
-            return wanted_size == current_size
-
-    return wanted == current
-
-
-def parse_size(value: str) -> int | None:
-    """Convert a zfs size such as 16k or 1M into bytes.
-
-    Args:
-        value (str): The size to convert.
-
-    Returns:
-        int | None: The size in bytes, or None if it is not a size.
-    """
-    value = value.strip()
-    if value.isdigit():
-        return int(value)
-
-    number, suffix = value[:-1], value[-1:].lower()
-    if suffix in SIZE_SUFFIXES and number.isdigit():
-        return int(number) * SIZE_SUFFIXES[suffix]
-
-    return None
-
-
-def cli() -> None:
-    """CLI."""
-    typer.run(main)
-
-
-if __name__ == "__main__":
-    cli()

rust/van_weather/Cargo.toml

@@ -1,21 +0,0 @@
-[package]
-name = "van-weather"
-version = "0.1.0"
-edition = "2024"
-description = "Fetch privacy-masked weather for a van and publish it to Home Assistant"
-license = "MIT"
-
-[dependencies]
-anyhow = "1.0"
-chrono = "0.4"
-clap = { version = "4.5", features = ["derive", "env"] }
-env_logger = "0.11"
-log = "0.4"
-reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] }
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
-
-[lints.clippy]
-all = "deny"
-pedantic = "deny"
-

rust/van_weather/package.nix

@@ -1,13 +0,0 @@
-{ rustPlatform }:
-rustPlatform.buildRustPackage {
-  pname = "van-weather";
-  version = "0.1.0";
-  src = ./.;
-
-  cargoLock.lockFile = ./Cargo.lock;
-
-  meta = {
-    description = "Privacy-masked van weather publisher for Home Assistant";
-    mainProgram = "van-weather";
-  };
-}

rust/van_weather/src/main.rs

@@ -1,565 +0,0 @@
-use std::{thread, time::Duration};
-
-use anyhow::{Context, Result, bail};
-use chrono::{DateTime, Utc};
-use clap::Parser;
-use log::{error, info};
-use reqwest::{
-    StatusCode, Url,
-    blocking::{Client, ClientBuilder},
-    header::{AUTHORIZATION, HeaderMap, HeaderValue},
-    retry,
-};
-use serde::Deserialize;
-use serde_json::{Value, json};
-
-const LAT_ENTITY: &str = "sensor.van_last_known_latitude";
-const LON_ENTITY: &str = "sensor.van_last_known_longitude";
-const PIRATE_WEATHER_HOST: &str = "api.pirateweather.net";
-const MASK_DECIMALS: u32 = 1;
-const MASK_FACTOR: f64 = decimal_factor(MASK_DECIMALS);
-const RETRIES_PER_REQUEST: u32 = 2;
-
-const fn decimal_factor(decimals: u32) -> f64 {
-    let mut factor = 1.0;
-    let mut remaining = decimals;
-    while remaining > 0 {
-        factor *= 10.0;
-        remaining -= 1;
-    }
-    factor
-}
-
-#[derive(Debug, Parser)]
-#[command(about, version)]
-struct Args {
-    #[arg(long, env = "HA_URL")]
-    ha_url: String,
-
-    #[arg(long, env = "HA_TOKEN", hide_env_values = true)]
-    ha_token: String,
-
-    #[arg(long, env = "PIRATE_WEATHER_API_KEY", hide_env_values = true)]
-    pirate_weather_api_key: String,
-
-    #[arg(
-        long,
-        default_value_t = 900,
-        value_parser = clap::value_parser!(u64).range(1..)
-    )]
-    interval: u64,
-
-    #[arg(long, default_value = "info", env = "RUST_LOG")]
-    log_level: String,
-}
-
-#[derive(Debug, Deserialize)]
-struct HaState {
-    state: String,
-}
-
-#[derive(Debug)]
-struct HttpClients {
-    home_assistant: Client,
-    pirate_weather: Client,
-}
-
-#[derive(Debug, Default, Deserialize)]
-struct ApiResponse {
-    #[serde(default)]
-    currently: CurrentWeather,
-    #[serde(default)]
-    daily: ForecastBlock<DailyApiForecast>,
-    #[serde(default)]
-    hourly: ForecastBlock<HourlyApiForecast>,
-}
-
-#[derive(Debug, Default, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct CurrentWeather {
-    temperature: Option<f64>,
-    apparent_temperature: Option<f64>,
-    humidity: Option<f64>,
-    wind_speed: Option<f64>,
-    wind_bearing: Option<f64>,
-    icon: Option<String>,
-    pressure: Option<f64>,
-    visibility: Option<f64>,
-    uv_index: Option<f64>,
-    ozone: Option<f64>,
-    nearest_storm_distance: Option<f64>,
-    nearest_storm_bearing: Option<f64>,
-    precip_probability: Option<f64>,
-    cloud_cover: Option<f64>,
-}
-
-#[derive(Debug, Deserialize)]
-struct ForecastBlock<T> {
-    #[serde(default)]
-    data: Vec<T>,
-}
-
-impl<T> Default for ForecastBlock<T> {
-    fn default() -> Self {
-        Self { data: Vec::new() }
-    }
-}
-
-#[derive(Debug, Default, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct DailyApiForecast {
-    time: Option<i64>,
-    icon: Option<String>,
-    temperature_high: Option<f64>,
-    temperature_low: Option<f64>,
-    precip_probability: Option<f64>,
-}
-
-#[derive(Debug, Default, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct HourlyApiForecast {
-    time: Option<i64>,
-    icon: Option<String>,
-    temperature: Option<f64>,
-    precip_probability: Option<f64>,
-}
-
-#[derive(Debug)]
-struct Weather {
-    current: CurrentWeather,
-    daily: Vec<DailyForecast>,
-    hourly: Vec<HourlyForecast>,
-}
-
-#[derive(Debug)]
-struct DailyForecast {
-    datetime: DateTime<Utc>,
-    condition: &'static str,
-    temperature: Option<f64>,
-    templow: Option<f64>,
-    precipitation_probability: Option<f64>,
-}
-
-#[derive(Debug)]
-struct HourlyForecast {
-    datetime: DateTime<Utc>,
-    condition: &'static str,
-    temperature: Option<f64>,
-    precipitation_probability: Option<f64>,
-}
-
-fn main() -> Result<()> {
-    let Args {
-        ha_url,
-        ha_token,
-        pirate_weather_api_key,
-        interval,
-        log_level,
-    } = Args::parse();
-    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or(log_level.as_str()))
-        .init();
-
-    let ha_url = ha_url.trim_end_matches('/').to_owned();
-    let ha_host = Url::parse(&ha_url)
-        .context("HA_URL is not a valid URL")?
-        .host_str()
-        .context("HA_URL has no host")?
-        .to_owned();
-    let clients = HttpClients {
-        home_assistant: build_client(&ha_host, Some(&ha_token))?,
-        pirate_weather: build_client(PIRATE_WEATHER_HOST, None)?,
-    };
-
-    info!("Starting van weather service, polling every {interval}s");
-    loop {
-        if let Err(err) = update_weather(&clients, &ha_url, &pirate_weather_api_key) {
-            error!("Weather update failed: {err:#}");
-        }
-        thread::sleep(Duration::from_secs(interval));
-    }
-}
-
-fn build_client(host: &str, bearer_token: Option<&str>) -> Result<Client> {
-    let policy = retry::for_host(host.to_owned())
-        .max_retries_per_request(RETRIES_PER_REQUEST)
-        .classify_fn(|request| {
-            let retryable = request.error().is_some()
-                || request.status().is_some_and(|status| {
-                    status == StatusCode::REQUEST_TIMEOUT
-                        || status == StatusCode::TOO_MANY_REQUESTS
-                        || status.is_server_error()
-                });
-            if retryable {
-                request.retryable()
-            } else {
-                request.success()
-            }
-        });
-
-    let mut builder = ClientBuilder::new()
-        .timeout(Duration::from_secs(30))
-        .retry(policy);
-
-    if let Some(token) = bearer_token {
-        let mut authorization = HeaderValue::from_str(&format!("Bearer {token}"))
-            .context("HA_TOKEN contains invalid header characters")?;
-        authorization.set_sensitive(true);
-        let mut headers = HeaderMap::new();
-        headers.insert(AUTHORIZATION, authorization);
-        builder = builder.default_headers(headers);
-    }
-
-    builder
-        .build()
-        .with_context(|| format!("failed to create HTTP client for {host}"))
-}
-
-fn update_weather(clients: &HttpClients, ha_url: &str, api_key: &str) -> Result<()> {
-    let lat = get_ha_state(&clients.home_assistant, ha_url, LAT_ENTITY)?;
-    let lon = get_ha_state(&clients.home_assistant, ha_url, LON_ENTITY)?;
-    let masked_lat = mask_coordinate(lat);
-    let masked_lon = mask_coordinate(lon);
-    info!("Masked location: {masked_lat}, {masked_lon}");
-
-    let weather = fetch_weather(&clients.pirate_weather, api_key, masked_lat, masked_lon)?;
-    info!(
-        "Weather: {}°F, {}",
-        weather
-            .current
-            .temperature
-            .map_or_else(|| "unknown".to_owned(), |value| value.to_string()),
-        condition(weather.current.icon.as_deref())
-    );
-    post_to_ha(&clients.home_assistant, ha_url, &weather)?;
-    info!("Posted weather to Home Assistant");
-    Ok(())
-}
-
-fn mask_coordinate(value: f64) -> f64 {
-    (value * MASK_FACTOR).round() / MASK_FACTOR
-}
-
-fn get_ha_state(client: &Client, ha_url: &str, entity_id: &str) -> Result<f64> {
-    let HaState { state } = client
-        .get(format!("{ha_url}/api/states/{entity_id}"))
-        .send()
-        .with_context(|| format!("request for {entity_id} failed"))?
-        .error_for_status()
-        .with_context(|| format!("Home Assistant rejected {entity_id} request"))?
-        .json()
-        .context("Home Assistant returned invalid JSON")?;
-
-    if matches!(state.as_str(), "unavailable" | "unknown") {
-        bail!("{entity_id} is {state}");
-    }
-
-    state
-        .parse::<f64>()
-        .with_context(|| format!("{entity_id} state is not numeric: {state}"))
-}
-
-fn fetch_weather(client: &Client, api_key: &str, lat: f64, lon: f64) -> Result<Weather> {
-    let response = client
-        .get(format!(
-            "https://{PIRATE_WEATHER_HOST}/forecast/{api_key}/{lat},{lon}"
-        ))
-        .query(&[("units", "us")])
-        .send()
-        .context("Pirate Weather request failed")?
-        .error_for_status()
-        .context("Pirate Weather rejected request")?;
-    let data = response
-        .json::<ApiResponse>()
-        .context("Pirate Weather returned invalid JSON")?;
-    Ok(parse_weather(data))
-}
-
-fn parse_weather(data: ApiResponse) -> Weather {
-    let daily = data
-        .daily
-        .data
-        .into_iter()
-        .take(8)
-        .filter_map(|day| {
-            timestamp(day.time).map(|datetime| DailyForecast {
-                datetime,
-                condition: condition(day.icon.as_deref()),
-                temperature: day.temperature_high,
-                templow: day.temperature_low,
-                precipitation_probability: day.precip_probability,
-            })
-        })
-        .collect();
-    let hourly = data
-        .hourly
-        .data
-        .into_iter()
-        .take(48)
-        .filter_map(|hour| {
-            timestamp(hour.time).map(|datetime| HourlyForecast {
-                datetime,
-                condition: condition(hour.icon.as_deref()),
-                temperature: hour.temperature,
-                precipitation_probability: hour.precip_probability,
-            })
-        })
-        .collect();
-    Weather {
-        current: data.currently,
-        daily,
-        hourly,
-    }
-}
-
-fn timestamp(value: Option<i64>) -> Option<DateTime<Utc>> {
-    value
-        .filter(|value| *value != 0)
-        .and_then(DateTime::from_timestamp_secs)
-}
-
-fn condition(icon: Option<&str>) -> &'static str {
-    match icon.unwrap_or_default() {
-        "clear-day" => "sunny",
-        "clear-night" => "clear-night",
-        "rain" => "rainy",
-        "snow" => "snowy",
-        "sleet" => "snowy-rainy",
-        "wind" => "windy",
-        "fog" => "fog",
-        "partly-cloudy-day" | "partly-cloudy-night" => "partlycloudy",
-        _ => "cloudy",
-    }
-}
-
-fn post_to_ha(client: &Client, ha_url: &str, weather: &Weather) -> Result<()> {
-    for (entity_id, payload) in weather_updates(weather) {
-        let response = client
-            .post(format!("{ha_url}/api/states/{entity_id}"))
-            .json(&payload)
-            .send()
-            .with_context(|| format!("failed to post {entity_id}"))?;
-        ensure_success(response.status(), &entity_id)?;
-    }
-    Ok(())
-}
-
-fn ensure_success(status: StatusCode, entity_id: &str) -> Result<()> {
-    if status.is_success() {
-        Ok(())
-    } else {
-        bail!("Home Assistant rejected {entity_id} update with {status}")
-    }
-}
-
-#[allow(clippy::too_many_lines)]
-fn weather_updates(weather: &Weather) -> Vec<(String, Value)> {
-    let current = &weather.current;
-    let mut updates = vec![
-        sensor(
-            "sensor.van_weather_condition",
-            Some(json!(condition(current.icon.as_deref()))),
-            json!({"friendly_name": "Van Weather Condition"}),
-        ),
-        sensor(
-            "sensor.van_weather_temperature",
-            current.temperature.map(|value| json!(value)),
-            json!({"unit_of_measurement": "°F", "device_class": "temperature"}),
-        ),
-        sensor(
-            "sensor.van_weather_apparent_temperature",
-            current.apparent_temperature.map(|value| json!(value)),
-            json!({"unit_of_measurement": "°F", "device_class": "temperature"}),
-        ),
-        sensor(
-            "sensor.van_weather_humidity",
-            Some(json!(percent(current.humidity))),
-            json!({"unit_of_measurement": "%", "device_class": "humidity"}),
-        ),
-        sensor(
-            "sensor.van_weather_pressure",
-            current.pressure.map(|value| json!(value)),
-            json!({"unit_of_measurement": "mbar", "device_class": "pressure"}),
-        ),
-        sensor(
-            "sensor.van_weather_wind_speed",
-            current.wind_speed.map(|value| json!(value)),
-            json!({"unit_of_measurement": "mph", "device_class": "wind_speed"}),
-        ),
-        sensor(
-            "sensor.van_weather_wind_bearing",
-            current.wind_bearing.map(|value| json!(value)),
-            json!({"unit_of_measurement": "°"}),
-        ),
-        sensor(
-            "sensor.van_weather_visibility",
-            current.visibility.map(|value| json!(value)),
-            json!({"unit_of_measurement": "mi"}),
-        ),
-        sensor(
-            "sensor.van_weather_uv_index",
-            current.uv_index.map(|value| json!(value)),
-            json!({"friendly_name": "Van Weather UV Index", "icon": "mdi:sun-wireless"}),
-        ),
-        sensor(
-            "sensor.van_weather_ozone",
-            current.ozone.map(|value| json!(value)),
-            json!({"unit_of_measurement": "DU", "icon": "mdi:earth"}),
-        ),
-        sensor(
-            "sensor.van_weather_nearest_storm_distance",
-            current.nearest_storm_distance.map(|value| json!(value)),
-            json!({"unit_of_measurement": "mi", "icon": "mdi:weather-lightning"}),
-        ),
-        sensor(
-            "sensor.van_weather_nearest_storm_bearing",
-            current.nearest_storm_bearing.map(|value| json!(value)),
-            json!({"unit_of_measurement": "°", "icon": "mdi:weather-lightning"}),
-        ),
-        sensor(
-            "sensor.van_weather_precip_probability",
-            Some(json!(percent(current.precip_probability))),
-            json!({"unit_of_measurement": "%", "icon": "mdi:weather-rainy"}),
-        ),
-        sensor(
-            "sensor.van_weather_cloud_cover",
-            Some(json!(percent(current.cloud_cover))),
-            json!({"unit_of_measurement": "%", "icon": "mdi:weather-cloudy"}),
-        ),
-    ]
-    .into_iter()
-    .flatten()
-    .collect::<Vec<_>>();
-
-    let daily = weather
-        .daily
-        .iter()
-        .map(|forecast| {
-            json!({
-                "datetime": forecast.datetime.to_rfc3339(),
-                "condition": forecast.condition,
-                "temperature": forecast.temperature,
-                "templow": forecast.templow,
-                "precipitation_probability": percent(forecast.precipitation_probability),
-            })
-        })
-        .collect::<Vec<_>>();
-    updates.push((
-        "sensor.van_weather_forecast_daily".to_owned(),
-        json!({"state": daily.len(), "attributes": {"forecast": daily}}),
-    ));
-
-    let hourly = weather
-        .hourly
-        .iter()
-        .map(|forecast| {
-            json!({
-                "datetime": forecast.datetime.to_rfc3339(),
-                "condition": forecast.condition,
-                "temperature": forecast.temperature,
-                "precipitation_probability": percent(forecast.precipitation_probability),
-            })
-        })
-        .collect::<Vec<_>>();
-    updates.push((
-        "sensor.van_weather_forecast_hourly".to_owned(),
-        json!({"state": hourly.len(), "attributes": {"forecast": hourly}}),
-    ));
-    updates
-}
-
-fn sensor(entity_id: &str, state: Option<Value>, attributes: Value) -> Option<(String, Value)> {
-    state.map(|state| {
-        let mut payload = serde_json::Map::new();
-        payload.insert("state".to_owned(), state);
-        payload.insert("attributes".to_owned(), attributes);
-        (entity_id.to_owned(), Value::Object(payload))
-    })
-}
-
-#[allow(clippy::cast_possible_truncation)]
-fn percent(value: Option<f64>) -> i64 {
-    // Preserve Python's int(probability * 100) behavior for Home Assistant.
-    (value.unwrap_or_default() * 100.0) as i64
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn maps_conditions_for_home_assistant() {
-        assert_eq!(condition(Some("clear-day")), "sunny");
-        assert_eq!(condition(Some("sleet")), "snowy-rainy");
-        assert_eq!(condition(Some("partly-cloudy-night")), "partlycloudy");
-        assert_eq!(condition(Some("unexpected")), "cloudy");
-        assert_eq!(condition(None), "cloudy");
-    }
-
-    #[test]
-    fn parses_api_response_and_builds_compatible_payloads() {
-        let response: ApiResponse = serde_json::from_value(json!({
-            "currently": {
-                "temperature": 72.5,
-                "humidity": 0.67,
-                "icon": "clear-day",
-                "precipProbability": 0.129,
-                "cloudCover": 0.4,
-                "summary": "Fine"
-            },
-            "daily": {"data": [{
-                "time": 1_700_000_000,
-                "icon": "rain",
-                "temperatureHigh": 75.0,
-                "temperatureLow": 52.0,
-                "precipProbability": 0.8
-            }]},
-            "hourly": {"data": [{
-                "time": 1_700_000_000,
-                "icon": "fog",
-                "temperature": 61.0,
-                "precipProbability": 0.05
-            }]}
-        }))
-        .unwrap();
-
-        let weather = parse_weather(response);
-        let updates = weather_updates(&weather);
-        let find = |id: &str| updates.iter().find(|(entity, _)| entity == id).unwrap();
-
-        assert_eq!(find("sensor.van_weather_condition").1["state"], "sunny");
-        assert_eq!(find("sensor.van_weather_humidity").1["state"], 67);
-        assert_eq!(find("sensor.van_weather_precip_probability").1["state"], 12);
-        assert_eq!(find("sensor.van_weather_forecast_daily").1["state"], 1);
-        assert_eq!(
-            find("sensor.van_weather_forecast_daily").1["attributes"]["forecast"][0]["condition"],
-            "rainy"
-        );
-        assert_eq!(find("sensor.van_weather_forecast_hourly").1["state"], 1);
-    }
-
-    #[test]
-    fn omits_missing_optional_current_sensors_but_keeps_percentage_sensors() {
-        let weather = parse_weather(ApiResponse::default());
-        let updates = weather_updates(&weather);
-
-        assert!(
-            !updates
-                .iter()
-                .any(|(id, _)| id == "sensor.van_weather_temperature")
-        );
-        assert_eq!(
-            updates
-                .iter()
-                .find(|(id, _)| id == "sensor.van_weather_humidity")
-                .unwrap()
-                .1["state"],
-            0
-        );
-    }
-
-    #[test]
-    fn masks_coordinates_to_about_eleven_kilometres() {
-        assert!((mask_coordinate(37.7749) - 37.8).abs() < f64::EPSILON);
-        assert!((mask_coordinate(-122.4194) - (-122.4)).abs() < f64::EPSILON);
-    }
-}

systems/brain/services/home_assistant.nix

@@ -1,7 +1,5 @@
 { pkgs, ... }:
 {
-  networking.firewall.allowedTCPPorts = [ 8123 ];
-
   users = {
     users.hass = {
       isSystemUser = true;
@@ -13,7 +11,9 @@
   services = {
     home-assistant = {
       enable = true;
+      openFirewall = true;
       config = {
+        http.server_port = 8123;
         homeassistant = {
           time_zone = "America/New_York";
           unit_system = "us_customary";
@@ -73,11 +73,10 @@
           uiprotect # for ubiquiti integration
           unifi-discovery # for ubiquiti integration
           jsonpath # for rest sensors
-          monarchmoneycommunity # for monarch
+          typedmonarchmoney # for monarch
         ];
       extraComponents = [ "isal" ];
       customComponents = with pkgs.home-assistant-custom-components; [
-        garmin_connect
         pirate-weather
       ];
 

systems/brain/services/van_weather.nix

@@ -1,10 +1,8 @@
 {
   pkgs,
+  inputs,
   ...
 }:
-let
-  van-weather = pkgs.callPackage ../../../rust/van_weather/package.nix { };
-in
 {
   systemd.services.van-weather = {
     description = "Van Weather Service";
@@ -15,9 +13,13 @@ in
     requires = [ "home-assistant.service" ];
     wantedBy = [ "multi-user.target" ];
 
+    environment = {
+      PYTHONPATH = "${inputs.self}/";
+    };
+
     serviceConfig = {
       Type = "simple";
-      ExecStart = "${van-weather}/bin/van-weather";
+      ExecStart = "${pkgs.my_python}/bin/python -m python.van_weather.main";
       EnvironmentFile = "/etc/van_weather.env";
       Restart = "on-failure";
       RestartSec = "5s";
@@ -27,6 +29,7 @@ in
       ProtectSystem = "strict";
       ProtectHome = "read-only";
       PrivateTmp = true;
+      ReadOnlyPaths = [ "${inputs.self}" ];
     };
   };
 }

systems/jeeves/datasets.nix

@@ -1,362 +0,0 @@
-# Dataset declarations for jeeves, kept as plain data rather than inside
-# zfs.nix so the dataset tree stays separate from the NixOS service wiring.
-# Datasets are nested the way zfs nests them: a pool holds datasets, which can
-# hold datasets of their own. The tree is flattened into "pool/parent/child"
-# names below, which is what zfs and services.zfs_manager work in.
-#
-# Consumed by ./zfs.nix, which feeds it to services.zfs_manager.
-let
-  # Every pool on jeeves was created with the same -O options.
-  poolDefaults = mountpoint: {
-    inherit mountpoint;
-    acltype = "posix"; # zfs reports posixacl back as posix
-    atime = "off";
-    compression = "zstd";
-    dnodesize = "auto";
-    xattr = "sa";
-  };
-
-  zfsKey = "file:///root/zfs.key";
-
-  # What a dataset gets when it is not called out below, kept identical to the
-  # "default" table so the datasets that used to fall through are unchanged.
-  standard = {
-    "15_min" = 8;
-    hourly = 24;
-  };
-
-  disabledSnapshots = {
-    "15_min" = 0;
-    hourly = 0;
-    daily = 0;
-    monthly = 0;
-  };
-
-  pools = {
-    # root_pool: retention only, its properties are not managed yet.
-    root_pool = {
-      manageProperties = false;
-      datasets = {
-        home = {
-          manageProperties = false;
-          snapshots = {
-            "15_min" = 8;
-            hourly = 24;
-            daily = 14;
-          };
-        };
-        root = {
-          manageProperties = false;
-          snapshots = standard;
-        };
-        nix = {
-          manageProperties = false;
-          snapshots."15_min" = 4;
-        };
-        var = {
-          manageProperties = false;
-          snapshots = {
-            "15_min" = 8;
-            hourly = 24;
-            daily = 30;
-            monthly = 6;
-          };
-        };
-      };
-    };
-
-    media = {
-      properties = poolDefaults "/zfs/media";
-      datasets = {
-        temp = {
-          properties = {
-            redundant_metadata = "none";
-            sync = "disabled";
-          };
-          snapshots."15_min" = 2;
-        };
-        secure = {
-          # An encryption root provisioned by scripts/zfs.sh. Its create-only
-          # properties are declared for verification, but zfs_manager must
-          # never create it automatically.
-          createIfMissing = true;
-          properties = {
-            encryption = "aes-256-gcm";
-            keyformat = "hex";
-            keylocation = zfsKey;
-          };
-          snapshots = disabledSnapshots;
-          datasets = {
-            docker = {
-              properties = {
-                mountpoint = "/zfs/media/docker";
-                compression = "zstd-9";
-              };
-              snapshots = {
-                "15_min" = 3;
-                hourly = 12;
-                daily = 14;
-                monthly = 2;
-              };
-            };
-            "github-runners" = {
-              properties = {
-                mountpoint = "/zfs/media/github-runners";
-                compression = "zstd-9";
-                sync = "disabled";
-              };
-              snapshots = {
-                "15_min" = 6;
-                hourly = 2;
-                daily = 1;
-              };
-            };
-            home_assistant = {
-              properties = {
-                mountpoint = "/zfs/media/home_assistant";
-                compression = "zstd-19";
-              };
-              snapshots = standard;
-            };
-            important = {
-              properties = {
-                compression = "zstd-9";
-                copies = "2";
-              };
-              snapshots = standard;
-            };
-            notes = {
-              properties = {
-                mountpoint = "/zfs/media/notes";
-                copies = "2";
-              };
-              snapshots = {
-                "15_min" = 8;
-                hourly = 24;
-                daily = 30;
-                monthly = 12;
-              };
-            };
-            postgres = {
-              properties = {
-                mountpoint = "/zfs/media/database/postgres";
-                primarycache = "metadata";
-                recordsize = "16K";
-              };
-              snapshots = {
-                "15_min" = 8;
-                hourly = 24;
-                daily = 7;
-              };
-            };
-            "postgres-wal" = {
-              properties = {
-                compression = "lz4";
-                logbias = "latency";
-                mountpoint = "/zfs/media/database/postgres-wal";
-                primarycache = "metadata";
-                recordsize = "32K";
-                secondarycache = "none";
-                special_small_blocks = "32K";
-              };
-              snapshots = {
-                "15_min" = 4;
-                hourly = 2;
-              };
-            };
-            prometheus = {
-              properties = {
-                mountpoint = "/zfs/media/database/prometheus";
-                compression = "lz4";
-              };
-              snapshots = standard;
-            };
-            services = {
-              properties = {
-                mountpoint = "/zfs/media/services";
-                compression = "zstd-9";
-              };
-              snapshots = standard;
-            };
-            share = {
-              properties = {
-                mountpoint = "/zfs/media/share";
-                exec = "off";
-              };
-              snapshots."15_min" = 4;
-            };
-          };
-        };
-      };
-    };
-
-    storage = {
-      properties = poolDefaults "/zfs/storage";
-      datasets = {
-        nomad = {
-          properties = {
-            mountpoint = "/zfs/storage/nomad";
-            compression = "zstd-9";
-          };
-          snapshots = standard;
-        };
-        ollama = {
-          properties = {
-            compression = "zstd-19";
-            recordsize = "1M";
-            sync = "disabled";
-          };
-          snapshots."15_min" = 2;
-        };
-        secure = {
-          # An encryption root provisioned by scripts/zfs.sh. Its create-only
-          # properties are declared for verification, but zfs_manager must
-          # never create it automatically.
-          createIfMissing = false;
-          properties = {
-            encryption = "aes-256-gcm";
-            keyformat = "hex";
-            keylocation = zfsKey;
-          };
-          snapshots = disabledSnapshots;
-          datasets = {
-            archive = {
-              properties = {
-                compression = "zstd-19";
-                mountpoint = "/zfs/storage/archive";
-                recordsize = "1M";
-              };
-              snapshots = standard;
-            };
-            important = {
-              properties = {
-                compression = "zstd-19";
-                copies = "2";
-                mountpoint = "/zfs/storage/important";
-              };
-              snapshots = standard;
-            };
-            library = {
-              properties = {
-                compression = "zstd-19";
-                mountpoint = "/zfs/storage/library";
-                recordsize = "1M";
-              };
-              snapshots = standard;
-            };
-            main = {
-              properties = {
-                compression = "zstd-19";
-                mountpoint = "/zfs/storage/main";
-              };
-              snapshots = standard;
-            };
-            photos = {
-              properties = {
-                compression = "zstd-19";
-                copies = "2";
-                mountpoint = "/zfs/storage/photos";
-                recordsize = "16K";
-              };
-              snapshots = standard;
-            };
-            plex = {
-              properties = {
-                compression = "zstd-19";
-                mountpoint = "/zfs/storage/plex";
-                recordsize = "1M";
-              };
-              snapshots = {
-                "15_min" = 6;
-                hourly = 2;
-                daily = 1;
-              };
-            };
-            secrets = {
-              properties = {
-                compression = "zstd-19";
-                copies = "3";
-                mountpoint = "/zfs/storage/secrets";
-              };
-              snapshots = {
-                "15_min" = 8;
-                hourly = 24;
-                daily = 30;
-                monthly = 12;
-              };
-            };
-            syncthing = {
-              properties = {
-                compression = "zstd-19";
-                mountpoint = "/zfs/storage/syncthing";
-              };
-              snapshots = standard;
-            };
-            transmission = {
-              properties = {
-                compression = "zstd-9";
-                exec = "off";
-                mountpoint = "/zfs/storage/transmission";
-                recordsize = "1M";
-                sync = "disabled";
-              };
-              snapshots."15_min" = 4;
-            };
-          };
-        };
-      };
-    };
-
-    scratch = {
-      properties = poolDefaults "/zfs/scratch" // {
-        encryption = "aes-256-gcm";
-        keyformat = "hex";
-        keylocation = zfsKey;
-      };
-      datasets = {
-        kafka = {
-          properties = {
-            mountpoint = "/zfs/scratch/kafka";
-            recordsize = "1M";
-          };
-          snapshots = standard;
-        };
-        kestra = {
-          properties = {
-            mountpoint = "/zfs/scratch/kestra";
-            sync = "disabled";
-          };
-          snapshots = standard;
-        };
-        transmission = {
-          properties = {
-            mountpoint = "/zfs/scratch/transmission";
-            recordsize = "16K";
-            sync = "disabled";
-          };
-          snapshots."15_min" = 2;
-        };
-        uv_cache = {
-          properties.mountpoint = "/zfs/scratch/uv_cache";
-          snapshots."15_min" = 2;
-        };
-      };
-    };
-  };
-
-  # Collapse the tree into the flat "pool/parent/child" names zfs uses. Each
-  # node keeps everything except its children.
-  flatten =
-    name: node:
-    builtins.foldl' (result: child: result // flatten "${name}/${child}" node.datasets.${child}) {
-      ${name} = builtins.removeAttrs node [ "datasets" ];
-    } (builtins.attrNames (node.datasets or { }));
-
-  datasets = builtins.foldl' (result: pool: result // flatten pool pools.${pool}) { } (
-    builtins.attrNames pools
-  );
-in
-{
-  inherit datasets;
-  defaultSnapshots = standard;
-}

systems/jeeves/default.nix

@@ -1,4 +1,7 @@
 { inputs, ... }:
+let
+  vars = import ./vars.nix;
+in
 {
   imports = [
     "${inputs.self}/users/dov"
@@ -12,7 +15,6 @@
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/update.nix"
     "${inputs.self}/common/optional/zerotier.nix"
-    "${inputs.self}/common/optional/zfs_manager.nix"
     ./monitoring
     ./docker
     ./services
@@ -22,7 +24,6 @@
     ./programs.nix
     ./runners
     ./syncthing.nix
-    ./zfs.nix
   ];
 
   services = {
@@ -30,6 +31,11 @@
 
     smartd.enable = true;
 
+    snapshot_manager = {
+      path = ./snapshot_config.toml;
+      EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
+    };
+
     zerotierone.joinNetworks = [ "a09acf02330d37b9" ];
   };
 

systems/jeeves/scripts/zfs.sh

@@ -1,10 +1,5 @@
 #!/bin/bash
 
-# Pool and vdev creation only. This is run by hand once per pool.
-#
-# Datasets and their properties are declared in systems/jeeves/zfs.nix and
-# reconciled by the zfs_manager service. Do not add zfs create lines here.
-
 # zpools
 
 # media
@@ -17,10 +12,35 @@ sudo zpool add storage -o ashift=12 special mirror
 sudo zpool add storage -o ashift=12 logs mirror
 
 # scratch
-sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnodesize=auto -O xattr=sa -O compression=zstd -O encryption=aes-256-gcm -O keyformat=hex -O keylocation=file:///root/zfs.key -m /zfs/scratch
+sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnodesize=auto -O xattr=sa -O compression=zstd -O encryption=aes-256-gcm -O keyformat=hex -O keylocation=file:///key -m /zfs/scratch
 
-# The two encrypted parent datasets have to exist before zfs_manager can create
-# anything under them, since encryption cannot be set after creation.
-# These will be removed if/when the media and storage pools are encrypted in the future.
+# media datasets
+sudo zfs create media/temp -o sync=disabled -o redundant_metadata=none
 sudo zfs create media/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key
+sudo zfs create media/secure/docker -o compression=zstd-9
+sudo zfs create media/secure/github-runners -o compression=zstd-9 -o sync=disabled
+sudo zfs create media/secure/home_assistant -o compression=zstd-19
+sudo zfs create media/secure/notes -o copies=2
+sudo zfs create media/secure/postgres -o mountpoint=/zfs/media/database/postgres -o recordsize=16k -o primarycache=metadata
+sudo zfs create media/secure/postgres-wal -o mountpoint=/zfs/media/database/postgres-wal -o recordsize=32k -o primarycache=metadata -o special_small_blocks=32K -o compression=lz4 -o secondarycache=none -o logbias=latency
+sudo zfs create media/secure/prometheus -o mountpoint=/zfs/media/database/prometheus -o compression=lz4
+sudo zfs create media/secure/services -o compression=zstd-9
+sudo zfs create media/secure/share -o mountpoint=/zfs/media/share -o exec=off
+
+# scratch datasets
+sudo zfs create scratch/kafka -o mountpoint=/zfs/scratch/kafka -o recordsize=1M
+sudo zfs create scratch/transmission -o mountpoint=/zfs/scratch/transmission -o recordsize=16k -o sync=disabled -o redundant_metadata=none
+sudo zfs create scratch/uv_cache -o mountpoint=/zfs/scratch/uv_cache
+
+# storage datasets
+sudo zfs create storage/ollama -o recordsize=1M -o compression=zstd-19 -o sync=disabled
 sudo zfs create storage/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key
+sudo zfs create storage/secure/archive -o recordsize=1M -o compression=zstd-19
+sudo zfs create storage/secure/library -o recordsize=1M -o compression=zstd-19
+sudo zfs create storage/secure/main -o compression=zstd-19
+sudo zfs create storage/secure/photos -o recordsize=16K -o compression=zstd-19 -o copies=2
+sudo zfs create storage/secure/plex -o recordsize=1M -o compression=zstd-19
+sudo zfs create storage/secure/secrets -o compression=zstd-19 -o copies=3
+sudo zfs create storage/secure/syncthing -o compression=zstd-19
+sudo zfs create storage/secure/transmission -o recordsize=1M -o compression=zstd-9 -o exec=off -o sync=disabled
+sudo zfs create storage/secure/important -o compression=zstd-19 -o copies=2 -o mountpoint=/zfs/storage/important

systems/jeeves/services/home_assistant.nix

@@ -0,0 +1,79 @@
+let
+  vars = import ../vars.nix;
+in
+{
+  users = {
+    users.hass = {
+      isSystemUser = true;
+      group = "hass";
+    };
+    groups.hass = { };
+  };
+
+  services = {
+    home-assistant = {
+      enable = true;
+      openFirewall = true;
+      configDir = vars.home_assistant;
+      config = {
+        http = {
+          server_port = 8123;
+          use_x_forwarded_for = true;
+          trusted_proxies = "127.0.0.1";
+        };
+        homeassistant = {
+          time_zone = "America/New_York";
+          unit_system = "us_customary";
+          temperature_unit = "F";
+        };
+        recorder = {
+          db_url = "postgresql://@/hass";
+          auto_purge = true;
+          purge_keep_days = 3650;
+          db_retry_wait = 15;
+        };
+        assist_pipeline = { };
+        backup = { };
+        bluetooth = { };
+        config = { };
+        dhcp = { };
+        energy = { };
+        history = { };
+        homeassistant_alerts = { };
+        image_upload = { };
+        logbook = { };
+        media_source = { };
+        mobile_app = { };
+        ssdp = { };
+        sun = { };
+        webhook = { };
+        zeroconf = { };
+        automation = "!include automations.yaml";
+        script = "!include scripts.yaml";
+        scene = "!include scenes.yaml";
+        group = "!include groups.yaml";
+      };
+      extraPackages =
+        python3Packages: with python3Packages; [
+          aioesphomeapi
+          aiounifi
+          bleak-esphome
+          esphome-dashboard-api
+          gtts
+          jellyfin-apiclient-python
+          psycopg2
+          pymetno
+          aio-ownet
+          rokuecp
+          uiprotect
+          wakeonlan
+        ];
+      extraComponents = [ "isal" ];
+    };
+    esphome = {
+      enable = true;
+      openFirewall = true;
+      address = "192.168.90.40";
+    };
+  };
+}

systems/jeeves/services/postgress.nix

@@ -33,6 +33,7 @@ in
 
 
       #type database DBuser origin-address auth-method
+      local hass     hass     trust
       local gitea    gitea    trust
 
       # signalbot
@@ -56,6 +57,7 @@ in
         superuser_map      postgres  postgres
         # Let other names login as themselves
         superuser_map      richie    postgres
+        superuser_map      hass      hass
     '';
     ensureUsers = [
       {
@@ -79,6 +81,16 @@ in
           replication = true;
         };
       }
+      {
+        name = "hass";
+        ensureDBOwnership = true;
+        ensureClauses = {
+          login = true;
+          createrole = true;
+          createdb = true;
+          replication = true;
+        };
+      }
       {
         name = "gitea";
         ensureDBOwnership = true;
@@ -109,6 +121,7 @@ in
     ];
     ensureDatabases = [
       "data_science_dev"
+      "hass"
       "gitea"
       "math"
       "n8n"

systems/jeeves/services/validate_system.toml

@@ -3,5 +3,6 @@ services = [
     "audiobookshelf",
     "haproxy",
     "docker",
+    "home-assistant",
     "jellyfin",
 ]

systems/jeeves/snapshot_config.toml

@@ -0,0 +1,129 @@
+["default"]
+15_min = 8
+hourly = 24
+daily = 0
+monthly = 0
+
+# root_pool
+["root_pool/home"]
+15_min = 8
+hourly = 24
+daily = 14
+monthly = 0
+
+["root_pool/root"]
+15_min = 8
+hourly = 24
+daily = 0
+monthly = 0
+
+["root_pool/nix"]
+15_min = 4
+hourly = 0
+daily = 0
+monthly = 0
+
+["root_pool/var"]
+15_min = 8
+hourly = 24
+daily = 30
+monthly = 6
+# storage
+["storage/ollama"]
+15_min = 2
+hourly = 0
+daily = 0
+monthly = 0
+
+["storage/secure"]
+15_min = 0
+hourly = 0
+daily = 0
+monthly = 0
+
+["storage/secure/plex"]
+15_min = 6
+hourly = 2
+daily = 1
+monthly = 0
+
+["storage/secure/transmission"]
+15_min = 4
+hourly = 0
+daily = 0
+monthly = 0
+
+["storage/secure/secrets"]
+15_min = 8
+hourly = 24
+daily = 30
+monthly = 12
+
+# media
+["media/temp"]
+15_min = 2
+hourly = 0
+daily = 0
+monthly = 0
+
+["media/secure"]
+15_min = 0
+hourly = 0
+daily = 0
+monthly = 0
+
+["media/secure/plex"]
+15_min = 6
+hourly = 2
+daily = 1
+monthly = 0
+
+["media/secure/postgres-wal"]
+15_min = 4
+hourly = 2
+daily = 0
+monthly = 0
+
+
+["media/secure/postgres"]
+15_min = 8
+hourly = 24
+daily = 7
+monthly = 0
+
+["media/secure/share"]
+15_min = 4
+hourly = 0
+daily = 0
+monthly = 0
+
+["media/secure/github-runners"]
+15_min = 6
+hourly = 2
+daily = 1
+monthly = 0
+
+["media/secure/notes"]
+15_min = 8
+hourly = 24
+daily = 30
+monthly = 12
+
+["media/secure/docker"]
+15_min = 3
+hourly = 12
+daily = 14
+monthly = 2
+
+# scratch
+["scratch/transmission"]
+15_min = 2
+hourly = 0
+daily = 0
+monthly = 0
+
+["scratch/uv_cache"]
+15_min = 2
+hourly = 0
+daily = 0
+monthly = 0

systems/jeeves/vars.nix

@@ -8,6 +8,7 @@ in
   database = "${zfs_media}/database";
   docker = "${zfs_media}/docker";
   docker_configs = "${zfs_media}/docker/configs";
+  home_assistant = "${zfs_media}/home_assistant";
   notes = "${zfs_media}/notes";
   secrets = "${zfs_storage}/secrets";
   services = "${zfs_media}/services";

systems/jeeves/zfs.nix

@@ -1,20 +0,0 @@
-{ inputs, ... }:
-let
-  vars = import ./vars.nix;
-  jeeves_zfs = import ./datasets.nix;
-in
-{
-  services = {
-    zfs_manager = {
-      enable = true;
-      PYTHONPATH = "${inputs.self}/";
-      EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
-
-      inherit (jeeves_zfs) datasets defaultSnapshots;
-    };
-
-    # Its retention config is generated from ./datasets.nix by
-    # common/optional/zfs_manager.nix, so only the credentials are set here.
-    snapshot_manager.EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
-  };
-}

tests/test_snapshot_manager.py

@@ -42,7 +42,7 @@ def test_main(mocker: MockerFixture, fs: FakeFilesystem) -> None:
     mock_dataset.create_snapshot.return_value = "snapshot created"
     mock_get_datasets = mocker.patch(f"{SNAPSHOT_MANAGER}.get_datasets", return_value=(mock_dataset,))
 
-    mock_get_snapshots_to_delete = mocker.patch(f"{SNAPSHOT_MANAGER}.get_snapshots_to_delete", return_value=[])
+    mock_get_snapshots_to_delete = mocker.patch(f"{SNAPSHOT_MANAGER}.get_snapshots_to_delete")
     mock_signal_alert = mocker.patch(f"{SNAPSHOT_MANAGER}.signal_alert")
     mock_snapshot_config_toml = '["default"]\n15_min = 8\nhourly = 24\ndaily = 0\nmonthly = 0\n'
     fs.create_file("/mock_snapshot_config.toml", contents=mock_snapshot_config_toml)
@@ -76,39 +76,13 @@ def test_main_create_snapshot_failure(mocker: MockerFixture, fs: FakeFilesystem)
     mock_signal_alert = mocker.patch(f"{SNAPSHOT_MANAGER}.signal_alert")
     mock_snapshot_config_toml = '["default"]\n15_min = 8\nhourly = 24\ndaily = 0\nmonthly = 0\n'
     fs.create_file("/mock_snapshot_config.toml", contents=mock_snapshot_config_toml)
-    with pytest.raises(SystemExit) as exit_info:
-        main(Path("/mock_snapshot_config.toml"))
+    main(Path("/mock_snapshot_config.toml"))
 
-    assert exit_info.value.code == 1
     mock_signal_alert.assert_called_once_with("test_dataset failed to create snapshot 2023-01-01T00:00:00")
     mock_get_datasets.assert_called_once()
     mock_get_snapshots_to_delete.assert_not_called()
 
 
-def test_main_delete_snapshot_failure(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    """Deletion failures make the service fail after processing the dataset."""
-    load_config_data.cache_clear()
-
-    mocker.patch(f"{SNAPSHOT_MANAGER}.get_time_stamp", return_value="2023-01-01T00:00:00")
-
-    mock_dataset = mocker.MagicMock(spec=Dataset)
-    mock_dataset.name = "test_dataset"
-    mock_dataset.create_snapshot.return_value = "snapshot created"
-    mocker.patch(f"{SNAPSHOT_MANAGER}.get_datasets", return_value=(mock_dataset,))
-    mocker.patch(
-        f"{SNAPSHOT_MANAGER}.get_snapshots_to_delete",
-        return_value=["test_dataset@auto_202301010000 failed to delete: busy"],
-    )
-    mocker.patch(f"{SNAPSHOT_MANAGER}.signal_alert")
-    mock_snapshot_config_toml = '["default"]\n15_min = 8\nhourly = 24\ndaily = 0\nmonthly = 0\n'
-    fs.create_file("/mock_snapshot_config.toml", contents=mock_snapshot_config_toml)
-
-    with pytest.raises(SystemExit) as exit_info:
-        main(Path("/mock_snapshot_config.toml"))
-
-    assert exit_info.value.code == 1
-
-
 def test_main_exception(mocker: MockerFixture, fs: FakeFilesystem) -> None:
     """Test main."""
     load_config_data.cache_clear()
@@ -167,18 +141,6 @@ def test_get_snapshots_to_delete_no_snapshot(mocker: MockerFixture) -> None:
     mock_dataset.delete_snapshot.assert_not_called()
 
 
-@pytest.mark.parametrize("invalid_count", [-1, "1", None, True])
-def test_invalid_retention_is_rejected_before_reading_snapshots(mocker: MockerFixture, invalid_count: object) -> None:
-    """Invalid standalone TOML values must never reach deletion logic."""
-    mock_dataset = mocker.MagicMock(spec=Dataset)
-    count_lookup = {"15_min": invalid_count, "hourly": 0, "daily": 0, "monthly": 0}
-
-    with pytest.raises(ValueError, match="15_min retention must be a non-negative integer"):
-        get_snapshots_to_delete(mock_dataset, count_lookup)  # type: ignore[arg-type]
-
-    mock_dataset.get_snapshots.assert_not_called()
-
-
 def test_get_snapshots_to_delete_errored(mocker: MockerFixture) -> None:
     """test_get_snapshots_to_delete_errored."""
     mock_snapshot_0 = create_mock_snapshot(mocker, "auto_202509150415")
@@ -191,12 +153,8 @@ def test_get_snapshots_to_delete_errored(mocker: MockerFixture) -> None:
 
     mock_signal_alert = mocker.patch(f"{SNAPSHOT_MANAGER}.signal_alert")
 
-    failures = get_snapshots_to_delete(
-        mock_dataset,
-        {"15_min": 1, "hourly": 0, "daily": 0, "monthly": 0},
-    )
+    get_snapshots_to_delete(mock_dataset, {"15_min": 1, "hourly": 0, "daily": 0, "monthly": 0})
 
-    assert failures == ["test_dataset@auto_202509150415 failed to delete: snapshot has dependent clones"]
     mock_signal_alert.assert_called_once_with(
         "test_dataset@auto_202509150415 failed to delete: snapshot has dependent clones"
     )

tests/test_zfs_manager.py

@@ -1,382 +0,0 @@
-"""test_zfs_manager."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-from typing import TYPE_CHECKING
-
-import pytest
-
-from python.tools.zfs_manager import main, parse_size, values_match
-
-if TYPE_CHECKING:
-    from pyfakefs.fake_filesystem import FakeFilesystem
-    from pytest_mock import MockerFixture
-
-ZFS_MANAGER = "python.tools.zfs_manager"
-CONFIG_PATH = "/mock_zfs_datasets.json"
-
-
-def write_config(
-    fs: FakeFilesystem,
-    datasets: dict[str, dict[str, str]],
-    unmanaged: list[str] | None = None,
-    never_create: list[str] | None = None,
-) -> Path:
-    """Write a dataset declaration to the fake filesystem."""
-    contents = {
-        "datasets": {
-            name: {
-                "manageProperties": True,
-                "createIfMissing": name not in (never_create or []),
-                "properties": props,
-            }
-            for name, props in datasets.items()
-        }
-        | {name: {"manageProperties": False, "createIfMissing": True, "properties": {}} for name in unmanaged or []},
-    }
-    fs.create_file(CONFIG_PATH, contents=json.dumps(contents))
-    return Path(CONFIG_PATH)
-
-
-def patch_zfs(
-    mocker: MockerFixture,
-    existing: list[str],
-    properties: dict[str, dict[str, tuple[str, str]]] | None = None,
-) -> dict[str, object]:
-    """Patch every zfs call zfs_manager makes."""
-    return {
-        "list": mocker.patch(f"{ZFS_MANAGER}.list_dataset_names", return_value=existing),
-        "get": mocker.patch(f"{ZFS_MANAGER}.get_properties", side_effect=lambda name: (properties or {}).get(name, {})),
-        "create": mocker.patch(f"{ZFS_MANAGER}.create_dataset", return_value=None),
-        "set": mocker.patch(f"{ZFS_MANAGER}.set_property", return_value=None),
-        "alert": mocker.patch(f"{ZFS_MANAGER}.signal_alert"),
-    }
-
-
-def test_creates_missing_dataset(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    zfs = patch_zfs(mocker, existing=["media", "media/secure"])
-    config = write_config(fs, {"media/secure/new": {"compression": "zstd-9"}})
-
-    main(config)
-
-    zfs["create"].assert_called_once_with("media/secure/new", {"compression": "zstd-9"})
-    zfs["set"].assert_not_called()
-    zfs["alert"].assert_not_called()
-
-
-def test_sets_drifted_property(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    zfs = patch_zfs(
-        mocker,
-        existing=["media", "media/temp"],
-        properties={"media/temp": {"compression": ("zstd", "inherited from media")}},
-    )
-    config = write_config(fs, {"media/temp": {"compression": "zstd-9"}})
-
-    main(config)
-
-    zfs["set"].assert_called_once_with("media/temp", "compression", "zstd-9")
-    zfs["create"].assert_not_called()
-
-
-def test_no_op_when_in_sync(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    zfs = patch_zfs(
-        mocker,
-        existing=["media", "media/temp"],
-        properties={"media/temp": {"sync": ("disabled", "local")}},
-    )
-    config = write_config(fs, {"media/temp": {"sync": "disabled"}})
-
-    main(config)
-
-    zfs["set"].assert_not_called()
-    zfs["create"].assert_not_called()
-    zfs["alert"].assert_not_called()
-
-
-def test_size_property_does_not_churn(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    """zfs get -p reports recordsize in bytes, the declaration uses a suffix."""
-    zfs = patch_zfs(
-        mocker,
-        existing=["media", "media/db"],
-        properties={"media/db": {"recordsize": ("16384", "local"), "special_small_blocks": ("32768", "local")}},
-    )
-    config = write_config(fs, {"media/db": {"recordsize": "16k", "special_small_blocks": "32K"}})
-
-    main(config)
-
-    zfs["set"].assert_not_called()
-
-
-def test_create_only_property_alerts_instead_of_setting(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    zfs = patch_zfs(
-        mocker,
-        existing=["media", "media/secure"],
-        properties={"media/secure": {"encryption": ("aes-256-gcm", "local")}},
-    )
-    config = write_config(fs, {"media/secure": {"encryption": "off"}})
-
-    # Nothing can fix a create-only mismatch at runtime, so it fails the run.
-    with pytest.raises(SystemExit) as exit_info:
-        main(config)
-
-    assert exit_info.value.code == 1
-    zfs["set"].assert_not_called()
-    assert zfs["alert"].call_count == 1
-    assert "can only be set when the dataset is created" in zfs["alert"].call_args.args[0]
-
-
-def test_undeclared_local_property_warns(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    zfs = patch_zfs(
-        mocker,
-        existing=["media", "media/temp"],
-        properties={"media/temp": {"exec": ("off", "local")}},
-    )
-    config = write_config(fs, {"media/temp": {}})
-
-    main(config)
-
-    zfs["alert"].assert_called_once_with("media/temp has exec=off set outside of nix")
-
-
-def test_undeclared_inherited_property_is_silent(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    zfs = patch_zfs(
-        mocker,
-        existing=["media", "media/temp"],
-        properties={
-            "media/temp": {
-                "compression": ("zstd", "inherited from media"),
-                "exec": ("on", "default"),
-                "nixos:shutdown-time": ("whenever", "local"),
-            },
-        },
-    )
-    config = write_config(fs, {"media/temp": {}})
-
-    main(config)
-
-    zfs["alert"].assert_not_called()
-
-
-def test_dry_run_makes_no_changes(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    zfs = patch_zfs(
-        mocker,
-        existing=["media", "media/temp"],
-        properties={"media/temp": {"compression": ("zstd", "local")}},
-    )
-    config = write_config(fs, {"media/temp": {"compression": "zstd-9"}, "media/new": {}})
-
-    main(config, dry_run=True)
-
-    zfs["set"].assert_not_called()
-    zfs["create"].assert_not_called()
-
-
-def test_pool_root_is_never_created(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    zfs = patch_zfs(mocker, existing=[])
-    config = write_config(fs, {"media": {"atime": "off"}})
-
-    with pytest.raises(SystemExit) as exit_info:
-        main(config)
-
-    assert exit_info.value.code == 1
-    zfs["create"].assert_not_called()
-    assert "does not create pools" in zfs["alert"].call_args.args[0]
-
-
-def test_children_skipped_when_parent_creation_fails(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    zfs = patch_zfs(mocker, existing=["media"])
-    zfs["create"].return_value = "Failed to create media/secure: key not loaded"
-    config = write_config(fs, {"media/secure": {}, "media/secure/child": {}})
-
-    with pytest.raises(SystemExit) as exit_info:
-        main(config)
-
-    assert exit_info.value.code == 1
-    zfs["create"].assert_called_once_with("media/secure", {})
-
-
-def test_unmanaged_dataset_properties_are_untouched(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    """A snapshots-only dataset is neither reconciled nor reported as unknown."""
-    zfs = patch_zfs(
-        mocker,
-        existing=["root_pool", "root_pool/var"],
-        properties={"root_pool/var": {"compression": ("lz4", "local")}},
-    )
-    config = write_config(fs, {}, unmanaged=["root_pool", "root_pool/var"])
-
-    main(config)
-
-    zfs["get"].assert_not_called()
-    zfs["set"].assert_not_called()
-    zfs["alert"].assert_not_called()
-
-
-def test_undeclared_dataset_is_left_alone(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    zfs = patch_zfs(mocker, existing=["media", "media/undeclared"])
-    config = write_config(fs, {})
-
-    main(config)
-
-    zfs["create"].assert_not_called()
-    zfs["set"].assert_not_called()
-    zfs["alert"].assert_not_called()
-
-
-def test_main_exception(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    zfs = patch_zfs(mocker, existing=[])
-    zfs["list"].side_effect = Exception("test")
-    config = write_config(fs, {})
-
-    with pytest.raises(SystemExit) as pytest_wrapped_e:
-        main(config)
-
-    assert pytest_wrapped_e.value.code == 1
-    zfs["alert"].assert_called_once_with("zfs_manager failed")
-
-
-@pytest.mark.parametrize(
-    ("value", "expected"),
-    [
-        ("16384", 16384),
-        ("16k", 16384),
-        ("16K", 16384),
-        ("1M", 1048576),
-        ("none", None),
-        ("", None),
-    ],
-)
-def test_parse_size(value, expected) -> None:
-    assert parse_size(value) == expected
-
-
-def test_values_match_falls_back_to_string_for_unparsable_sizes() -> None:
-    assert not values_match("recordsize", "none", "16384")
-    assert values_match("recordsize", "none", "none")
-    assert not values_match("compression", "zstd", "zstd-9")
-
-
-# -- failure handling: every one of these must exit non-zero -------------------
-
-
-def test_dataset_listing_failure_exits_nonzero(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    """A failed zfs list must abort, never be read as an empty system.
-
-    bash_wrapper hands back stderr as though it were output, so without the
-    return code check the reconciler would treat the error text as the dataset
-    list and conclude every declared dataset was missing.
-    """
-    zfs = patch_zfs(mocker, existing=[])
-    zfs["list"].side_effect = RuntimeError("Failed to list ZFS datasets: pool is busy")
-    config = write_config(fs, {"media/temp": {}})
-
-    with pytest.raises(SystemExit) as exit_info:
-        main(config)
-
-    assert exit_info.value.code == 1
-    zfs["create"].assert_not_called()
-    zfs["alert"].assert_called_once_with("zfs_manager failed")
-
-
-def test_create_failure_exits_nonzero(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    zfs = patch_zfs(mocker, existing=["media"])
-    zfs["create"].return_value = "Failed to create media/temp: out of space"
-    config = write_config(fs, {"media/temp": {}})
-
-    with pytest.raises(SystemExit) as exit_info:
-        main(config)
-
-    assert exit_info.value.code == 1
-    assert "out of space" in zfs["alert"].call_args.args[0]
-
-
-def test_set_failure_exits_nonzero(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    zfs = patch_zfs(
-        mocker,
-        existing=["media", "media/temp"],
-        properties={"media/temp": {"compression": ("zstd", "local")}},
-    )
-    zfs["set"].return_value = "Failed to set compression=zstd-9 on media/temp: permission denied"
-    config = write_config(fs, {"media/temp": {"compression": "zstd-9"}})
-
-    with pytest.raises(SystemExit) as exit_info:
-        main(config)
-
-    assert exit_info.value.code == 1
-    assert "permission denied" in zfs["alert"].call_args.args[0]
-
-
-def test_every_dataset_is_checked_before_failing(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    """One broken dataset must not hide the state of the others."""
-    zfs = patch_zfs(
-        mocker,
-        existing=["media", "media/one", "media/two", "media/three"],
-        properties={
-            "media/one": {"compression": ("zstd", "local")},
-            "media/two": {"compression": ("zstd", "local")},
-            "media/three": {"compression": ("zstd", "local")},
-        },
-    )
-    zfs["set"].return_value = "Failed to set compression: permission denied"
-    config = write_config(
-        fs,
-        {name: {"compression": "zstd-9"} for name in ("media/one", "media/two", "media/three")},
-    )
-
-    with pytest.raises(SystemExit) as exit_info:
-        main(config)
-
-    assert exit_info.value.code == 1
-    # All three were attempted and all three were reported, not just the first.
-    assert zfs["set"].call_count == 3
-    assert zfs["alert"].call_count == 3
-
-
-def test_dataset_marked_as_externally_created_is_never_created(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    """An encryption root must be reported as missing, not silently recreated.
-
-    Recreating it from this declaration would produce an unencrypted dataset,
-    since encryption is fixed at creation and is not declared here.
-    """
-    zfs = patch_zfs(mocker, existing=["media"])
-    config = write_config(fs, {"media/secure": {}}, never_create=["media/secure"])
-
-    with pytest.raises(SystemExit) as exit_info:
-        main(config)
-
-    assert exit_info.value.code == 1
-    zfs["create"].assert_not_called()
-    assert "created outside of nix" in zfs["alert"].call_args.args[0]
-
-
-def test_externally_created_dataset_is_still_property_checked(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    """When it does exist, it is reconciled like anything else."""
-    zfs = patch_zfs(
-        mocker,
-        existing=["media", "media/secure"],
-        properties={"media/secure": {"keylocation": ("prompt", "local")}},
-    )
-    config = write_config(
-        fs,
-        {"media/secure": {"keylocation": "file:///root/zfs.key"}},
-        never_create=["media/secure"],
-    )
-
-    main(config)
-
-    zfs["set"].assert_called_once_with("media/secure", "keylocation", "file:///root/zfs.key")
-
-
-def test_success_exits_cleanly(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    """The happy path must not raise SystemExit at all."""
-    zfs = patch_zfs(
-        mocker,
-        existing=["media", "media/temp"],
-        properties={"media/temp": {"sync": ("disabled", "local")}},
-    )
-    config = write_config(fs, {"media/temp": {"sync": "disabled"}})
-
-    main(config)
-
-    zfs["alert"].assert_not_called()

tests/zfs-integration.nix

@@ -1,135 +0,0 @@
-{ self }:
-{
-  name = "zfs-integration";
-
-  nodes.machine =
-    { pkgs, ... }:
-    let
-      testPython = pkgs.python314.withPackages (pythonPackages: [
-        pythonPackages.apprise
-        pythonPackages.typer
-      ]);
-    in
-    {
-      imports = [
-        ../common/global/snapshot_manager.nix
-        ../common/optional/zfs_manager.nix
-      ];
-
-      boot.supportedFilesystems = [ "zfs" ];
-      networking.hostId = "deadbeef";
-
-      virtualisation = {
-        emptyDiskImages = [ 2048 ];
-        memorySize = 2048;
-      };
-
-      environment.systemPackages = [
-        testPython
-        pkgs.zfs
-      ];
-
-      services = {
-        snapshot_manager = {
-          enable = true;
-          package = testPython;
-          PYTHONPATH = "${self}/";
-        };
-
-        zfs_manager = {
-          enable = true;
-          package = testPython;
-          PYTHONPATH = "${self}/";
-          defaultSnapshots = {
-            "15_min" = 2;
-            hourly = 2;
-            daily = 2;
-            monthly = 2;
-          };
-          datasets = {
-            testpool = {
-              properties = {
-                atime = "off";
-                compression = "lz4";
-                mountpoint = "/testpool";
-              };
-            };
-
-            "testpool/parent" = {
-              properties = {
-                compression = "zstd";
-                mountpoint = "/testpool/parent";
-              };
-            };
-
-            "testpool/parent/child" = {
-              properties = {
-                recordsize = "16K";
-                sync = "disabled";
-              };
-              snapshots = {
-                "15_min" = 1;
-                hourly = 1;
-                daily = 1;
-                monthly = 1;
-              };
-            };
-
-            "testpool/secure" = {
-              createIfMissing = false;
-              properties = {
-                encryption = "aes-256-gcm";
-                keyformat = "hex";
-                keylocation = "file:///root/zfs.key";
-              };
-              snapshots = {
-                "15_min" = 0;
-                hourly = 0;
-                daily = 0;
-                monthly = 0;
-              };
-            };
-
-            "testpool/secure/child" = {
-              properties.compression = "zstd-9";
-            };
-          };
-        };
-      };
-
-      systemd.services = {
-        prepare-zfs-integration = {
-          description = "Prepare the ZFS integration-test pool";
-          requiredBy = [ "zfs_manager.service" ];
-          before = [ "zfs_manager.service" ];
-          path = [ pkgs.zfs ];
-          serviceConfig = {
-            Type = "oneshot";
-            RemainAfterExit = true;
-          };
-          script = ''
-            printf '%064d\n' 0 > /root/zfs.key
-            chmod 0400 /root/zfs.key
-            zpool create -f -m /testpool testpool /dev/vdb
-            zfs create \
-              -o encryption=aes-256-gcm \
-              -o keyformat=hex \
-              -o keylocation=file:///root/zfs.key \
-              testpool/secure
-          '';
-        };
-
-        zfs_manager = {
-          requires = [ "prepare-zfs-integration.service" ];
-          after = [ "prepare-zfs-integration.service" ];
-        };
-      };
-    };
-
-  testScript =
-    { nodes, ... }:
-    builtins.replaceStrings
-      [ "@snapshot_config@" ]
-      [ (toString nodes.machine.services.snapshot_manager.path) ]
-      (builtins.readFile ./zfs_integration.py);
-}

tests/zfs_integration.py

@@ -1,56 +0,0 @@
-# The NixOS test driver provides these globals at runtime.
-# ruff: noqa: F821
-
-import tomllib
-from pathlib import Path
-
-snapshot_config_path = Path("@snapshot_config@")
-
-machine.start()
-machine.wait_for_unit("multi-user.target")
-machine.wait_for_unit("zfs_manager.service")
-
-with subtest("zfs_manager creates parents before children"):
-    machine.succeed("zfs list testpool/parent")
-    machine.succeed("zfs list testpool/parent/child")
-    machine.succeed("zfs list testpool/secure/child")
-
-with subtest("declared properties are reconciled"):
-    machine.succeed('test "$(zfs get -H -o value atime testpool)" = off')
-    machine.succeed('test "$(zfs get -H -o value compression testpool)" = lz4')
-    machine.succeed('test "$(zfs get -H -o value recordsize testpool/parent/child)" = 16K')
-    machine.succeed('test "$(zfs get -H -o value sync testpool/parent/child)" = disabled')
-
-    machine.succeed("zfs set sync=standard testpool/parent/child")
-    machine.succeed("systemctl restart zfs_manager.service")
-    machine.succeed('test "$(zfs get -H -o value sync testpool/parent/child)" = disabled')
-
-with subtest("externally created encryption roots are verified"):
-    machine.succeed('test "$(zfs get -H -o value encryption testpool/secure)" = aes-256-gcm')
-    machine.succeed('test "$(zfs get -H -o value keyformat testpool/secure)" = hex')
-    machine.succeed('test "$(zfs get -H -o value encryption testpool/secure/child)" = aes-256-gcm')
-
-with subtest("declared datasets inherit default snapshot retention"):
-    with snapshot_config_path.open("rb") as config_file:
-        snapshot_config = tomllib.load(config_file)
-
-    expected_default = {"15_min": 2, "hourly": 2, "daily": 2, "monthly": 2}
-    assert snapshot_config["default"] == expected_default
-    assert snapshot_config["testpool/parent"] == expected_default
-    assert snapshot_config["testpool/secure/child"] == expected_default
-    assert snapshot_config["testpool/secure"] == {
-        "15_min": 0,
-        "hourly": 0,
-        "daily": 0,
-        "monthly": 0,
-    }
-
-with subtest("snapshot deletion failures fail the systemd service"):
-    machine.succeed("zfs snapshot testpool/parent/child@auto_200001010015")
-    machine.succeed("zfs clone testpool/parent/child@auto_200001010015 testpool/dependent-clone")
-    machine.succeed("zfs snapshot testpool/parent/child@auto_200001010030")
-
-    machine.fail("systemctl start snapshot_manager.service")
-    machine.succeed("systemctl is-failed --quiet snapshot_manager.service")
-    machine.succeed("journalctl -u snapshot_manager.service --no-pager | grep -q 'snapshot has dependent clones'")
-    machine.fail("zfs list -H -t snapshot -o name | grep -q '^testpool/secure@auto_'")

users/richie/home/programs.nix

@@ -44,19 +44,12 @@
     # nodejs
     nodejs
     # Rust packages
-    bacon
-    cargo
-    cargo-audit
-    cargo-generate
-    cargo-machete
-    cargo-update
-    cargo-watch
-    clippy
-    rust-analyzer
-    rustc
-    rustfmt
     trunk
     wasm-pack
+    cargo-watch
+    cargo-generate
+    cargo-audit
+    cargo-update
     # cpp
     clang-tools
     clang_20