feat(zfs): add declarative management for ZFS datasets and integration tests

.github/workflows/zfs-integration.yml

@@ -0,0 +1,51 @@
+name: zfs integration
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [main]
+    paths:
+      - ".github/workflows/zfs-integration.yml"
+      - "flake.nix"
+      - "flake.lock"
+      - "overlays/default.nix"
+      - "common/global/snapshot_manager.nix"
+      - "common/optional/zfs_manager.nix"
+      - "python/signal_alert.py"
+      - "python/tools/snapshot_manager.py"
+      - "python/tools/zfs_manager.py"
+      - "python/zfs/**"
+      - "systems/jeeves/datasets.nix"
+      - "systems/jeeves/scripts/zfs.sh"
+      - "systems/jeeves/zfs.nix"
+      - "tests/zfs_integration.py"
+      - "tests/zfs-integration.nix"
+  push:
+    branches: [main]
+    paths:
+      - ".github/workflows/zfs-integration.yml"
+      - "flake.nix"
+      - "flake.lock"
+      - "overlays/default.nix"
+      - "common/global/snapshot_manager.nix"
+      - "common/optional/zfs_manager.nix"
+      - "python/signal_alert.py"
+      - "python/tools/snapshot_manager.py"
+      - "python/tools/zfs_manager.py"
+      - "python/zfs/**"
+      - "systems/jeeves/datasets.nix"
+      - "systems/jeeves/scripts/zfs.sh"
+      - "systems/jeeves/zfs.nix"
+      - "tests/zfs_integration.py"
+      - "tests/zfs-integration.nix"
+
+jobs:
+  zfs-integration:
+    runs-on: self-hosted
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build and run ZFS integration VM
+        run: >-
+          nix build --accept-flake-config --print-build-logs
+          .#packages.x86_64-linux.zfs-integration

common/global/snapshot_manager.nix

@@ -22,6 +22,11 @@ in
           the PYTHONPATH to use for the snapshot_manager service.
         '';
       };
+      package = lib.mkOption {
+        type = lib.types.package;
+        default = pkgs.my_python;
+        description = "Python environment used to run snapshot_manager.";
+      };
       EnvironmentFile = lib.mkOption {
         type = lib.types.nullOr (lib.types.coercedTo lib.types.path toString lib.types.str);
         default = null;
@@ -45,7 +50,7 @@ in
         };
         serviceConfig = {
           Type = "oneshot";
-          ExecStart = "${pkgs.my_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
+          ExecStart = "${cfg.package}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
         }
         // lib.optionalAttrs (cfg.EnvironmentFile != null) {
           EnvironmentFile = cfg.EnvironmentFile;

common/optional/zfs_manager.nix

@@ -142,6 +142,11 @@ in
           the PYTHONPATH to use for the zfs_manager service.
         '';
       };
+      package = lib.mkOption {
+        type = lib.types.package;
+        default = pkgs.my_python;
+        description = "Python environment used to run zfs_manager.";
+      };
       EnvironmentFile = lib.mkOption {
         type = lib.types.nullOr (lib.types.coercedTo lib.types.path toString lib.types.str);
         default = null;
@@ -175,7 +180,7 @@ in
         serviceConfig = {
           Type = "oneshot";
           RemainAfterExit = true;
-          ExecStart = "${pkgs.my_python}/bin/python -m python.tools.zfs_manager ${lib.escapeShellArg datasetConfig}${lib.optionalString cfg.dryRun " --dry-run"}";
+          ExecStart = "${cfg.package}/bin/python -m python.tools.zfs_manager ${lib.escapeShellArg datasetConfig}${lib.optionalString cfg.dryRun " --dry-run"}";
         }
         // lib.optionalAttrs (cfg.EnvironmentFile != null) {
           EnvironmentFile = cfg.EnvironmentFile;

flake.nix

@@ -77,6 +77,7 @@
         }
         // lib.optionalAttrs (pkgs.stdenv.hostPlatform.system == "x86_64-linux") {
           iso = self.nixosConfigurations.iso.config.system.build.isoImage;
+          zfs-integration = pkgs.testers.runNixOSTest (import ./tests/zfs-integration.nix { inherit self; });
         }
       );
       apps = forEachSystem (

tests/zfs-integration.nix

@@ -0,0 +1,135 @@
+{ self }:
+{
+  name = "zfs-integration";
+
+  nodes.machine =
+    { pkgs, ... }:
+    let
+      testPython = pkgs.python314.withPackages (pythonPackages: [
+        pythonPackages.apprise
+        pythonPackages.typer
+      ]);
+    in
+    {
+      imports = [
+        ../common/global/snapshot_manager.nix
+        ../common/optional/zfs_manager.nix
+      ];
+
+      boot.supportedFilesystems = [ "zfs" ];
+      networking.hostId = "deadbeef";
+
+      virtualisation = {
+        emptyDiskImages = [ 2048 ];
+        memorySize = 2048;
+      };
+
+      environment.systemPackages = [
+        testPython
+        pkgs.zfs
+      ];
+
+      services = {
+        snapshot_manager = {
+          enable = true;
+          package = testPython;
+          PYTHONPATH = "${self}/";
+        };
+
+        zfs_manager = {
+          enable = true;
+          package = testPython;
+          PYTHONPATH = "${self}/";
+          defaultSnapshots = {
+            "15_min" = 2;
+            hourly = 2;
+            daily = 2;
+            monthly = 2;
+          };
+          datasets = {
+            testpool = {
+              properties = {
+                atime = "off";
+                compression = "lz4";
+                mountpoint = "/testpool";
+              };
+            };
+
+            "testpool/parent" = {
+              properties = {
+                compression = "zstd";
+                mountpoint = "/testpool/parent";
+              };
+            };
+
+            "testpool/parent/child" = {
+              properties = {
+                recordsize = "16K";
+                sync = "disabled";
+              };
+              snapshots = {
+                "15_min" = 1;
+                hourly = 1;
+                daily = 1;
+                monthly = 1;
+              };
+            };
+
+            "testpool/secure" = {
+              createIfMissing = false;
+              properties = {
+                encryption = "aes-256-gcm";
+                keyformat = "hex";
+                keylocation = "file:///root/zfs.key";
+              };
+              snapshots = {
+                "15_min" = 0;
+                hourly = 0;
+                daily = 0;
+                monthly = 0;
+              };
+            };
+
+            "testpool/secure/child" = {
+              properties.compression = "zstd-9";
+            };
+          };
+        };
+      };
+
+      systemd.services = {
+        prepare-zfs-integration = {
+          description = "Prepare the ZFS integration-test pool";
+          requiredBy = [ "zfs_manager.service" ];
+          before = [ "zfs_manager.service" ];
+          path = [ pkgs.zfs ];
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+          };
+          script = ''
+            printf '%064d\n' 0 > /root/zfs.key
+            chmod 0400 /root/zfs.key
+            zpool create -f -m /testpool testpool /dev/vdb
+            zfs create \
+              -o encryption=aes-256-gcm \
+              -o keyformat=hex \
+              -o keylocation=file:///root/zfs.key \
+              testpool/secure
+          '';
+        };
+
+        zfs_manager = {
+          requires = [ "prepare-zfs-integration.service" ];
+          after = [ "prepare-zfs-integration.service" ];
+        };
+      };
+    };
+
+  testScript =
+    { nodes, ... }:
+    builtins.replaceStrings
+      [ "@snapshot_config@" ]
+      [ (toString nodes.machine.services.snapshot_manager.path) ]
+      (builtins.readFile ./zfs_integration.py);
+}

tests/zfs_integration.py

@@ -0,0 +1,56 @@
+# The NixOS test driver provides these globals at runtime.
+# ruff: noqa: F821
+
+import tomllib
+from pathlib import Path
+
+snapshot_config_path = Path("@snapshot_config@")
+
+machine.start()
+machine.wait_for_unit("multi-user.target")
+machine.wait_for_unit("zfs_manager.service")
+
+with subtest("zfs_manager creates parents before children"):
+    machine.succeed("zfs list testpool/parent")
+    machine.succeed("zfs list testpool/parent/child")
+    machine.succeed("zfs list testpool/secure/child")
+
+with subtest("declared properties are reconciled"):
+    machine.succeed('test "$(zfs get -H -o value atime testpool)" = off')
+    machine.succeed('test "$(zfs get -H -o value compression testpool)" = lz4')
+    machine.succeed('test "$(zfs get -H -o value recordsize testpool/parent/child)" = 16K')
+    machine.succeed('test "$(zfs get -H -o value sync testpool/parent/child)" = disabled')
+
+    machine.succeed("zfs set sync=standard testpool/parent/child")
+    machine.succeed("systemctl restart zfs_manager.service")
+    machine.succeed('test "$(zfs get -H -o value sync testpool/parent/child)" = disabled')
+
+with subtest("externally created encryption roots are verified"):
+    machine.succeed('test "$(zfs get -H -o value encryption testpool/secure)" = aes-256-gcm')
+    machine.succeed('test "$(zfs get -H -o value keyformat testpool/secure)" = hex')
+    machine.succeed('test "$(zfs get -H -o value encryption testpool/secure/child)" = aes-256-gcm')
+
+with subtest("declared datasets inherit default snapshot retention"):
+    with snapshot_config_path.open("rb") as config_file:
+        snapshot_config = tomllib.load(config_file)
+
+    expected_default = {"15_min": 2, "hourly": 2, "daily": 2, "monthly": 2}
+    assert snapshot_config["default"] == expected_default
+    assert snapshot_config["testpool/parent"] == expected_default
+    assert snapshot_config["testpool/secure/child"] == expected_default
+    assert snapshot_config["testpool/secure"] == {
+        "15_min": 0,
+        "hourly": 0,
+        "daily": 0,
+        "monthly": 0,
+    }
+
+with subtest("snapshot deletion failures fail the systemd service"):
+    machine.succeed("zfs snapshot testpool/parent/child@auto_200001010015")
+    machine.succeed("zfs clone testpool/parent/child@auto_200001010015 testpool/dependent-clone")
+    machine.succeed("zfs snapshot testpool/parent/child@auto_200001010030")
+
+    machine.fail("systemctl start snapshot_manager.service")
+    machine.succeed("systemctl is-failed --quiet snapshot_manager.service")
+    machine.succeed("journalctl -u snapshot_manager.service --no-pager | grep -q 'snapshot has dependent clones'")
+    machine.fail("zfs list -H -t snapshot -o name | grep -q '^testpool/secure@auto_'")