feat(zfs): add declarative management for ZFS datasets and integration tests

- Introduced a new `datasets.nix` file to define ZFS datasets and their properties, allowing for centralized management of dataset configurations.
- Updated `default.nix` to import and enable the `zfs_manager` service, integrating it into the system configuration.
- Refactored `zfs.sh` to remove dataset creation commands, delegating dataset management to the `zfs_manager` service.
- Removed the legacy `snapshot_config.toml` file, as snapshot configurations are now handled within `datasets.nix`.
- Modified `vars.nix` to derive mountpoint paths from `datasets.nix`, ensuring consistency across the configuration.
- Created a new `zfs.nix` file to define the `zfs_manager` service and its dependencies.
- Added comprehensive tests for the `zfs_manager` functionality, covering dataset creation, property management, and error handling.

.github/workflows/build_systems.yml

@@ -8,23 +8,8 @@ on:
     - cron: "0 22 * * *"
 
 jobs:
-  prebuild-common:
-    name: prebuild-common-x86-64-v3
-    runs-on: nix-cache-builder
-    steps:
-      - uses: actions/checkout@v4
-      # portal-1 is the smallest system closure: 95% of its derivations are
-      # shared by all five systems, so it is a maintainable common cache seed.
-      # Keep going so one failing package does not stop unrelated cache entries
-      # from being built.
-      - name: Build common packages
-        run: nixos-rebuild build --keep-going --accept-flake-config --flake ./#portal-1
-      - name: Copy common packages to nix-cache
-        run: nix copy --accept-flake-config --to unix:///host-nix/var/nix/daemon-socket/socket .#nixosConfigurations.portal-1.config.system.build.toplevel
-
   build:
     name: build-${{ matrix.system }}
-    needs: prebuild-common
     runs-on: self-hosted
     strategy:
       matrix:
@@ -33,7 +18,6 @@ jobs:
           - "brain"
           - "jeeves"
           - "rhapsody-in-green"
-          - "portal-1"
     continue-on-error: true
     steps:
       - uses: actions/checkout@v4

.github/workflows/merge_flake_lock_update.yml

@@ -15,8 +15,9 @@ jobs:
     steps:
       - name: merge_flake_lock_update
         run: >-
+          nix develop .#devShells.x86_64-linux.default -c
           python -m python.gitea_flake_lock merge
           --repo "${{ github.repository }}"
         env:
-          JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
           GITEA_URL: https://gitea.tmmworkshop.com

.github/workflows/pytest.yml

@@ -0,0 +1,19 @@
+name: pytest
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  pytest:
+    runs-on: self-hosted
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run tests
+        run: nix develop .#devShells.x86_64-linux.default -c pytest tests

.github/workflows/update-flake-lock.yml

@@ -18,8 +18,9 @@ jobs:
         run: nix flake update
       - name: Create or update flake.lock PR
         env:
-          JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
           GITEA_URL: https://gitea.tmmworkshop.com
         run: >-
+          nix develop .#devShells.x86_64-linux.default -c
           python -m python.gitea_flake_lock update
           --repo "${{ github.repository }}"

.github/workflows/zfs-integration.yml

@@ -0,0 +1,51 @@
+name: zfs integration
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [main]
+    paths:
+      - ".github/workflows/zfs-integration.yml"
+      - "flake.nix"
+      - "flake.lock"
+      - "overlays/default.nix"
+      - "common/global/snapshot_manager.nix"
+      - "common/optional/zfs_manager.nix"
+      - "python/signal_alert.py"
+      - "python/tools/snapshot_manager.py"
+      - "python/tools/zfs_manager.py"
+      - "python/zfs/**"
+      - "systems/jeeves/datasets.nix"
+      - "systems/jeeves/scripts/zfs.sh"
+      - "systems/jeeves/zfs.nix"
+      - "tests/zfs_integration.py"
+      - "tests/zfs-integration.nix"
+  push:
+    branches: [main]
+    paths:
+      - ".github/workflows/zfs-integration.yml"
+      - "flake.nix"
+      - "flake.lock"
+      - "overlays/default.nix"
+      - "common/global/snapshot_manager.nix"
+      - "common/optional/zfs_manager.nix"
+      - "python/signal_alert.py"
+      - "python/tools/snapshot_manager.py"
+      - "python/tools/zfs_manager.py"
+      - "python/zfs/**"
+      - "systems/jeeves/datasets.nix"
+      - "systems/jeeves/scripts/zfs.sh"
+      - "systems/jeeves/zfs.nix"
+      - "tests/zfs_integration.py"
+      - "tests/zfs-integration.nix"
+
+jobs:
+  zfs-integration:
+    runs-on: self-hosted
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build and run ZFS integration VM
+        run: >-
+          nix build --accept-flake-config --print-build-logs
+          .#packages.x86_64-linux.zfs-integration

.sops.yaml

@@ -1,9 +1,6 @@
 # Generate AGE keys from SSH keys with:
 #   ssh-keygen -A
 #   nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
-# update keys after addin/removing a key
-#   nix-shell -p sops --run "sops updatekeys users/secrets.yaml" users/secrets.yaml
-
 keys:
   - &admin_richie age1u8zj599elqqvcmhxn8zuwrufsz8w8w366d3ayrljjejljt2q45kq8mxw9c # cspell:disable-line
 
@@ -11,7 +8,6 @@ keys:
   - &system_brain age1jhf7vm0005j60mjq63696frrmjhpy8kpc2d66mw044lqap5mjv4snmwvwm # cspell:disable-line
   - &system_jeeves age13lmqgc3jvkyah5e3vcwmj4s5wsc2akctcga0lpc0x8v8du3fxprqp4ldkv # cspell:disable-line
   - &system_rhapsody age1ufnewppysaq2wwcl4ugngjz8pfzc5a35yg7luq0qmuqvctajcycs5lf6k4 # cspell:disable-line
-  - &system_portal_1 age1vyav6kxtvt3z4vtnkkjj38eu8hlts5m7ygyckhskvalg2gpjk52su53d0a # cspell:disable-line
 
 creation_rules:
   - path_regex: users/secrets\.yaml$
@@ -22,4 +18,3 @@ creation_rules:
           - *system_brain
           - *system_jeeves
           - *system_rhapsody
-          - *system_portal_1

.vscode/settings.json

@@ -10,6 +10,7 @@
     "aiounifi",
     "alsa",
     "apiclient",
+    "apscheduler",
     "archlinux",
     "ashift",
     "asrouter",
@@ -336,6 +337,8 @@
     "yubioath",
     "yzhang",
     "zeroconf",
+    "zerotier",
+    "zerotierone",
     "zoxide",
     "zram",
     "zstd"

common/global/default.nix

@@ -17,11 +17,16 @@
     ./nix.nix
     ./programs.nix
     ./ssh.nix
+    ./snapshot_manager.nix
   ];
 
   boot = {
-    tmp.useTmpfs = lib.mkDefault true;
+    tmp.useTmpfs = true;
     kernelPackages = lib.mkDefault pkgs.linuxPackages_6_12;
+    zfs = {
+      package = lib.mkDefault pkgs.zfs_2_4;
+      forceImportRoot = lib.mkDefault false;
+    };
   };
 
   hardware.enableRedistributableFirmware = true;
@@ -37,6 +42,9 @@
     overlays = builtins.attrValues outputs.overlays;
     config = {
       allowUnfree = true;
+      permittedInsecurePackages = [
+        "openssl-1.1.1w" # This is for discord-canary
+      ];
     };
   };
 
@@ -45,6 +53,16 @@
 
     # firmware update
     fwupd.enable = true;
+
+    snapshot_manager = {
+      enable = lib.mkDefault true;
+      PYTHONPATH = "${inputs.self}/";
+    };
+
+    zfs = {
+      trim.enable = lib.mkDefault true;
+      autoScrub.enable = lib.mkDefault true;
+    };
   };
 
   powerManagement.powertop.enable = lib.mkDefault true;

common/global/nix.nix

@@ -31,10 +31,6 @@ in
         "flakes"
         "ca-derivations"
       ];
-      system-features = lib.mkAfter [
-        "gccarch-x86-64-v2"
-        "gccarch-x86-64-v3"
-      ];
       warn-dirty = false;
       flake-registry = ""; # disable global flake registries
       connect-timeout = 10;

common/global/programs.nix

@@ -2,5 +2,6 @@
 {
   environment.systemPackages = with pkgs; [
     git
+    my_python
   ];
 }

common/global/snapshot_config.toml

@@ -22,12 +22,6 @@ hourly = 0
 daily = 0
 monthly = 0
 
-["root_pool/nix_build"]
-15_min = 1
-hourly = 0
-daily = 0
-monthly = 0
-
 ["root_pool/var"]
 15_min = 8
 hourly = 24

common/global/snapshot_manager.nix

@@ -1,5 +1,4 @@
 {
-  inputs,
   pkgs,
   lib,
   config,
@@ -7,11 +6,6 @@
 }:
 let
   cfg = config.services.snapshot_manager;
-  snapshotManagerPackages =
-    ps: with ps; [
-      httpx
-      typer
-    ];
 in
 {
   options = {
@@ -28,6 +22,11 @@ in
           the PYTHONPATH to use for the snapshot_manager service.
         '';
       };
+      package = lib.mkOption {
+        type = lib.types.package;
+        default = pkgs.my_python;
+        description = "Python environment used to run snapshot_manager.";
+      };
       EnvironmentFile = lib.mkOption {
         type = lib.types.nullOr (lib.types.coercedTo lib.types.path toString lib.types.str);
         default = null;
@@ -40,41 +39,6 @@ in
   };
 
   config = lib.mkIf cfg.enable {
-    nixpkgs.overlays = [
-      (final: _prev: {
-        snapshot_manager_python = final.python314.withPackages snapshotManagerPackages;
-        snapshot_manager_test_python = final.python314.withPackages (
-          ps:
-          snapshotManagerPackages ps
-          ++ (with ps; [
-            pyfakefs
-            pytest
-            pytest-asyncio
-            pytest-mock
-            pytest-xdist
-          ])
-        );
-        snapshot_manager_tests =
-          final.runCommand "snapshot-manager-tests"
-            {
-              nativeBuildInputs = [ final.snapshot_manager_test_python ];
-            }
-            ''
-              export HOME="$TMPDIR"
-              cd ${inputs.self}
-              pytest \
-                -o cache_dir="$TMPDIR/pytest-cache" \
-                tests/test_common.py \
-                tests/test_signal_alert.py \
-                tests/test_snapshot_manager.py \
-                tests/test_zfs.py
-              touch "$out"
-            '';
-      })
-    ];
-
-    system.checks = [ pkgs.snapshot_manager_tests ];
-
     systemd = {
       services.snapshot_manager = {
         description = "ZFS Snapshot Manager";
@@ -86,7 +50,7 @@ in
         };
         serviceConfig = {
           Type = "oneshot";
-          ExecStart = "${pkgs.snapshot_manager_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
+          ExecStart = "${cfg.package}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
         }
         // lib.optionalAttrs (cfg.EnvironmentFile != null) {
           EnvironmentFile = cfg.EnvironmentFile;

common/optional/docker.nix

@@ -17,7 +17,6 @@
     logDriver = "local";
     storageDriver = "overlay2";
     daemon.settings = {
-      live-restore = false;
       experimental = true;
       exec-opts = [ "native.cgroupdriver=systemd" ];
       log-opts = {

common/optional/monitoring-agent.nix

@@ -5,7 +5,7 @@
   ...
 }:
 let
-  monitoringInterface = "tailscale0";
+  monitoringInterface = "ztwfunumly";
   nodeTextfileDir = "/var/lib/prometheus-node-exporter-textfile";
 
   mkProcessNameTemplate =

common/optional/tailscale.nix

@@ -1,37 +0,0 @@
-{
-  config,
-  inputs,
-  ...
-}:
-{
-  nix.settings = {
-    trusted-substituters = [ "http://jeeves:5000" ];
-    substituters = [ "http://jeeves:5000/?priority=1&want-mass-query=true" ];
-    trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
-  };
-
-  services.tailscale = {
-    enable = true;
-    openFirewall = true;
-    authKeyFile = config.sops.secrets.tailscale_auth_key.path;
-
-    # OAuth client secrets create ephemeral nodes by default. NixOS machines
-    # are persistent and should enroll without interactive device approval.
-    authKeyParameters = {
-      ephemeral = false;
-      preauthorized = true;
-    };
-
-    extraUpFlags = [ "--advertise-tags=tag:nixos" ];
-  };
-
-  sops = {
-    age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
-
-    secrets.tailscale_auth_key = {
-      sopsFile = "${inputs.self}/users/secrets.yaml";
-      owner = "root";
-      mode = "0400";
-    };
-  };
-}

common/optional/x86-64-v3.nix

@@ -1,9 +0,0 @@
-{
-  nixpkgs.hostPlatform = {
-    system = "x86_64-linux";
-    gcc = {
-      arch = "x86-64-v3";
-      tune = "generic";
-    };
-  };
-}

common/optional/zerotier.nix

@@ -0,0 +1,11 @@
+{
+  services.zerotierone = {
+    enable = true;
+    joinNetworks = [ "e4da7455b2ae64ca" ];
+  };
+  nix.settings = {
+    trusted-substituters = [ "http://192.168.90.40:5000" ];
+    substituters = [ "http://192.168.90.40:5000/?priority=1&want-mass-query=true" ];
+    trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
+  };
+}

common/optional/zfs/default.nix

@@ -1,26 +0,0 @@
-{
-  inputs,
-  lib,
-  pkgs,
-  ...
-}:
-{
-  imports = [ ./snapshot.nix ];
-
-  boot.zfs = {
-    package = lib.mkDefault pkgs.zfs_2_4;
-    forceImportRoot = lib.mkDefault false;
-  };
-
-  services = {
-    snapshot_manager = {
-      enable = lib.mkDefault true;
-      PYTHONPATH = "${inputs.self}/";
-    };
-
-    zfs = {
-      trim.enable = lib.mkDefault true;
-      autoScrub.enable = lib.mkDefault true;
-    };
-  };
-}

common/optional/zfs_manager.nix

@@ -0,0 +1,191 @@
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  cfg = config.services.zfs_manager;
+
+  snapshotOptions = {
+    options = {
+      "15_min" = lib.mkOption {
+        type = lib.types.ints.unsigned;
+        default = 0;
+        description = "How many 15 minute snapshots to keep.";
+      };
+      hourly = lib.mkOption {
+        type = lib.types.ints.unsigned;
+        default = 0;
+        description = "How many hourly snapshots to keep.";
+      };
+      daily = lib.mkOption {
+        type = lib.types.ints.unsigned;
+        default = 0;
+        description = "How many daily snapshots to keep.";
+      };
+      monthly = lib.mkOption {
+        type = lib.types.ints.unsigned;
+        default = 0;
+        description = "How many monthly snapshots to keep.";
+      };
+    };
+  };
+
+  datasetOptions = {
+    options = {
+      manageProperties = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+        description = ''
+          Whether zfs_manager owns this dataset's properties. When false the
+          dataset only contributes its snapshot retention, which is how
+          root_pool datasets are declared.
+        '';
+      };
+      createIfMissing = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+        description = ''
+          Whether zfs_manager may create this dataset when it is absent.
+
+          Set it false for a dataset that has to be provisioned by hand, such
+          as an encryption root: encryption is fixed at creation time and
+          cannot be expressed here, so creating it automatically would silently
+          produce an unencrypted dataset where an encrypted one was intended.
+          The dataset is still property checked, and its absence is reported as
+          a failure rather than quietly fixed.
+        '';
+      };
+      properties = lib.mkOption {
+        type = lib.types.attrsOf lib.types.str;
+        default = { };
+        description = ''
+          The zfs properties this dataset should have. Values are compared
+          against the live dataset and corrected when they differ.
+        '';
+      };
+      snapshots = lib.mkOption {
+        type = lib.types.submodule snapshotOptions;
+        default = cfg.defaultSnapshots;
+        description = ''
+          Snapshot retention for this dataset. Defaults to defaultSnapshots.
+        '';
+      };
+    };
+  };
+
+  # snapshot_manager.py only ever walks datasets below a pool root, so pool
+  # roots are left out of the retention table. It also indexes the table
+  # directly, which is why every entry carries all four keys.
+  snapshotTable = lib.mapAttrs (_: dataset: dataset.snapshots) (
+    lib.filterAttrs (name: _: lib.hasInfix "/" name) cfg.datasets
+  );
+
+  snapshotConfig = (pkgs.formats.toml { }).generate "snapshot_config.toml" (
+    snapshotTable // { default = cfg.defaultSnapshots; }
+  );
+
+  # Every declared dataset is emitted, including the ones whose properties are
+  # not managed, so the tool can tell "deliberately hands off" apart from
+  # "nobody has written this down yet".
+  datasetConfig = (pkgs.formats.json { }).generate "zfs_datasets.json" {
+    datasets = lib.mapAttrs (_: dataset: {
+      inherit (dataset) manageProperties createIfMissing properties;
+    }) cfg.datasets;
+  };
+in
+{
+  options = {
+    services.zfs_manager = {
+      enable = lib.mkEnableOption "declarative ZFS dataset management";
+      datasets = lib.mkOption {
+        type = lib.types.attrsOf (lib.types.submodule datasetOptions);
+        default = { };
+        example = lib.literalExpression ''
+          {
+            "media/temp".properties = {
+              sync = "disabled";
+              redundant_metadata = "none";
+            };
+          }
+        '';
+        description = ''
+          The datasets to manage, keyed by full dataset name. Missing datasets
+          are created and drifted properties are corrected. Nothing is ever
+          destroyed, and datasets that are not declared are left alone.
+
+          A name without a "/" is a pool root filesystem. Its properties are
+          managed but it is never created, pool creation stays manual.
+        '';
+      };
+      defaultSnapshots = lib.mkOption {
+        type = lib.types.submodule snapshotOptions;
+        default = { };
+        description = ''
+          Retention for undeclared datasets and for declared datasets that do
+          not override their snapshots. Emitted as the "default" table of the
+          snapshot config.
+        '';
+      };
+      dryRun = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Log every change that would be made without touching zfs. Use this to
+          validate a new or heavily edited declaration before applying it.
+        '';
+      };
+      PYTHONPATH = lib.mkOption {
+        type = lib.types.str;
+        description = ''
+          the PYTHONPATH to use for the zfs_manager service.
+        '';
+      };
+      package = lib.mkOption {
+        type = lib.types.package;
+        default = pkgs.my_python;
+        description = "Python environment used to run zfs_manager.";
+      };
+      EnvironmentFile = lib.mkOption {
+        type = lib.types.nullOr (lib.types.coercedTo lib.types.path toString lib.types.str);
+        default = null;
+
+        description = ''
+          Single environment file for the service (e.g. /etc/zfs-manager/env).
+          Use a leading "-" to ignore if missing (systemd feature).
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.snapshot_manager.path = snapshotConfig;
+
+    systemd = {
+      services.zfs_manager = {
+        description = "ZFS Dataset Manager";
+        requires = [ "zfs-import.target" ];
+        after = [
+          "zfs-import.target"
+          "zfs-mount.service"
+        ];
+        wantedBy = [ "multi-user.target" ];
+        path = [ pkgs.zfs ];
+        # Re-run on nixos-rebuild switch whenever the declaration changes.
+        restartTriggers = [ datasetConfig ];
+        environment = {
+          PYTHONPATH = cfg.PYTHONPATH;
+        };
+        serviceConfig = {
+          Type = "oneshot";
+          RemainAfterExit = true;
+          ExecStart = "${cfg.package}/bin/python -m python.tools.zfs_manager ${lib.escapeShellArg datasetConfig}${lib.optionalString cfg.dryRun " --dry-run"}";
+        }
+        // lib.optionalAttrs (cfg.EnvironmentFile != null) {
+          EnvironmentFile = cfg.EnvironmentFile;
+        };
+      };
+    };
+  };
+}

flake.lock

@@ -1,23 +1,25 @@
 {
   "nodes": {
-    "disko": {
+    "firefox-addons": {
       "inputs": {
         "nixpkgs": [
           "nixpkgs"
         ]
       },
       "locked": {
-        "lastModified": 1781152676,
-        "narHash": "sha256-RxWs5ND31KzTG7wvMM+PMfUjyNpmIEr999lqNARaM5o=",
-        "owner": "nix-community",
-        "repo": "disko",
-        "rev": "ff8702b4de27f72b4c78573dfb89ec74e36abdf1",
-        "type": "github"
+        "dir": "pkgs/firefox-addons",
+        "lastModified": 1787025780,
+        "narHash": "sha256-NhyLP9G4DFOn/7aYr7K/D7hWrzEGr5EgUBV+lpdmJ24=",
+        "owner": "rycee",
+        "repo": "nur-expressions",
+        "rev": "5ad360b6d3cb0aa1b61f9cb27fef113ca9117c37",
+        "type": "gitlab"
       },
       "original": {
-        "owner": "nix-community",
-        "repo": "disko",
-        "type": "github"
+        "dir": "pkgs/firefox-addons",
+        "owner": "rycee",
+        "repo": "nur-expressions",
+        "type": "gitlab"
       }
     },
     "home-manager": {
@@ -27,11 +29,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1788651960,
-        "narHash": "sha256-v9wJd32eZ2bvhBzVOd7TIjLQd011P7nwOhjKtWlci5I=",
+        "lastModified": 1786999651,
+        "narHash": "sha256-MTGMFlLDTklsXhCp4r5GXB4VAVadPdalXLvUjd/K7h0=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "2c0350c759688177331b8f5242311fae8877bdb3",
+        "rev": "353742587cbaf079b3caee743115d037bc51fea6",
         "type": "github"
       },
       "original": {
@@ -45,11 +47,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1788860136,
-        "narHash": "sha256-MhPMOFV4pVkygWEbQ8t1De/uQ9cWF1u++tRe2L5tG48=",
+        "lastModified": 1786867632,
+        "narHash": "sha256-ez+ubZlA1RtdjCB18a6zJ9M4u8qoPDy08EcnsW5M3Xw=",
         "owner": "nixos",
         "repo": "nixos-hardware",
-        "rev": "62173785b9a18c78b4a15aca2623d02bceb9d077",
+        "rev": "ff17823245ab9ff7bcae6acf950bd89cba82c38c",
         "type": "github"
       },
       "original": {
@@ -74,11 +76,11 @@
     },
     "nixpkgs-master": {
       "locked": {
-        "lastModified": 1788892992,
-        "narHash": "sha256-cIMFh9gyU4/aLeB3JCcsWM3tTAvD9pAq9Smr1Wa8aIU=",
+        "lastModified": 1787081018,
+        "narHash": "sha256-K0uwZBtZsbBigHAMQW7YWti3gPe6a5ct5bcOw5F+Q9Y=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "dff6994123e257ec9901c271bc2b52e64d7c8f05",
+        "rev": "cacac5ac351a010599d9f9d106acfed25a8e4c77",
         "type": "github"
       },
       "original": {
@@ -106,11 +108,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1788752844,
-        "narHash": "sha256-VaWGJ6+cIYN2erfSecbRV+4ljI185Ty2wUrXyvQbgOw=",
+        "lastModified": 1787001381,
+        "narHash": "sha256-Ue1Yo8gfHdD4TMtNewhA4tkSYeFqXThju0nCyJc3ALo=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "dc5d91f840324650bac8c379428c7037a416959a",
+        "rev": "ec2d622de0773551768cf98f3fc50cbcc003b9c5",
         "type": "github"
       },
       "original": {
@@ -122,7 +124,7 @@
     },
     "root": {
       "inputs": {
-        "disko": "disko",
+        "firefox-addons": "firefox-addons",
         "home-manager": "home-manager",
         "nixos-hardware": "nixos-hardware",
         "nixpkgs": "nixpkgs_2",
@@ -139,11 +141,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1788337237,
-        "narHash": "sha256-gkSH8VUtCo6hnysNmb9DbTuDepH2t5pv+QWjP75xKAk=",
+        "lastModified": 1786629091,
+        "narHash": "sha256-gkig4nPi1CWc4Z50GBsjE4ygSE7hMpl/TwID2an2Cck=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "fbf759290e0cb0a98dfc813a4eb7d53ad1dacb57",
+        "rev": "a8627b21b9107c5711c96b84f32a9a4b3d45295f",
         "type": "github"
       },
       "original": {

flake.nix

@@ -26,13 +26,13 @@
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
-    sops-nix = {
-      url = "github:Mic92/sops-nix";
+    firefox-addons = {
+      url = "gitlab:rycee/nur-expressions?dir=pkgs/firefox-addons";
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
-    disko = {
-      url = "github:nix-community/disko";
+    sops-nix = {
+      url = "github:Mic92/sops-nix";
       inputs.nixpkgs.follows = "nixpkgs";
     };
   };
@@ -77,6 +77,7 @@
         }
         // lib.optionalAttrs (pkgs.stdenv.hostPlatform.system == "x86_64-linux") {
           iso = self.nixosConfigurations.iso.config.system.build.isoImage;
+          zfs-integration = pkgs.testers.runNixOSTest (import ./tests/zfs-integration.nix { inherit self; });
         }
       );
       apps = forEachSystem (

overlays/default.nix

@@ -15,5 +15,35 @@
     };
   };
 
-  patches = import ./patches;
+  python-env = final: _prev: {
+    my_python = final.python314.withPackages (
+      ps: with ps; [
+        alembic
+        apprise
+        apscheduler
+        fastapi
+        fastapi-cli
+        httpx
+        jinja2
+        mypy
+        pgvector
+        psycopg
+        pydantic
+        pyfakefs
+        pytest
+        pytest-cov
+        pytest-mock
+        pytest-xdist
+        python-multipart
+        pydantic-settings
+        ruff
+        sqlalchemy
+        tenacity
+        tinytuya
+        typer
+        uvicorn
+        websockets
+      ]
+    );
+  };
 }

overlays/patches/README.md

@@ -1,46 +0,0 @@
-# Package patches
-
-Each package follows the [GnuTLS layout](gnutls/README.md):
-
-- `default.nix` applies the patch through the package overlay.
-- A descriptive `.patch` file contains the standalone upstream source change.
-- `README.md` explains the problem, scope, reproduction, upstream status,
-  Nix integration, and recorded validation limits.
-- Companion `verify-*` tools live beside the patch when needed; otherwise
-  the README gives commands for the package's existing tests.
-
-Keep package-specific evidence in its directory. Patch headers explain the
-change independently of Nix, and `default.nix` preserves existing patches.
-
-| Package | Repair |
-| --- | --- |
-| [Abseil](abseil/README.md) | Public BMI2 header in Electron, Deno, and Signal's vendored copies |
-| [Backrefs](backrefs/README.md) | Match the regex timeout's CPU clock |
-| [GnuTLS](gnutls/README.md) | Wait for the UDP server socket before connecting |
-| [Jupyter Server](jupyter-server/README.md) | Exercise the correct shared future during reconnect |
-| [Prometheus](prometheus/README.md) | Complete parsing before inspecting the test editor state |
-| [pytest-xdist](pytest-xdist/README.md) | Check worker replacements and allow startup on loaded builders |
-| [SciPy](scipy/README.md) | Account for floating-point rounding in STFT tests |
-| [Sentry SDK](sentry-sdk/README.md) | Isolate SDK thread mocks from Python's threading module |
-| [Torchaudio](torchaudio/README.md) | Compare pitch-shift batches at appropriate precision |
-| [TorchCodec](torchcodec/README.md) | Match the reference MP3 encoder's sample format |
-
-## Local NixOS integration
-
-[`../default.nix`](../default.nix) imports this directory's
-[`default.nix`](default.nix), which wires each package's override into the
-package set. Abseil repairs several vendored copies and is gated on
-`x86-64-v3`; Prometheus patches its separate assets derivation; Python
-packages use `pythonPackagesExtensions`.
-
-The [pytest-xdist directory](pytest-xdist/README.md) also owns its outer-worker
-limit and remote-worker event timeout. These package overrides add no skipped
-tests. Existing nixpkgs exclusions remain separate from these repairs.
-
-The test-exclusion review used Python 3.14.7 and the pinned x86-64-v3 package
-set. Host-flake evaluation verified patch wiring, Python install checks,
-removal of the local skips, and Prometheus's reference to the patched assets.
-Jupyter and Sentry package tests used the preceding dependency set with the
-new package patch to avoid unrelated rebuilds after pytest-xdist changed.
-No complete NixOS rebuild was performed. Individual READMEs distinguish
-package builds, focused tests, and checks that have not been run.

overlays/patches/abseil/README.md

@@ -1,59 +0,0 @@
-# Abseil BMI2 public header
-
-Vendored Abseil includes `bmi2intrin.h` directly when `__BMI2__` is enabled.
-Compilers reject that internal header without the umbrella-header setup.
-`bmi2-public-header.patch` includes `immintrin.h` instead, allowing builds
-that enable BMI2 through `-march=x86-64-v3`.
-
-## Scope and behavior
-
-The patch changes one include in
-`third_party/abseil-cpp/absl/container/internal/raw_hash_set.h`.
-`default.nix` applies it to Electron 43's unwrapped package, Deno's
-`librusty_v8`, and Signal's WebRTC dependency. It also supplies the patched
-Electron package to Signal. These overrides apply only to `x86-64-v3`.
-
-The shared file path is relative to each vendoring project's source root,
-not the root of a standalone Abseil checkout. No hash-table algorithm or
-test exclusion changes.
-
-## Reproduction and focused checks
-
-From this directory, check and apply the patch to each vendored source tree:
-
-```sh
-patch --dry-run --fuzz=0 -d /path/to/vendor-source -p1 < bmi2-public-header.patch
-patch --fuzz=0 -d /path/to/vendor-source -p1 < bmi2-public-header.patch
-```
-
-A small compiler check isolates the header requirement. With GCC or Clang
-on x86-64, compile `#include <bmi2intrin.h>` using `-march=x86-64-v3`; the
-compiler rejects the direct include. Changing it to `#include <immintrin.h>`
-should compile. The full consumer builds below check integration with their
-actual toolchains.
-
-## Upstream status
-
-Abseil addressed this issue through
-[PR #2071](https://github.com/abseil/abseil-cpp/pull/2071), imported by its
-upstream workflow. That change uses `x86gprintrin.h`; this local variant uses
-the public `immintrin.h` umbrella header for the vendored toolchains.
-Keep the workaround until all three bundled copies include a compatible fix.
-This file is a local adaptation, not a verbatim copy of the upstream diff.
-
-## Local NixOS integration and build results
-
-[`../default.nix`](../default.nix) merges this directory's overlay fragment
-because it repairs multiple packages. From the repository root, the consumer
-build commands are:
-
-```sh
-nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.deno
-nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.electron_43
-nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.signal-desktop
-```
-
-The earlier extraction checked the vendored header snapshots and evaluated
-all three patch attachments. Those records do not establish successful full
-consumer rebuilds. No new compiler or consumer build was run for the layout
-change; the patch and override are unchanged.

overlays/patches/abseil/bmi2-public-header.patch

@@ -1,20 +0,0 @@
-Subject: [PATCH] abseil: include BMI2 intrinsics through the public header
-
-GCC and Clang reject direct inclusion of bmi2intrin.h. Include immintrin.h
-instead so that the compiler supplies the required intrinsic setup when
-BMI2 is enabled, including builds targeting x86-64-v3.
-
-This patch is shared by the vendored Abseil copies in Electron, rusty_v8
-(Deno), and Signal's WebRTC build.
-
---- a/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
-+++ b/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
-@@ -226,7 +226,7 @@
- #endif
- 
- #ifdef __BMI2__
--#include <bmi2intrin.h>
-+#include <immintrin.h>
- #endif  // __BMI2__
- 
- namespace absl {

overlays/patches/abseil/default.nix

@@ -1,38 +0,0 @@
-# Abseil accepted the upstream fix: https://github.com/abseil/abseil-cpp/pull/2071
-# Keep this workaround until Electron, Deno's rusty_v8, and Signal's WebRTC
-# update their bundled Abseil copies to include it.
-{ prev }:
-let
-  patchAbseilBmi2Include =
-    package:
-    package.overrideAttrs (old: {
-      # GCC and Clang require the public umbrella header for BMI2 intrinsics.
-      patches = (old.patches or [ ]) ++ [ ./bmi2-public-header.patch ];
-    });
-
-  electron43Unwrapped = patchAbseilBmi2Include prev.electron_43.unwrapped;
-  electron43 = prev.electron_43.override {
-    electron-unwrapped = electron43Unwrapped;
-  };
-
-  signalCallPackage =
-    path: args:
-    let
-      package = prev.callPackage path args;
-    in
-    if builtins.baseNameOf path == "webrtc.nix" then patchAbseilBmi2Include package else package;
-in
-prev.lib.optionalAttrs ((prev.stdenv.hostPlatform.gcc.arch or null) == "x86-64-v3") {
-  deno =
-    let
-      librusty_v8 = patchAbseilBmi2Include prev.deno.passthru.librusty_v8;
-    in
-    prev.deno.override { inherit librusty_v8; };
-
-  electron_43 = electron43;
-
-  signal-desktop = prev.signal-desktop.override {
-    electron_43 = electron43;
-    callPackage = signalCallPackage;
-  };
-}

overlays/patches/default.nix

@@ -1,14 +0,0 @@
-_final: prev:
-(import ./abseil { inherit prev; })
-// {
-  gnutls = import ./gnutls { inherit (prev) gnutls; };
-  prometheus = import ./prometheus { inherit (prev) prometheus; };
-
-  pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
-    (_pythonFinal: pythonPrev: {
-      pytest-xdist = import ./pytest-xdist { inherit (pythonPrev) pytest-xdist; };
-      scipy = import ./scipy { inherit (pythonPrev) scipy; };
-      sentry-sdk = import ./sentry-sdk { inherit (pythonPrev) sentry-sdk; };
-    })
-  ];
-}

overlays/patches/gnutls/README.md

@@ -1,160 +0,0 @@
-# GnuTLS UDP server readiness
-
-Under load, the test client can start before `gnutls-serv` binds its UDP
-socket, and the first handshake fails with `Connection refused`.
-`serv-udp.sh` currently waits a fixed four seconds; elapsed time does not
-establish server readiness. `udp-server-readiness.patch` replaces that wait
-with polling for the local IPv4 UDP endpoint.
-
-## Scope and waiting behavior
-
-The patch changes the existing `wait_udp_server()` and adds a new
-`check_if_udp_port_bound()` beside it in `tests/scripts/common.sh`.
-`serv-udp.sh` is its only caller in 3.8.13. The TCP helpers `wait_server()`
-and `wait_for_port()`, including their existing sleeps, are unchanged.
-Both original DTLS handshake checks remain unchanged.
-
-Each iteration checks process liveness and the socket **before sleeping**.
-A ready socket returns immediately. An unsuccessful check sleeps two
-seconds only if another attempt remains: at most 90 attempts, consistent
-with the existing `wait_server()` budget implemented by `wait_for_port()`,
-with no sleep after the final check. Server exit fails early; exhausting the
-budget fails and terminates the server. No handshake is retried, and no
-protocol timeout is changed. Once bound, the kernel can queue datagrams
-while the server is scheduled; the probe itself sends no packets.
-
-The existing `have_port_finder()` prefers `ss`, then `netstat`. If neither
-exists, it prints `neither ss nor netstat found` and exits **77 (skip)**.
-In the normal test flow, port selection calls it before launching a server.
-The probe runs in a subshell so that, even if this skip occurs after launch,
-the waiting helper can terminate and reap the server before exiting 77.
-
-## Why an IPv4 socket is expected
-
-This is specific to the server used by this test, not a general rule that
-IPv6 sockets cannot serve IPv4 clients. The client explicitly uses
-`127.0.0.1`. The server's `--udp` path calls `udp_server()`, which calls
-`listen_socket(..., SOCK_DGRAM)`. That function iterates the wildcard
-addresses returned by `getaddrinfo(NULL, ..., AI_PASSIVE)`:
-
-| Server build / Linux setting | Binding behavior |
-| --- | --- |
-| IPv6 enabled, `net.ipv6.bindv6only=0` | Requests `IPV6_V6ONLY=1` on the IPv6 socket, binds `[::]:PORT`, and separately binds `0.0.0.0:PORT`. It overrides the system's dual-stack default. |
-| IPv6 enabled, `net.ipv6.bindv6only=1` | The same explicit socket option and separate IPv4/IPv6 binds. |
-| `HAVE_IPV6` undefined | Skips every address family except `AF_INET`; only the IPv4 wildcard is attempted. |
-
-`udp_server()` uses `wait_for_connection()`, which puts **every listener**
-from that list into `select()` and returns a readable socket for `recvfrom()`;
-it does not permanently choose one socket based on `getaddrinfo()` order.
-
-The first two cases were traced with the actual GnuTLS 3.8.13 binary in
-separate Linux network namespaces: `setsockopt(IPV6_V6ONLY, [1])` and both
-UDP binds returned success under each setting. The no-IPv6 case was checked
-in source, not by building a second binary. The same bind implementation
-was checked directly on GitLab master.
-
-Thus, successful normal startup for this invocation provides an explicit
-IPv4 socket; a lone IPv6 wildcard is not the expected success path.
-There is one portability caveat: upstream discards the return value of
-`setsockopt(IPV6_V6ONLY)`. On a platform where that call fails and the server
-ends up with only a dual-stack socket, this helper would time out despite
-IPv4 reachability. Such a platform needs additional handling before this
-patch can claim support. Blindly accepting every IPv6 wildcard would also
-accept IPv6-only sockets before the separate IPv4 bind finishes.
-
-Source: [`src/serv.c`, `listen_socket()`](https://gitlab.com/gnutls/gnutls/-/blob/master/src/serv.c#L937),
-[`src/udp-serv.c`](https://gitlab.com/gnutls/gnutls/-/blob/master/src/udp-serv.c),
-and [`tests/serv-udp.sh`](https://gitlab.com/gnutls/gnutls/-/blob/master/tests/serv-udp.sh).
-
-## Port matching and ownership limit
-
-Only `-an` is passed to the socket-listing tool: BSD `netstat -u` selects
-Unix-domain sockets, whereas Linux `netstat -u` selects UDP. The parser
-handles the extra state column in `ss`, Linux colon-separated endpoints,
-and BSD dot-separated endpoints, including `*.PORT`. It matches the full
-local port and rejects TCP, IPv6 entries, peer ports, and longer numbers.
-
-A live PID plus a bound port does **not** prove that PID owns the socket.
-Existing `GETPORT` selection checks for an unused port and uses a test
-port-lock directory; `launch_bare_server()` also calls
-`wait_for_free_port()` before starting the process. These are advisory:
-the launcher does not enforce the latter's result, and another process
-can bind between the check and launch. The patch does not close that race
-or add nonportable PID parsing. An unrelated process can satisfy the
-socket check; the real handshakes remain the functional check and may
-fail (or reach the wrong server). This is a startup-order fix, not a
-socket-ownership guarantee.
-
-## Reproduction and focused checks
-
-Apply the patch to an unpacked source tree, then run the companion checks
-with Python's standard library and a shell:
-
-```sh
-patch --fuzz=0 -d /path/to/gnutls -p1 < udp-server-readiness.patch
-SHELL=/bin/sh python3 verify-readiness.py /path/to/gnutls/tests/scripts/common.sh -v
-```
-
-Set `NETSTAT=/path/to/netstat` to exercise one outside `PATH`. The checks
-cover Linux/BSD output samples, false matches, immediate readiness,
-missing tools, process exit, timeout cleanup, and real IPv4 UDP sockets
-whose bind is delayed six seconds. The missing-tools fixture is skipped
-if an absolute fallback `ss` path cannot be hidden with `PATH`. Native
-BSD execution remains untested.
-
-To reproduce with GnuTLS itself, run `tests/serv-udp.sh` with `SERV` pointing
-to a wrapper that sleeps six seconds, then `exec`s `gnutls-serv` with all
-arguments. Set `CLI` to the matching `gnutls-cli`, `srcdir` to the source
-`tests` directory, and `abs_top_builddir` to a writable build directory.
-With GnuTLS 3.8.13, the original helper failed the first handshake with
-`Connection refused`; the patched helper passed both with the same binaries.
-
-## GnuTLS submission
-
-Development and merge requests are on [GitLab](https://gitlab.com/gnutls/gnutls).
-[`CONTRIBUTING.md` on master](https://gitlab.com/gnutls/gnutls/-/blob/master/CONTRIBUTING.md)
-was read directly for this review. It requires the contributor's DCO
-`Signed-off-by`, successful and failure test coverage, consistent coding
-style, and adequate documentation; GitLab CI runs for merge requests.
-Its commenting guidance asks for comments explaining non-obvious behavior
-or protocol expectations. It does not prescribe an additional special
-test-suite comment. The patch now explains its IPv4 binding assumption
-next to the probe.
-
-The submission will contain the shell patch, without the Python verifier
-or a new Python test dependency. The existing `serv-udp.sh` supplies the
-functional success check. Running it through the six-second startup
-wrapper supplies a reproducible regression case: it fails before the fix
-and passes after it. The local verifier was used to validate socket-output
-parsing and the helper's success, process-exit, skip-cleanup, and timeout
-branches. Those branch checks are local evidence, not new automated
-coverage in the upstream suite; the MR must state that distinction.
-
-No dedicated unit-test harness for these shell helpers was found in the
-3.8.13 tests inspected. That does not establish that Python cannot be used
-upstream; keeping this submission dependency-free is a scope choice. Use
-the existing test and before/after reproduction as the submission's
-coverage argument, retaining the platform limitations above. Apply the
-patch in an upstream checkout and include those results with the
-contributor's own sign-off. No MR or sign-off has been created.
-
-## Local NixOS integration and build results
-
-`overlays/default.nix` imports the `overlays/patches` overlay, which loads
-`gnutls/default.nix` to apply the patch and keep `serv-udp.sh` enabled.
-The patch itself has no Nix dependencies and applies to 3.8.13 and GitLab
-master without fuzz.
-
-The final patch was rebuilt with:
-
-```sh
-nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.gnutls
-```
-
-That x86-64-v3 build passed: 927 tests, 796 passes, 131 existing skips,
-zero failures/errors, and `PASS: serv-udp.sh`. The patch bytes in the built
-derivation were compared with the repository artifact; both have SHA-256
-`59013d47fd446f2dd065012a2259ccc1898fedc8a053a630e13efa0076368760`.
-All seven local checks passed, including skip cleanup and exactly 90
-probes with 89 sleeps on timeout. The six-second before/after reproduction
-was also repeated successfully with the final helper.

overlays/patches/gnutls/default.nix

@@ -1,6 +0,0 @@
-{ gnutls }:
-gnutls.overrideAttrs (old: {
-  # Keep the UDP handshake test enabled on loaded builders by waiting for
-  # the server to bind its socket. Kept as a standalone patch for upstream.
-  patches = (old.patches or [ ]) ++ [ ./udp-server-readiness.patch ];
-})

overlays/patches/gnutls/udp-server-readiness.patch

@@ -1,70 +0,0 @@
-Subject: [PATCH] tests: wait for the UDP server socket before connecting
-
-A fixed four-second sleep does not guarantee that gnutls-serv has bound
-its UDP socket on a busy builder. Poll the local IPv4 UDP endpoint using
-the existing ss/netstat discovery, with the same retry budget as the TCP
-helper. Fail early if the server exits, and retain the original handshake
-checks in serv-udp.sh.
-
-Use flags common to ss and BSD/Linux netstat. Match the local endpoint
-and complete port number, excluding TCP, IPv6-only and peer endpoints.
-
---- a/tests/scripts/common.sh
-+++ b/tests/scripts/common.sh
-@@ -185,10 +185,55 @@
- 	fi
- }
- 
-+check_if_udp_port_bound() {
-+	local PORT=$1
-+	have_port_finder
-+	# Use only -an, which is shared by ss and BSD/Linux netstat. UDP has
-+	# no LISTEN state. Match the local IPv4 endpoint, not a peer port or
-+	# a longer port number. serv-udp.sh connects to 127.0.0.1;
-+	# listen_socket() in serv.c binds IPv4 separately and requests
-+	# IPV6_V6ONLY=1 for its IPv6 socket.
-+	$PFCMD -an | awk -v port="$PORT" '
-+		$1 == "udp" || $1 == "udp4" {
-+			# ss includes a state column; netstat does not.
-+			address = ($2 == "UNCONN" || $2 == "ESTAB") ? $5 : $4
-+			if (address ~ ("^[0-9.]+[.:]" port "$") ||
-+			    address == "*." port)
-+				found = 1
-+		}
-+		END { exit !found }
-+	'
-+}
-+
- wait_udp_server() {
- 	local PID=$1
-+	local ret
- 	trap "test -n \"${PID}\" && kill ${PID};exit 1" 1 15 2
--	sleep 4
-+	local i=0
-+	# Use the same retry budget as wait_for_port(), but also stop if the
-+	# server exits before binding its socket.
-+	while test $i -lt 90; do
-+		if ! kill -0 "$PID" 2>/dev/null; then
-+			fail "" "UDP server $PID exited before binding port $PORT"
-+		fi
-+		# Contain have_port_finder's exit so a skip also stops the server.
-+		if (check_if_udp_port_bound "$PORT"); then
-+			return 0
-+		else
-+			ret=$?
-+			if test "$ret" = 77; then
-+				kill "$PID" 2>/dev/null || :
-+				wait "$PID" 2>/dev/null || :
-+				exit 77
-+			fi
-+		fi
-+		i=$((i + 1))
-+		if test $i -lt 90; then
-+			echo "try $i: waiting for UDP port $PORT"
-+			sleep 2
-+		fi
-+	done
-+	fail "$PID" "UDP server $PORT did not come up"
- }
- 
- create_testdir() {

overlays/patches/gnutls/verify-readiness.py

@@ -1,180 +0,0 @@
-#!/usr/bin/env python3
-"""Exercise patched common.sh without building GnuTLS (Python standard library only).
-
-Usage: python3 verify-readiness.py /path/to/patched/tests/scripts/common.sh
-Set SHELL to test another shell, and NETSTAT to test a netstat outside PATH.
-"""
-
-# Use unittest so this upstream companion tool needs no pytest installation.
-# ruff: noqa: PT009
-
-import os
-import shutil
-import socket
-import subprocess
-import sys
-import tempfile
-import time
-import unittest
-from pathlib import Path
-
-COMMON = str(Path(sys.argv.pop(1)).resolve())
-SHELL = os.environ.get("SHELL", "/bin/sh")
-
-
-class ReadinessTests(unittest.TestCase):
-    """Check endpoint parsing and the server startup lifecycle."""
-
-    def setUp(self) -> None:
-        """Create a socket-listing fixture for each check."""
-        self.tmp = tempfile.TemporaryDirectory()
-        self.addCleanup(self.tmp.cleanup)
-        self.root = Path(self.tmp.name)
-        self.fixture = self.root / "sockets"
-        self.fixture.write_text("")
-        self.finder = self.root / "port-finder"
-        self.finder.write_text('#!/bin/sh\ncat "$SOCKET_FIXTURE"\n')
-        self.finder.chmod(0o755)
-
-    def run_shell(self, body: str, **env: str) -> subprocess.CompletedProcess[str]:
-        """Source the actual helper and run a shell scenario."""
-        return subprocess.run(
-            [SHELL, "-c", '. "$COMMON"\n' + body],
-            env={
-                **os.environ,
-                "COMMON": COMMON,
-                "SOCKET_FIXTURE": str(self.fixture),
-                "PFCMD": str(self.finder),
-                "PORT": "12345",
-                **env,
-            },
-            capture_output=True,
-            text=True,
-            timeout=20,
-            check=False,
-        )
-
-    def test_socket_formats_and_false_matches(self) -> None:
-        """Accept IPv4 UDP local endpoints and reject unrelated sockets."""
-        cases = [
-            ("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*", True),
-            ("udp UNCONN 0 0 127.0.0.1:12345 0.0.0.0:*", True),
-            ("udp 0 0 0.0.0.0:12345 0.0.0.0:*", True),
-            ("udp4 0 0 *.12345 *.*", True),
-            ("udp 0 0 127.0.0.1.12345 *.*", True),
-            ("udp 0 0 *.12345 *.*", True),
-            ("udp UNCONN 0 0 0.0.0.0:123456 0.0.0.0:*", False),
-            ("udp 0 0 0.0.0.0:123456 0.0.0.0:*", False),
-            ("udp ESTAB 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
-            ("udp 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
-            ("tcp LISTEN 0 128 0.0.0.0:12345 0.0.0.0:*", False),
-            ("tcp 0 0 0.0.0.0:12345 0.0.0.0:* LISTEN", False),
-            ("udp UNCONN 0 0 [::]:12345 [::]:*", False),
-            ("udp UNCONN 0 0 *:12345 *:*", False),
-            ("udp6 0 0 :::12345 :::*", False),
-            ("udp6 0 0 *.12345 *.*", False),
-            ("", False),
-        ]
-        for row, ready in cases:
-            with self.subTest(row=row):
-                self.fixture.write_text(row + "\n")
-                result = self.run_shell('check_if_udp_port_bound "$PORT"')
-                self.assertEqual(result.returncode, 0 if ready else 1, result.stderr)
-
-    def test_exited_server_fails_immediately(self) -> None:
-        """Fail without sleeping when the server has already exited."""
-        result = self.run_shell(
-            'true &\npid=$!\nwait "$pid"\nsleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$pid"'
-        )
-        self.assertEqual(result.returncode, 1)
-        self.assertIn("exited before binding", result.stderr)
-        self.assertNotIn("unexpected sleep", result.stderr)
-
-    def test_ready_socket_does_not_sleep(self) -> None:
-        """Check readiness before the first sleep."""
-        self.fixture.write_text("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*\n")
-        result = self.run_shell('sleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$$"')
-        self.assertEqual(result.returncode, 0, result.stderr)
-        self.assertNotIn("unexpected sleep", result.stderr)
-
-    def test_missing_port_finders_skip(self) -> None:
-        """Skip and stop the live server when no finder is available."""
-        # have_port_finder also tries these paths independently of PATH.
-        if any(os.access(f"{directory}/ss", os.X_OK) for directory in ("/sbin", "/usr/sbin", "/usr/local/sbin")):
-            self.skipTest("an absolute ss path cannot be hidden by this PATH-only fixture")
-        with subprocess.Popen(["sleep", "60"]) as server:
-            try:
-                result = self.run_shell(
-                    'unset PFCMD\nPATH=/nonexistent\nwait_udp_server "$SERVER_PID"',
-                    SERVER_PID=str(server.pid),
-                )
-                self.assertEqual(result.returncode, 77)
-                self.assertIn("neither ss nor netstat found", result.stderr)
-                server.wait(timeout=3)
-                self.assertLess(server.returncode, 0)
-            finally:
-                if server.poll() is None:
-                    server.kill()
-
-    def test_timeout_is_bounded_and_cleans_up(self) -> None:
-        """Stop polling after the retry budget and terminate the server."""
-        # Only accelerate the polling delay; keep a real live server process.
-        self.finder.write_text('#!/bin/sh\necho probe >&2\ncat "$SOCKET_FIXTURE"\n')
-        with subprocess.Popen(["sleep", "60"]) as server:
-            try:
-                result = self.run_shell(
-                    'sleep() { echo polling-sleep; }\nwait_udp_server "$SERVER_PID"',
-                    SERVER_PID=str(server.pid),
-                )
-                self.assertEqual(result.returncode, 1)
-                self.assertIn("did not come up", result.stderr)
-                self.assertEqual(result.stderr.count("probe\n"), 90)
-                self.assertEqual(result.stdout.count("polling-sleep"), 89)
-                server.wait(timeout=3)
-                self.assertLess(server.returncode, 0)
-            finally:
-                if server.poll() is None:
-                    server.kill()
-
-    def test_server_exits_while_waiting(self) -> None:
-        """Detect a startup failure that happens after polling begins."""
-        result = self.run_shell('sleep 1 &\npid=$!\nwait_udp_server "$pid"')
-        self.assertEqual(result.returncode, 1)
-        self.assertIn("exited before binding", result.stderr)
-        self.assertIn("waiting for UDP port", result.stdout)
-
-    def test_real_socket_delayed_beyond_four_seconds(self) -> None:
-        """Wait for a real delayed bind with each installed port finder."""
-        finders = [shutil.which("ss"), os.environ.get("NETSTAT") or shutil.which("netstat")]
-        finders = [finder for finder in finders if finder]
-        if not finders:
-            self.skipTest("neither ss nor netstat available")
-        for finder in finders:
-            with self.subTest(finder=finder):
-                with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
-                    sock.bind(("127.0.0.1", 0))
-                    port = sock.getsockname()[1]
-                code = (
-                    "import socket,time,sys; time.sleep(6); "
-                    "s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); "
-                    "s.bind(('127.0.0.1',int(sys.argv[1]))); time.sleep(30)"
-                )
-                with subprocess.Popen([sys.executable, "-c", code, str(port)]) as server:
-                    try:
-                        started = time.monotonic()
-                        result = self.run_shell(
-                            'wait_udp_server "$SERVER_PID"',
-                            SERVER_PID=str(server.pid),
-                            PORT=str(port),
-                            PFCMD=finder,
-                        )
-                        self.assertEqual(result.returncode, 0, result.stderr)
-                        self.assertGreaterEqual(time.monotonic() - started, 6)
-                        self.assertIsNone(server.poll())
-                    finally:
-                        server.terminate()
-                        server.wait(timeout=3)
-
-
-if __name__ == "__main__":
-    unittest.main()

overlays/patches/prometheus/README.md

@@ -1,66 +0,0 @@
-# Prometheus complete test parsing
-
-CodeMirror gives editor-state creation a 20 ms synchronous parsing budget.
-The shared `createEditorState()` test helper can therefore return an
-incomplete syntax tree when the process is descheduled. The completion and
-vector-matching tests immediately inspect that tree.
-
-## Scope and behavior
-
-`complete-test-parsing.patch` changes only
-`module/codemirror-promql/src/test/utils-test.ts` inside `web/ui`. It completes
-the small test expression with `ensureSyntaxTree(..., Infinity)` and publishes
-the completed parse through an empty transaction so `syntaxTree(state)` sees
-it. Failure to obtain a tree raises an error.
-
-The original assertions remain enabled, including `autocomplete topk params 2`
-and `foo * on(test,blub) bar`. The unlimited budget applies to the test helper;
-production editor parsing budgets are unchanged.
-
-## Reproduction and focused checks
-
-Use a disposable Prometheus 3.14.0 checkout. The patch root is `web/ui`, matching
-the Nix assets derivation. From this directory:
-
-```sh
-patch --fuzz=0 -d /path/to/prometheus/web/ui -p1 < complete-test-parsing.patch
-cd /path/to/prometheus/web/ui
-pnpm install --frozen-lockfile
-pnpm --filter @prometheus-io/lezer-promql build
-pnpm --filter @prometheus-io/codemirror-promql test
-```
-
-To force the scheduling condition, temporarily append this clock to
-`module/codemirror-promql/setupJest.cjs` in the disposable checkout:
-
-```js
-let parseClock = 0;
-Date.now = () => (parseClock += 25);
-```
-
-Each clock read crosses the editor's initial parsing budget. Against the
-original helper, the hybrid and vector suites have 186 failures, including
-both locally excluded cases. With the patch, all 386 CodeMirror tests pass
-under that same clock. Remove the injected clock before normal builds.
-
-## Upstream status
-
-This is a standalone test-helper patch for Prometheus 3.14.0. No upstream
-submission was made during this work. Recheck the helper when updating
-Prometheus or CodeMirror, including how an ensured parse becomes visible
-through the editor state.
-
-## Local NixOS integration and build results
-
-[`../default.nix`](../default.nix) loads `default.nix`, which patches the
-separate assets derivation. It updates both `passthru.assets` and the main
-Prometheus build's reference to those assets. From the repository root:
-
-```sh
-nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.prometheus.assets
-```
-
-The full x86-64-v3 assets build passed with the normal clock, including the
-CodeMirror and UI suites. Host-flake evaluation confirmed that the main
-Prometheus derivation refers to these patched assets. The Go server package
-was not rebuilt for this test-helper change.

overlays/patches/prometheus/complete-test-parsing.patch

@@ -1,36 +0,0 @@
-Subject: [PATCH] tests: finish parsing before inspecting editor state
-
-EditorState creation has a 20 ms parsing budget. A descheduled test can
-therefore observe an incomplete tree. Finish these small test documents
-without an interactive deadline and publish the result with a transaction.
-Keep the original completion and vector-matching assertions enabled.
-
---- a/module/codemirror-promql/src/test/utils-test.ts
-+++ b/module/codemirror-promql/src/test/utils-test.ts
-@@ -13,7 +13,7 @@
- 
- import { parser } from '@prometheus-io/lezer-promql';
- import { EditorState } from '@codemirror/state';
--import { LRLanguage } from '@codemirror/language';
-+import { ensureSyntaxTree, LRLanguage } from '@codemirror/language';
- import nock from 'nock';
- import path from 'path';
- import { fileURLToPath } from 'url';
-@@ -23,10 +23,16 @@
- const __dirname = path.dirname(fileURLToPath(import.meta.url));
- 
- export function createEditorState(expr: string): EditorState {
--  return EditorState.create({
-+  const state = EditorState.create({
-     doc: expr,
-     extensions: lightPromQLSyntax,
-   });
-+  // These tests need a complete tree, independent of the editor's time budget.
-+  if (!ensureSyntaxTree(state, state.doc.length, Infinity)) {
-+    throw new Error('Unable to parse the test expression');
-+  }
-+  // Publish the completed parse so syntaxTree(state) sees it too.
-+  return state.update({}).state;
- }
- 
- export function mockPrometheusServer(): void {

overlays/patches/prometheus/default.nix

@@ -1,17 +0,0 @@
-{ prometheus }:
-prometheus.overrideAttrs (
-  old:
-  let
-    assets = old.passthru.assets.overrideAttrs (assetsOld: {
-      patches = (assetsOld.patches or [ ]) ++ [ ./complete-test-parsing.patch ];
-    });
-  in
-  {
-    postPatch = builtins.replaceStrings [ "${old.passthru.assets}" ] [ "${assets}" ] (
-      builtins.unsafeDiscardStringContext old.postPatch
-    );
-    passthru = old.passthru // {
-      inherit assets;
-    };
-  }
-)

overlays/patches/pytest-xdist/README.md

@@ -1,79 +0,0 @@
-# pytest-xdist test fixes
-
-With two workers and a restart limit of three, the fourth worker crash
-requests shutdown while another test can still be running. That test may
-also crash. The original queued-work test requires exactly four failures,
-even though five failures can occur without exceeding the replacement limit.
-
-## Scope and behavior
-
-`concurrent-worker-crashes.patch` changes the assertions in
-`TestNodeFailure.test_max_worker_restart_tests_queued` in
-`testing/acceptance_test.py`. It requires exactly three replacements, four or
-five failed tests, the failed-tests exit status, the limit message, and no
-internal error. It retains the two-worker workload and ten queued tests.
-
-`worker-startup-timeout.patch` changes the remote-test helper's event timeout
-from 10 to 60 seconds so loaded builders have time to start workers. The
-helper returns immediately when an event arrives and still has a bounded wait.
-
-The existing nixpkgs pytest-9 compatibility patches remain in place.
-Production scheduling and worker-restart behavior are unchanged.
-
-## Reproduction and focused checks
-
-Use a disposable pytest-xdist 3.8.0 checkout with its test dependencies and
-the nixpkgs pytest-9 compatibility patches where required. From this directory:
-
-```sh
-patch --fuzz=0 -d /path/to/pytest-xdist -p1 < concurrent-worker-crashes.patch
-patch --fuzz=0 -d /path/to/pytest-xdist -p1 < worker-startup-timeout.patch
-cd /path/to/pytest-xdist
-python -m pytest testing/acceptance_test.py \
-  -k test_max_worker_restart_tests_queued -q
-python -m pytest testing/test_remote.py -q
-```
-
-Twenty unmodified runs passed during the review. To force the failing
-schedule, modify the generated crashing test in a disposable checkout to
-accept `worker_id`: make `gw3` wait for a marker created by `gw4`, and make
-`gw4` pause 0.1 seconds after creating the marker. Then both have in-flight
-tests when shutdown starts. Bound the marker wait so a reproduction failure
-cannot hang the suite. The original assertion fails on five reported
-failures; the patched test passes.
-
-## Worker startup and outer concurrency
-
-[`default.nix`](default.nix) runs the outer suite with one worker to limit
-nested process pools. This is a Nix test-runner setting; the source timeout
-change lives in [`worker-startup-timeout.patch`](worker-startup-timeout.patch).
-
-A separate reproduction inserts an 11-second `pytest_sessionstart` delay
-into the child created by `test_basic_collect_and_runtests` in
-`testing/test_remote.py`. The original 10-second channel wait fails; the
-60-second wait passes. This bounds waits for test worker events, including
-startup, rather than changing a product deadline.
-
-## Upstream status
-
-These are standalone test patches for pytest-xdist 3.8.0. No upstream submission
-was made during this work. Recheck the allowed in-flight failures, replacement
-count, and remote-test wait when updating the scheduler or worker behavior.
-
-## Local NixOS integration and build results
-
-[`../default.nix`](../default.nix) loads `default.nix` through
-`pythonPackagesExtensions`. From the repository root:
-
-```sh
-nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.python314Packages.pytest-xdist
-```
-
-After consolidating the settings in this directory, the full x86-64-v3 package
-build passed 185 tests, with 6 existing skips and 10 expected failures. Nix
-evaluation confirmed the same outer-worker limit and preserved existing
-patches, with the timeout now applied as a source patch.
-
-The earlier forced concurrent-crash and delayed-startup reproductions passed
-after their fixes; the focused crash test also passed after formatting its
-assertion.

overlays/patches/pytest-xdist/concurrent-worker-crashes.patch

@@ -1,29 +0,0 @@
-Subject: [PATCH] tests: count replacements when checking the worker restart limit
-
-With two workers, another in-flight test may crash after the fourth
-crash requests shutdown. Either four or five failed tests is valid.
-Require exactly three replacements and the failed-tests exit status,
-while preserving the queued-work and no-internal-error assertions.
-
---- a/testing/acceptance_test.py
-+++ b/testing/acceptance_test.py
-@@ -1011,9 +1011,18 @@
-                 "worker*crashed while running*",
-                 "worker*crashed while running*",
-                 "* xdist: maximum crashed workers reached: 3 *",
--                "* 4 failed in *",
-             ]
-         )
-+        # A second in-flight test may crash after shutdown is requested.
-+        # The restart limit constrains replacements, not concurrent failures.
-+        replacements = sum(
-+            line.startswith("replacing crashed worker ") for line in res.stdout.lines
-+        )
-+        assert replacements == 3
-+        failed = res.parseoutcomes()["failed"]
-+        assert failed in (4, 5)
-+        res.assert_outcomes(failed=failed)
-+        assert res.ret == pytest.ExitCode.TESTS_FAILED
-         assert "INTERNALERROR" not in res.stdout.str()
- 
-     def test_max_worker_restart_die(self, pytester: pytest.Pytester) -> None:

overlays/patches/pytest-xdist/default.nix

@@ -1,12 +0,0 @@
-{ pytest-xdist }:
-pytest-xdist.overridePythonAttrs (old: {
-  patches = (old.patches or [ ]) ++ [
-    ./concurrent-worker-crashes.patch
-    ./worker-startup-timeout.patch
-  ];
-
-  # The suite exercises its own worker pools. Limit the outer suite to one worker.
-  preCheck = builtins.replaceStrings [ "--numprocesses=$NIX_BUILD_CORES" ] [ "--numprocesses=1" ] (
-    old.preCheck or ""
-  );
-})

overlays/patches/pytest-xdist/worker-startup-timeout.patch

@@ -1,19 +0,0 @@
-Subject: [PATCH] tests: allow more time for remote worker events
-
-Worker startup can exceed ten seconds on heavily loaded builders. Allow
-the remote-test helper to wait up to sixty seconds for worker events.
-The wait still returns as soon as an event arrives and remains bounded.
-Production worker timeouts and test assertions are unchanged.
-
---- a/testing/test_remote.py
-+++ b/testing/test_remote.py
-@@ -17,7 +17,8 @@
- from xdist.workermanage import WorkerController
- 
- 
--WAIT_TIMEOUT = 10.0
-+# Allow worker events extra time on heavily loaded builders.
-+WAIT_TIMEOUT = 60.0
- 
- 
- def check_marshallable(d: object) -> None:

overlays/patches/scipy/README.md

@@ -1,70 +0,0 @@
-# SciPy STFT test tolerances
-
-The x86-64-v3 build can produce small floating-point residuals in inverse-STFT
-comparisons and scaling round trips. The original bounds reject these results,
-including residuals around `4e-17` where a round trip expects zero for a signal
-with amplitude 2.
-
-## Scope and behavior
-
-`stft-test-tolerances.patch` changes only the signal tests:
-
-- The inverse-STFT comparison in `_scipy_spectral_test_shim.py` uses
-  `max(1e-7, 2 * np.finfo(x.dtype).eps)` as its relative tolerance. Float64
-  keeps the original bound, and the existing i686 override remains.
-- Three scaling round trips in `test_spectral.py` gain an absolute tolerance
-  of one epsilon for the input dtype, allowing small residuals near zero.
-
-The tests remain enabled, and the production STFT implementation is unchanged.
-
-## Reproduction and focused checks
-
-From this directory, apply the patch to a disposable SciPy 1.18.0 checkout:
-
-```sh
-patch --fuzz=0 -d /path/to/scipy -p1 < stft-test-tolerances.patch
-```
-
-Build and install that tree with SciPy's test dependencies. From outside the
-source directory, run the installed tests:
-
-```sh
-python -m pytest --pyargs scipy.signal.tests.test_spectral \
-  -k 'roundtrip_float32 or roundtrip_scaling' -q
-```
-
-Use the same compiler flags and numerical libraries for before/after runs.
-The earlier reproduction called `TestSTFT.test_roundtrip_float32` and
-`TestSTFT.test_roundtrip_scaling` against the x86-64-v3 libraries, then loaded
-patched copies of the test modules. Both failed with the original bounds
-and passed with the adjusted bounds.
-
-## Upstream status
-
-[SciPy issue #25488](https://github.com/scipy/scipy/issues/25488) records
-related test failures with architecture-specific compiler flags. It is
-context for the local tolerance repair; this exact patch has not been
-submitted upstream during this work.
-
-## Local NixOS integration and build results
-
-[`../default.nix`](../default.nix) loads [`default.nix`](default.nix) through
-`pythonPackagesExtensions`, preserving the package's existing patches.
-The override also covers SciPy used to test other Python dependencies,
-including pgvector in portal's shared Python environment.
-
-From the repository root:
-
-```sh
-nix build --no-link -L .#nixosConfigurations.portal-1.pkgs.python314Packages.scipy
-```
-
-The original remote build of patched SciPy 1.18.0 passed 87,723 tests, with
-8,342 skips, 300 expected failures, and 22 unexpected passes. The patch and
-override have been restored byte-for-byte from commit `24cbf74f`; those counts
-describe the earlier full build.
-
-Restoration checks confirmed that the patch applies to the pinned source
-without fuzz, portal's evaluated SciPy retains its existing patch and install
-checks, and pgvector uses the patched SciPy. A full package or system rebuild
-was not repeated for this restoration.

overlays/patches/scipy/default.nix

@@ -1,5 +0,0 @@
-{ scipy }:
-scipy.overridePythonAttrs (old: {
-  # Keep the STFT tests enabled with tolerances for x86-64-v3 rounding.
-  patches = (old.patches or [ ]) ++ [ ./stft-test-tolerances.patch ];
-})

overlays/patches/scipy/stft-test-tolerances.patch

@@ -1,51 +0,0 @@
-Subject: [PATCH] signal: allow floating-point rounding in STFT tests
-
-Keep the STFT tests enabled for x86-64-v3 builds. Allow two float32
-epsilons of relative error when comparing inverse-STFT implementations;
-float64 and the existing i686 override remain unchanged. Allow one
-float64 epsilon of absolute error in all three scaling round trips,
-which otherwise require exact zeros (observed residual: 4e-17 for a
-signal with amplitude 2).
-
-Upstream issue: https://github.com/scipy/scipy/issues/25488
-
---- a/scipy/signal/tests/_scipy_spectral_test_shim.py
-+++ b/scipy/signal/tests/_scipy_spectral_test_shim.py
-@@ -294,7 +294,7 @@
- 
-     # Adapted tolerances to account for resolution loss:
-     atol = np.finfo(x.dtype).resolution*2  # instead of default atol = 0
--    rtol = 1e-7  # default for np.allclose()
-+    rtol = max(1e-7, 2 * np.finfo(x.dtype).eps)
- 
-     # Relax atol on 32-Bit platforms a bit to pass CI tests.
-     #  - Not clear why there are discrepancies (in the FFT maybe?)
---- a/scipy/signal/tests/test_spectral.py
-+++ b/scipy/signal/tests/test_spectral.py
-@@ -2044,7 +2044,7 @@
- 
-         # Test round trip:
-         x1 = istft(Zs, boundary=True, scaling='spectrum')[1]
--        assert_allclose(x1, x)
-+        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
- 
-         # For a Hann-windowed 256 sample length FFT, we expect a peak at
-         # frequency 64 (since it is 1/4 the length of X) with a height of 1
-@@ -2074,7 +2074,7 @@
- 
-         # Test round trip:
-         x1 = istft(Zp, input_onesided=False, boundary=True, scaling='psd')[1]
--        assert_allclose(x1, x)
-+        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
- 
-         # The power of the one-sided psd-scaled STFT can be determined
-         # analogously (note that the two sides are not of equal shape):
-@@ -2094,7 +2094,7 @@
- 
-         # Test round trip:
-         x1 = istft(Zp0, input_onesided=True, boundary=True, scaling='psd')[1]
--        assert_allclose(x1, x)
-+        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
- 
- 
- class TestSampledSpectralRepresentations:

overlays/patches/sentry-sdk/README.md

@@ -1,58 +0,0 @@
-# Sentry SDK thread-metadata test isolation
-
-The fallback tests globally mock `threading.current_thread` while a worker
-is running. Python 3.14's `Thread.join()` also calls that function. A one-use
-mock can therefore be consumed by the wrong caller or raise `StopIteration`
-when the main thread joins the worker.
-
-## Scope and behavior
-
-`isolate-threading-mocks.patch` changes three neighboring thread-metadata
-tests in `tests/test_utils.py`, including the formerly excluded
-`test_get_current_thread_meta_main_thread`.
-
-Each test replaces only `sentry_sdk.utils.threading`, wraps the real module
-for unmocked operations, and sets the SDK lookup's return value. The real
-`Thread.join()` continues using Python's unmodified `threading` module.
-The fallback-result assertions remain; SDK production code is unchanged.
-
-## Reproduction and focused checks
-
-Use a disposable Sentry SDK 2.66.0 checkout and its Python test dependencies.
-From this directory:
-
-```sh
-patch --fuzz=0 -d /path/to/sentry-python -p1 < isolate-threading-mocks.patch
-cd /path/to/sentry-python
-python -m pytest tests/test_utils.py -k get_current_thread_meta -q
-```
-
-To reproduce the race, hold the worker inside its mock just after
-`get_current_thread_meta()` returns, signal that point to the main thread,
-and call `Thread.join()` before releasing the worker. Use an independent
-bounded release so the patched join can finish. The original test raises
-`StopIteration` in `join`; the patched test passes under the same schedule.
-Perform this scheduling instrumentation only in a disposable checkout.
-
-## Upstream status
-
-This is a standalone test patch for Sentry SDK 2.66.0. No upstream submission
-was made during this work. Recheck mock isolation and Python threading
-behavior when upgrading the SDK or interpreter.
-
-## Local NixOS integration and build results
-
-[`../default.nix`](../default.nix) loads `default.nix` through
-`pythonPackagesExtensions`. From the repository root:
-
-```sh
-nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.python314Packages.sentry-sdk
-```
-
-The patched package passed 2,356 tests with 116 existing skips on Python
-3.14.7. The controlled join reproduction failed before the fix and passed
-after it.
-
-That package build used the preceding dependency set with this patch to avoid
-unrelated rebuilds after pytest-xdist changed. The integrated host derivation
-was evaluated; a complete NixOS rebuild was not performed.

overlays/patches/sentry-sdk/default.nix

@@ -1,4 +0,0 @@
-{ sentry-sdk }:
-sentry-sdk.overridePythonAttrs (old: {
-  patches = (old.patches or [ ]) ++ [ ./isolate-threading-mocks.patch ];
-})

overlays/patches/sentry-sdk/isolate-threading-mocks.patch

@@ -1,41 +0,0 @@
-Subject: [PATCH] tests: isolate SDK thread lookup mocks from Python threading
-
-Thread.join also calls threading.current_thread on Python 3.14. A global
-single-use side effect can be consumed by join instead of the SDK, or
-raise StopIteration in join after the SDK consumes it. Patch the SDK's
-module binding and delegate unmocked operations to the real module.
-Apply the same isolation to the adjacent invalid-thread fallback tests.
-
---- a/tests/test_utils.py
-+++ b/tests/test_utils.py
-@@ -914,7 +914,8 @@
-     results = Queue(maxsize=1)
- 
-     def target():
--        with mock.patch("threading.current_thread", side_effect=["fake thread"]):
-+        with mock.patch("sentry_sdk.utils.threading", wraps=threading) as sdk_threading:
-+            sdk_threading.current_thread.return_value = "fake thread"
-             results.put(get_current_thread_meta())
- 
-     thread = threading.Thread(target=target)
-@@ -930,7 +931,9 @@
- 
-     def target():
-         # mock that somehow the current thread doesn't exist
--        with mock.patch("threading.current_thread", side_effect=[None]):
-+        # Keep the real threading module intact for concurrent Thread.join calls.
-+        with mock.patch("sentry_sdk.utils.threading", wraps=threading) as sdk_threading:
-+            sdk_threading.current_thread.return_value = None
-             results.put(get_current_thread_meta())
- 
-     main_thread = threading.main_thread()
-@@ -945,7 +948,8 @@
-     results = Queue(maxsize=1)
- 
-     def target():
--        with mock.patch("threading.current_thread", return_value="fake thread"):
-+        with mock.patch("sentry_sdk.utils.threading", wraps=threading) as sdk_threading:
-+            sdk_threading.current_thread.return_value = "fake thread"
-             results.put(get_current_thread_meta())
- 
-     main_thread = threading.main_thread()

pyproject.toml

@@ -6,6 +6,51 @@ authors = [{ name = "Richie Cahill", email = "richie@tmmworkshop.com" }]
 requires-python = "~=3.14.0"
 readme = "README.md"
 license = "MIT"
+# these dependencies are a best effort and aren't guaranteed to work
+# for up-to-date dependencies, see overlays/default.nix
+dependencies = [
+    "alembic",
+    "apprise",
+    "apscheduler",
+    "beautifulsoup4",
+    "bm25s",
+    "ebooklib",
+    "fastapi",
+    "fastapi-cli",
+    "httpx",
+    "jinja2",
+    "pgvector",
+    "polars",
+    "psycopg[binary]",
+    "pydantic",
+    "pydantic-settings",
+    "python-multipart",
+    "sqlalchemy[asyncio]",
+    "tenacity",
+    "tiktoken",
+    "tinytuya",
+    "typer",
+    "uvicorn",
+    "websockets",
+    "yake",
+]
+
+[project.scripts]
+database = "python.database_cli:app"
+whisper-transcribe = "python.tools.whisper.transcribe:main"
+
+[dependency-groups]
+dev = [
+    "aiosqlite",
+    "mypy",
+    "pyfakefs",
+    "pytest-asyncio",
+    "pytest-cov",
+    "pytest-mock",
+    "pytest-xdist",
+    "pytest",
+    "ruff",
+]
 
 [tool.ruff]
 

python/ebook_search/config.py

@@ -37,7 +37,7 @@ class RerankConfig(BaseSettings):
     model_config = SettingsConfigDict(env_prefix="EBOOK_SEARCH_RERANK_", frozen=True, protected_namespaces=())
 
     enabled: bool = True
-    base_url: str = "http://bob:8001"
+    base_url: str = "http://192.168.90.25:8001"
     model: str = "qwen3-reranker-06b"
     candidates: int = 24
     timeout_seconds: float = 30.0
@@ -67,7 +67,7 @@ class EbookSearchConfig(BaseSettings):
     )
     chat_model: str = "deepseek-v4-flash"
     answer_enabled: bool = True
-    embedding_base_url: str = "http://bob:8000/v1"
+    embedding_base_url: str = "http://192.168.90.25:8000/v1"
     embedding_api_key: str = "not-needed"
     embedding_model: str = "qwen3-embedding-0.6b"
     embedding_batch_size: int = 32

python/ebook_search/docker/docker-compose.yml

@@ -9,6 +9,8 @@ services:
     restart: unless-stopped
     ports:
       - "${EBOOK_SEARCH_PORT:-8070}:8070"
+    extra_hosts:
+      - "jeeves:192.168.90.40"
     env_file:
       - ../../../.env
     environment:

python/gitea_flake_lock.py

@@ -14,7 +14,7 @@ DEFAULT_BASE_BRANCH = "main"
 DEFAULT_BRANCH = "automation/update-flake-lock"
 DEFAULT_GITEA_URL = "https://gitea.tmmworkshop.com"
 PR_LABELS = ["dependencies", "automated", "flake_lock_update"]
-PR_CHECK_WORKFLOWS = ["build_systems.yml", "treefmt.yml"]
+PR_CHECK_WORKFLOWS = ["build_systems.yml", "treefmt.yml", "pytest.yml"]
 PR_TITLE = "Update flake.lock"
 PR_BODY = "Automated flake.lock update."
 
@@ -84,13 +84,13 @@ def push_branch(*, branch: str) -> None:
     run_cmd(["git", "push", "origin", f"HEAD:{branch}", "--force"])
 
 
-def _required_jeeves_bot_token() -> str:
+def _required_gitea_token() -> str:
     """Read the required Gitea token from the environment."""
-    token = getenv("JEEVES_BOT_TOKEN")
+    token = getenv("GITEA_TOKEN")
     if token:
         return token
 
-    msg = "JEEVES_BOT_TOKEN environment variable is required"
+    msg = "GITEA_TOKEN environment variable is required"
     raise RuntimeError(msg)
 
 
@@ -111,7 +111,7 @@ def update(
     owner, repo_name = split_repo_name(repo)
     with GiteaClient(
         base_url=getenv("GITEA_URL", DEFAULT_GITEA_URL),
-        token=_required_jeeves_bot_token(),
+        token=_required_gitea_token(),
     ) as client:
         pull_request = ensure_flake_lock_pull_request(
             client,
@@ -134,7 +134,7 @@ def merge(
     owner, repo_name = split_repo_name(repo)
     with GiteaClient(
         base_url=getenv("GITEA_URL", DEFAULT_GITEA_URL),
-        token=_required_jeeves_bot_token(),
+        token=_required_gitea_token(),
     ) as client:
         pull_request = find_flake_lock_pull_request(client, owner=owner, repo=repo_name)
         if not pull_request:

python/installer/__main__.py

@@ -176,21 +176,6 @@ def create_zfs_datasets() -> None:
     run_output(("zfs", "create", "root_pool/home"))
     run_output(("zfs", "create", "-o", "reservation=1G", "root_pool/var"))
     run_output(("zfs", "create", "-o", "compression=zstd-9", "-o", "reservation=10G", "root_pool/nix"))
-    run_output(
-        (
-            "zfs",
-            "create",
-            "-o",
-            "sync=disabled",
-            "-o",
-            "redundant_metadata=some",
-            "-o",
-            "normalization=none",
-            "-o",
-            "utf8only=off",
-            "root_pool/nix_build",
-        )
-    )
     datasets = run_output(("zfs", "list", "-o", "name"))
 
     expected_datasets = {
@@ -198,7 +183,6 @@ def create_zfs_datasets() -> None:
         "root_pool/home",
         "root_pool/var",
         "root_pool/nix",
-        "root_pool/nix_build",
     }
     missing_datasets = expected_datasets.difference(datasets.splitlines())
     if missing_datasets:
@@ -264,7 +248,6 @@ def create_nix_hardware_file(mnt_dir: str, disks: Sequence[str], encrypt: str |
         '    "/home" = {\n      device = "root_pool/home";\n      fsType = "zfs";\n    };\n\n'
         '    "/var" = {\n      device = "root_pool/var";\n      fsType = "zfs";\n    };\n\n'
         '    "/nix" = {\n      device = "root_pool/nix";\n      fsType = "zfs";\n    };\n\n'
-        '    "/nix/var/nix/builds" = {\n      device = "root_pool/nix_build";\n      fsType = "zfs";\n    };\n\n'
         '    "/boot" = {\n'
         f'      device = "/dev/disk/by-uuid/{get_boot_drive_id(disks[0])}";\n'
         '      fsType = "vfat";\n      options = [\n        "fmask=0077"\n'
@@ -287,7 +270,6 @@ def install_nixos(mnt_dir: str, disks: Sequence[str], encrypt: str | None) -> No
     run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/home", f"{mnt_dir}/home"))
     run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/var", f"{mnt_dir}/var"))
     run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/nix", f"{mnt_dir}/nix"))
-    run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/nix_build", f"{mnt_dir}/nix/var/nix/builds"))
 
     for disk in disks:
         run_output(("mkfs.vfat", "-n", "EFI", f"{disk}-part1"))

python/signal_alert.py

@@ -5,13 +5,10 @@ from __future__ import annotations
 import logging
 from os import getenv
 
-import httpx
+from apprise import Apprise
 
 logger = logging.getLogger(__name__)
 
-SIGNAL_API_URL = "http://localhost:8989/v2/send"
-SIGNAL_API_TIMEOUT = 4.0
-
 
 def signal_alert(body: str, title: str = "") -> None:
     """Send a signal alert.
@@ -20,31 +17,14 @@ def signal_alert(body: str, title: str = "") -> None:
         body (str): The body of the alert.
         title (str, optional): The title of the alert. Defaults to "".
     """
+    apprise_client = Apprise()
+
     from_phone = getenv("SIGNAL_ALERT_FROM_PHONE")
     to_phone = getenv("SIGNAL_ALERT_TO_PHONE")
     if not from_phone or not to_phone:
         logger.info("SIGNAL_ALERT_FROM_PHONE or SIGNAL_ALERT_TO_PHONE not set")
         return
 
-    # Apprise's Signal integration did not support titles, so preserve that behavior.
-    if title:
-        logger.debug("Signal does not support notification titles; ignoring title")
+    apprise_client.add(f"signal://localhost:8989/{from_phone}/{to_phone}")
 
-    try:
-        response = httpx.post(
-            SIGNAL_API_URL,
-            json={
-                "message": body,
-                "number": from_phone,
-                "recipients": [to_phone],
-                "text_mode": "normal",
-            },
-            timeout=SIGNAL_API_TIMEOUT,
-            follow_redirects=True,
-        )
-    except httpx.HTTPError:
-        logger.exception("Unable to contact the Signal API")
-        return
-
-    if response.status_code not in {httpx.codes.OK, httpx.codes.CREATED}:
-        logger.error("Signal API returned HTTP status %d", response.status_code)
+    apprise_client.notify(title=title, body=body)

python/system_tests/__init__.py

@@ -0,0 +1 @@
+"""system_tests."""

python/system_tests/components.py

@@ -0,0 +1,99 @@
+"""Validate Jeeves."""
+
+from __future__ import annotations
+
+import logging
+from copy import copy
+from re import search
+from time import sleep
+from typing import TYPE_CHECKING
+
+from python.common import bash_wrapper
+from python.zfs import Zpool
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+logger = logging.getLogger(__name__)
+
+
+def zpool_tests(pool_names: Sequence[str], zpool_capacity_threshold: int = 90) -> list[str] | None:
+    """Check the zpool health and capacity.
+
+    Args:
+        pool_names (Sequence[str]): A list of pool names to test.
+        zpool_capacity_threshold (int, optional): The threshold for the zpool capacity. Defaults to 90.
+
+    Returns:
+        list[str] | None: A list of errors if any.
+    """
+    logger.info("Testing zpool")
+
+    errors: list[str] = []
+    for pool_name in pool_names:
+        pool = Zpool(pool_name)
+        if pool.health != "ONLINE":
+            errors.append(f"{pool.name} is {pool.health}")
+        if pool.capacity >= zpool_capacity_threshold:
+            errors.append(f"{pool.name} is low on space")
+
+    upgrade_status, _ = bash_wrapper("zpool upgrade")
+    if not search(r"Every feature flags pool has all supported and requested features enabled.", upgrade_status):
+        errors.append("ZPool out of date run `sudo zpool upgrade -a`")
+
+    return errors
+
+
+def systemd_tests(
+    service_names: Sequence[str],
+    max_retries: int = 30,
+    retry_delay_secs: int = 1,
+    retryable_statuses: Sequence[str] | None = None,
+    valid_statuses: Sequence[str] | None = None,
+) -> list[str] | None:
+    """Tests a systemd services.
+
+    Args:
+        service_names (Sequence[str]): A list of service names to test.
+        max_retries (int, optional): The maximum number of retries. Defaults to 30.
+            minimum value is 1.
+        retry_delay_secs (int, optional): The delay between retries in seconds. Defaults to 1.
+            minimum value is 1.
+        retryable_statuses (Sequence[str] | None, optional): A list of retryable statuses. Defaults to None.
+        valid_statuses (Sequence[str] | None, optional): A list of valid statuses. Defaults to None.
+
+    Returns:
+        list[str] | None: A list of errors if any.
+    """
+    logger.info("Testing systemd service")
+
+    max_retries = max(max_retries, 1)
+    retry_delay_secs = max(retry_delay_secs, 1)
+    last_try = max_retries - 1
+
+    if retryable_statuses is None:
+        retryable_statuses = ("inactive\n", "activating\n")
+
+    if valid_statuses is None:
+        valid_statuses = ("active\n",)
+
+    service_names_set = set(service_names)
+
+    errors: set[str] = set()
+    for retry in range(max_retries):
+        if not service_names_set:
+            break
+        logger.info(f"Testing systemd service in {retry + 1} of {max_retries}")
+        service_names_to_test = copy(service_names_set)
+        for service_name in service_names_to_test:
+            service_status, _ = bash_wrapper(f"systemctl is-active {service_name}")
+            if service_status in valid_statuses:
+                service_names_set.remove(service_name)
+                continue
+            if service_status in retryable_statuses and retry < last_try:
+                continue
+            errors.add(f"{service_name} is {service_status.strip()}")
+
+        sleep(retry_delay_secs)
+
+    return list(errors)

python/system_tests/validate_system.py

@@ -0,0 +1,67 @@
+"""Validate {server_name}."""
+
+import logging
+import sys
+import tomllib
+from os import environ
+from pathlib import Path  # noqa: TC003 This is required for the typer CLI
+from socket import gethostname
+
+import typer
+
+from python.common import configure_logger
+from python.signal_alert import signal_alert
+from python.system_tests.components import systemd_tests, zpool_tests
+
+logger = logging.getLogger(__name__)
+
+
+def load_config_data(config_file: Path) -> dict[str, list[str]]:
+    """Load a TOML configuration file.
+
+    Args:
+        config_file (Path): The path to the configuration file.
+
+    Returns:
+        dict: The configuration data.
+    """
+    return tomllib.loads(config_file.read_text())
+
+
+def main(config_file: Path) -> None:
+    """Main."""
+    configure_logger(level=environ.get("LOG_LEVEL", "INFO"))
+
+    server_name = gethostname()
+    logger.info(f"Starting {server_name} validation")
+
+    config_data = load_config_data(config_file)
+
+    errors: list[str] = []
+    try:
+        if config_data.get("zpools") and (zpool_errors := zpool_tests(config_data["zpools"])):
+            errors.extend(zpool_errors)
+
+        if config_data.get("services") and (systemd_errors := systemd_tests(config_data["services"])):
+            errors.extend(systemd_errors)
+
+    except Exception as error:
+        logger.exception(f"{server_name} validation failed")
+        errors.append(f"{server_name} validation failed: {error}")
+
+    if errors:
+        logger.error(f"{server_name} validation failed: \n{'\n'.join(errors)}")
+        signal_alert(f"{server_name} validation failed {errors}")
+
+        sys.exit(1)
+
+    logger.info(f"{server_name} validation passed")
+
+
+def cli() -> None:
+    """CLI."""
+    typer.run(main)
+
+
+if __name__ == "__main__":
+    cli()

python/tools/snapshot_manager.py

@@ -23,6 +23,7 @@ def main(config_file: Path) -> None:
     """Main."""
     configure_logger(level="DEBUG")
     logger.info("Starting snapshot_manager")
+    failures: list[str] = []
 
     try:
         time_stamp = get_time_stamp()
@@ -34,16 +35,23 @@ def main(config_file: Path) -> None:
                 msg = f"{dataset.name} failed to create snapshot {time_stamp}"
                 logger.error(msg)
                 signal_alert(msg)
+                failures.append(msg)
                 continue
             count_lookup = get_count_lookup(config_file, dataset.name)
             logger.info(f"using {count_lookup} for {dataset.name}")
-            get_snapshots_to_delete(dataset, count_lookup)
+            failures.extend(get_snapshots_to_delete(dataset, count_lookup))
     except Exception:
         logger.exception("snapshot_manager failed")
         signal_alert("snapshot_manager failed")
         sys.exit(1)
-    else:
-        logger.info("snapshot_manager completed")
+
+    if failures:
+        logger.error(f"snapshot_manager completed with {len(failures)} errors")
+        for failure in failures:
+            logger.error(f"  {failure}")
+        sys.exit(1)
+
+    logger.info("snapshot_manager completed")
 
 
 def get_count_lookup(config_file: Path, dataset_name: str) -> dict[str, int]:
@@ -92,19 +100,29 @@ def load_config_data(config_file: Path) -> dict[str, dict[str, int]]:
 def get_snapshots_to_delete(
     dataset: Dataset,
     count_lookup: dict[str, int],
-) -> None:
+) -> list[str]:
     """Get snapshots to delete.
 
     Args:
         dataset (Dataset): the dataset
         count_lookup (dict[str, int]): the count lookup
+
+    Returns:
+        list[str]: Snapshot deletion failures encountered while pruning.
     """
+    for retention_class in ("15_min", "hourly", "daily", "monthly"):
+        count = count_lookup.get(retention_class)
+        if not isinstance(count, int) or isinstance(count, bool) or count < 0:
+            error = f"{retention_class} retention must be a non-negative integer, got {count!r}"
+            raise ValueError(error)
+
+    failures: list[str] = []
     snapshots = dataset.get_snapshots()
 
     logger.info(f"calculating snapshots for {dataset.name} to be deleted")
     if not snapshots:
         logger.info(f"{dataset.name} has no snapshots")
-        return
+        return failures
 
     filters = (
         ("15_min", re_compile(r"auto_\d{10}(?:15|30|45)")),
@@ -129,6 +147,9 @@ def get_snapshots_to_delete(
                 error_message = f"{dataset.name}@{snapshot} failed to delete: {error}"
                 signal_alert(error_message)
                 logger.error(error_message)
+                failures.append(error_message)
+
+    return failures
 
 
 def get_time_stamp() -> str:

python/tools/zfs_manager.py

@@ -0,0 +1,340 @@
+"""zfs_manager.
+
+Reconciles the live zfs datasets against a declaration generated by
+common/optional/zfs_manager.nix. Datasets are created and properties are
+corrected, but nothing is ever destroyed or renamed.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sys
+from pathlib import Path  # noqa: TC003 This is required for the typer CLI
+
+import typer
+
+from python.common import configure_logger
+from python.signal_alert import signal_alert
+from python.zfs import create_dataset, get_properties, list_dataset_names, set_property
+
+logger = logging.getLogger(__name__)
+
+# Properties that can only be chosen at creation time. Attempting to zfs set
+# these fails on every run, so a mismatch is reported instead of retried.
+CREATE_ONLY_PROPERTIES = frozenset(
+    {
+        "casesensitivity",
+        "encryption",
+        "keyformat",
+        "normalization",
+        "utf8only",
+        "volblocksize",
+    },
+)
+
+# Properties whose values zfs reports in bytes but which are conventionally
+# declared with a size suffix, so "16k" and "16384" mean the same thing.
+SIZE_PROPERTIES = frozenset(
+    {
+        "quota",
+        "recordsize",
+        "refquota",
+        "refreservation",
+        "reservation",
+        "special_small_blocks",
+        "volblocksize",
+        "volsize",
+    },
+)
+
+SIZE_SUFFIXES = {"b": 1, "k": 1024, "m": 1024**2, "g": 1024**3, "t": 1024**4, "p": 1024**5}
+
+# Sources that mean the value was deliberately put on this dataset rather than
+# inherited from a parent or left at the zfs default.
+LOCAL_SOURCES = ("local", "received")
+
+
+class ReconciliationError(RuntimeError):
+    """One or more datasets could not be brought in line with the declaration."""
+
+    def __init__(self, failures: list[str]) -> None:
+        """Record the individual failures behind this run's exit code."""
+        self.failures = failures
+        super().__init__(f"ZFS reconciliation failed with {len(failures)} errors")
+
+
+def main(config_file: Path, *, dry_run: bool = False) -> None:
+    """Main.
+
+    Args:
+        config_file (Path): The path to the generated dataset declaration.
+        dry_run (bool): Log the changes that would be made without making them.
+    """
+    configure_logger(level="DEBUG")
+    logger.info(f"Starting zfs_manager {dry_run=}")
+
+    try:
+        reconcile(config_file, dry_run=dry_run)
+    except ReconciliationError as error:
+        summary = error
+    except Exception:
+        logger.exception("zfs_manager failed")
+        signal_alert("zfs_manager failed")
+        sys.exit(1)
+    else:
+        logger.info("zfs_manager completed")
+        return
+
+    # Each failure was logged and alerted as it happened. Repeating them
+    # together puts the whole picture at the end of the journal, which is what
+    # systemctl status shows. No traceback: this is an expected outcome, not a
+    # crash, and a stack trace would only bury the list.
+    logger.error(str(summary))
+    for failure in summary.failures:
+        logger.error(f"  {failure}")
+    sys.exit(1)
+
+
+def reconcile(config_file: Path, *, dry_run: bool) -> None:
+    """Bring every declared dataset in line, collecting problems as it goes.
+
+    One bad dataset must not hide the state of the others, so everything is
+    checked before anything is raised.
+
+    Args:
+        config_file (Path): The path to the generated dataset declaration.
+        dry_run (bool): Log the changes without making them.
+
+    Raises:
+        ReconciliationError: If anything could not be reconciled.
+    """
+    declared = json.loads(config_file.read_text())["datasets"]
+    existing = set(list_dataset_names())
+    unusable: set[str] = set()
+    failures: list[str] = []
+
+    # Parents before children so a newly created parent exists by the time its
+    # children are reconciled.
+    for name in sorted(declared, key=lambda name: (name.count("/"), name)):
+        entry = declared[name]
+
+        # Declared purely to record retention, its properties belong to
+        # whoever set them.
+        if not entry.get("manageProperties", True):
+            logger.debug(f"{name} is declared but its properties are not managed")
+            continue
+
+        if has_unusable_parent(name, unusable):
+            failures.append(fail(f"cannot reconcile {name}, its parent is missing"))
+            continue
+
+        if name in existing:
+            failures.extend(reconcile_dataset(name, entry["properties"], dry_run=dry_run))
+            continue
+
+        created, failure = handle_missing_dataset(name, entry, dry_run=dry_run)
+        if failure is not None:
+            failures.append(failure)
+        if created:
+            existing.add(name)
+        elif not dry_run:
+            unusable.add(name)
+
+    report_undeclared_datasets(existing, declared)
+
+    if failures:
+        raise ReconciliationError(failures)
+
+
+def fail(message: str) -> str:
+    """Log and alert a problem, and hand it back for the failure tally.
+
+    Args:
+        message (str): What went wrong.
+
+    Returns:
+        str: The same message, so the caller can collect it.
+    """
+    logger.error(message)
+    signal_alert(message)
+    return message
+
+
+def has_unusable_parent(name: str, unusable: set[str]) -> bool:
+    """Check whether an ancestor of a dataset is missing.
+
+    Args:
+        name (str): The name of the dataset.
+        unusable (set[str]): The datasets that do not exist and were not created.
+
+    Returns:
+        bool: True if any ancestor is unusable.
+    """
+    parts = name.split("/")
+    return any("/".join(parts[:depth]) in unusable for depth in range(1, len(parts)))
+
+
+def handle_missing_dataset(name: str, entry: dict, *, dry_run: bool) -> tuple[bool, str | None]:
+    """Deal with a declared dataset that is not on the system.
+
+    Pool roots are never created, and neither is anything the declaration marks
+    as provisioned outside of nix, such as an encryption root whose key
+    settings cannot be reproduced from the declaration.
+
+    Args:
+        name (str): The name of the dataset.
+        entry (dict): The declaration for this dataset.
+        dry_run (bool): Log the change without making it.
+
+    Returns:
+        tuple[bool, str | None]: Whether the dataset now exists, and a failure
+            message if there was one.
+    """
+    properties = entry["properties"]
+
+    if "/" not in name:
+        return False, fail(f"pool {name} is declared but does not exist, zfs_manager does not create pools")
+
+    if not entry.get("createIfMissing", True):
+        return False, fail(
+            f"{name} is declared but does not exist, and is marked as created outside of nix. "
+            "It has to be made by hand, see systems/jeeves/scripts/zfs.sh.",
+        )
+
+    if dry_run:
+        logger.info(f"would create {name} with {properties}")
+        return False, None
+
+    logger.info(f"creating {name} with {properties}")
+    if error := create_dataset(name, properties):
+        return False, fail(error)
+
+    return True, None
+
+
+def reconcile_dataset(name: str, properties: dict[str, str], *, dry_run: bool) -> list[str]:
+    """Bring an existing dataset in line with its declared properties.
+
+    Args:
+        name (str): The name of the dataset.
+        properties (dict[str, str]): The declared properties.
+        dry_run (bool): Log the changes without making them.
+
+    Returns:
+        list[str]: Anything that could not be put right.
+    """
+    failures: list[str] = []
+    current = get_properties(name)
+
+    for key, wanted in sorted(properties.items()):
+        current_value, _ = current.get(key, ("-", "-"))
+        if values_match(key, wanted, current_value):
+            continue
+
+        if key in CREATE_ONLY_PROPERTIES:
+            # Nothing can put this right while the dataset exists, so it is a
+            # hard failure rather than a warning that repeats unnoticed.
+            failures.append(
+                fail(
+                    f"{name} {key} is {current_value} but {wanted} is declared, "
+                    f"{key} can only be set when the dataset is created",
+                ),
+            )
+            continue
+
+        if dry_run:
+            logger.info(f"would set {key}={wanted} on {name}, currently {current_value}")
+            continue
+
+        logger.info(f"setting {key}={wanted} on {name}, was {current_value}")
+        if error := set_property(name, key, wanted):
+            failures.append(fail(error))
+
+    report_undeclared_properties(name, properties, current)
+    return failures
+
+
+def report_undeclared_properties(name: str, properties: dict[str, str], current: dict[str, tuple[str, str]]) -> None:
+    """Warn about properties set on the dataset but absent from the declaration.
+
+    Inherited and default values are silent, they are not drift. A locally set
+    value that nix does not know about was changed outside of this tool and
+    will be lost the next time the dataset is recreated, so it is worth saying.
+
+    Args:
+        name (str): The name of the dataset.
+        properties (dict[str, str]): The declared properties.
+        current (dict[str, tuple[str, str]]): The live properties keyed to (value, source).
+    """
+    for key, (value, source) in sorted(current.items()):
+        # User properties such as nixos:shutdown-time are written by other
+        # tools and are not something a dataset declaration should own.
+        if key in properties or ":" in key or source not in LOCAL_SOURCES:
+            continue
+
+        logger.warning(f"{name} has {key}={value} set outside of nix")
+        signal_alert(f"{name} has {key}={value} set outside of nix")
+
+
+def report_undeclared_datasets(existing: set[str], declared: dict[str, dict]) -> None:
+    """Warn about datasets that exist but are not declared.
+
+    These are left completely alone. They still get snapshots through the
+    default retention table.
+
+    Args:
+        existing (set[str]): The names of every live dataset.
+        declared (dict[str, dict]): The declaration.
+    """
+    for name in sorted(existing - set(declared)):
+        logger.warning(f"{name} exists but is not declared in nix")
+
+
+def values_match(key: str, wanted: str, current: str) -> bool:
+    """Compare a declared property value against the live one.
+
+    Args:
+        key (str): The property name.
+        wanted (str): The declared value.
+        current (str): The live value.
+
+    Returns:
+        bool: True if the two values mean the same thing.
+    """
+    if key in SIZE_PROPERTIES:
+        wanted_size = parse_size(wanted)
+        current_size = parse_size(current)
+        if wanted_size is not None and current_size is not None:
+            return wanted_size == current_size
+
+    return wanted == current
+
+
+def parse_size(value: str) -> int | None:
+    """Convert a zfs size such as 16k or 1M into bytes.
+
+    Args:
+        value (str): The size to convert.
+
+    Returns:
+        int | None: The size in bytes, or None if it is not a size.
+    """
+    value = value.strip()
+    if value.isdigit():
+        return int(value)
+
+    number, suffix = value[:-1], value[-1:].lower()
+    if suffix in SIZE_SUFFIXES and number.isdigit():
+        return int(number) * SIZE_SUFFIXES[suffix]
+
+    return None
+
+
+def cli() -> None:
+    """CLI."""
+    typer.run(main)
+
+
+if __name__ == "__main__":
+    cli()

python/van_weather/__init__.py

@@ -0,0 +1 @@
+"""Van weather service - fetches weather with masked GPS location."""

python/van_weather/main.py

@@ -0,0 +1,293 @@
+"""Van weather service - fetches weather with masked GPS for privacy."""
+
+import logging
+from datetime import UTC, datetime
+from typing import Annotated, Any
+
+import httpx
+import typer
+from apscheduler.schedulers.blocking import BlockingScheduler
+from tenacity import before_sleep_log, retry, stop_after_attempt, wait_fixed
+
+from python.common import configure_logger
+from python.van_weather.models import Config, DailyForecast, HourlyForecast, Weather
+
+# Map Pirate Weather icons to Home Assistant conditions
+CONDITION_MAP = {
+    "clear-day": "sunny",
+    "clear-night": "clear-night",
+    "rain": "rainy",
+    "snow": "snowy",
+    "sleet": "snowy-rainy",
+    "wind": "windy",
+    "fog": "fog",
+    "cloudy": "cloudy",
+    "partly-cloudy-day": "partlycloudy",
+    "partly-cloudy-night": "partlycloudy",
+}
+
+logger = logging.getLogger(__name__)
+
+
+@retry(
+    stop=stop_after_attempt(3),
+    wait=wait_fixed(5),
+    before_sleep=before_sleep_log(logger, logging.WARNING),
+    reraise=True,
+)
+def get_ha_state(url: str, token: str, entity_id: str) -> float:
+    """Get numeric state from Home Asasistant entity."""
+    response = httpx.get(
+        f"{url}/api/states/{entity_id}",
+        headers={"Authorization": f"Bearer {token}"},
+        timeout=30,
+    )
+    response.raise_for_status()
+    state = response.json()["state"]
+    if state in ("unavailable", "unknown"):
+        error = f"{entity_id} is {state}"
+        raise ValueError(error)
+    return float(state)
+
+
+def parse_daily_forecast(data: dict[str, dict[str, Any]]) -> list[DailyForecast]:
+    """Parse daily forecast from Pirate Weather API."""
+    daily = data.get("daily", {}).get("data", [])
+    daily_forecasts = []
+    for day in daily[:8]:  # Up to 8 days
+        time_stamp = day.get("time")
+        if time_stamp:
+            date_time = datetime.fromtimestamp(time_stamp, tz=UTC).isoformat()
+            daily_forecasts.append(
+                DailyForecast(
+                    date_time=date_time,
+                    condition=CONDITION_MAP.get(day.get("icon", ""), "cloudy"),
+                    temperature=day.get("temperatureHigh"),
+                    templow=day.get("temperatureLow"),
+                    precipitation_probability=day.get("precipProbability"),
+                    moon_phase=day.get("moonPhase"),
+                    wind_gust=day.get("windGust"),
+                    cloud_cover=day.get("cloudCover"),
+                )
+            )
+
+    return daily_forecasts
+
+
+def parse_hourly_forecast(data: dict[str, dict[str, Any]]) -> list[HourlyForecast]:
+    """Parse hourly forecast from Pirate Weather API."""
+    hourly = data.get("hourly", {}).get("data", [])
+    hourly_forecasts = []
+    for hour in hourly[:48]:  # Up to 48 hours
+        time_stamp = hour.get("time")
+        if time_stamp:
+            date_time = datetime.fromtimestamp(time_stamp, tz=UTC).isoformat()
+            hourly_forecasts.append(
+                HourlyForecast(
+                    date_time=date_time,
+                    condition=CONDITION_MAP.get(hour.get("icon", ""), "cloudy"),
+                    temperature=hour.get("temperature"),
+                    precipitation_probability=hour.get("precipProbability"),
+                )
+            )
+    return hourly_forecasts
+
+
+@retry(
+    stop=stop_after_attempt(3),
+    wait=wait_fixed(5),
+    before_sleep=before_sleep_log(logger, logging.WARNING),
+    reraise=True,
+)
+def fetch_weather(api_key: str, lat: float, lon: float) -> Weather:
+    """Fetch weather from Pirate Weather API."""
+    url = f"https://api.pirateweather.net/forecast/{api_key}/{lat},{lon}"
+    response = httpx.get(url, params={"units": "us"}, timeout=30)
+    response.raise_for_status()
+    data = response.json()
+
+    daily_forecasts = parse_daily_forecast(data)
+    hourly_forecasts = parse_hourly_forecast(data)
+
+    current = data.get("currently", {})
+    icon = current.get("icon", "")
+    return Weather(
+        temperature=current.get("temperature"),
+        feels_like=current.get("apparentTemperature"),
+        humidity=current.get("humidity"),
+        wind_speed=current.get("windSpeed"),
+        wind_bearing=current.get("windBearing"),
+        condition=CONDITION_MAP.get(icon, "cloudy"),
+        summary=current.get("summary"),
+        pressure=current.get("pressure"),
+        visibility=current.get("visibility"),
+        uv_index=current.get("uvIndex"),
+        ozone=current.get("ozone"),
+        nearest_storm_distance=current.get("nearestStormDistance"),
+        nearest_storm_bearing=current.get("nearestStormBearing"),
+        precip_probability=current.get("precipProbability"),
+        cloud_cover=current.get("cloudCover"),
+        daily_forecasts=daily_forecasts,
+        hourly_forecasts=hourly_forecasts,
+    )
+
+
+@retry(
+    stop=stop_after_attempt(3),
+    wait=wait_fixed(5),
+    before_sleep=before_sleep_log(logger, logging.WARNING),
+    reraise=True,
+)
+def post_to_ha(url: str, token: str, weather: Weather) -> None:
+    """Post weather data to Home Assistant as sensor entities."""
+    headers = {"Authorization": f"Bearer {token}"}
+
+    # Post current weather as individual sensors
+    sensors = {
+        "sensor.van_weather_condition": {
+            "state": weather.condition or "unknown",
+            "attributes": {"friendly_name": "Van Weather Condition"},
+        },
+        "sensor.van_weather_temperature": {
+            "state": weather.temperature,
+            "attributes": {"unit_of_measurement": "°F", "device_class": "temperature"},
+        },
+        "sensor.van_weather_apparent_temperature": {
+            "state": weather.feels_like,
+            "attributes": {"unit_of_measurement": "°F", "device_class": "temperature"},
+        },
+        "sensor.van_weather_humidity": {
+            "state": int((weather.humidity or 0) * 100),
+            "attributes": {"unit_of_measurement": "%", "device_class": "humidity"},
+        },
+        "sensor.van_weather_pressure": {
+            "state": weather.pressure,
+            "attributes": {"unit_of_measurement": "mbar", "device_class": "pressure"},
+        },
+        "sensor.van_weather_wind_speed": {
+            "state": weather.wind_speed,
+            "attributes": {"unit_of_measurement": "mph", "device_class": "wind_speed"},
+        },
+        "sensor.van_weather_wind_bearing": {
+            "state": weather.wind_bearing,
+            "attributes": {"unit_of_measurement": "°"},
+        },
+        "sensor.van_weather_visibility": {
+            "state": weather.visibility,
+            "attributes": {"unit_of_measurement": "mi"},
+        },
+        "sensor.van_weather_uv_index": {
+            "state": weather.uv_index,
+            "attributes": {"friendly_name": "Van Weather UV Index", "icon": "mdi:sun-wireless"},
+        },
+        "sensor.van_weather_ozone": {
+            "state": weather.ozone,
+            "attributes": {"unit_of_measurement": "DU", "icon": "mdi:earth"},
+        },
+        "sensor.van_weather_nearest_storm_distance": {
+            "state": weather.nearest_storm_distance,
+            "attributes": {"unit_of_measurement": "mi", "icon": "mdi:weather-lightning"},
+        },
+        "sensor.van_weather_nearest_storm_bearing": {
+            "state": weather.nearest_storm_bearing,
+            "attributes": {"unit_of_measurement": "°", "icon": "mdi:weather-lightning"},
+        },
+        "sensor.van_weather_precip_probability": {
+            "state": int((weather.precip_probability or 0) * 100),
+            "attributes": {"unit_of_measurement": "%", "icon": "mdi:weather-rainy"},
+        },
+        "sensor.van_weather_cloud_cover": {
+            "state": int((weather.cloud_cover or 0) * 100),
+            "attributes": {"unit_of_measurement": "%", "icon": "mdi:weather-cloudy"},
+        },
+    }
+
+    for entity_id, data in sensors.items():
+        if data["state"] is not None:
+            response = httpx.post(f"{url}/api/states/{entity_id}", headers=headers, json=data, timeout=30)
+            response.raise_for_status()
+
+    # Post daily forecast as JSON attribute sensor
+    daily_forecast = [
+        {
+            "datetime": daily_forecast.date_time.isoformat(),
+            "condition": daily_forecast.condition,
+            "temperature": daily_forecast.temperature,
+            "templow": daily_forecast.templow,
+            "precipitation_probability": int((daily_forecast.precipitation_probability or 0) * 100),
+        }
+        for daily_forecast in weather.daily_forecasts
+    ]
+
+    response = httpx.post(
+        f"{url}/api/states/sensor.van_weather_forecast_daily",
+        headers=headers,
+        json={"state": len(daily_forecast), "attributes": {"forecast": daily_forecast}},
+        timeout=30,
+    )
+    response.raise_for_status()
+
+    # Post hourly forecast as JSON attribute sensor
+    hourly_forecast = [
+        {
+            "datetime": hourly_forecast.date_time.isoformat(),
+            "condition": hourly_forecast.condition,
+            "temperature": hourly_forecast.temperature,
+            "precipitation_probability": int((hourly_forecast.precipitation_probability or 0) * 100),
+        }
+        for hourly_forecast in weather.hourly_forecasts
+    ]
+
+    response = httpx.post(
+        f"{url}/api/states/sensor.van_weather_forecast_hourly",
+        headers=headers,
+        json={"state": len(hourly_forecast), "attributes": {"forecast": hourly_forecast}},
+        timeout=30,
+    )
+    response.raise_for_status()
+
+
+def update_weather(config: Config) -> None:
+    """Fetch weather using last-known location, post to HA."""
+    lat = get_ha_state(config.ha_url, config.ha_token, config.lat_entity)
+    lon = get_ha_state(config.ha_url, config.ha_token, config.lon_entity)
+
+    masked_lat = round(lat, config.mask_decimals)
+    masked_lon = round(lon, config.mask_decimals)
+
+    logger.info(f"Masked location: {masked_lat}, {masked_lon}")
+
+    weather = fetch_weather(config.pirate_weather_api_key, masked_lat, masked_lon)
+    logger.info(f"Weather: {weather.temperature}°F, {weather.condition}")
+
+    post_to_ha(config.ha_url, config.ha_token, weather)
+    logger.info("Posted weather to HA")
+
+
+def main(
+    ha_url: Annotated[str, typer.Option(envvar="HA_URL")],
+    ha_token: Annotated[str, typer.Option(envvar="HA_TOKEN")],
+    api_key: Annotated[str, typer.Option(envvar="PIRATE_WEATHER_API_KEY")],
+    interval: Annotated[int, typer.Option(help="Poll interval in seconds")] = 900,
+    log_level: Annotated[str, typer.Option()] = "INFO",
+) -> None:
+    """Fetch weather for van using masked GPS location."""
+    configure_logger(log_level)
+
+    config = Config(ha_url=ha_url, ha_token=ha_token, pirate_weather_api_key=api_key)
+
+    logger.info(f"Starting van weather service, polling every {interval}s")
+
+    scheduler = BlockingScheduler()
+    scheduler.add_job(
+        update_weather,
+        "interval",
+        seconds=interval,
+        args=[config],
+        next_run_time=datetime.now(UTC),
+    )
+    scheduler.start()
+
+
+if __name__ == "__main__":
+    typer.run(main)

python/van_weather/models.py

@@ -0,0 +1,72 @@
+"""Models for van weather service."""
+
+from __future__ import annotations
+
+from datetime import datetime  # noqa: TC003 This is required for pydantic
+
+from pydantic import BaseModel, field_serializer
+
+
+class Config(BaseModel):
+    """Service configuration."""
+
+    ha_url: str
+    ha_token: str
+    pirate_weather_api_key: str
+    lat_entity: str = "sensor.van_last_known_latitude"
+    lon_entity: str = "sensor.van_last_known_longitude"
+    mask_decimals: int = 1  # ~11km accuracy
+
+
+class DailyForecast(BaseModel):
+    """Daily forecast entry."""
+
+    date_time: datetime
+    condition: str | None = None
+    temperature: float | None = None  # High
+    templow: float | None = None  # Low
+    precipitation_probability: float | None = None
+    moon_phase: float | None = None
+    wind_gust: float | None = None
+    cloud_cover: float | None = None
+
+    @field_serializer("date_time")
+    def serialize_date_time(self, date_time: datetime) -> str:
+        """Serialize datetime to ISO format."""
+        return date_time.isoformat()
+
+
+class HourlyForecast(BaseModel):
+    """Hourly forecast entry."""
+
+    date_time: datetime
+    condition: str | None = None
+    temperature: float | None = None
+    precipitation_probability: float | None = None
+
+    @field_serializer("date_time")
+    def serialize_date_time(self, date_time: datetime) -> str:
+        """Serialize datetime to ISO format."""
+        return date_time.isoformat()
+
+
+class Weather(BaseModel):
+    """Weather data from Pirate Weather."""
+
+    temperature: float | None = None
+    feels_like: float | None = None
+    humidity: float | None = None
+    wind_speed: float | None = None
+    wind_bearing: float | None = None
+    condition: str | None = None
+    summary: str | None = None
+    pressure: float | None = None
+    visibility: float | None = None
+    uv_index: float | None = None
+    ozone: float | None = None
+    nearest_storm_distance: float | None = None
+    nearest_storm_bearing: float | None = None
+    precip_probability: float | None = None
+    cloud_cover: float | None = None
+    daily_forecasts: list[DailyForecast] = []
+    hourly_forecasts: list[HourlyForecast] = []

shell.nix

@@ -8,7 +8,7 @@
   # loader path, which NixOS does not provide globally.
   ebook-search = pkgs.mkShell {
     nativeBuildInputs = with pkgs; [
-      python314
+      my_python
       uv
     ];
     LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
@@ -23,6 +23,7 @@
       nix
       home-manager
       git
+      my_python
 
       ssh-to-age
       gnupg

systems/bob/default.nix

@@ -3,18 +3,18 @@
   imports = [
     "${inputs.self}/users/math"
     "${inputs.self}/users/richie"
+    "${inputs.self}/users/steve"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/docker.nix"
-    "${inputs.self}/common/optional/monitoring-agent.nix"
-    "${inputs.self}/common/optional/nvidia.nix"
     "${inputs.self}/common/optional/scanner.nix"
+    "${inputs.self}/common/optional/monitoring-agent.nix"
     "${inputs.self}/common/optional/steam.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/systemd-boot.nix"
-    "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/x86-64-v3.nix"
-    "${inputs.self}/common/optional/zfs"
+    "${inputs.self}/common/optional/yubikey.nix"
+    "${inputs.self}/common/optional/zerotier.nix"
+    "${inputs.self}/common/optional/nvidia.nix"
     ./hardware.nix
     ./syncthing.nix
     ./llms.nix

systems/bob/hardware.nix

@@ -50,11 +50,6 @@
       fsType = "zfs";
     };
 
-    "/nix/var/nix/builds" = {
-      device = "root_pool/nix_build";
-      fsType = "zfs";
-    };
-
     "/var" = {
       device = "root_pool/var";
       fsType = "zfs";

systems/bob/snapshot_config.toml

@@ -22,12 +22,6 @@ hourly = 0
 daily = 0
 monthly = 0
 
-["root_pool/nix_build"]
-15_min = 1
-hourly = 0
-daily = 0
-monthly = 0
-
 ["root_pool/models"]
 15_min = 4
 hourly = 24

systems/brain/default.nix

@@ -7,10 +7,8 @@
     "${inputs.self}/common/optional/ssh_decrypt.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/systemd-boot.nix"
-    "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/x86-64-v3.nix"
-    "${inputs.self}/common/optional/zfs"
+    "${inputs.self}/common/optional/zerotier.nix"
     ./docker
     ./hardware.nix
     ./programs.nix

systems/brain/hardware.nix

@@ -53,11 +53,6 @@
       fsType = "zfs";
     };
 
-    "/nix/var/nix/builds" = {
-      device = "root_pool/nix_build";
-      fsType = "zfs";
-    };
-
     "/boot" = {
       device = "/dev/disk/by-uuid/12CE-A600";
       fsType = "vfat";

systems/brain/services/heater.nix

@@ -4,20 +4,6 @@
   ...
 }:
 {
-  nixpkgs.overlays = [
-    (final: _prev: {
-      heater_python = final.python314.withPackages (
-        ps: with ps; [
-          fastapi
-          pydantic
-          tinytuya
-          typer
-          uvicorn
-        ]
-      );
-    })
-  ];
-
   networking.firewall.allowedTCPPorts = [ 8124 ];
 
   systemd.services.heater-api = {
@@ -31,7 +17,7 @@
 
     serviceConfig = {
       Type = "simple";
-      ExecStart = "${pkgs.heater_python}/bin/python -m python.heater.main --host 0.0.0.0 --port 8124";
+      ExecStart = "${pkgs.my_python}/bin/python -m python.heater.main --host 0.0.0.0 --port 8124";
       EnvironmentFile = "/etc/heater.env";
       Restart = "on-failure";
       RestartSec = "5s";

systems/brain/services/home_assistant.nix

@@ -21,7 +21,6 @@
           packages = {
             victron_modbuss = "!include ${./home_assistant/victron_modbuss.yaml}";
             battery_sensors = "!include ${./home_assistant/battery_sensors.yaml}";
-            energy_meters = "!include ${./home_assistant/energy_meters.yaml}";
             gps_location = "!include ${./home_assistant/gps_location.yaml}";
             heater = "!include ${./home_assistant/heater.yaml}";
             van_weather = "!include ${./home_assistant/van_weather_template.yaml}";
@@ -68,6 +67,7 @@
           paho-mqtt # for mqtt
           psycopg2 # for postgresql
           py-improv-ble-client # for esphome
+          pymodbus # for modbus
           pyopenweathermap # for weather
           pymetno # for met.no weather
           uiprotect # for ubiquiti integration
@@ -75,19 +75,17 @@
           jsonpath # for rest sensors
           monarchmoneycommunity # for monarch
         ];
-      extraComponents = [
-        "isal"
-        "modbus" # for victron modbus integration
-      ];
+      extraComponents = [ "isal" ];
       customComponents = with pkgs.home-assistant-custom-components; [
         garmin_connect
+        pirate-weather
       ];
 
     };
     esphome = {
       enable = true;
       openFirewall = true;
-      address = "0.0.0.0";
+      address = "192.168.90.35";
     };
   };
 }

systems/brain/services/home_assistant/energy_meters.yaml

@@ -1,71 +0,0 @@
-template:
-  - sensor:
-      - name: "JK0 Energy In Clean"
-        unique_id: jk0_energy_in_clean
-        unit_of_measurement: "kWh"
-        device_class: energy
-        state_class: total_increasing
-        availability: >-
-          {{ is_number(states('sensor.jk0_energy_in')) }}
-        state: >-
-          {{ states('sensor.jk0_energy_in') | float }}
-
-      - name: "JK0 Energy Out Clean"
-        unique_id: jk0_energy_out_clean
-        unit_of_measurement: "kWh"
-        device_class: energy
-        state_class: total_increasing
-        availability: >-
-          {{ is_number(states('sensor.jk0_energy_out')) }}
-        state: >-
-          {{ states('sensor.jk0_energy_out') | float }}
-
-      - name: "JK1 Energy In Clean"
-        unique_id: jk1_energy_in_clean
-        unit_of_measurement: "kWh"
-        device_class: energy
-        state_class: total_increasing
-        availability: >-
-          {{ is_number(states('sensor.jk1_energy_in')) }}
-        state: >-
-          {{ states('sensor.jk1_energy_in') | float }}
-
-      - name: "JK1 Energy Out Clean"
-        unique_id: jk1_energy_out_clean
-        unit_of_measurement: "kWh"
-        device_class: energy
-        state_class: total_increasing
-        availability: >-
-          {{ is_number(states('sensor.jk1_energy_out')) }}
-        state: >-
-          {{ states('sensor.jk1_energy_out') | float }}
-
-      - name: "Solar Yield Clean"
-        unique_id: solar_yield_clean
-        unit_of_measurement: "kWh"
-        device_class: energy
-        state_class: total_increasing
-        availability: >-
-          {{ is_number(states('sensor.solar_yield_daily')) }}
-        state: >-
-          {{ states('sensor.solar_yield_daily') | float }}
-
-      - name: "DC Load Energy Clean"
-        unique_id: dc_load_energy_clean
-        unit_of_measurement: "kWh"
-        device_class: energy
-        state_class: total_increasing
-        availability: >-
-          {{ is_number(states('sensor.dc_load_energy_daily')) }}
-        state: >-
-          {{ states('sensor.dc_load_energy_daily') | float }}
-
-      - name: "DC Charger Energy Clean"
-        unique_id: dc_charger_energy_clean
-        unit_of_measurement: "kWh"
-        device_class: energy
-        state_class: total_increasing
-        availability: >-
-          {{ is_number(states('sensor.dc_charger_energy')) }}
-        state: >-
-          {{ states('sensor.dc_charger_energy') | float }}

systems/brain/services/postgress.nix

@@ -19,7 +19,7 @@
       local all  richie   trust
       host  all  richie   127.0.0.1/32    trust
       host  all  richie   ::1/128         trust
-      host  all  richie   100.64.0.0/10 trust
+      host  all  richie   192.168.90.1/24 trust
       host  all  richie   192.168.99.1/24 trust
 
       local vaninventory vaninventory trust
@@ -28,7 +28,7 @@
       local hass     hass     trust
 
       # ipv4
-      host  hass     hass     100.64.0.0/10 trust
+      host  hass     hass     192.168.90.1/24 trust
       host  hass     hass     127.0.0.1/32 trust
 
       # ipv6

systems/brain/syncthing.nix

@@ -3,7 +3,7 @@
 
   services.syncthing = {
     overrideFolders = false;
-    guiAddress = "0.0.0.0:8384";
+    guiAddress = "192.168.90.35:8384";
     settings = {
       "dotfiles" = {
         path = "/home/richie/dotfiles";

systems/jeeves/containers.nix

@@ -1,71 +0,0 @@
-{ pkgs, ... }:
-let
-  vars = import ./vars.nix;
-in
-{
-  virtualisation.docker.daemon.settings."data-root" = "${vars.containers}/docker";
-
-  # nixos-container hardcodes its state directory to /var/lib/nixos-containers,
-  # so route it to the shared container dataset with a bind mount.
-  fileSystems."/var/lib/nixos-containers" = {
-    device = "${vars.containers}/nixos-containers";
-    fsType = "none";
-    options = [ "bind" ];
-  };
-
-  systemd.services = {
-    prepare-docker-storage = {
-      description = "Create Docker storage directory";
-      requiredBy = [ "docker.service" ];
-      before = [ "docker.service" ];
-      requires = [ "zfs-mount.service" ];
-      after = [ "zfs-mount.service" ];
-      path = [
-        pkgs.coreutils
-        pkgs.util-linux
-      ];
-
-      unitConfig = {
-        DefaultDependencies = false;
-        RequiresMountsFor = [ "/nix" ];
-      };
-
-      serviceConfig = {
-        Type = "oneshot";
-        RemainAfterExit = true;
-      };
-
-      script = ''
-        mountpoint -q ${vars.containers}
-        install -d -m 0710 ${vars.containers}/docker
-      '';
-    };
-
-    prepare-nixos-container-storage = {
-      description = "Create NixOS container storage directory";
-      requiredBy = [ "var-lib-nixos\\x2dcontainers.mount" ];
-      before = [ "var-lib-nixos\\x2dcontainers.mount" ];
-      requires = [ "zfs-mount.service" ];
-      after = [ "zfs-mount.service" ];
-      path = [
-        pkgs.coreutils
-        pkgs.util-linux
-      ];
-
-      unitConfig = {
-        DefaultDependencies = false;
-        RequiresMountsFor = [ "/nix" ];
-      };
-
-      serviceConfig = {
-        Type = "oneshot";
-        RemainAfterExit = true;
-      };
-
-      script = ''
-        mountpoint -q ${vars.containers}
-        install -d -m 0755 ${vars.containers}/nixos-containers
-      '';
-    };
-  };
-}

systems/jeeves/datasets.nix

@@ -0,0 +1,362 @@
+# Dataset declarations for jeeves, kept as plain data rather than inside
+# zfs.nix so the dataset tree stays separate from the NixOS service wiring.
+# Datasets are nested the way zfs nests them: a pool holds datasets, which can
+# hold datasets of their own. The tree is flattened into "pool/parent/child"
+# names below, which is what zfs and services.zfs_manager work in.
+#
+# Consumed by ./zfs.nix, which feeds it to services.zfs_manager.
+let
+  # Every pool on jeeves was created with the same -O options.
+  poolDefaults = mountpoint: {
+    inherit mountpoint;
+    acltype = "posix"; # zfs reports posixacl back as posix
+    atime = "off";
+    compression = "zstd";
+    dnodesize = "auto";
+    xattr = "sa";
+  };
+
+  zfsKey = "file:///root/zfs.key";
+
+  # What a dataset gets when it is not called out below, kept identical to the
+  # "default" table so the datasets that used to fall through are unchanged.
+  standard = {
+    "15_min" = 8;
+    hourly = 24;
+  };
+
+  disabledSnapshots = {
+    "15_min" = 0;
+    hourly = 0;
+    daily = 0;
+    monthly = 0;
+  };
+
+  pools = {
+    # root_pool: retention only, its properties are not managed yet.
+    root_pool = {
+      manageProperties = false;
+      datasets = {
+        home = {
+          manageProperties = false;
+          snapshots = {
+            "15_min" = 8;
+            hourly = 24;
+            daily = 14;
+          };
+        };
+        root = {
+          manageProperties = false;
+          snapshots = standard;
+        };
+        nix = {
+          manageProperties = false;
+          snapshots."15_min" = 4;
+        };
+        var = {
+          manageProperties = false;
+          snapshots = {
+            "15_min" = 8;
+            hourly = 24;
+            daily = 30;
+            monthly = 6;
+          };
+        };
+      };
+    };
+
+    media = {
+      properties = poolDefaults "/zfs/media";
+      datasets = {
+        temp = {
+          properties = {
+            redundant_metadata = "none";
+            sync = "disabled";
+          };
+          snapshots."15_min" = 2;
+        };
+        secure = {
+          # An encryption root provisioned by scripts/zfs.sh. Its create-only
+          # properties are declared for verification, but zfs_manager must
+          # never create it automatically.
+          createIfMissing = true;
+          properties = {
+            encryption = "aes-256-gcm";
+            keyformat = "hex";
+            keylocation = zfsKey;
+          };
+          snapshots = disabledSnapshots;
+          datasets = {
+            docker = {
+              properties = {
+                mountpoint = "/zfs/media/docker";
+                compression = "zstd-9";
+              };
+              snapshots = {
+                "15_min" = 3;
+                hourly = 12;
+                daily = 14;
+                monthly = 2;
+              };
+            };
+            "github-runners" = {
+              properties = {
+                mountpoint = "/zfs/media/github-runners";
+                compression = "zstd-9";
+                sync = "disabled";
+              };
+              snapshots = {
+                "15_min" = 6;
+                hourly = 2;
+                daily = 1;
+              };
+            };
+            home_assistant = {
+              properties = {
+                mountpoint = "/zfs/media/home_assistant";
+                compression = "zstd-19";
+              };
+              snapshots = standard;
+            };
+            important = {
+              properties = {
+                compression = "zstd-9";
+                copies = "2";
+              };
+              snapshots = standard;
+            };
+            notes = {
+              properties = {
+                mountpoint = "/zfs/media/notes";
+                copies = "2";
+              };
+              snapshots = {
+                "15_min" = 8;
+                hourly = 24;
+                daily = 30;
+                monthly = 12;
+              };
+            };
+            postgres = {
+              properties = {
+                mountpoint = "/zfs/media/database/postgres";
+                primarycache = "metadata";
+                recordsize = "16K";
+              };
+              snapshots = {
+                "15_min" = 8;
+                hourly = 24;
+                daily = 7;
+              };
+            };
+            "postgres-wal" = {
+              properties = {
+                compression = "lz4";
+                logbias = "latency";
+                mountpoint = "/zfs/media/database/postgres-wal";
+                primarycache = "metadata";
+                recordsize = "32K";
+                secondarycache = "none";
+                special_small_blocks = "32K";
+              };
+              snapshots = {
+                "15_min" = 4;
+                hourly = 2;
+              };
+            };
+            prometheus = {
+              properties = {
+                mountpoint = "/zfs/media/database/prometheus";
+                compression = "lz4";
+              };
+              snapshots = standard;
+            };
+            services = {
+              properties = {
+                mountpoint = "/zfs/media/services";
+                compression = "zstd-9";
+              };
+              snapshots = standard;
+            };
+            share = {
+              properties = {
+                mountpoint = "/zfs/media/share";
+                exec = "off";
+              };
+              snapshots."15_min" = 4;
+            };
+          };
+        };
+      };
+    };
+
+    storage = {
+      properties = poolDefaults "/zfs/storage";
+      datasets = {
+        nomad = {
+          properties = {
+            mountpoint = "/zfs/storage/nomad";
+            compression = "zstd-9";
+          };
+          snapshots = standard;
+        };
+        ollama = {
+          properties = {
+            compression = "zstd-19";
+            recordsize = "1M";
+            sync = "disabled";
+          };
+          snapshots."15_min" = 2;
+        };
+        secure = {
+          # An encryption root provisioned by scripts/zfs.sh. Its create-only
+          # properties are declared for verification, but zfs_manager must
+          # never create it automatically.
+          createIfMissing = false;
+          properties = {
+            encryption = "aes-256-gcm";
+            keyformat = "hex";
+            keylocation = zfsKey;
+          };
+          snapshots = disabledSnapshots;
+          datasets = {
+            archive = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/archive";
+                recordsize = "1M";
+              };
+              snapshots = standard;
+            };
+            important = {
+              properties = {
+                compression = "zstd-19";
+                copies = "2";
+                mountpoint = "/zfs/storage/important";
+              };
+              snapshots = standard;
+            };
+            library = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/library";
+                recordsize = "1M";
+              };
+              snapshots = standard;
+            };
+            main = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/main";
+              };
+              snapshots = standard;
+            };
+            photos = {
+              properties = {
+                compression = "zstd-19";
+                copies = "2";
+                mountpoint = "/zfs/storage/photos";
+                recordsize = "16K";
+              };
+              snapshots = standard;
+            };
+            plex = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/plex";
+                recordsize = "1M";
+              };
+              snapshots = {
+                "15_min" = 6;
+                hourly = 2;
+                daily = 1;
+              };
+            };
+            secrets = {
+              properties = {
+                compression = "zstd-19";
+                copies = "3";
+                mountpoint = "/zfs/storage/secrets";
+              };
+              snapshots = {
+                "15_min" = 8;
+                hourly = 24;
+                daily = 30;
+                monthly = 12;
+              };
+            };
+            syncthing = {
+              properties = {
+                compression = "zstd-19";
+                mountpoint = "/zfs/storage/syncthing";
+              };
+              snapshots = standard;
+            };
+            transmission = {
+              properties = {
+                compression = "zstd-9";
+                exec = "off";
+                mountpoint = "/zfs/storage/transmission";
+                recordsize = "1M";
+                sync = "disabled";
+              };
+              snapshots."15_min" = 4;
+            };
+          };
+        };
+      };
+    };
+
+    scratch = {
+      properties = poolDefaults "/zfs/scratch" // {
+        encryption = "aes-256-gcm";
+        keyformat = "hex";
+        keylocation = zfsKey;
+      };
+      datasets = {
+        kafka = {
+          properties = {
+            mountpoint = "/zfs/scratch/kafka";
+            recordsize = "1M";
+          };
+          snapshots = standard;
+        };
+        kestra = {
+          properties = {
+            mountpoint = "/zfs/scratch/kestra";
+            sync = "disabled";
+          };
+          snapshots = standard;
+        };
+        transmission = {
+          properties = {
+            mountpoint = "/zfs/scratch/transmission";
+            recordsize = "16K";
+            sync = "disabled";
+          };
+          snapshots."15_min" = 2;
+        };
+        uv_cache = {
+          properties.mountpoint = "/zfs/scratch/uv_cache";
+          snapshots."15_min" = 2;
+        };
+      };
+    };
+  };
+
+  # Collapse the tree into the flat "pool/parent/child" names zfs uses. Each
+  # node keeps everything except its children.
+  flatten =
+    name: node:
+    builtins.foldl' (result: child: result // flatten "${name}/${child}" node.datasets.${child}) {
+      ${name} = builtins.removeAttrs node [ "datasets" ];
+    } (builtins.attrNames (node.datasets or { }));
+
+  datasets = builtins.foldl' (result: pool: result // flatten pool pools.${pool}) { } (
+    builtins.attrNames pools
+  );
+in
+{
+  inherit datasets;
+  defaultSnapshots = standard;
+}

systems/jeeves/default.nix

@@ -1,29 +1,28 @@
 { inputs, ... }:
-let
-  vars = import ./vars.nix;
-in
 {
   imports = [
+    "${inputs.self}/users/dov"
     "${inputs.self}/users/math"
     "${inputs.self}/users/richie"
+    "${inputs.self}/users/steve"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/docker.nix"
     "${inputs.self}/common/optional/monitoring-agent.nix"
     "${inputs.self}/common/optional/ssh_decrypt.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
-    "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/x86-64-v3.nix"
-    "${inputs.self}/common/optional/zfs"
+    "${inputs.self}/common/optional/zerotier.nix"
+    "${inputs.self}/common/optional/zfs_manager.nix"
     ./monitoring
     ./docker
     ./services
-    ./containers.nix
+    ./web_services
     ./hardware.nix
-    ./networking
+    ./networking.nix
     ./programs.nix
     ./runners
     ./syncthing.nix
+    ./zfs.nix
   ];
 
   services = {
@@ -31,10 +30,7 @@ in
 
     smartd.enable = true;
 
-    snapshot_manager = {
-      path = ./snapshot_config.toml;
-      EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
-    };
+    zerotierone.joinNetworks = [ "a09acf02330d37b9" ];
   };
 
   users.groups = {

systems/jeeves/docker/signal-cli-rest-api.nix

@@ -2,6 +2,9 @@ let
   vars = import ../vars.nix;
 in
 {
+  networking.firewall.allowedTCPPorts = [
+    8989
+  ];
   virtualisation.oci-containers.containers.signal_cli_rest_api = {
     image = "bbernhard/signal-cli-rest-api:0.199-dev";
     ports = [

systems/jeeves/hardware.nix

@@ -96,11 +96,6 @@ in
       fsType = "zfs";
     };
 
-    "/nix/var/nix/builds" = {
-      device = "root_pool/nix_build";
-      fsType = "zfs";
-    };
-
     "/var" = {
       device = "root_pool/var";
       fsType = "zfs";

systems/jeeves/monitoring/README.md

@@ -1,33 +0,0 @@
-# Monitoring
-
-## Vultr API metrics
-
-The `vultr-exporter` service reads its API token from:
-
-```text
-/zfs/storage/secrets/services/vultr-exporter
-```
-
-Create the file on Jeeves as root with the following contents:
-
-```text
-API_KEY=<Vultr API token>
-```
-
-The token needs read access to the Vultr Account and Billing APIs. Unrelated
-resource collectors are disabled in the packaged exporter.
-
-Restrict the file to root and ensure the public egress IP used by Jeeves is
-allowed for the token in the Vultr API settings:
-
-```console
-sudo chown root:root /zfs/storage/secrets/services/vultr-exporter
-sudo chmod 600 /zfs/storage/secrets/services/vultr-exporter
-```
-
-The exporter listens on `127.0.0.1:9188`; it is scraped by the local
-`prometheus-main` service every five minutes and is not exposed through the
-host firewall.
-
-Portal-1 exposes its node exporter only through `tailscale0` on port `9100`.
-Jeeves reaches it using the Portal-1 Tailscale hostname.

systems/jeeves/monitoring/dashboards/portal-1.json

@@ -1,798 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": false,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 1,
-  "links": [],
-  "panels": [
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "prom-main"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "unit": "short",
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "red",
-                "value": null
-              },
-              {
-                "color": "green",
-                "value": 1
-              }
-            ]
-          },
-          "mappings": [
-            {
-              "options": {
-                "0": {
-                  "color": "red",
-                  "index": 1,
-                  "text": "Offline"
-                },
-                "1": {
-                  "color": "green",
-                  "index": 0,
-                  "text": "Online"
-                }
-              },
-              "type": "value"
-            }
-          ]
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 4,
-        "x": 0,
-        "y": 0
-      },
-      "id": 1,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "prom-main"
-          },
-          "editorMode": "code",
-          "expr": "up{job=\"node\",instance=\"portal-1\"}",
-          "instant": true,
-          "legendFormat": "",
-          "range": false,
-          "refId": "A"
-        }
-      ],
-      "title": "Node Exporter",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "prom-main"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 70
-              },
-              {
-                "color": "red",
-                "value": 90
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 5,
-        "x": 4,
-        "y": 0
-      },
-      "id": 2,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "prom-main"
-          },
-          "editorMode": "code",
-          "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{job=\"node\",instance=\"portal-1\",mode=\"idle\"}[5m])))",
-          "instant": true,
-          "legendFormat": "",
-          "range": false,
-          "refId": "A"
-        }
-      ],
-      "title": "CPU Used",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "prom-main"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 70
-              },
-              {
-                "color": "red",
-                "value": 90
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 5,
-        "x": 9,
-        "y": 0
-      },
-      "id": 3,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "prom-main"
-          },
-          "editorMode": "code",
-          "expr": "100 * (1 - (node_memory_MemAvailable_bytes{job=\"node\",instance=\"portal-1\"} / node_memory_MemTotal_bytes{job=\"node\",instance=\"portal-1\"}))",
-          "instant": true,
-          "legendFormat": "",
-          "range": false,
-          "refId": "A"
-        }
-      ],
-      "title": "RAM Used",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "prom-main"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 70
-              },
-              {
-                "color": "red",
-                "value": 90
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 5,
-        "x": 14,
-        "y": 0
-      },
-      "id": 4,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "prom-main"
-          },
-          "editorMode": "code",
-          "expr": "100 * (1 - (node_filesystem_avail_bytes{job=\"node\",instance=\"portal-1\",mountpoint=\"/\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node\",instance=\"portal-1\",mountpoint=\"/\",fstype!=\"\"}))",
-          "instant": true,
-          "legendFormat": "",
-          "range": false,
-          "refId": "A"
-        }
-      ],
-      "title": "Root Filesystem Used",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "prom-main"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "unit": "short"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 5,
-        "x": 19,
-        "y": 0
-      },
-      "id": 5,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "prom-main"
-          },
-          "editorMode": "code",
-          "expr": "node_load1{job=\"node\",instance=\"portal-1\"}",
-          "instant": true,
-          "legendFormat": "",
-          "range": false,
-          "refId": "A"
-        }
-      ],
-      "title": "Load (1m)",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "prom-main"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 70
-              },
-              {
-                "color": "red",
-                "value": 90
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 5
-      },
-      "id": 6,
-      "options": {
-        "legend": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "prom-main"
-          },
-          "editorMode": "code",
-          "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{job=\"node\",instance=\"portal-1\",mode=\"idle\"}[5m])))",
-          "instant": false,
-          "legendFormat": "CPU used",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "CPU Usage",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "prom-main"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 70
-              },
-              {
-                "color": "red",
-                "value": 90
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 5
-      },
-      "id": 7,
-      "options": {
-        "legend": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "prom-main"
-          },
-          "editorMode": "code",
-          "expr": "100 * (1 - (node_memory_MemAvailable_bytes{job=\"node\",instance=\"portal-1\"} / node_memory_MemTotal_bytes{job=\"node\",instance=\"portal-1\"}))",
-          "instant": false,
-          "legendFormat": "RAM used",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "RAM Usage",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "prom-main"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 70
-              },
-              {
-                "color": "red",
-                "value": 90
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 13
-      },
-      "id": 8,
-      "options": {
-        "cellHeight": "sm",
-        "showHeader": true,
-        "sortBy": [
-          {
-            "desc": true,
-            "displayName": "Value"
-          }
-        ]
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "prom-main"
-          },
-          "editorMode": "code",
-          "expr": "100 * (1 - (node_filesystem_avail_bytes{job=\"node\",instance=\"portal-1\",fstype!~\"tmpfs|devtmpfs|overlay|squashfs\",mountpoint!~\"/run.*\"} / node_filesystem_size_bytes{job=\"node\",instance=\"portal-1\",fstype!~\"tmpfs|devtmpfs|overlay|squashfs\",mountpoint!~\"/run.*\"}))",
-          "format": "table",
-          "instant": true,
-          "legendFormat": "{{mountpoint}}",
-          "range": false,
-          "refId": "A"
-        }
-      ],
-      "title": "Filesystem Usage",
-      "type": "table"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "prom-main"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "unit": "Bps"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 13
-      },
-      "id": 9,
-      "options": {
-        "legend": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "prom-main"
-          },
-          "editorMode": "code",
-          "expr": "sum(rate(node_disk_read_bytes_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
-          "instant": false,
-          "legendFormat": "read",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "prom-main"
-          },
-          "editorMode": "code",
-          "expr": "sum(rate(node_disk_written_bytes_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
-          "instant": false,
-          "legendFormat": "write",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "Disk Throughput",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "prom-main"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "unit": "iops"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 21
-      },
-      "id": 10,
-      "options": {
-        "legend": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "prom-main"
-          },
-          "editorMode": "code",
-          "expr": "sum(rate(node_disk_reads_completed_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
-          "instant": false,
-          "legendFormat": "reads",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "prom-main"
-          },
-          "editorMode": "code",
-          "expr": "sum(rate(node_disk_writes_completed_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
-          "instant": false,
-          "legendFormat": "writes",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "Disk Operations",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "prom-main"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "unit": "Bps"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 21
-      },
-      "id": 11,
-      "options": {
-        "legend": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "prom-main"
-          },
-          "editorMode": "code",
-          "expr": "sum(rate(node_network_receive_bytes_total{job=\"node\",instance=\"portal-1\",device!=\"lo\"}[5m]))",
-          "instant": false,
-          "legendFormat": "received",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "prom-main"
-          },
-          "editorMode": "code",
-          "expr": "sum(rate(node_network_transmit_bytes_total{job=\"node\",instance=\"portal-1\",device!=\"lo\"}[5m]))",
-          "instant": false,
-          "legendFormat": "sent",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "Network Traffic",
-      "type": "timeseries"
-    }
-  ],
-  "refresh": "30s",
-  "schemaVersion": 39,
-  "tags": [
-    "monitoring",
-    "portal-1",
-    "vultr"
-  ],
-  "templating": {
-    "list": []
-  },
-  "time": {
-    "from": "now-24h",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "browser",
-  "title": "Portal-1 Host",
-  "uid": "portal-1-host",
-  "version": 1,
-  "weekStart": ""
-}

systems/jeeves/monitoring/default.nix

@@ -41,36 +41,29 @@ let
       {
         job_name = "node";
         static_configs = [
-          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9100")
-          (mkTarget "bob" "bob.taile39c31.ts.net:9100")
-          (mkTarget "portal-1" "portal-1.taile39c31.ts.net:9100")
+          (mkTarget "jeeves" "192.168.90.40:9100")
+          (mkTarget "bob" "192.168.90.25:9100")
         ];
       }
-      {
-        job_name = "vultr";
-        scrape_interval = "5m";
-        scrape_timeout = "45s";
-        static_configs = [ (mkTarget "vultr" "127.0.0.1:9188") ];
-      }
       {
         job_name = "process_grouped";
         static_configs = [
-          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9256")
-          (mkTarget "bob" "bob.taile39c31.ts.net:9256")
+          (mkTarget "jeeves" "192.168.90.40:9256")
+          (mkTarget "bob" "192.168.90.25:9256")
         ];
       }
       {
         job_name = "smartctl";
         static_configs = [
-          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9633")
-          (mkTarget "bob" "bob.taile39c31.ts.net:9633")
+          (mkTarget "jeeves" "192.168.90.40:9633")
+          (mkTarget "bob" "192.168.90.25:9633")
         ];
       }
       {
         job_name = "zfs";
         static_configs = [
-          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9134")
-          (mkTarget "bob" "bob.taile39c31.ts.net:9134")
+          (mkTarget "jeeves" "192.168.90.40:9134")
+          (mkTarget "bob" "192.168.90.25:9134")
         ];
       }
     ];
@@ -86,8 +79,8 @@ let
       {
         job_name = "process_pid";
         static_configs = [
-          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9257")
-          (mkTarget "bob" "bob.taile39c31.ts.net:9257")
+          (mkTarget "jeeves" "192.168.90.40:9257")
+          (mkTarget "bob" "192.168.90.25:9257")
         ];
       }
     ];
@@ -104,9 +97,7 @@ let
       after = [
         "zfs-media-database-prometheus.mount"
         "network.target"
-        "tailscaled-autoconnect.service"
       ];
-      wants = [ "tailscaled-autoconnect.service" ];
       requires = [ "zfs-media-database-prometheus.mount" ];
       wantedBy = [ "multi-user.target" ];
       unitConfig.RequiresMountsFor = [ dataDir ];
@@ -160,8 +151,6 @@ let
     };
 in
 {
-  imports = [ ./vultr-exporter.nix ];
-
   users = {
     groups.prometheus = { };
     users.prometheus = {

systems/jeeves/monitoring/vultr-exporter.nix

@@ -1,96 +0,0 @@
-{
-  lib,
-  pkgs,
-  ...
-}:
-let
-  vars = import ../vars.nix;
-
-  vultrExporter = pkgs.buildGoModule rec {
-    pname = "vultr-exporter";
-    version = "0-unstable-2025-10-14";
-
-    src = pkgs.fetchFromGitHub {
-      owner = "DazWilkin";
-      repo = "vultr-exporter";
-      rev = "d8b5d7107c2eda31566e6e895c8a67db32fdaf68";
-      hash = "sha256-3LpXh+mFMBRJfiY2ftELpM3AIE0LiZ9c6NFok0OeH5I=";
-    };
-
-    vendorHash = "sha256-qCHWmg6GxLIjqlf5UqRcuDRd1xqRFI3sl4ZduLajx+Y=";
-    subPackages = [ "cmd/server" ];
-
-    postPatch = ''
-      substituteInPlace cmd/server/main.go \
-        --replace-fail 'registry.MustRegister(collector.NewBlockStorageCollector(s, client, log))' "" \
-        --replace-fail 'registry.MustRegister(collector.NewKubernetesCollector(s, client, log))' "" \
-        --replace-fail 'registry.MustRegister(collector.NewLoadBalancerCollector(s, client, log))' "" \
-        --replace-fail 'registry.MustRegister(collector.NewReservedIPsCollector(s, client, log))' ""
-    '';
-
-    ldflags = [
-      "-s"
-      "-w"
-      "-X main.GitCommit=${src.rev}"
-      "-X main.OSVersion=NixOS"
-    ];
-
-    postInstall = ''
-      mv $out/bin/server $out/bin/vultr-exporter
-    '';
-
-    meta = {
-      description = "Prometheus exporter for Vultr account, billing, and bandwidth metrics";
-      homepage = "https://github.com/DazWilkin/vultr-exporter";
-      license = lib.licenses.asl20;
-      mainProgram = "vultr-exporter";
-    };
-  };
-in
-{
-  systemd.services.vultr-exporter = {
-    description = "Vultr Prometheus exporter";
-    documentation = [ "https://github.com/DazWilkin/vultr-exporter" ];
-    wants = [ "network-online.target" ];
-    after = [ "network-online.target" ];
-    wantedBy = [ "multi-user.target" ];
-
-    serviceConfig = {
-      ExecStart = "${lib.getExe vultrExporter} --endpoint=127.0.0.1:9188 --path=/metrics";
-      EnvironmentFile = "${vars.secrets}/services/vultr-exporter";
-      Restart = "on-failure";
-      RestartSec = "30s";
-
-      CapabilityBoundingSet = [ "" ];
-      DeviceAllow = [ "" ];
-      DynamicUser = true;
-      LockPersonality = true;
-      MemoryDenyWriteExecute = true;
-      NoNewPrivileges = true;
-      PrivateDevices = true;
-      PrivateTmp = true;
-      ProtectClock = true;
-      ProtectControlGroups = true;
-      ProtectHome = true;
-      ProtectHostname = true;
-      ProtectKernelLogs = true;
-      ProtectKernelModules = true;
-      ProtectKernelTunables = true;
-      ProtectSystem = "strict";
-      RemoveIPC = true;
-      RestrictAddressFamilies = [
-        "AF_INET"
-        "AF_INET6"
-      ];
-      RestrictNamespaces = true;
-      RestrictRealtime = true;
-      RestrictSUIDSGID = true;
-      SystemCallArchitectures = "native";
-      SystemCallFilter = [
-        "@system-service"
-        "~@privileged"
-      ];
-      UMask = "0077";
-    };
-  };
-}

systems/jeeves/networking.nix

@@ -8,6 +8,27 @@
     "net.bridge.bridge-nf-call-iptables" = 0;
   };
 
+  networking = {
+    hostName = "jeeves";
+    hostId = "0e15ce35";
+    firewall = {
+      enable = true;
+      interfaces.br-nix-builder = {
+        allowedTCPPorts = [ ];
+        allowedUDPPorts = [ ];
+      };
+      allowedTCPPorts = [
+        8070
+      ];
+    };
+    useNetworkd = true;
+    # Without this, unconfigured NICs get a catch-all DHCP network and
+    # join the LAN uninvited (how the dead atlantic card ended up
+    # ARP-poisoning 192.168.99.14). Every interface must be configured
+    # explicitly below.
+    useDHCP = false;
+  };
+
   systemd.network = {
     enable = true;
     wait-online = {

systems/jeeves/networking/default.nix

@@ -1,16 +0,0 @@
-{
-  imports = [
-    ./firewall.nix
-    ./interfaces.nix
-  ];
-  networking = {
-    hostName = "jeeves";
-    hostId = "0e15ce35";
-    useNetworkd = true;
-    # Without this, unconfigured NICs get a catch-all DHCP network and
-    # join the LAN uninvited (how the dead atlantic card ended up
-    # ARP-poisoning 192.168.99.14). Every interface must be configured
-    # explicitly below.
-    useDHCP = false;
-  };
-}

systems/jeeves/networking/firewall.nix

@@ -1,47 +0,0 @@
-let
-  ports = {
-    audiobookshelf = 8000;
-    ebook_searcher = 8070;
-    gems = 8002;
-    gitea_http = 6443;
-    gitea_ssh = 2223;
-    grafana = 3000;
-    jellyfin_http = 8096;
-    nix_binary_cache = 5000;
-    nornsight = 8001;
-    ollama = 11434;
-    open_webui = 8080;
-    postgresql = 5432;
-    share = 8091;
-    signal_cli = 8989;
-    syncthing = 8384;
-  };
-in
-{
-  networking.firewall = {
-    enable = true;
-    interfaces.br-nix-builder = {
-      allowedTCPPorts = [ ];
-      allowedUDPPorts = [ ];
-    };
-    interfaces.tailscale0.allowedTCPPorts = [
-      ports.audiobookshelf
-      ports.gems
-      ports.jellyfin_http
-      ports.nix_binary_cache
-      ports.nornsight
-      ports.share
-    ];
-    allowedTCPPorts = [
-      ports.ebook_searcher
-      ports.gitea_http
-      ports.gitea_ssh
-      ports.grafana
-      ports.ollama
-      ports.open_webui
-      ports.postgresql
-      ports.signal_cli
-      ports.syncthing
-    ];
-  };
-}

systems/jeeves/runners/default.nix

@@ -21,16 +21,5 @@
     nix-builder-12.enable = true;
     nix-builder-13.enable = true;
     nix-builder-14.enable = true;
-
-    # Warm the shared x86-64-v3 cache before the smaller per-system runners
-    # start. Eight jobs with eight cores each can use Jeeves' 64 logical CPUs,
-    # while the 6000% quota leaves some capacity for its normal services.
-    nix-cache-builder = {
-      enable = true;
-      labels = [ "nix-cache-builder:host" ];
-      cores = 8;
-      maxJobs = 8;
-      cpuQuota = "6000%";
-    };
   };
 }

systems/jeeves/runners/nix_builder.nix

@@ -1,7 +1,8 @@
 {
   config,
   lib,
-  pkgs,
+  outputs,
+  utils,
   ...
 }:
 
@@ -11,130 +12,6 @@ let
   cfg = config.services.nix_builder;
   runnerUsername = "gitea-runner";
   runnerUserid = 601;
-  containerConfig =
-    containerCfg:
-    {
-      config,
-      pkgs,
-      lib,
-      ...
-    }:
-    let
-      giteaAutomationPython = pkgs.python314.withPackages (
-        ps: with ps; [
-          httpx
-          pydantic
-          typer
-        ]
-      );
-      runnerConfigFile = (pkgs.formats.yaml { }).generate "gitea-runner.yaml" { };
-      registerRunner = pkgs.writeShellApplication {
-        name = "register-gitea-runner";
-        runtimeInputs = [
-          pkgs.coreutils
-          pkgs.hostname
-          config.services.gitea-actions-runner.package
-        ];
-        text = builtins.readFile ./register_gitea_runner.sh;
-      };
-    in
-    {
-      networking = {
-        hostName = "nix-builder";
-        useDHCP = lib.mkDefault true;
-        interfaces.eth0.useDHCP = true;
-        # Ensure containers don't inherit the host's stub resolver (127.0.0.53) which was causing issues
-        useHostResolvConf = false;
-      };
-      nix.settings = {
-        inherit (containerCfg) cores;
-        max-jobs = containerCfg.maxJobs;
-        system-features = lib.mkAfter [
-          "gccarch-x86-64-v2"
-          "gccarch-x86-64-v3"
-        ];
-        trusted-substituters = [
-          "https://cache.nixos.org"
-          "https://cache.tmmworkshop.com"
-          "https://nix-community.cachix.org"
-        ];
-        substituters = [
-          "https://cache.nixos.org/?priority=2&want-mass-query=true"
-          "https://cache.tmmworkshop.com/?priority=2&want-mass-query=true"
-          "https://nix-community.cachix.org/?priority=10&want-mass-query=true"
-        ];
-        trusted-public-keys = [
-          "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
-          "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA="
-          "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
-        ];
-        experimental-features = [
-          "flakes"
-          "nix-command"
-        ];
-        sandbox = true;
-        allowed-users = [ runnerUsername ];
-        trusted-users = [
-          "root"
-          runnerUsername
-        ];
-      };
-      users = {
-        users.${runnerUsername} = {
-          isSystemUser = true;
-          group = runnerUsername;
-          uid = runnerUserid;
-        };
-        groups.${runnerUsername}.gid = runnerUserid;
-      };
-      services.gitea-actions-runner.instances.builder = {
-        enable = true;
-        name = "jeeves-nix-builder";
-        url = "http://192.168.99.14:6443/";
-        labels = containerCfg.labels;
-        tokenFile = "/run/secrets/gitea-runners/registration-token";
-        settings.runner.timeout = "12h";
-        hostPackages = with pkgs; [
-          bash
-          coreutils
-          curl
-          gawk
-          gitMinimal
-          giteaAutomationPython
-          gnused
-          nix
-          nixfmt
-          nixos-rebuild
-          nodejs
-          ruff
-          treefmt
-          wget
-        ];
-      };
-      systemd.services."gitea-runner-builder" = {
-        serviceConfig = {
-          DynamicUser = mkForce false;
-          User = mkForce runnerUsername;
-          Group = mkForce runnerUsername;
-          ExecStartPre = mkForce [
-            "${getExe registerRunner} builder http://192.168.99.14:6443/ ${runnerConfigFile} ${escapeShellArgs containerCfg.labels}"
-          ];
-        };
-      };
-      system.stateVersion = "24.05";
-    };
-  mkContainerPath =
-    containerCfg:
-    (import "${pkgs.path}/nixos/lib/eval-config.nix" {
-      modules = [
-        {
-          boot.isNspawnContainer = true;
-          nixpkgs.pkgs = pkgs;
-        }
-        (containerConfig containerCfg)
-      ];
-      system = null;
-    }).config.system.build.toplevel;
 in
 {
   options.services.nix_builder = {
@@ -149,36 +26,7 @@ in
         types.submodule (
           { name, ... }:
           {
-            options = {
-              enable = mkEnableOption "Gitea runner container";
-
-              labels = mkOption {
-                type = types.listOf types.str;
-                default = [
-                  "self-hosted:host"
-                  "nixos:host"
-                ];
-                description = "Gitea Actions labels advertised by this runner.";
-              };
-
-              cores = mkOption {
-                type = types.ints.positive;
-                default = 8;
-                description = "Number of cores made available to each Nix build job.";
-              };
-
-              maxJobs = mkOption {
-                type = types.ints.positive;
-                default = 2;
-                description = "Maximum number of Nix build jobs run in parallel.";
-              };
-
-              cpuQuota = mkOption {
-                type = types.str;
-                default = "800%";
-                description = "systemd CPU quota for the runner container.";
-              };
-            };
+            options.enable = mkEnableOption "Gitea runner container";
           }
         )
       );
@@ -200,11 +48,9 @@ in
     containers = mapAttrs (
       name: containerCfg:
       mkIf containerCfg.enable {
-        path = mkContainerPath containerCfg;
         autoStart = true;
         privateNetwork = true;
         hostBridge = cfg.bridgeName;
-        extraFlags = [ "--hostname=${name}" ];
         bindMounts = {
           host-nix = {
             mountPoint = "/host-nix/var/nix/daemon-socket";
@@ -222,18 +68,106 @@ in
             isReadOnly = false;
           };
         };
+        config =
+          {
+            config,
+            pkgs,
+            lib,
+            ...
+          }:
+          {
+            networking = {
+              useDHCP = lib.mkDefault true;
+              interfaces.eth0.useDHCP = true;
+              # Ensure containers don't inherit the host's stub resolver (127.0.0.53) which was causing issues
+              useHostResolvConf = false;
+            };
+            nix.settings = {
+              trusted-substituters = [
+                "https://cache.nixos.org"
+                "https://cache.tmmworkshop.com"
+                "https://nix-community.cachix.org"
+              ];
+              substituters = [
+                "https://cache.nixos.org/?priority=2&want-mass-query=true"
+                "https://cache.tmmworkshop.com/?priority=2&want-mass-query=true"
+                "https://nix-community.cachix.org/?priority=10&want-mass-query=true"
+              ];
+              trusted-public-keys = [
+                "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
+                "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA="
+                "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
+              ];
+              experimental-features = [
+                "flakes"
+                "nix-command"
+              ];
+              sandbox = true;
+              allowed-users = [ "gitea-runner" ];
+              trusted-users = [
+                "root"
+                "gitea-runner"
+              ];
+            };
+            nixpkgs = {
+              overlays = builtins.attrValues outputs.overlays;
+              config.allowUnfree = true;
+            };
+            users = {
+              users.${runnerUsername} = {
+                isSystemUser = true;
+                group = runnerUsername;
+                uid = runnerUserid;
+              };
+              groups.${runnerUsername}.gid = runnerUserid;
+            };
+            services.gitea-actions-runner.instances.${name} = {
+              enable = true;
+              name = "jeeves-${name}";
+              url = "http://192.168.99.14:6443/";
+              labels = [
+                "self-hosted:host"
+                "nixos:host"
+              ];
+              tokenFile = "/run/secrets/gitea-runners/registration-token";
+              hostPackages = with pkgs; [
+                bash
+                coreutils
+                curl
+                gawk
+                gitMinimal
+                gnused
+                my_python
+                nix
+                nixfmt
+                nixos-rebuild
+                nodejs
+                treefmt
+                wget
+              ];
+            };
+            systemd.services."gitea-runner-${utils.escapeSystemdPath name}" = {
+              serviceConfig = {
+                DynamicUser = mkForce false;
+                User = mkForce runnerUsername;
+                Group = mkForce runnerUsername;
+              };
+            };
+            system.stateVersion = "24.05";
+          };
       }
     ) cfg.containers;
 
     systemd = {
-      services = mapAttrs' (
-        name: containerCfg:
-        nameValuePair "container@${name}" {
-          requires = [ "gitea.service" ];
-          after = [ "gitea.service" ];
-          serviceConfig.CPUQuota = containerCfg.cpuQuota;
-        }
-      ) (filterAttrs (_: c: c.enable) cfg.containers);
+      services = builtins.listToAttrs (
+        map (name: {
+          name = "container@${name}";
+          value = {
+            requires = [ "gitea.service" ];
+            after = [ "gitea.service" ];
+          };
+        }) (builtins.attrNames (filterAttrs (_: c: c.enable) cfg.containers))
+      );
 
       tmpfiles.rules = [
         "d ${vars.uv_cache} 0755 ${runnerUsername} ${runnerUsername} - -"

systems/jeeves/runners/register_gitea_runner.sh

@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-
-legacy_instance_dir="$STATE_DIRECTORY/$(hostname)"
-instance_dir="$STATE_DIRECTORY/$1"
-runner_url="$2"
-runner_config_file="$3"
-shift 3
-runner_labels="$(IFS=,; printf '%s' "$*")"
-
-if [ ! -e "$instance_dir" ] && [ -d "$legacy_instance_dir" ]; then
-  mv "$legacy_instance_dir" "$instance_dir"
-fi
-
-mkdir -vp "$instance_dir"
-cd "$instance_dir" || exit 1
-
-token_hash_file="$instance_dir/.token-hash"
-token_hash_current="$(printf '%s' "$TOKEN" | sha256sum | cut -d' ' -f1)"
-token_hash_stored="$(cat "$token_hash_file" 2>/dev/null || echo "")"
-labels_file="$instance_dir/.labels"
-labels_wanted="$(printf '%s\n' "$@" | sort)"
-labels_current="$(cat "$labels_file" 2>/dev/null || echo 0)"
-
-if [ ! -e "$instance_dir/.runner" ] || [ "$labels_wanted" != "$labels_current" ] || [ "$token_hash_current" != "$token_hash_stored" ]; then
-  rm -v "$instance_dir/.runner" || true
-
-  gitea-runner register --no-interactive \
-    --instance "$runner_url" \
-    --token "$TOKEN" \
-    --name "jeeves-$(hostname)" \
-    --labels "$runner_labels" \
-    --config "$runner_config_file"
-
-  printf '%s' "$token_hash_current" > "$token_hash_file"
-  printf '%s\n' "$labels_wanted" > "$labels_file"
-fi

systems/jeeves/scripts/zfs.sh

@@ -1,5 +1,10 @@
 #!/bin/bash
 
+# Pool and vdev creation only. This is run by hand once per pool.
+#
+# Datasets and their properties are declared in systems/jeeves/zfs.nix and
+# reconciled by the zfs_manager service. Do not add zfs create lines here.
+
 # zpools
 
 # media
@@ -12,35 +17,10 @@ sudo zpool add storage -o ashift=12 special mirror
 sudo zpool add storage -o ashift=12 logs mirror
 
 # scratch
-sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnodesize=auto -O xattr=sa -O compression=zstd -O encryption=aes-256-gcm -O keyformat=hex -O keylocation=file:///key -m /zfs/scratch
+sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnodesize=auto -O xattr=sa -O compression=zstd -O encryption=aes-256-gcm -O keyformat=hex -O keylocation=file:///root/zfs.key -m /zfs/scratch
 
-# media datasets
-sudo zfs create media/temp -o sync=disabled -o redundant_metadata=none
+# The two encrypted parent datasets have to exist before zfs_manager can create
+# anything under them, since encryption cannot be set after creation.
+# These will be removed if/when the media and storage pools are encrypted in the future.
 sudo zfs create media/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key
-sudo zfs create media/secure/containers -o mountpoint=/zfs/media/containers -o compression=lz4 -o sync=disabled -o redundant_metadata=some -o normalization=none -o utf8only=off
-sudo zfs create media/secure/docker -o compression=zstd-9
-sudo zfs create media/secure/github-runners -o compression=zstd-9 -o sync=disabled
-sudo zfs create media/secure/notes -o copies=2
-sudo zfs create media/secure/postgres -o mountpoint=/zfs/media/database/postgres -o recordsize=16k -o primarycache=metadata
-sudo zfs create media/secure/postgres-wal -o mountpoint=/zfs/media/database/postgres-wal -o recordsize=32k -o primarycache=metadata -o special_small_blocks=32K -o compression=lz4 -o secondarycache=none -o logbias=latency
-sudo zfs create media/secure/prometheus -o mountpoint=/zfs/media/database/prometheus -o compression=lz4
-sudo zfs create media/secure/services -o compression=zstd-9
-sudo zfs create media/secure/share -o mountpoint=/zfs/media/share -o exec=off
-
-# scratch datasets
-sudo zfs create scratch/kafka -o mountpoint=/zfs/scratch/kafka -o recordsize=1M
-sudo zfs create scratch/transmission -o mountpoint=/zfs/scratch/transmission -o recordsize=16k -o sync=disabled -o redundant_metadata=none
-sudo zfs create scratch/uv_cache -o mountpoint=/zfs/scratch/uv_cache
-
-# storage datasets
-sudo zfs create storage/ollama -o recordsize=1M -o compression=zstd-19 -o sync=disabled
 sudo zfs create storage/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key
-sudo zfs create storage/secure/archive -o recordsize=1M -o compression=zstd-19
-sudo zfs create storage/secure/library -o recordsize=1M -o compression=zstd-19
-sudo zfs create storage/secure/main -o compression=zstd-19
-sudo zfs create storage/secure/photos -o recordsize=16K -o compression=zstd-19 -o copies=2
-sudo zfs create storage/secure/plex -o recordsize=1M -o compression=zstd-19
-sudo zfs create storage/secure/secrets -o compression=zstd-19 -o copies=3
-sudo zfs create storage/secure/syncthing -o compression=zstd-19
-sudo zfs create storage/secure/transmission -o recordsize=1M -o compression=zstd-9 -o exec=off -o sync=disabled
-sudo zfs create storage/secure/important -o compression=zstd-19 -o copies=2 -o mountpoint=/zfs/storage/important

systems/jeeves/services/audiobookshelf.nix

@@ -6,7 +6,6 @@ in
   services.audiobookshelf = {
     enable = true;
     port = 8000;
-    host = "0.0.0.0";
   };
   systemd.services.audiobookshelf.serviceConfig.WorkingDirectory =
     lib.mkForce "${vars.docker_configs}/audiobookshelf";

systems/jeeves/services/camofox-browser.nix

@@ -0,0 +1,80 @@
+{
+  ...
+}:
+let
+  vars = import ../vars.nix;
+in
+{
+  systemd.tmpfiles.rules = [
+    "d ${vars.docker_configs}/camofox-browser 0750 root root - -"
+  ];
+
+  containers.camofox-browser = {
+    autoStart = true;
+    privateNetwork = false;
+    bindMounts = {
+      camofox-browser = {
+        hostPath = "${vars.docker_configs}/camofox-browser";
+        mountPoint = "/var/lib/camofox-browser";
+        isReadOnly = false;
+      };
+    };
+    config =
+      {
+        pkgs,
+        lib,
+        ...
+      }:
+      {
+        networking.hostName = "camofox-browser";
+
+        environment.systemPackages = with pkgs; [
+          ffmpeg
+          git
+          nodejs
+          python3Packages.yt-dlp
+        ];
+
+        systemd.services.camofox-browser = {
+          description = "Camofox browser server";
+          wantedBy = [ "multi-user.target" ];
+          after = [ "network.target" ];
+          environment = {
+            CAMOFOX_HOST = "127.0.0.1";
+            CAMOFOX_PORT = "9377";
+            HOME = "/var/lib/camofox-browser";
+          };
+          path = with pkgs; [
+            bash
+            coreutils
+            git
+            nodejs
+          ];
+          serviceConfig = {
+            Restart = "always";
+            RestartSec = "5s";
+            WorkingDirectory = "/var/lib/camofox-browser";
+          };
+          script = ''
+            set -eu
+
+            app_dir=/var/lib/camofox-browser/app
+
+            if [ ! -d "$app_dir/.git" ]; then
+              git clone --depth 1 https://github.com/jo-inc/camofox-browser "$app_dir"
+            fi
+
+            cd "$app_dir"
+
+            if [ ! -d node_modules ]; then
+              npm install
+            fi
+
+            exec npm start
+          '';
+        };
+
+        system.stateVersion = lib.mkDefault "24.05";
+      };
+  };
+}

systems/jeeves/services/gems.nix

@@ -6,47 +6,8 @@
 let
   vars = import ../vars.nix;
   stateDir = "${vars.services}/gems";
-  gemsPackages =
-    ps: with ps; [
-      fastapi
-      jinja2
-      pydantic
-      pydantic-settings
-      python-multipart
-      typer
-      uvicorn
-    ];
 in
 {
-  nixpkgs.overlays = [
-    (final: _prev: {
-      gems_python = final.python314.withPackages gemsPackages;
-      gems_test_python = final.python314.withPackages (
-        ps:
-        gemsPackages ps
-        ++ (with ps; [
-          httpx
-          pytest
-          pytest-asyncio
-          pytest-xdist
-        ])
-      );
-      gems_tests =
-        final.runCommand "gems-tests"
-          {
-            nativeBuildInputs = [ final.gems_test_python ];
-          }
-          ''
-            export HOME="$TMPDIR"
-            cd ${inputs.self}
-            pytest -o cache_dir="$TMPDIR/pytest-cache" tests/gems
-            touch "$out"
-          '';
-    })
-  ];
-
-  system.checks = [ pkgs.gems_tests ];
-
   users.groups.gems = { };
   users.users.gems = {
     isSystemUser = true;
@@ -75,7 +36,7 @@ in
       Type = "simple";
       User = "gems";
       Group = "gems";
-      ExecStart = "${pkgs.gems_python}/bin/python -m python.gems.main --host 0.0.0.0 --port 8002";
+      ExecStart = "${pkgs.my_python}/bin/python -m python.gems.main --host 127.0.0.1 --port 8002";
       Restart = "on-failure";
       RestartSec = "5s";
       StandardOutput = "journal";

systems/jeeves/services/gitea.nix

@@ -2,6 +2,11 @@ let
   vars = import ../vars.nix;
 in
 {
+  networking.firewall.allowedTCPPorts = [
+    6443
+    2223
+  ];
+
   services.gitea = {
     enable = true;
     appName = "TMM Workshop";
@@ -19,18 +24,13 @@ in
       actions = {
         ENABLED = true;
         DEFAULT_ACTIONS_URL = "github";
-        ENDLESS_TASK_TIMEOUT = "12h";
       };
       service.DISABLE_REGISTRATION = true;
       server = {
-        DOMAIN = "gitea.tmmworkshop.com";
+        DOMAIN = "tmmworkshop.com";
         ROOT_URL = "https://gitea.tmmworkshop.com/";
         HTTP_PORT = 6443;
-        BUILTIN_SSH_SERVER_USER = "gitea";
-        SSH_USER = "gitea";
-        SSH_DOMAIN = "gitea.tmmworkshop.com";
         SSH_PORT = 2223;
-        SSH_LISTEN_HOST = "0.0.0.0";
         SSH_LISTEN_PORT = 2223;
         START_SSH_SERVER = true;
         PUBLIC_URL_DETECTION = "auto";

systems/jeeves/services/grafana.nix

@@ -6,6 +6,8 @@ let
   grafanaDataDir = "${vars.services}/grafana";
 in
 {
+  networking.firewall.allowedTCPPorts = [ 3000 ];
+
   services.grafana = {
     enable = true;
     dataDir = grafanaDataDir;
@@ -17,9 +19,9 @@ in
         secret_key = "$__file{${vars.secrets}/services/grafana/secret_key}";
       };
       server = {
-        http_addr = "0.0.0.0";
+        http_addr = "192.168.90.40";
         http_port = 3000;
-        root_url = "http://jeeves:3000/";
+        root_url = "http://192.168.90.40:3000/";
       };
     };
     provision = {

systems/jeeves/services/jellyfin.nix

@@ -4,6 +4,7 @@ in
 {
   services.jellyfin = {
     enable = true;
+    openFirewall = true;
     dataDir = "${vars.services}/jellyfin";
     cacheDir = "${vars.services}/jellyfin/cache";
   };

systems/jeeves/services/llms.nix

@@ -38,6 +38,7 @@ in
       "qwen3.5:35b"
     ];
     modelsDir = vars.ollama;
+    openFirewall = true;
   };
   systemd.services = {
     ollama.serviceConfig = {

systems/jeeves/services/nix_serve.nix

@@ -7,5 +7,6 @@ in
     package = pkgs.nix-serve-ng;
     enable = true;
     secretKeyFile = "${vars.secrets}/services/nix-cache/cache-priv-key.pem";
+    openFirewall = true;
   };
 }

systems/jeeves/services/open_webui.nix

@@ -0,0 +1,13 @@
+{
+  services.open-webui = {
+    enable = true;
+    host = "0.0.0.0";
+    openFirewall = true;
+    environment = {
+      ANONYMIZED_TELEMETRY = "False";
+      DO_NOT_TRACK = "True";
+      SCARF_NO_ANALYTICS = "True";
+      OLLAMA_API_BASE_URL = "http://127.0.0.1:11434";
+    };
+  };
+}

systems/jeeves/services/postgress.nix

@@ -3,6 +3,8 @@ let
   vars = import ../vars.nix;
 in
 {
+  networking.firewall.allowedTCPPorts = [ 5432 ];
+
   # Symlink pg_wal to a ZFS dataset on the special (metadata) vdev for fast WAL writes
   # this is required for systemd sandboxing
   systemd.services.postgresql.serviceConfig.ReadWritePaths = [ "/zfs/media/database/postgres-wal" ];
@@ -25,7 +27,7 @@ in
       local all  richie   trust
       host  all  richie   127.0.0.1/32    trust
       host  all  richie   ::1/128         trust
-      host  all  richie   100.64.0.0/10 trust
+      host  all  richie   192.168.90.1/24 trust
       host  all  richie   192.168.99.1/24 trust
       host  all  richie   172.16.0.0/12   trust
 
@@ -40,12 +42,12 @@ in
       local postgres  math   trust
       host  postgres  math   127.0.0.1/32    trust
       host  postgres  math   ::1/128         trust
-      host  postgres  math   100.64.0.0/10 trust
+      host  postgres  math   192.168.90.1/24 trust
 
       local data_science_dev  math   trust
       host  data_science_dev  math   127.0.0.1/32    trust
       host  data_science_dev  math   ::1/128         trust
-      host  data_science_dev  math   100.64.0.0/10 trust
+      host  data_science_dev  math   192.168.90.1/24 trust
     '';
 
     identMap = ''

systems/jeeves/services/systemd.nix

@@ -1,7 +1,11 @@
 {
   pkgs,
+  inputs,
   ...
 }:
+let
+  vars = import ../vars.nix;
+in
 {
   systemd = {
     services = {
@@ -26,6 +30,21 @@
           ExecStart = "${pkgs.bash}/bin/bash -c 'echo 1 > /sys/bus/pci/devices/0000:61:00.0/remove'";
         };
       };
+      startup_validation = {
+        requires = [ "network-online.target" ];
+        after = [ "network-online.target" ];
+        wantedBy = [ "multi-user.target" ];
+        description = "validates startup";
+        path = [ pkgs.zfs ];
+        environment = {
+          PYTHONPATH = "${inputs.self}/";
+        };
+        serviceConfig = {
+          EnvironmentFile = "${vars.secrets}/services/server-validation";
+          Type = "oneshot";
+          ExecStart = "${pkgs.my_python}/bin/python -m python.system_tests.validate_system '${./validate_system.toml}'";
+        };
+      };
     };
     timers = {
       plex_permission = {
@@ -36,6 +55,13 @@
           Unit = "plex_permission.service";
         };
       };
+      startup_validation = {
+        wantedBy = [ "timers.target" ];
+        timerConfig = {
+          OnBootSec = "10min";
+          Unit = "startup_validation.service";
+        };
+      };
     };
   };
 }

systems/jeeves/services/transmission.nix

@@ -21,11 +21,11 @@ in
       message-level = 3;
       peer-port = 51413;
       rpc-bind-address = "0.0.0.0";
-      rpc-host-whitelist = "127.0.0.1,jeeves,jeeves.taile39c31.ts.net";
+      rpc-host-whitelist = "127.0.0.1,192.168.90.40";
       rpc-host-whitelist-enabled = true;
       rpc-port = 9091;
       rpc-whitelist-enabled = true;
-      rpc-whitelist = "127.0.0.1,100.*.*.*";
+      rpc-whitelist = "127.0.0.1,192.168.90.49";
       seed-queue-enabled = false;
     };
   };

systems/jeeves/services/validate_system.toml

@@ -0,0 +1,7 @@
+zpool = ["root_pool", "storage", "media"]
+services = [
+    "audiobookshelf",
+    "haproxy",
+    "docker",
+    "jellyfin",
+]

systems/jeeves/snapshot_config.toml

@@ -1,141 +0,0 @@
-["default"]
-15_min = 8
-hourly = 24
-daily = 0
-monthly = 0
-
-# root_pool
-["root_pool/home"]
-15_min = 8
-hourly = 24
-daily = 14
-monthly = 0
-
-["root_pool/root"]
-15_min = 8
-hourly = 24
-daily = 0
-monthly = 0
-
-["root_pool/nix"]
-15_min = 4
-hourly = 0
-daily = 0
-monthly = 0
-
-["root_pool/nix_build"]
-15_min = 1
-hourly = 0
-daily = 0
-monthly = 0
-
-["root_pool/var"]
-15_min = 8
-hourly = 24
-daily = 30
-monthly = 6
-# storage
-["storage/ollama"]
-15_min = 2
-hourly = 0
-daily = 0
-monthly = 0
-
-["storage/secure"]
-15_min = 0
-hourly = 0
-daily = 0
-monthly = 0
-
-["storage/secure/plex"]
-15_min = 6
-hourly = 2
-daily = 1
-monthly = 0
-
-["storage/secure/transmission"]
-15_min = 4
-hourly = 0
-daily = 0
-monthly = 0
-
-["storage/secure/secrets"]
-15_min = 8
-hourly = 24
-daily = 30
-monthly = 12
-
-# media
-["media/secure/containers"]
-15_min = 2
-hourly = 0
-daily = 0
-monthly = 0
-
-["media/temp"]
-15_min = 2
-hourly = 0
-daily = 0
-monthly = 0
-
-["media/secure"]
-15_min = 0
-hourly = 0
-daily = 0
-monthly = 0
-
-["media/secure/plex"]
-15_min = 6
-hourly = 2
-daily = 1
-monthly = 0
-
-["media/secure/postgres-wal"]
-15_min = 4
-hourly = 2
-daily = 0
-monthly = 0
-
-
-["media/secure/postgres"]
-15_min = 8
-hourly = 24
-daily = 7
-monthly = 0
-
-["media/secure/share"]
-15_min = 4
-hourly = 0
-daily = 0
-monthly = 0
-
-["media/secure/github-runners"]
-15_min = 6
-hourly = 2
-daily = 1
-monthly = 0
-
-["media/secure/notes"]
-15_min = 8
-hourly = 24
-daily = 30
-monthly = 12
-
-["media/secure/docker"]
-15_min = 3
-hourly = 12
-daily = 14
-monthly = 2
-
-# scratch
-["scratch/transmission"]
-15_min = 2
-hourly = 0
-daily = 0
-monthly = 0
-
-["scratch/uv_cache"]
-15_min = 2
-hourly = 0
-daily = 0
-monthly = 0