chore: update flake.lock

im not using firefox any more and with the move to x86 v3 1 less browser build is nice

.github/workflows/build_systems.yml

@@ -18,6 +18,7 @@ jobs:
           - "brain"
           - "jeeves"
           - "rhapsody-in-green"
+          - "portal-1"
     continue-on-error: true
     steps:
       - uses: actions/checkout@v4

.github/workflows/merge_flake_lock_update.yml

@@ -15,9 +15,8 @@ jobs:
     steps:
       - name: merge_flake_lock_update
         run: >-
-          nix develop .#devShells.x86_64-linux.default -c
           python -m python.gitea_flake_lock merge
           --repo "${{ github.repository }}"
         env:
-          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+          JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
           GITEA_URL: https://gitea.tmmworkshop.com

.github/workflows/pytest.yml

@@ -1,19 +0,0 @@
-name: pytest
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
-
-jobs:
-  pytest:
-    runs-on: self-hosted
-
-    steps:
-      - uses: actions/checkout@v4
-      - name: Run tests
-        run: nix develop .#devShells.x86_64-linux.default -c pytest tests

.github/workflows/update-flake-lock.yml

@@ -18,9 +18,8 @@ jobs:
         run: nix flake update
       - name: Create or update flake.lock PR
         env:
-          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+          JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
           GITEA_URL: https://gitea.tmmworkshop.com
         run: >-
-          nix develop .#devShells.x86_64-linux.default -c
           python -m python.gitea_flake_lock update
           --repo "${{ github.repository }}"

.sops.yaml

@@ -1,6 +1,9 @@
 # Generate AGE keys from SSH keys with:
 #   ssh-keygen -A
 #   nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
+# update keys after addin/removing a key
+#   nix-shell -p sops --run "sops updatekeys users/secrets.yaml" users/secrets.yaml
+
 keys:
   - &admin_richie age1u8zj599elqqvcmhxn8zuwrufsz8w8w366d3ayrljjejljt2q45kq8mxw9c # cspell:disable-line
 
@@ -8,6 +11,7 @@ keys:
   - &system_brain age1jhf7vm0005j60mjq63696frrmjhpy8kpc2d66mw044lqap5mjv4snmwvwm # cspell:disable-line
   - &system_jeeves age13lmqgc3jvkyah5e3vcwmj4s5wsc2akctcga0lpc0x8v8du3fxprqp4ldkv # cspell:disable-line
   - &system_rhapsody age1ufnewppysaq2wwcl4ugngjz8pfzc5a35yg7luq0qmuqvctajcycs5lf6k4 # cspell:disable-line
+  - &system_portal_1 age1vyav6kxtvt3z4vtnkkjj38eu8hlts5m7ygyckhskvalg2gpjk52su53d0a # cspell:disable-line
 
 creation_rules:
   - path_regex: users/secrets\.yaml$
@@ -18,3 +22,4 @@ creation_rules:
           - *system_brain
           - *system_jeeves
           - *system_rhapsody
+          - *system_portal_1

.vscode/settings.json

@@ -10,7 +10,6 @@
     "aiounifi",
     "alsa",
     "apiclient",
-    "apscheduler",
     "archlinux",
     "ashift",
     "asrouter",
@@ -337,8 +336,6 @@
     "yubioath",
     "yzhang",
     "zeroconf",
-    "zerotier",
-    "zerotierone",
     "zoxide",
     "zram",
     "zstd"

Cargo.toml

@@ -0,0 +1,4 @@
+[workspace]
+resolver = "2"
+
+members = ["rust/*"]

common/global/default.nix

@@ -17,16 +17,11 @@
     ./nix.nix
     ./programs.nix
     ./ssh.nix
-    ./snapshot_manager.nix
   ];
 
   boot = {
-    tmp.useTmpfs = true;
+    tmp.useTmpfs = lib.mkDefault true;
     kernelPackages = lib.mkDefault pkgs.linuxPackages_6_12;
-    zfs = {
-      package = lib.mkDefault pkgs.zfs_2_4;
-      forceImportRoot = lib.mkDefault false;
-    };
   };
 
   hardware.enableRedistributableFirmware = true;
@@ -42,9 +37,6 @@
     overlays = builtins.attrValues outputs.overlays;
     config = {
       allowUnfree = true;
-      permittedInsecurePackages = [
-        "openssl-1.1.1w" # This is for discord-canary
-      ];
     };
   };
 
@@ -53,16 +45,6 @@
 
     # firmware update
     fwupd.enable = true;
-
-    snapshot_manager = {
-      enable = lib.mkDefault true;
-      PYTHONPATH = "${inputs.self}/";
-    };
-
-    zfs = {
-      trim.enable = lib.mkDefault true;
-      autoScrub.enable = lib.mkDefault true;
-    };
   };
 
   powerManagement.powertop.enable = lib.mkDefault true;

common/global/nix.nix

@@ -31,6 +31,10 @@ in
         "flakes"
         "ca-derivations"
       ];
+      system-features = lib.mkAfter [
+        "gccarch-x86-64-v2"
+        "gccarch-x86-64-v3"
+      ];
       warn-dirty = false;
       flake-registry = ""; # disable global flake registries
       connect-timeout = 10;

common/global/programs.nix

@@ -2,6 +2,5 @@
 {
   environment.systemPackages = with pkgs; [
     git
-    my_python
   ];
 }

common/optional/docker.nix

@@ -17,6 +17,7 @@
     logDriver = "local";
     storageDriver = "overlay2";
     daemon.settings = {
+      live-restore = false;
       experimental = true;
       exec-opts = [ "native.cgroupdriver=systemd" ];
       log-opts = {

common/optional/monitoring-agent.nix

@@ -5,7 +5,7 @@
   ...
 }:
 let
-  monitoringInterface = "ztwfunumly";
+  monitoringInterface = "tailscale0";
   nodeTextfileDir = "/var/lib/prometheus-node-exporter-textfile";
 
   mkProcessNameTemplate =

common/optional/tailscale.nix

@@ -0,0 +1,37 @@
+{
+  config,
+  inputs,
+  ...
+}:
+{
+  nix.settings = {
+    trusted-substituters = [ "http://jeeves:5000" ];
+    substituters = [ "http://jeeves:5000/?priority=1&want-mass-query=true" ];
+    trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
+  };
+
+  services.tailscale = {
+    enable = true;
+    openFirewall = true;
+    authKeyFile = config.sops.secrets.tailscale_auth_key.path;
+
+    # OAuth client secrets create ephemeral nodes by default. NixOS machines
+    # are persistent and should enroll without interactive device approval.
+    authKeyParameters = {
+      ephemeral = false;
+      preauthorized = true;
+    };
+
+    extraUpFlags = [ "--advertise-tags=tag:nixos" ];
+  };
+
+  sops = {
+    age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
+
+    secrets.tailscale_auth_key = {
+      sopsFile = "${inputs.self}/users/secrets.yaml";
+      owner = "root";
+      mode = "0400";
+    };
+  };
+}

common/optional/x86-64-v3.nix

@@ -0,0 +1,9 @@
+{
+  nixpkgs.hostPlatform = {
+    system = "x86_64-linux";
+    gcc = {
+      arch = "x86-64-v3";
+      tune = "generic";
+    };
+  };
+}

common/optional/zerotier.nix

@@ -1,11 +0,0 @@
-{
-  services.zerotierone = {
-    enable = true;
-    joinNetworks = [ "e4da7455b2ae64ca" ];
-  };
-  nix.settings = {
-    trusted-substituters = [ "http://192.168.90.40:5000" ];
-    substituters = [ "http://192.168.90.40:5000/?priority=1&want-mass-query=true" ];
-    trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
-  };
-}

common/optional/zfs/default.nix

@@ -0,0 +1,26 @@
+{
+  inputs,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  imports = [ ./snapshot.nix ];
+
+  boot.zfs = {
+    package = lib.mkDefault pkgs.zfs_2_4;
+    forceImportRoot = lib.mkDefault false;
+  };
+
+  services = {
+    snapshot_manager = {
+      enable = lib.mkDefault true;
+      PYTHONPATH = "${inputs.self}/";
+    };
+
+    zfs = {
+      trim.enable = lib.mkDefault true;
+      autoScrub.enable = lib.mkDefault true;
+    };
+  };
+}

common/optional/zfs/snapshot.nix

@@ -1,4 +1,5 @@
 {
+  inputs,
   pkgs,
   lib,
   config,
@@ -6,6 +7,11 @@
 }:
 let
   cfg = config.services.snapshot_manager;
+  snapshotManagerPackages =
+    ps: with ps; [
+      httpx
+      typer
+    ];
 in
 {
   options = {
@@ -34,6 +40,41 @@ in
   };
 
   config = lib.mkIf cfg.enable {
+    nixpkgs.overlays = [
+      (final: _prev: {
+        snapshot_manager_python = final.python314.withPackages snapshotManagerPackages;
+        snapshot_manager_test_python = final.python314.withPackages (
+          ps:
+          snapshotManagerPackages ps
+          ++ (with ps; [
+            pyfakefs
+            pytest
+            pytest-asyncio
+            pytest-mock
+            pytest-xdist
+          ])
+        );
+        snapshot_manager_tests =
+          final.runCommand "snapshot-manager-tests"
+            {
+              nativeBuildInputs = [ final.snapshot_manager_test_python ];
+            }
+            ''
+              export HOME="$TMPDIR"
+              cd ${inputs.self}
+              pytest \
+                -o cache_dir="$TMPDIR/pytest-cache" \
+                tests/test_common.py \
+                tests/test_signal_alert.py \
+                tests/test_snapshot_manager.py \
+                tests/test_zfs.py
+              touch "$out"
+            '';
+      })
+    ];
+
+    system.checks = [ pkgs.snapshot_manager_tests ];
+
     systemd = {
       services.snapshot_manager = {
         description = "ZFS Snapshot Manager";
@@ -45,7 +86,7 @@ in
         };
         serviceConfig = {
           Type = "oneshot";
-          ExecStart = "${pkgs.my_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
+          ExecStart = "${pkgs.snapshot_manager_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
         }
         // lib.optionalAttrs (cfg.EnvironmentFile != null) {
           EnvironmentFile = cfg.EnvironmentFile;

common/optional/zfs/snapshot_config.toml

@@ -22,6 +22,12 @@ hourly = 0
 daily = 0
 monthly = 0
 
+["root_pool/nix_build"]
+15_min = 1
+hourly = 0
+daily = 0
+monthly = 0
+
 ["root_pool/var"]
 15_min = 8
 hourly = 24

flake.lock

@@ -1,25 +1,23 @@
 {
   "nodes": {
-    "firefox-addons": {
+    "disko": {
       "inputs": {
         "nixpkgs": [
           "nixpkgs"
         ]
       },
       "locked": {
-        "dir": "pkgs/firefox-addons",
-        "lastModified": 1785729742,
-        "narHash": "sha256-PBavY37OTsIM7VJMUYPv2Rz/gSpbtJGxAGl9iXCaMU4=",
-        "owner": "rycee",
-        "repo": "nur-expressions",
-        "rev": "1529ecae5978cd2ac18a8edbf27967350bf4b80e",
-        "type": "gitlab"
+        "lastModified": 1789770686,
+        "narHash": "sha256-uZkBR7yHdIKUFB5SZdfgh1qkGfI3XmYmI/lTiquxbck=",
+        "owner": "nix-community",
+        "repo": "disko",
+        "rev": "725ea35e410ad83be4931d1bff7e090eacaf3563",
+        "type": "github"
       },
       "original": {
-        "dir": "pkgs/firefox-addons",
-        "owner": "rycee",
-        "repo": "nur-expressions",
-        "type": "gitlab"
+        "owner": "nix-community",
+        "repo": "disko",
+        "type": "github"
       }
     },
     "home-manager": {
@@ -29,11 +27,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1785531816,
-        "narHash": "sha256-vkMnV0JIyw+g/NmcfoajlGaAO+9a0ezia+FZohQJrik=",
+        "lastModified": 1790365885,
+        "narHash": "sha256-iQ0ebhiVo64NktTnwft9hNxlFu4j5cvljVxyJaEVIP4=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "bf9ce9fec78f95f374e8dd3b503863a3ec128ebe",
+        "rev": "7b4c5ec4bedaf1e062bbc1bcaeddbc6bd242aa1b",
         "type": "github"
       },
       "original": {
@@ -47,11 +45,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1785232496,
-        "narHash": "sha256-65EQYIRRpTdpH8lUiB6Mvo5uBkG60aBIzAJuALfx+O0=",
+        "lastModified": 1790321948,
+        "narHash": "sha256-mZGHLGi217oLIwzDM4PNRZeCNa6g2WBXURyXbNMFs7Y=",
         "owner": "nixos",
         "repo": "nixos-hardware",
-        "rev": "2e790b0a6be8ec2b76174ac0931b8ff11919ec98",
+        "rev": "30d48a0ec6035f8140d0125af274f0de95f1e9b5",
         "type": "github"
       },
       "original": {
@@ -63,11 +61,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1767892417,
-        "narHash": "sha256-8bW3q88CEg2u4hSP66Vf4lpbLonHz7hqDNBMcCY7E9U=",
-        "rev": "3497aa5c9457a9d88d71fa93a4a8368816fbeeba",
+        "lastModified": 1789546076,
+        "narHash": "sha256-vWkSk5bbfTqdtMoSgD9FshACO8JCvXTFi+3cqEp0mH0=",
+        "rev": "b1b875982b17dabde9b4a37f3e229e74913e6db3",
         "type": "tarball",
-        "url": "https://releases.nixos.org/nixos/unstable/nixos-26.05pre924538.3497aa5c9457/nixexprs.tar.xz"
+        "url": "https://releases.nixos.org/nixos/unstable/nixos-26.11pre1074753.b1b875982b17/nixexprs.tar.xz"
       },
       "original": {
         "type": "tarball",
@@ -76,11 +74,11 @@
     },
     "nixpkgs-master": {
       "locked": {
-        "lastModified": 1785777863,
-        "narHash": "sha256-BPLjbZgQ7hud+zRbLDqyFAW4qTKD2MBLR+TRI7oOg/w=",
+        "lastModified": 1790378376,
+        "narHash": "sha256-pTfAyxq8916OZDMlXUqH9q7o41nGXaedCjsNy/EYjNI=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "0adbd05f9f410c2b26d550d344ac019efa6b7224",
+        "rev": "4775b2ca8bed8ec8f07a06a815e9c9ab94b64403",
         "type": "github"
       },
       "original": {
@@ -108,11 +106,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1785692966,
-        "narHash": "sha256-vUfIeBEfpbAfZ5zjgIkYk7eHBeVfCYVjLbWnMkseYnk=",
+        "lastModified": 1790185690,
+        "narHash": "sha256-xJ+X4hBtOcAFGBOe5nAMyMUeF9foJBmIOu3NjBqBycU=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "643809054d65fdd466a63e3155b8c498cb483c04",
+        "rev": "4975466d324710c576dc11ad614684e6bd8cad8e",
         "type": "github"
       },
       "original": {
@@ -124,7 +122,7 @@
     },
     "root": {
       "inputs": {
-        "firefox-addons": "firefox-addons",
+        "disko": "disko",
         "home-manager": "home-manager",
         "nixos-hardware": "nixos-hardware",
         "nixpkgs": "nixpkgs_2",
@@ -141,11 +139,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1783174389,
-        "narHash": "sha256-aCWC8ngycU7OdJrU2+Je3qf+1a2ykuBvpPhZT/9tXMc=",
+        "lastModified": 1790247521,
+        "narHash": "sha256-TNfgoHsqsYYvaJImlWctfRkh4PseTagzegU1Dgdbehw=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "f1406619a3884cd5c47992a70b8b35c9c0fcb4c9",
+        "rev": "2bd00bd9bb35fe6d114888c8f1c2e946c541dd8f",
         "type": "github"
       },
       "original": {

flake.nix

@@ -26,13 +26,13 @@
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
-    firefox-addons = {
-      url = "gitlab:rycee/nur-expressions?dir=pkgs/firefox-addons";
+    sops-nix = {
+      url = "github:Mic92/sops-nix";
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
-    sops-nix = {
-      url = "github:Mic92/sops-nix";
+    disko = {
+      url = "github:nix-community/disko";
       inputs.nixpkgs.follows = "nixpkgs";
     };
   };

overlays/default.nix

@@ -15,35 +15,6 @@
     };
   };
 
-  python-env = final: _prev: {
-    my_python = final.python314.withPackages (
-      ps: with ps; [
-        alembic
-        apprise
-        apscheduler
-        fastapi
-        fastapi-cli
-        httpx
-        jinja2
-        mypy
-        pgvector
-        psycopg
-        pydantic
-        pyfakefs
-        pytest
-        pytest-cov
-        pytest-mock
-        pytest-xdist
-        python-multipart
-        pydantic-settings
-        ruff
-        sqlalchemy
-        tenacity
-        tinytuya
-        typer
-        uvicorn
-        websockets
-      ]
-    );
-  };
+  test-exclusions = import ./test-exclusions.nix;
+  x86-64-v3-workarounds = import ./x86-64-v3-workarounds.nix;
 }

overlays/test-exclusions.nix

@@ -0,0 +1,133 @@
+# Test exclusions for the locally rebuilt x86-64-v3 package set.
+#
+# Selecting x86-64-v3 changes every affected derivation, so the normal
+# nixpkgs binary cache cannot be used and upstream test suites run locally.
+# The jeeves builder uses /tmp/nix-builds so filesystem tests run on tmpfs
+# instead of ZFS with normalization=formD and utf8only=on; those tests remain
+# enabled. The remaining workarounds cover UDP readiness, resource-sensitive
+# parser and nested-worker races, and mismatched timeout clocks, plus
+# architecture-dependent floating-point differences whose risk we accept for
+# our workloads. Keep these exceptions visible until their causes are fixed.
+_final: prev: {
+  gnutls = prev.gnutls.overrideAttrs (old: {
+    # This test uses a fixed four-second sleep instead of checking UDP
+    # readiness; the client saw no listener in the x86-64-v3 build.
+    postPatch = (old.postPatch or "") + ''
+      sed '2iexit 77' -i tests/serv-udp.sh
+    '';
+  });
+
+  prometheus = prev.prometheus.overrideAttrs (
+    old:
+    let
+      assets = old.passthru.assets.overrideAttrs (assetsOld: {
+        # CodeMirror's bounded synchronous parser can return an incomplete tree
+        # when these cases run on a heavily loaded builder.
+        postPatch = (assetsOld.postPatch or "") + ''
+          substituteInPlace module/codemirror-promql/src/complete/hybrid.test.ts \
+            --replace-fail "it(value.title, () => {" \
+            "(value.title === 'autocomplete topk params 2' ? it.skip : it)(value.title, () => {"
+          substituteInPlace module/codemirror-promql/src/parser/vector.test.ts \
+            --replace-fail "it(value.binaryExpr, () => {" \
+            "(value.binaryExpr === 'foo * on(test,blub) bar' ? it.skip : it)(value.binaryExpr, () => {"
+        '';
+      });
+    in
+    {
+      postPatch = builtins.replaceStrings [ "${old.passthru.assets}" ] [ "${assets}" ] (
+        builtins.unsafeDiscardStringContext old.postPatch
+      );
+      passthru = old.passthru // {
+        inherit assets;
+      };
+    }
+  );
+
+  pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
+    (_pythonFinal: pythonPrev: {
+      backrefs = pythonPrev.backrefs.overridePythonAttrs (old: {
+        # regex measures its timeout in process CPU time, while this test used
+        # wall time and could miss the timeout when a busy builder descheduled it.
+        postPatch = (old.postPatch or "") + ''
+          substituteInPlace tests/test_bregex.py \
+            --replace-fail "time.time()" "time.process_time()"
+        '';
+      });
+
+      pytest-xdist = pythonPrev.pytest-xdist.overridePythonAttrs (old: {
+        # The suite exercises its own worker pools. Run the outer suite with one
+        # worker and allow inner workers more time on heavily loaded builders.
+        postPatch = (old.postPatch or "") + ''
+          substituteInPlace testing/test_remote.py \
+            --replace-fail "WAIT_TIMEOUT = 10.0" "WAIT_TIMEOUT = 60.0"
+        '';
+        preCheck = builtins.replaceStrings [ "--numprocesses=$NIX_BUILD_CORES" ] [ "--numprocesses=1" ] (
+          old.preCheck or ""
+        );
+        # This test deliberately crashes workers past the restart limit and
+        # races while checking which replacement message was emitted.
+        disabledTests = (old.disabledTests or [ ]) ++ [
+          "test_max_worker_restart_tests_queued"
+        ];
+      });
+
+      jupyter-server = pythonPrev.jupyter-server.overridePythonAttrs (old: {
+        # The kernel reply arrived after the one-second outer deadline when the
+        # builder was heavily loaded. Keep the regression test but allow it the
+        # same margin as the other resource-sensitive tests.
+        postPatch = (old.postPatch or "") + ''
+          substituteInPlace tests/services/kernels/test_connection.py \
+            --replace-fail \
+            "await asyncio.wait_for(asyncio.wrap_future(conn2.request_kernel_info()), timeout=1.0)" \
+            "await asyncio.wait_for(asyncio.wrap_future(conn2.request_kernel_info()), timeout=10.0)"
+        '';
+      });
+
+      scipy = pythonPrev.scipy.overridePythonAttrs (old: {
+        # x86-64-v3 FFT implementations produce rounding differences outside
+        # these tests' strict tolerances. We accept the numerical-precision
+        # risk for our workloads.
+        disabledTests = (old.disabledTests or [ ]) ++ [
+          "test_roundtrip_float32"
+          "test_roundtrip_scaling"
+        ];
+      });
+
+      sentry-sdk = pythonPrev.sentry-sdk.overridePythonAttrs (old: {
+        # This test globally mocks threading.current_thread while another
+        # thread is running. On Python 3.14, Thread.join can race with that
+        # mock and exhaust its single side effect before the worker removes it.
+        disabledTests = (old.disabledTests or [ ]) ++ [
+          "test_get_current_thread_meta_main_thread"
+        ];
+      });
+
+      torchaudio = pythonPrev.torchaudio.overridePythonAttrs (old: {
+        # x86-64-v3 pitch shifting produces batch-versus-single-item numerical
+        # differences up to 2.9e-6. We accept that audio-precision risk for our
+        # workloads.
+        disabledTests = (old.disabledTests or [ ]) ++ [
+          "test_batch_pitch_shift"
+        ];
+      });
+
+      torchcodec = pythonPrev.torchcodec.overridePythonAttrs (old: {
+        # For these 8 kHz MP3 cases, the x86-64-v3 API and CLI codec paths
+        # differ in 0.8% of decoded samples. Retain the original tolerance for
+        # 99% of samples and accept the localized audio-precision risk.
+        postPatch = (old.postPatch or "") + ''
+          substituteInPlace test/test_encoders.py \
+            --replace-fail \
+            'if sys.platform == "darwin":' \
+            'if sys.platform == "darwin" or (
+                format == "mp3"
+                and sample_rate == 8_000
+                and asset is SINE_MONO_S32
+                and bit_rate in (None, 0)
+                and num_channels in (None, 1)
+            ):'
+        '';
+      });
+    })
+  ];
+}

overlays/x86-64-v3-workarounds.nix

@@ -0,0 +1,60 @@
+# Compatibility fixes for packages rebuilt with x86-64-v3.
+#
+# The v3 baseline enables instructions that expose source assumptions hidden
+# by the generic x86-64 build. Keep compile fixes here, separate from test
+# exclusions, until upstream or nixpkgs incorporates them.
+_final: prev:
+let
+  patchAbseilBmi2Include =
+    package:
+    package.overrideAttrs (old: {
+      # GCC and Clang prohibit including their internal BMI2 header directly.
+      # The public umbrella provides the same intrinsics with the required
+      # compiler setup.
+      postPatch = (old.postPatch or "") + ''
+        substituteInPlace third_party/abseil-cpp/absl/container/internal/raw_hash_set.h \
+          --replace-fail "#include <bmi2intrin.h>" "#include <immintrin.h>"
+      '';
+    });
+
+  removeSiblingOutputChecks =
+    package:
+    package.overrideAttrs (old: {
+      # Nix 2.34 can validate a partial multi-output rebuild against only the
+      # outputs still being realised. PostgreSQL's checks then reject valid
+      # sibling names such as "out" and "lib". Keep the test suite and
+      # disallowed-requisite checks; accept the loss of cross-output checks.
+      outputChecks = builtins.mapAttrs (
+        _output: checks: builtins.removeAttrs checks [ "disallowedReferences" ]
+      ) (old.outputChecks or { });
+    });
+
+  electron43Unwrapped = patchAbseilBmi2Include prev.electron_43.unwrapped;
+  electron43 = prev.electron_43.override {
+    electron-unwrapped = electron43Unwrapped;
+  };
+
+  signalCallPackage =
+    path: args:
+    let
+      package = prev.callPackage path args;
+    in
+    if builtins.baseNameOf path == "webrtc.nix" then patchAbseilBmi2Include package else package;
+in
+prev.lib.optionalAttrs ((prev.stdenv.hostPlatform.gcc.arch or null) == "x86-64-v3") {
+  deno =
+    let
+      librusty_v8 = patchAbseilBmi2Include prev.deno.passthru.librusty_v8;
+    in
+    prev.deno.override { inherit librusty_v8; };
+
+  electron_43 = electron43;
+
+  postgresql = removeSiblingOutputChecks prev.postgresql;
+  postgresql_18 = removeSiblingOutputChecks prev.postgresql_18;
+
+  signal-desktop = prev.signal-desktop.override {
+    electron_43 = electron43;
+    callPackage = signalCallPackage;
+  };
+}

pyproject.toml

@@ -6,51 +6,6 @@ authors = [{ name = "Richie Cahill", email = "richie@tmmworkshop.com" }]
 requires-python = "~=3.14.0"
 readme = "README.md"
 license = "MIT"
-# these dependencies are a best effort and aren't guaranteed to work
-# for up-to-date dependencies, see overlays/default.nix
-dependencies = [
-    "alembic",
-    "apprise",
-    "apscheduler",
-    "beautifulsoup4",
-    "bm25s",
-    "ebooklib",
-    "fastapi",
-    "fastapi-cli",
-    "httpx",
-    "jinja2",
-    "pgvector",
-    "polars",
-    "psycopg[binary]",
-    "pydantic",
-    "pydantic-settings",
-    "python-multipart",
-    "sqlalchemy[asyncio]",
-    "tenacity",
-    "tiktoken",
-    "tinytuya",
-    "typer",
-    "uvicorn",
-    "websockets",
-    "yake",
-]
-
-[project.scripts]
-database = "python.database_cli:app"
-whisper-transcribe = "python.tools.whisper.transcribe:main"
-
-[dependency-groups]
-dev = [
-    "aiosqlite",
-    "mypy",
-    "pyfakefs",
-    "pytest-asyncio",
-    "pytest-cov",
-    "pytest-mock",
-    "pytest-xdist",
-    "pytest",
-    "ruff",
-]
 
 [tool.ruff]
 

python/ebook_search/config.py

@@ -37,7 +37,7 @@ class RerankConfig(BaseSettings):
     model_config = SettingsConfigDict(env_prefix="EBOOK_SEARCH_RERANK_", frozen=True, protected_namespaces=())
 
     enabled: bool = True
-    base_url: str = "http://192.168.90.25:8001"
+    base_url: str = "http://bob:8001"
     model: str = "qwen3-reranker-06b"
     candidates: int = 24
     timeout_seconds: float = 30.0
@@ -67,7 +67,7 @@ class EbookSearchConfig(BaseSettings):
     )
     chat_model: str = "deepseek-v4-flash"
     answer_enabled: bool = True
-    embedding_base_url: str = "http://192.168.90.25:8000/v1"
+    embedding_base_url: str = "http://bob:8000/v1"
     embedding_api_key: str = "not-needed"
     embedding_model: str = "qwen3-embedding-0.6b"
     embedding_batch_size: int = 32

python/ebook_search/docker/docker-compose.yml

@@ -9,8 +9,6 @@ services:
     restart: unless-stopped
     ports:
       - "${EBOOK_SEARCH_PORT:-8070}:8070"
-    extra_hosts:
-      - "jeeves:192.168.90.40"
     env_file:
       - ../../../.env
     environment:

python/gitea_flake_lock.py

@@ -14,7 +14,7 @@ DEFAULT_BASE_BRANCH = "main"
 DEFAULT_BRANCH = "automation/update-flake-lock"
 DEFAULT_GITEA_URL = "https://gitea.tmmworkshop.com"
 PR_LABELS = ["dependencies", "automated", "flake_lock_update"]
-PR_CHECK_WORKFLOWS = ["build_systems.yml", "treefmt.yml", "pytest.yml"]
+PR_CHECK_WORKFLOWS = ["build_systems.yml", "treefmt.yml"]
 PR_TITLE = "Update flake.lock"
 PR_BODY = "Automated flake.lock update."
 
@@ -84,13 +84,13 @@ def push_branch(*, branch: str) -> None:
     run_cmd(["git", "push", "origin", f"HEAD:{branch}", "--force"])
 
 
-def _required_gitea_token() -> str:
+def _required_jeeves_bot_token() -> str:
     """Read the required Gitea token from the environment."""
-    token = getenv("GITEA_TOKEN")
+    token = getenv("JEEVES_BOT_TOKEN")
     if token:
         return token
 
-    msg = "GITEA_TOKEN environment variable is required"
+    msg = "JEEVES_BOT_TOKEN environment variable is required"
     raise RuntimeError(msg)
 
 
@@ -111,7 +111,7 @@ def update(
     owner, repo_name = split_repo_name(repo)
     with GiteaClient(
         base_url=getenv("GITEA_URL", DEFAULT_GITEA_URL),
-        token=_required_gitea_token(),
+        token=_required_jeeves_bot_token(),
     ) as client:
         pull_request = ensure_flake_lock_pull_request(
             client,
@@ -134,7 +134,7 @@ def merge(
     owner, repo_name = split_repo_name(repo)
     with GiteaClient(
         base_url=getenv("GITEA_URL", DEFAULT_GITEA_URL),
-        token=_required_gitea_token(),
+        token=_required_jeeves_bot_token(),
     ) as client:
         pull_request = find_flake_lock_pull_request(client, owner=owner, repo=repo_name)
         if not pull_request:

python/installer/__main__.py

@@ -176,6 +176,21 @@ def create_zfs_datasets() -> None:
     run_output(("zfs", "create", "root_pool/home"))
     run_output(("zfs", "create", "-o", "reservation=1G", "root_pool/var"))
     run_output(("zfs", "create", "-o", "compression=zstd-9", "-o", "reservation=10G", "root_pool/nix"))
+    run_output(
+        (
+            "zfs",
+            "create",
+            "-o",
+            "sync=disabled",
+            "-o",
+            "redundant_metadata=some",
+            "-o",
+            "normalization=none",
+            "-o",
+            "utf8only=off",
+            "root_pool/nix_build",
+        )
+    )
     datasets = run_output(("zfs", "list", "-o", "name"))
 
     expected_datasets = {
@@ -183,6 +198,7 @@ def create_zfs_datasets() -> None:
         "root_pool/home",
         "root_pool/var",
         "root_pool/nix",
+        "root_pool/nix_build",
     }
     missing_datasets = expected_datasets.difference(datasets.splitlines())
     if missing_datasets:
@@ -248,6 +264,7 @@ def create_nix_hardware_file(mnt_dir: str, disks: Sequence[str], encrypt: str |
         '    "/home" = {\n      device = "root_pool/home";\n      fsType = "zfs";\n    };\n\n'
         '    "/var" = {\n      device = "root_pool/var";\n      fsType = "zfs";\n    };\n\n'
         '    "/nix" = {\n      device = "root_pool/nix";\n      fsType = "zfs";\n    };\n\n'
+        '    "/nix/var/nix/builds" = {\n      device = "root_pool/nix_build";\n      fsType = "zfs";\n    };\n\n'
         '    "/boot" = {\n'
         f'      device = "/dev/disk/by-uuid/{get_boot_drive_id(disks[0])}";\n'
         '      fsType = "vfat";\n      options = [\n        "fmask=0077"\n'
@@ -270,6 +287,7 @@ def install_nixos(mnt_dir: str, disks: Sequence[str], encrypt: str | None) -> No
     run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/home", f"{mnt_dir}/home"))
     run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/var", f"{mnt_dir}/var"))
     run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/nix", f"{mnt_dir}/nix"))
+    run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/nix_build", f"{mnt_dir}/nix/var/nix/builds"))
 
     for disk in disks:
         run_output(("mkfs.vfat", "-n", "EFI", f"{disk}-part1"))

python/signal_alert.py

@@ -5,10 +5,13 @@ from __future__ import annotations
 import logging
 from os import getenv
 
-from apprise import Apprise
+import httpx
 
 logger = logging.getLogger(__name__)
 
+SIGNAL_API_URL = "http://localhost:8989/v2/send"
+SIGNAL_API_TIMEOUT = 4.0
+
 
 def signal_alert(body: str, title: str = "") -> None:
     """Send a signal alert.
@@ -17,14 +20,31 @@ def signal_alert(body: str, title: str = "") -> None:
         body (str): The body of the alert.
         title (str, optional): The title of the alert. Defaults to "".
     """
-    apprise_client = Apprise()
-
     from_phone = getenv("SIGNAL_ALERT_FROM_PHONE")
     to_phone = getenv("SIGNAL_ALERT_TO_PHONE")
     if not from_phone or not to_phone:
         logger.info("SIGNAL_ALERT_FROM_PHONE or SIGNAL_ALERT_TO_PHONE not set")
         return
 
-    apprise_client.add(f"signal://localhost:8989/{from_phone}/{to_phone}")
+    # Apprise's Signal integration did not support titles, so preserve that behavior.
+    if title:
+        logger.debug("Signal does not support notification titles; ignoring title")
 
-    apprise_client.notify(title=title, body=body)
+    try:
+        response = httpx.post(
+            SIGNAL_API_URL,
+            json={
+                "message": body,
+                "number": from_phone,
+                "recipients": [to_phone],
+                "text_mode": "normal",
+            },
+            timeout=SIGNAL_API_TIMEOUT,
+            follow_redirects=True,
+        )
+    except httpx.HTTPError:
+        logger.exception("Unable to contact the Signal API")
+        return
+
+    if response.status_code not in {httpx.codes.OK, httpx.codes.CREATED}:
+        logger.error("Signal API returned HTTP status %d", response.status_code)

python/system_tests/__init__.py

@@ -1 +0,0 @@
-"""system_tests."""

python/system_tests/components.py

@@ -1,99 +0,0 @@
-"""Validate Jeeves."""
-
-from __future__ import annotations
-
-import logging
-from copy import copy
-from re import search
-from time import sleep
-from typing import TYPE_CHECKING
-
-from python.common import bash_wrapper
-from python.zfs import Zpool
-
-if TYPE_CHECKING:
-    from collections.abc import Sequence
-
-logger = logging.getLogger(__name__)
-
-
-def zpool_tests(pool_names: Sequence[str], zpool_capacity_threshold: int = 90) -> list[str] | None:
-    """Check the zpool health and capacity.
-
-    Args:
-        pool_names (Sequence[str]): A list of pool names to test.
-        zpool_capacity_threshold (int, optional): The threshold for the zpool capacity. Defaults to 90.
-
-    Returns:
-        list[str] | None: A list of errors if any.
-    """
-    logger.info("Testing zpool")
-
-    errors: list[str] = []
-    for pool_name in pool_names:
-        pool = Zpool(pool_name)
-        if pool.health != "ONLINE":
-            errors.append(f"{pool.name} is {pool.health}")
-        if pool.capacity >= zpool_capacity_threshold:
-            errors.append(f"{pool.name} is low on space")
-
-    upgrade_status, _ = bash_wrapper("zpool upgrade")
-    if not search(r"Every feature flags pool has all supported and requested features enabled.", upgrade_status):
-        errors.append("ZPool out of date run `sudo zpool upgrade -a`")
-
-    return errors
-
-
-def systemd_tests(
-    service_names: Sequence[str],
-    max_retries: int = 30,
-    retry_delay_secs: int = 1,
-    retryable_statuses: Sequence[str] | None = None,
-    valid_statuses: Sequence[str] | None = None,
-) -> list[str] | None:
-    """Tests a systemd services.
-
-    Args:
-        service_names (Sequence[str]): A list of service names to test.
-        max_retries (int, optional): The maximum number of retries. Defaults to 30.
-            minimum value is 1.
-        retry_delay_secs (int, optional): The delay between retries in seconds. Defaults to 1.
-            minimum value is 1.
-        retryable_statuses (Sequence[str] | None, optional): A list of retryable statuses. Defaults to None.
-        valid_statuses (Sequence[str] | None, optional): A list of valid statuses. Defaults to None.
-
-    Returns:
-        list[str] | None: A list of errors if any.
-    """
-    logger.info("Testing systemd service")
-
-    max_retries = max(max_retries, 1)
-    retry_delay_secs = max(retry_delay_secs, 1)
-    last_try = max_retries - 1
-
-    if retryable_statuses is None:
-        retryable_statuses = ("inactive\n", "activating\n")
-
-    if valid_statuses is None:
-        valid_statuses = ("active\n",)
-
-    service_names_set = set(service_names)
-
-    errors: set[str] = set()
-    for retry in range(max_retries):
-        if not service_names_set:
-            break
-        logger.info(f"Testing systemd service in {retry + 1} of {max_retries}")
-        service_names_to_test = copy(service_names_set)
-        for service_name in service_names_to_test:
-            service_status, _ = bash_wrapper(f"systemctl is-active {service_name}")
-            if service_status in valid_statuses:
-                service_names_set.remove(service_name)
-                continue
-            if service_status in retryable_statuses and retry < last_try:
-                continue
-            errors.add(f"{service_name} is {service_status.strip()}")
-
-        sleep(retry_delay_secs)
-
-    return list(errors)

python/system_tests/validate_system.py

@@ -1,67 +0,0 @@
-"""Validate {server_name}."""
-
-import logging
-import sys
-import tomllib
-from os import environ
-from pathlib import Path  # noqa: TC003 This is required for the typer CLI
-from socket import gethostname
-
-import typer
-
-from python.common import configure_logger
-from python.signal_alert import signal_alert
-from python.system_tests.components import systemd_tests, zpool_tests
-
-logger = logging.getLogger(__name__)
-
-
-def load_config_data(config_file: Path) -> dict[str, list[str]]:
-    """Load a TOML configuration file.
-
-    Args:
-        config_file (Path): The path to the configuration file.
-
-    Returns:
-        dict: The configuration data.
-    """
-    return tomllib.loads(config_file.read_text())
-
-
-def main(config_file: Path) -> None:
-    """Main."""
-    configure_logger(level=environ.get("LOG_LEVEL", "INFO"))
-
-    server_name = gethostname()
-    logger.info(f"Starting {server_name} validation")
-
-    config_data = load_config_data(config_file)
-
-    errors: list[str] = []
-    try:
-        if config_data.get("zpools") and (zpool_errors := zpool_tests(config_data["zpools"])):
-            errors.extend(zpool_errors)
-
-        if config_data.get("services") and (systemd_errors := systemd_tests(config_data["services"])):
-            errors.extend(systemd_errors)
-
-    except Exception as error:
-        logger.exception(f"{server_name} validation failed")
-        errors.append(f"{server_name} validation failed: {error}")
-
-    if errors:
-        logger.error(f"{server_name} validation failed: \n{'\n'.join(errors)}")
-        signal_alert(f"{server_name} validation failed {errors}")
-
-        sys.exit(1)
-
-    logger.info(f"{server_name} validation passed")
-
-
-def cli() -> None:
-    """CLI."""
-    typer.run(main)
-
-
-if __name__ == "__main__":
-    cli()

python/van_weather/__init__.py

@@ -1 +0,0 @@
-"""Van weather service - fetches weather with masked GPS location."""

python/van_weather/main.py

@@ -1,293 +0,0 @@
-"""Van weather service - fetches weather with masked GPS for privacy."""
-
-import logging
-from datetime import UTC, datetime
-from typing import Annotated, Any
-
-import httpx
-import typer
-from apscheduler.schedulers.blocking import BlockingScheduler
-from tenacity import before_sleep_log, retry, stop_after_attempt, wait_fixed
-
-from python.common import configure_logger
-from python.van_weather.models import Config, DailyForecast, HourlyForecast, Weather
-
-# Map Pirate Weather icons to Home Assistant conditions
-CONDITION_MAP = {
-    "clear-day": "sunny",
-    "clear-night": "clear-night",
-    "rain": "rainy",
-    "snow": "snowy",
-    "sleet": "snowy-rainy",
-    "wind": "windy",
-    "fog": "fog",
-    "cloudy": "cloudy",
-    "partly-cloudy-day": "partlycloudy",
-    "partly-cloudy-night": "partlycloudy",
-}
-
-logger = logging.getLogger(__name__)
-
-
-@retry(
-    stop=stop_after_attempt(3),
-    wait=wait_fixed(5),
-    before_sleep=before_sleep_log(logger, logging.WARNING),
-    reraise=True,
-)
-def get_ha_state(url: str, token: str, entity_id: str) -> float:
-    """Get numeric state from Home Asasistant entity."""
-    response = httpx.get(
-        f"{url}/api/states/{entity_id}",
-        headers={"Authorization": f"Bearer {token}"},
-        timeout=30,
-    )
-    response.raise_for_status()
-    state = response.json()["state"]
-    if state in ("unavailable", "unknown"):
-        error = f"{entity_id} is {state}"
-        raise ValueError(error)
-    return float(state)
-
-
-def parse_daily_forecast(data: dict[str, dict[str, Any]]) -> list[DailyForecast]:
-    """Parse daily forecast from Pirate Weather API."""
-    daily = data.get("daily", {}).get("data", [])
-    daily_forecasts = []
-    for day in daily[:8]:  # Up to 8 days
-        time_stamp = day.get("time")
-        if time_stamp:
-            date_time = datetime.fromtimestamp(time_stamp, tz=UTC).isoformat()
-            daily_forecasts.append(
-                DailyForecast(
-                    date_time=date_time,
-                    condition=CONDITION_MAP.get(day.get("icon", ""), "cloudy"),
-                    temperature=day.get("temperatureHigh"),
-                    templow=day.get("temperatureLow"),
-                    precipitation_probability=day.get("precipProbability"),
-                    moon_phase=day.get("moonPhase"),
-                    wind_gust=day.get("windGust"),
-                    cloud_cover=day.get("cloudCover"),
-                )
-            )
-
-    return daily_forecasts
-
-
-def parse_hourly_forecast(data: dict[str, dict[str, Any]]) -> list[HourlyForecast]:
-    """Parse hourly forecast from Pirate Weather API."""
-    hourly = data.get("hourly", {}).get("data", [])
-    hourly_forecasts = []
-    for hour in hourly[:48]:  # Up to 48 hours
-        time_stamp = hour.get("time")
-        if time_stamp:
-            date_time = datetime.fromtimestamp(time_stamp, tz=UTC).isoformat()
-            hourly_forecasts.append(
-                HourlyForecast(
-                    date_time=date_time,
-                    condition=CONDITION_MAP.get(hour.get("icon", ""), "cloudy"),
-                    temperature=hour.get("temperature"),
-                    precipitation_probability=hour.get("precipProbability"),
-                )
-            )
-    return hourly_forecasts
-
-
-@retry(
-    stop=stop_after_attempt(3),
-    wait=wait_fixed(5),
-    before_sleep=before_sleep_log(logger, logging.WARNING),
-    reraise=True,
-)
-def fetch_weather(api_key: str, lat: float, lon: float) -> Weather:
-    """Fetch weather from Pirate Weather API."""
-    url = f"https://api.pirateweather.net/forecast/{api_key}/{lat},{lon}"
-    response = httpx.get(url, params={"units": "us"}, timeout=30)
-    response.raise_for_status()
-    data = response.json()
-
-    daily_forecasts = parse_daily_forecast(data)
-    hourly_forecasts = parse_hourly_forecast(data)
-
-    current = data.get("currently", {})
-    icon = current.get("icon", "")
-    return Weather(
-        temperature=current.get("temperature"),
-        feels_like=current.get("apparentTemperature"),
-        humidity=current.get("humidity"),
-        wind_speed=current.get("windSpeed"),
-        wind_bearing=current.get("windBearing"),
-        condition=CONDITION_MAP.get(icon, "cloudy"),
-        summary=current.get("summary"),
-        pressure=current.get("pressure"),
-        visibility=current.get("visibility"),
-        uv_index=current.get("uvIndex"),
-        ozone=current.get("ozone"),
-        nearest_storm_distance=current.get("nearestStormDistance"),
-        nearest_storm_bearing=current.get("nearestStormBearing"),
-        precip_probability=current.get("precipProbability"),
-        cloud_cover=current.get("cloudCover"),
-        daily_forecasts=daily_forecasts,
-        hourly_forecasts=hourly_forecasts,
-    )
-
-
-@retry(
-    stop=stop_after_attempt(3),
-    wait=wait_fixed(5),
-    before_sleep=before_sleep_log(logger, logging.WARNING),
-    reraise=True,
-)
-def post_to_ha(url: str, token: str, weather: Weather) -> None:
-    """Post weather data to Home Assistant as sensor entities."""
-    headers = {"Authorization": f"Bearer {token}"}
-
-    # Post current weather as individual sensors
-    sensors = {
-        "sensor.van_weather_condition": {
-            "state": weather.condition or "unknown",
-            "attributes": {"friendly_name": "Van Weather Condition"},
-        },
-        "sensor.van_weather_temperature": {
-            "state": weather.temperature,
-            "attributes": {"unit_of_measurement": "°F", "device_class": "temperature"},
-        },
-        "sensor.van_weather_apparent_temperature": {
-            "state": weather.feels_like,
-            "attributes": {"unit_of_measurement": "°F", "device_class": "temperature"},
-        },
-        "sensor.van_weather_humidity": {
-            "state": int((weather.humidity or 0) * 100),
-            "attributes": {"unit_of_measurement": "%", "device_class": "humidity"},
-        },
-        "sensor.van_weather_pressure": {
-            "state": weather.pressure,
-            "attributes": {"unit_of_measurement": "mbar", "device_class": "pressure"},
-        },
-        "sensor.van_weather_wind_speed": {
-            "state": weather.wind_speed,
-            "attributes": {"unit_of_measurement": "mph", "device_class": "wind_speed"},
-        },
-        "sensor.van_weather_wind_bearing": {
-            "state": weather.wind_bearing,
-            "attributes": {"unit_of_measurement": "°"},
-        },
-        "sensor.van_weather_visibility": {
-            "state": weather.visibility,
-            "attributes": {"unit_of_measurement": "mi"},
-        },
-        "sensor.van_weather_uv_index": {
-            "state": weather.uv_index,
-            "attributes": {"friendly_name": "Van Weather UV Index", "icon": "mdi:sun-wireless"},
-        },
-        "sensor.van_weather_ozone": {
-            "state": weather.ozone,
-            "attributes": {"unit_of_measurement": "DU", "icon": "mdi:earth"},
-        },
-        "sensor.van_weather_nearest_storm_distance": {
-            "state": weather.nearest_storm_distance,
-            "attributes": {"unit_of_measurement": "mi", "icon": "mdi:weather-lightning"},
-        },
-        "sensor.van_weather_nearest_storm_bearing": {
-            "state": weather.nearest_storm_bearing,
-            "attributes": {"unit_of_measurement": "°", "icon": "mdi:weather-lightning"},
-        },
-        "sensor.van_weather_precip_probability": {
-            "state": int((weather.precip_probability or 0) * 100),
-            "attributes": {"unit_of_measurement": "%", "icon": "mdi:weather-rainy"},
-        },
-        "sensor.van_weather_cloud_cover": {
-            "state": int((weather.cloud_cover or 0) * 100),
-            "attributes": {"unit_of_measurement": "%", "icon": "mdi:weather-cloudy"},
-        },
-    }
-
-    for entity_id, data in sensors.items():
-        if data["state"] is not None:
-            response = httpx.post(f"{url}/api/states/{entity_id}", headers=headers, json=data, timeout=30)
-            response.raise_for_status()
-
-    # Post daily forecast as JSON attribute sensor
-    daily_forecast = [
-        {
-            "datetime": daily_forecast.date_time.isoformat(),
-            "condition": daily_forecast.condition,
-            "temperature": daily_forecast.temperature,
-            "templow": daily_forecast.templow,
-            "precipitation_probability": int((daily_forecast.precipitation_probability or 0) * 100),
-        }
-        for daily_forecast in weather.daily_forecasts
-    ]
-
-    response = httpx.post(
-        f"{url}/api/states/sensor.van_weather_forecast_daily",
-        headers=headers,
-        json={"state": len(daily_forecast), "attributes": {"forecast": daily_forecast}},
-        timeout=30,
-    )
-    response.raise_for_status()
-
-    # Post hourly forecast as JSON attribute sensor
-    hourly_forecast = [
-        {
-            "datetime": hourly_forecast.date_time.isoformat(),
-            "condition": hourly_forecast.condition,
-            "temperature": hourly_forecast.temperature,
-            "precipitation_probability": int((hourly_forecast.precipitation_probability or 0) * 100),
-        }
-        for hourly_forecast in weather.hourly_forecasts
-    ]
-
-    response = httpx.post(
-        f"{url}/api/states/sensor.van_weather_forecast_hourly",
-        headers=headers,
-        json={"state": len(hourly_forecast), "attributes": {"forecast": hourly_forecast}},
-        timeout=30,
-    )
-    response.raise_for_status()
-
-
-def update_weather(config: Config) -> None:
-    """Fetch weather using last-known location, post to HA."""
-    lat = get_ha_state(config.ha_url, config.ha_token, config.lat_entity)
-    lon = get_ha_state(config.ha_url, config.ha_token, config.lon_entity)
-
-    masked_lat = round(lat, config.mask_decimals)
-    masked_lon = round(lon, config.mask_decimals)
-
-    logger.info(f"Masked location: {masked_lat}, {masked_lon}")
-
-    weather = fetch_weather(config.pirate_weather_api_key, masked_lat, masked_lon)
-    logger.info(f"Weather: {weather.temperature}°F, {weather.condition}")
-
-    post_to_ha(config.ha_url, config.ha_token, weather)
-    logger.info("Posted weather to HA")
-
-
-def main(
-    ha_url: Annotated[str, typer.Option(envvar="HA_URL")],
-    ha_token: Annotated[str, typer.Option(envvar="HA_TOKEN")],
-    api_key: Annotated[str, typer.Option(envvar="PIRATE_WEATHER_API_KEY")],
-    interval: Annotated[int, typer.Option(help="Poll interval in seconds")] = 900,
-    log_level: Annotated[str, typer.Option()] = "INFO",
-) -> None:
-    """Fetch weather for van using masked GPS location."""
-    configure_logger(log_level)
-
-    config = Config(ha_url=ha_url, ha_token=ha_token, pirate_weather_api_key=api_key)
-
-    logger.info(f"Starting van weather service, polling every {interval}s")
-
-    scheduler = BlockingScheduler()
-    scheduler.add_job(
-        update_weather,
-        "interval",
-        seconds=interval,
-        args=[config],
-        next_run_time=datetime.now(UTC),
-    )
-    scheduler.start()
-
-
-if __name__ == "__main__":
-    typer.run(main)

python/van_weather/models.py

@@ -1,72 +0,0 @@
-"""Models for van weather service."""
-
-from __future__ import annotations
-
-from datetime import datetime  # noqa: TC003 This is required for pydantic
-
-from pydantic import BaseModel, field_serializer
-
-
-class Config(BaseModel):
-    """Service configuration."""
-
-    ha_url: str
-    ha_token: str
-    pirate_weather_api_key: str
-    lat_entity: str = "sensor.van_last_known_latitude"
-    lon_entity: str = "sensor.van_last_known_longitude"
-    mask_decimals: int = 1  # ~11km accuracy
-
-
-class DailyForecast(BaseModel):
-    """Daily forecast entry."""
-
-    date_time: datetime
-    condition: str | None = None
-    temperature: float | None = None  # High
-    templow: float | None = None  # Low
-    precipitation_probability: float | None = None
-    moon_phase: float | None = None
-    wind_gust: float | None = None
-    cloud_cover: float | None = None
-
-    @field_serializer("date_time")
-    def serialize_date_time(self, date_time: datetime) -> str:
-        """Serialize datetime to ISO format."""
-        return date_time.isoformat()
-
-
-class HourlyForecast(BaseModel):
-    """Hourly forecast entry."""
-
-    date_time: datetime
-    condition: str | None = None
-    temperature: float | None = None
-    precipitation_probability: float | None = None
-
-    @field_serializer("date_time")
-    def serialize_date_time(self, date_time: datetime) -> str:
-        """Serialize datetime to ISO format."""
-        return date_time.isoformat()
-
-
-class Weather(BaseModel):
-    """Weather data from Pirate Weather."""
-
-    temperature: float | None = None
-    feels_like: float | None = None
-    humidity: float | None = None
-    wind_speed: float | None = None
-    wind_bearing: float | None = None
-    condition: str | None = None
-    summary: str | None = None
-    pressure: float | None = None
-    visibility: float | None = None
-    uv_index: float | None = None
-    ozone: float | None = None
-    nearest_storm_distance: float | None = None
-    nearest_storm_bearing: float | None = None
-    precip_probability: float | None = None
-    cloud_cover: float | None = None
-    daily_forecasts: list[DailyForecast] = []
-    hourly_forecasts: list[HourlyForecast] = []

rust/van_weather/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "van-weather"
+version = "0.1.0"
+edition = "2024"
+description = "Fetch privacy-masked weather for a van and publish it to Home Assistant"
+license = "MIT"
+
+[dependencies]
+anyhow = "1.0"
+chrono = "0.4"
+clap = { version = "4.5", features = ["derive", "env"] }
+env_logger = "0.11"
+log = "0.4"
+reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+
+[lints.clippy]
+all = "deny"
+pedantic = "deny"
+

rust/van_weather/package.nix

@@ -0,0 +1,13 @@
+{ rustPlatform }:
+rustPlatform.buildRustPackage {
+  pname = "van-weather";
+  version = "0.1.0";
+  src = ./.;
+
+  cargoLock.lockFile = ./Cargo.lock;
+
+  meta = {
+    description = "Privacy-masked van weather publisher for Home Assistant";
+    mainProgram = "van-weather";
+  };
+}

rust/van_weather/src/main.rs

@@ -0,0 +1,565 @@
+use std::{thread, time::Duration};
+
+use anyhow::{Context, Result, bail};
+use chrono::{DateTime, Utc};
+use clap::Parser;
+use log::{error, info};
+use reqwest::{
+    StatusCode, Url,
+    blocking::{Client, ClientBuilder},
+    header::{AUTHORIZATION, HeaderMap, HeaderValue},
+    retry,
+};
+use serde::Deserialize;
+use serde_json::{Value, json};
+
+const LAT_ENTITY: &str = "sensor.van_last_known_latitude";
+const LON_ENTITY: &str = "sensor.van_last_known_longitude";
+const PIRATE_WEATHER_HOST: &str = "api.pirateweather.net";
+const MASK_DECIMALS: u32 = 1;
+const MASK_FACTOR: f64 = decimal_factor(MASK_DECIMALS);
+const RETRIES_PER_REQUEST: u32 = 2;
+
+const fn decimal_factor(decimals: u32) -> f64 {
+    let mut factor = 1.0;
+    let mut remaining = decimals;
+    while remaining > 0 {
+        factor *= 10.0;
+        remaining -= 1;
+    }
+    factor
+}
+
+#[derive(Debug, Parser)]
+#[command(about, version)]
+struct Args {
+    #[arg(long, env = "HA_URL")]
+    ha_url: String,
+
+    #[arg(long, env = "HA_TOKEN", hide_env_values = true)]
+    ha_token: String,
+
+    #[arg(long, env = "PIRATE_WEATHER_API_KEY", hide_env_values = true)]
+    pirate_weather_api_key: String,
+
+    #[arg(
+        long,
+        default_value_t = 900,
+        value_parser = clap::value_parser!(u64).range(1..)
+    )]
+    interval: u64,
+
+    #[arg(long, default_value = "info", env = "RUST_LOG")]
+    log_level: String,
+}
+
+#[derive(Debug, Deserialize)]
+struct HaState {
+    state: String,
+}
+
+#[derive(Debug)]
+struct HttpClients {
+    home_assistant: Client,
+    pirate_weather: Client,
+}
+
+#[derive(Debug, Default, Deserialize)]
+struct ApiResponse {
+    #[serde(default)]
+    currently: CurrentWeather,
+    #[serde(default)]
+    daily: ForecastBlock<DailyApiForecast>,
+    #[serde(default)]
+    hourly: ForecastBlock<HourlyApiForecast>,
+}
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct CurrentWeather {
+    temperature: Option<f64>,
+    apparent_temperature: Option<f64>,
+    humidity: Option<f64>,
+    wind_speed: Option<f64>,
+    wind_bearing: Option<f64>,
+    icon: Option<String>,
+    pressure: Option<f64>,
+    visibility: Option<f64>,
+    uv_index: Option<f64>,
+    ozone: Option<f64>,
+    nearest_storm_distance: Option<f64>,
+    nearest_storm_bearing: Option<f64>,
+    precip_probability: Option<f64>,
+    cloud_cover: Option<f64>,
+}
+
+#[derive(Debug, Deserialize)]
+struct ForecastBlock<T> {
+    #[serde(default)]
+    data: Vec<T>,
+}
+
+impl<T> Default for ForecastBlock<T> {
+    fn default() -> Self {
+        Self { data: Vec::new() }
+    }
+}
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct DailyApiForecast {
+    time: Option<i64>,
+    icon: Option<String>,
+    temperature_high: Option<f64>,
+    temperature_low: Option<f64>,
+    precip_probability: Option<f64>,
+}
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct HourlyApiForecast {
+    time: Option<i64>,
+    icon: Option<String>,
+    temperature: Option<f64>,
+    precip_probability: Option<f64>,
+}
+
+#[derive(Debug)]
+struct Weather {
+    current: CurrentWeather,
+    daily: Vec<DailyForecast>,
+    hourly: Vec<HourlyForecast>,
+}
+
+#[derive(Debug)]
+struct DailyForecast {
+    datetime: DateTime<Utc>,
+    condition: &'static str,
+    temperature: Option<f64>,
+    templow: Option<f64>,
+    precipitation_probability: Option<f64>,
+}
+
+#[derive(Debug)]
+struct HourlyForecast {
+    datetime: DateTime<Utc>,
+    condition: &'static str,
+    temperature: Option<f64>,
+    precipitation_probability: Option<f64>,
+}
+
+fn main() -> Result<()> {
+    let Args {
+        ha_url,
+        ha_token,
+        pirate_weather_api_key,
+        interval,
+        log_level,
+    } = Args::parse();
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or(log_level.as_str()))
+        .init();
+
+    let ha_url = ha_url.trim_end_matches('/').to_owned();
+    let ha_host = Url::parse(&ha_url)
+        .context("HA_URL is not a valid URL")?
+        .host_str()
+        .context("HA_URL has no host")?
+        .to_owned();
+    let clients = HttpClients {
+        home_assistant: build_client(&ha_host, Some(&ha_token))?,
+        pirate_weather: build_client(PIRATE_WEATHER_HOST, None)?,
+    };
+
+    info!("Starting van weather service, polling every {interval}s");
+    loop {
+        if let Err(err) = update_weather(&clients, &ha_url, &pirate_weather_api_key) {
+            error!("Weather update failed: {err:#}");
+        }
+        thread::sleep(Duration::from_secs(interval));
+    }
+}
+
+fn build_client(host: &str, bearer_token: Option<&str>) -> Result<Client> {
+    let policy = retry::for_host(host.to_owned())
+        .max_retries_per_request(RETRIES_PER_REQUEST)
+        .classify_fn(|request| {
+            let retryable = request.error().is_some()
+                || request.status().is_some_and(|status| {
+                    status == StatusCode::REQUEST_TIMEOUT
+                        || status == StatusCode::TOO_MANY_REQUESTS
+                        || status.is_server_error()
+                });
+            if retryable {
+                request.retryable()
+            } else {
+                request.success()
+            }
+        });
+
+    let mut builder = ClientBuilder::new()
+        .timeout(Duration::from_secs(30))
+        .retry(policy);
+
+    if let Some(token) = bearer_token {
+        let mut authorization = HeaderValue::from_str(&format!("Bearer {token}"))
+            .context("HA_TOKEN contains invalid header characters")?;
+        authorization.set_sensitive(true);
+        let mut headers = HeaderMap::new();
+        headers.insert(AUTHORIZATION, authorization);
+        builder = builder.default_headers(headers);
+    }
+
+    builder
+        .build()
+        .with_context(|| format!("failed to create HTTP client for {host}"))
+}
+
+fn update_weather(clients: &HttpClients, ha_url: &str, api_key: &str) -> Result<()> {
+    let lat = get_ha_state(&clients.home_assistant, ha_url, LAT_ENTITY)?;
+    let lon = get_ha_state(&clients.home_assistant, ha_url, LON_ENTITY)?;
+    let masked_lat = mask_coordinate(lat);
+    let masked_lon = mask_coordinate(lon);
+    info!("Masked location: {masked_lat}, {masked_lon}");
+
+    let weather = fetch_weather(&clients.pirate_weather, api_key, masked_lat, masked_lon)?;
+    info!(
+        "Weather: {}°F, {}",
+        weather
+            .current
+            .temperature
+            .map_or_else(|| "unknown".to_owned(), |value| value.to_string()),
+        condition(weather.current.icon.as_deref())
+    );
+    post_to_ha(&clients.home_assistant, ha_url, &weather)?;
+    info!("Posted weather to Home Assistant");
+    Ok(())
+}
+
+fn mask_coordinate(value: f64) -> f64 {
+    (value * MASK_FACTOR).round() / MASK_FACTOR
+}
+
+fn get_ha_state(client: &Client, ha_url: &str, entity_id: &str) -> Result<f64> {
+    let HaState { state } = client
+        .get(format!("{ha_url}/api/states/{entity_id}"))
+        .send()
+        .with_context(|| format!("request for {entity_id} failed"))?
+        .error_for_status()
+        .with_context(|| format!("Home Assistant rejected {entity_id} request"))?
+        .json()
+        .context("Home Assistant returned invalid JSON")?;
+
+    if matches!(state.as_str(), "unavailable" | "unknown") {
+        bail!("{entity_id} is {state}");
+    }
+
+    state
+        .parse::<f64>()
+        .with_context(|| format!("{entity_id} state is not numeric: {state}"))
+}
+
+fn fetch_weather(client: &Client, api_key: &str, lat: f64, lon: f64) -> Result<Weather> {
+    let response = client
+        .get(format!(
+            "https://{PIRATE_WEATHER_HOST}/forecast/{api_key}/{lat},{lon}"
+        ))
+        .query(&[("units", "us")])
+        .send()
+        .context("Pirate Weather request failed")?
+        .error_for_status()
+        .context("Pirate Weather rejected request")?;
+    let data = response
+        .json::<ApiResponse>()
+        .context("Pirate Weather returned invalid JSON")?;
+    Ok(parse_weather(data))
+}
+
+fn parse_weather(data: ApiResponse) -> Weather {
+    let daily = data
+        .daily
+        .data
+        .into_iter()
+        .take(8)
+        .filter_map(|day| {
+            timestamp(day.time).map(|datetime| DailyForecast {
+                datetime,
+                condition: condition(day.icon.as_deref()),
+                temperature: day.temperature_high,
+                templow: day.temperature_low,
+                precipitation_probability: day.precip_probability,
+            })
+        })
+        .collect();
+    let hourly = data
+        .hourly
+        .data
+        .into_iter()
+        .take(48)
+        .filter_map(|hour| {
+            timestamp(hour.time).map(|datetime| HourlyForecast {
+                datetime,
+                condition: condition(hour.icon.as_deref()),
+                temperature: hour.temperature,
+                precipitation_probability: hour.precip_probability,
+            })
+        })
+        .collect();
+    Weather {
+        current: data.currently,
+        daily,
+        hourly,
+    }
+}
+
+fn timestamp(value: Option<i64>) -> Option<DateTime<Utc>> {
+    value
+        .filter(|value| *value != 0)
+        .and_then(DateTime::from_timestamp_secs)
+}
+
+fn condition(icon: Option<&str>) -> &'static str {
+    match icon.unwrap_or_default() {
+        "clear-day" => "sunny",
+        "clear-night" => "clear-night",
+        "rain" => "rainy",
+        "snow" => "snowy",
+        "sleet" => "snowy-rainy",
+        "wind" => "windy",
+        "fog" => "fog",
+        "partly-cloudy-day" | "partly-cloudy-night" => "partlycloudy",
+        _ => "cloudy",
+    }
+}
+
+fn post_to_ha(client: &Client, ha_url: &str, weather: &Weather) -> Result<()> {
+    for (entity_id, payload) in weather_updates(weather) {
+        let response = client
+            .post(format!("{ha_url}/api/states/{entity_id}"))
+            .json(&payload)
+            .send()
+            .with_context(|| format!("failed to post {entity_id}"))?;
+        ensure_success(response.status(), &entity_id)?;
+    }
+    Ok(())
+}
+
+fn ensure_success(status: StatusCode, entity_id: &str) -> Result<()> {
+    if status.is_success() {
+        Ok(())
+    } else {
+        bail!("Home Assistant rejected {entity_id} update with {status}")
+    }
+}
+
+#[allow(clippy::too_many_lines)]
+fn weather_updates(weather: &Weather) -> Vec<(String, Value)> {
+    let current = &weather.current;
+    let mut updates = vec![
+        sensor(
+            "sensor.van_weather_condition",
+            Some(json!(condition(current.icon.as_deref()))),
+            json!({"friendly_name": "Van Weather Condition"}),
+        ),
+        sensor(
+            "sensor.van_weather_temperature",
+            current.temperature.map(|value| json!(value)),
+            json!({"unit_of_measurement": "°F", "device_class": "temperature"}),
+        ),
+        sensor(
+            "sensor.van_weather_apparent_temperature",
+            current.apparent_temperature.map(|value| json!(value)),
+            json!({"unit_of_measurement": "°F", "device_class": "temperature"}),
+        ),
+        sensor(
+            "sensor.van_weather_humidity",
+            Some(json!(percent(current.humidity))),
+            json!({"unit_of_measurement": "%", "device_class": "humidity"}),
+        ),
+        sensor(
+            "sensor.van_weather_pressure",
+            current.pressure.map(|value| json!(value)),
+            json!({"unit_of_measurement": "mbar", "device_class": "pressure"}),
+        ),
+        sensor(
+            "sensor.van_weather_wind_speed",
+            current.wind_speed.map(|value| json!(value)),
+            json!({"unit_of_measurement": "mph", "device_class": "wind_speed"}),
+        ),
+        sensor(
+            "sensor.van_weather_wind_bearing",
+            current.wind_bearing.map(|value| json!(value)),
+            json!({"unit_of_measurement": "°"}),
+        ),
+        sensor(
+            "sensor.van_weather_visibility",
+            current.visibility.map(|value| json!(value)),
+            json!({"unit_of_measurement": "mi"}),
+        ),
+        sensor(
+            "sensor.van_weather_uv_index",
+            current.uv_index.map(|value| json!(value)),
+            json!({"friendly_name": "Van Weather UV Index", "icon": "mdi:sun-wireless"}),
+        ),
+        sensor(
+            "sensor.van_weather_ozone",
+            current.ozone.map(|value| json!(value)),
+            json!({"unit_of_measurement": "DU", "icon": "mdi:earth"}),
+        ),
+        sensor(
+            "sensor.van_weather_nearest_storm_distance",
+            current.nearest_storm_distance.map(|value| json!(value)),
+            json!({"unit_of_measurement": "mi", "icon": "mdi:weather-lightning"}),
+        ),
+        sensor(
+            "sensor.van_weather_nearest_storm_bearing",
+            current.nearest_storm_bearing.map(|value| json!(value)),
+            json!({"unit_of_measurement": "°", "icon": "mdi:weather-lightning"}),
+        ),
+        sensor(
+            "sensor.van_weather_precip_probability",
+            Some(json!(percent(current.precip_probability))),
+            json!({"unit_of_measurement": "%", "icon": "mdi:weather-rainy"}),
+        ),
+        sensor(
+            "sensor.van_weather_cloud_cover",
+            Some(json!(percent(current.cloud_cover))),
+            json!({"unit_of_measurement": "%", "icon": "mdi:weather-cloudy"}),
+        ),
+    ]
+    .into_iter()
+    .flatten()
+    .collect::<Vec<_>>();
+
+    let daily = weather
+        .daily
+        .iter()
+        .map(|forecast| {
+            json!({
+                "datetime": forecast.datetime.to_rfc3339(),
+                "condition": forecast.condition,
+                "temperature": forecast.temperature,
+                "templow": forecast.templow,
+                "precipitation_probability": percent(forecast.precipitation_probability),
+            })
+        })
+        .collect::<Vec<_>>();
+    updates.push((
+        "sensor.van_weather_forecast_daily".to_owned(),
+        json!({"state": daily.len(), "attributes": {"forecast": daily}}),
+    ));
+
+    let hourly = weather
+        .hourly
+        .iter()
+        .map(|forecast| {
+            json!({
+                "datetime": forecast.datetime.to_rfc3339(),
+                "condition": forecast.condition,
+                "temperature": forecast.temperature,
+                "precipitation_probability": percent(forecast.precipitation_probability),
+            })
+        })
+        .collect::<Vec<_>>();
+    updates.push((
+        "sensor.van_weather_forecast_hourly".to_owned(),
+        json!({"state": hourly.len(), "attributes": {"forecast": hourly}}),
+    ));
+    updates
+}
+
+fn sensor(entity_id: &str, state: Option<Value>, attributes: Value) -> Option<(String, Value)> {
+    state.map(|state| {
+        let mut payload = serde_json::Map::new();
+        payload.insert("state".to_owned(), state);
+        payload.insert("attributes".to_owned(), attributes);
+        (entity_id.to_owned(), Value::Object(payload))
+    })
+}
+
+#[allow(clippy::cast_possible_truncation)]
+fn percent(value: Option<f64>) -> i64 {
+    // Preserve Python's int(probability * 100) behavior for Home Assistant.
+    (value.unwrap_or_default() * 100.0) as i64
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn maps_conditions_for_home_assistant() {
+        assert_eq!(condition(Some("clear-day")), "sunny");
+        assert_eq!(condition(Some("sleet")), "snowy-rainy");
+        assert_eq!(condition(Some("partly-cloudy-night")), "partlycloudy");
+        assert_eq!(condition(Some("unexpected")), "cloudy");
+        assert_eq!(condition(None), "cloudy");
+    }
+
+    #[test]
+    fn parses_api_response_and_builds_compatible_payloads() {
+        let response: ApiResponse = serde_json::from_value(json!({
+            "currently": {
+                "temperature": 72.5,
+                "humidity": 0.67,
+                "icon": "clear-day",
+                "precipProbability": 0.129,
+                "cloudCover": 0.4,
+                "summary": "Fine"
+            },
+            "daily": {"data": [{
+                "time": 1_700_000_000,
+                "icon": "rain",
+                "temperatureHigh": 75.0,
+                "temperatureLow": 52.0,
+                "precipProbability": 0.8
+            }]},
+            "hourly": {"data": [{
+                "time": 1_700_000_000,
+                "icon": "fog",
+                "temperature": 61.0,
+                "precipProbability": 0.05
+            }]}
+        }))
+        .unwrap();
+
+        let weather = parse_weather(response);
+        let updates = weather_updates(&weather);
+        let find = |id: &str| updates.iter().find(|(entity, _)| entity == id).unwrap();
+
+        assert_eq!(find("sensor.van_weather_condition").1["state"], "sunny");
+        assert_eq!(find("sensor.van_weather_humidity").1["state"], 67);
+        assert_eq!(find("sensor.van_weather_precip_probability").1["state"], 12);
+        assert_eq!(find("sensor.van_weather_forecast_daily").1["state"], 1);
+        assert_eq!(
+            find("sensor.van_weather_forecast_daily").1["attributes"]["forecast"][0]["condition"],
+            "rainy"
+        );
+        assert_eq!(find("sensor.van_weather_forecast_hourly").1["state"], 1);
+    }
+
+    #[test]
+    fn omits_missing_optional_current_sensors_but_keeps_percentage_sensors() {
+        let weather = parse_weather(ApiResponse::default());
+        let updates = weather_updates(&weather);
+
+        assert!(
+            !updates
+                .iter()
+                .any(|(id, _)| id == "sensor.van_weather_temperature")
+        );
+        assert_eq!(
+            updates
+                .iter()
+                .find(|(id, _)| id == "sensor.van_weather_humidity")
+                .unwrap()
+                .1["state"],
+            0
+        );
+    }
+
+    #[test]
+    fn masks_coordinates_to_about_eleven_kilometres() {
+        assert!((mask_coordinate(37.7749) - 37.8).abs() < f64::EPSILON);
+        assert!((mask_coordinate(-122.4194) - (-122.4)).abs() < f64::EPSILON);
+    }
+}

shell.nix

@@ -8,7 +8,7 @@
   # loader path, which NixOS does not provide globally.
   ebook-search = pkgs.mkShell {
     nativeBuildInputs = with pkgs; [
-      my_python
+      python314
       uv
     ];
     LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
@@ -23,7 +23,6 @@
       nix
       home-manager
       git
-      my_python
 
       ssh-to-age
       gnupg

systems/bob/default.nix

@@ -3,18 +3,18 @@
   imports = [
     "${inputs.self}/users/math"
     "${inputs.self}/users/richie"
-    "${inputs.self}/users/steve"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/docker.nix"
-    "${inputs.self}/common/optional/scanner.nix"
     "${inputs.self}/common/optional/monitoring-agent.nix"
+    "${inputs.self}/common/optional/nvidia.nix"
+    "${inputs.self}/common/optional/scanner.nix"
     "${inputs.self}/common/optional/steam.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/systemd-boot.nix"
+    "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/yubikey.nix"
-    "${inputs.self}/common/optional/zerotier.nix"
-    "${inputs.self}/common/optional/nvidia.nix"
+    "${inputs.self}/common/optional/x86-64-v3.nix"
+    "${inputs.self}/common/optional/zfs"
     ./hardware.nix
     ./syncthing.nix
     ./llms.nix

systems/bob/hardware.nix

@@ -50,6 +50,11 @@
       fsType = "zfs";
     };
 
+    "/nix/var/nix/builds" = {
+      device = "root_pool/nix_build";
+      fsType = "zfs";
+    };
+
     "/var" = {
       device = "root_pool/var";
       fsType = "zfs";

systems/bob/snapshot_config.toml

@@ -22,6 +22,12 @@ hourly = 0
 daily = 0
 monthly = 0
 
+["root_pool/nix_build"]
+15_min = 1
+hourly = 0
+daily = 0
+monthly = 0
+
 ["root_pool/models"]
 15_min = 4
 hourly = 24

systems/brain/default.nix

@@ -7,8 +7,10 @@
     "${inputs.self}/common/optional/ssh_decrypt.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/systemd-boot.nix"
+    "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/zerotier.nix"
+    "${inputs.self}/common/optional/x86-64-v3.nix"
+    "${inputs.self}/common/optional/zfs"
     ./docker
     ./hardware.nix
     ./programs.nix

systems/brain/hardware.nix

@@ -53,6 +53,11 @@
       fsType = "zfs";
     };
 
+    "/nix/var/nix/builds" = {
+      device = "root_pool/nix_build";
+      fsType = "zfs";
+    };
+
     "/boot" = {
       device = "/dev/disk/by-uuid/12CE-A600";
       fsType = "vfat";

systems/brain/services/heater.nix

@@ -4,6 +4,20 @@
   ...
 }:
 {
+  nixpkgs.overlays = [
+    (final: _prev: {
+      heater_python = final.python314.withPackages (
+        ps: with ps; [
+          fastapi
+          pydantic
+          tinytuya
+          typer
+          uvicorn
+        ]
+      );
+    })
+  ];
+
   networking.firewall.allowedTCPPorts = [ 8124 ];
 
   systemd.services.heater-api = {
@@ -17,7 +31,7 @@
 
     serviceConfig = {
       Type = "simple";
-      ExecStart = "${pkgs.my_python}/bin/python -m python.heater.main --host 0.0.0.0 --port 8124";
+      ExecStart = "${pkgs.heater_python}/bin/python -m python.heater.main --host 0.0.0.0 --port 8124";
       EnvironmentFile = "/etc/heater.env";
       Restart = "on-failure";
       RestartSec = "5s";

systems/brain/services/home_assistant.nix

@@ -1,5 +1,7 @@
 { pkgs, ... }:
 {
+  networking.firewall.allowedTCPPorts = [ 8123 ];
+
   users = {
     users.hass = {
       isSystemUser = true;
@@ -11,9 +13,7 @@
   services = {
     home-assistant = {
       enable = true;
-      openFirewall = true;
       config = {
-        http.server_port = 8123;
         homeassistant = {
           time_zone = "America/New_York";
           unit_system = "us_customary";
@@ -21,6 +21,7 @@
           packages = {
             victron_modbuss = "!include ${./home_assistant/victron_modbuss.yaml}";
             battery_sensors = "!include ${./home_assistant/battery_sensors.yaml}";
+            energy_meters = "!include ${./home_assistant/energy_meters.yaml}";
             gps_location = "!include ${./home_assistant/gps_location.yaml}";
             heater = "!include ${./home_assistant/heater.yaml}";
             van_weather = "!include ${./home_assistant/van_weather_template.yaml}";
@@ -67,24 +68,26 @@
           paho-mqtt # for mqtt
           psycopg2 # for postgresql
           py-improv-ble-client # for esphome
-          pymodbus # for modbus
           pyopenweathermap # for weather
           pymetno # for met.no weather
           uiprotect # for ubiquiti integration
           unifi-discovery # for ubiquiti integration
           jsonpath # for rest sensors
-          typedmonarchmoney # for monarch
+          monarchmoneycommunity # for monarch
         ];
-      extraComponents = [ "isal" ];
+      extraComponents = [
+        "isal"
+        "modbus" # for victron modbus integration
+      ];
       customComponents = with pkgs.home-assistant-custom-components; [
-        pirate-weather
+        garmin_connect
       ];
 
     };
     esphome = {
       enable = true;
       openFirewall = true;
-      address = "192.168.90.35";
+      address = "0.0.0.0";
     };
   };
 }

systems/brain/services/home_assistant/energy_meters.yaml

@@ -0,0 +1,71 @@
+template:
+  - sensor:
+      - name: "JK0 Energy In Clean"
+        unique_id: jk0_energy_in_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.jk0_energy_in')) }}
+        state: >-
+          {{ states('sensor.jk0_energy_in') | float }}
+
+      - name: "JK0 Energy Out Clean"
+        unique_id: jk0_energy_out_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.jk0_energy_out')) }}
+        state: >-
+          {{ states('sensor.jk0_energy_out') | float }}
+
+      - name: "JK1 Energy In Clean"
+        unique_id: jk1_energy_in_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.jk1_energy_in')) }}
+        state: >-
+          {{ states('sensor.jk1_energy_in') | float }}
+
+      - name: "JK1 Energy Out Clean"
+        unique_id: jk1_energy_out_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.jk1_energy_out')) }}
+        state: >-
+          {{ states('sensor.jk1_energy_out') | float }}
+
+      - name: "Solar Yield Clean"
+        unique_id: solar_yield_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.solar_yield_daily')) }}
+        state: >-
+          {{ states('sensor.solar_yield_daily') | float }}
+
+      - name: "DC Load Energy Clean"
+        unique_id: dc_load_energy_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.dc_load_energy_daily')) }}
+        state: >-
+          {{ states('sensor.dc_load_energy_daily') | float }}
+
+      - name: "DC Charger Energy Clean"
+        unique_id: dc_charger_energy_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.dc_charger_energy')) }}
+        state: >-
+          {{ states('sensor.dc_charger_energy') | float }}

systems/brain/services/postgress.nix

@@ -19,7 +19,7 @@
       local all  richie   trust
       host  all  richie   127.0.0.1/32    trust
       host  all  richie   ::1/128         trust
-      host  all  richie   192.168.90.1/24 trust
+      host  all  richie   100.64.0.0/10 trust
       host  all  richie   192.168.99.1/24 trust
 
       local vaninventory vaninventory trust
@@ -28,7 +28,7 @@
       local hass     hass     trust
 
       # ipv4
-      host  hass     hass     192.168.90.1/24 trust
+      host  hass     hass     100.64.0.0/10 trust
       host  hass     hass     127.0.0.1/32 trust
 
       # ipv6

systems/brain/services/van_weather.nix

@@ -1,8 +1,10 @@
 {
   pkgs,
-  inputs,
   ...
 }:
+let
+  van-weather = pkgs.callPackage ../../../rust/van_weather/package.nix { };
+in
 {
   systemd.services.van-weather = {
     description = "Van Weather Service";
@@ -13,13 +15,9 @@
     requires = [ "home-assistant.service" ];
     wantedBy = [ "multi-user.target" ];
 
-    environment = {
-      PYTHONPATH = "${inputs.self}/";
-    };
-
     serviceConfig = {
       Type = "simple";
-      ExecStart = "${pkgs.my_python}/bin/python -m python.van_weather.main";
+      ExecStart = "${van-weather}/bin/van-weather";
       EnvironmentFile = "/etc/van_weather.env";
       Restart = "on-failure";
       RestartSec = "5s";
@@ -29,7 +27,6 @@
       ProtectSystem = "strict";
       ProtectHome = "read-only";
       PrivateTmp = true;
-      ReadOnlyPaths = [ "${inputs.self}" ];
     };
   };
 }

systems/brain/syncthing.nix

@@ -3,7 +3,7 @@
 
   services.syncthing = {
     overrideFolders = false;
-    guiAddress = "192.168.90.35:8384";
+    guiAddress = "0.0.0.0:8384";
     settings = {
       "dotfiles" = {
         path = "/home/richie/dotfiles";

systems/jeeves/containers.nix

@@ -0,0 +1,71 @@
+{ pkgs, ... }:
+let
+  vars = import ./vars.nix;
+in
+{
+  virtualisation.docker.daemon.settings."data-root" = "${vars.containers}/docker";
+
+  # nixos-container hardcodes its state directory to /var/lib/nixos-containers,
+  # so route it to the shared container dataset with a bind mount.
+  fileSystems."/var/lib/nixos-containers" = {
+    device = "${vars.containers}/nixos-containers";
+    fsType = "none";
+    options = [ "bind" ];
+  };
+
+  systemd.services = {
+    prepare-docker-storage = {
+      description = "Create Docker storage directory";
+      requiredBy = [ "docker.service" ];
+      before = [ "docker.service" ];
+      requires = [ "zfs-mount.service" ];
+      after = [ "zfs-mount.service" ];
+      path = [
+        pkgs.coreutils
+        pkgs.util-linux
+      ];
+
+      unitConfig = {
+        DefaultDependencies = false;
+        RequiresMountsFor = [ "/nix" ];
+      };
+
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+
+      script = ''
+        mountpoint -q ${vars.containers}
+        install -d -m 0710 ${vars.containers}/docker
+      '';
+    };
+
+    prepare-nixos-container-storage = {
+      description = "Create NixOS container storage directory";
+      requiredBy = [ "var-lib-nixos\\x2dcontainers.mount" ];
+      before = [ "var-lib-nixos\\x2dcontainers.mount" ];
+      requires = [ "zfs-mount.service" ];
+      after = [ "zfs-mount.service" ];
+      path = [
+        pkgs.coreutils
+        pkgs.util-linux
+      ];
+
+      unitConfig = {
+        DefaultDependencies = false;
+        RequiresMountsFor = [ "/nix" ];
+      };
+
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+
+      script = ''
+        mountpoint -q ${vars.containers}
+        install -d -m 0755 ${vars.containers}/nixos-containers
+      '';
+    };
+  };
+}

systems/jeeves/default.nix

@@ -4,23 +4,23 @@ let
 in
 {
   imports = [
-    "${inputs.self}/users/dov"
     "${inputs.self}/users/math"
     "${inputs.self}/users/richie"
-    "${inputs.self}/users/steve"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/docker.nix"
     "${inputs.self}/common/optional/monitoring-agent.nix"
     "${inputs.self}/common/optional/ssh_decrypt.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
+    "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/zerotier.nix"
+    "${inputs.self}/common/optional/x86-64-v3.nix"
+    "${inputs.self}/common/optional/zfs"
     ./monitoring
     ./docker
     ./services
-    ./web_services
+    ./containers.nix
     ./hardware.nix
-    ./networking.nix
+    ./networking
     ./programs.nix
     ./runners
     ./syncthing.nix
@@ -35,8 +35,6 @@ in
       path = ./snapshot_config.toml;
       EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
     };
-
-    zerotierone.joinNetworks = [ "a09acf02330d37b9" ];
   };
 
   users.groups = {

systems/jeeves/docker/signal-cli-rest-api.nix

@@ -2,9 +2,6 @@ let
   vars = import ../vars.nix;
 in
 {
-  networking.firewall.allowedTCPPorts = [
-    8989
-  ];
   virtualisation.oci-containers.containers.signal_cli_rest_api = {
     image = "bbernhard/signal-cli-rest-api:0.199-dev";
     ports = [

systems/jeeves/hardware.nix

@@ -96,6 +96,11 @@ in
       fsType = "zfs";
     };
 
+    "/nix/var/nix/builds" = {
+      device = "root_pool/nix_build";
+      fsType = "zfs";
+    };
+
     "/var" = {
       device = "root_pool/var";
       fsType = "zfs";

systems/jeeves/monitoring/README.md

@@ -0,0 +1,33 @@
+# Monitoring
+
+## Vultr API metrics
+
+The `vultr-exporter` service reads its API token from:
+
+```text
+/zfs/storage/secrets/services/vultr-exporter
+```
+
+Create the file on Jeeves as root with the following contents:
+
+```text
+API_KEY=<Vultr API token>
+```
+
+The token needs read access to the Vultr Account and Billing APIs. Unrelated
+resource collectors are disabled in the packaged exporter.
+
+Restrict the file to root and ensure the public egress IP used by Jeeves is
+allowed for the token in the Vultr API settings:
+
+```console
+sudo chown root:root /zfs/storage/secrets/services/vultr-exporter
+sudo chmod 600 /zfs/storage/secrets/services/vultr-exporter
+```
+
+The exporter listens on `127.0.0.1:9188`; it is scraped by the local
+`prometheus-main` service every five minutes and is not exposed through the
+host firewall.
+
+Portal-1 exposes its node exporter only through `tailscale0` on port `9100`.
+Jeeves reaches it using the Portal-1 Tailscale hostname.

systems/jeeves/monitoring/dashboards/portal-1.json

@@ -0,0 +1,798 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": false,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "short",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "red",
+                  "index": 1,
+                  "text": "Offline"
+                },
+                "1": {
+                  "color": "green",
+                  "index": 0,
+                  "text": "Online"
+                }
+              },
+              "type": "value"
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 4,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "up{job=\"node\",instance=\"portal-1\"}",
+          "instant": true,
+          "legendFormat": "",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Node Exporter",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 5,
+        "x": 4,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{job=\"node\",instance=\"portal-1\",mode=\"idle\"}[5m])))",
+          "instant": true,
+          "legendFormat": "",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "CPU Used",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 5,
+        "x": 9,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - (node_memory_MemAvailable_bytes{job=\"node\",instance=\"portal-1\"} / node_memory_MemTotal_bytes{job=\"node\",instance=\"portal-1\"}))",
+          "instant": true,
+          "legendFormat": "",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "RAM Used",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 5,
+        "x": 14,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - (node_filesystem_avail_bytes{job=\"node\",instance=\"portal-1\",mountpoint=\"/\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node\",instance=\"portal-1\",mountpoint=\"/\",fstype!=\"\"}))",
+          "instant": true,
+          "legendFormat": "",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Root Filesystem Used",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 5,
+        "x": 19,
+        "y": 0
+      },
+      "id": 5,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "node_load1{job=\"node\",instance=\"portal-1\"}",
+          "instant": true,
+          "legendFormat": "",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Load (1m)",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 5
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{job=\"node\",instance=\"portal-1\",mode=\"idle\"}[5m])))",
+          "instant": false,
+          "legendFormat": "CPU used",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "CPU Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 5
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - (node_memory_MemAvailable_bytes{job=\"node\",instance=\"portal-1\"} / node_memory_MemTotal_bytes{job=\"node\",instance=\"portal-1\"}))",
+          "instant": false,
+          "legendFormat": "RAM used",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "RAM Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 13
+      },
+      "id": 8,
+      "options": {
+        "cellHeight": "sm",
+        "showHeader": true,
+        "sortBy": [
+          {
+            "desc": true,
+            "displayName": "Value"
+          }
+        ]
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - (node_filesystem_avail_bytes{job=\"node\",instance=\"portal-1\",fstype!~\"tmpfs|devtmpfs|overlay|squashfs\",mountpoint!~\"/run.*\"} / node_filesystem_size_bytes{job=\"node\",instance=\"portal-1\",fstype!~\"tmpfs|devtmpfs|overlay|squashfs\",mountpoint!~\"/run.*\"}))",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "{{mountpoint}}",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Filesystem Usage",
+      "type": "table"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "Bps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 13
+      },
+      "id": 9,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(node_disk_read_bytes_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
+          "instant": false,
+          "legendFormat": "read",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(node_disk_written_bytes_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
+          "instant": false,
+          "legendFormat": "write",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Disk Throughput",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "iops"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 21
+      },
+      "id": 10,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(node_disk_reads_completed_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
+          "instant": false,
+          "legendFormat": "reads",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(node_disk_writes_completed_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
+          "instant": false,
+          "legendFormat": "writes",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Disk Operations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "Bps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 21
+      },
+      "id": 11,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(node_network_receive_bytes_total{job=\"node\",instance=\"portal-1\",device!=\"lo\"}[5m]))",
+          "instant": false,
+          "legendFormat": "received",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(node_network_transmit_bytes_total{job=\"node\",instance=\"portal-1\",device!=\"lo\"}[5m]))",
+          "instant": false,
+          "legendFormat": "sent",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Network Traffic",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 39,
+  "tags": [
+    "monitoring",
+    "portal-1",
+    "vultr"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Portal-1 Host",
+  "uid": "portal-1-host",
+  "version": 1,
+  "weekStart": ""
+}

systems/jeeves/monitoring/default.nix

@@ -41,29 +41,36 @@ let
       {
         job_name = "node";
         static_configs = [
-          (mkTarget "jeeves" "192.168.90.40:9100")
-          (mkTarget "bob" "192.168.90.25:9100")
+          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9100")
+          (mkTarget "bob" "bob.taile39c31.ts.net:9100")
+          (mkTarget "portal-1" "portal-1.taile39c31.ts.net:9100")
         ];
       }
+      {
+        job_name = "vultr";
+        scrape_interval = "5m";
+        scrape_timeout = "45s";
+        static_configs = [ (mkTarget "vultr" "127.0.0.1:9188") ];
+      }
       {
         job_name = "process_grouped";
         static_configs = [
-          (mkTarget "jeeves" "192.168.90.40:9256")
-          (mkTarget "bob" "192.168.90.25:9256")
+          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9256")
+          (mkTarget "bob" "bob.taile39c31.ts.net:9256")
         ];
       }
       {
         job_name = "smartctl";
         static_configs = [
-          (mkTarget "jeeves" "192.168.90.40:9633")
-          (mkTarget "bob" "192.168.90.25:9633")
+          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9633")
+          (mkTarget "bob" "bob.taile39c31.ts.net:9633")
         ];
       }
       {
         job_name = "zfs";
         static_configs = [
-          (mkTarget "jeeves" "192.168.90.40:9134")
-          (mkTarget "bob" "192.168.90.25:9134")
+          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9134")
+          (mkTarget "bob" "bob.taile39c31.ts.net:9134")
         ];
       }
     ];
@@ -79,8 +86,8 @@ let
       {
         job_name = "process_pid";
         static_configs = [
-          (mkTarget "jeeves" "192.168.90.40:9257")
-          (mkTarget "bob" "192.168.90.25:9257")
+          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9257")
+          (mkTarget "bob" "bob.taile39c31.ts.net:9257")
         ];
       }
     ];
@@ -97,7 +104,9 @@ let
       after = [
         "zfs-media-database-prometheus.mount"
         "network.target"
+        "tailscaled-autoconnect.service"
       ];
+      wants = [ "tailscaled-autoconnect.service" ];
       requires = [ "zfs-media-database-prometheus.mount" ];
       wantedBy = [ "multi-user.target" ];
       unitConfig.RequiresMountsFor = [ dataDir ];
@@ -151,6 +160,8 @@ let
     };
 in
 {
+  imports = [ ./vultr-exporter.nix ];
+
   users = {
     groups.prometheus = { };
     users.prometheus = {

systems/jeeves/monitoring/vultr-exporter.nix

@@ -0,0 +1,96 @@
+{
+  lib,
+  pkgs,
+  ...
+}:
+let
+  vars = import ../vars.nix;
+
+  vultrExporter = pkgs.buildGoModule rec {
+    pname = "vultr-exporter";
+    version = "0-unstable-2025-10-14";
+
+    src = pkgs.fetchFromGitHub {
+      owner = "DazWilkin";
+      repo = "vultr-exporter";
+      rev = "d8b5d7107c2eda31566e6e895c8a67db32fdaf68";
+      hash = "sha256-3LpXh+mFMBRJfiY2ftELpM3AIE0LiZ9c6NFok0OeH5I=";
+    };
+
+    vendorHash = "sha256-qCHWmg6GxLIjqlf5UqRcuDRd1xqRFI3sl4ZduLajx+Y=";
+    subPackages = [ "cmd/server" ];
+
+    postPatch = ''
+      substituteInPlace cmd/server/main.go \
+        --replace-fail 'registry.MustRegister(collector.NewBlockStorageCollector(s, client, log))' "" \
+        --replace-fail 'registry.MustRegister(collector.NewKubernetesCollector(s, client, log))' "" \
+        --replace-fail 'registry.MustRegister(collector.NewLoadBalancerCollector(s, client, log))' "" \
+        --replace-fail 'registry.MustRegister(collector.NewReservedIPsCollector(s, client, log))' ""
+    '';
+
+    ldflags = [
+      "-s"
+      "-w"
+      "-X main.GitCommit=${src.rev}"
+      "-X main.OSVersion=NixOS"
+    ];
+
+    postInstall = ''
+      mv $out/bin/server $out/bin/vultr-exporter
+    '';
+
+    meta = {
+      description = "Prometheus exporter for Vultr account, billing, and bandwidth metrics";
+      homepage = "https://github.com/DazWilkin/vultr-exporter";
+      license = lib.licenses.asl20;
+      mainProgram = "vultr-exporter";
+    };
+  };
+in
+{
+  systemd.services.vultr-exporter = {
+    description = "Vultr Prometheus exporter";
+    documentation = [ "https://github.com/DazWilkin/vultr-exporter" ];
+    wants = [ "network-online.target" ];
+    after = [ "network-online.target" ];
+    wantedBy = [ "multi-user.target" ];
+
+    serviceConfig = {
+      ExecStart = "${lib.getExe vultrExporter} --endpoint=127.0.0.1:9188 --path=/metrics";
+      EnvironmentFile = "${vars.secrets}/services/vultr-exporter";
+      Restart = "on-failure";
+      RestartSec = "30s";
+
+      CapabilityBoundingSet = [ "" ];
+      DeviceAllow = [ "" ];
+      DynamicUser = true;
+      LockPersonality = true;
+      MemoryDenyWriteExecute = true;
+      NoNewPrivileges = true;
+      PrivateDevices = true;
+      PrivateTmp = true;
+      ProtectClock = true;
+      ProtectControlGroups = true;
+      ProtectHome = true;
+      ProtectHostname = true;
+      ProtectKernelLogs = true;
+      ProtectKernelModules = true;
+      ProtectKernelTunables = true;
+      ProtectSystem = "strict";
+      RemoveIPC = true;
+      RestrictAddressFamilies = [
+        "AF_INET"
+        "AF_INET6"
+      ];
+      RestrictNamespaces = true;
+      RestrictRealtime = true;
+      RestrictSUIDSGID = true;
+      SystemCallArchitectures = "native";
+      SystemCallFilter = [
+        "@system-service"
+        "~@privileged"
+      ];
+      UMask = "0077";
+    };
+  };
+}

systems/jeeves/networking/default.nix

@@ -0,0 +1,16 @@
+{
+  imports = [
+    ./firewall.nix
+    ./interfaces.nix
+  ];
+  networking = {
+    hostName = "jeeves";
+    hostId = "0e15ce35";
+    useNetworkd = true;
+    # Without this, unconfigured NICs get a catch-all DHCP network and
+    # join the LAN uninvited (how the dead atlantic card ended up
+    # ARP-poisoning 192.168.99.14). Every interface must be configured
+    # explicitly below.
+    useDHCP = false;
+  };
+}

systems/jeeves/networking/firewall.nix

@@ -0,0 +1,47 @@
+let
+  ports = {
+    audiobookshelf = 8000;
+    ebook_searcher = 8070;
+    gems = 8002;
+    gitea_http = 6443;
+    gitea_ssh = 2223;
+    grafana = 3000;
+    jellyfin_http = 8096;
+    nix_binary_cache = 5000;
+    nornsight = 8001;
+    ollama = 11434;
+    open_webui = 8080;
+    postgresql = 5432;
+    share = 8091;
+    signal_cli = 8989;
+    syncthing = 8384;
+  };
+in
+{
+  networking.firewall = {
+    enable = true;
+    interfaces.br-nix-builder = {
+      allowedTCPPorts = [ ];
+      allowedUDPPorts = [ ];
+    };
+    interfaces.tailscale0.allowedTCPPorts = [
+      ports.audiobookshelf
+      ports.gems
+      ports.jellyfin_http
+      ports.nix_binary_cache
+      ports.nornsight
+      ports.share
+    ];
+    allowedTCPPorts = [
+      ports.ebook_searcher
+      ports.gitea_http
+      ports.gitea_ssh
+      ports.grafana
+      ports.ollama
+      ports.open_webui
+      ports.postgresql
+      ports.signal_cli
+      ports.syncthing
+    ];
+  };
+}

systems/jeeves/networking/interfaces.nix

@@ -8,27 +8,6 @@
     "net.bridge.bridge-nf-call-iptables" = 0;
   };
 
-  networking = {
-    hostName = "jeeves";
-    hostId = "0e15ce35";
-    firewall = {
-      enable = true;
-      interfaces.br-nix-builder = {
-        allowedTCPPorts = [ ];
-        allowedUDPPorts = [ ];
-      };
-      allowedTCPPorts = [
-        8070
-      ];
-    };
-    useNetworkd = true;
-    # Without this, unconfigured NICs get a catch-all DHCP network and
-    # join the LAN uninvited (how the dead atlantic card ended up
-    # ARP-poisoning 192.168.99.14). Every interface must be configured
-    # explicitly below.
-    useDHCP = false;
-  };
-
   systemd.network = {
     enable = true;
     wait-online = {

systems/jeeves/runners/nix_builder.nix

@@ -1,8 +1,7 @@
 {
   config,
   lib,
-  outputs,
-  utils,
+  pkgs,
   ...
 }:
 
@@ -12,6 +11,132 @@ let
   cfg = config.services.nix_builder;
   runnerUsername = "gitea-runner";
   runnerUserid = 601;
+  runnerLabels = [
+    "self-hosted:host"
+    "nixos:host"
+  ];
+  containerConfig =
+    {
+      config,
+      pkgs,
+      lib,
+      ...
+    }:
+    let
+      giteaAutomationPython = pkgs.python314.withPackages (
+        ps: with ps; [
+          httpx
+          pydantic
+          typer
+        ]
+      );
+      runnerConfigFile = (pkgs.formats.yaml { }).generate "gitea-runner.yaml" { };
+      registerRunner = pkgs.writeShellApplication {
+        name = "register-gitea-runner";
+        runtimeInputs = [
+          pkgs.coreutils
+          pkgs.hostname
+          config.services.gitea-actions-runner.package
+        ];
+        text = builtins.readFile ./register_gitea_runner.sh;
+      };
+    in
+    {
+      networking = {
+        hostName = "nix-builder";
+        useDHCP = lib.mkDefault true;
+        interfaces.eth0.useDHCP = true;
+        # Ensure containers don't inherit the host's stub resolver (127.0.0.53) which was causing issues
+        useHostResolvConf = false;
+      };
+      nix.settings = {
+        cores = 8;
+        max-jobs = 2;
+        system-features = lib.mkAfter [
+          "gccarch-x86-64-v2"
+          "gccarch-x86-64-v3"
+        ];
+        trusted-substituters = [
+          "https://cache.nixos.org"
+          "https://cache.tmmworkshop.com"
+          "https://nix-community.cachix.org"
+        ];
+        substituters = [
+          "https://cache.nixos.org/?priority=2&want-mass-query=true"
+          "https://cache.tmmworkshop.com/?priority=2&want-mass-query=true"
+          "https://nix-community.cachix.org/?priority=10&want-mass-query=true"
+        ];
+        trusted-public-keys = [
+          "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
+          "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA="
+          "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
+        ];
+        experimental-features = [
+          "flakes"
+          "nix-command"
+        ];
+        sandbox = true;
+        allowed-users = [ runnerUsername ];
+        trusted-users = [
+          "root"
+          runnerUsername
+        ];
+      };
+      users = {
+        users.${runnerUsername} = {
+          isSystemUser = true;
+          group = runnerUsername;
+          uid = runnerUserid;
+        };
+        groups.${runnerUsername}.gid = runnerUserid;
+      };
+      services.gitea-actions-runner.instances.builder = {
+        enable = true;
+        name = "jeeves-nix-builder";
+        url = "http://192.168.99.14:6443/";
+        labels = runnerLabels;
+        tokenFile = "/run/secrets/gitea-runners/registration-token";
+        settings.runner.timeout = "12h";
+        hostPackages = with pkgs; [
+          bash
+          coreutils
+          curl
+          gawk
+          gitMinimal
+          giteaAutomationPython
+          gnused
+          nix
+          nixfmt
+          nixos-rebuild
+          nodejs
+          ruff
+          treefmt
+          wget
+        ];
+      };
+      systemd.services."gitea-runner-builder" = {
+        serviceConfig = {
+          DynamicUser = mkForce false;
+          User = mkForce runnerUsername;
+          Group = mkForce runnerUsername;
+          ExecStartPre = mkForce [
+            "${getExe registerRunner} builder http://192.168.99.14:6443/ ${runnerConfigFile} ${escapeShellArgs runnerLabels}"
+          ];
+        };
+      };
+      system.stateVersion = "24.05";
+    };
+  sharedContainerPath =
+    (import "${pkgs.path}/nixos/lib/eval-config.nix" {
+      modules = [
+        {
+          boot.isNspawnContainer = true;
+          nixpkgs.pkgs = pkgs;
+        }
+        containerConfig
+      ];
+      system = null;
+    }).config.system.build.toplevel;
 in
 {
   options.services.nix_builder = {
@@ -48,9 +173,11 @@ in
     containers = mapAttrs (
       name: containerCfg:
       mkIf containerCfg.enable {
+        path = sharedContainerPath;
         autoStart = true;
         privateNetwork = true;
         hostBridge = cfg.bridgeName;
+        extraFlags = [ "--hostname=${name}" ];
         bindMounts = {
           host-nix = {
             mountPoint = "/host-nix/var/nix/daemon-socket";
@@ -68,93 +195,6 @@ in
             isReadOnly = false;
           };
         };
-        config =
-          {
-            config,
-            pkgs,
-            lib,
-            ...
-          }:
-          {
-            networking = {
-              useDHCP = lib.mkDefault true;
-              interfaces.eth0.useDHCP = true;
-              # Ensure containers don't inherit the host's stub resolver (127.0.0.53) which was causing issues
-              useHostResolvConf = false;
-            };
-            nix.settings = {
-              trusted-substituters = [
-                "https://cache.nixos.org"
-                "https://cache.tmmworkshop.com"
-                "https://nix-community.cachix.org"
-              ];
-              substituters = [
-                "https://cache.nixos.org/?priority=2&want-mass-query=true"
-                "https://cache.tmmworkshop.com/?priority=2&want-mass-query=true"
-                "https://nix-community.cachix.org/?priority=10&want-mass-query=true"
-              ];
-              trusted-public-keys = [
-                "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
-                "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA="
-                "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
-              ];
-              experimental-features = [
-                "flakes"
-                "nix-command"
-              ];
-              sandbox = true;
-              allowed-users = [ "gitea-runner" ];
-              trusted-users = [
-                "root"
-                "gitea-runner"
-              ];
-            };
-            nixpkgs = {
-              overlays = builtins.attrValues outputs.overlays;
-              config.allowUnfree = true;
-            };
-            users = {
-              users.${runnerUsername} = {
-                isSystemUser = true;
-                group = runnerUsername;
-                uid = runnerUserid;
-              };
-              groups.${runnerUsername}.gid = runnerUserid;
-            };
-            services.gitea-actions-runner.instances.${name} = {
-              enable = true;
-              name = "jeeves-${name}";
-              url = "http://192.168.99.14:6443/";
-              labels = [
-                "self-hosted:host"
-                "nixos:host"
-              ];
-              tokenFile = "/run/secrets/gitea-runners/registration-token";
-              hostPackages = with pkgs; [
-                bash
-                coreutils
-                curl
-                gawk
-                gitMinimal
-                gnused
-                my_python
-                nix
-                nixfmt
-                nixos-rebuild
-                nodejs
-                treefmt
-                wget
-              ];
-            };
-            systemd.services."gitea-runner-${utils.escapeSystemdPath name}" = {
-              serviceConfig = {
-                DynamicUser = mkForce false;
-                User = mkForce runnerUsername;
-                Group = mkForce runnerUsername;
-              };
-            };
-            system.stateVersion = "24.05";
-          };
       }
     ) cfg.containers;
 
@@ -165,6 +205,7 @@ in
           value = {
             requires = [ "gitea.service" ];
             after = [ "gitea.service" ];
+            serviceConfig.CPUQuota = "800%";
           };
         }) (builtins.attrNames (filterAttrs (_: c: c.enable) cfg.containers))
       );

systems/jeeves/runners/register_gitea_runner.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+legacy_instance_dir="$STATE_DIRECTORY/$(hostname)"
+instance_dir="$STATE_DIRECTORY/$1"
+runner_url="$2"
+runner_config_file="$3"
+shift 3
+runner_labels="$(IFS=,; printf '%s' "$*")"
+
+if [ ! -e "$instance_dir" ] && [ -d "$legacy_instance_dir" ]; then
+  mv "$legacy_instance_dir" "$instance_dir"
+fi
+
+mkdir -vp "$instance_dir"
+cd "$instance_dir" || exit 1
+
+token_hash_file="$instance_dir/.token-hash"
+token_hash_current="$(printf '%s' "$TOKEN" | sha256sum | cut -d' ' -f1)"
+token_hash_stored="$(cat "$token_hash_file" 2>/dev/null || echo "")"
+labels_file="$instance_dir/.labels"
+labels_wanted="$(printf '%s\n' "$@" | sort)"
+labels_current="$(cat "$labels_file" 2>/dev/null || echo 0)"
+
+if [ ! -e "$instance_dir/.runner" ] || [ "$labels_wanted" != "$labels_current" ] || [ "$token_hash_current" != "$token_hash_stored" ]; then
+  rm -v "$instance_dir/.runner" || true
+
+  gitea-runner register --no-interactive \
+    --instance "$runner_url" \
+    --token "$TOKEN" \
+    --name "jeeves-$(hostname)" \
+    --labels "$runner_labels" \
+    --config "$runner_config_file"
+
+  printf '%s' "$token_hash_current" > "$token_hash_file"
+  printf '%s\n' "$labels_wanted" > "$labels_file"
+fi

systems/jeeves/scripts/zfs.sh

@@ -17,9 +17,9 @@ sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnode
 # media datasets
 sudo zfs create media/temp -o sync=disabled -o redundant_metadata=none
 sudo zfs create media/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key
+sudo zfs create media/secure/containers -o mountpoint=/zfs/media/containers -o compression=lz4 -o sync=disabled -o redundant_metadata=some -o normalization=none -o utf8only=off
 sudo zfs create media/secure/docker -o compression=zstd-9
 sudo zfs create media/secure/github-runners -o compression=zstd-9 -o sync=disabled
-sudo zfs create media/secure/home_assistant -o compression=zstd-19
 sudo zfs create media/secure/notes -o copies=2
 sudo zfs create media/secure/postgres -o mountpoint=/zfs/media/database/postgres -o recordsize=16k -o primarycache=metadata
 sudo zfs create media/secure/postgres-wal -o mountpoint=/zfs/media/database/postgres-wal -o recordsize=32k -o primarycache=metadata -o special_small_blocks=32K -o compression=lz4 -o secondarycache=none -o logbias=latency

systems/jeeves/services/audiobookshelf.nix

@@ -6,6 +6,7 @@ in
   services.audiobookshelf = {
     enable = true;
     port = 8000;
+    host = "0.0.0.0";
   };
   systemd.services.audiobookshelf.serviceConfig.WorkingDirectory =
     lib.mkForce "${vars.docker_configs}/audiobookshelf";

systems/jeeves/services/camofox-browser.nix

@@ -1,80 +0,0 @@
-{
-  ...
-}:
-let
-  vars = import ../vars.nix;
-in
-{
-  systemd.tmpfiles.rules = [
-    "d ${vars.docker_configs}/camofox-browser 0750 root root - -"
-  ];
-
-  containers.camofox-browser = {
-    autoStart = true;
-    privateNetwork = false;
-    bindMounts = {
-      camofox-browser = {
-        hostPath = "${vars.docker_configs}/camofox-browser";
-        mountPoint = "/var/lib/camofox-browser";
-        isReadOnly = false;
-      };
-    };
-    config =
-      {
-        pkgs,
-        lib,
-        ...
-      }:
-      {
-        networking.hostName = "camofox-browser";
-
-        environment.systemPackages = with pkgs; [
-          ffmpeg
-          git
-          nodejs
-          python3Packages.yt-dlp
-        ];
-
-        systemd.services.camofox-browser = {
-          description = "Camofox browser server";
-          wantedBy = [ "multi-user.target" ];
-          after = [ "network.target" ];
-          environment = {
-            CAMOFOX_HOST = "127.0.0.1";
-            CAMOFOX_PORT = "9377";
-            HOME = "/var/lib/camofox-browser";
-          };
-          path = with pkgs; [
-            bash
-            coreutils
-            git
-            nodejs
-          ];
-          serviceConfig = {
-            Restart = "always";
-            RestartSec = "5s";
-            WorkingDirectory = "/var/lib/camofox-browser";
-          };
-          script = ''
-            set -eu
-
-            app_dir=/var/lib/camofox-browser/app
-
-            if [ ! -d "$app_dir/.git" ]; then
-              git clone --depth 1 https://github.com/jo-inc/camofox-browser "$app_dir"
-            fi
-
-            cd "$app_dir"
-
-            if [ ! -d node_modules ]; then
-              npm install
-            fi
-
-            exec npm start
-          '';
-        };
-
-        system.stateVersion = lib.mkDefault "24.05";
-      };
-  };
-}

systems/jeeves/services/gems.nix

@@ -6,8 +6,47 @@
 let
   vars = import ../vars.nix;
   stateDir = "${vars.services}/gems";
+  gemsPackages =
+    ps: with ps; [
+      fastapi
+      jinja2
+      pydantic
+      pydantic-settings
+      python-multipart
+      typer
+      uvicorn
+    ];
 in
 {
+  nixpkgs.overlays = [
+    (final: _prev: {
+      gems_python = final.python314.withPackages gemsPackages;
+      gems_test_python = final.python314.withPackages (
+        ps:
+        gemsPackages ps
+        ++ (with ps; [
+          httpx
+          pytest
+          pytest-asyncio
+          pytest-xdist
+        ])
+      );
+      gems_tests =
+        final.runCommand "gems-tests"
+          {
+            nativeBuildInputs = [ final.gems_test_python ];
+          }
+          ''
+            export HOME="$TMPDIR"
+            cd ${inputs.self}
+            pytest -o cache_dir="$TMPDIR/pytest-cache" tests/gems
+            touch "$out"
+          '';
+    })
+  ];
+
+  system.checks = [ pkgs.gems_tests ];
+
   users.groups.gems = { };
   users.users.gems = {
     isSystemUser = true;
@@ -36,7 +75,7 @@ in
       Type = "simple";
       User = "gems";
       Group = "gems";
-      ExecStart = "${pkgs.my_python}/bin/python -m python.gems.main --host 127.0.0.1 --port 8002";
+      ExecStart = "${pkgs.gems_python}/bin/python -m python.gems.main --host 0.0.0.0 --port 8002";
       Restart = "on-failure";
       RestartSec = "5s";
       StandardOutput = "journal";

systems/jeeves/services/gitea.nix

@@ -2,11 +2,6 @@ let
   vars = import ../vars.nix;
 in
 {
-  networking.firewall.allowedTCPPorts = [
-    6443
-    2223
-  ];
-
   services.gitea = {
     enable = true;
     appName = "TMM Workshop";
@@ -24,13 +19,18 @@ in
       actions = {
         ENABLED = true;
         DEFAULT_ACTIONS_URL = "github";
+        ENDLESS_TASK_TIMEOUT = "12h";
       };
       service.DISABLE_REGISTRATION = true;
       server = {
-        DOMAIN = "tmmworkshop.com";
+        DOMAIN = "gitea.tmmworkshop.com";
         ROOT_URL = "https://gitea.tmmworkshop.com/";
         HTTP_PORT = 6443;
+        BUILTIN_SSH_SERVER_USER = "gitea";
+        SSH_USER = "gitea";
+        SSH_DOMAIN = "gitea.tmmworkshop.com";
         SSH_PORT = 2223;
+        SSH_LISTEN_HOST = "0.0.0.0";
         SSH_LISTEN_PORT = 2223;
         START_SSH_SERVER = true;
         PUBLIC_URL_DETECTION = "auto";

systems/jeeves/services/grafana.nix

@@ -6,8 +6,6 @@ let
   grafanaDataDir = "${vars.services}/grafana";
 in
 {
-  networking.firewall.allowedTCPPorts = [ 3000 ];
-
   services.grafana = {
     enable = true;
     dataDir = grafanaDataDir;
@@ -19,9 +17,9 @@ in
         secret_key = "$__file{${vars.secrets}/services/grafana/secret_key}";
       };
       server = {
-        http_addr = "192.168.90.40";
+        http_addr = "0.0.0.0";
         http_port = 3000;
-        root_url = "http://192.168.90.40:3000/";
+        root_url = "http://jeeves:3000/";
       };
     };
     provision = {

systems/jeeves/services/home_assistant.nix

@@ -1,79 +0,0 @@
-let
-  vars = import ../vars.nix;
-in
-{
-  users = {
-    users.hass = {
-      isSystemUser = true;
-      group = "hass";
-    };
-    groups.hass = { };
-  };
-
-  services = {
-    home-assistant = {
-      enable = true;
-      openFirewall = true;
-      configDir = vars.home_assistant;
-      config = {
-        http = {
-          server_port = 8123;
-          use_x_forwarded_for = true;
-          trusted_proxies = "127.0.0.1";
-        };
-        homeassistant = {
-          time_zone = "America/New_York";
-          unit_system = "us_customary";
-          temperature_unit = "F";
-        };
-        recorder = {
-          db_url = "postgresql://@/hass";
-          auto_purge = true;
-          purge_keep_days = 3650;
-          db_retry_wait = 15;
-        };
-        assist_pipeline = { };
-        backup = { };
-        bluetooth = { };
-        config = { };
-        dhcp = { };
-        energy = { };
-        history = { };
-        homeassistant_alerts = { };
-        image_upload = { };
-        logbook = { };
-        media_source = { };
-        mobile_app = { };
-        ssdp = { };
-        sun = { };
-        webhook = { };
-        zeroconf = { };
-        automation = "!include automations.yaml";
-        script = "!include scripts.yaml";
-        scene = "!include scenes.yaml";
-        group = "!include groups.yaml";
-      };
-      extraPackages =
-        python3Packages: with python3Packages; [
-          aioesphomeapi
-          aiounifi
-          bleak-esphome
-          esphome-dashboard-api
-          gtts
-          jellyfin-apiclient-python
-          psycopg2
-          pymetno
-          aio-ownet
-          rokuecp
-          uiprotect
-          wakeonlan
-        ];
-      extraComponents = [ "isal" ];
-    };
-    esphome = {
-      enable = true;
-      openFirewall = true;
-      address = "192.168.90.40";
-    };
-  };
-}

systems/jeeves/services/jellyfin.nix

@@ -4,7 +4,6 @@ in
 {
   services.jellyfin = {
     enable = true;
-    openFirewall = true;
     dataDir = "${vars.services}/jellyfin";
     cacheDir = "${vars.services}/jellyfin/cache";
   };

systems/jeeves/services/llms.nix

@@ -38,7 +38,6 @@ in
       "qwen3.5:35b"
     ];
     modelsDir = vars.ollama;
-    openFirewall = true;
   };
   systemd.services = {
     ollama.serviceConfig = {

systems/jeeves/services/nix_serve.nix

@@ -7,6 +7,5 @@ in
     package = pkgs.nix-serve-ng;
     enable = true;
     secretKeyFile = "${vars.secrets}/services/nix-cache/cache-priv-key.pem";
-    openFirewall = true;
   };
 }

systems/jeeves/services/open_webui.nix

@@ -1,13 +0,0 @@
-{
-  services.open-webui = {
-    enable = true;
-    host = "0.0.0.0";
-    openFirewall = true;
-    environment = {
-      ANONYMIZED_TELEMETRY = "False";
-      DO_NOT_TRACK = "True";
-      SCARF_NO_ANALYTICS = "True";
-      OLLAMA_API_BASE_URL = "http://127.0.0.1:11434";
-    };
-  };
-}

systems/jeeves/services/postgress.nix

@@ -3,8 +3,6 @@ let
   vars = import ../vars.nix;
 in
 {
-  networking.firewall.allowedTCPPorts = [ 5432 ];
-
   # Symlink pg_wal to a ZFS dataset on the special (metadata) vdev for fast WAL writes
   # this is required for systemd sandboxing
   systemd.services.postgresql.serviceConfig.ReadWritePaths = [ "/zfs/media/database/postgres-wal" ];
@@ -27,13 +25,12 @@ in
       local all  richie   trust
       host  all  richie   127.0.0.1/32    trust
       host  all  richie   ::1/128         trust
-      host  all  richie   192.168.90.1/24 trust
+      host  all  richie   100.64.0.0/10 trust
       host  all  richie   192.168.99.1/24 trust
       host  all  richie   172.16.0.0/12   trust
 
 
       #type database DBuser origin-address auth-method
-      local hass     hass     trust
       local gitea    gitea    trust
 
       # signalbot
@@ -43,12 +40,12 @@ in
       local postgres  math   trust
       host  postgres  math   127.0.0.1/32    trust
       host  postgres  math   ::1/128         trust
-      host  postgres  math   192.168.90.1/24 trust
+      host  postgres  math   100.64.0.0/10 trust
 
       local data_science_dev  math   trust
       host  data_science_dev  math   127.0.0.1/32    trust
       host  data_science_dev  math   ::1/128         trust
-      host  data_science_dev  math   192.168.90.1/24 trust
+      host  data_science_dev  math   100.64.0.0/10 trust
     '';
 
     identMap = ''
@@ -57,7 +54,6 @@ in
         superuser_map      postgres  postgres
         # Let other names login as themselves
         superuser_map      richie    postgres
-        superuser_map      hass      hass
     '';
     ensureUsers = [
       {
@@ -81,16 +77,6 @@ in
           replication = true;
         };
       }
-      {
-        name = "hass";
-        ensureDBOwnership = true;
-        ensureClauses = {
-          login = true;
-          createrole = true;
-          createdb = true;
-          replication = true;
-        };
-      }
       {
         name = "gitea";
         ensureDBOwnership = true;
@@ -121,7 +107,6 @@ in
     ];
     ensureDatabases = [
       "data_science_dev"
-      "hass"
       "gitea"
       "math"
       "n8n"

systems/jeeves/services/systemd.nix

@@ -1,11 +1,7 @@
 {
   pkgs,
-  inputs,
   ...
 }:
-let
-  vars = import ../vars.nix;
-in
 {
   systemd = {
     services = {
@@ -30,21 +26,6 @@ in
           ExecStart = "${pkgs.bash}/bin/bash -c 'echo 1 > /sys/bus/pci/devices/0000:61:00.0/remove'";
         };
       };
-      startup_validation = {
-        requires = [ "network-online.target" ];
-        after = [ "network-online.target" ];
-        wantedBy = [ "multi-user.target" ];
-        description = "validates startup";
-        path = [ pkgs.zfs ];
-        environment = {
-          PYTHONPATH = "${inputs.self}/";
-        };
-        serviceConfig = {
-          EnvironmentFile = "${vars.secrets}/services/server-validation";
-          Type = "oneshot";
-          ExecStart = "${pkgs.my_python}/bin/python -m python.system_tests.validate_system '${./validate_system.toml}'";
-        };
-      };
     };
     timers = {
       plex_permission = {
@@ -55,13 +36,6 @@ in
           Unit = "plex_permission.service";
         };
       };
-      startup_validation = {
-        wantedBy = [ "timers.target" ];
-        timerConfig = {
-          OnBootSec = "10min";
-          Unit = "startup_validation.service";
-        };
-      };
     };
   };
 }

systems/jeeves/services/transmission.nix

@@ -21,11 +21,11 @@ in
       message-level = 3;
       peer-port = 51413;
       rpc-bind-address = "0.0.0.0";
-      rpc-host-whitelist = "127.0.0.1,192.168.90.40";
+      rpc-host-whitelist = "127.0.0.1,jeeves,jeeves.taile39c31.ts.net";
       rpc-host-whitelist-enabled = true;
       rpc-port = 9091;
       rpc-whitelist-enabled = true;
-      rpc-whitelist = "127.0.0.1,192.168.90.49";
+      rpc-whitelist = "127.0.0.1,100.*.*.*";
       seed-queue-enabled = false;
     };
   };

systems/jeeves/services/validate_system.toml

@@ -1,8 +0,0 @@
-zpool = ["root_pool", "storage", "media"]
-services = [
-    "audiobookshelf",
-    "haproxy",
-    "docker",
-    "home-assistant",
-    "jellyfin",
-]

systems/jeeves/snapshot_config.toml

@@ -23,6 +23,12 @@ hourly = 0
 daily = 0
 monthly = 0
 
+["root_pool/nix_build"]
+15_min = 1
+hourly = 0
+daily = 0
+monthly = 0
+
 ["root_pool/var"]
 15_min = 8
 hourly = 24
@@ -60,6 +66,12 @@ daily = 30
 monthly = 12
 
 # media
+["media/secure/containers"]
+15_min = 2
+hourly = 0
+daily = 0
+monthly = 0
+
 ["media/temp"]
 15_min = 2
 hourly = 0

systems/jeeves/syncthing.nix

@@ -2,11 +2,9 @@ let
   vars = import ./vars.nix;
 in
 {
-  networking.firewall.allowedTCPPorts = [ 8384 ];
-
   services.syncthing = {
     overrideFolders = false;
-    guiAddress = "192.168.90.40:8384";
+    guiAddress = "0.0.0.0:8384";
     settings = {
       devices.davids-server.id = "7GXTDGR-AOXFW2O-K6J7NM3-XYZNRRW-AKHAFWM-GBOWUPQ-OA6JIWD-ER7RDQL"; # cspell:disable-line
       folders = {

systems/jeeves/vars.nix

@@ -5,10 +5,10 @@ let
 in
 {
   inherit zfs_media zfs_storage zfs_scratch;
+  containers = "${zfs_media}/containers";
   database = "${zfs_media}/database";
   docker = "${zfs_media}/docker";
   docker_configs = "${zfs_media}/docker/configs";
-  home_assistant = "${zfs_media}/home_assistant";
   notes = "${zfs_media}/notes";
   secrets = "${zfs_storage}/secrets";
   services = "${zfs_media}/services";

systems/jeeves/web_services/acme.nix

@@ -1,75 +0,0 @@
-let
-  domains = [
-    "audiobookshelf"
-    "cache"
-    "gitea"
-    "gems"
-    "jellyfin"
-    "share"
-  ];
-  extraDomains = [ "www.norn-sight.com" ];
-
-  makeCert = name: {
-    name = "${name}.tmmworkshop.com";
-    value = {
-      webroot = "/var/lib/acme/.challenges";
-      group = "acme";
-      reloadServices = [ "haproxy.service" ];
-    };
-  };
-
-  makeExtraCert = name: {
-    inherit name;
-    value = {
-      webroot = "/var/lib/acme/.challenges";
-      group = "acme";
-      reloadServices = [ "haproxy.service" ];
-    };
-  };
-
-  acmeServices =
-    map (domain: "acme-${domain}.tmmworkshop.com.service") domains
-    ++ map (domain: "acme-${domain}.service") extraDomains;
-in
-{
-  users.users.haproxy.extraGroups = [ "acme" ];
-
-  security.acme = {
-    acceptTerms = true;
-    defaults.email = "Richie@tmmworkshop.com";
-    certs = builtins.listToAttrs ((map makeCert domains) ++ (map makeExtraCert extraDomains));
-  };
-
-  # Minimal nginx to serve ACME HTTP-01 challenge files for HAProxy
-  services.nginx = {
-    enable = true;
-    virtualHosts."acme-challenge" = {
-      listen = [
-        {
-          addr = "127.0.0.1";
-          port = 8402;
-        }
-      ];
-      locations."/.well-known/acme-challenge/" = {
-        root = "/var/lib/acme/.challenges";
-      };
-    };
-  };
-
-  # Ensure the challenge directory exists with correct permissions
-  systemd.tmpfiles.rules = [
-    "d /var/lib/acme/.challenges 0750 acme acme - -"
-    "d /var/lib/acme/.challenges/.well-known 0750 acme acme - -"
-    "d /var/lib/acme/.challenges/.well-known/acme-challenge 0750 acme acme - -"
-  ];
-
-  users.users.nginx.extraGroups = [ "acme" ];
-
-  # HAProxy needs certs to exist before it can bind :443.
-  # NixOS's acme module generates self-signed placeholders on first boot
-  # via acme-<domain>.service — just make HAProxy wait for them.
-  systemd.services.haproxy = {
-    after = acmeServices;
-    wants = acmeServices;
-  };
-}

systems/jeeves/web_services/default.nix

@@ -1,9 +0,0 @@
-{ lib, ... }:
-{
-  imports =
-    let
-      files = builtins.attrNames (builtins.readDir ./.);
-      nixFiles = builtins.filter (name: lib.hasSuffix ".nix" name && name != "default.nix") files;
-    in
-    map (file: ./. + "/${file}") nixFiles;
-}

systems/jeeves/web_services/haproxy.nix

@@ -1,14 +0,0 @@
-{
-  networking.firewall.allowedTCPPorts = [
-    80
-    443
-  ];
-
-  # Global robots.txt served by HAProxy for every vhost (see haproxy.cfg).
-  environment.etc."haproxy/robots.txt".source = ./robots.txt;
-
-  services.haproxy = {
-    enable = true;
-    config = builtins.readFile ./haproxy.cfg;
-  };
-}

systems/portal-1/README.md

@@ -0,0 +1,83 @@
+# portal_1
+
+Minimal NixOS target for a Vultr VM, installed with nixos-anywhere. The Nix
+flake target is `portal_1`; the machine hostname is `portal-1` because DNS
+hostnames cannot contain underscores.
+
+## Before deploying
+
+1. Confirm the VM's system disk is `/dev/vda`. If it is not, update both
+   references in `disk-config.nix`.
+2. Confirm the SSH public key in `default.nix` is the key that should have
+   administrator access.
+3. Boot the VM into a NixOS installer or another nixos-anywhere-compatible
+   Linux rescue environment with root SSH access. Keep this environment
+   running while completing the SOPS bootstrap below.
+
+## Bootstrap SOPS
+
+Use the rescue environment's SSH host key as the permanent portal identity.
+Replace `VM_IP` below:
+
+```console
+ssh root@VM_IP 'cat /etc/ssh/ssh_host_ed25519_key.pub' | \
+  nix shell nixpkgs#ssh-to-age --command ssh-to-age
+```
+
+This prints an `age1...` recipient; it does not copy the private key. Add the
+recipient to `.sops.yaml`:
+
+```yaml
+- &system_portal_1 age1...
+```
+
+Then add `*system_portal_1` to the age recipients for
+`users/secrets.yaml`. Re-encrypt the existing file for the new recipient and
+add the Tailscale key:
+
+```console
+nix shell nixpkgs#sops --command sops updatekeys users/secrets.yaml
+nix shell nixpkgs#sops --command sops users/secrets.yaml
+```
+
+Add the OAuth client secret from the `Auth Keys: Write` credential in the SOPS
+editor and save it:
+
+```yaml
+tailscale_auth_key: tskey-client-...
+```
+
+## Deploy
+
+From the repository root, replace `VM_IP` with the VM's public IP:
+
+```console
+nix run github:nix-community/nixos-anywhere -- \
+  --copy-host-keys --flake .#portal_1 root@VM_IP
+```
+
+This repartitions `/dev/vda`, so anything already on that disk is erased. The
+layout reserves 8 GiB for swap and assigns the remaining space to the root
+filesystem.
+
+`--copy-host-keys` preserves the same private SSH host key at
+`/etc/ssh/ssh_host_ed25519_key` on the installed system. SOPS-Nix converts that
+key to an age identity during activation. After the reboot, connect as
+`richie` and verify that automatic Tailscale enrollment succeeded:
+
+```console
+ssh -p 278 richie@VM_IP
+sudo tailscale status
+```
+
+The installed OpenSSH service listens on port 278. Port 22 is served by
+Endlessh and will not provide an SSH login.
+
+HAProxy uses the same frontend, routing, and rate-limiting configuration as
+Jeeves. Portal manages the ACME certificates for the existing public domains;
+their DNS records must resolve to Portal for HTTP-01 issuance and renewal.
+
+The application backends still use Jeeves' original `127.0.0.1` addresses.
+Replace them with the corresponding Tailscale addresses before directing
+application traffic through Portal. Ports 80 and 443 are allowed through the
+firewall.

systems/portal-1/default.nix

@@ -0,0 +1,56 @@
+{
+  inputs,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  imports = [
+    inputs.disko.nixosModules.disko
+    "${inputs.self}/users/richie"
+    "${inputs.self}/common/global"
+    "${inputs.self}/common/optional/tailscale.nix"
+    "${inputs.self}/common/optional/x86-64-v3.nix"
+    ./disk-config.nix
+    ./haproxy
+    ./monitoring.nix
+  ];
+
+  boot = {
+    # Avoid consuming the VM's limited memory for /tmp.
+    tmp.useTmpfs = false;
+
+    # The Vultr system disk and NIC are exposed as virtio devices.
+    initrd.availableKernelModules = [
+      "virtio_pci"
+      "virtio_blk"
+      "virtio_scsi"
+      "sd_mod"
+    ];
+  };
+
+  networking = {
+    hostName = "portal-1";
+    useDHCP = lib.mkDefault true;
+
+    firewall = {
+      enable = true;
+      allowedTCPPorts = [ 278 ];
+    };
+  };
+
+  services.openssh.ports = [ 278 ];
+
+  environment.systemPackages = with pkgs; [
+    curl
+    htop
+    vim
+  ];
+
+  # The VM uses its regular swap instead of compressed RAM swap.
+  zramSwap.enable = false;
+
+  time.timeZone = "Etc/UTC";
+
+  system.stateVersion = "24.05";
+}

systems/portal-1/disk-config.nix

@@ -0,0 +1,59 @@
+{ ... }:
+{
+  # Vultr's first virtio disk is normally /dev/vda. Change this before
+  # deployment if the selected image exposes its system disk differently.
+  disko.devices.disk.main = {
+    type = "disk";
+    device = "/dev/vda";
+    content = {
+      type = "gpt";
+      partitions = {
+        bios = {
+          size = "1M";
+          type = "EF02";
+        };
+
+        ESP = {
+          size = "512M";
+          type = "EF00";
+          content = {
+            type = "filesystem";
+            format = "vfat";
+            mountpoint = "/boot";
+            mountOptions = [
+              "fmask=0077"
+              "dmask=0077"
+            ];
+          };
+        };
+
+        swap = {
+          size = "8G";
+          content = {
+            type = "swap";
+          };
+        };
+
+        root = {
+          size = "100%";
+          content = {
+            type = "filesystem";
+            format = "ext4";
+            mountpoint = "/";
+          };
+        };
+      };
+    };
+  };
+
+  # Install GRUB for both legacy BIOS and EFI-capable Vultr plans.
+  boot.loader = {
+    grub = {
+      enable = true;
+      devices = [ "/dev/vda" ];
+      efiSupport = true;
+      efiInstallAsRemovable = true;
+    };
+    efi.canTouchEfiVariables = false;
+  };
+}

systems/portal-1/haproxy/default.nix

@@ -0,0 +1,89 @@
+let
+  domains = [
+    "audiobookshelf"
+    "cache"
+    "gitea"
+    "gems"
+    "jellyfin"
+    "share"
+  ];
+  extraDomains = [ "www.norn-sight.com" ];
+
+  makeCert = name: {
+    name = "${name}.tmmworkshop.com";
+    value = {
+      webroot = "/var/lib/acme/.challenges";
+      group = "acme";
+      reloadServices = [ "haproxy.service" ];
+    };
+  };
+
+  makeExtraCert = name: {
+    inherit name;
+    value = {
+      webroot = "/var/lib/acme/.challenges";
+      group = "acme";
+      reloadServices = [ "haproxy.service" ];
+    };
+  };
+
+  acmeServices =
+    map (domain: "acme-${domain}.tmmworkshop.com.service") domains
+    ++ map (domain: "acme-${domain}.service") extraDomains;
+in
+{
+  networking.firewall.allowedTCPPorts = [
+    80
+    443
+    2223
+  ];
+
+  # Global robots.txt served by HAProxy for every vhost (see haproxy.cfg).
+  environment.etc."haproxy/robots.txt".source = ./robots.txt;
+
+  services = {
+    haproxy = {
+      enable = true;
+      config = builtins.readFile ./haproxy.cfg;
+    };
+    nginx = {
+      enable = true;
+      virtualHosts."acme-challenge" = {
+        listen = [
+          {
+            addr = "127.0.0.1";
+            port = 8402;
+          }
+        ];
+        locations."/.well-known/acme-challenge/" = {
+          root = "/var/lib/acme/.challenges";
+        };
+      };
+    };
+  };
+
+  security.acme = {
+    acceptTerms = true;
+    defaults.email = "Richie@tmmworkshop.com";
+    certs = builtins.listToAttrs ((map makeCert domains) ++ (map makeExtraCert extraDomains));
+  };
+
+  systemd = {
+    services.haproxy = {
+      # HAProxy needs the initial certificates and Tailscale connectivity before
+      # it can serve requests to its backends.
+      after = acmeServices ++ [ "tailscaled-autoconnect.service" ];
+      wants = acmeServices ++ [ "tailscaled-autoconnect.service" ];
+    };
+    tmpfiles.rules = [
+      "d /var/lib/acme/.challenges 0750 acme acme - -"
+      "d /var/lib/acme/.challenges/.well-known 0750 acme acme - -"
+      "d /var/lib/acme/.challenges/.well-known/acme-challenge 0750 acme acme - -"
+    ];
+  };
+
+  users.users = {
+    haproxy.extraGroups = [ "acme" ];
+    nginx.extraGroups = [ "acme" ];
+  };
+}

systems/portal-1/haproxy/haproxy.cfg

@@ -22,8 +22,8 @@ defaults
 
 #Application Setup
 frontend ContentSwitching
-  bind *:80 v4v6
-  bind *:443 v4v6 ssl crt /var/lib/acme/audiobookshelf.tmmworkshop.com/full.pem crt /var/lib/acme/cache.tmmworkshop.com/full.pem crt /var/lib/acme/gems.tmmworkshop.com/full.pem crt /var/lib/acme/jellyfin.tmmworkshop.com/full.pem crt /var/lib/acme/share.tmmworkshop.com/full.pem crt /var/lib/acme/gitea.tmmworkshop.com/full.pem crt /var/lib/acme/www.norn-sight.com/full.pem
+  bind [::]:80 v4v6
+  bind [::]:443 v4v6 ssl crt /var/lib/acme/audiobookshelf.tmmworkshop.com/full.pem crt /var/lib/acme/cache.tmmworkshop.com/full.pem crt /var/lib/acme/gems.tmmworkshop.com/full.pem crt /var/lib/acme/jellyfin.tmmworkshop.com/full.pem crt /var/lib/acme/share.tmmworkshop.com/full.pem crt /var/lib/acme/gitea.tmmworkshop.com/full.pem crt /var/lib/acme/www.norn-sight.com/full.pem
   mode  http
 
   # ACME challenge routing (must be first)
@@ -110,6 +110,14 @@ frontend ContentSwitching
   use_backend gems  if host_gems
   use_backend norn_sight  if host_norn_sight
 
+# Proxy Gitea's built-in SSH server over Tailscale to Jeeves. Port 22 remains
+# available to Endlessh on Portal.
+frontend GiteaSSH
+  bind [::]:2223 v4v6
+  mode tcp
+  option tcplog
+  default_backend gitea_ssh
+
 # Stick-table only (no servers): tracks per-IP request rate to Gitea's compare
 # endpoint over a 5-minute window so the frontend can cap it at 1 per 5 min.
 backend st_compare
@@ -124,31 +132,35 @@ backend acme_challenge
 
 backend audiobookshelf_nodes
   mode http
-  server server 127.0.0.1:8000
+  server server jeeves:8000
 
 backend cache_nodes
   mode http
-  server server 127.0.0.1:5000
+  server server jeeves:5000
 
 backend jellyfin
   mode http
   option forwardfor
-  server jellyfin 127.0.0.1:8096
+  server jellyfin jeeves:8096
 
 backend share_nodes
   mode http
-  server server 127.0.0.1:8091
+  server server jeeves:8091
 
 backend gitea
   mode http
-  server server 127.0.0.1:6443
+  server server jeeves:6443
+
+backend gitea_ssh
+  mode tcp
+  server gitea jeeves:2223
 
 backend gems
   mode http
   option forwardfor
   timeout server 1h
-  server gems 127.0.0.1:8002
+  server gems jeeves:8002
 
 backend norn_sight
   mode http
-  server server 127.0.0.1:8001
+  server server jeeves:8001

systems/portal-1/monitoring.nix

@@ -0,0 +1,15 @@
+{ config, ... }:
+{
+  networking.firewall.interfaces.tailscale0.allowedTCPPorts = [
+    config.services.prometheus.exporters.node.port
+  ];
+
+  services.prometheus.exporters.node = {
+    enable = true;
+    enabledCollectors = [
+      "pressure"
+      "processes"
+      "systemd"
+    ];
+  };
+}

systems/rhapsody-in-green/agent_logger.nix

@@ -1,35 +0,0 @@
-{
-  pkgs,
-  inputs,
-  ...
-}:
-{
-  systemd.services.agent-logger = {
-    description = "Unified agent logger";
-    after = [ "local-fs.target" ];
-    wantedBy = [ "multi-user.target" ];
-
-    environment = {
-      AGENT_LOG_DB = "/var/lib/agent-logger/agent_log.sqlite";
-      HOME = "/home/richie";
-      PYTHONPATH = "${inputs.self}";
-    };
-
-    serviceConfig = {
-      Type = "simple";
-      User = "richie";
-      WorkingDirectory = "/home/richie";
-      ExecStart = "${pkgs.my_python}/bin/python -m python.agent_logger.main";
-      StateDirectory = "agent-logger";
-      Restart = "on-failure";
-      RestartSec = "5s";
-      StandardOutput = "journal";
-      StandardError = "journal";
-      NoNewPrivileges = true;
-      ProtectSystem = "strict";
-      ProtectHome = "read-only";
-      PrivateTmp = true;
-      ReadOnlyPaths = [ "${inputs.self}" ];
-    };
-  };
-}

systems/rhapsody-in-green/default.nix

@@ -8,10 +8,11 @@
     "${inputs.self}/common/optional/steam.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/systemd-boot.nix"
+    "${inputs.self}/common/optional/tailscale.nix"
+    "${inputs.self}/common/optional/x86-64-v3.nix"
     "${inputs.self}/common/optional/yubikey.nix"
-    "${inputs.self}/common/optional/zerotier.nix"
+    "${inputs.self}/common/optional/zfs"
     ./hardware.nix
-    ./open_webui.nix
     ./programs.nix
     ./qmk.nix
     ./syncthing.nix

systems/rhapsody-in-green/hardware.nix

@@ -34,14 +34,22 @@
       device = "root_pool/root";
       fsType = "zfs";
     };
+
     "/home" = {
       device = "root_pool/home";
       fsType = "zfs";
     };
+
+    "/nix/var/nix/builds" = {
+      device = "root_pool/nix_build";
+      fsType = "zfs";
+    };
+
     "/var" = {
       device = "root_pool/var";
       fsType = "zfs";
     };
+
     "/boot" = {
       device = "/dev/disk/by-uuid/12CE-A600";
       fsType = "vfat";

systems/rhapsody-in-green/open_webui.nix

@@ -1,13 +0,0 @@
-{
-  services.open-webui = {
-    enable = true;
-    host = "0.0.0.0";
-    environment = {
-      ANONYMIZED_TELEMETRY = "False";
-      DO_NOT_TRACK = "True";
-      SCARF_NO_ANALYTICS = "True";
-      OLLAMA_API_BASE_URL = "https://ollama.com";
-      WEBUI_AUTH = "False";
-    };
-  };
-}

tests/ebook_search/test_rerank.py

@@ -51,7 +51,7 @@ def test_config_defaults_enable_reranking(mocker: MockerFixture) -> None:
     config = load_rerank_config()
 
     assert config.enabled is True
-    assert config.base_url == "http://192.168.90.25:8001"
+    assert config.base_url == "http://bob:8001"
     assert config.model == "qwen3-reranker-06b"
     assert config.candidates == 24
     assert config.timeout_seconds == 30

tests/test_components.py

@@ -1,104 +0,0 @@
-"""test_components."""
-
-from pytest_mock import MockerFixture
-
-from python.system_tests.components import systemd_tests, zpool_tests
-from python.zfs import Zpool
-
-temp = "Every feature flags pool has all supported and requested features enabled.\n"
-
-SYSTEM_TESTS_COMPONENTS = "python.system_tests.components"
-
-
-def test_zpool_tests(mocker: MockerFixture) -> None:
-    """test_zpool_tests."""
-    mock_zpool = mocker.MagicMock(spec=Zpool)
-    mock_zpool.health = "ONLINE"
-    mock_zpool.capacity = 70
-    mock_zpool.name = "Main"
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=(temp, ""))
-    errors = zpool_tests(("Main",))
-    assert errors == []
-
-
-def test_zpool_tests_out_of_date(mocker: MockerFixture) -> None:
-    """test_zpool_tests_out_of_date."""
-    mock_zpool = mocker.MagicMock(spec=Zpool)
-    mock_zpool.health = "ONLINE"
-    mock_zpool.capacity = 70
-    mock_zpool.name = "Main"
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=("", ""))
-    errors = zpool_tests(("Main",))
-    assert errors == ["ZPool out of date run `sudo zpool upgrade -a`"]
-
-
-def test_zpool_tests_out_of_space(mocker: MockerFixture) -> None:
-    """test_zpool_tests_out_of_space."""
-    mock_zpool = mocker.MagicMock(spec=Zpool)
-    mock_zpool.health = "ONLINE"
-    mock_zpool.capacity = 100
-    mock_zpool.name = "Main"
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=(temp, ""))
-    errors = zpool_tests(("Main",))
-    assert errors == ["Main is low on space"]
-
-
-def test_zpool_tests_offline(mocker: MockerFixture) -> None:
-    """test_zpool_tests_offline."""
-    mock_zpool = mocker.MagicMock(spec=Zpool)
-    mock_zpool.health = "OFFLINE"
-    mock_zpool.capacity = 70
-    mock_zpool.name = "Main"
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=(temp, ""))
-    errors = zpool_tests(("Main",))
-    assert errors == ["Main is OFFLINE"]
-
-
-def test_systemd_tests(mocker: MockerFixture) -> None:
-    """test_systemd_tests."""
-    mocker.patch(
-        f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper",
-        side_effect=[
-            ("inactive\n", ""),
-            ("active\n", ""),
-        ],
-    )
-    errors = systemd_tests(("docker",))
-    assert errors == []
-    """test_systemd_tests."""
-
-
-def test_systemd_tests_multiple_negative_retries(mocker: MockerFixture) -> None:
-    """test_systemd_tests_fail."""
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=("active\n", ""))
-    errors = systemd_tests(("docker",), max_retries=-1, retry_delay_secs=-1)
-    assert errors == []
-
-
-def test_systemd_tests_multiple_pass(mocker: MockerFixture) -> None:
-    """test_systemd_tests_fail."""
-    mocker.patch(
-        f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper",
-        side_effect=[
-            ("inactive\n", ""),
-            ("activating\n", ""),
-            ("active\n", ""),
-        ],
-    )
-    errors = systemd_tests(
-        ("docker",),
-        retryable_statuses=("inactive\n", "activating\n"),
-        valid_statuses=("active\n",),
-    )
-    assert errors == []
-
-
-def test_systemd_tests_fail(mocker: MockerFixture) -> None:
-    """test_systemd_tests_fail."""
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=("inactive\n", ""))
-    errors = systemd_tests(("docker",), max_retries=5)
-    assert errors == ["docker is inactive"]