flake update

Failed assertions:
       - `services.gitea.settings.server.DOMAIN` was removed upstream and replaced by `services.gitea.settings.server.ROOT_URL`.

.github/workflows/build_systems.yml

@@ -8,8 +8,23 @@ on:
     - cron: "0 22 * * *"
 
 jobs:
+  prebuild-common:
+    name: prebuild-common-x86-64-v3
+    runs-on: nix-cache-builder
+    steps:
+      - uses: actions/checkout@v4
+      # portal-1 is the smallest system closure: 95% of its derivations are
+      # shared by all five systems, so it is a maintainable common cache seed.
+      # Keep going so one failing package does not stop unrelated cache entries
+      # from being built.
+      - name: Build common packages
+        run: nixos-rebuild build --keep-going --accept-flake-config --flake ./#portal-1
+      - name: Copy common packages to nix-cache
+        run: nix copy --accept-flake-config --to unix:///host-nix/var/nix/daemon-socket/socket .#nixosConfigurations.portal-1.config.system.build.toplevel
+
   build:
     name: build-${{ matrix.system }}
+    needs: prebuild-common
     runs-on: self-hosted
     strategy:
       matrix:
@@ -18,6 +33,7 @@ jobs:
           - "brain"
           - "jeeves"
           - "rhapsody-in-green"
+          - "portal-1"
     continue-on-error: true
     steps:
       - uses: actions/checkout@v4

.github/workflows/merge_flake_lock_update.yml

@@ -15,9 +15,8 @@ jobs:
     steps:
       - name: merge_flake_lock_update
         run: >-
-          nix develop .#devShells.x86_64-linux.default -c
           python -m python.gitea_flake_lock merge
           --repo "${{ github.repository }}"
         env:
-          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+          JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
           GITEA_URL: https://gitea.tmmworkshop.com

.github/workflows/pytest.yml

@@ -1,19 +0,0 @@
-name: pytest
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
-
-jobs:
-  pytest:
-    runs-on: self-hosted
-
-    steps:
-      - uses: actions/checkout@v4
-      - name: Run tests
-        run: nix develop .#devShells.x86_64-linux.default -c pytest tests

.github/workflows/update-flake-lock.yml

@@ -18,9 +18,8 @@ jobs:
         run: nix flake update
       - name: Create or update flake.lock PR
         env:
-          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+          JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
           GITEA_URL: https://gitea.tmmworkshop.com
         run: >-
-          nix develop .#devShells.x86_64-linux.default -c
           python -m python.gitea_flake_lock update
           --repo "${{ github.repository }}"

.sops.yaml

@@ -1,6 +1,9 @@
 # Generate AGE keys from SSH keys with:
 #   ssh-keygen -A
 #   nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
+# update keys after addin/removing a key
+#   nix-shell -p sops --run "sops updatekeys users/secrets.yaml" users/secrets.yaml
+
 keys:
   - &admin_richie age1u8zj599elqqvcmhxn8zuwrufsz8w8w366d3ayrljjejljt2q45kq8mxw9c # cspell:disable-line
 
@@ -8,6 +11,7 @@ keys:
   - &system_brain age1jhf7vm0005j60mjq63696frrmjhpy8kpc2d66mw044lqap5mjv4snmwvwm # cspell:disable-line
   - &system_jeeves age13lmqgc3jvkyah5e3vcwmj4s5wsc2akctcga0lpc0x8v8du3fxprqp4ldkv # cspell:disable-line
   - &system_rhapsody age1ufnewppysaq2wwcl4ugngjz8pfzc5a35yg7luq0qmuqvctajcycs5lf6k4 # cspell:disable-line
+  - &system_portal_1 age1vyav6kxtvt3z4vtnkkjj38eu8hlts5m7ygyckhskvalg2gpjk52su53d0a # cspell:disable-line
 
 creation_rules:
   - path_regex: users/secrets\.yaml$
@@ -18,3 +22,4 @@ creation_rules:
           - *system_brain
           - *system_jeeves
           - *system_rhapsody
+          - *system_portal_1

.vscode/settings.json

@@ -10,7 +10,6 @@
     "aiounifi",
     "alsa",
     "apiclient",
-    "apscheduler",
     "archlinux",
     "ashift",
     "asrouter",
@@ -337,8 +336,6 @@
     "yubioath",
     "yzhang",
     "zeroconf",
-    "zerotier",
-    "zerotierone",
     "zoxide",
     "zram",
     "zstd"

common/global/default.nix

@@ -17,16 +17,11 @@
     ./nix.nix
     ./programs.nix
     ./ssh.nix
-    ./snapshot_manager.nix
   ];
 
   boot = {
-    tmp.useTmpfs = true;
+    tmp.useTmpfs = lib.mkDefault true;
     kernelPackages = lib.mkDefault pkgs.linuxPackages_6_12;
-    zfs = {
-      package = lib.mkDefault pkgs.zfs_2_4;
-      forceImportRoot = lib.mkDefault false;
-    };
   };
 
   hardware.enableRedistributableFirmware = true;
@@ -42,9 +37,6 @@
     overlays = builtins.attrValues outputs.overlays;
     config = {
       allowUnfree = true;
-      permittedInsecurePackages = [
-        "openssl-1.1.1w" # This is for discord-canary
-      ];
     };
   };
 
@@ -53,16 +45,6 @@
 
     # firmware update
     fwupd.enable = true;
-
-    snapshot_manager = {
-      enable = lib.mkDefault true;
-      PYTHONPATH = "${inputs.self}/";
-    };
-
-    zfs = {
-      trim.enable = lib.mkDefault true;
-      autoScrub.enable = lib.mkDefault true;
-    };
   };
 
   powerManagement.powertop.enable = lib.mkDefault true;

common/global/nix.nix

@@ -31,6 +31,10 @@ in
         "flakes"
         "ca-derivations"
       ];
+      system-features = lib.mkAfter [
+        "gccarch-x86-64-v2"
+        "gccarch-x86-64-v3"
+      ];
       warn-dirty = false;
       flake-registry = ""; # disable global flake registries
       connect-timeout = 10;

common/global/programs.nix

@@ -2,6 +2,5 @@
 {
   environment.systemPackages = with pkgs; [
     git
-    my_python
   ];
 }

common/optional/desktop.nix

@@ -1,42 +0,0 @@
-{ pkgs, ... }:
-{
-  boot = {
-    kernelPackages = pkgs.linuxPackages_6_18;
-    zfs.package = pkgs.zfs_2_4;
-  };
-
-  hardware.bluetooth = {
-    enable = true;
-    powerOnBoot = true;
-  };
-
-  # rtkit is optional but recommended for pipewire
-  security.rtkit.enable = true;
-
-  services = {
-    displayManager.sddm = {
-      enable = true;
-      wayland.enable = true;
-    };
-
-    desktopManager.plasma6.enable = true;
-
-    xserver = {
-      enable = true;
-      xkb = {
-        layout = "us";
-        variant = "";
-      };
-    };
-
-    pulseaudio.enable = false;
-
-    pipewire = {
-      enable = true;
-      alsa.enable = true;
-      alsa.support32Bit = true;
-      pulse.enable = true;
-      wireplumber.enable = true;
-    };
-  };
-}

common/optional/desktop/default.nix

@@ -0,0 +1,17 @@
+{ pkgs, ... }:
+{
+  imports = [
+    ./kde.nix
+    ./pipewire.nix
+  ];
+
+  boot = {
+    kernelPackages = pkgs.linuxPackages_6_18;
+    zfs.package = pkgs.zfs_2_4;
+  };
+
+  hardware.bluetooth = {
+    enable = true;
+    powerOnBoot = true;
+  };
+}

common/optional/desktop/kde.nix

@@ -0,0 +1,45 @@
+{ pkgs, ... }:
+{
+  environment.plasma6.excludePackages = with pkgs.kdePackages; [
+    aurorae # theme
+    elisa # music player
+    kate # text editor
+    kconfig # config editor
+    khelpcenter # help center
+    kinfocenter # system info
+    konsole # terminal
+    krdp # remote desktop
+    ktexteditor # thing for kate
+    okular # pdf reader
+    plasma-keyboard # used by plasma-keyboard KCM
+    plasma-sdk # plasma development tools
+    plasma-workspace-wallpapers # wallpapers
+    qrca # qr code scanner
+    qtvirtualkeyboard # virtual keyboard
+    union # theme
+  ];
+
+  # disable KDE PIM (Personal Information Management) applications
+  programs.kde-pim.enable = false;
+
+  services = {
+    # removes screen reader and speech dispatcher from the system
+    orca.enable = false;
+    speechd.enable = false;
+
+    displayManager.sddm = {
+      enable = true;
+      wayland.enable = true;
+    };
+
+    desktopManager.plasma6.enable = true;
+
+    xserver = {
+      enable = true;
+      xkb = {
+        layout = "us";
+        variant = "";
+      };
+    };
+  };
+}

common/optional/desktop/pipewire.nix

@@ -0,0 +1,16 @@
+{
+  # rtkit is optional but recommended for pipewire
+  security.rtkit.enable = true;
+
+  services = {
+    pulseaudio.enable = false;
+
+    pipewire = {
+      enable = true;
+      alsa.enable = true;
+      alsa.support32Bit = true;
+      pulse.enable = true;
+      wireplumber.enable = true;
+    };
+  };
+}

common/optional/docker.nix

@@ -17,6 +17,7 @@
     logDriver = "local";
     storageDriver = "overlay2";
     daemon.settings = {
+      live-restore = false;
       experimental = true;
       exec-opts = [ "native.cgroupdriver=systemd" ];
       log-opts = {

common/optional/monitoring-agent.nix

@@ -5,7 +5,7 @@
   ...
 }:
 let
-  monitoringInterface = "ztwfunumly";
+  monitoringInterface = "tailscale0";
   nodeTextfileDir = "/var/lib/prometheus-node-exporter-textfile";
 
   mkProcessNameTemplate =

common/optional/signal-cli.nix

@@ -0,0 +1,46 @@
+{ pkgs, ... }:
+{
+  environment.systemPackages = [
+    pkgs.signal-cli
+  ];
+
+  users = {
+    groups.signal-cli = { };
+    users.signal-cli = {
+      isSystemUser = true;
+      group = "signal-cli";
+      home = "/var/lib/signal-cli";
+    };
+  };
+
+  systemd.tmpfiles.rules = [
+    "d /var/lib/signal-cli 0700 signal-cli signal-cli - -"
+    "Z /var/lib/signal-cli - signal-cli signal-cli - -"
+  ];
+
+  systemd.services.signal-cli = {
+    description = "Signal CLI JSON-RPC service";
+    after = [ "network-online.target" ];
+    wants = [ "network-online.target" ];
+    wantedBy = [ "multi-user.target" ];
+    unitConfig.RequiresMountsFor = [ "/var/lib/signal-cli" ];
+
+    serviceConfig = {
+      Type = "simple";
+      User = "signal-cli";
+      Group = "signal-cli";
+      ExecStart = "${pkgs.signal-cli}/bin/signal-cli --data-dir /var/lib/signal-cli daemon --socket /run/signal-cli/socket";
+      Restart = "on-failure";
+      RestartSec = "5s";
+      SuccessExitStatus = 143;
+      RuntimeDirectory = "signal-cli";
+      RuntimeDirectoryMode = "0750";
+      UMask = "0007";
+      NoNewPrivileges = true;
+      PrivateTmp = true;
+      ProtectHome = true;
+      ProtectSystem = "strict";
+      ReadWritePaths = [ "/var/lib/signal-cli" ];
+    };
+  };
+}

common/optional/steam.nix

@@ -1,7 +1,6 @@
 { pkgs, ... }:
 {
   environment.systemPackages = with pkgs; [
-    mangohud
     steam-run
   ];
   hardware.steam-hardware.enable = true;

common/optional/tailscale.nix

@@ -0,0 +1,37 @@
+{
+  config,
+  inputs,
+  ...
+}:
+{
+  nix.settings = {
+    trusted-substituters = [ "http://jeeves:5000" ];
+    substituters = [ "http://jeeves:5000/?priority=1&want-mass-query=true" ];
+    trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
+  };
+
+  services.tailscale = {
+    enable = true;
+    openFirewall = true;
+    authKeyFile = config.sops.secrets.tailscale_auth_key.path;
+
+    # OAuth client secrets create ephemeral nodes by default. NixOS machines
+    # are persistent and should enroll without interactive device approval.
+    authKeyParameters = {
+      ephemeral = false;
+      preauthorized = true;
+    };
+
+    extraUpFlags = [ "--advertise-tags=tag:nixos" ];
+  };
+
+  sops = {
+    age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
+
+    secrets.tailscale_auth_key = {
+      sopsFile = "${inputs.self}/users/secrets.yaml";
+      owner = "root";
+      mode = "0400";
+    };
+  };
+}

common/optional/x86-64-v3/default.nix

@@ -0,0 +1,15 @@
+{
+  nixpkgs = {
+    hostPlatform = {
+      system = "x86_64-linux";
+      gcc = {
+        arch = "x86-64-v3";
+        tune = "generic";
+      };
+    };
+
+    # These patches repair tests and bundled dependencies that are sensitive
+    # to the compiler flags used by the x86-64-v3 package set.
+    overlays = [ (import ./patches) ];
+  };
+}

common/optional/x86-64-v3/patches/abseil/bmi2-public-header.patch

@@ -0,0 +1,20 @@
+Subject: [PATCH] abseil: include BMI2 intrinsics through the public header
+
+GCC and Clang reject direct inclusion of bmi2intrin.h. Include immintrin.h
+instead so that the compiler supplies the required intrinsic setup when
+BMI2 is enabled, including builds targeting x86-64-v3.
+
+This patch is shared by the vendored Abseil copies in Electron, rusty_v8
+(Deno), and Signal's WebRTC build.
+
+--- a/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
++++ b/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
+@@ -226,7 +226,7 @@
+ #endif
+ 
+ #ifdef __BMI2__
+-#include <bmi2intrin.h>
++#include <immintrin.h>
+ #endif  // __BMI2__
+ 
+ namespace absl {

common/optional/x86-64-v3/patches/abseil/default.nix

@@ -0,0 +1,38 @@
+# Abseil accepted the upstream fix: https://github.com/abseil/abseil-cpp/pull/2071
+# Keep this workaround until Electron, Deno's rusty_v8, and Signal's WebRTC
+# update their bundled Abseil copies to include it.
+{ prev }:
+let
+  patchAbseilBmi2Include =
+    package:
+    package.overrideAttrs (old: {
+      # GCC and Clang require the public umbrella header for BMI2 intrinsics.
+      patches = (old.patches or [ ]) ++ [ ./bmi2-public-header.patch ];
+    });
+
+  electron43Unwrapped = patchAbseilBmi2Include prev.electron_43.unwrapped;
+  electron43 = prev.electron_43.override {
+    electron-unwrapped = electron43Unwrapped;
+  };
+
+  signalCallPackage =
+    path: args:
+    let
+      package = prev.callPackage path args;
+    in
+    if builtins.baseNameOf path == "webrtc.nix" then patchAbseilBmi2Include package else package;
+in
+prev.lib.optionalAttrs ((prev.stdenv.hostPlatform.gcc.arch or null) == "x86-64-v3") {
+  deno =
+    let
+      librusty_v8 = patchAbseilBmi2Include prev.deno.passthru.librusty_v8;
+    in
+    prev.deno.override { inherit librusty_v8; };
+
+  electron_43 = electron43;
+
+  signal-desktop = prev.signal-desktop.override {
+    electron_43 = electron43;
+    callPackage = signalCallPackage;
+  };
+}

common/optional/x86-64-v3/patches/afdko/check-overlap-font-content.patch

@@ -0,0 +1,19 @@
+Subject: [PATCH] Compare font content in the overlap-removal regression
+
+PFA_SKIP ends with a separator, producing an empty prefix that matches every
+line. Remove the empty prefix for this test so glyph differences are checked,
+and assert that tx exits successfully before inspecting the output.
+
+--- a/tests/tx_test.py
++++ b/tests/tx_test.py
+@@ -1064,6 +1064,8 @@
+     output_path = get_temp_file_path()
+     args = [TOOL, '-t1', '+V', '-o', output_path, input_path]
+-    subprocess.call(args)
+-    assert differ([expected_path, output_path, '-s', PFA_SKIP[0]])
++    subprocess.check_call(args)
++    # An empty skip prefix matches every line and hides font differences.
++    skip_headers = SPLIT_MARKER.join(filter(None, PFA_SKIP[0].split(SPLIT_MARKER)))
++    assert differ([expected_path, output_path, '-s', skip_headers])
+ 
+ 

common/optional/x86-64-v3/patches/afdko/default.nix

@@ -0,0 +1,10 @@
+{ afdko }:
+afdko.overridePythonAttrs (old: {
+  # FMA changes overlap-removal coordinates by 0.01 units on x86-64-v3.
+  # Separate multiply/add rounding reproduces the reference Type 1 font exactly.
+  env = (old.env or { }) // {
+    NIX_CFLAGS_COMPILE = (old.env.NIX_CFLAGS_COMPILE or "") + " -ffp-contract=off";
+  };
+
+  patches = (old.patches or [ ]) ++ [ ./check-overlap-font-content.patch ];
+})

common/optional/x86-64-v3/patches/default.nix

@@ -0,0 +1,16 @@
+_final: prev:
+(import ./abseil { inherit prev; })
+// {
+  gnutls = import ./gnutls { inherit (prev) gnutls; };
+  prometheus = import ./prometheus { inherit (prev) prometheus; };
+  zopfli = import ./zopfli { inherit (prev) zopfli; };
+
+  pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
+    (_pythonFinal: pythonPrev: {
+      afdko = import ./afdko { inherit (pythonPrev) afdko; };
+      psutil = import ./psutil { inherit (pythonPrev) psutil; };
+      pytest-xdist = import ./pytest-xdist { inherit (pythonPrev) pytest-xdist; };
+      scipy = import ./scipy { inherit (pythonPrev) scipy; };
+    })
+  ];
+}

common/optional/x86-64-v3/patches/gnutls/default.nix

@@ -0,0 +1,6 @@
+{ gnutls }:
+gnutls.overrideAttrs (old: {
+  # Keep the UDP handshake test enabled on loaded builders by waiting for
+  # the server to bind its socket. Kept as a standalone patch for upstream.
+  patches = (old.patches or [ ]) ++ [ ./udp-server-readiness.patch ];
+})

common/optional/x86-64-v3/patches/gnutls/udp-server-readiness.patch

@@ -0,0 +1,70 @@
+Subject: [PATCH] tests: wait for the UDP server socket before connecting
+
+A fixed four-second sleep does not guarantee that gnutls-serv has bound
+its UDP socket on a busy builder. Poll the local IPv4 UDP endpoint using
+the existing ss/netstat discovery, with the same retry budget as the TCP
+helper. Fail early if the server exits, and retain the original handshake
+checks in serv-udp.sh.
+
+Use flags common to ss and BSD/Linux netstat. Match the local endpoint
+and complete port number, excluding TCP, IPv6-only and peer endpoints.
+
+--- a/tests/scripts/common.sh
++++ b/tests/scripts/common.sh
+@@ -185,10 +185,55 @@
+ 	fi
+ }
+ 
++check_if_udp_port_bound() {
++	local PORT=$1
++	have_port_finder
++	# Use only -an, which is shared by ss and BSD/Linux netstat. UDP has
++	# no LISTEN state. Match the local IPv4 endpoint, not a peer port or
++	# a longer port number. serv-udp.sh connects to 127.0.0.1;
++	# listen_socket() in serv.c binds IPv4 separately and requests
++	# IPV6_V6ONLY=1 for its IPv6 socket.
++	$PFCMD -an | awk -v port="$PORT" '
++		$1 == "udp" || $1 == "udp4" {
++			# ss includes a state column; netstat does not.
++			address = ($2 == "UNCONN" || $2 == "ESTAB") ? $5 : $4
++			if (address ~ ("^[0-9.]+[.:]" port "$") ||
++			    address == "*." port)
++				found = 1
++		}
++		END { exit !found }
++	'
++}
++
+ wait_udp_server() {
+ 	local PID=$1
++	local ret
+ 	trap "test -n \"${PID}\" && kill ${PID};exit 1" 1 15 2
+-	sleep 4
++	local i=0
++	# Use the same retry budget as wait_for_port(), but also stop if the
++	# server exits before binding its socket.
++	while test $i -lt 90; do
++		if ! kill -0 "$PID" 2>/dev/null; then
++			fail "" "UDP server $PID exited before binding port $PORT"
++		fi
++		# Contain have_port_finder's exit so a skip also stops the server.
++		if (check_if_udp_port_bound "$PORT"); then
++			return 0
++		else
++			ret=$?
++			if test "$ret" = 77; then
++				kill "$PID" 2>/dev/null || :
++				wait "$PID" 2>/dev/null || :
++				exit 77
++			fi
++		fi
++		i=$((i + 1))
++		if test $i -lt 90; then
++			echo "try $i: waiting for UDP port $PORT"
++			sleep 2
++		fi
++	done
++	fail "$PID" "UDP server $PORT did not come up"
+ }
+ 
+ create_testdir() {

common/optional/x86-64-v3/patches/gnutls/verify-readiness.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""Exercise patched common.sh without building GnuTLS (Python standard library only).
+
+Usage: python3 verify-readiness.py /path/to/patched/tests/scripts/common.sh
+Set SHELL to test another shell, and NETSTAT to test a netstat outside PATH.
+"""
+
+# Use unittest so this upstream companion tool needs no pytest installation.
+# ruff: noqa: PT009
+
+import os
+import shutil
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+from pathlib import Path
+
+COMMON = str(Path(sys.argv.pop(1)).resolve())
+SHELL = os.environ.get("SHELL", "/bin/sh")
+
+
+class ReadinessTests(unittest.TestCase):
+    """Check endpoint parsing and the server startup lifecycle."""
+
+    def setUp(self) -> None:
+        """Create a socket-listing fixture for each check."""
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.root = Path(self.tmp.name)
+        self.fixture = self.root / "sockets"
+        self.fixture.write_text("")
+        self.finder = self.root / "port-finder"
+        self.finder.write_text('#!/bin/sh\ncat "$SOCKET_FIXTURE"\n')
+        self.finder.chmod(0o755)
+
+    def run_shell(self, body: str, **env: str) -> subprocess.CompletedProcess[str]:
+        """Source the actual helper and run a shell scenario."""
+        return subprocess.run(
+            [SHELL, "-c", '. "$COMMON"\n' + body],
+            env={
+                **os.environ,
+                "COMMON": COMMON,
+                "SOCKET_FIXTURE": str(self.fixture),
+                "PFCMD": str(self.finder),
+                "PORT": "12345",
+                **env,
+            },
+            capture_output=True,
+            text=True,
+            timeout=20,
+            check=False,
+        )
+
+    def test_socket_formats_and_false_matches(self) -> None:
+        """Accept IPv4 UDP local endpoints and reject unrelated sockets."""
+        cases = [
+            ("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*", True),
+            ("udp UNCONN 0 0 127.0.0.1:12345 0.0.0.0:*", True),
+            ("udp 0 0 0.0.0.0:12345 0.0.0.0:*", True),
+            ("udp4 0 0 *.12345 *.*", True),
+            ("udp 0 0 127.0.0.1.12345 *.*", True),
+            ("udp 0 0 *.12345 *.*", True),
+            ("udp UNCONN 0 0 0.0.0.0:123456 0.0.0.0:*", False),
+            ("udp 0 0 0.0.0.0:123456 0.0.0.0:*", False),
+            ("udp ESTAB 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
+            ("udp 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
+            ("tcp LISTEN 0 128 0.0.0.0:12345 0.0.0.0:*", False),
+            ("tcp 0 0 0.0.0.0:12345 0.0.0.0:* LISTEN", False),
+            ("udp UNCONN 0 0 [::]:12345 [::]:*", False),
+            ("udp UNCONN 0 0 *:12345 *:*", False),
+            ("udp6 0 0 :::12345 :::*", False),
+            ("udp6 0 0 *.12345 *.*", False),
+            ("", False),
+        ]
+        for row, ready in cases:
+            with self.subTest(row=row):
+                self.fixture.write_text(row + "\n")
+                result = self.run_shell('check_if_udp_port_bound "$PORT"')
+                self.assertEqual(result.returncode, 0 if ready else 1, result.stderr)
+
+    def test_exited_server_fails_immediately(self) -> None:
+        """Fail without sleeping when the server has already exited."""
+        result = self.run_shell(
+            'true &\npid=$!\nwait "$pid"\nsleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$pid"'
+        )
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("exited before binding", result.stderr)
+        self.assertNotIn("unexpected sleep", result.stderr)
+
+    def test_ready_socket_does_not_sleep(self) -> None:
+        """Check readiness before the first sleep."""
+        self.fixture.write_text("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*\n")
+        result = self.run_shell('sleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$$"')
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertNotIn("unexpected sleep", result.stderr)
+
+    def test_missing_port_finders_skip(self) -> None:
+        """Skip and stop the live server when no finder is available."""
+        # have_port_finder also tries these paths independently of PATH.
+        if any(os.access(f"{directory}/ss", os.X_OK) for directory in ("/sbin", "/usr/sbin", "/usr/local/sbin")):
+            self.skipTest("an absolute ss path cannot be hidden by this PATH-only fixture")
+        with subprocess.Popen(["sleep", "60"]) as server:
+            try:
+                result = self.run_shell(
+                    'unset PFCMD\nPATH=/nonexistent\nwait_udp_server "$SERVER_PID"',
+                    SERVER_PID=str(server.pid),
+                )
+                self.assertEqual(result.returncode, 77)
+                self.assertIn("neither ss nor netstat found", result.stderr)
+                server.wait(timeout=3)
+                self.assertLess(server.returncode, 0)
+            finally:
+                if server.poll() is None:
+                    server.kill()
+
+    def test_timeout_is_bounded_and_cleans_up(self) -> None:
+        """Stop polling after the retry budget and terminate the server."""
+        # Only accelerate the polling delay; keep a real live server process.
+        self.finder.write_text('#!/bin/sh\necho probe >&2\ncat "$SOCKET_FIXTURE"\n')
+        with subprocess.Popen(["sleep", "60"]) as server:
+            try:
+                result = self.run_shell(
+                    'sleep() { echo polling-sleep; }\nwait_udp_server "$SERVER_PID"',
+                    SERVER_PID=str(server.pid),
+                )
+                self.assertEqual(result.returncode, 1)
+                self.assertIn("did not come up", result.stderr)
+                self.assertEqual(result.stderr.count("probe\n"), 90)
+                self.assertEqual(result.stdout.count("polling-sleep"), 89)
+                server.wait(timeout=3)
+                self.assertLess(server.returncode, 0)
+            finally:
+                if server.poll() is None:
+                    server.kill()
+
+    def test_server_exits_while_waiting(self) -> None:
+        """Detect a startup failure that happens after polling begins."""
+        result = self.run_shell('sleep 1 &\npid=$!\nwait_udp_server "$pid"')
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("exited before binding", result.stderr)
+        self.assertIn("waiting for UDP port", result.stdout)
+
+    def test_real_socket_delayed_beyond_four_seconds(self) -> None:
+        """Wait for a real delayed bind with each installed port finder."""
+        finders = [shutil.which("ss"), os.environ.get("NETSTAT") or shutil.which("netstat")]
+        finders = [finder for finder in finders if finder]
+        if not finders:
+            self.skipTest("neither ss nor netstat available")
+        for finder in finders:
+            with self.subTest(finder=finder):
+                with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
+                    sock.bind(("127.0.0.1", 0))
+                    port = sock.getsockname()[1]
+                code = (
+                    "import socket,time,sys; time.sleep(6); "
+                    "s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); "
+                    "s.bind(('127.0.0.1',int(sys.argv[1]))); time.sleep(30)"
+                )
+                with subprocess.Popen([sys.executable, "-c", code, str(port)]) as server:
+                    try:
+                        started = time.monotonic()
+                        result = self.run_shell(
+                            'wait_udp_server "$SERVER_PID"',
+                            SERVER_PID=str(server.pid),
+                            PORT=str(port),
+                            PFCMD=finder,
+                        )
+                        self.assertEqual(result.returncode, 0, result.stderr)
+                        self.assertGreaterEqual(time.monotonic() - started, 6)
+                        self.assertIsNone(server.poll())
+                    finally:
+                        server.terminate()
+                        server.wait(timeout=3)
+
+
+if __name__ == "__main__":
+    unittest.main()

common/optional/x86-64-v3/patches/prometheus/complete-test-parsing.patch

@@ -0,0 +1,36 @@
+Subject: [PATCH] tests: finish parsing before inspecting editor state
+
+EditorState creation has a 20 ms parsing budget. A descheduled test can
+therefore observe an incomplete tree. Finish these small test documents
+without an interactive deadline and publish the result with a transaction.
+Keep the original completion and vector-matching assertions enabled.
+
+--- a/module/codemirror-promql/src/test/utils-test.ts
++++ b/module/codemirror-promql/src/test/utils-test.ts
+@@ -13,7 +13,7 @@
+ 
+ import { parser } from '@prometheus-io/lezer-promql';
+ import { EditorState } from '@codemirror/state';
+-import { LRLanguage } from '@codemirror/language';
++import { ensureSyntaxTree, LRLanguage } from '@codemirror/language';
+ import nock from 'nock';
+ import path from 'path';
+ import { fileURLToPath } from 'url';
+@@ -23,10 +23,16 @@
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
+ 
+ export function createEditorState(expr: string): EditorState {
+-  return EditorState.create({
++  const state = EditorState.create({
+     doc: expr,
+     extensions: lightPromQLSyntax,
+   });
++  // These tests need a complete tree, independent of the editor's time budget.
++  if (!ensureSyntaxTree(state, state.doc.length, Infinity)) {
++    throw new Error('Unable to parse the test expression');
++  }
++  // Publish the completed parse so syntaxTree(state) sees it too.
++  return state.update({}).state;
+ }
+ 
+ export function mockPrometheusServer(): void {

common/optional/x86-64-v3/patches/prometheus/default.nix

@@ -0,0 +1,17 @@
+{ prometheus }:
+prometheus.overrideAttrs (
+  old:
+  let
+    assets = old.passthru.assets.overrideAttrs (assetsOld: {
+      patches = (assetsOld.patches or [ ]) ++ [ ./complete-test-parsing.patch ];
+    });
+  in
+  {
+    postPatch = builtins.replaceStrings [ "${old.passthru.assets}" ] [ "${assets}" ] (
+      builtins.unsafeDiscardStringContext old.postPatch
+    );
+    passthru = old.passthru // {
+      inherit assets;
+    };
+  }
+)

common/optional/x86-64-v3/patches/psutil/default.nix

@@ -0,0 +1,4 @@
+{ psutil }:
+psutil.overridePythonAttrs (old: {
+  patches = (old.patches or [ ]) ++ [ ./heap-info-zero-mmap.patch ];
+})

common/optional/x86-64-v3/patches/psutil/heap-info-zero-mmap.patch

@@ -0,0 +1,17 @@
+Subject: [PATCH] Allow an empty mmap allocation total in heap_info
+
+The allocator may satisfy all live allocations from the heap. A zero
+mmap_used value is valid and depends on the worker allocation history.
+Keep the heap and platform checks without requiring an mmap allocation.
+
+--- a/tests/test_system.py
++++ b/tests/test_system.py
+@@ -267,6 +267,7 @@
+         if MACOS:
+             assert m.mmap_used == 0  # not supported
+         else:
+-            assert m.mmap_used > 0
++            # A process can have no live mmap-backed malloc allocations.
++            assert m.mmap_used >= 0
+         if WINDOWS:
+             assert m.heap_count >= 0

common/optional/x86-64-v3/patches/pytest-xdist/concurrent-worker-crashes.patch

@@ -0,0 +1,29 @@
+Subject: [PATCH] tests: count replacements when checking the worker restart limit
+
+With two workers, another in-flight test may crash after the fourth
+crash requests shutdown. Either four or five failed tests is valid.
+Require exactly three replacements and the failed-tests exit status,
+while preserving the queued-work and no-internal-error assertions.
+
+--- a/testing/acceptance_test.py
++++ b/testing/acceptance_test.py
+@@ -1011,9 +1011,18 @@
+                 "worker*crashed while running*",
+                 "worker*crashed while running*",
+                 "* xdist: maximum crashed workers reached: 3 *",
+-                "* 4 failed in *",
+             ]
+         )
++        # A second in-flight test may crash after shutdown is requested.
++        # The restart limit constrains replacements, not concurrent failures.
++        replacements = sum(
++            line.startswith("replacing crashed worker ") for line in res.stdout.lines
++        )
++        assert replacements == 3
++        failed = res.parseoutcomes()["failed"]
++        assert failed in (4, 5)
++        res.assert_outcomes(failed=failed)
++        assert res.ret == pytest.ExitCode.TESTS_FAILED
+         assert "INTERNALERROR" not in res.stdout.str()
+ 
+     def test_max_worker_restart_die(self, pytester: pytest.Pytester) -> None:

common/optional/x86-64-v3/patches/pytest-xdist/default.nix

@@ -0,0 +1,12 @@
+{ pytest-xdist }:
+pytest-xdist.overridePythonAttrs (old: {
+  patches = (old.patches or [ ]) ++ [
+    ./concurrent-worker-crashes.patch
+    ./worker-startup-timeout.patch
+  ];
+
+  # The suite exercises its own worker pools. Limit the outer suite to one worker.
+  preCheck = builtins.replaceStrings [ "--numprocesses=$NIX_BUILD_CORES" ] [ "--numprocesses=1" ] (
+    old.preCheck or ""
+  );
+})

common/optional/x86-64-v3/patches/pytest-xdist/worker-startup-timeout.patch

@@ -0,0 +1,19 @@
+Subject: [PATCH] tests: allow more time for remote worker events
+
+Worker startup can exceed ten seconds on heavily loaded builders. Allow
+the remote-test helper to wait up to sixty seconds for worker events.
+The wait still returns as soon as an event arrives and remains bounded.
+Production worker timeouts and test assertions are unchanged.
+
+--- a/testing/test_remote.py
++++ b/testing/test_remote.py
+@@ -17,7 +17,8 @@
+ from xdist.workermanage import WorkerController
+ 
+ 
+-WAIT_TIMEOUT = 10.0
++# Allow worker events extra time on heavily loaded builders.
++WAIT_TIMEOUT = 60.0
+ 
+ 
+ def check_marshallable(d: object) -> None:

common/optional/x86-64-v3/patches/scipy/default.nix

@@ -0,0 +1,5 @@
+{ scipy }:
+scipy.overridePythonAttrs (old: {
+  # Keep the STFT tests enabled with tolerances for x86-64-v3 rounding.
+  patches = (old.patches or [ ]) ++ [ ./stft-test-tolerances.patch ];
+})

common/optional/x86-64-v3/patches/scipy/stft-test-tolerances.patch

@@ -0,0 +1,51 @@
+Subject: [PATCH] signal: allow floating-point rounding in STFT tests
+
+Keep the STFT tests enabled for x86-64-v3 builds. Allow two float32
+epsilons of relative error when comparing inverse-STFT implementations;
+float64 and the existing i686 override remain unchanged. Allow one
+float64 epsilon of absolute error in all three scaling round trips,
+which otherwise require exact zeros (observed residual: 4e-17 for a
+signal with amplitude 2).
+
+Upstream issue: https://github.com/scipy/scipy/issues/25488
+
+--- a/scipy/signal/tests/_scipy_spectral_test_shim.py
++++ b/scipy/signal/tests/_scipy_spectral_test_shim.py
+@@ -294,7 +294,7 @@
+ 
+     # Adapted tolerances to account for resolution loss:
+     atol = np.finfo(x.dtype).resolution*2  # instead of default atol = 0
+-    rtol = 1e-7  # default for np.allclose()
++    rtol = max(1e-7, 2 * np.finfo(x.dtype).eps)
+ 
+     # Relax atol on 32-Bit platforms a bit to pass CI tests.
+     #  - Not clear why there are discrepancies (in the FFT maybe?)
+--- a/scipy/signal/tests/test_spectral.py
++++ b/scipy/signal/tests/test_spectral.py
+@@ -2044,7 +2044,7 @@
+ 
+         # Test round trip:
+         x1 = istft(Zs, boundary=True, scaling='spectrum')[1]
+-        assert_allclose(x1, x)
++        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
+ 
+         # For a Hann-windowed 256 sample length FFT, we expect a peak at
+         # frequency 64 (since it is 1/4 the length of X) with a height of 1
+@@ -2074,7 +2074,7 @@
+ 
+         # Test round trip:
+         x1 = istft(Zp, input_onesided=False, boundary=True, scaling='psd')[1]
+-        assert_allclose(x1, x)
++        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
+ 
+         # The power of the one-sided psd-scaled STFT can be determined
+         # analogously (note that the two sides are not of equal shape):
+@@ -2094,7 +2094,7 @@
+ 
+         # Test round trip:
+         x1 = istft(Zp0, input_onesided=True, boundary=True, scaling='psd')[1]
+-        assert_allclose(x1, x)
++        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
+ 
+ 
+ class TestSampledSpectralRepresentations:

common/optional/x86-64-v3/patches/zopfli/default.nix

@@ -0,0 +1,4 @@
+{ zopfli }:
+zopfli.overrideAttrs (old: {
+  patches = (old.patches or [ ]) ++ [ ./unaligned-match-loads.patch ];
+})

common/optional/x86-64-v3/patches/zopfli/unaligned-match-loads.patch

@@ -0,0 +1,32 @@
+Subject: [PATCH] Read unaligned match buffers with memcpy
+
+Byte buffers need not satisfy integer alignment or aliasing requirements.
+GCC 16 with x86-64-v3 vectorizes the cast loads using aligned AVX reads,
+causing a segmentation fault. Copy into local integers instead.
+
+--- a/src/zopfli/lz77.c
++++ b/src/zopfli/lz77.c
+@@ -302,13 +302,20 @@
+   if (sizeof(size_t) == 8) {
+     /* 8 checks at once per array bounds check (size_t is 64-bit). */
+-    while (scan < safe_end && *((size_t*)scan) == *((size_t*)match)) {
++    while (scan < safe_end) {
++      size_t scan_word, match_word;
++      memcpy(&scan_word, scan, sizeof(scan_word));
++      memcpy(&match_word, match, sizeof(match_word));
++      if (scan_word != match_word) break;
+       scan += 8;
+       match += 8;
+     }
+   } else if (sizeof(unsigned int) == 4) {
+     /* 4 checks at once per array bounds check (unsigned int is 32-bit). */
+-    while (scan < safe_end
+-        && *((unsigned int*)scan) == *((unsigned int*)match)) {
++    while (scan < safe_end) {
++      unsigned int scan_word, match_word;
++      memcpy(&scan_word, scan, sizeof(scan_word));
++      memcpy(&match_word, match, sizeof(match_word));
++      if (scan_word != match_word) break;
+       scan += 4;
+       match += 4;
+     }

common/optional/zerotier.nix

@@ -1,11 +0,0 @@
-{
-  services.zerotierone = {
-    enable = true;
-    joinNetworks = [ "e4da7455b2ae64ca" ];
-  };
-  nix.settings = {
-    trusted-substituters = [ "http://192.168.90.40:5000" ];
-    substituters = [ "http://192.168.90.40:5000/?priority=1&want-mass-query=true" ];
-    trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
-  };
-}

common/optional/zfs/default.nix

@@ -0,0 +1,26 @@
+{
+  inputs,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  imports = [ ./snapshot.nix ];
+
+  boot.zfs = {
+    package = lib.mkDefault pkgs.zfs_2_4;
+    forceImportRoot = lib.mkDefault false;
+  };
+
+  services = {
+    snapshot_manager = {
+      enable = lib.mkDefault true;
+      PYTHONPATH = "${inputs.self}/";
+    };
+
+    zfs = {
+      trim.enable = lib.mkDefault true;
+      autoScrub.enable = lib.mkDefault true;
+    };
+  };
+}

common/optional/zfs/snapshot.nix

@@ -1,4 +1,5 @@
 {
+  inputs,
   pkgs,
   lib,
   config,
@@ -6,6 +7,7 @@
 }:
 let
   cfg = config.services.snapshot_manager;
+  snapshotManagerPackages = ps: with ps; [ typer ];
 in
 {
   options = {
@@ -34,6 +36,41 @@ in
   };
 
   config = lib.mkIf cfg.enable {
+    nixpkgs.overlays = [
+      (final: _prev: {
+        snapshot_manager_python = final.python314.withPackages snapshotManagerPackages;
+        snapshot_manager_test_python = final.python314.withPackages (
+          ps:
+          snapshotManagerPackages ps
+          ++ (with ps; [
+            pyfakefs
+            pytest
+            pytest-asyncio
+            pytest-mock
+            pytest-xdist
+          ])
+        );
+        snapshot_manager_tests =
+          final.runCommand "snapshot-manager-tests"
+            {
+              nativeBuildInputs = [ final.snapshot_manager_test_python ];
+            }
+            ''
+              export HOME="$TMPDIR"
+              cd ${inputs.self}
+              pytest \
+                -o cache_dir="$TMPDIR/pytest-cache" \
+                tests/test_common.py \
+                tests/test_signal_alert.py \
+                tests/test_snapshot_manager.py \
+                tests/test_zfs.py
+              touch "$out"
+            '';
+      })
+    ];
+
+    system.checks = [ pkgs.snapshot_manager_tests ];
+
     systemd = {
       services.snapshot_manager = {
         description = "ZFS Snapshot Manager";
@@ -45,7 +82,7 @@ in
         };
         serviceConfig = {
           Type = "oneshot";
-          ExecStart = "${pkgs.my_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
+          ExecStart = "${pkgs.snapshot_manager_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
         }
         // lib.optionalAttrs (cfg.EnvironmentFile != null) {
           EnvironmentFile = cfg.EnvironmentFile;

common/optional/zfs/snapshot_config.toml

@@ -22,6 +22,12 @@ hourly = 0
 daily = 0
 monthly = 0
 
+["root_pool/nix_build"]
+15_min = 1
+hourly = 0
+daily = 0
+monthly = 0
+
 ["root_pool/var"]
 15_min = 8
 hourly = 24

flake.lock

@@ -1,25 +1,41 @@
 {
   "nodes": {
-    "firefox-addons": {
+    "disko": {
       "inputs": {
         "nixpkgs": [
           "nixpkgs"
         ]
       },
       "locked": {
-        "dir": "pkgs/firefox-addons",
+        "lastModified": 1789770686,
-        "lastModified": 1787025780,
+        "narHash": "sha256-uZkBR7yHdIKUFB5SZdfgh1qkGfI3XmYmI/lTiquxbck=",
-        "narHash": "sha256-NhyLP9G4DFOn/7aYr7K/D7hWrzEGr5EgUBV+lpdmJ24=",
+        "owner": "nix-community",
-        "owner": "rycee",
+        "repo": "disko",
-        "repo": "nur-expressions",
+        "rev": "725ea35e410ad83be4931d1bff7e090eacaf3563",
-        "rev": "5ad360b6d3cb0aa1b61f9cb27fef113ca9117c37",
+        "type": "github"
-        "type": "gitlab"
       },
       "original": {
-        "dir": "pkgs/firefox-addons",
+        "owner": "nix-community",
-        "owner": "rycee",
+        "repo": "disko",
-        "repo": "nur-expressions",
+        "type": "github"
-        "type": "gitlab"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
       }
     },
     "home-manager": {
@@ -29,11 +45,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1786999651,
+        "lastModified": 1791293500,
-        "narHash": "sha256-MTGMFlLDTklsXhCp4r5GXB4VAVadPdalXLvUjd/K7h0=",
+        "narHash": "sha256-3hwJd2/vG/VYV1P/g7oDgdbNsWODhN5F//Od+hPMm2k=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "353742587cbaf079b3caee743115d037bc51fea6",
+        "rev": "0e11b7bcba1d74ef8ea10bfdfb1e1a5811a21087",
         "type": "github"
       },
       "original": {
@@ -42,16 +58,38 @@
         "type": "github"
       }
     },
+    "jellyswarrm": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-overlay": "rust-overlay"
+      },
+      "locked": {
+        "lastModified": 1791127101,
+        "narHash": "sha256-SmrBBhcv9iNHA2vlafi7Zpe8bkcsMjVNSmCCVf5/jrQ=",
+        "owner": "LLukas22",
+        "repo": "Jellyswarrm",
+        "rev": "0b4301c6c74efff9c3634296913032579c98faf6",
+        "type": "github"
+      },
+      "original": {
+        "owner": "LLukas22",
+        "repo": "Jellyswarrm",
+        "type": "github"
+      }
+    },
     "nixos-hardware": {
       "inputs": {
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1786867632,
+        "lastModified": 1791128201,
-        "narHash": "sha256-ez+ubZlA1RtdjCB18a6zJ9M4u8qoPDy08EcnsW5M3Xw=",
+        "narHash": "sha256-rlsYY/Dg/LhbIj51Aa6TLvRpZeb3KZgUTd7llQ5XcJE=",
         "owner": "nixos",
         "repo": "nixos-hardware",
-        "rev": "ff17823245ab9ff7bcae6acf950bd89cba82c38c",
+        "rev": "31cc5f4d9b9ba601071e8b8504601b9b176e2756",
         "type": "github"
       },
       "original": {
@@ -63,11 +101,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1767892417,
+        "lastModified": 1789546076,
-        "narHash": "sha256-8bW3q88CEg2u4hSP66Vf4lpbLonHz7hqDNBMcCY7E9U=",
+        "narHash": "sha256-vWkSk5bbfTqdtMoSgD9FshACO8JCvXTFi+3cqEp0mH0=",
-        "rev": "3497aa5c9457a9d88d71fa93a4a8368816fbeeba",
+        "rev": "b1b875982b17dabde9b4a37f3e229e74913e6db3",
         "type": "tarball",
-        "url": "https://releases.nixos.org/nixos/unstable/nixos-26.05pre924538.3497aa5c9457/nixexprs.tar.xz"
+        "url": "https://releases.nixos.org/nixos/unstable/nixos-26.11pre1074753.b1b875982b17/nixexprs.tar.xz"
       },
       "original": {
         "type": "tarball",
@@ -76,11 +114,11 @@
     },
     "nixpkgs-master": {
       "locked": {
-        "lastModified": 1787081018,
+        "lastModified": 1791296764,
-        "narHash": "sha256-K0uwZBtZsbBigHAMQW7YWti3gPe6a5ct5bcOw5F+Q9Y=",
+        "narHash": "sha256-N2b6Uxdr1MFhscEPqDyntTTtGZASgq3XYE0ujRzfeNI=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "cacac5ac351a010599d9f9d106acfed25a8e4c77",
+        "rev": "a15afac8691bd24f746255ffbd76306693e4cf7a",
         "type": "github"
       },
       "original": {
@@ -108,11 +146,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1787001381,
+        "lastModified": 1791260994,
-        "narHash": "sha256-Ue1Yo8gfHdD4TMtNewhA4tkSYeFqXThju0nCyJc3ALo=",
+        "narHash": "sha256-Miqqk/ammqnTUxaoCyvtwoPeLbI1ksFyLxi/CazZpWY=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "ec2d622de0773551768cf98f3fc50cbcc003b9c5",
+        "rev": "151fa4e8ddfdd8dd25d945ad94ed54a13de9f6e4",
         "type": "github"
       },
       "original": {
@@ -124,14 +162,36 @@
     },
     "root": {
       "inputs": {
-        "firefox-addons": "firefox-addons",
+        "disko": "disko",
         "home-manager": "home-manager",
+        "jellyswarrm": "jellyswarrm",
         "nixos-hardware": "nixos-hardware",
         "nixpkgs": "nixpkgs_2",
         "nixpkgs-master": "nixpkgs-master",
         "nixpkgs-stable": "nixpkgs-stable",
         "sops-nix": "sops-nix",
-        "systems": "systems"
+        "systems": "systems_2"
+      }
+    },
+    "rust-overlay": {
+      "inputs": {
+        "nixpkgs": [
+          "jellyswarrm",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1779419951,
+        "narHash": "sha256-dMX0PUslUHPajP6o8FEoRdFv9afq/dec4POR0vVfjK4=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "5b5c521d6cae9ef4aa32f888eb2c0ce595c9be52",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
       }
     },
     "sops-nix": {
@@ -141,11 +201,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1786629091,
+        "lastModified": 1791103873,
-        "narHash": "sha256-gkig4nPi1CWc4Z50GBsjE4ygSE7hMpl/TwID2an2Cck=",
+        "narHash": "sha256-nFxM+pKoZ8LJAEnUXARyCaOAloWgaW9kZQOSjWzKcTE=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "a8627b21b9107c5711c96b84f32a9a4b3d45295f",
+        "rev": "dcd241ba97088c22569d1573286e1b9daad340c0",
         "type": "github"
       },
       "original": {
@@ -155,6 +215,21 @@
       }
     },
     "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
       "locked": {
         "lastModified": 1689347949,
         "narHash": "sha256-12tWmuL2zgBgZkdoB6qXZsgJEH9LR3oUgpaQq2RbI80=",

flake.nix

@@ -21,18 +21,23 @@
 
     nixos-hardware.url = "github:nixos/nixos-hardware/master";
 
+    jellyswarrm = {
+      url = "github:LLukas22/Jellyswarrm";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
     home-manager = {
       url = "github:nix-community/home-manager";
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
-    firefox-addons = {
+    sops-nix = {
-      url = "gitlab:rycee/nur-expressions?dir=pkgs/firefox-addons";
+      url = "github:Mic92/sops-nix";
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
-    sops-nix = {
+    disko = {
-      url = "github:Mic92/sops-nix";
+      url = "github:nix-community/disko";
       inputs.nixpkgs.follows = "nixpkgs";
     };
   };

overlays/default.nix

@@ -1,13 +1,13 @@
 { inputs, ... }:
 {
-  # When applied, the stable nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.stable'
+  # Additional package sets are accessible through `pkgs.<name>`.
   stable = final: _prev: {
     stable = import inputs.nixpkgs-stable {
       system = final.stdenv.hostPlatform.system;
       config.allowUnfree = true;
     };
   };
-  # When applied, the master nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.master'
+
   master = final: _prev: {
     master = import inputs.nixpkgs-master {
       system = final.stdenv.hostPlatform.system;
@@ -15,35 +15,12 @@
     };
   };
 
-  python-env = final: _prev: {
+  # Baseline x86-64 (v1) packages for prebuilt applications that should not
-    my_python = final.python314.withPackages (
+  # inherit an x86-64-v3 host platform.
-      ps: with ps; [
+  x86-v1 = final: _prev: {
-        alembic
+    x86-v1 = import inputs.nixpkgs {
-        apprise
+      system = final.stdenv.hostPlatform.system;
-        apscheduler
+      config.allowUnfree = true;
-        fastapi
+    };
-        fastapi-cli
-        httpx
-        jinja2
-        mypy
-        pgvector
-        psycopg
-        pydantic
-        pyfakefs
-        pytest
-        pytest-cov
-        pytest-mock
-        pytest-xdist
-        python-multipart
-        pydantic-settings
-        ruff
-        sqlalchemy
-        tenacity
-        tinytuya
-        typer
-        uvicorn
-        websockets
-      ]
-    );
   };
 }

pyproject.toml

@@ -6,51 +6,6 @@ authors = [{ name = "Richie Cahill", email = "richie@tmmworkshop.com" }]
 requires-python = "~=3.14.0"
 readme = "README.md"
 license = "MIT"
-# these dependencies are a best effort and aren't guaranteed to work
-# for up-to-date dependencies, see overlays/default.nix
-dependencies = [
-    "alembic",
-    "apprise",
-    "apscheduler",
-    "beautifulsoup4",
-    "bm25s",
-    "ebooklib",
-    "fastapi",
-    "fastapi-cli",
-    "httpx",
-    "jinja2",
-    "pgvector",
-    "polars",
-    "psycopg[binary]",
-    "pydantic",
-    "pydantic-settings",
-    "python-multipart",
-    "sqlalchemy[asyncio]",
-    "tenacity",
-    "tiktoken",
-    "tinytuya",
-    "typer",
-    "uvicorn",
-    "websockets",
-    "yake",
-]
-
-[project.scripts]
-database = "python.database_cli:app"
-whisper-transcribe = "python.tools.whisper.transcribe:main"
-
-[dependency-groups]
-dev = [
-    "aiosqlite",
-    "mypy",
-    "pyfakefs",
-    "pytest-asyncio",
-    "pytest-cov",
-    "pytest-mock",
-    "pytest-xdist",
-    "pytest",
-    "ruff",
-]
 
 [tool.ruff]
 

python/ebook_search/config.py

@@ -37,7 +37,7 @@ class RerankConfig(BaseSettings):
     model_config = SettingsConfigDict(env_prefix="EBOOK_SEARCH_RERANK_", frozen=True, protected_namespaces=())
 
     enabled: bool = True
-    base_url: str = "http://192.168.90.25:8001"
+    base_url: str = "http://bob:8001"
     model: str = "qwen3-reranker-06b"
     candidates: int = 24
     timeout_seconds: float = 30.0
@@ -67,7 +67,7 @@ class EbookSearchConfig(BaseSettings):
     )
     chat_model: str = "deepseek-v4-flash"
     answer_enabled: bool = True
-    embedding_base_url: str = "http://192.168.90.25:8000/v1"
+    embedding_base_url: str = "http://bob:8000/v1"
     embedding_api_key: str = "not-needed"
     embedding_model: str = "qwen3-embedding-0.6b"
     embedding_batch_size: int = 32

python/ebook_search/docker/README.md

@@ -1,77 +0,0 @@
-# Ebook Search Docker
-
-Run the EPUB search app against the existing Postgres database on `jeeves`:
-
-```sh
-python -m python.ebook_search.docker.containers start --library-path /path/to/epubs --build
-```
-
-All ebook-search Docker files live in this directory:
-
-- `Dockerfile` — multi-stage: `test` (runs pytest) and `runtime` (default target, the app image)
-- `docker-compose.yml`
-- `containers.py` — Typer lifecycle CLI
-- `pyproject.toml` / `uv.lock` — the container's uv-locked dependencies
-
-The app listens on `http://localhost:8070`.
-
-Useful lifecycle commands:
-
-```sh
-python -m python.ebook_search.docker.containers build
-python -m python.ebook_search.docker.containers start --library-path /path/to/epubs
-python -m python.ebook_search.docker.containers test
-python -m python.ebook_search.docker.containers logs
-python -m python.ebook_search.docker.containers ps
-python -m python.ebook_search.docker.containers stop
-```
-
-Direct compose usage from the repo root:
-
-```sh
-docker compose -f python/ebook_search/docker/docker-compose.yml ps
-```
-
-## Dependencies
-
-The image builds its environment with uv from `pyproject.toml` + `uv.lock` in this
-directory — this is the source of truth for the container's dependencies. To add or
-update a dependency, edit `pyproject.toml` here and regenerate the lock (uv is
-available in the `ebook-search` dev shell):
-
-```sh
-nix develop .#ebook-search -c uv lock --project python/ebook_search/docker
-```
-
-## Tests
-
-The main pytest suite excludes `tests/ebook_search` (its dependencies are no longer
-in the nix dev shell). The `test ebook search` CI workflow runs them in a uv env
-built from the lockfile in this directory — same commands work locally from the
-repo root (the `--override-ini` drops the main suite's ignore):
-
-```sh
-uv sync --locked --project python/ebook_search/docker
-uv run --project python/ebook_search/docker --no-sync pytest tests/ebook_search --override-ini addopts="-n auto -ra"
-```
-
-They can also run inside the Docker `test` image, which validates the image itself:
-
-```sh
-python -m python.ebook_search.docker.containers test
-```
-
-or the raw docker equivalent:
-
-```sh
-docker build --file python/ebook_search/docker/Dockerfile --target test --tag ebook-search:test .
-docker run --rm ebook-search:test
-```
-
-## Configuration
-
-The compose service loads the repo root `.env` into the container via `env_file`.
-
-Mount your EPUB directory by setting `EBOOK_LIBRARY_HOST_PATH` in an env file or on the command line. The container sees it as `/library`, and `EBOOK_SEARCH_LIBRARY_PATHS` is set to `/library` inside the container.
-
-Database connection settings are controlled by `RICHIE_DB`, `RICHIE_HOST`, `RICHIE_PORT`, `RICHIE_USER`, and `RICHIE_PASSWORD`. The default host is `jeeves`.

python/ebook_search/docker/docker-compose.yml

@@ -9,8 +9,6 @@ services:
     restart: unless-stopped
     ports:
       - "${EBOOK_SEARCH_PORT:-8070}:8070"
-    extra_hosts:
-      - "jeeves:192.168.90.40"
     env_file:
       - ../../../.env
     environment:

python/gems/README.md

@@ -1,54 +0,0 @@
-# Gems
-
-Gems is a server-rendered, turn-based resource-engine game for one to four human or AI players. It uses FastAPI,
-Jinja, HTMX, server-sent events, and SQLite.
-
-The application deliberately contains no playable card deck, patron/governor set, objective set, official artwork,
-or copied rulebook text. A room host must upload a content pack they are entitled to use before starting a game.
-
-## Run locally
-
-```shell
-uv run gems --host 127.0.0.1 --port 8082
-```
-
-The default database and installation key are created under `.gems/`. The following environment variables override
-runtime behavior:
-
-- `GEMS_DATABASE_PATH`
-- `GEMS_KEY_PATH`
-- `GEMS_PUBLIC_ORIGIN`
-- `GEMS_SECURE_COOKIES`
-- `GEMS_HOST`
-- `GEMS_PORT`
-
-## Content packs
-
-The current schema is available from a running server at `/schemas/content-pack-v1.json`. A pack defines exactly
-five normal resources, one wild resource, cards, and optional patrons, objectives, and outposts. `patrons` is the
-canonical field name; `governors` is accepted as an input alias.
-
-Cards may use only the built-in, bounded effect vocabulary:
-
-- `none`
-- `virtual_wild`
-- `copy_bonus`
-- `copy_and_claim`
-- `multi_bonus`
-- `claim_free`
-- an optional discard-cards alternate cost
-
-Unknown fields, resource references, executable expressions, HTML, artwork URLs, and files larger than 512 KiB are
-rejected. The normalized pack is private to its room and becomes immutable when play starts.
-
-## Neutral module mapping
-
-Gems calls the four optional mechanics Objectives, Outposts, Eastern Decks, and Fortifications. Lobby presets combine
-these mechanics into the familiar base, objective-race, objective-plus-outpost, eastern-plus-fortification, and
-all-module configurations. Component identities and values always come from the uploaded pack.
-
-## Jeeves
-
-The NixOS module runs one Uvicorn worker on `127.0.0.1:8002`, stores state in
-`/zfs/media/services/gems`, and publishes it through HAProxy at `https://gems.tmmworkshop.com`. The DNS record must
-point to Jeeves before ACME can issue the certificate.

python/gitea_flake_lock.py

@@ -14,7 +14,7 @@ DEFAULT_BASE_BRANCH = "main"
 DEFAULT_BRANCH = "automation/update-flake-lock"
 DEFAULT_GITEA_URL = "https://gitea.tmmworkshop.com"
 PR_LABELS = ["dependencies", "automated", "flake_lock_update"]
-PR_CHECK_WORKFLOWS = ["build_systems.yml", "treefmt.yml", "pytest.yml"]
+PR_CHECK_WORKFLOWS = ["build_systems.yml", "treefmt.yml"]
 PR_TITLE = "Update flake.lock"
 PR_BODY = "Automated flake.lock update."
 
@@ -84,13 +84,13 @@ def push_branch(*, branch: str) -> None:
     run_cmd(["git", "push", "origin", f"HEAD:{branch}", "--force"])
 
 
-def _required_gitea_token() -> str:
+def _required_jeeves_bot_token() -> str:
     """Read the required Gitea token from the environment."""
-    token = getenv("GITEA_TOKEN")
+    token = getenv("JEEVES_BOT_TOKEN")
     if token:
         return token
 
-    msg = "GITEA_TOKEN environment variable is required"
+    msg = "JEEVES_BOT_TOKEN environment variable is required"
     raise RuntimeError(msg)
 
 
@@ -111,7 +111,7 @@ def update(
     owner, repo_name = split_repo_name(repo)
     with GiteaClient(
         base_url=getenv("GITEA_URL", DEFAULT_GITEA_URL),
-        token=_required_gitea_token(),
+        token=_required_jeeves_bot_token(),
     ) as client:
         pull_request = ensure_flake_lock_pull_request(
             client,
@@ -134,7 +134,7 @@ def merge(
     owner, repo_name = split_repo_name(repo)
     with GiteaClient(
         base_url=getenv("GITEA_URL", DEFAULT_GITEA_URL),
-        token=_required_gitea_token(),
+        token=_required_jeeves_bot_token(),
     ) as client:
         pull_request = find_flake_lock_pull_request(client, owner=owner, repo=repo_name)
         if not pull_request:

python/installer/__main__.py

@@ -176,6 +176,21 @@ def create_zfs_datasets() -> None:
     run_output(("zfs", "create", "root_pool/home"))
     run_output(("zfs", "create", "-o", "reservation=1G", "root_pool/var"))
     run_output(("zfs", "create", "-o", "compression=zstd-9", "-o", "reservation=10G", "root_pool/nix"))
+    run_output(
+        (
+            "zfs",
+            "create",
+            "-o",
+            "sync=disabled",
+            "-o",
+            "redundant_metadata=some",
+            "-o",
+            "normalization=none",
+            "-o",
+            "utf8only=off",
+            "root_pool/nix_build",
+        )
+    )
     datasets = run_output(("zfs", "list", "-o", "name"))
 
     expected_datasets = {
@@ -183,6 +198,7 @@ def create_zfs_datasets() -> None:
         "root_pool/home",
         "root_pool/var",
         "root_pool/nix",
+        "root_pool/nix_build",
     }
     missing_datasets = expected_datasets.difference(datasets.splitlines())
     if missing_datasets:
@@ -248,6 +264,7 @@ def create_nix_hardware_file(mnt_dir: str, disks: Sequence[str], encrypt: str |
         '    "/home" = {\n      device = "root_pool/home";\n      fsType = "zfs";\n    };\n\n'
         '    "/var" = {\n      device = "root_pool/var";\n      fsType = "zfs";\n    };\n\n'
         '    "/nix" = {\n      device = "root_pool/nix";\n      fsType = "zfs";\n    };\n\n'
+        '    "/nix/var/nix/builds" = {\n      device = "root_pool/nix_build";\n      fsType = "zfs";\n    };\n\n'
         '    "/boot" = {\n'
         f'      device = "/dev/disk/by-uuid/{get_boot_drive_id(disks[0])}";\n'
         '      fsType = "vfat";\n      options = [\n        "fmask=0077"\n'
@@ -270,6 +287,7 @@ def install_nixos(mnt_dir: str, disks: Sequence[str], encrypt: str | None) -> No
     run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/home", f"{mnt_dir}/home"))
     run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/var", f"{mnt_dir}/var"))
     run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/nix", f"{mnt_dir}/nix"))
+    run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/nix_build", f"{mnt_dir}/nix/var/nix/builds"))
 
     for disk in disks:
         run_output(("mkfs.vfat", "-n", "EFI", f"{disk}-part1"))

python/signal_alert.py

@@ -2,29 +2,63 @@
 
 from __future__ import annotations
 
+import json
 import logging
-from os import getenv
+import socket
-
-from apprise import Apprise
 
 logger = logging.getLogger(__name__)
 
 
-def signal_alert(body: str, title: str = "") -> None:
+class SignalRPCError(RuntimeError):
-    """Send a signal alert.
+    """signal-cli returned an error or closed without responding."""
+
+
+def signal_alert(body: str, from_phone: str, to_phone: str, *, timeout: float = 4.0) -> None:
+    """Send a Signal alert through the local signal-cli daemon.
 
     Args:
-        body (str): The body of the alert.
+        body: The body of the alert.
-        title (str, optional): The title of the alert. Defaults to "".
+        from_phone: The Signal account sending the alert.
+        to_phone: The Signal account receiving the alert.
+        timeout: Seconds to wait on each socket operation.
+
+    Raises:
+        SignalRPCError: If signal-cli returns an error or closes the
+            connection before responding.
+        OSError: If the socket is unreachable or an operation times out.
     """
-    apprise_client = Apprise()
+    signal_rpc_id = "signal-alert"
+    request = {
+        "jsonrpc": "2.0",
+        "method": "send",
+        "params": {
+            "account": from_phone,
+            "recipient": [to_phone],
+            "message": body,
+        },
+        "id": signal_rpc_id,
+    }
 
-    from_phone = getenv("SIGNAL_ALERT_FROM_PHONE")
+    try:
-    to_phone = getenv("SIGNAL_ALERT_TO_PHONE")
+        with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client:
-    if not from_phone or not to_phone:
+            client.settimeout(timeout)
-        logger.info("SIGNAL_ALERT_FROM_PHONE or SIGNAL_ALERT_TO_PHONE not set")
+            client.connect("/run/signal-cli/socket")
-        return
+            client.sendall(json.dumps(request).encode() + b"\n")
 
-    apprise_client.add(f"signal://localhost:8989/{from_phone}/{to_phone}")
+            buffer = b""
-
+            while chunk := client.recv(65536):
-    apprise_client.notify(title=title, body=body)
+                buffer += chunk
+                while b"\n" in buffer:
+                    line, buffer = buffer.split(b"\n", maxsplit=1)
+                    response = json.loads(line)
+                    if response.get("id") != signal_rpc_id:
+                        continue
+                    if rpc_error := response.get("error"):
+                        error = f"Signal RPC returned an error: {rpc_error}"
+                        raise SignalRPCError(error)
+                    return
+            error = "Signal RPC socket closed before returning a response"
+            raise SignalRPCError(error)
+    except (OSError, json.JSONDecodeError) as exc:
+        error = f"Signal RPC failed: {exc!r}"
+        raise SignalRPCError(error) from exc

python/system_tests/__init__.py

@@ -1 +0,0 @@
-"""system_tests."""

python/system_tests/components.py

@@ -1,99 +0,0 @@
-"""Validate Jeeves."""
-
-from __future__ import annotations
-
-import logging
-from copy import copy
-from re import search
-from time import sleep
-from typing import TYPE_CHECKING
-
-from python.common import bash_wrapper
-from python.zfs import Zpool
-
-if TYPE_CHECKING:
-    from collections.abc import Sequence
-
-logger = logging.getLogger(__name__)
-
-
-def zpool_tests(pool_names: Sequence[str], zpool_capacity_threshold: int = 90) -> list[str] | None:
-    """Check the zpool health and capacity.
-
-    Args:
-        pool_names (Sequence[str]): A list of pool names to test.
-        zpool_capacity_threshold (int, optional): The threshold for the zpool capacity. Defaults to 90.
-
-    Returns:
-        list[str] | None: A list of errors if any.
-    """
-    logger.info("Testing zpool")
-
-    errors: list[str] = []
-    for pool_name in pool_names:
-        pool = Zpool(pool_name)
-        if pool.health != "ONLINE":
-            errors.append(f"{pool.name} is {pool.health}")
-        if pool.capacity >= zpool_capacity_threshold:
-            errors.append(f"{pool.name} is low on space")
-
-    upgrade_status, _ = bash_wrapper("zpool upgrade")
-    if not search(r"Every feature flags pool has all supported and requested features enabled.", upgrade_status):
-        errors.append("ZPool out of date run `sudo zpool upgrade -a`")
-
-    return errors
-
-
-def systemd_tests(
-    service_names: Sequence[str],
-    max_retries: int = 30,
-    retry_delay_secs: int = 1,
-    retryable_statuses: Sequence[str] | None = None,
-    valid_statuses: Sequence[str] | None = None,
-) -> list[str] | None:
-    """Tests a systemd services.
-
-    Args:
-        service_names (Sequence[str]): A list of service names to test.
-        max_retries (int, optional): The maximum number of retries. Defaults to 30.
-            minimum value is 1.
-        retry_delay_secs (int, optional): The delay between retries in seconds. Defaults to 1.
-            minimum value is 1.
-        retryable_statuses (Sequence[str] | None, optional): A list of retryable statuses. Defaults to None.
-        valid_statuses (Sequence[str] | None, optional): A list of valid statuses. Defaults to None.
-
-    Returns:
-        list[str] | None: A list of errors if any.
-    """
-    logger.info("Testing systemd service")
-
-    max_retries = max(max_retries, 1)
-    retry_delay_secs = max(retry_delay_secs, 1)
-    last_try = max_retries - 1
-
-    if retryable_statuses is None:
-        retryable_statuses = ("inactive\n", "activating\n")
-
-    if valid_statuses is None:
-        valid_statuses = ("active\n",)
-
-    service_names_set = set(service_names)
-
-    errors: set[str] = set()
-    for retry in range(max_retries):
-        if not service_names_set:
-            break
-        logger.info(f"Testing systemd service in {retry + 1} of {max_retries}")
-        service_names_to_test = copy(service_names_set)
-        for service_name in service_names_to_test:
-            service_status, _ = bash_wrapper(f"systemctl is-active {service_name}")
-            if service_status in valid_statuses:
-                service_names_set.remove(service_name)
-                continue
-            if service_status in retryable_statuses and retry < last_try:
-                continue
-            errors.add(f"{service_name} is {service_status.strip()}")
-
-        sleep(retry_delay_secs)
-
-    return list(errors)

python/system_tests/validate_system.py

@@ -1,67 +0,0 @@
-"""Validate {server_name}."""
-
-import logging
-import sys
-import tomllib
-from os import environ
-from pathlib import Path  # noqa: TC003 This is required for the typer CLI
-from socket import gethostname
-
-import typer
-
-from python.common import configure_logger
-from python.signal_alert import signal_alert
-from python.system_tests.components import systemd_tests, zpool_tests
-
-logger = logging.getLogger(__name__)
-
-
-def load_config_data(config_file: Path) -> dict[str, list[str]]:
-    """Load a TOML configuration file.
-
-    Args:
-        config_file (Path): The path to the configuration file.
-
-    Returns:
-        dict: The configuration data.
-    """
-    return tomllib.loads(config_file.read_text())
-
-
-def main(config_file: Path) -> None:
-    """Main."""
-    configure_logger(level=environ.get("LOG_LEVEL", "INFO"))
-
-    server_name = gethostname()
-    logger.info(f"Starting {server_name} validation")
-
-    config_data = load_config_data(config_file)
-
-    errors: list[str] = []
-    try:
-        if config_data.get("zpools") and (zpool_errors := zpool_tests(config_data["zpools"])):
-            errors.extend(zpool_errors)
-
-        if config_data.get("services") and (systemd_errors := systemd_tests(config_data["services"])):
-            errors.extend(systemd_errors)
-
-    except Exception as error:
-        logger.exception(f"{server_name} validation failed")
-        errors.append(f"{server_name} validation failed: {error}")
-
-    if errors:
-        logger.error(f"{server_name} validation failed: \n{'\n'.join(errors)}")
-        signal_alert(f"{server_name} validation failed {errors}")
-
-        sys.exit(1)
-
-    logger.info(f"{server_name} validation passed")
-
-
-def cli() -> None:
-    """CLI."""
-    typer.run(main)
-
-
-if __name__ == "__main__":
-    cli()

python/tools/snapshot_manager.py

@@ -6,6 +6,7 @@ import logging
 import sys
 import tomllib
 from functools import cache
+from os import getenv
 from pathlib import Path  # noqa: TC003 This is required for the typer CLI
 from re import compile as re_compile
 from re import search
@@ -24,6 +25,15 @@ def main(config_file: Path) -> None:
     configure_logger(level="DEBUG")
     logger.info("Starting snapshot_manager")
 
+    from_phone = getenv("SIGNAL_ALERT_FROM_PHONE")
+    to_phone = getenv("SIGNAL_ALERT_TO_PHONE")
+    if not from_phone:
+        error = "SIGNAL_ALERT_FROM_PHONE environment variable is not set"
+        raise ValueError(error)
+    if not to_phone:
+        error = "SIGNAL_ALERT_TO_PHONE environment variable is not set"
+        raise ValueError(error)
+
     try:
         time_stamp = get_time_stamp()
 
@@ -33,14 +43,14 @@ def main(config_file: Path) -> None:
             if status != "snapshot created":
                 msg = f"{dataset.name} failed to create snapshot {time_stamp}"
                 logger.error(msg)
-                signal_alert(msg)
+                signal_alert(msg, from_phone, to_phone)
                 continue
             count_lookup = get_count_lookup(config_file, dataset.name)
             logger.info(f"using {count_lookup} for {dataset.name}")
-            get_snapshots_to_delete(dataset, count_lookup)
+            get_snapshots_to_delete(dataset, count_lookup, from_phone, to_phone)
     except Exception:
         logger.exception("snapshot_manager failed")
-        signal_alert("snapshot_manager failed")
+        signal_alert("snapshot_manager failed", from_phone, to_phone)
         sys.exit(1)
     else:
         logger.info("snapshot_manager completed")
@@ -92,12 +102,16 @@ def load_config_data(config_file: Path) -> dict[str, dict[str, int]]:
 def get_snapshots_to_delete(
     dataset: Dataset,
     count_lookup: dict[str, int],
+    from_phone: str,
+    to_phone: str,
 ) -> None:
     """Get snapshots to delete.
 
     Args:
         dataset (Dataset): the dataset
         count_lookup (dict[str, int]): the count lookup
+        from_phone (str): The Signal account sending alerts.
+        to_phone (str): The Signal account receiving alerts.
     """
     snapshots = dataset.get_snapshots()
 
@@ -127,7 +141,7 @@ def get_snapshots_to_delete(
         for snapshot in snapshots_being_deleted:
             if error := dataset.delete_snapshot(snapshot):
                 error_message = f"{dataset.name}@{snapshot} failed to delete: {error}"
-                signal_alert(error_message)
+                signal_alert(error_message, from_phone, to_phone)
                 logger.error(error_message)
 
 

python/van_weather/__init__.py

@@ -1 +0,0 @@
-"""Van weather service - fetches weather with masked GPS location."""

python/van_weather/main.py

@@ -1,293 +0,0 @@
-"""Van weather service - fetches weather with masked GPS for privacy."""
-
-import logging
-from datetime import UTC, datetime
-from typing import Annotated, Any
-
-import httpx
-import typer
-from apscheduler.schedulers.blocking import BlockingScheduler
-from tenacity import before_sleep_log, retry, stop_after_attempt, wait_fixed
-
-from python.common import configure_logger
-from python.van_weather.models import Config, DailyForecast, HourlyForecast, Weather
-
-# Map Pirate Weather icons to Home Assistant conditions
-CONDITION_MAP = {
-    "clear-day": "sunny",
-    "clear-night": "clear-night",
-    "rain": "rainy",
-    "snow": "snowy",
-    "sleet": "snowy-rainy",
-    "wind": "windy",
-    "fog": "fog",
-    "cloudy": "cloudy",
-    "partly-cloudy-day": "partlycloudy",
-    "partly-cloudy-night": "partlycloudy",
-}
-
-logger = logging.getLogger(__name__)
-
-
-@retry(
-    stop=stop_after_attempt(3),
-    wait=wait_fixed(5),
-    before_sleep=before_sleep_log(logger, logging.WARNING),
-    reraise=True,
-)
-def get_ha_state(url: str, token: str, entity_id: str) -> float:
-    """Get numeric state from Home Asasistant entity."""
-    response = httpx.get(
-        f"{url}/api/states/{entity_id}",
-        headers={"Authorization": f"Bearer {token}"},
-        timeout=30,
-    )
-    response.raise_for_status()
-    state = response.json()["state"]
-    if state in ("unavailable", "unknown"):
-        error = f"{entity_id} is {state}"
-        raise ValueError(error)
-    return float(state)
-
-
-def parse_daily_forecast(data: dict[str, dict[str, Any]]) -> list[DailyForecast]:
-    """Parse daily forecast from Pirate Weather API."""
-    daily = data.get("daily", {}).get("data", [])
-    daily_forecasts = []
-    for day in daily[:8]:  # Up to 8 days
-        time_stamp = day.get("time")
-        if time_stamp:
-            date_time = datetime.fromtimestamp(time_stamp, tz=UTC).isoformat()
-            daily_forecasts.append(
-                DailyForecast(
-                    date_time=date_time,
-                    condition=CONDITION_MAP.get(day.get("icon", ""), "cloudy"),
-                    temperature=day.get("temperatureHigh"),
-                    templow=day.get("temperatureLow"),
-                    precipitation_probability=day.get("precipProbability"),
-                    moon_phase=day.get("moonPhase"),
-                    wind_gust=day.get("windGust"),
-                    cloud_cover=day.get("cloudCover"),
-                )
-            )
-
-    return daily_forecasts
-
-
-def parse_hourly_forecast(data: dict[str, dict[str, Any]]) -> list[HourlyForecast]:
-    """Parse hourly forecast from Pirate Weather API."""
-    hourly = data.get("hourly", {}).get("data", [])
-    hourly_forecasts = []
-    for hour in hourly[:48]:  # Up to 48 hours
-        time_stamp = hour.get("time")
-        if time_stamp:
-            date_time = datetime.fromtimestamp(time_stamp, tz=UTC).isoformat()
-            hourly_forecasts.append(
-                HourlyForecast(
-                    date_time=date_time,
-                    condition=CONDITION_MAP.get(hour.get("icon", ""), "cloudy"),
-                    temperature=hour.get("temperature"),
-                    precipitation_probability=hour.get("precipProbability"),
-                )
-            )
-    return hourly_forecasts
-
-
-@retry(
-    stop=stop_after_attempt(3),
-    wait=wait_fixed(5),
-    before_sleep=before_sleep_log(logger, logging.WARNING),
-    reraise=True,
-)
-def fetch_weather(api_key: str, lat: float, lon: float) -> Weather:
-    """Fetch weather from Pirate Weather API."""
-    url = f"https://api.pirateweather.net/forecast/{api_key}/{lat},{lon}"
-    response = httpx.get(url, params={"units": "us"}, timeout=30)
-    response.raise_for_status()
-    data = response.json()
-
-    daily_forecasts = parse_daily_forecast(data)
-    hourly_forecasts = parse_hourly_forecast(data)
-
-    current = data.get("currently", {})
-    icon = current.get("icon", "")
-    return Weather(
-        temperature=current.get("temperature"),
-        feels_like=current.get("apparentTemperature"),
-        humidity=current.get("humidity"),
-        wind_speed=current.get("windSpeed"),
-        wind_bearing=current.get("windBearing"),
-        condition=CONDITION_MAP.get(icon, "cloudy"),
-        summary=current.get("summary"),
-        pressure=current.get("pressure"),
-        visibility=current.get("visibility"),
-        uv_index=current.get("uvIndex"),
-        ozone=current.get("ozone"),
-        nearest_storm_distance=current.get("nearestStormDistance"),
-        nearest_storm_bearing=current.get("nearestStormBearing"),
-        precip_probability=current.get("precipProbability"),
-        cloud_cover=current.get("cloudCover"),
-        daily_forecasts=daily_forecasts,
-        hourly_forecasts=hourly_forecasts,
-    )
-
-
-@retry(
-    stop=stop_after_attempt(3),
-    wait=wait_fixed(5),
-    before_sleep=before_sleep_log(logger, logging.WARNING),
-    reraise=True,
-)
-def post_to_ha(url: str, token: str, weather: Weather) -> None:
-    """Post weather data to Home Assistant as sensor entities."""
-    headers = {"Authorization": f"Bearer {token}"}
-
-    # Post current weather as individual sensors
-    sensors = {
-        "sensor.van_weather_condition": {
-            "state": weather.condition or "unknown",
-            "attributes": {"friendly_name": "Van Weather Condition"},
-        },
-        "sensor.van_weather_temperature": {
-            "state": weather.temperature,
-            "attributes": {"unit_of_measurement": "°F", "device_class": "temperature"},
-        },
-        "sensor.van_weather_apparent_temperature": {
-            "state": weather.feels_like,
-            "attributes": {"unit_of_measurement": "°F", "device_class": "temperature"},
-        },
-        "sensor.van_weather_humidity": {
-            "state": int((weather.humidity or 0) * 100),
-            "attributes": {"unit_of_measurement": "%", "device_class": "humidity"},
-        },
-        "sensor.van_weather_pressure": {
-            "state": weather.pressure,
-            "attributes": {"unit_of_measurement": "mbar", "device_class": "pressure"},
-        },
-        "sensor.van_weather_wind_speed": {
-            "state": weather.wind_speed,
-            "attributes": {"unit_of_measurement": "mph", "device_class": "wind_speed"},
-        },
-        "sensor.van_weather_wind_bearing": {
-            "state": weather.wind_bearing,
-            "attributes": {"unit_of_measurement": "°"},
-        },
-        "sensor.van_weather_visibility": {
-            "state": weather.visibility,
-            "attributes": {"unit_of_measurement": "mi"},
-        },
-        "sensor.van_weather_uv_index": {
-            "state": weather.uv_index,
-            "attributes": {"friendly_name": "Van Weather UV Index", "icon": "mdi:sun-wireless"},
-        },
-        "sensor.van_weather_ozone": {
-            "state": weather.ozone,
-            "attributes": {"unit_of_measurement": "DU", "icon": "mdi:earth"},
-        },
-        "sensor.van_weather_nearest_storm_distance": {
-            "state": weather.nearest_storm_distance,
-            "attributes": {"unit_of_measurement": "mi", "icon": "mdi:weather-lightning"},
-        },
-        "sensor.van_weather_nearest_storm_bearing": {
-            "state": weather.nearest_storm_bearing,
-            "attributes": {"unit_of_measurement": "°", "icon": "mdi:weather-lightning"},
-        },
-        "sensor.van_weather_precip_probability": {
-            "state": int((weather.precip_probability or 0) * 100),
-            "attributes": {"unit_of_measurement": "%", "icon": "mdi:weather-rainy"},
-        },
-        "sensor.van_weather_cloud_cover": {
-            "state": int((weather.cloud_cover or 0) * 100),
-            "attributes": {"unit_of_measurement": "%", "icon": "mdi:weather-cloudy"},
-        },
-    }
-
-    for entity_id, data in sensors.items():
-        if data["state"] is not None:
-            response = httpx.post(f"{url}/api/states/{entity_id}", headers=headers, json=data, timeout=30)
-            response.raise_for_status()
-
-    # Post daily forecast as JSON attribute sensor
-    daily_forecast = [
-        {
-            "datetime": daily_forecast.date_time.isoformat(),
-            "condition": daily_forecast.condition,
-            "temperature": daily_forecast.temperature,
-            "templow": daily_forecast.templow,
-            "precipitation_probability": int((daily_forecast.precipitation_probability or 0) * 100),
-        }
-        for daily_forecast in weather.daily_forecasts
-    ]
-
-    response = httpx.post(
-        f"{url}/api/states/sensor.van_weather_forecast_daily",
-        headers=headers,
-        json={"state": len(daily_forecast), "attributes": {"forecast": daily_forecast}},
-        timeout=30,
-    )
-    response.raise_for_status()
-
-    # Post hourly forecast as JSON attribute sensor
-    hourly_forecast = [
-        {
-            "datetime": hourly_forecast.date_time.isoformat(),
-            "condition": hourly_forecast.condition,
-            "temperature": hourly_forecast.temperature,
-            "precipitation_probability": int((hourly_forecast.precipitation_probability or 0) * 100),
-        }
-        for hourly_forecast in weather.hourly_forecasts
-    ]
-
-    response = httpx.post(
-        f"{url}/api/states/sensor.van_weather_forecast_hourly",
-        headers=headers,
-        json={"state": len(hourly_forecast), "attributes": {"forecast": hourly_forecast}},
-        timeout=30,
-    )
-    response.raise_for_status()
-
-
-def update_weather(config: Config) -> None:
-    """Fetch weather using last-known location, post to HA."""
-    lat = get_ha_state(config.ha_url, config.ha_token, config.lat_entity)
-    lon = get_ha_state(config.ha_url, config.ha_token, config.lon_entity)
-
-    masked_lat = round(lat, config.mask_decimals)
-    masked_lon = round(lon, config.mask_decimals)
-
-    logger.info(f"Masked location: {masked_lat}, {masked_lon}")
-
-    weather = fetch_weather(config.pirate_weather_api_key, masked_lat, masked_lon)
-    logger.info(f"Weather: {weather.temperature}°F, {weather.condition}")
-
-    post_to_ha(config.ha_url, config.ha_token, weather)
-    logger.info("Posted weather to HA")
-
-
-def main(
-    ha_url: Annotated[str, typer.Option(envvar="HA_URL")],
-    ha_token: Annotated[str, typer.Option(envvar="HA_TOKEN")],
-    api_key: Annotated[str, typer.Option(envvar="PIRATE_WEATHER_API_KEY")],
-    interval: Annotated[int, typer.Option(help="Poll interval in seconds")] = 900,
-    log_level: Annotated[str, typer.Option()] = "INFO",
-) -> None:
-    """Fetch weather for van using masked GPS location."""
-    configure_logger(log_level)
-
-    config = Config(ha_url=ha_url, ha_token=ha_token, pirate_weather_api_key=api_key)
-
-    logger.info(f"Starting van weather service, polling every {interval}s")
-
-    scheduler = BlockingScheduler()
-    scheduler.add_job(
-        update_weather,
-        "interval",
-        seconds=interval,
-        args=[config],
-        next_run_time=datetime.now(UTC),
-    )
-    scheduler.start()
-
-
-if __name__ == "__main__":
-    typer.run(main)

python/van_weather/models.py

@@ -1,72 +0,0 @@
-"""Models for van weather service."""
-
-from __future__ import annotations
-
-from datetime import datetime  # noqa: TC003 This is required for pydantic
-
-from pydantic import BaseModel, field_serializer
-
-
-class Config(BaseModel):
-    """Service configuration."""
-
-    ha_url: str
-    ha_token: str
-    pirate_weather_api_key: str
-    lat_entity: str = "sensor.van_last_known_latitude"
-    lon_entity: str = "sensor.van_last_known_longitude"
-    mask_decimals: int = 1  # ~11km accuracy
-
-
-class DailyForecast(BaseModel):
-    """Daily forecast entry."""
-
-    date_time: datetime
-    condition: str | None = None
-    temperature: float | None = None  # High
-    templow: float | None = None  # Low
-    precipitation_probability: float | None = None
-    moon_phase: float | None = None
-    wind_gust: float | None = None
-    cloud_cover: float | None = None
-
-    @field_serializer("date_time")
-    def serialize_date_time(self, date_time: datetime) -> str:
-        """Serialize datetime to ISO format."""
-        return date_time.isoformat()
-
-
-class HourlyForecast(BaseModel):
-    """Hourly forecast entry."""
-
-    date_time: datetime
-    condition: str | None = None
-    temperature: float | None = None
-    precipitation_probability: float | None = None
-
-    @field_serializer("date_time")
-    def serialize_date_time(self, date_time: datetime) -> str:
-        """Serialize datetime to ISO format."""
-        return date_time.isoformat()
-
-
-class Weather(BaseModel):
-    """Weather data from Pirate Weather."""
-
-    temperature: float | None = None
-    feels_like: float | None = None
-    humidity: float | None = None
-    wind_speed: float | None = None
-    wind_bearing: float | None = None
-    condition: str | None = None
-    summary: str | None = None
-    pressure: float | None = None
-    visibility: float | None = None
-    uv_index: float | None = None
-    ozone: float | None = None
-    nearest_storm_distance: float | None = None
-    nearest_storm_bearing: float | None = None
-    precip_probability: float | None = None
-    cloud_cover: float | None = None
-    daily_forecasts: list[DailyForecast] = []
-    hourly_forecasts: list[HourlyForecast] = []

scripts/signal/authorize.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+if [[ "$(hostname)" != "jeeves" ]]; then
+  echo "Signal device authorization must run on jeeves." >&2
+  exit 1
+fi
+if [[ "$EUID" -ne 0 ]]; then
+  echo "Signal device authorization must run as root." >&2
+  exit 1
+fi
+if [[ "$#" -ne 1 ]]; then
+  echo "Usage: signal_authorize.sh DEVICE_URI" >&2
+  exit 2
+fi
+if [[ "$1" != sgnl://linkdevice\?* ]]; then
+  echo "Invalid Signal device link URI." >&2
+  exit 2
+fi
+
+service_name="signal-cli"
+data_dir="/var/lib/signal-cli"
+device_uri="$1"
+restart_service=false
+
+cleanup() {
+  if [[ "$restart_service" == true ]]; then
+    systemctl start "$service_name"
+  fi
+}
+trap cleanup EXIT
+
+restart_service=true
+systemctl stop "$service_name"
+runuser -u signal-cli -- \
+  signal-cli --data-dir "$data_dir" -a +12016554378 addDevice --uri "$device_uri"

scripts/signal/link.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+host_name="$(hostname)"
+if [[ "$#" -ne 0 ]]; then
+  echo "Usage: signal_link.sh" >&2
+  exit 2
+fi
+
+service_name="signal-cli"
+data_dir="/var/lib/signal-cli"
+link_pid=""
+temp_dir=""
+restart_service=false
+
+cleanup() {
+  if [[ -n "$link_pid" ]] && kill -0 "$link_pid" 2>/dev/null; then
+    kill "$link_pid" 2>/dev/null || true
+    wait "$link_pid" 2>/dev/null || true
+  fi
+
+  if [[ -n "$temp_dir" ]]; then
+    rm -f -- "$temp_dir/link-uri"
+    rmdir -- "$temp_dir"
+  fi
+
+  if [[ "$restart_service" == true ]]; then
+    sudo systemctl start "$service_name"
+  fi
+}
+trap cleanup EXIT
+
+restart_service=true
+sudo systemctl stop "$service_name"
+
+temp_dir="$(mktemp -d)"
+link_fifo="$temp_dir/link-uri"
+mkfifo "$link_fifo"
+
+sudo -u signal-cli \
+  signal-cli --data-dir "$data_dir" link --name "$host_name" >"$link_fifo" &
+link_pid="$!"
+
+if ! IFS= read -r device_uri <"$link_fifo"; then
+  wait "$link_pid"
+  exit 1
+fi
+if [[ "$device_uri" != sgnl://linkdevice\?* ]]; then
+  echo "signal-cli returned an invalid device link URI." >&2
+  exit 1
+fi
+
+echo "Run this command manually on jeeves in another dotfiles dir:"
+printf "sudo ./scripts/signal/authorize.sh %q\n" "$device_uri"
+echo "Waiting for jeeves to authorize this device..."
+
+wait "$link_pid"
+link_pid=""
+echo "$host_name is now linked to the Signal account on jeeves."

shell.nix

@@ -8,7 +8,7 @@
   # loader path, which NixOS does not provide globally.
   ebook-search = pkgs.mkShell {
     nativeBuildInputs = with pkgs; [
-      my_python
+      python314
       uv
     ];
     LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
@@ -23,7 +23,6 @@
       nix
       home-manager
       git
-      my_python
 
       ssh-to-age
       gnupg

systems/bob/default.nix

@@ -3,18 +3,18 @@
   imports = [
     "${inputs.self}/users/math"
     "${inputs.self}/users/richie"
-    "${inputs.self}/users/steve"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/docker.nix"
-    "${inputs.self}/common/optional/scanner.nix"
     "${inputs.self}/common/optional/monitoring-agent.nix"
-    "${inputs.self}/common/optional/steam.nix"
+    "${inputs.self}/common/optional/nvidia.nix"
+    "${inputs.self}/common/optional/scanner.nix"
+    "${inputs.self}/common/optional/signal-cli.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/systemd-boot.nix"
+    "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/yubikey.nix"
+    "${inputs.self}/common/optional/x86-64-v3"
-    "${inputs.self}/common/optional/zerotier.nix"
+    "${inputs.self}/common/optional/zfs"
-    "${inputs.self}/common/optional/nvidia.nix"
     ./hardware.nix
     ./syncthing.nix
     ./llms.nix

systems/bob/hardware.nix

@@ -50,6 +50,11 @@
       fsType = "zfs";
     };
 
+    "/nix/var/nix/builds" = {
+      device = "root_pool/nix_build";
+      fsType = "zfs";
+    };
+
     "/var" = {
       device = "root_pool/var";
       fsType = "zfs";

systems/bob/snapshot_config.toml

@@ -22,6 +22,12 @@ hourly = 0
 daily = 0
 monthly = 0
 
+["root_pool/nix_build"]
+15_min = 1
+hourly = 0
+daily = 0
+monthly = 0
+
 ["root_pool/models"]
 15_min = 4
 hourly = 24

systems/brain/default.nix

@@ -4,11 +4,14 @@
     "${inputs.self}/users/richie"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/docker.nix"
+    "${inputs.self}/common/optional/signal-cli.nix"
     "${inputs.self}/common/optional/ssh_decrypt.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/systemd-boot.nix"
+    "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/zerotier.nix"
+    "${inputs.self}/common/optional/x86-64-v3"
+    "${inputs.self}/common/optional/zfs"
     ./docker
     ./hardware.nix
     ./programs.nix

systems/brain/docker/docker_networks.md

@@ -1,3 +0,0 @@
-# docker_networks
-
-docker network create -d bridge web

systems/brain/hardware.nix

@@ -53,6 +53,11 @@
       fsType = "zfs";
     };
 
+    "/nix/var/nix/builds" = {
+      device = "root_pool/nix_build";
+      fsType = "zfs";
+    };
+
     "/boot" = {
       device = "/dev/disk/by-uuid/12CE-A600";
       fsType = "vfat";

systems/brain/services/heater.nix

@@ -4,6 +4,20 @@
   ...
 }:
 {
+  nixpkgs.overlays = [
+    (final: _prev: {
+      heater_python = final.python314.withPackages (
+        ps: with ps; [
+          fastapi
+          pydantic
+          tinytuya
+          typer
+          uvicorn
+        ]
+      );
+    })
+  ];
+
   networking.firewall.allowedTCPPorts = [ 8124 ];
 
   systemd.services.heater-api = {
@@ -17,7 +31,7 @@
 
     serviceConfig = {
       Type = "simple";
-      ExecStart = "${pkgs.my_python}/bin/python -m python.heater.main --host 0.0.0.0 --port 8124";
+      ExecStart = "${pkgs.heater_python}/bin/python -m python.heater.main --host 0.0.0.0 --port 8124";
       EnvironmentFile = "/etc/heater.env";
       Restart = "on-failure";
       RestartSec = "5s";

systems/brain/services/home_assistant.nix

@@ -21,6 +21,7 @@
           packages = {
             victron_modbuss = "!include ${./home_assistant/victron_modbuss.yaml}";
             battery_sensors = "!include ${./home_assistant/battery_sensors.yaml}";
+            energy_meters = "!include ${./home_assistant/energy_meters.yaml}";
             gps_location = "!include ${./home_assistant/gps_location.yaml}";
             heater = "!include ${./home_assistant/heater.yaml}";
             van_weather = "!include ${./home_assistant/van_weather_template.yaml}";
@@ -67,7 +68,6 @@
           paho-mqtt # for mqtt
           psycopg2 # for postgresql
           py-improv-ble-client # for esphome
-          pymodbus # for modbus
           pyopenweathermap # for weather
           pymetno # for met.no weather
           uiprotect # for ubiquiti integration
@@ -75,17 +75,19 @@
           jsonpath # for rest sensors
           monarchmoneycommunity # for monarch
         ];
-      extraComponents = [ "isal" ];
+      extraComponents = [
+        "isal"
+        "modbus" # for victron modbus integration
+      ];
       customComponents = with pkgs.home-assistant-custom-components; [
         garmin_connect
-        pirate-weather
       ];
 
     };
     esphome = {
       enable = true;
       openFirewall = true;
-      address = "192.168.90.35";
+      address = "0.0.0.0";
     };
   };
 }

systems/brain/services/home_assistant/energy_meters.yaml

@@ -0,0 +1,71 @@
+template:
+  - sensor:
+      - name: "JK0 Energy In Clean"
+        unique_id: jk0_energy_in_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.jk0_energy_in')) }}
+        state: >-
+          {{ states('sensor.jk0_energy_in') | float }}
+
+      - name: "JK0 Energy Out Clean"
+        unique_id: jk0_energy_out_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.jk0_energy_out')) }}
+        state: >-
+          {{ states('sensor.jk0_energy_out') | float }}
+
+      - name: "JK1 Energy In Clean"
+        unique_id: jk1_energy_in_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.jk1_energy_in')) }}
+        state: >-
+          {{ states('sensor.jk1_energy_in') | float }}
+
+      - name: "JK1 Energy Out Clean"
+        unique_id: jk1_energy_out_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.jk1_energy_out')) }}
+        state: >-
+          {{ states('sensor.jk1_energy_out') | float }}
+
+      - name: "Solar Yield Clean"
+        unique_id: solar_yield_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.solar_yield_daily')) }}
+        state: >-
+          {{ states('sensor.solar_yield_daily') | float }}
+
+      - name: "DC Load Energy Clean"
+        unique_id: dc_load_energy_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.dc_load_energy_daily')) }}
+        state: >-
+          {{ states('sensor.dc_load_energy_daily') | float }}
+
+      - name: "DC Charger Energy Clean"
+        unique_id: dc_charger_energy_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.dc_charger_energy')) }}
+        state: >-
+          {{ states('sensor.dc_charger_energy') | float }}

systems/brain/services/postgress.nix

@@ -19,7 +19,7 @@
       local all  richie   trust
       host  all  richie   127.0.0.1/32    trust
       host  all  richie   ::1/128         trust
-      host  all  richie   192.168.90.1/24 trust
+      host  all  richie   100.64.0.0/10 trust
       host  all  richie   192.168.99.1/24 trust
 
       local vaninventory vaninventory trust
@@ -28,7 +28,7 @@
       local hass     hass     trust
 
       # ipv4
-      host  hass     hass     192.168.90.1/24 trust
+      host  hass     hass     100.64.0.0/10 trust
       host  hass     hass     127.0.0.1/32 trust
 
       # ipv6

systems/brain/syncthing.nix

@@ -3,7 +3,7 @@
 
   services.syncthing = {
     overrideFolders = false;
-    guiAddress = "192.168.90.35:8384";
+    guiAddress = "0.0.0.0:8384";
     settings = {
       "dotfiles" = {
         path = "/home/richie/dotfiles";

systems/jeeves/containers.nix

@@ -0,0 +1,71 @@
+{ pkgs, ... }:
+let
+  vars = import ./vars.nix;
+in
+{
+  virtualisation.docker.daemon.settings."data-root" = "${vars.containers}/docker";
+
+  # nixos-container hardcodes its state directory to /var/lib/nixos-containers,
+  # so route it to the shared container dataset with a bind mount.
+  fileSystems."/var/lib/nixos-containers" = {
+    device = "${vars.containers}/nixos-containers";
+    fsType = "none";
+    options = [ "bind" ];
+  };
+
+  systemd.services = {
+    prepare-docker-storage = {
+      description = "Create Docker storage directory";
+      requiredBy = [ "docker.service" ];
+      before = [ "docker.service" ];
+      requires = [ "zfs-mount.service" ];
+      after = [ "zfs-mount.service" ];
+      path = [
+        pkgs.coreutils
+        pkgs.util-linux
+      ];
+
+      unitConfig = {
+        DefaultDependencies = false;
+        RequiresMountsFor = [ "/nix" ];
+      };
+
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+
+      script = ''
+        mountpoint -q ${vars.containers}
+        install -d -m 0710 ${vars.containers}/docker
+      '';
+    };
+
+    prepare-nixos-container-storage = {
+      description = "Create NixOS container storage directory";
+      requiredBy = [ "var-lib-nixos\\x2dcontainers.mount" ];
+      before = [ "var-lib-nixos\\x2dcontainers.mount" ];
+      requires = [ "zfs-mount.service" ];
+      after = [ "zfs-mount.service" ];
+      path = [
+        pkgs.coreutils
+        pkgs.util-linux
+      ];
+
+      unitConfig = {
+        DefaultDependencies = false;
+        RequiresMountsFor = [ "/nix" ];
+      };
+
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+
+      script = ''
+        mountpoint -q ${vars.containers}
+        install -d -m 0755 ${vars.containers}/nixos-containers
+      '';
+    };
+  };
+}

systems/jeeves/default.nix

@@ -4,23 +4,24 @@ let
 in
 {
   imports = [
-    "${inputs.self}/users/dov"
     "${inputs.self}/users/math"
     "${inputs.self}/users/richie"
-    "${inputs.self}/users/steve"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/docker.nix"
     "${inputs.self}/common/optional/monitoring-agent.nix"
+    "${inputs.self}/common/optional/signal-cli.nix"
     "${inputs.self}/common/optional/ssh_decrypt.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
+    "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/zerotier.nix"
+    "${inputs.self}/common/optional/x86-64-v3"
+    "${inputs.self}/common/optional/zfs"
     ./monitoring
     ./docker
     ./services
-    ./web_services
+    ./containers.nix
     ./hardware.nix
-    ./networking.nix
+    ./networking
     ./programs.nix
     ./runners
     ./syncthing.nix
@@ -35,8 +36,6 @@ in
       path = ./snapshot_config.toml;
       EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
     };
-
-    zerotierone.joinNetworks = [ "a09acf02330d37b9" ];
   };
 
   users.groups = {

systems/jeeves/docker/signal-cli-rest-api.nix

@@ -1,21 +0,0 @@
-let
-  vars = import ../vars.nix;
-in
-{
-  networking.firewall.allowedTCPPorts = [
-    8989
-  ];
-  virtualisation.oci-containers.containers.signal_cli_rest_api = {
-    image = "bbernhard/signal-cli-rest-api:0.199-dev";
-    ports = [
-      "8989:8080"
-    ];
-    volumes = [
-      "${vars.docker_configs}/signal-cli-config:/home/.local/share/signal-cli"
-    ];
-    environment = {
-      MODE = "json-rpc";
-    };
-    autoStart = true;
-  };
-}

systems/jeeves/hardware.nix

@@ -96,6 +96,11 @@ in
       fsType = "zfs";
     };
 
+    "/nix/var/nix/builds" = {
+      device = "root_pool/nix_build";
+      fsType = "zfs";
+    };
+
     "/var" = {
       device = "root_pool/var";
       fsType = "zfs";

systems/jeeves/monitoring/dashboards/portal-1.json

@@ -0,0 +1,798 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": false,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "short",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "red",
+                  "index": 1,
+                  "text": "Offline"
+                },
+                "1": {
+                  "color": "green",
+                  "index": 0,
+                  "text": "Online"
+                }
+              },
+              "type": "value"
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 4,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "up{job=\"node\",instance=\"portal-1\"}",
+          "instant": true,
+          "legendFormat": "",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Node Exporter",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 5,
+        "x": 4,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{job=\"node\",instance=\"portal-1\",mode=\"idle\"}[5m])))",
+          "instant": true,
+          "legendFormat": "",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "CPU Used",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 5,
+        "x": 9,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - (node_memory_MemAvailable_bytes{job=\"node\",instance=\"portal-1\"} / node_memory_MemTotal_bytes{job=\"node\",instance=\"portal-1\"}))",
+          "instant": true,
+          "legendFormat": "",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "RAM Used",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 5,
+        "x": 14,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - (node_filesystem_avail_bytes{job=\"node\",instance=\"portal-1\",mountpoint=\"/\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node\",instance=\"portal-1\",mountpoint=\"/\",fstype!=\"\"}))",
+          "instant": true,
+          "legendFormat": "",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Root Filesystem Used",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 5,
+        "x": 19,
+        "y": 0
+      },
+      "id": 5,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "node_load1{job=\"node\",instance=\"portal-1\"}",
+          "instant": true,
+          "legendFormat": "",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Load (1m)",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 5
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{job=\"node\",instance=\"portal-1\",mode=\"idle\"}[5m])))",
+          "instant": false,
+          "legendFormat": "CPU used",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "CPU Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 5
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - (node_memory_MemAvailable_bytes{job=\"node\",instance=\"portal-1\"} / node_memory_MemTotal_bytes{job=\"node\",instance=\"portal-1\"}))",
+          "instant": false,
+          "legendFormat": "RAM used",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "RAM Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 13
+      },
+      "id": 8,
+      "options": {
+        "cellHeight": "sm",
+        "showHeader": true,
+        "sortBy": [
+          {
+            "desc": true,
+            "displayName": "Value"
+          }
+        ]
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - (node_filesystem_avail_bytes{job=\"node\",instance=\"portal-1\",fstype!~\"tmpfs|devtmpfs|overlay|squashfs\",mountpoint!~\"/run.*\"} / node_filesystem_size_bytes{job=\"node\",instance=\"portal-1\",fstype!~\"tmpfs|devtmpfs|overlay|squashfs\",mountpoint!~\"/run.*\"}))",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "{{mountpoint}}",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Filesystem Usage",
+      "type": "table"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "Bps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 13
+      },
+      "id": 9,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(node_disk_read_bytes_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
+          "instant": false,
+          "legendFormat": "read",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(node_disk_written_bytes_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
+          "instant": false,
+          "legendFormat": "write",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Disk Throughput",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "iops"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 21
+      },
+      "id": 10,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(node_disk_reads_completed_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
+          "instant": false,
+          "legendFormat": "reads",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(node_disk_writes_completed_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
+          "instant": false,
+          "legendFormat": "writes",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Disk Operations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "Bps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 21
+      },
+      "id": 11,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(node_network_receive_bytes_total{job=\"node\",instance=\"portal-1\",device!=\"lo\"}[5m]))",
+          "instant": false,
+          "legendFormat": "received",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(node_network_transmit_bytes_total{job=\"node\",instance=\"portal-1\",device!=\"lo\"}[5m]))",
+          "instant": false,
+          "legendFormat": "sent",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Network Traffic",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 39,
+  "tags": [
+    "monitoring",
+    "portal-1",
+    "vultr"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Portal-1 Host",
+  "uid": "portal-1-host",
+  "version": 1,
+  "weekStart": ""
+}

systems/jeeves/monitoring/default.nix

@@ -41,29 +41,36 @@ let
       {
         job_name = "node";
         static_configs = [
-          (mkTarget "jeeves" "192.168.90.40:9100")
+          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9100")
-          (mkTarget "bob" "192.168.90.25:9100")
+          (mkTarget "bob" "bob.taile39c31.ts.net:9100")
+          (mkTarget "portal-1" "portal-1.taile39c31.ts.net:9100")
         ];
       }
+      {
+        job_name = "vultr";
+        scrape_interval = "5m";
+        scrape_timeout = "45s";
+        static_configs = [ (mkTarget "vultr" "127.0.0.1:9188") ];
+      }
       {
         job_name = "process_grouped";
         static_configs = [
-          (mkTarget "jeeves" "192.168.90.40:9256")
+          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9256")
-          (mkTarget "bob" "192.168.90.25:9256")
+          (mkTarget "bob" "bob.taile39c31.ts.net:9256")
         ];
       }
       {
         job_name = "smartctl";
         static_configs = [
-          (mkTarget "jeeves" "192.168.90.40:9633")
+          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9633")
-          (mkTarget "bob" "192.168.90.25:9633")
+          (mkTarget "bob" "bob.taile39c31.ts.net:9633")
         ];
       }
       {
         job_name = "zfs";
         static_configs = [
-          (mkTarget "jeeves" "192.168.90.40:9134")
+          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9134")
-          (mkTarget "bob" "192.168.90.25:9134")
+          (mkTarget "bob" "bob.taile39c31.ts.net:9134")
         ];
       }
     ];
@@ -79,8 +86,8 @@ let
       {
         job_name = "process_pid";
         static_configs = [
-          (mkTarget "jeeves" "192.168.90.40:9257")
+          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9257")
-          (mkTarget "bob" "192.168.90.25:9257")
+          (mkTarget "bob" "bob.taile39c31.ts.net:9257")
         ];
       }
     ];
@@ -97,7 +104,9 @@ let
       after = [
         "zfs-media-database-prometheus.mount"
         "network.target"
+        "tailscaled-autoconnect.service"
       ];
+      wants = [ "tailscaled-autoconnect.service" ];
       requires = [ "zfs-media-database-prometheus.mount" ];
       wantedBy = [ "multi-user.target" ];
       unitConfig.RequiresMountsFor = [ dataDir ];
@@ -151,6 +160,8 @@ let
     };
 in
 {
+  imports = [ ./vultr-exporter.nix ];
+
   users = {
     groups.prometheus = { };
     users.prometheus = {

systems/jeeves/monitoring/vultr-exporter.nix

@@ -0,0 +1,96 @@
+{
+  lib,
+  pkgs,
+  ...
+}:
+let
+  vars = import ../vars.nix;
+
+  vultrExporter = pkgs.buildGoModule rec {
+    pname = "vultr-exporter";
+    version = "0-unstable-2025-10-14";
+
+    src = pkgs.fetchFromGitHub {
+      owner = "DazWilkin";
+      repo = "vultr-exporter";
+      rev = "d8b5d7107c2eda31566e6e895c8a67db32fdaf68";
+      hash = "sha256-3LpXh+mFMBRJfiY2ftELpM3AIE0LiZ9c6NFok0OeH5I=";
+    };
+
+    vendorHash = "sha256-qCHWmg6GxLIjqlf5UqRcuDRd1xqRFI3sl4ZduLajx+Y=";
+    subPackages = [ "cmd/server" ];
+
+    postPatch = ''
+      substituteInPlace cmd/server/main.go \
+        --replace-fail 'registry.MustRegister(collector.NewBlockStorageCollector(s, client, log))' "" \
+        --replace-fail 'registry.MustRegister(collector.NewKubernetesCollector(s, client, log))' "" \
+        --replace-fail 'registry.MustRegister(collector.NewLoadBalancerCollector(s, client, log))' "" \
+        --replace-fail 'registry.MustRegister(collector.NewReservedIPsCollector(s, client, log))' ""
+    '';
+
+    ldflags = [
+      "-s"
+      "-w"
+      "-X main.GitCommit=${src.rev}"
+      "-X main.OSVersion=NixOS"
+    ];
+
+    postInstall = ''
+      mv $out/bin/server $out/bin/vultr-exporter
+    '';
+
+    meta = {
+      description = "Prometheus exporter for Vultr account, billing, and bandwidth metrics";
+      homepage = "https://github.com/DazWilkin/vultr-exporter";
+      license = lib.licenses.asl20;
+      mainProgram = "vultr-exporter";
+    };
+  };
+in
+{
+  systemd.services.vultr-exporter = {
+    description = "Vultr Prometheus exporter";
+    documentation = [ "https://github.com/DazWilkin/vultr-exporter" ];
+    wants = [ "network-online.target" ];
+    after = [ "network-online.target" ];
+    wantedBy = [ "multi-user.target" ];
+
+    serviceConfig = {
+      ExecStart = "${lib.getExe vultrExporter} --endpoint=127.0.0.1:9188 --path=/metrics";
+      EnvironmentFile = "${vars.secrets}/services/vultr-exporter";
+      Restart = "on-failure";
+      RestartSec = "30s";
+
+      CapabilityBoundingSet = [ "" ];
+      DeviceAllow = [ "" ];
+      DynamicUser = true;
+      LockPersonality = true;
+      MemoryDenyWriteExecute = true;
+      NoNewPrivileges = true;
+      PrivateDevices = true;
+      PrivateTmp = true;
+      ProtectClock = true;
+      ProtectControlGroups = true;
+      ProtectHome = true;
+      ProtectHostname = true;
+      ProtectKernelLogs = true;
+      ProtectKernelModules = true;
+      ProtectKernelTunables = true;
+      ProtectSystem = "strict";
+      RemoveIPC = true;
+      RestrictAddressFamilies = [
+        "AF_INET"
+        "AF_INET6"
+      ];
+      RestrictNamespaces = true;
+      RestrictRealtime = true;
+      RestrictSUIDSGID = true;
+      SystemCallArchitectures = "native";
+      SystemCallFilter = [
+        "@system-service"
+        "~@privileged"
+      ];
+      UMask = "0077";
+    };
+  };
+}

systems/jeeves/networking/default.nix

@@ -0,0 +1,16 @@
+{
+  imports = [
+    ./firewall.nix
+    ./interfaces.nix
+  ];
+  networking = {
+    hostName = "jeeves";
+    hostId = "0e15ce35";
+    useNetworkd = true;
+    # Without this, unconfigured NICs get a catch-all DHCP network and
+    # join the LAN uninvited (how the dead atlantic card ended up
+    # ARP-poisoning 192.168.99.14). Every interface must be configured
+    # explicitly below.
+    useDHCP = false;
+  };
+}

systems/jeeves/networking/firewall.nix

@@ -0,0 +1,47 @@
+let
+  ports = {
+    audiobookshelf = 8000;
+    ebook_searcher = 8070;
+    gems = 8002;
+    gitea_http = 6443;
+    gitea_ssh = 2223;
+    grafana = 3000;
+    jellyfin_http = 8096;
+    jellyswarrm = 3100;
+    nix_binary_cache = 5000;
+    nornsight = 8001;
+    ollama = 11434;
+    open_webui = 8080;
+    postgresql = 5432;
+    share = 8091;
+    syncthing = 8384;
+  };
+in
+{
+  networking.firewall = {
+    enable = true;
+    interfaces.br-nix-builder = {
+      allowedTCPPorts = [ ];
+      allowedUDPPorts = [ ];
+    };
+    interfaces.tailscale0.allowedTCPPorts = [
+      ports.audiobookshelf
+      ports.gems
+      ports.jellyfin_http
+      ports.jellyswarrm
+      ports.nix_binary_cache
+      ports.nornsight
+      ports.share
+    ];
+    allowedTCPPorts = [
+      ports.ebook_searcher
+      ports.gitea_http
+      ports.gitea_ssh
+      ports.grafana
+      ports.ollama
+      ports.open_webui
+      ports.postgresql
+      ports.syncthing
+    ];
+  };
+}

systems/jeeves/networking/interfaces.nix

@@ -8,27 +8,6 @@
     "net.bridge.bridge-nf-call-iptables" = 0;
   };
 
-  networking = {
-    hostName = "jeeves";
-    hostId = "0e15ce35";
-    firewall = {
-      enable = true;
-      interfaces.br-nix-builder = {
-        allowedTCPPorts = [ ];
-        allowedUDPPorts = [ ];
-      };
-      allowedTCPPorts = [
-        8070
-      ];
-    };
-    useNetworkd = true;
-    # Without this, unconfigured NICs get a catch-all DHCP network and
-    # join the LAN uninvited (how the dead atlantic card ended up
-    # ARP-poisoning 192.168.99.14). Every interface must be configured
-    # explicitly below.
-    useDHCP = false;
-  };
-
   systemd.network = {
     enable = true;
     wait-online = {

systems/jeeves/runners/default.nix

@@ -21,5 +21,16 @@
     nix-builder-12.enable = true;
     nix-builder-13.enable = true;
     nix-builder-14.enable = true;
+
+    # Warm the shared x86-64-v3 cache before the smaller per-system runners
+    # start. Eight jobs with eight cores each can use Jeeves' 64 logical CPUs,
+    # while the 6000% quota leaves some capacity for its normal services.
+    nix-cache-builder = {
+      enable = true;
+      labels = [ "nix-cache-builder:host" ];
+      cores = 8;
+      maxJobs = 8;
+      cpuQuota = "6000%";
+    };
   };
 }

systems/jeeves/runners/nix_builder.nix

@@ -1,8 +1,7 @@
 {
   config,
   lib,
-  outputs,
+  pkgs,
-  utils,
   ...
 }:
 
@@ -12,6 +11,130 @@ let
   cfg = config.services.nix_builder;
   runnerUsername = "gitea-runner";
   runnerUserid = 601;
+  containerConfig =
+    containerCfg:
+    {
+      config,
+      pkgs,
+      lib,
+      ...
+    }:
+    let
+      giteaAutomationPython = pkgs.python314.withPackages (
+        ps: with ps; [
+          httpx
+          pydantic
+          typer
+        ]
+      );
+      runnerConfigFile = (pkgs.formats.yaml { }).generate "gitea-runner.yaml" { };
+      registerRunner = pkgs.writeShellApplication {
+        name = "register-gitea-runner";
+        runtimeInputs = [
+          pkgs.coreutils
+          pkgs.hostname
+          config.services.gitea-actions-runner.package
+        ];
+        text = builtins.readFile ./register_gitea_runner.sh;
+      };
+    in
+    {
+      networking = {
+        hostName = "nix-builder";
+        useDHCP = lib.mkDefault true;
+        interfaces.eth0.useDHCP = true;
+        # Ensure containers don't inherit the host's stub resolver (127.0.0.53) which was causing issues
+        useHostResolvConf = false;
+      };
+      nix.settings = {
+        inherit (containerCfg) cores;
+        max-jobs = containerCfg.maxJobs;
+        system-features = lib.mkAfter [
+          "gccarch-x86-64-v2"
+          "gccarch-x86-64-v3"
+        ];
+        trusted-substituters = [
+          "https://cache.nixos.org"
+          "https://cache.tmmworkshop.com"
+          "https://nix-community.cachix.org"
+        ];
+        substituters = [
+          "https://cache.nixos.org/?priority=2&want-mass-query=true"
+          "https://cache.tmmworkshop.com/?priority=2&want-mass-query=true"
+          "https://nix-community.cachix.org/?priority=10&want-mass-query=true"
+        ];
+        trusted-public-keys = [
+          "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
+          "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA="
+          "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
+        ];
+        experimental-features = [
+          "flakes"
+          "nix-command"
+        ];
+        sandbox = true;
+        allowed-users = [ runnerUsername ];
+        trusted-users = [
+          "root"
+          runnerUsername
+        ];
+      };
+      users = {
+        users.${runnerUsername} = {
+          isSystemUser = true;
+          group = runnerUsername;
+          uid = runnerUserid;
+        };
+        groups.${runnerUsername}.gid = runnerUserid;
+      };
+      services.gitea-actions-runner.instances.builder = {
+        enable = true;
+        name = "jeeves-nix-builder";
+        url = "http://192.168.99.14:6443/";
+        labels = containerCfg.labels;
+        tokenFile = "/run/secrets/gitea-runners/registration-token";
+        settings.runner.timeout = "12h";
+        hostPackages = with pkgs; [
+          bash
+          coreutils
+          curl
+          gawk
+          gitMinimal
+          giteaAutomationPython
+          gnused
+          nix
+          nixfmt
+          nixos-rebuild
+          nodejs
+          ruff
+          treefmt
+          wget
+        ];
+      };
+      systemd.services."gitea-runner-builder" = {
+        serviceConfig = {
+          DynamicUser = mkForce false;
+          User = mkForce runnerUsername;
+          Group = mkForce runnerUsername;
+          ExecStartPre = mkForce [
+            "${getExe registerRunner} builder http://192.168.99.14:6443/ ${runnerConfigFile} ${escapeShellArgs containerCfg.labels}"
+          ];
+        };
+      };
+      system.stateVersion = "24.05";
+    };
+  mkContainerPath =
+    containerCfg:
+    (import "${pkgs.path}/nixos/lib/eval-config.nix" {
+      modules = [
+        {
+          boot.isNspawnContainer = true;
+          nixpkgs.pkgs = pkgs;
+        }
+        (containerConfig containerCfg)
+      ];
+      system = null;
+    }).config.system.build.toplevel;
 in
 {
   options.services.nix_builder = {
@@ -26,7 +149,36 @@ in
         types.submodule (
           { name, ... }:
           {
-            options.enable = mkEnableOption "Gitea runner container";
+            options = {
+              enable = mkEnableOption "Gitea runner container";
+
+              labels = mkOption {
+                type = types.listOf types.str;
+                default = [
+                  "self-hosted:host"
+                  "nixos:host"
+                ];
+                description = "Gitea Actions labels advertised by this runner.";
+              };
+
+              cores = mkOption {
+                type = types.ints.positive;
+                default = 8;
+                description = "Number of cores made available to each Nix build job.";
+              };
+
+              maxJobs = mkOption {
+                type = types.ints.positive;
+                default = 2;
+                description = "Maximum number of Nix build jobs run in parallel.";
+              };
+
+              cpuQuota = mkOption {
+                type = types.str;
+                default = "800%";
+                description = "systemd CPU quota for the runner container.";
+              };
+            };
           }
         )
       );
@@ -48,9 +200,11 @@ in
     containers = mapAttrs (
       name: containerCfg:
       mkIf containerCfg.enable {
+        path = mkContainerPath containerCfg;
         autoStart = true;
         privateNetwork = true;
         hostBridge = cfg.bridgeName;
+        extraFlags = [ "--hostname=${name}" ];
         bindMounts = {
           host-nix = {
             mountPoint = "/host-nix/var/nix/daemon-socket";
@@ -68,106 +222,18 @@ in
             isReadOnly = false;
           };
         };
-        config =
-          {
-            config,
-            pkgs,
-            lib,
-            ...
-          }:
-          {
-            networking = {
-              useDHCP = lib.mkDefault true;
-              interfaces.eth0.useDHCP = true;
-              # Ensure containers don't inherit the host's stub resolver (127.0.0.53) which was causing issues
-              useHostResolvConf = false;
-            };
-            nix.settings = {
-              trusted-substituters = [
-                "https://cache.nixos.org"
-                "https://cache.tmmworkshop.com"
-                "https://nix-community.cachix.org"
-              ];
-              substituters = [
-                "https://cache.nixos.org/?priority=2&want-mass-query=true"
-                "https://cache.tmmworkshop.com/?priority=2&want-mass-query=true"
-                "https://nix-community.cachix.org/?priority=10&want-mass-query=true"
-              ];
-              trusted-public-keys = [
-                "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
-                "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA="
-                "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
-              ];
-              experimental-features = [
-                "flakes"
-                "nix-command"
-              ];
-              sandbox = true;
-              allowed-users = [ "gitea-runner" ];
-              trusted-users = [
-                "root"
-                "gitea-runner"
-              ];
-            };
-            nixpkgs = {
-              overlays = builtins.attrValues outputs.overlays;
-              config.allowUnfree = true;
-            };
-            users = {
-              users.${runnerUsername} = {
-                isSystemUser = true;
-                group = runnerUsername;
-                uid = runnerUserid;
-              };
-              groups.${runnerUsername}.gid = runnerUserid;
-            };
-            services.gitea-actions-runner.instances.${name} = {
-              enable = true;
-              name = "jeeves-${name}";
-              url = "http://192.168.99.14:6443/";
-              labels = [
-                "self-hosted:host"
-                "nixos:host"
-              ];
-              tokenFile = "/run/secrets/gitea-runners/registration-token";
-              hostPackages = with pkgs; [
-                bash
-                coreutils
-                curl
-                gawk
-                gitMinimal
-                gnused
-                my_python
-                nix
-                nixfmt
-                nixos-rebuild
-                nodejs
-                treefmt
-                wget
-              ];
-            };
-            systemd.services."gitea-runner-${utils.escapeSystemdPath name}" = {
-              serviceConfig = {
-                DynamicUser = mkForce false;
-                User = mkForce runnerUsername;
-                Group = mkForce runnerUsername;
-              };
-            };
-            system.stateVersion = "24.05";
-          };
       }
     ) cfg.containers;
 
     systemd = {
-      services = builtins.listToAttrs (
+      services = mapAttrs' (
-        map (name: {
+        name: containerCfg:
-          name = "container@${name}";
+        nameValuePair "container@${name}" {
-          value = {
+          requires = [ "gitea.service" ];
-            requires = [ "gitea.service" ];
+          after = [ "gitea.service" ];
-            after = [ "gitea.service" ];
+          serviceConfig.CPUQuota = containerCfg.cpuQuota;
-          };
+        }
-        }) (builtins.attrNames (filterAttrs (_: c: c.enable) cfg.containers))
+      ) (filterAttrs (_: c: c.enable) cfg.containers);
-      );
 
       tmpfiles.rules = [
         "d ${vars.uv_cache} 0755 ${runnerUsername} ${runnerUsername} - -"

systems/jeeves/runners/register_gitea_runner.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+legacy_instance_dir="$STATE_DIRECTORY/$(hostname)"
+instance_dir="$STATE_DIRECTORY/$1"
+runner_url="$2"
+runner_config_file="$3"
+shift 3
+runner_labels="$(IFS=,; printf '%s' "$*")"
+
+if [ ! -e "$instance_dir" ] && [ -d "$legacy_instance_dir" ]; then
+  mv "$legacy_instance_dir" "$instance_dir"
+fi
+
+mkdir -vp "$instance_dir"
+cd "$instance_dir" || exit 1
+
+token_hash_file="$instance_dir/.token-hash"
+token_hash_current="$(printf '%s' "$TOKEN" | sha256sum | cut -d' ' -f1)"
+token_hash_stored="$(cat "$token_hash_file" 2>/dev/null || echo "")"
+labels_file="$instance_dir/.labels"
+labels_wanted="$(printf '%s\n' "$@" | sort)"
+labels_current="$(cat "$labels_file" 2>/dev/null || echo 0)"
+
+if [ ! -e "$instance_dir/.runner" ] || [ "$labels_wanted" != "$labels_current" ] || [ "$token_hash_current" != "$token_hash_stored" ]; then
+  rm -v "$instance_dir/.runner" || true
+
+  gitea-runner register --no-interactive \
+    --instance "$runner_url" \
+    --token "$TOKEN" \
+    --name "jeeves-$(hostname)" \
+    --labels "$runner_labels" \
+    --config "$runner_config_file"
+
+  printf '%s' "$token_hash_current" > "$token_hash_file"
+  printf '%s\n' "$labels_wanted" > "$labels_file"
+fi

systems/jeeves/scripts/zfs.sh

@@ -17,6 +17,7 @@ sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnode
 # media datasets
 sudo zfs create media/temp -o sync=disabled -o redundant_metadata=none
 sudo zfs create media/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key
+sudo zfs create media/secure/containers -o mountpoint=/zfs/media/containers -o compression=lz4 -o sync=disabled -o redundant_metadata=some -o normalization=none -o utf8only=off
 sudo zfs create media/secure/docker -o compression=zstd-9
 sudo zfs create media/secure/github-runners -o compression=zstd-9 -o sync=disabled
 sudo zfs create media/secure/notes -o copies=2

systems/jeeves/services/audiobookshelf.nix

@@ -6,6 +6,7 @@ in
   services.audiobookshelf = {
     enable = true;
     port = 8000;
+    host = "0.0.0.0";
   };
   systemd.services.audiobookshelf.serviceConfig.WorkingDirectory =
     lib.mkForce "${vars.docker_configs}/audiobookshelf";

systems/jeeves/services/camofox-browser.nix

@@ -1,80 +0,0 @@
-{
-  ...
-}:
-let
-  vars = import ../vars.nix;
-in
-{
-  systemd.tmpfiles.rules = [
-    "d ${vars.docker_configs}/camofox-browser 0750 root root - -"
-  ];
-
-  containers.camofox-browser = {
-    autoStart = true;
-    privateNetwork = false;
-    bindMounts = {
-      camofox-browser = {
-        hostPath = "${vars.docker_configs}/camofox-browser";
-        mountPoint = "/var/lib/camofox-browser";
-        isReadOnly = false;
-      };
-    };
-    config =
-      {
-        pkgs,
-        lib,
-        ...
-      }:
-      {
-        networking.hostName = "camofox-browser";
-
-        environment.systemPackages = with pkgs; [
-          ffmpeg
-          git
-          nodejs
-          python3Packages.yt-dlp
-        ];
-
-        systemd.services.camofox-browser = {
-          description = "Camofox browser server";
-          wantedBy = [ "multi-user.target" ];
-          after = [ "network.target" ];
-          environment = {
-            CAMOFOX_HOST = "127.0.0.1";
-            CAMOFOX_PORT = "9377";
-            HOME = "/var/lib/camofox-browser";
-          };
-          path = with pkgs; [
-            bash
-            coreutils
-            git
-            nodejs
-          ];
-          serviceConfig = {
-            Restart = "always";
-            RestartSec = "5s";
-            WorkingDirectory = "/var/lib/camofox-browser";
-          };
-          script = ''
-            set -eu
-
-            app_dir=/var/lib/camofox-browser/app
-
-            if [ ! -d "$app_dir/.git" ]; then
-              git clone --depth 1 https://github.com/jo-inc/camofox-browser "$app_dir"
-            fi
-
-            cd "$app_dir"
-
-            if [ ! -d node_modules ]; then
-              npm install
-            fi
-
-            exec npm start
-          '';
-        };
-
-        system.stateVersion = lib.mkDefault "24.05";
-      };
-  };
-}

systems/jeeves/services/gems.nix

@@ -6,8 +6,47 @@
 let
   vars = import ../vars.nix;
   stateDir = "${vars.services}/gems";
+  gemsPackages =
+    ps: with ps; [
+      fastapi
+      jinja2
+      pydantic
+      pydantic-settings
+      python-multipart
+      typer
+      uvicorn
+    ];
 in
 {
+  nixpkgs.overlays = [
+    (final: _prev: {
+      gems_python = final.python314.withPackages gemsPackages;
+      gems_test_python = final.python314.withPackages (
+        ps:
+        gemsPackages ps
+        ++ (with ps; [
+          httpx
+          pytest
+          pytest-asyncio
+          pytest-xdist
+        ])
+      );
+      gems_tests =
+        final.runCommand "gems-tests"
+          {
+            nativeBuildInputs = [ final.gems_test_python ];
+          }
+          ''
+            export HOME="$TMPDIR"
+            cd ${inputs.self}
+            pytest -o cache_dir="$TMPDIR/pytest-cache" tests/gems
+            touch "$out"
+          '';
+    })
+  ];
+
+  system.checks = [ pkgs.gems_tests ];
+
   users.groups.gems = { };
   users.users.gems = {
     isSystemUser = true;
@@ -36,7 +75,7 @@ in
       Type = "simple";
       User = "gems";
       Group = "gems";
-      ExecStart = "${pkgs.my_python}/bin/python -m python.gems.main --host 127.0.0.1 --port 8002";
+      ExecStart = "${pkgs.gems_python}/bin/python -m python.gems.main --host 0.0.0.0 --port 8002";
       Restart = "on-failure";
       RestartSec = "5s";
       StandardOutput = "journal";

systems/jeeves/services/gitea.nix

@@ -2,11 +2,6 @@ let
   vars = import ../vars.nix;
 in
 {
-  networking.firewall.allowedTCPPorts = [
-    6443
-    2223
-  ];
-
   services.gitea = {
     enable = true;
     appName = "TMM Workshop";
@@ -24,13 +19,17 @@ in
       actions = {
         ENABLED = true;
         DEFAULT_ACTIONS_URL = "github";
+        ENDLESS_TASK_TIMEOUT = "12h";
       };
       service.DISABLE_REGISTRATION = true;
       server = {
-        DOMAIN = "tmmworkshop.com";
         ROOT_URL = "https://gitea.tmmworkshop.com/";
         HTTP_PORT = 6443;
+        BUILTIN_SSH_SERVER_USER = "gitea";
+        SSH_USER = "gitea";
+        SSH_DOMAIN = "gitea.tmmworkshop.com";
         SSH_PORT = 2223;
+        SSH_LISTEN_HOST = "0.0.0.0";
         SSH_LISTEN_PORT = 2223;
         START_SSH_SERVER = true;
         PUBLIC_URL_DETECTION = "auto";

systems/jeeves/services/grafana.nix

@@ -6,8 +6,6 @@ let
   grafanaDataDir = "${vars.services}/grafana";
 in
 {
-  networking.firewall.allowedTCPPorts = [ 3000 ];
-
   services.grafana = {
     enable = true;
     dataDir = grafanaDataDir;
@@ -19,9 +17,9 @@ in
         secret_key = "$__file{${vars.secrets}/services/grafana/secret_key}";
       };
       server = {
-        http_addr = "192.168.90.40";
+        http_addr = "0.0.0.0";
         http_port = 3000;
-        root_url = "http://192.168.90.40:3000/";
+        root_url = "http://jeeves:3000/";
       };
     };
     provision = {

systems/jeeves/services/jellyfin.nix

@@ -4,7 +4,6 @@ in
 {
   services.jellyfin = {
     enable = true;
-    openFirewall = true;
     dataDir = "${vars.services}/jellyfin";
     cacheDir = "${vars.services}/jellyfin/cache";
   };

systems/jeeves/services/jellyswarrm.nix

@@ -0,0 +1,16 @@
+{ inputs, ... }:
+let
+  vars = import ../vars.nix;
+in
+{
+  imports = [ inputs.jellyswarrm.nixosModules.default ];
+
+  services.jellyswarrm = {
+    enable = true;
+    host = "0.0.0.0";
+    port = 3100;
+    dataDir = "${vars.services}/jellyswarrm";
+    username = "admin";
+    passwordFile = "${vars.secrets}/services/jellyswarrm-password";
+  };
+}

systems/jeeves/services/llms.nix

@@ -38,7 +38,6 @@ in
       "qwen3.5:35b"
     ];
     modelsDir = vars.ollama;
-    openFirewall = true;
   };
   systemd.services = {
     ollama.serviceConfig = {

systems/jeeves/services/nix_serve.nix

@@ -7,6 +7,5 @@ in
     package = pkgs.nix-serve-ng;
     enable = true;
     secretKeyFile = "${vars.secrets}/services/nix-cache/cache-priv-key.pem";
-    openFirewall = true;
   };
 }

systems/jeeves/services/open_webui.nix

@@ -1,13 +0,0 @@
-{
-  services.open-webui = {
-    enable = true;
-    host = "0.0.0.0";
-    openFirewall = true;
-    environment = {
-      ANONYMIZED_TELEMETRY = "False";
-      DO_NOT_TRACK = "True";
-      SCARF_NO_ANALYTICS = "True";
-      OLLAMA_API_BASE_URL = "http://127.0.0.1:11434";
-    };
-  };
-}

systems/jeeves/services/postgress.nix

@@ -3,8 +3,6 @@ let
   vars = import ../vars.nix;
 in
 {
-  networking.firewall.allowedTCPPorts = [ 5432 ];
-
   # Symlink pg_wal to a ZFS dataset on the special (metadata) vdev for fast WAL writes
   # this is required for systemd sandboxing
   systemd.services.postgresql.serviceConfig.ReadWritePaths = [ "/zfs/media/database/postgres-wal" ];
@@ -27,7 +25,7 @@ in
       local all  richie   trust
       host  all  richie   127.0.0.1/32    trust
       host  all  richie   ::1/128         trust
-      host  all  richie   192.168.90.1/24 trust
+      host  all  richie   100.64.0.0/10 trust
       host  all  richie   192.168.99.1/24 trust
       host  all  richie   172.16.0.0/12   trust
 
@@ -42,12 +40,12 @@ in
       local postgres  math   trust
       host  postgres  math   127.0.0.1/32    trust
       host  postgres  math   ::1/128         trust
-      host  postgres  math   192.168.90.1/24 trust
+      host  postgres  math   100.64.0.0/10 trust
 
       local data_science_dev  math   trust
       host  data_science_dev  math   127.0.0.1/32    trust
       host  data_science_dev  math   ::1/128         trust
-      host  data_science_dev  math   192.168.90.1/24 trust
+      host  data_science_dev  math   100.64.0.0/10 trust
     '';
 
     identMap = ''