feat(ci): add dedicated Nix cache prebuild runner

Extract the header fix into a standalone patch and colocate the
Electron, Deno, and Signal overrides in patches/abseil. Preserve the
x86-64-v3 restriction and document the accepted upstream fix while
waiting for bundled dependency updates.

Import the Abseil overrides through the patches overlay and register
the torchcodec override. Leave PostgreSQL output checks in
x86-64-v3-workarounds.nix.

.github/workflows/build_systems.yml

@@ -8,8 +8,23 @@ on:
     - cron: "0 22 * * *"
 
 jobs:
+  prebuild-common:
+    name: prebuild-common-x86-64-v3
+    runs-on: nix-cache-builder
+    steps:
+      - uses: actions/checkout@v4
+      # portal-1 is the smallest system closure: 95% of its derivations are
+      # shared by all five systems, so it is a maintainable common cache seed.
+      # Keep going so one failing package does not stop unrelated cache entries
+      # from being built.
+      - name: Build common packages
+        run: nixos-rebuild build --keep-going --accept-flake-config --flake ./#portal-1
+      - name: Copy common packages to nix-cache
+        run: nix copy --accept-flake-config --to unix:///host-nix/var/nix/daemon-socket/socket .#nixosConfigurations.portal-1.config.system.build.toplevel
+
   build:
     name: build-${{ matrix.system }}
+    needs: prebuild-common
     runs-on: self-hosted
     strategy:
       matrix:
@@ -18,6 +33,7 @@ jobs:
           - "brain"
           - "jeeves"
           - "rhapsody-in-green"
+          - "portal-1"
     continue-on-error: true
     steps:
       - uses: actions/checkout@v4

.vscode/settings.json

@@ -10,7 +10,6 @@
     "aiounifi",
     "alsa",
     "apiclient",
-    "apscheduler",
     "archlinux",
     "ashift",
     "asrouter",
@@ -337,8 +336,6 @@
     "yubioath",
     "yzhang",
     "zeroconf",
-    "zerotier",
-    "zerotierone",
     "zoxide",
     "zram",
     "zstd"

common/global/nix.nix

@@ -31,6 +31,10 @@ in
         "flakes"
         "ca-derivations"
       ];
+      system-features = lib.mkAfter [
+        "gccarch-x86-64-v2"
+        "gccarch-x86-64-v3"
+      ];
       warn-dirty = false;
       flake-registry = ""; # disable global flake registries
       connect-timeout = 10;

common/optional/docker.nix

@@ -17,6 +17,7 @@
     logDriver = "local";
     storageDriver = "overlay2";
     daemon.settings = {
+      live-restore = false;
       experimental = true;
       exec-opts = [ "native.cgroupdriver=systemd" ];
       log-opts = {

common/optional/x86-64-v3.nix

@@ -0,0 +1,9 @@
+{
+  nixpkgs.hostPlatform = {
+    system = "x86_64-linux";
+    gcc = {
+      arch = "x86-64-v3";
+      tune = "generic";
+    };
+  };
+}

common/optional/zfs/snapshot_config.toml

@@ -22,6 +22,12 @@ hourly = 0
 daily = 0
 monthly = 0
 
+["root_pool/nix_build"]
+15_min = 1
+hourly = 0
+daily = 0
+monthly = 0
+
 ["root_pool/var"]
 15_min = 8
 hourly = 24

flake.lock

@@ -20,28 +20,6 @@
         "type": "github"
       }
     },
-    "firefox-addons": {
-      "inputs": {
-        "nixpkgs": [
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "dir": "pkgs/firefox-addons",
-        "lastModified": 1788494578,
-        "narHash": "sha256-u++Hq1RJDYMvQw2WEBFlW029n5Y9RimOpYrsdK8V/EY=",
-        "owner": "rycee",
-        "repo": "nur-expressions",
-        "rev": "f6f8cd6a1d31925cbf0e0e8b6420321b9bd5790e",
-        "type": "gitlab"
-      },
-      "original": {
-        "dir": "pkgs/firefox-addons",
-        "owner": "rycee",
-        "repo": "nur-expressions",
-        "type": "gitlab"
-      }
-    },
     "home-manager": {
       "inputs": {
         "nixpkgs": [
@@ -49,11 +27,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1788487777,
-        "narHash": "sha256-Ro/e1N4ZR8/XaFF+sF9SgfPK79HM5YNEppzFj8p+0Ak=",
+        "lastModified": 1788651960,
+        "narHash": "sha256-v9wJd32eZ2bvhBzVOd7TIjLQd011P7nwOhjKtWlci5I=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "693e8ce0fb240a73c116a03cfd7b19269c87af88",
+        "rev": "2c0350c759688177331b8f5242311fae8877bdb3",
         "type": "github"
       },
       "original": {
@@ -67,11 +45,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1788335262,
-        "narHash": "sha256-3jBEq8avfzlrsY4UpW4d5TOmm7ocseZfnitEJhqauyc=",
+        "lastModified": 1788860136,
+        "narHash": "sha256-MhPMOFV4pVkygWEbQ8t1De/uQ9cWF1u++tRe2L5tG48=",
         "owner": "nixos",
         "repo": "nixos-hardware",
-        "rev": "44d95795ee2d475b3d687325e26dcf4ca9104557",
+        "rev": "62173785b9a18c78b4a15aca2623d02bceb9d077",
         "type": "github"
       },
       "original": {
@@ -96,11 +74,11 @@
     },
     "nixpkgs-master": {
       "locked": {
-        "lastModified": 1788564290,
-        "narHash": "sha256-5aciwtB/CEwIz6iC7pX2l++MFzTvinNT7BXF+AqOgfU=",
+        "lastModified": 1788892992,
+        "narHash": "sha256-cIMFh9gyU4/aLeB3JCcsWM3tTAvD9pAq9Smr1Wa8aIU=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "e361c89231b87a289715d7580619082182583241",
+        "rev": "dff6994123e257ec9901c271bc2b52e64d7c8f05",
         "type": "github"
       },
       "original": {
@@ -128,11 +106,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1788404924,
-        "narHash": "sha256-lhEhY8X5EgkQ/eg6IFz4cc8jRuSYSvnxx1al7d1dvZ0=",
+        "lastModified": 1788752844,
+        "narHash": "sha256-VaWGJ6+cIYN2erfSecbRV+4ljI185Ty2wUrXyvQbgOw=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "0968519e14f7aa7d3e9b389682bd74d2b51c8ce8",
+        "rev": "dc5d91f840324650bac8c379428c7037a416959a",
         "type": "github"
       },
       "original": {
@@ -145,7 +123,6 @@
     "root": {
       "inputs": {
         "disko": "disko",
-        "firefox-addons": "firefox-addons",
         "home-manager": "home-manager",
         "nixos-hardware": "nixos-hardware",
         "nixpkgs": "nixpkgs_2",

flake.nix

@@ -26,11 +26,6 @@
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
-    firefox-addons = {
-      url = "gitlab:rycee/nur-expressions?dir=pkgs/firefox-addons";
-      inputs.nixpkgs.follows = "nixpkgs";
-    };
-
     sops-nix = {
       url = "github:Mic92/sops-nix";
       inputs.nixpkgs.follows = "nixpkgs";

overlays/default.nix

@@ -15,12 +15,15 @@
     };
   };
 
+  patches = import ./patches;
+  test-exclusions = import ./test-exclusions.nix;
+  # x86-64-v3-workarounds = import ./x86-64-v3-workarounds.nix;
+
   python-env = final: _prev: {
     my_python = final.python314.withPackages (
       ps: with ps; [
         alembic
         apprise
-        apscheduler
         fastapi
         fastapi-cli
         httpx

overlays/patches/README.md

@@ -0,0 +1,46 @@
+# Package patches
+
+Each package follows the [GnuTLS layout](gnutls/README.md):
+
+- `default.nix` applies the patch through the package overlay.
+- A descriptive `.patch` file contains the standalone upstream source change.
+- `README.md` explains the problem, scope, reproduction, upstream status,
+  Nix integration, and recorded validation limits.
+- Companion `verify-*` tools live beside the patch when needed; otherwise
+  the README gives commands for the package's existing tests.
+
+Keep package-specific evidence in its directory. Patch headers explain the
+change independently of Nix, and `default.nix` preserves existing patches.
+
+| Package | Repair |
+| --- | --- |
+| [Abseil](abseil/README.md) | Public BMI2 header in Electron, Deno, and Signal's vendored copies |
+| [Backrefs](backrefs/README.md) | Match the regex timeout's CPU clock |
+| [GnuTLS](gnutls/README.md) | Wait for the UDP server socket before connecting |
+| [Jupyter Server](jupyter-server/README.md) | Exercise the correct shared future during reconnect |
+| [Prometheus](prometheus/README.md) | Complete parsing before inspecting the test editor state |
+| [pytest-xdist](pytest-xdist/README.md) | Check worker replacements despite concurrent crashes |
+| [SciPy](scipy/README.md) | Account for floating-point rounding in STFT tests |
+| [Sentry SDK](sentry-sdk/README.md) | Isolate SDK thread mocks from Python's threading module |
+| [Torchaudio](torchaudio/README.md) | Compare pitch-shift batches at appropriate precision |
+| [TorchCodec](torchcodec/README.md) | Match the reference MP3 encoder's sample format |
+
+## Local NixOS integration
+
+[`../default.nix`](../default.nix) imports this directory's
+[`default.nix`](default.nix), which wires each package's override into the
+package set. Abseil repairs several vendored copies and is gated on
+`x86-64-v3`; Prometheus patches its separate assets derivation; Python
+packages use `pythonPackagesExtensions`.
+
+[`../test-exclusions.nix`](../test-exclusions.nix) retains only pytest-xdist's
+outer-worker limit and inner-worker startup allowance. It adds no skipped
+tests. Existing nixpkgs exclusions remain separate from these repairs.
+
+The test-exclusion review used Python 3.14.7 and the pinned x86-64-v3 package
+set. Host-flake evaluation verified patch wiring, Python install checks,
+removal of the local skips, and Prometheus's reference to the patched assets.
+Jupyter and Sentry package tests used the preceding dependency set with the
+new package patch to avoid unrelated rebuilds after pytest-xdist changed.
+No complete NixOS rebuild was performed. Individual READMEs distinguish
+package builds, focused tests, and checks that have not been run.

overlays/patches/abseil/README.md

@@ -0,0 +1,59 @@
+# Abseil BMI2 public header
+
+Vendored Abseil includes `bmi2intrin.h` directly when `__BMI2__` is enabled.
+Compilers reject that internal header without the umbrella-header setup.
+`bmi2-public-header.patch` includes `immintrin.h` instead, allowing builds
+that enable BMI2 through `-march=x86-64-v3`.
+
+## Scope and behavior
+
+The patch changes one include in
+`third_party/abseil-cpp/absl/container/internal/raw_hash_set.h`.
+`default.nix` applies it to Electron 43's unwrapped package, Deno's
+`librusty_v8`, and Signal's WebRTC dependency. It also supplies the patched
+Electron package to Signal. These overrides apply only to `x86-64-v3`.
+
+The shared file path is relative to each vendoring project's source root,
+not the root of a standalone Abseil checkout. No hash-table algorithm or
+test exclusion changes.
+
+## Reproduction and focused checks
+
+From this directory, check and apply the patch to each vendored source tree:
+
+```sh
+patch --dry-run --fuzz=0 -d /path/to/vendor-source -p1 < bmi2-public-header.patch
+patch --fuzz=0 -d /path/to/vendor-source -p1 < bmi2-public-header.patch
+```
+
+A small compiler check isolates the header requirement. With GCC or Clang
+on x86-64, compile `#include <bmi2intrin.h>` using `-march=x86-64-v3`; the
+compiler rejects the direct include. Changing it to `#include <immintrin.h>`
+should compile. The full consumer builds below check integration with their
+actual toolchains.
+
+## Upstream status
+
+Abseil addressed this issue through
+[PR #2071](https://github.com/abseil/abseil-cpp/pull/2071), imported by its
+upstream workflow. That change uses `x86gprintrin.h`; this local variant uses
+the public `immintrin.h` umbrella header for the vendored toolchains.
+Keep the workaround until all three bundled copies include a compatible fix.
+This file is a local adaptation, not a verbatim copy of the upstream diff.
+
+## Local NixOS integration and build results
+
+[`../default.nix`](../default.nix) merges this directory's overlay fragment
+because it repairs multiple packages. From the repository root, the consumer
+build commands are:
+
+```sh
+nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.deno
+nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.electron_43
+nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.signal-desktop
+```
+
+The earlier extraction checked the vendored header snapshots and evaluated
+all three patch attachments. Those records do not establish successful full
+consumer rebuilds. No new compiler or consumer build was run for the layout
+change; the patch and override are unchanged.

overlays/patches/abseil/bmi2-public-header.patch

@@ -0,0 +1,20 @@
+Subject: [PATCH] abseil: include BMI2 intrinsics through the public header
+
+GCC and Clang reject direct inclusion of bmi2intrin.h. Include immintrin.h
+instead so that the compiler supplies the required intrinsic setup when
+BMI2 is enabled, including builds targeting x86-64-v3.
+
+This patch is shared by the vendored Abseil copies in Electron, rusty_v8
+(Deno), and Signal's WebRTC build.
+
+--- a/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
++++ b/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
+@@ -226,7 +226,7 @@
+ #endif
+ 
+ #ifdef __BMI2__
+-#include <bmi2intrin.h>
++#include <immintrin.h>
+ #endif  // __BMI2__
+ 
+ namespace absl {

overlays/patches/abseil/default.nix

@@ -0,0 +1,38 @@
+# Abseil accepted the upstream fix: https://github.com/abseil/abseil-cpp/pull/2071
+# Keep this workaround until Electron, Deno's rusty_v8, and Signal's WebRTC
+# update their bundled Abseil copies to include it.
+{ prev }:
+let
+  patchAbseilBmi2Include =
+    package:
+    package.overrideAttrs (old: {
+      # GCC and Clang require the public umbrella header for BMI2 intrinsics.
+      patches = (old.patches or [ ]) ++ [ ./bmi2-public-header.patch ];
+    });
+
+  electron43Unwrapped = patchAbseilBmi2Include prev.electron_43.unwrapped;
+  electron43 = prev.electron_43.override {
+    electron-unwrapped = electron43Unwrapped;
+  };
+
+  signalCallPackage =
+    path: args:
+    let
+      package = prev.callPackage path args;
+    in
+    if builtins.baseNameOf path == "webrtc.nix" then patchAbseilBmi2Include package else package;
+in
+prev.lib.optionalAttrs ((prev.stdenv.hostPlatform.gcc.arch or null) == "x86-64-v3") {
+  deno =
+    let
+      librusty_v8 = patchAbseilBmi2Include prev.deno.passthru.librusty_v8;
+    in
+    prev.deno.override { inherit librusty_v8; };
+
+  electron_43 = electron43;
+
+  signal-desktop = prev.signal-desktop.override {
+    electron_43 = electron43;
+    callPackage = signalCallPackage;
+  };
+}

overlays/patches/default.nix

@@ -0,0 +1,15 @@
+_final: prev:
+(import ./abseil { inherit prev; })
+// {
+  gnutls = import ./gnutls { inherit (prev) gnutls; };
+  prometheus = import ./prometheus { inherit (prev) prometheus; };
+
+  pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
+    (_pythonFinal: pythonPrev: {
+      backrefs = import ./backrefs { inherit (pythonPrev) backrefs; };
+      pytest-xdist = import ./pytest-xdist { inherit (pythonPrev) pytest-xdist; };
+      sentry-sdk = import ./sentry-sdk { inherit (pythonPrev) sentry-sdk; };
+      torchcodec = import ./torchcodec { inherit (pythonPrev) torchcodec; };
+    })
+  ];
+}

overlays/patches/gnutls/README.md

@@ -0,0 +1,160 @@
+# GnuTLS UDP server readiness
+
+Under load, the test client can start before `gnutls-serv` binds its UDP
+socket, and the first handshake fails with `Connection refused`.
+`serv-udp.sh` currently waits a fixed four seconds; elapsed time does not
+establish server readiness. `udp-server-readiness.patch` replaces that wait
+with polling for the local IPv4 UDP endpoint.
+
+## Scope and waiting behavior
+
+The patch changes the existing `wait_udp_server()` and adds a new
+`check_if_udp_port_bound()` beside it in `tests/scripts/common.sh`.
+`serv-udp.sh` is its only caller in 3.8.13. The TCP helpers `wait_server()`
+and `wait_for_port()`, including their existing sleeps, are unchanged.
+Both original DTLS handshake checks remain unchanged.
+
+Each iteration checks process liveness and the socket **before sleeping**.
+A ready socket returns immediately. An unsuccessful check sleeps two
+seconds only if another attempt remains: at most 90 attempts, consistent
+with the existing `wait_server()` budget implemented by `wait_for_port()`,
+with no sleep after the final check. Server exit fails early; exhausting the
+budget fails and terminates the server. No handshake is retried, and no
+protocol timeout is changed. Once bound, the kernel can queue datagrams
+while the server is scheduled; the probe itself sends no packets.
+
+The existing `have_port_finder()` prefers `ss`, then `netstat`. If neither
+exists, it prints `neither ss nor netstat found` and exits **77 (skip)**.
+In the normal test flow, port selection calls it before launching a server.
+The probe runs in a subshell so that, even if this skip occurs after launch,
+the waiting helper can terminate and reap the server before exiting 77.
+
+## Why an IPv4 socket is expected
+
+This is specific to the server used by this test, not a general rule that
+IPv6 sockets cannot serve IPv4 clients. The client explicitly uses
+`127.0.0.1`. The server's `--udp` path calls `udp_server()`, which calls
+`listen_socket(..., SOCK_DGRAM)`. That function iterates the wildcard
+addresses returned by `getaddrinfo(NULL, ..., AI_PASSIVE)`:
+
+| Server build / Linux setting | Binding behavior |
+| --- | --- |
+| IPv6 enabled, `net.ipv6.bindv6only=0` | Requests `IPV6_V6ONLY=1` on the IPv6 socket, binds `[::]:PORT`, and separately binds `0.0.0.0:PORT`. It overrides the system's dual-stack default. |
+| IPv6 enabled, `net.ipv6.bindv6only=1` | The same explicit socket option and separate IPv4/IPv6 binds. |
+| `HAVE_IPV6` undefined | Skips every address family except `AF_INET`; only the IPv4 wildcard is attempted. |
+
+`udp_server()` uses `wait_for_connection()`, which puts **every listener**
+from that list into `select()` and returns a readable socket for `recvfrom()`;
+it does not permanently choose one socket based on `getaddrinfo()` order.
+
+The first two cases were traced with the actual GnuTLS 3.8.13 binary in
+separate Linux network namespaces: `setsockopt(IPV6_V6ONLY, [1])` and both
+UDP binds returned success under each setting. The no-IPv6 case was checked
+in source, not by building a second binary. The same bind implementation
+was checked directly on GitLab master.
+
+Thus, successful normal startup for this invocation provides an explicit
+IPv4 socket; a lone IPv6 wildcard is not the expected success path.
+There is one portability caveat: upstream discards the return value of
+`setsockopt(IPV6_V6ONLY)`. On a platform where that call fails and the server
+ends up with only a dual-stack socket, this helper would time out despite
+IPv4 reachability. Such a platform needs additional handling before this
+patch can claim support. Blindly accepting every IPv6 wildcard would also
+accept IPv6-only sockets before the separate IPv4 bind finishes.
+
+Source: [`src/serv.c`, `listen_socket()`](https://gitlab.com/gnutls/gnutls/-/blob/master/src/serv.c#L937),
+[`src/udp-serv.c`](https://gitlab.com/gnutls/gnutls/-/blob/master/src/udp-serv.c),
+and [`tests/serv-udp.sh`](https://gitlab.com/gnutls/gnutls/-/blob/master/tests/serv-udp.sh).
+
+## Port matching and ownership limit
+
+Only `-an` is passed to the socket-listing tool: BSD `netstat -u` selects
+Unix-domain sockets, whereas Linux `netstat -u` selects UDP. The parser
+handles the extra state column in `ss`, Linux colon-separated endpoints,
+and BSD dot-separated endpoints, including `*.PORT`. It matches the full
+local port and rejects TCP, IPv6 entries, peer ports, and longer numbers.
+
+A live PID plus a bound port does **not** prove that PID owns the socket.
+Existing `GETPORT` selection checks for an unused port and uses a test
+port-lock directory; `launch_bare_server()` also calls
+`wait_for_free_port()` before starting the process. These are advisory:
+the launcher does not enforce the latter's result, and another process
+can bind between the check and launch. The patch does not close that race
+or add nonportable PID parsing. An unrelated process can satisfy the
+socket check; the real handshakes remain the functional check and may
+fail (or reach the wrong server). This is a startup-order fix, not a
+socket-ownership guarantee.
+
+## Reproduction and focused checks
+
+Apply the patch to an unpacked source tree, then run the companion checks
+with Python's standard library and a shell:
+
+```sh
+patch --fuzz=0 -d /path/to/gnutls -p1 < udp-server-readiness.patch
+SHELL=/bin/sh python3 verify-readiness.py /path/to/gnutls/tests/scripts/common.sh -v
+```
+
+Set `NETSTAT=/path/to/netstat` to exercise one outside `PATH`. The checks
+cover Linux/BSD output samples, false matches, immediate readiness,
+missing tools, process exit, timeout cleanup, and real IPv4 UDP sockets
+whose bind is delayed six seconds. The missing-tools fixture is skipped
+if an absolute fallback `ss` path cannot be hidden with `PATH`. Native
+BSD execution remains untested.
+
+To reproduce with GnuTLS itself, run `tests/serv-udp.sh` with `SERV` pointing
+to a wrapper that sleeps six seconds, then `exec`s `gnutls-serv` with all
+arguments. Set `CLI` to the matching `gnutls-cli`, `srcdir` to the source
+`tests` directory, and `abs_top_builddir` to a writable build directory.
+With GnuTLS 3.8.13, the original helper failed the first handshake with
+`Connection refused`; the patched helper passed both with the same binaries.
+
+## GnuTLS submission
+
+Development and merge requests are on [GitLab](https://gitlab.com/gnutls/gnutls).
+[`CONTRIBUTING.md` on master](https://gitlab.com/gnutls/gnutls/-/blob/master/CONTRIBUTING.md)
+was read directly for this review. It requires the contributor's DCO
+`Signed-off-by`, successful and failure test coverage, consistent coding
+style, and adequate documentation; GitLab CI runs for merge requests.
+Its commenting guidance asks for comments explaining non-obvious behavior
+or protocol expectations. It does not prescribe an additional special
+test-suite comment. The patch now explains its IPv4 binding assumption
+next to the probe.
+
+The submission will contain the shell patch, without the Python verifier
+or a new Python test dependency. The existing `serv-udp.sh` supplies the
+functional success check. Running it through the six-second startup
+wrapper supplies a reproducible regression case: it fails before the fix
+and passes after it. The local verifier was used to validate socket-output
+parsing and the helper's success, process-exit, skip-cleanup, and timeout
+branches. Those branch checks are local evidence, not new automated
+coverage in the upstream suite; the MR must state that distinction.
+
+No dedicated unit-test harness for these shell helpers was found in the
+3.8.13 tests inspected. That does not establish that Python cannot be used
+upstream; keeping this submission dependency-free is a scope choice. Use
+the existing test and before/after reproduction as the submission's
+coverage argument, retaining the platform limitations above. Apply the
+patch in an upstream checkout and include those results with the
+contributor's own sign-off. No MR or sign-off has been created.
+
+## Local NixOS integration and build results
+
+`overlays/default.nix` imports the `overlays/patches` overlay, which loads
+`gnutls/default.nix` to apply the patch and keep `serv-udp.sh` enabled.
+The patch itself has no Nix dependencies and applies to 3.8.13 and GitLab
+master without fuzz.
+
+The final patch was rebuilt with:
+
+```sh
+nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.gnutls
+```
+
+That x86-64-v3 build passed: 927 tests, 796 passes, 131 existing skips,
+zero failures/errors, and `PASS: serv-udp.sh`. The patch bytes in the built
+derivation were compared with the repository artifact; both have SHA-256
+`59013d47fd446f2dd065012a2259ccc1898fedc8a053a630e13efa0076368760`.
+All seven local checks passed, including skip cleanup and exactly 90
+probes with 89 sleeps on timeout. The six-second before/after reproduction
+was also repeated successfully with the final helper.

overlays/patches/gnutls/default.nix

@@ -0,0 +1,6 @@
+{ gnutls }:
+gnutls.overrideAttrs (old: {
+  # Keep the UDP handshake test enabled on loaded builders by waiting for
+  # the server to bind its socket. Kept as a standalone patch for upstream.
+  patches = (old.patches or [ ]) ++ [ ./udp-server-readiness.patch ];
+})

overlays/patches/gnutls/udp-server-readiness.patch

@@ -0,0 +1,70 @@
+Subject: [PATCH] tests: wait for the UDP server socket before connecting
+
+A fixed four-second sleep does not guarantee that gnutls-serv has bound
+its UDP socket on a busy builder. Poll the local IPv4 UDP endpoint using
+the existing ss/netstat discovery, with the same retry budget as the TCP
+helper. Fail early if the server exits, and retain the original handshake
+checks in serv-udp.sh.
+
+Use flags common to ss and BSD/Linux netstat. Match the local endpoint
+and complete port number, excluding TCP, IPv6-only and peer endpoints.
+
+--- a/tests/scripts/common.sh
++++ b/tests/scripts/common.sh
+@@ -185,10 +185,55 @@
+ 	fi
+ }
+ 
++check_if_udp_port_bound() {
++	local PORT=$1
++	have_port_finder
++	# Use only -an, which is shared by ss and BSD/Linux netstat. UDP has
++	# no LISTEN state. Match the local IPv4 endpoint, not a peer port or
++	# a longer port number. serv-udp.sh connects to 127.0.0.1;
++	# listen_socket() in serv.c binds IPv4 separately and requests
++	# IPV6_V6ONLY=1 for its IPv6 socket.
++	$PFCMD -an | awk -v port="$PORT" '
++		$1 == "udp" || $1 == "udp4" {
++			# ss includes a state column; netstat does not.
++			address = ($2 == "UNCONN" || $2 == "ESTAB") ? $5 : $4
++			if (address ~ ("^[0-9.]+[.:]" port "$") ||
++			    address == "*." port)
++				found = 1
++		}
++		END { exit !found }
++	'
++}
++
+ wait_udp_server() {
+ 	local PID=$1
++	local ret
+ 	trap "test -n \"${PID}\" && kill ${PID};exit 1" 1 15 2
+-	sleep 4
++	local i=0
++	# Use the same retry budget as wait_for_port(), but also stop if the
++	# server exits before binding its socket.
++	while test $i -lt 90; do
++		if ! kill -0 "$PID" 2>/dev/null; then
++			fail "" "UDP server $PID exited before binding port $PORT"
++		fi
++		# Contain have_port_finder's exit so a skip also stops the server.
++		if (check_if_udp_port_bound "$PORT"); then
++			return 0
++		else
++			ret=$?
++			if test "$ret" = 77; then
++				kill "$PID" 2>/dev/null || :
++				wait "$PID" 2>/dev/null || :
++				exit 77
++			fi
++		fi
++		i=$((i + 1))
++		if test $i -lt 90; then
++			echo "try $i: waiting for UDP port $PORT"
++			sleep 2
++		fi
++	done
++	fail "$PID" "UDP server $PORT did not come up"
+ }
+ 
+ create_testdir() {

overlays/patches/gnutls/verify-readiness.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""Exercise patched common.sh without building GnuTLS (Python standard library only).
+
+Usage: python3 verify-readiness.py /path/to/patched/tests/scripts/common.sh
+Set SHELL to test another shell, and NETSTAT to test a netstat outside PATH.
+"""
+
+# Use unittest so this upstream companion tool needs no pytest installation.
+# ruff: noqa: PT009
+
+import os
+import shutil
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+from pathlib import Path
+
+COMMON = str(Path(sys.argv.pop(1)).resolve())
+SHELL = os.environ.get("SHELL", "/bin/sh")
+
+
+class ReadinessTests(unittest.TestCase):
+    """Check endpoint parsing and the server startup lifecycle."""
+
+    def setUp(self) -> None:
+        """Create a socket-listing fixture for each check."""
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.root = Path(self.tmp.name)
+        self.fixture = self.root / "sockets"
+        self.fixture.write_text("")
+        self.finder = self.root / "port-finder"
+        self.finder.write_text('#!/bin/sh\ncat "$SOCKET_FIXTURE"\n')
+        self.finder.chmod(0o755)
+
+    def run_shell(self, body: str, **env: str) -> subprocess.CompletedProcess[str]:
+        """Source the actual helper and run a shell scenario."""
+        return subprocess.run(
+            [SHELL, "-c", '. "$COMMON"\n' + body],
+            env={
+                **os.environ,
+                "COMMON": COMMON,
+                "SOCKET_FIXTURE": str(self.fixture),
+                "PFCMD": str(self.finder),
+                "PORT": "12345",
+                **env,
+            },
+            capture_output=True,
+            text=True,
+            timeout=20,
+            check=False,
+        )
+
+    def test_socket_formats_and_false_matches(self) -> None:
+        """Accept IPv4 UDP local endpoints and reject unrelated sockets."""
+        cases = [
+            ("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*", True),
+            ("udp UNCONN 0 0 127.0.0.1:12345 0.0.0.0:*", True),
+            ("udp 0 0 0.0.0.0:12345 0.0.0.0:*", True),
+            ("udp4 0 0 *.12345 *.*", True),
+            ("udp 0 0 127.0.0.1.12345 *.*", True),
+            ("udp 0 0 *.12345 *.*", True),
+            ("udp UNCONN 0 0 0.0.0.0:123456 0.0.0.0:*", False),
+            ("udp 0 0 0.0.0.0:123456 0.0.0.0:*", False),
+            ("udp ESTAB 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
+            ("udp 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
+            ("tcp LISTEN 0 128 0.0.0.0:12345 0.0.0.0:*", False),
+            ("tcp 0 0 0.0.0.0:12345 0.0.0.0:* LISTEN", False),
+            ("udp UNCONN 0 0 [::]:12345 [::]:*", False),
+            ("udp UNCONN 0 0 *:12345 *:*", False),
+            ("udp6 0 0 :::12345 :::*", False),
+            ("udp6 0 0 *.12345 *.*", False),
+            ("", False),
+        ]
+        for row, ready in cases:
+            with self.subTest(row=row):
+                self.fixture.write_text(row + "\n")
+                result = self.run_shell('check_if_udp_port_bound "$PORT"')
+                self.assertEqual(result.returncode, 0 if ready else 1, result.stderr)
+
+    def test_exited_server_fails_immediately(self) -> None:
+        """Fail without sleeping when the server has already exited."""
+        result = self.run_shell(
+            'true &\npid=$!\nwait "$pid"\nsleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$pid"'
+        )
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("exited before binding", result.stderr)
+        self.assertNotIn("unexpected sleep", result.stderr)
+
+    def test_ready_socket_does_not_sleep(self) -> None:
+        """Check readiness before the first sleep."""
+        self.fixture.write_text("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*\n")
+        result = self.run_shell('sleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$$"')
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertNotIn("unexpected sleep", result.stderr)
+
+    def test_missing_port_finders_skip(self) -> None:
+        """Skip and stop the live server when no finder is available."""
+        # have_port_finder also tries these paths independently of PATH.
+        if any(os.access(f"{directory}/ss", os.X_OK) for directory in ("/sbin", "/usr/sbin", "/usr/local/sbin")):
+            self.skipTest("an absolute ss path cannot be hidden by this PATH-only fixture")
+        with subprocess.Popen(["sleep", "60"]) as server:
+            try:
+                result = self.run_shell(
+                    'unset PFCMD\nPATH=/nonexistent\nwait_udp_server "$SERVER_PID"',
+                    SERVER_PID=str(server.pid),
+                )
+                self.assertEqual(result.returncode, 77)
+                self.assertIn("neither ss nor netstat found", result.stderr)
+                server.wait(timeout=3)
+                self.assertLess(server.returncode, 0)
+            finally:
+                if server.poll() is None:
+                    server.kill()
+
+    def test_timeout_is_bounded_and_cleans_up(self) -> None:
+        """Stop polling after the retry budget and terminate the server."""
+        # Only accelerate the polling delay; keep a real live server process.
+        self.finder.write_text('#!/bin/sh\necho probe >&2\ncat "$SOCKET_FIXTURE"\n')
+        with subprocess.Popen(["sleep", "60"]) as server:
+            try:
+                result = self.run_shell(
+                    'sleep() { echo polling-sleep; }\nwait_udp_server "$SERVER_PID"',
+                    SERVER_PID=str(server.pid),
+                )
+                self.assertEqual(result.returncode, 1)
+                self.assertIn("did not come up", result.stderr)
+                self.assertEqual(result.stderr.count("probe\n"), 90)
+                self.assertEqual(result.stdout.count("polling-sleep"), 89)
+                server.wait(timeout=3)
+                self.assertLess(server.returncode, 0)
+            finally:
+                if server.poll() is None:
+                    server.kill()
+
+    def test_server_exits_while_waiting(self) -> None:
+        """Detect a startup failure that happens after polling begins."""
+        result = self.run_shell('sleep 1 &\npid=$!\nwait_udp_server "$pid"')
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("exited before binding", result.stderr)
+        self.assertIn("waiting for UDP port", result.stdout)
+
+    def test_real_socket_delayed_beyond_four_seconds(self) -> None:
+        """Wait for a real delayed bind with each installed port finder."""
+        finders = [shutil.which("ss"), os.environ.get("NETSTAT") or shutil.which("netstat")]
+        finders = [finder for finder in finders if finder]
+        if not finders:
+            self.skipTest("neither ss nor netstat available")
+        for finder in finders:
+            with self.subTest(finder=finder):
+                with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
+                    sock.bind(("127.0.0.1", 0))
+                    port = sock.getsockname()[1]
+                code = (
+                    "import socket,time,sys; time.sleep(6); "
+                    "s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); "
+                    "s.bind(('127.0.0.1',int(sys.argv[1]))); time.sleep(30)"
+                )
+                with subprocess.Popen([sys.executable, "-c", code, str(port)]) as server:
+                    try:
+                        started = time.monotonic()
+                        result = self.run_shell(
+                            'wait_udp_server "$SERVER_PID"',
+                            SERVER_PID=str(server.pid),
+                            PORT=str(port),
+                            PFCMD=finder,
+                        )
+                        self.assertEqual(result.returncode, 0, result.stderr)
+                        self.assertGreaterEqual(time.monotonic() - started, 6)
+                        self.assertIsNone(server.poll())
+                    finally:
+                        server.terminate()
+                        server.wait(timeout=3)
+
+
+if __name__ == "__main__":
+    unittest.main()

overlays/patches/prometheus/README.md

@@ -0,0 +1,66 @@
+# Prometheus complete test parsing
+
+CodeMirror gives editor-state creation a 20 ms synchronous parsing budget.
+The shared `createEditorState()` test helper can therefore return an
+incomplete syntax tree when the process is descheduled. The completion and
+vector-matching tests immediately inspect that tree.
+
+## Scope and behavior
+
+`complete-test-parsing.patch` changes only
+`module/codemirror-promql/src/test/utils-test.ts` inside `web/ui`. It completes
+the small test expression with `ensureSyntaxTree(..., Infinity)` and publishes
+the completed parse through an empty transaction so `syntaxTree(state)` sees
+it. Failure to obtain a tree raises an error.
+
+The original assertions remain enabled, including `autocomplete topk params 2`
+and `foo * on(test,blub) bar`. The unlimited budget applies to the test helper;
+production editor parsing budgets are unchanged.
+
+## Reproduction and focused checks
+
+Use a disposable Prometheus 3.14.0 checkout. The patch root is `web/ui`, matching
+the Nix assets derivation. From this directory:
+
+```sh
+patch --fuzz=0 -d /path/to/prometheus/web/ui -p1 < complete-test-parsing.patch
+cd /path/to/prometheus/web/ui
+pnpm install --frozen-lockfile
+pnpm --filter @prometheus-io/lezer-promql build
+pnpm --filter @prometheus-io/codemirror-promql test
+```
+
+To force the scheduling condition, temporarily append this clock to
+`module/codemirror-promql/setupJest.cjs` in the disposable checkout:
+
+```js
+let parseClock = 0;
+Date.now = () => (parseClock += 25);
+```
+
+Each clock read crosses the editor's initial parsing budget. Against the
+original helper, the hybrid and vector suites have 186 failures, including
+both locally excluded cases. With the patch, all 386 CodeMirror tests pass
+under that same clock. Remove the injected clock before normal builds.
+
+## Upstream status
+
+This is a standalone test-helper patch for Prometheus 3.14.0. No upstream
+submission was made during this work. Recheck the helper when updating
+Prometheus or CodeMirror, including how an ensured parse becomes visible
+through the editor state.
+
+## Local NixOS integration and build results
+
+[`../default.nix`](../default.nix) loads `default.nix`, which patches the
+separate assets derivation. It updates both `passthru.assets` and the main
+Prometheus build's reference to those assets. From the repository root:
+
+```sh
+nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.prometheus.assets
+```
+
+The full x86-64-v3 assets build passed with the normal clock, including the
+CodeMirror and UI suites. Host-flake evaluation confirmed that the main
+Prometheus derivation refers to these patched assets. The Go server package
+was not rebuilt for this test-helper change.

overlays/patches/prometheus/complete-test-parsing.patch

@@ -0,0 +1,36 @@
+Subject: [PATCH] tests: finish parsing before inspecting editor state
+
+EditorState creation has a 20 ms parsing budget. A descheduled test can
+therefore observe an incomplete tree. Finish these small test documents
+without an interactive deadline and publish the result with a transaction.
+Keep the original completion and vector-matching assertions enabled.
+
+--- a/module/codemirror-promql/src/test/utils-test.ts
++++ b/module/codemirror-promql/src/test/utils-test.ts
+@@ -13,7 +13,7 @@
+ 
+ import { parser } from '@prometheus-io/lezer-promql';
+ import { EditorState } from '@codemirror/state';
+-import { LRLanguage } from '@codemirror/language';
++import { ensureSyntaxTree, LRLanguage } from '@codemirror/language';
+ import nock from 'nock';
+ import path from 'path';
+ import { fileURLToPath } from 'url';
+@@ -23,10 +23,16 @@
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
+ 
+ export function createEditorState(expr: string): EditorState {
+-  return EditorState.create({
++  const state = EditorState.create({
+     doc: expr,
+     extensions: lightPromQLSyntax,
+   });
++  // These tests need a complete tree, independent of the editor's time budget.
++  if (!ensureSyntaxTree(state, state.doc.length, Infinity)) {
++    throw new Error('Unable to parse the test expression');
++  }
++  // Publish the completed parse so syntaxTree(state) sees it too.
++  return state.update({}).state;
+ }
+ 
+ export function mockPrometheusServer(): void {

overlays/patches/prometheus/default.nix

@@ -0,0 +1,17 @@
+{ prometheus }:
+prometheus.overrideAttrs (
+  old:
+  let
+    assets = old.passthru.assets.overrideAttrs (assetsOld: {
+      patches = (assetsOld.patches or [ ]) ++ [ ./complete-test-parsing.patch ];
+    });
+  in
+  {
+    postPatch = builtins.replaceStrings [ "${old.passthru.assets}" ] [ "${assets}" ] (
+      builtins.unsafeDiscardStringContext old.postPatch
+    );
+    passthru = old.passthru // {
+      inherit assets;
+    };
+  }
+)

overlays/patches/pytest-xdist/README.md

@@ -0,0 +1,67 @@
+# pytest-xdist concurrent worker crashes
+
+With two workers and a restart limit of three, the fourth worker crash
+requests shutdown while another test can still be running. That test may
+also crash. The original queued-work test requires exactly four failures,
+even though five failures can occur without exceeding the replacement limit.
+
+## Scope and behavior
+
+`concurrent-worker-crashes.patch` changes the assertions in
+`TestNodeFailure.test_max_worker_restart_tests_queued` in
+`testing/acceptance_test.py`. It requires exactly three replacements, four or
+five failed tests, the failed-tests exit status, the limit message, and no
+internal error. It retains the two-worker workload and ten queued tests.
+
+The existing nixpkgs pytest-9 compatibility patches remain in place.
+Production scheduling and worker-restart behavior are unchanged.
+
+## Reproduction and focused checks
+
+Use a disposable pytest-xdist 3.8.0 checkout with its test dependencies and
+the nixpkgs pytest-9 compatibility patches where required. From this directory:
+
+```sh
+patch --fuzz=0 -d /path/to/pytest-xdist -p1 < concurrent-worker-crashes.patch
+cd /path/to/pytest-xdist
+python -m pytest testing/acceptance_test.py \
+  -k test_max_worker_restart_tests_queued -q
+```
+
+Twenty unmodified runs passed during the review. To force the failing
+schedule, modify the generated crashing test in a disposable checkout to
+accept `worker_id`: make `gw3` wait for a marker created by `gw4`, and make
+`gw4` pause 0.1 seconds after creating the marker. Then both have in-flight
+tests when shutdown starts. Bound the marker wait so a reproduction failure
+cannot hang the suite. The original assertion fails on five reported
+failures; the patched test passes.
+
+## Remaining resource settings
+
+[`../../test-exclusions.nix`](../../test-exclusions.nix) runs the outer suite
+with one worker and sets the inner-worker wait to 60 seconds. These settings
+limit nested process pools and allow worker startup on loaded builders.
+
+A separate reproduction inserts an 11-second `pytest_sessionstart` delay
+into the child created by `test_basic_collect_and_runtests` in
+`testing/test_remote.py`. The original 10-second channel wait fails; the
+60-second wait passes. This is a worker-startup bound, not a product deadline.
+
+## Upstream status
+
+This is a standalone test patch for pytest-xdist 3.8.0. No upstream submission
+was made during this work. Recheck the allowed in-flight failures and
+replacement count when updating the scheduler or shutdown behavior.
+
+## Local NixOS integration and build results
+
+[`../default.nix`](../default.nix) loads `default.nix` through
+`pythonPackagesExtensions`. From the repository root:
+
+```sh
+nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.python314Packages.pytest-xdist
+```
+
+The patched x86-64-v3 package passed 185 tests, with 6 existing skips and
+10 expected failures. The forced concurrent-crash reproduction passed after
+the fix, and the focused test passed again after formatting the assertion.

overlays/patches/pytest-xdist/concurrent-worker-crashes.patch

@@ -0,0 +1,29 @@
+Subject: [PATCH] tests: count replacements when checking the worker restart limit
+
+With two workers, another in-flight test may crash after the fourth
+crash requests shutdown. Either four or five failed tests is valid.
+Require exactly three replacements and the failed-tests exit status,
+while preserving the queued-work and no-internal-error assertions.
+
+--- a/testing/acceptance_test.py
++++ b/testing/acceptance_test.py
+@@ -1011,9 +1011,18 @@
+                 "worker*crashed while running*",
+                 "worker*crashed while running*",
+                 "* xdist: maximum crashed workers reached: 3 *",
+-                "* 4 failed in *",
+             ]
+         )
++        # A second in-flight test may crash after shutdown is requested.
++        # The restart limit constrains replacements, not concurrent failures.
++        replacements = sum(
++            line.startswith("replacing crashed worker ") for line in res.stdout.lines
++        )
++        assert replacements == 3
++        failed = res.parseoutcomes()["failed"]
++        assert failed in (4, 5)
++        res.assert_outcomes(failed=failed)
++        assert res.ret == pytest.ExitCode.TESTS_FAILED
+         assert "INTERNALERROR" not in res.stdout.str()
+ 
+     def test_max_worker_restart_die(self, pytester: pytest.Pytester) -> None:

overlays/patches/pytest-xdist/default.nix

@@ -0,0 +1,4 @@
+{ pytest-xdist }:
+pytest-xdist.overridePythonAttrs (old: {
+  patches = (old.patches or [ ]) ++ [ ./concurrent-worker-crashes.patch ];
+})

overlays/patches/sentry-sdk/README.md

@@ -0,0 +1,58 @@
+# Sentry SDK thread-metadata test isolation
+
+The fallback tests globally mock `threading.current_thread` while a worker
+is running. Python 3.14's `Thread.join()` also calls that function. A one-use
+mock can therefore be consumed by the wrong caller or raise `StopIteration`
+when the main thread joins the worker.
+
+## Scope and behavior
+
+`isolate-threading-mocks.patch` changes three neighboring thread-metadata
+tests in `tests/test_utils.py`, including the formerly excluded
+`test_get_current_thread_meta_main_thread`.
+
+Each test replaces only `sentry_sdk.utils.threading`, wraps the real module
+for unmocked operations, and sets the SDK lookup's return value. The real
+`Thread.join()` continues using Python's unmodified `threading` module.
+The fallback-result assertions remain; SDK production code is unchanged.
+
+## Reproduction and focused checks
+
+Use a disposable Sentry SDK 2.66.0 checkout and its Python test dependencies.
+From this directory:
+
+```sh
+patch --fuzz=0 -d /path/to/sentry-python -p1 < isolate-threading-mocks.patch
+cd /path/to/sentry-python
+python -m pytest tests/test_utils.py -k get_current_thread_meta -q
+```
+
+To reproduce the race, hold the worker inside its mock just after
+`get_current_thread_meta()` returns, signal that point to the main thread,
+and call `Thread.join()` before releasing the worker. Use an independent
+bounded release so the patched join can finish. The original test raises
+`StopIteration` in `join`; the patched test passes under the same schedule.
+Perform this scheduling instrumentation only in a disposable checkout.
+
+## Upstream status
+
+This is a standalone test patch for Sentry SDK 2.66.0. No upstream submission
+was made during this work. Recheck mock isolation and Python threading
+behavior when upgrading the SDK or interpreter.
+
+## Local NixOS integration and build results
+
+[`../default.nix`](../default.nix) loads `default.nix` through
+`pythonPackagesExtensions`. From the repository root:
+
+```sh
+nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.python314Packages.sentry-sdk
+```
+
+The patched package passed 2,356 tests with 116 existing skips on Python
+3.14.7. The controlled join reproduction failed before the fix and passed
+after it.
+
+That package build used the preceding dependency set with this patch to avoid
+unrelated rebuilds after pytest-xdist changed. The integrated host derivation
+was evaluated; a complete NixOS rebuild was not performed.

overlays/patches/sentry-sdk/default.nix

@@ -0,0 +1,4 @@
+{ sentry-sdk }:
+sentry-sdk.overridePythonAttrs (old: {
+  patches = (old.patches or [ ]) ++ [ ./isolate-threading-mocks.patch ];
+})

overlays/patches/sentry-sdk/isolate-threading-mocks.patch

@@ -0,0 +1,41 @@
+Subject: [PATCH] tests: isolate SDK thread lookup mocks from Python threading
+
+Thread.join also calls threading.current_thread on Python 3.14. A global
+single-use side effect can be consumed by join instead of the SDK, or
+raise StopIteration in join after the SDK consumes it. Patch the SDK's
+module binding and delegate unmocked operations to the real module.
+Apply the same isolation to the adjacent invalid-thread fallback tests.
+
+--- a/tests/test_utils.py
++++ b/tests/test_utils.py
+@@ -914,7 +914,8 @@
+     results = Queue(maxsize=1)
+ 
+     def target():
+-        with mock.patch("threading.current_thread", side_effect=["fake thread"]):
++        with mock.patch("sentry_sdk.utils.threading", wraps=threading) as sdk_threading:
++            sdk_threading.current_thread.return_value = "fake thread"
+             results.put(get_current_thread_meta())
+ 
+     thread = threading.Thread(target=target)
+@@ -930,7 +931,9 @@
+ 
+     def target():
+         # mock that somehow the current thread doesn't exist
+-        with mock.patch("threading.current_thread", side_effect=[None]):
++        # Keep the real threading module intact for concurrent Thread.join calls.
++        with mock.patch("sentry_sdk.utils.threading", wraps=threading) as sdk_threading:
++            sdk_threading.current_thread.return_value = None
+             results.put(get_current_thread_meta())
+ 
+     main_thread = threading.main_thread()
+@@ -945,7 +948,8 @@
+     results = Queue(maxsize=1)
+ 
+     def target():
+-        with mock.patch("threading.current_thread", return_value="fake thread"):
++        with mock.patch("sentry_sdk.utils.threading", wraps=threading) as sdk_threading:
++            sdk_threading.current_thread.return_value = "fake thread"
+             results.put(get_current_thread_meta())
+ 
+     main_thread = threading.main_thread()

overlays/test-exclusions.nix

@@ -0,0 +1,27 @@
+# Test resource settings for the locally rebuilt x86-64-v3 package set.
+#
+# Selecting x86-64-v3 changes every affected derivation, so the normal
+# nixpkgs binary cache cannot be used and upstream test suites run locally.
+# The jeeves builder uses /tmp/nix-builds so filesystem tests run on tmpfs
+# instead of ZFS with normalization=formD and utf8only=on; those tests remain
+# enabled. This overlay no longer excludes any tests. The remaining settings
+# bound nested worker concurrency and allow time for worker startup under load.
+# Test repairs and their validation are indexed in patches/README.md.
+_final: prev: {
+  pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
+    (_pythonFinal: pythonPrev: {
+      pytest-xdist = pythonPrev.pytest-xdist.overridePythonAttrs (old: {
+        # The suite exercises its own worker pools. Run the outer suite with one
+        # worker and allow inner workers more time on heavily loaded builders.
+        postPatch = (old.postPatch or "") + ''
+          substituteInPlace testing/test_remote.py \
+            --replace-fail "WAIT_TIMEOUT = 10.0" "WAIT_TIMEOUT = 60.0"
+        '';
+        preCheck = builtins.replaceStrings [ "--numprocesses=$NIX_BUILD_CORES" ] [ "--numprocesses=1" ] (
+          old.preCheck or ""
+        );
+      });
+
+    })
+  ];
+}

overlays/x86-64-v3-workarounds.nix

@@ -0,0 +1,20 @@
+# Output-validation workarounds for packages rebuilt with x86-64-v3.
+_final: prev:
+let
+  removeSiblingOutputChecks =
+    package:
+    package.overrideAttrs (old: {
+      # Nix 2.34 can validate a partial multi-output rebuild against only the
+      # outputs still being realised. PostgreSQL's checks then reject valid
+      # sibling names such as "out" and "lib". Keep the test suite and
+      # disallowed-requisite checks; accept the loss of cross-output checks.
+      outputChecks = builtins.mapAttrs (
+        _output: checks: builtins.removeAttrs checks [ "disallowedReferences" ]
+      ) (old.outputChecks or { });
+    });
+
+in
+prev.lib.optionalAttrs ((prev.stdenv.hostPlatform.gcc.arch or null) == "x86-64-v3") {
+  postgresql = removeSiblingOutputChecks prev.postgresql;
+  postgresql_18 = removeSiblingOutputChecks prev.postgresql_18;
+}

pyproject.toml

@@ -11,7 +11,6 @@ license = "MIT"
 dependencies = [
     "alembic",
     "apprise",
-    "apscheduler",
     "beautifulsoup4",
     "bm25s",
     "ebooklib",
@@ -20,7 +19,6 @@ dependencies = [
     "httpx",
     "jinja2",
     "pgvector",
-    "polars",
     "psycopg[binary]",
     "pydantic",
     "pydantic-settings",

python/installer/__main__.py

@@ -176,6 +176,21 @@ def create_zfs_datasets() -> None:
     run_output(("zfs", "create", "root_pool/home"))
     run_output(("zfs", "create", "-o", "reservation=1G", "root_pool/var"))
     run_output(("zfs", "create", "-o", "compression=zstd-9", "-o", "reservation=10G", "root_pool/nix"))
+    run_output(
+        (
+            "zfs",
+            "create",
+            "-o",
+            "sync=disabled",
+            "-o",
+            "redundant_metadata=some",
+            "-o",
+            "normalization=none",
+            "-o",
+            "utf8only=off",
+            "root_pool/nix_build",
+        )
+    )
     datasets = run_output(("zfs", "list", "-o", "name"))
 
     expected_datasets = {
@@ -183,6 +198,7 @@ def create_zfs_datasets() -> None:
         "root_pool/home",
         "root_pool/var",
         "root_pool/nix",
+        "root_pool/nix_build",
     }
     missing_datasets = expected_datasets.difference(datasets.splitlines())
     if missing_datasets:
@@ -248,6 +264,7 @@ def create_nix_hardware_file(mnt_dir: str, disks: Sequence[str], encrypt: str |
         '    "/home" = {\n      device = "root_pool/home";\n      fsType = "zfs";\n    };\n\n'
         '    "/var" = {\n      device = "root_pool/var";\n      fsType = "zfs";\n    };\n\n'
         '    "/nix" = {\n      device = "root_pool/nix";\n      fsType = "zfs";\n    };\n\n'
+        '    "/nix/var/nix/builds" = {\n      device = "root_pool/nix_build";\n      fsType = "zfs";\n    };\n\n'
         '    "/boot" = {\n'
         f'      device = "/dev/disk/by-uuid/{get_boot_drive_id(disks[0])}";\n'
         '      fsType = "vfat";\n      options = [\n        "fmask=0077"\n'
@@ -270,6 +287,7 @@ def install_nixos(mnt_dir: str, disks: Sequence[str], encrypt: str | None) -> No
     run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/home", f"{mnt_dir}/home"))
     run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/var", f"{mnt_dir}/var"))
     run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/nix", f"{mnt_dir}/nix"))
+    run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/nix_build", f"{mnt_dir}/nix/var/nix/builds"))
 
     for disk in disks:
         run_output(("mkfs.vfat", "-n", "EFI", f"{disk}-part1"))

python/van_weather/__init__.py

@@ -1 +0,0 @@
-"""Van weather service - fetches weather with masked GPS location."""

python/van_weather/main.py

@@ -1,293 +0,0 @@
-"""Van weather service - fetches weather with masked GPS for privacy."""
-
-import logging
-from datetime import UTC, datetime
-from typing import Annotated, Any
-
-import httpx
-import typer
-from apscheduler.schedulers.blocking import BlockingScheduler
-from tenacity import before_sleep_log, retry, stop_after_attempt, wait_fixed
-
-from python.common import configure_logger
-from python.van_weather.models import Config, DailyForecast, HourlyForecast, Weather
-
-# Map Pirate Weather icons to Home Assistant conditions
-CONDITION_MAP = {
-    "clear-day": "sunny",
-    "clear-night": "clear-night",
-    "rain": "rainy",
-    "snow": "snowy",
-    "sleet": "snowy-rainy",
-    "wind": "windy",
-    "fog": "fog",
-    "cloudy": "cloudy",
-    "partly-cloudy-day": "partlycloudy",
-    "partly-cloudy-night": "partlycloudy",
-}
-
-logger = logging.getLogger(__name__)
-
-
-@retry(
-    stop=stop_after_attempt(3),
-    wait=wait_fixed(5),
-    before_sleep=before_sleep_log(logger, logging.WARNING),
-    reraise=True,
-)
-def get_ha_state(url: str, token: str, entity_id: str) -> float:
-    """Get numeric state from Home Asasistant entity."""
-    response = httpx.get(
-        f"{url}/api/states/{entity_id}",
-        headers={"Authorization": f"Bearer {token}"},
-        timeout=30,
-    )
-    response.raise_for_status()
-    state = response.json()["state"]
-    if state in ("unavailable", "unknown"):
-        error = f"{entity_id} is {state}"
-        raise ValueError(error)
-    return float(state)
-
-
-def parse_daily_forecast(data: dict[str, dict[str, Any]]) -> list[DailyForecast]:
-    """Parse daily forecast from Pirate Weather API."""
-    daily = data.get("daily", {}).get("data", [])
-    daily_forecasts = []
-    for day in daily[:8]:  # Up to 8 days
-        time_stamp = day.get("time")
-        if time_stamp:
-            date_time = datetime.fromtimestamp(time_stamp, tz=UTC).isoformat()
-            daily_forecasts.append(
-                DailyForecast(
-                    date_time=date_time,
-                    condition=CONDITION_MAP.get(day.get("icon", ""), "cloudy"),
-                    temperature=day.get("temperatureHigh"),
-                    templow=day.get("temperatureLow"),
-                    precipitation_probability=day.get("precipProbability"),
-                    moon_phase=day.get("moonPhase"),
-                    wind_gust=day.get("windGust"),
-                    cloud_cover=day.get("cloudCover"),
-                )
-            )
-
-    return daily_forecasts
-
-
-def parse_hourly_forecast(data: dict[str, dict[str, Any]]) -> list[HourlyForecast]:
-    """Parse hourly forecast from Pirate Weather API."""
-    hourly = data.get("hourly", {}).get("data", [])
-    hourly_forecasts = []
-    for hour in hourly[:48]:  # Up to 48 hours
-        time_stamp = hour.get("time")
-        if time_stamp:
-            date_time = datetime.fromtimestamp(time_stamp, tz=UTC).isoformat()
-            hourly_forecasts.append(
-                HourlyForecast(
-                    date_time=date_time,
-                    condition=CONDITION_MAP.get(hour.get("icon", ""), "cloudy"),
-                    temperature=hour.get("temperature"),
-                    precipitation_probability=hour.get("precipProbability"),
-                )
-            )
-    return hourly_forecasts
-
-
-@retry(
-    stop=stop_after_attempt(3),
-    wait=wait_fixed(5),
-    before_sleep=before_sleep_log(logger, logging.WARNING),
-    reraise=True,
-)
-def fetch_weather(api_key: str, lat: float, lon: float) -> Weather:
-    """Fetch weather from Pirate Weather API."""
-    url = f"https://api.pirateweather.net/forecast/{api_key}/{lat},{lon}"
-    response = httpx.get(url, params={"units": "us"}, timeout=30)
-    response.raise_for_status()
-    data = response.json()
-
-    daily_forecasts = parse_daily_forecast(data)
-    hourly_forecasts = parse_hourly_forecast(data)
-
-    current = data.get("currently", {})
-    icon = current.get("icon", "")
-    return Weather(
-        temperature=current.get("temperature"),
-        feels_like=current.get("apparentTemperature"),
-        humidity=current.get("humidity"),
-        wind_speed=current.get("windSpeed"),
-        wind_bearing=current.get("windBearing"),
-        condition=CONDITION_MAP.get(icon, "cloudy"),
-        summary=current.get("summary"),
-        pressure=current.get("pressure"),
-        visibility=current.get("visibility"),
-        uv_index=current.get("uvIndex"),
-        ozone=current.get("ozone"),
-        nearest_storm_distance=current.get("nearestStormDistance"),
-        nearest_storm_bearing=current.get("nearestStormBearing"),
-        precip_probability=current.get("precipProbability"),
-        cloud_cover=current.get("cloudCover"),
-        daily_forecasts=daily_forecasts,
-        hourly_forecasts=hourly_forecasts,
-    )
-
-
-@retry(
-    stop=stop_after_attempt(3),
-    wait=wait_fixed(5),
-    before_sleep=before_sleep_log(logger, logging.WARNING),
-    reraise=True,
-)
-def post_to_ha(url: str, token: str, weather: Weather) -> None:
-    """Post weather data to Home Assistant as sensor entities."""
-    headers = {"Authorization": f"Bearer {token}"}
-
-    # Post current weather as individual sensors
-    sensors = {
-        "sensor.van_weather_condition": {
-            "state": weather.condition or "unknown",
-            "attributes": {"friendly_name": "Van Weather Condition"},
-        },
-        "sensor.van_weather_temperature": {
-            "state": weather.temperature,
-            "attributes": {"unit_of_measurement": "°F", "device_class": "temperature"},
-        },
-        "sensor.van_weather_apparent_temperature": {
-            "state": weather.feels_like,
-            "attributes": {"unit_of_measurement": "°F", "device_class": "temperature"},
-        },
-        "sensor.van_weather_humidity": {
-            "state": int((weather.humidity or 0) * 100),
-            "attributes": {"unit_of_measurement": "%", "device_class": "humidity"},
-        },
-        "sensor.van_weather_pressure": {
-            "state": weather.pressure,
-            "attributes": {"unit_of_measurement": "mbar", "device_class": "pressure"},
-        },
-        "sensor.van_weather_wind_speed": {
-            "state": weather.wind_speed,
-            "attributes": {"unit_of_measurement": "mph", "device_class": "wind_speed"},
-        },
-        "sensor.van_weather_wind_bearing": {
-            "state": weather.wind_bearing,
-            "attributes": {"unit_of_measurement": "°"},
-        },
-        "sensor.van_weather_visibility": {
-            "state": weather.visibility,
-            "attributes": {"unit_of_measurement": "mi"},
-        },
-        "sensor.van_weather_uv_index": {
-            "state": weather.uv_index,
-            "attributes": {"friendly_name": "Van Weather UV Index", "icon": "mdi:sun-wireless"},
-        },
-        "sensor.van_weather_ozone": {
-            "state": weather.ozone,
-            "attributes": {"unit_of_measurement": "DU", "icon": "mdi:earth"},
-        },
-        "sensor.van_weather_nearest_storm_distance": {
-            "state": weather.nearest_storm_distance,
-            "attributes": {"unit_of_measurement": "mi", "icon": "mdi:weather-lightning"},
-        },
-        "sensor.van_weather_nearest_storm_bearing": {
-            "state": weather.nearest_storm_bearing,
-            "attributes": {"unit_of_measurement": "°", "icon": "mdi:weather-lightning"},
-        },
-        "sensor.van_weather_precip_probability": {
-            "state": int((weather.precip_probability or 0) * 100),
-            "attributes": {"unit_of_measurement": "%", "icon": "mdi:weather-rainy"},
-        },
-        "sensor.van_weather_cloud_cover": {
-            "state": int((weather.cloud_cover or 0) * 100),
-            "attributes": {"unit_of_measurement": "%", "icon": "mdi:weather-cloudy"},
-        },
-    }
-
-    for entity_id, data in sensors.items():
-        if data["state"] is not None:
-            response = httpx.post(f"{url}/api/states/{entity_id}", headers=headers, json=data, timeout=30)
-            response.raise_for_status()
-
-    # Post daily forecast as JSON attribute sensor
-    daily_forecast = [
-        {
-            "datetime": daily_forecast.date_time.isoformat(),
-            "condition": daily_forecast.condition,
-            "temperature": daily_forecast.temperature,
-            "templow": daily_forecast.templow,
-            "precipitation_probability": int((daily_forecast.precipitation_probability or 0) * 100),
-        }
-        for daily_forecast in weather.daily_forecasts
-    ]
-
-    response = httpx.post(
-        f"{url}/api/states/sensor.van_weather_forecast_daily",
-        headers=headers,
-        json={"state": len(daily_forecast), "attributes": {"forecast": daily_forecast}},
-        timeout=30,
-    )
-    response.raise_for_status()
-
-    # Post hourly forecast as JSON attribute sensor
-    hourly_forecast = [
-        {
-            "datetime": hourly_forecast.date_time.isoformat(),
-            "condition": hourly_forecast.condition,
-            "temperature": hourly_forecast.temperature,
-            "precipitation_probability": int((hourly_forecast.precipitation_probability or 0) * 100),
-        }
-        for hourly_forecast in weather.hourly_forecasts
-    ]
-
-    response = httpx.post(
-        f"{url}/api/states/sensor.van_weather_forecast_hourly",
-        headers=headers,
-        json={"state": len(hourly_forecast), "attributes": {"forecast": hourly_forecast}},
-        timeout=30,
-    )
-    response.raise_for_status()
-
-
-def update_weather(config: Config) -> None:
-    """Fetch weather using last-known location, post to HA."""
-    lat = get_ha_state(config.ha_url, config.ha_token, config.lat_entity)
-    lon = get_ha_state(config.ha_url, config.ha_token, config.lon_entity)
-
-    masked_lat = round(lat, config.mask_decimals)
-    masked_lon = round(lon, config.mask_decimals)
-
-    logger.info(f"Masked location: {masked_lat}, {masked_lon}")
-
-    weather = fetch_weather(config.pirate_weather_api_key, masked_lat, masked_lon)
-    logger.info(f"Weather: {weather.temperature}°F, {weather.condition}")
-
-    post_to_ha(config.ha_url, config.ha_token, weather)
-    logger.info("Posted weather to HA")
-
-
-def main(
-    ha_url: Annotated[str, typer.Option(envvar="HA_URL")],
-    ha_token: Annotated[str, typer.Option(envvar="HA_TOKEN")],
-    api_key: Annotated[str, typer.Option(envvar="PIRATE_WEATHER_API_KEY")],
-    interval: Annotated[int, typer.Option(help="Poll interval in seconds")] = 900,
-    log_level: Annotated[str, typer.Option()] = "INFO",
-) -> None:
-    """Fetch weather for van using masked GPS location."""
-    configure_logger(log_level)
-
-    config = Config(ha_url=ha_url, ha_token=ha_token, pirate_weather_api_key=api_key)
-
-    logger.info(f"Starting van weather service, polling every {interval}s")
-
-    scheduler = BlockingScheduler()
-    scheduler.add_job(
-        update_weather,
-        "interval",
-        seconds=interval,
-        args=[config],
-        next_run_time=datetime.now(UTC),
-    )
-    scheduler.start()
-
-
-if __name__ == "__main__":
-    typer.run(main)

python/van_weather/models.py

@@ -1,72 +0,0 @@
-"""Models for van weather service."""
-
-from __future__ import annotations
-
-from datetime import datetime  # noqa: TC003 This is required for pydantic
-
-from pydantic import BaseModel, field_serializer
-
-
-class Config(BaseModel):
-    """Service configuration."""
-
-    ha_url: str
-    ha_token: str
-    pirate_weather_api_key: str
-    lat_entity: str = "sensor.van_last_known_latitude"
-    lon_entity: str = "sensor.van_last_known_longitude"
-    mask_decimals: int = 1  # ~11km accuracy
-
-
-class DailyForecast(BaseModel):
-    """Daily forecast entry."""
-
-    date_time: datetime
-    condition: str | None = None
-    temperature: float | None = None  # High
-    templow: float | None = None  # Low
-    precipitation_probability: float | None = None
-    moon_phase: float | None = None
-    wind_gust: float | None = None
-    cloud_cover: float | None = None
-
-    @field_serializer("date_time")
-    def serialize_date_time(self, date_time: datetime) -> str:
-        """Serialize datetime to ISO format."""
-        return date_time.isoformat()
-
-
-class HourlyForecast(BaseModel):
-    """Hourly forecast entry."""
-
-    date_time: datetime
-    condition: str | None = None
-    temperature: float | None = None
-    precipitation_probability: float | None = None
-
-    @field_serializer("date_time")
-    def serialize_date_time(self, date_time: datetime) -> str:
-        """Serialize datetime to ISO format."""
-        return date_time.isoformat()
-
-
-class Weather(BaseModel):
-    """Weather data from Pirate Weather."""
-
-    temperature: float | None = None
-    feels_like: float | None = None
-    humidity: float | None = None
-    wind_speed: float | None = None
-    wind_bearing: float | None = None
-    condition: str | None = None
-    summary: str | None = None
-    pressure: float | None = None
-    visibility: float | None = None
-    uv_index: float | None = None
-    ozone: float | None = None
-    nearest_storm_distance: float | None = None
-    nearest_storm_bearing: float | None = None
-    precip_probability: float | None = None
-    cloud_cover: float | None = None
-    daily_forecasts: list[DailyForecast] = []
-    hourly_forecasts: list[HourlyForecast] = []

systems/bob/default.nix

@@ -3,7 +3,6 @@
   imports = [
     "${inputs.self}/users/math"
     "${inputs.self}/users/richie"
-    "${inputs.self}/users/steve"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/docker.nix"
     "${inputs.self}/common/optional/monitoring-agent.nix"
@@ -14,7 +13,7 @@
     "${inputs.self}/common/optional/systemd-boot.nix"
     "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/yubikey.nix"
+    "${inputs.self}/common/optional/x86-64-v3.nix"
     "${inputs.self}/common/optional/zfs"
     ./hardware.nix
     ./syncthing.nix

systems/bob/hardware.nix

@@ -50,6 +50,11 @@
       fsType = "zfs";
     };
 
+    "/nix/var/nix/builds" = {
+      device = "root_pool/nix_build";
+      fsType = "zfs";
+    };
+
     "/var" = {
       device = "root_pool/var";
       fsType = "zfs";

systems/bob/snapshot_config.toml

@@ -22,6 +22,12 @@ hourly = 0
 daily = 0
 monthly = 0
 
+["root_pool/nix_build"]
+15_min = 1
+hourly = 0
+daily = 0
+monthly = 0
+
 ["root_pool/models"]
 15_min = 4
 hourly = 24

systems/brain/default.nix

@@ -9,6 +9,7 @@
     "${inputs.self}/common/optional/systemd-boot.nix"
     "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
+    "${inputs.self}/common/optional/x86-64-v3.nix"
     "${inputs.self}/common/optional/zfs"
     ./docker
     ./hardware.nix

systems/brain/hardware.nix

@@ -53,6 +53,11 @@
       fsType = "zfs";
     };
 
+    "/nix/var/nix/builds" = {
+      device = "root_pool/nix_build";
+      fsType = "zfs";
+    };
+
     "/boot" = {
       device = "/dev/disk/by-uuid/12CE-A600";
       fsType = "vfat";

systems/brain/services/home_assistant.nix

@@ -68,7 +68,6 @@
           paho-mqtt # for mqtt
           psycopg2 # for postgresql
           py-improv-ble-client # for esphome
-          pymodbus # for modbus
           pyopenweathermap # for weather
           pymetno # for met.no weather
           uiprotect # for ubiquiti integration
@@ -76,10 +75,12 @@
           jsonpath # for rest sensors
           monarchmoneycommunity # for monarch
         ];
-      extraComponents = [ "isal" ];
+      extraComponents = [
+        "isal"
+        "modbus" # for victron modbus integration
+      ];
       customComponents = with pkgs.home-assistant-custom-components; [
         garmin_connect
-        pirate-weather
       ];
 
     };

systems/jeeves/containers.nix

@@ -0,0 +1,71 @@
+{ pkgs, ... }:
+let
+  vars = import ./vars.nix;
+in
+{
+  virtualisation.docker.daemon.settings."data-root" = "${vars.containers}/docker";
+
+  # nixos-container hardcodes its state directory to /var/lib/nixos-containers,
+  # so route it to the shared container dataset with a bind mount.
+  fileSystems."/var/lib/nixos-containers" = {
+    device = "${vars.containers}/nixos-containers";
+    fsType = "none";
+    options = [ "bind" ];
+  };
+
+  systemd.services = {
+    prepare-docker-storage = {
+      description = "Create Docker storage directory";
+      requiredBy = [ "docker.service" ];
+      before = [ "docker.service" ];
+      requires = [ "zfs-mount.service" ];
+      after = [ "zfs-mount.service" ];
+      path = [
+        pkgs.coreutils
+        pkgs.util-linux
+      ];
+
+      unitConfig = {
+        DefaultDependencies = false;
+        RequiresMountsFor = [ "/nix" ];
+      };
+
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+
+      script = ''
+        mountpoint -q ${vars.containers}
+        install -d -m 0710 ${vars.containers}/docker
+      '';
+    };
+
+    prepare-nixos-container-storage = {
+      description = "Create NixOS container storage directory";
+      requiredBy = [ "var-lib-nixos\\x2dcontainers.mount" ];
+      before = [ "var-lib-nixos\\x2dcontainers.mount" ];
+      requires = [ "zfs-mount.service" ];
+      after = [ "zfs-mount.service" ];
+      path = [
+        pkgs.coreutils
+        pkgs.util-linux
+      ];
+
+      unitConfig = {
+        DefaultDependencies = false;
+        RequiresMountsFor = [ "/nix" ];
+      };
+
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+
+      script = ''
+        mountpoint -q ${vars.containers}
+        install -d -m 0755 ${vars.containers}/nixos-containers
+      '';
+    };
+  };
+}

systems/jeeves/default.nix

@@ -4,10 +4,8 @@ let
 in
 {
   imports = [
-    "${inputs.self}/users/dov"
     "${inputs.self}/users/math"
     "${inputs.self}/users/richie"
-    "${inputs.self}/users/steve"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/docker.nix"
     "${inputs.self}/common/optional/monitoring-agent.nix"
@@ -15,10 +13,12 @@ in
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
+    "${inputs.self}/common/optional/x86-64-v3.nix"
     "${inputs.self}/common/optional/zfs"
     ./monitoring
     ./docker
     ./services
+    ./containers.nix
     ./hardware.nix
     ./networking
     ./programs.nix
@@ -35,14 +35,6 @@ in
       path = ./snapshot_config.toml;
       EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
     };
-
-    zerotierone = {
-      enable = true;
-      joinNetworks = [
-        "e4da7455b2ae64ca"
-        "a09acf02330d37b9"
-      ];
-    };
   };
 
   users.groups = {

systems/jeeves/hardware.nix

@@ -96,6 +96,11 @@ in
       fsType = "zfs";
     };
 
+    "/nix/var/nix/builds" = {
+      device = "root_pool/nix_build";
+      fsType = "zfs";
+    };
+
     "/var" = {
       device = "root_pool/var";
       fsType = "zfs";

systems/jeeves/runners/default.nix

@@ -21,5 +21,16 @@
     nix-builder-12.enable = true;
     nix-builder-13.enable = true;
     nix-builder-14.enable = true;
+
+    # Warm the shared x86-64-v3 cache before the smaller per-system runners
+    # start. Eight jobs with eight cores each can use Jeeves' 64 logical CPUs,
+    # while the 6000% quota leaves some capacity for its normal services.
+    nix-cache-builder = {
+      enable = true;
+      labels = [ "nix-cache-builder:host" ];
+      cores = 8;
+      maxJobs = 8;
+      cpuQuota = "6000%";
+    };
   };
 }

systems/jeeves/runners/nix_builder.nix

@@ -1,8 +1,7 @@
 {
   config,
   lib,
-  outputs,
-  utils,
+  pkgs,
   ...
 }:
 
@@ -12,6 +11,122 @@ let
   cfg = config.services.nix_builder;
   runnerUsername = "gitea-runner";
   runnerUserid = 601;
+  containerConfig =
+    containerCfg:
+    {
+      config,
+      pkgs,
+      lib,
+      ...
+    }:
+    let
+      runnerConfigFile = (pkgs.formats.yaml { }).generate "gitea-runner.yaml" { };
+      registerRunner = pkgs.writeShellApplication {
+        name = "register-gitea-runner";
+        runtimeInputs = [
+          pkgs.coreutils
+          pkgs.hostname
+          config.services.gitea-actions-runner.package
+        ];
+        text = builtins.readFile ./register_gitea_runner.sh;
+      };
+    in
+    {
+      networking = {
+        hostName = "nix-builder";
+        useDHCP = lib.mkDefault true;
+        interfaces.eth0.useDHCP = true;
+        # Ensure containers don't inherit the host's stub resolver (127.0.0.53) which was causing issues
+        useHostResolvConf = false;
+      };
+      nix.settings = {
+        inherit (containerCfg) cores;
+        max-jobs = containerCfg.maxJobs;
+        system-features = lib.mkAfter [
+          "gccarch-x86-64-v2"
+          "gccarch-x86-64-v3"
+        ];
+        trusted-substituters = [
+          "https://cache.nixos.org"
+          "https://cache.tmmworkshop.com"
+          "https://nix-community.cachix.org"
+        ];
+        substituters = [
+          "https://cache.nixos.org/?priority=2&want-mass-query=true"
+          "https://cache.tmmworkshop.com/?priority=2&want-mass-query=true"
+          "https://nix-community.cachix.org/?priority=10&want-mass-query=true"
+        ];
+        trusted-public-keys = [
+          "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
+          "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA="
+          "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
+        ];
+        experimental-features = [
+          "flakes"
+          "nix-command"
+        ];
+        sandbox = true;
+        allowed-users = [ runnerUsername ];
+        trusted-users = [
+          "root"
+          runnerUsername
+        ];
+      };
+      users = {
+        users.${runnerUsername} = {
+          isSystemUser = true;
+          group = runnerUsername;
+          uid = runnerUserid;
+        };
+        groups.${runnerUsername}.gid = runnerUserid;
+      };
+      services.gitea-actions-runner.instances.builder = {
+        enable = true;
+        name = "jeeves-nix-builder";
+        url = "http://192.168.99.14:6443/";
+        labels = containerCfg.labels;
+        tokenFile = "/run/secrets/gitea-runners/registration-token";
+        settings.runner.timeout = "12h";
+        hostPackages = with pkgs; [
+          bash
+          coreutils
+          curl
+          gawk
+          gitMinimal
+          gnused
+          my_python
+          nix
+          nixfmt
+          nixos-rebuild
+          nodejs
+          treefmt
+          wget
+        ];
+      };
+      systemd.services."gitea-runner-builder" = {
+        serviceConfig = {
+          DynamicUser = mkForce false;
+          User = mkForce runnerUsername;
+          Group = mkForce runnerUsername;
+          ExecStartPre = mkForce [
+            "${getExe registerRunner} builder http://192.168.99.14:6443/ ${runnerConfigFile} ${escapeShellArgs containerCfg.labels}"
+          ];
+        };
+      };
+      system.stateVersion = "24.05";
+    };
+  mkContainerPath =
+    containerCfg:
+    (import "${pkgs.path}/nixos/lib/eval-config.nix" {
+      modules = [
+        {
+          boot.isNspawnContainer = true;
+          nixpkgs.pkgs = pkgs;
+        }
+        (containerConfig containerCfg)
+      ];
+      system = null;
+    }).config.system.build.toplevel;
 in
 {
   options.services.nix_builder = {
@@ -26,7 +141,36 @@ in
         types.submodule (
           { name, ... }:
           {
-            options.enable = mkEnableOption "Gitea runner container";
+            options = {
+              enable = mkEnableOption "Gitea runner container";
+
+              labels = mkOption {
+                type = types.listOf types.str;
+                default = [
+                  "self-hosted:host"
+                  "nixos:host"
+                ];
+                description = "Gitea Actions labels advertised by this runner.";
+              };
+
+              cores = mkOption {
+                type = types.ints.positive;
+                default = 8;
+                description = "Number of cores made available to each Nix build job.";
+              };
+
+              maxJobs = mkOption {
+                type = types.ints.positive;
+                default = 2;
+                description = "Maximum number of Nix build jobs run in parallel.";
+              };
+
+              cpuQuota = mkOption {
+                type = types.str;
+                default = "800%";
+                description = "systemd CPU quota for the runner container.";
+              };
+            };
           }
         )
       );
@@ -48,9 +192,11 @@ in
     containers = mapAttrs (
       name: containerCfg:
       mkIf containerCfg.enable {
+        path = mkContainerPath containerCfg;
         autoStart = true;
         privateNetwork = true;
         hostBridge = cfg.bridgeName;
+        extraFlags = [ "--hostname=${name}" ];
         bindMounts = {
           host-nix = {
             mountPoint = "/host-nix/var/nix/daemon-socket";
@@ -68,106 +214,18 @@ in
             isReadOnly = false;
           };
         };
-        config =
-          {
-            config,
-            pkgs,
-            lib,
-            ...
-          }:
-          {
-            networking = {
-              useDHCP = lib.mkDefault true;
-              interfaces.eth0.useDHCP = true;
-              # Ensure containers don't inherit the host's stub resolver (127.0.0.53) which was causing issues
-              useHostResolvConf = false;
-            };
-            nix.settings = {
-              trusted-substituters = [
-                "https://cache.nixos.org"
-                "https://cache.tmmworkshop.com"
-                "https://nix-community.cachix.org"
-              ];
-              substituters = [
-                "https://cache.nixos.org/?priority=2&want-mass-query=true"
-                "https://cache.tmmworkshop.com/?priority=2&want-mass-query=true"
-                "https://nix-community.cachix.org/?priority=10&want-mass-query=true"
-              ];
-              trusted-public-keys = [
-                "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
-                "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA="
-                "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
-              ];
-              experimental-features = [
-                "flakes"
-                "nix-command"
-              ];
-              sandbox = true;
-              allowed-users = [ "gitea-runner" ];
-              trusted-users = [
-                "root"
-                "gitea-runner"
-              ];
-            };
-            nixpkgs = {
-              overlays = builtins.attrValues outputs.overlays;
-              config.allowUnfree = true;
-            };
-            users = {
-              users.${runnerUsername} = {
-                isSystemUser = true;
-                group = runnerUsername;
-                uid = runnerUserid;
-              };
-              groups.${runnerUsername}.gid = runnerUserid;
-            };
-            services.gitea-actions-runner.instances.${name} = {
-              enable = true;
-              name = "jeeves-${name}";
-              url = "http://192.168.99.14:6443/";
-              labels = [
-                "self-hosted:host"
-                "nixos:host"
-              ];
-              tokenFile = "/run/secrets/gitea-runners/registration-token";
-              hostPackages = with pkgs; [
-                bash
-                coreutils
-                curl
-                gawk
-                gitMinimal
-                gnused
-                my_python
-                nix
-                nixfmt
-                nixos-rebuild
-                nodejs
-                treefmt
-                wget
-              ];
-            };
-            systemd.services."gitea-runner-${utils.escapeSystemdPath name}" = {
-              serviceConfig = {
-                DynamicUser = mkForce false;
-                User = mkForce runnerUsername;
-                Group = mkForce runnerUsername;
-              };
-            };
-            system.stateVersion = "24.05";
-          };
       }
     ) cfg.containers;
 
     systemd = {
-      services = builtins.listToAttrs (
-        map (name: {
-          name = "container@${name}";
-          value = {
-            requires = [ "gitea.service" ];
-            after = [ "gitea.service" ];
-          };
-        }) (builtins.attrNames (filterAttrs (_: c: c.enable) cfg.containers))
-      );
+      services = mapAttrs' (
+        name: containerCfg:
+        nameValuePair "container@${name}" {
+          requires = [ "gitea.service" ];
+          after = [ "gitea.service" ];
+          serviceConfig.CPUQuota = containerCfg.cpuQuota;
+        }
+      ) (filterAttrs (_: c: c.enable) cfg.containers);
 
       tmpfiles.rules = [
         "d ${vars.uv_cache} 0755 ${runnerUsername} ${runnerUsername} - -"

systems/jeeves/runners/register_gitea_runner.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+legacy_instance_dir="$STATE_DIRECTORY/$(hostname)"
+instance_dir="$STATE_DIRECTORY/$1"
+runner_url="$2"
+runner_config_file="$3"
+shift 3
+runner_labels="$(IFS=,; printf '%s' "$*")"
+
+if [ ! -e "$instance_dir" ] && [ -d "$legacy_instance_dir" ]; then
+  mv "$legacy_instance_dir" "$instance_dir"
+fi
+
+mkdir -vp "$instance_dir"
+cd "$instance_dir" || exit 1
+
+token_hash_file="$instance_dir/.token-hash"
+token_hash_current="$(printf '%s' "$TOKEN" | sha256sum | cut -d' ' -f1)"
+token_hash_stored="$(cat "$token_hash_file" 2>/dev/null || echo "")"
+labels_file="$instance_dir/.labels"
+labels_wanted="$(printf '%s\n' "$@" | sort)"
+labels_current="$(cat "$labels_file" 2>/dev/null || echo 0)"
+
+if [ ! -e "$instance_dir/.runner" ] || [ "$labels_wanted" != "$labels_current" ] || [ "$token_hash_current" != "$token_hash_stored" ]; then
+  rm -v "$instance_dir/.runner" || true
+
+  gitea-runner register --no-interactive \
+    --instance "$runner_url" \
+    --token "$TOKEN" \
+    --name "jeeves-$(hostname)" \
+    --labels "$runner_labels" \
+    --config "$runner_config_file"
+
+  printf '%s' "$token_hash_current" > "$token_hash_file"
+  printf '%s\n' "$labels_wanted" > "$labels_file"
+fi

systems/jeeves/scripts/zfs.sh

@@ -17,6 +17,7 @@ sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnode
 # media datasets
 sudo zfs create media/temp -o sync=disabled -o redundant_metadata=none
 sudo zfs create media/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key
+sudo zfs create media/secure/containers -o mountpoint=/zfs/media/containers -o compression=lz4 -o sync=disabled -o redundant_metadata=some -o normalization=none -o utf8only=off
 sudo zfs create media/secure/docker -o compression=zstd-9
 sudo zfs create media/secure/github-runners -o compression=zstd-9 -o sync=disabled
 sudo zfs create media/secure/notes -o copies=2

systems/jeeves/services/camofox-browser.nix

@@ -1,80 +0,0 @@
-{
-  ...
-}:
-let
-  vars = import ../vars.nix;
-in
-{
-  systemd.tmpfiles.rules = [
-    "d ${vars.docker_configs}/camofox-browser 0750 root root - -"
-  ];
-
-  containers.camofox-browser = {
-    autoStart = true;
-    privateNetwork = false;
-    bindMounts = {
-      camofox-browser = {
-        hostPath = "${vars.docker_configs}/camofox-browser";
-        mountPoint = "/var/lib/camofox-browser";
-        isReadOnly = false;
-      };
-    };
-    config =
-      {
-        pkgs,
-        lib,
-        ...
-      }:
-      {
-        networking.hostName = "camofox-browser";
-
-        environment.systemPackages = with pkgs; [
-          ffmpeg
-          git
-          nodejs
-          python3Packages.yt-dlp
-        ];
-
-        systemd.services.camofox-browser = {
-          description = "Camofox browser server";
-          wantedBy = [ "multi-user.target" ];
-          after = [ "network.target" ];
-          environment = {
-            CAMOFOX_HOST = "127.0.0.1";
-            CAMOFOX_PORT = "9377";
-            HOME = "/var/lib/camofox-browser";
-          };
-          path = with pkgs; [
-            bash
-            coreutils
-            git
-            nodejs
-          ];
-          serviceConfig = {
-            Restart = "always";
-            RestartSec = "5s";
-            WorkingDirectory = "/var/lib/camofox-browser";
-          };
-          script = ''
-            set -eu
-
-            app_dir=/var/lib/camofox-browser/app
-
-            if [ ! -d "$app_dir/.git" ]; then
-              git clone --depth 1 https://github.com/jo-inc/camofox-browser "$app_dir"
-            fi
-
-            cd "$app_dir"
-
-            if [ ! -d node_modules ]; then
-              npm install
-            fi
-
-            exec npm start
-          '';
-        };
-
-        system.stateVersion = lib.mkDefault "24.05";
-      };
-  };
-}

systems/jeeves/services/gitea.nix

@@ -19,6 +19,7 @@ in
       actions = {
         ENABLED = true;
         DEFAULT_ACTIONS_URL = "github";
+        ENDLESS_TASK_TIMEOUT = "12h";
       };
       service.DISABLE_REGISTRATION = true;
       server = {

systems/jeeves/services/open_webui.nix

@@ -1,12 +0,0 @@
-{
-  services.open-webui = {
-    enable = true;
-    host = "0.0.0.0";
-    environment = {
-      ANONYMIZED_TELEMETRY = "False";
-      DO_NOT_TRACK = "True";
-      SCARF_NO_ANALYTICS = "True";
-      OLLAMA_API_BASE_URL = "http://127.0.0.1:11434";
-    };
-  };
-}

systems/jeeves/snapshot_config.toml

@@ -23,6 +23,12 @@ hourly = 0
 daily = 0
 monthly = 0
 
+["root_pool/nix_build"]
+15_min = 1
+hourly = 0
+daily = 0
+monthly = 0
+
 ["root_pool/var"]
 15_min = 8
 hourly = 24
@@ -60,6 +66,12 @@ daily = 30
 monthly = 12
 
 # media
+["media/secure/containers"]
+15_min = 2
+hourly = 0
+daily = 0
+monthly = 0
+
 ["media/temp"]
 15_min = 2
 hourly = 0

systems/jeeves/vars.nix

@@ -5,6 +5,7 @@ let
 in
 {
   inherit zfs_media zfs_storage zfs_scratch;
+  containers = "${zfs_media}/containers";
   database = "${zfs_media}/database";
   docker = "${zfs_media}/docker";
   docker_configs = "${zfs_media}/docker/configs";

systems/portal-1/default.nix

@@ -10,13 +10,12 @@
     "${inputs.self}/users/richie"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/tailscale.nix"
+    "${inputs.self}/common/optional/x86-64-v3.nix"
     ./disk-config.nix
     ./haproxy
     ./monitoring.nix
   ];
 
-  nixpkgs.hostPlatform = "x86_64-linux";
-
   boot = {
     # Avoid consuming the VM's limited memory for /tmp.
     tmp.useTmpfs = false;

systems/rhapsody-in-green/default.nix

@@ -9,10 +9,10 @@
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/systemd-boot.nix"
     "${inputs.self}/common/optional/tailscale.nix"
+    "${inputs.self}/common/optional/x86-64-v3.nix"
     "${inputs.self}/common/optional/yubikey.nix"
     "${inputs.self}/common/optional/zfs"
     ./hardware.nix
-    ./open_webui.nix
     ./programs.nix
     ./qmk.nix
     ./syncthing.nix

systems/rhapsody-in-green/hardware.nix

@@ -34,14 +34,22 @@
       device = "root_pool/root";
       fsType = "zfs";
     };
+
     "/home" = {
       device = "root_pool/home";
       fsType = "zfs";
     };
+
+    "/nix/var/nix/builds" = {
+      device = "root_pool/nix_build";
+      fsType = "zfs";
+    };
+
     "/var" = {
       device = "root_pool/var";
       fsType = "zfs";
     };
+
     "/boot" = {
       device = "/dev/disk/by-uuid/12CE-A600";
       fsType = "vfat";

systems/rhapsody-in-green/open_webui.nix

@@ -1,13 +0,0 @@
-{
-  services.open-webui = {
-    enable = true;
-    host = "0.0.0.0";
-    environment = {
-      ANONYMIZED_TELEMETRY = "False";
-      DO_NOT_TRACK = "True";
-      SCARF_NO_ANALYTICS = "True";
-      OLLAMA_API_BASE_URL = "https://ollama.com";
-      WEBUI_AUTH = "False";
-    };
-  };
-}

users/dov/default.nix

@@ -1,26 +0,0 @@
-{
-  pkgs,
-  ...
-}:
-{
-
-  users = {
-    users.dov = {
-      isNormalUser = true;
-
-      shell = pkgs.zsh;
-      group = "dov";
-      openssh.authorizedKeys.keys = [
-        "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCoSBmMfvp6aThkrfkLQ4TDwZJD0HCC0lsTIRNREIAWeduNkVFdkO3T1SMLmCKJ+zlL9xziNBEqB8NFl4TabAhptIGlKgTOc8C9eDaPQMQV8PB/4DxZhCt7O2qb4Vfcz82IHPtdwwaIsGpssgh81mQ4tPtP8BI0TluRBy+2v095s20j+PYRGrAXJtOWpVY9zaXxEJ8dXDhHDF2xzxvUcwu9NsoY8t+4/ZJ1mrTTG+eGp5gLAKnSVhAGgsmqCY577Nkso1jSzKer1XcCiaGIedpLuTzmUbOFFEVhhTSy+Ki1NLEcjGW2e6Vyg5Pm8VGN75MqyDZsi5igv9Grxq62EHQ4mFea9rns48B74O/bNQ1OoLVO9u/HwdLCgBTZzptrCmNwML6kBWrmCogoH3ueVbHwDCW5kTTMBCoVV+HaZ+qDWu7xZVx49MuCK29QGZj/IrN0N7h78KL0NYajdei87R0mcmWGP0YaJTdCQ4iKi9c77BUYQp+Qpqt+mnIX0cgjQOU= dkruger@kzin" # cspell:disable-line
-      ];
-      extraGroups = [
-        "audio"
-        "video"
-        "users"
-      ];
-      uid = 1004;
-    };
-
-    groups.dov.gid = 1004;
-  };
-}

users/dov/home/cli/default.nix

@@ -1,9 +0,0 @@
-{
-  imports = [
-    ./direnv.nix
-    ./git.nix
-    ./zsh.nix
-  ];
-
-  programs.starship.enable = true;
-}

users/dov/home/cli/direnv.nix

@@ -1,8 +0,0 @@
-{
-
-  programs.direnv = {
-    enable = true;
-    enableZshIntegration = true;
-    nix-direnv.enable = true;
-  };
-}

users/dov/home/cli/git.nix

@@ -1,15 +0,0 @@
-{
-  programs.git = {
-    enable = true;
-    signing.format = null;
-    settings = {
-      user = {
-        email = "dov.kruger@gmail.com";
-        name = "Dov Kruger";
-      };
-      pull.rebase = true;
-      color.ui = true;
-    };
-    lfs.enable = true;
-  };
-}

users/dov/home/cli/zsh.nix

@@ -1,27 +0,0 @@
-{
-  programs.zsh = {
-    enable = true;
-    syntaxHighlighting.enable = true;
-    history.size = 10000;
-    oh-my-zsh = {
-      enable = true;
-      plugins = [
-        "git"
-        "docker"
-        "docker-compose"
-        "colored-man-pages"
-        "rust"
-        "systemd"
-        "tmux"
-        "ufw"
-        "z"
-      ];
-    };
-    shellAliases = {
-      "lrt" = "eza --icons -lsnew";
-      "ls" = "eza";
-      "ll" = "eza --long --group";
-      "la" = "eza --all";
-    };
-  };
-}

users/dov/home/global.nix

@@ -1,22 +0,0 @@
-{ config, ... }:
-{
-  imports = [
-    ./cli
-    ./programs.nix
-    ./ssh_config.nix
-  ];
-
-  programs = {
-    home-manager.enable = true;
-    git.enable = true;
-  };
-
-  home = {
-    username = "dov";
-    homeDirectory = "/home/${config.home.username}";
-    stateVersion = "24.05";
-    sessionVariables = {
-      FLAKE = "$HOME/dotfiles";
-    };
-  };
-}

users/dov/home/programs.nix

@@ -1,56 +0,0 @@
-{ pkgs, ... }:
-{
-  home.packages = with pkgs; [
-    # cli
-    bat
-    btop
-    eza
-    fd
-    ffmpegthumbnailer
-    fzf
-    git
-    gnupg
-    imagemagick
-    jq
-    ncdu
-    ouch
-    p7zip
-    poppler
-    rar
-    ripgrep
-    starship
-    tmux
-    unzip
-    yazi
-    zoxide
-    # system info
-    hwloc
-    lynis
-    pciutils
-    smartmontools
-    usbutils
-    # networking
-    iperf3
-    nmap
-    wget
-    # python
-    ruff
-    uv
-    # nodejs
-    nodejs
-    # Rust packages
-    trunk
-    wasm-pack
-    cargo-watch
-    cargo-generate
-    cargo-audit
-    cargo-update
-    # nix
-    nix-init
-    nix-output-monitor
-    nix-prefetch
-    nix-tree
-    nixfmt
-    treefmt
-  ];
-}

users/dov/home/ssh_config.nix

@@ -1,6 +0,0 @@
-{
-  programs.ssh = {
-    enable = true;
-    enableDefaultConfig = false;
-  };
-}

users/dov/systems/jeeves.nix

@@ -1,5 +0,0 @@
-{
-  imports = [
-    ../home/global.nix
-  ];
-}

users/math/home/programs.nix

@@ -3,54 +3,17 @@
   home.packages = with pkgs; [
     # cli
     bat
-    btop
-    eza
-    fd
-    ffmpegthumbnailer
     fzf
     git
-    gnupg
-    imagemagick
     jq
     ncdu
-    ouch
+    fastfetch
     p7zip
-    poppler
     rar
-    ripgrep
-    starship
-    tmux
     unzip
-    yazi
     zoxide
-    # system info
-    hwloc
-    lynis
-    pciutils
-    smartmontools
-    usbutils
-    # networking
-    iperf3
-    nmap
-    wget
     # python
     ruff
     uv
-    # nodejs
-    nodejs
-    # Rust packages
-    trunk
-    wasm-pack
-    cargo-watch
-    cargo-generate
-    cargo-audit
-    cargo-update
-    # nix
-    nix-init
-    nix-output-monitor
-    nix-prefetch
-    nix-tree
-    nixfmt
-    treefmt
   ];
 }

users/richie/home/gui/default.nix

@@ -1,12 +1,11 @@
-{ inputs, pkgs, ... }:
+{ pkgs, ... }:
 {
   imports = [
-    "${inputs.self}/users/shared/comms.nix"
-    "${inputs.self}/users/shared/games.nix"
-    "${inputs.self}/users/shared/sweet.nix"
-    ./firefox
+    ./comms.nix
+    ./games.nix
     ./kitty.nix
     ./llm_tools.nix
+    ./sweet.nix
     ./t3_code
     ./vscode
   ];

users/richie/home/gui/firefox/default.nix

@@ -1,257 +0,0 @@
-{ config, inputs, ... }:
-{
-  imports = [ ./search_engines.nix ];
-
-  programs.firefox = {
-    configPath = "${config.xdg.configHome}/mozilla/firefox";
-    enable = true;
-    profiles.richie = {
-      extensions.packages = with inputs.firefox-addons.packages.x86_64-linux; [
-        bitwarden
-        darkreader
-        dearrow
-        fastforwardteam
-        return-youtube-dislikes
-        sponsorblock
-        ublock-origin
-      ];
-      search = {
-        force = true;
-        default = "kagi";
-        order = [
-          "kagi"
-          "ddg"
-          "google"
-        ];
-      };
-      settings = {
-        # SECTION: FASTFOX
-        # GENERAL
-        "content.notify.interval" = 100000;
-
-        # GFX
-        "gfx.canvas.accelerated.cache-items" = 4096;
-        "gfx.canvas.accelerated.cache-size" = 512;
-        "gfx.content.skia-font-cache-size" = 20;
-
-        # DISK CACHE
-        "browser.cache.jsbc_compression_level" = 3;
-
-        # MEDIA CACHE
-        "media.memory_cache_max_size" = 65536;
-        "media.cache_readahead_limit" = 7200;
-        "media.cache_resume_threshold" = 3600;
-
-        # IMAGE CACHE
-        "image.mem.decode_bytes_at_a_time" = 32768;
-
-        # NETWORK
-        "network.buffer.cache.size" = 262144;
-        "network.buffer.cache.count" = 128;
-        "network.http.max-connections" = 1800;
-        "network.http.max-persistent-connections-per-server" = 10;
-        "network.http.max-urgent-start-excessive-connections-per-host" = 5;
-        "network.http.accept-encoding" = "gzip, deflate, br, zstd";
-        "network.http.pacing.requests.enabled" = false;
-        "network.dnsCacheExpiration" = 3600;
-        "network.dns.max_high_priority_threads" = 8;
-        "network.ssl_tokens_cache_capacity" = 10240;
-
-        # SPECULATIVE LOADING
-        "network.dns.disablePrefetch" = true;
-        "network.prefetch-next" = false;
-        "network.predictor.enabled" = false;
-
-        # EXPERIMENTAL
-        "layout.css.grid-template-masonry-value.enabled" = true;
-        "dom.enable_web_task_scheduling" = true;
-        "layout.css.has-selector.enabled" = true;
-        "dom.security.sanitizer.enabled" = true;
-
-        # SECTION: SECUREFOX
-        # TRACKING PROTECTION
-        "browser.contentblocking.category" = "strict";
-        "urlclassifier.trackingSkipURLs" = "*.reddit.com, *.twitter.com, *.twimg.com, *.tiktok.com";
-        "urlclassifier.features.socialtracking.skipURLs" = "*.instagram.com, *.twitter.com, *.twimg.com";
-        "network.cookie.sameSite.noneRequiresSecure" = true;
-        "browser.download.start_downloads_in_tmp_dir" = true;
-        "browser.helperApps.deleteTempFileOnExit" = true;
-        "browser.uitour.enabled" = false;
-        "privacy.globalprivacycontrol.enabled" = true;
-
-        # OCSP & CERTS / HPKP
-        "security.OCSP.enabled" = 0;
-        "security.remote_settings.crlite_filters.enabled" = true;
-        "security.pki.crlite_mode" = 2;
-
-        # SSL / TLS
-        "security.ssl.treat_unsafe_negotiation_as_broken" = true;
-        "browser.xul.error_pages.expert_bad_cert" = true;
-        "security.tls.enable_0rtt_data" = false;
-
-        # DISK AVOIDANCE
-        "browser.privatebrowsing.forceMediaMemoryCache" = true;
-        "browser.sessionstore.interval" = 60000;
-
-        # SHUTDOWN & SANITIZING
-        "privacy.history.custom" = true;
-
-        # SEARCH / URL BAR
-        "browser.search.separatePrivateDefault.ui.enabled" = true;
-        "browser.urlbar.update2.engineAliasRefresh" = true;
-        # PREF: restore search engine suggestions
-        "browser.search.suggest.enabled" = true;
-        "browser.urlbar.suggest.quicksuggest.sponsored" = false;
-        "browser.urlbar.suggest.quicksuggest.nonsponsored" = false;
-        "browser.formfill.enable" = false;
-        "security.insecure_connection_text.enabled" = true;
-        "security.insecure_connection_text.pbmode.enabled" = true;
-        "network.IDN_show_punycode" = true;
-
-        # HTTPS-FIRST POLICY
-        "dom.security.https_first" = true;
-        "dom.security.https_first_schemeless" = true;
-
-        # PASSWORDS
-        "signon.formlessCapture.enabled" = false;
-        "signon.rememberSignons" = false;
-        "signon.privateBrowsingCapture.enabled" = false;
-        "network.auth.subresource-http-auth-allow" = 1;
-        "editor.truncate_user_pastes" = false;
-
-        # MIXED CONTENT + CROSS-SITE
-        "security.mixed_content.block_display_content" = true;
-        "security.mixed_content.upgrade_display_content" = true;
-        "security.mixed_content.upgrade_display_content.image" = true;
-        "pdfjs.enableScripting" = false;
-        "extensions.postDownloadThirdPartyPrompt" = false;
-
-        # HEADERS / REFERERS
-        "network.http.referer.XOriginTrimmingPolicy" = 2;
-
-        # CONTAINERS
-        "privacy.userContext.ui.enabled" = true;
-
-        # WEBRTC
-        "media.peerconnection.ice.proxy_only_if_behind_proxy" = true;
-        "media.peerconnection.ice.default_address_only" = true;
-
-        # SAFE BROWSING
-        "browser.safebrowsing.downloads.remote.enabled" = false;
-
-        # MOZILLA
-        # PREF: allow websites to ask you to receive site notifications
-        "permissions.default.desktop-notification" = 0; # allow websites to ask
-        # PREF: allow websites to ask you for your location
-        "permissions.default.geo" = 0;
-        "geo.provider.network.url" =
-          "https://location.services.mozilla.com/v1/geolocate?key=%MOZILLA_API_KEY%";
-        "permissions.manager.defaultsUrl" = "";
-        "webchannel.allowObject.urlWhitelist" = "";
-
-        # TELEMETRY
-        "datareporting.policy.dataSubmissionEnabled" = false;
-        "datareporting.healthreport.uploadEnabled" = false;
-        "toolkit.telemetry.unified" = false;
-        "toolkit.telemetry.enabled" = false;
-        "toolkit.telemetry.server" = "data:,";
-        "toolkit.telemetry.archive.enabled" = false;
-        "toolkit.telemetry.newProfilePing.enabled" = false;
-        "toolkit.telemetry.shutdownPingSender.enabled" = false;
-        "toolkit.telemetry.updatePing.enabled" = false;
-        "toolkit.telemetry.bhrPing.enabled" = false;
-        "toolkit.telemetry.firstShutdownPing.enabled" = false;
-        "toolkit.telemetry.coverage.opt-out" = true;
-        "toolkit.coverage.opt-out" = true;
-        "toolkit.coverage.endpoint.base" = "";
-        "browser.ping-centre.telemetry" = false;
-        "browser.newtabpage.activity-stream.feeds.telemetry" = false;
-        "browser.newtabpage.activity-stream.telemetry" = false;
-
-        # EXPERIMENTS
-        "app.shield.optoutstudies.enabled" = false;
-        "app.normandy.enabled" = false;
-        "app.normandy.api_url" = "";
-
-        # CRASH REPORTS
-        "breakpad.reportURL" = "";
-        "browser.tabs.crashReporting.sendReport" = false;
-        "browser.crashReports.unsubmittedCheck.autoSubmit2" = false;
-
-        # DETECTION
-        "captivedetect.canonicalURL" = "";
-        "network.captive-portal-service.enabled" = false;
-        "network.connectivity-service.enabled" = false;
-
-        # SECTION: PESKYFOX
-        # MOZILLA UI
-        "browser.privatebrowsing.vpnpromourl" = "";
-        "extensions.getAddons.showPane" = false;
-        "extensions.htmlaboutaddons.recommendations.enabled" = false;
-        "browser.discovery.enabled" = false;
-        "browser.shell.checkDefaultBrowser" = false;
-        "browser.newtabpage.activity-stream.asrouter.userprefs.cfr.addons" = false;
-        "browser.newtabpage.activity-stream.asrouter.userprefs.cfr.features" = false;
-        "browser.preferences.moreFromMozilla" = false;
-        "browser.tabs.tabmanager.enabled" = false;
-        "browser.aboutConfig.showWarning" = false;
-        "browser.aboutwelcome.enabled" = false;
-
-        # THEME ADJUSTMENTS
-        "toolkit.legacyUserProfileCustomizations.stylesheets" = true;
-        "browser.compactmode.show" = true;
-        "browser.display.focus_ring_on_anything" = true;
-        "browser.display.focus_ring_style" = 0;
-        "browser.display.focus_ring_width" = 0;
-        "layout.css.prefers-color-scheme.content-override" = 2;
-
-        # COOKIE BANNER HANDLING
-        "cookiebanners.service.mode" = 1;
-        "cookiebanners.service.mode.privateBrowsing" = 1;
-
-        # FULLSCREEN NOTICE
-        "full-screen-api.transition-duration.enter" = "0 0";
-        "full-screen-api.transition-duration.leave" = "0 0";
-        "full-screen-api.warning.delay" = -1;
-        "full-screen-api.warning.timeout" = 0;
-
-        # URL BAR
-        "browser.urlbar.suggest.calculator" = true;
-        "browser.urlbar.unitConversion.enabled" = true;
-        "browser.urlbar.trending.featureGate" = false;
-
-        # NEW TAB PAGE
-        "browser.newtabpage.activity-stream.feeds.topsites" = false;
-        "browser.newtabpage.activity-stream.feeds.section.topstories" = false;
-
-        # POCKET
-        "extensions.pocket.enabled" = false;
-
-        # DOWNLOADS
-        "browser.download.always_ask_before_handling_new_types" = true;
-        "browser.download.manager.addToRecentDocs" = false;
-
-        # PDF
-        "browser.download.open_pdf_attachments_inline" = true;
-
-        # TAB BEHAVIOR
-        "browser.bookmarks.openInTabClosesMenu" = false;
-        "browser.menu.showViewImageInfo" = true;
-        "findbar.highlightAll" = true;
-        "layout.word_select.eat_space_to_next_word" = false;
-
-        # SECTION: MY OVERRIDES
-        "browser.startup.homepage" = "https://google.com";
-        "identity.fxaccounts.enabled" = false;
-
-        # SECTION SMOOTHFOX
-        # OPTION: SHARPEN SCROLLING                                                           *
-        "apz.overscroll.enabled" = true; # DEFAULT NON-LINUX
-        "mousewheel.min_line_scroll_amount" = 10; # 10-40; adjust this number to your liking; default=5
-        "general.smoothScroll.mouseWheel.durationMinMS" = 80; # default=50
-        "general.smoothScroll.currentVelocityWeighting" = "0.15"; # default=.25
-        "general.smoothScroll.stopDecelerationWeighting" = "0.6"; # default=.4
-      };
-    };
-  };
-}

users/richie/home/gui/firefox/github.svg

@@ -1,3 +0,0 @@
-<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path fill-rule="evenodd" clip-rule="evenodd" d="M16 0C7.16 0 0 7.16 0 16C0 23.08 4.58 29.06 10.94 31.18C11.74 31.32 12.04 30.84 12.04 30.42C12.04 30.04 12.02 28.78 12.02 27.44C8 28.18 6.96 26.46 6.64 25.56C6.46 25.1 5.68 23.68 5 23.3C4.44 23 3.64 22.26 4.98 22.24C6.24 22.22 7.14 23.4 7.44 23.88C8.88 26.3 11.18 25.62 12.1 25.2C12.24 24.16 12.66 23.46 13.12 23.06C9.56 22.66 5.84 21.28 5.84 15.16C5.84 13.42 6.46 11.98 7.48 10.86C7.32 10.46 6.76 8.82 7.64 6.62C7.64 6.62 8.98 6.2 12.04 8.26C13.32 7.9 14.68 7.72 16.04 7.72C17.4 7.72 18.76 7.9 20.04 8.26C23.1 6.18 24.44 6.62 24.44 6.62C25.32 8.82 24.76 10.46 24.6 10.86C25.62 11.98 26.24 13.4 26.24 15.16C26.24 21.3 22.5 22.66 18.94 23.06C19.52 23.56 20.02 24.52 20.02 26.02C20.02 28.16 20 29.88 20 30.42C20 30.84 20.3 31.34 21.1 31.18C27.42 29.06 32 23.06 32 16C32 7.16 24.84 0 16 0V0Z" fill="white"/>
-</svg>

users/richie/home/gui/firefox/search_engines.nix

@@ -1,99 +0,0 @@
-{ pkgs, ... }:
-{
-  programs.firefox.profiles.richie.search.engines = {
-    "Nix Options" = {
-      urls = [
-        {
-          template = "https://search.nixos.org/options";
-          params = [
-            {
-              name = "type";
-              value = "packages";
-            }
-            {
-              name = "channel";
-              value = "unstable";
-            }
-            {
-              name = "query";
-              value = "{searchTerms}";
-            }
-          ];
-        }
-      ];
-      icon = "${pkgs.nixos-icons}/share/icons/hicolor/scalable/apps/nix-snowflake.svg";
-      definedAliases = [ "@o" ];
-    };
-    "Nix Packages" = {
-      urls = [
-        {
-          template = "https://search.nixos.org/packages";
-          params = [
-            {
-              name = "type";
-              value = "packages";
-            }
-            {
-              name = "channel";
-              value = "unstable";
-            }
-            {
-              name = "query";
-              value = "{searchTerms}";
-            }
-          ];
-        }
-      ];
-      icon = "${pkgs.nixos-icons}/share/icons/hicolor/scalable/apps/nix-snowflake.svg";
-      definedAliases = [ "@n" ];
-    };
-    "Nix Packages pr-tracker" = {
-      urls = [
-        {
-          template = "https://nixpk.gs/pr-tracker.html?";
-          params = [
-            {
-              name = "pr";
-              value = "{searchTerms}";
-            }
-          ];
-        }
-      ];
-      icon = "${pkgs.nixos-icons}/share/icons/hicolor/scalable/apps/nix-snowflake.svg";
-      definedAliases = [ "@nprt" ];
-    };
-    "kagi" = {
-      urls = [
-        {
-          template = "https://kagi.com/search?";
-          params = [
-            {
-              name = "q";
-              value = "{searchTerms}";
-            }
-          ];
-        }
-      ];
-      icon = ./kagi.png;
-    };
-    github = {
-      urls = [
-        {
-          template = "https://github.com/search?";
-          params = [
-            {
-              name = "q";
-              value = "{searchTerms}";
-            }
-            {
-              name = "type";
-              value = "code";
-            }
-          ];
-        }
-      ];
-      icon = ./github.svg;
-      definedAliases = [ "@g" ];
-    };
-  };
-}

users/richie/home/programs.nix

@@ -3,21 +3,14 @@
   home.packages = with pkgs; [
     # cli
     bat
-    fd
-    ffmpegthumbnailer
     fzf
     git
-    gnupg
-    imagemagick
     jq
     ncdu
     fastfetch
-    ouch
     p7zip
-    poppler
     rar
     unzip
-    yazi
     zoxide
     # Home Assistant
     esphome
@@ -35,8 +28,6 @@
     # python
     ruff
     uv
-    # nodejs
-    nodejs
     # Rust packages
     bacon
     cargo
@@ -51,9 +42,6 @@
     rustfmt
     trunk
     wasm-pack
-    # cpp
-    clang-tools
-    clang_20
     # nix
     nix-init
     nix-output-monitor
@@ -61,6 +49,5 @@
     nix-tree
     nixfmt
     treefmt
-    codebase-memory-mcp
   ];
 }

users/steve/default.nix

@@ -1,44 +0,0 @@
-{
-  pkgs,
-  config,
-  ...
-}:
-let
-  ifTheyExist = groups: builtins.filter (group: builtins.hasAttr group config.users.groups) groups;
-in
-{
-
-  users = {
-    users.steve = {
-      isNormalUser = true;
-
-      shell = pkgs.zsh;
-      group = "steve";
-      openssh.authorizedKeys.keys = [
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJH03VzDbUhzfhvwD+OsYh6GobODYaI9jdNdzWQoqFsp matth@Jove" # cspell:disable-line
-      ];
-      extraGroups = [
-        "audio"
-        "video"
-        "wheel"
-        "users"
-      ]
-      ++ ifTheyExist [
-        "dialout"
-        "docker"
-        "hass"
-        "libvirtd"
-        "networkmanager"
-        "plugdev"
-        "scanner"
-        "transmission"
-        "uaccess"
-        "wireshark"
-      ];
-      uid = 1005;
-    };
-
-    groups.steve.gid = 1005;
-  };
-  home-manager.users.steve = import ./systems/${config.networking.hostName}.nix;
-}

users/steve/home/cli/default.nix

@@ -1,9 +0,0 @@
-{
-  imports = [
-    ./direnv.nix
-    ./git.nix
-    ./zsh.nix
-  ];
-
-  programs.starship.enable = true;
-}

users/steve/home/cli/direnv.nix

@@ -1,8 +0,0 @@
-{
-
-  programs.direnv = {
-    enable = true;
-    enableZshIntegration = true;
-    nix-direnv.enable = true;
-  };
-}

users/steve/home/cli/git.nix

@@ -1,15 +0,0 @@
-{
-  programs.git = {
-    enable = true;
-    signing.format = null;
-    settings = {
-      user = {
-        email = "matthew.michal11@gmail.com";
-        name = "Matthew Michal";
-      };
-      pull.rebase = true;
-      color.ui = true;
-    };
-    lfs.enable = true;
-  };
-}

users/steve/home/cli/zsh.nix

@@ -1,28 +0,0 @@
-{
-  programs.zsh = {
-    enable = true;
-    syntaxHighlighting.enable = true;
-    history.size = 10000;
-    oh-my-zsh = {
-      enable = true;
-      plugins = [
-        "git"
-        "docker"
-        "docker-compose"
-        "colored-man-pages"
-        "rust"
-        "systemd"
-        "tmux"
-        "ufw"
-        "z"
-      ];
-    };
-    shellAliases = {
-      "lrt" = "eza --icons -lsnew";
-      "ls" = "eza";
-      "ll" = "eza --long --group";
-      "la" = "eza --all";
-
-    };
-  };
-}

users/steve/home/global.nix

@@ -1,22 +0,0 @@
-{ config, ... }:
-{
-  imports = [
-    ./cli
-    ./programs.nix
-    ./ssh_config.nix
-  ];
-
-  programs = {
-    home-manager.enable = true;
-    git.enable = true;
-  };
-
-  home = {
-    username = "steve";
-    homeDirectory = "/home/${config.home.username}";
-    stateVersion = "24.05";
-    sessionVariables = {
-      FLAKE = "$HOME/dotfiles";
-    };
-  };
-}

users/steve/home/programs.nix

@@ -1,56 +0,0 @@
-{ pkgs, ... }:
-{
-  home.packages = with pkgs; [
-    # cli
-    bat
-    btop
-    eza
-    fd
-    ffmpegthumbnailer
-    fzf
-    git
-    gnupg
-    imagemagick
-    jq
-    ncdu
-    ouch
-    p7zip
-    poppler
-    rar
-    ripgrep
-    starship
-    tmux
-    unzip
-    yazi
-    zoxide
-    # system info
-    hwloc
-    lynis
-    pciutils
-    smartmontools
-    usbutils
-    # networking
-    iperf3
-    nmap
-    wget
-    # python
-    ruff
-    uv
-    # nodejs
-    nodejs
-    # Rust packages
-    trunk
-    wasm-pack
-    cargo-watch
-    cargo-generate
-    cargo-audit
-    cargo-update
-    # nix
-    nix-init
-    nix-output-monitor
-    nix-prefetch
-    nix-tree
-    nixfmt
-    treefmt
-  ];
-}

users/steve/home/ssh_config.nix

@@ -1,6 +0,0 @@
-{
-  programs.ssh = {
-    enable = true;
-    enableDefaultConfig = false;
-  };
-}

users/steve/systems/bob.nix

@@ -1,5 +0,0 @@
-{
-  imports = [
-    ../home/global.nix
-  ];
-}

users/steve/systems/jeeves.nix

@@ -1,5 +0,0 @@
-{
-  imports = [
-    ../home/global.nix
-  ];
-}