updated read me

it was only required for rhapsody-in-green

.github/workflows/build_systems.yml

@@ -8,8 +8,23 @@ on:
     - cron: "0 22 * * *"
 
 jobs:
+  prebuild-common:
+    name: prebuild-common-x86-64-v3
+    runs-on: nix-cache-builder
+    steps:
+      - uses: actions/checkout@v4
+      # portal-1 is the smallest system closure: 95% of its derivations are
+      # shared by all five systems, so it is a maintainable common cache seed.
+      # Keep going so one failing package does not stop unrelated cache entries
+      # from being built.
+      - name: Build common packages
+        run: nixos-rebuild build --keep-going --accept-flake-config --flake ./#portal-1
+      - name: Copy common packages to nix-cache
+        run: nix copy --accept-flake-config --to unix:///host-nix/var/nix/daemon-socket/socket .#nixosConfigurations.portal-1.config.system.build.toplevel
+
   build:
     name: build-${{ matrix.system }}
+    needs: prebuild-common
     runs-on: self-hosted
     strategy:
       matrix:

.github/workflows/merge_flake_lock_update.yml

@@ -15,7 +15,6 @@ jobs:
     steps:
       - name: merge_flake_lock_update
         run: >-
-          nix develop .#devShells.x86_64-linux.default -c
           python -m python.gitea_flake_lock merge
           --repo "${{ github.repository }}"
         env:

.github/workflows/pytest.yml

@@ -1,19 +0,0 @@
-name: pytest
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
-
-jobs:
-  pytest:
-    runs-on: self-hosted
-
-    steps:
-      - uses: actions/checkout@v4
-      - name: Run tests
-        run: nix develop .#devShells.x86_64-linux.default -c pytest tests

.github/workflows/update-flake-lock.yml

@@ -21,6 +21,5 @@ jobs:
           JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
           GITEA_URL: https://gitea.tmmworkshop.com
         run: >-
-          nix develop .#devShells.x86_64-linux.default -c
           python -m python.gitea_flake_lock update
           --repo "${{ github.repository }}"

common/global/nix.nix

@@ -31,6 +31,10 @@ in
         "flakes"
         "ca-derivations"
       ];
+      system-features = lib.mkAfter [
+        "gccarch-x86-64-v2"
+        "gccarch-x86-64-v3"
+      ];
       warn-dirty = false;
       flake-registry = ""; # disable global flake registries
       connect-timeout = 10;

common/global/programs.nix

@@ -2,6 +2,5 @@
 {
   environment.systemPackages = with pkgs; [
     git
-    my_python
   ];
 }

common/optional/x86-64-v3/default.nix

@@ -0,0 +1,15 @@
+{
+  nixpkgs = {
+    hostPlatform = {
+      system = "x86_64-linux";
+      gcc = {
+        arch = "x86-64-v3";
+        tune = "generic";
+      };
+    };
+
+    # These patches repair tests and bundled dependencies that are sensitive
+    # to the compiler flags used by the x86-64-v3 package set.
+    overlays = [ (import ./patches) ];
+  };
+}

common/optional/x86-64-v3/patches/README.md

@@ -0,0 +1,38 @@
+# Package patches
+
+Each package follows the [GnuTLS layout](gnutls/README.md):
+
+- `default.nix` applies the patch through the package overlay.
+- A descriptive `.patch` file contains the standalone upstream source change.
+- `README.md` explains the problem, scope, reproduction, upstream status,
+  Nix integration, and recorded validation limits.
+- Companion `verify-*` tools live beside the patch when needed; otherwise
+  the README gives commands for the package's existing tests.
+
+Keep package-specific evidence in its directory. Patch headers explain the
+change independently of Nix, and `default.nix` preserves existing patches.
+
+| Package | Repair |
+| --- | --- |
+| [Abseil](abseil/README.md) | Public BMI2 header in Electron, Deno, and Signal's vendored copies |
+| [GnuTLS](gnutls/README.md) | Wait for the UDP server socket before connecting |
+| [Prometheus](prometheus/README.md) | Complete parsing before inspecting the test editor state |
+| [pytest-xdist](pytest-xdist/README.md) | Check worker replacements and allow startup on loaded builders |
+| [SciPy](scipy/README.md) | Account for floating-point rounding in STFT tests |
+
+## Local NixOS integration
+
+The [`x86-64-v3` optional module](../default.nix)
+imports this directory's [`default.nix`](default.nix) directly, so the patch
+overlay applies only to hosts using that package set. Prometheus patches its
+separate assets derivation; Python packages use `pythonPackagesExtensions`.
+
+The [pytest-xdist directory](pytest-xdist/README.md) also owns its outer-worker
+limit and remote-worker event timeout. These package overrides add no skipped
+tests. Existing nixpkgs exclusions remain separate from these repairs.
+
+The review used Python 3.14.7 and the pinned x86-64-v3 package set. Host-flake
+evaluation verified patch wiring, Python install checks, and Prometheus's
+reference to the patched assets. No complete NixOS rebuild was performed.
+Individual READMEs distinguish package builds, focused tests, and checks that
+have not been run.

common/optional/x86-64-v3/patches/abseil/README.md

@@ -0,0 +1,59 @@
+# Abseil BMI2 public header
+
+Vendored Abseil includes `bmi2intrin.h` directly when `__BMI2__` is enabled.
+Compilers reject that internal header without the umbrella-header setup.
+`bmi2-public-header.patch` includes `immintrin.h` instead, allowing builds
+that enable BMI2 through `-march=x86-64-v3`.
+
+## Scope and behavior
+
+The patch changes one include in
+`third_party/abseil-cpp/absl/container/internal/raw_hash_set.h`.
+`default.nix` applies it to Electron 43's unwrapped package, Deno's
+`librusty_v8`, and Signal's WebRTC dependency. It also supplies the patched
+Electron package to Signal. These overrides apply only to `x86-64-v3`.
+
+The shared file path is relative to each vendoring project's source root,
+not the root of a standalone Abseil checkout. No hash-table algorithm or
+test exclusion changes.
+
+## Reproduction and focused checks
+
+From this directory, check and apply the patch to each vendored source tree:
+
+```sh
+patch --dry-run --fuzz=0 -d /path/to/vendor-source -p1 < bmi2-public-header.patch
+patch --fuzz=0 -d /path/to/vendor-source -p1 < bmi2-public-header.patch
+```
+
+A small compiler check isolates the header requirement. With GCC or Clang
+on x86-64, compile `#include <bmi2intrin.h>` using `-march=x86-64-v3`; the
+compiler rejects the direct include. Changing it to `#include <immintrin.h>`
+should compile. The full consumer builds below check integration with their
+actual toolchains.
+
+## Upstream status
+
+Abseil addressed this issue through
+[PR #2071](https://github.com/abseil/abseil-cpp/pull/2071), imported by its
+upstream workflow. That change uses `x86gprintrin.h`; this local variant uses
+the public `immintrin.h` umbrella header for the vendored toolchains.
+Keep the workaround until all three bundled copies include a compatible fix.
+This file is a local adaptation, not a verbatim copy of the upstream diff.
+
+## Local NixOS integration and build results
+
+[`../default.nix`](../default.nix) merges this directory's overlay fragment
+because it repairs multiple packages. From the repository root, the consumer
+build commands are:
+
+```sh
+nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.deno
+nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.electron_43
+nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.signal-desktop
+```
+
+The earlier extraction checked the vendored header snapshots and evaluated
+all three patch attachments. Those records do not establish successful full
+consumer rebuilds. No new compiler or consumer build was run for the layout
+change; the patch and override are unchanged.

common/optional/x86-64-v3/patches/abseil/bmi2-public-header.patch

@@ -0,0 +1,20 @@
+Subject: [PATCH] abseil: include BMI2 intrinsics through the public header
+
+GCC and Clang reject direct inclusion of bmi2intrin.h. Include immintrin.h
+instead so that the compiler supplies the required intrinsic setup when
+BMI2 is enabled, including builds targeting x86-64-v3.
+
+This patch is shared by the vendored Abseil copies in Electron, rusty_v8
+(Deno), and Signal's WebRTC build.
+
+--- a/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
++++ b/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
+@@ -226,7 +226,7 @@
+ #endif
+ 
+ #ifdef __BMI2__
+-#include <bmi2intrin.h>
++#include <immintrin.h>
+ #endif  // __BMI2__
+ 
+ namespace absl {

common/optional/x86-64-v3/patches/abseil/default.nix

@@ -0,0 +1,38 @@
+# Abseil accepted the upstream fix: https://github.com/abseil/abseil-cpp/pull/2071
+# Keep this workaround until Electron, Deno's rusty_v8, and Signal's WebRTC
+# update their bundled Abseil copies to include it.
+{ prev }:
+let
+  patchAbseilBmi2Include =
+    package:
+    package.overrideAttrs (old: {
+      # GCC and Clang require the public umbrella header for BMI2 intrinsics.
+      patches = (old.patches or [ ]) ++ [ ./bmi2-public-header.patch ];
+    });
+
+  electron43Unwrapped = patchAbseilBmi2Include prev.electron_43.unwrapped;
+  electron43 = prev.electron_43.override {
+    electron-unwrapped = electron43Unwrapped;
+  };
+
+  signalCallPackage =
+    path: args:
+    let
+      package = prev.callPackage path args;
+    in
+    if builtins.baseNameOf path == "webrtc.nix" then patchAbseilBmi2Include package else package;
+in
+prev.lib.optionalAttrs ((prev.stdenv.hostPlatform.gcc.arch or null) == "x86-64-v3") {
+  deno =
+    let
+      librusty_v8 = patchAbseilBmi2Include prev.deno.passthru.librusty_v8;
+    in
+    prev.deno.override { inherit librusty_v8; };
+
+  electron_43 = electron43;
+
+  signal-desktop = prev.signal-desktop.override {
+    electron_43 = electron43;
+    callPackage = signalCallPackage;
+  };
+}

common/optional/x86-64-v3/patches/default.nix

@@ -0,0 +1,13 @@
+_final: prev:
+(import ./abseil { inherit prev; })
+// {
+  gnutls = import ./gnutls { inherit (prev) gnutls; };
+  prometheus = import ./prometheus { inherit (prev) prometheus; };
+
+  pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
+    (_pythonFinal: pythonPrev: {
+      pytest-xdist = import ./pytest-xdist { inherit (pythonPrev) pytest-xdist; };
+      scipy = import ./scipy { inherit (pythonPrev) scipy; };
+    })
+  ];
+}

common/optional/x86-64-v3/patches/gnutls/README.md

@@ -0,0 +1,160 @@
+# GnuTLS UDP server readiness
+
+Under load, the test client can start before `gnutls-serv` binds its UDP
+socket, and the first handshake fails with `Connection refused`.
+`serv-udp.sh` currently waits a fixed four seconds; elapsed time does not
+establish server readiness. `udp-server-readiness.patch` replaces that wait
+with polling for the local IPv4 UDP endpoint.
+
+## Scope and waiting behavior
+
+The patch changes the existing `wait_udp_server()` and adds a new
+`check_if_udp_port_bound()` beside it in `tests/scripts/common.sh`.
+`serv-udp.sh` is its only caller in 3.8.13. The TCP helpers `wait_server()`
+and `wait_for_port()`, including their existing sleeps, are unchanged.
+Both original DTLS handshake checks remain unchanged.
+
+Each iteration checks process liveness and the socket **before sleeping**.
+A ready socket returns immediately. An unsuccessful check sleeps two
+seconds only if another attempt remains: at most 90 attempts, consistent
+with the existing `wait_server()` budget implemented by `wait_for_port()`,
+with no sleep after the final check. Server exit fails early; exhausting the
+budget fails and terminates the server. No handshake is retried, and no
+protocol timeout is changed. Once bound, the kernel can queue datagrams
+while the server is scheduled; the probe itself sends no packets.
+
+The existing `have_port_finder()` prefers `ss`, then `netstat`. If neither
+exists, it prints `neither ss nor netstat found` and exits **77 (skip)**.
+In the normal test flow, port selection calls it before launching a server.
+The probe runs in a subshell so that, even if this skip occurs after launch,
+the waiting helper can terminate and reap the server before exiting 77.
+
+## Why an IPv4 socket is expected
+
+This is specific to the server used by this test, not a general rule that
+IPv6 sockets cannot serve IPv4 clients. The client explicitly uses
+`127.0.0.1`. The server's `--udp` path calls `udp_server()`, which calls
+`listen_socket(..., SOCK_DGRAM)`. That function iterates the wildcard
+addresses returned by `getaddrinfo(NULL, ..., AI_PASSIVE)`:
+
+| Server build / Linux setting | Binding behavior |
+| --- | --- |
+| IPv6 enabled, `net.ipv6.bindv6only=0` | Requests `IPV6_V6ONLY=1` on the IPv6 socket, binds `[::]:PORT`, and separately binds `0.0.0.0:PORT`. It overrides the system's dual-stack default. |
+| IPv6 enabled, `net.ipv6.bindv6only=1` | The same explicit socket option and separate IPv4/IPv6 binds. |
+| `HAVE_IPV6` undefined | Skips every address family except `AF_INET`; only the IPv4 wildcard is attempted. |
+
+`udp_server()` uses `wait_for_connection()`, which puts **every listener**
+from that list into `select()` and returns a readable socket for `recvfrom()`;
+it does not permanently choose one socket based on `getaddrinfo()` order.
+
+The first two cases were traced with the actual GnuTLS 3.8.13 binary in
+separate Linux network namespaces: `setsockopt(IPV6_V6ONLY, [1])` and both
+UDP binds returned success under each setting. The no-IPv6 case was checked
+in source, not by building a second binary. The same bind implementation
+was checked directly on GitLab master.
+
+Thus, successful normal startup for this invocation provides an explicit
+IPv4 socket; a lone IPv6 wildcard is not the expected success path.
+There is one portability caveat: upstream discards the return value of
+`setsockopt(IPV6_V6ONLY)`. On a platform where that call fails and the server
+ends up with only a dual-stack socket, this helper would time out despite
+IPv4 reachability. Such a platform needs additional handling before this
+patch can claim support. Blindly accepting every IPv6 wildcard would also
+accept IPv6-only sockets before the separate IPv4 bind finishes.
+
+Source: [`src/serv.c`, `listen_socket()`](https://gitlab.com/gnutls/gnutls/-/blob/master/src/serv.c#L937),
+[`src/udp-serv.c`](https://gitlab.com/gnutls/gnutls/-/blob/master/src/udp-serv.c),
+and [`tests/serv-udp.sh`](https://gitlab.com/gnutls/gnutls/-/blob/master/tests/serv-udp.sh).
+
+## Port matching and ownership limit
+
+Only `-an` is passed to the socket-listing tool: BSD `netstat -u` selects
+Unix-domain sockets, whereas Linux `netstat -u` selects UDP. The parser
+handles the extra state column in `ss`, Linux colon-separated endpoints,
+and BSD dot-separated endpoints, including `*.PORT`. It matches the full
+local port and rejects TCP, IPv6 entries, peer ports, and longer numbers.
+
+A live PID plus a bound port does **not** prove that PID owns the socket.
+Existing `GETPORT` selection checks for an unused port and uses a test
+port-lock directory; `launch_bare_server()` also calls
+`wait_for_free_port()` before starting the process. These are advisory:
+the launcher does not enforce the latter's result, and another process
+can bind between the check and launch. The patch does not close that race
+or add nonportable PID parsing. An unrelated process can satisfy the
+socket check; the real handshakes remain the functional check and may
+fail (or reach the wrong server). This is a startup-order fix, not a
+socket-ownership guarantee.
+
+## Reproduction and focused checks
+
+Apply the patch to an unpacked source tree, then run the companion checks
+with Python's standard library and a shell:
+
+```sh
+patch --fuzz=0 -d /path/to/gnutls -p1 < udp-server-readiness.patch
+SHELL=/bin/sh python3 verify-readiness.py /path/to/gnutls/tests/scripts/common.sh -v
+```
+
+Set `NETSTAT=/path/to/netstat` to exercise one outside `PATH`. The checks
+cover Linux/BSD output samples, false matches, immediate readiness,
+missing tools, process exit, timeout cleanup, and real IPv4 UDP sockets
+whose bind is delayed six seconds. The missing-tools fixture is skipped
+if an absolute fallback `ss` path cannot be hidden with `PATH`. Native
+BSD execution remains untested.
+
+To reproduce with GnuTLS itself, run `tests/serv-udp.sh` with `SERV` pointing
+to a wrapper that sleeps six seconds, then `exec`s `gnutls-serv` with all
+arguments. Set `CLI` to the matching `gnutls-cli`, `srcdir` to the source
+`tests` directory, and `abs_top_builddir` to a writable build directory.
+With GnuTLS 3.8.13, the original helper failed the first handshake with
+`Connection refused`; the patched helper passed both with the same binaries.
+
+## GnuTLS submission
+
+Development and merge requests are on [GitLab](https://gitlab.com/gnutls/gnutls).
+[`CONTRIBUTING.md` on master](https://gitlab.com/gnutls/gnutls/-/blob/master/CONTRIBUTING.md)
+was read directly for this review. It requires the contributor's DCO
+`Signed-off-by`, successful and failure test coverage, consistent coding
+style, and adequate documentation; GitLab CI runs for merge requests.
+Its commenting guidance asks for comments explaining non-obvious behavior
+or protocol expectations. It does not prescribe an additional special
+test-suite comment. The patch now explains its IPv4 binding assumption
+next to the probe.
+
+The submission will contain the shell patch, without the Python verifier
+or a new Python test dependency. The existing `serv-udp.sh` supplies the
+functional success check. Running it through the six-second startup
+wrapper supplies a reproducible regression case: it fails before the fix
+and passes after it. The local verifier was used to validate socket-output
+parsing and the helper's success, process-exit, skip-cleanup, and timeout
+branches. Those branch checks are local evidence, not new automated
+coverage in the upstream suite; the MR must state that distinction.
+
+No dedicated unit-test harness for these shell helpers was found in the
+3.8.13 tests inspected. That does not establish that Python cannot be used
+upstream; keeping this submission dependency-free is a scope choice. Use
+the existing test and before/after reproduction as the submission's
+coverage argument, retaining the platform limitations above. Apply the
+patch in an upstream checkout and include those results with the
+contributor's own sign-off. No MR or sign-off has been created.
+
+## Local NixOS integration and build results
+
+`overlays/default.nix` imports the `overlays/patches` overlay, which loads
+`gnutls/default.nix` to apply the patch and keep `serv-udp.sh` enabled.
+The patch itself has no Nix dependencies and applies to 3.8.13 and GitLab
+master without fuzz.
+
+The final patch was rebuilt with:
+
+```sh
+nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.gnutls
+```
+
+That x86-64-v3 build passed: 927 tests, 796 passes, 131 existing skips,
+zero failures/errors, and `PASS: serv-udp.sh`. The patch bytes in the built
+derivation were compared with the repository artifact; both have SHA-256
+`59013d47fd446f2dd065012a2259ccc1898fedc8a053a630e13efa0076368760`.
+All seven local checks passed, including skip cleanup and exactly 90
+probes with 89 sleeps on timeout. The six-second before/after reproduction
+was also repeated successfully with the final helper.

common/optional/x86-64-v3/patches/gnutls/default.nix

@@ -0,0 +1,6 @@
+{ gnutls }:
+gnutls.overrideAttrs (old: {
+  # Keep the UDP handshake test enabled on loaded builders by waiting for
+  # the server to bind its socket. Kept as a standalone patch for upstream.
+  patches = (old.patches or [ ]) ++ [ ./udp-server-readiness.patch ];
+})

common/optional/x86-64-v3/patches/gnutls/udp-server-readiness.patch

@@ -0,0 +1,70 @@
+Subject: [PATCH] tests: wait for the UDP server socket before connecting
+
+A fixed four-second sleep does not guarantee that gnutls-serv has bound
+its UDP socket on a busy builder. Poll the local IPv4 UDP endpoint using
+the existing ss/netstat discovery, with the same retry budget as the TCP
+helper. Fail early if the server exits, and retain the original handshake
+checks in serv-udp.sh.
+
+Use flags common to ss and BSD/Linux netstat. Match the local endpoint
+and complete port number, excluding TCP, IPv6-only and peer endpoints.
+
+--- a/tests/scripts/common.sh
++++ b/tests/scripts/common.sh
+@@ -185,10 +185,55 @@
+ 	fi
+ }
+ 
++check_if_udp_port_bound() {
++	local PORT=$1
++	have_port_finder
++	# Use only -an, which is shared by ss and BSD/Linux netstat. UDP has
++	# no LISTEN state. Match the local IPv4 endpoint, not a peer port or
++	# a longer port number. serv-udp.sh connects to 127.0.0.1;
++	# listen_socket() in serv.c binds IPv4 separately and requests
++	# IPV6_V6ONLY=1 for its IPv6 socket.
++	$PFCMD -an | awk -v port="$PORT" '
++		$1 == "udp" || $1 == "udp4" {
++			# ss includes a state column; netstat does not.
++			address = ($2 == "UNCONN" || $2 == "ESTAB") ? $5 : $4
++			if (address ~ ("^[0-9.]+[.:]" port "$") ||
++			    address == "*." port)
++				found = 1
++		}
++		END { exit !found }
++	'
++}
++
+ wait_udp_server() {
+ 	local PID=$1
++	local ret
+ 	trap "test -n \"${PID}\" && kill ${PID};exit 1" 1 15 2
+-	sleep 4
++	local i=0
++	# Use the same retry budget as wait_for_port(), but also stop if the
++	# server exits before binding its socket.
++	while test $i -lt 90; do
++		if ! kill -0 "$PID" 2>/dev/null; then
++			fail "" "UDP server $PID exited before binding port $PORT"
++		fi
++		# Contain have_port_finder's exit so a skip also stops the server.
++		if (check_if_udp_port_bound "$PORT"); then
++			return 0
++		else
++			ret=$?
++			if test "$ret" = 77; then
++				kill "$PID" 2>/dev/null || :
++				wait "$PID" 2>/dev/null || :
++				exit 77
++			fi
++		fi
++		i=$((i + 1))
++		if test $i -lt 90; then
++			echo "try $i: waiting for UDP port $PORT"
++			sleep 2
++		fi
++	done
++	fail "$PID" "UDP server $PORT did not come up"
+ }
+ 
+ create_testdir() {

common/optional/x86-64-v3/patches/gnutls/verify-readiness.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""Exercise patched common.sh without building GnuTLS (Python standard library only).
+
+Usage: python3 verify-readiness.py /path/to/patched/tests/scripts/common.sh
+Set SHELL to test another shell, and NETSTAT to test a netstat outside PATH.
+"""
+
+# Use unittest so this upstream companion tool needs no pytest installation.
+# ruff: noqa: PT009
+
+import os
+import shutil
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+from pathlib import Path
+
+COMMON = str(Path(sys.argv.pop(1)).resolve())
+SHELL = os.environ.get("SHELL", "/bin/sh")
+
+
+class ReadinessTests(unittest.TestCase):
+    """Check endpoint parsing and the server startup lifecycle."""
+
+    def setUp(self) -> None:
+        """Create a socket-listing fixture for each check."""
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.root = Path(self.tmp.name)
+        self.fixture = self.root / "sockets"
+        self.fixture.write_text("")
+        self.finder = self.root / "port-finder"
+        self.finder.write_text('#!/bin/sh\ncat "$SOCKET_FIXTURE"\n')
+        self.finder.chmod(0o755)
+
+    def run_shell(self, body: str, **env: str) -> subprocess.CompletedProcess[str]:
+        """Source the actual helper and run a shell scenario."""
+        return subprocess.run(
+            [SHELL, "-c", '. "$COMMON"\n' + body],
+            env={
+                **os.environ,
+                "COMMON": COMMON,
+                "SOCKET_FIXTURE": str(self.fixture),
+                "PFCMD": str(self.finder),
+                "PORT": "12345",
+                **env,
+            },
+            capture_output=True,
+            text=True,
+            timeout=20,
+            check=False,
+        )
+
+    def test_socket_formats_and_false_matches(self) -> None:
+        """Accept IPv4 UDP local endpoints and reject unrelated sockets."""
+        cases = [
+            ("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*", True),
+            ("udp UNCONN 0 0 127.0.0.1:12345 0.0.0.0:*", True),
+            ("udp 0 0 0.0.0.0:12345 0.0.0.0:*", True),
+            ("udp4 0 0 *.12345 *.*", True),
+            ("udp 0 0 127.0.0.1.12345 *.*", True),
+            ("udp 0 0 *.12345 *.*", True),
+            ("udp UNCONN 0 0 0.0.0.0:123456 0.0.0.0:*", False),
+            ("udp 0 0 0.0.0.0:123456 0.0.0.0:*", False),
+            ("udp ESTAB 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
+            ("udp 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
+            ("tcp LISTEN 0 128 0.0.0.0:12345 0.0.0.0:*", False),
+            ("tcp 0 0 0.0.0.0:12345 0.0.0.0:* LISTEN", False),
+            ("udp UNCONN 0 0 [::]:12345 [::]:*", False),
+            ("udp UNCONN 0 0 *:12345 *:*", False),
+            ("udp6 0 0 :::12345 :::*", False),
+            ("udp6 0 0 *.12345 *.*", False),
+            ("", False),
+        ]
+        for row, ready in cases:
+            with self.subTest(row=row):
+                self.fixture.write_text(row + "\n")
+                result = self.run_shell('check_if_udp_port_bound "$PORT"')
+                self.assertEqual(result.returncode, 0 if ready else 1, result.stderr)
+
+    def test_exited_server_fails_immediately(self) -> None:
+        """Fail without sleeping when the server has already exited."""
+        result = self.run_shell(
+            'true &\npid=$!\nwait "$pid"\nsleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$pid"'
+        )
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("exited before binding", result.stderr)
+        self.assertNotIn("unexpected sleep", result.stderr)
+
+    def test_ready_socket_does_not_sleep(self) -> None:
+        """Check readiness before the first sleep."""
+        self.fixture.write_text("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*\n")
+        result = self.run_shell('sleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$$"')
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertNotIn("unexpected sleep", result.stderr)
+
+    def test_missing_port_finders_skip(self) -> None:
+        """Skip and stop the live server when no finder is available."""
+        # have_port_finder also tries these paths independently of PATH.
+        if any(os.access(f"{directory}/ss", os.X_OK) for directory in ("/sbin", "/usr/sbin", "/usr/local/sbin")):
+            self.skipTest("an absolute ss path cannot be hidden by this PATH-only fixture")
+        with subprocess.Popen(["sleep", "60"]) as server:
+            try:
+                result = self.run_shell(
+                    'unset PFCMD\nPATH=/nonexistent\nwait_udp_server "$SERVER_PID"',
+                    SERVER_PID=str(server.pid),
+                )
+                self.assertEqual(result.returncode, 77)
+                self.assertIn("neither ss nor netstat found", result.stderr)
+                server.wait(timeout=3)
+                self.assertLess(server.returncode, 0)
+            finally:
+                if server.poll() is None:
+                    server.kill()
+
+    def test_timeout_is_bounded_and_cleans_up(self) -> None:
+        """Stop polling after the retry budget and terminate the server."""
+        # Only accelerate the polling delay; keep a real live server process.
+        self.finder.write_text('#!/bin/sh\necho probe >&2\ncat "$SOCKET_FIXTURE"\n')
+        with subprocess.Popen(["sleep", "60"]) as server:
+            try:
+                result = self.run_shell(
+                    'sleep() { echo polling-sleep; }\nwait_udp_server "$SERVER_PID"',
+                    SERVER_PID=str(server.pid),
+                )
+                self.assertEqual(result.returncode, 1)
+                self.assertIn("did not come up", result.stderr)
+                self.assertEqual(result.stderr.count("probe\n"), 90)
+                self.assertEqual(result.stdout.count("polling-sleep"), 89)
+                server.wait(timeout=3)
+                self.assertLess(server.returncode, 0)
+            finally:
+                if server.poll() is None:
+                    server.kill()
+
+    def test_server_exits_while_waiting(self) -> None:
+        """Detect a startup failure that happens after polling begins."""
+        result = self.run_shell('sleep 1 &\npid=$!\nwait_udp_server "$pid"')
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("exited before binding", result.stderr)
+        self.assertIn("waiting for UDP port", result.stdout)
+
+    def test_real_socket_delayed_beyond_four_seconds(self) -> None:
+        """Wait for a real delayed bind with each installed port finder."""
+        finders = [shutil.which("ss"), os.environ.get("NETSTAT") or shutil.which("netstat")]
+        finders = [finder for finder in finders if finder]
+        if not finders:
+            self.skipTest("neither ss nor netstat available")
+        for finder in finders:
+            with self.subTest(finder=finder):
+                with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
+                    sock.bind(("127.0.0.1", 0))
+                    port = sock.getsockname()[1]
+                code = (
+                    "import socket,time,sys; time.sleep(6); "
+                    "s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); "
+                    "s.bind(('127.0.0.1',int(sys.argv[1]))); time.sleep(30)"
+                )
+                with subprocess.Popen([sys.executable, "-c", code, str(port)]) as server:
+                    try:
+                        started = time.monotonic()
+                        result = self.run_shell(
+                            'wait_udp_server "$SERVER_PID"',
+                            SERVER_PID=str(server.pid),
+                            PORT=str(port),
+                            PFCMD=finder,
+                        )
+                        self.assertEqual(result.returncode, 0, result.stderr)
+                        self.assertGreaterEqual(time.monotonic() - started, 6)
+                        self.assertIsNone(server.poll())
+                    finally:
+                        server.terminate()
+                        server.wait(timeout=3)
+
+
+if __name__ == "__main__":
+    unittest.main()

common/optional/x86-64-v3/patches/prometheus/README.md

@@ -0,0 +1,66 @@
+# Prometheus complete test parsing
+
+CodeMirror gives editor-state creation a 20 ms synchronous parsing budget.
+The shared `createEditorState()` test helper can therefore return an
+incomplete syntax tree when the process is descheduled. The completion and
+vector-matching tests immediately inspect that tree.
+
+## Scope and behavior
+
+`complete-test-parsing.patch` changes only
+`module/codemirror-promql/src/test/utils-test.ts` inside `web/ui`. It completes
+the small test expression with `ensureSyntaxTree(..., Infinity)` and publishes
+the completed parse through an empty transaction so `syntaxTree(state)` sees
+it. Failure to obtain a tree raises an error.
+
+The original assertions remain enabled, including `autocomplete topk params 2`
+and `foo * on(test,blub) bar`. The unlimited budget applies to the test helper;
+production editor parsing budgets are unchanged.
+
+## Reproduction and focused checks
+
+Use a disposable Prometheus 3.14.0 checkout. The patch root is `web/ui`, matching
+the Nix assets derivation. From this directory:
+
+```sh
+patch --fuzz=0 -d /path/to/prometheus/web/ui -p1 < complete-test-parsing.patch
+cd /path/to/prometheus/web/ui
+pnpm install --frozen-lockfile
+pnpm --filter @prometheus-io/lezer-promql build
+pnpm --filter @prometheus-io/codemirror-promql test
+```
+
+To force the scheduling condition, temporarily append this clock to
+`module/codemirror-promql/setupJest.cjs` in the disposable checkout:
+
+```js
+let parseClock = 0;
+Date.now = () => (parseClock += 25);
+```
+
+Each clock read crosses the editor's initial parsing budget. Against the
+original helper, the hybrid and vector suites have 186 failures, including
+both locally excluded cases. With the patch, all 386 CodeMirror tests pass
+under that same clock. Remove the injected clock before normal builds.
+
+## Upstream status
+
+This is a standalone test-helper patch for Prometheus 3.14.0. No upstream
+submission was made during this work. Recheck the helper when updating
+Prometheus or CodeMirror, including how an ensured parse becomes visible
+through the editor state.
+
+## Local NixOS integration and build results
+
+[`../default.nix`](../default.nix) loads `default.nix`, which patches the
+separate assets derivation. It updates both `passthru.assets` and the main
+Prometheus build's reference to those assets. From the repository root:
+
+```sh
+nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.prometheus.assets
+```
+
+The full x86-64-v3 assets build passed with the normal clock, including the
+CodeMirror and UI suites. Host-flake evaluation confirmed that the main
+Prometheus derivation refers to these patched assets. The Go server package
+was not rebuilt for this test-helper change.

common/optional/x86-64-v3/patches/prometheus/complete-test-parsing.patch

@@ -0,0 +1,36 @@
+Subject: [PATCH] tests: finish parsing before inspecting editor state
+
+EditorState creation has a 20 ms parsing budget. A descheduled test can
+therefore observe an incomplete tree. Finish these small test documents
+without an interactive deadline and publish the result with a transaction.
+Keep the original completion and vector-matching assertions enabled.
+
+--- a/module/codemirror-promql/src/test/utils-test.ts
++++ b/module/codemirror-promql/src/test/utils-test.ts
+@@ -13,7 +13,7 @@
+ 
+ import { parser } from '@prometheus-io/lezer-promql';
+ import { EditorState } from '@codemirror/state';
+-import { LRLanguage } from '@codemirror/language';
++import { ensureSyntaxTree, LRLanguage } from '@codemirror/language';
+ import nock from 'nock';
+ import path from 'path';
+ import { fileURLToPath } from 'url';
+@@ -23,10 +23,16 @@
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
+ 
+ export function createEditorState(expr: string): EditorState {
+-  return EditorState.create({
++  const state = EditorState.create({
+     doc: expr,
+     extensions: lightPromQLSyntax,
+   });
++  // These tests need a complete tree, independent of the editor's time budget.
++  if (!ensureSyntaxTree(state, state.doc.length, Infinity)) {
++    throw new Error('Unable to parse the test expression');
++  }
++  // Publish the completed parse so syntaxTree(state) sees it too.
++  return state.update({}).state;
+ }
+ 
+ export function mockPrometheusServer(): void {

common/optional/x86-64-v3/patches/prometheus/default.nix

@@ -0,0 +1,17 @@
+{ prometheus }:
+prometheus.overrideAttrs (
+  old:
+  let
+    assets = old.passthru.assets.overrideAttrs (assetsOld: {
+      patches = (assetsOld.patches or [ ]) ++ [ ./complete-test-parsing.patch ];
+    });
+  in
+  {
+    postPatch = builtins.replaceStrings [ "${old.passthru.assets}" ] [ "${assets}" ] (
+      builtins.unsafeDiscardStringContext old.postPatch
+    );
+    passthru = old.passthru // {
+      inherit assets;
+    };
+  }
+)

common/optional/x86-64-v3/patches/pytest-xdist/README.md

@@ -0,0 +1,79 @@
+# pytest-xdist test fixes
+
+With two workers and a restart limit of three, the fourth worker crash
+requests shutdown while another test can still be running. That test may
+also crash. The original queued-work test requires exactly four failures,
+even though five failures can occur without exceeding the replacement limit.
+
+## Scope and behavior
+
+`concurrent-worker-crashes.patch` changes the assertions in
+`TestNodeFailure.test_max_worker_restart_tests_queued` in
+`testing/acceptance_test.py`. It requires exactly three replacements, four or
+five failed tests, the failed-tests exit status, the limit message, and no
+internal error. It retains the two-worker workload and ten queued tests.
+
+`worker-startup-timeout.patch` changes the remote-test helper's event timeout
+from 10 to 60 seconds so loaded builders have time to start workers. The
+helper returns immediately when an event arrives and still has a bounded wait.
+
+The existing nixpkgs pytest-9 compatibility patches remain in place.
+Production scheduling and worker-restart behavior are unchanged.
+
+## Reproduction and focused checks
+
+Use a disposable pytest-xdist 3.8.0 checkout with its test dependencies and
+the nixpkgs pytest-9 compatibility patches where required. From this directory:
+
+```sh
+patch --fuzz=0 -d /path/to/pytest-xdist -p1 < concurrent-worker-crashes.patch
+patch --fuzz=0 -d /path/to/pytest-xdist -p1 < worker-startup-timeout.patch
+cd /path/to/pytest-xdist
+python -m pytest testing/acceptance_test.py \
+  -k test_max_worker_restart_tests_queued -q
+python -m pytest testing/test_remote.py -q
+```
+
+Twenty unmodified runs passed during the review. To force the failing
+schedule, modify the generated crashing test in a disposable checkout to
+accept `worker_id`: make `gw3` wait for a marker created by `gw4`, and make
+`gw4` pause 0.1 seconds after creating the marker. Then both have in-flight
+tests when shutdown starts. Bound the marker wait so a reproduction failure
+cannot hang the suite. The original assertion fails on five reported
+failures; the patched test passes.
+
+## Worker startup and outer concurrency
+
+[`default.nix`](default.nix) runs the outer suite with one worker to limit
+nested process pools. This is a Nix test-runner setting; the source timeout
+change lives in [`worker-startup-timeout.patch`](worker-startup-timeout.patch).
+
+A separate reproduction inserts an 11-second `pytest_sessionstart` delay
+into the child created by `test_basic_collect_and_runtests` in
+`testing/test_remote.py`. The original 10-second channel wait fails; the
+60-second wait passes. This bounds waits for test worker events, including
+startup, rather than changing a product deadline.
+
+## Upstream status
+
+These are standalone test patches for pytest-xdist 3.8.0. No upstream submission
+was made during this work. Recheck the allowed in-flight failures, replacement
+count, and remote-test wait when updating the scheduler or worker behavior.
+
+## Local NixOS integration and build results
+
+[`../default.nix`](../default.nix) loads `default.nix` through
+`pythonPackagesExtensions`. From the repository root:
+
+```sh
+nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.python314Packages.pytest-xdist
+```
+
+After consolidating the settings in this directory, the full x86-64-v3 package
+build passed 185 tests, with 6 existing skips and 10 expected failures. Nix
+evaluation confirmed the same outer-worker limit and preserved existing
+patches, with the timeout now applied as a source patch.
+
+The earlier forced concurrent-crash and delayed-startup reproductions passed
+after their fixes; the focused crash test also passed after formatting its
+assertion.

common/optional/x86-64-v3/patches/pytest-xdist/concurrent-worker-crashes.patch

@@ -0,0 +1,29 @@
+Subject: [PATCH] tests: count replacements when checking the worker restart limit
+
+With two workers, another in-flight test may crash after the fourth
+crash requests shutdown. Either four or five failed tests is valid.
+Require exactly three replacements and the failed-tests exit status,
+while preserving the queued-work and no-internal-error assertions.
+
+--- a/testing/acceptance_test.py
++++ b/testing/acceptance_test.py
+@@ -1011,9 +1011,18 @@
+                 "worker*crashed while running*",
+                 "worker*crashed while running*",
+                 "* xdist: maximum crashed workers reached: 3 *",
+-                "* 4 failed in *",
+             ]
+         )
++        # A second in-flight test may crash after shutdown is requested.
++        # The restart limit constrains replacements, not concurrent failures.
++        replacements = sum(
++            line.startswith("replacing crashed worker ") for line in res.stdout.lines
++        )
++        assert replacements == 3
++        failed = res.parseoutcomes()["failed"]
++        assert failed in (4, 5)
++        res.assert_outcomes(failed=failed)
++        assert res.ret == pytest.ExitCode.TESTS_FAILED
+         assert "INTERNALERROR" not in res.stdout.str()
+ 
+     def test_max_worker_restart_die(self, pytester: pytest.Pytester) -> None:

common/optional/x86-64-v3/patches/pytest-xdist/default.nix

@@ -0,0 +1,12 @@
+{ pytest-xdist }:
+pytest-xdist.overridePythonAttrs (old: {
+  patches = (old.patches or [ ]) ++ [
+    ./concurrent-worker-crashes.patch
+    ./worker-startup-timeout.patch
+  ];
+
+  # The suite exercises its own worker pools. Limit the outer suite to one worker.
+  preCheck = builtins.replaceStrings [ "--numprocesses=$NIX_BUILD_CORES" ] [ "--numprocesses=1" ] (
+    old.preCheck or ""
+  );
+})

common/optional/x86-64-v3/patches/pytest-xdist/worker-startup-timeout.patch

@@ -0,0 +1,19 @@
+Subject: [PATCH] tests: allow more time for remote worker events
+
+Worker startup can exceed ten seconds on heavily loaded builders. Allow
+the remote-test helper to wait up to sixty seconds for worker events.
+The wait still returns as soon as an event arrives and remains bounded.
+Production worker timeouts and test assertions are unchanged.
+
+--- a/testing/test_remote.py
++++ b/testing/test_remote.py
+@@ -17,7 +17,8 @@
+ from xdist.workermanage import WorkerController
+ 
+ 
+-WAIT_TIMEOUT = 10.0
++# Allow worker events extra time on heavily loaded builders.
++WAIT_TIMEOUT = 60.0
+ 
+ 
+ def check_marshallable(d: object) -> None:

common/optional/x86-64-v3/patches/scipy/README.md

@@ -0,0 +1,70 @@
+# SciPy STFT test tolerances
+
+The x86-64-v3 build can produce small floating-point residuals in inverse-STFT
+comparisons and scaling round trips. The original bounds reject these results,
+including residuals around `4e-17` where a round trip expects zero for a signal
+with amplitude 2.
+
+## Scope and behavior
+
+`stft-test-tolerances.patch` changes only the signal tests:
+
+- The inverse-STFT comparison in `_scipy_spectral_test_shim.py` uses
+  `max(1e-7, 2 * np.finfo(x.dtype).eps)` as its relative tolerance. Float64
+  keeps the original bound, and the existing i686 override remains.
+- Three scaling round trips in `test_spectral.py` gain an absolute tolerance
+  of one epsilon for the input dtype, allowing small residuals near zero.
+
+The tests remain enabled, and the production STFT implementation is unchanged.
+
+## Reproduction and focused checks
+
+From this directory, apply the patch to a disposable SciPy 1.18.0 checkout:
+
+```sh
+patch --fuzz=0 -d /path/to/scipy -p1 < stft-test-tolerances.patch
+```
+
+Build and install that tree with SciPy's test dependencies. From outside the
+source directory, run the installed tests:
+
+```sh
+python -m pytest --pyargs scipy.signal.tests.test_spectral \
+  -k 'roundtrip_float32 or roundtrip_scaling' -q
+```
+
+Use the same compiler flags and numerical libraries for before/after runs.
+The earlier reproduction called `TestSTFT.test_roundtrip_float32` and
+`TestSTFT.test_roundtrip_scaling` against the x86-64-v3 libraries, then loaded
+patched copies of the test modules. Both failed with the original bounds
+and passed with the adjusted bounds.
+
+## Upstream status
+
+[SciPy issue #25488](https://github.com/scipy/scipy/issues/25488) records
+related test failures with architecture-specific compiler flags. It is
+context for the local tolerance repair; this exact patch has not been
+submitted upstream during this work.
+
+## Local NixOS integration and build results
+
+[`../default.nix`](../default.nix) loads [`default.nix`](default.nix) through
+`pythonPackagesExtensions`, preserving the package's existing patches.
+The override also covers SciPy used to test other Python dependencies,
+including pgvector in portal's shared Python environment.
+
+From the repository root:
+
+```sh
+nix build --no-link -L .#nixosConfigurations.portal-1.pkgs.python314Packages.scipy
+```
+
+The original remote build of patched SciPy 1.18.0 passed 87,723 tests, with
+8,342 skips, 300 expected failures, and 22 unexpected passes. The patch and
+override have been restored byte-for-byte from commit `24cbf74f`; those counts
+describe the earlier full build.
+
+Restoration checks confirmed that the patch applies to the pinned source
+without fuzz, portal's evaluated SciPy retains its existing patch and install
+checks, and pgvector uses the patched SciPy. A full package or system rebuild
+was not repeated for this restoration.

common/optional/x86-64-v3/patches/scipy/default.nix

@@ -0,0 +1,5 @@
+{ scipy }:
+scipy.overridePythonAttrs (old: {
+  # Keep the STFT tests enabled with tolerances for x86-64-v3 rounding.
+  patches = (old.patches or [ ]) ++ [ ./stft-test-tolerances.patch ];
+})

common/optional/x86-64-v3/patches/scipy/stft-test-tolerances.patch

@@ -0,0 +1,51 @@
+Subject: [PATCH] signal: allow floating-point rounding in STFT tests
+
+Keep the STFT tests enabled for x86-64-v3 builds. Allow two float32
+epsilons of relative error when comparing inverse-STFT implementations;
+float64 and the existing i686 override remain unchanged. Allow one
+float64 epsilon of absolute error in all three scaling round trips,
+which otherwise require exact zeros (observed residual: 4e-17 for a
+signal with amplitude 2).
+
+Upstream issue: https://github.com/scipy/scipy/issues/25488
+
+--- a/scipy/signal/tests/_scipy_spectral_test_shim.py
++++ b/scipy/signal/tests/_scipy_spectral_test_shim.py
+@@ -294,7 +294,7 @@
+ 
+     # Adapted tolerances to account for resolution loss:
+     atol = np.finfo(x.dtype).resolution*2  # instead of default atol = 0
+-    rtol = 1e-7  # default for np.allclose()
++    rtol = max(1e-7, 2 * np.finfo(x.dtype).eps)
+ 
+     # Relax atol on 32-Bit platforms a bit to pass CI tests.
+     #  - Not clear why there are discrepancies (in the FFT maybe?)
+--- a/scipy/signal/tests/test_spectral.py
++++ b/scipy/signal/tests/test_spectral.py
+@@ -2044,7 +2044,7 @@
+ 
+         # Test round trip:
+         x1 = istft(Zs, boundary=True, scaling='spectrum')[1]
+-        assert_allclose(x1, x)
++        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
+ 
+         # For a Hann-windowed 256 sample length FFT, we expect a peak at
+         # frequency 64 (since it is 1/4 the length of X) with a height of 1
+@@ -2074,7 +2074,7 @@
+ 
+         # Test round trip:
+         x1 = istft(Zp, input_onesided=False, boundary=True, scaling='psd')[1]
+-        assert_allclose(x1, x)
++        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
+ 
+         # The power of the one-sided psd-scaled STFT can be determined
+         # analogously (note that the two sides are not of equal shape):
+@@ -2094,7 +2094,7 @@
+ 
+         # Test round trip:
+         x1 = istft(Zp0, input_onesided=True, boundary=True, scaling='psd')[1]
+-        assert_allclose(x1, x)
++        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
+ 
+ 
+ class TestSampledSpectralRepresentations:

common/optional/zfs/snapshot.nix

@@ -1,4 +1,5 @@
 {
+  inputs,
   pkgs,
   lib,
   config,
@@ -6,6 +7,11 @@
 }:
 let
   cfg = config.services.snapshot_manager;
+  snapshotManagerPackages =
+    ps: with ps; [
+      httpx
+      typer
+    ];
 in
 {
   options = {
@@ -34,6 +40,41 @@ in
   };
 
   config = lib.mkIf cfg.enable {
+    nixpkgs.overlays = [
+      (final: _prev: {
+        snapshot_manager_python = final.python314.withPackages snapshotManagerPackages;
+        snapshot_manager_test_python = final.python314.withPackages (
+          ps:
+          snapshotManagerPackages ps
+          ++ (with ps; [
+            pyfakefs
+            pytest
+            pytest-asyncio
+            pytest-mock
+            pytest-xdist
+          ])
+        );
+        snapshot_manager_tests =
+          final.runCommand "snapshot-manager-tests"
+            {
+              nativeBuildInputs = [ final.snapshot_manager_test_python ];
+            }
+            ''
+              export HOME="$TMPDIR"
+              cd ${inputs.self}
+              pytest \
+                -o cache_dir="$TMPDIR/pytest-cache" \
+                tests/test_common.py \
+                tests/test_signal_alert.py \
+                tests/test_snapshot_manager.py \
+                tests/test_zfs.py
+              touch "$out"
+            '';
+      })
+    ];
+
+    system.checks = [ pkgs.snapshot_manager_tests ];
+
     systemd = {
       services.snapshot_manager = {
         description = "ZFS Snapshot Manager";
@@ -45,7 +86,7 @@ in
         };
         serviceConfig = {
           Type = "oneshot";
-          ExecStart = "${pkgs.my_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
+          ExecStart = "${pkgs.snapshot_manager_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
         }
         // lib.optionalAttrs (cfg.EnvironmentFile != null) {
           EnvironmentFile = cfg.EnvironmentFile;

common/optional/zfs/snapshot_config.toml

@@ -22,6 +22,12 @@ hourly = 0
 daily = 0
 monthly = 0
 
+["root_pool/nix_build"]
+15_min = 1
+hourly = 0
+daily = 0
+monthly = 0
+
 ["root_pool/var"]
 15_min = 8
 hourly = 24

flake.lock

@@ -20,28 +20,6 @@
         "type": "github"
       }
     },
-    "firefox-addons": {
-      "inputs": {
-        "nixpkgs": [
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "dir": "pkgs/firefox-addons",
-        "lastModified": 1788840136,
-        "narHash": "sha256-ej5jnQIfjbw4wwPzy8Y4ntG9F3asdF6YBkLqpYeGUxc=",
-        "owner": "rycee",
-        "repo": "nur-expressions",
-        "rev": "555b23e68256c5abfb8a85a385231830d1d7ad1f",
-        "type": "gitlab"
-      },
-      "original": {
-        "dir": "pkgs/firefox-addons",
-        "owner": "rycee",
-        "repo": "nur-expressions",
-        "type": "gitlab"
-      }
-    },
     "home-manager": {
       "inputs": {
         "nixpkgs": [
@@ -145,7 +123,6 @@
     "root": {
       "inputs": {
         "disko": "disko",
-        "firefox-addons": "firefox-addons",
         "home-manager": "home-manager",
         "nixos-hardware": "nixos-hardware",
         "nixpkgs": "nixpkgs_2",

flake.nix

@@ -26,11 +26,6 @@
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
-    firefox-addons = {
-      url = "gitlab:rycee/nur-expressions?dir=pkgs/firefox-addons";
-      inputs.nixpkgs.follows = "nixpkgs";
-    };
-
     sops-nix = {
       url = "github:Mic92/sops-nix";
       inputs.nixpkgs.follows = "nixpkgs";

overlays/default.nix

@@ -1,13 +1,13 @@
 { inputs, ... }:
 {
-  # When applied, the stable nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.stable'
+  # Additional package sets are accessible through `pkgs.<name>`.
   stable = final: _prev: {
     stable = import inputs.nixpkgs-stable {
       system = final.stdenv.hostPlatform.system;
       config.allowUnfree = true;
     };
   };
-  # When applied, the master nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.master'
+
   master = final: _prev: {
     master = import inputs.nixpkgs-master {
       system = final.stdenv.hostPlatform.system;
@@ -15,34 +15,12 @@
     };
   };
 
-  python-env = final: _prev: {
+  # Baseline x86-64 (v1) packages for prebuilt applications that should not
-    my_python = final.python314.withPackages (
+  # inherit an x86-64-v3 host platform.
-      ps: with ps; [
+  x86-v1 = final: _prev: {
-        alembic
+    x86-v1 = import inputs.nixpkgs {
-        apprise
+      system = final.stdenv.hostPlatform.system;
-        fastapi
+      config.allowUnfree = true;
-        fastapi-cli
+    };
-        httpx
-        jinja2
-        mypy
-        pgvector
-        psycopg
-        pydantic
-        pyfakefs
-        pytest
-        pytest-cov
-        pytest-mock
-        pytest-xdist
-        python-multipart
-        pydantic-settings
-        ruff
-        sqlalchemy
-        tenacity
-        tinytuya
-        typer
-        uvicorn
-        websockets
-      ]
-    );
   };
 }

pyproject.toml

@@ -6,49 +6,6 @@ authors = [{ name = "Richie Cahill", email = "richie@tmmworkshop.com" }]
 requires-python = "~=3.14.0"
 readme = "README.md"
 license = "MIT"
-# these dependencies are a best effort and aren't guaranteed to work
-# for up-to-date dependencies, see overlays/default.nix
-dependencies = [
-    "alembic",
-    "apprise",
-    "beautifulsoup4",
-    "bm25s",
-    "ebooklib",
-    "fastapi",
-    "fastapi-cli",
-    "httpx",
-    "jinja2",
-    "pgvector",
-    "psycopg[binary]",
-    "pydantic",
-    "pydantic-settings",
-    "python-multipart",
-    "sqlalchemy[asyncio]",
-    "tenacity",
-    "tiktoken",
-    "tinytuya",
-    "typer",
-    "uvicorn",
-    "websockets",
-    "yake",
-]
-
-[project.scripts]
-database = "python.database_cli:app"
-whisper-transcribe = "python.tools.whisper.transcribe:main"
-
-[dependency-groups]
-dev = [
-    "aiosqlite",
-    "mypy",
-    "pyfakefs",
-    "pytest-asyncio",
-    "pytest-cov",
-    "pytest-mock",
-    "pytest-xdist",
-    "pytest",
-    "ruff",
-]
 
 [tool.ruff]
 

python/gitea_flake_lock.py

@@ -14,7 +14,7 @@ DEFAULT_BASE_BRANCH = "main"
 DEFAULT_BRANCH = "automation/update-flake-lock"
 DEFAULT_GITEA_URL = "https://gitea.tmmworkshop.com"
 PR_LABELS = ["dependencies", "automated", "flake_lock_update"]
-PR_CHECK_WORKFLOWS = ["build_systems.yml", "treefmt.yml", "pytest.yml"]
+PR_CHECK_WORKFLOWS = ["build_systems.yml", "treefmt.yml"]
 PR_TITLE = "Update flake.lock"
 PR_BODY = "Automated flake.lock update."
 

python/installer/__main__.py

@@ -264,7 +264,7 @@ def create_nix_hardware_file(mnt_dir: str, disks: Sequence[str], encrypt: str |
         '    "/home" = {\n      device = "root_pool/home";\n      fsType = "zfs";\n    };\n\n'
         '    "/var" = {\n      device = "root_pool/var";\n      fsType = "zfs";\n    };\n\n'
         '    "/nix" = {\n      device = "root_pool/nix";\n      fsType = "zfs";\n    };\n\n'
-        '    "/nix_build" = {\n      device = "root_pool/nix_build";\n      fsType = "zfs";\n    };\n\n'
+        '    "/nix/var/nix/builds" = {\n      device = "root_pool/nix_build";\n      fsType = "zfs";\n    };\n\n'
         '    "/boot" = {\n'
         f'      device = "/dev/disk/by-uuid/{get_boot_drive_id(disks[0])}";\n'
         '      fsType = "vfat";\n      options = [\n        "fmask=0077"\n'
@@ -287,7 +287,7 @@ def install_nixos(mnt_dir: str, disks: Sequence[str], encrypt: str | None) -> No
     run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/home", f"{mnt_dir}/home"))
     run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/var", f"{mnt_dir}/var"))
     run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/nix", f"{mnt_dir}/nix"))
-    run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/nix_build", f"{mnt_dir}/nix_build"))
+    run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/nix_build", f"{mnt_dir}/nix/var/nix/builds"))
 
     for disk in disks:
         run_output(("mkfs.vfat", "-n", "EFI", f"{disk}-part1"))

python/signal_alert.py

@@ -5,10 +5,13 @@ from __future__ import annotations
 import logging
 from os import getenv
 
-from apprise import Apprise
+import httpx
 
 logger = logging.getLogger(__name__)
 
+SIGNAL_API_URL = "http://localhost:8989/v2/send"
+SIGNAL_API_TIMEOUT = 4.0
+
 
 def signal_alert(body: str, title: str = "") -> None:
     """Send a signal alert.
@@ -17,14 +20,31 @@ def signal_alert(body: str, title: str = "") -> None:
         body (str): The body of the alert.
         title (str, optional): The title of the alert. Defaults to "".
     """
-    apprise_client = Apprise()
-
     from_phone = getenv("SIGNAL_ALERT_FROM_PHONE")
     to_phone = getenv("SIGNAL_ALERT_TO_PHONE")
     if not from_phone or not to_phone:
         logger.info("SIGNAL_ALERT_FROM_PHONE or SIGNAL_ALERT_TO_PHONE not set")
         return
 
-    apprise_client.add(f"signal://localhost:8989/{from_phone}/{to_phone}")
+    # Apprise's Signal integration did not support titles, so preserve that behavior.
+    if title:
+        logger.debug("Signal does not support notification titles; ignoring title")
 
-    apprise_client.notify(title=title, body=body)
+    try:
+        response = httpx.post(
+            SIGNAL_API_URL,
+            json={
+                "message": body,
+                "number": from_phone,
+                "recipients": [to_phone],
+                "text_mode": "normal",
+            },
+            timeout=SIGNAL_API_TIMEOUT,
+            follow_redirects=True,
+        )
+    except httpx.HTTPError:
+        logger.exception("Unable to contact the Signal API")
+        return
+
+    if response.status_code not in {httpx.codes.OK, httpx.codes.CREATED}:
+        logger.error("Signal API returned HTTP status %d", response.status_code)

python/system_tests/__init__.py

@@ -1 +0,0 @@
-"""system_tests."""

python/system_tests/components.py

@@ -1,99 +0,0 @@
-"""Validate Jeeves."""
-
-from __future__ import annotations
-
-import logging
-from copy import copy
-from re import search
-from time import sleep
-from typing import TYPE_CHECKING
-
-from python.common import bash_wrapper
-from python.zfs import Zpool
-
-if TYPE_CHECKING:
-    from collections.abc import Sequence
-
-logger = logging.getLogger(__name__)
-
-
-def zpool_tests(pool_names: Sequence[str], zpool_capacity_threshold: int = 90) -> list[str] | None:
-    """Check the zpool health and capacity.
-
-    Args:
-        pool_names (Sequence[str]): A list of pool names to test.
-        zpool_capacity_threshold (int, optional): The threshold for the zpool capacity. Defaults to 90.
-
-    Returns:
-        list[str] | None: A list of errors if any.
-    """
-    logger.info("Testing zpool")
-
-    errors: list[str] = []
-    for pool_name in pool_names:
-        pool = Zpool(pool_name)
-        if pool.health != "ONLINE":
-            errors.append(f"{pool.name} is {pool.health}")
-        if pool.capacity >= zpool_capacity_threshold:
-            errors.append(f"{pool.name} is low on space")
-
-    upgrade_status, _ = bash_wrapper("zpool upgrade")
-    if not search(r"Every feature flags pool has all supported and requested features enabled.", upgrade_status):
-        errors.append("ZPool out of date run `sudo zpool upgrade -a`")
-
-    return errors
-
-
-def systemd_tests(
-    service_names: Sequence[str],
-    max_retries: int = 30,
-    retry_delay_secs: int = 1,
-    retryable_statuses: Sequence[str] | None = None,
-    valid_statuses: Sequence[str] | None = None,
-) -> list[str] | None:
-    """Tests a systemd services.
-
-    Args:
-        service_names (Sequence[str]): A list of service names to test.
-        max_retries (int, optional): The maximum number of retries. Defaults to 30.
-            minimum value is 1.
-        retry_delay_secs (int, optional): The delay between retries in seconds. Defaults to 1.
-            minimum value is 1.
-        retryable_statuses (Sequence[str] | None, optional): A list of retryable statuses. Defaults to None.
-        valid_statuses (Sequence[str] | None, optional): A list of valid statuses. Defaults to None.
-
-    Returns:
-        list[str] | None: A list of errors if any.
-    """
-    logger.info("Testing systemd service")
-
-    max_retries = max(max_retries, 1)
-    retry_delay_secs = max(retry_delay_secs, 1)
-    last_try = max_retries - 1
-
-    if retryable_statuses is None:
-        retryable_statuses = ("inactive\n", "activating\n")
-
-    if valid_statuses is None:
-        valid_statuses = ("active\n",)
-
-    service_names_set = set(service_names)
-
-    errors: set[str] = set()
-    for retry in range(max_retries):
-        if not service_names_set:
-            break
-        logger.info(f"Testing systemd service in {retry + 1} of {max_retries}")
-        service_names_to_test = copy(service_names_set)
-        for service_name in service_names_to_test:
-            service_status, _ = bash_wrapper(f"systemctl is-active {service_name}")
-            if service_status in valid_statuses:
-                service_names_set.remove(service_name)
-                continue
-            if service_status in retryable_statuses and retry < last_try:
-                continue
-            errors.add(f"{service_name} is {service_status.strip()}")
-
-        sleep(retry_delay_secs)
-
-    return list(errors)

python/system_tests/validate_system.py

@@ -1,67 +0,0 @@
-"""Validate {server_name}."""
-
-import logging
-import sys
-import tomllib
-from os import environ
-from pathlib import Path  # noqa: TC003 This is required for the typer CLI
-from socket import gethostname
-
-import typer
-
-from python.common import configure_logger
-from python.signal_alert import signal_alert
-from python.system_tests.components import systemd_tests, zpool_tests
-
-logger = logging.getLogger(__name__)
-
-
-def load_config_data(config_file: Path) -> dict[str, list[str]]:
-    """Load a TOML configuration file.
-
-    Args:
-        config_file (Path): The path to the configuration file.
-
-    Returns:
-        dict: The configuration data.
-    """
-    return tomllib.loads(config_file.read_text())
-
-
-def main(config_file: Path) -> None:
-    """Main."""
-    configure_logger(level=environ.get("LOG_LEVEL", "INFO"))
-
-    server_name = gethostname()
-    logger.info(f"Starting {server_name} validation")
-
-    config_data = load_config_data(config_file)
-
-    errors: list[str] = []
-    try:
-        if config_data.get("zpools") and (zpool_errors := zpool_tests(config_data["zpools"])):
-            errors.extend(zpool_errors)
-
-        if config_data.get("services") and (systemd_errors := systemd_tests(config_data["services"])):
-            errors.extend(systemd_errors)
-
-    except Exception as error:
-        logger.exception(f"{server_name} validation failed")
-        errors.append(f"{server_name} validation failed: {error}")
-
-    if errors:
-        logger.error(f"{server_name} validation failed: \n{'\n'.join(errors)}")
-        signal_alert(f"{server_name} validation failed {errors}")
-
-        sys.exit(1)
-
-    logger.info(f"{server_name} validation passed")
-
-
-def cli() -> None:
-    """CLI."""
-    typer.run(main)
-
-
-if __name__ == "__main__":
-    cli()

shell.nix

@@ -8,7 +8,7 @@
   # loader path, which NixOS does not provide globally.
   ebook-search = pkgs.mkShell {
     nativeBuildInputs = with pkgs; [
-      my_python
+      python314
       uv
     ];
     LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
@@ -23,7 +23,6 @@
       nix
       home-manager
       git
-      my_python
 
       ssh-to-age
       gnupg

systems/bob/default.nix

@@ -3,7 +3,6 @@
   imports = [
     "${inputs.self}/users/math"
     "${inputs.self}/users/richie"
-    "${inputs.self}/users/steve"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/docker.nix"
     "${inputs.self}/common/optional/monitoring-agent.nix"
@@ -14,7 +13,7 @@
     "${inputs.self}/common/optional/systemd-boot.nix"
     "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/yubikey.nix"
+    "${inputs.self}/common/optional/x86-64-v3"
     "${inputs.self}/common/optional/zfs"
     ./hardware.nix
     ./syncthing.nix

systems/bob/hardware.nix

@@ -50,6 +50,11 @@
       fsType = "zfs";
     };
 
+    "/nix/var/nix/builds" = {
+      device = "root_pool/nix_build";
+      fsType = "zfs";
+    };
+
     "/var" = {
       device = "root_pool/var";
       fsType = "zfs";

systems/bob/snapshot_config.toml

@@ -22,6 +22,12 @@ hourly = 0
 daily = 0
 monthly = 0
 
+["root_pool/nix_build"]
+15_min = 1
+hourly = 0
+daily = 0
+monthly = 0
+
 ["root_pool/models"]
 15_min = 4
 hourly = 24

systems/brain/default.nix

@@ -9,6 +9,7 @@
     "${inputs.self}/common/optional/systemd-boot.nix"
     "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
+    "${inputs.self}/common/optional/x86-64-v3"
     "${inputs.self}/common/optional/zfs"
     ./docker
     ./hardware.nix

systems/brain/hardware.nix

@@ -53,6 +53,11 @@
       fsType = "zfs";
     };
 
+    "/nix/var/nix/builds" = {
+      device = "root_pool/nix_build";
+      fsType = "zfs";
+    };
+
     "/boot" = {
       device = "/dev/disk/by-uuid/12CE-A600";
       fsType = "vfat";

systems/brain/services/heater.nix

@@ -4,6 +4,20 @@
   ...
 }:
 {
+  nixpkgs.overlays = [
+    (final: _prev: {
+      heater_python = final.python314.withPackages (
+        ps: with ps; [
+          fastapi
+          pydantic
+          tinytuya
+          typer
+          uvicorn
+        ]
+      );
+    })
+  ];
+
   networking.firewall.allowedTCPPorts = [ 8124 ];
 
   systemd.services.heater-api = {
@@ -17,7 +31,7 @@
 
     serviceConfig = {
       Type = "simple";
-      ExecStart = "${pkgs.my_python}/bin/python -m python.heater.main --host 0.0.0.0 --port 8124";
+      ExecStart = "${pkgs.heater_python}/bin/python -m python.heater.main --host 0.0.0.0 --port 8124";
       EnvironmentFile = "/etc/heater.env";
       Restart = "on-failure";
       RestartSec = "5s";

systems/brain/services/home_assistant.nix

@@ -68,7 +68,6 @@
           paho-mqtt # for mqtt
           psycopg2 # for postgresql
           py-improv-ble-client # for esphome
-          pymodbus # for modbus
           pyopenweathermap # for weather
           pymetno # for met.no weather
           uiprotect # for ubiquiti integration
@@ -76,7 +75,10 @@
           jsonpath # for rest sensors
           monarchmoneycommunity # for monarch
         ];
-      extraComponents = [ "isal" ];
+      extraComponents = [
+        "isal"
+        "modbus" # for victron modbus integration
+      ];
       customComponents = with pkgs.home-assistant-custom-components; [
         garmin_connect
       ];

systems/jeeves/containers.nix

@@ -0,0 +1,71 @@
+{ pkgs, ... }:
+let
+  vars = import ./vars.nix;
+in
+{
+  virtualisation.docker.daemon.settings."data-root" = "${vars.containers}/docker";
+
+  # nixos-container hardcodes its state directory to /var/lib/nixos-containers,
+  # so route it to the shared container dataset with a bind mount.
+  fileSystems."/var/lib/nixos-containers" = {
+    device = "${vars.containers}/nixos-containers";
+    fsType = "none";
+    options = [ "bind" ];
+  };
+
+  systemd.services = {
+    prepare-docker-storage = {
+      description = "Create Docker storage directory";
+      requiredBy = [ "docker.service" ];
+      before = [ "docker.service" ];
+      requires = [ "zfs-mount.service" ];
+      after = [ "zfs-mount.service" ];
+      path = [
+        pkgs.coreutils
+        pkgs.util-linux
+      ];
+
+      unitConfig = {
+        DefaultDependencies = false;
+        RequiresMountsFor = [ "/nix" ];
+      };
+
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+
+      script = ''
+        mountpoint -q ${vars.containers}
+        install -d -m 0710 ${vars.containers}/docker
+      '';
+    };
+
+    prepare-nixos-container-storage = {
+      description = "Create NixOS container storage directory";
+      requiredBy = [ "var-lib-nixos\\x2dcontainers.mount" ];
+      before = [ "var-lib-nixos\\x2dcontainers.mount" ];
+      requires = [ "zfs-mount.service" ];
+      after = [ "zfs-mount.service" ];
+      path = [
+        pkgs.coreutils
+        pkgs.util-linux
+      ];
+
+      unitConfig = {
+        DefaultDependencies = false;
+        RequiresMountsFor = [ "/nix" ];
+      };
+
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+
+      script = ''
+        mountpoint -q ${vars.containers}
+        install -d -m 0755 ${vars.containers}/nixos-containers
+      '';
+    };
+  };
+}

systems/jeeves/default.nix

@@ -1,4 +1,4 @@
-{ inputs, lib, ... }:
+{ inputs, ... }:
 let
   vars = import ./vars.nix;
 in
@@ -6,7 +6,6 @@ in
   imports = [
     "${inputs.self}/users/math"
     "${inputs.self}/users/richie"
-    "${inputs.self}/users/steve"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/docker.nix"
     "${inputs.self}/common/optional/monitoring-agent.nix"
@@ -14,20 +13,18 @@ in
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
+    "${inputs.self}/common/optional/x86-64-v3"
     "${inputs.self}/common/optional/zfs"
     ./monitoring
     ./docker
     ./services
+    ./containers.nix
     ./hardware.nix
     ./networking
     ./programs.nix
     ./runners
     ./syncthing.nix
   ];
-  nix.settings.system-features = lib.mkAfter [
-    "gccarch-x86-64-v2"
-    "gccarch-x86-64-v3"
-  ];
 
   services = {
     openssh.ports = [ 629 ];

systems/jeeves/hardware.nix

@@ -96,6 +96,11 @@ in
       fsType = "zfs";
     };
 
+    "/nix/var/nix/builds" = {
+      device = "root_pool/nix_build";
+      fsType = "zfs";
+    };
+
     "/var" = {
       device = "root_pool/var";
       fsType = "zfs";

systems/jeeves/runners/default.nix

@@ -21,5 +21,16 @@
     nix-builder-12.enable = true;
     nix-builder-13.enable = true;
     nix-builder-14.enable = true;
+
+    # Warm the shared x86-64-v3 cache before the smaller per-system runners
+    # start. Eight jobs with eight cores each can use Jeeves' 64 logical CPUs,
+    # while the 6000% quota leaves some capacity for its normal services.
+    nix-cache-builder = {
+      enable = true;
+      labels = [ "nix-cache-builder:host" ];
+      cores = 8;
+      maxJobs = 8;
+      cpuQuota = "6000%";
+    };
   };
 }

systems/jeeves/runners/nix_builder.nix

@@ -11,11 +11,8 @@ let
   cfg = config.services.nix_builder;
   runnerUsername = "gitea-runner";
   runnerUserid = 601;
-  runnerLabels = [
-    "self-hosted:host"
-    "nixos:host"
-  ];
   containerConfig =
+    containerCfg:
     {
       config,
       pkgs,
@@ -23,6 +20,13 @@ let
       ...
     }:
     let
+      giteaAutomationPython = pkgs.python314.withPackages (
+        ps: with ps; [
+          httpx
+          pydantic
+          typer
+        ]
+      );
       runnerConfigFile = (pkgs.formats.yaml { }).generate "gitea-runner.yaml" { };
       registerRunner = pkgs.writeShellApplication {
         name = "register-gitea-runner";
@@ -43,6 +47,8 @@ let
         useHostResolvConf = false;
       };
       nix.settings = {
+        inherit (containerCfg) cores;
+        max-jobs = containerCfg.maxJobs;
         system-features = lib.mkAfter [
           "gccarch-x86-64-v2"
           "gccarch-x86-64-v3"
@@ -85,20 +91,22 @@ let
         enable = true;
         name = "jeeves-nix-builder";
         url = "http://192.168.99.14:6443/";
-        labels = runnerLabels;
+        labels = containerCfg.labels;
         tokenFile = "/run/secrets/gitea-runners/registration-token";
+        settings.runner.timeout = "12h";
         hostPackages = with pkgs; [
           bash
           coreutils
           curl
           gawk
           gitMinimal
+          giteaAutomationPython
           gnused
-          my_python
           nix
           nixfmt
           nixos-rebuild
           nodejs
+          ruff
           treefmt
           wget
         ];
@@ -109,20 +117,21 @@ let
           User = mkForce runnerUsername;
           Group = mkForce runnerUsername;
           ExecStartPre = mkForce [
-            "${getExe registerRunner} builder http://192.168.99.14:6443/ ${runnerConfigFile} ${escapeShellArgs runnerLabels}"
+            "${getExe registerRunner} builder http://192.168.99.14:6443/ ${runnerConfigFile} ${escapeShellArgs containerCfg.labels}"
           ];
         };
       };
       system.stateVersion = "24.05";
     };
-  sharedContainerPath =
+  mkContainerPath =
+    containerCfg:
     (import "${pkgs.path}/nixos/lib/eval-config.nix" {
       modules = [
         {
           boot.isNspawnContainer = true;
           nixpkgs.pkgs = pkgs;
         }
-        containerConfig
+        (containerConfig containerCfg)
       ];
       system = null;
     }).config.system.build.toplevel;
@@ -140,7 +149,36 @@ in
         types.submodule (
           { name, ... }:
           {
-            options.enable = mkEnableOption "Gitea runner container";
+            options = {
+              enable = mkEnableOption "Gitea runner container";
+
+              labels = mkOption {
+                type = types.listOf types.str;
+                default = [
+                  "self-hosted:host"
+                  "nixos:host"
+                ];
+                description = "Gitea Actions labels advertised by this runner.";
+              };
+
+              cores = mkOption {
+                type = types.ints.positive;
+                default = 8;
+                description = "Number of cores made available to each Nix build job.";
+              };
+
+              maxJobs = mkOption {
+                type = types.ints.positive;
+                default = 2;
+                description = "Maximum number of Nix build jobs run in parallel.";
+              };
+
+              cpuQuota = mkOption {
+                type = types.str;
+                default = "800%";
+                description = "systemd CPU quota for the runner container.";
+              };
+            };
           }
         )
       );
@@ -162,7 +200,7 @@ in
     containers = mapAttrs (
       name: containerCfg:
       mkIf containerCfg.enable {
-        path = sharedContainerPath;
+        path = mkContainerPath containerCfg;
         autoStart = true;
         privateNetwork = true;
         hostBridge = cfg.bridgeName;
@@ -188,15 +226,14 @@ in
     ) cfg.containers;
 
     systemd = {
-      services = builtins.listToAttrs (
+      services = mapAttrs' (
-        map (name: {
+        name: containerCfg:
-          name = "container@${name}";
+        nameValuePair "container@${name}" {
-          value = {
+          requires = [ "gitea.service" ];
-            requires = [ "gitea.service" ];
+          after = [ "gitea.service" ];
-            after = [ "gitea.service" ];
+          serviceConfig.CPUQuota = containerCfg.cpuQuota;
-          };
+        }
-        }) (builtins.attrNames (filterAttrs (_: c: c.enable) cfg.containers))
+      ) (filterAttrs (_: c: c.enable) cfg.containers);
-      );
 
       tmpfiles.rules = [
         "d ${vars.uv_cache} 0755 ${runnerUsername} ${runnerUsername} - -"

systems/jeeves/scripts/zfs.sh

@@ -17,6 +17,7 @@ sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnode
 # media datasets
 sudo zfs create media/temp -o sync=disabled -o redundant_metadata=none
 sudo zfs create media/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key
+sudo zfs create media/secure/containers -o mountpoint=/zfs/media/containers -o compression=lz4 -o sync=disabled -o redundant_metadata=some -o normalization=none -o utf8only=off
 sudo zfs create media/secure/docker -o compression=zstd-9
 sudo zfs create media/secure/github-runners -o compression=zstd-9 -o sync=disabled
 sudo zfs create media/secure/notes -o copies=2

systems/jeeves/services/camofox-browser.nix

@@ -1,80 +0,0 @@
-{
-  ...
-}:
-let
-  vars = import ../vars.nix;
-in
-{
-  systemd.tmpfiles.rules = [
-    "d ${vars.docker_configs}/camofox-browser 0750 root root - -"
-  ];
-
-  containers.camofox-browser = {
-    autoStart = true;
-    privateNetwork = false;
-    bindMounts = {
-      camofox-browser = {
-        hostPath = "${vars.docker_configs}/camofox-browser";
-        mountPoint = "/var/lib/camofox-browser";
-        isReadOnly = false;
-      };
-    };
-    config =
-      {
-        pkgs,
-        lib,
-        ...
-      }:
-      {
-        networking.hostName = "camofox-browser";
-
-        environment.systemPackages = with pkgs; [
-          ffmpeg
-          git
-          nodejs
-          python3Packages.yt-dlp
-        ];
-
-        systemd.services.camofox-browser = {
-          description = "Camofox browser server";
-          wantedBy = [ "multi-user.target" ];
-          after = [ "network.target" ];
-          environment = {
-            CAMOFOX_HOST = "127.0.0.1";
-            CAMOFOX_PORT = "9377";
-            HOME = "/var/lib/camofox-browser";
-          };
-          path = with pkgs; [
-            bash
-            coreutils
-            git
-            nodejs
-          ];
-          serviceConfig = {
-            Restart = "always";
-            RestartSec = "5s";
-            WorkingDirectory = "/var/lib/camofox-browser";
-          };
-          script = ''
-            set -eu
-
-            app_dir=/var/lib/camofox-browser/app
-
-            if [ ! -d "$app_dir/.git" ]; then
-              git clone --depth 1 https://github.com/jo-inc/camofox-browser "$app_dir"
-            fi
-
-            cd "$app_dir"
-
-            if [ ! -d node_modules ]; then
-              npm install
-            fi
-
-            exec npm start
-          '';
-        };
-
-        system.stateVersion = lib.mkDefault "24.05";
-      };
-  };
-}

systems/jeeves/services/gems.nix

@@ -6,8 +6,47 @@
 let
   vars = import ../vars.nix;
   stateDir = "${vars.services}/gems";
+  gemsPackages =
+    ps: with ps; [
+      fastapi
+      jinja2
+      pydantic
+      pydantic-settings
+      python-multipart
+      typer
+      uvicorn
+    ];
 in
 {
+  nixpkgs.overlays = [
+    (final: _prev: {
+      gems_python = final.python314.withPackages gemsPackages;
+      gems_test_python = final.python314.withPackages (
+        ps:
+        gemsPackages ps
+        ++ (with ps; [
+          httpx
+          pytest
+          pytest-asyncio
+          pytest-xdist
+        ])
+      );
+      gems_tests =
+        final.runCommand "gems-tests"
+          {
+            nativeBuildInputs = [ final.gems_test_python ];
+          }
+          ''
+            export HOME="$TMPDIR"
+            cd ${inputs.self}
+            pytest -o cache_dir="$TMPDIR/pytest-cache" tests/gems
+            touch "$out"
+          '';
+    })
+  ];
+
+  system.checks = [ pkgs.gems_tests ];
+
   users.groups.gems = { };
   users.users.gems = {
     isSystemUser = true;
@@ -36,7 +75,7 @@ in
       Type = "simple";
       User = "gems";
       Group = "gems";
-      ExecStart = "${pkgs.my_python}/bin/python -m python.gems.main --host 0.0.0.0 --port 8002";
+      ExecStart = "${pkgs.gems_python}/bin/python -m python.gems.main --host 0.0.0.0 --port 8002";
       Restart = "on-failure";
       RestartSec = "5s";
       StandardOutput = "journal";

systems/jeeves/services/gitea.nix

@@ -19,6 +19,7 @@ in
       actions = {
         ENABLED = true;
         DEFAULT_ACTIONS_URL = "github";
+        ENDLESS_TASK_TIMEOUT = "12h";
       };
       service.DISABLE_REGISTRATION = true;
       server = {

systems/jeeves/services/open_webui.nix

@@ -1,12 +0,0 @@
-{
-  services.open-webui = {
-    enable = true;
-    host = "0.0.0.0";
-    environment = {
-      ANONYMIZED_TELEMETRY = "False";
-      DO_NOT_TRACK = "True";
-      SCARF_NO_ANALYTICS = "True";
-      OLLAMA_API_BASE_URL = "http://127.0.0.1:11434";
-    };
-  };
-}

systems/jeeves/services/systemd.nix

@@ -1,11 +1,7 @@
 {
   pkgs,
-  inputs,
   ...
 }:
-let
-  vars = import ../vars.nix;
-in
 {
   systemd = {
     services = {
@@ -30,21 +26,6 @@ in
           ExecStart = "${pkgs.bash}/bin/bash -c 'echo 1 > /sys/bus/pci/devices/0000:61:00.0/remove'";
         };
       };
-      startup_validation = {
-        requires = [ "network-online.target" ];
-        after = [ "network-online.target" ];
-        wantedBy = [ "multi-user.target" ];
-        description = "validates startup";
-        path = [ pkgs.zfs ];
-        environment = {
-          PYTHONPATH = "${inputs.self}/";
-        };
-        serviceConfig = {
-          EnvironmentFile = "${vars.secrets}/services/server-validation";
-          Type = "oneshot";
-          ExecStart = "${pkgs.my_python}/bin/python -m python.system_tests.validate_system '${./validate_system.toml}'";
-        };
-      };
     };
     timers = {
       plex_permission = {
@@ -55,13 +36,6 @@ in
           Unit = "plex_permission.service";
         };
       };
-      startup_validation = {
-        wantedBy = [ "timers.target" ];
-        timerConfig = {
-          OnBootSec = "10min";
-          Unit = "startup_validation.service";
-        };
-      };
     };
   };
 }

systems/jeeves/services/validate_system.toml

@@ -1,6 +0,0 @@
-zpool = ["root_pool", "storage", "media"]
-services = [
-    "audiobookshelf",
-    "docker",
-    "jellyfin",
-]

systems/jeeves/snapshot_config.toml

@@ -23,6 +23,12 @@ hourly = 0
 daily = 0
 monthly = 0
 
+["root_pool/nix_build"]
+15_min = 1
+hourly = 0
+daily = 0
+monthly = 0
+
 ["root_pool/var"]
 15_min = 8
 hourly = 24
@@ -60,6 +66,12 @@ daily = 30
 monthly = 12
 
 # media
+["media/secure/containers"]
+15_min = 2
+hourly = 0
+daily = 0
+monthly = 0
+
 ["media/temp"]
 15_min = 2
 hourly = 0

systems/jeeves/vars.nix

@@ -5,6 +5,7 @@ let
 in
 {
   inherit zfs_media zfs_storage zfs_scratch;
+  containers = "${zfs_media}/containers";
   database = "${zfs_media}/database";
   docker = "${zfs_media}/docker";
   docker_configs = "${zfs_media}/docker/configs";

systems/portal-1/default.nix

@@ -10,13 +10,12 @@
     "${inputs.self}/users/richie"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/tailscale.nix"
+    "${inputs.self}/common/optional/x86-64-v3"
     ./disk-config.nix
     ./haproxy
     ./monitoring.nix
   ];
 
-  nixpkgs.hostPlatform = "x86_64-linux";
-
   boot = {
     # Avoid consuming the VM's limited memory for /tmp.
     tmp.useTmpfs = false;

systems/rhapsody-in-green/agent_logger.nix

@@ -1,35 +0,0 @@
-{
-  pkgs,
-  inputs,
-  ...
-}:
-{
-  systemd.services.agent-logger = {
-    description = "Unified agent logger";
-    after = [ "local-fs.target" ];
-    wantedBy = [ "multi-user.target" ];
-
-    environment = {
-      AGENT_LOG_DB = "/var/lib/agent-logger/agent_log.sqlite";
-      HOME = "/home/richie";
-      PYTHONPATH = "${inputs.self}";
-    };
-
-    serviceConfig = {
-      Type = "simple";
-      User = "richie";
-      WorkingDirectory = "/home/richie";
-      ExecStart = "${pkgs.my_python}/bin/python -m python.agent_logger.main";
-      StateDirectory = "agent-logger";
-      Restart = "on-failure";
-      RestartSec = "5s";
-      StandardOutput = "journal";
-      StandardError = "journal";
-      NoNewPrivileges = true;
-      ProtectSystem = "strict";
-      ProtectHome = "read-only";
-      PrivateTmp = true;
-      ReadOnlyPaths = [ "${inputs.self}" ];
-    };
-  };
-}

systems/rhapsody-in-green/default.nix

@@ -12,7 +12,6 @@
     "${inputs.self}/common/optional/yubikey.nix"
     "${inputs.self}/common/optional/zfs"
     ./hardware.nix
-    ./open_webui.nix
     ./programs.nix
     ./qmk.nix
     ./syncthing.nix

systems/rhapsody-in-green/hardware.nix

@@ -34,14 +34,22 @@
       device = "root_pool/root";
       fsType = "zfs";
     };
+
     "/home" = {
       device = "root_pool/home";
       fsType = "zfs";
     };
+
+    "/nix/var/nix/builds" = {
+      device = "root_pool/nix_build";
+      fsType = "zfs";
+    };
+
     "/var" = {
       device = "root_pool/var";
       fsType = "zfs";
     };
+
     "/boot" = {
       device = "/dev/disk/by-uuid/12CE-A600";
       fsType = "vfat";

systems/rhapsody-in-green/open_webui.nix

@@ -1,13 +0,0 @@
-{
-  services.open-webui = {
-    enable = true;
-    host = "0.0.0.0";
-    environment = {
-      ANONYMIZED_TELEMETRY = "False";
-      DO_NOT_TRACK = "True";
-      SCARF_NO_ANALYTICS = "True";
-      OLLAMA_API_BASE_URL = "https://ollama.com";
-      WEBUI_AUTH = "False";
-    };
-  };
-}

tests/test_components.py

@@ -1,104 +0,0 @@
-"""test_components."""
-
-from pytest_mock import MockerFixture
-
-from python.system_tests.components import systemd_tests, zpool_tests
-from python.zfs import Zpool
-
-temp = "Every feature flags pool has all supported and requested features enabled.\n"
-
-SYSTEM_TESTS_COMPONENTS = "python.system_tests.components"
-
-
-def test_zpool_tests(mocker: MockerFixture) -> None:
-    """test_zpool_tests."""
-    mock_zpool = mocker.MagicMock(spec=Zpool)
-    mock_zpool.health = "ONLINE"
-    mock_zpool.capacity = 70
-    mock_zpool.name = "Main"
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=(temp, ""))
-    errors = zpool_tests(("Main",))
-    assert errors == []
-
-
-def test_zpool_tests_out_of_date(mocker: MockerFixture) -> None:
-    """test_zpool_tests_out_of_date."""
-    mock_zpool = mocker.MagicMock(spec=Zpool)
-    mock_zpool.health = "ONLINE"
-    mock_zpool.capacity = 70
-    mock_zpool.name = "Main"
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=("", ""))
-    errors = zpool_tests(("Main",))
-    assert errors == ["ZPool out of date run `sudo zpool upgrade -a`"]
-
-
-def test_zpool_tests_out_of_space(mocker: MockerFixture) -> None:
-    """test_zpool_tests_out_of_space."""
-    mock_zpool = mocker.MagicMock(spec=Zpool)
-    mock_zpool.health = "ONLINE"
-    mock_zpool.capacity = 100
-    mock_zpool.name = "Main"
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=(temp, ""))
-    errors = zpool_tests(("Main",))
-    assert errors == ["Main is low on space"]
-
-
-def test_zpool_tests_offline(mocker: MockerFixture) -> None:
-    """test_zpool_tests_offline."""
-    mock_zpool = mocker.MagicMock(spec=Zpool)
-    mock_zpool.health = "OFFLINE"
-    mock_zpool.capacity = 70
-    mock_zpool.name = "Main"
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=(temp, ""))
-    errors = zpool_tests(("Main",))
-    assert errors == ["Main is OFFLINE"]
-
-
-def test_systemd_tests(mocker: MockerFixture) -> None:
-    """test_systemd_tests."""
-    mocker.patch(
-        f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper",
-        side_effect=[
-            ("inactive\n", ""),
-            ("active\n", ""),
-        ],
-    )
-    errors = systemd_tests(("docker",))
-    assert errors == []
-    """test_systemd_tests."""
-
-
-def test_systemd_tests_multiple_negative_retries(mocker: MockerFixture) -> None:
-    """test_systemd_tests_fail."""
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=("active\n", ""))
-    errors = systemd_tests(("docker",), max_retries=-1, retry_delay_secs=-1)
-    assert errors == []
-
-
-def test_systemd_tests_multiple_pass(mocker: MockerFixture) -> None:
-    """test_systemd_tests_fail."""
-    mocker.patch(
-        f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper",
-        side_effect=[
-            ("inactive\n", ""),
-            ("activating\n", ""),
-            ("active\n", ""),
-        ],
-    )
-    errors = systemd_tests(
-        ("docker",),
-        retryable_statuses=("inactive\n", "activating\n"),
-        valid_statuses=("active\n",),
-    )
-    assert errors == []
-
-
-def test_systemd_tests_fail(mocker: MockerFixture) -> None:
-    """test_systemd_tests_fail."""
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=("inactive\n", ""))
-    errors = systemd_tests(("docker",), max_retries=5)
-    assert errors == ["docker is inactive"]

tests/test_server_validate_scripts.py

@@ -1,63 +0,0 @@
-"""test_server_validate_scripts."""
-
-from __future__ import annotations
-
-from pathlib import Path
-from typing import TYPE_CHECKING
-
-import pytest
-from pytest_mock import MockerFixture
-
-from python.system_tests.validate_system import main
-
-if TYPE_CHECKING:
-    from pyfakefs.fake_filesystem import FakeFilesystem
-    from pytest_mock import MockerFixture
-
-VALIDATE_SYSTEM = "python.system_tests.validate_system"
-
-
-def test_validate_system(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    """test_validate_system."""
-    fs.create_file(
-        "/mock_snapshot_config.toml",
-        contents='zpools = ["root_pool", "storage", "media"]\nservices = ["docker"]\n',
-    )
-
-    mocker.patch(f"{VALIDATE_SYSTEM}.systemd_tests", return_value=None)
-    mocker.patch(f"{VALIDATE_SYSTEM}.zpool_tests", return_value=None)
-    main(Path("/mock_snapshot_config.toml"))
-
-
-def test_validate_system_errors(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    """test_validate_system_errors."""
-    fs.create_file(
-        "/mock_snapshot_config.toml",
-        contents='zpools = ["root_pool", "storage", "media"]\nservices = ["docker"]\n',
-    )
-
-    mocker.patch(f"{VALIDATE_SYSTEM}.signal_alert")
-    mocker.patch(f"{VALIDATE_SYSTEM}.systemd_tests", return_value=["systemd_tests error"])
-    mocker.patch(f"{VALIDATE_SYSTEM}.zpool_tests", return_value=["zpool_tests error"])
-
-    with pytest.raises(SystemExit) as exception_info:
-        main(Path("/mock_snapshot_config.toml"))
-
-    assert exception_info.value.code == 1
-
-
-def test_validate_system_execution(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    """test_validate_system_execution."""
-    fs.create_file(
-        "/mock_snapshot_config.toml",
-        contents='zpools = ["root_pool", "storage", "media"]\nservices = ["docker"]\n',
-    )
-
-    mocker.patch(f"{VALIDATE_SYSTEM}.signal_alert")
-    mocker.patch(f"{VALIDATE_SYSTEM}.systemd_tests", return_value=None)
-    mocker.patch(f"{VALIDATE_SYSTEM}.zpool_tests", side_effect=RuntimeError("zpool_tests error"))
-
-    with pytest.raises(SystemExit) as exception_info:
-        main(Path("/mock_snapshot_config.toml"))
-
-    assert exception_info.value.code == 1

tests/test_signal_alert.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 from os import environ
 from typing import TYPE_CHECKING
 
-from apprise import Apprise
+import httpx
 
 from python.signal_alert import signal_alert
 
@@ -19,14 +19,23 @@ def test_signal_alert(mocker: MockerFixture) -> None:
     environ["SIGNAL_ALERT_TO_PHONE"] = "0987654321"
 
     mock_logger = mocker.patch("python.signal_alert.logger")
-    mock_apprise_client = mocker.MagicMock(spec=Apprise)
+    mock_response = mocker.MagicMock(spec=httpx.Response, status_code=httpx.codes.CREATED)
-    mocker.patch("python.signal_alert.Apprise", return_value=mock_apprise_client)
+    mock_post = mocker.patch("python.signal_alert.httpx.post", return_value=mock_response)
 
     signal_alert("test")
 
     mock_logger.info.assert_not_called()
-    mock_apprise_client.add.assert_called_once_with("signal://localhost:8989/1234567890/0987654321")
+    mock_post.assert_called_once_with(
-    mock_apprise_client.notify.assert_called_once_with(title="", body="test")
+        "http://localhost:8989/v2/send",
+        json={
+            "message": "test",
+            "number": "1234567890",
+            "recipients": ["0987654321"],
+            "text_mode": "normal",
+        },
+        timeout=4.0,
+        follow_redirects=True,
+    )
 
 
 def test_signal_alert_no_phones(mocker: MockerFixture) -> None:
@@ -39,3 +48,34 @@ def test_signal_alert_no_phones(mocker: MockerFixture) -> None:
     signal_alert("test")
 
     mock_logger.info.assert_called_once_with("SIGNAL_ALERT_FROM_PHONE or SIGNAL_ALERT_TO_PHONE not set")
+
+
+def test_signal_alert_http_error(mocker: MockerFixture) -> None:
+    """HTTP errors are logged rather than propagated."""
+    environ["SIGNAL_ALERT_FROM_PHONE"] = "1234567890"
+    environ["SIGNAL_ALERT_TO_PHONE"] = "0987654321"
+
+    mock_logger = mocker.patch("python.signal_alert.logger")
+    request = httpx.Request("POST", "http://localhost:8989/v2/send")
+    mocker.patch(
+        "python.signal_alert.httpx.post",
+        side_effect=httpx.ConnectError("connection failed", request=request),
+    )
+
+    signal_alert("test")
+
+    mock_logger.exception.assert_called_once_with("Unable to contact the Signal API")
+
+
+def test_signal_alert_unsuccessful_response(mocker: MockerFixture) -> None:
+    """Unexpected response statuses are logged."""
+    environ["SIGNAL_ALERT_FROM_PHONE"] = "1234567890"
+    environ["SIGNAL_ALERT_TO_PHONE"] = "0987654321"
+
+    mock_logger = mocker.patch("python.signal_alert.logger")
+    mock_response = mocker.MagicMock(spec=httpx.Response, status_code=httpx.codes.BAD_GATEWAY)
+    mocker.patch("python.signal_alert.httpx.post", return_value=mock_response)
+
+    signal_alert("test")
+
+    mock_logger.error.assert_called_once_with("Signal API returned HTTP status %d", httpx.codes.BAD_GATEWAY)

users/math/home/programs.nix

@@ -3,54 +3,17 @@
   home.packages = with pkgs; [
     # cli
     bat
-    btop
-    eza
-    fd
-    ffmpegthumbnailer
     fzf
     git
-    gnupg
-    imagemagick
     jq
     ncdu
-    ouch
+    fastfetch
     p7zip
-    poppler
     rar
-    ripgrep
-    starship
-    tmux
     unzip
-    yazi
     zoxide
-    # system info
-    hwloc
-    lynis
-    pciutils
-    smartmontools
-    usbutils
-    # networking
-    iperf3
-    nmap
-    wget
     # python
     ruff
     uv
-    # nodejs
-    nodejs
-    # Rust packages
-    trunk
-    wasm-pack
-    cargo-watch
-    cargo-generate
-    cargo-audit
-    cargo-update
-    # nix
-    nix-init
-    nix-output-monitor
-    nix-prefetch
-    nix-tree
-    nixfmt
-    treefmt
   ];
 }

users/richie/home/gui/default.nix

@@ -1,12 +1,11 @@
-{ inputs, pkgs, ... }:
+{ pkgs, ... }:
 {
   imports = [
-    "${inputs.self}/users/shared/comms.nix"
+    ./comms.nix
-    "${inputs.self}/users/shared/games.nix"
+    ./games.nix
-    "${inputs.self}/users/shared/sweet.nix"
-    ./firefox
     ./kitty.nix
     ./llm_tools.nix
+    ./sweet.nix
     ./t3_code
     ./vscode
   ];
@@ -22,7 +21,6 @@
     vlc
     # browser
     brave
-    chromium
     # dev tools
     gparted
     jetbrains.datagrip

users/richie/home/gui/firefox/default.nix

@@ -1,257 +0,0 @@
-{ config, inputs, ... }:
-{
-  imports = [ ./search_engines.nix ];
-
-  programs.firefox = {
-    configPath = "${config.xdg.configHome}/mozilla/firefox";
-    enable = true;
-    profiles.richie = {
-      extensions.packages = with inputs.firefox-addons.packages.x86_64-linux; [
-        bitwarden
-        darkreader
-        dearrow
-        fastforwardteam
-        return-youtube-dislikes
-        sponsorblock
-        ublock-origin
-      ];
-      search = {
-        force = true;
-        default = "kagi";
-        order = [
-          "kagi"
-          "ddg"
-          "google"
-        ];
-      };
-      settings = {
-        # SECTION: FASTFOX
-        # GENERAL
-        "content.notify.interval" = 100000;
-
-        # GFX
-        "gfx.canvas.accelerated.cache-items" = 4096;
-        "gfx.canvas.accelerated.cache-size" = 512;
-        "gfx.content.skia-font-cache-size" = 20;
-
-        # DISK CACHE
-        "browser.cache.jsbc_compression_level" = 3;
-
-        # MEDIA CACHE
-        "media.memory_cache_max_size" = 65536;
-        "media.cache_readahead_limit" = 7200;
-        "media.cache_resume_threshold" = 3600;
-
-        # IMAGE CACHE
-        "image.mem.decode_bytes_at_a_time" = 32768;
-
-        # NETWORK
-        "network.buffer.cache.size" = 262144;
-        "network.buffer.cache.count" = 128;
-        "network.http.max-connections" = 1800;
-        "network.http.max-persistent-connections-per-server" = 10;
-        "network.http.max-urgent-start-excessive-connections-per-host" = 5;
-        "network.http.accept-encoding" = "gzip, deflate, br, zstd";
-        "network.http.pacing.requests.enabled" = false;
-        "network.dnsCacheExpiration" = 3600;
-        "network.dns.max_high_priority_threads" = 8;
-        "network.ssl_tokens_cache_capacity" = 10240;
-
-        # SPECULATIVE LOADING
-        "network.dns.disablePrefetch" = true;
-        "network.prefetch-next" = false;
-        "network.predictor.enabled" = false;
-
-        # EXPERIMENTAL
-        "layout.css.grid-template-masonry-value.enabled" = true;
-        "dom.enable_web_task_scheduling" = true;
-        "layout.css.has-selector.enabled" = true;
-        "dom.security.sanitizer.enabled" = true;
-
-        # SECTION: SECUREFOX
-        # TRACKING PROTECTION
-        "browser.contentblocking.category" = "strict";
-        "urlclassifier.trackingSkipURLs" = "*.reddit.com, *.twitter.com, *.twimg.com, *.tiktok.com";
-        "urlclassifier.features.socialtracking.skipURLs" = "*.instagram.com, *.twitter.com, *.twimg.com";
-        "network.cookie.sameSite.noneRequiresSecure" = true;
-        "browser.download.start_downloads_in_tmp_dir" = true;
-        "browser.helperApps.deleteTempFileOnExit" = true;
-        "browser.uitour.enabled" = false;
-        "privacy.globalprivacycontrol.enabled" = true;
-
-        # OCSP & CERTS / HPKP
-        "security.OCSP.enabled" = 0;
-        "security.remote_settings.crlite_filters.enabled" = true;
-        "security.pki.crlite_mode" = 2;
-
-        # SSL / TLS
-        "security.ssl.treat_unsafe_negotiation_as_broken" = true;
-        "browser.xul.error_pages.expert_bad_cert" = true;
-        "security.tls.enable_0rtt_data" = false;
-
-        # DISK AVOIDANCE
-        "browser.privatebrowsing.forceMediaMemoryCache" = true;
-        "browser.sessionstore.interval" = 60000;
-
-        # SHUTDOWN & SANITIZING
-        "privacy.history.custom" = true;
-
-        # SEARCH / URL BAR
-        "browser.search.separatePrivateDefault.ui.enabled" = true;
-        "browser.urlbar.update2.engineAliasRefresh" = true;
-        # PREF: restore search engine suggestions
-        "browser.search.suggest.enabled" = true;
-        "browser.urlbar.suggest.quicksuggest.sponsored" = false;
-        "browser.urlbar.suggest.quicksuggest.nonsponsored" = false;
-        "browser.formfill.enable" = false;
-        "security.insecure_connection_text.enabled" = true;
-        "security.insecure_connection_text.pbmode.enabled" = true;
-        "network.IDN_show_punycode" = true;
-
-        # HTTPS-FIRST POLICY
-        "dom.security.https_first" = true;
-        "dom.security.https_first_schemeless" = true;
-
-        # PASSWORDS
-        "signon.formlessCapture.enabled" = false;
-        "signon.rememberSignons" = false;
-        "signon.privateBrowsingCapture.enabled" = false;
-        "network.auth.subresource-http-auth-allow" = 1;
-        "editor.truncate_user_pastes" = false;
-
-        # MIXED CONTENT + CROSS-SITE
-        "security.mixed_content.block_display_content" = true;
-        "security.mixed_content.upgrade_display_content" = true;
-        "security.mixed_content.upgrade_display_content.image" = true;
-        "pdfjs.enableScripting" = false;
-        "extensions.postDownloadThirdPartyPrompt" = false;
-
-        # HEADERS / REFERERS
-        "network.http.referer.XOriginTrimmingPolicy" = 2;
-
-        # CONTAINERS
-        "privacy.userContext.ui.enabled" = true;
-
-        # WEBRTC
-        "media.peerconnection.ice.proxy_only_if_behind_proxy" = true;
-        "media.peerconnection.ice.default_address_only" = true;
-
-        # SAFE BROWSING
-        "browser.safebrowsing.downloads.remote.enabled" = false;
-
-        # MOZILLA
-        # PREF: allow websites to ask you to receive site notifications
-        "permissions.default.desktop-notification" = 0; # allow websites to ask
-        # PREF: allow websites to ask you for your location
-        "permissions.default.geo" = 0;
-        "geo.provider.network.url" =
-          "https://location.services.mozilla.com/v1/geolocate?key=%MOZILLA_API_KEY%";
-        "permissions.manager.defaultsUrl" = "";
-        "webchannel.allowObject.urlWhitelist" = "";
-
-        # TELEMETRY
-        "datareporting.policy.dataSubmissionEnabled" = false;
-        "datareporting.healthreport.uploadEnabled" = false;
-        "toolkit.telemetry.unified" = false;
-        "toolkit.telemetry.enabled" = false;
-        "toolkit.telemetry.server" = "data:,";
-        "toolkit.telemetry.archive.enabled" = false;
-        "toolkit.telemetry.newProfilePing.enabled" = false;
-        "toolkit.telemetry.shutdownPingSender.enabled" = false;
-        "toolkit.telemetry.updatePing.enabled" = false;
-        "toolkit.telemetry.bhrPing.enabled" = false;
-        "toolkit.telemetry.firstShutdownPing.enabled" = false;
-        "toolkit.telemetry.coverage.opt-out" = true;
-        "toolkit.coverage.opt-out" = true;
-        "toolkit.coverage.endpoint.base" = "";
-        "browser.ping-centre.telemetry" = false;
-        "browser.newtabpage.activity-stream.feeds.telemetry" = false;
-        "browser.newtabpage.activity-stream.telemetry" = false;
-
-        # EXPERIMENTS
-        "app.shield.optoutstudies.enabled" = false;
-        "app.normandy.enabled" = false;
-        "app.normandy.api_url" = "";
-
-        # CRASH REPORTS
-        "breakpad.reportURL" = "";
-        "browser.tabs.crashReporting.sendReport" = false;
-        "browser.crashReports.unsubmittedCheck.autoSubmit2" = false;
-
-        # DETECTION
-        "captivedetect.canonicalURL" = "";
-        "network.captive-portal-service.enabled" = false;
-        "network.connectivity-service.enabled" = false;
-
-        # SECTION: PESKYFOX
-        # MOZILLA UI
-        "browser.privatebrowsing.vpnpromourl" = "";
-        "extensions.getAddons.showPane" = false;
-        "extensions.htmlaboutaddons.recommendations.enabled" = false;
-        "browser.discovery.enabled" = false;
-        "browser.shell.checkDefaultBrowser" = false;
-        "browser.newtabpage.activity-stream.asrouter.userprefs.cfr.addons" = false;
-        "browser.newtabpage.activity-stream.asrouter.userprefs.cfr.features" = false;
-        "browser.preferences.moreFromMozilla" = false;
-        "browser.tabs.tabmanager.enabled" = false;
-        "browser.aboutConfig.showWarning" = false;
-        "browser.aboutwelcome.enabled" = false;
-
-        # THEME ADJUSTMENTS
-        "toolkit.legacyUserProfileCustomizations.stylesheets" = true;
-        "browser.compactmode.show" = true;
-        "browser.display.focus_ring_on_anything" = true;
-        "browser.display.focus_ring_style" = 0;
-        "browser.display.focus_ring_width" = 0;
-        "layout.css.prefers-color-scheme.content-override" = 2;
-
-        # COOKIE BANNER HANDLING
-        "cookiebanners.service.mode" = 1;
-        "cookiebanners.service.mode.privateBrowsing" = 1;
-
-        # FULLSCREEN NOTICE
-        "full-screen-api.transition-duration.enter" = "0 0";
-        "full-screen-api.transition-duration.leave" = "0 0";
-        "full-screen-api.warning.delay" = -1;
-        "full-screen-api.warning.timeout" = 0;
-
-        # URL BAR
-        "browser.urlbar.suggest.calculator" = true;
-        "browser.urlbar.unitConversion.enabled" = true;
-        "browser.urlbar.trending.featureGate" = false;
-
-        # NEW TAB PAGE
-        "browser.newtabpage.activity-stream.feeds.topsites" = false;
-        "browser.newtabpage.activity-stream.feeds.section.topstories" = false;
-
-        # POCKET
-        "extensions.pocket.enabled" = false;
-
-        # DOWNLOADS
-        "browser.download.always_ask_before_handling_new_types" = true;
-        "browser.download.manager.addToRecentDocs" = false;
-
-        # PDF
-        "browser.download.open_pdf_attachments_inline" = true;
-
-        # TAB BEHAVIOR
-        "browser.bookmarks.openInTabClosesMenu" = false;
-        "browser.menu.showViewImageInfo" = true;
-        "findbar.highlightAll" = true;
-        "layout.word_select.eat_space_to_next_word" = false;
-
-        # SECTION: MY OVERRIDES
-        "browser.startup.homepage" = "https://google.com";
-        "identity.fxaccounts.enabled" = false;
-
-        # SECTION SMOOTHFOX
-        # OPTION: SHARPEN SCROLLING                                                           *
-        "apz.overscroll.enabled" = true; # DEFAULT NON-LINUX
-        "mousewheel.min_line_scroll_amount" = 10; # 10-40; adjust this number to your liking; default=5
-        "general.smoothScroll.mouseWheel.durationMinMS" = 80; # default=50
-        "general.smoothScroll.currentVelocityWeighting" = "0.15"; # default=.25
-        "general.smoothScroll.stopDecelerationWeighting" = "0.6"; # default=.4
-      };
-    };
-  };
-}

users/richie/home/gui/firefox/github.svg

@@ -1,3 +0,0 @@
-<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path fill-rule="evenodd" clip-rule="evenodd" d="M16 0C7.16 0 0 7.16 0 16C0 23.08 4.58 29.06 10.94 31.18C11.74 31.32 12.04 30.84 12.04 30.42C12.04 30.04 12.02 28.78 12.02 27.44C8 28.18 6.96 26.46 6.64 25.56C6.46 25.1 5.68 23.68 5 23.3C4.44 23 3.64 22.26 4.98 22.24C6.24 22.22 7.14 23.4 7.44 23.88C8.88 26.3 11.18 25.62 12.1 25.2C12.24 24.16 12.66 23.46 13.12 23.06C9.56 22.66 5.84 21.28 5.84 15.16C5.84 13.42 6.46 11.98 7.48 10.86C7.32 10.46 6.76 8.82 7.64 6.62C7.64 6.62 8.98 6.2 12.04 8.26C13.32 7.9 14.68 7.72 16.04 7.72C17.4 7.72 18.76 7.9 20.04 8.26C23.1 6.18 24.44 6.62 24.44 6.62C25.32 8.82 24.76 10.46 24.6 10.86C25.62 11.98 26.24 13.4 26.24 15.16C26.24 21.3 22.5 22.66 18.94 23.06C19.52 23.56 20.02 24.52 20.02 26.02C20.02 28.16 20 29.88 20 30.42C20 30.84 20.3 31.34 21.1 31.18C27.42 29.06 32 23.06 32 16C32 7.16 24.84 0 16 0V0Z" fill="white"/>
-</svg>

users/richie/home/gui/firefox/search_engines.nix

@@ -1,99 +0,0 @@
-{ pkgs, ... }:
-{
-  programs.firefox.profiles.richie.search.engines = {
-    "Nix Options" = {
-      urls = [
-        {
-          template = "https://search.nixos.org/options";
-          params = [
-            {
-              name = "type";
-              value = "packages";
-            }
-            {
-              name = "channel";
-              value = "unstable";
-            }
-            {
-              name = "query";
-              value = "{searchTerms}";
-            }
-          ];
-        }
-      ];
-      icon = "${pkgs.nixos-icons}/share/icons/hicolor/scalable/apps/nix-snowflake.svg";
-      definedAliases = [ "@o" ];
-    };
-    "Nix Packages" = {
-      urls = [
-        {
-          template = "https://search.nixos.org/packages";
-          params = [
-            {
-              name = "type";
-              value = "packages";
-            }
-            {
-              name = "channel";
-              value = "unstable";
-            }
-            {
-              name = "query";
-              value = "{searchTerms}";
-            }
-          ];
-        }
-      ];
-      icon = "${pkgs.nixos-icons}/share/icons/hicolor/scalable/apps/nix-snowflake.svg";
-      definedAliases = [ "@n" ];
-    };
-    "Nix Packages pr-tracker" = {
-      urls = [
-        {
-          template = "https://nixpk.gs/pr-tracker.html?";
-          params = [
-            {
-              name = "pr";
-              value = "{searchTerms}";
-            }
-          ];
-        }
-      ];
-      icon = "${pkgs.nixos-icons}/share/icons/hicolor/scalable/apps/nix-snowflake.svg";
-      definedAliases = [ "@nprt" ];
-    };
-    "kagi" = {
-      urls = [
-        {
-          template = "https://kagi.com/search?";
-          params = [
-            {
-              name = "q";
-              value = "{searchTerms}";
-            }
-          ];
-        }
-      ];
-      icon = ./kagi.png;
-    };
-    github = {
-      urls = [
-        {
-          template = "https://github.com/search?";
-          params = [
-            {
-              name = "q";
-              value = "{searchTerms}";
-            }
-            {
-              name = "type";
-              value = "code";
-            }
-          ];
-        }
-      ];
-      icon = ./github.svg;
-      definedAliases = [ "@g" ];
-    };
-  };
-}

users/richie/home/programs.nix

@@ -3,21 +3,14 @@
   home.packages = with pkgs; [
     # cli
     bat
-    fd
-    ffmpegthumbnailer
     fzf
     git
-    gnupg
-    imagemagick
     jq
     ncdu
     fastfetch
-    ouch
     p7zip
-    poppler
     rar
     unzip
-    yazi
     zoxide
     # Home Assistant
     esphome
@@ -35,8 +28,6 @@
     # python
     ruff
     uv
-    # nodejs
-    nodejs
     # Rust packages
     bacon
     cargo
@@ -51,9 +42,6 @@
     rustfmt
     trunk
     wasm-pack
-    # cpp
-    clang-tools
-    clang_20
     # nix
     nix-init
     nix-output-monitor
@@ -61,6 +49,5 @@
     nix-tree
     nixfmt
     treefmt
-    codebase-memory-mcp
   ];
 }

users/steve/default.nix

@@ -1,44 +0,0 @@
-{
-  pkgs,
-  config,
-  ...
-}:
-let
-  ifTheyExist = groups: builtins.filter (group: builtins.hasAttr group config.users.groups) groups;
-in
-{
-
-  users = {
-    users.steve = {
-      isNormalUser = true;
-
-      shell = pkgs.zsh;
-      group = "steve";
-      openssh.authorizedKeys.keys = [
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJH03VzDbUhzfhvwD+OsYh6GobODYaI9jdNdzWQoqFsp matth@Jove" # cspell:disable-line
-      ];
-      extraGroups = [
-        "audio"
-        "video"
-        "wheel"
-        "users"
-      ]
-      ++ ifTheyExist [
-        "dialout"
-        "docker"
-        "hass"
-        "libvirtd"
-        "networkmanager"
-        "plugdev"
-        "scanner"
-        "transmission"
-        "uaccess"
-        "wireshark"
-      ];
-      uid = 1005;
-    };
-
-    groups.steve.gid = 1005;
-  };
-  home-manager.users.steve = import ./systems/${config.networking.hostName}.nix;
-}

users/steve/home/cli/default.nix

@@ -1,9 +0,0 @@
-{
-  imports = [
-    ./direnv.nix
-    ./git.nix
-    ./zsh.nix
-  ];
-
-  programs.starship.enable = true;
-}

users/steve/home/cli/direnv.nix

@@ -1,8 +0,0 @@
-{
-
-  programs.direnv = {
-    enable = true;
-    enableZshIntegration = true;
-    nix-direnv.enable = true;
-  };
-}

users/steve/home/cli/git.nix

@@ -1,15 +0,0 @@
-{
-  programs.git = {
-    enable = true;
-    signing.format = null;
-    settings = {
-      user = {
-        email = "matthew.michal11@gmail.com";
-        name = "Matthew Michal";
-      };
-      pull.rebase = true;
-      color.ui = true;
-    };
-    lfs.enable = true;
-  };
-}

users/steve/home/cli/zsh.nix

@@ -1,28 +0,0 @@
-{
-  programs.zsh = {
-    enable = true;
-    syntaxHighlighting.enable = true;
-    history.size = 10000;
-    oh-my-zsh = {
-      enable = true;
-      plugins = [
-        "git"
-        "docker"
-        "docker-compose"
-        "colored-man-pages"
-        "rust"
-        "systemd"
-        "tmux"
-        "ufw"
-        "z"
-      ];
-    };
-    shellAliases = {
-      "lrt" = "eza --icons -lsnew";
-      "ls" = "eza";
-      "ll" = "eza --long --group";
-      "la" = "eza --all";
-
-    };
-  };
-}

users/steve/home/global.nix

@@ -1,22 +0,0 @@
-{ config, ... }:
-{
-  imports = [
-    ./cli
-    ./programs.nix
-    ./ssh_config.nix
-  ];
-
-  programs = {
-    home-manager.enable = true;
-    git.enable = true;
-  };
-
-  home = {
-    username = "steve";
-    homeDirectory = "/home/${config.home.username}";
-    stateVersion = "24.05";
-    sessionVariables = {
-      FLAKE = "$HOME/dotfiles";
-    };
-  };
-}

users/steve/home/programs.nix

@@ -1,56 +0,0 @@
-{ pkgs, ... }:
-{
-  home.packages = with pkgs; [
-    # cli
-    bat
-    btop
-    eza
-    fd
-    ffmpegthumbnailer
-    fzf
-    git
-    gnupg
-    imagemagick
-    jq
-    ncdu
-    ouch
-    p7zip
-    poppler
-    rar
-    ripgrep
-    starship
-    tmux
-    unzip
-    yazi
-    zoxide
-    # system info
-    hwloc
-    lynis
-    pciutils
-    smartmontools
-    usbutils
-    # networking
-    iperf3
-    nmap
-    wget
-    # python
-    ruff
-    uv
-    # nodejs
-    nodejs
-    # Rust packages
-    trunk
-    wasm-pack
-    cargo-watch
-    cargo-generate
-    cargo-audit
-    cargo-update
-    # nix
-    nix-init
-    nix-output-monitor
-    nix-prefetch
-    nix-tree
-    nixfmt
-    treefmt
-  ];
-}

users/steve/home/ssh_config.nix

@@ -1,6 +0,0 @@
-{
-  programs.ssh = {
-    enable = true;
-    enableDefaultConfig = false;
-  };
-}

users/steve/systems/bob.nix

@@ -1,5 +0,0 @@
-{
-  imports = [
-    ../home/global.nix
-  ];
-}

users/steve/systems/jeeves.nix

@@ -1,5 +0,0 @@
-{
-  imports = [
-    ../home/global.nix
-  ];
-}