more patches

it was only required for rhapsody-in-green

.github/workflows/build_systems.yml

@@ -8,8 +8,23 @@ on:
     - cron: "0 22 * * *"
 
 jobs:
+  prebuild-common:
+    name: prebuild-common-x86-64-v3
+    runs-on: nix-cache-builder
+    steps:
+      - uses: actions/checkout@v4
+      # portal-1 is the smallest system closure: 95% of its derivations are
+      # shared by all five systems, so it is a maintainable common cache seed.
+      # Keep going so one failing package does not stop unrelated cache entries
+      # from being built.
+      - name: Build common packages
+        run: nixos-rebuild build --keep-going --accept-flake-config --flake ./#portal-1
+      - name: Copy common packages to nix-cache
+        run: nix copy --accept-flake-config --to unix:///host-nix/var/nix/daemon-socket/socket .#nixosConfigurations.portal-1.config.system.build.toplevel
+
   build:
     name: build-${{ matrix.system }}
+    needs: prebuild-common
     runs-on: self-hosted
     strategy:
       matrix:

common/optional/signal-cli.nix

@@ -0,0 +1,46 @@
+{ pkgs, ... }:
+{
+  environment.systemPackages = [
+    pkgs.signal-cli
+  ];
+
+  users = {
+    groups.signal-cli = { };
+    users.signal-cli = {
+      isSystemUser = true;
+      group = "signal-cli";
+      home = "/var/lib/signal-cli";
+    };
+  };
+
+  systemd.tmpfiles.rules = [
+    "d /var/lib/signal-cli 0700 signal-cli signal-cli - -"
+    "Z /var/lib/signal-cli - signal-cli signal-cli - -"
+  ];
+
+  systemd.services.signal-cli = {
+    description = "Signal CLI JSON-RPC service";
+    after = [ "network-online.target" ];
+    wants = [ "network-online.target" ];
+    wantedBy = [ "multi-user.target" ];
+    unitConfig.RequiresMountsFor = [ "/var/lib/signal-cli" ];
+
+    serviceConfig = {
+      Type = "simple";
+      User = "signal-cli";
+      Group = "signal-cli";
+      ExecStart = "${pkgs.signal-cli}/bin/signal-cli --data-dir /var/lib/signal-cli daemon --socket /run/signal-cli/socket";
+      Restart = "on-failure";
+      RestartSec = "5s";
+      SuccessExitStatus = 143;
+      RuntimeDirectory = "signal-cli";
+      RuntimeDirectoryMode = "0750";
+      UMask = "0007";
+      NoNewPrivileges = true;
+      PrivateTmp = true;
+      ProtectHome = true;
+      ProtectSystem = "strict";
+      ReadWritePaths = [ "/var/lib/signal-cli" ];
+    };
+  };
+}

common/optional/x86-64-v3.nix

@@ -1,9 +0,0 @@
-{
-  nixpkgs.hostPlatform = {
-    system = "x86_64-linux";
-    gcc = {
-      arch = "x86-64-v3";
-      tune = "generic";
-    };
-  };
-}

common/optional/x86-64-v3/default.nix

@@ -0,0 +1,15 @@
+{
+  nixpkgs = {
+    hostPlatform = {
+      system = "x86_64-linux";
+      gcc = {
+        arch = "x86-64-v3";
+        tune = "generic";
+      };
+    };
+
+    # These patches repair tests and bundled dependencies that are sensitive
+    # to the compiler flags used by the x86-64-v3 package set.
+    overlays = [ (import ./patches) ];
+  };
+}

common/optional/x86-64-v3/patches/abseil/bmi2-public-header.patch

@@ -0,0 +1,20 @@
+Subject: [PATCH] abseil: include BMI2 intrinsics through the public header
+
+GCC and Clang reject direct inclusion of bmi2intrin.h. Include immintrin.h
+instead so that the compiler supplies the required intrinsic setup when
+BMI2 is enabled, including builds targeting x86-64-v3.
+
+This patch is shared by the vendored Abseil copies in Electron, rusty_v8
+(Deno), and Signal's WebRTC build.
+
+--- a/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
++++ b/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
+@@ -226,7 +226,7 @@
+ #endif
+ 
+ #ifdef __BMI2__
+-#include <bmi2intrin.h>
++#include <immintrin.h>
+ #endif  // __BMI2__
+ 
+ namespace absl {

common/optional/x86-64-v3/patches/abseil/default.nix

@@ -0,0 +1,38 @@
+# Abseil accepted the upstream fix: https://github.com/abseil/abseil-cpp/pull/2071
+# Keep this workaround until Electron, Deno's rusty_v8, and Signal's WebRTC
+# update their bundled Abseil copies to include it.
+{ prev }:
+let
+  patchAbseilBmi2Include =
+    package:
+    package.overrideAttrs (old: {
+      # GCC and Clang require the public umbrella header for BMI2 intrinsics.
+      patches = (old.patches or [ ]) ++ [ ./bmi2-public-header.patch ];
+    });
+
+  electron43Unwrapped = patchAbseilBmi2Include prev.electron_43.unwrapped;
+  electron43 = prev.electron_43.override {
+    electron-unwrapped = electron43Unwrapped;
+  };
+
+  signalCallPackage =
+    path: args:
+    let
+      package = prev.callPackage path args;
+    in
+    if builtins.baseNameOf path == "webrtc.nix" then patchAbseilBmi2Include package else package;
+in
+prev.lib.optionalAttrs ((prev.stdenv.hostPlatform.gcc.arch or null) == "x86-64-v3") {
+  deno =
+    let
+      librusty_v8 = patchAbseilBmi2Include prev.deno.passthru.librusty_v8;
+    in
+    prev.deno.override { inherit librusty_v8; };
+
+  electron_43 = electron43;
+
+  signal-desktop = prev.signal-desktop.override {
+    electron_43 = electron43;
+    callPackage = signalCallPackage;
+  };
+}

common/optional/x86-64-v3/patches/afdko/check-overlap-font-content.patch

@@ -0,0 +1,19 @@
+Subject: [PATCH] Compare font content in the overlap-removal regression
+
+PFA_SKIP ends with a separator, producing an empty prefix that matches every
+line. Remove the empty prefix for this test so glyph differences are checked,
+and assert that tx exits successfully before inspecting the output.
+
+--- a/tests/tx_test.py
++++ b/tests/tx_test.py
+@@ -1064,6 +1064,8 @@
+     output_path = get_temp_file_path()
+     args = [TOOL, '-t1', '+V', '-o', output_path, input_path]
+-    subprocess.call(args)
+-    assert differ([expected_path, output_path, '-s', PFA_SKIP[0]])
++    subprocess.check_call(args)
++    # An empty skip prefix matches every line and hides font differences.
++    skip_headers = SPLIT_MARKER.join(filter(None, PFA_SKIP[0].split(SPLIT_MARKER)))
++    assert differ([expected_path, output_path, '-s', skip_headers])
+ 
+ 

common/optional/x86-64-v3/patches/afdko/default.nix

@@ -0,0 +1,10 @@
+{ afdko }:
+afdko.overridePythonAttrs (old: {
+  # FMA changes overlap-removal coordinates by 0.01 units on x86-64-v3.
+  # Separate multiply/add rounding reproduces the reference Type 1 font exactly.
+  env = (old.env or { }) // {
+    NIX_CFLAGS_COMPILE = (old.env.NIX_CFLAGS_COMPILE or "") + " -ffp-contract=off";
+  };
+
+  patches = (old.patches or [ ]) ++ [ ./check-overlap-font-content.patch ];
+})

common/optional/x86-64-v3/patches/cuda/default.nix

@@ -0,0 +1,19 @@
+{ prev }:
+let
+  buildRedistHook = prev.runCommand "buildRedistHook.bash" { } ''
+    cp ${prev.path}/pkgs/development/cuda-modules/buildRedist/buildRedistHook.bash "$out"
+    chmod u+w "$out"
+    patch "$out" < ${./propagated-build-output-arrays.patch}
+  '';
+in
+prev.cudaPackages_12_9.overrideScope (
+  _cudaFinal: cudaPrev: {
+    buildRedist =
+      args:
+      (cudaPrev.buildRedist args).overrideAttrs (old: {
+        nativeBuildInputs = map (
+          input: if builtins.baseNameOf input == "buildRedistHook.bash" then buildRedistHook else input
+        ) old.nativeBuildInputs;
+      });
+  }
+)

common/optional/x86-64-v3/patches/cuda/propagated-build-output-arrays.patch

@@ -0,0 +1,44 @@
+Subject: [PATCH] Preserve structured CUDA output arrays
+
+The multiple-outputs hook now accepts arrays. The old compatibility export
+collapses include and lib into one array element, breaking indirect expansion.
+Keep the array and iterate over each output when propagating to out.
+
+--- a/buildRedistHook.bash
++++ b/buildRedistHook.bash
+@@ -23,9 +23,6 @@
+ 
+   postInstallCheckHooks+=(checkCudaNonEmptyOutputs)
+   nixLog "added checkCudaNonEmptyOutputs to postInstallCheckHooks"
+-
+-  preFixupHooks+=(fixupPropagatedBuildOutputsForMultipleOutputs)
+-  nixLog "added fixupPropagatedBuildOutputsForMultipleOutputs to preFixupHooks"
+ 
+   postFixupHooks+=(fixupCudaPropagatedBuildOutputsToOut)
+   nixLog "added fixupCudaPropagatedBuildOutputsToOut to postFixupHooks"
+@@ -193,16 +190,6 @@
+   return 0
+ }
+ 
+-# TODO(@connorbaker): https://github.com/NixOS/nixpkgs/issues/323126.
+-# _multioutPropagateDev() currently expects a space-separated string rather than an array.
+-# NOTE: Because _multioutPropagateDev is a postFixup hook, we correct it in preFixup.
+-fixupPropagatedBuildOutputsForMultipleOutputs() {
+-  nixLog "converting propagatedBuildOutputs to a space-separated string"
+-  # shellcheck disable=SC2124
+-  export propagatedBuildOutputs="${propagatedBuildOutputs[@]}"
+-  return 0
+-}
+-
+ # The multiple outputs setup hook only propagates build outputs to dev.
+ # We want to propagate them to out as well, in case the user interpolates
+ # the package into a string -- in such a case, the dev output is not selected
+@@ -217,7 +204,7 @@
+   mkdir -p "${out:?}/nix-support"
+ 
+   # NOTE: We must use printWords to ensure the output is a single line.
+-  for output in $propagatedBuildOutputs; do
++  for output in "${propagatedBuildOutputs[@]}"; do
+     # Propagate the other components to the out output
+     nixLog "adding ${!output:?} to propagatedBuildInputs of ${out:?}"
+     printWords "${!output:?}" >>"${out:?}/nix-support/propagated-build-inputs"

common/optional/x86-64-v3/patches/default.nix

@@ -0,0 +1,18 @@
+_final: prev:
+(import ./abseil { inherit prev; })
+// {
+  cudaPackages_12_9 = import ./cuda { inherit prev; };
+  trunk = import ./trunk { inherit (prev) trunk jq; };
+  gnutls = import ./gnutls { inherit (prev) gnutls; };
+  prometheus = import ./prometheus { inherit (prev) prometheus; };
+  zopfli = import ./zopfli { inherit (prev) zopfli; };
+
+  pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
+    (_pythonFinal: pythonPrev: {
+      afdko = import ./afdko { inherit (pythonPrev) afdko; };
+      psutil = import ./psutil { inherit (pythonPrev) psutil; };
+      pytest-xdist = import ./pytest-xdist { inherit (pythonPrev) pytest-xdist; };
+      scipy = import ./scipy { inherit (pythonPrev) scipy; };
+    })
+  ];
+}

common/optional/x86-64-v3/patches/gnutls/default.nix

@@ -0,0 +1,6 @@
+{ gnutls }:
+gnutls.overrideAttrs (old: {
+  # Keep the UDP handshake test enabled on loaded builders by waiting for
+  # the server to bind its socket. Kept as a standalone patch for upstream.
+  patches = (old.patches or [ ]) ++ [ ./udp-server-readiness.patch ];
+})

common/optional/x86-64-v3/patches/gnutls/udp-server-readiness.patch

@@ -0,0 +1,70 @@
+Subject: [PATCH] tests: wait for the UDP server socket before connecting
+
+A fixed four-second sleep does not guarantee that gnutls-serv has bound
+its UDP socket on a busy builder. Poll the local IPv4 UDP endpoint using
+the existing ss/netstat discovery, with the same retry budget as the TCP
+helper. Fail early if the server exits, and retain the original handshake
+checks in serv-udp.sh.
+
+Use flags common to ss and BSD/Linux netstat. Match the local endpoint
+and complete port number, excluding TCP, IPv6-only and peer endpoints.
+
+--- a/tests/scripts/common.sh
++++ b/tests/scripts/common.sh
+@@ -185,10 +185,55 @@
+ 	fi
+ }
+ 
++check_if_udp_port_bound() {
++	local PORT=$1
++	have_port_finder
++	# Use only -an, which is shared by ss and BSD/Linux netstat. UDP has
++	# no LISTEN state. Match the local IPv4 endpoint, not a peer port or
++	# a longer port number. serv-udp.sh connects to 127.0.0.1;
++	# listen_socket() in serv.c binds IPv4 separately and requests
++	# IPV6_V6ONLY=1 for its IPv6 socket.
++	$PFCMD -an | awk -v port="$PORT" '
++		$1 == "udp" || $1 == "udp4" {
++			# ss includes a state column; netstat does not.
++			address = ($2 == "UNCONN" || $2 == "ESTAB") ? $5 : $4
++			if (address ~ ("^[0-9.]+[.:]" port "$") ||
++			    address == "*." port)
++				found = 1
++		}
++		END { exit !found }
++	'
++}
++
+ wait_udp_server() {
+ 	local PID=$1
++	local ret
+ 	trap "test -n \"${PID}\" && kill ${PID};exit 1" 1 15 2
+-	sleep 4
++	local i=0
++	# Use the same retry budget as wait_for_port(), but also stop if the
++	# server exits before binding its socket.
++	while test $i -lt 90; do
++		if ! kill -0 "$PID" 2>/dev/null; then
++			fail "" "UDP server $PID exited before binding port $PORT"
++		fi
++		# Contain have_port_finder's exit so a skip also stops the server.
++		if (check_if_udp_port_bound "$PORT"); then
++			return 0
++		else
++			ret=$?
++			if test "$ret" = 77; then
++				kill "$PID" 2>/dev/null || :
++				wait "$PID" 2>/dev/null || :
++				exit 77
++			fi
++		fi
++		i=$((i + 1))
++		if test $i -lt 90; then
++			echo "try $i: waiting for UDP port $PORT"
++			sleep 2
++		fi
++	done
++	fail "$PID" "UDP server $PORT did not come up"
+ }
+ 
+ create_testdir() {

common/optional/x86-64-v3/patches/gnutls/verify-readiness.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""Exercise patched common.sh without building GnuTLS (Python standard library only).
+
+Usage: python3 verify-readiness.py /path/to/patched/tests/scripts/common.sh
+Set SHELL to test another shell, and NETSTAT to test a netstat outside PATH.
+"""
+
+# Use unittest so this upstream companion tool needs no pytest installation.
+# ruff: noqa: PT009
+
+import os
+import shutil
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+from pathlib import Path
+
+COMMON = str(Path(sys.argv.pop(1)).resolve())
+SHELL = os.environ.get("SHELL", "/bin/sh")
+
+
+class ReadinessTests(unittest.TestCase):
+    """Check endpoint parsing and the server startup lifecycle."""
+
+    def setUp(self) -> None:
+        """Create a socket-listing fixture for each check."""
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.root = Path(self.tmp.name)
+        self.fixture = self.root / "sockets"
+        self.fixture.write_text("")
+        self.finder = self.root / "port-finder"
+        self.finder.write_text('#!/bin/sh\ncat "$SOCKET_FIXTURE"\n')
+        self.finder.chmod(0o755)
+
+    def run_shell(self, body: str, **env: str) -> subprocess.CompletedProcess[str]:
+        """Source the actual helper and run a shell scenario."""
+        return subprocess.run(
+            [SHELL, "-c", '. "$COMMON"\n' + body],
+            env={
+                **os.environ,
+                "COMMON": COMMON,
+                "SOCKET_FIXTURE": str(self.fixture),
+                "PFCMD": str(self.finder),
+                "PORT": "12345",
+                **env,
+            },
+            capture_output=True,
+            text=True,
+            timeout=20,
+            check=False,
+        )
+
+    def test_socket_formats_and_false_matches(self) -> None:
+        """Accept IPv4 UDP local endpoints and reject unrelated sockets."""
+        cases = [
+            ("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*", True),
+            ("udp UNCONN 0 0 127.0.0.1:12345 0.0.0.0:*", True),
+            ("udp 0 0 0.0.0.0:12345 0.0.0.0:*", True),
+            ("udp4 0 0 *.12345 *.*", True),
+            ("udp 0 0 127.0.0.1.12345 *.*", True),
+            ("udp 0 0 *.12345 *.*", True),
+            ("udp UNCONN 0 0 0.0.0.0:123456 0.0.0.0:*", False),
+            ("udp 0 0 0.0.0.0:123456 0.0.0.0:*", False),
+            ("udp ESTAB 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
+            ("udp 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
+            ("tcp LISTEN 0 128 0.0.0.0:12345 0.0.0.0:*", False),
+            ("tcp 0 0 0.0.0.0:12345 0.0.0.0:* LISTEN", False),
+            ("udp UNCONN 0 0 [::]:12345 [::]:*", False),
+            ("udp UNCONN 0 0 *:12345 *:*", False),
+            ("udp6 0 0 :::12345 :::*", False),
+            ("udp6 0 0 *.12345 *.*", False),
+            ("", False),
+        ]
+        for row, ready in cases:
+            with self.subTest(row=row):
+                self.fixture.write_text(row + "\n")
+                result = self.run_shell('check_if_udp_port_bound "$PORT"')
+                self.assertEqual(result.returncode, 0 if ready else 1, result.stderr)
+
+    def test_exited_server_fails_immediately(self) -> None:
+        """Fail without sleeping when the server has already exited."""
+        result = self.run_shell(
+            'true &\npid=$!\nwait "$pid"\nsleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$pid"'
+        )
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("exited before binding", result.stderr)
+        self.assertNotIn("unexpected sleep", result.stderr)
+
+    def test_ready_socket_does_not_sleep(self) -> None:
+        """Check readiness before the first sleep."""
+        self.fixture.write_text("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*\n")
+        result = self.run_shell('sleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$$"')
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertNotIn("unexpected sleep", result.stderr)
+
+    def test_missing_port_finders_skip(self) -> None:
+        """Skip and stop the live server when no finder is available."""
+        # have_port_finder also tries these paths independently of PATH.
+        if any(os.access(f"{directory}/ss", os.X_OK) for directory in ("/sbin", "/usr/sbin", "/usr/local/sbin")):
+            self.skipTest("an absolute ss path cannot be hidden by this PATH-only fixture")
+        with subprocess.Popen(["sleep", "60"]) as server:
+            try:
+                result = self.run_shell(
+                    'unset PFCMD\nPATH=/nonexistent\nwait_udp_server "$SERVER_PID"',
+                    SERVER_PID=str(server.pid),
+                )
+                self.assertEqual(result.returncode, 77)
+                self.assertIn("neither ss nor netstat found", result.stderr)
+                server.wait(timeout=3)
+                self.assertLess(server.returncode, 0)
+            finally:
+                if server.poll() is None:
+                    server.kill()
+
+    def test_timeout_is_bounded_and_cleans_up(self) -> None:
+        """Stop polling after the retry budget and terminate the server."""
+        # Only accelerate the polling delay; keep a real live server process.
+        self.finder.write_text('#!/bin/sh\necho probe >&2\ncat "$SOCKET_FIXTURE"\n')
+        with subprocess.Popen(["sleep", "60"]) as server:
+            try:
+                result = self.run_shell(
+                    'sleep() { echo polling-sleep; }\nwait_udp_server "$SERVER_PID"',
+                    SERVER_PID=str(server.pid),
+                )
+                self.assertEqual(result.returncode, 1)
+                self.assertIn("did not come up", result.stderr)
+                self.assertEqual(result.stderr.count("probe\n"), 90)
+                self.assertEqual(result.stdout.count("polling-sleep"), 89)
+                server.wait(timeout=3)
+                self.assertLess(server.returncode, 0)
+            finally:
+                if server.poll() is None:
+                    server.kill()
+
+    def test_server_exits_while_waiting(self) -> None:
+        """Detect a startup failure that happens after polling begins."""
+        result = self.run_shell('sleep 1 &\npid=$!\nwait_udp_server "$pid"')
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("exited before binding", result.stderr)
+        self.assertIn("waiting for UDP port", result.stdout)
+
+    def test_real_socket_delayed_beyond_four_seconds(self) -> None:
+        """Wait for a real delayed bind with each installed port finder."""
+        finders = [shutil.which("ss"), os.environ.get("NETSTAT") or shutil.which("netstat")]
+        finders = [finder for finder in finders if finder]
+        if not finders:
+            self.skipTest("neither ss nor netstat available")
+        for finder in finders:
+            with self.subTest(finder=finder):
+                with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
+                    sock.bind(("127.0.0.1", 0))
+                    port = sock.getsockname()[1]
+                code = (
+                    "import socket,time,sys; time.sleep(6); "
+                    "s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); "
+                    "s.bind(('127.0.0.1',int(sys.argv[1]))); time.sleep(30)"
+                )
+                with subprocess.Popen([sys.executable, "-c", code, str(port)]) as server:
+                    try:
+                        started = time.monotonic()
+                        result = self.run_shell(
+                            'wait_udp_server "$SERVER_PID"',
+                            SERVER_PID=str(server.pid),
+                            PORT=str(port),
+                            PFCMD=finder,
+                        )
+                        self.assertEqual(result.returncode, 0, result.stderr)
+                        self.assertGreaterEqual(time.monotonic() - started, 6)
+                        self.assertIsNone(server.poll())
+                    finally:
+                        server.terminate()
+                        server.wait(timeout=3)
+
+
+if __name__ == "__main__":
+    unittest.main()

common/optional/x86-64-v3/patches/prometheus/complete-test-parsing.patch

@@ -0,0 +1,36 @@
+Subject: [PATCH] tests: finish parsing before inspecting editor state
+
+EditorState creation has a 20 ms parsing budget. A descheduled test can
+therefore observe an incomplete tree. Finish these small test documents
+without an interactive deadline and publish the result with a transaction.
+Keep the original completion and vector-matching assertions enabled.
+
+--- a/module/codemirror-promql/src/test/utils-test.ts
++++ b/module/codemirror-promql/src/test/utils-test.ts
+@@ -13,7 +13,7 @@
+ 
+ import { parser } from '@prometheus-io/lezer-promql';
+ import { EditorState } from '@codemirror/state';
+-import { LRLanguage } from '@codemirror/language';
++import { ensureSyntaxTree, LRLanguage } from '@codemirror/language';
+ import nock from 'nock';
+ import path from 'path';
+ import { fileURLToPath } from 'url';
+@@ -23,10 +23,16 @@
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
+ 
+ export function createEditorState(expr: string): EditorState {
+-  return EditorState.create({
++  const state = EditorState.create({
+     doc: expr,
+     extensions: lightPromQLSyntax,
+   });
++  // These tests need a complete tree, independent of the editor's time budget.
++  if (!ensureSyntaxTree(state, state.doc.length, Infinity)) {
++    throw new Error('Unable to parse the test expression');
++  }
++  // Publish the completed parse so syntaxTree(state) sees it too.
++  return state.update({}).state;
+ }
+ 
+ export function mockPrometheusServer(): void {

common/optional/x86-64-v3/patches/prometheus/default.nix

@@ -0,0 +1,17 @@
+{ prometheus }:
+prometheus.overrideAttrs (
+  old:
+  let
+    assets = old.passthru.assets.overrideAttrs (assetsOld: {
+      patches = (assetsOld.patches or [ ]) ++ [ ./complete-test-parsing.patch ];
+    });
+  in
+  {
+    postPatch = builtins.replaceStrings [ "${old.passthru.assets}" ] [ "${assets}" ] (
+      builtins.unsafeDiscardStringContext old.postPatch
+    );
+    passthru = old.passthru // {
+      inherit assets;
+    };
+  }
+)

common/optional/x86-64-v3/patches/psutil/default.nix

@@ -0,0 +1,4 @@
+{ psutil }:
+psutil.overridePythonAttrs (old: {
+  patches = (old.patches or [ ]) ++ [ ./heap-info-zero-mmap.patch ];
+})

common/optional/x86-64-v3/patches/psutil/heap-info-zero-mmap.patch

@@ -0,0 +1,17 @@
+Subject: [PATCH] Allow an empty mmap allocation total in heap_info
+
+The allocator may satisfy all live allocations from the heap. A zero
+mmap_used value is valid and depends on the worker allocation history.
+Keep the heap and platform checks without requiring an mmap allocation.
+
+--- a/tests/test_system.py
++++ b/tests/test_system.py
+@@ -267,6 +267,7 @@
+         if MACOS:
+             assert m.mmap_used == 0  # not supported
+         else:
+-            assert m.mmap_used > 0
++            # A process can have no live mmap-backed malloc allocations.
++            assert m.mmap_used >= 0
+         if WINDOWS:
+             assert m.heap_count >= 0

common/optional/x86-64-v3/patches/pytest-xdist/concurrent-worker-crashes.patch

@@ -0,0 +1,29 @@
+Subject: [PATCH] tests: count replacements when checking the worker restart limit
+
+With two workers, another in-flight test may crash after the fourth
+crash requests shutdown. Either four or five failed tests is valid.
+Require exactly three replacements and the failed-tests exit status,
+while preserving the queued-work and no-internal-error assertions.
+
+--- a/testing/acceptance_test.py
++++ b/testing/acceptance_test.py
+@@ -1011,9 +1011,18 @@
+                 "worker*crashed while running*",
+                 "worker*crashed while running*",
+                 "* xdist: maximum crashed workers reached: 3 *",
+-                "* 4 failed in *",
+             ]
+         )
++        # A second in-flight test may crash after shutdown is requested.
++        # The restart limit constrains replacements, not concurrent failures.
++        replacements = sum(
++            line.startswith("replacing crashed worker ") for line in res.stdout.lines
++        )
++        assert replacements == 3
++        failed = res.parseoutcomes()["failed"]
++        assert failed in (4, 5)
++        res.assert_outcomes(failed=failed)
++        assert res.ret == pytest.ExitCode.TESTS_FAILED
+         assert "INTERNALERROR" not in res.stdout.str()
+ 
+     def test_max_worker_restart_die(self, pytester: pytest.Pytester) -> None:

common/optional/x86-64-v3/patches/pytest-xdist/default.nix

@@ -0,0 +1,12 @@
+{ pytest-xdist }:
+pytest-xdist.overridePythonAttrs (old: {
+  patches = (old.patches or [ ]) ++ [
+    ./concurrent-worker-crashes.patch
+    ./worker-startup-timeout.patch
+  ];
+
+  # The suite exercises its own worker pools. Limit the outer suite to one worker.
+  preCheck = builtins.replaceStrings [ "--numprocesses=$NIX_BUILD_CORES" ] [ "--numprocesses=1" ] (
+    old.preCheck or ""
+  );
+})

common/optional/x86-64-v3/patches/pytest-xdist/worker-startup-timeout.patch

@@ -0,0 +1,19 @@
+Subject: [PATCH] tests: allow more time for remote worker events
+
+Worker startup can exceed ten seconds on heavily loaded builders. Allow
+the remote-test helper to wait up to sixty seconds for worker events.
+The wait still returns as soon as an event arrives and remains bounded.
+Production worker timeouts and test assertions are unchanged.
+
+--- a/testing/test_remote.py
++++ b/testing/test_remote.py
+@@ -17,7 +17,8 @@
+ from xdist.workermanage import WorkerController
+ 
+ 
+-WAIT_TIMEOUT = 10.0
++# Allow worker events extra time on heavily loaded builders.
++WAIT_TIMEOUT = 60.0
+ 
+ 
+ def check_marshallable(d: object) -> None:

common/optional/x86-64-v3/patches/scipy/default.nix

@@ -0,0 +1,5 @@
+{ scipy }:
+scipy.overridePythonAttrs (old: {
+  # Keep the STFT tests enabled with tolerances for x86-64-v3 rounding.
+  patches = (old.patches or [ ]) ++ [ ./stft-test-tolerances.patch ];
+})

common/optional/x86-64-v3/patches/scipy/stft-test-tolerances.patch

@@ -0,0 +1,51 @@
+Subject: [PATCH] signal: allow floating-point rounding in STFT tests
+
+Keep the STFT tests enabled for x86-64-v3 builds. Allow two float32
+epsilons of relative error when comparing inverse-STFT implementations;
+float64 and the existing i686 override remain unchanged. Allow one
+float64 epsilon of absolute error in all three scaling round trips,
+which otherwise require exact zeros (observed residual: 4e-17 for a
+signal with amplitude 2).
+
+Upstream issue: https://github.com/scipy/scipy/issues/25488
+
+--- a/scipy/signal/tests/_scipy_spectral_test_shim.py
++++ b/scipy/signal/tests/_scipy_spectral_test_shim.py
+@@ -294,7 +294,7 @@
+ 
+     # Adapted tolerances to account for resolution loss:
+     atol = np.finfo(x.dtype).resolution*2  # instead of default atol = 0
+-    rtol = 1e-7  # default for np.allclose()
++    rtol = max(1e-7, 2 * np.finfo(x.dtype).eps)
+ 
+     # Relax atol on 32-Bit platforms a bit to pass CI tests.
+     #  - Not clear why there are discrepancies (in the FFT maybe?)
+--- a/scipy/signal/tests/test_spectral.py
++++ b/scipy/signal/tests/test_spectral.py
+@@ -2044,7 +2044,7 @@
+ 
+         # Test round trip:
+         x1 = istft(Zs, boundary=True, scaling='spectrum')[1]
+-        assert_allclose(x1, x)
++        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
+ 
+         # For a Hann-windowed 256 sample length FFT, we expect a peak at
+         # frequency 64 (since it is 1/4 the length of X) with a height of 1
+@@ -2074,7 +2074,7 @@
+ 
+         # Test round trip:
+         x1 = istft(Zp, input_onesided=False, boundary=True, scaling='psd')[1]
+-        assert_allclose(x1, x)
++        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
+ 
+         # The power of the one-sided psd-scaled STFT can be determined
+         # analogously (note that the two sides are not of equal shape):
+@@ -2094,7 +2094,7 @@
+ 
+         # Test round trip:
+         x1 = istft(Zp0, input_onesided=True, boundary=True, scaling='psd')[1]
+-        assert_allclose(x1, x)
++        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
+ 
+ 
+ class TestSampledSpectralRepresentations:

common/optional/x86-64-v3/patches/trunk/default.nix

@@ -0,0 +1,21 @@
+{ trunk, jq }:
+trunk.overrideAttrs (old: {
+  nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ jq ];
+
+  # Patch the bundled libdeflate before Cargo validates its vendor checksums.
+  postPatch = (old.postPatch or "") + ''
+    libdeflateVendors=( "$cargoDepsCopy"/source-*/libdeflate-sys-1.23.1 )
+    if [ "''${#libdeflateVendors[@]}" -ne 1 ] || [ ! -d "''${libdeflateVendors[0]}" ]; then
+      echo "Expected exactly one vendored libdeflate-sys 1.23.1" >&2
+      exit 1
+    fi
+    libdeflateVendor="''${libdeflateVendors[0]}"
+    patch -d "$libdeflateVendor/libdeflate" -p1 < ${./libdeflate-gcc16.patch}
+    libdeflateHeader=libdeflate/lib/x86/cpu_features.h
+    libdeflateChecksum=$(sha256sum "$libdeflateVendor/$libdeflateHeader" | cut -d ' ' -f 1)
+    jq --arg path "$libdeflateHeader" --arg hash "$libdeflateChecksum" \
+      '.files[$path] = $hash' "$libdeflateVendor/.cargo-checksum.json" \
+      > "$libdeflateVendor/.cargo-checksum.json.new"
+    mv "$libdeflateVendor/.cargo-checksum.json.new" "$libdeflateVendor/.cargo-checksum.json"
+  '';
+})

common/optional/x86-64-v3/patches/trunk/libdeflate-gcc16.patch

@@ -0,0 +1,33 @@
+From 2a3762cf736aff9aa5dda329fdec3c08a816eb9b Mon Sep 17 00:00:00 2001
+From: Sergei Trofimovich <slyich@gmail.com>
+Date: Tue, 20 May 2025 21:18:28 +0100
+Subject: [PATCH] lib/x86/cpu_features.h: drop evex512 on gcc-16
+
+gcc-16 `master` branch dropped support for `evex512` as:
+    https://gcc.gnu.org/git/?p=gcc.git;a=commitdiff;h=c052a6f4a1c803cb92147ff98fb91cf3511e0856
+
+As a result libdeflate build started failing as:
+
+    In file included from /build/source/lib/adler32.c:28:
+    /build/source/lib/x86/adler32_template.h:135:12: error: attribute 'target' argument 'no-evex512' is unknown
+      135 | ADD_SUFFIX(reduce_to_32bits)(vec_t v_s1, vec_t v_s2, u32 *s1_p, u32 *s2_p)
+          |            ^~~~~~~~~~~~~~~~
+
+The change adds upper build to `evex512` for `gcc`.
+---
+ lib/x86/cpu_features.h | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/lib/x86/cpu_features.h b/lib/x86/cpu_features.h
+index fbfdb0eb..f9d69d72 100644
+--- a/lib/x86/cpu_features.h
++++ b/lib/x86/cpu_features.h
+@@ -165,7 +165,7 @@ static inline u32 get_x86_cpu_features(void) { return 0; }
+ #  define HAVE_AVXVNNI(features)	((features) & X86_CPU_FEATURE_AVXVNNI)
+ #endif
+ 
+-#if (GCC_PREREQ(14, 0) || CLANG_PREREQ(18, 0, 18000000)) \
++#if ((GCC_PREREQ(14, 0) && !(GCC_PREREQ(16, 0))) || CLANG_PREREQ(18, 0, 18000000)) \
+ 	&& !defined(__EVEX512__) /* avoid subtracting the evex512 feature */
+ #  define EVEX512	",evex512"	/* needed to override potential -mno-evex512 */
+ #  define NO_EVEX512	",no-evex512"

common/optional/x86-64-v3/patches/zopfli/default.nix

@@ -0,0 +1,4 @@
+{ zopfli }:
+zopfli.overrideAttrs (old: {
+  patches = (old.patches or [ ]) ++ [ ./unaligned-match-loads.patch ];
+})

common/optional/x86-64-v3/patches/zopfli/unaligned-match-loads.patch

@@ -0,0 +1,32 @@
+Subject: [PATCH] Read unaligned match buffers with memcpy
+
+Byte buffers need not satisfy integer alignment or aliasing requirements.
+GCC 16 with x86-64-v3 vectorizes the cast loads using aligned AVX reads,
+causing a segmentation fault. Copy into local integers instead.
+
+--- a/src/zopfli/lz77.c
++++ b/src/zopfli/lz77.c
+@@ -302,13 +302,20 @@
+   if (sizeof(size_t) == 8) {
+     /* 8 checks at once per array bounds check (size_t is 64-bit). */
+-    while (scan < safe_end && *((size_t*)scan) == *((size_t*)match)) {
++    while (scan < safe_end) {
++      size_t scan_word, match_word;
++      memcpy(&scan_word, scan, sizeof(scan_word));
++      memcpy(&match_word, match, sizeof(match_word));
++      if (scan_word != match_word) break;
+       scan += 8;
+       match += 8;
+     }
+   } else if (sizeof(unsigned int) == 4) {
+     /* 4 checks at once per array bounds check (unsigned int is 32-bit). */
+-    while (scan < safe_end
+-        && *((unsigned int*)scan) == *((unsigned int*)match)) {
++    while (scan < safe_end) {
++      unsigned int scan_word, match_word;
++      memcpy(&scan_word, scan, sizeof(scan_word));
++      memcpy(&match_word, match, sizeof(match_word));
++      if (scan_word != match_word) break;
+       scan += 4;
+       match += 4;
+     }

common/optional/zfs/snapshot.nix

@@ -7,11 +7,7 @@
 }:
 let
   cfg = config.services.snapshot_manager;
-  snapshotManagerPackages =
-    ps: with ps; [
-      httpx
-      typer
-    ];
+  snapshotManagerPackages = ps: with ps; [ typer ];
 in
 {
   options = {

flake.lock

@@ -7,11 +7,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1781152676,
-        "narHash": "sha256-RxWs5ND31KzTG7wvMM+PMfUjyNpmIEr999lqNARaM5o=",
+        "lastModified": 1789770686,
+        "narHash": "sha256-uZkBR7yHdIKUFB5SZdfgh1qkGfI3XmYmI/lTiquxbck=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "ff8702b4de27f72b4c78573dfb89ec74e36abdf1",
+        "rev": "725ea35e410ad83be4931d1bff7e090eacaf3563",
         "type": "github"
       },
       "original": {
@@ -20,6 +20,24 @@
         "type": "github"
       }
     },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "home-manager": {
       "inputs": {
         "nixpkgs": [
@@ -27,11 +45,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1788651960,
-        "narHash": "sha256-v9wJd32eZ2bvhBzVOd7TIjLQd011P7nwOhjKtWlci5I=",
+        "lastModified": 1790819692,
+        "narHash": "sha256-rvubAPbh3QCliic3sAK+CXTrPCGgJstY52OTs/5H1J4=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "2c0350c759688177331b8f5242311fae8877bdb3",
+        "rev": "c8ecc29e5175452bee4a2aa1ba2383856d795338",
         "type": "github"
       },
       "original": {
@@ -40,16 +58,38 @@
         "type": "github"
       }
     },
+    "jellyswarrm": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-overlay": "rust-overlay"
+      },
+      "locked": {
+        "lastModified": 1790427786,
+        "narHash": "sha256-8+Ip0HFW2oZiRr0zCZmPPIHfTdxxHy4CmSF1uOkM3Bw=",
+        "owner": "LLukas22",
+        "repo": "Jellyswarrm",
+        "rev": "9e67c5ad12116add867ba6c1bf69f767879e9147",
+        "type": "github"
+      },
+      "original": {
+        "owner": "LLukas22",
+        "repo": "Jellyswarrm",
+        "type": "github"
+      }
+    },
     "nixos-hardware": {
       "inputs": {
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1788860136,
-        "narHash": "sha256-MhPMOFV4pVkygWEbQ8t1De/uQ9cWF1u++tRe2L5tG48=",
+        "lastModified": 1790722812,
+        "narHash": "sha256-XwFt1uLyaBCQU1/nUeGqeBnqNXPVgGRnz20YVYdmF6Y=",
         "owner": "nixos",
         "repo": "nixos-hardware",
-        "rev": "62173785b9a18c78b4a15aca2623d02bceb9d077",
+        "rev": "06f9ecaea5f64b6ff61cf42cb32f21621c4fa14a",
         "type": "github"
       },
       "original": {
@@ -61,11 +101,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1767892417,
-        "narHash": "sha256-8bW3q88CEg2u4hSP66Vf4lpbLonHz7hqDNBMcCY7E9U=",
-        "rev": "3497aa5c9457a9d88d71fa93a4a8368816fbeeba",
+        "lastModified": 1789546076,
+        "narHash": "sha256-vWkSk5bbfTqdtMoSgD9FshACO8JCvXTFi+3cqEp0mH0=",
+        "rev": "b1b875982b17dabde9b4a37f3e229e74913e6db3",
         "type": "tarball",
-        "url": "https://releases.nixos.org/nixos/unstable/nixos-26.05pre924538.3497aa5c9457/nixexprs.tar.xz"
+        "url": "https://releases.nixos.org/nixos/unstable/nixos-26.11pre1074753.b1b875982b17/nixexprs.tar.xz"
       },
       "original": {
         "type": "tarball",
@@ -74,11 +114,11 @@
     },
     "nixpkgs-master": {
       "locked": {
-        "lastModified": 1788892992,
-        "narHash": "sha256-cIMFh9gyU4/aLeB3JCcsWM3tTAvD9pAq9Smr1Wa8aIU=",
+        "lastModified": 1790862528,
+        "narHash": "sha256-00jiK7QCLGuVe4mj3kDa6P/iDFKb87aK9ZtJDZCOvTc=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "dff6994123e257ec9901c271bc2b52e64d7c8f05",
+        "rev": "86f3f30260edec38090733d0667e818e6e2ea6f5",
         "type": "github"
       },
       "original": {
@@ -106,11 +146,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1788752844,
-        "narHash": "sha256-VaWGJ6+cIYN2erfSecbRV+4ljI185Ty2wUrXyvQbgOw=",
+        "lastModified": 1790689126,
+        "narHash": "sha256-ilerN1WLSvF+HMjziC/Wv99J5y02maDH+6hZPwsORKg=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "dc5d91f840324650bac8c379428c7037a416959a",
+        "rev": "b4fd65b198c599cbe814fcb9f42d25d021595ec9",
         "type": "github"
       },
       "original": {
@@ -124,12 +164,34 @@
       "inputs": {
         "disko": "disko",
         "home-manager": "home-manager",
+        "jellyswarrm": "jellyswarrm",
         "nixos-hardware": "nixos-hardware",
         "nixpkgs": "nixpkgs_2",
         "nixpkgs-master": "nixpkgs-master",
         "nixpkgs-stable": "nixpkgs-stable",
         "sops-nix": "sops-nix",
-        "systems": "systems"
+        "systems": "systems_2"
+      }
+    },
+    "rust-overlay": {
+      "inputs": {
+        "nixpkgs": [
+          "jellyswarrm",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1779419951,
+        "narHash": "sha256-dMX0PUslUHPajP6o8FEoRdFv9afq/dec4POR0vVfjK4=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "5b5c521d6cae9ef4aa32f888eb2c0ce595c9be52",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
       }
     },
     "sops-nix": {
@@ -139,11 +201,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1788337237,
-        "narHash": "sha256-gkSH8VUtCo6hnysNmb9DbTuDepH2t5pv+QWjP75xKAk=",
+        "lastModified": 1790498116,
+        "narHash": "sha256-rs9meAYxW3zzrh43yaW7htrqCD+X9+pupDPHN86fumI=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "fbf759290e0cb0a98dfc813a4eb7d53ad1dacb57",
+        "rev": "5efb5a6f4f5ab192817d28557dd4d650fa14d866",
         "type": "github"
       },
       "original": {
@@ -153,6 +215,21 @@
       }
     },
     "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
       "locked": {
         "lastModified": 1689347949,
         "narHash": "sha256-12tWmuL2zgBgZkdoB6qXZsgJEH9LR3oUgpaQq2RbI80=",

flake.nix

@@ -21,6 +21,11 @@
 
     nixos-hardware.url = "github:nixos/nixos-hardware/master";
 
+    jellyswarrm = {
+      url = "github:LLukas22/Jellyswarrm";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
     home-manager = {
       url = "github:nix-community/home-manager";
       inputs.nixpkgs.follows = "nixpkgs";

overlays/default.nix

@@ -1,13 +1,13 @@
 { inputs, ... }:
 {
-  # When applied, the stable nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.stable'
+  # Additional package sets are accessible through `pkgs.<name>`.
   stable = final: _prev: {
     stable = import inputs.nixpkgs-stable {
       system = final.stdenv.hostPlatform.system;
       config.allowUnfree = true;
     };
   };
-  # When applied, the master nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.master'
+
   master = final: _prev: {
     master = import inputs.nixpkgs-master {
       system = final.stdenv.hostPlatform.system;
@@ -15,6 +15,12 @@
     };
   };
 
-  test-exclusions = import ./test-exclusions.nix;
-  x86-64-v3-workarounds = import ./x86-64-v3-workarounds.nix;
+  # Baseline x86-64 (v1) packages for prebuilt applications that should not
+  # inherit an x86-64-v3 host platform.
+  x86-v1 = final: _prev: {
+    x86-v1 = import inputs.nixpkgs {
+      system = final.stdenv.hostPlatform.system;
+      config.allowUnfree = true;
+    };
+  };
 }

overlays/test-exclusions.nix

@@ -1,133 +0,0 @@
-# Test exclusions for the locally rebuilt x86-64-v3 package set.
-#
-# Selecting x86-64-v3 changes every affected derivation, so the normal
-# nixpkgs binary cache cannot be used and upstream test suites run locally.
-# The jeeves builder uses /tmp/nix-builds so filesystem tests run on tmpfs
-# instead of ZFS with normalization=formD and utf8only=on; those tests remain
-# enabled. The remaining workarounds cover UDP readiness, resource-sensitive
-# parser and nested-worker races, and mismatched timeout clocks, plus
-# architecture-dependent floating-point differences whose risk we accept for
-# our workloads. Keep these exceptions visible until their causes are fixed.
-_final: prev: {
-  gnutls = prev.gnutls.overrideAttrs (old: {
-    # This test uses a fixed four-second sleep instead of checking UDP
-    # readiness; the client saw no listener in the x86-64-v3 build.
-    postPatch = (old.postPatch or "") + ''
-      sed '2iexit 77' -i tests/serv-udp.sh
-    '';
-  });
-
-  prometheus = prev.prometheus.overrideAttrs (
-    old:
-    let
-      assets = old.passthru.assets.overrideAttrs (assetsOld: {
-        # CodeMirror's bounded synchronous parser can return an incomplete tree
-        # when these cases run on a heavily loaded builder.
-        postPatch = (assetsOld.postPatch or "") + ''
-          substituteInPlace module/codemirror-promql/src/complete/hybrid.test.ts \
-            --replace-fail "it(value.title, () => {" \
-            "(value.title === 'autocomplete topk params 2' ? it.skip : it)(value.title, () => {"
-          substituteInPlace module/codemirror-promql/src/parser/vector.test.ts \
-            --replace-fail "it(value.binaryExpr, () => {" \
-            "(value.binaryExpr === 'foo * on(test,blub) bar' ? it.skip : it)(value.binaryExpr, () => {"
-        '';
-      });
-    in
-    {
-      postPatch = builtins.replaceStrings [ "${old.passthru.assets}" ] [ "${assets}" ] (
-        builtins.unsafeDiscardStringContext old.postPatch
-      );
-      passthru = old.passthru // {
-        inherit assets;
-      };
-    }
-  );
-
-  pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
-    (_pythonFinal: pythonPrev: {
-      backrefs = pythonPrev.backrefs.overridePythonAttrs (old: {
-        # regex measures its timeout in process CPU time, while this test used
-        # wall time and could miss the timeout when a busy builder descheduled it.
-        postPatch = (old.postPatch or "") + ''
-          substituteInPlace tests/test_bregex.py \
-            --replace-fail "time.time()" "time.process_time()"
-        '';
-      });
-
-      pytest-xdist = pythonPrev.pytest-xdist.overridePythonAttrs (old: {
-        # The suite exercises its own worker pools. Run the outer suite with one
-        # worker and allow inner workers more time on heavily loaded builders.
-        postPatch = (old.postPatch or "") + ''
-          substituteInPlace testing/test_remote.py \
-            --replace-fail "WAIT_TIMEOUT = 10.0" "WAIT_TIMEOUT = 60.0"
-        '';
-        preCheck = builtins.replaceStrings [ "--numprocesses=$NIX_BUILD_CORES" ] [ "--numprocesses=1" ] (
-          old.preCheck or ""
-        );
-        # This test deliberately crashes workers past the restart limit and
-        # races while checking which replacement message was emitted.
-        disabledTests = (old.disabledTests or [ ]) ++ [
-          "test_max_worker_restart_tests_queued"
-        ];
-      });
-
-      jupyter-server = pythonPrev.jupyter-server.overridePythonAttrs (old: {
-        # The kernel reply arrived after the one-second outer deadline when the
-        # builder was heavily loaded. Keep the regression test but allow it the
-        # same margin as the other resource-sensitive tests.
-        postPatch = (old.postPatch or "") + ''
-          substituteInPlace tests/services/kernels/test_connection.py \
-            --replace-fail \
-            "await asyncio.wait_for(asyncio.wrap_future(conn2.request_kernel_info()), timeout=1.0)" \
-            "await asyncio.wait_for(asyncio.wrap_future(conn2.request_kernel_info()), timeout=10.0)"
-        '';
-      });
-
-      scipy = pythonPrev.scipy.overridePythonAttrs (old: {
-        # x86-64-v3 FFT implementations produce rounding differences outside
-        # these tests' strict tolerances. We accept the numerical-precision
-        # risk for our workloads.
-        disabledTests = (old.disabledTests or [ ]) ++ [
-          "test_roundtrip_float32"
-          "test_roundtrip_scaling"
-        ];
-      });
-
-      sentry-sdk = pythonPrev.sentry-sdk.overridePythonAttrs (old: {
-        # This test globally mocks threading.current_thread while another
-        # thread is running. On Python 3.14, Thread.join can race with that
-        # mock and exhaust its single side effect before the worker removes it.
-        disabledTests = (old.disabledTests or [ ]) ++ [
-          "test_get_current_thread_meta_main_thread"
-        ];
-      });
-
-      torchaudio = pythonPrev.torchaudio.overridePythonAttrs (old: {
-        # x86-64-v3 pitch shifting produces batch-versus-single-item numerical
-        # differences up to 2.9e-6. We accept that audio-precision risk for our
-        # workloads.
-        disabledTests = (old.disabledTests or [ ]) ++ [
-          "test_batch_pitch_shift"
-        ];
-      });
-
-      torchcodec = pythonPrev.torchcodec.overridePythonAttrs (old: {
-        # For these 8 kHz MP3 cases, the x86-64-v3 API and CLI codec paths
-        # differ in 0.8% of decoded samples. Retain the original tolerance for
-        # 99% of samples and accept the localized audio-precision risk.
-        postPatch = (old.postPatch or "") + ''
-          substituteInPlace test/test_encoders.py \
-            --replace-fail \
-            'if sys.platform == "darwin":' \
-            'if sys.platform == "darwin" or (
-                format == "mp3"
-                and sample_rate == 8_000
-                and asset is SINE_MONO_S32
-                and bit_rate in (None, 0)
-                and num_channels in (None, 1)
-            ):'
-        '';
-      });
-    })
-  ];
-}

overlays/x86-64-v3-workarounds.nix

@@ -1,60 +0,0 @@
-# Compatibility fixes for packages rebuilt with x86-64-v3.
-#
-# The v3 baseline enables instructions that expose source assumptions hidden
-# by the generic x86-64 build. Keep compile fixes here, separate from test
-# exclusions, until upstream or nixpkgs incorporates them.
-_final: prev:
-let
-  patchAbseilBmi2Include =
-    package:
-    package.overrideAttrs (old: {
-      # GCC and Clang prohibit including their internal BMI2 header directly.
-      # The public umbrella provides the same intrinsics with the required
-      # compiler setup.
-      postPatch = (old.postPatch or "") + ''
-        substituteInPlace third_party/abseil-cpp/absl/container/internal/raw_hash_set.h \
-          --replace-fail "#include <bmi2intrin.h>" "#include <immintrin.h>"
-      '';
-    });
-
-  removeSiblingOutputChecks =
-    package:
-    package.overrideAttrs (old: {
-      # Nix 2.34 can validate a partial multi-output rebuild against only the
-      # outputs still being realised. PostgreSQL's checks then reject valid
-      # sibling names such as "out" and "lib". Keep the test suite and
-      # disallowed-requisite checks; accept the loss of cross-output checks.
-      outputChecks = builtins.mapAttrs (
-        _output: checks: builtins.removeAttrs checks [ "disallowedReferences" ]
-      ) (old.outputChecks or { });
-    });
-
-  electron43Unwrapped = patchAbseilBmi2Include prev.electron_43.unwrapped;
-  electron43 = prev.electron_43.override {
-    electron-unwrapped = electron43Unwrapped;
-  };
-
-  signalCallPackage =
-    path: args:
-    let
-      package = prev.callPackage path args;
-    in
-    if builtins.baseNameOf path == "webrtc.nix" then patchAbseilBmi2Include package else package;
-in
-prev.lib.optionalAttrs ((prev.stdenv.hostPlatform.gcc.arch or null) == "x86-64-v3") {
-  deno =
-    let
-      librusty_v8 = patchAbseilBmi2Include prev.deno.passthru.librusty_v8;
-    in
-    prev.deno.override { inherit librusty_v8; };
-
-  electron_43 = electron43;
-
-  postgresql = removeSiblingOutputChecks prev.postgresql;
-  postgresql_18 = removeSiblingOutputChecks prev.postgresql_18;
-
-  signal-desktop = prev.signal-desktop.override {
-    electron_43 = electron43;
-    callPackage = signalCallPackage;
-  };
-}

python/ebook_search/docker/README.md

@@ -1,77 +0,0 @@
-# Ebook Search Docker
-
-Run the EPUB search app against the existing Postgres database on `jeeves`:
-
-```sh
-python -m python.ebook_search.docker.containers start --library-path /path/to/epubs --build
-```
-
-All ebook-search Docker files live in this directory:
-
-- `Dockerfile` — multi-stage: `test` (runs pytest) and `runtime` (default target, the app image)
-- `docker-compose.yml`
-- `containers.py` — Typer lifecycle CLI
-- `pyproject.toml` / `uv.lock` — the container's uv-locked dependencies
-
-The app listens on `http://localhost:8070`.
-
-Useful lifecycle commands:
-
-```sh
-python -m python.ebook_search.docker.containers build
-python -m python.ebook_search.docker.containers start --library-path /path/to/epubs
-python -m python.ebook_search.docker.containers test
-python -m python.ebook_search.docker.containers logs
-python -m python.ebook_search.docker.containers ps
-python -m python.ebook_search.docker.containers stop
-```
-
-Direct compose usage from the repo root:
-
-```sh
-docker compose -f python/ebook_search/docker/docker-compose.yml ps
-```
-
-## Dependencies
-
-The image builds its environment with uv from `pyproject.toml` + `uv.lock` in this
-directory — this is the source of truth for the container's dependencies. To add or
-update a dependency, edit `pyproject.toml` here and regenerate the lock (uv is
-available in the `ebook-search` dev shell):
-
-```sh
-nix develop .#ebook-search -c uv lock --project python/ebook_search/docker
-```
-
-## Tests
-
-The main pytest suite excludes `tests/ebook_search` (its dependencies are no longer
-in the nix dev shell). The `test ebook search` CI workflow runs them in a uv env
-built from the lockfile in this directory — same commands work locally from the
-repo root (the `--override-ini` drops the main suite's ignore):
-
-```sh
-uv sync --locked --project python/ebook_search/docker
-uv run --project python/ebook_search/docker --no-sync pytest tests/ebook_search --override-ini addopts="-n auto -ra"
-```
-
-They can also run inside the Docker `test` image, which validates the image itself:
-
-```sh
-python -m python.ebook_search.docker.containers test
-```
-
-or the raw docker equivalent:
-
-```sh
-docker build --file python/ebook_search/docker/Dockerfile --target test --tag ebook-search:test .
-docker run --rm ebook-search:test
-```
-
-## Configuration
-
-The compose service loads the repo root `.env` into the container via `env_file`.
-
-Mount your EPUB directory by setting `EBOOK_LIBRARY_HOST_PATH` in an env file or on the command line. The container sees it as `/library`, and `EBOOK_SEARCH_LIBRARY_PATHS` is set to `/library` inside the container.
-
-Database connection settings are controlled by `RICHIE_DB`, `RICHIE_HOST`, `RICHIE_PORT`, `RICHIE_USER`, and `RICHIE_PASSWORD`. The default host is `jeeves`.

python/gems/README.md

@@ -1,54 +0,0 @@
-# Gems
-
-Gems is a server-rendered, turn-based resource-engine game for one to four human or AI players. It uses FastAPI,
-Jinja, HTMX, server-sent events, and SQLite.
-
-The application deliberately contains no playable card deck, patron/governor set, objective set, official artwork,
-or copied rulebook text. A room host must upload a content pack they are entitled to use before starting a game.
-
-## Run locally
-
-```shell
-uv run gems --host 127.0.0.1 --port 8082
-```
-
-The default database and installation key are created under `.gems/`. The following environment variables override
-runtime behavior:
-
-- `GEMS_DATABASE_PATH`
-- `GEMS_KEY_PATH`
-- `GEMS_PUBLIC_ORIGIN`
-- `GEMS_SECURE_COOKIES`
-- `GEMS_HOST`
-- `GEMS_PORT`
-
-## Content packs
-
-The current schema is available from a running server at `/schemas/content-pack-v1.json`. A pack defines exactly
-five normal resources, one wild resource, cards, and optional patrons, objectives, and outposts. `patrons` is the
-canonical field name; `governors` is accepted as an input alias.
-
-Cards may use only the built-in, bounded effect vocabulary:
-
-- `none`
-- `virtual_wild`
-- `copy_bonus`
-- `copy_and_claim`
-- `multi_bonus`
-- `claim_free`
-- an optional discard-cards alternate cost
-
-Unknown fields, resource references, executable expressions, HTML, artwork URLs, and files larger than 512 KiB are
-rejected. The normalized pack is private to its room and becomes immutable when play starts.
-
-## Neutral module mapping
-
-Gems calls the four optional mechanics Objectives, Outposts, Eastern Decks, and Fortifications. Lobby presets combine
-these mechanics into the familiar base, objective-race, objective-plus-outpost, eastern-plus-fortification, and
-all-module configurations. Component identities and values always come from the uploaded pack.
-
-## Jeeves
-
-The NixOS module runs one Uvicorn worker on `127.0.0.1:8002`, stores state in
-`/zfs/media/services/gems`, and publishes it through HAProxy at `https://gems.tmmworkshop.com`. The DNS record must
-point to Jeeves before ACME can issue the certificate.

python/signal_alert.py

@@ -2,49 +2,63 @@
 
 from __future__ import annotations
 
+import json
 import logging
-from os import getenv
-
-import httpx
+import socket
 
 logger = logging.getLogger(__name__)
 
-SIGNAL_API_URL = "http://localhost:8989/v2/send"
-SIGNAL_API_TIMEOUT = 4.0
+
+class SignalRPCError(RuntimeError):
+    """signal-cli returned an error or closed without responding."""
 
 
-def signal_alert(body: str, title: str = "") -> None:
-    """Send a signal alert.
+def signal_alert(body: str, from_phone: str, to_phone: str, *, timeout: float = 4.0) -> None:
+    """Send a Signal alert through the local signal-cli daemon.
 
     Args:
-        body (str): The body of the alert.
-        title (str, optional): The title of the alert. Defaults to "".
-    """
-    from_phone = getenv("SIGNAL_ALERT_FROM_PHONE")
-    to_phone = getenv("SIGNAL_ALERT_TO_PHONE")
-    if not from_phone or not to_phone:
-        logger.info("SIGNAL_ALERT_FROM_PHONE or SIGNAL_ALERT_TO_PHONE not set")
-        return
+        body: The body of the alert.
+        from_phone: The Signal account sending the alert.
+        to_phone: The Signal account receiving the alert.
+        timeout: Seconds to wait on each socket operation.
 
-    # Apprise's Signal integration did not support titles, so preserve that behavior.
-    if title:
-        logger.debug("Signal does not support notification titles; ignoring title")
+    Raises:
+        SignalRPCError: If signal-cli returns an error or closes the
+            connection before responding.
+        OSError: If the socket is unreachable or an operation times out.
+    """
+    signal_rpc_id = "signal-alert"
+    request = {
+        "jsonrpc": "2.0",
+        "method": "send",
+        "params": {
+            "account": from_phone,
+            "recipient": [to_phone],
+            "message": body,
+        },
+        "id": signal_rpc_id,
+    }
 
     try:
-        response = httpx.post(
-            SIGNAL_API_URL,
-            json={
-                "message": body,
-                "number": from_phone,
-                "recipients": [to_phone],
-                "text_mode": "normal",
-            },
-            timeout=SIGNAL_API_TIMEOUT,
-            follow_redirects=True,
-        )
-    except httpx.HTTPError:
-        logger.exception("Unable to contact the Signal API")
-        return
+        with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client:
+            client.settimeout(timeout)
+            client.connect("/run/signal-cli/socket")
+            client.sendall(json.dumps(request).encode() + b"\n")
 
-    if response.status_code not in {httpx.codes.OK, httpx.codes.CREATED}:
-        logger.error("Signal API returned HTTP status %d", response.status_code)
+            buffer = b""
+            while chunk := client.recv(65536):
+                buffer += chunk
+                while b"\n" in buffer:
+                    line, buffer = buffer.split(b"\n", maxsplit=1)
+                    response = json.loads(line)
+                    if response.get("id") != signal_rpc_id:
+                        continue
+                    if rpc_error := response.get("error"):
+                        error = f"Signal RPC returned an error: {rpc_error}"
+                        raise SignalRPCError(error)
+                    return
+            error = "Signal RPC socket closed before returning a response"
+            raise SignalRPCError(error)
+    except (OSError, json.JSONDecodeError) as exc:
+        error = f"Signal RPC failed: {exc!r}"
+        raise SignalRPCError(error) from exc

python/tools/snapshot_manager.py

@@ -6,6 +6,7 @@ import logging
 import sys
 import tomllib
 from functools import cache
+from os import getenv
 from pathlib import Path  # noqa: TC003 This is required for the typer CLI
 from re import compile as re_compile
 from re import search
@@ -24,6 +25,15 @@ def main(config_file: Path) -> None:
     configure_logger(level="DEBUG")
     logger.info("Starting snapshot_manager")
 
+    from_phone = getenv("SIGNAL_ALERT_FROM_PHONE")
+    to_phone = getenv("SIGNAL_ALERT_TO_PHONE")
+    if not from_phone:
+        error = "SIGNAL_ALERT_FROM_PHONE environment variable is not set"
+        raise ValueError(error)
+    if not to_phone:
+        error = "SIGNAL_ALERT_TO_PHONE environment variable is not set"
+        raise ValueError(error)
+
     try:
         time_stamp = get_time_stamp()
 
@@ -33,14 +43,14 @@ def main(config_file: Path) -> None:
             if status != "snapshot created":
                 msg = f"{dataset.name} failed to create snapshot {time_stamp}"
                 logger.error(msg)
-                signal_alert(msg)
+                signal_alert(msg, from_phone, to_phone)
                 continue
             count_lookup = get_count_lookup(config_file, dataset.name)
             logger.info(f"using {count_lookup} for {dataset.name}")
-            get_snapshots_to_delete(dataset, count_lookup)
+            get_snapshots_to_delete(dataset, count_lookup, from_phone, to_phone)
     except Exception:
         logger.exception("snapshot_manager failed")
-        signal_alert("snapshot_manager failed")
+        signal_alert("snapshot_manager failed", from_phone, to_phone)
         sys.exit(1)
     else:
         logger.info("snapshot_manager completed")
@@ -92,12 +102,16 @@ def load_config_data(config_file: Path) -> dict[str, dict[str, int]]:
 def get_snapshots_to_delete(
     dataset: Dataset,
     count_lookup: dict[str, int],
+    from_phone: str,
+    to_phone: str,
 ) -> None:
     """Get snapshots to delete.
 
     Args:
         dataset (Dataset): the dataset
         count_lookup (dict[str, int]): the count lookup
+        from_phone (str): The Signal account sending alerts.
+        to_phone (str): The Signal account receiving alerts.
     """
     snapshots = dataset.get_snapshots()
 
@@ -127,7 +141,7 @@ def get_snapshots_to_delete(
         for snapshot in snapshots_being_deleted:
             if error := dataset.delete_snapshot(snapshot):
                 error_message = f"{dataset.name}@{snapshot} failed to delete: {error}"
-                signal_alert(error_message)
+                signal_alert(error_message, from_phone, to_phone)
                 logger.error(error_message)
 
 

scripts/signal/authorize.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+if [[ "$(hostname)" != "jeeves" ]]; then
+  echo "Signal device authorization must run on jeeves." >&2
+  exit 1
+fi
+if [[ "$EUID" -ne 0 ]]; then
+  echo "Signal device authorization must run as root." >&2
+  exit 1
+fi
+if [[ "$#" -ne 1 ]]; then
+  echo "Usage: signal_authorize.sh DEVICE_URI" >&2
+  exit 2
+fi
+if [[ "$1" != sgnl://linkdevice\?* ]]; then
+  echo "Invalid Signal device link URI." >&2
+  exit 2
+fi
+
+service_name="signal-cli"
+data_dir="/var/lib/signal-cli"
+device_uri="$1"
+restart_service=false
+
+cleanup() {
+  if [[ "$restart_service" == true ]]; then
+    systemctl start "$service_name"
+  fi
+}
+trap cleanup EXIT
+
+restart_service=true
+systemctl stop "$service_name"
+runuser -u signal-cli -- \
+  signal-cli --data-dir "$data_dir" -a +12016554378 addDevice --uri "$device_uri"

scripts/signal/link.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+host_name="$(hostname)"
+if [[ "$#" -ne 0 ]]; then
+  echo "Usage: signal_link.sh" >&2
+  exit 2
+fi
+
+service_name="signal-cli"
+data_dir="/var/lib/signal-cli"
+link_pid=""
+temp_dir=""
+restart_service=false
+
+cleanup() {
+  if [[ -n "$link_pid" ]] && kill -0 "$link_pid" 2>/dev/null; then
+    kill "$link_pid" 2>/dev/null || true
+    wait "$link_pid" 2>/dev/null || true
+  fi
+
+  if [[ -n "$temp_dir" ]]; then
+    rm -f -- "$temp_dir/link-uri"
+    rmdir -- "$temp_dir"
+  fi
+
+  if [[ "$restart_service" == true ]]; then
+    sudo systemctl start "$service_name"
+  fi
+}
+trap cleanup EXIT
+
+restart_service=true
+sudo systemctl stop "$service_name"
+
+temp_dir="$(mktemp -d)"
+link_fifo="$temp_dir/link-uri"
+mkfifo "$link_fifo"
+
+sudo -u signal-cli \
+  signal-cli --data-dir "$data_dir" link --name "$host_name" >"$link_fifo" &
+link_pid="$!"
+
+if ! IFS= read -r device_uri <"$link_fifo"; then
+  wait "$link_pid"
+  exit 1
+fi
+if [[ "$device_uri" != sgnl://linkdevice\?* ]]; then
+  echo "signal-cli returned an invalid device link URI." >&2
+  exit 1
+fi
+
+echo "Run this command manually on jeeves in another dotfiles dir:"
+printf "sudo ./scripts/signal/authorize.sh %q\n" "$device_uri"
+echo "Waiting for jeeves to authorize this device..."
+
+wait "$link_pid"
+link_pid=""
+echo "$host_name is now linked to the Signal account on jeeves."

systems/bob/default.nix

@@ -8,12 +8,12 @@
     "${inputs.self}/common/optional/monitoring-agent.nix"
     "${inputs.self}/common/optional/nvidia.nix"
     "${inputs.self}/common/optional/scanner.nix"
-    "${inputs.self}/common/optional/steam.nix"
+    "${inputs.self}/common/optional/signal-cli.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/systemd-boot.nix"
     "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/x86-64-v3.nix"
+    "${inputs.self}/common/optional/x86-64-v3"
     "${inputs.self}/common/optional/zfs"
     ./hardware.nix
     ./syncthing.nix

systems/brain/default.nix

@@ -4,12 +4,13 @@
     "${inputs.self}/users/richie"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/docker.nix"
+    "${inputs.self}/common/optional/signal-cli.nix"
     "${inputs.self}/common/optional/ssh_decrypt.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/systemd-boot.nix"
     "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/x86-64-v3.nix"
+    "${inputs.self}/common/optional/x86-64-v3"
     "${inputs.self}/common/optional/zfs"
     ./docker
     ./hardware.nix

systems/brain/docker/docker_networks.md

@@ -1,3 +0,0 @@
-# docker_networks
-
-docker network create -d bridge web

systems/jeeves/default.nix

@@ -9,11 +9,12 @@ in
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/docker.nix"
     "${inputs.self}/common/optional/monitoring-agent.nix"
+    "${inputs.self}/common/optional/signal-cli.nix"
     "${inputs.self}/common/optional/ssh_decrypt.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/x86-64-v3.nix"
+    "${inputs.self}/common/optional/x86-64-v3"
     "${inputs.self}/common/optional/zfs"
     ./monitoring
     ./docker

systems/jeeves/docker/signal-cli-rest-api.nix

@@ -1,18 +0,0 @@
-let
-  vars = import ../vars.nix;
-in
-{
-  virtualisation.oci-containers.containers.signal_cli_rest_api = {
-    image = "bbernhard/signal-cli-rest-api:0.199-dev";
-    ports = [
-      "8989:8080"
-    ];
-    volumes = [
-      "${vars.docker_configs}/signal-cli-config:/home/.local/share/signal-cli"
-    ];
-    environment = {
-      MODE = "json-rpc";
-    };
-    autoStart = true;
-  };
-}

systems/jeeves/monitoring/README.md

@@ -1,33 +0,0 @@
-# Monitoring
-
-## Vultr API metrics
-
-The `vultr-exporter` service reads its API token from:
-
-```text
-/zfs/storage/secrets/services/vultr-exporter
-```
-
-Create the file on Jeeves as root with the following contents:
-
-```text
-API_KEY=<Vultr API token>
-```
-
-The token needs read access to the Vultr Account and Billing APIs. Unrelated
-resource collectors are disabled in the packaged exporter.
-
-Restrict the file to root and ensure the public egress IP used by Jeeves is
-allowed for the token in the Vultr API settings:
-
-```console
-sudo chown root:root /zfs/storage/secrets/services/vultr-exporter
-sudo chmod 600 /zfs/storage/secrets/services/vultr-exporter
-```
-
-The exporter listens on `127.0.0.1:9188`; it is scraped by the local
-`prometheus-main` service every five minutes and is not exposed through the
-host firewall.
-
-Portal-1 exposes its node exporter only through `tailscale0` on port `9100`.
-Jeeves reaches it using the Portal-1 Tailscale hostname.

systems/jeeves/networking/firewall.nix

@@ -7,13 +7,13 @@ let
     gitea_ssh = 2223;
     grafana = 3000;
     jellyfin_http = 8096;
+    jellyswarrm = 3100;
     nix_binary_cache = 5000;
     nornsight = 8001;
     ollama = 11434;
     open_webui = 8080;
     postgresql = 5432;
     share = 8091;
-    signal_cli = 8989;
     syncthing = 8384;
   };
 in
@@ -28,6 +28,7 @@ in
       ports.audiobookshelf
       ports.gems
       ports.jellyfin_http
+      ports.jellyswarrm
       ports.nix_binary_cache
       ports.nornsight
       ports.share
@@ -40,7 +41,6 @@ in
       ports.ollama
       ports.open_webui
       ports.postgresql
-      ports.signal_cli
       ports.syncthing
     ];
   };

systems/jeeves/runners/default.nix

@@ -21,5 +21,16 @@
     nix-builder-12.enable = true;
     nix-builder-13.enable = true;
     nix-builder-14.enable = true;
+
+    # Warm the shared x86-64-v3 cache before the smaller per-system runners
+    # start. Eight jobs with eight cores each can use Jeeves' 64 logical CPUs,
+    # while the 6000% quota leaves some capacity for its normal services.
+    nix-cache-builder = {
+      enable = true;
+      labels = [ "nix-cache-builder:host" ];
+      cores = 8;
+      maxJobs = 8;
+      cpuQuota = "6000%";
+    };
   };
 }

systems/jeeves/runners/nix_builder.nix

@@ -11,11 +11,8 @@ let
   cfg = config.services.nix_builder;
   runnerUsername = "gitea-runner";
   runnerUserid = 601;
-  runnerLabels = [
-    "self-hosted:host"
-    "nixos:host"
-  ];
   containerConfig =
+    containerCfg:
     {
       config,
       pkgs,
@@ -50,8 +47,8 @@ let
         useHostResolvConf = false;
       };
       nix.settings = {
-        cores = 8;
-        max-jobs = 2;
+        inherit (containerCfg) cores;
+        max-jobs = containerCfg.maxJobs;
         system-features = lib.mkAfter [
           "gccarch-x86-64-v2"
           "gccarch-x86-64-v3"
@@ -94,7 +91,7 @@ let
         enable = true;
         name = "jeeves-nix-builder";
         url = "http://192.168.99.14:6443/";
-        labels = runnerLabels;
+        labels = containerCfg.labels;
         tokenFile = "/run/secrets/gitea-runners/registration-token";
         settings.runner.timeout = "12h";
         hostPackages = with pkgs; [
@@ -120,20 +117,21 @@ let
           User = mkForce runnerUsername;
           Group = mkForce runnerUsername;
           ExecStartPre = mkForce [
-            "${getExe registerRunner} builder http://192.168.99.14:6443/ ${runnerConfigFile} ${escapeShellArgs runnerLabels}"
+            "${getExe registerRunner} builder http://192.168.99.14:6443/ ${runnerConfigFile} ${escapeShellArgs containerCfg.labels}"
           ];
         };
       };
       system.stateVersion = "24.05";
     };
-  sharedContainerPath =
+  mkContainerPath =
+    containerCfg:
     (import "${pkgs.path}/nixos/lib/eval-config.nix" {
       modules = [
         {
           boot.isNspawnContainer = true;
           nixpkgs.pkgs = pkgs;
         }
-        containerConfig
+        (containerConfig containerCfg)
       ];
       system = null;
     }).config.system.build.toplevel;
@@ -151,7 +149,36 @@ in
         types.submodule (
           { name, ... }:
           {
-            options.enable = mkEnableOption "Gitea runner container";
+            options = {
+              enable = mkEnableOption "Gitea runner container";
+
+              labels = mkOption {
+                type = types.listOf types.str;
+                default = [
+                  "self-hosted:host"
+                  "nixos:host"
+                ];
+                description = "Gitea Actions labels advertised by this runner.";
+              };
+
+              cores = mkOption {
+                type = types.ints.positive;
+                default = 8;
+                description = "Number of cores made available to each Nix build job.";
+              };
+
+              maxJobs = mkOption {
+                type = types.ints.positive;
+                default = 2;
+                description = "Maximum number of Nix build jobs run in parallel.";
+              };
+
+              cpuQuota = mkOption {
+                type = types.str;
+                default = "800%";
+                description = "systemd CPU quota for the runner container.";
+              };
+            };
           }
         )
       );
@@ -173,7 +200,7 @@ in
     containers = mapAttrs (
       name: containerCfg:
       mkIf containerCfg.enable {
-        path = sharedContainerPath;
+        path = mkContainerPath containerCfg;
         autoStart = true;
         privateNetwork = true;
         hostBridge = cfg.bridgeName;
@@ -199,16 +226,14 @@ in
     ) cfg.containers;
 
     systemd = {
-      services = builtins.listToAttrs (
-        map (name: {
-          name = "container@${name}";
-          value = {
-            requires = [ "gitea.service" ];
-            after = [ "gitea.service" ];
-            serviceConfig.CPUQuota = "800%";
-          };
-        }) (builtins.attrNames (filterAttrs (_: c: c.enable) cfg.containers))
-      );
+      services = mapAttrs' (
+        name: containerCfg:
+        nameValuePair "container@${name}" {
+          requires = [ "gitea.service" ];
+          after = [ "gitea.service" ];
+          serviceConfig.CPUQuota = containerCfg.cpuQuota;
+        }
+      ) (filterAttrs (_: c: c.enable) cfg.containers);
 
       tmpfiles.rules = [
         "d ${vars.uv_cache} 0755 ${runnerUsername} ${runnerUsername} - -"

systems/jeeves/services/jellyswarrm.nix

@@ -0,0 +1,16 @@
+{ inputs, ... }:
+let
+  vars = import ../vars.nix;
+in
+{
+  imports = [ inputs.jellyswarrm.nixosModules.default ];
+
+  services.jellyswarrm = {
+    enable = true;
+    host = "0.0.0.0";
+    port = 3100;
+    dataDir = "${vars.services}/jellyswarrm";
+    username = "admin";
+    passwordFile = "${vars.secrets}/services/jellyswarrm-password";
+  };
+}

systems/portal-1/README.md

@@ -1,83 +0,0 @@
-# portal_1
-
-Minimal NixOS target for a Vultr VM, installed with nixos-anywhere. The Nix
-flake target is `portal_1`; the machine hostname is `portal-1` because DNS
-hostnames cannot contain underscores.
-
-## Before deploying
-
-1. Confirm the VM's system disk is `/dev/vda`. If it is not, update both
-   references in `disk-config.nix`.
-2. Confirm the SSH public key in `default.nix` is the key that should have
-   administrator access.
-3. Boot the VM into a NixOS installer or another nixos-anywhere-compatible
-   Linux rescue environment with root SSH access. Keep this environment
-   running while completing the SOPS bootstrap below.
-
-## Bootstrap SOPS
-
-Use the rescue environment's SSH host key as the permanent portal identity.
-Replace `VM_IP` below:
-
-```console
-ssh root@VM_IP 'cat /etc/ssh/ssh_host_ed25519_key.pub' | \
-  nix shell nixpkgs#ssh-to-age --command ssh-to-age
-```
-
-This prints an `age1...` recipient; it does not copy the private key. Add the
-recipient to `.sops.yaml`:
-
-```yaml
-- &system_portal_1 age1...
-```
-
-Then add `*system_portal_1` to the age recipients for
-`users/secrets.yaml`. Re-encrypt the existing file for the new recipient and
-add the Tailscale key:
-
-```console
-nix shell nixpkgs#sops --command sops updatekeys users/secrets.yaml
-nix shell nixpkgs#sops --command sops users/secrets.yaml
-```
-
-Add the OAuth client secret from the `Auth Keys: Write` credential in the SOPS
-editor and save it:
-
-```yaml
-tailscale_auth_key: tskey-client-...
-```
-
-## Deploy
-
-From the repository root, replace `VM_IP` with the VM's public IP:
-
-```console
-nix run github:nix-community/nixos-anywhere -- \
-  --copy-host-keys --flake .#portal_1 root@VM_IP
-```
-
-This repartitions `/dev/vda`, so anything already on that disk is erased. The
-layout reserves 8 GiB for swap and assigns the remaining space to the root
-filesystem.
-
-`--copy-host-keys` preserves the same private SSH host key at
-`/etc/ssh/ssh_host_ed25519_key` on the installed system. SOPS-Nix converts that
-key to an age identity during activation. After the reboot, connect as
-`richie` and verify that automatic Tailscale enrollment succeeded:
-
-```console
-ssh -p 278 richie@VM_IP
-sudo tailscale status
-```
-
-The installed OpenSSH service listens on port 278. Port 22 is served by
-Endlessh and will not provide an SSH login.
-
-HAProxy uses the same frontend, routing, and rate-limiting configuration as
-Jeeves. Portal manages the ACME certificates for the existing public domains;
-their DNS records must resolve to Portal for HTTP-01 issuance and renewal.
-
-The application backends still use Jeeves' original `127.0.0.1` addresses.
-Replace them with the corresponding Tailscale addresses before directing
-application traffic through Portal. Ports 80 and 443 are allowed through the
-firewall.

systems/portal-1/default.nix

@@ -9,8 +9,9 @@
     inputs.disko.nixosModules.disko
     "${inputs.self}/users/richie"
     "${inputs.self}/common/global"
+    "${inputs.self}/common/optional/signal-cli.nix"
     "${inputs.self}/common/optional/tailscale.nix"
-    "${inputs.self}/common/optional/x86-64-v3.nix"
+    "${inputs.self}/common/optional/x86-64-v3"
     ./disk-config.nix
     ./haproxy
     ./monitoring.nix

systems/rhapsody-in-green/default.nix

@@ -9,7 +9,6 @@
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/systemd-boot.nix"
     "${inputs.self}/common/optional/tailscale.nix"
-    "${inputs.self}/common/optional/x86-64-v3.nix"
     "${inputs.self}/common/optional/yubikey.nix"
     "${inputs.self}/common/optional/zfs"
     ./hardware.nix

tests/test_signal_alert.py

@@ -2,80 +2,68 @@
 
 from __future__ import annotations
 
-from os import environ
+import json
+import socket
 from typing import TYPE_CHECKING
 
-import httpx
+import pytest
 
-from python.signal_alert import signal_alert
+from python.signal_alert import SignalRPCError, signal_alert
 
 if TYPE_CHECKING:
     from pytest_mock import MockerFixture
 
 
 def test_signal_alert(mocker: MockerFixture) -> None:
-    """test_signal_alert."""
-    environ["SIGNAL_ALERT_FROM_PHONE"] = "1234567890"
-    environ["SIGNAL_ALERT_TO_PHONE"] = "0987654321"
-
     mock_logger = mocker.patch("python.signal_alert.logger")
-    mock_response = mocker.MagicMock(spec=httpx.Response, status_code=httpx.codes.CREATED)
-    mock_post = mocker.patch("python.signal_alert.httpx.post", return_value=mock_response)
+    mock_socket = mocker.patch("python.signal_alert.socket.socket")
+    client = mock_socket.return_value.__enter__.return_value
+    client.recv.side_effect = [b'{"jsonrpc":"2.0","result":{},"id":"signal-alert"}\n']
 
-    signal_alert("test")
+    assert signal_alert("test", "1234567890", "0987654321") is None
 
     mock_logger.info.assert_not_called()
-    mock_post.assert_called_once_with(
-        "http://localhost:8989/v2/send",
-        json={
+    mock_socket.assert_called_once_with(socket.AF_UNIX, socket.SOCK_STREAM)
+    client.settimeout.assert_called_once_with(4.0)
+    client.connect.assert_called_once_with("/run/signal-cli/socket")
+    request = json.loads(client.sendall.call_args.args[0])
+    assert request == {
+        "jsonrpc": "2.0",
+        "method": "send",
+        "params": {
+            "account": "1234567890",
+            "recipient": ["0987654321"],
             "message": "test",
-            "number": "1234567890",
-            "recipients": ["0987654321"],
-            "text_mode": "normal",
         },
-        timeout=4.0,
-        follow_redirects=True,
-    )
+        "id": "signal-alert",
+    }
 
 
-def test_signal_alert_no_phones(mocker: MockerFixture) -> None:
-    """test_signal_alert_no_phones."""
-    if "SIGNAL_ALERT_FROM_PHONE" in environ:
-        del environ["SIGNAL_ALERT_FROM_PHONE"]
-    if "SIGNAL_ALERT_TO_PHONE" in environ:
-        del environ["SIGNAL_ALERT_TO_PHONE"]
-    mock_logger = mocker.patch("python.signal_alert.logger")
-    signal_alert("test")
+def test_signal_alert_socket_error(mocker: MockerFixture) -> None:
+    mocker.patch("python.signal_alert.socket.socket", side_effect=ConnectionError("connection failed"))
 
-    mock_logger.info.assert_called_once_with("SIGNAL_ALERT_FROM_PHONE or SIGNAL_ALERT_TO_PHONE not set")
+    with pytest.raises(SignalRPCError, match="Signal RPC failed") as exc_info:
+        signal_alert("test", "1234567890", "0987654321")
+
+    assert isinstance(exc_info.value.__cause__, ConnectionError)
 
 
-def test_signal_alert_http_error(mocker: MockerFixture) -> None:
-    """HTTP errors are logged rather than propagated."""
-    environ["SIGNAL_ALERT_FROM_PHONE"] = "1234567890"
-    environ["SIGNAL_ALERT_TO_PHONE"] = "0987654321"
+def test_signal_alert_rpc_error(mocker: MockerFixture) -> None:
+    mock_socket = mocker.patch("python.signal_alert.socket.socket")
+    client = mock_socket.return_value.__enter__.return_value
+    client.recv.side_effect = [b'{"jsonrpc":"2.0","error":{"code":-1,"message":"failed"},"id":"signal-alert"}\n']
 
-    mock_logger = mocker.patch("python.signal_alert.logger")
-    request = httpx.Request("POST", "http://localhost:8989/v2/send")
-    mocker.patch(
-        "python.signal_alert.httpx.post",
-        side_effect=httpx.ConnectError("connection failed", request=request),
-    )
-
-    signal_alert("test")
-
-    mock_logger.exception.assert_called_once_with("Unable to contact the Signal API")
+    with pytest.raises(SignalRPCError, match=r"Signal RPC returned an error:.*failed"):
+        signal_alert("test", "1234567890", "0987654321")
 
 
-def test_signal_alert_unsuccessful_response(mocker: MockerFixture) -> None:
-    """Unexpected response statuses are logged."""
-    environ["SIGNAL_ALERT_FROM_PHONE"] = "1234567890"
-    environ["SIGNAL_ALERT_TO_PHONE"] = "0987654321"
+def test_signal_alert_ignores_notifications(mocker: MockerFixture) -> None:
+    mock_socket = mocker.patch("python.signal_alert.socket.socket")
+    client = mock_socket.return_value.__enter__.return_value
+    client.recv.side_effect = [
+        b'{"jsonrpc":"2.0","method":"receive"}\n{"jsonrpc":"2.0","result":{},"id":"signal-alert"}\n'
+    ]
 
-    mock_logger = mocker.patch("python.signal_alert.logger")
-    mock_response = mocker.MagicMock(spec=httpx.Response, status_code=httpx.codes.BAD_GATEWAY)
-    mocker.patch("python.signal_alert.httpx.post", return_value=mock_response)
+    assert signal_alert("test", "1234567890", "0987654321") is None
 
-    signal_alert("test")
-
-    mock_logger.error.assert_called_once_with("Signal API returned HTTP status %d", httpx.codes.BAD_GATEWAY)
+    assert client.recv.call_count == 1

tests/test_snapshot_manager.py

@@ -16,6 +16,17 @@ if TYPE_CHECKING:
     from pytest_mock import MockerFixture
 
 SNAPSHOT_MANAGER = "python.tools.snapshot_manager"
+FROM_PHONE = "1234567890"
+TO_PHONE = "0987654321"
+
+
+def patch_phone_numbers(mocker: MockerFixture) -> None:
+    """Patch the Signal phone number environment variables."""
+    phone_numbers = {
+        "SIGNAL_ALERT_FROM_PHONE": FROM_PHONE,
+        "SIGNAL_ALERT_TO_PHONE": TO_PHONE,
+    }
+    mocker.patch(f"{SNAPSHOT_MANAGER}.getenv", side_effect=phone_numbers.get)
 
 
 def patch_utcnow(mocker: MockerFixture, datetime_value: datetime) -> None:
@@ -34,6 +45,7 @@ def create_mock_snapshot(mocker: MockerFixture, name: str) -> Snapshot:
 def test_main(mocker: MockerFixture, fs: FakeFilesystem) -> None:
     """Test main."""
     load_config_data.cache_clear()
+    patch_phone_numbers(mocker)
 
     mocker.patch(f"{SNAPSHOT_MANAGER}.get_time_stamp", return_value="2023-01-01T00:00:00")
 
@@ -58,12 +70,15 @@ def test_main(mocker: MockerFixture, fs: FakeFilesystem) -> None:
             "daily": 0,
             "monthly": 0,
         },
+        FROM_PHONE,
+        TO_PHONE,
     )
 
 
 def test_main_create_snapshot_failure(mocker: MockerFixture, fs: FakeFilesystem) -> None:
     """Test main."""
     load_config_data.cache_clear()
+    patch_phone_numbers(mocker)
 
     mocker.patch(f"{SNAPSHOT_MANAGER}.get_time_stamp", return_value="2023-01-01T00:00:00")
 
@@ -78,7 +93,11 @@ def test_main_create_snapshot_failure(mocker: MockerFixture, fs: FakeFilesystem)
     fs.create_file("/mock_snapshot_config.toml", contents=mock_snapshot_config_toml)
     main(Path("/mock_snapshot_config.toml"))
 
-    mock_signal_alert.assert_called_once_with("test_dataset failed to create snapshot 2023-01-01T00:00:00")
+    mock_signal_alert.assert_called_once_with(
+        "test_dataset failed to create snapshot 2023-01-01T00:00:00",
+        FROM_PHONE,
+        TO_PHONE,
+    )
     mock_get_datasets.assert_called_once()
     mock_get_snapshots_to_delete.assert_not_called()
 
@@ -86,6 +105,7 @@ def test_main_create_snapshot_failure(mocker: MockerFixture, fs: FakeFilesystem)
 def test_main_exception(mocker: MockerFixture, fs: FakeFilesystem) -> None:
     """Test main."""
     load_config_data.cache_clear()
+    patch_phone_numbers(mocker)
 
     mocker.patch(f"{SNAPSHOT_MANAGER}.get_time_stamp", return_value="2023-01-01T00:00:00")
 
@@ -103,7 +123,7 @@ def test_main_exception(mocker: MockerFixture, fs: FakeFilesystem) -> None:
 
     assert isinstance(pytest_wrapped_e.value, SystemExit)
     assert pytest_wrapped_e.value.code == 1
-    mock_signal_alert.assert_called_once_with("snapshot_manager failed")
+    mock_signal_alert.assert_called_once_with("snapshot_manager failed", FROM_PHONE, TO_PHONE)
     mock_get_datasets.assert_called_once()
     mock_get_snapshots_to_delete.assert_not_called()
 
@@ -120,7 +140,12 @@ def test_get_snapshots_to_delete(mocker: MockerFixture) -> None:
 
     mock_signal_alert = mocker.patch(f"{SNAPSHOT_MANAGER}.signal_alert")
 
-    get_snapshots_to_delete(mock_dataset, {"15_min": 1, "hourly": 0, "daily": 0, "monthly": 0})
+    get_snapshots_to_delete(
+        mock_dataset,
+        {"15_min": 1, "hourly": 0, "daily": 0, "monthly": 0},
+        FROM_PHONE,
+        TO_PHONE,
+    )
 
     mock_signal_alert.assert_not_called()
     mock_dataset.delete_snapshot.assert_called_once_with("auto_202509150415")
@@ -135,7 +160,12 @@ def test_get_snapshots_to_delete_no_snapshot(mocker: MockerFixture) -> None:
 
     mock_signal_alert = mocker.patch(f"{SNAPSHOT_MANAGER}.signal_alert")
 
-    get_snapshots_to_delete(mock_dataset, {"15_min": 1, "hourly": 0, "daily": 0, "monthly": 0})
+    get_snapshots_to_delete(
+        mock_dataset,
+        {"15_min": 1, "hourly": 0, "daily": 0, "monthly": 0},
+        FROM_PHONE,
+        TO_PHONE,
+    )
 
     mock_signal_alert.assert_not_called()
     mock_dataset.delete_snapshot.assert_not_called()
@@ -153,10 +183,17 @@ def test_get_snapshots_to_delete_errored(mocker: MockerFixture) -> None:
 
     mock_signal_alert = mocker.patch(f"{SNAPSHOT_MANAGER}.signal_alert")
 
-    get_snapshots_to_delete(mock_dataset, {"15_min": 1, "hourly": 0, "daily": 0, "monthly": 0})
+    get_snapshots_to_delete(
+        mock_dataset,
+        {"15_min": 1, "hourly": 0, "daily": 0, "monthly": 0},
+        FROM_PHONE,
+        TO_PHONE,
+    )
 
     mock_signal_alert.assert_called_once_with(
-        "test_dataset@auto_202509150415 failed to delete: snapshot has dependent clones"
+        "test_dataset@auto_202509150415 failed to delete: snapshot has dependent clones",
+        FROM_PHONE,
+        TO_PHONE,
     )
     mock_dataset.delete_snapshot.assert_called_once_with("auto_202509150415")
 

users/richie/default.nix

@@ -41,6 +41,7 @@ in
         "ollama"
         "plugdev"
         "scanner"
+        "signal-cli"
         "transmission"
         "uaccess"
         "uucp"

users/richie/home/gui/default.nix

@@ -21,7 +21,6 @@
     vlc
     # browser
     brave
-    chromium
     # dev tools
     gparted
     jetbrains.datagrip