feat(haproxy): implement HAProxy log shipper with initial configuration and dependencies

- Introduced a new Rust-based service for ingesting HAProxy logs, including the necessary database schema and API endpoints.
- Implemented database interactions for managing source status, inserting events, and handling ingestion batches.
- Added validation for incoming log batches to ensure data integrity.
- Configured systemd service for the HAProxy ingestion server with appropriate security settings.
- Updated firewall rules to allow traffic on the ingestion server's port (8444).
- Created necessary PostgreSQL database and user configurations for the ingestion service.

.github/workflows/build_systems.yml

@@ -8,8 +8,23 @@ on:
     - cron: "0 22 * * *"
 
 jobs:
+  prebuild-common:
+    name: prebuild-common-x86-64-v3
+    runs-on: nix-cache-builder
+    steps:
+      - uses: actions/checkout@v4
+      # portal-1 is the smallest system closure: 95% of its derivations are
+      # shared by all five systems, so it is a maintainable common cache seed.
+      # Keep going so one failing package does not stop unrelated cache entries
+      # from being built.
+      - name: Build common packages
+        run: nixos-rebuild build --keep-going --accept-flake-config --flake ./#portal-1
+      - name: Copy common packages to nix-cache
+        run: nix copy --accept-flake-config --to unix:///host-nix/var/nix/daemon-socket/socket .#nixosConfigurations.portal-1.config.system.build.toplevel
+
   build:
     name: build-${{ matrix.system }}
+    needs: prebuild-common
     runs-on: self-hosted
     strategy:
       matrix:

.github/workflows/merge_flake_lock_update.yml

@@ -15,7 +15,6 @@ jobs:
     steps:
       - name: merge_flake_lock_update
         run: >-
-          nix develop .#devShells.x86_64-linux.default -c
           python -m python.gitea_flake_lock merge
           --repo "${{ github.repository }}"
         env:

.github/workflows/pytest.yml

@@ -1,19 +0,0 @@
-name: pytest
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
-
-jobs:
-  pytest:
-    runs-on: self-hosted
-
-    steps:
-      - uses: actions/checkout@v4
-      - name: Run tests
-        run: nix develop .#devShells.x86_64-linux.default -c pytest tests

.github/workflows/update-flake-lock.yml

@@ -21,6 +21,5 @@ jobs:
           JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
           GITEA_URL: https://gitea.tmmworkshop.com
         run: >-
-          nix develop .#devShells.x86_64-linux.default -c
           python -m python.gitea_flake_lock update
           --repo "${{ github.repository }}"

.sops.yaml

@@ -11,7 +11,7 @@ keys:
   - &system_brain age1jhf7vm0005j60mjq63696frrmjhpy8kpc2d66mw044lqap5mjv4snmwvwm # cspell:disable-line
   - &system_jeeves age13lmqgc3jvkyah5e3vcwmj4s5wsc2akctcga0lpc0x8v8du3fxprqp4ldkv # cspell:disable-line
   - &system_rhapsody age1ufnewppysaq2wwcl4ugngjz8pfzc5a35yg7luq0qmuqvctajcycs5lf6k4 # cspell:disable-line
-  - &system_portal_1 age1vyav6kxtvt3z4vtnkkjj38eu8hlts5m7ygyckhskvalg2gpjk52su53d0a # cspell:disable-line
+  - &system_portal_1 age1cm7ujz4t9cyc3nyy0w30l2addggk0gy8h3shd9yq47jg845tndnsa3utfl # cspell:disable-line
 
 creation_rules:
   - path_regex: users/secrets\.yaml$
@@ -23,3 +23,33 @@ creation_rules:
           - *system_jeeves
           - *system_rhapsody
           - *system_portal_1
+
+  - path_regex: systems/bob/secrets\.yaml$
+    key_groups:
+      - age:
+          - *admin_richie
+          - *system_bob
+
+  - path_regex: systems/brain/secrets\.yaml$
+    key_groups:
+      - age:
+          - *admin_richie
+          - *system_brain
+
+  - path_regex: systems/jeeves/secrets\.yaml$
+    key_groups:
+      - age:
+          - *admin_richie
+          - *system_jeeves
+
+  - path_regex: systems/rhapsody-in-green/secrets\.yaml$
+    key_groups:
+      - age:
+          - *admin_richie
+          - *system_rhapsody
+
+  - path_regex: systems/portal-1/secrets\.yaml$
+    key_groups:
+      - age:
+          - *admin_richie
+          - *system_portal_1

Cargo.toml

@@ -2,3 +2,66 @@
 resolver = "2"
 
 members = ["rust/*"]
+
+[workspace.dependencies]
+anyhow = "1.0"
+axum = "0.8"
+bytes = "1.10"
+chrono = "0.4"
+clap = { version = "4.5", features = ["derive", "env"] }
+env_logger = "0.11"
+garde = { version = "0.23", features = ["derive", "serde"] }
+hex = "0.4"
+log = "0.4"
+prost = "0.14"
+prost-build = "0.14"
+protoc-bin-vendored = "3.2"
+# HTTP clients use the shared tls crate to install AWS-LC before building a reqwest client.
+reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-webpki-roots-no-provider"] }
+rustls = { version = "0.23", default-features = false, features = ["aws_lc_rs", "std", "tls12"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+sha2 = "0.10"
+signal = { path = "rust/signal" }
+sqlx = { version = "0.9", default-features = false, features = [
+    "chrono",
+    "postgres",
+    "runtime-tokio",
+    "tls-none",
+] }
+subtle = "2.6"
+systemd = { version = "0.10", features = ["journal"] }
+thiserror = "2.0"
+tls = { path = "rust/tls" }
+tokio = { version = "1.47", features = [
+    "macros",
+    "net",
+    "rt-multi-thread",
+    "signal",
+    "time",
+] }
+tokio-retry = "0.3"
+tower-http = { version = "0.6", features = ["limit", "trace"] }
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
+zfs = { path = "rust/zfs" }
+zstd = "0.13"
+
+[workspace.lints.clippy]
+all = { level = "deny", priority = -1 }
+pedantic = { level = "deny", priority = -1 } # UM, ACTUALLY
+nursery = { level = "deny", priority = -1 }  # BETA LINTS
+# DENY PANICS
+unwrap_used = "deny"
+expect_used = "deny"
+indexing_slicing = "deny"
+arithmetic_side_effects = "deny"
+unreachable = "deny"
+unimplemented = "deny"
+unchecked_time_subtraction = "deny"
+todo = "deny"
+string_slice = "deny"
+panic_in_result_fn = "deny"
+panic = "deny"
+exit = "deny"
+as_conversions = "deny"

clippy.toml

@@ -0,0 +1,4 @@
+allow-unwrap-in-tests = true
+allow-expect-in-tests = true
+allow-panic-in-tests = true
+allow-indexing-slicing-in-tests = true

common/global/default.nix

@@ -21,7 +21,7 @@
 
   boot = {
     tmp.useTmpfs = lib.mkDefault true;
-    kernelPackages = lib.mkDefault pkgs.linuxPackages_6_12;
+    kernelPackages = lib.mkDefault pkgs.linuxPackages_6_18;
   };
 
   hardware.enableRedistributableFirmware = true;

common/global/programs.nix

@@ -2,6 +2,5 @@
 {
   environment.systemPackages = with pkgs; [
     git
-    my_python
   ];
 }

common/optional/desktop.nix

@@ -1,42 +0,0 @@
-{ pkgs, ... }:
-{
-  boot = {
-    kernelPackages = pkgs.linuxPackages_6_18;
-    zfs.package = pkgs.zfs_2_4;
-  };
-
-  hardware.bluetooth = {
-    enable = true;
-    powerOnBoot = true;
-  };
-
-  # rtkit is optional but recommended for pipewire
-  security.rtkit.enable = true;
-
-  services = {
-    displayManager.sddm = {
-      enable = true;
-      wayland.enable = true;
-    };
-
-    desktopManager.plasma6.enable = true;
-
-    xserver = {
-      enable = true;
-      xkb = {
-        layout = "us";
-        variant = "";
-      };
-    };
-
-    pulseaudio.enable = false;
-
-    pipewire = {
-      enable = true;
-      alsa.enable = true;
-      alsa.support32Bit = true;
-      pulse.enable = true;
-      wireplumber.enable = true;
-    };
-  };
-}

common/optional/desktop/default.nix

@@ -0,0 +1,17 @@
+{ pkgs, ... }:
+{
+  imports = [
+    ./kde.nix
+    ./pipewire.nix
+  ];
+
+  boot = {
+    kernelPackages = pkgs.linuxPackages_6_18;
+    zfs.package = pkgs.zfs_2_4;
+  };
+
+  hardware.bluetooth = {
+    enable = true;
+    powerOnBoot = true;
+  };
+}

common/optional/desktop/kde.nix

@@ -0,0 +1,45 @@
+{ pkgs, ... }:
+{
+  environment.plasma6.excludePackages = with pkgs.kdePackages; [
+    aurorae # theme
+    elisa # music player
+    kate # text editor
+    kconfig # config editor
+    khelpcenter # help center
+    kinfocenter # system info
+    konsole # terminal
+    krdp # remote desktop
+    ktexteditor # thing for kate
+    okular # pdf reader
+    plasma-keyboard # used by plasma-keyboard KCM
+    plasma-sdk # plasma development tools
+    plasma-workspace-wallpapers # wallpapers
+    qrca # qr code scanner
+    qtvirtualkeyboard # virtual keyboard
+    union # theme
+  ];
+
+  # disable KDE PIM (Personal Information Management) applications
+  programs.kde-pim.enable = false;
+
+  services = {
+    # removes screen reader and speech dispatcher from the system
+    orca.enable = false;
+    speechd.enable = false;
+
+    displayManager.sddm = {
+      enable = true;
+      wayland.enable = true;
+    };
+
+    desktopManager.plasma6.enable = true;
+
+    xserver = {
+      enable = true;
+      xkb = {
+        layout = "us";
+        variant = "";
+      };
+    };
+  };
+}

common/optional/desktop/pipewire.nix

@@ -0,0 +1,16 @@
+{
+  # rtkit is optional but recommended for pipewire
+  security.rtkit.enable = true;
+
+  services = {
+    pulseaudio.enable = false;
+
+    pipewire = {
+      enable = true;
+      alsa.enable = true;
+      alsa.support32Bit = true;
+      pulse.enable = true;
+      wireplumber.enable = true;
+    };
+  };
+}

common/optional/signal-cli.nix

@@ -0,0 +1,46 @@
+{ pkgs, ... }:
+{
+  environment.systemPackages = [
+    pkgs.signal-cli
+  ];
+
+  users = {
+    groups.signal-cli = { };
+    users.signal-cli = {
+      isSystemUser = true;
+      group = "signal-cli";
+      home = "/var/lib/signal-cli";
+    };
+  };
+
+  systemd.tmpfiles.rules = [
+    "d /var/lib/signal-cli 0700 signal-cli signal-cli - -"
+    "Z /var/lib/signal-cli - signal-cli signal-cli - -"
+  ];
+
+  systemd.services.signal-cli = {
+    description = "Signal CLI JSON-RPC service";
+    after = [ "network-online.target" ];
+    wants = [ "network-online.target" ];
+    wantedBy = [ "multi-user.target" ];
+    unitConfig.RequiresMountsFor = [ "/var/lib/signal-cli" ];
+
+    serviceConfig = {
+      Type = "simple";
+      User = "signal-cli";
+      Group = "signal-cli";
+      ExecStart = "${pkgs.signal-cli}/bin/signal-cli --data-dir /var/lib/signal-cli daemon --socket /run/signal-cli/socket";
+      Restart = "on-failure";
+      RestartSec = "5s";
+      SuccessExitStatus = 143;
+      RuntimeDirectory = "signal-cli";
+      RuntimeDirectoryMode = "0750";
+      UMask = "0007";
+      NoNewPrivileges = true;
+      PrivateTmp = true;
+      ProtectHome = true;
+      ProtectSystem = "strict";
+      ReadWritePaths = [ "/var/lib/signal-cli" ];
+    };
+  };
+}

common/optional/steam.nix

@@ -1,7 +1,6 @@
 { pkgs, ... }:
 {
   environment.systemPackages = with pkgs; [
-    mangohud
     steam-run
   ];
   hardware.steam-hardware.enable = true;

common/optional/x86-64-v3/default.nix

@@ -0,0 +1,15 @@
+{
+  nixpkgs = {
+    hostPlatform = {
+      system = "x86_64-linux";
+      gcc = {
+        arch = "x86-64-v3";
+        tune = "generic";
+      };
+    };
+
+    # These patches repair tests and bundled dependencies that are sensitive
+    # to the compiler flags used by the x86-64-v3 package set.
+    overlays = [ (import ./patches) ];
+  };
+}

common/optional/x86-64-v3/patches/abseil/bmi2-public-header.patch

@@ -0,0 +1,20 @@
+Subject: [PATCH] abseil: include BMI2 intrinsics through the public header
+
+GCC and Clang reject direct inclusion of bmi2intrin.h. Include immintrin.h
+instead so that the compiler supplies the required intrinsic setup when
+BMI2 is enabled, including builds targeting x86-64-v3.
+
+This patch is shared by the vendored Abseil copies in Electron, rusty_v8
+(Deno), and Signal's WebRTC build.
+
+--- a/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
++++ b/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
+@@ -226,7 +226,7 @@
+ #endif
+ 
+ #ifdef __BMI2__
+-#include <bmi2intrin.h>
++#include <immintrin.h>
+ #endif  // __BMI2__
+ 
+ namespace absl {

common/optional/x86-64-v3/patches/abseil/default.nix

@@ -0,0 +1,38 @@
+# Abseil accepted the upstream fix: https://github.com/abseil/abseil-cpp/pull/2071
+# Keep this workaround until Electron, Deno's rusty_v8, and Signal's WebRTC
+# update their bundled Abseil copies to include it.
+{ prev }:
+let
+  patchAbseilBmi2Include =
+    package:
+    package.overrideAttrs (old: {
+      # GCC and Clang require the public umbrella header for BMI2 intrinsics.
+      patches = (old.patches or [ ]) ++ [ ./bmi2-public-header.patch ];
+    });
+
+  electron43Unwrapped = patchAbseilBmi2Include prev.electron_43.unwrapped;
+  electron43 = prev.electron_43.override {
+    electron-unwrapped = electron43Unwrapped;
+  };
+
+  signalCallPackage =
+    path: args:
+    let
+      package = prev.callPackage path args;
+    in
+    if builtins.baseNameOf path == "webrtc.nix" then patchAbseilBmi2Include package else package;
+in
+prev.lib.optionalAttrs ((prev.stdenv.hostPlatform.gcc.arch or null) == "x86-64-v3") {
+  deno =
+    let
+      librusty_v8 = patchAbseilBmi2Include prev.deno.passthru.librusty_v8;
+    in
+    prev.deno.override { inherit librusty_v8; };
+
+  electron_43 = electron43;
+
+  signal-desktop = prev.signal-desktop.override {
+    electron_43 = electron43;
+    callPackage = signalCallPackage;
+  };
+}

common/optional/x86-64-v3/patches/afdko/check-overlap-font-content.patch

@@ -0,0 +1,19 @@
+Subject: [PATCH] Compare font content in the overlap-removal regression
+
+PFA_SKIP ends with a separator, producing an empty prefix that matches every
+line. Remove the empty prefix for this test so glyph differences are checked,
+and assert that tx exits successfully before inspecting the output.
+
+--- a/tests/tx_test.py
++++ b/tests/tx_test.py
+@@ -1064,6 +1064,8 @@
+     output_path = get_temp_file_path()
+     args = [TOOL, '-t1', '+V', '-o', output_path, input_path]
+-    subprocess.call(args)
+-    assert differ([expected_path, output_path, '-s', PFA_SKIP[0]])
++    subprocess.check_call(args)
++    # An empty skip prefix matches every line and hides font differences.
++    skip_headers = SPLIT_MARKER.join(filter(None, PFA_SKIP[0].split(SPLIT_MARKER)))
++    assert differ([expected_path, output_path, '-s', skip_headers])
+ 
+ 

common/optional/x86-64-v3/patches/afdko/default.nix

@@ -0,0 +1,10 @@
+{ afdko }:
+afdko.overridePythonAttrs (old: {
+  # FMA changes overlap-removal coordinates by 0.01 units on x86-64-v3.
+  # Separate multiply/add rounding reproduces the reference Type 1 font exactly.
+  env = (old.env or { }) // {
+    NIX_CFLAGS_COMPILE = (old.env.NIX_CFLAGS_COMPILE or "") + " -ffp-contract=off";
+  };
+
+  patches = (old.patches or [ ]) ++ [ ./check-overlap-font-content.patch ];
+})

common/optional/x86-64-v3/patches/default.nix

@@ -0,0 +1,16 @@
+_final: prev:
+(import ./abseil { inherit prev; })
+// {
+  gnutls = import ./gnutls { inherit (prev) gnutls; };
+  prometheus = import ./prometheus { inherit (prev) prometheus; };
+  zopfli = import ./zopfli { inherit (prev) zopfli; };
+
+  pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
+    (_pythonFinal: pythonPrev: {
+      afdko = import ./afdko { inherit (pythonPrev) afdko; };
+      psutil = import ./psutil { inherit (pythonPrev) psutil; };
+      pytest-xdist = import ./pytest-xdist { inherit (pythonPrev) pytest-xdist; };
+      scipy = import ./scipy { inherit (pythonPrev) scipy; };
+    })
+  ];
+}

common/optional/x86-64-v3/patches/gnutls/default.nix

@@ -0,0 +1,6 @@
+{ gnutls }:
+gnutls.overrideAttrs (old: {
+  # Keep the UDP handshake test enabled on loaded builders by waiting for
+  # the server to bind its socket. Kept as a standalone patch for upstream.
+  patches = (old.patches or [ ]) ++ [ ./udp-server-readiness.patch ];
+})

common/optional/x86-64-v3/patches/gnutls/udp-server-readiness.patch

@@ -0,0 +1,70 @@
+Subject: [PATCH] tests: wait for the UDP server socket before connecting
+
+A fixed four-second sleep does not guarantee that gnutls-serv has bound
+its UDP socket on a busy builder. Poll the local IPv4 UDP endpoint using
+the existing ss/netstat discovery, with the same retry budget as the TCP
+helper. Fail early if the server exits, and retain the original handshake
+checks in serv-udp.sh.
+
+Use flags common to ss and BSD/Linux netstat. Match the local endpoint
+and complete port number, excluding TCP, IPv6-only and peer endpoints.
+
+--- a/tests/scripts/common.sh
++++ b/tests/scripts/common.sh
+@@ -185,10 +185,55 @@
+ 	fi
+ }
+ 
++check_if_udp_port_bound() {
++	local PORT=$1
++	have_port_finder
++	# Use only -an, which is shared by ss and BSD/Linux netstat. UDP has
++	# no LISTEN state. Match the local IPv4 endpoint, not a peer port or
++	# a longer port number. serv-udp.sh connects to 127.0.0.1;
++	# listen_socket() in serv.c binds IPv4 separately and requests
++	# IPV6_V6ONLY=1 for its IPv6 socket.
++	$PFCMD -an | awk -v port="$PORT" '
++		$1 == "udp" || $1 == "udp4" {
++			# ss includes a state column; netstat does not.
++			address = ($2 == "UNCONN" || $2 == "ESTAB") ? $5 : $4
++			if (address ~ ("^[0-9.]+[.:]" port "$") ||
++			    address == "*." port)
++				found = 1
++		}
++		END { exit !found }
++	'
++}
++
+ wait_udp_server() {
+ 	local PID=$1
++	local ret
+ 	trap "test -n \"${PID}\" && kill ${PID};exit 1" 1 15 2
+-	sleep 4
++	local i=0
++	# Use the same retry budget as wait_for_port(), but also stop if the
++	# server exits before binding its socket.
++	while test $i -lt 90; do
++		if ! kill -0 "$PID" 2>/dev/null; then
++			fail "" "UDP server $PID exited before binding port $PORT"
++		fi
++		# Contain have_port_finder's exit so a skip also stops the server.
++		if (check_if_udp_port_bound "$PORT"); then
++			return 0
++		else
++			ret=$?
++			if test "$ret" = 77; then
++				kill "$PID" 2>/dev/null || :
++				wait "$PID" 2>/dev/null || :
++				exit 77
++			fi
++		fi
++		i=$((i + 1))
++		if test $i -lt 90; then
++			echo "try $i: waiting for UDP port $PORT"
++			sleep 2
++		fi
++	done
++	fail "$PID" "UDP server $PORT did not come up"
+ }
+ 
+ create_testdir() {

common/optional/x86-64-v3/patches/gnutls/verify-readiness.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""Exercise patched common.sh without building GnuTLS (Python standard library only).
+
+Usage: python3 verify-readiness.py /path/to/patched/tests/scripts/common.sh
+Set SHELL to test another shell, and NETSTAT to test a netstat outside PATH.
+"""
+
+# Use unittest so this upstream companion tool needs no pytest installation.
+# ruff: noqa: PT009
+
+import os
+import shutil
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+from pathlib import Path
+
+COMMON = str(Path(sys.argv.pop(1)).resolve())
+SHELL = os.environ.get("SHELL", "/bin/sh")
+
+
+class ReadinessTests(unittest.TestCase):
+    """Check endpoint parsing and the server startup lifecycle."""
+
+    def setUp(self) -> None:
+        """Create a socket-listing fixture for each check."""
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.root = Path(self.tmp.name)
+        self.fixture = self.root / "sockets"
+        self.fixture.write_text("")
+        self.finder = self.root / "port-finder"
+        self.finder.write_text('#!/bin/sh\ncat "$SOCKET_FIXTURE"\n')
+        self.finder.chmod(0o755)
+
+    def run_shell(self, body: str, **env: str) -> subprocess.CompletedProcess[str]:
+        """Source the actual helper and run a shell scenario."""
+        return subprocess.run(
+            [SHELL, "-c", '. "$COMMON"\n' + body],
+            env={
+                **os.environ,
+                "COMMON": COMMON,
+                "SOCKET_FIXTURE": str(self.fixture),
+                "PFCMD": str(self.finder),
+                "PORT": "12345",
+                **env,
+            },
+            capture_output=True,
+            text=True,
+            timeout=20,
+            check=False,
+        )
+
+    def test_socket_formats_and_false_matches(self) -> None:
+        """Accept IPv4 UDP local endpoints and reject unrelated sockets."""
+        cases = [
+            ("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*", True),
+            ("udp UNCONN 0 0 127.0.0.1:12345 0.0.0.0:*", True),
+            ("udp 0 0 0.0.0.0:12345 0.0.0.0:*", True),
+            ("udp4 0 0 *.12345 *.*", True),
+            ("udp 0 0 127.0.0.1.12345 *.*", True),
+            ("udp 0 0 *.12345 *.*", True),
+            ("udp UNCONN 0 0 0.0.0.0:123456 0.0.0.0:*", False),
+            ("udp 0 0 0.0.0.0:123456 0.0.0.0:*", False),
+            ("udp ESTAB 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
+            ("udp 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
+            ("tcp LISTEN 0 128 0.0.0.0:12345 0.0.0.0:*", False),
+            ("tcp 0 0 0.0.0.0:12345 0.0.0.0:* LISTEN", False),
+            ("udp UNCONN 0 0 [::]:12345 [::]:*", False),
+            ("udp UNCONN 0 0 *:12345 *:*", False),
+            ("udp6 0 0 :::12345 :::*", False),
+            ("udp6 0 0 *.12345 *.*", False),
+            ("", False),
+        ]
+        for row, ready in cases:
+            with self.subTest(row=row):
+                self.fixture.write_text(row + "\n")
+                result = self.run_shell('check_if_udp_port_bound "$PORT"')
+                self.assertEqual(result.returncode, 0 if ready else 1, result.stderr)
+
+    def test_exited_server_fails_immediately(self) -> None:
+        """Fail without sleeping when the server has already exited."""
+        result = self.run_shell(
+            'true &\npid=$!\nwait "$pid"\nsleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$pid"'
+        )
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("exited before binding", result.stderr)
+        self.assertNotIn("unexpected sleep", result.stderr)
+
+    def test_ready_socket_does_not_sleep(self) -> None:
+        """Check readiness before the first sleep."""
+        self.fixture.write_text("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*\n")
+        result = self.run_shell('sleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$$"')
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertNotIn("unexpected sleep", result.stderr)
+
+    def test_missing_port_finders_skip(self) -> None:
+        """Skip and stop the live server when no finder is available."""
+        # have_port_finder also tries these paths independently of PATH.
+        if any(os.access(f"{directory}/ss", os.X_OK) for directory in ("/sbin", "/usr/sbin", "/usr/local/sbin")):
+            self.skipTest("an absolute ss path cannot be hidden by this PATH-only fixture")
+        with subprocess.Popen(["sleep", "60"]) as server:
+            try:
+                result = self.run_shell(
+                    'unset PFCMD\nPATH=/nonexistent\nwait_udp_server "$SERVER_PID"',
+                    SERVER_PID=str(server.pid),
+                )
+                self.assertEqual(result.returncode, 77)
+                self.assertIn("neither ss nor netstat found", result.stderr)
+                server.wait(timeout=3)
+                self.assertLess(server.returncode, 0)
+            finally:
+                if server.poll() is None:
+                    server.kill()
+
+    def test_timeout_is_bounded_and_cleans_up(self) -> None:
+        """Stop polling after the retry budget and terminate the server."""
+        # Only accelerate the polling delay; keep a real live server process.
+        self.finder.write_text('#!/bin/sh\necho probe >&2\ncat "$SOCKET_FIXTURE"\n')
+        with subprocess.Popen(["sleep", "60"]) as server:
+            try:
+                result = self.run_shell(
+                    'sleep() { echo polling-sleep; }\nwait_udp_server "$SERVER_PID"',
+                    SERVER_PID=str(server.pid),
+                )
+                self.assertEqual(result.returncode, 1)
+                self.assertIn("did not come up", result.stderr)
+                self.assertEqual(result.stderr.count("probe\n"), 90)
+                self.assertEqual(result.stdout.count("polling-sleep"), 89)
+                server.wait(timeout=3)
+                self.assertLess(server.returncode, 0)
+            finally:
+                if server.poll() is None:
+                    server.kill()
+
+    def test_server_exits_while_waiting(self) -> None:
+        """Detect a startup failure that happens after polling begins."""
+        result = self.run_shell('sleep 1 &\npid=$!\nwait_udp_server "$pid"')
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("exited before binding", result.stderr)
+        self.assertIn("waiting for UDP port", result.stdout)
+
+    def test_real_socket_delayed_beyond_four_seconds(self) -> None:
+        """Wait for a real delayed bind with each installed port finder."""
+        finders = [shutil.which("ss"), os.environ.get("NETSTAT") or shutil.which("netstat")]
+        finders = [finder for finder in finders if finder]
+        if not finders:
+            self.skipTest("neither ss nor netstat available")
+        for finder in finders:
+            with self.subTest(finder=finder):
+                with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
+                    sock.bind(("127.0.0.1", 0))
+                    port = sock.getsockname()[1]
+                code = (
+                    "import socket,time,sys; time.sleep(6); "
+                    "s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); "
+                    "s.bind(('127.0.0.1',int(sys.argv[1]))); time.sleep(30)"
+                )
+                with subprocess.Popen([sys.executable, "-c", code, str(port)]) as server:
+                    try:
+                        started = time.monotonic()
+                        result = self.run_shell(
+                            'wait_udp_server "$SERVER_PID"',
+                            SERVER_PID=str(server.pid),
+                            PORT=str(port),
+                            PFCMD=finder,
+                        )
+                        self.assertEqual(result.returncode, 0, result.stderr)
+                        self.assertGreaterEqual(time.monotonic() - started, 6)
+                        self.assertIsNone(server.poll())
+                    finally:
+                        server.terminate()
+                        server.wait(timeout=3)
+
+
+if __name__ == "__main__":
+    unittest.main()

common/optional/x86-64-v3/patches/prometheus/complete-test-parsing.patch

@@ -0,0 +1,36 @@
+Subject: [PATCH] tests: finish parsing before inspecting editor state
+
+EditorState creation has a 20 ms parsing budget. A descheduled test can
+therefore observe an incomplete tree. Finish these small test documents
+without an interactive deadline and publish the result with a transaction.
+Keep the original completion and vector-matching assertions enabled.
+
+--- a/module/codemirror-promql/src/test/utils-test.ts
++++ b/module/codemirror-promql/src/test/utils-test.ts
+@@ -13,7 +13,7 @@
+ 
+ import { parser } from '@prometheus-io/lezer-promql';
+ import { EditorState } from '@codemirror/state';
+-import { LRLanguage } from '@codemirror/language';
++import { ensureSyntaxTree, LRLanguage } from '@codemirror/language';
+ import nock from 'nock';
+ import path from 'path';
+ import { fileURLToPath } from 'url';
+@@ -23,10 +23,16 @@
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
+ 
+ export function createEditorState(expr: string): EditorState {
+-  return EditorState.create({
++  const state = EditorState.create({
+     doc: expr,
+     extensions: lightPromQLSyntax,
+   });
++  // These tests need a complete tree, independent of the editor's time budget.
++  if (!ensureSyntaxTree(state, state.doc.length, Infinity)) {
++    throw new Error('Unable to parse the test expression');
++  }
++  // Publish the completed parse so syntaxTree(state) sees it too.
++  return state.update({}).state;
+ }
+ 
+ export function mockPrometheusServer(): void {

common/optional/x86-64-v3/patches/prometheus/default.nix

@@ -0,0 +1,17 @@
+{ prometheus }:
+prometheus.overrideAttrs (
+  old:
+  let
+    assets = old.passthru.assets.overrideAttrs (assetsOld: {
+      patches = (assetsOld.patches or [ ]) ++ [ ./complete-test-parsing.patch ];
+    });
+  in
+  {
+    postPatch = builtins.replaceStrings [ "${old.passthru.assets}" ] [ "${assets}" ] (
+      builtins.unsafeDiscardStringContext old.postPatch
+    );
+    passthru = old.passthru // {
+      inherit assets;
+    };
+  }
+)

common/optional/x86-64-v3/patches/psutil/default.nix

@@ -0,0 +1,4 @@
+{ psutil }:
+psutil.overridePythonAttrs (old: {
+  patches = (old.patches or [ ]) ++ [ ./heap-info-zero-mmap.patch ];
+})

common/optional/x86-64-v3/patches/psutil/heap-info-zero-mmap.patch

@@ -0,0 +1,17 @@
+Subject: [PATCH] Allow an empty mmap allocation total in heap_info
+
+The allocator may satisfy all live allocations from the heap. A zero
+mmap_used value is valid and depends on the worker allocation history.
+Keep the heap and platform checks without requiring an mmap allocation.
+
+--- a/tests/test_system.py
++++ b/tests/test_system.py
+@@ -267,6 +267,7 @@
+         if MACOS:
+             assert m.mmap_used == 0  # not supported
+         else:
+-            assert m.mmap_used > 0
++            # A process can have no live mmap-backed malloc allocations.
++            assert m.mmap_used >= 0
+         if WINDOWS:
+             assert m.heap_count >= 0

common/optional/x86-64-v3/patches/pytest-xdist/concurrent-worker-crashes.patch

@@ -0,0 +1,29 @@
+Subject: [PATCH] tests: count replacements when checking the worker restart limit
+
+With two workers, another in-flight test may crash after the fourth
+crash requests shutdown. Either four or five failed tests is valid.
+Require exactly three replacements and the failed-tests exit status,
+while preserving the queued-work and no-internal-error assertions.
+
+--- a/testing/acceptance_test.py
++++ b/testing/acceptance_test.py
+@@ -1011,9 +1011,18 @@
+                 "worker*crashed while running*",
+                 "worker*crashed while running*",
+                 "* xdist: maximum crashed workers reached: 3 *",
+-                "* 4 failed in *",
+             ]
+         )
++        # A second in-flight test may crash after shutdown is requested.
++        # The restart limit constrains replacements, not concurrent failures.
++        replacements = sum(
++            line.startswith("replacing crashed worker ") for line in res.stdout.lines
++        )
++        assert replacements == 3
++        failed = res.parseoutcomes()["failed"]
++        assert failed in (4, 5)
++        res.assert_outcomes(failed=failed)
++        assert res.ret == pytest.ExitCode.TESTS_FAILED
+         assert "INTERNALERROR" not in res.stdout.str()
+ 
+     def test_max_worker_restart_die(self, pytester: pytest.Pytester) -> None:

common/optional/x86-64-v3/patches/pytest-xdist/default.nix

@@ -0,0 +1,12 @@
+{ pytest-xdist }:
+pytest-xdist.overridePythonAttrs (old: {
+  patches = (old.patches or [ ]) ++ [
+    ./concurrent-worker-crashes.patch
+    ./worker-startup-timeout.patch
+  ];
+
+  # The suite exercises its own worker pools. Limit the outer suite to one worker.
+  preCheck = builtins.replaceStrings [ "--numprocesses=$NIX_BUILD_CORES" ] [ "--numprocesses=1" ] (
+    old.preCheck or ""
+  );
+})

common/optional/x86-64-v3/patches/pytest-xdist/worker-startup-timeout.patch

@@ -0,0 +1,19 @@
+Subject: [PATCH] tests: allow more time for remote worker events
+
+Worker startup can exceed ten seconds on heavily loaded builders. Allow
+the remote-test helper to wait up to sixty seconds for worker events.
+The wait still returns as soon as an event arrives and remains bounded.
+Production worker timeouts and test assertions are unchanged.
+
+--- a/testing/test_remote.py
++++ b/testing/test_remote.py
+@@ -17,7 +17,8 @@
+ from xdist.workermanage import WorkerController
+ 
+ 
+-WAIT_TIMEOUT = 10.0
++# Allow worker events extra time on heavily loaded builders.
++WAIT_TIMEOUT = 60.0
+ 
+ 
+ def check_marshallable(d: object) -> None:

common/optional/x86-64-v3/patches/scipy/default.nix

@@ -0,0 +1,5 @@
+{ scipy }:
+scipy.overridePythonAttrs (old: {
+  # Keep the STFT tests enabled with tolerances for x86-64-v3 rounding.
+  patches = (old.patches or [ ]) ++ [ ./stft-test-tolerances.patch ];
+})

common/optional/x86-64-v3/patches/scipy/stft-test-tolerances.patch

@@ -0,0 +1,51 @@
+Subject: [PATCH] signal: allow floating-point rounding in STFT tests
+
+Keep the STFT tests enabled for x86-64-v3 builds. Allow two float32
+epsilons of relative error when comparing inverse-STFT implementations;
+float64 and the existing i686 override remain unchanged. Allow one
+float64 epsilon of absolute error in all three scaling round trips,
+which otherwise require exact zeros (observed residual: 4e-17 for a
+signal with amplitude 2).
+
+Upstream issue: https://github.com/scipy/scipy/issues/25488
+
+--- a/scipy/signal/tests/_scipy_spectral_test_shim.py
++++ b/scipy/signal/tests/_scipy_spectral_test_shim.py
+@@ -294,7 +294,7 @@
+ 
+     # Adapted tolerances to account for resolution loss:
+     atol = np.finfo(x.dtype).resolution*2  # instead of default atol = 0
+-    rtol = 1e-7  # default for np.allclose()
++    rtol = max(1e-7, 2 * np.finfo(x.dtype).eps)
+ 
+     # Relax atol on 32-Bit platforms a bit to pass CI tests.
+     #  - Not clear why there are discrepancies (in the FFT maybe?)
+--- a/scipy/signal/tests/test_spectral.py
++++ b/scipy/signal/tests/test_spectral.py
+@@ -2044,7 +2044,7 @@
+ 
+         # Test round trip:
+         x1 = istft(Zs, boundary=True, scaling='spectrum')[1]
+-        assert_allclose(x1, x)
++        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
+ 
+         # For a Hann-windowed 256 sample length FFT, we expect a peak at
+         # frequency 64 (since it is 1/4 the length of X) with a height of 1
+@@ -2074,7 +2074,7 @@
+ 
+         # Test round trip:
+         x1 = istft(Zp, input_onesided=False, boundary=True, scaling='psd')[1]
+-        assert_allclose(x1, x)
++        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
+ 
+         # The power of the one-sided psd-scaled STFT can be determined
+         # analogously (note that the two sides are not of equal shape):
+@@ -2094,7 +2094,7 @@
+ 
+         # Test round trip:
+         x1 = istft(Zp0, input_onesided=True, boundary=True, scaling='psd')[1]
+-        assert_allclose(x1, x)
++        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
+ 
+ 
+ class TestSampledSpectralRepresentations:

common/optional/x86-64-v3/patches/zopfli/default.nix

@@ -0,0 +1,4 @@
+{ zopfli }:
+zopfli.overrideAttrs (old: {
+  patches = (old.patches or [ ]) ++ [ ./unaligned-match-loads.patch ];
+})

common/optional/x86-64-v3/patches/zopfli/unaligned-match-loads.patch

@@ -0,0 +1,32 @@
+Subject: [PATCH] Read unaligned match buffers with memcpy
+
+Byte buffers need not satisfy integer alignment or aliasing requirements.
+GCC 16 with x86-64-v3 vectorizes the cast loads using aligned AVX reads,
+causing a segmentation fault. Copy into local integers instead.
+
+--- a/src/zopfli/lz77.c
++++ b/src/zopfli/lz77.c
+@@ -302,13 +302,20 @@
+   if (sizeof(size_t) == 8) {
+     /* 8 checks at once per array bounds check (size_t is 64-bit). */
+-    while (scan < safe_end && *((size_t*)scan) == *((size_t*)match)) {
++    while (scan < safe_end) {
++      size_t scan_word, match_word;
++      memcpy(&scan_word, scan, sizeof(scan_word));
++      memcpy(&match_word, match, sizeof(match_word));
++      if (scan_word != match_word) break;
+       scan += 8;
+       match += 8;
+     }
+   } else if (sizeof(unsigned int) == 4) {
+     /* 4 checks at once per array bounds check (unsigned int is 32-bit). */
+-    while (scan < safe_end
+-        && *((unsigned int*)scan) == *((unsigned int*)match)) {
++    while (scan < safe_end) {
++      unsigned int scan_word, match_word;
++      memcpy(&scan_word, scan, sizeof(scan_word));
++      memcpy(&match_word, match, sizeof(match_word));
++      if (scan_word != match_word) break;
+       scan += 4;
+       match += 4;
+     }

common/optional/zfs/default.nix

@@ -1,5 +1,4 @@
 {
-  inputs,
   lib,
   pkgs,
   ...
@@ -15,7 +14,6 @@
   services = {
     snapshot_manager = {
       enable = lib.mkDefault true;
-      PYTHONPATH = "${inputs.self}/";
     };
 
     zfs = {

common/optional/zfs/snapshot.nix

@@ -6,22 +6,18 @@
 }:
 let
   cfg = config.services.snapshot_manager;
+  snapshotManager = pkgs.callPackage ../../../rust/snapshot_manager/package.nix { };
 in
 {
   options = {
     services.snapshot_manager = {
       enable = lib.mkEnableOption "ZFS snapshot manager";
+      signalAlerts = lib.mkEnableOption "Signal failure alerts for the snapshot manager";
       path = lib.mkOption {
         type = lib.types.path;
         default = ./snapshot_config.toml;
         description = "Path to the snapshot_manager TOML config.";
       };
-      PYTHONPATH = lib.mkOption {
-        type = lib.types.str;
-        description = ''
-          the PYTHONPATH to use for the snapshot_manager service.
-        '';
-      };
       EnvironmentFile = lib.mkOption {
         type = lib.types.nullOr (lib.types.coercedTo lib.types.path toString lib.types.str);
         default = null;
@@ -34,18 +30,17 @@ in
   };
 
   config = lib.mkIf cfg.enable {
+    system.checks = [ snapshotManager ];
+
     systemd = {
       services.snapshot_manager = {
         description = "ZFS Snapshot Manager";
         requires = [ "zfs-import.target" ];
         after = [ "zfs-import.target" ];
         path = [ pkgs.zfs ];
-        environment = {
-          PYTHONPATH = cfg.PYTHONPATH;
-        };
         serviceConfig = {
           Type = "oneshot";
-          ExecStart = "${pkgs.my_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
+          ExecStart = "${snapshotManager}/bin/snapshot-manager ${lib.escapeShellArg cfg.path} --signal-alerts ${lib.boolToString cfg.signalAlerts}";
         }
         // lib.optionalAttrs (cfg.EnvironmentFile != null) {
           EnvironmentFile = cfg.EnvironmentFile;

common/optional/zfs/snapshot_config.toml

@@ -22,6 +22,12 @@ hourly = 0
 daily = 0
 monthly = 0
 
+["root_pool/nix_build"]
+15_min = 1
+hourly = 0
+daily = 0
+monthly = 0
+
 ["root_pool/var"]
 15_min = 8
 hourly = 24

flake.lock

@@ -7,11 +7,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1781152676,
-        "narHash": "sha256-RxWs5ND31KzTG7wvMM+PMfUjyNpmIEr999lqNARaM5o=",
+        "lastModified": 1789770686,
+        "narHash": "sha256-uZkBR7yHdIKUFB5SZdfgh1qkGfI3XmYmI/lTiquxbck=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "ff8702b4de27f72b4c78573dfb89ec74e36abdf1",
+        "rev": "725ea35e410ad83be4931d1bff7e090eacaf3563",
         "type": "github"
       },
       "original": {
@@ -20,26 +20,22 @@
         "type": "github"
       }
     },
-    "firefox-addons": {
+    "flake-utils": {
       "inputs": {
-        "nixpkgs": [
-          "nixpkgs"
-        ]
+        "systems": "systems"
       },
       "locked": {
-        "dir": "pkgs/firefox-addons",
-        "lastModified": 1788840136,
-        "narHash": "sha256-ej5jnQIfjbw4wwPzy8Y4ntG9F3asdF6YBkLqpYeGUxc=",
-        "owner": "rycee",
-        "repo": "nur-expressions",
-        "rev": "555b23e68256c5abfb8a85a385231830d1d7ad1f",
-        "type": "gitlab"
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
       },
       "original": {
-        "dir": "pkgs/firefox-addons",
-        "owner": "rycee",
-        "repo": "nur-expressions",
-        "type": "gitlab"
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
       }
     },
     "home-manager": {
@@ -49,11 +45,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1788651960,
-        "narHash": "sha256-v9wJd32eZ2bvhBzVOd7TIjLQd011P7nwOhjKtWlci5I=",
+        "lastModified": 1791293500,
+        "narHash": "sha256-3hwJd2/vG/VYV1P/g7oDgdbNsWODhN5F//Od+hPMm2k=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "2c0350c759688177331b8f5242311fae8877bdb3",
+        "rev": "0e11b7bcba1d74ef8ea10bfdfb1e1a5811a21087",
         "type": "github"
       },
       "original": {
@@ -62,16 +58,38 @@
         "type": "github"
       }
     },
+    "jellyswarrm": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-overlay": "rust-overlay"
+      },
+      "locked": {
+        "lastModified": 1791127101,
+        "narHash": "sha256-SmrBBhcv9iNHA2vlafi7Zpe8bkcsMjVNSmCCVf5/jrQ=",
+        "owner": "LLukas22",
+        "repo": "Jellyswarrm",
+        "rev": "0b4301c6c74efff9c3634296913032579c98faf6",
+        "type": "github"
+      },
+      "original": {
+        "owner": "LLukas22",
+        "repo": "Jellyswarrm",
+        "type": "github"
+      }
+    },
     "nixos-hardware": {
       "inputs": {
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1788860136,
-        "narHash": "sha256-MhPMOFV4pVkygWEbQ8t1De/uQ9cWF1u++tRe2L5tG48=",
+        "lastModified": 1791128201,
+        "narHash": "sha256-rlsYY/Dg/LhbIj51Aa6TLvRpZeb3KZgUTd7llQ5XcJE=",
         "owner": "nixos",
         "repo": "nixos-hardware",
-        "rev": "62173785b9a18c78b4a15aca2623d02bceb9d077",
+        "rev": "31cc5f4d9b9ba601071e8b8504601b9b176e2756",
         "type": "github"
       },
       "original": {
@@ -83,11 +101,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1767892417,
-        "narHash": "sha256-8bW3q88CEg2u4hSP66Vf4lpbLonHz7hqDNBMcCY7E9U=",
-        "rev": "3497aa5c9457a9d88d71fa93a4a8368816fbeeba",
+        "lastModified": 1789546076,
+        "narHash": "sha256-vWkSk5bbfTqdtMoSgD9FshACO8JCvXTFi+3cqEp0mH0=",
+        "rev": "b1b875982b17dabde9b4a37f3e229e74913e6db3",
         "type": "tarball",
-        "url": "https://releases.nixos.org/nixos/unstable/nixos-26.05pre924538.3497aa5c9457/nixexprs.tar.xz"
+        "url": "https://releases.nixos.org/nixos/unstable/nixos-26.11pre1074753.b1b875982b17/nixexprs.tar.xz"
       },
       "original": {
         "type": "tarball",
@@ -96,11 +114,11 @@
     },
     "nixpkgs-master": {
       "locked": {
-        "lastModified": 1788892992,
-        "narHash": "sha256-cIMFh9gyU4/aLeB3JCcsWM3tTAvD9pAq9Smr1Wa8aIU=",
+        "lastModified": 1791296764,
+        "narHash": "sha256-N2b6Uxdr1MFhscEPqDyntTTtGZASgq3XYE0ujRzfeNI=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "dff6994123e257ec9901c271bc2b52e64d7c8f05",
+        "rev": "a15afac8691bd24f746255ffbd76306693e4cf7a",
         "type": "github"
       },
       "original": {
@@ -128,11 +146,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1788752844,
-        "narHash": "sha256-VaWGJ6+cIYN2erfSecbRV+4ljI185Ty2wUrXyvQbgOw=",
+        "lastModified": 1791260994,
+        "narHash": "sha256-Miqqk/ammqnTUxaoCyvtwoPeLbI1ksFyLxi/CazZpWY=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "dc5d91f840324650bac8c379428c7037a416959a",
+        "rev": "151fa4e8ddfdd8dd25d945ad94ed54a13de9f6e4",
         "type": "github"
       },
       "original": {
@@ -145,14 +163,35 @@
     "root": {
       "inputs": {
         "disko": "disko",
-        "firefox-addons": "firefox-addons",
         "home-manager": "home-manager",
+        "jellyswarrm": "jellyswarrm",
         "nixos-hardware": "nixos-hardware",
         "nixpkgs": "nixpkgs_2",
         "nixpkgs-master": "nixpkgs-master",
         "nixpkgs-stable": "nixpkgs-stable",
         "sops-nix": "sops-nix",
-        "systems": "systems"
+        "systems": "systems_2"
+      }
+    },
+    "rust-overlay": {
+      "inputs": {
+        "nixpkgs": [
+          "jellyswarrm",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1779419951,
+        "narHash": "sha256-dMX0PUslUHPajP6o8FEoRdFv9afq/dec4POR0vVfjK4=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "5b5c521d6cae9ef4aa32f888eb2c0ce595c9be52",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
       }
     },
     "sops-nix": {
@@ -162,11 +201,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1788337237,
-        "narHash": "sha256-gkSH8VUtCo6hnysNmb9DbTuDepH2t5pv+QWjP75xKAk=",
+        "lastModified": 1791103873,
+        "narHash": "sha256-nFxM+pKoZ8LJAEnUXARyCaOAloWgaW9kZQOSjWzKcTE=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "fbf759290e0cb0a98dfc813a4eb7d53ad1dacb57",
+        "rev": "dcd241ba97088c22569d1573286e1b9daad340c0",
         "type": "github"
       },
       "original": {
@@ -176,6 +215,21 @@
       }
     },
     "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
       "locked": {
         "lastModified": 1689347949,
         "narHash": "sha256-12tWmuL2zgBgZkdoB6qXZsgJEH9LR3oUgpaQq2RbI80=",

flake.nix

@@ -21,13 +21,13 @@
 
     nixos-hardware.url = "github:nixos/nixos-hardware/master";
 
-    home-manager = {
-      url = "github:nix-community/home-manager";
+    jellyswarrm = {
+      url = "github:LLukas22/Jellyswarrm";
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
-    firefox-addons = {
-      url = "gitlab:rycee/nur-expressions?dir=pkgs/firefox-addons";
+    home-manager = {
+      url = "github:nix-community/home-manager";
       inputs.nixpkgs.follows = "nixpkgs";
     };
 

overlays/default.nix

@@ -1,13 +1,13 @@
 { inputs, ... }:
 {
-  # When applied, the stable nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.stable'
+  # Additional package sets are accessible through `pkgs.<name>`.
   stable = final: _prev: {
     stable = import inputs.nixpkgs-stable {
       system = final.stdenv.hostPlatform.system;
       config.allowUnfree = true;
     };
   };
-  # When applied, the master nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.master'
+
   master = final: _prev: {
     master = import inputs.nixpkgs-master {
       system = final.stdenv.hostPlatform.system;
@@ -15,34 +15,12 @@
     };
   };
 
-  python-env = final: _prev: {
-    my_python = final.python314.withPackages (
-      ps: with ps; [
-        alembic
-        apprise
-        fastapi
-        fastapi-cli
-        httpx
-        jinja2
-        mypy
-        pgvector
-        psycopg
-        pydantic
-        pyfakefs
-        pytest
-        pytest-cov
-        pytest-mock
-        pytest-xdist
-        python-multipart
-        pydantic-settings
-        ruff
-        sqlalchemy
-        tenacity
-        tinytuya
-        typer
-        uvicorn
-        websockets
-      ]
-    );
+  # Baseline x86-64 (v1) packages for prebuilt applications that should not
+  # inherit an x86-64-v3 host platform.
+  x86-v1 = final: _prev: {
+    x86-v1 = import inputs.nixpkgs {
+      system = final.stdenv.hostPlatform.system;
+      config.allowUnfree = true;
+    };
   };
 }

pyproject.toml

@@ -6,49 +6,6 @@ authors = [{ name = "Richie Cahill", email = "richie@tmmworkshop.com" }]
 requires-python = "~=3.14.0"
 readme = "README.md"
 license = "MIT"
-# these dependencies are a best effort and aren't guaranteed to work
-# for up-to-date dependencies, see overlays/default.nix
-dependencies = [
-    "alembic",
-    "apprise",
-    "beautifulsoup4",
-    "bm25s",
-    "ebooklib",
-    "fastapi",
-    "fastapi-cli",
-    "httpx",
-    "jinja2",
-    "pgvector",
-    "psycopg[binary]",
-    "pydantic",
-    "pydantic-settings",
-    "python-multipart",
-    "sqlalchemy[asyncio]",
-    "tenacity",
-    "tiktoken",
-    "tinytuya",
-    "typer",
-    "uvicorn",
-    "websockets",
-    "yake",
-]
-
-[project.scripts]
-database = "python.database_cli:app"
-whisper-transcribe = "python.tools.whisper.transcribe:main"
-
-[dependency-groups]
-dev = [
-    "aiosqlite",
-    "mypy",
-    "pyfakefs",
-    "pytest-asyncio",
-    "pytest-cov",
-    "pytest-mock",
-    "pytest-xdist",
-    "pytest",
-    "ruff",
-]
 
 [tool.ruff]
 

python/ebook_search/docker/README.md

@@ -1,77 +0,0 @@
-# Ebook Search Docker
-
-Run the EPUB search app against the existing Postgres database on `jeeves`:
-
-```sh
-python -m python.ebook_search.docker.containers start --library-path /path/to/epubs --build
-```
-
-All ebook-search Docker files live in this directory:
-
-- `Dockerfile` — multi-stage: `test` (runs pytest) and `runtime` (default target, the app image)
-- `docker-compose.yml`
-- `containers.py` — Typer lifecycle CLI
-- `pyproject.toml` / `uv.lock` — the container's uv-locked dependencies
-
-The app listens on `http://localhost:8070`.
-
-Useful lifecycle commands:
-
-```sh
-python -m python.ebook_search.docker.containers build
-python -m python.ebook_search.docker.containers start --library-path /path/to/epubs
-python -m python.ebook_search.docker.containers test
-python -m python.ebook_search.docker.containers logs
-python -m python.ebook_search.docker.containers ps
-python -m python.ebook_search.docker.containers stop
-```
-
-Direct compose usage from the repo root:
-
-```sh
-docker compose -f python/ebook_search/docker/docker-compose.yml ps
-```
-
-## Dependencies
-
-The image builds its environment with uv from `pyproject.toml` + `uv.lock` in this
-directory — this is the source of truth for the container's dependencies. To add or
-update a dependency, edit `pyproject.toml` here and regenerate the lock (uv is
-available in the `ebook-search` dev shell):
-
-```sh
-nix develop .#ebook-search -c uv lock --project python/ebook_search/docker
-```
-
-## Tests
-
-The main pytest suite excludes `tests/ebook_search` (its dependencies are no longer
-in the nix dev shell). The `test ebook search` CI workflow runs them in a uv env
-built from the lockfile in this directory — same commands work locally from the
-repo root (the `--override-ini` drops the main suite's ignore):
-
-```sh
-uv sync --locked --project python/ebook_search/docker
-uv run --project python/ebook_search/docker --no-sync pytest tests/ebook_search --override-ini addopts="-n auto -ra"
-```
-
-They can also run inside the Docker `test` image, which validates the image itself:
-
-```sh
-python -m python.ebook_search.docker.containers test
-```
-
-or the raw docker equivalent:
-
-```sh
-docker build --file python/ebook_search/docker/Dockerfile --target test --tag ebook-search:test .
-docker run --rm ebook-search:test
-```
-
-## Configuration
-
-The compose service loads the repo root `.env` into the container via `env_file`.
-
-Mount your EPUB directory by setting `EBOOK_LIBRARY_HOST_PATH` in an env file or on the command line. The container sees it as `/library`, and `EBOOK_SEARCH_LIBRARY_PATHS` is set to `/library` inside the container.
-
-Database connection settings are controlled by `RICHIE_DB`, `RICHIE_HOST`, `RICHIE_PORT`, `RICHIE_USER`, and `RICHIE_PASSWORD`. The default host is `jeeves`.

python/gems/README.md

@@ -1,54 +0,0 @@
-# Gems
-
-Gems is a server-rendered, turn-based resource-engine game for one to four human or AI players. It uses FastAPI,
-Jinja, HTMX, server-sent events, and SQLite.
-
-The application deliberately contains no playable card deck, patron/governor set, objective set, official artwork,
-or copied rulebook text. A room host must upload a content pack they are entitled to use before starting a game.
-
-## Run locally
-
-```shell
-uv run gems --host 127.0.0.1 --port 8082
-```
-
-The default database and installation key are created under `.gems/`. The following environment variables override
-runtime behavior:
-
-- `GEMS_DATABASE_PATH`
-- `GEMS_KEY_PATH`
-- `GEMS_PUBLIC_ORIGIN`
-- `GEMS_SECURE_COOKIES`
-- `GEMS_HOST`
-- `GEMS_PORT`
-
-## Content packs
-
-The current schema is available from a running server at `/schemas/content-pack-v1.json`. A pack defines exactly
-five normal resources, one wild resource, cards, and optional patrons, objectives, and outposts. `patrons` is the
-canonical field name; `governors` is accepted as an input alias.
-
-Cards may use only the built-in, bounded effect vocabulary:
-
-- `none`
-- `virtual_wild`
-- `copy_bonus`
-- `copy_and_claim`
-- `multi_bonus`
-- `claim_free`
-- an optional discard-cards alternate cost
-
-Unknown fields, resource references, executable expressions, HTML, artwork URLs, and files larger than 512 KiB are
-rejected. The normalized pack is private to its room and becomes immutable when play starts.
-
-## Neutral module mapping
-
-Gems calls the four optional mechanics Objectives, Outposts, Eastern Decks, and Fortifications. Lobby presets combine
-these mechanics into the familiar base, objective-race, objective-plus-outpost, eastern-plus-fortification, and
-all-module configurations. Component identities and values always come from the uploaded pack.
-
-## Jeeves
-
-The NixOS module runs one Uvicorn worker on `127.0.0.1:8002`, stores state in
-`/zfs/media/services/gems`, and publishes it through HAProxy at `https://gems.tmmworkshop.com`. The DNS record must
-point to Jeeves before ACME can issue the certificate.

python/gitea_flake_lock.py

@@ -14,7 +14,7 @@ DEFAULT_BASE_BRANCH = "main"
 DEFAULT_BRANCH = "automation/update-flake-lock"
 DEFAULT_GITEA_URL = "https://gitea.tmmworkshop.com"
 PR_LABELS = ["dependencies", "automated", "flake_lock_update"]
-PR_CHECK_WORKFLOWS = ["build_systems.yml", "treefmt.yml", "pytest.yml"]
+PR_CHECK_WORKFLOWS = ["build_systems.yml", "treefmt.yml"]
 PR_TITLE = "Update flake.lock"
 PR_BODY = "Automated flake.lock update."
 

python/signal_alert.py

@@ -2,29 +2,63 @@
 
 from __future__ import annotations
 
+import json
 import logging
-from os import getenv
-
-from apprise import Apprise
+import socket
 
 logger = logging.getLogger(__name__)
 
 
-def signal_alert(body: str, title: str = "") -> None:
-    """Send a signal alert.
+class SignalRPCError(RuntimeError):
+    """signal-cli returned an error or closed without responding."""
+
+
+def signal_alert(body: str, from_phone: str, to_phone: str, *, timeout: float = 4.0) -> None:
+    """Send a Signal alert through the local signal-cli daemon.
 
     Args:
-        body (str): The body of the alert.
-        title (str, optional): The title of the alert. Defaults to "".
+        body: The body of the alert.
+        from_phone: The Signal account sending the alert.
+        to_phone: The Signal account receiving the alert.
+        timeout: Seconds to wait on each socket operation.
+
+    Raises:
+        SignalRPCError: If signal-cli returns an error or closes the
+            connection before responding.
+        OSError: If the socket is unreachable or an operation times out.
     """
-    apprise_client = Apprise()
+    signal_rpc_id = "signal-alert"
+    request = {
+        "jsonrpc": "2.0",
+        "method": "send",
+        "params": {
+            "account": from_phone,
+            "recipient": [to_phone],
+            "message": body,
+        },
+        "id": signal_rpc_id,
+    }
 
-    from_phone = getenv("SIGNAL_ALERT_FROM_PHONE")
-    to_phone = getenv("SIGNAL_ALERT_TO_PHONE")
-    if not from_phone or not to_phone:
-        logger.info("SIGNAL_ALERT_FROM_PHONE or SIGNAL_ALERT_TO_PHONE not set")
-        return
+    try:
+        with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client:
+            client.settimeout(timeout)
+            client.connect("/run/signal-cli/socket")
+            client.sendall(json.dumps(request).encode() + b"\n")
 
-    apprise_client.add(f"signal://localhost:8989/{from_phone}/{to_phone}")
-
-    apprise_client.notify(title=title, body=body)
+            buffer = b""
+            while chunk := client.recv(65536):
+                buffer += chunk
+                while b"\n" in buffer:
+                    line, buffer = buffer.split(b"\n", maxsplit=1)
+                    response = json.loads(line)
+                    if response.get("id") != signal_rpc_id:
+                        continue
+                    if rpc_error := response.get("error"):
+                        error = f"Signal RPC returned an error: {rpc_error}"
+                        raise SignalRPCError(error)
+                    return
+            error = "Signal RPC socket closed before returning a response"
+            raise SignalRPCError(error)
+    except (OSError, json.JSONDecodeError) as exc:
+        error = f"Signal RPC failed: {exc!r}"
+        raise SignalRPCError(error) from exc

python/system_tests/__init__.py

@@ -1 +0,0 @@
-"""system_tests."""

python/system_tests/components.py

@@ -1,99 +0,0 @@
-"""Validate Jeeves."""
-
-from __future__ import annotations
-
-import logging
-from copy import copy
-from re import search
-from time import sleep
-from typing import TYPE_CHECKING
-
-from python.common import bash_wrapper
-from python.zfs import Zpool
-
-if TYPE_CHECKING:
-    from collections.abc import Sequence
-
-logger = logging.getLogger(__name__)
-
-
-def zpool_tests(pool_names: Sequence[str], zpool_capacity_threshold: int = 90) -> list[str] | None:
-    """Check the zpool health and capacity.
-
-    Args:
-        pool_names (Sequence[str]): A list of pool names to test.
-        zpool_capacity_threshold (int, optional): The threshold for the zpool capacity. Defaults to 90.
-
-    Returns:
-        list[str] | None: A list of errors if any.
-    """
-    logger.info("Testing zpool")
-
-    errors: list[str] = []
-    for pool_name in pool_names:
-        pool = Zpool(pool_name)
-        if pool.health != "ONLINE":
-            errors.append(f"{pool.name} is {pool.health}")
-        if pool.capacity >= zpool_capacity_threshold:
-            errors.append(f"{pool.name} is low on space")
-
-    upgrade_status, _ = bash_wrapper("zpool upgrade")
-    if not search(r"Every feature flags pool has all supported and requested features enabled.", upgrade_status):
-        errors.append("ZPool out of date run `sudo zpool upgrade -a`")
-
-    return errors
-
-
-def systemd_tests(
-    service_names: Sequence[str],
-    max_retries: int = 30,
-    retry_delay_secs: int = 1,
-    retryable_statuses: Sequence[str] | None = None,
-    valid_statuses: Sequence[str] | None = None,
-) -> list[str] | None:
-    """Tests a systemd services.
-
-    Args:
-        service_names (Sequence[str]): A list of service names to test.
-        max_retries (int, optional): The maximum number of retries. Defaults to 30.
-            minimum value is 1.
-        retry_delay_secs (int, optional): The delay between retries in seconds. Defaults to 1.
-            minimum value is 1.
-        retryable_statuses (Sequence[str] | None, optional): A list of retryable statuses. Defaults to None.
-        valid_statuses (Sequence[str] | None, optional): A list of valid statuses. Defaults to None.
-
-    Returns:
-        list[str] | None: A list of errors if any.
-    """
-    logger.info("Testing systemd service")
-
-    max_retries = max(max_retries, 1)
-    retry_delay_secs = max(retry_delay_secs, 1)
-    last_try = max_retries - 1
-
-    if retryable_statuses is None:
-        retryable_statuses = ("inactive\n", "activating\n")
-
-    if valid_statuses is None:
-        valid_statuses = ("active\n",)
-
-    service_names_set = set(service_names)
-
-    errors: set[str] = set()
-    for retry in range(max_retries):
-        if not service_names_set:
-            break
-        logger.info(f"Testing systemd service in {retry + 1} of {max_retries}")
-        service_names_to_test = copy(service_names_set)
-        for service_name in service_names_to_test:
-            service_status, _ = bash_wrapper(f"systemctl is-active {service_name}")
-            if service_status in valid_statuses:
-                service_names_set.remove(service_name)
-                continue
-            if service_status in retryable_statuses and retry < last_try:
-                continue
-            errors.add(f"{service_name} is {service_status.strip()}")
-
-        sleep(retry_delay_secs)
-
-    return list(errors)

python/system_tests/validate_system.py

@@ -1,67 +0,0 @@
-"""Validate {server_name}."""
-
-import logging
-import sys
-import tomllib
-from os import environ
-from pathlib import Path  # noqa: TC003 This is required for the typer CLI
-from socket import gethostname
-
-import typer
-
-from python.common import configure_logger
-from python.signal_alert import signal_alert
-from python.system_tests.components import systemd_tests, zpool_tests
-
-logger = logging.getLogger(__name__)
-
-
-def load_config_data(config_file: Path) -> dict[str, list[str]]:
-    """Load a TOML configuration file.
-
-    Args:
-        config_file (Path): The path to the configuration file.
-
-    Returns:
-        dict: The configuration data.
-    """
-    return tomllib.loads(config_file.read_text())
-
-
-def main(config_file: Path) -> None:
-    """Main."""
-    configure_logger(level=environ.get("LOG_LEVEL", "INFO"))
-
-    server_name = gethostname()
-    logger.info(f"Starting {server_name} validation")
-
-    config_data = load_config_data(config_file)
-
-    errors: list[str] = []
-    try:
-        if config_data.get("zpools") and (zpool_errors := zpool_tests(config_data["zpools"])):
-            errors.extend(zpool_errors)
-
-        if config_data.get("services") and (systemd_errors := systemd_tests(config_data["services"])):
-            errors.extend(systemd_errors)
-
-    except Exception as error:
-        logger.exception(f"{server_name} validation failed")
-        errors.append(f"{server_name} validation failed: {error}")
-
-    if errors:
-        logger.error(f"{server_name} validation failed: \n{'\n'.join(errors)}")
-        signal_alert(f"{server_name} validation failed {errors}")
-
-        sys.exit(1)
-
-    logger.info(f"{server_name} validation passed")
-
-
-def cli() -> None:
-    """CLI."""
-    typer.run(main)
-
-
-if __name__ == "__main__":
-    cli()

python/tools/snapshot_manager.py

@@ -1,147 +0,0 @@
-"""snapshot_manager."""
-
-from __future__ import annotations
-
-import logging
-import sys
-import tomllib
-from functools import cache
-from pathlib import Path  # noqa: TC003 This is required for the typer CLI
-from re import compile as re_compile
-from re import search
-
-import typer
-
-from python.common import configure_logger, utcnow
-from python.signal_alert import signal_alert
-from python.zfs import Dataset, get_datasets
-
-logger = logging.getLogger(__name__)
-
-
-def main(config_file: Path) -> None:
-    """Main."""
-    configure_logger(level="DEBUG")
-    logger.info("Starting snapshot_manager")
-
-    try:
-        time_stamp = get_time_stamp()
-
-        for dataset in get_datasets():
-            status = dataset.create_snapshot(time_stamp)
-            logger.debug(f"{status=}")
-            if status != "snapshot created":
-                msg = f"{dataset.name} failed to create snapshot {time_stamp}"
-                logger.error(msg)
-                signal_alert(msg)
-                continue
-            count_lookup = get_count_lookup(config_file, dataset.name)
-            logger.info(f"using {count_lookup} for {dataset.name}")
-            get_snapshots_to_delete(dataset, count_lookup)
-    except Exception:
-        logger.exception("snapshot_manager failed")
-        signal_alert("snapshot_manager failed")
-        sys.exit(1)
-    else:
-        logger.info("snapshot_manager completed")
-
-
-def get_count_lookup(config_file: Path, dataset_name: str) -> dict[str, int]:
-    """Get the count lookup.
-
-    Args:
-        config_file (Path): The path to the configuration file.
-        dataset_name (str): The name of the dataset.
-
-    Returns:
-        dict[str, int]: The count lookup.
-    """
-    config_data = load_config_data(config_file)
-
-    return config_data.get(dataset_name, get_default_config(config_data))
-
-
-def get_default_config(config_data: dict[str, dict[str, int]]) -> dict[str, int]:
-    """Get the default configuration.
-
-    Args:
-        config_data (dict[str, dict[str, int]]): The configuration data.
-
-    Returns:
-        dict[str, int]: The default configuration.
-    """
-    return config_data.get(
-        "default",
-        {"15_min": 4, "hourly": 12, "daily": 0, "monthly": 0},
-    )
-
-
-@cache
-def load_config_data(config_file: Path) -> dict[str, dict[str, int]]:
-    """Load a TOML configuration file.
-
-    Args:
-        config_file (Path): The path to the configuration file.
-
-    Returns:
-        dict: The configuration data.
-    """
-    return tomllib.loads(config_file.read_text())
-
-
-def get_snapshots_to_delete(
-    dataset: Dataset,
-    count_lookup: dict[str, int],
-) -> None:
-    """Get snapshots to delete.
-
-    Args:
-        dataset (Dataset): the dataset
-        count_lookup (dict[str, int]): the count lookup
-    """
-    snapshots = dataset.get_snapshots()
-
-    logger.info(f"calculating snapshots for {dataset.name} to be deleted")
-    if not snapshots:
-        logger.info(f"{dataset.name} has no snapshots")
-        return
-
-    filters = (
-        ("15_min", re_compile(r"auto_\d{10}(?:15|30|45)")),
-        ("hourly", re_compile(r"auto_\d{8}(?!00)\d{2}00")),
-        ("daily", re_compile(r"auto_\d{6}(?!01)\d{2}0000")),
-        ("monthly", re_compile(r"auto_\d{6}010000")),
-    )
-
-    for filter_name, snapshot_filter in filters:
-        logger.debug(f"{filter_name=}\n{snapshot_filter=}")
-
-        filtered_snapshots = sorted(snapshot.name for snapshot in snapshots if search(snapshot_filter, snapshot.name))
-
-        logger.debug(f"{filtered_snapshots=}")
-
-        snapshots_wanted = count_lookup[filter_name]
-        snapshots_being_deleted = filtered_snapshots[:-snapshots_wanted] if snapshots_wanted > 0 else filtered_snapshots
-
-        logger.info(f"{snapshots_being_deleted} are being deleted")
-        for snapshot in snapshots_being_deleted:
-            if error := dataset.delete_snapshot(snapshot):
-                error_message = f"{dataset.name}@{snapshot} failed to delete: {error}"
-                signal_alert(error_message)
-                logger.error(error_message)
-
-
-def get_time_stamp() -> str:
-    """Get the time stamp."""
-    now = utcnow()
-    nearest_15_min = now.replace(minute=(now.minute - (now.minute % 15)))
-    return nearest_15_min.strftime("auto_%Y%m%d%H%M")
-
-
-def cli() -> None:
-    """CLI."""
-    typer.run(main)
-
-
-if __name__ == "__main__":
-    cli()

rust/haproxy_ingest_proto/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "haproxy-ingest-proto"
+version = "0.1.0"
+edition = "2024"
+description = "Shared protocol types and limits for HAProxy ingestion"
+license = "MIT"
+build = "build.rs"
+
+[lints]
+workspace = true
+
+[dependencies]
+prost.workspace = true
+
+[build-dependencies]
+prost-build.workspace = true
+protoc-bin-vendored.workspace = true

rust/haproxy_ingest_proto/README.md

@@ -0,0 +1,96 @@
+# HAProxy cloud log ingestion
+
+This workspace is split into three crates:
+
+- `haproxy-ingest-proto` owns the shared Protobuf schema and protocol limits.
+- `haproxy-log-shipper` is a short-lived client launched by a systemd timer on
+  `portal-1`. It reads only local `haproxy.service` journal records and drains
+  them in cursor order.
+- `haproxy-ingest-server` is the private HTTP API on Jeeves. It authenticates
+  the source, commits records to PostgreSQL, and advances the journal cursor
+  atomically.
+
+The wire contract is in `proto/haproxy_ingest.proto`; the initial PostgreSQL
+schema is in `../haproxy_ingest_server/migrations/001_initial.sql`.
+
+## One-time setup
+
+The shipper endpoint is supplied by the `haproxy-log-shipper` service as
+`http://jeeves.taile39c31.ts.net:8444`; the environment file only needs the
+shared token. Ingestion uses HTTP over Tailscale and requires no certificates.
+
+Create one random token and install the same value on the shipper and server.
+A VM rebuild with a new `/etc/machine-id` is still treated as a new log source.
+
+On `portal-1`, create `/var/lib/haproxy-log-shipper/environment`:
+
+```text
+HAPROXY_INGEST_TOKEN=<at-least-32-character-random-token>
+```
+
+Set the file ownership and mode to `root:root` and `0600`.
+
+On Jeeves, create
+`/zfs/storage/secrets/services/haproxy-ingest-server`:
+
+```text
+HAPROXY_INGEST_TOKEN=<the-same-random-token>
+```
+
+Set this file to `root:root` and `0600`. systemd loads it before switching the
+API to its unprivileged service account.
+
+A suitable token can be generated without writing it to shell history:
+
+```console
+openssl rand -hex 32
+```
+
+Deploy Jeeves before `portal-1`, then verify:
+
+```console
+systemctl status haproxy-ingest-server.service
+systemctl status haproxy-log-shipper.timer
+journalctl -u haproxy-log-shipper.service
+```
+
+The server listens over IPv4 on TCP 8444, which Jeeves' firewall permits only
+through `tailscale0`. Both service sandboxes disable IPv6. Tailscale encrypts
+traffic between the hosts. The bearer token authorizes access to the ingestion API;
+the machine ID identifies a log source but is not an authorization boundary.
+
+## Protocol behavior
+
+- Maximum sender batch: 1,000 records or 20 MiB serialized Protobuf.
+- Maximum receiver request: 24 MiB compressed, 20 MiB decompressed, 10,000 records.
+- Compression: one Zstandard frame over the complete Protobuf body.
+- Request timeouts: 3 seconds to connect and 10 seconds total.
+- Retry delays: 1, 2, and 4 seconds plus bounded jitter.
+- `409`: fetch the authoritative server cursor and rebuild.
+- `413`: fail because a sender batch within the configured limits was rejected.
+- `422`: replace only the reported offending record with a durable rejection.
+- Ambiguous failures: query the status endpoint before retrying.
+
+Journal retention on `portal-1` is persistent, bounded to 512 MiB, and retained
+for up to 14 days. If an acknowledged cursor has already been vacuumed, the
+shipper records a `JournalGap` before resuming from the oldest retained matching
+entry.
+
+## Database inspection
+
+The API owns the `haproxy_ingest` database and creates daily
+`haproxy_logs_YYYYMMDD` partitions before using chunked SQLx inserts.
+Useful checks include:
+
+```sql
+SELECT machine_id, hostname, last_journal_timestamp, last_commit_time
+FROM ingest_source_state;
+
+SELECT journal_timestamp, client_ip, http_method, request_path, status_code
+FROM haproxy_logs
+ORDER BY journal_timestamp DESC
+LIMIT 100;
+
+SELECT * FROM ingest_journal_gaps ORDER BY detected_at DESC;
+SELECT * FROM ingest_rejected_events ORDER BY ingest_time DESC;
+```

rust/haproxy_ingest_proto/build.rs

@@ -0,0 +1,11 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let protoc = protoc_bin_vendored::protoc_bin_path()?;
+    unsafe {
+        std::env::set_var("PROTOC", protoc);
+    }
+    let mut config = prost_build::Config::new();
+    config.type_attribute(".haproxy.ingest.v1.LogBatch", "#[derive(Eq)]");
+    config.compile_protos(&["proto/haproxy_ingest.proto"], &["proto"])?;
+    println!("cargo:rerun-if-changed=proto/haproxy_ingest.proto");
+    Ok(())
+}

rust/haproxy_ingest_proto/proto/haproxy_ingest.proto

@@ -0,0 +1,71 @@
+syntax = "proto3";
+
+package haproxy.ingest.v1;
+
+message LogBatch {
+  uint32 schema_version = 1;
+  string machine_id = 2;
+  string hostname = 3;
+  optional string previous_cursor = 4;
+  string last_cursor = 5;
+  int64 first_journal_timestamp_us = 6;
+  int64 last_journal_timestamp_us = 7;
+  repeated LogRecord records = 8;
+}
+
+message LogRecord {
+  int64 journal_timestamp_us = 1;
+
+  oneof payload {
+    HAProxyEvent event = 2;
+    RejectedEvent rejected = 3;
+    JournalGap gap = 4;
+  }
+}
+
+message HAProxyEvent {
+  reserved 22;
+
+  optional string client_ip = 1;
+  optional uint32 client_port = 2;
+  optional string frontend_name = 3;
+  optional string backend_name = 4;
+  optional string server_name = 5;
+  optional string http_method = 6;
+  optional string request_path = 7;
+  optional string http_version = 8;
+  int32 status_code = 9;
+  uint64 bytes_sent = 10;
+  int32 time_request_ms = 11;
+  int32 time_wait_ms = 12;
+  int32 time_connect_ms = 13;
+  int32 time_response_ms = 14;
+  int32 time_total_ms = 15;
+  optional string termination_state = 16;
+  uint32 retries = 17;
+  optional string tls_sni = 18;
+  optional string tls_version = 19;
+  optional string tls_cipher = 20;
+  optional bool tls_resumed = 21;
+  repeated string normalization_flags = 23;
+  optional uint32 active_connections = 24;
+  optional uint32 frontend_connections = 25;
+  optional uint32 backend_connections = 26;
+  optional uint32 server_connections = 27;
+  optional uint32 server_queue = 28;
+  optional uint32 backend_queue = 29;
+}
+
+message RejectedEvent {
+  string reason = 1;
+  bytes message_sha256 = 2;
+  optional string raw_message_prefix = 3;
+  bool raw_message_truncated = 4;
+}
+
+message JournalGap {
+  string expected_cursor = 1;
+  string first_available_cursor = 2;
+  optional int64 first_available_timestamp_us = 3;
+  string reason = 4;
+}

rust/haproxy_ingest_proto/src/lib.rs

@@ -0,0 +1,12 @@
+pub mod proto {
+    #![allow(clippy::large_enum_variant)]
+
+    include!(concat!(env!("OUT_DIR"), "/haproxy.ingest.v1.rs"));
+}
+
+pub const SCHEMA_VERSION: u32 = 1;
+pub const MAX_BATCH_RECORDS: usize = 1_000;
+pub const MAX_BATCH_PROTO_BYTES: usize = 20 * 1024 * 1024;
+pub const MAX_COMPRESSED_BYTES: usize = 24 * 1024 * 1024;
+pub const MAX_DECOMPRESSED_BYTES: usize = MAX_BATCH_PROTO_BYTES;
+pub const SERVER_MAX_RECORDS: usize = 10_000;

rust/haproxy_ingest_server/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "haproxy-ingest-server"
+version = "0.1.0"
+edition = "2024"
+description = "HAProxy log ingestion API"
+license = "MIT"
+
+[lints]
+workspace = true
+
+[dependencies]
+anyhow.workspace = true
+axum.workspace = true
+bytes.workspace = true
+chrono = { workspace = true, features = ["serde"] }
+clap.workspace = true
+haproxy-ingest-proto = { path = "../haproxy_ingest_proto" }
+hex.workspace = true
+prost.workspace = true
+serde.workspace = true
+sha2.workspace = true
+sqlx.workspace = true
+subtle.workspace = true
+thiserror.workspace = true
+tokio.workspace = true
+tower-http.workspace = true
+tracing.workspace = true
+tracing-subscriber.workspace = true
+zstd.workspace = true

rust/haproxy_ingest_server/migrations/001_initial.sql

@@ -0,0 +1,102 @@
+CREATE TABLE IF NOT EXISTS ingest_source_state (
+    machine_id                TEXT PRIMARY KEY,
+    hostname                  TEXT NOT NULL,
+    last_cursor               TEXT,
+    last_journal_timestamp    TIMESTAMPTZ,
+    last_source_ip            INET,
+    last_commit_time          TIMESTAMPTZ
+);
+
+CREATE TABLE IF NOT EXISTS haproxy_logs (
+    id                        BIGINT GENERATED ALWAYS AS IDENTITY,
+    machine_id                TEXT NOT NULL,
+    hostname                  TEXT NOT NULL,
+    source_ip                 INET NOT NULL,
+    journal_timestamp         TIMESTAMPTZ NOT NULL,
+    request_time              TIMESTAMPTZ,
+    client_ip                 INET,
+    client_port               INTEGER,
+    frontend_name             TEXT,
+    backend_name              TEXT,
+    server_name               TEXT,
+    http_method               TEXT,
+    request_path              TEXT,
+    http_version              TEXT,
+    status_code               INTEGER NOT NULL,
+    bytes_sent                BIGINT NOT NULL,
+    time_request_ms           INTEGER NOT NULL,
+    time_wait_ms              INTEGER NOT NULL,
+    time_connect_ms           INTEGER NOT NULL,
+    time_response_ms          INTEGER NOT NULL,
+    time_total_ms             INTEGER NOT NULL,
+    termination_state         TEXT,
+    retries                   INTEGER NOT NULL,
+    tls_sni                   TEXT,
+    tls_version               TEXT,
+    tls_cipher                TEXT,
+    tls_resumed               BOOLEAN,
+    normalization_flags       TEXT[] NOT NULL DEFAULT '{}',
+    active_connections        INTEGER,
+    frontend_connections      INTEGER,
+    backend_connections       INTEGER,
+    server_connections        INTEGER,
+    server_queue              INTEGER,
+    backend_queue             INTEGER,
+    ingest_time               TIMESTAMPTZ NOT NULL DEFAULT now(),
+    PRIMARY KEY (journal_timestamp, id)
+) PARTITION BY RANGE (journal_timestamp);
+
+CREATE INDEX IF NOT EXISTS haproxy_logs_machine_time_idx
+ON haproxy_logs (machine_id, journal_timestamp);
+
+CREATE INDEX IF NOT EXISTS haproxy_logs_status_time_idx
+ON haproxy_logs (status_code, journal_timestamp);
+
+CREATE TABLE IF NOT EXISTS ingest_rejected_events (
+    id                        BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    machine_id                TEXT NOT NULL,
+    hostname                  TEXT NOT NULL,
+    journal_timestamp         TIMESTAMPTZ NOT NULL,
+    reason                    TEXT NOT NULL,
+    message_sha256            BYTEA NOT NULL,
+    raw_message_prefix        TEXT,
+    raw_message_truncated     BOOLEAN NOT NULL,
+    source_ip                 INET NOT NULL,
+    ingest_time               TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS ingest_rejected_events_machine_time_idx
+ON ingest_rejected_events (machine_id, journal_timestamp);
+
+CREATE TABLE IF NOT EXISTS ingest_journal_gaps (
+    id                         BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    machine_id                 TEXT NOT NULL,
+    hostname                   TEXT NOT NULL,
+    expected_cursor            TEXT NOT NULL,
+    first_available_cursor     TEXT NOT NULL,
+    first_available_timestamp  TIMESTAMPTZ,
+    reason                     TEXT NOT NULL,
+    source_ip                  INET NOT NULL,
+    detected_at                TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS ingest_journal_gaps_machine_time_idx
+ON ingest_journal_gaps (machine_id, detected_at);
+
+CREATE TABLE IF NOT EXISTS ingest_batches (
+    batch_id                   TEXT PRIMARY KEY,
+    machine_id                 TEXT NOT NULL,
+    hostname                   TEXT NOT NULL,
+    source_ip                  INET NOT NULL,
+    previous_cursor            TEXT,
+    last_cursor                TEXT NOT NULL,
+    first_journal_timestamp    TIMESTAMPTZ,
+    last_journal_timestamp     TIMESTAMPTZ,
+    record_count               INTEGER NOT NULL,
+    compressed_bytes           BIGINT NOT NULL,
+    uncompressed_bytes         BIGINT NOT NULL,
+    received_at                TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS ingest_batches_machine_time_idx
+ON ingest_batches (machine_id, received_at);

rust/haproxy_ingest_server/package.nix

@@ -0,0 +1,19 @@
+{
+  lib,
+  rustPlatform,
+}:
+rustPlatform.buildRustPackage {
+  pname = "haproxy-ingest-server";
+  version = "0.1.0";
+  src = import ../rust-source.nix { inherit lib; };
+
+  cargoLock.lockFile = ../../Cargo.lock;
+  cargoBuildFlags = [ "-p haproxy-ingest-server" ];
+  cargoTestFlags = [ "-p haproxy-ingest-server" ];
+
+  meta = {
+    description = "HAProxy log ingestion API";
+    license = lib.licenses.mit;
+    platforms = lib.platforms.linux;
+  };
+}

rust/haproxy_ingest_server/src/db.rs

@@ -0,0 +1,376 @@
+//! `SQLx` statements and Rust-to-PostgreSQL value conversion.
+
+use std::net::IpAddr;
+
+use anyhow::{Context, Result};
+use chrono::{DateTime, NaiveDate, Utc};
+use haproxy_ingest_proto::proto::{HaProxyEvent, JournalGap, LogBatch, RejectedEvent, log_record};
+use sha2::{Digest, Sha256};
+use sqlx::postgres::PgRow;
+use sqlx::{AssertSqlSafe, Decode, PgPool, Postgres, QueryBuilder, Row, Transaction, Type};
+
+use crate::{ApiError, timestamp};
+
+const MIGRATION: &str = include_str!("../migrations/001_initial.sql");
+const EVENT_INSERT_CHUNK_SIZE: usize = 1_000;
+
+pub struct SourceStatus {
+    pub hostname: Option<String>,
+    pub last_cursor: Option<String>,
+    pub last_journal_timestamp: Option<DateTime<Utc>>,
+    pub last_commit_time: Option<DateTime<Utc>>,
+}
+
+pub async fn apply_ingestion_schema(pool: &PgPool) -> Result<()> {
+    sqlx::raw_sql(MIGRATION)
+        .execute(pool)
+        .await
+        .context("failed to apply ingestion schema")?;
+    Ok(())
+}
+
+pub async fn select_source_status(
+    pool: &PgPool,
+    machine_id: &str,
+) -> Result<Option<SourceStatus>, ApiError> {
+    let row = sqlx::query(
+        "SELECT hostname, last_cursor, last_journal_timestamp, last_commit_time \
+         FROM ingest_source_state WHERE machine_id = $1",
+    )
+    .bind(machine_id)
+    .fetch_optional(pool)
+    .await
+    .map_err(ApiError::internal)?;
+    row.map(|row| {
+        Ok(SourceStatus {
+            hostname: get(&row, "hostname")?,
+            last_cursor: get(&row, "last_cursor")?,
+            last_journal_timestamp: get(&row, "last_journal_timestamp")?,
+            last_commit_time: get(&row, "last_commit_time")?,
+        })
+    })
+    .transpose()
+}
+
+fn get<'r, T>(row: &'r PgRow, column: &str) -> Result<T, ApiError>
+where
+    T: Decode<'r, Postgres> + Type<Postgres>,
+{
+    row.try_get(column).map_err(ApiError::internal)
+}
+
+pub async fn set_lock_timeout(transaction: &mut Transaction<'_, Postgres>) -> Result<(), ApiError> {
+    sqlx::query("SET LOCAL lock_timeout = '5s'")
+        .execute(&mut **transaction)
+        .await
+        .map_err(ApiError::internal)?;
+    Ok(())
+}
+
+pub async fn set_statement_timeout(
+    transaction: &mut Transaction<'_, Postgres>,
+) -> Result<(), ApiError> {
+    sqlx::query("SET LOCAL statement_timeout = '8s'")
+        .execute(&mut **transaction)
+        .await
+        .map_err(ApiError::internal)?;
+    Ok(())
+}
+
+pub async fn insert_source_state(
+    transaction: &mut Transaction<'_, Postgres>,
+    batch: &LogBatch,
+) -> Result<(), ApiError> {
+    sqlx::query(
+        "INSERT INTO ingest_source_state (machine_id, hostname) VALUES ($1, $2) \
+         ON CONFLICT (machine_id) DO NOTHING",
+    )
+    .bind(&batch.machine_id)
+    .bind(&batch.hostname)
+    .execute(&mut **transaction)
+    .await
+    .map_err(ApiError::internal)?;
+    Ok(())
+}
+
+pub async fn select_committed_cursor(
+    transaction: &mut Transaction<'_, Postgres>,
+    machine_id: &str,
+) -> Result<Option<String>, ApiError> {
+    sqlx::query_scalar(
+        "SELECT last_cursor FROM ingest_source_state WHERE machine_id = $1 FOR UPDATE",
+    )
+    .bind(machine_id)
+    .fetch_one(&mut **transaction)
+    .await
+    .map_err(ApiError::internal)
+}
+
+pub async fn create_daily_partition(
+    transaction: &mut Transaction<'_, Postgres>,
+    date: NaiveDate,
+    next: NaiveDate,
+) -> Result<(), ApiError> {
+    let name = partition_name(date);
+    let statement = format!(
+        "CREATE TABLE IF NOT EXISTS {name} PARTITION OF haproxy_logs \
+         FOR VALUES FROM ('{date}') TO ('{next}')"
+    );
+    sqlx::query(AssertSqlSafe(statement.as_str()))
+        .execute(&mut **transaction)
+        .await
+        .map_err(ApiError::internal)?;
+    Ok(())
+}
+
+fn partition_name(date: NaiveDate) -> String {
+    format!("haproxy_logs_{}", date.format("%Y%m%d"))
+}
+
+struct PreparedEvent<'a> {
+    event: &'a HaProxyEvent,
+    journal_timestamp: DateTime<Utc>,
+    client_ip: Option<String>,
+    client_port: Option<i32>,
+    bytes_sent: i64,
+    retries: i32,
+    active_connections: Option<i32>,
+    frontend_connections: Option<i32>,
+    backend_connections: Option<i32>,
+    server_connections: Option<i32>,
+    server_queue: Option<i32>,
+    backend_queue: Option<i32>,
+}
+
+fn prepare_events(batch: &LogBatch) -> Result<Vec<PreparedEvent<'_>>, ApiError> {
+    let mut prepared = Vec::new();
+    for record in &batch.records {
+        let Some(log_record::Payload::Event(event)) = &record.payload else {
+            continue;
+        };
+        prepared.push(PreparedEvent {
+            event,
+            journal_timestamp: timestamp(record.journal_timestamp_us)?,
+            client_ip: event
+                .client_ip
+                .as_deref()
+                .and_then(|value| value.parse::<IpAddr>().ok())
+                .map(|value| value.to_string()),
+            client_port: event
+                .client_port
+                .map(i32::try_from)
+                .transpose()
+                .map_err(ApiError::internal)?,
+            bytes_sent: i64::try_from(event.bytes_sent).map_err(ApiError::internal)?,
+            retries: i32::try_from(event.retries).map_err(ApiError::internal)?,
+            active_connections: sql_i32(event.active_connections).map_err(ApiError::internal)?,
+            frontend_connections: sql_i32(event.frontend_connections)
+                .map_err(ApiError::internal)?,
+            backend_connections: sql_i32(event.backend_connections).map_err(ApiError::internal)?,
+            server_connections: sql_i32(event.server_connections).map_err(ApiError::internal)?,
+            server_queue: sql_i32(event.server_queue).map_err(ApiError::internal)?,
+            backend_queue: sql_i32(event.backend_queue).map_err(ApiError::internal)?,
+        });
+    }
+    Ok(prepared)
+}
+
+#[allow(clippy::too_many_lines)]
+pub async fn insert_events(
+    transaction: &mut Transaction<'_, Postgres>,
+    batch: &LogBatch,
+    source_ip: IpAddr,
+) -> Result<(), ApiError> {
+    let prepared = prepare_events(batch)?;
+    let source_ip = source_ip.to_string();
+    for chunk in prepared.chunks(EVENT_INSERT_CHUNK_SIZE) {
+        let mut query = QueryBuilder::<Postgres>::new(
+            "INSERT INTO haproxy_logs (machine_id,hostname,source_ip,journal_timestamp,client_ip,\
+             client_port,frontend_name,backend_name,server_name,http_method,request_path,http_version,\
+             status_code,bytes_sent,time_request_ms,time_wait_ms,time_connect_ms,time_response_ms,\
+             time_total_ms,termination_state,retries,tls_sni,tls_version,tls_cipher,tls_resumed,\
+             normalization_flags,active_connections,frontend_connections,\
+             backend_connections,server_connections,server_queue,backend_queue) ",
+        );
+        query.push_values(chunk, |mut values, row| {
+            values
+                .push_bind(&batch.machine_id)
+                .push_bind(&batch.hostname)
+                .push_bind(source_ip.as_str())
+                .push_unseparated("::inet")
+                .push_bind(row.journal_timestamp)
+                .push_bind(row.client_ip.as_deref())
+                .push_unseparated("::inet")
+                .push_bind(row.client_port)
+                .push_bind(row.event.frontend_name.as_deref())
+                .push_bind(row.event.backend_name.as_deref())
+                .push_bind(row.event.server_name.as_deref())
+                .push_bind(row.event.http_method.as_deref())
+                .push_bind(row.event.request_path.as_deref())
+                .push_bind(row.event.http_version.as_deref())
+                .push_bind(row.event.status_code)
+                .push_bind(row.bytes_sent)
+                .push_bind(row.event.time_request_ms)
+                .push_bind(row.event.time_wait_ms)
+                .push_bind(row.event.time_connect_ms)
+                .push_bind(row.event.time_response_ms)
+                .push_bind(row.event.time_total_ms)
+                .push_bind(row.event.termination_state.as_deref())
+                .push_bind(row.retries)
+                .push_bind(row.event.tls_sni.as_deref())
+                .push_bind(row.event.tls_version.as_deref())
+                .push_bind(row.event.tls_cipher.as_deref())
+                .push_bind(row.event.tls_resumed)
+                .push_bind(row.event.normalization_flags.as_slice())
+                .push_bind(row.active_connections)
+                .push_bind(row.frontend_connections)
+                .push_bind(row.backend_connections)
+                .push_bind(row.server_connections)
+                .push_bind(row.server_queue)
+                .push_bind(row.backend_queue);
+        });
+        query
+            .build()
+            .execute(&mut **transaction)
+            .await
+            .map_err(ApiError::internal)?;
+    }
+    Ok(())
+}
+
+pub async fn insert_rejected_event(
+    transaction: &mut Transaction<'_, Postgres>,
+    batch: &LogBatch,
+    source_ip: IpAddr,
+    journal_timestamp: DateTime<Utc>,
+    rejected: &RejectedEvent,
+) -> Result<(), ApiError> {
+    sqlx::query(
+        "INSERT INTO ingest_rejected_events (machine_id,hostname,journal_timestamp,reason,\
+         message_sha256,raw_message_prefix,raw_message_truncated,source_ip) \
+         VALUES ($1,$2,$3,$4,$5,$6,$7,$8::inet)",
+    )
+    .bind(&batch.machine_id)
+    .bind(&batch.hostname)
+    .bind(journal_timestamp)
+    .bind(&rejected.reason)
+    .bind(&rejected.message_sha256)
+    .bind(&rejected.raw_message_prefix)
+    .bind(rejected.raw_message_truncated)
+    .bind(source_ip.to_string())
+    .execute(&mut **transaction)
+    .await
+    .map_err(ApiError::internal)?;
+    Ok(())
+}
+
+pub async fn insert_journal_gap(
+    transaction: &mut Transaction<'_, Postgres>,
+    batch: &LogBatch,
+    source_ip: IpAddr,
+    first_available_timestamp: Option<DateTime<Utc>>,
+    gap: &JournalGap,
+) -> Result<(), ApiError> {
+    sqlx::query(
+        "INSERT INTO ingest_journal_gaps (machine_id,hostname,expected_cursor,\
+         first_available_cursor,first_available_timestamp,reason,source_ip) \
+         VALUES ($1,$2,$3,$4,$5,$6,$7::inet)",
+    )
+    .bind(&batch.machine_id)
+    .bind(&batch.hostname)
+    .bind(&gap.expected_cursor)
+    .bind(&gap.first_available_cursor)
+    .bind(first_available_timestamp)
+    .bind(&gap.reason)
+    .bind(source_ip.to_string())
+    .execute(&mut **transaction)
+    .await
+    .map_err(ApiError::internal)?;
+    Ok(())
+}
+
+pub async fn insert_ingest_batch(
+    transaction: &mut Transaction<'_, Postgres>,
+    batch: &LogBatch,
+    source_ip: IpAddr,
+    compressed_bytes: usize,
+    uncompressed_bytes: usize,
+) -> Result<(), ApiError> {
+    let first_timestamp = timestamp(batch.first_journal_timestamp_us)?;
+    let last_timestamp = timestamp(batch.last_journal_timestamp_us)?;
+    let batch_id = batch_id(batch);
+    let record_count = i32::try_from(batch.records.len()).map_err(ApiError::internal)?;
+    let compressed_bytes = i64::try_from(compressed_bytes).map_err(ApiError::internal)?;
+    let uncompressed_bytes = i64::try_from(uncompressed_bytes).map_err(ApiError::internal)?;
+    sqlx::query(
+        "INSERT INTO ingest_batches (batch_id, machine_id, hostname, source_ip, previous_cursor, \
+         last_cursor, first_journal_timestamp, last_journal_timestamp, record_count, \
+         compressed_bytes, uncompressed_bytes) \
+         VALUES ($1,$2,$3,$4::inet,$5,$6,$7,$8,$9,$10,$11) ON CONFLICT (batch_id) DO NOTHING",
+    )
+    .bind(&batch_id)
+    .bind(&batch.machine_id)
+    .bind(&batch.hostname)
+    .bind(source_ip.to_string())
+    .bind(&batch.previous_cursor)
+    .bind(&batch.last_cursor)
+    .bind(first_timestamp)
+    .bind(last_timestamp)
+    .bind(record_count)
+    .bind(compressed_bytes)
+    .bind(uncompressed_bytes)
+    .execute(&mut **transaction)
+    .await
+    .map_err(ApiError::internal)?;
+    Ok(())
+}
+
+pub async fn update_source_state(
+    transaction: &mut Transaction<'_, Postgres>,
+    batch: &LogBatch,
+    source_ip: IpAddr,
+) -> Result<(), ApiError> {
+    let last_timestamp = timestamp(batch.last_journal_timestamp_us)?;
+    sqlx::query(
+        "UPDATE ingest_source_state SET hostname=$2, last_cursor=$3, \
+         last_journal_timestamp=$4, last_source_ip=$5::inet, last_commit_time=now() \
+         WHERE machine_id=$1",
+    )
+    .bind(&batch.machine_id)
+    .bind(&batch.hostname)
+    .bind(&batch.last_cursor)
+    .bind(last_timestamp)
+    .bind(source_ip.to_string())
+    .execute(&mut **transaction)
+    .await
+    .map_err(ApiError::internal)?;
+    Ok(())
+}
+
+fn batch_id(batch: &LogBatch) -> String {
+    let mut hash = Sha256::new();
+    hash.update(batch.machine_id.as_bytes());
+    hash.update([0]);
+    if let Some(previous) = &batch.previous_cursor {
+        hash.update(previous.as_bytes());
+    }
+    hash.update([0]);
+    hash.update(batch.last_cursor.as_bytes());
+    hex::encode(hash.finalize())
+}
+
+fn sql_i32(value: Option<u32>) -> Result<Option<i32>, std::num::TryFromIntError> {
+    value.map(i32::try_from).transpose()
+}
+
+#[cfg(test)]
+#[allow(clippy::unwrap_used)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn partition_names_are_date_only() {
+        let date = NaiveDate::from_ymd_opt(2026, 8, 29).unwrap();
+        assert_eq!(partition_name(date), "haproxy_logs_20260829");
+    }
+}

rust/haproxy_ingest_server/src/main.rs

@@ -0,0 +1,530 @@
+use std::{
+    collections::BTreeSet,
+    io::Read,
+    net::{IpAddr, SocketAddr},
+};
+
+use anyhow::{Context, Result, bail};
+use axum::{
+    Router,
+    body::Bytes,
+    extract::{ConnectInfo, DefaultBodyLimit, Path, State},
+    http::{HeaderMap, StatusCode, header},
+    response::{IntoResponse, Response},
+    routing::{get as get_route, post},
+};
+use chrono::{DateTime, Days, Utc};
+use clap::Parser;
+use haproxy_ingest_proto::{
+    MAX_COMPRESSED_BYTES, MAX_DECOMPRESSED_BYTES,
+    proto::{LogBatch, log_record},
+};
+use prost::Message;
+use serde::Serialize;
+use sha2::{Digest, Sha256};
+use sqlx::{PgPool, Postgres, Transaction, postgres::PgPoolOptions};
+use subtle::ConstantTimeEq;
+use tower_http::trace::TraceLayer;
+use tracing::{info, warn};
+use tracing_subscriber::EnvFilter;
+
+mod db;
+mod validation;
+
+use validation::{ValidationError, validate_batch};
+
+#[derive(Debug, Parser)]
+#[command(about, version)]
+struct Args {
+    #[arg(long, env = "HAPROXY_INGEST_LISTEN", default_value = "0.0.0.0:8444")]
+    listen: SocketAddr,
+
+    #[arg(long, env = "HAPROXY_INGEST_DATABASE_URL", hide_env_values = true)]
+    database_url: String,
+
+    #[arg(long, env = "HAPROXY_INGEST_TOKEN", hide_env_values = true)]
+    token: String,
+}
+
+#[derive(Clone)]
+struct AppState {
+    pool: PgPool,
+    auth: SourceToken,
+}
+
+#[derive(Clone)]
+struct SourceToken {
+    hash: [u8; 32],
+}
+
+impl SourceToken {
+    fn parse(value: &str) -> Result<Self> {
+        let token = value.trim();
+        if token.len() < 32 {
+            bail!("source token must be at least 32 characters");
+        }
+        Ok(Self {
+            hash: Sha256::digest(token.as_bytes()).into(),
+        })
+    }
+
+    fn authenticate(&self, headers: &HeaderMap) -> Result<(), ApiError> {
+        let authorization = headers
+            .get(header::AUTHORIZATION)
+            .and_then(|value| value.to_str().ok())
+            .and_then(|value| value.strip_prefix("Bearer "))
+            .ok_or_else(|| ApiError::new(StatusCode::UNAUTHORIZED, "missing bearer token"))?;
+        let presented: [u8; 32] = Sha256::digest(authorization.as_bytes()).into();
+        (presented.ct_eq(&self.hash).unwrap_u8() == 1)
+            .then_some(())
+            .ok_or_else(|| ApiError::new(StatusCode::UNAUTHORIZED, "invalid bearer token"))
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct StatusResponse {
+    machine_id: String,
+    hostname: Option<String>,
+    last_cursor: Option<String>,
+    last_journal_timestamp: Option<DateTime<Utc>>,
+    last_commit_time: Option<DateTime<Utc>>,
+}
+
+#[derive(Debug, Serialize)]
+struct CommitResponse {
+    machine_id: String,
+    last_cursor: String,
+    records_committed: usize,
+    already_committed: bool,
+}
+
+#[derive(Debug, Serialize)]
+struct ErrorResponse {
+    error: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    record_index: Option<usize>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    last_cursor: Option<String>,
+}
+
+#[derive(Debug)]
+struct ApiError {
+    status: StatusCode,
+    message: String,
+    record_index: Option<usize>,
+    last_cursor: Option<String>,
+}
+
+impl ApiError {
+    fn new(status: StatusCode, message: impl Into<String>) -> Self {
+        Self {
+            status,
+            message: message.into(),
+            record_index: None,
+            last_cursor: None,
+        }
+    }
+
+    fn invalid(error: &ValidationError) -> Self {
+        Self {
+            status: StatusCode::UNPROCESSABLE_ENTITY,
+            record_index: error.record_index(),
+            message: error.to_string(),
+            last_cursor: None,
+        }
+    }
+
+    fn conflict(last_cursor: Option<String>) -> Self {
+        Self {
+            status: StatusCode::CONFLICT,
+            message: "batch previous_cursor does not match the committed cursor".to_owned(),
+            record_index: None,
+            last_cursor,
+        }
+    }
+
+    fn internal(error: impl std::fmt::Display) -> Self {
+        warn!(error = %error, "ingestion request failed");
+        Self::new(
+            StatusCode::SERVICE_UNAVAILABLE,
+            "ingestion backend unavailable",
+        )
+    }
+}
+
+impl IntoResponse for ApiError {
+    fn into_response(self) -> Response {
+        (
+            self.status,
+            axum::Json(ErrorResponse {
+                error: self.message,
+                record_index: self.record_index,
+                last_cursor: self.last_cursor,
+            }),
+        )
+            .into_response()
+    }
+}
+
+#[derive(Debug, Eq, PartialEq)]
+enum CursorDecision {
+    Accept,
+    AlreadyCommitted,
+    Conflict,
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    tracing_subscriber::fmt()
+        .json()
+        .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()))
+        .init();
+    let args = Args::parse();
+    let pool = PgPoolOptions::new()
+        .max_connections(8)
+        .connect(&args.database_url)
+        .await
+        .context("failed to connect to PostgreSQL")?;
+
+    db::apply_ingestion_schema(&pool).await?;
+
+    let state = AppState {
+        pool,
+        auth: SourceToken::parse(&args.token)?,
+    };
+
+    let app = Router::new()
+        .route("/v1/logs/status/{machine_id}", get_route(get_status))
+        .route("/v1/logs/batch", post(post_batch))
+        .layer(DefaultBodyLimit::max(MAX_COMPRESSED_BYTES))
+        .layer(TraceLayer::new_for_http())
+        .with_state(state);
+    let listener = tokio::net::TcpListener::bind(args.listen)
+        .await
+        .context("failed to bind ingestion HTTP listener")?;
+
+    info!(listen = %args.listen, "HAProxy ingestion API listening over HTTP");
+    axum::serve(
+        listener,
+        app.into_make_service_with_connect_info::<SocketAddr>(),
+    )
+    .await
+    .context("ingestion server failed")
+}
+
+async fn get_status(
+    State(state): State<AppState>,
+    Path(machine_id): Path<String>,
+    headers: HeaderMap,
+) -> Result<axum::Json<StatusResponse>, ApiError> {
+    state.auth.authenticate(&headers)?;
+    let status = db::select_source_status(&state.pool, &machine_id).await?;
+    let response = match status {
+        None => StatusResponse {
+            machine_id: machine_id.clone(),
+            hostname: None,
+            last_cursor: None,
+            last_journal_timestamp: None,
+            last_commit_time: None,
+        },
+        Some(status) => StatusResponse {
+            machine_id: machine_id.clone(),
+            hostname: status.hostname,
+            last_cursor: status.last_cursor,
+            last_journal_timestamp: status.last_journal_timestamp,
+            last_commit_time: status.last_commit_time,
+        },
+    };
+    Ok(axum::Json(response))
+}
+
+async fn post_batch(
+    State(state): State<AppState>,
+    ConnectInfo(peer): ConnectInfo<SocketAddr>,
+    headers: HeaderMap,
+    body: Bytes,
+) -> Result<axum::Json<CommitResponse>, ApiError> {
+    state.auth.authenticate(&headers)?;
+    check_protobuf(&headers)?;
+    check_zstd(&headers)?;
+    let compressed_bytes = body.len();
+    let decoded = decompress_bounded(&body)?;
+    let uncompressed_bytes = decoded.len();
+    let batch = LogBatch::decode(decoded.as_slice()).map_err(|error| {
+        ApiError::new(
+            StatusCode::BAD_REQUEST,
+            format!("invalid protobuf: {error}"),
+        )
+    })?;
+    validate_batch(&batch).map_err(|error| ApiError::invalid(&error))?;
+    let response = commit_batch(
+        &state.pool,
+        &batch,
+        peer.ip(),
+        compressed_bytes,
+        uncompressed_bytes,
+    )
+    .await?;
+    Ok(axum::Json(response))
+}
+
+fn check_protobuf(headers: &HeaderMap) -> Result<(), ApiError> {
+    if headers
+        .get(header::CONTENT_TYPE)
+        .and_then(|value| value.to_str().ok())
+        != Some("application/x-protobuf")
+    {
+        return Err(ApiError::new(
+            StatusCode::UNSUPPORTED_MEDIA_TYPE,
+            "Content-Type must be application/x-protobuf",
+        ));
+    }
+    Ok(())
+}
+
+fn check_zstd(headers: &HeaderMap) -> Result<(), ApiError> {
+    if headers
+        .get(header::CONTENT_ENCODING)
+        .and_then(|value| value.to_str().ok())
+        != Some("zstd")
+    {
+        return Err(ApiError::new(
+            StatusCode::UNSUPPORTED_MEDIA_TYPE,
+            "Content-Encoding must be zstd",
+        ));
+    }
+    Ok(())
+}
+
+fn decompress_bounded(body: &[u8]) -> Result<Vec<u8>, ApiError> {
+    let decoder = zstd::stream::read::Decoder::new(body).map_err(|error| {
+        ApiError::new(
+            StatusCode::BAD_REQUEST,
+            format!("invalid zstd body: {error}"),
+        )
+    })?;
+    let mut output = Vec::new();
+    let read_limit = u64::try_from(MAX_DECOMPRESSED_BYTES)
+        .map_err(ApiError::internal)?
+        .saturating_add(1);
+    decoder
+        .take(read_limit)
+        .read_to_end(&mut output)
+        .map_err(|error| {
+            ApiError::new(
+                StatusCode::BAD_REQUEST,
+                format!("invalid zstd body: {error}"),
+            )
+        })?;
+    if output.len() > MAX_DECOMPRESSED_BYTES {
+        return Err(ApiError::new(
+            StatusCode::PAYLOAD_TOO_LARGE,
+            "decompressed protobuf exceeds 20 MiB",
+        ));
+    }
+    Ok(output)
+}
+
+async fn commit_batch(
+    pool: &PgPool,
+    batch: &LogBatch,
+    source_ip: IpAddr,
+    compressed_bytes: usize,
+    uncompressed_bytes: usize,
+) -> Result<CommitResponse, ApiError> {
+    let mut transaction = pool.begin().await.map_err(ApiError::internal)?;
+    db::set_lock_timeout(&mut transaction).await?;
+    db::set_statement_timeout(&mut transaction).await?;
+    db::insert_source_state(&mut transaction, batch).await?;
+    let committed_cursor = db::select_committed_cursor(&mut transaction, &batch.machine_id).await?;
+    match classify_cursor_chain(
+        committed_cursor.as_deref(),
+        batch.previous_cursor.as_deref(),
+        &batch.last_cursor,
+    ) {
+        CursorDecision::Accept => {}
+        CursorDecision::AlreadyCommitted => {
+            transaction.commit().await.map_err(ApiError::internal)?;
+            return Ok(CommitResponse {
+                machine_id: batch.machine_id.clone(),
+                last_cursor: batch.last_cursor.clone(),
+                records_committed: batch.records.len(),
+                already_committed: true,
+            });
+        }
+        CursorDecision::Conflict => return Err(ApiError::conflict(committed_cursor)),
+    }
+
+    ensure_daily_partitions(&mut transaction, batch).await?;
+    db::insert_events(&mut transaction, batch, source_ip).await?;
+    insert_non_event_records(&mut transaction, batch, source_ip).await?;
+    db::insert_ingest_batch(
+        &mut transaction,
+        batch,
+        source_ip,
+        compressed_bytes,
+        uncompressed_bytes,
+    )
+    .await?;
+    db::update_source_state(&mut transaction, batch, source_ip).await?;
+    transaction.commit().await.map_err(ApiError::internal)?;
+    info!(
+        machine_id = %batch.machine_id,
+        records = batch.records.len(),
+        last_cursor = %batch.last_cursor,
+        "committed HAProxy log batch"
+    );
+    Ok(CommitResponse {
+        machine_id: batch.machine_id.clone(),
+        last_cursor: batch.last_cursor.clone(),
+        records_committed: batch.records.len(),
+        already_committed: false,
+    })
+}
+
+async fn ensure_daily_partitions(
+    transaction: &mut Transaction<'_, Postgres>,
+    batch: &LogBatch,
+) -> Result<(), ApiError> {
+    let mut dates = BTreeSet::new();
+    for record in &batch.records {
+        if matches!(record.payload, Some(log_record::Payload::Event(_))) {
+            dates.insert(timestamp(record.journal_timestamp_us)?.date_naive());
+        }
+    }
+    for date in dates {
+        let next = date.checked_add_days(Days::new(1)).ok_or_else(|| {
+            ApiError::new(StatusCode::UNPROCESSABLE_ENTITY, "partition date overflow")
+        })?;
+        db::create_daily_partition(transaction, date, next).await?;
+    }
+    Ok(())
+}
+
+async fn insert_non_event_records(
+    transaction: &mut Transaction<'_, Postgres>,
+    batch: &LogBatch,
+    source_ip: IpAddr,
+) -> Result<(), ApiError> {
+    for record in &batch.records {
+        match &record.payload {
+            Some(log_record::Payload::Rejected(rejected)) => {
+                let journal_timestamp = timestamp(record.journal_timestamp_us)?;
+                db::insert_rejected_event(
+                    transaction,
+                    batch,
+                    source_ip,
+                    journal_timestamp,
+                    rejected,
+                )
+                .await?;
+            }
+            Some(log_record::Payload::Gap(gap)) => {
+                let first_available_timestamp = gap
+                    .first_available_timestamp_us
+                    .map(timestamp)
+                    .transpose()?;
+                db::insert_journal_gap(
+                    transaction,
+                    batch,
+                    source_ip,
+                    first_available_timestamp,
+                    gap,
+                )
+                .await?;
+            }
+            Some(log_record::Payload::Event(_)) | None => {}
+        }
+    }
+    Ok(())
+}
+
+fn timestamp(micros: i64) -> Result<DateTime<Utc>, ApiError> {
+    DateTime::from_timestamp_micros(micros).ok_or_else(|| {
+        ApiError::new(
+            StatusCode::UNPROCESSABLE_ENTITY,
+            "journal timestamp is outside supported range",
+        )
+    })
+}
+
+fn classify_cursor_chain(
+    committed: Option<&str>,
+    previous: Option<&str>,
+    last: &str,
+) -> CursorDecision {
+    if committed == Some(last) {
+        CursorDecision::AlreadyCommitted
+    } else if committed == previous {
+        CursorDecision::Accept
+    } else {
+        CursorDecision::Conflict
+    }
+}
+
+#[cfg(test)]
+#[allow(clippy::unwrap_used)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn starts_without_certificate_arguments() {
+        Args::try_parse_from([
+            "haproxy-ingest-server",
+            "--database-url",
+            "postgres://localhost/haproxy_ingest",
+            "--token",
+            "0123456789abcdef0123456789abcdef",
+        ])
+        .unwrap();
+    }
+
+    #[test]
+    fn shared_token_authenticates_requests() {
+        let token = SourceToken::parse("0123456789abcdef0123456789abcdef").unwrap();
+        let mut headers = HeaderMap::new();
+        headers.insert(
+            header::AUTHORIZATION,
+            "Bearer 0123456789abcdef0123456789abcdef".parse().unwrap(),
+        );
+        assert!(token.authenticate(&headers).is_ok());
+        headers.insert(
+            header::AUTHORIZATION,
+            "Bearer fedcba9876543210fedcba9876543210".parse().unwrap(),
+        );
+        assert!(token.authenticate(&headers).is_err());
+    }
+
+    #[test]
+    fn decompression_limit_is_enforced() {
+        let input = vec![0_u8; MAX_DECOMPRESSED_BYTES + 1];
+        let compressed = zstd::stream::encode_all(input.as_slice(), 1).unwrap();
+        assert!(matches!(
+            decompress_bounded(&compressed),
+            Err(ApiError {
+                status: StatusCode::PAYLOAD_TOO_LARGE,
+                ..
+            })
+        ));
+    }
+
+    #[test]
+    fn cursor_chain_handles_bootstrap_continuation_duplicate_and_conflict() {
+        assert_eq!(
+            classify_cursor_chain(None, None, "c1"),
+            CursorDecision::Accept
+        );
+        assert_eq!(
+            classify_cursor_chain(Some("c1"), Some("c1"), "c2"),
+            CursorDecision::Accept
+        );
+        assert_eq!(
+            classify_cursor_chain(Some("c2"), Some("c1"), "c2"),
+            CursorDecision::AlreadyCommitted
+        );
+        assert_eq!(
+            classify_cursor_chain(Some("other"), Some("c1"), "c2"),
+            CursorDecision::Conflict
+        );
+    }
+}

rust/haproxy_ingest_server/src/validation.rs

@@ -0,0 +1,214 @@
+use std::net::IpAddr;
+
+use chrono::{DateTime, Utc};
+use haproxy_ingest_proto::{
+    SCHEMA_VERSION, SERVER_MAX_RECORDS,
+    proto::{LogBatch, LogRecord, log_record},
+};
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum ValidationError {
+    #[error("unsupported schema_version {0}")]
+    SchemaVersion(u32),
+    #[error("machine_id must be 32 lowercase hexadecimal characters")]
+    MachineId,
+    #[error("hostname is empty or longer than 255 bytes")]
+    Hostname,
+    #[error("last_cursor is empty")]
+    LastCursor,
+    #[error("batch contains no records or more than {SERVER_MAX_RECORDS} records")]
+    RecordCount,
+    #[error("batch journal timestamp bounds are invalid")]
+    TimestampBounds,
+    #[error("record {index}: {reason}")]
+    Record { index: usize, reason: String },
+}
+
+impl ValidationError {
+    #[must_use]
+    pub const fn record_index(&self) -> Option<usize> {
+        match self {
+            Self::Record { index, .. } => Some(*index),
+            _ => None,
+        }
+    }
+}
+
+/// Validates protocol-level and per-record invariants before database access.
+///
+/// # Errors
+///
+/// Returns [`ValidationError`] when the batch cannot be committed safely.
+pub fn validate_batch(batch: &LogBatch) -> Result<(), ValidationError> {
+    if batch.schema_version != SCHEMA_VERSION {
+        return Err(ValidationError::SchemaVersion(batch.schema_version));
+    }
+    if batch.machine_id.len() != 32
+        || !batch
+            .machine_id
+            .bytes()
+            .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
+    {
+        return Err(ValidationError::MachineId);
+    }
+    if batch.hostname.is_empty() || batch.hostname.len() > 255 {
+        return Err(ValidationError::Hostname);
+    }
+    if batch.last_cursor.is_empty() {
+        return Err(ValidationError::LastCursor);
+    }
+    if batch.records.is_empty() || batch.records.len() > SERVER_MAX_RECORDS {
+        return Err(ValidationError::RecordCount);
+    }
+    // Journal cursors define record order; wall-clock timestamps may move backward.
+    if batch
+        .records
+        .first()
+        .map(|record| record.journal_timestamp_us)
+        != Some(batch.first_journal_timestamp_us)
+        || batch
+            .records
+            .last()
+            .map(|record| record.journal_timestamp_us)
+            != Some(batch.last_journal_timestamp_us)
+    {
+        return Err(ValidationError::TimestampBounds);
+    }
+
+    for (index, record) in batch.records.iter().enumerate() {
+        validate_record(index, record)?;
+    }
+    Ok(())
+}
+
+fn validate_record(index: usize, record: &LogRecord) -> Result<(), ValidationError> {
+    if DateTime::<Utc>::from_timestamp_micros(record.journal_timestamp_us).is_none() {
+        return record_error(
+            index,
+            "journal timestamp is outside PostgreSQL's supported range",
+        );
+    }
+    let Some(payload) = &record.payload else {
+        return record_error(index, "payload is absent");
+    };
+    match payload {
+        log_record::Payload::Event(event) => {
+            if event.bytes_sent > i64::MAX.cast_unsigned() {
+                return record_error(index, "bytes_sent exceeds PostgreSQL BIGINT");
+            }
+            if event.retries > i32::MAX.cast_unsigned()
+                || [
+                    event.active_connections,
+                    event.frontend_connections,
+                    event.backend_connections,
+                    event.server_connections,
+                    event.server_queue,
+                    event.backend_queue,
+                ]
+                .into_iter()
+                .flatten()
+                .any(|value| value > i32::MAX.cast_unsigned())
+            {
+                return record_error(index, "counter exceeds PostgreSQL INTEGER");
+            }
+            if event.client_port.is_some_and(|port| port > u16::MAX.into()) {
+                return record_error(index, "client_port exceeds 65535");
+            }
+            if let Some(ip) = &event.client_ip
+                && ip.parse::<IpAddr>().is_err()
+            {
+                return record_error(index, "client_ip is invalid");
+            }
+        }
+        log_record::Payload::Rejected(rejected) => {
+            if rejected.reason.is_empty() || rejected.message_sha256.len() != 32 {
+                return record_error(index, "rejected event reason or SHA-256 is invalid");
+            }
+        }
+        log_record::Payload::Gap(gap) => {
+            if gap.expected_cursor.is_empty()
+                || gap.first_available_cursor.is_empty()
+                || gap.reason.is_empty()
+            {
+                return record_error(index, "journal gap fields are incomplete");
+            }
+        }
+    }
+    Ok(())
+}
+
+fn record_error<T>(index: usize, reason: &str) -> Result<T, ValidationError> {
+    Err(ValidationError::Record {
+        index,
+        reason: reason.to_owned(),
+    })
+}
+
+#[cfg(test)]
+#[allow(clippy::indexing_slicing, clippy::unwrap_used)]
+mod tests {
+    use haproxy_ingest_proto::proto::{HaProxyEvent, LogRecord, log_record};
+
+    use super::*;
+
+    fn batch() -> LogBatch {
+        LogBatch {
+            schema_version: 1,
+            machine_id: "0123456789abcdef0123456789abcdef".to_owned(),
+            hostname: "portal-1".to_owned(),
+            previous_cursor: None,
+            last_cursor: "cursor-1".to_owned(),
+            first_journal_timestamp_us: 1_700_000_000_000_000,
+            last_journal_timestamp_us: 1_700_000_000_000_000,
+            records: vec![LogRecord {
+                journal_timestamp_us: 1_700_000_000_000_000,
+                payload: Some(log_record::Payload::Event(HaProxyEvent::default())),
+            }],
+        }
+    }
+
+    #[test]
+    fn accepts_a_valid_batch() {
+        validate_batch(&batch()).unwrap();
+    }
+
+    #[test]
+    fn accepts_a_batch_with_decreasing_journal_timestamps() {
+        let mut batch = batch();
+        batch.previous_cursor = Some("cursor-0".to_owned());
+        batch.last_cursor = "cursor-2".to_owned();
+        batch.last_journal_timestamp_us = 1_699_999_999_000_000;
+        batch.records.push(LogRecord {
+            journal_timestamp_us: batch.last_journal_timestamp_us,
+            payload: Some(log_record::Payload::Event(HaProxyEvent::default())),
+        });
+
+        validate_batch(&batch).unwrap();
+    }
+
+    #[test]
+    fn rejects_batch_timestamps_that_do_not_match_the_endpoint_records() {
+        for (first, last) in [
+            (1_699_999_999_000_000, 1_700_000_000_000_000),
+            (1_700_000_000_000_000, 1_700_000_001_000_000),
+        ] {
+            let mut batch = batch();
+            batch.first_journal_timestamp_us = first;
+            batch.last_journal_timestamp_us = last;
+
+            assert!(matches!(
+                validate_batch(&batch),
+                Err(ValidationError::TimestampBounds)
+            ));
+        }
+    }
+
+    #[test]
+    fn reports_the_invalid_record_index() {
+        let mut batch = batch();
+        batch.records[0].payload = None;
+        let error = validate_batch(&batch).unwrap_err();
+        assert_eq!(error.record_index(), Some(0));
+    }
+}

rust/haproxy_log_shipper/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "haproxy-log-shipper"
+version = "0.1.0"
+edition = "2024"
+description = "Ordered HAProxy journald ingestion client"
+license = "MIT"
+
+[lints]
+workspace = true
+
+[dependencies]
+anyhow.workspace = true
+chrono = { workspace = true, features = ["serde"] }
+clap.workspace = true
+garde.workspace = true
+haproxy-ingest-proto = { path = "../haproxy_ingest_proto" }
+prost.workspace = true
+reqwest = { workspace = true, features = ["json"] }
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+systemd.workspace = true
+tls.workspace = true
+tokio.workspace = true
+tokio-retry.workspace = true
+tracing.workspace = true
+tracing-subscriber.workspace = true
+zstd.workspace = true

rust/haproxy_log_shipper/package.nix

@@ -0,0 +1,24 @@
+{
+  lib,
+  pkg-config,
+  rustPlatform,
+  systemd,
+}:
+rustPlatform.buildRustPackage {
+  pname = "haproxy-log-shipper";
+  version = "0.1.0";
+  src = import ../rust-source.nix { inherit lib; };
+
+  cargoLock.lockFile = ../../Cargo.lock;
+  cargoBuildFlags = [ "-p haproxy-log-shipper" ];
+  cargoTestFlags = [ "-p haproxy-log-shipper" ];
+
+  nativeBuildInputs = [ pkg-config ];
+  buildInputs = [ systemd ];
+
+  meta = {
+    description = "Ordered HAProxy journald ingestion client";
+    license = lib.licenses.mit;
+    platforms = lib.platforms.linux;
+  };
+}

rust/haproxy_log_shipper/src/api.rs

@@ -0,0 +1,360 @@
+use std::time::Duration;
+
+use anyhow::{Context, Result, bail};
+use haproxy_ingest_proto::{
+    SCHEMA_VERSION,
+    proto::{LogBatch, LogRecord, log_record},
+};
+use prost::Message;
+use reqwest::{Client, StatusCode, header};
+use serde::Deserialize;
+use tokio_retry::strategy::{ExponentialBackoff, jitter};
+use tracing::{Instrument, info, warn};
+
+use crate::mapper::rejected_event;
+
+const INITIAL_RETRIES: usize = 3;
+
+#[derive(Debug, Deserialize)]
+pub struct StatusResponse {
+    machine_id: String,
+    pub last_cursor: Option<String>,
+    last_journal_timestamp: Option<chrono::DateTime<chrono::Utc>>,
+}
+
+#[derive(Debug, Deserialize)]
+struct CommitResponse {
+    last_cursor: String,
+    records_committed: usize,
+    already_committed: bool,
+}
+
+#[derive(Debug, Deserialize)]
+struct ErrorResponse {
+    error: String,
+    record_index: Option<usize>,
+    last_cursor: Option<String>,
+}
+
+#[derive(Clone, Debug)]
+pub struct PendingRecord {
+    pub record: LogRecord,
+    pub cursor: String,
+    pub raw_message: Option<String>,
+}
+
+#[derive(Debug)]
+pub struct PendingBatch {
+    pub machine_id: String,
+    pub hostname: String,
+    pub previous_cursor: Option<String>,
+    pub records: Vec<PendingRecord>,
+}
+
+impl PendingBatch {
+    fn last_cursor(&self) -> Result<&str> {
+        self.records
+            .last()
+            .map(|record| record.cursor.as_str())
+            .context("pending batch is empty")
+    }
+
+    pub fn to_wire(&self) -> Result<LogBatch> {
+        let first_journal_timestamp_us = self
+            .records
+            .first()
+            .context("pending batch is empty")?
+            .record
+            .journal_timestamp_us;
+        let last_journal_timestamp_us = self
+            .records
+            .last()
+            .context("pending batch is empty")?
+            .record
+            .journal_timestamp_us;
+        Ok(LogBatch {
+            schema_version: SCHEMA_VERSION,
+            machine_id: self.machine_id.clone(),
+            hostname: self.hostname.clone(),
+            previous_cursor: self.previous_cursor.clone(),
+            last_cursor: self.last_cursor()?.to_owned(),
+            first_journal_timestamp_us,
+            last_journal_timestamp_us,
+            records: self
+                .records
+                .iter()
+                .map(|pending| pending.record.clone())
+                .collect(),
+        })
+    }
+
+    fn replace_with_rejection(&mut self, index: usize, reason: &str) -> Result<()> {
+        let pending = self
+            .records
+            .get_mut(index)
+            .with_context(|| format!("receiver returned out-of-range record index {index}"))?;
+        if matches!(
+            pending.record.payload,
+            Some(log_record::Payload::Rejected(_))
+        ) {
+            bail!("receiver rejected an already-isolated RejectedEvent at index {index}");
+        }
+        let raw = pending
+            .raw_message
+            .as_deref()
+            .context("receiver rejected a journal gap marker")?;
+        pending.record.payload = Some(log_record::Payload::Rejected(rejected_event(
+            raw,
+            &format!("receiver validation rejected record: {reason}"),
+        )));
+        Ok(())
+    }
+}
+
+pub enum SendResult {
+    Committed(String),
+    Resync,
+}
+
+enum PostBatchResult {
+    Complete(SendResult),
+    RetryNow,
+    RetryAfterStatusCheck,
+}
+
+enum StatusRequestResult {
+    Complete(StatusResponse),
+    Retry,
+}
+
+pub async fn post_batch_with_retry(
+    client: &Client,
+    endpoint: &str,
+    pending: &mut PendingBatch,
+) -> Result<SendResult> {
+    for (attempt, delay) in retry_schedule().enumerate() {
+        if let Some(delay) = delay {
+            tokio::time::sleep(delay).await;
+        }
+        let last_cursor = loop {
+            let batch = pending.to_wire()?;
+            let last_cursor = batch.last_cursor.clone();
+            let encoded = batch.encode_to_vec();
+            let compressed = zstd::stream::encode_all(encoded.as_slice(), 3)
+                .context("failed to compress protobuf batch")?;
+            let result = post_batch(client, endpoint, pending, compressed, &last_cursor)
+                .instrument(tracing::info_span!("post_batch", attempt))
+                .await?;
+            match result {
+                PostBatchResult::Complete(send_result) => return Ok(send_result),
+                PostBatchResult::RetryNow => {}
+                PostBatchResult::RetryAfterStatusCheck => break last_cursor,
+            }
+        };
+
+        if let Ok(status) = get_status_with_retry(client, endpoint, &pending.machine_id).await {
+            if status.last_cursor.as_deref() == Some(last_cursor.as_str()) {
+                info!(last_cursor, "status check confirmed ambiguous batch commit");
+                return Ok(SendResult::Committed(last_cursor));
+            }
+            if status.last_cursor != pending.previous_cursor {
+                return Ok(SendResult::Resync);
+            }
+        }
+    }
+    bail!("batch failed after initial attempt and {INITIAL_RETRIES} retries")
+}
+
+async fn post_batch(
+    client: &Client,
+    endpoint: &str,
+    pending: &mut PendingBatch,
+    compressed: Vec<u8>,
+    last_cursor: &str,
+) -> Result<PostBatchResult> {
+    let response = client
+        .post(format!("{endpoint}/v1/logs/batch"))
+        .header(header::CONTENT_TYPE, "application/x-protobuf")
+        .header(header::CONTENT_ENCODING, "zstd")
+        .body(compressed)
+        .send()
+        .await;
+
+    match response {
+        Ok(response) if response.status().is_success() => {
+            let committed: CommitResponse = response
+                .json()
+                .await
+                .context("receiver returned an invalid commit response")?;
+            if committed.last_cursor != last_cursor
+                || committed.records_committed != pending.records.len()
+            {
+                bail!("receiver commit response did not match the sent batch");
+            }
+            info!(
+                already_committed = committed.already_committed,
+                "batch acknowledged"
+            );
+            Ok(PostBatchResult::Complete(SendResult::Committed(
+                last_cursor.to_owned(),
+            )))
+        }
+        Ok(response) if response.status() == StatusCode::CONFLICT => {
+            let error = decode_error(response).await;
+            warn!(server_cursor = ?error.last_cursor, error = %error.error, "receiver rejected cursor chain");
+            Ok(PostBatchResult::Complete(SendResult::Resync))
+        }
+        Ok(response) if response.status() == StatusCode::PAYLOAD_TOO_LARGE => {
+            let error = decode_error(response).await;
+            bail!(
+                "receiver rejected a batch within the sender's configured limits as too large: {}",
+                error.error
+            )
+        }
+        Ok(response) if response.status() == StatusCode::UNPROCESSABLE_ENTITY => {
+            let error = decode_error(response).await;
+            let index = error
+                .record_index
+                .context("receiver returned 422 without an offending record index")?;
+            pending.replace_with_rejection(index, &error.error)?;
+            warn!(record_index = index, error = %error.error, "isolated receiver-rejected record");
+            Ok(PostBatchResult::RetryNow)
+        }
+        Ok(response)
+            if matches!(
+                response.status(),
+                StatusCode::UNAUTHORIZED
+                    | StatusCode::FORBIDDEN
+                    | StatusCode::BAD_REQUEST
+                    | StatusCode::UNSUPPORTED_MEDIA_TYPE
+            ) =>
+        {
+            let status = response.status();
+            let error = decode_error(response).await;
+            bail!("permanent receiver error {status}: {}", error.error);
+        }
+        Ok(response) if !response.status().is_server_error() => {
+            let status = response.status();
+            let error = decode_error(response).await;
+            bail!("unexpected receiver response {status}: {}", error.error);
+        }
+        Ok(response) => {
+            warn!(status = %response.status(), "transient receiver failure");
+            Ok(PostBatchResult::RetryAfterStatusCheck)
+        }
+        Err(error) => {
+            warn!(error = %error, "ambiguous batch request failure");
+            Ok(PostBatchResult::RetryAfterStatusCheck)
+        }
+    }
+}
+
+pub async fn get_status_with_retry(
+    client: &Client,
+    endpoint: &str,
+    machine_id: &str,
+) -> Result<StatusResponse> {
+    for (attempt, delay) in retry_schedule().enumerate() {
+        if let Some(delay) = delay {
+            tokio::time::sleep(delay).await;
+        }
+        let result = get_status(client, endpoint, machine_id)
+            .instrument(tracing::info_span!("get_status", attempt))
+            .await?;
+        match result {
+            StatusRequestResult::Complete(status) => return Ok(status),
+            StatusRequestResult::Retry => {}
+        }
+    }
+    bail!("status request failed after initial attempt and {INITIAL_RETRIES} retries")
+}
+
+async fn get_status(
+    client: &Client,
+    endpoint: &str,
+    machine_id: &str,
+) -> Result<StatusRequestResult> {
+    let response = match client
+        .get(format!("{endpoint}/v1/logs/status/{machine_id}"))
+        .send()
+        .await
+    {
+        Ok(response) => response,
+        Err(error) => {
+            warn!(error = %error, "transient receiver status failure");
+            return Ok(StatusRequestResult::Retry);
+        }
+    };
+    if !response.status().is_success() {
+        let status = response.status();
+        let error = decode_error(response).await;
+        if status.is_server_error() {
+            warn!(status = %status, error = %error.error, "transient receiver status failure");
+            return Ok(StatusRequestResult::Retry);
+        }
+        bail!(
+            "receiver status request failed with {status}: {}",
+            error.error
+        );
+    }
+    let status: StatusResponse = response
+        .json()
+        .await
+        .context("receiver returned an invalid status response")?;
+    if status.machine_id != machine_id {
+        bail!("receiver status response contains the wrong machine_id");
+    }
+    info!(
+        last_cursor = ?status.last_cursor,
+        last_journal_timestamp = ?status.last_journal_timestamp,
+        "loaded receiver checkpoint"
+    );
+    Ok(StatusRequestResult::Complete(status))
+}
+
+fn retry_delays() -> impl Iterator<Item = Duration> {
+    // The library raises the base to each attempt, then multiplies by the factor.
+    // 2^n * 500 milliseconds gives delays of 1, 2, and 4 seconds.
+    ExponentialBackoff::from_millis(2)
+        .factor(500)
+        .take(INITIAL_RETRIES)
+}
+
+fn retry_schedule() -> impl Iterator<Item = Option<Duration>> {
+    std::iter::once(None).chain(retry_delays().map(jitter).map(Some))
+}
+
+async fn decode_error(response: reqwest::Response) -> ErrorResponse {
+    response.json().await.unwrap_or_else(|error| ErrorResponse {
+        error: format!("response body was not valid error JSON: {error}"),
+        record_index: None,
+        last_cursor: None,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn retries_use_one_two_and_four_second_delays() {
+        assert_eq!(
+            retry_delays().collect::<Vec<_>>(),
+            [
+                Duration::from_secs(1),
+                Duration::from_secs(2),
+                Duration::from_secs(4),
+            ]
+        );
+    }
+
+    #[test]
+    fn retry_schedule_starts_immediately_and_bounds_jitter() {
+        let mut schedule = retry_schedule();
+        assert_eq!(schedule.next(), Some(None));
+        for limit in retry_delays() {
+            assert!(matches!(schedule.next(), Some(Some(delay)) if delay <= limit));
+        }
+        assert_eq!(schedule.next(), None);
+    }
+}

rust/haproxy_log_shipper/src/main.rs

@@ -0,0 +1,271 @@
+use std::{fs, time::Duration};
+
+use anyhow::{Context, Result, anyhow, bail};
+use clap::Parser;
+use haproxy_ingest_proto::{
+    MAX_BATCH_PROTO_BYTES, MAX_BATCH_RECORDS,
+    proto::{JournalGap, LogRecord, log_record},
+};
+use prost::Message;
+use reqwest::{Client, header};
+use systemd::journal::{Journal, OpenOptions};
+use tls::install_tls_provider;
+use tracing::{info, warn};
+use tracing_subscriber::EnvFilter;
+
+mod api;
+mod mapper;
+
+use api::{PendingBatch, PendingRecord, SendResult, get_status_with_retry, post_batch_with_retry};
+use mapper::map_message;
+
+#[derive(Debug, Parser)]
+#[command(about, version)]
+struct Args {
+    #[arg(long, env = "HAPROXY_INGEST_TOKEN", hide_env_values = true)]
+    token: String,
+
+    #[arg(long, env = "HAPROXY_INGEST_ENDPOINT")]
+    endpoint: String,
+
+    #[arg(long, env = "HAPROXY_INGEST_UNIT", default_value = "haproxy.service")]
+    unit: String,
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    tracing_subscriber::fmt()
+        .json()
+        .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()))
+        .init();
+    let args = Args::parse();
+    install_tls_provider()?;
+    let machine_id = get_valid_machine_id()?;
+    let hostname = read_trimmed("/etc/hostname")?;
+    let client = build_client(&args.token)?;
+
+    let mut committed_cursor = get_status_with_retry(&client, &args.endpoint, &machine_id)
+        .await?
+        .last_cursor;
+    loop {
+        let Some(mut batch) =
+            read_batch(&machine_id, &hostname, committed_cursor.clone(), &args.unit)?
+        else {
+            info!(machine_id, "no uncommitted HAProxy journal records");
+            return Ok(());
+        };
+
+        committed_cursor = match post_batch_with_retry(&client, &args.endpoint, &mut batch).await? {
+            SendResult::Committed(cursor) => {
+                info!(
+                    records = batch.records.len(),
+                    last_cursor = %cursor,
+                    "receiver committed HAProxy batch"
+                );
+                Some(cursor)
+            }
+            SendResult::Resync => {
+                let status = get_status_with_retry(&client, &args.endpoint, &machine_id).await?;
+
+                warn!(
+                    server_cursor = ?status.last_cursor,
+                    "cursor conflict; rebuilding from receiver checkpoint"
+                );
+
+                status.last_cursor
+            }
+        };
+    }
+}
+
+fn build_client(token: &str) -> Result<Client> {
+    let authorization = format!("Bearer {token}")
+        .parse()
+        .context("HAPROXY_INGEST_TOKEN contains invalid HTTP header characters")?;
+    Client::builder()
+        .connect_timeout(Duration::from_secs(3))
+        .timeout(Duration::from_secs(10))
+        .default_headers({
+            let mut headers = reqwest::header::HeaderMap::new();
+            headers.insert(header::AUTHORIZATION, authorization);
+            headers
+        })
+        .build()
+        .context("failed to build HTTP client")
+}
+
+fn read_batch(
+    machine_id: &str,
+    hostname: &str,
+    previous_cursor: Option<String>,
+    unit: &str,
+) -> Result<Option<PendingBatch>> {
+    let (mut journal, setup) = setup_journal(machine_id, unit, previous_cursor.as_deref())?;
+    let mut pending_batch = PendingBatch {
+        machine_id: machine_id.to_owned(),
+        hostname: hostname.to_owned(),
+        previous_cursor,
+        records: Vec::new(),
+    };
+    match setup {
+        JournalSetup::Empty => return Ok(None),
+        JournalSetup::Ready => {}
+        JournalSetup::Gap(pending_record) => pending_batch.records.push(*pending_record),
+    }
+    while pending_batch.records.len() < MAX_BATCH_RECORDS {
+        let Some(entry) = journal
+            .next_entry()
+            .context("failed to read journal entry")?
+        else {
+            break;
+        };
+        let cursor = journal.cursor().context("journal entry has no cursor")?;
+        let journal_timestamp_us = i64::try_from(
+            journal
+                .timestamp_usec()
+                .context("journal entry has no timestamp")?,
+        )
+        .context("journal timestamp exceeds int64")?;
+        let message = entry.get("MESSAGE").cloned().unwrap_or_default();
+        pending_batch.records.push(PendingRecord {
+            record: map_message(&message, journal_timestamp_us),
+            cursor,
+            raw_message: Some(message),
+        });
+        if pending_batch.to_wire()?.encoded_len() > MAX_BATCH_PROTO_BYTES {
+            if pending_batch.records.len() == 1 {
+                bail!("single mapped journal record exceeds the 20 MiB protobuf limit");
+            }
+            pending_batch.records.pop();
+            break;
+        }
+    }
+    if pending_batch.records.is_empty() {
+        Ok(None)
+    } else {
+        Ok(Some(pending_batch))
+    }
+}
+
+enum JournalSetup {
+    Empty,
+    Ready,
+    Gap(Box<PendingRecord>),
+}
+
+fn setup_journal(
+    machine_id: &str,
+    unit: &str,
+    previous_cursor: Option<&str>,
+) -> Result<(Journal, JournalSetup)> {
+    let mut journal = OpenOptions::default()
+        .system(true)
+        .local_only(true)
+        .open()
+        .context("failed to open the local system journal")?;
+    journal
+        .match_add("_SYSTEMD_UNIT", unit.as_bytes())
+        .context("failed to filter journal by systemd unit")?;
+    journal
+        .match_add("_MACHINE_ID", machine_id.as_bytes())
+        .context("failed to filter journal by machine ID")?;
+    let setup = position_journal(&mut journal, previous_cursor)?;
+    Ok((journal, setup))
+}
+
+fn position_journal(journal: &mut Journal, previous_cursor: Option<&str>) -> Result<JournalSetup> {
+    let Some(cursor) = previous_cursor else {
+        journal
+            .seek_head()
+            .context("failed to seek to journal head")?;
+        return Ok(JournalSetup::Ready);
+    };
+    let exact = journal.seek_cursor(cursor).is_ok()
+        && journal
+            .next()
+            .context("failed to move to saved journal cursor")?
+            > 0
+        && journal
+            .test_cursor(cursor)
+            .context("failed to test saved journal cursor")?;
+    if exact {
+        return Ok(JournalSetup::Ready);
+    }
+
+    journal
+        .seek_head()
+        .context("failed to seek to retained journal head")?;
+    if journal
+        .next()
+        .context("failed to read retained journal head")?
+        == 0
+    {
+        return Ok(JournalSetup::Empty);
+    }
+    let first_cursor = journal
+        .cursor()
+        .context("retained journal entry has no cursor")?;
+    let first_timestamp = i64::try_from(
+        journal
+            .timestamp_usec()
+            .context("retained journal entry has no timestamp")?,
+    )
+    .context("journal timestamp exceeds int64")?;
+    let gap = PendingRecord {
+        record: LogRecord {
+            journal_timestamp_us: first_timestamp,
+            payload: Some(log_record::Payload::Gap(JournalGap {
+                expected_cursor: cursor.to_owned(),
+                first_available_cursor: first_cursor.clone(),
+                first_available_timestamp_us: Some(first_timestamp),
+                reason: "saved cursor is no longer present in the filtered journal".to_owned(),
+            })),
+        },
+        cursor: first_cursor,
+        raw_message: None,
+    };
+    journal
+        .seek_head()
+        .context("failed to rewind retained journal")?;
+    Ok(JournalSetup::Gap(Box::new(gap)))
+}
+
+fn get_valid_machine_id() -> Result<String> {
+    let machine_id = read_trimmed("/etc/machine-id")?;
+    if machine_id.len() != 32
+        || !machine_id
+            .bytes()
+            .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
+    {
+        bail!("/etc/machine-id is not 32 lowercase hexadecimal characters");
+    }
+    Ok(machine_id)
+}
+
+fn read_trimmed(path: &str) -> Result<String> {
+    let value = fs::read_to_string(path).with_context(|| format!("failed to read {path}"))?;
+    let value = value.trim();
+    if value.is_empty() {
+        return Err(anyhow!("{path} is empty"));
+    }
+    Ok(value.to_owned())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn builds_http_client_without_certificate() -> Result<()> {
+        let args = Args::try_parse_from([
+            "haproxy-log-shipper",
+            "--endpoint",
+            "http://127.0.0.1:8444",
+            "--token",
+            "test-token",
+        ])?;
+        install_tls_provider()?;
+        build_client(&args.token)?;
+        Ok(())
+    }
+}

rust/haproxy_log_shipper/src/mapper.rs

@@ -0,0 +1,354 @@
+use std::{fmt::Display, net::IpAddr, str::FromStr};
+
+use anyhow::{Context, Result};
+use garde::{Unvalidated, Valid};
+use haproxy_ingest_proto::proto::{HaProxyEvent, LogRecord, RejectedEvent, log_record};
+use serde::{Deserialize, Deserializer, de::Error as _};
+use sha2::{Digest, Sha256};
+
+const RAW_PREFIX_LIMIT: usize = 4 * 1024;
+
+#[derive(Debug, Deserialize, garde::Validate)]
+#[garde(allow_unvalidated)]
+struct RawHaProxyEvent {
+    #[serde(default, deserialize_with = "deserialize_optional_string")]
+    client_ip: Option<String>,
+    #[serde(default, deserialize_with = "deserialize_optional_number")]
+    #[garde(range(max = 65_535))]
+    client_port: Option<u32>,
+    #[serde(default, deserialize_with = "deserialize_optional_string")]
+    frontend_name: Option<String>,
+    #[serde(default, deserialize_with = "deserialize_optional_string")]
+    backend_name: Option<String>,
+    #[serde(default, deserialize_with = "deserialize_optional_string")]
+    server_name: Option<String>,
+    #[serde(default, deserialize_with = "deserialize_optional_string")]
+    http_method: Option<String>,
+    #[serde(default, deserialize_with = "deserialize_optional_string")]
+    request_path: Option<String>,
+    #[serde(default, deserialize_with = "deserialize_optional_string")]
+    http_version: Option<String>,
+    #[serde(default, deserialize_with = "deserialize_optional_number")]
+    #[garde(required, range(min = 100, max = 599))]
+    status_code: Option<i32>,
+    #[serde(default, deserialize_with = "deserialize_optional_number")]
+    #[garde(required, range(max = 9_223_372_036_854_775_807))]
+    bytes_sent: Option<u64>,
+    #[serde(default, deserialize_with = "deserialize_optional_number")]
+    #[garde(required, range(min = -1))]
+    time_request_ms: Option<i32>,
+    #[serde(default, deserialize_with = "deserialize_optional_number")]
+    #[garde(required, range(min = -1))]
+    time_wait_ms: Option<i32>,
+    #[serde(default, deserialize_with = "deserialize_optional_number")]
+    #[garde(required, range(min = -1))]
+    time_connect_ms: Option<i32>,
+    #[serde(default, deserialize_with = "deserialize_optional_number")]
+    #[garde(required, range(min = -1))]
+    time_response_ms: Option<i32>,
+    #[serde(default, deserialize_with = "deserialize_optional_number")]
+    #[garde(required, range(min = -1))]
+    time_total_ms: Option<i32>,
+    #[serde(default, deserialize_with = "deserialize_optional_string")]
+    termination_state: Option<String>,
+    #[serde(default, deserialize_with = "deserialize_optional_number")]
+    #[garde(range(max = 2_147_483_647))]
+    retries: Option<u32>,
+    #[serde(default, deserialize_with = "deserialize_optional_string")]
+    tls_sni: Option<String>,
+    #[serde(default, deserialize_with = "deserialize_optional_string")]
+    tls_version: Option<String>,
+    #[serde(default, deserialize_with = "deserialize_optional_string")]
+    tls_cipher: Option<String>,
+    #[serde(default, deserialize_with = "deserialize_optional_bool")]
+    tls_resumed: Option<bool>,
+    #[serde(default, deserialize_with = "deserialize_optional_number")]
+    #[garde(range(max = 2_147_483_647))]
+    active_connections: Option<u32>,
+    #[serde(default, deserialize_with = "deserialize_optional_number")]
+    #[garde(range(max = 2_147_483_647))]
+    frontend_connections: Option<u32>,
+    #[serde(default, deserialize_with = "deserialize_optional_number")]
+    #[garde(range(max = 2_147_483_647))]
+    backend_connections: Option<u32>,
+    #[serde(default, deserialize_with = "deserialize_optional_number")]
+    #[garde(range(max = 2_147_483_647))]
+    server_connections: Option<u32>,
+    #[serde(default, deserialize_with = "deserialize_optional_number")]
+    #[garde(range(max = 2_147_483_647))]
+    server_queue: Option<u32>,
+    #[serde(default, deserialize_with = "deserialize_optional_number")]
+    #[garde(range(max = 2_147_483_647))]
+    backend_queue: Option<u32>,
+}
+
+#[must_use]
+pub fn map_message(message: &str, journal_timestamp_us: i64) -> LogRecord {
+    let payload = match parse_event(message) {
+        Ok(event) => log_record::Payload::Event(event),
+        Err(error) => log_record::Payload::Rejected(rejected_event(message, &format!("{error:#}"))),
+    };
+    LogRecord {
+        journal_timestamp_us,
+        payload: Some(payload),
+    }
+}
+
+fn parse_event(message: &str) -> Result<HaProxyEvent> {
+    let event: Unvalidated<RawHaProxyEvent> =
+        serde_json::from_str(message).context("invalid HAProxy JSON")?;
+    let event = event.validate().context("invalid HAProxy event")?;
+    normalize_event(event)
+}
+
+fn normalize_event(event: Valid<RawHaProxyEvent>) -> Result<HaProxyEvent> {
+    let event = event.into_inner();
+    let mut normalization_flags = Vec::new();
+    let client_ip = event.client_ip.and_then(|raw| {
+        raw.parse::<IpAddr>().map_or_else(
+            |_| {
+                normalization_flags.push("invalid_client_ip".to_owned());
+                None
+            },
+            |ip| Some(ip.to_string()),
+        )
+    });
+
+    Ok(HaProxyEvent {
+        client_ip,
+        client_port: event.client_port,
+        frontend_name: event.frontend_name,
+        backend_name: event.backend_name,
+        server_name: event.server_name,
+        http_method: event.http_method,
+        request_path: event.request_path,
+        http_version: event.http_version,
+        status_code: event
+            .status_code
+            .context("validated status_code is absent")?,
+        bytes_sent: event.bytes_sent.context("validated bytes_sent is absent")?,
+        time_request_ms: event
+            .time_request_ms
+            .context("validated time_request_ms is absent")?,
+        time_wait_ms: event
+            .time_wait_ms
+            .context("validated time_wait_ms is absent")?,
+        time_connect_ms: event
+            .time_connect_ms
+            .context("validated time_connect_ms is absent")?,
+        time_response_ms: event
+            .time_response_ms
+            .context("validated time_response_ms is absent")?,
+        time_total_ms: event
+            .time_total_ms
+            .context("validated time_total_ms is absent")?,
+        termination_state: event.termination_state,
+        retries: event.retries.unwrap_or_default(),
+        tls_sni: event.tls_sni,
+        tls_version: event.tls_version,
+        tls_cipher: event.tls_cipher,
+        tls_resumed: event.tls_resumed,
+        normalization_flags,
+        active_connections: event.active_connections,
+        frontend_connections: event.frontend_connections,
+        backend_connections: event.backend_connections,
+        server_connections: event.server_connections,
+        server_queue: event.server_queue,
+        backend_queue: event.backend_queue,
+    })
+}
+
+#[must_use]
+pub fn rejected_event(message: &str, reason: &str) -> RejectedEvent {
+    let mut hasher = Sha256::new();
+    hasher.update(message.as_bytes());
+    let prefix = truncate_utf8(message, RAW_PREFIX_LIMIT);
+    RejectedEvent {
+        reason: truncate_utf8(reason, 1_024).to_owned(),
+        message_sha256: hasher.finalize().to_vec(),
+        raw_message_prefix: Some(prefix.to_owned()),
+        raw_message_truncated: prefix.len() != message.len(),
+    }
+}
+
+fn deserialize_optional_string<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    match Option::<String>::deserialize(deserializer)? {
+        Some(value) if is_empty_or_dash(&value) => Ok(None),
+        value => Ok(value),
+    }
+}
+
+#[derive(Deserialize)]
+#[serde(untagged)]
+enum NumberOrString<T> {
+    Number(T),
+    String(String),
+}
+
+fn deserialize_optional_number<'de, D, T>(deserializer: D) -> Result<Option<T>, D::Error>
+where
+    D: Deserializer<'de>,
+    T: Deserialize<'de> + FromStr,
+    T::Err: Display,
+{
+    match Option::<NumberOrString<T>>::deserialize(deserializer)? {
+        None => Ok(None),
+        Some(NumberOrString::Number(value)) => Ok(Some(value)),
+        Some(NumberOrString::String(value)) if is_empty_or_dash(&value) => Ok(None),
+        Some(NumberOrString::String(value)) => value.parse().map(Some).map_err(D::Error::custom),
+    }
+}
+
+#[derive(Deserialize)]
+#[serde(untagged)]
+enum BoolValue {
+    Bool(bool),
+    Integer(i64),
+    String(String),
+}
+
+fn deserialize_optional_bool<'de, D>(deserializer: D) -> Result<Option<bool>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    match Option::<BoolValue>::deserialize(deserializer)? {
+        None => Ok(None),
+        Some(BoolValue::Bool(value)) => Ok(Some(value)),
+        Some(BoolValue::Integer(0)) => Ok(Some(false)),
+        Some(BoolValue::Integer(1)) => Ok(Some(true)),
+        Some(BoolValue::String(value)) if is_empty_or_dash(&value) => Ok(None),
+        Some(BoolValue::String(value)) if matches!(value.as_str(), "0" | "false") => {
+            Ok(Some(false))
+        }
+        Some(BoolValue::String(value)) if matches!(value.as_str(), "1" | "true") => Ok(Some(true)),
+        Some(BoolValue::Integer(_) | BoolValue::String(_)) => {
+            Err(D::Error::custom("expected a boolean, 0, 1, null, or '-'"))
+        }
+    }
+}
+
+fn is_empty_or_dash(value: &str) -> bool {
+    value.is_empty() || value == "-"
+}
+
+fn truncate_utf8(value: &str, max_bytes: usize) -> &str {
+    if value.len() <= max_bytes {
+        return value;
+    }
+    let mut end = max_bytes;
+    while !value.is_char_boundary(end) {
+        end = end.saturating_sub(1);
+    }
+    value.get(..end).unwrap_or_default()
+}
+
+#[cfg(test)]
+#[allow(clippy::panic, clippy::unwrap_used)]
+mod tests {
+    use haproxy_ingest_proto::proto::log_record;
+
+    use super::*;
+
+    fn valid_json(extra: &str) -> String {
+        format!(
+            r#"{{"client_ip":"192.0.2.10","status_code":200,"bytes_sent":12,"time_request_ms":-1,"time_wait_ms":0,"time_connect_ms":3,"time_response_ms":4,"time_total_ms":7{extra}}}"#
+        )
+    }
+
+    #[test]
+    fn preserves_negative_timer_sentinels() {
+        let record = map_message(&valid_json(""), 10);
+        let Some(log_record::Payload::Event(event)) = record.payload else {
+            panic!("expected mapped event");
+        };
+        assert_eq!(event.time_request_ms, -1);
+    }
+
+    #[test]
+    fn serde_parses_the_full_u64_range() {
+        let message =
+            valid_json("").replace("\"bytes_sent\":12", "\"bytes_sent\":18446744073709551615");
+        let event: RawHaProxyEvent = serde_json::from_str(&message).unwrap();
+        assert_eq!(event.bytes_sent, Some(u64::MAX));
+    }
+
+    #[test]
+    fn serde_coerces_haproxy_values() {
+        let message = r#"{
+            "client_port":"443",
+            "frontend_name":"-",
+            "status_code":"200",
+            "bytes_sent":"12",
+            "time_request_ms":"-1",
+            "time_wait_ms":"0",
+            "time_connect_ms":"3",
+            "time_response_ms":"4",
+            "time_total_ms":"7",
+            "retries":"1",
+            "tls_resumed":"true",
+            "server_queue":null
+        }"#;
+        let record = map_message(message, 10);
+        let Some(log_record::Payload::Event(event)) = record.payload else {
+            panic!("expected mapped event");
+        };
+        assert_eq!(event.client_port, Some(443));
+        assert_eq!(event.frontend_name, None);
+        assert_eq!(event.status_code, 200);
+        assert_eq!(event.time_request_ms, -1);
+        assert_eq!(event.retries, 1);
+        assert_eq!(event.tls_resumed, Some(true));
+        assert_eq!(event.server_queue, None);
+    }
+
+    #[test]
+    fn garde_rejects_invalid_semantics() {
+        let messages = [
+            valid_json("").replace("\"status_code\":200", "\"status_code\":99"),
+            valid_json("").replace("\"time_total_ms\":7", "\"time_total_ms\":-2"),
+            valid_json(",\"client_port\":65536"),
+        ];
+        for message in messages {
+            let record = map_message(&message, 10);
+            let Some(log_record::Payload::Rejected(rejected)) = record.payload else {
+                panic!("expected rejected event");
+            };
+            assert!(rejected.reason.contains("invalid HAProxy event"));
+        }
+    }
+
+    #[test]
+    fn invalid_ip_is_non_fatal_and_flagged() {
+        let message = valid_json("").replace("192.0.2.10", "not-an-ip");
+        let record = map_message(&message, 10);
+        let Some(log_record::Payload::Event(event)) = record.payload else {
+            panic!("expected mapped event");
+        };
+        assert_eq!(event.client_ip, None);
+        assert_eq!(event.normalization_flags, ["invalid_client_ip"]);
+    }
+
+    #[test]
+    fn preserves_request_path() {
+        let path = format!("/{}é", "a".repeat(20_000));
+        let message = valid_json(&format!(",\"request_path\":{path:?}"));
+        let record = map_message(&message, 10);
+        let Some(log_record::Payload::Event(event)) = record.payload else {
+            panic!("expected mapped event");
+        };
+        assert_eq!(event.request_path.as_deref(), Some(path.as_str()));
+    }
+
+    #[test]
+    fn malformed_event_becomes_durable_rejection() {
+        let record = map_message("not-json", 10);
+        let Some(log_record::Payload::Rejected(rejected)) = record.payload else {
+            panic!("expected rejected event");
+        };
+        assert_eq!(rejected.message_sha256.len(), 32);
+        assert!(rejected.reason.contains("invalid HAProxy JSON"));
+    }
+}

rust/rust-source.nix

@@ -0,0 +1,10 @@
+{ lib }:
+lib.fileset.toSource {
+  root = ../.;
+  fileset = lib.fileset.unions [
+    ../Cargo.toml
+    ../Cargo.lock
+    ../clippy.toml
+    ./.
+  ];
+}

rust/signal/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "signal"
+version = "0.1.0"
+edition = "2024"
+description = "Send Signal alerts through the local signal-cli JSON-RPC daemon"
+license = "MIT"
+publish = false
+
+[dependencies]
+anyhow.workspace = true
+serde_json.workspace = true
+
+[target.'cfg(target_os = "linux")'.dev-dependencies]
+libc = "0.2"
+
+[lints]
+workspace = true

rust/signal/src/lib.rs

@@ -0,0 +1,78 @@
+//! Send alerts through the local signal-cli daemon on Unix systems.
+//!
+//! ```no_run
+//! use signal::Signal;
+//!
+//! let client = Signal {
+//!     from_phone: "+15555550100".into(),
+//! };
+//! client.send("+15555550101", "Service failed")?;
+//! client.send("+15555550102", "Service recovered")?;
+//! # Ok::<(), anyhow::Error>(())
+//! ```
+
+use std::{
+    io::{BufRead, BufReader, Write},
+    os::unix::net::UnixStream,
+    time::Duration,
+};
+
+use anyhow::{Context, Result, bail};
+use serde_json::{Value, json};
+
+/// A sending account for alerts through `/run/signal-cli/socket`.
+pub struct Signal {
+    /// The phone number of an account registered with the local daemon.
+    pub from_phone: String,
+}
+
+impl Signal {
+    /// Send a message to `to_phone` and wait for the corresponding JSON-RPC response.
+    ///
+    /// Socket reads and writes use a four-second timeout.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if connecting, writing, or reading fails, the daemon
+    /// returns invalid JSON or an RPC error, or it closes before responding.
+    pub fn send(&self, to_phone: &str, body: &str) -> Result<()> {
+        let mut stream = UnixStream::connect("/run/signal-cli/socket")
+            .context("Failed to connect to Signal RPC socket")?;
+        let timeout = Duration::from_secs(4);
+        stream.set_read_timeout(Some(timeout))?;
+        stream.set_write_timeout(Some(timeout))?;
+        let request = json!({
+            "jsonrpc": "2.0",
+            "method": "send",
+            "params": {
+                "account": self.from_phone,
+                "recipient": [to_phone],
+                "message": body,
+            },
+            "id": "signal-alert",
+        });
+        writeln!(stream, "{request}").context("Failed to send Signal RPC request")?;
+        let mut reader = BufReader::new(stream);
+        let mut line = String::new();
+        loop {
+            line.clear();
+            if reader
+                .read_line(&mut line)
+                .context("Failed to read Signal RPC response")?
+                == 0
+                || !line.ends_with('\n')
+            {
+                bail!("Signal RPC socket closed before returning a response");
+            }
+            let response: Value =
+                serde_json::from_str(&line).context("Invalid Signal RPC response")?;
+            if response.get("id").and_then(Value::as_str) != Some("signal-alert") {
+                continue;
+            }
+            if let Some(error) = response.get("error").filter(|value| !value.is_null()) {
+                bail!("Signal RPC returned an error: {error}");
+            }
+            return Ok(());
+        }
+    }
+}

rust/signal/tests/send.rs

@@ -0,0 +1,147 @@
+#![cfg(test)]
+#![cfg(target_os = "linux")]
+
+use std::{
+    cell::RefCell,
+    io::{self, BufRead, BufReader, Write},
+    os::{fd::AsRawFd, unix::net::UnixStream},
+    thread,
+    time::{Duration, Instant},
+};
+
+use serde_json::{Value, json};
+use signal::Signal;
+
+thread_local! {
+    static CONNECTION: RefCell<Option<UnixStream>> = const { RefCell::new(None) };
+}
+
+// Patch libc's connect only in this integration-test executable. Each test
+// supplies its own socket pair; no call can reach the real Signal daemon.
+// SAFETY: This symbol uses libc's connect ABI and preserves its return/errno
+// convention. It never delegates to a real connection attempt.
+#[unsafe(no_mangle)]
+unsafe extern "C" fn connect(
+    socket: libc::c_int,
+    address: *const libc::sockaddr,
+    length: libc::socklen_t,
+) -> libc::c_int {
+    let path_offset = std::mem::offset_of!(libc::sockaddr_un, sun_path);
+    let expected = b"/run/signal-cli/socket\0";
+    if !address.is_null()
+        && let Some(path_end) = path_offset.checked_add(expected.len())
+        && usize::try_from(length).is_ok_and(|length| length >= path_end)
+    {
+        // SAFETY: connect's caller supplies an address valid for length bytes.
+        // The bounds check above covers the entire pathname slice.
+        let path = unsafe {
+            std::slice::from_raw_parts(address.cast::<u8>().add(path_offset), expected.len())
+        };
+        if path == expected
+            && let Some(stream) = CONNECTION.with_borrow_mut(Option::take)
+        {
+            // SAFETY: Both descriptors are live and owned by this thread.
+            // dup2 replaces the unconnected socket with the test endpoint.
+            return if unsafe { libc::dup2(stream.as_raw_fd(), socket) } < 0 {
+                -1
+            } else {
+                0
+            };
+        }
+    }
+    // SAFETY: __errno_location returns this thread's valid errno pointer.
+    unsafe { *libc::__errno_location() = libc::ECONNREFUSED };
+    -1
+}
+
+fn test_connection() -> UnixStream {
+    let (client, server) = UnixStream::pair().unwrap();
+    CONNECTION.with_borrow_mut(|connection| assert!(connection.replace(client).is_none()));
+    server
+}
+
+fn signal() -> Signal {
+    Signal {
+        from_phone: "1234567890".into(),
+    }
+}
+
+#[test]
+fn reuses_sender_for_different_recipients_and_ignores_unrelated_responses() {
+    let client = signal();
+    for (to_phone, body) in [
+        ("0987654321", "failure\nwith details"),
+        ("+15555550102", "another failure"),
+    ] {
+        let server = test_connection();
+        let task = thread::spawn(move || {
+            let mut reader = BufReader::new(server);
+            let mut line = String::new();
+            reader.read_line(&mut line).unwrap();
+            let request: Value = serde_json::from_str(&line).unwrap();
+            assert_eq!(
+                request,
+                json!({
+                    "jsonrpc": "2.0", "method": "send", "id": "signal-alert",
+                    "params": { "account": "1234567890", "recipient": [to_phone], "message": body },
+                })
+            );
+            reader
+                .get_mut()
+                .write_all(b"{\"method\":\"receive\"}\n{\"id\":\"other\"}\n{\"id\":\"signal-")
+                .unwrap();
+            reader
+                .get_mut()
+                .write_all(b"alert\",\"result\":{}}\n")
+                .unwrap();
+        });
+        client.send(to_phone, body).unwrap();
+        task.join().unwrap();
+    }
+}
+
+#[test]
+fn rejects_rpc_errors_invalid_json_and_premature_close() {
+    for (reply, expected) in [
+        (
+            "{\"id\":\"signal-alert\",\"error\":{\"code\":-1}}\n",
+            "Signal RPC returned an error",
+        ),
+        ("invalid\n", "Invalid Signal RPC response"),
+        ("", "socket closed"),
+        ("{\"id\":\"signal-alert\"}", "socket closed"),
+    ] {
+        let server = test_connection();
+        let task = thread::spawn(move || {
+            let mut reader = BufReader::new(server);
+            reader.read_line(&mut String::new()).unwrap();
+            reader.get_mut().write_all(reply.as_bytes()).unwrap();
+        });
+        let error = signal().send("0987654321", "failure").unwrap_err();
+        assert!(format!("{error:#}").contains(expected), "{error:#}");
+        task.join().unwrap();
+    }
+}
+
+#[test]
+fn reports_connection_failure() {
+    let error = signal().send("0987654321", "failure").unwrap_err();
+    assert_eq!(error.to_string(), "Failed to connect to Signal RPC socket");
+    assert_eq!(
+        error.downcast_ref::<io::Error>().unwrap().kind(),
+        io::ErrorKind::ConnectionRefused
+    );
+}
+
+#[test]
+fn unresponsive_server_times_out() {
+    let _server = test_connection();
+    let start = Instant::now();
+    let error = signal().send("0987654321", "failure").unwrap_err();
+    assert_eq!(error.to_string(), "Failed to read Signal RPC response");
+    assert!(matches!(
+        error.downcast_ref::<io::Error>().unwrap().kind(),
+        io::ErrorKind::TimedOut | io::ErrorKind::WouldBlock
+    ));
+    assert!(start.elapsed() >= Duration::from_secs(4));
+}

rust/snapshot_manager/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "snapshot-manager"
+version = "0.1.0"
+edition = "2024"
+description = "Create and rotate ZFS snapshots with Signal failure alerts"
+license = "MIT"
+
+[dependencies]
+anyhow.workspace = true
+chrono.workspace = true
+clap = { workspace = true, features = ["derive", "env"] }
+env_logger.workspace = true
+log.workspace = true
+serde = { workspace = true, features = ["derive"] }
+signal.workspace = true
+toml = "0.9"
+zfs.workspace = true
+
+[dev-dependencies]
+tempfile = "3"
+
+[lints]
+workspace = true

rust/snapshot_manager/package.nix

@@ -0,0 +1,25 @@
+{ lib, rustPlatform }:
+rustPlatform.buildRustPackage {
+  pname = "snapshot-manager";
+  version = "0.1.0";
+  src = import ../rust-source.nix { inherit lib; };
+
+  cargoLock.lockFile = ../../Cargo.lock;
+  cargoBuildFlags = [
+    "-p"
+    "snapshot-manager"
+  ];
+  cargoTestFlags = [
+    "-p"
+    "snapshot-manager"
+    "-p"
+    "signal"
+    "-p"
+    "zfs"
+  ];
+
+  meta = {
+    description = "ZFS snapshot manager with Signal failure alerts";
+    mainProgram = "snapshot-manager";
+  };
+}

rust/snapshot_manager/src/main.rs

@@ -0,0 +1,383 @@
+use std::{
+    collections::HashMap,
+    fs,
+    path::{Path, PathBuf},
+    process::ExitCode,
+};
+
+use anyhow::{Context, Result};
+use chrono::{DateTime, Datelike, NaiveDate, NaiveTime, Timelike, Utc};
+use clap::Parser;
+use log::{error, info};
+use serde::Deserialize;
+use signal::Signal;
+
+#[derive(Parser)]
+#[command(version, about)]
+struct Args {
+    /// TOML file containing default and per-dataset snapshot retention counts.
+    config_file: PathBuf,
+
+    /// Send Signal failure alerts (requires both phone numbers).
+    #[arg(long, env = "SIGNAL_ALERTS", default_value_t = false, action = clap::ArgAction::Set)]
+    signal_alerts: bool,
+
+    /// Signal account sending failure alerts.
+    #[arg(
+        long,
+        env = "SIGNAL_ALERT_FROM_PHONE",
+        hide_env_values = true,
+        required_if_eq("signal_alerts", "true"),
+        value_parser = clap::builder::NonEmptyStringValueParser::new()
+    )]
+    signal_alert_from_phone: Option<String>,
+
+    /// Signal recipient for failure alerts.
+    #[arg(
+        long,
+        env = "SIGNAL_ALERT_TO_PHONE",
+        hide_env_values = true,
+        required_if_eq("signal_alerts", "true"),
+        value_parser = clap::builder::NonEmptyStringValueParser::new()
+    )]
+    signal_alert_to_phone: Option<String>,
+}
+
+#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+struct Retention {
+    #[serde(rename = "15_min")]
+    quarter_hourly: usize,
+    hourly: usize,
+    daily: usize,
+    monthly: usize,
+}
+
+impl Default for Retention {
+    fn default() -> Self {
+        Self {
+            quarter_hourly: 4,
+            hourly: 12,
+            daily: 0,
+            monthly: 0,
+        }
+    }
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(transparent)]
+struct Config(HashMap<String, Retention>);
+
+impl Config {
+    fn load(path: &Path) -> Result<Self> {
+        let contents = fs::read_to_string(path)
+            .with_context(|| format!("Failed to read configuration {}", path.display()))?;
+        toml::from_str(&contents)
+            .with_context(|| format!("Invalid configuration {}", path.display()))
+    }
+
+    fn retention_for(&self, dataset: &str) -> Retention {
+        self.0
+            .get(dataset)
+            .or_else(|| self.0.get("default"))
+            .copied()
+            .unwrap_or_default()
+    }
+}
+
+fn main() -> ExitCode {
+    let args = Args::parse();
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
+    info!("Starting snapshot_manager");
+    let signal = if args.signal_alerts {
+        args.signal_alert_from_phone
+            .map(|from_phone| Signal { from_phone })
+    } else {
+        None
+    };
+    let alerts = signal.as_ref().zip(args.signal_alert_to_phone.as_deref());
+    match run(&args.config_file, alerts) {
+        Ok(()) => {
+            info!("snapshot_manager completed");
+            ExitCode::SUCCESS
+        }
+        Err(error) => {
+            error!("{error:#}");
+            ExitCode::FAILURE
+        }
+    }
+}
+
+fn run(config_file: &Path, alerts: Option<(&Signal, &str)>) -> Result<()> {
+    let result = (|| {
+        // Validate all retention settings before making any changes to ZFS.
+        let config = Config::load(config_file)?;
+        let snapshot_time = timestamp(Utc::now());
+        for dataset_name in zfs::list_datasets()? {
+            if let Err(error) = zfs::create_snapshot(&dataset_name, &snapshot_time) {
+                let message = format!("{dataset_name} failed to create snapshot {snapshot_time}");
+                error!("{message}: {error:#}");
+                send_alert(alerts, &message)?;
+                continue;
+            }
+            remove_old_snapshots(&config, alerts, &dataset_name)
+                .context("Failed to remove old snapshots")?;
+        }
+        Ok(())
+    })();
+    if let Err(error) = &result {
+        error!("snapshot_manager failed: {error:#}");
+        if let Err(alert_error) = send_alert(alerts, "snapshot_manager failed") {
+            error!("Failed to send failure alert: {alert_error:#}");
+        }
+    }
+    result
+}
+
+fn timestamp(now: DateTime<Utc>) -> String {
+    let minute = now.minute().saturating_sub(now.minute() % 15);
+    format!("auto_{}{minute:02}", now.format("%Y%m%d%H"))
+}
+
+fn remove_old_snapshots(
+    config: &Config,
+    alerts: Option<(&Signal, &str)>,
+    dataset_name: &str,
+) -> Result<()> {
+    let retention = config.retention_for(dataset_name);
+    info!("Using {retention:?} for {dataset_name}");
+    let snapshots = zfs::list_snapshots(dataset_name)?;
+    for snapshot in snapshots_to_delete(&snapshots, retention) {
+        info!("Deleting {dataset_name}@{snapshot}");
+        if let Some(error) = zfs::delete_snapshot(dataset_name, snapshot)? {
+            let message = format!("{dataset_name}@{snapshot} failed to delete: {error}");
+            error!("{message}");
+            send_alert(alerts, &message)?;
+        }
+    }
+    Ok(())
+}
+
+fn snapshots_to_delete(snapshots: &[String], retention: Retention) -> Vec<&str> {
+    let mut tiers: [Vec<&str>; 4] = Default::default();
+    for snapshot in snapshots {
+        if let Some(index) = bucket(snapshot)
+            && let Some(tier) = tiers.get_mut(index)
+        {
+            tier.push(snapshot);
+        }
+    }
+
+    let counts = [
+        retention.quarter_hourly,
+        retention.hourly,
+        retention.daily,
+        retention.monthly,
+    ];
+    let mut deletions = Vec::new();
+    for (mut tier, count) in tiers.into_iter().zip(counts) {
+        tier.sort_unstable();
+        let delete_count = tier.len().saturating_sub(count);
+        deletions.extend(tier.into_iter().take(delete_count));
+    }
+    deletions
+}
+
+fn bucket(name: &str) -> Option<usize> {
+    let digits = name.strip_prefix("auto_")?;
+    if digits.len() != 12 || !digits.bytes().all(|byte| byte.is_ascii_digit()) {
+        return None;
+    }
+    let date = NaiveDate::from_ymd_opt(
+        digits.get(..4)?.parse().ok()?,
+        digits.get(4..6)?.parse().ok()?,
+        digits.get(6..8)?.parse().ok()?,
+    )?;
+    let time = NaiveTime::from_hms_opt(
+        digits.get(8..10)?.parse().ok()?,
+        digits.get(10..)?.parse().ok()?,
+        0,
+    )?;
+
+    // The tiers are disjoint: midnight belongs to daily, except on the first
+    // of the month, when it belongs to monthly.
+    match (time.minute(), time.hour(), date.day()) {
+        (15 | 30 | 45, _, _) => Some(0),
+        (0, 1..=23, _) => Some(1),
+        (0, 0, 2..=31) => Some(2),
+        (0, 0, 1) => Some(3),
+        _ => None,
+    }
+}
+
+fn send_alert(alerts: Option<(&Signal, &str)>, body: &str) -> Result<()> {
+    if let Some((signal, to_phone)) = alerts {
+        signal.send(to_phone, body)?;
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use chrono::TimeZone;
+
+    use super::*;
+
+    #[test]
+    fn dataset_overrides_default_and_other_datasets_fall_back() {
+        let config: Config = toml::from_str(
+            r#"
+            [default]
+            15_min = 8
+            hourly = 24
+            daily = 0
+            monthly = 0
+
+            ["pool/home"]
+            15_min = 2
+            hourly = 3
+            daily = 4
+            monthly = 5
+            "#,
+        )
+        .unwrap();
+        assert_eq!(
+            config.retention_for("pool/home"),
+            Retention {
+                quarter_hourly: 2,
+                hourly: 3,
+                daily: 4,
+                monthly: 5,
+            }
+        );
+        assert_eq!(config.retention_for("pool/other").hourly, 24);
+        assert_eq!(config.retention_for("pool/home/child").quarter_hourly, 8);
+    }
+
+    #[test]
+    fn absent_default_uses_original_builtin_counts() {
+        let config: Config = toml::from_str("").unwrap();
+        assert_eq!(config.retention_for("pool/home"), Retention::default());
+    }
+
+    #[test]
+    fn rejects_incomplete_negative_and_unknown_retention_settings() {
+        for settings in [
+            "15_min = 1\nhourly = 2\ndaily = 3",
+            "15_min = -1\nhourly = 2\ndaily = 3\nmonthly = 4",
+            "15_min = 1\nhourly = 2\ndaily = 3\nmonthly = 4\nweekly = 5",
+            "15_min = 1.5\nhourly = 2\ndaily = 3\nmonthly = 4",
+        ] {
+            assert!(toml::from_str::<Config>(&format!("[default]\n{settings}")).is_err());
+        }
+    }
+
+    #[test]
+    fn timestamps_round_down_at_quarter_hour_boundaries() {
+        for (minute, expected) in [
+            (0, "00"),
+            (14, "00"),
+            (15, "15"),
+            (29, "15"),
+            (30, "30"),
+            (44, "30"),
+            (45, "45"),
+            (59, "45"),
+        ] {
+            let now = Utc.with_ymd_and_hms(2026, 1, 1, 0, minute, 59).unwrap();
+            assert_eq!(timestamp(now), format!("auto_2026010100{expected}"));
+        }
+    }
+
+    #[test]
+    fn tiers_are_disjoint_at_day_and_month_boundaries() {
+        for (name, expected) in [
+            ("auto_202602010015", 0),
+            ("auto_202602010030", 0),
+            ("auto_202602010045", 0),
+            ("auto_202602010100", 1),
+            ("auto_202602012300", 1),
+            ("auto_202602020000", 2),
+            ("auto_202402290000", 2),
+            ("auto_202602010000", 3),
+            ("auto_202601010000", 3),
+        ] {
+            assert_eq!(bucket(name), Some(expected), "{name}");
+        }
+    }
+
+    #[test]
+    fn only_exact_valid_manager_names_are_eligible() {
+        for name in [
+            "manual",
+            "manual_auto_202602010015",
+            "auto_202602010015_backup",
+            "auto_20260201001",
+            "auto_202602300000",
+            "auto_202602012400",
+            "auto_202602010060",
+            "auto_202602010005",
+            "auto_２０２６02010015",
+        ] {
+            assert_eq!(bucket(name), None, "{name}");
+        }
+    }
+
+    #[test]
+    fn keeps_newest_in_each_tier_regardless_of_input_order() {
+        let snapshots = [
+            "auto_202601010000",
+            "auto_202602020000",
+            "auto_202602020115",
+            "auto_202602020215",
+            "auto_202602020200",
+            "auto_202602020100",
+            "auto_202602030000",
+            "auto_202602010000",
+            "manual",
+        ]
+        .map(String::from);
+        let retention = Retention {
+            quarter_hourly: 1,
+            hourly: 1,
+            daily: 1,
+            monthly: 1,
+        };
+        assert_eq!(
+            snapshots_to_delete(&snapshots, retention),
+            [
+                "auto_202602020115",
+                "auto_202602020100",
+                "auto_202602020000",
+                "auto_202601010000",
+            ]
+        );
+    }
+
+    #[test]
+    fn zero_counts_delete_all_managed_snapshots_and_large_counts_keep_them() {
+        let snapshots = [
+            "auto_202601010015",
+            "auto_202601010100",
+            "auto_202601020000",
+            "auto_202601010000",
+            "manual",
+        ]
+        .map(String::from);
+        let none = Retention {
+            quarter_hourly: 0,
+            hourly: 0,
+            daily: 0,
+            monthly: 0,
+        };
+        assert_eq!(snapshots_to_delete(&snapshots, none).len(), 4);
+        let all = Retention {
+            quarter_hourly: 100,
+            hourly: 100,
+            daily: 100,
+            monthly: 100,
+        };
+        assert!(snapshots_to_delete(&snapshots, all).is_empty());
+        assert!(snapshots_to_delete(&[], none).is_empty());
+    }
+}

rust/snapshot_manager/tests/cli.rs

@@ -0,0 +1,202 @@
+#![cfg(test)]
+
+use std::{fs, os::unix::fs::symlink, path::Path, process::Command};
+
+use tempfile::tempdir;
+
+fn command() -> Command {
+    let mut command = Command::new(env!("CARGO_BIN_EXE_snapshot-manager"));
+    command
+        .env_remove("SIGNAL_ALERTS")
+        .env_remove("SIGNAL_ALERT_FROM_PHONE")
+        .env_remove("SIGNAL_ALERT_TO_PHONE")
+        .env_remove("SNAPSHOT_TEST_SCRIPT")
+        .env_remove("RUST_LOG");
+    command
+}
+
+#[test]
+fn help_works_without_phone_numbers() {
+    let output = command().arg("--help").output().unwrap();
+    assert!(output.status.success());
+    assert!(
+        String::from_utf8(output.stdout)
+            .unwrap()
+            .contains("<CONFIG_FILE>")
+    );
+}
+
+#[test]
+fn help_documents_environment_variables_without_exposing_phone_numbers() {
+    let output = command()
+        .env("SIGNAL_ALERT_FROM_PHONE", "+15555550101")
+        .env("SIGNAL_ALERT_TO_PHONE", "+15555550102")
+        .arg("--help")
+        .output()
+        .unwrap();
+    assert!(output.status.success());
+    let stdout = String::from_utf8(output.stdout).unwrap();
+    for name in [
+        "SIGNAL_ALERTS",
+        "SIGNAL_ALERT_FROM_PHONE",
+        "SIGNAL_ALERT_TO_PHONE",
+    ] {
+        assert!(stdout.contains(name), "{stdout}");
+    }
+    assert!(!stdout.contains("+15555550101"), "{stdout}");
+    assert!(!stdout.contains("+15555550102"), "{stdout}");
+}
+
+#[test]
+fn config_argument_is_required() {
+    let output = command().output().unwrap();
+    assert_eq!(output.status.code(), Some(2));
+}
+
+#[test]
+fn enabled_alerts_require_phone_numbers_before_running() {
+    for use_env in [false, true] {
+        for (from, to, expected) in [
+            (None, None, "--signal-alert-from-phone"),
+            (Some(""), None, "--signal-alert-from-phone"),
+            (Some("+15555550101"), None, "--signal-alert-to-phone"),
+            (Some("+15555550101"), Some(""), "--signal-alert-to-phone"),
+            (None, Some("+15555550102"), "--signal-alert-from-phone"),
+        ] {
+            let mut command = command();
+            if use_env {
+                command.env("SIGNAL_ALERTS", "true");
+            } else {
+                command.args(["--signal-alerts", "true"]);
+            }
+            if let Some(from) = from {
+                command.env("SIGNAL_ALERT_FROM_PHONE", from);
+            }
+            if let Some(to) = to {
+                command.env("SIGNAL_ALERT_TO_PHONE", to);
+            }
+            let output = command.arg("unused.toml").output().unwrap();
+            let stderr = String::from_utf8(output.stderr).unwrap();
+            assert_eq!(output.status.code(), Some(2), "{stderr}");
+            assert!(stderr.contains(expected), "{stderr}");
+            assert!(!stderr.contains("Starting snapshot_manager"), "{stderr}");
+        }
+    }
+}
+
+#[test]
+fn enabled_alerts_accept_environment_variables_and_cli_overrides() {
+    let dir = tempdir().unwrap();
+    let config = dir.path().join("config.toml");
+    let scenario = dir.path().join("scenario");
+    fs::write(&config, "").unwrap();
+    // An empty dataset list completes successfully without sending an alert.
+    fs::write(&scenario, "").unwrap();
+    symlink(
+        Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/zfs"),
+        dir.path().join("zfs"),
+    )
+    .unwrap();
+
+    for use_cli in [false, true] {
+        let mut command = command();
+        if use_cli {
+            command
+                .env("SIGNAL_ALERTS", "invalid")
+                .env("SIGNAL_ALERT_FROM_PHONE", "")
+                .env("SIGNAL_ALERT_TO_PHONE", "")
+                .args([
+                    "--signal-alerts",
+                    "true",
+                    "--signal-alert-from-phone",
+                    "+15555550101",
+                    "--signal-alert-to-phone",
+                    "+15555550102",
+                ]);
+        } else {
+            command
+                .env("SIGNAL_ALERTS", "true")
+                .env("SIGNAL_ALERT_FROM_PHONE", "+15555550101")
+                .env("SIGNAL_ALERT_TO_PHONE", "+15555550102");
+        }
+        let output = command
+            .current_dir(dir.path())
+            .env("PATH", dir.path())
+            .env("SNAPSHOT_TEST_SCRIPT", &scenario)
+            .arg(&config)
+            .output()
+            .unwrap();
+        let stderr = String::from_utf8(output.stderr).unwrap();
+        assert!(output.status.success(), "{stderr}");
+        assert!(stderr.contains("snapshot_manager completed"), "{stderr}");
+    }
+}
+
+#[test]
+fn disabled_alerts_allow_snapshot_failures_without_phone_numbers() {
+    let dir = tempdir().unwrap();
+    let config = dir.path().join("config.toml");
+    fs::write(&config, "").unwrap();
+    symlink(
+        Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/zfs"),
+        dir.path().join("zfs"),
+    )
+    .unwrap();
+
+    for (env_value, args) in [
+        (None, vec![]),
+        (Some("false"), vec![]),
+        (None, vec!["--signal-alerts", "false"]),
+        (Some("true"), vec!["--signal-alerts", "false"]),
+    ] {
+        let mut command = command();
+        if let Some(value) = env_value {
+            command.env("SIGNAL_ALERTS", value);
+        }
+        let output = command
+            .env("PATH", dir.path())
+            .args(args)
+            .arg(&config)
+            .output()
+            .unwrap();
+        let stderr = String::from_utf8(output.stderr).unwrap();
+        assert!(output.status.success(), "{stderr}");
+        assert!(
+            stderr.contains("pool/home failed to create snapshot"),
+            "{stderr}"
+        );
+        assert!(stderr.contains("snapshot_manager completed"), "{stderr}");
+        assert!(!stderr.contains("Signal"), "{stderr}");
+    }
+}
+
+#[test]
+fn disabled_alerts_preserve_fatal_errors_without_phone_numbers() {
+    let dir = tempdir().unwrap();
+    for args in [vec![], vec!["--signal-alerts", "false"]] {
+        let output = command()
+            .args(args)
+            .arg(dir.path().join("missing.toml"))
+            .output()
+            .unwrap();
+        let stderr = String::from_utf8(output.stderr).unwrap();
+        assert_eq!(output.status.code(), Some(1), "{stderr}");
+        assert!(stderr.contains("Failed to read configuration"), "{stderr}");
+        assert!(!stderr.contains("SIGNAL_ALERT_"), "{stderr}");
+        assert!(!stderr.contains("Failed to send failure alert"), "{stderr}");
+    }
+}
+
+#[test]
+fn signal_alerts_requires_a_boolean_value() {
+    for use_env in [false, true] {
+        let mut command = command();
+        if use_env {
+            command.env("SIGNAL_ALERTS", "invalid");
+        } else {
+            command.args(["--signal-alerts", "invalid"]);
+        }
+        let output = command.arg("unused.toml").output().unwrap();
+        assert_eq!(output.status.code(), Some(2));
+    }
+}

rust/snapshot_manager/tests/fixtures/zfs

@@ -0,0 +1,12 @@
+#!/bin/sh
+if [ -n "${SNAPSHOT_TEST_SCRIPT:-}" ]; then
+  printf '%s\n' "$*" >> calls
+  . "$SNAPSHOT_TEST_SCRIPT"
+  exit
+fi
+
+case "$*" in
+  'list -Hp -t filesystem -o name') printf '%s\n' pool/home ;;
+  'snapshot pool/home@'*) printf '%s\n' 'permission denied' >&2; exit 1 ;;
+  *) exit 99 ;;
+esac

rust/snapshot_manager/tests/manager.rs

@@ -0,0 +1,290 @@
+#![cfg(test)]
+
+use std::{
+    fs,
+    os::unix::fs::symlink,
+    path::{Path, PathBuf},
+    process::{Command, Output},
+};
+
+use tempfile::{TempDir, tempdir};
+
+struct Fixture {
+    dir: TempDir,
+    config: PathBuf,
+}
+
+impl Fixture {
+    fn new(script: &str) -> Self {
+        let dir = tempdir().unwrap();
+        // Only the child process sees this PATH. The real zfs cannot be found.
+        symlink(
+            Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/zfs"),
+            dir.path().join("zfs"),
+        )
+        .unwrap();
+        // The executable is a checked-in fixture; scenario files are read by
+        // its shell, so parallel tests never execute a file being written.
+        fs::write(dir.path().join("scenario"), script).unwrap();
+        let config = dir.path().join("config.toml");
+        fs::write(
+            &config,
+            "[default]\n15_min = 1\nhourly = 0\ndaily = 0\nmonthly = 0\n",
+        )
+        .unwrap();
+        Self { dir, config }
+    }
+
+    fn calls(&self) -> Vec<String> {
+        fs::read_to_string(self.dir.path().join("calls"))
+            .unwrap_or_default()
+            .lines()
+            .map(String::from)
+            .collect()
+    }
+
+    fn normalized_calls(&self) -> Vec<String> {
+        self.calls()
+            .into_iter()
+            .map(|call| {
+                if call.starts_with("snapshot ") {
+                    let (dataset, name) = call.split_once('@').unwrap();
+                    assert!(name.starts_with("auto_"), "{call}");
+                    format!("{dataset}@<timestamp>")
+                } else {
+                    call
+                }
+            })
+            .collect()
+    }
+
+    fn run(&self) -> Output {
+        Command::new(env!("CARGO_BIN_EXE_snapshot-manager"))
+            .current_dir(self.dir.path())
+            .env("PATH", self.dir.path())
+            .env("SNAPSHOT_TEST_SCRIPT", self.dir.path().join("scenario"))
+            .env_remove("SIGNAL_ALERTS")
+            .env_remove("SIGNAL_ALERT_FROM_PHONE")
+            .env_remove("SIGNAL_ALERT_TO_PHONE")
+            .env_remove("RUST_LOG")
+            .arg(&self.config)
+            .output()
+            .unwrap()
+    }
+}
+
+fn stderr(output: &Output) -> &str {
+    std::str::from_utf8(&output.stderr).unwrap()
+}
+
+#[test]
+fn creates_then_prunes_only_this_datasets_managed_snapshots() {
+    let fixture = Fixture::new(
+        r#"
+case "$*" in
+  'list -Hp -t filesystem -o name')
+    printf '%s\n' pool pool/home
+    printf '%s\n' 'warning on stderr' >&2 ;;
+  'snapshot pool/home@auto_'*) ;;
+  'list -Hp -t snapshot -o name -d 1 pool/home')
+    printf '%s\n' pool/home@auto_200002020145 pool/home@auto_200002020115 pool/home@manual pool/home@manual_auto_200001010015 pool/home/child@auto_200001010015 pool/other@auto_200001010015 ;;
+  'destroy pool/home@auto_200002020115') ;;
+  *) exit 99 ;;
+esac
+"#,
+    );
+    let output = fixture.run();
+    assert!(output.status.success(), "{}", stderr(&output));
+    assert_eq!(
+        fixture.normalized_calls(),
+        [
+            "list -Hp -t filesystem -o name",
+            "snapshot pool/home@<timestamp>",
+            "list -Hp -t snapshot -o name -d 1 pool/home",
+            "destroy pool/home@auto_200002020115",
+        ]
+    );
+}
+
+#[test]
+fn failed_or_duplicate_creation_skips_pruning_but_continues() {
+    for error in ["snapshot already exists", "permission denied"] {
+        let fixture = Fixture::new(&format!(
+            r#"
+case "$*" in
+  'list -Hp -t filesystem -o name') printf '%s\n' pool/home pool/other ;;
+  'snapshot pool/home@auto_'*) printf '%s\n' '{error}' >&2; exit 1 ;;
+  'snapshot pool/other@auto_'*) ;;
+  'list -Hp -t snapshot -o name -d 1 pool/other') ;;
+  *) exit 99 ;;
+esac
+"#
+        ));
+        let output = fixture.run();
+        assert!(output.status.success(), "{}", stderr(&output));
+        assert!(stderr(&output).contains("pool/home failed to create snapshot auto_"));
+        assert!(stderr(&output).contains(error));
+        assert_eq!(
+            fixture.normalized_calls(),
+            [
+                "list -Hp -t filesystem -o name",
+                "snapshot pool/home@<timestamp>",
+                "snapshot pool/other@<timestamp>",
+                "list -Hp -t snapshot -o name -d 1 pool/other",
+            ]
+        );
+        let calls = fixture.calls();
+        assert_eq!(
+            calls[1].split_once('@').unwrap().1,
+            calls[2].split_once('@').unwrap().1
+        );
+    }
+}
+
+#[test]
+fn dependent_clone_failure_is_reported_and_remaining_deletions_continue() {
+    let fixture = Fixture::new(
+        r#"
+case "$*" in
+  'list -Hp -t filesystem -o name') printf '%s\n' pool/home ;;
+  'snapshot pool/home@auto_'*) ;;
+  'list -Hp -t snapshot -o name -d 1 pool/home')
+    printf '%s\n' pool/home@auto_200002020115 pool/home@auto_200002020130 pool/home@auto_200002020145 ;;
+  'destroy pool/home@auto_200002020115')
+    printf '%s\n' "cannot destroy 'pool/home@auto_200002020115': snapshot has dependent clones" >&2; exit 1 ;;
+  'destroy pool/home@auto_200002020130') ;;
+  *) exit 99 ;;
+esac
+"#,
+    );
+    let output = fixture.run();
+    assert!(output.status.success(), "{}", stderr(&output));
+    assert!(
+        stderr(&output).contains(
+            "pool/home@auto_200002020115 failed to delete: snapshot has dependent clones"
+        )
+    );
+    assert_eq!(
+        fixture.calls().last().unwrap(),
+        "destroy pool/home@auto_200002020130"
+    );
+}
+
+#[test]
+fn unexpected_destroy_failure_aborts_and_reports_the_error() {
+    let fixture = Fixture::new(
+        r#"
+case "$1" in
+  list)
+    if [ "$4" = filesystem ]; then
+      printf '%s\n' pool/home
+    else
+      printf '%s\n' pool/home@auto_200002020115 pool/home@auto_200002020130 pool/home@auto_200002020145
+    fi ;;
+  snapshot) ;;
+  destroy) printf '%s\n' 'permission denied' >&2; exit 1 ;;
+esac
+"#,
+    );
+    let output = fixture.run();
+    assert_eq!(output.status.code(), Some(1), "{}", stderr(&output));
+    assert!(stderr(&output).contains("permission denied"));
+    assert!(stderr(&output).contains("snapshot_manager failed"));
+    assert_eq!(
+        fixture.calls().last().unwrap(),
+        "destroy pool/home@auto_200002020115"
+    );
+}
+
+#[test]
+fn failed_list_never_uses_partial_output() {
+    for stream in ["", " >&2"] {
+        let fixture = Fixture::new(&format!(
+            "printf '%s\\n' pool/home\nprintf '%s\\n' 'cannot list datasets'{stream}\nexit 1"
+        ));
+        let output = fixture.run();
+        assert_eq!(output.status.code(), Some(1), "{}", stderr(&output));
+        assert!(stderr(&output).contains("cannot list datasets"));
+        assert!(stderr(&output).contains("snapshot_manager failed"));
+        assert_eq!(fixture.calls(), ["list -Hp -t filesystem -o name"]);
+    }
+}
+
+#[test]
+fn failed_snapshot_listing_aborts_without_destroying_anything() {
+    let fixture = Fixture::new(
+        r#"
+case "$*" in
+  'list -Hp -t filesystem -o name') printf '%s\n' pool/home ;;
+  'snapshot pool/home@auto_'*) ;;
+  *) printf '%s\n' pool/home@auto_200002020115; exit 1 ;;
+esac
+"#,
+    );
+    let output = fixture.run();
+    assert_eq!(output.status.code(), Some(1), "{}", stderr(&output));
+    assert!(stderr(&output).contains("snapshot_manager failed"));
+    assert_eq!(fixture.calls().len(), 3);
+}
+
+#[test]
+fn invalid_config_stops_before_any_zfs_commands() {
+    let fixture = Fixture::new("exit 99");
+    fs::write(
+        &fixture.config,
+        "[default]\n15_min = -1\nhourly = 1\ndaily = 0\nmonthly = 0",
+    )
+    .unwrap();
+    let output = fixture.run();
+    assert_eq!(output.status.code(), Some(1), "{}", stderr(&output));
+    assert!(stderr(&output).contains("Invalid configuration"));
+    assert!(fixture.calls().is_empty());
+}
+
+#[test]
+fn missing_zfs_executable_fails_and_reports_the_error() {
+    let fixture = Fixture::new("exit 99");
+    fs::remove_file(fixture.dir.path().join("zfs")).unwrap();
+    let output = fixture.run();
+    assert_eq!(output.status.code(), Some(1), "{}", stderr(&output));
+    assert!(stderr(&output).contains("Failed to execute zfs"));
+    assert!(stderr(&output).contains("snapshot_manager failed"));
+    assert!(fixture.calls().is_empty());
+}
+
+#[test]
+fn dataset_names_are_passed_as_one_argument_without_a_shell() {
+    let fixture = Fixture::new(
+        r#"
+case "$1" in
+  list)
+    [ "$*" = 'list -Hp -t filesystem -o name' ] || exit 98
+    printf '%s\n' 'pool/two words;touch injected' ;;
+  snapshot)
+    [ "$#" -eq 2 ] || exit 97
+    case "$2" in
+      'pool/two words;touch injected@auto_'*) printf '%s\n' 'argument preserved' >&2; exit 1 ;;
+      *) exit 96 ;;
+    esac ;;
+  *) exit 99 ;;
+esac
+"#,
+    );
+    let output = fixture.run();
+    assert!(output.status.success(), "{}", stderr(&output));
+    assert!(stderr(&output).contains("argument preserved"));
+    assert_eq!(
+        fixture.normalized_calls(),
+        [
+            "list -Hp -t filesystem -o name",
+            "snapshot pool/two words;touch injected@<timestamp>"
+        ]
+    );
+    assert!(fs::read_dir(fixture.dir.path()).unwrap().all(|entry| {
+        !entry
+            .unwrap()
+            .file_name()
+            .to_string_lossy()
+            .starts_with("injected")
+    }));
+}

rust/tls/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "tls"
+version = "0.1.0"
+edition = "2024"
+description = "Shared AWS-LC TLS provider initialization"
+license = "MIT"
+publish = false
+
+[dependencies]
+anyhow.workspace = true
+rustls.workspace = true
+
+[lints]
+workspace = true

rust/tls/src/lib.rs

@@ -0,0 +1,16 @@
+//! Shared TLS configuration for workspace applications.
+
+use anyhow::{Result, anyhow};
+
+/// Install AWS-LC as the process-wide default TLS crypto provider.
+///
+/// Call once at startup, before creating HTTP clients.
+///
+/// # Errors
+///
+/// Returns an error if a TLS crypto provider has already been initialized.
+pub fn install_tls_provider() -> Result<()> {
+    rustls::crypto::aws_lc_rs::default_provider()
+        .install_default()
+        .map_err(|_| anyhow!("TLS crypto provider was already initialized"))
+}

rust/van_weather/Cargo.toml

@@ -6,16 +6,16 @@ description = "Fetch privacy-masked weather for a van and publish it to Home Ass
 license = "MIT"
 
 [dependencies]
-anyhow = "1.0"
-chrono = "0.4"
-clap = { version = "4.5", features = ["derive", "env"] }
-env_logger = "0.11"
-log = "0.4"
-reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] }
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
-
-[lints.clippy]
-all = "deny"
-pedantic = "deny"
+anyhow.workspace = true
+chrono.workspace = true
+clap = { workspace = true, features = ["derive", "env"] }
+env_logger.workspace = true
+log.workspace = true
+num-traits = "0.2"
+reqwest = { workspace = true, features = ["blocking", "json"] }
+serde = { workspace = true, features = ["derive"] }
+serde_json.workspace = true
+tls.workspace = true
 
+[lints]
+workspace = true

rust/van_weather/package.nix

@@ -1,10 +1,18 @@
-{ rustPlatform }:
+{ lib, rustPlatform }:
 rustPlatform.buildRustPackage {
   pname = "van-weather";
   version = "0.1.0";
-  src = ./.;
+  src = import ../rust-source.nix { inherit lib; };
 
-  cargoLock.lockFile = ./Cargo.lock;
+  cargoLock.lockFile = ../../Cargo.lock;
+  cargoBuildFlags = [
+    "-p"
+    "van-weather"
+  ];
+  cargoTestFlags = [
+    "-p"
+    "van-weather"
+  ];
 
   meta = {
     description = "Privacy-masked van weather publisher for Home Assistant";

rust/van_weather/src/main.rs

@@ -4,6 +4,7 @@ use anyhow::{Context, Result, bail};
 use chrono::{DateTime, Utc};
 use clap::Parser;
 use log::{error, info};
+use num_traits::ToPrimitive;
 use reqwest::{
     StatusCode, Url,
     blocking::{Client, ClientBuilder},
@@ -12,6 +13,7 @@ use reqwest::{
 };
 use serde::Deserialize;
 use serde_json::{Value, json};
+use tls::install_tls_provider;
 
 const LAT_ENTITY: &str = "sensor.van_last_known_latitude";
 const LON_ENTITY: &str = "sensor.van_last_known_longitude";
@@ -25,7 +27,7 @@ const fn decimal_factor(decimals: u32) -> f64 {
     let mut remaining = decimals;
     while remaining > 0 {
         factor *= 10.0;
-        remaining -= 1;
+        remaining = remaining.saturating_sub(1);
     }
     factor
 }
@@ -158,6 +160,7 @@ fn main() -> Result<()> {
     } = Args::parse();
     env_logger::Builder::from_env(env_logger::Env::default().default_filter_or(log_level.as_str()))
         .init();
+    install_tls_provider()?;
 
     let ha_url = ha_url.trim_end_matches('/').to_owned();
     let ha_host = Url::parse(&ha_url)
@@ -468,24 +471,39 @@ fn weather_updates(weather: &Weather) -> Vec<(String, Value)> {
 }
 
 fn sensor(entity_id: &str, state: Option<Value>, attributes: Value) -> Option<(String, Value)> {
-    state.map(|state| {
-        let mut payload = serde_json::Map::new();
-        payload.insert("state".to_owned(), state);
-        payload.insert("attributes".to_owned(), attributes);
-        (entity_id.to_owned(), Value::Object(payload))
-    })
+    let state = state?;
+    let mut payload = serde_json::Map::new();
+    payload.insert("state".to_owned(), state);
+    payload.insert("attributes".to_owned(), attributes);
+    Some((entity_id.to_owned(), Value::Object(payload)))
 }
 
-#[allow(clippy::cast_possible_truncation)]
 fn percent(value: Option<f64>) -> i64 {
     // Preserve Python's int(probability * 100) behavior for Home Assistant.
-    (value.unwrap_or_default() * 100.0) as i64
+    let value = value.unwrap_or_default() * 100.0;
+    value.to_i64().unwrap_or_else(|| {
+        if value.is_nan() {
+            0
+        } else if value.is_sign_negative() {
+            i64::MIN
+        } else {
+            i64::MAX
+        }
+    })
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    #[test]
+    fn builds_http_clients_with_aws_lc() -> Result<()> {
+        install_tls_provider()?;
+        build_client("home-assistant.example", Some("test-token"))?;
+        build_client(PIRATE_WEATHER_HOST, None)?;
+        Ok(())
+    }
+
     #[test]
     fn maps_conditions_for_home_assistant() {
         assert_eq!(condition(Some("clear-day")), "sunny");
@@ -562,4 +580,21 @@ mod tests {
         assert!((mask_coordinate(37.7749) - 37.8).abs() < f64::EPSILON);
         assert!((mask_coordinate(-122.4194) - (-122.4)).abs() < f64::EPSILON);
     }
+
+    #[test]
+    fn percentages_truncate_and_preserve_saturation_for_extreme_values() {
+        for (value, expected) in [
+            (None, 0),
+            (Some(0.129), 12),
+            (Some(-0.129), -12),
+            (Some(1.0), 100),
+            (Some(f64::NAN), 0),
+            (Some(f64::MAX), i64::MAX),
+            (Some(f64::MIN), i64::MIN),
+            (Some(f64::INFINITY), i64::MAX),
+            (Some(f64::NEG_INFINITY), i64::MIN),
+        ] {
+            assert_eq!(percent(value), expected);
+        }
+    }
 }

rust/zfs/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "zfs"
+version = "0.1.0"
+edition = "2024"
+description = "List ZFS datasets and manage snapshots through the zfs executable"
+license = "MIT"
+publish = false
+
+[dependencies]
+anyhow.workspace = true
+
+[lints]
+workspace = true

rust/zfs/src/lib.rs

@@ -0,0 +1,100 @@
+//! Manage ZFS datasets and snapshots using the `zfs` executable on `PATH`.
+//!
+//! ```no_run
+//! for snapshot in zfs::list_snapshots("pool/home")? {
+//!     println!("{snapshot}");
+//! }
+//! # Ok::<(), anyhow::Error>(())
+//! ```
+
+use std::process::Command;
+
+use anyhow::{Context, Result, bail};
+
+const ZFS_EXECUTABLE: &str = "zfs";
+
+fn zfs_command(args: &[&str]) -> Result<String> {
+    let output = Command::new(ZFS_EXECUTABLE)
+        .args(args)
+        .env("LC_ALL", "C")
+        .output()
+        .context("Failed to execute zfs")?;
+    if !output.status.success() {
+        let message = if output.stderr.is_empty() {
+            &output.stdout
+        } else {
+            &output.stderr
+        };
+        bail!(
+            "zfs {} failed ({}): {}",
+            args.join(" "),
+            output.status,
+            String::from_utf8_lossy(message).trim()
+        );
+    }
+    String::from_utf8(output.stdout).context("zfs returned invalid UTF-8")
+}
+
+/// List filesystem datasets, excluding pool roots.
+///
+/// # Errors
+///
+/// Returns an error if executing ZFS or decoding its output fails.
+pub fn list_datasets() -> Result<Vec<String>> {
+    let output = zfs_command(&["list", "-Hp", "-t", "filesystem", "-o", "name"])?;
+    Ok(output
+        .lines()
+        .filter(|name| name.contains('/'))
+        .map(String::from)
+        .collect())
+}
+
+/// List snapshot names belonging directly to `dataset`, excluding child datasets.
+///
+/// Returned names omit the `dataset@` prefix.
+///
+/// # Errors
+///
+/// Returns an error if executing ZFS or decoding its output fails.
+pub fn list_snapshots(dataset: &str) -> Result<Vec<String>> {
+    let output = zfs_command(&[
+        "list", "-Hp", "-t", "snapshot", "-o", "name", "-d", "1", dataset,
+    ])?;
+    Ok(output
+        .lines()
+        .filter_map(|name| {
+            let (owner, snapshot) = name.split_once('@')?;
+            (owner == dataset).then(|| snapshot.to_owned())
+        })
+        .collect())
+}
+
+/// Create `dataset@snapshot`.
+///
+/// # Errors
+///
+/// Returns an error if executing ZFS or decoding its output fails.
+pub fn create_snapshot(dataset: &str, snapshot: &str) -> Result<()> {
+    zfs_command(&["snapshot", &format!("{dataset}@{snapshot}")])?;
+    Ok(())
+}
+
+/// Delete `dataset@snapshot`.
+///
+/// Returns `None` on success, or `Some("snapshot has dependent clones")` when
+/// dependent clones prevent deletion.
+///
+/// # Errors
+///
+/// Returns an error for other ZFS failures, execution failures, or invalid UTF-8 output.
+pub fn delete_snapshot(dataset: &str, snapshot: &str) -> Result<Option<&'static str>> {
+    if let Err(error) = zfs_command(&["destroy", &format!("{dataset}@{snapshot}")]) {
+        if error.to_string().contains(&format!(
+            "cannot destroy '{dataset}@{snapshot}': snapshot has dependent clones"
+        )) {
+            return Ok(Some("snapshot has dependent clones"));
+        }
+        return Err(error);
+    }
+    Ok(None)
+}

scripts/signal/authorize.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+if [[ "$(hostname)" != "jeeves" ]]; then
+  echo "Signal device authorization must run on jeeves." >&2
+  exit 1
+fi
+if [[ "$EUID" -ne 0 ]]; then
+  echo "Signal device authorization must run as root." >&2
+  exit 1
+fi
+if [[ "$#" -ne 1 ]]; then
+  echo "Usage: signal_authorize.sh DEVICE_URI" >&2
+  exit 2
+fi
+if [[ "$1" != sgnl://linkdevice\?* ]]; then
+  echo "Invalid Signal device link URI." >&2
+  exit 2
+fi
+
+service_name="signal-cli"
+data_dir="/var/lib/signal-cli"
+device_uri="$1"
+restart_service=false
+
+cleanup() {
+  if [[ "$restart_service" == true ]]; then
+    systemctl start "$service_name"
+  fi
+}
+trap cleanup EXIT
+
+restart_service=true
+systemctl stop "$service_name"
+runuser -u signal-cli -- \
+  signal-cli --data-dir "$data_dir" -a +12016554378 addDevice --uri "$device_uri"

scripts/signal/link.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+host_name="$(hostname)"
+if [[ "$#" -ne 0 ]]; then
+  echo "Usage: signal_link.sh" >&2
+  exit 2
+fi
+
+service_name="signal-cli"
+data_dir="/var/lib/signal-cli"
+link_pid=""
+temp_dir=""
+restart_service=false
+
+cleanup() {
+  if [[ -n "$link_pid" ]] && kill -0 "$link_pid" 2>/dev/null; then
+    kill "$link_pid" 2>/dev/null || true
+    wait "$link_pid" 2>/dev/null || true
+  fi
+
+  if [[ -n "$temp_dir" ]]; then
+    rm -f -- "$temp_dir/link-uri"
+    rmdir -- "$temp_dir"
+  fi
+
+  if [[ "$restart_service" == true ]]; then
+    sudo systemctl start "$service_name"
+  fi
+}
+trap cleanup EXIT
+
+restart_service=true
+sudo systemctl stop "$service_name"
+
+temp_dir="$(mktemp -d)"
+link_fifo="$temp_dir/link-uri"
+mkfifo "$link_fifo"
+
+sudo -u signal-cli \
+  signal-cli --data-dir "$data_dir" link --name "$host_name" >"$link_fifo" &
+link_pid="$!"
+
+if ! IFS= read -r device_uri <"$link_fifo"; then
+  wait "$link_pid"
+  exit 1
+fi
+if [[ "$device_uri" != sgnl://linkdevice\?* ]]; then
+  echo "signal-cli returned an invalid device link URI." >&2
+  exit 1
+fi
+
+echo "Run this command manually on jeeves in another dotfiles dir:"
+printf "sudo ./scripts/signal/authorize.sh %q\n" "$device_uri"
+echo "Waiting for jeeves to authorize this device..."
+
+wait "$link_pid"
+link_pid=""
+echo "$host_name is now linked to the Signal account on jeeves."

shell.nix

@@ -8,7 +8,7 @@
   # loader path, which NixOS does not provide globally.
   ebook-search = pkgs.mkShell {
     nativeBuildInputs = with pkgs; [
-      my_python
+      python314
       uv
     ];
     LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
@@ -23,7 +23,6 @@
       nix
       home-manager
       git
-      my_python
 
       ssh-to-age
       gnupg

systems/bob/default.nix

@@ -3,18 +3,17 @@
   imports = [
     "${inputs.self}/users/math"
     "${inputs.self}/users/richie"
-    "${inputs.self}/users/steve"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/docker.nix"
     "${inputs.self}/common/optional/monitoring-agent.nix"
     "${inputs.self}/common/optional/nvidia.nix"
     "${inputs.self}/common/optional/scanner.nix"
-    "${inputs.self}/common/optional/steam.nix"
+    "${inputs.self}/common/optional/signal-cli.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/systemd-boot.nix"
     "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/yubikey.nix"
+    "${inputs.self}/common/optional/x86-64-v3"
     "${inputs.self}/common/optional/zfs"
     ./hardware.nix
     ./syncthing.nix

systems/bob/secrets.yaml

@@ -0,0 +1,25 @@
+system_token: ENC[AES256_GCM,data:zpyosU2oJTljDBbjeQs79gpzPtfL6Y41nZ3X1d+IgdI1fczx/F6MUOaBN6xVkVGYNHhaaCKaPmS/X7J8TDJHQGWPxM0ZsPluGGCnxhQjsqyY4oc3jyBkye7IbnnWmj1yhisq2V1ti5W7uewCQBioGqJYCNo4PfEjp6ipcgaI5no=,iv:Ee6nw88SUL8+80RR0YA4vAAGilGf7bY6EPhE81jD1Ic=,tag:NtRwWqCntdQN9BlakxvslQ==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBtdjNXQ2VpUlBYMjZJbmRT
+            ajk2WEY5VndyU0o0aWdYbXRUSnE0QWxGNlYwCnBkOVd1VkMweTdIZHZBTmp5akMv
+            TE5SRStHYndXWm0wOG82TDlRa2dHV1kKLS0tIEwveXkxMkpULzhBZWQ5WXJQOGFm
+            TjVmQUkzQmJHYXgwSjF1T1dqMlNUVUEKCz8bv8tdtNPRLSXCNoyEY9yTdpFBg1uG
+            EpZ/VgsBV0LktJkqkv6ZSRWp2HiEC61305GEFxrE3u4w7ha6FLJMrQ==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1u8zj599elqqvcmhxn8zuwrufsz8w8w366d3ayrljjejljt2q45kq8mxw9c
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrLzVQSFZQWTZ2ZnVLdEta
+            S0tkMElVMzZJQ2xSeElQL3N3dy9hQWxzVld3ClcySjhvT0RyUUk4SlNPd2Jpdzhv
+            QU5samVSQnFEdk80M3diazVEVUVFT1UKLS0tIEpLdE82dUFuT0t2NGxqYmxSbis1
+            MFAxN3BtRHVKMWxTeVBpblJDTUcvK0UKZTfYCDTAijn0UPK+Z7UjWOoODPf746Q9
+            ya085jHs39Z1597tgJzWIgydkHl6x9VYUkDu+1Yq4LOeyHFscELnJQ==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1q47vup0tjhulkg7d6xwmdsgrw64h4ax3la3evzqpxyy4adsmk9fs56qz3y
+    lastmodified: "2026-10-14T02:07:23Z"
+    mac: ENC[AES256_GCM,data:o21tNt4Cs5DpCMcWCFjCQ3DRZZBCftFgm2P80y849sFF8rfFVZy99Rda0fiPpohRs45GsvDxPaDAxcCJl2N31pQ3Zg8VCKNS1LbBoDI3iqhuUtmKeudGfOUY6Gd1+j/E4ML29Cfrzm6r3WiXNPa7q8Jmq/kspwoubpQF2q9xp3o=,iv:6hbw+cRlEAdSaPvfh1sMoORI1uhQkqmpOK0aIhL+f+c=,tag:wTa/39ToJSOmn7b31+m34Q==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.3

systems/bob/snapshot_config.toml

@@ -22,6 +22,12 @@ hourly = 0
 daily = 0
 monthly = 0
 
+["root_pool/nix_build"]
+15_min = 1
+hourly = 0
+daily = 0
+monthly = 0
+
 ["root_pool/models"]
 15_min = 4
 hourly = 24

systems/brain/default.nix

@@ -4,11 +4,13 @@
     "${inputs.self}/users/richie"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/docker.nix"
+    "${inputs.self}/common/optional/signal-cli.nix"
     "${inputs.self}/common/optional/ssh_decrypt.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/systemd-boot.nix"
     "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
+    "${inputs.self}/common/optional/x86-64-v3"
     "${inputs.self}/common/optional/zfs"
     ./docker
     ./hardware.nix

systems/brain/docker/docker_networks.md

@@ -1,3 +0,0 @@
-# docker_networks
-
-docker network create -d bridge web

systems/brain/secrets.yaml

@@ -0,0 +1,25 @@
+system_token: ENC[AES256_GCM,data:qub1LgQ89K1EmKk8y/sE1OcPdEB5Zu/S/iZENBA+5d3NQUxJwhX3f3xTo4syBsYFk3b4X1AVx8X0JK27fyr36f9QKPc8IR3Kt4foNp+96gJs9wzskJ188yGFGn1Tle9BLkF1/l5Pzt8Xwe4VCOGuT9uh74q2sCUgcC52u+aez6E=,iv:AbAunuRTY9lhcnd2MwOyVv4in0MfyxwH+LsNK0DoMbQ=,tag:VAmVTSpPh734cZt5Vy0mTg==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjWGNFR01XRXBJUDZ0cFd6
+            dzdWVUlHT1NQU2VrV3VRSlF2bThYdWRtTERJClI1cFFTZzRNSlc1QmlnQUNxSHZB
+            dVhqejBMcDZVVXdIeU0yM3NxelBhMTgKLS0tIFJWRHdqTW5UY3ZOZUZ5dXJKRity
+            YXhHOWl0Y0dQOS9uN0VCSmJHWkxXd0EKV5LbdNZ5oujX+rey8i/XUxoOKw99JKJJ
+            N1sIIc4LmXrqeqRRhXAakQYoWEqQa12Vu3OZrPXkvoQ0KQHKcKbPPA==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1u8zj599elqqvcmhxn8zuwrufsz8w8w366d3ayrljjejljt2q45kq8mxw9c
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhVkNnUkZEbk8rcm90dFAy
+            Ty9vYWN4YkIxdVUyTlJCak4yMmx2dkZXUFFRCmZTRkJReGRwTXB1ZThYSjVmUC9D
+            L1JkVEhHUjRjenpBS0hkQWRiVXY1dEEKLS0tIGM1S2VYbHBjbDU2RG5MUC9HcDNW
+            VEFCR3ZXVzNOMVY3Q0VEM0Y4dnROZUEKKpULO8QALBkItUgkT7IQgPsC/3gQKfid
+            1dgv+CpFGLXDtS8Ocm/r0y6WL2ds1E5rwoQ4UphnxWJGLIAo8cs36g==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1jhf7vm0005j60mjq63696frrmjhpy8kpc2d66mw044lqap5mjv4snmwvwm
+    lastmodified: "2026-10-14T02:08:28Z"
+    mac: ENC[AES256_GCM,data:fBhvyzmnzQtLZ7QNELrJvFfnfymLjfzb7Bi61+6dCqNNgrMnYI4gU+9xDSepHskWQ+9ZS3fHZ8ZQCMC+xjhEvI8PWVe4nQYFKHkNOOrfZSBb8osWyzChbQCQXJtK1zalfo/Yk9gvOzeGhqM+XOqMt15Wr56eETSPpPNwVdG2QPY=,iv:TRCrl90dFGJiP0RsYpEIuvknCh8tslIVWTu+Loeklbw=,tag:LkyzzSEnytMkz40IPeBM6g==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.3

systems/brain/services/heater.nix

@@ -4,6 +4,20 @@
   ...
 }:
 {
+  nixpkgs.overlays = [
+    (final: _prev: {
+      heater_python = final.python314.withPackages (
+        ps: with ps; [
+          fastapi
+          pydantic
+          tinytuya
+          typer
+          uvicorn
+        ]
+      );
+    })
+  ];
+
   networking.firewall.allowedTCPPorts = [ 8124 ];
 
   systemd.services.heater-api = {
@@ -17,7 +31,7 @@
 
     serviceConfig = {
       Type = "simple";
-      ExecStart = "${pkgs.my_python}/bin/python -m python.heater.main --host 0.0.0.0 --port 8124";
+      ExecStart = "${pkgs.heater_python}/bin/python -m python.heater.main --host 0.0.0.0 --port 8124";
       EnvironmentFile = "/etc/heater.env";
       Restart = "on-failure";
       RestartSec = "5s";

systems/brain/services/home_assistant.nix

@@ -13,6 +13,10 @@
   services = {
     home-assistant = {
       enable = true;
+      # Keep Home Assistant's Python dependencies on baseline x86-64.
+      package = pkgs.x86-v1.home-assistant.overrideAttrs (_: {
+        doInstallCheck = false;
+      });
       config = {
         homeassistant = {
           time_zone = "America/New_York";
@@ -68,7 +72,6 @@
           paho-mqtt # for mqtt
           psycopg2 # for postgresql
           py-improv-ble-client # for esphome
-          pymodbus # for modbus
           pyopenweathermap # for weather
           pymetno # for met.no weather
           uiprotect # for ubiquiti integration
@@ -76,8 +79,11 @@
           jsonpath # for rest sensors
           monarchmoneycommunity # for monarch
         ];
-      extraComponents = [ "isal" ];
-      customComponents = with pkgs.home-assistant-custom-components; [
+      extraComponents = [
+        "isal"
+        "modbus" # for victron modbus integration
+      ];
+      customComponents = with pkgs.x86-v1.home-assistant-custom-components; [
         garmin_connect
       ];
 

systems/jeeves/containers.nix

@@ -0,0 +1,71 @@
+{ pkgs, ... }:
+let
+  vars = import ./vars.nix;
+in
+{
+  virtualisation.docker.daemon.settings."data-root" = "${vars.containers}/docker";
+
+  # nixos-container hardcodes its state directory to /var/lib/nixos-containers,
+  # so route it to the shared container dataset with a bind mount.
+  fileSystems."/var/lib/nixos-containers" = {
+    device = "${vars.containers}/nixos-containers";
+    fsType = "none";
+    options = [ "bind" ];
+  };
+
+  systemd.services = {
+    prepare-docker-storage = {
+      description = "Create Docker storage directory";
+      requiredBy = [ "docker.service" ];
+      before = [ "docker.service" ];
+      requires = [ "zfs-mount.service" ];
+      after = [ "zfs-mount.service" ];
+      path = [
+        pkgs.coreutils
+        pkgs.util-linux
+      ];
+
+      unitConfig = {
+        DefaultDependencies = false;
+        RequiresMountsFor = [ "/nix" ];
+      };
+
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+
+      script = ''
+        mountpoint -q ${vars.containers}
+        install -d -m 0710 ${vars.containers}/docker
+      '';
+    };
+
+    prepare-nixos-container-storage = {
+      description = "Create NixOS container storage directory";
+      requiredBy = [ "var-lib-nixos\\x2dcontainers.mount" ];
+      before = [ "var-lib-nixos\\x2dcontainers.mount" ];
+      requires = [ "zfs-mount.service" ];
+      after = [ "zfs-mount.service" ];
+      path = [
+        pkgs.coreutils
+        pkgs.util-linux
+      ];
+
+      unitConfig = {
+        DefaultDependencies = false;
+        RequiresMountsFor = [ "/nix" ];
+      };
+
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+
+      script = ''
+        mountpoint -q ${vars.containers}
+        install -d -m 0755 ${vars.containers}/nixos-containers
+      '';
+    };
+  };
+}

systems/jeeves/default.nix

@@ -1,4 +1,4 @@
-{ inputs, lib, ... }:
+{ inputs, ... }:
 let
   vars = import ./vars.nix;
 in
@@ -6,18 +6,20 @@ in
   imports = [
     "${inputs.self}/users/math"
     "${inputs.self}/users/richie"
-    "${inputs.self}/users/steve"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/docker.nix"
     "${inputs.self}/common/optional/monitoring-agent.nix"
+    "${inputs.self}/common/optional/signal-cli.nix"
     "${inputs.self}/common/optional/ssh_decrypt.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
+    "${inputs.self}/common/optional/x86-64-v3"
     "${inputs.self}/common/optional/zfs"
     ./monitoring
     ./docker
     ./services
+    ./containers.nix
     ./hardware.nix
     ./networking
     ./programs.nix
@@ -36,10 +38,5 @@ in
     };
   };
 
-  users.groups = {
-    nornsight = { };
-    nornsight-admin = { };
-  };
-
   system.stateVersion = "24.05";
 }

systems/jeeves/docker/signal-cli-rest-api.nix

@@ -1,18 +0,0 @@
-let
-  vars = import ../vars.nix;
-in
-{
-  virtualisation.oci-containers.containers.signal_cli_rest_api = {
-    image = "bbernhard/signal-cli-rest-api:0.199-dev";
-    ports = [
-      "8989:8080"
-    ];
-    volumes = [
-      "${vars.docker_configs}/signal-cli-config:/home/.local/share/signal-cli"
-    ];
-    environment = {
-      MODE = "json-rpc";
-    };
-    autoStart = true;
-  };
-}

systems/jeeves/monitoring/README.md

@@ -1,33 +0,0 @@
-# Monitoring
-
-## Vultr API metrics
-
-The `vultr-exporter` service reads its API token from:
-
-```text
-/zfs/storage/secrets/services/vultr-exporter
-```
-
-Create the file on Jeeves as root with the following contents:
-
-```text
-API_KEY=<Vultr API token>
-```
-
-The token needs read access to the Vultr Account and Billing APIs. Unrelated
-resource collectors are disabled in the packaged exporter.
-
-Restrict the file to root and ensure the public egress IP used by Jeeves is
-allowed for the token in the Vultr API settings:
-
-```console
-sudo chown root:root /zfs/storage/secrets/services/vultr-exporter
-sudo chmod 600 /zfs/storage/secrets/services/vultr-exporter
-```
-
-The exporter listens on `127.0.0.1:9188`; it is scraped by the local
-`prometheus-main` service every five minutes and is not exposed through the
-host firewall.
-
-Portal-1 exposes its node exporter only through `tailscale0` on port `9100`.
-Jeeves reaches it using the Portal-1 Tailscale hostname.

systems/jeeves/networking/firewall.nix

@@ -6,14 +6,14 @@ let
     gitea_http = 6443;
     gitea_ssh = 2223;
     grafana = 3000;
+    haproxy_ingest = 8444;
     jellyfin_http = 8096;
+    jellyswarrm = 3100;
     nix_binary_cache = 5000;
-    nornsight = 8001;
     ollama = 11434;
     open_webui = 8080;
     postgresql = 5432;
     share = 8091;
-    signal_cli = 8989;
     syncthing = 8384;
   };
 in
@@ -27,9 +27,10 @@ in
     interfaces.tailscale0.allowedTCPPorts = [
       ports.audiobookshelf
       ports.gems
+      ports.haproxy_ingest
       ports.jellyfin_http
+      ports.jellyswarrm
       ports.nix_binary_cache
-      ports.nornsight
       ports.share
     ];
     allowedTCPPorts = [
@@ -40,7 +41,6 @@ in
       ports.ollama
       ports.open_webui
       ports.postgresql
-      ports.signal_cli
       ports.syncthing
     ];
   };