flank update

it was only required for rhapsody-in-green

.dockerignore

@@ -3,6 +3,19 @@
 .mypy_cache
 .pytest_cache
 .ruff_cache
+.venv
+**/.venv
+.env
+.cache
+.claude
+.coverage
+.vscode
+.stfolder
+.literotica_data
+esphome
+htmlcov
+data
+ebooks
 __pycache__
 **/__pycache__
 *.pyc

.github/workflows/build_systems.yml

@@ -8,8 +8,23 @@ on:
     - cron: "0 22 * * *"
 
 jobs:
+  prebuild-common:
+    name: prebuild-common-x86-64-v3
+    runs-on: nix-cache-builder
+    steps:
+      - uses: actions/checkout@v4
+      # portal-1 is the smallest system closure: 95% of its derivations are
+      # shared by all five systems, so it is a maintainable common cache seed.
+      # Keep going so one failing package does not stop unrelated cache entries
+      # from being built.
+      - name: Build common packages
+        run: nixos-rebuild build --keep-going --accept-flake-config --flake ./#portal-1
+      - name: Copy common packages to nix-cache
+        run: nix copy --accept-flake-config --to unix:///host-nix/var/nix/daemon-socket/socket .#nixosConfigurations.portal-1.config.system.build.toplevel
+
   build:
     name: build-${{ matrix.system }}
+    needs: prebuild-common
     runs-on: self-hosted
     strategy:
       matrix:
@@ -18,6 +33,7 @@ jobs:
           - "brain"
           - "jeeves"
           - "rhapsody-in-green"
+          - "portal-1"
     continue-on-error: true
     steps:
       - uses: actions/checkout@v4

.github/workflows/merge_flake_lock_update.yml

@@ -15,9 +15,8 @@ jobs:
     steps:
       - name: merge_flake_lock_update
         run: >-
-          nix develop .#devShells.x86_64-linux.default -c
           python -m python.gitea_flake_lock merge
           --repo "${{ github.repository }}"
         env:
-          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+          JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
           GITEA_URL: https://gitea.tmmworkshop.com

.github/workflows/pytest.yml

@@ -1,19 +0,0 @@
-name: pytest
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
-
-jobs:
-  pytest:
-    runs-on: self-hosted
-
-    steps:
-      - uses: actions/checkout@v4
-      - name: Run tests
-        run: nix develop .#devShells.x86_64-linux.default -c pytest tests

.github/workflows/test_ebook_search.yml

@@ -0,0 +1,26 @@
+name: test ebook search
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+env:
+  UV_PYTHON_DOWNLOADS: never
+  UV_CACHE_DIR: /var/cache/uv
+  UV_LINK_MODE: copy
+
+jobs:
+  test-ebook-search:
+    runs-on: self-hosted
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install dependencies
+        run: nix develop .#devShells.x86_64-linux.ebook-search -c uv sync --locked --project python/ebook_search/docker
+      - name: Run ebook search tests
+        run: nix develop .#devShells.x86_64-linux.ebook-search -c uv run --project python/ebook_search/docker --no-sync pytest tests/ebook_search --override-ini addopts="-n auto -ra"

.github/workflows/update-flake-lock.yml

@@ -18,9 +18,8 @@ jobs:
         run: nix flake update
       - name: Create or update flake.lock PR
         env:
-          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+          JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
           GITEA_URL: https://gitea.tmmworkshop.com
         run: >-
-          nix develop .#devShells.x86_64-linux.default -c
           python -m python.gitea_flake_lock update
           --repo "${{ github.repository }}"

.gitignore

@@ -173,3 +173,6 @@ frontend/node_modules/
 # data from testing llms
 data/*
 .ebook_search_bm25
+
+# gems data
+.gems

.sops.yaml

@@ -1,6 +1,9 @@
 # Generate AGE keys from SSH keys with:
 #   ssh-keygen -A
 #   nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
+# update keys after addin/removing a key
+#   nix-shell -p sops --run "sops updatekeys users/secrets.yaml" users/secrets.yaml
+
 keys:
   - &admin_richie age1u8zj599elqqvcmhxn8zuwrufsz8w8w366d3ayrljjejljt2q45kq8mxw9c # cspell:disable-line
 
@@ -8,6 +11,7 @@ keys:
   - &system_brain age1jhf7vm0005j60mjq63696frrmjhpy8kpc2d66mw044lqap5mjv4snmwvwm # cspell:disable-line
   - &system_jeeves age13lmqgc3jvkyah5e3vcwmj4s5wsc2akctcga0lpc0x8v8du3fxprqp4ldkv # cspell:disable-line
   - &system_rhapsody age1ufnewppysaq2wwcl4ugngjz8pfzc5a35yg7luq0qmuqvctajcycs5lf6k4 # cspell:disable-line
+  - &system_portal_1 age1vyav6kxtvt3z4vtnkkjj38eu8hlts5m7ygyckhskvalg2gpjk52su53d0a # cspell:disable-line
 
 creation_rules:
   - path_regex: users/secrets\.yaml$
@@ -18,3 +22,4 @@ creation_rules:
           - *system_brain
           - *system_jeeves
           - *system_rhapsody
+          - *system_portal_1

.vscode/settings.json

@@ -10,7 +10,6 @@
     "aiounifi",
     "alsa",
     "apiclient",
-    "apscheduler",
     "archlinux",
     "ashift",
     "asrouter",
@@ -337,8 +336,6 @@
     "yubioath",
     "yzhang",
     "zeroconf",
-    "zerotier",
-    "zerotierone",
     "zoxide",
     "zram",
     "zstd"

Cargo.toml

@@ -0,0 +1,4 @@
+[workspace]
+resolver = "2"
+
+members = ["rust/*"]

common/global/default.nix

@@ -17,16 +17,11 @@
     ./nix.nix
     ./programs.nix
     ./ssh.nix
-    ./snapshot_manager.nix
   ];
 
   boot = {
-    tmp.useTmpfs = true;
+    tmp.useTmpfs = lib.mkDefault true;
     kernelPackages = lib.mkDefault pkgs.linuxPackages_6_12;
-    zfs = {
-      package = lib.mkDefault pkgs.zfs_2_4;
-      forceImportRoot = lib.mkDefault false;
-    };
   };
 
   hardware.enableRedistributableFirmware = true;
@@ -42,9 +37,6 @@
     overlays = builtins.attrValues outputs.overlays;
     config = {
       allowUnfree = true;
-      permittedInsecurePackages = [
-        "openssl-1.1.1w" # This is for discord-canary
-      ];
     };
   };
 
@@ -53,16 +45,6 @@
 
     # firmware update
     fwupd.enable = true;
-
-    snapshot_manager = {
-      enable = lib.mkDefault true;
-      PYTHONPATH = "${inputs.self}/";
-    };
-
-    zfs = {
-      trim.enable = lib.mkDefault true;
-      autoScrub.enable = lib.mkDefault true;
-    };
   };
 
   powerManagement.powertop.enable = lib.mkDefault true;

common/global/nix.nix

@@ -31,6 +31,10 @@ in
         "flakes"
         "ca-derivations"
       ];
+      system-features = lib.mkAfter [
+        "gccarch-x86-64-v2"
+        "gccarch-x86-64-v3"
+      ];
       warn-dirty = false;
       flake-registry = ""; # disable global flake registries
       connect-timeout = 10;
@@ -40,6 +44,6 @@ in
 
     # Add each flake input as a registry and nix_path
     registry = lib.mapAttrs (_: flake: { inherit flake; }) flakeInputs;
-    nixPath = lib.mapAttrsToList (n: _: "${n}=flake:${n}") flakeInputs;
+    nix-path = lib.mapAttrsToList (n: _: "${n}=flake:${n}") flakeInputs;
   };
 }

common/global/programs.nix

@@ -2,6 +2,5 @@
 {
   environment.systemPackages = with pkgs; [
     git
-    my_python
   ];
 }

common/optional/docker.nix

@@ -17,6 +17,7 @@
     logDriver = "local";
     storageDriver = "overlay2";
     daemon.settings = {
+      live-restore = false;
       experimental = true;
       exec-opts = [ "native.cgroupdriver=systemd" ];
       log-opts = {

common/optional/monitoring-agent.nix

@@ -5,7 +5,7 @@
   ...
 }:
 let
-  monitoringInterface = "ztwfunumly";
+  monitoringInterface = "tailscale0";
   nodeTextfileDir = "/var/lib/prometheus-node-exporter-textfile";
 
   mkProcessNameTemplate =

common/optional/signal-cli.nix

@@ -0,0 +1,46 @@
+{ pkgs, ... }:
+{
+  environment.systemPackages = [
+    pkgs.signal-cli
+  ];
+
+  users = {
+    groups.signal-cli = { };
+    users.signal-cli = {
+      isSystemUser = true;
+      group = "signal-cli";
+      home = "/var/lib/signal-cli";
+    };
+  };
+
+  systemd.tmpfiles.rules = [
+    "d /var/lib/signal-cli 0700 signal-cli signal-cli - -"
+    "Z /var/lib/signal-cli - signal-cli signal-cli - -"
+  ];
+
+  systemd.services.signal-cli = {
+    description = "Signal CLI JSON-RPC service";
+    after = [ "network-online.target" ];
+    wants = [ "network-online.target" ];
+    wantedBy = [ "multi-user.target" ];
+    unitConfig.RequiresMountsFor = [ "/var/lib/signal-cli" ];
+
+    serviceConfig = {
+      Type = "simple";
+      User = "signal-cli";
+      Group = "signal-cli";
+      ExecStart = "${pkgs.signal-cli}/bin/signal-cli --data-dir /var/lib/signal-cli daemon --socket /run/signal-cli/socket";
+      Restart = "on-failure";
+      RestartSec = "5s";
+      SuccessExitStatus = 143;
+      RuntimeDirectory = "signal-cli";
+      RuntimeDirectoryMode = "0750";
+      UMask = "0007";
+      NoNewPrivileges = true;
+      PrivateTmp = true;
+      ProtectHome = true;
+      ProtectSystem = "strict";
+      ReadWritePaths = [ "/var/lib/signal-cli" ];
+    };
+  };
+}

common/optional/tailscale.nix

@@ -0,0 +1,37 @@
+{
+  config,
+  inputs,
+  ...
+}:
+{
+  nix.settings = {
+    trusted-substituters = [ "http://jeeves:5000" ];
+    substituters = [ "http://jeeves:5000/?priority=1&want-mass-query=true" ];
+    trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
+  };
+
+  services.tailscale = {
+    enable = true;
+    openFirewall = true;
+    authKeyFile = config.sops.secrets.tailscale_auth_key.path;
+
+    # OAuth client secrets create ephemeral nodes by default. NixOS machines
+    # are persistent and should enroll without interactive device approval.
+    authKeyParameters = {
+      ephemeral = false;
+      preauthorized = true;
+    };
+
+    extraUpFlags = [ "--advertise-tags=tag:nixos" ];
+  };
+
+  sops = {
+    age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
+
+    secrets.tailscale_auth_key = {
+      sopsFile = "${inputs.self}/users/secrets.yaml";
+      owner = "root";
+      mode = "0400";
+    };
+  };
+}

common/optional/x86-64-v3/default.nix

@@ -0,0 +1,15 @@
+{
+  nixpkgs = {
+    hostPlatform = {
+      system = "x86_64-linux";
+      gcc = {
+        arch = "x86-64-v3";
+        tune = "generic";
+      };
+    };
+
+    # These patches repair tests and bundled dependencies that are sensitive
+    # to the compiler flags used by the x86-64-v3 package set.
+    overlays = [ (import ./patches) ];
+  };
+}

common/optional/x86-64-v3/patches/abseil/bmi2-public-header.patch

@@ -0,0 +1,20 @@
+Subject: [PATCH] abseil: include BMI2 intrinsics through the public header
+
+GCC and Clang reject direct inclusion of bmi2intrin.h. Include immintrin.h
+instead so that the compiler supplies the required intrinsic setup when
+BMI2 is enabled, including builds targeting x86-64-v3.
+
+This patch is shared by the vendored Abseil copies in Electron, rusty_v8
+(Deno), and Signal's WebRTC build.
+
+--- a/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
++++ b/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
+@@ -226,7 +226,7 @@
+ #endif
+ 
+ #ifdef __BMI2__
+-#include <bmi2intrin.h>
++#include <immintrin.h>
+ #endif  // __BMI2__
+ 
+ namespace absl {

common/optional/x86-64-v3/patches/abseil/default.nix

@@ -0,0 +1,38 @@
+# Abseil accepted the upstream fix: https://github.com/abseil/abseil-cpp/pull/2071
+# Keep this workaround until Electron, Deno's rusty_v8, and Signal's WebRTC
+# update their bundled Abseil copies to include it.
+{ prev }:
+let
+  patchAbseilBmi2Include =
+    package:
+    package.overrideAttrs (old: {
+      # GCC and Clang require the public umbrella header for BMI2 intrinsics.
+      patches = (old.patches or [ ]) ++ [ ./bmi2-public-header.patch ];
+    });
+
+  electron43Unwrapped = patchAbseilBmi2Include prev.electron_43.unwrapped;
+  electron43 = prev.electron_43.override {
+    electron-unwrapped = electron43Unwrapped;
+  };
+
+  signalCallPackage =
+    path: args:
+    let
+      package = prev.callPackage path args;
+    in
+    if builtins.baseNameOf path == "webrtc.nix" then patchAbseilBmi2Include package else package;
+in
+prev.lib.optionalAttrs ((prev.stdenv.hostPlatform.gcc.arch or null) == "x86-64-v3") {
+  deno =
+    let
+      librusty_v8 = patchAbseilBmi2Include prev.deno.passthru.librusty_v8;
+    in
+    prev.deno.override { inherit librusty_v8; };
+
+  electron_43 = electron43;
+
+  signal-desktop = prev.signal-desktop.override {
+    electron_43 = electron43;
+    callPackage = signalCallPackage;
+  };
+}

common/optional/x86-64-v3/patches/afdko/check-overlap-font-content.patch

@@ -0,0 +1,19 @@
+Subject: [PATCH] Compare font content in the overlap-removal regression
+
+PFA_SKIP ends with a separator, producing an empty prefix that matches every
+line. Remove the empty prefix for this test so glyph differences are checked,
+and assert that tx exits successfully before inspecting the output.
+
+--- a/tests/tx_test.py
++++ b/tests/tx_test.py
+@@ -1064,6 +1064,8 @@
+     output_path = get_temp_file_path()
+     args = [TOOL, '-t1', '+V', '-o', output_path, input_path]
+-    subprocess.call(args)
+-    assert differ([expected_path, output_path, '-s', PFA_SKIP[0]])
++    subprocess.check_call(args)
++    # An empty skip prefix matches every line and hides font differences.
++    skip_headers = SPLIT_MARKER.join(filter(None, PFA_SKIP[0].split(SPLIT_MARKER)))
++    assert differ([expected_path, output_path, '-s', skip_headers])
+ 
+ 

common/optional/x86-64-v3/patches/afdko/default.nix

@@ -0,0 +1,10 @@
+{ afdko }:
+afdko.overridePythonAttrs (old: {
+  # FMA changes overlap-removal coordinates by 0.01 units on x86-64-v3.
+  # Separate multiply/add rounding reproduces the reference Type 1 font exactly.
+  env = (old.env or { }) // {
+    NIX_CFLAGS_COMPILE = (old.env.NIX_CFLAGS_COMPILE or "") + " -ffp-contract=off";
+  };
+
+  patches = (old.patches or [ ]) ++ [ ./check-overlap-font-content.patch ];
+})

common/optional/x86-64-v3/patches/cuda/default.nix

@@ -0,0 +1,19 @@
+{ prev }:
+let
+  buildRedistHook = prev.runCommand "buildRedistHook.bash" { } ''
+    cp ${prev.path}/pkgs/development/cuda-modules/buildRedist/buildRedistHook.bash "$out"
+    chmod u+w "$out"
+    patch "$out" < ${./propagated-build-output-arrays.patch}
+  '';
+in
+prev.cudaPackages_12_9.overrideScope (
+  _cudaFinal: cudaPrev: {
+    buildRedist =
+      args:
+      (cudaPrev.buildRedist args).overrideAttrs (old: {
+        nativeBuildInputs = map (
+          input: if builtins.baseNameOf input == "buildRedistHook.bash" then buildRedistHook else input
+        ) old.nativeBuildInputs;
+      });
+  }
+)

common/optional/x86-64-v3/patches/cuda/propagated-build-output-arrays.patch

@@ -0,0 +1,44 @@
+Subject: [PATCH] Preserve structured CUDA output arrays
+
+The multiple-outputs hook now accepts arrays. The old compatibility export
+collapses include and lib into one array element, breaking indirect expansion.
+Keep the array and iterate over each output when propagating to out.
+
+--- a/buildRedistHook.bash
++++ b/buildRedistHook.bash
+@@ -23,9 +23,6 @@
+ 
+   postInstallCheckHooks+=(checkCudaNonEmptyOutputs)
+   nixLog "added checkCudaNonEmptyOutputs to postInstallCheckHooks"
+-
+-  preFixupHooks+=(fixupPropagatedBuildOutputsForMultipleOutputs)
+-  nixLog "added fixupPropagatedBuildOutputsForMultipleOutputs to preFixupHooks"
+ 
+   postFixupHooks+=(fixupCudaPropagatedBuildOutputsToOut)
+   nixLog "added fixupCudaPropagatedBuildOutputsToOut to postFixupHooks"
+@@ -193,16 +190,6 @@
+   return 0
+ }
+ 
+-# TODO(@connorbaker): https://github.com/NixOS/nixpkgs/issues/323126.
+-# _multioutPropagateDev() currently expects a space-separated string rather than an array.
+-# NOTE: Because _multioutPropagateDev is a postFixup hook, we correct it in preFixup.
+-fixupPropagatedBuildOutputsForMultipleOutputs() {
+-  nixLog "converting propagatedBuildOutputs to a space-separated string"
+-  # shellcheck disable=SC2124
+-  export propagatedBuildOutputs="${propagatedBuildOutputs[@]}"
+-  return 0
+-}
+-
+ # The multiple outputs setup hook only propagates build outputs to dev.
+ # We want to propagate them to out as well, in case the user interpolates
+ # the package into a string -- in such a case, the dev output is not selected
+@@ -217,7 +204,7 @@
+   mkdir -p "${out:?}/nix-support"
+ 
+   # NOTE: We must use printWords to ensure the output is a single line.
+-  for output in $propagatedBuildOutputs; do
++  for output in "${propagatedBuildOutputs[@]}"; do
+     # Propagate the other components to the out output
+     nixLog "adding ${!output:?} to propagatedBuildInputs of ${out:?}"
+     printWords "${!output:?}" >>"${out:?}/nix-support/propagated-build-inputs"

common/optional/x86-64-v3/patches/default.nix

@@ -0,0 +1,18 @@
+_final: prev:
+(import ./abseil { inherit prev; })
+// {
+  cudaPackages_12_9 = import ./cuda { inherit prev; };
+  trunk = import ./trunk { inherit (prev) trunk jq; };
+  gnutls = import ./gnutls { inherit (prev) gnutls; };
+  prometheus = import ./prometheus { inherit (prev) prometheus; };
+  zopfli = import ./zopfli { inherit (prev) zopfli; };
+
+  pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
+    (_pythonFinal: pythonPrev: {
+      afdko = import ./afdko { inherit (pythonPrev) afdko; };
+      psutil = import ./psutil { inherit (pythonPrev) psutil; };
+      pytest-xdist = import ./pytest-xdist { inherit (pythonPrev) pytest-xdist; };
+      scipy = import ./scipy { inherit (pythonPrev) scipy; };
+    })
+  ];
+}

common/optional/x86-64-v3/patches/gnutls/default.nix

@@ -0,0 +1,6 @@
+{ gnutls }:
+gnutls.overrideAttrs (old: {
+  # Keep the UDP handshake test enabled on loaded builders by waiting for
+  # the server to bind its socket. Kept as a standalone patch for upstream.
+  patches = (old.patches or [ ]) ++ [ ./udp-server-readiness.patch ];
+})

common/optional/x86-64-v3/patches/gnutls/udp-server-readiness.patch

@@ -0,0 +1,70 @@
+Subject: [PATCH] tests: wait for the UDP server socket before connecting
+
+A fixed four-second sleep does not guarantee that gnutls-serv has bound
+its UDP socket on a busy builder. Poll the local IPv4 UDP endpoint using
+the existing ss/netstat discovery, with the same retry budget as the TCP
+helper. Fail early if the server exits, and retain the original handshake
+checks in serv-udp.sh.
+
+Use flags common to ss and BSD/Linux netstat. Match the local endpoint
+and complete port number, excluding TCP, IPv6-only and peer endpoints.
+
+--- a/tests/scripts/common.sh
++++ b/tests/scripts/common.sh
+@@ -185,10 +185,55 @@
+ 	fi
+ }
+ 
++check_if_udp_port_bound() {
++	local PORT=$1
++	have_port_finder
++	# Use only -an, which is shared by ss and BSD/Linux netstat. UDP has
++	# no LISTEN state. Match the local IPv4 endpoint, not a peer port or
++	# a longer port number. serv-udp.sh connects to 127.0.0.1;
++	# listen_socket() in serv.c binds IPv4 separately and requests
++	# IPV6_V6ONLY=1 for its IPv6 socket.
++	$PFCMD -an | awk -v port="$PORT" '
++		$1 == "udp" || $1 == "udp4" {
++			# ss includes a state column; netstat does not.
++			address = ($2 == "UNCONN" || $2 == "ESTAB") ? $5 : $4
++			if (address ~ ("^[0-9.]+[.:]" port "$") ||
++			    address == "*." port)
++				found = 1
++		}
++		END { exit !found }
++	'
++}
++
+ wait_udp_server() {
+ 	local PID=$1
++	local ret
+ 	trap "test -n \"${PID}\" && kill ${PID};exit 1" 1 15 2
+-	sleep 4
++	local i=0
++	# Use the same retry budget as wait_for_port(), but also stop if the
++	# server exits before binding its socket.
++	while test $i -lt 90; do
++		if ! kill -0 "$PID" 2>/dev/null; then
++			fail "" "UDP server $PID exited before binding port $PORT"
++		fi
++		# Contain have_port_finder's exit so a skip also stops the server.
++		if (check_if_udp_port_bound "$PORT"); then
++			return 0
++		else
++			ret=$?
++			if test "$ret" = 77; then
++				kill "$PID" 2>/dev/null || :
++				wait "$PID" 2>/dev/null || :
++				exit 77
++			fi
++		fi
++		i=$((i + 1))
++		if test $i -lt 90; then
++			echo "try $i: waiting for UDP port $PORT"
++			sleep 2
++		fi
++	done
++	fail "$PID" "UDP server $PORT did not come up"
+ }
+ 
+ create_testdir() {

common/optional/x86-64-v3/patches/gnutls/verify-readiness.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""Exercise patched common.sh without building GnuTLS (Python standard library only).
+
+Usage: python3 verify-readiness.py /path/to/patched/tests/scripts/common.sh
+Set SHELL to test another shell, and NETSTAT to test a netstat outside PATH.
+"""
+
+# Use unittest so this upstream companion tool needs no pytest installation.
+# ruff: noqa: PT009
+
+import os
+import shutil
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+from pathlib import Path
+
+COMMON = str(Path(sys.argv.pop(1)).resolve())
+SHELL = os.environ.get("SHELL", "/bin/sh")
+
+
+class ReadinessTests(unittest.TestCase):
+    """Check endpoint parsing and the server startup lifecycle."""
+
+    def setUp(self) -> None:
+        """Create a socket-listing fixture for each check."""
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.root = Path(self.tmp.name)
+        self.fixture = self.root / "sockets"
+        self.fixture.write_text("")
+        self.finder = self.root / "port-finder"
+        self.finder.write_text('#!/bin/sh\ncat "$SOCKET_FIXTURE"\n')
+        self.finder.chmod(0o755)
+
+    def run_shell(self, body: str, **env: str) -> subprocess.CompletedProcess[str]:
+        """Source the actual helper and run a shell scenario."""
+        return subprocess.run(
+            [SHELL, "-c", '. "$COMMON"\n' + body],
+            env={
+                **os.environ,
+                "COMMON": COMMON,
+                "SOCKET_FIXTURE": str(self.fixture),
+                "PFCMD": str(self.finder),
+                "PORT": "12345",
+                **env,
+            },
+            capture_output=True,
+            text=True,
+            timeout=20,
+            check=False,
+        )
+
+    def test_socket_formats_and_false_matches(self) -> None:
+        """Accept IPv4 UDP local endpoints and reject unrelated sockets."""
+        cases = [
+            ("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*", True),
+            ("udp UNCONN 0 0 127.0.0.1:12345 0.0.0.0:*", True),
+            ("udp 0 0 0.0.0.0:12345 0.0.0.0:*", True),
+            ("udp4 0 0 *.12345 *.*", True),
+            ("udp 0 0 127.0.0.1.12345 *.*", True),
+            ("udp 0 0 *.12345 *.*", True),
+            ("udp UNCONN 0 0 0.0.0.0:123456 0.0.0.0:*", False),
+            ("udp 0 0 0.0.0.0:123456 0.0.0.0:*", False),
+            ("udp ESTAB 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
+            ("udp 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
+            ("tcp LISTEN 0 128 0.0.0.0:12345 0.0.0.0:*", False),
+            ("tcp 0 0 0.0.0.0:12345 0.0.0.0:* LISTEN", False),
+            ("udp UNCONN 0 0 [::]:12345 [::]:*", False),
+            ("udp UNCONN 0 0 *:12345 *:*", False),
+            ("udp6 0 0 :::12345 :::*", False),
+            ("udp6 0 0 *.12345 *.*", False),
+            ("", False),
+        ]
+        for row, ready in cases:
+            with self.subTest(row=row):
+                self.fixture.write_text(row + "\n")
+                result = self.run_shell('check_if_udp_port_bound "$PORT"')
+                self.assertEqual(result.returncode, 0 if ready else 1, result.stderr)
+
+    def test_exited_server_fails_immediately(self) -> None:
+        """Fail without sleeping when the server has already exited."""
+        result = self.run_shell(
+            'true &\npid=$!\nwait "$pid"\nsleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$pid"'
+        )
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("exited before binding", result.stderr)
+        self.assertNotIn("unexpected sleep", result.stderr)
+
+    def test_ready_socket_does_not_sleep(self) -> None:
+        """Check readiness before the first sleep."""
+        self.fixture.write_text("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*\n")
+        result = self.run_shell('sleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$$"')
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertNotIn("unexpected sleep", result.stderr)
+
+    def test_missing_port_finders_skip(self) -> None:
+        """Skip and stop the live server when no finder is available."""
+        # have_port_finder also tries these paths independently of PATH.
+        if any(os.access(f"{directory}/ss", os.X_OK) for directory in ("/sbin", "/usr/sbin", "/usr/local/sbin")):
+            self.skipTest("an absolute ss path cannot be hidden by this PATH-only fixture")
+        with subprocess.Popen(["sleep", "60"]) as server:
+            try:
+                result = self.run_shell(
+                    'unset PFCMD\nPATH=/nonexistent\nwait_udp_server "$SERVER_PID"',
+                    SERVER_PID=str(server.pid),
+                )
+                self.assertEqual(result.returncode, 77)
+                self.assertIn("neither ss nor netstat found", result.stderr)
+                server.wait(timeout=3)
+                self.assertLess(server.returncode, 0)
+            finally:
+                if server.poll() is None:
+                    server.kill()
+
+    def test_timeout_is_bounded_and_cleans_up(self) -> None:
+        """Stop polling after the retry budget and terminate the server."""
+        # Only accelerate the polling delay; keep a real live server process.
+        self.finder.write_text('#!/bin/sh\necho probe >&2\ncat "$SOCKET_FIXTURE"\n')
+        with subprocess.Popen(["sleep", "60"]) as server:
+            try:
+                result = self.run_shell(
+                    'sleep() { echo polling-sleep; }\nwait_udp_server "$SERVER_PID"',
+                    SERVER_PID=str(server.pid),
+                )
+                self.assertEqual(result.returncode, 1)
+                self.assertIn("did not come up", result.stderr)
+                self.assertEqual(result.stderr.count("probe\n"), 90)
+                self.assertEqual(result.stdout.count("polling-sleep"), 89)
+                server.wait(timeout=3)
+                self.assertLess(server.returncode, 0)
+            finally:
+                if server.poll() is None:
+                    server.kill()
+
+    def test_server_exits_while_waiting(self) -> None:
+        """Detect a startup failure that happens after polling begins."""
+        result = self.run_shell('sleep 1 &\npid=$!\nwait_udp_server "$pid"')
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("exited before binding", result.stderr)
+        self.assertIn("waiting for UDP port", result.stdout)
+
+    def test_real_socket_delayed_beyond_four_seconds(self) -> None:
+        """Wait for a real delayed bind with each installed port finder."""
+        finders = [shutil.which("ss"), os.environ.get("NETSTAT") or shutil.which("netstat")]
+        finders = [finder for finder in finders if finder]
+        if not finders:
+            self.skipTest("neither ss nor netstat available")
+        for finder in finders:
+            with self.subTest(finder=finder):
+                with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
+                    sock.bind(("127.0.0.1", 0))
+                    port = sock.getsockname()[1]
+                code = (
+                    "import socket,time,sys; time.sleep(6); "
+                    "s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); "
+                    "s.bind(('127.0.0.1',int(sys.argv[1]))); time.sleep(30)"
+                )
+                with subprocess.Popen([sys.executable, "-c", code, str(port)]) as server:
+                    try:
+                        started = time.monotonic()
+                        result = self.run_shell(
+                            'wait_udp_server "$SERVER_PID"',
+                            SERVER_PID=str(server.pid),
+                            PORT=str(port),
+                            PFCMD=finder,
+                        )
+                        self.assertEqual(result.returncode, 0, result.stderr)
+                        self.assertGreaterEqual(time.monotonic() - started, 6)
+                        self.assertIsNone(server.poll())
+                    finally:
+                        server.terminate()
+                        server.wait(timeout=3)
+
+
+if __name__ == "__main__":
+    unittest.main()

common/optional/x86-64-v3/patches/prometheus/complete-test-parsing.patch

@@ -0,0 +1,36 @@
+Subject: [PATCH] tests: finish parsing before inspecting editor state
+
+EditorState creation has a 20 ms parsing budget. A descheduled test can
+therefore observe an incomplete tree. Finish these small test documents
+without an interactive deadline and publish the result with a transaction.
+Keep the original completion and vector-matching assertions enabled.
+
+--- a/module/codemirror-promql/src/test/utils-test.ts
++++ b/module/codemirror-promql/src/test/utils-test.ts
+@@ -13,7 +13,7 @@
+ 
+ import { parser } from '@prometheus-io/lezer-promql';
+ import { EditorState } from '@codemirror/state';
+-import { LRLanguage } from '@codemirror/language';
++import { ensureSyntaxTree, LRLanguage } from '@codemirror/language';
+ import nock from 'nock';
+ import path from 'path';
+ import { fileURLToPath } from 'url';
+@@ -23,10 +23,16 @@
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
+ 
+ export function createEditorState(expr: string): EditorState {
+-  return EditorState.create({
++  const state = EditorState.create({
+     doc: expr,
+     extensions: lightPromQLSyntax,
+   });
++  // These tests need a complete tree, independent of the editor's time budget.
++  if (!ensureSyntaxTree(state, state.doc.length, Infinity)) {
++    throw new Error('Unable to parse the test expression');
++  }
++  // Publish the completed parse so syntaxTree(state) sees it too.
++  return state.update({}).state;
+ }
+ 
+ export function mockPrometheusServer(): void {

common/optional/x86-64-v3/patches/prometheus/default.nix

@@ -0,0 +1,17 @@
+{ prometheus }:
+prometheus.overrideAttrs (
+  old:
+  let
+    assets = old.passthru.assets.overrideAttrs (assetsOld: {
+      patches = (assetsOld.patches or [ ]) ++ [ ./complete-test-parsing.patch ];
+    });
+  in
+  {
+    postPatch = builtins.replaceStrings [ "${old.passthru.assets}" ] [ "${assets}" ] (
+      builtins.unsafeDiscardStringContext old.postPatch
+    );
+    passthru = old.passthru // {
+      inherit assets;
+    };
+  }
+)

common/optional/x86-64-v3/patches/psutil/default.nix

@@ -0,0 +1,4 @@
+{ psutil }:
+psutil.overridePythonAttrs (old: {
+  patches = (old.patches or [ ]) ++ [ ./heap-info-zero-mmap.patch ];
+})

common/optional/x86-64-v3/patches/psutil/heap-info-zero-mmap.patch

@@ -0,0 +1,17 @@
+Subject: [PATCH] Allow an empty mmap allocation total in heap_info
+
+The allocator may satisfy all live allocations from the heap. A zero
+mmap_used value is valid and depends on the worker allocation history.
+Keep the heap and platform checks without requiring an mmap allocation.
+
+--- a/tests/test_system.py
++++ b/tests/test_system.py
+@@ -267,6 +267,7 @@
+         if MACOS:
+             assert m.mmap_used == 0  # not supported
+         else:
+-            assert m.mmap_used > 0
++            # A process can have no live mmap-backed malloc allocations.
++            assert m.mmap_used >= 0
+         if WINDOWS:
+             assert m.heap_count >= 0

common/optional/x86-64-v3/patches/pytest-xdist/concurrent-worker-crashes.patch

@@ -0,0 +1,29 @@
+Subject: [PATCH] tests: count replacements when checking the worker restart limit
+
+With two workers, another in-flight test may crash after the fourth
+crash requests shutdown. Either four or five failed tests is valid.
+Require exactly three replacements and the failed-tests exit status,
+while preserving the queued-work and no-internal-error assertions.
+
+--- a/testing/acceptance_test.py
++++ b/testing/acceptance_test.py
+@@ -1011,9 +1011,18 @@
+                 "worker*crashed while running*",
+                 "worker*crashed while running*",
+                 "* xdist: maximum crashed workers reached: 3 *",
+-                "* 4 failed in *",
+             ]
+         )
++        # A second in-flight test may crash after shutdown is requested.
++        # The restart limit constrains replacements, not concurrent failures.
++        replacements = sum(
++            line.startswith("replacing crashed worker ") for line in res.stdout.lines
++        )
++        assert replacements == 3
++        failed = res.parseoutcomes()["failed"]
++        assert failed in (4, 5)
++        res.assert_outcomes(failed=failed)
++        assert res.ret == pytest.ExitCode.TESTS_FAILED
+         assert "INTERNALERROR" not in res.stdout.str()
+ 
+     def test_max_worker_restart_die(self, pytester: pytest.Pytester) -> None:

common/optional/x86-64-v3/patches/pytest-xdist/default.nix

@@ -0,0 +1,12 @@
+{ pytest-xdist }:
+pytest-xdist.overridePythonAttrs (old: {
+  patches = (old.patches or [ ]) ++ [
+    ./concurrent-worker-crashes.patch
+    ./worker-startup-timeout.patch
+  ];
+
+  # The suite exercises its own worker pools. Limit the outer suite to one worker.
+  preCheck = builtins.replaceStrings [ "--numprocesses=$NIX_BUILD_CORES" ] [ "--numprocesses=1" ] (
+    old.preCheck or ""
+  );
+})

common/optional/x86-64-v3/patches/pytest-xdist/worker-startup-timeout.patch

@@ -0,0 +1,19 @@
+Subject: [PATCH] tests: allow more time for remote worker events
+
+Worker startup can exceed ten seconds on heavily loaded builders. Allow
+the remote-test helper to wait up to sixty seconds for worker events.
+The wait still returns as soon as an event arrives and remains bounded.
+Production worker timeouts and test assertions are unchanged.
+
+--- a/testing/test_remote.py
++++ b/testing/test_remote.py
+@@ -17,7 +17,8 @@
+ from xdist.workermanage import WorkerController
+ 
+ 
+-WAIT_TIMEOUT = 10.0
++# Allow worker events extra time on heavily loaded builders.
++WAIT_TIMEOUT = 60.0
+ 
+ 
+ def check_marshallable(d: object) -> None:

common/optional/x86-64-v3/patches/scipy/default.nix

@@ -0,0 +1,5 @@
+{ scipy }:
+scipy.overridePythonAttrs (old: {
+  # Keep the STFT tests enabled with tolerances for x86-64-v3 rounding.
+  patches = (old.patches or [ ]) ++ [ ./stft-test-tolerances.patch ];
+})

common/optional/x86-64-v3/patches/scipy/stft-test-tolerances.patch

@@ -0,0 +1,51 @@
+Subject: [PATCH] signal: allow floating-point rounding in STFT tests
+
+Keep the STFT tests enabled for x86-64-v3 builds. Allow two float32
+epsilons of relative error when comparing inverse-STFT implementations;
+float64 and the existing i686 override remain unchanged. Allow one
+float64 epsilon of absolute error in all three scaling round trips,
+which otherwise require exact zeros (observed residual: 4e-17 for a
+signal with amplitude 2).
+
+Upstream issue: https://github.com/scipy/scipy/issues/25488
+
+--- a/scipy/signal/tests/_scipy_spectral_test_shim.py
++++ b/scipy/signal/tests/_scipy_spectral_test_shim.py
+@@ -294,7 +294,7 @@
+ 
+     # Adapted tolerances to account for resolution loss:
+     atol = np.finfo(x.dtype).resolution*2  # instead of default atol = 0
+-    rtol = 1e-7  # default for np.allclose()
++    rtol = max(1e-7, 2 * np.finfo(x.dtype).eps)
+ 
+     # Relax atol on 32-Bit platforms a bit to pass CI tests.
+     #  - Not clear why there are discrepancies (in the FFT maybe?)
+--- a/scipy/signal/tests/test_spectral.py
++++ b/scipy/signal/tests/test_spectral.py
+@@ -2044,7 +2044,7 @@
+ 
+         # Test round trip:
+         x1 = istft(Zs, boundary=True, scaling='spectrum')[1]
+-        assert_allclose(x1, x)
++        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
+ 
+         # For a Hann-windowed 256 sample length FFT, we expect a peak at
+         # frequency 64 (since it is 1/4 the length of X) with a height of 1
+@@ -2074,7 +2074,7 @@
+ 
+         # Test round trip:
+         x1 = istft(Zp, input_onesided=False, boundary=True, scaling='psd')[1]
+-        assert_allclose(x1, x)
++        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
+ 
+         # The power of the one-sided psd-scaled STFT can be determined
+         # analogously (note that the two sides are not of equal shape):
+@@ -2094,7 +2094,7 @@
+ 
+         # Test round trip:
+         x1 = istft(Zp0, input_onesided=True, boundary=True, scaling='psd')[1]
+-        assert_allclose(x1, x)
++        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
+ 
+ 
+ class TestSampledSpectralRepresentations:

common/optional/x86-64-v3/patches/trunk/default.nix

@@ -0,0 +1,21 @@
+{ trunk, jq }:
+trunk.overrideAttrs (old: {
+  nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ jq ];
+
+  # Patch the bundled libdeflate before Cargo validates its vendor checksums.
+  postPatch = (old.postPatch or "") + ''
+    libdeflateVendors=( "$cargoDepsCopy"/source-*/libdeflate-sys-1.23.1 )
+    if [ "''${#libdeflateVendors[@]}" -ne 1 ] || [ ! -d "''${libdeflateVendors[0]}" ]; then
+      echo "Expected exactly one vendored libdeflate-sys 1.23.1" >&2
+      exit 1
+    fi
+    libdeflateVendor="''${libdeflateVendors[0]}"
+    patch -d "$libdeflateVendor/libdeflate" -p1 < ${./libdeflate-gcc16.patch}
+    libdeflateHeader=libdeflate/lib/x86/cpu_features.h
+    libdeflateChecksum=$(sha256sum "$libdeflateVendor/$libdeflateHeader" | cut -d ' ' -f 1)
+    jq --arg path "$libdeflateHeader" --arg hash "$libdeflateChecksum" \
+      '.files[$path] = $hash' "$libdeflateVendor/.cargo-checksum.json" \
+      > "$libdeflateVendor/.cargo-checksum.json.new"
+    mv "$libdeflateVendor/.cargo-checksum.json.new" "$libdeflateVendor/.cargo-checksum.json"
+  '';
+})

common/optional/x86-64-v3/patches/trunk/libdeflate-gcc16.patch

@@ -0,0 +1,33 @@
+From 2a3762cf736aff9aa5dda329fdec3c08a816eb9b Mon Sep 17 00:00:00 2001
+From: Sergei Trofimovich <slyich@gmail.com>
+Date: Tue, 20 May 2025 21:18:28 +0100
+Subject: [PATCH] lib/x86/cpu_features.h: drop evex512 on gcc-16
+
+gcc-16 `master` branch dropped support for `evex512` as:
+    https://gcc.gnu.org/git/?p=gcc.git;a=commitdiff;h=c052a6f4a1c803cb92147ff98fb91cf3511e0856
+
+As a result libdeflate build started failing as:
+
+    In file included from /build/source/lib/adler32.c:28:
+    /build/source/lib/x86/adler32_template.h:135:12: error: attribute 'target' argument 'no-evex512' is unknown
+      135 | ADD_SUFFIX(reduce_to_32bits)(vec_t v_s1, vec_t v_s2, u32 *s1_p, u32 *s2_p)
+          |            ^~~~~~~~~~~~~~~~
+
+The change adds upper build to `evex512` for `gcc`.
+---
+ lib/x86/cpu_features.h | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/lib/x86/cpu_features.h b/lib/x86/cpu_features.h
+index fbfdb0eb..f9d69d72 100644
+--- a/lib/x86/cpu_features.h
++++ b/lib/x86/cpu_features.h
+@@ -165,7 +165,7 @@ static inline u32 get_x86_cpu_features(void) { return 0; }
+ #  define HAVE_AVXVNNI(features)	((features) & X86_CPU_FEATURE_AVXVNNI)
+ #endif
+ 
+-#if (GCC_PREREQ(14, 0) || CLANG_PREREQ(18, 0, 18000000)) \
++#if ((GCC_PREREQ(14, 0) && !(GCC_PREREQ(16, 0))) || CLANG_PREREQ(18, 0, 18000000)) \
+ 	&& !defined(__EVEX512__) /* avoid subtracting the evex512 feature */
+ #  define EVEX512	",evex512"	/* needed to override potential -mno-evex512 */
+ #  define NO_EVEX512	",no-evex512"

common/optional/x86-64-v3/patches/zopfli/default.nix

@@ -0,0 +1,4 @@
+{ zopfli }:
+zopfli.overrideAttrs (old: {
+  patches = (old.patches or [ ]) ++ [ ./unaligned-match-loads.patch ];
+})

common/optional/x86-64-v3/patches/zopfli/unaligned-match-loads.patch

@@ -0,0 +1,32 @@
+Subject: [PATCH] Read unaligned match buffers with memcpy
+
+Byte buffers need not satisfy integer alignment or aliasing requirements.
+GCC 16 with x86-64-v3 vectorizes the cast loads using aligned AVX reads,
+causing a segmentation fault. Copy into local integers instead.
+
+--- a/src/zopfli/lz77.c
++++ b/src/zopfli/lz77.c
+@@ -302,13 +302,20 @@
+   if (sizeof(size_t) == 8) {
+     /* 8 checks at once per array bounds check (size_t is 64-bit). */
+-    while (scan < safe_end && *((size_t*)scan) == *((size_t*)match)) {
++    while (scan < safe_end) {
++      size_t scan_word, match_word;
++      memcpy(&scan_word, scan, sizeof(scan_word));
++      memcpy(&match_word, match, sizeof(match_word));
++      if (scan_word != match_word) break;
+       scan += 8;
+       match += 8;
+     }
+   } else if (sizeof(unsigned int) == 4) {
+     /* 4 checks at once per array bounds check (unsigned int is 32-bit). */
+-    while (scan < safe_end
+-        && *((unsigned int*)scan) == *((unsigned int*)match)) {
++    while (scan < safe_end) {
++      unsigned int scan_word, match_word;
++      memcpy(&scan_word, scan, sizeof(scan_word));
++      memcpy(&match_word, match, sizeof(match_word));
++      if (scan_word != match_word) break;
+       scan += 4;
+       match += 4;
+     }

common/optional/zerotier.nix

@@ -1,11 +0,0 @@
-{
-  services.zerotierone = {
-    enable = true;
-    joinNetworks = [ "e4da7455b2ae64ca" ];
-  };
-  nix.settings = {
-    trusted-substituters = [ "http://192.168.90.40:5000" ];
-    substituters = [ "http://192.168.90.40:5000/?priority=1&want-mass-query=true" ];
-    trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
-  };
-}

common/optional/zfs/default.nix

@@ -0,0 +1,26 @@
+{
+  inputs,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  imports = [ ./snapshot.nix ];
+
+  boot.zfs = {
+    package = lib.mkDefault pkgs.zfs_2_4;
+    forceImportRoot = lib.mkDefault false;
+  };
+
+  services = {
+    snapshot_manager = {
+      enable = lib.mkDefault true;
+      PYTHONPATH = "${inputs.self}/";
+    };
+
+    zfs = {
+      trim.enable = lib.mkDefault true;
+      autoScrub.enable = lib.mkDefault true;
+    };
+  };
+}

common/optional/zfs/snapshot.nix

@@ -1,4 +1,5 @@
 {
+  inputs,
   pkgs,
   lib,
   config,
@@ -6,6 +7,7 @@
 }:
 let
   cfg = config.services.snapshot_manager;
+  snapshotManagerPackages = ps: with ps; [ typer ];
 in
 {
   options = {
@@ -34,6 +36,41 @@ in
   };
 
   config = lib.mkIf cfg.enable {
+    nixpkgs.overlays = [
+      (final: _prev: {
+        snapshot_manager_python = final.python314.withPackages snapshotManagerPackages;
+        snapshot_manager_test_python = final.python314.withPackages (
+          ps:
+          snapshotManagerPackages ps
+          ++ (with ps; [
+            pyfakefs
+            pytest
+            pytest-asyncio
+            pytest-mock
+            pytest-xdist
+          ])
+        );
+        snapshot_manager_tests =
+          final.runCommand "snapshot-manager-tests"
+            {
+              nativeBuildInputs = [ final.snapshot_manager_test_python ];
+            }
+            ''
+              export HOME="$TMPDIR"
+              cd ${inputs.self}
+              pytest \
+                -o cache_dir="$TMPDIR/pytest-cache" \
+                tests/test_common.py \
+                tests/test_signal_alert.py \
+                tests/test_snapshot_manager.py \
+                tests/test_zfs.py
+              touch "$out"
+            '';
+      })
+    ];
+
+    system.checks = [ pkgs.snapshot_manager_tests ];
+
     systemd = {
       services.snapshot_manager = {
         description = "ZFS Snapshot Manager";
@@ -45,7 +82,7 @@ in
         };
         serviceConfig = {
           Type = "oneshot";
-          ExecStart = "${pkgs.my_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
+          ExecStart = "${pkgs.snapshot_manager_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
         }
         // lib.optionalAttrs (cfg.EnvironmentFile != null) {
           EnvironmentFile = cfg.EnvironmentFile;

common/optional/zfs/snapshot_config.toml

@@ -22,6 +22,12 @@ hourly = 0
 daily = 0
 monthly = 0
 
+["root_pool/nix_build"]
+15_min = 1
+hourly = 0
+daily = 0
+monthly = 0
+
 ["root_pool/var"]
 15_min = 8
 hourly = 24

flake.lock

@@ -1,25 +1,41 @@
 {
   "nodes": {
-    "firefox-addons": {
+    "disko": {
       "inputs": {
         "nixpkgs": [
           "nixpkgs"
         ]
       },
       "locked": {
-        "dir": "pkgs/firefox-addons",
-        "lastModified": 1782964936,
-        "narHash": "sha256-wXEBDr7/dFQYhVpDwCKc9fkrYQQE4x0bdirX1bsLBGA=",
-        "owner": "rycee",
-        "repo": "nur-expressions",
-        "rev": "64feee871e0373dd6121e412c3fb12e372d1bfb5",
-        "type": "gitlab"
+        "lastModified": 1789770686,
+        "narHash": "sha256-uZkBR7yHdIKUFB5SZdfgh1qkGfI3XmYmI/lTiquxbck=",
+        "owner": "nix-community",
+        "repo": "disko",
+        "rev": "725ea35e410ad83be4931d1bff7e090eacaf3563",
+        "type": "github"
       },
       "original": {
-        "dir": "pkgs/firefox-addons",
-        "owner": "rycee",
-        "repo": "nur-expressions",
-        "type": "gitlab"
+        "owner": "nix-community",
+        "repo": "disko",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
       }
     },
     "home-manager": {
@@ -29,11 +45,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1783005591,
-        "narHash": "sha256-NcLHV5uBAeggDUE2wPbKszjfyaSLsoqaYt7izOphkZw=",
+        "lastModified": 1790989254,
+        "narHash": "sha256-ZN+riaeFuE+DZ5zfqJiXlM12jLwZzeIXiNkU1YeHMXI=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "f469c79b955609d6a8fdd9e689be76a93b1621d7",
+        "rev": "acd21c5a3420a9d5fd0ed06299b10828267ef9ba",
         "type": "github"
       },
       "original": {
@@ -42,16 +58,38 @@
         "type": "github"
       }
     },
+    "jellyswarrm": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-overlay": "rust-overlay"
+      },
+      "locked": {
+        "lastModified": 1790427786,
+        "narHash": "sha256-8+Ip0HFW2oZiRr0zCZmPPIHfTdxxHy4CmSF1uOkM3Bw=",
+        "owner": "LLukas22",
+        "repo": "Jellyswarrm",
+        "rev": "9e67c5ad12116add867ba6c1bf69f767879e9147",
+        "type": "github"
+      },
+      "original": {
+        "owner": "LLukas22",
+        "repo": "Jellyswarrm",
+        "type": "github"
+      }
+    },
     "nixos-hardware": {
       "inputs": {
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1782562157,
-        "narHash": "sha256-a7+T6QSeowynwZ1ZJJbP8T8ntAytvrui8kFGJmIZt2c=",
+        "lastModified": 1790948122,
+        "narHash": "sha256-fDMYyzY1lI5b8RuCMrvoaIgTeRwNnTG4dNk1ucROlQE=",
         "owner": "nixos",
         "repo": "nixos-hardware",
-        "rev": "a9cf7546a938c737b079e738de73934a13de9784",
+        "rev": "0953bb1a609df63013176e3463393f8df993d3ef",
         "type": "github"
       },
       "original": {
@@ -63,11 +101,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1767892417,
-        "narHash": "sha256-8bW3q88CEg2u4hSP66Vf4lpbLonHz7hqDNBMcCY7E9U=",
-        "rev": "3497aa5c9457a9d88d71fa93a4a8368816fbeeba",
+        "lastModified": 1789546076,
+        "narHash": "sha256-vWkSk5bbfTqdtMoSgD9FshACO8JCvXTFi+3cqEp0mH0=",
+        "rev": "b1b875982b17dabde9b4a37f3e229e74913e6db3",
         "type": "tarball",
-        "url": "https://releases.nixos.org/nixos/unstable/nixos-26.05pre924538.3497aa5c9457/nixexprs.tar.xz"
+        "url": "https://releases.nixos.org/nixos/unstable/nixos-26.11pre1074753.b1b875982b17/nixexprs.tar.xz"
       },
       "original": {
         "type": "tarball",
@@ -76,11 +114,11 @@
     },
     "nixpkgs-master": {
       "locked": {
-        "lastModified": 1783021952,
-        "narHash": "sha256-8PghAtSGGZ0umfVI8Qbd7ZbFrfZPiH1UwtVbgLeikDA=",
+        "lastModified": 1791033472,
+        "narHash": "sha256-IRpPnIfZ0drkkyMtuuvXaada2RjH+G2WYv14Ap1W8q4=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "f136374c679c54171a3ace589d15e9e79a8bd086",
+        "rev": "ad70d223e208533b7ee8cc8e336a34aad134ab19",
         "type": "github"
       },
       "original": {
@@ -108,11 +146,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1782723713,
-        "narHash": "sha256-oPXCU/SSUokcGaJREHibG1CBX3+s/W7orDWQOZDsEeQ=",
+        "lastModified": 1790822859,
+        "narHash": "sha256-69xHQhAeMAD2wDXO7T2pcOZIF9Sga2W+JkmY2a11Ops=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "b5aa0fbd538984f6e3d201be0005b4463d8b09f8",
+        "rev": "c59305bab2065cfecc4944690d9eedbb56f3a9fa",
         "type": "github"
       },
       "original": {
@@ -124,14 +162,36 @@
     },
     "root": {
       "inputs": {
-        "firefox-addons": "firefox-addons",
+        "disko": "disko",
         "home-manager": "home-manager",
+        "jellyswarrm": "jellyswarrm",
         "nixos-hardware": "nixos-hardware",
         "nixpkgs": "nixpkgs_2",
         "nixpkgs-master": "nixpkgs-master",
         "nixpkgs-stable": "nixpkgs-stable",
         "sops-nix": "sops-nix",
-        "systems": "systems"
+        "systems": "systems_2"
+      }
+    },
+    "rust-overlay": {
+      "inputs": {
+        "nixpkgs": [
+          "jellyswarrm",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1779419951,
+        "narHash": "sha256-dMX0PUslUHPajP6o8FEoRdFv9afq/dec4POR0vVfjK4=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "5b5c521d6cae9ef4aa32f888eb2c0ce595c9be52",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
       }
     },
     "sops-nix": {
@@ -141,11 +201,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1782165805,
-        "narHash": "sha256-478kKQBvK6SYTOdN2h9jhKJv94nbXRbFMfuL1WshErg=",
+        "lastModified": 1790498116,
+        "narHash": "sha256-rs9meAYxW3zzrh43yaW7htrqCD+X9+pupDPHN86fumI=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "56b24064fdcaedca53553b1a6d607fd23b613a24",
+        "rev": "5efb5a6f4f5ab192817d28557dd4d650fa14d866",
         "type": "github"
       },
       "original": {
@@ -155,6 +215,21 @@
       }
     },
     "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
       "locked": {
         "lastModified": 1689347949,
         "narHash": "sha256-12tWmuL2zgBgZkdoB6qXZsgJEH9LR3oUgpaQq2RbI80=",

flake.nix

@@ -21,18 +21,23 @@
 
     nixos-hardware.url = "github:nixos/nixos-hardware/master";
 
+    jellyswarrm = {
+      url = "github:LLukas22/Jellyswarrm";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
     home-manager = {
       url = "github:nix-community/home-manager";
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
-    firefox-addons = {
-      url = "gitlab:rycee/nur-expressions?dir=pkgs/firefox-addons";
+    sops-nix = {
+      url = "github:Mic92/sops-nix";
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
-    sops-nix = {
-      url = "github:Mic92/sops-nix";
+    disko = {
+      url = "github:nix-community/disko";
       inputs.nixpkgs.follows = "nixpkgs";
     };
   };

overlays/default.nix

@@ -1,13 +1,13 @@
 { inputs, ... }:
 {
-  # When applied, the stable nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.stable'
+  # Additional package sets are accessible through `pkgs.<name>`.
   stable = final: _prev: {
     stable = import inputs.nixpkgs-stable {
       system = final.stdenv.hostPlatform.system;
       config.allowUnfree = true;
     };
   };
-  # When applied, the master nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.master'
+
   master = final: _prev: {
     master = import inputs.nixpkgs-master {
       system = final.stdenv.hostPlatform.system;
@@ -15,34 +15,12 @@
     };
   };
 
-  python-env = final: _prev: {
-    my_python = final.python314.withPackages (
-      ps:
-      with ps;
-      [
-        alembic
-        apprise
-        apscheduler
-        fastapi
-        fastapi-cli
-        httpx
-        mypy
-        pgvector
-        psycopg
-        pydantic
-        pyfakefs
-        pytest
-        pytest-cov
-        pytest-mock
-        pytest-xdist
-        python-multipart
-        ruff
-        sqlalchemy
-        tenacity
-        tinytuya
-        typer
-        websockets
-      ]
-    );
+  # Baseline x86-64 (v1) packages for prebuilt applications that should not
+  # inherit an x86-64-v3 host platform.
+  x86-v1 = final: _prev: {
+    x86-v1 = import inputs.nixpkgs {
+      system = final.stdenv.hostPlatform.system;
+      config.allowUnfree = true;
+    };
   };
 }

pyproject.toml

@@ -6,51 +6,6 @@ authors = [{ name = "Richie Cahill", email = "richie@tmmworkshop.com" }]
 requires-python = "~=3.14.0"
 readme = "README.md"
 license = "MIT"
-# these dependencies are a best effort and aren't guaranteed to work
-# for up-to-date dependencies, see overlays/default.nix
-dependencies = [
-    "alembic",
-    "apprise",
-    "apscheduler",
-    "beautifulsoup4",
-    "bm25s",
-    "ebooklib",
-    "fastapi",
-    "fastapi-cli",
-    "httpx",
-    "jinja2",
-    "pgvector",
-    "polars",
-    "psycopg[binary]",
-    "pydantic",
-    "pydantic-settings",
-    "python-multipart",
-    "sqlalchemy[asyncio]",
-    "tenacity",
-    "tiktoken",
-    "tinytuya",
-    "typer",
-    "uvicorn",
-    "websockets",
-    "yake",
-]
-
-[project.scripts]
-database = "python.database_cli:app"
-whisper-transcribe = "python.tools.whisper.transcribe:main"
-
-[dependency-groups]
-dev = [
-    "aiosqlite",
-    "mypy",
-    "pyfakefs",
-    "pytest-asyncio",
-    "pytest-cov",
-    "pytest-mock",
-    "pytest-xdist",
-    "pytest",
-    "ruff",
-]
 
 [tool.ruff]
 
@@ -65,6 +20,7 @@ lint.ignore = [
     "ISC001", # (TEMP) conflicts when used with the formatter
     "S603",   # (PERM) This is known to cause a false positive
     "S607",   # (PERM) This is becoming a consistent annoyance
+    "CPY001",  # (PERM) I don't include the license in every file
 ]
 
 [tool.ruff.lint.per-file-ignores]
@@ -119,7 +75,7 @@ exclude_lines = [
 ]
 
 [tool.pytest.ini_options]
-addopts = "-n auto -ra"
+addopts = "-n auto -ra --ignore=tests/ebook_search"
 asyncio_mode = "auto"
 testpaths = ["tests"]
 # --cov=system_tools --cov-report=term-missing --cov-report=xml --cov-report=html --cov-branch

python/alembic/richie/versions/2026_07_09-remove_spacy_ner_751260fc3228.py

@@ -0,0 +1,55 @@
+"""remove spaCy-ner.
+
+Revision ID: 751260fc3228
+Revises: dddee09eddcc
+Create Date: 2026-07-09 23:03:39.554083
+
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import sqlalchemy as sa
+from alembic import op
+
+from python.orm import RichieBase
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+# revision identifiers, used by Alembic.
+revision: str = "751260fc3228"
+down_revision: str | None = "dddee09eddcc"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+schema = RichieBase.schema_name
+
+
+def upgrade() -> None:
+    """Upgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("candidate_phrases", "source_spacy_noun_chunk", schema=schema)
+    op.drop_column("candidate_phrases", "source_spacy_ner", schema=schema)
+    op.drop_column("candidate_phrases", "spacy_label", schema=schema)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "candidate_phrases", sa.Column("spacy_label", sa.VARCHAR(), autoincrement=False, nullable=True), schema=schema
+    )
+    op.add_column(
+        "candidate_phrases",
+        sa.Column("source_spacy_ner", sa.BOOLEAN(), autoincrement=False, nullable=False),
+        schema=schema,
+    )
+    op.add_column(
+        "candidate_phrases",
+        sa.Column("source_spacy_noun_chunk", sa.BOOLEAN(), autoincrement=False, nullable=False),
+        schema=schema,
+    )
+    # ### end Alembic commands ###

python/api/__init__.py

@@ -1 +0,0 @@
-"""FastAPI applications."""

python/api/main.py

@@ -1,56 +0,0 @@
-"""FastAPI interface for Contact database."""
-
-from __future__ import annotations
-
-import logging
-from contextlib import asynccontextmanager
-from typing import TYPE_CHECKING, Annotated
-
-import typer
-import uvicorn
-from fastapi import FastAPI
-
-from python.api.routers import contact_router, views_router
-from python.common import configure_logger
-from python.fastapi_tools import ZstdMiddleware
-from python.orm.common import get_postgres_engine
-
-if TYPE_CHECKING:
-    from collections.abc import AsyncIterator
-
-logger = logging.getLogger(__name__)
-
-
-def create_app() -> FastAPI:
-    """Create and configure the FastAPI application."""
-
-    @asynccontextmanager
-    async def lifespan(app: FastAPI) -> AsyncIterator[None]:
-        """Manage application lifespan."""
-        app.state.engine = get_postgres_engine()
-        yield
-        app.state.engine.dispose()
-
-    app = FastAPI(title="Contact Database API", lifespan=lifespan)
-    app.add_middleware(ZstdMiddleware)
-
-    app.include_router(contact_router)
-    app.include_router(views_router)
-
-    return app
-
-
-def serve(
-    host: Annotated[str, typer.Option("--host", "-h", help="Host to bind to")],
-    port: Annotated[int, typer.Option("--port", "-p", help="Port to bind to")] = 8000,
-    log_level: Annotated[str, typer.Option("--log-level", "-l", help="Log level")] = "INFO",
-) -> None:
-    """Start the Contact API server."""
-    configure_logger(log_level)
-
-    app = create_app()
-    uvicorn.run(app, host=host, port=port)
-
-
-if __name__ == "__main__":
-    typer.run(serve)

python/api/routers/__init__.py

@@ -1,6 +0,0 @@
-"""API routers."""
-
-from python.api.routers.contact import router as contact_router
-from python.api.routers.views import router as views_router
-
-__all__ = ["contact_router", "views_router"]

python/api/routers/contact.py

@@ -1,481 +0,0 @@
-"""Contact API router."""
-
-from pathlib import Path
-
-from fastapi import APIRouter, HTTPException, Request
-from fastapi.responses import HTMLResponse
-from fastapi.templating import Jinja2Templates
-from pydantic import BaseModel
-from sqlalchemy import select
-from sqlalchemy.orm import selectinload
-
-from python.fastapi_tools.db import DbSession  # noqa: TC001 this is a FastAPI needed at runtime
-from python.orm.richie.contact import Contact, ContactRelationship, Need, RelationshipType
-
-TEMPLATES_DIR = Path(__file__).parent.parent / "templates"
-templates = Jinja2Templates(directory=TEMPLATES_DIR)
-
-
-def _is_htmx(request: Request) -> bool:
-    """Check if the request is from HTMX."""
-    return request.headers.get("HX-Request") == "true"
-
-
-class NeedBase(BaseModel):
-    """Base schema for Need."""
-
-    name: str
-    description: str | None = None
-
-
-class NeedCreate(NeedBase):
-    """Schema for creating a Need."""
-
-
-class NeedResponse(NeedBase):
-    """Schema for Need response."""
-
-    id: int
-
-    model_config = {"from_attributes": True}
-
-
-class ContactRelationshipCreate(BaseModel):
-    """Schema for creating a contact relationship."""
-
-    related_contact_id: int
-    relationship_type: RelationshipType
-    closeness_weight: int | None = None
-
-
-class ContactRelationshipUpdate(BaseModel):
-    """Schema for updating a contact relationship."""
-
-    relationship_type: RelationshipType | None = None
-    closeness_weight: int | None = None
-
-
-class ContactRelationshipResponse(BaseModel):
-    """Schema for contact relationship response."""
-
-    contact_id: int
-    related_contact_id: int
-    relationship_type: str
-    closeness_weight: int
-
-    model_config = {"from_attributes": True}
-
-
-class RelationshipTypeInfo(BaseModel):
-    """Information about a relationship type."""
-
-    value: str
-    display_name: str
-    default_weight: int
-
-
-class GraphNode(BaseModel):
-    """Node in the relationship graph."""
-
-    id: int
-    name: str
-    current_job: str | None = None
-
-
-class GraphEdge(BaseModel):
-    """Edge in the relationship graph."""
-
-    source: int
-    target: int
-    relationship_type: str
-    closeness_weight: int
-
-
-class GraphData(BaseModel):
-    """Complete graph data for visualization."""
-
-    nodes: list[GraphNode]
-    edges: list[GraphEdge]
-
-
-class ContactBase(BaseModel):
-    """Base schema for Contact."""
-
-    name: str
-    age: int | None = None
-    bio: str | None = None
-    current_job: str | None = None
-    gender: str | None = None
-    goals: str | None = None
-    legal_name: str | None = None
-    profile_pic: str | None = None
-    safe_conversation_starters: str | None = None
-    self_sufficiency_score: int | None = None
-    social_structure_style: str | None = None
-    ssn: str | None = None
-    suffix: str | None = None
-    timezone: str | None = None
-    topics_to_avoid: str | None = None
-
-
-class ContactCreate(ContactBase):
-    """Schema for creating a Contact."""
-
-    need_ids: list[int] = []
-
-
-class ContactUpdate(BaseModel):
-    """Schema for updating a Contact."""
-
-    name: str | None = None
-    age: int | None = None
-    bio: str | None = None
-    current_job: str | None = None
-    gender: str | None = None
-    goals: str | None = None
-    legal_name: str | None = None
-    profile_pic: str | None = None
-    safe_conversation_starters: str | None = None
-    self_sufficiency_score: int | None = None
-    social_structure_style: str | None = None
-    ssn: str | None = None
-    suffix: str | None = None
-    timezone: str | None = None
-    topics_to_avoid: str | None = None
-    need_ids: list[int] | None = None
-
-
-class ContactResponse(ContactBase):
-    """Schema for Contact response with relationships."""
-
-    id: int
-    needs: list[NeedResponse] = []
-    related_to: list[ContactRelationshipResponse] = []
-    related_from: list[ContactRelationshipResponse] = []
-
-    model_config = {"from_attributes": True}
-
-
-class ContactListResponse(ContactBase):
-    """Schema for Contact list response."""
-
-    id: int
-
-    model_config = {"from_attributes": True}
-
-
-router = APIRouter(prefix="/api", tags=["contacts"])
-
-
-@router.post("/needs", response_model=NeedResponse)
-def create_need(need: NeedCreate, db: DbSession) -> Need:
-    """Create a new need."""
-    db_need = Need(name=need.name, description=need.description)
-    db.add(db_need)
-    db.commit()
-    db.refresh(db_need)
-    return db_need
-
-
-@router.get("/needs", response_model=list[NeedResponse])
-def list_needs(db: DbSession) -> list[Need]:
-    """List all needs."""
-    return list(db.scalars(select(Need)).all())
-
-
-@router.get("/needs/{need_id}", response_model=NeedResponse)
-def get_need(need_id: int, db: DbSession) -> Need:
-    """Get a need by ID."""
-    need = db.get(Need, need_id)
-    if not need:
-        raise HTTPException(status_code=404, detail="Need not found")
-    return need
-
-
-@router.delete("/needs/{need_id}", response_model=None)
-def delete_need(need_id: int, request: Request, db: DbSession) -> dict[str, bool] | HTMLResponse:
-    """Delete a need by ID."""
-    need = db.get(Need, need_id)
-    if not need:
-        raise HTTPException(status_code=404, detail="Need not found")
-    db.delete(need)
-    db.commit()
-    if _is_htmx(request):
-        return HTMLResponse("")
-    return {"deleted": True}
-
-
-@router.post("/contacts", response_model=ContactResponse)
-def create_contact(contact: ContactCreate, db: DbSession) -> Contact:
-    """Create a new contact."""
-    need_ids = contact.need_ids
-    contact_data = contact.model_dump(exclude={"need_ids"})
-    db_contact = Contact(**contact_data)
-
-    if need_ids:
-        needs = list(db.scalars(select(Need).where(Need.id.in_(need_ids))).all())
-        db_contact.needs = needs
-
-    db.add(db_contact)
-    db.commit()
-    db.refresh(db_contact)
-    return db_contact
-
-
-@router.get("/contacts", response_model=list[ContactListResponse])
-def list_contacts(
-    db: DbSession,
-    skip: int = 0,
-    limit: int = 100,
-) -> list[Contact]:
-    """List all contacts with pagination."""
-    return list(db.scalars(select(Contact).offset(skip).limit(limit)).all())
-
-
-@router.get("/contacts/{contact_id}", response_model=ContactResponse)
-def get_contact(contact_id: int, db: DbSession) -> Contact:
-    """Get a contact by ID with all relationships."""
-    contact = db.scalar(
-        select(Contact)
-        .where(Contact.id == contact_id)
-        .options(
-            selectinload(Contact.needs),
-            selectinload(Contact.related_to),
-            selectinload(Contact.related_from),
-        )
-    )
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-    return contact
-
-
-@router.patch("/contacts/{contact_id}", response_model=ContactResponse)
-def update_contact(
-    contact_id: int,
-    contact: ContactUpdate,
-    db: DbSession,
-) -> Contact:
-    """Update a contact by ID."""
-    db_contact = db.get(Contact, contact_id)
-    if not db_contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    update_data = contact.model_dump(exclude_unset=True)
-    need_ids = update_data.pop("need_ids", None)
-
-    for key, value in update_data.items():
-        setattr(db_contact, key, value)
-
-    if need_ids is not None:
-        needs = list(db.scalars(select(Need).where(Need.id.in_(need_ids))).all())
-        db_contact.needs = needs
-
-    db.commit()
-    db.refresh(db_contact)
-    return db_contact
-
-
-@router.delete("/contacts/{contact_id}", response_model=None)
-def delete_contact(contact_id: int, request: Request, db: DbSession) -> dict[str, bool] | HTMLResponse:
-    """Delete a contact by ID."""
-    contact = db.get(Contact, contact_id)
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-    db.delete(contact)
-    db.commit()
-    if _is_htmx(request):
-        return HTMLResponse("")
-    return {"deleted": True}
-
-
-@router.post("/contacts/{contact_id}/needs/{need_id}")
-def add_need_to_contact(
-    contact_id: int,
-    need_id: int,
-    db: DbSession,
-) -> dict[str, bool]:
-    """Add a need to a contact."""
-    contact = db.get(Contact, contact_id)
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    need = db.get(Need, need_id)
-    if not need:
-        raise HTTPException(status_code=404, detail="Need not found")
-
-    if need not in contact.needs:
-        contact.needs.append(need)
-        db.commit()
-
-    return {"added": True}
-
-
-@router.delete("/contacts/{contact_id}/needs/{need_id}", response_model=None)
-def remove_need_from_contact(
-    contact_id: int,
-    need_id: int,
-    request: Request,
-    db: DbSession,
-) -> dict[str, bool] | HTMLResponse:
-    """Remove a need from a contact."""
-    contact = db.get(Contact, contact_id)
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    need = db.get(Need, need_id)
-    if not need:
-        raise HTTPException(status_code=404, detail="Need not found")
-
-    if need in contact.needs:
-        contact.needs.remove(need)
-        db.commit()
-
-    if _is_htmx(request):
-        return HTMLResponse("")
-    return {"removed": True}
-
-
-@router.post(
-    "/contacts/{contact_id}/relationships",
-    response_model=ContactRelationshipResponse,
-)
-def add_contact_relationship(
-    contact_id: int,
-    relationship: ContactRelationshipCreate,
-    db: DbSession,
-) -> ContactRelationship:
-    """Add a relationship between two contacts."""
-    contact = db.get(Contact, contact_id)
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    related_contact = db.get(Contact, relationship.related_contact_id)
-    if not related_contact:
-        raise HTTPException(status_code=404, detail="Related contact not found")
-
-    if contact_id == relationship.related_contact_id:
-        raise HTTPException(status_code=400, detail="Cannot relate contact to itself")
-
-    # Use provided weight or default from relationship type
-    weight = relationship.closeness_weight
-    if weight is None:
-        weight = relationship.relationship_type.default_weight
-
-    db_relationship = ContactRelationship(
-        contact_id=contact_id,
-        related_contact_id=relationship.related_contact_id,
-        relationship_type=relationship.relationship_type.value,
-        closeness_weight=weight,
-    )
-    db.add(db_relationship)
-    db.commit()
-    db.refresh(db_relationship)
-    return db_relationship
-
-
-@router.get(
-    "/contacts/{contact_id}/relationships",
-    response_model=list[ContactRelationshipResponse],
-)
-def get_contact_relationships(
-    contact_id: int,
-    db: DbSession,
-) -> list[ContactRelationship]:
-    """Get all relationships for a contact."""
-    contact = db.get(Contact, contact_id)
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    outgoing = list(db.scalars(select(ContactRelationship).where(ContactRelationship.contact_id == contact_id)).all())
-    incoming = list(
-        db.scalars(select(ContactRelationship).where(ContactRelationship.related_contact_id == contact_id)).all()
-    )
-    return outgoing + incoming
-
-
-@router.patch(
-    "/contacts/{contact_id}/relationships/{related_contact_id}",
-    response_model=ContactRelationshipResponse,
-)
-def update_contact_relationship(
-    contact_id: int,
-    related_contact_id: int,
-    update: ContactRelationshipUpdate,
-    db: DbSession,
-) -> ContactRelationship:
-    """Update a relationship between two contacts."""
-    relationship = db.scalar(
-        select(ContactRelationship).where(
-            ContactRelationship.contact_id == contact_id,
-            ContactRelationship.related_contact_id == related_contact_id,
-        )
-    )
-    if not relationship:
-        raise HTTPException(status_code=404, detail="Relationship not found")
-
-    if update.relationship_type is not None:
-        relationship.relationship_type = update.relationship_type.value
-    if update.closeness_weight is not None:
-        relationship.closeness_weight = update.closeness_weight
-
-    db.commit()
-    db.refresh(relationship)
-    return relationship
-
-
-@router.delete("/contacts/{contact_id}/relationships/{related_contact_id}", response_model=None)
-def remove_contact_relationship(
-    contact_id: int,
-    related_contact_id: int,
-    request: Request,
-    db: DbSession,
-) -> dict[str, bool] | HTMLResponse:
-    """Remove a relationship between two contacts."""
-    relationship = db.scalar(
-        select(ContactRelationship).where(
-            ContactRelationship.contact_id == contact_id,
-            ContactRelationship.related_contact_id == related_contact_id,
-        )
-    )
-    if not relationship:
-        raise HTTPException(status_code=404, detail="Relationship not found")
-
-    db.delete(relationship)
-    db.commit()
-    if _is_htmx(request):
-        return HTMLResponse("")
-    return {"deleted": True}
-
-
-@router.get("/relationship-types")
-def list_relationship_types() -> list[RelationshipTypeInfo]:
-    """List all available relationship types with their default weights."""
-    return [
-        RelationshipTypeInfo(
-            value=rt.value,
-            display_name=rt.display_name,
-            default_weight=rt.default_weight,
-        )
-        for rt in RelationshipType
-    ]
-
-
-@router.get("/graph")
-def get_relationship_graph(db: DbSession) -> GraphData:
-    """Get all contacts and relationships as graph data for visualization."""
-    contacts = list(db.scalars(select(Contact)).all())
-    relationships = list(db.scalars(select(ContactRelationship)).all())
-
-    nodes = [GraphNode(id=c.id, name=c.name, current_job=c.current_job) for c in contacts]
-
-    edges = [
-        GraphEdge(
-            source=rel.contact_id,
-            target=rel.related_contact_id,
-            relationship_type=rel.relationship_type,
-            closeness_weight=rel.closeness_weight,
-        )
-        for rel in relationships
-    ]
-
-    return GraphData(nodes=nodes, edges=edges)

python/api/routers/views.py

@@ -1,345 +0,0 @@
-"""HTMX server-rendered view router."""
-
-from pathlib import Path
-from typing import Annotated, Any
-
-from fastapi import APIRouter, Form, HTTPException, Request
-from fastapi.responses import HTMLResponse, RedirectResponse
-from fastapi.templating import Jinja2Templates
-from sqlalchemy import select
-from sqlalchemy.orm import Session, selectinload
-
-from python.fastapi_tools.db import DbSession  # noqa: TC001 this is a FastAPI needed at runtime
-from python.orm.richie.contact import Contact, ContactRelationship, Need, RelationshipType
-
-TEMPLATES_DIR = Path(__file__).parent.parent / "templates"
-templates = Jinja2Templates(directory=TEMPLATES_DIR)
-
-router = APIRouter(tags=["views"])
-
-FAMILIAL_TYPES = {
-    "parent",
-    "child",
-    "sibling",
-    "grandparent",
-    "grandchild",
-    "aunt_uncle",
-    "niece_nephew",
-    "cousin",
-    "in_law",
-}
-FRIEND_TYPES = {"best_friend", "close_friend", "friend", "acquaintance", "neighbor"}
-PARTNER_TYPES = {"spouse", "partner"}
-PROFESSIONAL_TYPES = {"mentor", "mentee", "business_partner", "colleague", "manager", "direct_report", "client"}
-
-CONTACT_STRING_FIELDS = (
-    "name",
-    "legal_name",
-    "suffix",
-    "gender",
-    "current_job",
-    "timezone",
-    "profile_pic",
-    "bio",
-    "goals",
-    "social_structure_style",
-    "safe_conversation_starters",
-    "topics_to_avoid",
-    "ssn",
-)
-
-CONTACT_INT_FIELDS = ("age", "self_sufficiency_score")
-
-
-def _group_relationships(relationships: list[ContactRelationship]) -> dict[str, list[ContactRelationship]]:
-    """Group relationships by category."""
-    groups: dict[str, list[ContactRelationship]] = {
-        "familial": [],
-        "partners": [],
-        "friends": [],
-        "professional": [],
-        "other": [],
-    }
-    for rel in relationships:
-        if rel.relationship_type in FAMILIAL_TYPES:
-            groups["familial"].append(rel)
-        elif rel.relationship_type in PARTNER_TYPES:
-            groups["partners"].append(rel)
-        elif rel.relationship_type in FRIEND_TYPES:
-            groups["friends"].append(rel)
-        elif rel.relationship_type in PROFESSIONAL_TYPES:
-            groups["professional"].append(rel)
-        else:
-            groups["other"].append(rel)
-    return groups
-
-
-def _build_contact_name_map(database: Session, contact: Contact) -> dict[int, str]:
-    """Build a mapping of contact IDs to names for relationship display."""
-    related_ids = {rel.related_contact_id for rel in contact.related_to}
-    related_ids |= {rel.contact_id for rel in contact.related_from}
-    related_ids.discard(contact.id)
-
-    if not related_ids:
-        return {}
-
-    related_contacts = list(database.scalars(select(Contact).where(Contact.id.in_(related_ids))).all())
-    return {related.id: related.name for related in related_contacts}
-
-
-def _get_relationship_type_display() -> dict[str, str]:
-    """Build a mapping of relationship type values to display names."""
-    return {rel_type.value: rel_type.display_name for rel_type in RelationshipType}
-
-
-async def _parse_contact_form(request: Request) -> dict[str, Any]:
-    """Parse contact form data from a multipart/form request."""
-    form_data = await request.form()
-    result: dict[str, Any] = {}
-
-    for field in CONTACT_STRING_FIELDS:
-        value = form_data.get(field, "")
-        result[field] = str(value) if value else None
-
-    for field in CONTACT_INT_FIELDS:
-        value = form_data.get(field, "")
-        result[field] = int(value) if value else None
-
-    result["need_ids"] = [int(value) for value in form_data.getlist("need_ids")]
-    return result
-
-
-def _save_contact_from_form(database: Session, contact: Contact, form_result: dict[str, Any]) -> None:
-    """Apply parsed form data to a Contact and save associated needs."""
-    need_ids = form_result.pop("need_ids")
-
-    for key, value in form_result.items():
-        setattr(contact, key, value)
-
-    if need_ids:
-        contact.needs = list(database.scalars(select(Need).where(Need.id.in_(need_ids))).all())
-    else:
-        contact.needs = []
-
-
-@router.get("/", response_class=HTMLResponse)
-@router.get("/contacts", response_class=HTMLResponse)
-def contact_list_page(request: Request, database: DbSession) -> HTMLResponse:
-    """Render the contacts list page."""
-    contacts = list(database.scalars(select(Contact)).all())
-    return templates.TemplateResponse(request, "contact_list.html", {"contacts": contacts})
-
-
-@router.get("/contacts/new", response_class=HTMLResponse)
-def new_contact_page(request: Request, database: DbSession) -> HTMLResponse:
-    """Render the new contact form page."""
-    all_needs = list(database.scalars(select(Need)).all())
-    return templates.TemplateResponse(request, "contact_form.html", {"contact": None, "all_needs": all_needs})
-
-
-@router.post("/htmx/contacts/new")
-async def create_contact_form(request: Request, database: DbSession) -> RedirectResponse:
-    """Handle the create contact form submission."""
-    form_result = await _parse_contact_form(request)
-    contact = Contact()
-    _save_contact_from_form(database, contact, form_result)
-
-    database.add(contact)
-    database.commit()
-    database.refresh(contact)
-    return RedirectResponse(url=f"/contacts/{contact.id}", status_code=303)
-
-
-@router.get("/contacts/{contact_id}", response_class=HTMLResponse)
-def contact_detail_page(contact_id: int, request: Request, database: DbSession) -> HTMLResponse:
-    """Render the contact detail page."""
-    contact = database.scalar(
-        select(Contact)
-        .where(Contact.id == contact_id)
-        .options(
-            selectinload(Contact.needs),
-            selectinload(Contact.related_to),
-            selectinload(Contact.related_from),
-        )
-    )
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    contact_names = _build_contact_name_map(database, contact)
-    grouped_relationships = _group_relationships(contact.related_to)
-    all_contacts = list(database.scalars(select(Contact)).all())
-    all_needs = list(database.scalars(select(Need)).all())
-    available_needs = [need for need in all_needs if need not in contact.needs]
-
-    return templates.TemplateResponse(
-        request,
-        "contact_detail.html",
-        {
-            "contact": contact,
-            "contact_names": contact_names,
-            "grouped_relationships": grouped_relationships,
-            "all_contacts": all_contacts,
-            "available_needs": available_needs,
-            "relationship_types": list(RelationshipType),
-        },
-    )
-
-
-@router.get("/contacts/{contact_id}/edit", response_class=HTMLResponse)
-def edit_contact_page(contact_id: int, request: Request, database: DbSession) -> HTMLResponse:
-    """Render the edit contact form page."""
-    contact = database.scalar(select(Contact).where(Contact.id == contact_id).options(selectinload(Contact.needs)))
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    all_needs = list(database.scalars(select(Need)).all())
-    return templates.TemplateResponse(request, "contact_form.html", {"contact": contact, "all_needs": all_needs})
-
-
-@router.post("/htmx/contacts/{contact_id}/edit")
-async def update_contact_form(contact_id: int, request: Request, database: DbSession) -> RedirectResponse:
-    """Handle the edit contact form submission."""
-    contact = database.get(Contact, contact_id)
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    form_result = await _parse_contact_form(request)
-    _save_contact_from_form(database, contact, form_result)
-
-    database.commit()
-    return RedirectResponse(url=f"/contacts/{contact_id}", status_code=303)
-
-
-@router.post("/htmx/contacts/{contact_id}/add-need", response_class=HTMLResponse)
-def add_need_to_contact_htmx(
-    contact_id: int,
-    request: Request,
-    database: DbSession,
-    need_id: Annotated[int, Form()],
-) -> HTMLResponse:
-    """Add a need to a contact and return updated manage-needs partial."""
-    contact = database.scalar(select(Contact).where(Contact.id == contact_id).options(selectinload(Contact.needs)))
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    need = database.get(Need, need_id)
-    if not need:
-        raise HTTPException(status_code=404, detail="Need not found")
-
-    if need not in contact.needs:
-        contact.needs.append(need)
-        database.commit()
-        database.refresh(contact)
-
-    return templates.TemplateResponse(request, "partials/manage_needs.html", {"contact": contact})
-
-
-@router.post("/htmx/contacts/{contact_id}/add-relationship", response_class=HTMLResponse)
-def add_relationship_htmx(
-    contact_id: int,
-    request: Request,
-    database: DbSession,
-    related_contact_id: Annotated[int, Form()],
-    relationship_type: Annotated[str, Form()],
-) -> HTMLResponse:
-    """Add a relationship and return updated manage-relationships partial."""
-    contact = database.scalar(select(Contact).where(Contact.id == contact_id).options(selectinload(Contact.related_to)))
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    related_contact = database.get(Contact, related_contact_id)
-    if not related_contact:
-        raise HTTPException(status_code=404, detail="Related contact not found")
-
-    rel_type = RelationshipType(relationship_type)
-    weight = rel_type.default_weight
-
-    relationship = ContactRelationship(
-        contact_id=contact_id,
-        related_contact_id=related_contact_id,
-        relationship_type=relationship_type,
-        closeness_weight=weight,
-    )
-    database.add(relationship)
-    database.commit()
-    database.refresh(contact)
-
-    contact_names = _build_contact_name_map(database, contact)
-    return templates.TemplateResponse(
-        request,
-        "partials/manage_relationships.html",
-        {"contact": contact, "contact_names": contact_names},
-    )
-
-
-@router.post("/htmx/contacts/{contact_id}/relationships/{related_contact_id}/weight")
-def update_relationship_weight_htmx(
-    contact_id: int,
-    related_contact_id: int,
-    database: DbSession,
-    closeness_weight: Annotated[int, Form()],
-) -> HTMLResponse:
-    """Update a relationship's closeness weight from HTMX range input."""
-    relationship = database.scalar(
-        select(ContactRelationship).where(
-            ContactRelationship.contact_id == contact_id,
-            ContactRelationship.related_contact_id == related_contact_id,
-        )
-    )
-    if not relationship:
-        raise HTTPException(status_code=404, detail="Relationship not found")
-
-    relationship.closeness_weight = closeness_weight
-    database.commit()
-    return HTMLResponse("")
-
-
-@router.post("/htmx/needs", response_class=HTMLResponse)
-def create_need_htmx(
-    request: Request,
-    database: DbSession,
-    name: Annotated[str, Form()],
-    description: Annotated[str, Form()] = "",
-) -> HTMLResponse:
-    """Create a need via form data and return updated needs list."""
-    need = Need(name=name, description=description or None)
-    database.add(need)
-    database.commit()
-    needs = list(database.scalars(select(Need)).all())
-    return templates.TemplateResponse(request, "partials/need_items.html", {"needs": needs})
-
-
-@router.get("/needs", response_class=HTMLResponse)
-def needs_page(request: Request, database: DbSession) -> HTMLResponse:
-    """Render the needs list page."""
-    needs = list(database.scalars(select(Need)).all())
-    return templates.TemplateResponse(request, "need_list.html", {"needs": needs})
-
-
-@router.get("/graph", response_class=HTMLResponse)
-def graph_page(request: Request, database: DbSession) -> HTMLResponse:
-    """Render the relationship graph page."""
-    contacts = list(database.scalars(select(Contact)).all())
-    relationships = list(database.scalars(select(ContactRelationship)).all())
-
-    graph_data = {
-        "nodes": [{"id": contact.id, "name": contact.name, "current_job": contact.current_job} for contact in contacts],
-        "edges": [
-            {
-                "source": rel.contact_id,
-                "target": rel.related_contact_id,
-                "relationship_type": rel.relationship_type,
-                "closeness_weight": rel.closeness_weight,
-            }
-            for rel in relationships
-        ],
-    }
-
-    return templates.TemplateResponse(
-        request,
-        "graph.html",
-        {
-            "graph_data": graph_data,
-            "relationship_type_display": _get_relationship_type_display(),
-        },
-    )

python/api/templates/base.html

@@ -1,198 +0,0 @@
-<!DOCTYPE html>
-<html lang="en" data-theme="light">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>{% block title %}Contact Database{% endblock %}</title>
-    <script src="https://unpkg.com/htmx.org@2.0.4"></script>
-    <style>
-        :root {
-            --color-bg: #f5f5f5;
-            --color-bg-card: #ffffff;
-            --color-bg-hover: #f0f0f0;
-            --color-bg-muted: #f9f9f9;
-            --color-bg-error: #ffe0e0;
-            --color-text: #333333;
-            --color-text-muted: #666666;
-            --color-text-error: #cc0000;
-            --color-border: #dddddd;
-            --color-border-light: #eeeeee;
-            --color-border-lighter: #f0f0f0;
-            --color-primary: #0066cc;
-            --color-primary-hover: #0055aa;
-            --color-danger: #cc3333;
-            --color-danger-hover: #aa2222;
-            --color-tag-bg: #e0e0e0;
-            --shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
-            font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
-            line-height: 1.5;
-            color: var(--color-text);
-            background-color: var(--color-bg);
-        }
-        [data-theme="dark"] {
-            --color-bg: #1a1a1a;
-            --color-bg-card: #2d2d2d;
-            --color-bg-hover: #3d3d3d;
-            --color-bg-muted: #252525;
-            --color-bg-error: #4a2020;
-            --color-text: #e0e0e0;
-            --color-text-muted: #a0a0a0;
-            --color-text-error: #ff6b6b;
-            --color-border: #404040;
-            --color-border-light: #353535;
-            --color-border-lighter: #303030;
-            --color-primary: #4da6ff;
-            --color-primary-hover: #7dbfff;
-            --color-danger: #ff6b6b;
-            --color-danger-hover: #ff8a8a;
-            --color-tag-bg: #404040;
-            --shadow: 0 1px 3px rgba(0, 0, 0, 0.3);
-        }
-        * { box-sizing: border-box; }
-        body { margin: 0; background: var(--color-bg); color: var(--color-text); }
-        .app { max-width: 1000px; margin: 0 auto; padding: 20px; }
-        nav { display: flex; align-items: center; gap: 20px; padding: 15px 0; border-bottom: 1px solid var(--color-border); margin-bottom: 20px; }
-        nav a { color: var(--color-primary); text-decoration: none; font-weight: 500; }
-        nav a:hover { text-decoration: underline; }
-        .theme-toggle { margin-left: auto; }
-        main { background: var(--color-bg-card); padding: 20px; border-radius: 8px; box-shadow: var(--shadow); }
-        .header { display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px; }
-        .header h1 { margin: 0; }
-        a { color: var(--color-primary); }
-        a:hover { text-decoration: underline; }
-
-        .btn { display: inline-block; padding: 8px 16px; border: 1px solid var(--color-border); border-radius: 4px; background: var(--color-bg-card); color: var(--color-text); text-decoration: none; cursor: pointer; font-size: 14px; margin-left: 8px; }
-        .btn:hover { background: var(--color-bg-hover); }
-        .btn-primary { background: var(--color-primary); border-color: var(--color-primary); color: white; }
-        .btn-primary:hover { background: var(--color-primary-hover); }
-        .btn-danger { background: var(--color-danger); border-color: var(--color-danger); color: white; }
-        .btn-danger:hover { background: var(--color-danger-hover); }
-        .btn-small { padding: 4px 8px; font-size: 12px; }
-        .btn:disabled { opacity: 0.6; cursor: not-allowed; }
-
-        table { width: 100%; border-collapse: collapse; }
-        th, td { padding: 12px; text-align: left; border-bottom: 1px solid var(--color-border-light); }
-        th { font-weight: 600; background: var(--color-bg-muted); }
-        tr:hover { background: var(--color-bg-muted); }
-
-        .error { background: var(--color-bg-error); color: var(--color-text-error); padding: 10px; border-radius: 4px; margin-bottom: 20px; }
-        .tag { display: inline-block; background: var(--color-tag-bg); padding: 2px 8px; border-radius: 12px; font-size: 12px; color: var(--color-text-muted); }
-
-        .add-form { display: flex; gap: 10px; margin-top: 15px; flex-wrap: wrap; }
-        .add-form select, .add-form input { padding: 8px; border: 1px solid var(--color-border); border-radius: 4px; min-width: 200px; background: var(--color-bg-card); color: var(--color-text); }
-
-        .form-group { margin-bottom: 20px; }
-        .form-group label { display: block; font-weight: 500; margin-bottom: 5px; }
-        .form-group input, .form-group textarea, .form-group select { width: 100%; padding: 10px; border: 1px solid var(--color-border); border-radius: 4px; font-size: 14px; background: var(--color-bg-card); color: var(--color-text); }
-        .form-group textarea { resize: vertical; }
-        .form-row { display: grid; grid-template-columns: 1fr 1fr; gap: 20px; }
-        .checkbox-group { display: flex; flex-wrap: wrap; gap: 15px; }
-        .checkbox-label { display: flex; align-items: center; gap: 5px; cursor: pointer; }
-        .form-actions { display: flex; gap: 10px; margin-top: 30px; padding-top: 20px; border-top: 1px solid var(--color-border-light); }
-
-        .need-form { background: var(--color-bg-muted); padding: 20px; border-radius: 4px; margin-bottom: 20px; }
-        .need-items { list-style: none; padding: 0; }
-        .need-items li { display: flex; justify-content: space-between; align-items: flex-start; padding: 15px; border: 1px solid var(--color-border-light); border-radius: 4px; margin-bottom: 10px; }
-        .need-info p { margin: 5px 0 0; color: var(--color-text-muted); font-size: 14px; }
-
-        .graph-container { width: 100%; }
-        .graph-hint { color: var(--color-text-muted); font-size: 14px; margin-bottom: 15px; }
-        .selected-info { margin-top: 15px; padding: 15px; background: var(--color-bg-muted); border-radius: 8px; }
-        .selected-info h3 { margin: 0 0 10px; }
-        .selected-info p { margin: 5px 0; color: var(--color-text-muted); }
-        .legend { margin-top: 20px; padding: 15px; background: var(--color-bg-muted); border-radius: 8px; }
-        .legend h4 { margin: 0 0 10px; font-size: 14px; }
-        .legend-items { display: flex; flex-wrap: wrap; gap: 15px; }
-        .legend-item { display: flex; align-items: center; gap: 8px; font-size: 12px; color: var(--color-text-muted); }
-        .legend-line { width: 30px; border-radius: 2px; }
-
-        .id-card { width: 100%; }
-        .id-card-inner { background: linear-gradient(135deg, #0a0a0f 0%, #1a1a2e 50%, #0a0a0f 100%); background-image: radial-gradient(white 1px, transparent 1px), linear-gradient(135deg, #0a0a0f 0%, #1a1a2e 50%, #0a0a0f 100%); background-size: 50px 50px, 100% 100%; color: #fff; border-radius: 12px; padding: 25px; min-height: 500px; position: relative; overflow: hidden; }
-        .id-card-header { display: flex; justify-content: space-between; align-items: flex-start; margin-bottom: 15px; }
-        .id-card-header-left { flex: 1; }
-        .id-card-header-right { display: flex; flex-direction: column; align-items: flex-end; gap: 10px; }
-        .id-card-title { font-size: 2.5rem; font-weight: 700; margin: 0; color: #fff; text-shadow: 2px 2px 4px rgba(0,0,0,0.5); }
-        .id-profile-pic { width: 80px; height: 80px; border-radius: 8px; object-fit: cover; border: 2px solid rgba(255,255,255,0.3); }
-        .id-profile-placeholder { width: 80px; height: 80px; border-radius: 8px; background: linear-gradient(135deg, #4ecdc4 0%, #44a8a0 100%); display: flex; align-items: center; justify-content: center; border: 2px solid rgba(255,255,255,0.3); }
-        .id-profile-placeholder span { font-size: 2rem; font-weight: 700; color: #fff; text-shadow: 1px 1px 2px rgba(0,0,0,0.3); }
-        .id-card-actions { display: flex; gap: 8px; }
-        .id-card-actions .btn { background: rgba(255,255,255,0.1); border-color: rgba(255,255,255,0.3); color: #fff; }
-        .id-card-actions .btn:hover { background: rgba(255,255,255,0.2); }
-        .id-card-body { display: grid; grid-template-columns: 1fr 1.5fr; gap: 30px; }
-        .id-card-left { display: flex; flex-direction: column; gap: 8px; }
-        .id-field { font-size: 1rem; line-height: 1.4; }
-        .id-field-block { margin-top: 15px; font-size: 0.95rem; line-height: 1.5; }
-        .id-label { color: #4ecdc4; font-weight: 500; }
-        .id-card-right { display: flex; flex-direction: column; gap: 20px; }
-        .id-bio { font-size: 0.9rem; line-height: 1.6; color: #e0e0e0; }
-        .id-relationships { margin-top: 10px; }
-        .id-section-title { font-size: 1.5rem; margin: 0 0 15px; color: #fff; border-bottom: 1px solid rgba(255,255,255,0.2); padding-bottom: 8px; }
-        .id-rel-group { margin-bottom: 12px; font-size: 0.9rem; line-height: 1.6; }
-        .id-rel-label { color: #a0a0a0; }
-        .id-rel-group a { color: #4ecdc4; text-decoration: none; }
-        .id-rel-group a:hover { text-decoration: underline; }
-        .id-rel-type { color: #888; font-size: 0.85em; }
-        .id-card-warnings { margin-top: 30px; padding-top: 20px; border-top: 1px solid rgba(255,255,255,0.2); display: flex; flex-wrap: wrap; gap: 20px; }
-        .id-warning { display: flex; align-items: center; gap: 8px; font-size: 0.9rem; color: #ff6b6b; }
-        .warning-dot { width: 8px; height: 8px; background: #ff6b6b; border-radius: 50%; flex-shrink: 0; }
-        .warning-desc { color: #ccc; }
-
-        .id-card-manage { margin-top: 20px; background: var(--color-bg-muted); border-radius: 8px; padding: 15px; }
-        .id-card-manage summary { cursor: pointer; font-weight: 600; font-size: 1.1rem; padding: 5px 0; }
-        .id-card-manage[open] summary { margin-bottom: 15px; border-bottom: 1px solid var(--color-border-light); padding-bottom: 10px; }
-        .manage-section { margin-bottom: 25px; }
-        .manage-section h3 { margin: 0 0 15px; font-size: 1rem; }
-        .manage-relationships { display: flex; flex-direction: column; gap: 10px; margin-bottom: 15px; }
-        .manage-rel-item { display: flex; align-items: center; gap: 12px; padding: 10px; background: var(--color-bg-card); border-radius: 6px; flex-wrap: wrap; }
-        .manage-rel-item a { font-weight: 500; min-width: 120px; }
-        .weight-control { display: flex; align-items: center; gap: 8px; font-size: 12px; color: var(--color-text-muted); }
-        .weight-control input[type="range"] { width: 80px; cursor: pointer; }
-        .weight-value { min-width: 20px; text-align: center; font-weight: 600; }
-        .manage-needs-list { list-style: none; padding: 0; margin: 0 0 15px; }
-        .manage-needs-list li { display: flex; align-items: center; gap: 12px; padding: 10px; background: var(--color-bg-card); border-radius: 6px; margin-bottom: 8px; }
-        .manage-needs-list li .btn { margin-left: auto; }
-
-        .htmx-indicator { display: none; }
-        .htmx-request .htmx-indicator { display: inline; }
-        .htmx-request.htmx-indicator { display: inline; }
-
-        @media (max-width: 768px) {
-            .id-card-body { grid-template-columns: 1fr; }
-            .id-card-title { font-size: 1.8rem; }
-            .id-card-header { flex-direction: column; gap: 15px; }
-        }
-    </style>
-</head>
-<body>
-    <div class="app">
-        <nav>
-            <a href="/contacts">Contacts</a>
-            <a href="/graph">Graph</a>
-            <a href="/needs">Needs</a>
-            <button class="btn btn-small theme-toggle" onclick="toggleTheme()">
-                <span id="theme-label">Dark</span>
-            </button>
-        </nav>
-
-        <main id="main-content">
-            {% block content %}{% endblock %}
-        </main>
-    </div>
-
-    <script>
-        function toggleTheme() {
-            const html = document.documentElement;
-            const current = html.getAttribute('data-theme');
-            const next = current === 'light' ? 'dark' : 'light';
-            html.setAttribute('data-theme', next);
-            localStorage.setItem('theme', next);
-            document.getElementById('theme-label').textContent = next === 'light' ? 'Dark' : 'Light';
-        }
-        (function() {
-            const saved = localStorage.getItem('theme') || 'light';
-            document.documentElement.setAttribute('data-theme', saved);
-            document.getElementById('theme-label').textContent = saved === 'light' ? 'Dark' : 'Light';
-        })();
-    </script>
-</body>
-</html>

python/api/templates/contact_detail.html

@@ -1,204 +0,0 @@
-{% extends "base.html" %}
-{% block title %}{{ contact.name }}{% endblock %}
-{% block content %}
-<div class="id-card">
-    <div class="id-card-inner">
-        <div class="id-card-header">
-            <div class="id-card-header-left">
-                <h1 class="id-card-title">I.D.: {{ contact.name }}</h1>
-            </div>
-            <div class="id-card-header-right">
-                {% if contact.profile_pic %}
-                <img src="{{ contact.profile_pic }}" alt="{{ contact.name }}'s profile" class="id-profile-pic">
-                {% else %}
-                <div class="id-profile-placeholder">
-                    <span>{{ contact.name[0]|upper }}</span>
-                </div>
-                {% endif %}
-                <div class="id-card-actions">
-                    <a href="/contacts/{{ contact.id }}/edit" class="btn btn-small">Edit</a>
-                    <a href="/contacts" class="btn btn-small">Back</a>
-                </div>
-            </div>
-        </div>
-
-        <div class="id-card-body">
-            <div class="id-card-left">
-                {% if contact.legal_name %}
-                <div class="id-field">Legal name: {{ contact.legal_name }}</div>
-                {% endif %}
-                {% if contact.suffix %}
-                <div class="id-field">Suffix: {{ contact.suffix }}</div>
-                {% endif %}
-                {% if contact.gender %}
-                <div class="id-field">Gender: {{ contact.gender }}</div>
-                {% endif %}
-                {% if contact.age %}
-                <div class="id-field">Age: {{ contact.age }}</div>
-                {% endif %}
-                {% if contact.current_job %}
-                <div class="id-field">Job: {{ contact.current_job }}</div>
-                {% endif %}
-                {% if contact.social_structure_style %}
-                <div class="id-field">Social style: {{ contact.social_structure_style }}</div>
-                {% endif %}
-                {% if contact.self_sufficiency_score is not none %}
-                <div class="id-field">Self-Sufficiency: {{ contact.self_sufficiency_score }}</div>
-                {% endif %}
-                {% if contact.timezone %}
-                <div class="id-field">Timezone: {{ contact.timezone }}</div>
-                {% endif %}
-                {% if contact.safe_conversation_starters %}
-                <div class="id-field-block">
-                    <span class="id-label">Safe con starters:</span> {{ contact.safe_conversation_starters }}
-                </div>
-                {% endif %}
-                {% if contact.topics_to_avoid %}
-                <div class="id-field-block">
-                    <span class="id-label">Topics to avoid:</span> {{ contact.topics_to_avoid }}
-                </div>
-                {% endif %}
-                {% if contact.goals %}
-                <div class="id-field-block">
-                    <span class="id-label">Goals:</span> {{ contact.goals }}
-                </div>
-                {% endif %}
-            </div>
-
-            <div class="id-card-right">
-                {% if contact.bio %}
-                <div class="id-bio">
-                    <span class="id-label">Bio:</span> {{ contact.bio }}
-                </div>
-                {% endif %}
-
-                <div class="id-relationships">
-                    <h2 class="id-section-title">Relationships</h2>
-
-                    {% if grouped_relationships.familial %}
-                    <div class="id-rel-group">
-                        <span class="id-rel-label">Familial:</span>
-                        {% for rel in grouped_relationships.familial %}
-                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a><span class="id-rel-type">({{ rel.relationship_type|replace("_", " ")|title }})</span>{% if not loop.last %}, {% endif %}
-                        {% endfor %}
-                    </div>
-                    {% endif %}
-
-                    {% if grouped_relationships.partners %}
-                    <div class="id-rel-group">
-                        <span class="id-rel-label">Partners:</span>
-                        {% for rel in grouped_relationships.partners %}
-                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a>{% if not loop.last %}, {% endif %}
-                        {% endfor %}
-                    </div>
-                    {% endif %}
-
-                    {% if grouped_relationships.friends %}
-                    <div class="id-rel-group">
-                        <span class="id-rel-label">Friends:</span>
-                        {% for rel in grouped_relationships.friends %}
-                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a>{% if not loop.last %}, {% endif %}
-                        {% endfor %}
-                    </div>
-                    {% endif %}
-
-                    {% if grouped_relationships.professional %}
-                    <div class="id-rel-group">
-                        <span class="id-rel-label">Professional:</span>
-                        {% for rel in grouped_relationships.professional %}
-                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a><span class="id-rel-type">({{ rel.relationship_type|replace("_", " ")|title }})</span>{% if not loop.last %}, {% endif %}
-                        {% endfor %}
-                    </div>
-                    {% endif %}
-
-                    {% if grouped_relationships.other %}
-                    <div class="id-rel-group">
-                        <span class="id-rel-label">Other:</span>
-                        {% for rel in grouped_relationships.other %}
-                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a><span class="id-rel-type">({{ rel.relationship_type|replace("_", " ")|title }})</span>{% if not loop.last %}, {% endif %}
-                        {% endfor %}
-                    </div>
-                    {% endif %}
-
-                    {% if contact.related_from %}
-                    <div class="id-rel-group">
-                        <span class="id-rel-label">Known by:</span>
-                        {% for rel in contact.related_from %}
-                        <a href="/contacts/{{ rel.contact_id }}">{{ contact_names[rel.contact_id] }}</a>{% if not loop.last %}, {% endif %}
-                        {% endfor %}
-                    </div>
-                    {% endif %}
-                </div>
-            </div>
-        </div>
-
-        {% if contact.needs %}
-        <div class="id-card-warnings">
-            {% for need in contact.needs %}
-            <div class="id-warning">
-                <span class="warning-dot"></span>
-                Warning: {{ need.name }}
-                {% if need.description %}<span class="warning-desc"> - {{ need.description }}</span>{% endif %}
-            </div>
-            {% endfor %}
-        </div>
-        {% endif %}
-    </div>
-
-    <details class="id-card-manage">
-        <summary>Manage Contact</summary>
-
-        <div class="manage-section">
-            <h3>Manage Relationships</h3>
-            <div id="manage-relationships" class="manage-relationships">
-                {% include "partials/manage_relationships.html" %}
-            </div>
-
-            {% if all_contacts %}
-            <form hx-post="/htmx/contacts/{{ contact.id }}/add-relationship"
-                  hx-target="#manage-relationships"
-                  hx-swap="innerHTML"
-                  class="add-form">
-                <select name="related_contact_id" required>
-                    <option value="">Select contact...</option>
-                    {% for other in all_contacts %}
-                    {% if other.id != contact.id %}
-                    <option value="{{ other.id }}">{{ other.name }}</option>
-                    {% endif %}
-                    {% endfor %}
-                </select>
-                <select name="relationship_type" required>
-                    <option value="">Select relationship type...</option>
-                    {% for rel_type in relationship_types %}
-                    <option value="{{ rel_type.value }}">{{ rel_type.display_name }}</option>
-                    {% endfor %}
-                </select>
-                <button type="submit" class="btn btn-primary">Add Relationship</button>
-            </form>
-            {% endif %}
-        </div>
-
-        <div class="manage-section">
-            <h3>Manage Needs/Warnings</h3>
-            <div id="manage-needs">
-                {% include "partials/manage_needs.html" %}
-            </div>
-
-            {% if available_needs %}
-            <form hx-post="/htmx/contacts/{{ contact.id }}/add-need"
-                  hx-target="#manage-needs"
-                  hx-swap="innerHTML"
-                  class="add-form">
-                <select name="need_id" required>
-                    <option value="">Select a need...</option>
-                    {% for need in available_needs %}
-                    <option value="{{ need.id }}">{{ need.name }}</option>
-                    {% endfor %}
-                </select>
-                <button type="submit" class="btn btn-primary">Add Need</button>
-            </form>
-            {% endif %}
-        </div>
-    </details>
-</div>
-{% endblock %}

python/api/templates/contact_form.html

@@ -1,115 +0,0 @@
-{% extends "base.html" %}
-{% block title %}{{ "Edit " + contact.name if contact else "New Contact" }}{% endblock %}
-{% block content %}
-<div class="contact-form">
-    <h1>{{ "Edit Contact" if contact else "New Contact" }}</h1>
-
-    {% if contact %}
-    <form method="post" action="/htmx/contacts/{{ contact.id }}/edit">
-    {% else %}
-    <form method="post" action="/htmx/contacts/new">
-    {% endif %}
-
-        <div class="form-group">
-            <label for="name">Name *</label>
-            <input id="name" name="name" type="text" value="{{ contact.name if contact else '' }}" required>
-        </div>
-
-        <div class="form-row">
-            <div class="form-group">
-                <label for="legal_name">Legal Name</label>
-                <input id="legal_name" name="legal_name" type="text" value="{{ contact.legal_name or '' }}">
-            </div>
-            <div class="form-group">
-                <label for="suffix">Suffix</label>
-                <input id="suffix" name="suffix" type="text" value="{{ contact.suffix or '' }}">
-            </div>
-        </div>
-
-        <div class="form-row">
-            <div class="form-group">
-                <label for="age">Age</label>
-                <input id="age" name="age" type="number" value="{{ contact.age if contact and contact.age is not none else '' }}">
-            </div>
-            <div class="form-group">
-                <label for="gender">Gender</label>
-                <input id="gender" name="gender" type="text" value="{{ contact.gender or '' }}">
-            </div>
-        </div>
-
-        <div class="form-group">
-            <label for="current_job">Current Job</label>
-            <input id="current_job" name="current_job" type="text" value="{{ contact.current_job or '' }}">
-        </div>
-
-        <div class="form-group">
-            <label for="timezone">Timezone</label>
-            <input id="timezone" name="timezone" type="text" value="{{ contact.timezone or '' }}">
-        </div>
-
-        <div class="form-group">
-            <label for="profile_pic">Profile Picture URL</label>
-            <input id="profile_pic" name="profile_pic" type="url" placeholder="https://example.com/photo.jpg" value="{{ contact.profile_pic or '' }}">
-        </div>
-
-        <div class="form-group">
-            <label for="bio">Bio</label>
-            <textarea id="bio" name="bio" rows="3">{{ contact.bio or '' }}</textarea>
-        </div>
-
-        <div class="form-group">
-            <label for="goals">Goals</label>
-            <textarea id="goals" name="goals" rows="3">{{ contact.goals or '' }}</textarea>
-        </div>
-
-        <div class="form-group">
-            <label for="social_structure_style">Social Structure Style</label>
-            <input id="social_structure_style" name="social_structure_style" type="text" value="{{ contact.social_structure_style or '' }}">
-        </div>
-
-        <div class="form-group">
-            <label for="self_sufficiency_score">Self-Sufficiency Score (1-10)</label>
-            <input id="self_sufficiency_score" name="self_sufficiency_score" type="number" min="1" max="10" value="{{ contact.self_sufficiency_score if contact and contact.self_sufficiency_score is not none else '' }}">
-        </div>
-
-        <div class="form-group">
-            <label for="safe_conversation_starters">Safe Conversation Starters</label>
-            <textarea id="safe_conversation_starters" name="safe_conversation_starters" rows="2">{{ contact.safe_conversation_starters or '' }}</textarea>
-        </div>
-
-        <div class="form-group">
-            <label for="topics_to_avoid">Topics to Avoid</label>
-            <textarea id="topics_to_avoid" name="topics_to_avoid" rows="2">{{ contact.topics_to_avoid or '' }}</textarea>
-        </div>
-
-        <div class="form-group">
-            <label for="ssn">SSN</label>
-            <input id="ssn" name="ssn" type="text" value="{{ contact.ssn or '' }}">
-        </div>
-
-        {% if all_needs %}
-        <div class="form-group">
-            <label>Needs/Accommodations</label>
-            <div class="checkbox-group">
-                {% for need in all_needs %}
-                <label class="checkbox-label">
-                    <input type="checkbox" name="need_ids" value="{{ need.id }}"
-                           {% if contact and need in contact.needs %}checked{% endif %}>
-                    {{ need.name }}
-                </label>
-                {% endfor %}
-            </div>
-        </div>
-        {% endif %}
-
-        <div class="form-actions">
-            <button type="submit" class="btn btn-primary">Save</button>
-            {% if contact %}
-            <a href="/contacts/{{ contact.id }}" class="btn">Cancel</a>
-            {% else %}
-            <a href="/contacts" class="btn">Cancel</a>
-            {% endif %}
-        </div>
-    </form>
-</div>
-{% endblock %}

python/api/templates/contact_list.html

@@ -1,14 +0,0 @@
-{% extends "base.html" %}
-{% block title %}Contacts{% endblock %}
-{% block content %}
-<div class="contact-list">
-    <div class="header">
-        <h1>Contacts</h1>
-        <a href="/contacts/new" class="btn btn-primary">Add Contact</a>
-    </div>
-
-    <div id="contact-table">
-        {% include "partials/contact_table.html" %}
-    </div>
-</div>
-{% endblock %}

python/api/templates/graph.html

@@ -1,198 +0,0 @@
-{% extends "base.html" %}
-{% block title %}Relationship Graph{% endblock %}
-{% block content %}
-<div class="graph-container">
-    <div class="header">
-        <h1>Relationship Graph</h1>
-    </div>
-    <p class="graph-hint">Drag nodes to reposition. Closer relationships have shorter, darker edges.</p>
-    <canvas id="graph-canvas" width="900" height="600"
-            style="border: 1px solid var(--color-border); border-radius: 8px; background: var(--color-bg); cursor: grab;">
-    </canvas>
-    <div id="selected-info"></div>
-    <div class="legend">
-        <h4>Relationship Closeness (1-10)</h4>
-        <div class="legend-items">
-            <div class="legend-item">
-                <span class="legend-line" style="background: hsl(220, 70%, 40%); height: 4px; display: inline-block;"></span>
-                <span>10 - Very Close (Spouse, Partner)</span>
-            </div>
-            <div class="legend-item">
-                <span class="legend-line" style="background: hsl(220, 70%, 52%); height: 3px; display: inline-block;"></span>
-                <span>7 - Close (Family, Best Friend)</span>
-            </div>
-            <div class="legend-item">
-                <span class="legend-line" style="background: hsl(220, 70%, 64%); height: 2px; display: inline-block;"></span>
-                <span>4 - Moderate (Friend, Colleague)</span>
-            </div>
-            <div class="legend-item">
-                <span class="legend-line" style="background: hsl(220, 70%, 72%); height: 1px; display: inline-block;"></span>
-                <span>2 - Distant (Acquaintance)</span>
-            </div>
-        </div>
-    </div>
-</div>
-
-<script>
-(function() {
-    const RELATIONSHIP_DISPLAY = {{ relationship_type_display|tojson }};
-    const graphData = {{ graph_data|tojson }};
-
-    const canvas = document.getElementById('graph-canvas');
-    const ctx = canvas.getContext('2d');
-    const width = canvas.width;
-    const height = canvas.height;
-    const centerX = width / 2;
-    const centerY = height / 2;
-
-    const nodes = graphData.nodes.map(function(node) {
-        return Object.assign({}, node, {
-            x: centerX + (Math.random() - 0.5) * 300,
-            y: centerY + (Math.random() - 0.5) * 300,
-            vx: 0,
-            vy: 0
-        });
-    });
-
-    const nodeMap = new Map(nodes.map(function(node) { return [node.id, node]; }));
-
-    const edges = graphData.edges.map(function(edge) {
-        const sourceNode = nodeMap.get(edge.source);
-        const targetNode = nodeMap.get(edge.target);
-        if (!sourceNode || !targetNode) return null;
-        return Object.assign({}, edge, { sourceNode: sourceNode, targetNode: targetNode });
-    }).filter(function(edge) { return edge !== null; });
-
-    let dragNode = null;
-    let selectedNode = null;
-
-    const repulsion = 5000;
-    const springStrength = 0.05;
-    const baseSpringLength = 150;
-    const damping = 0.9;
-    const centerPull = 0.01;
-
-    function simulate() {
-        for (const node of nodes) { node.vx = 0; node.vy = 0; }
-        for (let i = 0; i < nodes.length; i++) {
-            for (let j = i + 1; j < nodes.length; j++) {
-                const dx = nodes[j].x - nodes[i].x;
-                const dy = nodes[j].y - nodes[i].y;
-                const dist = Math.sqrt(dx * dx + dy * dy) || 1;
-                const force = repulsion / (dist * dist);
-                const fx = (dx / dist) * force;
-                const fy = (dy / dist) * force;
-                nodes[i].vx -= fx; nodes[i].vy -= fy;
-                nodes[j].vx += fx; nodes[j].vy += fy;
-            }
-        }
-        for (const edge of edges) {
-            const dx = edge.targetNode.x - edge.sourceNode.x;
-            const dy = edge.targetNode.y - edge.sourceNode.y;
-            const dist = Math.sqrt(dx * dx + dy * dy) || 1;
-            const normalizedWeight = edge.closeness_weight / 10;
-            const idealLength = baseSpringLength * (1.5 - normalizedWeight);
-            const displacement = dist - idealLength;
-            const force = springStrength * displacement;
-            const fx = (dx / dist) * force;
-            const fy = (dy / dist) * force;
-            edge.sourceNode.vx += fx; edge.sourceNode.vy += fy;
-            edge.targetNode.vx -= fx; edge.targetNode.vy -= fy;
-        }
-        for (const node of nodes) {
-            node.vx += (centerX - node.x) * centerPull;
-            node.vy += (centerY - node.y) * centerPull;
-        }
-        for (const node of nodes) {
-            if (node === dragNode) continue;
-            node.x += node.vx * damping;
-            node.y += node.vy * damping;
-            node.x = Math.max(30, Math.min(width - 30, node.x));
-            node.y = Math.max(30, Math.min(height - 30, node.y));
-        }
-    }
-
-    function getEdgeColor(weight) {
-        const normalized = weight / 10;
-        return 'hsl(220, 70%, ' + (80 - normalized * 40) + '%)';
-    }
-
-    function draw() {
-        ctx.clearRect(0, 0, width, height);
-        for (const edge of edges) {
-            const lineWidth = 1 + (edge.closeness_weight / 10) * 3;
-            ctx.strokeStyle = getEdgeColor(edge.closeness_weight);
-            ctx.lineWidth = lineWidth;
-            ctx.beginPath();
-            ctx.moveTo(edge.sourceNode.x, edge.sourceNode.y);
-            ctx.lineTo(edge.targetNode.x, edge.targetNode.y);
-            ctx.stroke();
-            const midX = (edge.sourceNode.x + edge.targetNode.x) / 2;
-            const midY = (edge.sourceNode.y + edge.targetNode.y) / 2;
-            ctx.fillStyle = '#666';
-            ctx.font = '10px sans-serif';
-            ctx.textAlign = 'center';
-            const label = RELATIONSHIP_DISPLAY[edge.relationship_type] || edge.relationship_type;
-            ctx.fillText(label, midX, midY - 5);
-        }
-        for (const node of nodes) {
-            const isSelected = node === selectedNode;
-            const radius = isSelected ? 25 : 20;
-            ctx.beginPath();
-            ctx.arc(node.x, node.y, radius, 0, Math.PI * 2);
-            ctx.fillStyle = isSelected ? '#0066cc' : '#fff';
-            ctx.fill();
-            ctx.strokeStyle = '#0066cc';
-            ctx.lineWidth = 2;
-            ctx.stroke();
-            ctx.fillStyle = isSelected ? '#fff' : '#333';
-            ctx.font = '12px sans-serif';
-            ctx.textAlign = 'center';
-            ctx.textBaseline = 'middle';
-            const name = node.name.length > 10 ? node.name.slice(0, 9) + '\u2026' : node.name;
-            ctx.fillText(name, node.x, node.y);
-        }
-    }
-
-    function animate() {
-        simulate();
-        draw();
-        requestAnimationFrame(animate);
-    }
-    animate();
-
-    function getNodeAt(x, y) {
-        for (const node of nodes) {
-            const dx = x - node.x;
-            const dy = y - node.y;
-            if (dx * dx + dy * dy < 400) return node;
-        }
-        return null;
-    }
-
-    canvas.addEventListener('mousedown', function(event) {
-        const rect = canvas.getBoundingClientRect();
-        const node = getNodeAt(event.clientX - rect.left, event.clientY - rect.top);
-        if (node) {
-            dragNode = node;
-            selectedNode = node;
-            const infoDiv = document.getElementById('selected-info');
-            let html = '<div class="selected-info"><h3>' + node.name + '</h3>';
-            if (node.current_job) html += '<p>Job: ' + node.current_job + '</p>';
-            html += '<a href="/contacts/' + node.id + '">View details</a></div>';
-            infoDiv.innerHTML = html;
-        }
-    });
-
-    canvas.addEventListener('mousemove', function(event) {
-        if (!dragNode) return;
-        const rect = canvas.getBoundingClientRect();
-        dragNode.x = event.clientX - rect.left;
-        dragNode.y = event.clientY - rect.top;
-    });
-
-    canvas.addEventListener('mouseup', function() { dragNode = null; });
-    canvas.addEventListener('mouseleave', function() { dragNode = null; });
-})();
-</script>
-{% endblock %}

python/api/templates/need_list.html

@@ -1,31 +0,0 @@
-{% extends "base.html" %}
-{% block title %}Needs{% endblock %}
-{% block content %}
-<div class="need-list">
-    <div class="header">
-        <h1>Needs / Accommodations</h1>
-        <button class="btn btn-primary" onclick="document.getElementById('need-form').toggleAttribute('hidden')">Add Need</button>
-    </div>
-
-    <form id="need-form" hidden
-          hx-post="/htmx/needs"
-          hx-target="#need-items"
-          hx-swap="innerHTML"
-          hx-on::after-request="if(event.detail.successful) this.reset()"
-          class="need-form">
-        <div class="form-group">
-            <label for="name">Name *</label>
-            <input id="name" name="name" type="text" placeholder="e.g., Light Sensitive, ADHD" required>
-        </div>
-        <div class="form-group">
-            <label for="description">Description</label>
-            <textarea id="description" name="description" placeholder="Optional description..." rows="2"></textarea>
-        </div>
-        <button type="submit" class="btn btn-primary">Create</button>
-    </form>
-
-    <div id="need-items">
-        {% include "partials/need_items.html" %}
-    </div>
-</div>
-{% endblock %}

python/api/templates/partials/contact_table.html

@@ -1,33 +0,0 @@
-{% if contacts %}
-<table>
-    <thead>
-        <tr>
-            <th>Name</th>
-            <th>Job</th>
-            <th>Timezone</th>
-            <th>Actions</th>
-        </tr>
-    </thead>
-    <tbody>
-        {% for contact in contacts %}
-        <tr id="contact-row-{{ contact.id }}">
-            <td><a href="/contacts/{{ contact.id }}">{{ contact.name }}</a></td>
-            <td>{{ contact.current_job or "-" }}</td>
-            <td>{{ contact.timezone or "-" }}</td>
-            <td>
-                <a href="/contacts/{{ contact.id }}/edit" class="btn">Edit</a>
-                <button class="btn btn-danger"
-                        hx-delete="/api/contacts/{{ contact.id }}"
-                        hx-target="#contact-row-{{ contact.id }}"
-                        hx-swap="outerHTML"
-                        hx-confirm="Delete this contact?">
-                    Delete
-                </button>
-            </td>
-        </tr>
-        {% endfor %}
-    </tbody>
-</table>
-{% else %}
-<p>No contacts yet.</p>
-{% endif %}

python/api/templates/partials/manage_needs.html

@@ -1,14 +0,0 @@
-<ul class="manage-needs-list">
-    {% for need in contact.needs %}
-    <li id="contact-need-{{ need.id }}">
-        <strong>{{ need.name }}</strong>
-        {% if need.description %}<span> - {{ need.description }}</span>{% endif %}
-        <button class="btn btn-small btn-danger"
-                hx-delete="/api/contacts/{{ contact.id }}/needs/{{ need.id }}"
-                hx-target="#contact-need-{{ need.id }}"
-                hx-swap="outerHTML">
-            Remove
-        </button>
-    </li>
-    {% endfor %}
-</ul>

python/api/templates/partials/manage_relationships.html

@@ -1,23 +0,0 @@
-{% for rel in contact.related_to %}
-<div class="manage-rel-item" id="rel-{{ contact.id }}-{{ rel.related_contact_id }}">
-    <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a>
-    <span class="tag">{{ rel.relationship_type|replace("_", " ")|title }}</span>
-    <label class="weight-control">
-        <span>Closeness:</span>
-        <input type="range" min="1" max="10" value="{{ rel.closeness_weight }}"
-               hx-post="/htmx/contacts/{{ contact.id }}/relationships/{{ rel.related_contact_id }}/weight"
-               hx-trigger="change"
-               hx-include="this"
-               name="closeness_weight"
-               hx-swap="none"
-               oninput="this.nextElementSibling.textContent = this.value">
-        <span class="weight-value">{{ rel.closeness_weight }}</span>
-    </label>
-    <button class="btn btn-small btn-danger"
-            hx-delete="/api/contacts/{{ contact.id }}/relationships/{{ rel.related_contact_id }}"
-            hx-target="#rel-{{ contact.id }}-{{ rel.related_contact_id }}"
-            hx-swap="outerHTML">
-        Remove
-    </button>
-</div>
-{% endfor %}

python/api/templates/partials/need_items.html

@@ -1,21 +0,0 @@
-{% if needs %}
-<ul class="need-items">
-    {% for need in needs %}
-    <li id="need-item-{{ need.id }}">
-        <div class="need-info">
-            <strong>{{ need.name }}</strong>
-            {% if need.description %}<p>{{ need.description }}</p>{% endif %}
-        </div>
-        <button class="btn btn-danger"
-                hx-delete="/api/needs/{{ need.id }}"
-                hx-target="#need-item-{{ need.id }}"
-                hx-swap="outerHTML"
-                hx-confirm="Delete this need?">
-            Delete
-        </button>
-    </li>
-    {% endfor %}
-</ul>
-{% else %}
-<p>No needs defined yet.</p>
-{% endif %}

python/ebook_search/answer.py

@@ -6,6 +6,7 @@ import logging
 from typing import TYPE_CHECKING
 
 from python.ebook_search.llm_interface import request_chat_completion
+from python.ebook_search.prompts import load_prompt
 
 if TYPE_CHECKING:
     import httpx
@@ -32,11 +33,8 @@ async def answer_query(
         return "No relevant sources were found."
 
     logger.info(
-        "ebook_answer_request_start base_url=%s model=%s sources=%s query_length=%s",
-        config.vllm_base_url,
-        config.chat_model,
-        len(results),
-        len(query),
+        f"ebook_answer_request_start {config.vllm_base_url=} {config.chat_model=} sources={len(results)} "
+        f"query_length={len(query)}"
     )
     context = "\n\n".join(
         f"[{index}] {result.source_title}{' - ' + result.chapter_title if result.chapter_title else ''}\n{result.text}"
@@ -45,21 +43,8 @@ async def answer_query(
     content = await request_chat_completion(
         client,
         config,
-        [
-            {
-                "role": "system",
-                "content": (
-                    "Answer only from the provided context. Cite sources with bracketed numbers like [1]. "
-                    "If the context is insufficient, say so."
-                ),
-            },
-            {"role": "user", "content": f"Question:\n{query}\n\nContext:\n{context}"},
-        ],
+        load_prompt("answer").messages(query=query, context=context),
     )
 
-    logger.info(
-        "ebook_answer_request_complete model=%s answer_length=%s",
-        config.chat_model,
-        len(content),
-    )
+    logger.info(f"ebook_answer_request_complete {config.chat_model=} answer_length={len(content)}")
     return content or "The model returned an empty answer."

python/ebook_search/api/bm25_tasks.py

@@ -36,10 +36,7 @@ def schedule_bm25_refresh(app: FastAPI) -> None:
         app.state.bm25_refresh_task = loop.create_task(refresh_bm25_for_app(app))
 
     app.state.bm25_refresh_timer = loop.call_later(app.state.config.bm25_refresh_delay_seconds, start_refresh)
-    logger.info(
-        "ebook_bm25_refresh_scheduled delay_seconds=%s",
-        app.state.config.bm25_refresh_delay_seconds,
-    )
+    logger.info(f"ebook_bm25_refresh_scheduled {app.state.config.bm25_refresh_delay_seconds=}")
 
 
 def cancel_bm25_refresh(app: FastAPI) -> None:

python/ebook_search/api/dependencies.py

@@ -6,7 +6,6 @@ from typing import Annotated
 
 import httpx
 from fastapi import Depends, Request
-from sqlalchemy.ext.asyncio import AsyncEngine
 
 from python.ebook_search.config import EbookSearchConfig
 
@@ -16,16 +15,10 @@ def get_config(request: Request) -> EbookSearchConfig:
     return request.app.state.config
 
 
-def get_engine(request: Request) -> AsyncEngine:
-    """Get the database engine from app state."""
-    return request.app.state.engine
-
-
 def get_http_client(request: Request) -> httpx.AsyncClient:
     """Get the shared LLM HTTP client from app state."""
     return request.app.state.http_client
 
 
 AppConfig = Annotated[EbookSearchConfig, Depends(get_config)]
-AppEngine = Annotated[AsyncEngine, Depends(get_engine)]
 AppHttpClient = Annotated[httpx.AsyncClient, Depends(get_http_client)]

python/ebook_search/api/judge_tasks.py

@@ -65,12 +65,12 @@ def start_book_phrase_judgment(app: FastAPI, background_tasks: BackgroundTasks,
     """
     state = get_judge_task_state(app)
     if source_id in state.running_book_ids:
-        logger.info("ebook_book_phrase_judgment_already_running source_id=%s", source_id)
+        logger.info(f"ebook_book_phrase_judgment_already_running {source_id=}")
         return False
     state.running_book_ids.add(source_id)
     state.outcome_messages.pop(source_id, None)
     background_tasks.add_task(judge_book_phrases_for_app, app, source_id)
-    logger.info("ebook_book_phrase_judgment_queued source_id=%s", source_id)
+    logger.info(f"ebook_book_phrase_judgment_queued {source_id=}")
     return True
 
 
@@ -85,12 +85,8 @@ async def judge_book_phrases_for_app(app: FastAPI, source_id: int) -> None:
     try:
         result = await judge_candidate_phrases_for_books(app.state.engine, app.state.config, source_ids=[source_id])
         logger.info(
-            "ebook_book_phrase_judgment_complete source_id=%s judged=%s protected=%s mentions=%s failed=%s",
-            source_id,
-            result.candidates_judged,
-            result.protected_phrases,
-            result.phrase_mentions,
-            result.books_failed,
+            f"ebook_book_phrase_judgment_complete {source_id=} {result.candidates_judged=} {result.protected_phrases=} "
+            f"{result.phrase_mentions=} {result.books_failed=}"
         )
         if result.books_failed:
             message = "Judging failed; see server logs for details"
@@ -99,7 +95,7 @@ async def judge_book_phrases_for_app(app: FastAPI, source_id: int) -> None:
                 f"Judged {result.candidates_judged} candidates; {result.protected_phrases} protected phrases promoted"
             )
     except Exception:
-        logger.exception("ebook_book_phrase_judgment_task_failed source_id=%s", source_id)
+        logger.exception(f"ebook_book_phrase_judgment_task_failed {source_id=}")
         message = "Judging failed; see server logs for details"
     state.running_book_ids.discard(source_id)
     state.outcome_messages[source_id] = message

python/ebook_search/api/main.py

@@ -37,16 +37,9 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     config = load_config()
     app.state.config = config
     logger.info(
-        "ebook_search_config_loaded top_k=%s embedding_model=%s embedding_base_url=%s vllm_base_url=%s "
-        "rerank_enabled=%s phrase_matching_enabled=%s answer_enabled=%s library_paths=%s",
-        config.top_k,
-        config.embedding_model,
-        config.embedding_base_url,
-        config.vllm_base_url,
-        config.rerank.enabled,
-        config.phrase_matching_enabled,
-        config.answer_enabled,
-        len(config.library_paths),
+        f"ebook_search_config_loaded {config.top_k=} {config.embedding_model=} {config.embedding_base_url=} "
+        f"{config.vllm_base_url=} {config.rerank.enabled=} {config.phrase_matching_enabled=} {config.answer_enabled=} "
+        f"library_paths={len(config.library_paths)}"
     )
     if not config.library_paths:
         logger.warning("ebook_search_no_library_paths_configured")

python/ebook_search/api/routes/admin.py

@@ -10,16 +10,18 @@ from fastapi.responses import HTMLResponse
 from python.ebook_search.api.bm25_tasks import schedule_bm25_refresh
 from python.ebook_search.api.dependencies import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
     AppConfig,
-    AppEngine,
     AppHttpClient,
 )
-from python.ebook_search.api.web import templates
+from python.ebook_search.api.web import error_response, templates
 from python.ebook_search.embeddings import embed_missing_chunks, embedding_model_stats
 from python.ebook_search.ingest import ingest_configured_paths
 from python.ebook_search.protected_phrases.generate_ngrams import generate_candidate_phrases_for_books
 from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
 from python.ebook_search.protected_phrases.store import book_ids_pending_first_judgment, corpus_phrase_stats
-from python.fastapi_tools import AsyncDbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+from python.fastapi_tools import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
+    AppAsyncEngine,
+    AsyncDbSession,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -32,10 +34,8 @@ async def admin(request: Request, config: AppConfig, session: AsyncDbSession) ->
     stats = await embedding_model_stats(session)
     phrase_stats = await corpus_phrase_stats(session)
     logger.info(
-        "ebook_admin_page_loaded models=%s candidate_phrases=%s protected_phrases=%s",
-        len(stats),
-        phrase_stats.candidate_phrases,
-        phrase_stats.protected_phrases,
+        f"ebook_admin_page_loaded models={len(stats)} {phrase_stats.candidate_phrases=} "
+        f"{phrase_stats.protected_phrases=}"
     )
     return templates.TemplateResponse(
         request,
@@ -52,65 +52,26 @@ async def scan_library(request: Request, config: AppConfig, session: AsyncDbSess
         await session.commit()
     except Exception as error:
         logger.exception("ebook_admin_scan_failed")
-        return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
+        return error_response(request, error)
 
-    logger.info("ebook_admin_scan_complete changed_files=%s", count)
+    logger.info(f"ebook_admin_scan_complete {count=}")
     if count > 0:
         schedule_bm25_refresh(request.app)
     return templates.TemplateResponse(request, "partials/admin_status.html", {"message": f"Indexed {count} EPUBs"})
 
 
 @router.post("/phrases/generate-all", response_class=HTMLResponse)
-async def generate_all_phrases(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
+async def generate_all_phrases(request: Request, config: AppConfig, engine: AppAsyncEngine) -> HTMLResponse:
     """Regenerate candidate phrases for every indexed book without LLM judging."""
-    return await run_phrase_generation(request, config, session, only_missing=False)
-
-
-@router.post("/phrases/generate-missing", response_class=HTMLResponse)
-async def generate_missing_phrases(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
-    """Generate candidate phrases only for books that have none yet."""
-    return await run_phrase_generation(request, config, session, only_missing=True)
-
-
-async def run_phrase_generation(
-    request: Request,
-    config: AppConfig,
-    session: AsyncDbSession,
-    *,
-    only_missing: bool,
-) -> HTMLResponse:
-    """Run candidate phrase generation and render the outcome as an admin status partial.
-
-    Args:
-        request (Request): Current request, for template rendering.
-        config (AppConfig): Runtime phrase-tuning settings.
-        session (AsyncDbSession): Active database session.
-        only_missing (bool): Only generate for books without candidates instead of every book.
-
-    Returns:
-        HTMLResponse: Status partial describing the generation outcome.
-    """
     try:
-        result = await generate_candidate_phrases_for_books(session, config, only_missing=only_missing)
-        await session.commit()
+        result = await generate_candidate_phrases_for_books(engine, config)
     except Exception as error:
-        await session.rollback()
-        logger.exception("ebook_admin_generate_phrases_failed only_missing=%s", only_missing)
-        return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
+        logger.exception("ebook_admin_generate_phrases_failed")
+        return error_response(request, error)
 
     logger.info(
-        "ebook_admin_generate_phrases_complete only_missing=%s books_seen=%s books_built=%s candidates=%s",
-        only_missing,
-        result.books_seen,
-        result.books_built,
-        result.candidate_phrases,
+        f"ebook_admin_generate_phrases_complete {result.books_seen=} {result.books_built=} {result.candidate_phrases=}"
     )
-    if only_missing and result.books_seen == 0:
-        return templates.TemplateResponse(
-            request,
-            "partials/admin_status.html",
-            {"message": "All books already have candidate phrases"},
-        )
     return templates.TemplateResponse(
         request,
         "partials/admin_status.html",
@@ -124,7 +85,7 @@ async def run_phrase_generation(
 
 
 @router.post("/phrases/judge-all", response_class=HTMLResponse)
-async def judge_all_phrases(request: Request, engine: AppEngine, config: AppConfig) -> HTMLResponse:
+async def judge_all_phrases(request: Request, engine: AppAsyncEngine, config: AppConfig) -> HTMLResponse:
     """Judge unjudged candidate phrases across every indexed book."""
     return await run_phrase_judgment(request, engine, config, source_ids=None)
 
@@ -132,7 +93,7 @@ async def judge_all_phrases(request: Request, engine: AppEngine, config: AppConf
 @router.post("/phrases/judge-missing", response_class=HTMLResponse)
 async def judge_missing_phrases(
     request: Request,
-    engine: AppEngine,
+    engine: AppAsyncEngine,
     config: AppConfig,
     session: AsyncDbSession,
 ) -> HTMLResponse:
@@ -149,7 +110,7 @@ async def judge_missing_phrases(
 
 async def run_phrase_judgment(
     request: Request,
-    engine: AppEngine,
+    engine: AppAsyncEngine,
     config: AppConfig,
     *,
     source_ids: list[int] | None,
@@ -158,7 +119,7 @@ async def run_phrase_judgment(
 
     Args:
         request (Request): Current request, for template rendering.
-        engine (AppEngine): Engine used to open per-book judging sessions.
+        engine (AppAsyncEngine): Engine used to open per-book judging sessions.
         config (AppConfig): Runtime phrase-tuning settings.
         source_ids (list[int] | None): Books to judge; ``None`` judges every indexed book.
 
@@ -169,17 +130,11 @@ async def run_phrase_judgment(
         result = await judge_candidate_phrases_for_books(engine, config, source_ids=source_ids)
     except Exception as error:
         logger.exception("ebook_admin_judge_phrases_failed")
-        return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
+        return error_response(request, error)
 
     logger.info(
-        "ebook_admin_judge_phrases_complete books_seen=%s books_judged=%s books_failed=%s candidates_judged=%s "
-        "protected=%s mentions=%s",
-        result.books_seen,
-        result.books_judged,
-        result.books_failed,
-        result.candidates_judged,
-        result.protected_phrases,
-        result.phrase_mentions,
+        f"ebook_admin_judge_phrases_complete {result.books_seen=} {result.books_judged=} {result.books_failed=} "
+        f"{result.candidates_judged=} {result.protected_phrases=} {result.phrase_mentions=}"
     )
     return templates.TemplateResponse(
         request,
@@ -208,9 +163,9 @@ async def embed_missing(
         await session.commit()
     except Exception as error:
         logger.exception("ebook_admin_embed_missing_failed")
-        return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
+        return error_response(request, error)
 
-    logger.info("ebook_admin_embed_missing_complete chunks=%s", count)
+    logger.info(f"ebook_admin_embed_missing_complete {count=}")
     return templates.TemplateResponse(
         request,
         "partials/admin_status.html",
@@ -236,26 +191,12 @@ async def embed_all(
             await session.commit()
             total += count
             batches += 1
-            logger.info(
-                "ebook_admin_embed_all_batch_complete batch=%s chunks=%s total_chunks=%s",
-                batches,
-                count,
-                total,
-            )
+            logger.info(f"ebook_admin_embed_all_batch_complete {batches=} {count=} {total=}")
     except Exception as error:
-        logger.exception(
-            "ebook_admin_embed_all_failed batches=%s chunks=%s",
-            batches,
-            total,
-        )
-        return templates.TemplateResponse(
-            request,
-            "partials/error.html",
-            {"message": f"Embed all failed after {total} chunks in {batches} batches: {error}"},
-            status_code=500,
-        )
+        logger.exception(f"ebook_admin_embed_all_failed {batches=} {total=}")
+        return error_response(request, f"Embed all failed after {total} chunks in {batches} batches: {error}")
 
-    logger.info("ebook_admin_embed_all_complete batches=%s chunks=%s", batches, total)
+    logger.info(f"ebook_admin_embed_all_complete {batches=} {total=}")
     return templates.TemplateResponse(
         request,
         "partials/admin_status.html",

python/ebook_search/api/routes/health.py

@@ -60,14 +60,7 @@ async def ready(config: AppConfig, session: AsyncDbSession, client: AppHttpClien
         status = "ready"
         status_code = HTTPStatus.OK
 
-    logger.info(
-        "ebook_ready_check status=%s database=%s embedding=%s chat=%s bm25=%s",
-        status,
-        database_ok,
-        embedding_ok,
-        chat_status,
-        bm25_status,
-    )
+    logger.info(f"ebook_ready_check {status=} {database_ok=} {embedding_ok=} {chat_status=} {bm25_status=}")
     return JSONResponse(content={"status": status, "checks": checks}, status_code=status_code)
 
 
@@ -83,7 +76,7 @@ async def check_database(session: AsyncSession) -> bool:
     try:
         await session.execute(select(literal(1)))
     except SQLAlchemyError as error:
-        logger.warning("ebook_ready_database_unavailable error=%s", error)
+        logger.warning(f"ebook_ready_database_unavailable {error=}")
         return False
     return True
 

python/ebook_search/api/routes/page.py

@@ -15,6 +15,7 @@ from python.ebook_search.api.dependencies import (
 from python.ebook_search.api.judge_tasks import is_judging_book, pop_book_judgment_outcome, start_book_phrase_judgment
 from python.ebook_search.api.web import templates
 from python.ebook_search.protected_phrases.generate_ngrams import recalculate_candidate_phrases_for_book
+from python.ebook_search.protected_phrases.store import count_protected_phrases
 from python.fastapi_tools import AsyncDbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
 from python.orm.richie import EbookCandidatePhrase, EbookChapter, EbookChunk, EbookProtectedPhrase, EbookSource
 
@@ -36,7 +37,7 @@ async def index(request: Request, config: AppConfig) -> HTMLResponse:
 async def books(request: Request, session: AsyncDbSession) -> HTMLResponse:
     """Render the indexed books page."""
     sources = list((await session.scalars(select(EbookSource).order_by(EbookSource.title))).all())
-    logger.info("ebook_books_page_loaded count=%s", len(sources))
+    logger.info(f"ebook_books_page_loaded count={len(sources)}")
     return templates.TemplateResponse(request, "books.html", {"sources": sources})
 
 
@@ -71,14 +72,6 @@ async def get_judged_candidate_count(session: AsyncSession, book_id: int) -> int
     )
 
 
-async def get_protected_count(session: AsyncSession, book_id: int) -> int:
-    """Return the number of protected phrases for one book."""
-    return (
-        await session.scalar(select(func.count(EbookProtectedPhrase.id)).where(EbookProtectedPhrase.book_id == book_id))
-        or 0
-    )
-
-
 async def get_candidates(session: AsyncSession, book_id: int) -> list[EbookCandidatePhrase]:
     """Return the indexed candidates for one book."""
     return list(
@@ -122,7 +115,7 @@ async def book_detail(source_id: int, request: Request, session: AsyncDbSession)
         chunk_count = await get_chunk_count(session, source.id)
         candidate_count = await get_candidate_count(session, source.id)
         judged_candidate_count = await get_judged_candidate_count(session, source.id)
-        protected_count = await get_protected_count(session, source.id)
+        protected_count = await count_protected_phrases(session, source.id)
         candidates = await get_candidates(session, source.id)
         protected_phrases = await get_protected_phrases(session, source.id)
     else:
@@ -134,14 +127,8 @@ async def book_detail(source_id: int, request: Request, session: AsyncDbSession)
         candidates = []
         protected_phrases = []
     logger.info(
-        "ebook_book_detail_loaded source_id=%s found=%s chapters=%s chunks=%s candidates=%s judged=%s protected=%s",
-        source_id,
-        source is not None,
-        chapter_count,
-        chunk_count,
-        candidate_count,
-        judged_candidate_count,
-        protected_count,
+        f"ebook_book_detail_loaded {source_id=} found={source is not None} {chapter_count=} {chunk_count=} "
+        f"{candidate_count=} {judged_candidate_count=} {protected_count=}"
     )
     return templates.TemplateResponse(
         request,
@@ -168,16 +155,14 @@ async def recalculate_book_phrases(source_id: int, config: AppConfig, session: A
     if source is None:
         raise HTTPException(status_code=404, detail="Book not found")
 
-    result = await recalculate_candidate_phrases_for_book(session, source, config, use_process_pool=True)
+    try:
+        result = await recalculate_candidate_phrases_for_book(session, source, config)
+    except ValueError as error:
+        raise HTTPException(status_code=409, detail=str(error)) from error
     logger.info(
-        "ebook_book_phrase_recalculation_complete source_id=%s candidates=%s deleted_candidates=%s "
-        "deleted_protected=%s deleted_aliases=%s deleted_mentions=%s",
-        source_id,
-        result.candidate_phrases,
-        result.deleted_candidates,
-        result.deleted_protected_phrases,
-        result.deleted_aliases,
-        result.deleted_mentions,
+        f"ebook_book_phrase_recalculation_complete {source_id=} {result.candidate_phrases=} "
+        f"{result.deleted_candidates=} {result.deleted_protected_phrases=} {result.deleted_aliases=} "
+        f"{result.deleted_mentions=}"
     )
     return RedirectResponse(
         url=f"/books/{source_id}?phrases_recalculated={result.candidate_phrases}",
@@ -198,5 +183,5 @@ async def judge_book_phrases(
         raise HTTPException(status_code=404, detail="Book not found")
 
     started = start_book_phrase_judgment(request.app, background_tasks, source.id)
-    logger.info("ebook_book_phrase_judgment_requested source_id=%s started=%s", source_id, started)
+    logger.info(f"ebook_book_phrase_judgment_requested {source_id=} {started=}")
     return RedirectResponse(url=f"/books/{source_id}", status_code=303)

python/ebook_search/api/routes/search.py

@@ -13,10 +13,9 @@ from fastapi.responses import HTMLResponse
 from python.ebook_search.answer import answer_query
 from python.ebook_search.api.dependencies import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
     AppConfig,
-    AppEngine,
     AppHttpClient,
 )
-from python.ebook_search.api.web import templates
+from python.ebook_search.api.web import error_response, templates
 from python.ebook_search.guardrails import (
     CitationReport,
     is_confident,
@@ -25,6 +24,7 @@ from python.ebook_search.guardrails import (
 )
 from python.ebook_search.search import SearchResponse, search_ebooks
 from python.ebook_search.timing import runtime_step_from_start
+from python.fastapi_tools import AppAsyncEngine  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
 
 if TYPE_CHECKING:
     import httpx
@@ -49,9 +49,8 @@ async def build_answer(
 
     if not is_confident(response.results, config):
         logger.info(
-            "ebook_answer_low_confidence confidence=%.4f threshold=%.4f",
-            retrieval_confidence(response.results),
-            config.min_retrieval_confidence,
+            f"ebook_answer_low_confidence confidence={retrieval_confidence(response.results):.4f} "
+            f"{config.min_retrieval_confidence=:.4f}"
         )
         answer = (
             "Retrieval confidence is low for this query, so answer generation was skipped. "
@@ -62,18 +61,14 @@ async def build_answer(
     try:
         answer = await answer_query(client, query, response.results, config)
     except RuntimeError as error:
-        logger.warning("ebook_answer_request_failed_falling_back error=%s", error)
+        logger.warning(f"ebook_answer_request_failed_falling_back {error=}")
         return "Answer generation failed. Source chunks are still shown below.", False, None
 
     citation_report = None
     if config.validate_citations_enabled and response.results:
         citation_report = validate_citations(answer, len(response.results))
         if citation_report.invalid or not citation_report.grounded:
-            logger.warning(
-                "ebook_answer_citation_issue invalid=%s grounded=%s",
-                citation_report.invalid,
-                citation_report.grounded,
-            )
+            logger.warning(f"ebook_answer_citation_issue {citation_report.invalid=} {citation_report.grounded=}")
     return answer, False, citation_report
 
 
@@ -81,11 +76,12 @@ async def build_answer(
 async def search(
     request: Request,
     config: AppConfig,
-    engine: AppEngine,
+    engine: AppAsyncEngine,
     client: AppHttpClient,
     query: Annotated[str, Form()],
-    rerank: Annotated[str | None, Form()] = None,
-    phrase_matching: Annotated[str | None, Form()] = None,
+    *,
+    rerank: Annotated[bool, Form()] = False,
+    phrase_matching: Annotated[bool, Form()] = False,
 ) -> HTMLResponse:
     """Run a search and render HTMX results."""
     try:
@@ -94,12 +90,12 @@ async def search(
             client,
             query,
             config,
-            rerank=rerank == "true",
-            phrase_matching=phrase_matching == "true",
+            rerank=rerank,
+            phrase_matching=phrase_matching,
         )
     except Exception as error:
         logger.exception("ebook_search_request_failed")
-        return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
+        return error_response(request, error)
 
     answer_start = perf_counter()
     answer, low_confidence, citation_report = await build_answer(client, query, response, config)
@@ -110,12 +106,10 @@ async def search(
     )
 
     for step in response.timings:
-        logger.info("ebook_search_timing step=%r runtime_ms=%.1f", step.name, step.duration_ms)
+        logger.info(f"ebook_search_timing {step.name=} {step.duration_ms=:.1f}")
     logger.info(
-        "ebook_search_request_complete results=%s rank_label=%s runtime_ms=%.1f",
-        len(response.results),
-        response.rank_label,
-        response.total_runtime_ms,
+        f"ebook_search_request_complete results={len(response.results)} {response.rank_label=} "
+        f"{response.total_runtime_ms=:.1f}"
     )
     return templates.TemplateResponse(
         request,

python/ebook_search/api/templates/admin.html

@@ -60,13 +60,6 @@ head %}
     >
       <button type="submit">Regenerate all phrases</button>
     </form>
-    <form
-      hx-post="/admin/phrases/generate-missing"
-      hx-target="#admin-status"
-      hx-swap="innerHTML"
-    >
-      <button type="submit">Add missing phrases</button>
-    </form>
     <form
       hx-post="/admin/phrases/judge-all"
       hx-target="#admin-status"

python/ebook_search/api/web.py

@@ -3,9 +3,14 @@
 from __future__ import annotations
 
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from fastapi.templating import Jinja2Templates
 
+if TYPE_CHECKING:
+    from fastapi import Request
+    from fastapi.responses import HTMLResponse
+
 PACKAGE_DIR = Path(__file__).resolve().parent
 TEMPLATE_DIR = PACKAGE_DIR / "templates"
 STATIC_DIR = PACKAGE_DIR / "static"
@@ -21,3 +26,8 @@ def static_version(filename: str) -> int:
 
 templates = Jinja2Templates(directory=TEMPLATE_DIR)
 templates.env.globals["static_version"] = static_version
+
+
+def error_response(request: Request, message: object) -> HTMLResponse:
+    """Render the shared error partial for a failed UI request."""
+    return templates.TemplateResponse(request, "partials/error.html", {"message": str(message)}, status_code=500)

python/ebook_search/bm25_corpus.py

@@ -15,6 +15,7 @@ from typing import TYPE_CHECKING
 import bm25s
 from sqlalchemy import func, select, union_all
 
+from python.ebook_search.chunk_records import CHUNK_RECORD_COLUMNS
 from python.orm.richie import EbookChapter, EbookChunk, EbookSource
 
 if TYPE_CHECKING:
@@ -80,23 +81,19 @@ async def ensure_bm25_corpus(session: AsyncSession, config: EbookSearchConfig) -
     manifest = read_bm25_manifest(index_path)
     db_updated_at = await corpus_last_updated_at(session)
     if not bm25_index_exists(index_path, manifest):
-        logger.info("ebook_bm25_index_missing path=%s", index_path)
+        logger.info(f"ebook_bm25_index_missing {index_path=}")
         await refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
         return
     if db_updated_at is not None and manifest is not None and manifest.created_at < db_updated_at:
         logger.info(
-            "ebook_bm25_index_stale path=%s created_at=%s db_updated_at=%s",
-            index_path,
-            manifest.created_at.isoformat(),
-            db_updated_at.isoformat(),
+            f"ebook_bm25_index_stale {index_path=} created_at={manifest.created_at.isoformat()} "
+            f"db_updated_at={db_updated_at.isoformat()}"
         )
         await refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
         return
     logger.info(
-        "ebook_bm25_index_current path=%s chunks=%s created_at=%s",
-        index_path,
-        manifest.chunk_count if manifest else 0,
-        manifest.created_at.isoformat() if manifest else None,
+        f"ebook_bm25_index_current {index_path=} chunks={manifest.chunk_count if manifest else 0} "
+        f"created_at={manifest.created_at.isoformat() if manifest else None}"
     )
 
 
@@ -119,10 +116,7 @@ async def refresh_bm25_corpus(
     )
     await asyncio.to_thread(write_bm25_corpus, index_path, records, texts, manifest)
     logger.info(
-        "ebook_bm25_index_refreshed path=%s chunks=%s created_at=%s",
-        index_path,
-        manifest.chunk_count,
-        manifest.created_at.isoformat(),
+        f"ebook_bm25_index_refreshed {index_path=} {manifest.chunk_count=} created_at={manifest.created_at.isoformat()}"
     )
     return manifest
 
@@ -135,7 +129,7 @@ def load_bm25_corpus(config: EbookSearchConfig) -> BM25Corpus:
     """
     index_path = bm25_index_path(config)
     active_index_path = get_current_bm25_index(index_path)
-    logger.info("ebook_bm25_corpus_cache_load path=%s active_path=%s", index_path, active_index_path)
+    logger.info(f"ebook_bm25_corpus_cache_load {index_path=} {active_index_path=}")
     manifest = read_bm25_manifest(index_path)
     if manifest is None or not bm25_index_exists(index_path, manifest):
         msg = f"BM25 corpus is not available: {index_path}"
@@ -176,13 +170,7 @@ async def fetch_bm25_corpus_records(session: AsyncSession) -> tuple[list[dict[st
     """
     statement = (
         select(
-            EbookChunk.id.label("chunk_id"),
-            EbookChunk.text.label("text"),
-            EbookSource.id.label("source_id"),
-            EbookSource.title.label("source_title"),
-            EbookSource.author.label("source_author"),
-            EbookChapter.title.label("chapter_title"),
-            EbookChunk.page_label.label("page_label"),
+            *CHUNK_RECORD_COLUMNS,
             EbookChunk.search_text.label("bm25_text"),
         )
         .select_from(EbookChunk)

python/ebook_search/chunk_records.py

@@ -0,0 +1,13 @@
+"""Shared database columns used to build search-result records."""
+
+from python.orm.richie import EbookChapter, EbookChunk, EbookSource
+
+CHUNK_RECORD_COLUMNS = (
+    EbookChunk.id.label("chunk_id"),
+    EbookChunk.text.label("text"),
+    EbookSource.id.label("source_id"),
+    EbookSource.title.label("source_title"),
+    EbookSource.author.label("source_author"),
+    EbookChapter.title.label("chapter_title"),
+    EbookChunk.page_label.label("page_label"),
+)

python/ebook_search/config.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-from os import getenv
 from typing import Annotated, Self
 
 from pydantic import AliasChoices, Field, field_validator, model_validator
@@ -32,18 +31,13 @@ def normalize_embedding_alias(model: str) -> str:
     return standard_model
 
 
-def normalize_embedding_model(default: str = "qwen3-embedding-0.6b") -> str:
-    """Normalize the configured embedding alias to its provider model name."""
-    return normalize_embedding_alias(getenv("EBOOK_SEARCH_EMBEDDING_MODEL", default))
-
-
 class RerankConfig(BaseSettings):
     """vLLM reranker settings."""
 
     model_config = SettingsConfigDict(env_prefix="EBOOK_SEARCH_RERANK_", frozen=True, protected_namespaces=())
 
     enabled: bool = True
-    base_url: str = "http://192.168.90.25:8001"
+    base_url: str = "http://bob:8001"
     model: str = "qwen3-reranker-06b"
     candidates: int = 24
     timeout_seconds: float = 30.0
@@ -73,7 +67,7 @@ class EbookSearchConfig(BaseSettings):
     )
     chat_model: str = "deepseek-v4-flash"
     answer_enabled: bool = True
-    embedding_base_url: str = "http://192.168.90.25:8000/v1"
+    embedding_base_url: str = "http://bob:8000/v1"
     embedding_api_key: str = "not-needed"
     embedding_model: str = "qwen3-embedding-0.6b"
     embedding_batch_size: int = 32
@@ -97,6 +91,8 @@ class EbookSearchConfig(BaseSettings):
     phrase_min_tokens: int = 2
     phrase_max_tokens: int = 5
     phrase_max_entity_tokens: int = 8
+    phrase_yake_top_k: int = 1000
+    phrase_yake_dedup_limit: float = 0.85
     phrase_raw_ngram_min_count: int = 2
     phrase_raw_count_score_threshold: int = 3
     phrase_raw_count_high_score_threshold: int = 10
@@ -105,6 +101,10 @@ class EbookSearchConfig(BaseSettings):
     phrase_target_protected_per_book: int = 100
     phrase_default_allow_nested: bool = False
     phrase_default_suppress_children: bool = True
+    phrase_bad_start_score_penalty: float = 10.0
+    phrase_bad_end_score_penalty: float = 10.0
+    phrase_multi_source_score_bonus: float = 2.0
+    phrase_multi_source_min_sources: int = 2
 
     @field_validator("library_paths", mode="before")
     @classmethod

python/ebook_search/docker/Dockerfile

@@ -1,12 +1,17 @@
-FROM python:3.14-slim
+FROM python:3.14-slim AS base
+
+COPY --from=ghcr.io/astral-sh/uv:0.11.26 /uv /uvx /bin/
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
-    PIP_NO_CACHE_DIR=1 \
     APP_DIR=/home/richie/dotfiles \
-    EBOOK_SEARCH_HOST=0.0.0.0 \
-    EBOOK_SEARCH_PORT=8070 \
-    EBOOK_SEARCH_BM25_INDEX_DIR=/data/bm25
+    UV_PROJECT_ENVIRONMENT=/opt/venv \
+    UV_PYTHON_DOWNLOADS=never \
+    UV_NO_CACHE=1
+
+# Separate ENV instruction so ${APP_DIR} and ${PATH} from above resolve.
+ENV PYTHONPATH=${APP_DIR} \
+    PATH=/opt/venv/bin:${PATH}
 
 WORKDIR ${APP_DIR}
 
@@ -14,29 +19,29 @@ RUN apt-get update \
     && apt-get install -y --no-install-recommends build-essential curl \
     && rm -rf /var/lib/apt/lists/*
 
-COPY pyproject.toml README.md LICENSE ./
-COPY python ./python
+COPY python/ebook_search/docker/pyproject.toml python/ebook_search/docker/uv.lock ./
 
-RUN python -m pip install --upgrade pip \
-    && python -m pip install \
-        "alembic" \
-        "beautifulsoup4" \
-        "bm25s" \
-        "ebooklib" \
-        "fastapi" \
-        "httpx" \
-        "jinja2" \
-        "pgvector" \
-        "psycopg[binary]" \
-        "pydantic" \
-        "pydantic-settings" \
-        "python-multipart" \
-        "sqlalchemy[asyncio]" \
-        "tiktoken" \
-        "typer" \
-        "uvicorn[standard]" \
-        "yake" \
-    && python -m pip install --no-deps --editable "${APP_DIR}"
+RUN uv sync --locked --no-dev
+
+
+FROM base AS test
+
+RUN uv sync --locked
+
+COPY python ./python
+COPY tests/__init__.py ./tests/__init__.py
+COPY tests/ebook_search ./tests/ebook_search
+
+CMD ["pytest"]
+
+
+FROM base AS runtime
+
+ENV EBOOK_SEARCH_HOST=0.0.0.0 \
+    EBOOK_SEARCH_PORT=8070 \
+    EBOOK_SEARCH_BM25_INDEX_DIR=/data/bm25
+
+COPY python ./python
 
 RUN useradd --create-home --uid 10001 app \
     && mkdir -p /data \

python/ebook_search/docker/README.md

@@ -1,40 +0,0 @@
-# Ebook Search Docker
-
-Run the EPUB search app against the existing Postgres database on `jeeves`:
-
-```sh
-ebook-search-containers start --library-path /path/to/epubs --build
-```
-
-All ebook-search Docker files live in this directory:
-
-- `Dockerfile`
-- `docker-compose.yml`
-- `containers.py`
-- `container.py`
-
-The app listens on `http://localhost:8070`.
-
-Useful lifecycle commands:
-
-```sh
-ebook-search-containers build
-ebook-search-containers start --library-path /path/to/epubs
-ebook-search-containers logs
-ebook-search-containers ps
-ebook-search-containers stop
-```
-
-Direct compose usage from the repo root:
-
-```sh
-docker compose -f python/ebook_search/docker/docker-compose.yml ps
-```
-
-The compose service also loads the repo root `.env` into the container via `env_file`.
-
-Mount your EPUB directory by setting `EBOOK_LIBRARY_HOST_PATH` in an env file or on the command line. The container sees it as `/library`, and `EBOOK_SEARCH_LIBRARY_PATHS` is set to `/library` inside the container.
-
-Database connection settings are controlled by `RICHIE_DB`, `RICHIE_HOST`, `RICHIE_PORT`, `RICHIE_USER`, and `RICHIE_PASSWORD`. The default host is `jeeves`.
-
-Startup runs the Richie Alembic migrations automatically after creating the `main` schema and `vector` extension.

python/ebook_search/docker/containers.py

@@ -32,7 +32,7 @@ def docker_run(
     capture_output: bool = False,
 ) -> subprocess.CompletedProcess[str]:
     """Run docker with repo-root cwd and consistent error handling."""
-    logger.info("docker %s", " ".join(arguments))
+    logger.info(f"docker {' '.join(arguments)}")
     return subprocess.run(
         ["docker", *arguments],
         cwd=get_repo_dir(),
@@ -73,6 +73,23 @@ def build_image() -> None:
         raise RuntimeError(msg)
 
 
+def build_test_image() -> None:
+    """Build the ebook search test Docker image."""
+    dockerfile = Path(__file__).resolve().with_name("Dockerfile")
+    result = docker_run(["build", "--file", str(dockerfile), "--target", "test", "--tag", "ebook-search:test", "."])
+    if result.returncode != 0:
+        msg = "Failed to build ebook search test image"
+        raise RuntimeError(msg)
+
+
+def run_test_image() -> None:
+    """Run the ebook search test suite inside Docker."""
+    result = docker_run(["run", "--rm", "ebook-search:test"])
+    if result.returncode != 0:
+        msg = f"Ebook search tests failed with code {result.returncode}"
+        raise RuntimeError(msg)
+
+
 def start_stack(
     *,
     library_path: Path | None = None,
@@ -210,6 +227,19 @@ def logs(
         typer.echo(output)
 
 
+@app.command("test")
+def run_tests(
+    *,
+    build: Annotated[bool, typer.Option("--build/--no-build", help="Build the test image before running.")] = True,
+    log_level: Annotated[str, typer.Option(help="Log level.")] = "INFO",
+) -> None:
+    """Run ebook search tests inside the Docker test image."""
+    configure_logger(log_level)
+    if build:
+        build_test_image()
+    run_test_image()
+
+
 @app.command("ps")
 def ps() -> None:
     """Show ebook search container status."""

python/ebook_search/docker/docker-compose.yml

@@ -9,8 +9,6 @@ services:
     restart: unless-stopped
     ports:
       - "${EBOOK_SEARCH_PORT:-8070}:8070"
-    extra_hosts:
-      - "jeeves:192.168.90.40"
     env_file:
       - ../../../.env
     environment:

python/ebook_search/docker/pyproject.toml

@@ -0,0 +1,41 @@
+[project]
+name = "ebook-search"
+version = "0.1.0"
+description = "Locked runtime environment for the ebook search container."
+requires-python = "~=3.14.0"
+dependencies = [
+    "alembic",
+    "beautifulsoup4",
+    "bm25s",
+    "ebooklib",
+    "fastapi",
+    "httpx",
+    "jinja2",
+    "pgvector",
+    "psycopg[binary]",
+    "pydantic",
+    "pydantic-settings",
+    "python-multipart",
+    "sqlalchemy[asyncio]",
+    "tiktoken",
+    "typer",
+    "uvicorn[standard]",
+    "yake",
+]
+
+[dependency-groups]
+dev = [
+    "aiosqlite",
+    "pytest",
+    "pytest-asyncio",
+    "pytest-mock",
+    "pytest-xdist",
+]
+
+[tool.uv]
+package = false
+
+[tool.pytest.ini_options]
+addopts = "-n auto -ra"
+asyncio_mode = "auto"
+testpaths = ["tests/ebook_search"]

python/ebook_search/embeddings.py

@@ -72,24 +72,14 @@ async def embed_texts(
     config: EbookSearchConfig,
 ) -> list[list[float]]:
     """Embed text with the configured vLLM embedding model."""
-    logger.info(
-        "ebook_embed_request_start base_url=%s model=%s count=%s",
-        config.embedding_base_url,
-        config.embedding_model,
-        len(texts),
-    )
+    logger.info(f"ebook_embed_request_start {config.embedding_base_url=} {config.embedding_model=} count={len(texts)}")
     vectors = await request_embeddings(client, texts, config)
     expected_dimension = MODEL_DIMENSIONS[config.embedding_model]
     for vector in vectors:
         if len(vector) != expected_dimension:
             msg = f"Expected {expected_dimension} dimensions, got {len(vector)}"
             raise ValueError(msg)
-    logger.info(
-        "ebook_embed_request_complete model=%s count=%s dimension=%s",
-        config.embedding_model,
-        len(vectors),
-        expected_dimension,
-    )
+    logger.info(f"ebook_embed_request_complete {config.embedding_model=} count={len(vectors)} {expected_dimension=}")
     return vectors
 
 
@@ -105,7 +95,7 @@ async def ensure_embedding_models(session: AsyncSession) -> None:
         existing = await session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == name))
         if existing is None:
             session.add(EbookEmbeddingModel(name=name, dimension=dimension, is_default=name == "qwen3-embedding-0.6b"))
-            logger.info("ebook_embedding_model_created model=%s dimension=%s", name, dimension)
+            logger.info(f"ebook_embedding_model_created {name=} {dimension=}")
     await session.flush()
 
 
@@ -159,10 +149,10 @@ async def embed_missing_chunks(session: AsyncSession, client: httpx.AsyncClient,
         )
     )
     if not chunks:
-        logger.info("ebook_embed_missing_none model=%s", config.embedding_model)
+        logger.info(f"ebook_embed_missing_none {config.embedding_model=}")
         return 0
 
-    logger.info("ebook_embed_missing_batch_start model=%s count=%s", config.embedding_model, len(chunks))
+    logger.info(f"ebook_embed_missing_batch_start {config.embedding_model=} count={len(chunks)}")
     vectors = await embed_texts(client, [chunk.text for chunk in chunks], config)
     rows = [
         {"chunk_id": chunk.id, "model_id": model.id, "embedding": vector}
@@ -171,5 +161,5 @@ async def embed_missing_chunks(session: AsyncSession, client: httpx.AsyncClient,
     statement = insert(table).values(rows).on_conflict_do_nothing(index_elements=["chunk_id", "model_id"])
     await session.execute(statement)
     await session.flush()
-    logger.info("ebook_embed_missing_batch_complete model=%s count=%s", config.embedding_model, len(rows))
+    logger.info(f"ebook_embed_missing_batch_complete {config.embedding_model=} count={len(rows)}")
     return len(rows)

python/ebook_search/ingest.py

@@ -11,6 +11,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING
 
 import tiktoken
+from anyio import Path as AsyncPath
 from sqlalchemy import or_, select
 
 from python.ebook_search.epub_parse import parse_epub
@@ -74,18 +75,18 @@ def chunk_text(
     return [chunk for chunk in chunks if chunk.text]
 
 
-def find_library_epubs(library_path: str) -> tuple[Path, list[Path] | None]:
-    """Resolve one configured library path and collect its EPUB files (blocking filesystem walk).
+async def find_library_epubs(library_path: str) -> tuple[AsyncPath, list[AsyncPath] | None]:
+    """Resolve one configured library path and collect its EPUB files asynchronously.
 
     Returns:
         tuple[Path, list[Path] | None]: The expanded path and its EPUB files, or ``None`` when
         the path is neither an EPUB file nor a directory.
     """
-    path = Path(library_path).expanduser()
-    if path.is_file() and path.suffix.lower() == ".epub":
+    path = await AsyncPath(library_path).expanduser()
+    if await path.is_file() and path.suffix.lower() == ".epub":
         return path, [path]
-    if path.is_dir():
-        return path, sorted(path.rglob("*.epub"))
+    if await path.is_dir():
+        return path, sorted([epub_path async for epub_path in path.rglob("*.epub")])
     return path, None
 
 
@@ -93,44 +94,45 @@ async def ingest_configured_paths(session: AsyncSession, config: EbookSearchConf
     """Ingest every EPUB found under configured library paths."""
     count = 0
     for library_path in config.library_paths:
-        path, epub_paths = await asyncio.to_thread(find_library_epubs, library_path)
-        logger.info("ebook_ingest_path_start path=%s", path)
+        path, epub_paths = await find_library_epubs(library_path)
+        logger.info(f"ebook_ingest_path_start {path=}")
         if epub_paths is None:
-            logger.warning("ebook_ingest_path_missing path=%s", path)
+            logger.warning(f"ebook_ingest_path_missing {path=}")
             continue
         for epub_path in epub_paths:
             count += int(await ingest_file(session, epub_path, config))
-    logger.info("ebook_ingest_paths_complete changed_files=%s configured_paths=%s", count, len(config.library_paths))
+    logger.info(f"ebook_ingest_paths_complete {count=} configured_paths={len(config.library_paths)}")
     return count
 
 
-def resolve_ingest_path(path: Path) -> Path:
-    """Expand and resolve an ingest path (blocking filesystem call)."""
-    return path.expanduser().resolve()
+async def resolve_ingest_path(path: Path | AsyncPath) -> AsyncPath:
+    """Expand and resolve an ingest path without blocking the event loop."""
+    expanded_path = await AsyncPath(path).expanduser()
+    return await expanded_path.resolve()
 
 
-async def ingest_file(session: AsyncSession, path: Path, config: EbookSearchConfig) -> bool:
+async def ingest_file(session: AsyncSession, path: Path | AsyncPath, config: EbookSearchConfig) -> bool:
     """Ingest one EPUB file. Return True when the database changed."""
     try:
-        resolved_path = await asyncio.to_thread(resolve_ingest_path, path)
-        logger.info("ebook_ingest_file_start path=%s", resolved_path)
-        file_hash = await asyncio.to_thread(sha256_file, resolved_path)
+        resolved_path = await resolve_ingest_path(path)
+        logger.info(f"ebook_ingest_file_start {resolved_path=}")
+        file_hash = await sha256_file(resolved_path)
         existing = await find_existing_source(session, resolved_path, file_hash)
         if existing is not None and existing.file_sha256 == file_hash:
-            stat = resolved_path.stat()
+            stat = await resolved_path.stat()
             existing.file_path = str(resolved_path)
             existing.file_mtime = datetime.fromtimestamp(stat.st_mtime, tz=UTC)
             existing.file_size = stat.st_size
             await session.flush()
-            logger.info("ebook_ingest_file_unchanged source_id=%s path=%s", existing.id, resolved_path)
+            logger.info(f"ebook_ingest_file_unchanged {existing.id=} {resolved_path=}")
             return False
         if existing is not None:
-            logger.info("ebook_ingest_file_replacing source_id=%s path=%s", existing.id, resolved_path)
+            logger.info(f"ebook_ingest_file_replacing {existing.id=} {resolved_path=}")
             await session.delete(existing)
             await session.flush()
 
-        stat = resolved_path.stat()
-        parsed = await asyncio.to_thread(parse_epub, resolved_path)
+        stat = await resolved_path.stat()
+        parsed = await asyncio.to_thread(parse_epub, Path(resolved_path))
         source = EbookSource(
             title=parsed.title,
             author=parsed.author,
@@ -157,24 +159,21 @@ async def ingest_file(session: AsyncSession, path: Path, config: EbookSearchConf
             await session.flush()
             chunk_index = add_chapter_chunks(session, source, chapter, parsed_chapter, chunk_index, config)
 
-        await session.commit()
         mention_count = await index_chunk_phrase_mentions_for_book(session, source.id, config)
+        await session.commit()
         logger.info(
-            "ebook_ingest_file_complete source_id=%s path=%s chapters=%s chunks=%s phrase_mentions=%s",
-            source.id,
-            resolved_path,
-            len(parsed.chapters),
-            chunk_index,
-            mention_count,
+            f"ebook_ingest_file_complete {source.id=} {resolved_path=} chapters={len(parsed.chapters)} {chunk_index=} "
+            f"{mention_count=}"
         )
     except Exception:
-        logger.exception(f"ebook_ingest_file_error path={path}")
+        await session.rollback()
+        logger.exception(f"ebook_ingest_file_error {path=}")
         return False
     else:
         return True
 
 
-async def find_existing_source(session: AsyncSession, path: Path, file_hash: str) -> EbookSource | None:
+async def find_existing_source(session: AsyncSession, path: Path | AsyncPath, file_hash: str) -> EbookSource | None:
     """Find an existing source by canonical path or file hash."""
     return await session.scalar(
         select(EbookSource).where(or_(EbookSource.file_path == str(path), EbookSource.file_sha256 == file_hash))
@@ -213,10 +212,10 @@ def add_chapter_chunks(
     return chunk_index
 
 
-def sha256_file(path: Path) -> str:
-    """Calculate the SHA-256 digest for a file."""
+async def sha256_file(path: AsyncPath) -> str:
+    """Calculate the SHA-256 digest for a file without blocking the event loop."""
     digest = hashlib.sha256()
-    with path.open("rb") as file:
-        for block in iter(lambda: file.read(1024 * 1024), b""):
+    async with await path.open("rb") as file:
+        while block := await file.read(1024 * 1024):
             digest.update(block)
     return digest.hexdigest()

python/ebook_search/llm_interface.py

@@ -51,10 +51,7 @@ async def request_embeddings(
         return embedding_vectors_from_response(response.json())
     except (httpx.HTTPError, ValueError, KeyError, TypeError) as error:
         logger.exception(
-            "ebook_embed_request_failed base_url=%s model=%s count=%s",
-            config.embedding_base_url,
-            config.embedding_model,
-            len(texts),
+            f"ebook_embed_request_failed {config.embedding_base_url=} {config.embedding_model=} count={len(texts)}"
         )
         msg = f"Embedding request failed. base_url={config.embedding_base_url} model={config.embedding_model}"
         raise RuntimeError(msg) from error
@@ -67,17 +64,13 @@ async def check_embedding_endpoint(
     timeout_seconds: float = 5.0,
 ) -> bool:
     """Return whether the configured embedding endpoint answers a model listing."""
-    try:
-        response = await client.get(
-            f"{config.embedding_base_url.rstrip('/')}/models",
-            headers=auth_headers(config.embedding_api_key),
-            timeout=timeout_seconds,
-        )
-        response.raise_for_status()
-    except httpx.HTTPError as error:
-        logger.warning("ebook_embedding_endpoint_unreachable base_url=%s error=%s", config.embedding_base_url, error)
-        return False
-    return True
+    return await _check_endpoint(
+        client,
+        base_url=config.embedding_base_url,
+        api_key=config.embedding_api_key,
+        timeout_seconds=timeout_seconds,
+        unavailable_log=f"ebook_embedding_endpoint_unreachable {config.embedding_base_url=}",
+    )
 
 
 async def check_chat_endpoint(
@@ -87,15 +80,33 @@ async def check_chat_endpoint(
     timeout_seconds: float = 5.0,
 ) -> bool:
     """Return whether the configured chat (answering) endpoint answers a model listing."""
+    return await _check_endpoint(
+        client,
+        base_url=config.vllm_base_url,
+        api_key=config.vllm_api_key,
+        timeout_seconds=timeout_seconds,
+        unavailable_log=f"ebook_chat_endpoint_unreachable {config.vllm_base_url=}",
+    )
+
+
+async def _check_endpoint(
+    client: httpx.AsyncClient,
+    *,
+    base_url: str,
+    api_key: str,
+    timeout_seconds: float,
+    unavailable_log: str,
+) -> bool:
+    """Return whether an OpenAI-compatible endpoint answers a model listing."""
     try:
         response = await client.get(
-            f"{config.vllm_base_url.rstrip('/')}/models",
-            headers=auth_headers(config.vllm_api_key),
+            f"{base_url.rstrip('/')}/models",
+            headers=auth_headers(api_key),
             timeout=timeout_seconds,
         )
         response.raise_for_status()
     except httpx.HTTPError as error:
-        logger.warning("ebook_chat_endpoint_unreachable base_url=%s error=%s", config.vllm_base_url, error)
+        logger.warning(f"{unavailable_log} {error=}")
         return False
     return True
 
@@ -163,6 +174,8 @@ async def request_chat_completion(
     client: httpx.AsyncClient,
     config: EbookSearchConfig,
     messages: Sequence[dict[str, str]],
+    *,
+    response_format: dict[str, object] | None = None,
 ) -> str:
     """Request a chat completion over a shared async client.
 
@@ -170,6 +183,7 @@ async def request_chat_completion(
         client (httpx.AsyncClient): Shared async client whose connection pool bounds concurrency.
         config (EbookSearchConfig): Runtime settings supplying the endpoint, model, and auth.
         messages (Sequence[dict[str, str]]): OpenAI-style chat messages.
+        response_format (dict[str, object] | None): Optional OpenAI-compatible structured output constraint.
 
     Returns:
         str: The assistant message text.
@@ -181,11 +195,8 @@ async def request_chat_completion(
         response = await client.post(
             f"{config.vllm_base_url.rstrip('/')}/chat/completions",
             headers=auth_headers(config.vllm_api_key),
-            json={
-                "model": config.chat_model,
-                "messages": list(messages),
-                "temperature": 0,
-            },
+            json={"model": config.chat_model, "messages": list(messages), "temperature": 0}
+            | ({"response_format": response_format} if response_format is not None else {}),
             timeout=config.chat_timeout_seconds,
         )
         response.raise_for_status()

python/ebook_search/loadtest.py

@@ -112,7 +112,7 @@ async def send_search(client: httpx.AsyncClient, query: str, *, rerank: bool) ->
     try:
         response = await client.post("/search", data=data)
     except httpx.HTTPError as error:
-        logger.warning("ebook_loadtest_request_failed error=%s", error)
+        logger.warning(f"ebook_loadtest_request_failed {error=}")
         return RequestResult(status_code=0, latency_ms=(time.perf_counter() - start) * 1000, ok=False)
     return RequestResult(
         status_code=response.status_code,
@@ -192,14 +192,7 @@ def main(
     """Load test the search endpoint and report latency and throughput."""
     configure_logger(log_level)
     queries = load_queries(queries_file)
-    logger.info(
-        "ebook_loadtest_start base_url=%s requests=%s concurrency=%s rerank=%s queries=%s",
-        base_url,
-        request_count,
-        concurrency,
-        rerank,
-        len(queries),
-    )
+    logger.info(f"ebook_loadtest_start {base_url=} {request_count=} {concurrency=} {rerank=} queries={len(queries)}")
     summary = asyncio.run(
         run_load(
             base_url=base_url,

python/ebook_search/prompts/__init__.py

@@ -0,0 +1,8 @@
+"""LLM prompt templates for EPUB search."""
+
+from python.ebook_search.prompts.lib import (
+    Prompt,
+    load_prompt,
+)
+
+__all__ = ["Prompt", "load_prompt"]

python/ebook_search/prompts/answer.toml

@@ -0,0 +1,9 @@
+system = """\
+Answer only from the provided context. Cite sources with bracketed numbers like [1]. \
+If the context is insufficient, say so."""
+user = """\
+Question:
+{query}
+
+Context:
+{context}"""

python/ebook_search/prompts/lib.py

@@ -0,0 +1,44 @@
+"""Load and render TOML-backed LLM prompt templates."""
+
+from __future__ import annotations
+
+import tomllib
+from dataclasses import dataclass
+from functools import cache
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class Prompt:
+    """A system and user prompt pair loaded from TOML."""
+
+    system: str
+    user: str
+
+    def messages(self, **values: str) -> list[dict[str, str]]:
+        """Render this prompt as OpenAI-style chat messages."""
+        return [
+            {"role": "system", "content": self.system.format(**values)},
+            {"role": "user", "content": self.user.format(**values)},
+        ]
+
+
+@cache
+def _get_prompt_dir() -> Path:
+    """Return the directory containing prompt template files."""
+    return Path(__file__).resolve().parent
+
+
+@cache
+def load_prompt(name: str) -> Prompt:
+    """Load and validate a named system and user prompt pair from TOML."""
+    path = _get_prompt_dir() / f"{name}.toml"
+    with path.open("rb") as file:
+        body = tomllib.load(file)
+
+    system = body.get("system")
+    user = body.get("user")
+    if not isinstance(system, str) or not isinstance(user, str):
+        msg = f"{path} must define string system and user prompts"
+        raise TypeError(msg)
+    return Prompt(system=system, user=user)

python/ebook_search/prompts/phrase_judge.toml

@@ -0,0 +1,10 @@
+system = """\
+Judge whether a candidate phrase from a book should be protected for RAG retrieval. \
+Do not extract new phrases. Reject common grammar fragments, ordinary nonspecific \
+phrases, unstable fragments, and phrases kept only because they are frequent. Keep \
+people, places, organizations, factions, events, technologies, fictional conditions, \
+magic systems, formal titles, named concepts, and recurring world-specific terms. \
+Return only a JSON object with keys: keep, canonical, category, aliases, confidence, \
+importance, allow_nested, suppress_children, reason."""
+
+user = "{candidate_json}"

python/ebook_search/protected_phrases/__init__.py

@@ -1 +1 @@
-"""Init."""
+"""Protected phrase extraction and matching for ebook search."""

python/ebook_search/protected_phrases/extraction.py

@@ -29,39 +29,9 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-BAD_START_SCORE_PENALTY = 10.0
-BAD_END_SCORE_PENALTY = 10.0
-MULTI_SOURCE_SCORE_BONUS = 2.0
-MULTI_SOURCE_MIN_SOURCES = 2
 CAPITALIZED_PHRASE_RE = re.compile(r"\b(?:[A-Z][a-zA-Z']+)(?:\s+(?:of|the|and|in|on|for|[A-Z][a-zA-Z']+)){0,6}")
 
 
-class SpacySpan(Protocol):
-    """Small protocol for the spaCy span attributes used by this module."""
-
-    text: str
-
-
-class SpacyEntity(SpacySpan, Protocol):
-    """Small protocol for the spaCy entity attributes used by this module."""
-
-    label_: str
-
-
-class SpacyDoc(Protocol):
-    """Small protocol for the spaCy doc attributes used by this module."""
-
-    ents: Iterable[SpacyEntity]
-    noun_chunks: Iterable[SpacySpan]
-
-
-class SpacyLanguage(Protocol):
-    """Small protocol for a callable spaCy language pipeline."""
-
-    def __call__(self, text: str) -> SpacyDoc:
-        """Parse text into a spaCy-like doc."""
-
-
 class YakeExtractor(Protocol):
     """Small protocol for the YAKE extractor used by this module."""
 
@@ -90,40 +60,30 @@ def normalize_candidate_phrase(
     phrase_text: str,
     config: EbookSearchConfig,
     *,
-    min_tokens: int | None = None,
     max_tokens: int | None = None,
-    strip_leading_article: bool = False,
 ) -> tuple[str, str, int] | None:
     """Normalize a candidate phrase and validate token bounds.
 
     Args:
         phrase_text (str): Raw phrase text to normalize.
         config (EbookSearchConfig): Runtime phrase-tuning settings.
-        min_tokens (int | None): Minimum token count override; defaults to ``config.phrase_min_tokens``.
         max_tokens (int | None): Maximum token count override; defaults to ``config.phrase_max_tokens``.
-        strip_leading_article (bool): Whether to drop a single leading English article.
 
     Returns:
         tuple[str, str, int] | None: Display text, normalized phrase, and token count, or ``None``
         when the phrase falls outside the token bounds or is ignored.
     """
     normalized_tokens = tokenize_with_offsets(phrase_text)
-    start = 0
-    if strip_leading_article and normalized_tokens and normalized_tokens[0].text in {"the", "a", "an"}:
-        start = 1
-
-    selected_tokens = normalized_tokens[start:]
-    min_count = config.phrase_min_tokens if min_tokens is None else min_tokens
     max_count = config.phrase_max_tokens if max_tokens is None else max_tokens
-    if len(selected_tokens) < min_count or len(selected_tokens) > max_count:
+    if len(normalized_tokens) < config.phrase_min_tokens or len(normalized_tokens) > max_count:
         return None
 
-    phrase_norm = " ".join(token.text for token in selected_tokens)
+    phrase_norm = " ".join(token.text for token in normalized_tokens)
     if phrase_norm in get_ignored_phrases():
         return None
 
-    display_text = phrase_text[selected_tokens[0].start_char : selected_tokens[-1].end_char].strip()
-    return display_text or phrase_norm, phrase_norm, len(selected_tokens)
+    display_text = phrase_text[normalized_tokens[0].start_char : normalized_tokens[-1].end_char].strip()
+    return display_text or phrase_norm, phrase_norm, len(normalized_tokens)
 
 
 def count_raw_ngrams(tokens: Sequence[str], config: EbookSearchConfig) -> Counter[str]:
@@ -194,7 +154,7 @@ def extract_raw_ngrams_by_chapter(
 
 
 @lru_cache(maxsize=2)
-def get_yake_extractor(max_ngram: int, top_k: int) -> KeywordExtractor:
+def get_yake_extractor(max_ngram: int, top_k: int, dedup_limit: float) -> KeywordExtractor:
     """Return a cached YAKE extractor for the given settings.
 
     Constructing a ``KeywordExtractor`` loads the language's stopword list from disk, so it is
@@ -203,29 +163,32 @@ def get_yake_extractor(max_ngram: int, top_k: int) -> KeywordExtractor:
     Args:
         max_ngram (int): Maximum n-gram size to extract.
         top_k (int): Maximum number of keyphrases to request.
+        dedup_limit (float): Deduplication similarity threshold.
 
     Returns:
         KeywordExtractor: A shared extractor instance for the given settings.
     """
-    return KeywordExtractor(lan="en", n=max_ngram, dedupLim=0.85, top=top_k)
+    return KeywordExtractor(lan="en", n=max_ngram, dedupLim=dedup_limit, top=top_k)
 
 
 def extract_yake_candidates(
     book_text: str,
     config: EbookSearchConfig,
-    top_k: int = 1000,
 ) -> dict[str, PhraseCandidate]:
     """Extract YAKE keyphrases when the optional YAKE package is installed.
 
     Args:
         book_text (str): Full book text to extract keyphrases from.
         config (EbookSearchConfig): Runtime phrase-tuning settings.
-        top_k (int): Maximum number of YAKE keyphrases to request.
 
     Returns:
         dict[str, PhraseCandidate]: Candidates keyed by normalized phrase, with YAKE scores.
     """
-    extractor = get_yake_extractor(config.phrase_max_tokens, top_k)
+    extractor = get_yake_extractor(
+        config.phrase_max_tokens,
+        config.phrase_yake_top_k,
+        config.phrase_yake_dedup_limit,
+    )
     out: dict[str, PhraseCandidate] = {}
     for phrase_text, yake_score in extractor.extract_keywords(book_text):
         normalized = normalize_candidate_phrase(phrase_text, config)
@@ -242,55 +205,6 @@ def extract_yake_candidates(
     return out
 
 
-def extract_spacy_candidates(
-    book_text: str,
-    nlp: SpacyLanguage,
-    config: EbookSearchConfig,
-) -> dict[str, PhraseCandidate]:
-    """Extract spaCy named entities and noun chunks from one text block.
-
-    Args:
-        book_text (str): Text block to parse with spaCy.
-        nlp (SpacyLanguage): Callable spaCy language pipeline.
-        config (EbookSearchConfig): Runtime phrase-tuning settings.
-
-    Returns:
-        dict[str, PhraseCandidate]: Candidates keyed by normalized phrase from entities and noun chunks.
-    """
-    out: dict[str, PhraseCandidate] = {}
-    doc = nlp(book_text)
-
-    for ent in doc.ents:
-        normalized = normalize_candidate_phrase(
-            ent.text,
-            config,
-            max_tokens=config.phrase_max_entity_tokens,
-        )
-        if normalized is None:
-            continue
-        phrase_text, phrase_norm, token_count = normalized
-        out[phrase_norm] = PhraseCandidate(
-            phrase_text=phrase_text,
-            phrase_norm=phrase_norm,
-            token_count=token_count,
-            source_spacy_ner=True,
-            spacy_label=ent.label_,
-        )
-
-    for chunk in doc.noun_chunks:
-        normalized = normalize_candidate_phrase(chunk.text, config, strip_leading_article=True)
-        if normalized is None:
-            continue
-        phrase_text, phrase_norm, token_count = normalized
-        out[phrase_norm] = PhraseCandidate(
-            phrase_text=phrase_text,
-            phrase_norm=phrase_norm,
-            token_count=token_count,
-            source_spacy_noun_chunk=True,
-        )
-    return out
-
-
 def extract_capitalized_phrases(original_text: str, config: EbookSearchConfig) -> dict[str, PhraseCandidate]:
     """Extract capitalized phrase runs that often carry fictional terms.
 
@@ -392,16 +306,12 @@ def merge_candidate(existing: PhraseCandidate, item: PhraseCandidate) -> None:
     """
     existing.source_raw_ngram = existing.source_raw_ngram or item.source_raw_ngram
     existing.source_yake = existing.source_yake or item.source_yake
-    existing.source_spacy_ner = existing.source_spacy_ner or item.source_spacy_ner
-    existing.source_spacy_noun_chunk = existing.source_spacy_noun_chunk or item.source_spacy_noun_chunk
     existing.source_capitalized = existing.source_capitalized or item.source_capitalized
     existing.source_metadata = existing.source_metadata or item.source_metadata
     existing.raw_count += item.raw_count
     existing.chapter_count = max(existing.chapter_count, item.chapter_count)
     if item.yake_score is not None:
         existing.yake_score = item.yake_score
-    if item.spacy_label:
-        existing.spacy_label = item.spacy_label
 
 
 def enrich_with_frequency_and_chapter_counts(
@@ -576,14 +486,14 @@ def score_candidate(candidate: PhraseCandidate, config: EbookSearchConfig) -> fl
         float: Combined score from sources, frequency, and length, less any penalties.
     """
     score = source_score(candidate) + frequency_score(candidate, config) + token_count_score(candidate, config)
-    if non_raw_source_count(candidate) >= MULTI_SOURCE_MIN_SOURCES:
-        score += MULTI_SOURCE_SCORE_BONUS
+    if non_raw_source_count(candidate) >= config.phrase_multi_source_min_sources:
+        score += config.phrase_multi_source_score_bonus
     if candidate.phrase_norm in get_ignored_phrases():
         score -= 100.0
     if has_bad_start(candidate.phrase_norm):
-        score -= BAD_START_SCORE_PENALTY
+        score -= config.phrase_bad_start_score_penalty
     if has_bad_end(candidate.phrase_norm):
-        score -= BAD_END_SCORE_PENALTY
+        score -= config.phrase_bad_end_score_penalty
     return score
 
 
@@ -599,8 +509,6 @@ def non_raw_source_count(candidate: PhraseCandidate) -> int:
     return sum(
         (
             candidate.source_yake,
-            candidate.source_spacy_ner,
-            candidate.source_spacy_noun_chunk,
             candidate.source_capitalized,
             candidate.source_metadata,
         )
@@ -646,8 +554,6 @@ def source_score(candidate: PhraseCandidate) -> float:
         weight
         for enabled, weight in (
             (candidate.source_yake, 2.0),
-            (candidate.source_spacy_ner, 2.5),
-            (candidate.source_spacy_noun_chunk, 1.5),
             (candidate.source_capitalized, 2.0),
             (candidate.source_metadata, 2.0),
             (candidate.source_raw_ngram, 0.5),
@@ -738,10 +644,6 @@ def candidate_source_names(candidate: PhraseCandidate) -> list[str]:
         names.append("raw_ngram")
     if candidate.source_yake:
         names.append("yake")
-    if candidate.source_spacy_ner:
-        names.append("spacy_ner")
-    if candidate.source_spacy_noun_chunk:
-        names.append("spacy_noun_chunk")
     if candidate.source_capitalized:
         names.append("capitalized")
     if candidate.source_metadata:
@@ -754,16 +656,14 @@ def extract_phrase_candidates_for_book(
     chapters: Sequence[str],
     config: EbookSearchConfig,
     *,
-    nlp: SpacyLanguage | None = None,
     metadata: Mapping[str, object] | None = None,
 ) -> list[PhraseCandidate]:
     """Extract, score, and limit phrase candidates for one book.
 
     Args:
         book_text (str): Full book text used for most extraction sources.
-        chapters (Sequence[str]): Chapter-like text blocks used for spaCy and frequency counts.
+        chapters (Sequence[str]): Chapter-like text blocks used for frequency counts.
         config (EbookSearchConfig): Runtime phrase-tuning settings.
-        nlp (SpacyLanguage | None): Optional spaCy pipeline for entity and noun-chunk sources.
         metadata (Mapping[str, object] | None): Optional book metadata used as a candidate source.
 
     Returns:
@@ -771,47 +671,30 @@ def extract_phrase_candidates_for_book(
     """
     started_at = perf_counter()
     logger.info(
-        "ebook_phrase_candidate_extract_start chapters=%s chars=%s min_tokens=%s max_tokens=%s max_candidates=%s",
-        len(chapters),
-        len(book_text),
-        config.phrase_min_tokens,
-        config.phrase_max_tokens,
-        config.protected_phrase_max_candidates_per_book,
+        f"ebook_phrase_candidate_extract_start chapters={len(chapters)} chars={len(book_text)} "
+        f"{config.phrase_min_tokens=} {config.phrase_max_tokens=} {config.protected_phrase_max_candidates_per_book=}"
     )
     raw_started_at = perf_counter()
     raw = extract_raw_ngrams_by_chapter(chapters, config)
     logger.info(
-        "ebook_phrase_candidate_extract_raw_complete candidates=%s duration_ms=%.1f",
-        len(raw),
-        (perf_counter() - raw_started_at) * 1000,
+        f"ebook_phrase_candidate_extract_raw_complete candidates={len(raw)} "
+        f"duration_ms={(perf_counter() - raw_started_at) * 1000:.1f}"
     )
     yake_started_at = perf_counter()
     yake_candidates = extract_yake_candidates(book_text, config)
     logger.info(
-        "ebook_phrase_candidate_extract_yake_complete candidates=%s duration_ms=%.1f",
-        len(yake_candidates),
-        (perf_counter() - yake_started_at) * 1000,
+        f"ebook_phrase_candidate_extract_yake_complete candidates={len(yake_candidates)} "
+        f"duration_ms={(perf_counter() - yake_started_at) * 1000:.1f}"
     )
-    spacy_candidates: dict[str, PhraseCandidate] = {}
-    if nlp is not None:
-        spacy_started_at = perf_counter()
-        for chapter in chapters:
-            spacy_candidates = merge_candidate_sources(spacy_candidates, extract_spacy_candidates(chapter, nlp, config))
-        logger.info(
-            "ebook_phrase_candidate_extract_spacy_complete candidates=%s duration_ms=%.1f",
-            len(spacy_candidates),
-            (perf_counter() - spacy_started_at) * 1000,
-        )
     capitalized_started_at = perf_counter()
     capitalized = extract_capitalized_phrases(book_text, config)
     logger.info(
-        "ebook_phrase_candidate_extract_capitalized_complete candidates=%s duration_ms=%.1f",
-        len(capitalized),
-        (perf_counter() - capitalized_started_at) * 1000,
+        f"ebook_phrase_candidate_extract_capitalized_complete candidates={len(capitalized)} "
+        f"duration_ms={(perf_counter() - capitalized_started_at) * 1000:.1f}"
     )
     metadata_candidates = extract_metadata_candidates(metadata, config)
 
-    candidates = merge_candidate_sources(raw, yake_candidates, spacy_candidates, capitalized, metadata_candidates)
+    candidates = merge_candidate_sources(raw, yake_candidates, capitalized, metadata_candidates)
     enriched_started_at = perf_counter()
     # Raw n-gram sizes were already counted per chapter above, so only enrich the remaining
     # (entity-length) sizes here instead of re-sliding every size over the whole book.
@@ -831,23 +714,11 @@ def extract_phrase_candidates_for_book(
         : config.protected_phrase_max_candidates_per_book
     ]
     logger.info(
-        "ebook_phrase_candidate_extract_complete raw=%s yake=%s spacy=%s capitalized=%s metadata=%s "
-        "merged=%s filtered_too_short=%s filtered_too_rare=%s filtered_too_common=%s filtered_junk=%s "
-        "min_uses=%s storable=%s limited=%s enrich_score_ms=%.1f duration_ms=%.1f",
-        len(raw),
-        len(yake_candidates),
-        len(spacy_candidates),
-        len(capitalized),
-        len(metadata_candidates),
-        pre_filter_count,
-        filtered_too_short,
-        filtered_too_rare,
-        filtered_too_common,
-        filtered_junk,
-        minimum_candidate_raw_count(config),
-        len(candidates),
-        len(limited),
-        (perf_counter() - enriched_started_at) * 1000,
-        (perf_counter() - started_at) * 1000,
+        f"ebook_phrase_candidate_extract_complete raw={len(raw)} yake={len(yake_candidates)} "
+        f"capitalized={len(capitalized)} metadata={len(metadata_candidates)} {pre_filter_count=} {filtered_too_short=} "
+        f"{filtered_too_rare=} {filtered_too_common=} {filtered_junk=} min_uses={minimum_candidate_raw_count(config)} "
+        f"storable={len(candidates)} limited={len(limited)} "
+        f"enrich_score_ms={(perf_counter() - enriched_started_at) * 1000:.1f} "
+        f"duration_ms={(perf_counter() - started_at) * 1000:.1f}"
     )
     return limited

python/ebook_search/protected_phrases/generate_ngrams.py

@@ -4,11 +4,11 @@ from __future__ import annotations
 
 import asyncio
 import logging
-from collections import deque
 from time import perf_counter
 from typing import TYPE_CHECKING
 
 from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
 
 from python.ebook_search.protected_phrases.extraction import extract_phrase_candidates_for_book
 from python.ebook_search.protected_phrases.models import (
@@ -16,65 +16,81 @@ from python.ebook_search.protected_phrases.models import (
     PhraseCandidateGenerationResult,
     PhraseRecalculationResult,
 )
-from python.ebook_search.protected_phrases.pool import extract_phrase_candidates_in_pool, get_extraction_pool
+from python.ebook_search.protected_phrases.pool import get_extraction_pool
 from python.ebook_search.protected_phrases.store import (
     bulk_upsert_unjudged_candidates,
     delete_phrase_data_for_book,
     load_book_chapter_texts,
-    metadata_for_source,
+    metadata_for_source_id,
     new_candidate_row,
     prune_unstorable_unjudged_candidate_phrases,
 )
-from python.orm.richie import EbookCandidatePhrase, EbookSource
+from python.orm.common import get_async_postgres_engine
+from python.orm.richie import EbookSource
 
 if TYPE_CHECKING:
-    from collections.abc import Mapping, Sequence
-    from concurrent.futures import Future
-
-    from sqlalchemy.ext.asyncio import AsyncSession
+    from sqlalchemy.ext.asyncio import AsyncEngine
 
     from python.ebook_search.config import EbookSearchConfig
-    from python.ebook_search.protected_phrases.extraction import SpacyLanguage
     from python.ebook_search.protected_phrases.models import PhraseCandidate
 
 logger = logging.getLogger(__name__)
 
 
+class BookHasNoChaptersError(ValueError):
+    """Raised when a book has no indexed chapter text to generate phrases from."""
+
+
 async def generate_candidate_phrases_for_books(
-    session: AsyncSession,
+    engine: AsyncEngine,
     config: EbookSearchConfig,
-    *,
-    only_missing: bool = False,
 ) -> PhraseCandidateGenerationResult:
     """Create or refresh candidate phrases for indexed books without calling the LLM judge.
 
-    Extraction always runs concurrently in the shared process pool so a full backfill uses
-    multiple cores.
+    Every book is submitted to the shared process pool up front and runs in parallel across the
+    pool's workers; the call blocks until all books have finished. Each worker opens its own
+    database engine from environment variables, loads the book's chapters, and commits the
+    book's candidates independently.
 
     Args:
-        session (Session): Active database session.
+        engine (AsyncEngine): Engine used to read the book list in this process.
         config (EbookSearchConfig): Runtime phrase-tuning settings.
-        only_missing (bool): When True, only generate for books that have no candidate phrases
-            yet instead of refreshing every book.
 
     Returns:
         PhraseCandidateGenerationResult: Per-corpus counts of books seen, built, and candidates stored.
+
+    Results are collected in book order while the pool keeps working. A book failure (including
+    a book with no indexed chapters) is logged and counted as not built; the remaining books
+    are unaffected.
     """
-    source_query = select(EbookSource).order_by(EbookSource.id)
-    if only_missing:
-        has_candidates = select(EbookCandidatePhrase.id).where(EbookCandidatePhrase.book_id == EbookSource.id)
-        source_query = source_query.where(~has_candidates.exists())
-    sources = (await session.scalars(source_query)).all()
-    books_seen = len(sources)
+    async with AsyncSession(engine, expire_on_commit=False) as session:
+        source_query = select(EbookSource.id).order_by(EbookSource.id)
+        source_ids = (await session.scalars(source_query)).all()
+    books_seen = len(source_ids)
     logger.info(
-        "ebook_candidate_phrase_generation_start books_seen=%s min_tokens=%s max_tokens=%s max_candidates_per_book=%s",
-        books_seen,
-        config.phrase_min_tokens,
-        config.phrase_max_tokens,
-        config.protected_phrase_max_candidates_per_book,
+        f"ebook_candidate_phrase_generation_start {books_seen=} {config.phrase_min_tokens=} "
+        f"{config.phrase_max_tokens=} {config.protected_phrase_max_candidates_per_book=}"
     )
 
-    outcomes = await generate_candidates_for_sources_pooled(session, sources, config)
+    pool = get_extraction_pool(config.protected_phrase_extraction_workers)
+    wrapped_futures = [
+        (
+            source_id,
+            asyncio.wrap_future(pool.submit(generate_candidate_phrases_for_book_in_worker, source_id, None, config)),
+        )
+        for source_id in source_ids
+    ]
+    outcomes: list[BookCandidateResult] = []
+    for source_id, wrapped_future in wrapped_futures:
+        await asyncio.wait([wrapped_future])
+        exception = wrapped_future.exception()
+        if exception is not None:
+            logger.error(f"ebook_candidate_phrase_generation_book_failed {source_id=}")
+            outcomes.append(BookCandidateResult())
+            continue
+        saved_count = wrapped_future.result()
+        logger.info(f"ebook_candidate_phrase_generation_book_committed {source_id=} {saved_count=}")
+        outcomes.append(BookCandidateResult(candidates=saved_count, built=True))
 
     result = PhraseCandidateGenerationResult(
         books_seen=books_seen,
@@ -82,159 +98,44 @@ async def generate_candidate_phrases_for_books(
         candidate_phrases=sum(outcome.candidates for outcome in outcomes),
     )
     logger.info(
-        "ebook_candidate_phrase_generation_complete books_seen=%s books_built=%s candidate_total=%s",
-        result.books_seen,
-        result.books_built,
-        result.candidate_phrases,
+        f"ebook_candidate_phrase_generation_complete {result.books_seen=} {result.books_built=} "
+        f"{result.candidate_phrases=}"
     )
     return result
 
 
-async def generate_candidates_for_sources_pooled(
-    session: AsyncSession,
-    sources: Sequence[EbookSource],
-    config: EbookSearchConfig,
-) -> list[BookCandidateResult]:
-    """Generate candidate phrases for many books, extracting them concurrently in worker processes.
-
-    Chapter loading and row persistence stay on the caller's session (serial), while the CPU-bound
-    extraction runs in the shared process pool. A bounded window of in-flight books overlaps
-    extraction across cores without loading every book's candidates into memory at once.
-
-    Args:
-        session (Session): Active database session.
-        sources (Sequence[EbookSource]): Indexed books to generate candidates for.
-        config (EbookSearchConfig): Runtime phrase-tuning settings.
-
-    Returns:
-        list[BookCandidateResult]: One result per book.
-    """
-    pool = get_extraction_pool(config.protected_phrase_extraction_workers)
-    max_in_flight = max(1, config.protected_phrase_extraction_workers) * 2
-    pending: deque[tuple[EbookSource, Future[list[PhraseCandidate]]]] = deque()
-    outcomes: list[BookCandidateResult] = []
-
-    async def drain_one() -> None:
-        source, future = pending.popleft()
-        extracted = await asyncio.wrap_future(future)
-        outcomes.append(await store_source_candidates(session, source, extracted, config))
-
-    try:
-        for source in sources:
-            chapters = await load_book_chapter_texts(session, source.id)
-            if not chapters:
-                logger.warning("ebook_candidate_phrase_generation_book_empty source_id=%s", source.id)
-                outcomes.append(BookCandidateResult())
-                continue
-            future = pool.submit(
-                extract_phrase_candidates_for_book,
-                "\n\n".join(chapters),
-                chapters,
-                config,
-                metadata=metadata_for_source(source),
-            )
-            pending.append((source, future))
-            if len(pending) >= max_in_flight:
-                await drain_one()
-        while pending:
-            await drain_one()
-    except Exception:
-        for _, future in pending:
-            future.cancel()
-        await session.rollback()
-        logger.exception("ebook_candidate_phrase_generation_pooled_failed")
-        raise
-    return outcomes
-
-
-async def store_source_candidates(
-    session: AsyncSession,
-    source: EbookSource,
-    limited_candidates: list[PhraseCandidate],
-    config: EbookSearchConfig,
-) -> BookCandidateResult:
-    """Persist and commit one book's already-extracted candidates.
-
-    Args:
-        session (AsyncSession): Active database session.
-        source (EbookSource): Book the candidates belong to.
-        limited_candidates (list[PhraseCandidate]): Scored candidates to persist.
-        config (EbookSearchConfig): Runtime phrase-tuning settings.
-
-    Returns:
-        BookCandidateResult: Candidate count and that the book was committed.
-    """
-    book_started_at = perf_counter()
-    saved_count = await store_candidate_phrases_for_book(session, source.id, None, limited_candidates, config)
-    await session.commit()
-    logger.info(
-        "ebook_candidate_phrase_generation_book_committed source_id=%s candidates=%s duration_ms=%.1f",
-        source.id,
-        saved_count,
-        (perf_counter() - book_started_at) * 1000,
-    )
-    return BookCandidateResult(candidates=saved_count, built=True)
-
-
 async def recalculate_candidate_phrases_for_book(
     session: AsyncSession,
     source: EbookSource,
     config: EbookSearchConfig,
-    *,
-    nlp: SpacyLanguage | None = None,
-    use_process_pool: bool = False,
 ) -> PhraseRecalculationResult:
     """Remove all book phrase data, regenerate candidates, and commit the completed book.
 
     Args:
-        session (Session): Active database session.
+        session (AsyncSession): Active database session; deletion and regeneration commit on it.
         source (EbookSource): Indexed book to recalculate.
         config (EbookSearchConfig): Runtime phrase-tuning settings.
-        nlp (SpacyLanguage | None): Optional spaCy pipeline for entity and noun-chunk sources.
-        use_process_pool (bool): Run the CPU-bound extraction in a worker process so concurrent
-            recalculations do not serialize behind the GIL. Defaults to in-process for callers
-            (tests, backfills) that do not need it.
 
     Returns:
         PhraseRecalculationResult: Deleted-row counts and the number of candidates regenerated.
+
+    Raises:
+        BookHasNoChaptersError: If the book has no indexed chapters. The deletion is rolled
+            back, so the book's existing phrases stay intact.
+
+    The deletion and regeneration share the caller's session, so they commit together; a
+    regeneration failure rolls the deletion back.
     """
     started_at = perf_counter()
-    logger.info(
-        "ebook_candidate_phrase_recalculation_start source_id=%s title=%r",
+    logger.info(f"ebook_candidate_phrase_recalculation_start {source.id=} {source.title=}")
+    deleted = await delete_phrase_data_for_book(session, source.id)
+    candidate_count = await generate_candidate_phrases_for_book(
+        session,
         source.id,
-        source.title,
+        series_id=None,
+        config=config,
+        replace_all=True,
     )
-    try:
-        deleted = await delete_phrase_data_for_book(session, source.id)
-        chapters = await load_book_chapter_texts(session, source.id)
-        if not chapters:
-            logger.warning("ebook_candidate_phrase_recalculation_book_empty source_id=%s", source.id)
-            await session.commit()
-            return PhraseRecalculationResult(
-                book_id=source.id,
-                deleted_candidates=deleted.deleted_candidates,
-                deleted_protected_phrases=deleted.deleted_protected_phrases,
-                deleted_aliases=deleted.deleted_aliases,
-                deleted_mentions=deleted.deleted_mentions,
-                candidate_phrases=0,
-            )
-
-        candidate_count = await generate_candidate_phrases_for_book(
-            session,
-            source.id,
-            series_id=None,
-            chapters=chapters,
-            config=config,
-            nlp=nlp,
-            metadata=metadata_for_source(source),
-            replace_all=True,
-            use_process_pool=use_process_pool,
-        )
-        await session.commit()
-    except Exception:
-        await session.rollback()
-        logger.exception("ebook_candidate_phrase_recalculation_failed source_id=%s", source.id)
-        raise
 
     result = PhraseRecalculationResult(
         book_id=source.id,
@@ -245,75 +146,106 @@ async def recalculate_candidate_phrases_for_book(
         candidate_phrases=candidate_count,
     )
     logger.info(
-        "ebook_candidate_phrase_recalculation_complete source_id=%s deleted_candidates=%s "
-        "deleted_protected=%s deleted_aliases=%s deleted_mentions=%s candidates=%s duration_ms=%.1f",
-        source.id,
-        result.deleted_candidates,
-        result.deleted_protected_phrases,
-        result.deleted_aliases,
-        result.deleted_mentions,
-        result.candidate_phrases,
-        (perf_counter() - started_at) * 1000,
+        f"ebook_candidate_phrase_recalculation_complete {source.id=} {result.deleted_candidates=} "
+        f"{result.deleted_protected_phrases=} {result.deleted_aliases=} {result.deleted_mentions=} "
+        f"{result.candidate_phrases=} duration_ms={(perf_counter() - started_at) * 1000:.1f}"
     )
     return result
 
 
+def generate_candidate_phrases_for_book_in_worker(
+    book_id: int,
+    series_id: int | None,
+    config: EbookSearchConfig,
+) -> int:
+    """Run one book's candidate generation in a pooled worker process.
+
+    The worker has no engine or session to inherit (neither can cross process boundaries), so
+    it creates its own engine from environment variables, opens the book's session on it, and
+    disposes the engine once the book is stored.
+
+    Args:
+        book_id (int): Book the candidates belong to.
+        series_id (int | None): Series scope for the stored candidates.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        int: Number of candidate phrase rows stored.
+    """
+
+    async def generate_with_worker_engine() -> int:
+        engine = get_async_postgres_engine(name="RICHIE", vector_engine=True, pool_size=1)
+        try:
+            async with AsyncSession(engine, expire_on_commit=False) as session:
+                return await generate_candidate_phrases_for_book(
+                    session,
+                    book_id,
+                    series_id,
+                    config,
+                )
+        finally:
+            await engine.dispose()
+
+    return asyncio.run(generate_with_worker_engine())
+
+
 async def generate_candidate_phrases_for_book(
     session: AsyncSession,
     book_id: int,
     series_id: int | None,
-    chapters: Sequence[str],
     config: EbookSearchConfig,
     *,
-    nlp: SpacyLanguage | None = None,
-    metadata: Mapping[str, object] | None = None,
     replace_all: bool = False,
-    use_process_pool: bool = False,
 ) -> int:
-    """Extract and store candidate phrases for one book without LLM judging.
+    """Load a book's chapters and metadata, extract candidate phrases, and store them without LLM judging.
+
+    The session commits only when the whole book succeeds; any failure rolls the session back,
+    which also restores rows the caller deleted in the same transaction (e.g. a recalculation).
 
     Args:
-        session (Session): Active database session.
+        session (AsyncSession): Active database session; committed on success, rolled back on failure.
         book_id (int): Book the candidates belong to.
         series_id (int | None): Series scope for the stored candidates.
-        chapters (Sequence[str]): Chapter-like text blocks used for extraction and frequency counts.
         config (EbookSearchConfig): Runtime phrase-tuning settings.
-        nlp (SpacyLanguage | None): Optional spaCy pipeline for entity and noun-chunk sources.
-        metadata (Mapping[str, object] | None): Optional book metadata used as a candidate source.
         replace_all (bool): When the caller has already cleared this book's candidates (e.g. a
             recalculation), skip the per-candidate existence lookup and bulk-insert new rows.
-        use_process_pool (bool): Run the CPU-bound extraction in a worker process to avoid
-            serializing concurrent requests behind the GIL. Ignored when ``nlp`` is set, since
-            the spaCy pipeline cannot be sent to a worker process.
 
     Returns:
         int: Number of candidate phrase rows stored.
+
+    Raises:
+        BookHasNoChaptersError: If the book has no indexed chapter text.
     """
     started_at = perf_counter()
-    book_text = "\n\n".join(chapters)
-    if use_process_pool and nlp is None:
-        limited_candidates = await extract_phrase_candidates_in_pool(book_text, chapters, config, metadata=metadata)
-    else:
-        limited_candidates = extract_phrase_candidates_for_book(
+    chapters = await load_book_chapter_texts(session, book_id)
+    if not chapters:
+        await session.rollback()
+        message = f"book {book_id} has no indexed chapters"
+        raise BookHasNoChaptersError(message)
+    metadata = await metadata_for_source_id(session, book_id)
+    try:
+        book_text = "\n\n".join(chapters)
+        candidates = extract_phrase_candidates_for_book(
             book_text,
             chapters,
             config,
-            nlp=nlp,
             metadata=metadata,
         )
-    saved_count = await store_candidate_phrases_for_book(
-        session,
-        book_id,
-        series_id,
-        limited_candidates,
-        config,
-        replace_all=replace_all,
-    )
+        saved_count = await store_candidate_phrases_for_book(
+            session,
+            book_id,
+            series_id,
+            candidates,
+            config,
+            replace_all=replace_all,
+        )
+        await session.commit()
+    except Exception:
+        await session.rollback()
+        raise
     logger.info(
-        "ebook_candidate_phrase_generation_book_duration book_id=%s candidates=%s duration_ms=%.1f",
-        book_id,
-        saved_count,
-        (perf_counter() - started_at) * 1000,
+        f"ebook_candidate_phrase_generation_book_duration {book_id=} {saved_count=} "
+        f"duration_ms={(perf_counter() - started_at) * 1000:.1f}"
     )
     return saved_count
 
@@ -348,23 +280,16 @@ async def store_candidate_phrases_for_book(
         await session.flush()
         saved_count = len(rows)
         logger.info(
-            "ebook_candidate_phrase_save_start book_id=%s candidates=%s mode=bulk_insert",
-            book_id,
-            len(limited_candidates),
+            f"ebook_candidate_phrase_save_start {book_id=} candidates={len(limited_candidates)} mode=bulk_insert"
         )
     else:
         pruned_count = await prune_unstorable_unjudged_candidate_phrases(session, book_id, config)
         logger.info(
-            "ebook_candidate_phrase_save_start book_id=%s candidates=%s pruned_unstorable=%s",
-            book_id,
-            len(limited_candidates),
-            pruned_count,
+            f"ebook_candidate_phrase_save_start {book_id=} candidates={len(limited_candidates)} {pruned_count=}"
         )
         saved_count = await bulk_upsert_unjudged_candidates(session, book_id, series_id, limited_candidates)
     logger.info(
-        "ebook_candidate_phrase_save_complete book_id=%s candidates=%s save_ms=%.1f",
-        book_id,
-        saved_count,
-        (perf_counter() - save_started_at) * 1000,
+        f"ebook_candidate_phrase_save_complete {book_id=} {saved_count=} "
+        f"save_ms={(perf_counter() - save_started_at) * 1000:.1f}"
     )
     return saved_count

python/ebook_search/protected_phrases/judge_ngrams.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import asyncio
 import json
 import logging
-import re
+from dataclasses import replace
 from time import perf_counter
 from typing import TYPE_CHECKING
 
@@ -14,6 +14,7 @@ from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from python.ebook_search.llm_interface import request_chat_completion
+from python.ebook_search.prompts import load_prompt
 from python.ebook_search.protected_phrases.extraction import (
     candidate_source_names,
     get_sample_contexts,
@@ -44,9 +45,6 @@ if TYPE_CHECKING:
     from python.ebook_search.protected_phrases.models import PhraseCandidate
     from python.orm.richie import EbookProtectedPhrase
 
-JSON_OBJECT_RE = re.compile(r"\{.*\}", re.DOTALL)
-
-
 logger = logging.getLogger(__name__)
 
 
@@ -80,12 +78,8 @@ async def judge_candidate_phrases_for_books(
     book_workers = max(1, config.phrase_judge_book_workers)
     phrase_workers = max(1, config.phrase_judge_phrase_workers)
     logger.info(
-        "ebook_candidate_phrase_judgment_start books_seen=%s book_workers=%s phrase_workers=%s "
-        "confidence_threshold=%.2f",
-        books_seen,
-        book_workers,
-        phrase_workers,
-        config.protected_phrase_confidence_threshold,
+        f"ebook_candidate_phrase_judgment_start {books_seen=} {book_workers=} {phrase_workers=} "
+        f"{config.protected_phrase_confidence_threshold=:.2f}"
     )
 
     book_semaphore = asyncio.Semaphore(book_workers)
@@ -105,14 +99,8 @@ async def judge_candidate_phrases_for_books(
         phrase_mentions=sum(outcome.mentions for outcome in outcomes),
     )
     logger.info(
-        "ebook_candidate_phrase_judgment_complete books_seen=%s books_judged=%s books_failed=%s "
-        "candidates_judged=%s protected=%s mentions=%s",
-        result.books_seen,
-        result.books_judged,
-        result.books_failed,
-        result.candidates_judged,
-        result.protected_phrases,
-        result.phrase_mentions,
+        f"ebook_candidate_phrase_judgment_complete {result.books_seen=} {result.books_judged=} {result.books_failed=} "
+        f"{result.candidates_judged=} {result.protected_phrases=} {result.phrase_mentions=}"
     )
     return result
 
@@ -147,7 +135,7 @@ async def judge_one_book_async(
                 return BookJudgmentResult()
             return await persist_book_judgments(engine, source_id, config, judged)
         except Exception:
-            logger.exception("ebook_candidate_phrase_judgment_book_failed source_id=%s", source_id)
+            logger.exception(f"ebook_candidate_phrase_judgment_book_failed {source_id=}")
             return BookJudgmentResult(failed=True)
 
 
@@ -173,7 +161,7 @@ async def prepare_book_judgment(
         return None
     async with AsyncSession(engine) as session:
         if not await count_unjudged_candidates(session, source_id, config):
-            logger.info("ebook_candidate_phrase_judgment_book_skip_no_unjudged source_id=%s", source_id)
+            logger.info(f"ebook_candidate_phrase_judgment_book_skip_no_unjudged {source_id=}")
             return None
         existing_protected = await count_protected_phrases(session, source_id)
         target_remaining: int | None = None
@@ -181,15 +169,13 @@ async def prepare_book_judgment(
             target_remaining = max(config.phrase_target_protected_per_book - existing_protected, 0)
             if target_remaining == 0:
                 logger.info(
-                    "ebook_candidate_phrase_judgment_skipped_target_met source_id=%s existing_protected=%s target=%s",
-                    source_id,
-                    existing_protected,
-                    config.phrase_target_protected_per_book,
+                    f"ebook_candidate_phrase_judgment_skipped_target_met {source_id=} {existing_protected=} "
+                    f"{config.phrase_target_protected_per_book=}"
                 )
                 return None
         book_text = await load_book_text(session, source_id)
         if not book_text:
-            logger.warning("ebook_candidate_phrase_judgment_book_empty source_id=%s", source_id)
+            logger.warning(f"ebook_candidate_phrase_judgment_book_empty {source_id=}")
             return None
         normalized_book_text = normalize_text(book_text)
         # Stored rows may predate the current junk filters and score weights, so re-filter and
@@ -211,15 +197,8 @@ async def prepare_book_judgment(
                 normalized_book_text, candidate.phrase_norm
             )
     logger.info(
-        "ebook_candidate_phrase_judgment_candidates_loaded source_id=%s candidates=%s skipped_junk=%s "
-        "unjudged_rows=%s existing_protected=%s target_remaining=%s judgment_limit=%s",
-        source_id,
-        len(work_items),
-        skipped_junk,
-        len(rows),
-        existing_protected,
-        target_remaining,
-        judgment_limit,
+        f"ebook_candidate_phrase_judgment_candidates_loaded {source_id=} candidates={len(work_items)} {skipped_junk=} "
+        f"unjudged_rows={len(rows)} {existing_protected=} {target_remaining=} {judgment_limit=}"
     )
     return work_items, target_remaining
 
@@ -280,7 +259,9 @@ async def judge_candidate_async(
     Returns:
         LLMJudgment: The parsed judgment.
     """
-    content = await request_chat_completion(client, config, build_judge_messages(candidate))
+    content = await request_chat_completion(
+        client, config, build_judge_messages(candidate), response_format={"type": "json_object"}
+    )
     return parse_llm_judgment(content, config)
 
 
@@ -305,39 +286,39 @@ async def persist_book_judgments(
     book_started_at = perf_counter()
     async with AsyncSession(engine, expire_on_commit=False) as session:
         try:
+            normalized_book_text = normalize_text(await load_book_text(session, source_id))
             protected: list[EbookProtectedPhrase] = []
             for candidate_id, candidate, judgment, promote in judged:
-                candidate_row = await save_candidate_to_db(session, source_id, None, candidate, judgment=judgment)
+                filtered_judgment = replace(
+                    judgment,
+                    aliases=tuple(
+                        alias for alias in judgment.aliases if alias_occurs_in_book(alias, normalized_book_text)
+                    ),
+                )
+                candidate_row = await save_candidate_to_db(
+                    session, source_id, None, candidate, judgment=filtered_judgment
+                )
                 if promote:
                     protected.append(
-                        await upsert_protected_phrase(session, source_id, None, candidate, judgment, candidate_row)
+                        await upsert_protected_phrase(
+                            session, source_id, None, candidate, filtered_judgment, candidate_row
+                        )
                     )
                 logger.info(
-                    "ebook_candidate_phrase_judgment_candidate_complete source_id=%s candidate_id=%s phrase=%r "
-                    "keep=%s confidence=%.3f category=%r promoted=%s",
-                    source_id,
-                    candidate_id,
-                    candidate.phrase_norm,
-                    judgment.keep,
-                    judgment.confidence,
-                    judgment.category,
-                    promote,
+                    f"ebook_candidate_phrase_judgment_candidate_complete {source_id=} {candidate_id=} "
+                    f"{candidate.phrase_norm=} {judgment.keep=} {judgment.confidence=:.3f} {judgment.category=} "
+                    f"{promote=}"
                 )
             await session.flush()
             mentions = await index_chunk_phrase_mentions_for_book(session, source_id, config) if protected else 0
             await session.commit()
         except Exception:
             await session.rollback()
-            logger.exception("ebook_candidate_phrase_judgment_book_persist_failed source_id=%s", source_id)
+            logger.exception(f"ebook_candidate_phrase_judgment_book_persist_failed {source_id=}")
             return BookJudgmentResult(failed=True)
     logger.info(
-        "ebook_candidate_phrase_judgment_book_committed source_id=%s judged=%s protected=%s mentions=%s "
-        "duration_ms=%.1f",
-        source_id,
-        len(judged),
-        len(protected),
-        mentions,
-        (perf_counter() - book_started_at) * 1000,
+        f"ebook_candidate_phrase_judgment_book_committed {source_id=} judged={len(judged)} protected={len(protected)} "
+        f"{mentions=} duration_ms={(perf_counter() - book_started_at) * 1000:.1f}"
     )
     return BookJudgmentResult(judged=len(judged), protected=len(protected), mentions=mentions, committed=True)
 
@@ -369,24 +350,14 @@ def should_protect_judged_candidate(
     accepted_token_count = len(accepted_tokens)
     if accepted_token_count < config.phrase_min_tokens:
         logger.info(
-            "ebook_candidate_phrase_judgment_candidate_skip_short_canonical book_id=%s candidate_id=%s "
-            "phrase=%r canonical=%r token_count=%s min_tokens=%s",
-            book_id,
-            candidate_id,
-            candidate.phrase_norm,
-            accepted_norm,
-            accepted_token_count,
-            config.phrase_min_tokens,
+            f"ebook_candidate_phrase_judgment_candidate_skip_short_canonical {book_id=} {candidate_id=} "
+            f"{candidate.phrase_norm=} {accepted_norm=} {accepted_token_count=} {config.phrase_min_tokens=}"
         )
         return False
     if is_most_common_word_phrase(accepted_tokens):
         logger.info(
-            "ebook_candidate_phrase_judgment_candidate_skip_common_canonical book_id=%s candidate_id=%s "
-            "phrase=%r canonical=%r",
-            book_id,
-            candidate_id,
-            candidate.phrase_norm,
-            accepted_norm,
+            f"ebook_candidate_phrase_judgment_candidate_skip_common_canonical {book_id=} {candidate_id=} "
+            f"{candidate.phrase_norm=} {accepted_norm=}"
         )
         return False
     return True
@@ -409,20 +380,7 @@ def build_judge_messages(candidate: PhraseCandidate) -> list[dict[str, str]]:
         "chapter_count": candidate.chapter_count,
         "contexts": candidate.sample_contexts,
     }
-    return [
-        {
-            "role": "system",
-            "content": (
-                "Judge whether a candidate phrase from a book should be protected for RAG retrieval. "
-                "Do not extract new phrases. Reject common grammar fragments, ordinary nonspecific phrases, "
-                "unstable fragments, and phrases kept only because they are frequent. Keep people, places, "
-                "organizations, factions, events, technologies, fictional conditions, magic systems, formal titles, "
-                "named concepts, and recurring world-specific terms. Return only a JSON object with keys: keep, "
-                "canonical, category, aliases, confidence, importance, allow_nested, suppress_children, reason."
-            ),
-        },
-        {"role": "user", "content": json.dumps(payload, ensure_ascii=True)},
-    ]
+    return load_prompt("phrase_judge").messages(candidate_json=json.dumps(payload, ensure_ascii=True))
 
 
 def parse_llm_judgment(content: str, config: EbookSearchConfig) -> LLMJudgment:
@@ -447,14 +405,14 @@ def parse_llm_judgment(content: str, config: EbookSearchConfig) -> LLMJudgment:
     if not isinstance(aliases, list | tuple):
         aliases = ()
     return LLMJudgment(
-        keep=bool(body.get("keep", False)),
+        keep=strict_bool(body.get("keep"), default=False),
         canonical=optional_text(body.get("canonical")),
         category=optional_text(body.get("category")),
         aliases=tuple(str(alias) for alias in aliases if isinstance(alias, str) and alias.strip()),
         confidence=clamped_float(body.get("confidence"), default=0.0),
         importance=clamped_float(body.get("importance"), default=0.5),
-        allow_nested=bool(body.get("allow_nested", config.phrase_default_allow_nested)),
-        suppress_children=bool(body.get("suppress_children", config.phrase_default_suppress_children)),
+        allow_nested=strict_bool(body.get("allow_nested"), default=config.phrase_default_allow_nested),
+        suppress_children=strict_bool(body.get("suppress_children"), default=config.phrase_default_suppress_children),
         reason=optional_text(body.get("reason")),
     )
 
@@ -471,14 +429,13 @@ def extract_json_object(content: str) -> str:
     Raises:
         ValueError: If no JSON object is found in the response.
     """
-    stripped = content.strip()
-    if stripped.startswith("{") and stripped.endswith("}"):
-        return stripped
-    match = JSON_OBJECT_RE.search(stripped)
-    if match is None:
-        msg = "LLM phrase judge response did not contain a JSON object"
-        raise ValueError(msg)
-    return match.group(0)
+    return content.strip()
+
+
+def alias_occurs_in_book(alias: str, normalized_book_text: str) -> bool:
+    """Return whether a normalized alias occurs as a complete phrase in the source book."""
+    alias_norm = normalize_text(alias)
+    return bool(alias_norm) and f" {alias_norm} " in f" {normalized_book_text} "
 
 
 def optional_text(value: object) -> str | None:
@@ -496,6 +453,11 @@ def optional_text(value: object) -> str | None:
     return stripped or None
 
 
+def strict_bool(value: object, *, default: bool) -> bool:
+    """Return a JSON boolean, falling back when the value has another type."""
+    return value if isinstance(value, bool) else default
+
+
 def clamped_float(value: object, *, default: float) -> float:
     """Coerce a JSON number into the 0.0 to 1.0 range.
 

python/ebook_search/protected_phrases/matching.py

@@ -6,12 +6,11 @@ import logging
 from collections import defaultdict
 from typing import TYPE_CHECKING
 
-from sqlalchemy import and_, delete, func, or_, select
+from sqlalchemy import and_, delete, func, or_, select, union
 
 from python.ebook_search.protected_phrases.config import get_ignored_phrases
 from python.ebook_search.protected_phrases.models import (
     ChunkPhraseHit,
-    HydratedPhraseMatch,
     PhraseLookup,
     PhraseMatch,
 )
@@ -29,11 +28,107 @@ if TYPE_CHECKING:
     from sqlalchemy.ext.asyncio import AsyncSession
 
     from python.ebook_search.config import EbookSearchConfig
-    from python.ebook_search.protected_phrases.text_normalization import NormalizedToken
 
 logger = logging.getLogger(__name__)
 
 
+async def detect_protected_phrases_for_query(
+    session: AsyncSession,
+    query_text: str,
+    config: EbookSearchConfig,
+) -> list[PhraseMatch]:
+    """Find query phrases with indexed exact matches on canonical and alias norms.
+
+    Args:
+        session (AsyncSession): Active database session.
+        query_text (str): User query text to detect phrases in.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        list[PhraseMatch]: Metadata-backed, overlap-resolved phrase matches for the query.
+    """
+    tokens_ = tokenize_with_offsets(query_text)
+    windows_by_norm: defaultdict[str, list[tuple[int, int]]] = defaultdict(list)
+    token_texts = [token.text for token in tokens_]
+    max_tokens = max(config.phrase_max_tokens, config.phrase_max_entity_tokens)
+    for phrase_norm, start, end in generate_query_ngrams(
+        token_texts,
+        min_n=config.phrase_min_tokens,
+        max_n=max_tokens,
+    ):
+        windows_by_norm[phrase_norm].append((start, end))
+    if not windows_by_norm:
+        return []
+
+    query_norms = tuple(windows_by_norm)
+    matched_norms = union(
+        select(
+            EbookProtectedPhrase.id.label("phrase_id"),
+            EbookProtectedPhrase.phrase_norm.label("matched_norm"),
+        ).where(EbookProtectedPhrase.phrase_norm.in_(query_norms)),
+        select(
+            EbookPhraseAlias.phrase_id.label("phrase_id"),
+            EbookPhraseAlias.alias_norm.label("matched_norm"),
+        ).where(EbookPhraseAlias.alias_norm.in_(query_norms)),
+    ).subquery()
+    statement = select(EbookProtectedPhrase, matched_norms.c.matched_norm).join(
+        matched_norms,
+        matched_norms.c.phrase_id == EbookProtectedPhrase.id,
+    )
+
+    matches: list[PhraseMatch] = []
+    for phrase, matched_norm in await session.execute(statement):
+        for start, end in windows_by_norm[matched_norm]:
+            matches.append(
+                PhraseMatch(
+                    phrase_id=phrase.id,
+                    matched_norm=matched_norm,
+                    phrase_text=phrase.phrase_text,
+                    phrase_norm=phrase.phrase_norm,
+                    canonical_id=phrase.canonical_id,
+                    phrase_type=phrase.phrase_type,
+                    token_count=end - start,
+                    confidence=phrase.confidence,
+                    importance=phrase.importance,
+                    allow_nested=phrase.allow_nested,
+                    suppress_children=phrase.suppress_children,
+                    start_token=start,
+                    end_token=end,
+                    start_char=tokens_[start].start_char,
+                    end_char=tokens_[end - 1].end_char,
+                    book_id=phrase.book_id,
+                    series_id=phrase.series_id,
+                )
+            )
+    return resolve_overlaps(matches)
+
+
+async def index_chunk_phrase_mentions_for_book(
+    session: AsyncSession,
+    book_id: int,
+    config: EbookSearchConfig,
+) -> int:
+    """Rebuild chunk phrase mentions for all chunks in one book.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose chunk mentions are rebuilt.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        int: Total number of chunk phrase mentions indexed for the book.
+    """
+    lookup = await load_phrase_lookup(session, config, book_id=book_id)
+    await session.execute(delete(EbookChunkPhraseMention).where(EbookChunkPhraseMention.book_id == book_id))
+    chunks = await session.scalars(select(EbookChunk).where(EbookChunk.source_id == book_id).order_by(EbookChunk.id))
+    count = 0
+    for chunk in chunks:
+        count += index_chunk_phrase_mentions(session, chunk, lookup=lookup)
+    await session.flush()
+    logger.info(f"ebook_chunk_phrase_mentions_indexed {book_id=} {count=}")
+    return count
+
+
 async def load_phrase_lookup(
     session: AsyncSession,
     config: EbookSearchConfig,
@@ -52,40 +147,29 @@ async def load_phrase_lookup(
     Returns:
         PhraseLookup: Normalized phrase and alias maps with the token-window bounds to test.
     """
-    norm_to_ids: defaultdict[str, list[int]] = defaultdict(list)
-    alias_to_ids: defaultdict[str, list[int]] = defaultdict(list)
+    phrase_ids_by_norm: defaultdict[str, set[int]] = defaultdict(set)
+    phrases_by_id: dict[int, EbookProtectedPhrase] = {}
     max_tokens = config.phrase_max_tokens
 
-    phrase_statement = select(
-        EbookProtectedPhrase.id,
-        EbookProtectedPhrase.phrase_norm,
-        EbookProtectedPhrase.token_count,
-    )
+    statement = select(
+        EbookProtectedPhrase,
+        EbookPhraseAlias.alias_norm,
+    ).outerjoin(EbookPhraseAlias, EbookPhraseAlias.phrase_id == EbookProtectedPhrase.id)
     scope_filter = protected_phrase_scope_filter(book_id=book_id, series_id=series_id)
     if scope_filter is not None:
-        phrase_statement = phrase_statement.where(scope_filter)
+        statement = statement.where(scope_filter)
 
-    for row in await session.execute(phrase_statement):
-        phrase_id = int(row.id)
-        phrase_norm = str(row.phrase_norm)
-        norm_to_ids[phrase_norm].append(phrase_id)
-        max_tokens = max(max_tokens, int(row.token_count))
-
-    alias_statement = select(
-        EbookPhraseAlias.alias_norm,
-        EbookPhraseAlias.phrase_id,
-    ).join(EbookProtectedPhrase, EbookProtectedPhrase.id == EbookPhraseAlias.phrase_id)
-    if scope_filter is not None:
-        alias_statement = alias_statement.where(scope_filter)
-
-    for row in await session.execute(alias_statement):
-        alias_norm = str(row.alias_norm)
-        alias_to_ids[alias_norm].append(int(row.phrase_id))
-        max_tokens = max(max_tokens, len(alias_norm.split()))
+    for phrase, alias_norm in await session.execute(statement):
+        phrases_by_id[phrase.id] = phrase
+        phrase_ids_by_norm[phrase.phrase_norm].add(phrase.id)
+        max_tokens = max(max_tokens, phrase.token_count)
+        if alias_norm is not None:
+            phrase_ids_by_norm[alias_norm].add(phrase.id)
+            max_tokens = max(max_tokens, len(alias_norm.split()))
 
     return PhraseLookup(
-        norm_to_phrase_ids={key: tuple(values) for key, values in norm_to_ids.items()},
-        alias_to_phrase_ids={key: tuple(values) for key, values in alias_to_ids.items()},
+        phrase_ids_by_norm={key: tuple(sorted(values)) for key, values in phrase_ids_by_norm.items()},
+        phrases_by_id=phrases_by_id,
         min_tokens=config.phrase_min_tokens,
         max_tokens=max_tokens,
     )
@@ -111,6 +195,144 @@ def protected_phrase_scope_filter(*, book_id: int | None, series_id: int | None)
     return and_(*conditions)
 
 
+def is_inside(child: PhraseMatch, parent: PhraseMatch) -> bool:
+    """Return whether one token span is strictly inside another.
+
+    Args:
+        child (PhraseMatch): Candidate nested match.
+        parent (PhraseMatch): Candidate enclosing match.
+
+    Returns:
+        bool: True when ``child`` lies within ``parent`` and is not the same span.
+    """
+    return (
+        child.start_token >= parent.start_token
+        and child.end_token <= parent.end_token
+        and (child.start_token, child.end_token, child.phrase_id)
+        != (parent.start_token, parent.end_token, parent.phrase_id)
+    )
+
+
+def index_chunk_phrase_mentions(session: AsyncSession, chunk: EbookChunk, *, lookup: PhraseLookup) -> int:
+    """Store protected phrase mentions for one chunk.
+
+    Args:
+        session (AsyncSession): Active database session.
+        chunk (EbookChunk): Chunk whose text is scanned for phrase mentions.
+        lookup (PhraseLookup): In-memory phrase and alias lookup maps.
+
+    Returns:
+        int: Number of phrase mentions stored for the chunk.
+    """
+    tokens_ = tokenize_with_offsets(chunk.text)
+    token_texts = [token.text for token in tokens_]
+    raw_matches: list[PhraseMatch] = []
+    phrase_windows = generate_query_ngrams(token_texts, min_n=lookup.min_tokens, max_n=lookup.max_tokens)
+    for matched_norm, start, end in phrase_windows:
+        for phrase_id in lookup.phrase_ids_by_norm.get(matched_norm, ()):
+            phrase = lookup.phrases_by_id[phrase_id]
+            raw_matches.append(
+                PhraseMatch(
+                    phrase_id=phrase_id,
+                    matched_norm=matched_norm,
+                    phrase_text=phrase.phrase_text,
+                    phrase_norm=phrase.phrase_norm,
+                    canonical_id=phrase.canonical_id,
+                    phrase_type=phrase.phrase_type,
+                    confidence=phrase.confidence,
+                    importance=phrase.importance,
+                    allow_nested=phrase.allow_nested,
+                    suppress_children=phrase.suppress_children,
+                    start_token=start,
+                    end_token=end,
+                    token_count=end - start,
+                    start_char=tokens_[start].start_char,
+                    end_char=tokens_[end - 1].end_char,
+                    book_id=phrase.book_id,
+                    series_id=phrase.series_id,
+                )
+            )
+
+    matches = resolve_overlaps(raw_matches)
+    for match in matches:
+        session.add(
+            EbookChunkPhraseMention(
+                chunk_id=chunk.id,
+                phrase_id=match.phrase_id,
+                book_id=match.book_id if match.book_id is not None else chunk.source_id,
+                series_id=match.series_id,
+                start_char=match.start_char if match.start_char is not None else 0,
+                end_char=match.end_char,
+            )
+        )
+    return len(matches)
+
+
+def resolve_overlaps(matches: Sequence[PhraseMatch]) -> list[PhraseMatch]:
+    """Resolve overlapping phrase matches without relying only on longest match.
+
+    Args:
+        matches (Sequence[PhraseMatch]): Metadata-backed matches that may overlap.
+
+    Returns:
+        list[PhraseMatch]: The kept, non-suppressed matches.
+    """
+    sorted_matches = sorted(
+        matches,
+        key=lambda match: (match.start_token, -match.token_count, -match.importance, -match.confidence),
+    )
+    kept: list[PhraseMatch] = []
+    for candidate in sorted_matches:
+        if any(should_suppress(candidate, existing) for existing in kept):
+            continue
+        kept.append(candidate)
+    return kept
+
+
+def should_suppress(candidate: PhraseMatch, kept: PhraseMatch) -> bool:
+    """Return whether an already-kept match should suppress a candidate.
+
+    Args:
+        candidate (PhraseMatch): Match being considered for keeping.
+        kept (PhraseMatch): Match already kept that may suppress the candidate.
+
+    Returns:
+        bool: True when the candidate should be dropped in favor of the kept match.
+    """
+    if not overlaps(candidate, kept):
+        return False
+    if candidate.canonical_id == kept.canonical_id:
+        return rank_match(kept) >= rank_match(candidate)
+    if is_inside(candidate, kept) and kept.suppress_children and not candidate.allow_nested:
+        return True
+    return not candidate.allow_nested and rank_match(kept) > rank_match(candidate)
+
+
+def overlaps(first: PhraseMatch, second: PhraseMatch) -> bool:
+    """Return whether two token spans overlap.
+
+    Args:
+        first (PhraseMatch): First match to compare.
+        second (PhraseMatch): Second match to compare.
+
+    Returns:
+        bool: True when the two token spans share at least one token position.
+    """
+    return not (first.end_token <= second.start_token or first.start_token >= second.end_token)
+
+
+def rank_match(match: PhraseMatch) -> tuple[float, float, int]:
+    """Rank phrase matches by importance, confidence, then token count.
+
+    Args:
+        match (PhraseMatch): Match to build a sort key for.
+
+    Returns:
+        tuple[float, float, int]: A comparable key of importance, confidence, and token count.
+    """
+    return (match.importance, match.confidence, match.token_count)
+
+
 def generate_query_ngrams(
     tokens_: Sequence[str],
     min_n: int,
@@ -136,301 +358,13 @@ def generate_query_ngrams(
             yield phrase_norm, start, end
 
 
-def detect_phrase_candidates(query_text: str, lookup: PhraseLookup) -> list[PhraseMatch]:
-    """Detect protected phrase windows in a user query using RAM hash lookups.
-
-    Args:
-        query_text (str): User query text to scan.
-        lookup (PhraseLookup): In-memory phrase and alias lookup maps.
-
-    Returns:
-        list[PhraseMatch]: Unhydrated phrase matches found in the query.
-    """
-    return detect_phrase_candidates_from_tokens(tokenize_with_offsets(query_text), lookup)
-
-
-def detect_phrase_candidates_in_text(text: str, lookup: PhraseLookup) -> list[PhraseMatch]:
-    """Detect protected phrase windows in arbitrary text with character offsets.
-
-    Args:
-        text (str): Arbitrary text, such as a chunk, to scan.
-        lookup (PhraseLookup): In-memory phrase and alias lookup maps.
-
-    Returns:
-        list[PhraseMatch]: Unhydrated phrase matches found in the text.
-    """
-    return detect_phrase_candidates_from_tokens(tokenize_with_offsets(text), lookup)
-
-
-def detect_phrase_candidates_from_tokens(tokens_: Sequence[NormalizedToken], lookup: PhraseLookup) -> list[PhraseMatch]:
-    """Detect protected phrase windows from already-normalized tokens.
-
-    Args:
-        tokens_ (Sequence[NormalizedToken]): Normalized tokens with character offsets.
-        lookup (PhraseLookup): In-memory phrase and alias lookup maps.
-
-    Returns:
-        list[PhraseMatch]: Deduplicated unhydrated phrase matches with token and character spans.
-    """
-    matches: list[PhraseMatch] = []
-    seen: set[tuple[int | None, str, int, int]] = set()
-    token_texts = [token.text for token in tokens_]
-    for phrase_norm, start, end in generate_query_ngrams(token_texts, min_n=lookup.min_tokens, max_n=lookup.max_tokens):
-        phrase_ids = lookup.norm_to_phrase_ids.get(phrase_norm, ())
-        alias_ids = lookup.alias_to_phrase_ids.get(phrase_norm, ())
-        for phrase_id in (*phrase_ids, *alias_ids):
-            key = (phrase_id, phrase_norm, start, end)
-            if key in seen:
-                continue
-            seen.add(key)
-            matches.append(
-                PhraseMatch(
-                    phrase_norm=phrase_norm,
-                    phrase_id=phrase_id,
-                    start_token=start,
-                    end_token=end,
-                    token_count=end - start,
-                    start_char=tokens_[start].start_char,
-                    end_char=tokens_[end - 1].end_char,
-                )
-            )
-    return matches
-
-
-async def hydrate_matches(session: AsyncSession, matches: Sequence[PhraseMatch]) -> list[HydratedPhraseMatch]:
-    """Fetch protected phrase metadata for raw phrase matches.
-
-    Args:
-        session (AsyncSession): Active database session.
-        matches (Sequence[PhraseMatch]): Unhydrated matches to enrich.
-
-    Returns:
-        list[HydratedPhraseMatch]: Matches with protected-phrase metadata attached.
-    """
-    if not matches:
-        return []
-
-    phrase_ids = sorted({match.phrase_id for match in matches if match.phrase_id is not None})
-    if not phrase_ids:
-        return []
-
-    rows = {
-        row.id: row
-        for row in await session.scalars(select(EbookProtectedPhrase).where(EbookProtectedPhrase.id.in_(phrase_ids)))
-    }
-    hydrated: list[HydratedPhraseMatch] = []
-    for match in matches:
-        if match.phrase_id is None:
-            continue
-        phrase = rows.get(match.phrase_id)
-        if phrase is None:
-            continue
-        hydrated.append(
-            HydratedPhraseMatch(
-                phrase_id=phrase.id,
-                matched_norm=match.phrase_norm,
-                phrase_text=phrase.phrase_text,
-                phrase_norm=phrase.phrase_norm,
-                canonical_id=phrase.canonical_id,
-                phrase_type=phrase.phrase_type,
-                token_count=match.token_count,
-                confidence=phrase.confidence,
-                importance=phrase.importance,
-                allow_nested=phrase.allow_nested,
-                suppress_children=phrase.suppress_children,
-                start_token=match.start_token,
-                end_token=match.end_token,
-                start_char=match.start_char,
-                end_char=match.end_char,
-                book_id=phrase.book_id,
-                series_id=phrase.series_id,
-            )
-        )
-    return hydrated
-
-
-def overlaps(first: HydratedPhraseMatch, second: HydratedPhraseMatch) -> bool:
-    """Return whether two token spans overlap.
-
-    Args:
-        first (HydratedPhraseMatch): First match to compare.
-        second (HydratedPhraseMatch): Second match to compare.
-
-    Returns:
-        bool: True when the two token spans share at least one token position.
-    """
-    return not (first.end_token <= second.start_token or first.start_token >= second.end_token)
-
-
-def is_inside(child: HydratedPhraseMatch, parent: HydratedPhraseMatch) -> bool:
-    """Return whether one token span is strictly inside another.
-
-    Args:
-        child (HydratedPhraseMatch): Candidate nested match.
-        parent (HydratedPhraseMatch): Candidate enclosing match.
-
-    Returns:
-        bool: True when ``child`` lies within ``parent`` and is not the same span.
-    """
-    return (
-        child.start_token >= parent.start_token
-        and child.end_token <= parent.end_token
-        and (child.start_token, child.end_token, child.phrase_id)
-        != (parent.start_token, parent.end_token, parent.phrase_id)
-    )
-
-
-def rank_match(match: HydratedPhraseMatch) -> tuple[float, float, int]:
-    """Rank phrase matches by importance, confidence, then token count.
-
-    Args:
-        match (HydratedPhraseMatch): Match to build a sort key for.
-
-    Returns:
-        tuple[float, float, int]: A comparable key of importance, confidence, and token count.
-    """
-    return (match.importance, match.confidence, match.token_count)
-
-
-def should_suppress(candidate: HydratedPhraseMatch, kept: HydratedPhraseMatch) -> bool:
-    """Return whether an already-kept match should suppress a candidate.
-
-    Args:
-        candidate (HydratedPhraseMatch): Match being considered for keeping.
-        kept (HydratedPhraseMatch): Match already kept that may suppress the candidate.
-
-    Returns:
-        bool: True when the candidate should be dropped in favor of the kept match.
-    """
-    if not overlaps(candidate, kept):
-        return False
-    if candidate.canonical_id == kept.canonical_id:
-        return rank_match(kept) >= rank_match(candidate)
-    if is_inside(candidate, kept) and kept.suppress_children and not candidate.allow_nested:
-        return True
-    return not candidate.allow_nested and rank_match(kept) > rank_match(candidate)
-
-
-def resolve_overlaps(matches: Sequence[HydratedPhraseMatch]) -> list[HydratedPhraseMatch]:
-    """Resolve overlapping phrase matches without relying only on longest match.
-
-    Args:
-        matches (Sequence[HydratedPhraseMatch]): Hydrated matches that may overlap.
-
-    Returns:
-        list[HydratedPhraseMatch]: The kept, non-suppressed matches.
-    """
-    sorted_matches = sorted(
-        matches,
-        key=lambda match: (match.start_token, -match.token_count, -match.importance, -match.confidence),
-    )
-    kept: list[HydratedPhraseMatch] = []
-    for candidate in sorted_matches:
-        if any(should_suppress(candidate, existing) for existing in kept):
-            continue
-        kept.append(candidate)
-    return kept
-
-
-async def detect_protected_phrases_for_query(
-    session: AsyncSession,
-    query_text: str,
-    config: EbookSearchConfig,
-    *,
-    lookup: PhraseLookup | None = None,
-    book_id: int | None = None,
-    series_id: int | None = None,
-) -> list[HydratedPhraseMatch]:
-    """Run the full online protected-phrase query-detection pipeline.
-
-    Args:
-        session (AsyncSession): Active database session.
-        query_text (str): User query text to detect phrases in.
-        config (EbookSearchConfig): Runtime phrase-tuning settings.
-        lookup (PhraseLookup | None): Optional preloaded lookup; loaded on demand when ``None``.
-        book_id (int | None): Optional book scope for lookup loading.
-        series_id (int | None): Optional series scope for lookup loading.
-
-    Returns:
-        list[HydratedPhraseMatch]: Hydrated, overlap-resolved phrase matches for the query.
-    """
-    active_lookup = (
-        lookup
-        if lookup is not None
-        else await load_phrase_lookup(session, config, book_id=book_id, series_id=series_id)
-    )
-    return resolve_overlaps(await hydrate_matches(session, detect_phrase_candidates(query_text, active_lookup)))
-
-
-async def index_chunk_phrase_mentions_for_book(
-    session: AsyncSession,
-    book_id: int,
-    config: EbookSearchConfig,
-    *,
-    series_id: int | None = None,
-    lookup: PhraseLookup | None = None,
-) -> int:
-    """Rebuild chunk phrase mentions for all chunks in one book.
-
-    Args:
-        session (AsyncSession): Active database session.
-        book_id (int): Book whose chunk mentions are rebuilt.
-        config (EbookSearchConfig): Runtime phrase-tuning settings.
-        series_id (int | None): Optional series scope for lookup loading.
-        lookup (PhraseLookup | None): Optional preloaded lookup; loaded on demand when ``None``.
-
-    Returns:
-        int: Total number of chunk phrase mentions indexed for the book.
-    """
-    active_lookup = (
-        lookup
-        if lookup is not None
-        else await load_phrase_lookup(session, config, book_id=book_id, series_id=series_id)
-    )
-    await session.execute(delete(EbookChunkPhraseMention).where(EbookChunkPhraseMention.book_id == book_id))
-    chunks = await session.scalars(select(EbookChunk).where(EbookChunk.source_id == book_id).order_by(EbookChunk.id))
-    count = 0
-    for chunk in chunks:
-        count += await index_chunk_phrase_mentions(session, chunk, lookup=active_lookup)
-    await session.flush()
-    logger.info("ebook_chunk_phrase_mentions_indexed book_id=%s mentions=%s", book_id, count)
-    return count
-
-
-async def index_chunk_phrase_mentions(session: AsyncSession, chunk: EbookChunk, *, lookup: PhraseLookup) -> int:
-    """Store protected phrase mentions for one chunk.
-
-    Args:
-        session (AsyncSession): Active database session.
-        chunk (EbookChunk): Chunk whose text is scanned for phrase mentions.
-        lookup (PhraseLookup): In-memory phrase and alias lookup maps.
-
-    Returns:
-        int: Number of phrase mentions stored for the chunk.
-    """
-    raw_matches = detect_phrase_candidates_in_text(chunk.text, lookup)
-    hydrated = resolve_overlaps(await hydrate_matches(session, raw_matches))
-    for match in hydrated:
-        session.add(
-            EbookChunkPhraseMention(
-                chunk_id=chunk.id,
-                phrase_id=match.phrase_id,
-                book_id=match.book_id if match.book_id is not None else chunk.source_id,
-                series_id=match.series_id,
-                start_char=match.start_char if match.start_char is not None else 0,
-                end_char=match.end_char,
-            )
-        )
-    return len(hydrated)
-
-
 async def phrase_hits_for_chunks(
     session: AsyncSession,
     *,
     chunk_ids: Sequence[int],
     phrase_ids: Sequence[int],
 ) -> dict[int, tuple[ChunkPhraseHit, ...]]:
-    """Return matched protected phrases with mention counts by chunk id using indexed chunk mentions.
+    """Return matched protected phrases with mention counts by chunk id.
 
     Args:
         session (AsyncSession): Active database session.
@@ -461,31 +395,11 @@ async def phrase_hits_for_chunks(
     )
     hits: defaultdict[int, list[ChunkPhraseHit]] = defaultdict(list)
     for row in await session.execute(statement):
-        hits[int(row.chunk_id)].append(
+        hits[row.chunk_id].append(
             ChunkPhraseHit(
-                phrase_id=int(row.phrase_id),
-                phrase_text=str(row.phrase_text),
-                mention_count=int(row.mention_count),
+                phrase_id=row.phrase_id,
+                phrase_text=row.phrase_text,
+                mention_count=row.mention_count,
             )
         )
     return {chunk_id: tuple(chunk_hits) for chunk_id, chunk_hits in hits.items()}
-
-
-async def phrase_hit_counts_for_chunks(
-    session: AsyncSession,
-    *,
-    chunk_ids: Sequence[int],
-    phrase_ids: Sequence[int],
-) -> dict[int, int]:
-    """Return phrase-hit counts by chunk id using indexed chunk mentions.
-
-    Args:
-        session (AsyncSession): Active database session.
-        chunk_ids (Sequence[int]): Chunk ids to count mentions for.
-        phrase_ids (Sequence[int]): Protected phrase ids to restrict the counts to.
-
-    Returns:
-        dict[int, int]: Total mention count per chunk id.
-    """
-    hits = await phrase_hits_for_chunks(session, chunk_ids=chunk_ids, phrase_ids=phrase_ids)
-    return {chunk_id: sum(hit.mention_count for hit in chunk_hits) for chunk_id, chunk_hits in hits.items()}

python/ebook_search/protected_phrases/models.py

@@ -8,6 +8,8 @@ from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from collections.abc import Mapping
 
+    from python.orm.richie import EbookProtectedPhrase
+
 
 @dataclass(slots=True)
 class PhraseCandidate:
@@ -19,11 +21,8 @@ class PhraseCandidate:
         token_count (int): Number of normalized tokens in the phrase.
         source_raw_ngram (bool): Whether the raw n-gram extractor produced the phrase.
         source_yake (bool): Whether YAKE keyword extraction produced the phrase.
-        source_spacy_ner (bool): Whether spaCy named-entity recognition produced the phrase.
-        source_spacy_noun_chunk (bool): Whether spaCy noun chunking produced the phrase.
         source_capitalized (bool): Whether the capitalized-run extractor produced the phrase.
         source_metadata (bool): Whether book metadata produced the phrase.
-        spacy_label (str | None): spaCy entity label when NER produced the phrase.
         raw_count (int): Occurrences counted across the book text.
         chapter_count (int): Number of chapters containing the phrase.
         yake_score (float | None): Raw YAKE score when available; lower is better.
@@ -36,11 +35,8 @@ class PhraseCandidate:
     token_count: int
     source_raw_ngram: bool = False
     source_yake: bool = False
-    source_spacy_ner: bool = False
-    source_spacy_noun_chunk: bool = False
     source_capitalized: bool = False
     source_metadata: bool = False
-    spacy_label: str | None = None
     raw_count: int = 0
     chapter_count: int = 0
     yake_score: float | None = None
@@ -77,47 +73,24 @@ class LLMJudgment:
 
 @dataclass(frozen=True, slots=True)
 class PhraseLookup:
-    """In-memory lookup maps used for constant-time phrase-window checks.
+    """In-memory phrase metadata used for constant-time text-window checks.
 
     Attributes:
-        norm_to_phrase_ids (Mapping[str, tuple[int, ...]]): Normalized phrase to protected phrase ids.
-        alias_to_phrase_ids (Mapping[str, tuple[int, ...]]): Normalized alias to protected phrase ids.
+        phrase_ids_by_norm (Mapping[str, tuple[int, ...]]): Canonical and alias norms to phrase ids.
+        phrases_by_id (Mapping[int, EbookProtectedPhrase]): Protected phrase metadata by id.
         min_tokens (int): Smallest token-window size to test.
         max_tokens (int): Largest token-window size to test.
     """
 
-    norm_to_phrase_ids: Mapping[str, tuple[int, ...]]
-    alias_to_phrase_ids: Mapping[str, tuple[int, ...]]
+    phrase_ids_by_norm: Mapping[str, tuple[int, ...]]
+    phrases_by_id: Mapping[int, EbookProtectedPhrase]
     min_tokens: int
     max_tokens: int
 
 
 @dataclass(frozen=True, slots=True)
 class PhraseMatch:
-    """An unhydrated query or chunk phrase match.
-
-    Attributes:
-        phrase_norm (str): Normalized text of the matched window.
-        start_token (int): Index of the first matched token.
-        end_token (int): Index one past the last matched token.
-        token_count (int): Number of tokens in the match.
-        phrase_id (int | None): Matched protected phrase id when known.
-        start_char (int | None): Start character offset in the source text.
-        end_char (int | None): End character offset in the source text.
-    """
-
-    phrase_norm: str
-    start_token: int
-    end_token: int
-    token_count: int
-    phrase_id: int | None = None
-    start_char: int | None = None
-    end_char: int | None = None
-
-
-@dataclass(frozen=True, slots=True)
-class HydratedPhraseMatch:
-    """A phrase match with protected-phrase metadata attached.
+    """A detected phrase match with protected-phrase metadata attached.
 
     Attributes:
         phrase_id (int): Protected phrase id.
@@ -164,8 +137,8 @@ class ChunkPhraseHit:
 
     Attributes:
         phrase_id (int): Protected phrase id.
-        phrase_text (str): Display text of the protected phrase.
-        mention_count (int): Indexed mentions of the phrase in the chunk.
+        phrase_text (str): Display text for the phrase.
+        mention_count (int): Indexed mentions inside the chunk.
     """
 
     phrase_id: int