feat(richie user env): adding app_image_path configuration

Add test_protected_phrases.py covering phrase-matching behavior in the
RAG engine, and update the existing ebook_search tests to use the async
SQLAlchemy engine/session (create_async_engine, AsyncSession) and async
HTTP paths.

.dockerignore

@@ -0,0 +1,15 @@
+.git
+.direnv
+.mypy_cache
+.pytest_cache
+.ruff_cache
+__pycache__
+**/__pycache__
+*.pyc
+*.pyo
+.ebook_search_bm25
+result
+result-*
+*.egg-info
+dist
+build

.vscode/settings.json

@@ -71,6 +71,7 @@
     "ehci",
     "emerg",
     "endlessh",
+    "ents",
     "errorlens",
     "esbenp",
     "esphome",
@@ -172,6 +173,8 @@
     "Networkd",
     "networkmanager",
     "newtabpage",
+    "ngram",
+    "ngrams",
     "nixfmt",
     "nixos",
     "nixpkgs",
@@ -298,7 +301,9 @@
     "uiprotect",
     "uitour",
     "unifi",
+    "unjudged",
     "unrar",
+    "unstorable",
     "unsubmitted",
     "uptimekuma",
     "urlbar",
@@ -326,6 +331,7 @@
     "xcursorgen",
     "xdist",
     "xhci",
+    "yake",
     "yazi",
     "yubikey",
     "yubioath",

README.md

@@ -1 +1,51 @@
 # dotfiles
+
+## Installer ISO
+
+Build a bootable NixOS ISO with the installer preinstalled:
+
+```sh
+nix build .#iso
+```
+
+Write `result/iso/nixos-zfs-installer.iso` to a USB stick (for example with `dd`) or boot it in a VM. The image is the minimal NixOS installation CD with ZFS enabled and `nixos-installer` on `PATH`. SSH is enabled and the `nixos` and `root` accounts use the password `nixos`, so you can also run the installer remotely. Once booted:
+
+```sh
+sudo nixos-installer
+```
+
+The ISO bundles the `.#installer-nixos` package, a variant of the binary that keeps its Nix store linkage instead of being patched for foreign distributions.
+
+## Installer binary
+
+Build the self-contained installer executable with:
+
+```sh
+nix build .#installer
+```
+
+The flake package (defined in `python/installer/package.nix`) uses the Python builder in `python/installer/build.py`, which stages only the installer modules before running PyInstaller. You can also call it directly when `pyinstaller` and `patchelf` are on `PATH`:
+
+```sh
+python -m python.installer.build --output ./nixos-installer
+```
+
+Copy `result/bin/nixos-installer` to the installer USB stick and run it as root from the NixOS live environment:
+
+```sh
+sudo ./nixos-installer
+```
+
+Validate the live environment first with:
+
+```sh
+./nixos-installer --check
+```
+
+Paste a value into the TUI encryption password field to enable LUKS during install, or set `ENCRYPT_KEY`:
+
+```sh
+sudo env ENCRYPT_KEY='change-me' ./nixos-installer
+```
+
+The binary bundles the Python runtime and only the installer modules it imports. It still expects the NixOS installer environment to provide system install tools such as `parted`, `zfs`, `zpool`, `cryptsetup`, `nixos-generate-config`, and `nixos-install`.

flake.lock

@@ -8,11 +8,11 @@
       },
       "locked": {
         "dir": "pkgs/firefox-addons",
-        "lastModified": 1781928171,
+        "lastModified": 1782964936,
-        "narHash": "sha256-2IIxdVe7afJ8HyTeR/MD9Qw5iIJ68o1iRYkiXn7LDag=",
+        "narHash": "sha256-wXEBDr7/dFQYhVpDwCKc9fkrYQQE4x0bdirX1bsLBGA=",
         "owner": "rycee",
         "repo": "nur-expressions",
-        "rev": "1777920f3688105a47b95b229dcefc85ae1bb42e",
+        "rev": "64feee871e0373dd6121e412c3fb12e372d1bfb5",
         "type": "gitlab"
       },
       "original": {
@@ -29,11 +29,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1781989573,
+        "lastModified": 1783005591,
-        "narHash": "sha256-npfH7Zv7t1akX/ArqCNro4zU4ViPlghLaPnbEfHbCxk=",
+        "narHash": "sha256-NcLHV5uBAeggDUE2wPbKszjfyaSLsoqaYt7izOphkZw=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "78e7d8b13ecd7f5256a5c11ce216876164099d9f",
+        "rev": "f469c79b955609d6a8fdd9e689be76a93b1621d7",
         "type": "github"
       },
       "original": {
@@ -47,11 +47,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1781622756,
+        "lastModified": 1782562157,
-        "narHash": "sha256-JrPh4M6S7aPsEE9tOENuZrxC6o2szSLlK+t4+nLke9s=",
+        "narHash": "sha256-a7+T6QSeowynwZ1ZJJbP8T8ntAytvrui8kFGJmIZt2c=",
         "owner": "nixos",
         "repo": "nixos-hardware",
-        "rev": "08018c72174a4df5657f8d94178ac69fb9c243e5",
+        "rev": "a9cf7546a938c737b079e738de73934a13de9784",
         "type": "github"
       },
       "original": {
@@ -76,11 +76,11 @@
     },
     "nixpkgs-master": {
       "locked": {
-        "lastModified": 1782009160,
+        "lastModified": 1783021952,
-        "narHash": "sha256-BRto9JC5S8uaapULr/n+tLbZf896K4kbtVZ0PnFra0U=",
+        "narHash": "sha256-8PghAtSGGZ0umfVI8Qbd7ZbFrfZPiH1UwtVbgLeikDA=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "a7ef34422d8e3cd1f9ffaec949735adb983d4ffc",
+        "rev": "f136374c679c54171a3ace589d15e9e79a8bd086",
         "type": "github"
       },
       "original": {
@@ -108,11 +108,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1781577229,
+        "lastModified": 1782723713,
-        "narHash": "sha256-lrp67w8AulE9Ks53n27I45ADSzbOCn4H+CNW1Ck8B+8=",
+        "narHash": "sha256-oPXCU/SSUokcGaJREHibG1CBX3+s/W7orDWQOZDsEeQ=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "567a49d1913ce81ac6e9582e3553dd90a955875f",
+        "rev": "b5aa0fbd538984f6e3d201be0005b4463d8b09f8",
         "type": "github"
       },
       "original": {
@@ -141,11 +141,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1781943681,
+        "lastModified": 1782165805,
-        "narHash": "sha256-NFHmA7H47adqiyp+0iEOyZOQhmigDqA/NBAlf4imB6U=",
+        "narHash": "sha256-478kKQBvK6SYTOdN2h9jhKJv94nbXRbFMfuL1WshErg=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "420f8d2e9882911f65cfac15cc706f639ba96cca",
+        "rev": "56b24064fdcaedca53553b1a6d607fd23b613a24",
         "type": "github"
       },
       "original": {

flake.nix

@@ -65,32 +65,48 @@
 
       devShells = forEachSystem (pkgs: import ./shell.nix { inherit pkgs; });
       formatter = forEachSystem (pkgs: pkgs.treefmt);
+      packages = forEachSystem (
+        pkgs:
+        let
+          installer = pkgs.callPackage ./python/installer/package.nix { };
+          installer-nixos = pkgs.callPackage ./python/installer/package.nix { patchElf = false; };
+        in
+        {
+          inherit installer installer-nixos;
+          default = installer;
+        }
+        // lib.optionalAttrs (pkgs.stdenv.hostPlatform.system == "x86_64-linux") {
+          iso = self.nixosConfigurations.iso.config.system.build.isoImage;
+        }
+      );
+      apps = forEachSystem (
+        pkgs:
+        let
+          system = pkgs.stdenv.hostPlatform.system;
+          installer = {
+            type = "app";
+            program = "${self.packages.${system}.installer}/bin/nixos-installer";
+            meta.description = "One-file NixOS ZFS installer.";
+          };
+        in
+        {
+          inherit installer;
+          default = installer;
+        }
+      );
 
-      nixosConfigurations = {
+      nixosConfigurations =
-        bob = lib.nixosSystem {
+        let
-          modules = [
+          hosts = builtins.attrNames (
-            ./systems/bob
+            lib.filterAttrs (_: type: type == "directory") (builtins.readDir ./systems)
-          ];
+          );
-          specialArgs = { inherit inputs outputs; };
+          mkHost =
-        };
+            name:
-        brain = lib.nixosSystem {
+            lib.nixosSystem {
-          modules = [
+              modules = [ ./systems/${name} ];
-            ./systems/brain
+              specialArgs = { inherit inputs outputs; };
-          ];
+            };
-          specialArgs = { inherit inputs outputs; };
+        in
-        };
+        lib.genAttrs hosts mkHost;
-        jeeves = lib.nixosSystem {
-          modules = [
-            ./systems/jeeves
-          ];
-          specialArgs = { inherit inputs outputs; };
-        };
-        rhapsody-in-green = lib.nixosSystem {
-          modules = [
-            ./systems/rhapsody-in-green
-          ];
-          specialArgs = { inherit inputs outputs; };
-        };
-      };
     };
 }

overlays/default.nix

@@ -18,44 +18,18 @@
   python-env = final: _prev: {
     my_python = final.python314.withPackages (
       ps:
-      let
-        bm25s = ps.buildPythonPackage rec {
-          pname = "bm25s";
-          version = "0.3.9";
-          pyproject = true;
-
-          src = final.fetchPypi {
-            inherit pname version;
-            hash = "sha256-iVxnnZUrfeg1XttfPhpiCh4vKU0dQrkZvwghzOLi9Zc=";
-          };
-
-          build-system = [ ps.setuptools ];
-          dependencies = with ps; [
-            numpy
-            scipy
-          ];
-
-          pythonImportsCheck = [ "bm25s" ];
-        };
-      in
       with ps;
       [
         alembic
         apprise
         apscheduler
-        beautifulsoup4
-        ebooklib
         fastapi
         fastapi-cli
         httpx
         mypy
-        numpy
-        orjson
         pgvector
-        polars
         psycopg
         pydantic
-        pydantic-settings
         pyfakefs
         pytest
         pytest-cov
@@ -63,12 +37,8 @@
         pytest-xdist
         python-multipart
         ruff
-        scalene
         sqlalchemy
-        bm25s
         tenacity
-        textual
-        tiktoken
         tinytuya
         typer
         websockets

pyproject.toml

@@ -12,19 +12,27 @@ dependencies = [
     "alembic",
     "apprise",
     "apscheduler",
+    "beautifulsoup4",
+    "bm25s",
+    "ebooklib",
     "fastapi",
     "fastapi-cli",
     "httpx",
+    "jinja2",
+    "pgvector",
     "polars",
     "psycopg[binary]",
     "pydantic",
     "pydantic-settings",
     "python-multipart",
-    "sqlalchemy",
+    "sqlalchemy[asyncio]",
     "tenacity",
+    "tiktoken",
     "tinytuya",
     "typer",
+    "uvicorn",
     "websockets",
+    "yake",
 ]
 
 [project.scripts]
@@ -33,8 +41,10 @@ whisper-transcribe = "python.tools.whisper.transcribe:main"
 
 [dependency-groups]
 dev = [
+    "aiosqlite",
     "mypy",
     "pyfakefs",
+    "pytest-asyncio",
     "pytest-cov",
     "pytest-mock",
     "pytest-xdist",
@@ -110,5 +120,6 @@ exclude_lines = [
 
 [tool.pytest.ini_options]
 addopts = "-n auto -ra"
+asyncio_mode = "auto"
 testpaths = ["tests"]
 # --cov=system_tools --cov-report=term-missing --cov-report=xml --cov-report=html --cov-branch

python/alembic/richie/versions/2026_06_29-adding_phrase_metadata_tables_dddee09eddcc.py

@@ -0,0 +1,206 @@
+"""adding Phrase metadata tables.
+
+Revision ID: dddee09eddcc
+Revises: 96d72c748c24
+Create Date: 2026-06-29 00:49:07.344159
+
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+from python.orm import RichieBase
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+# revision identifiers, used by Alembic.
+revision: str = "dddee09eddcc"
+down_revision: str | None = "96d72c748c24"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+schema = RichieBase.schema_name
+
+
+def upgrade() -> None:
+    """Upgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "candidate_phrases",
+        sa.Column("book_id", sa.Integer(), nullable=False),
+        sa.Column("series_id", sa.Integer(), nullable=True),
+        sa.Column("phrase_text", sa.Text(), nullable=False),
+        sa.Column("phrase_norm", sa.Text(), nullable=False),
+        sa.Column("token_count", sa.Integer(), nullable=False),
+        sa.Column("source_raw_ngram", sa.Boolean(), nullable=False),
+        sa.Column("source_yake", sa.Boolean(), nullable=False),
+        sa.Column("source_spacy_ner", sa.Boolean(), nullable=False),
+        sa.Column("source_spacy_noun_chunk", sa.Boolean(), nullable=False),
+        sa.Column("source_capitalized", sa.Boolean(), nullable=False),
+        sa.Column("source_metadata", sa.Boolean(), nullable=False),
+        sa.Column("spacy_label", sa.String(), nullable=True),
+        sa.Column("raw_count", sa.Integer(), nullable=False),
+        sa.Column("chapter_count", sa.Integer(), nullable=False),
+        sa.Column("yake_score", sa.Float(), nullable=True),
+        sa.Column("candidate_score", sa.Float(), nullable=False),
+        sa.Column(
+            "sample_contexts",
+            sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), "postgresql"),
+            nullable=True,
+        ),
+        sa.Column("llm_judged", sa.Boolean(), nullable=False),
+        sa.Column("llm_keep", sa.Boolean(), nullable=True),
+        sa.Column("llm_confidence", sa.Float(), nullable=True),
+        sa.Column("llm_category", sa.String(), nullable=True),
+        sa.Column("llm_reason", sa.Text(), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["book_id"],
+            [f"{schema}.ebook_source.id"],
+            name=op.f("fk_candidate_phrases_book_id_ebook_source"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_candidate_phrases")),
+        sa.UniqueConstraint("book_id", "phrase_norm", name="uq_candidate_phrases_book_id_phrase_norm"),
+        schema=schema,
+    )
+    op.create_index(
+        "candidate_phrases_book_norm_idx", "candidate_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
+    )
+    op.create_index(
+        "candidate_phrases_book_score_idx",
+        "candidate_phrases",
+        ["book_id", "candidate_score"],
+        unique=False,
+        schema=schema,
+    )
+    op.create_table(
+        "protected_phrases",
+        sa.Column("book_id", sa.Integer(), nullable=True),
+        sa.Column("series_id", sa.Integer(), nullable=True),
+        sa.Column("phrase_text", sa.Text(), nullable=False),
+        sa.Column("phrase_norm", sa.Text(), nullable=False),
+        sa.Column("canonical_id", sa.String(), nullable=False),
+        sa.Column("phrase_type", sa.String(), nullable=True),
+        sa.Column("token_count", sa.Integer(), nullable=False),
+        sa.Column("confidence", sa.Float(), nullable=False),
+        sa.Column("importance", sa.Float(), nullable=False),
+        sa.Column("allow_nested", sa.Boolean(), nullable=False),
+        sa.Column("suppress_children", sa.Boolean(), nullable=False),
+        sa.Column("source_candidate_id", sa.Integer(), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["book_id"],
+            [f"{schema}.ebook_source.id"],
+            name=op.f("fk_protected_phrases_book_id_ebook_source"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["source_candidate_id"],
+            [f"{schema}.candidate_phrases.id"],
+            name=op.f("fk_protected_phrases_source_candidate_id_candidate_phrases"),
+            ondelete="SET NULL",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_protected_phrases")),
+        sa.UniqueConstraint("book_id", "phrase_norm", name="uq_protected_phrases_book_id_phrase_norm"),
+        schema=schema,
+    )
+    op.create_index(
+        "protected_phrases_book_norm_idx", "protected_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
+    )
+    op.create_index("protected_phrases_norm_idx", "protected_phrases", ["phrase_norm"], unique=False, schema=schema)
+    op.create_index(
+        "protected_phrases_series_norm_idx",
+        "protected_phrases",
+        ["series_id", "phrase_norm"],
+        unique=False,
+        schema=schema,
+    )
+    op.create_table(
+        "chunk_phrase_mentions",
+        sa.Column("chunk_id", sa.BigInteger(), nullable=False),
+        sa.Column("phrase_id", sa.Integer(), nullable=False),
+        sa.Column("book_id", sa.Integer(), nullable=True),
+        sa.Column("series_id", sa.Integer(), nullable=True),
+        sa.Column("start_char", sa.Integer(), nullable=False),
+        sa.Column("end_char", sa.Integer(), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["book_id"],
+            [f"{schema}.ebook_source.id"],
+            name=op.f("fk_chunk_phrase_mentions_book_id_ebook_source"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["chunk_id"],
+            [f"{schema}.ebook_chunk.id"],
+            name=op.f("fk_chunk_phrase_mentions_chunk_id_ebook_chunk"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["phrase_id"],
+            [f"{schema}.protected_phrases.id"],
+            name=op.f("fk_chunk_phrase_mentions_phrase_id_protected_phrases"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_chunk_phrase_mentions")),
+        sa.UniqueConstraint("chunk_id", "phrase_id", "start_char", name="uq_chunk_phrase_mentions_chunk_phrase_start"),
+        schema=schema,
+    )
+    op.create_index(
+        "chunk_phrase_mentions_chunk_idx", "chunk_phrase_mentions", ["chunk_id"], unique=False, schema=schema
+    )
+    op.create_index(
+        "chunk_phrase_mentions_phrase_idx", "chunk_phrase_mentions", ["phrase_id"], unique=False, schema=schema
+    )
+    op.create_table(
+        "phrase_aliases",
+        sa.Column("phrase_id", sa.Integer(), nullable=False),
+        sa.Column("alias_text", sa.Text(), nullable=False),
+        sa.Column("alias_norm", sa.Text(), nullable=False),
+        sa.Column("confidence", sa.Float(), nullable=False),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["phrase_id"],
+            [f"{schema}.protected_phrases.id"],
+            name=op.f("fk_phrase_aliases_phrase_id_protected_phrases"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_phrase_aliases")),
+        sa.UniqueConstraint("phrase_id", "alias_norm", name="uq_phrase_aliases_phrase_id_alias_norm"),
+        schema=schema,
+    )
+    op.create_index("phrase_aliases_norm_idx", "phrase_aliases", ["alias_norm"], unique=False, schema=schema)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index("phrase_aliases_norm_idx", table_name="phrase_aliases", schema=schema)
+    op.drop_table("phrase_aliases", schema=schema)
+    op.drop_index("chunk_phrase_mentions_phrase_idx", table_name="chunk_phrase_mentions", schema=schema)
+    op.drop_index("chunk_phrase_mentions_chunk_idx", table_name="chunk_phrase_mentions", schema=schema)
+    op.drop_table("chunk_phrase_mentions", schema=schema)
+    op.drop_index("protected_phrases_series_norm_idx", table_name="protected_phrases", schema=schema)
+    op.drop_index("protected_phrases_norm_idx", table_name="protected_phrases", schema=schema)
+    op.drop_index("protected_phrases_book_norm_idx", table_name="protected_phrases", schema=schema)
+    op.drop_table("protected_phrases", schema=schema)
+    op.drop_index("candidate_phrases_book_score_idx", table_name="candidate_phrases", schema=schema)
+    op.drop_index("candidate_phrases_book_norm_idx", table_name="candidate_phrases", schema=schema)
+    op.drop_table("candidate_phrases", schema=schema)
+    # ### end Alembic commands ###

python/common.py

@@ -3,28 +3,23 @@
 from __future__ import annotations
 
 import logging
-import sys
 from datetime import UTC, datetime
-from os import getenv
+from pathlib import Path
 from subprocess import PIPE, Popen
 
-from apprise import Apprise
+from python.logging_config import configure_logger as _configure_logger
 
 logger = logging.getLogger(__name__)
 
 
-def configure_logger(level: str = "INFO") -> None:
+def get_repo_dir() -> Path:
-    """Configure the logger.
+    """Return the repository root directory."""
+    return Path(__file__).resolve().parents[1]
 
-    Args:
+
-        level (str, optional): The logging level. Defaults to "INFO".
+def configure_logger(level: str = "INFO") -> None:
-    """
+    """Configure the logger."""
-    logging.basicConfig(
+    _configure_logger(level)
-        level=level,
-        datefmt="%Y-%m-%dT%H:%M:%S%z",
-        format="%(asctime)s %(levelname)s %(filename)s:%(lineno)d - %(message)s",
-        handlers=[logging.StreamHandler(sys.stdout)],
-    )
 
 
 def bash_wrapper(command: str) -> tuple[str, int]:
@@ -47,26 +42,6 @@ def bash_wrapper(command: str) -> tuple[str, int]:
     return output.decode(), process.returncode
 
 
-def signal_alert(body: str, title: str = "") -> None:
-    """Send a signal alert.
-
-    Args:
-        body (str): The body of the alert.
-        title (str, optional): The title of the alert. Defaults to "".
-    """
-    apprise_client = Apprise()
-
-    from_phone = getenv("SIGNAL_ALERT_FROM_PHONE")
-    to_phone = getenv("SIGNAL_ALERT_TO_PHONE")
-    if not from_phone or not to_phone:
-        logger.info("SIGNAL_ALERT_FROM_PHONE or SIGNAL_ALERT_TO_PHONE not set")
-        return
-
-    apprise_client.add(f"signal://localhost:8989/{from_phone}/{to_phone}")
-
-    apprise_client.notify(title=title, body=body)
-
-
 def utcnow() -> datetime:
     """Get the current UTC time."""
     return datetime.now(tz=UTC)

python/ebook_search/answer.py

@@ -8,13 +8,20 @@ from typing import TYPE_CHECKING
 from python.ebook_search.llm_interface import request_chat_completion
 
 if TYPE_CHECKING:
+    import httpx
+
     from python.ebook_search.config import EbookSearchConfig
     from python.ebook_search.search import SearchResult
 
 logger = logging.getLogger(__name__)
 
 
-def answer_query(query: str, results: list[SearchResult], config: EbookSearchConfig) -> str:
+async def answer_query(
+    client: httpx.AsyncClient,
+    query: str,
+    results: list[SearchResult],
+    config: EbookSearchConfig,
+) -> str:
     """Answer a question using only retrieved chunks."""
     if not config.answer_enabled:
         logger.info("ebook_answer_skipped_disabled")
@@ -35,7 +42,8 @@ def answer_query(query: str, results: list[SearchResult], config: EbookSearchCon
         f"[{index}] {result.source_title}{' - ' + result.chapter_title if result.chapter_title else ''}\n{result.text}"
         for index, result in enumerate(results, start=1)
     )
-    content = request_chat_completion(
+    content = await request_chat_completion(
+        client,
         config,
         [
             {

python/ebook_search/api/bm25_tasks.py

@@ -1,18 +1,22 @@
-"""Background BM25 refresh tasks for the web app."""
+"""Background BM25 refresh tasks for the web app.
+
+The refresh is scheduled on the event loop instead of a thread because the async psycopg
+driver only works from the loop; a bare thread cannot open a session on the async engine.
+"""
 
 from __future__ import annotations
 
+import asyncio
 import logging
-from threading import Timer
 from typing import TYPE_CHECKING
 
-from sqlalchemy.orm import Session
+from sqlalchemy.ext.asyncio import AsyncSession
 
 from python.ebook_search.bm25_corpus import load_bm25_corpus, refresh_bm25_corpus
 
 if TYPE_CHECKING:
     from fastapi import FastAPI
-    from sqlalchemy.engine import Engine
+    from sqlalchemy.ext.asyncio import AsyncEngine
 
     from python.ebook_search.config import EbookSearchConfig
 
@@ -20,15 +24,18 @@ logger = logging.getLogger(__name__)
 
 
 def schedule_bm25_refresh(app: FastAPI) -> None:
-    """Schedule a delayed BM25 corpus refresh, replacing any pending refresh."""
+    """Schedule a delayed BM25 corpus refresh, replacing any pending refresh.
-    existing_timer = getattr(app.state, "bm25_refresh_timer", None)
-    if existing_timer is not None:
-        existing_timer.cancel()
 
-    timer = Timer(app.state.config.bm25_refresh_delay_seconds, refresh_bm25_for_app, args=(app,))
+    Only called from route handlers, so a running event loop is guaranteed.
-    timer.daemon = True
+    """
-    timer.start()
+    cancel_bm25_refresh(app)
-    app.state.bm25_refresh_timer = timer
+
+    loop = asyncio.get_running_loop()
+
+    def start_refresh() -> None:
+        app.state.bm25_refresh_task = loop.create_task(refresh_bm25_for_app(app))
+
+    app.state.bm25_refresh_timer = loop.call_later(app.state.config.bm25_refresh_delay_seconds, start_refresh)
     logger.info(
         "ebook_bm25_refresh_scheduled delay_seconds=%s",
         app.state.config.bm25_refresh_delay_seconds,
@@ -36,25 +43,31 @@ def schedule_bm25_refresh(app: FastAPI) -> None:
 
 
 def cancel_bm25_refresh(app: FastAPI) -> None:
-    """Cancel any pending BM25 corpus refresh."""
+    """Cancel any pending BM25 corpus refresh timer and in-flight refresh task."""
     existing_timer = getattr(app.state, "bm25_refresh_timer", None)
     if existing_timer is not None:
         existing_timer.cancel()
         app.state.bm25_refresh_timer = None
         logger.info("ebook_bm25_refresh_cancelled")
 
+    existing_task = getattr(app.state, "bm25_refresh_task", None)
+    if existing_task is not None:
+        if not existing_task.done():
+            existing_task.cancel()
+        app.state.bm25_refresh_task = None
 
-def refresh_bm25_for_app(app: FastAPI) -> None:
+
+async def refresh_bm25_for_app(app: FastAPI) -> None:
     """Refresh the BM25 corpus using the app engine and config."""
     try:
-        refresh_bm25_for_engine(app.state.engine, app.state.config)
+        await refresh_bm25_for_engine(app.state.engine, app.state.config)
     except Exception:
         logger.exception("ebook_bm25_refresh_failed")
 
 
-def refresh_bm25_for_engine(engine: Engine, config: EbookSearchConfig) -> None:
+async def refresh_bm25_for_engine(engine: AsyncEngine, config: EbookSearchConfig) -> None:
-    """Refresh the BM25 corpus using a SQLAlchemy engine."""
+    """Refresh the BM25 corpus using an async SQLAlchemy engine."""
-    with Session(engine) as session:
+    async with AsyncSession(engine) as session:
-        refresh_bm25_corpus(session, config)
+        await refresh_bm25_corpus(session, config)
     load_bm25_corpus.cache_clear()
     logger.info("ebook_bm25_corpus_cache_cleared_after_refresh")

python/ebook_search/api/dependencies.py

@@ -4,8 +4,9 @@ from __future__ import annotations
 
 from typing import Annotated
 
+import httpx
 from fastapi import Depends, Request
-from sqlalchemy.engine import Engine
+from sqlalchemy.ext.asyncio import AsyncEngine
 
 from python.ebook_search.config import EbookSearchConfig
 
@@ -15,10 +16,16 @@ def get_config(request: Request) -> EbookSearchConfig:
     return request.app.state.config
 
 
-def get_engine(request: Request) -> Engine:
+def get_engine(request: Request) -> AsyncEngine:
     """Get the database engine from app state."""
     return request.app.state.engine
 
 
+def get_http_client(request: Request) -> httpx.AsyncClient:
+    """Get the shared LLM HTTP client from app state."""
+    return request.app.state.http_client
+
+
 AppConfig = Annotated[EbookSearchConfig, Depends(get_config)]
-AppEngine = Annotated[Engine, Depends(get_engine)]
+AppEngine = Annotated[AsyncEngine, Depends(get_engine)]
+AppHttpClient = Annotated[httpx.AsyncClient, Depends(get_http_client)]

python/ebook_search/api/judge_tasks.py

@@ -0,0 +1,131 @@
+"""Background phrase-judging tasks for the web app.
+
+Judging a book sends one LLM request per candidate phrase, which can take minutes, so it must
+not run inside the request where it would block the UI. Judgments run as async FastAPI
+background tasks, awaited on the event loop after the response is sent, and are tracked per
+book in app state so a second judge request for a book that is already being judged is
+rejected instead of doubling the work.
+
+State is loop-confined: every read and mutation happens on the event loop (async route
+handlers and async background tasks) and no critical section contains an ``await``, so each
+mutation is atomic per loop iteration and no locking is needed.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
+
+if TYPE_CHECKING:
+    from fastapi import BackgroundTasks, FastAPI
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class JudgeTaskState:
+    """Running book judgments and last outcome messages, keyed by book id."""
+
+    running_book_ids: set[int] = field(default_factory=set)
+    outcome_messages: dict[int, str] = field(default_factory=dict)
+
+
+def get_judge_task_state(app: FastAPI) -> JudgeTaskState:
+    """Return the app's judge task state, creating it on first use.
+
+    Args:
+        app (FastAPI): App whose state holds the judge task registry.
+
+    Returns:
+        JudgeTaskState: The shared judge task state for this app.
+    """
+    state = getattr(app.state, "judge_tasks", None)
+    if state is None:
+        state = JudgeTaskState()
+        app.state.judge_tasks = state
+    return state
+
+
+def start_book_phrase_judgment(app: FastAPI, background_tasks: BackgroundTasks, source_id: int) -> bool:
+    """Queue judging of one book's candidate phrases as a FastAPI background task.
+
+    The book is claimed before the response returns, so a repeated judge request cannot queue
+    a second run while one is pending or running.
+
+    Args:
+        app (FastAPI): App supplying the engine, config, and judge task state.
+        background_tasks (BackgroundTasks): Request's background tasks to queue the judgment on.
+        source_id (int): Book to judge candidates for.
+
+    Returns:
+        bool: True when a judgment was queued, False when one is already running for this book.
+    """
+    state = get_judge_task_state(app)
+    if source_id in state.running_book_ids:
+        logger.info("ebook_book_phrase_judgment_already_running source_id=%s", source_id)
+        return False
+    state.running_book_ids.add(source_id)
+    state.outcome_messages.pop(source_id, None)
+    background_tasks.add_task(judge_book_phrases_for_app, app, source_id)
+    logger.info("ebook_book_phrase_judgment_queued source_id=%s", source_id)
+    return True
+
+
+async def judge_book_phrases_for_app(app: FastAPI, source_id: int) -> None:
+    """Judge one book using the app engine and config, recording the outcome message.
+
+    Args:
+        app (FastAPI): App supplying the engine, config, and judge task state.
+        source_id (int): Book to judge candidates for.
+    """
+    state = get_judge_task_state(app)
+    try:
+        result = await judge_candidate_phrases_for_books(app.state.engine, app.state.config, source_ids=[source_id])
+        logger.info(
+            "ebook_book_phrase_judgment_complete source_id=%s judged=%s protected=%s mentions=%s failed=%s",
+            source_id,
+            result.candidates_judged,
+            result.protected_phrases,
+            result.phrase_mentions,
+            result.books_failed,
+        )
+        if result.books_failed:
+            message = "Judging failed; see server logs for details"
+        else:
+            message = (
+                f"Judged {result.candidates_judged} candidates; {result.protected_phrases} protected phrases promoted"
+            )
+    except Exception:
+        logger.exception("ebook_book_phrase_judgment_task_failed source_id=%s", source_id)
+        message = "Judging failed; see server logs for details"
+    state.running_book_ids.discard(source_id)
+    state.outcome_messages[source_id] = message
+
+
+def is_judging_book(app: FastAPI, source_id: int) -> bool:
+    """Report whether a judgment is currently queued or running for one book.
+
+    Args:
+        app (FastAPI): App supplying the judge task state.
+        source_id (int): Book to check.
+
+    Returns:
+        bool: True while the book's judgment is pending or running.
+    """
+    return source_id in get_judge_task_state(app).running_book_ids
+
+
+def pop_book_judgment_outcome(app: FastAPI, source_id: int) -> str | None:
+    """Return and clear the outcome message from one book's last finished judgment.
+
+    Args:
+        app (FastAPI): App supplying the judge task state.
+        source_id (int): Book to fetch the outcome for.
+
+    Returns:
+        str | None: The outcome message, or None when there is nothing new to report.
+    """
+    return get_judge_task_state(app).outcome_messages.pop(source_id, None)

python/ebook_search/api/main.py

@@ -6,11 +6,12 @@ import logging
 from contextlib import asynccontextmanager
 from typing import TYPE_CHECKING, Annotated
 
+import httpx
 import typer
 import uvicorn
 from fastapi import FastAPI
 from fastapi.staticfiles import StaticFiles
-from sqlalchemy.orm import Session
+from sqlalchemy.ext.asyncio import AsyncSession
 
 from python.common import configure_logger
 from python.ebook_search.api.bm25_tasks import cancel_bm25_refresh
@@ -18,8 +19,9 @@ from python.ebook_search.api.routes import admin_router, health_router, page_rou
 from python.ebook_search.api.web import STATIC_DIR
 from python.ebook_search.bm25_corpus import ensure_bm25_corpus
 from python.ebook_search.config import load_config
+from python.ebook_search.protected_phrases.pool import shutdown_extraction_pool
 from python.fastapi_tools import ZstdMiddleware
-from python.orm.common import get_postgres_engine
+from python.orm.common import get_async_postgres_engine
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
@@ -36,26 +38,36 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     app.state.config = config
     logger.info(
         "ebook_search_config_loaded top_k=%s embedding_model=%s embedding_base_url=%s vllm_base_url=%s "
-        "rerank_enabled=%s answer_enabled=%s library_paths=%s",
+        "rerank_enabled=%s phrase_matching_enabled=%s answer_enabled=%s library_paths=%s",
         config.top_k,
         config.embedding_model,
         config.embedding_base_url,
         config.vllm_base_url,
         config.rerank.enabled,
+        config.phrase_matching_enabled,
         config.answer_enabled,
         len(config.library_paths),
     )
     if not config.library_paths:
         logger.warning("ebook_search_no_library_paths_configured")
-    app.state.engine = get_postgres_engine(name="RICHIE", vector_engine=True)
+    # Concurrent phrase judging opens one session per book worker on this engine, so size the pool
-    with Session(app.state.engine) as session:
+    # to cover those plus headroom for ordinary web requests.
-        ensure_bm25_corpus(session, config)
+    app.state.engine = get_async_postgres_engine(
+        name="RICHIE",
+        vector_engine=True,
+        pool_size=config.phrase_judge_book_workers + 10,
+    )
+    app.state.http_client = httpx.AsyncClient()
+    async with AsyncSession(app.state.engine, expire_on_commit=False) as session:
+        await ensure_bm25_corpus(session, config)
     try:
         yield
     finally:
         logger.info("ebook_search_shutdown")
         cancel_bm25_refresh(app)
-        app.state.engine.dispose()
+        shutdown_extraction_pool()
+        await app.state.http_client.aclose()
+        await app.state.engine.dispose()
 
 
 def create_app() -> FastAPI:

python/ebook_search/api/routes/admin.py

@@ -8,13 +8,18 @@ from fastapi import APIRouter, Request
 from fastapi.responses import HTMLResponse
 
 from python.ebook_search.api.bm25_tasks import schedule_bm25_refresh
-from python.ebook_search.api.dependencies import (
+from python.ebook_search.api.dependencies import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
-    AppConfig,  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+    AppConfig,
+    AppEngine,
+    AppHttpClient,
 )
 from python.ebook_search.api.web import templates
 from python.ebook_search.embeddings import embed_missing_chunks, embedding_model_stats
 from python.ebook_search.ingest import ingest_configured_paths
-from python.fastapi_tools import DbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+from python.ebook_search.protected_phrases.generate_ngrams import generate_candidate_phrases_for_books
+from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
+from python.ebook_search.protected_phrases.store import book_ids_pending_first_judgment, corpus_phrase_stats
+from python.fastapi_tools import AsyncDbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
 
 logger = logging.getLogger(__name__)
 
@@ -22,19 +27,29 @@ router = APIRouter(prefix="/admin")
 
 
 @router.get("", response_class=HTMLResponse)
-def admin(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
+async def admin(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
     """Render the admin page."""
-    stats = embedding_model_stats(session)
+    stats = await embedding_model_stats(session)
-    logger.info("ebook_admin_page_loaded models=%s", len(stats))
+    phrase_stats = await corpus_phrase_stats(session)
-    return templates.TemplateResponse(request, "admin.html", {"config": config, "stats": stats})
+    logger.info(
+        "ebook_admin_page_loaded models=%s candidate_phrases=%s protected_phrases=%s",
+        len(stats),
+        phrase_stats.candidate_phrases,
+        phrase_stats.protected_phrases,
+    )
+    return templates.TemplateResponse(
+        request,
+        "admin.html",
+        {"config": config, "stats": stats, "phrase_stats": phrase_stats},
+    )
 
 
 @router.post("/scan", response_class=HTMLResponse)
-def scan_library(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
+async def scan_library(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
     """Scan configured library paths for EPUB changes."""
     try:
-        count = ingest_configured_paths(session, config)
+        count = await ingest_configured_paths(session, config)
-        session.commit()
+        await session.commit()
     except Exception as error:
         logger.exception("ebook_admin_scan_failed")
         return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
@@ -45,12 +60,152 @@ def scan_library(request: Request, config: AppConfig, session: DbSession) -> HTM
     return templates.TemplateResponse(request, "partials/admin_status.html", {"message": f"Indexed {count} EPUBs"})
 
 
+@router.post("/phrases/generate-all", response_class=HTMLResponse)
+async def generate_all_phrases(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
+    """Regenerate candidate phrases for every indexed book without LLM judging."""
+    return await run_phrase_generation(request, config, session, only_missing=False)
+
+
+@router.post("/phrases/generate-missing", response_class=HTMLResponse)
+async def generate_missing_phrases(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
+    """Generate candidate phrases only for books that have none yet."""
+    return await run_phrase_generation(request, config, session, only_missing=True)
+
+
+async def run_phrase_generation(
+    request: Request,
+    config: AppConfig,
+    session: AsyncDbSession,
+    *,
+    only_missing: bool,
+) -> HTMLResponse:
+    """Run candidate phrase generation and render the outcome as an admin status partial.
+
+    Args:
+        request (Request): Current request, for template rendering.
+        config (AppConfig): Runtime phrase-tuning settings.
+        session (AsyncDbSession): Active database session.
+        only_missing (bool): Only generate for books without candidates instead of every book.
+
+    Returns:
+        HTMLResponse: Status partial describing the generation outcome.
+    """
+    try:
+        result = await generate_candidate_phrases_for_books(session, config, only_missing=only_missing)
+        await session.commit()
+    except Exception as error:
+        await session.rollback()
+        logger.exception("ebook_admin_generate_phrases_failed only_missing=%s", only_missing)
+        return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
+
+    logger.info(
+        "ebook_admin_generate_phrases_complete only_missing=%s books_seen=%s books_built=%s candidates=%s",
+        only_missing,
+        result.books_seen,
+        result.books_built,
+        result.candidate_phrases,
+    )
+    if only_missing and result.books_seen == 0:
+        return templates.TemplateResponse(
+            request,
+            "partials/admin_status.html",
+            {"message": "All books already have candidate phrases"},
+        )
+    return templates.TemplateResponse(
+        request,
+        "partials/admin_status.html",
+        {
+            "message": (
+                f"Generated phrases for {result.books_built} of {result.books_seen} books; "
+                f"{result.candidate_phrases} candidates stored"
+            )
+        },
+    )
+
+
+@router.post("/phrases/judge-all", response_class=HTMLResponse)
+async def judge_all_phrases(request: Request, engine: AppEngine, config: AppConfig) -> HTMLResponse:
+    """Judge unjudged candidate phrases across every indexed book."""
+    return await run_phrase_judgment(request, engine, config, source_ids=None)
+
+
+@router.post("/phrases/judge-missing", response_class=HTMLResponse)
+async def judge_missing_phrases(
+    request: Request,
+    engine: AppEngine,
+    config: AppConfig,
+    session: AsyncDbSession,
+) -> HTMLResponse:
+    """Judge candidate phrases only for books where judging has never run."""
+    source_ids = await book_ids_pending_first_judgment(session)
+    if not source_ids:
+        return templates.TemplateResponse(
+            request,
+            "partials/admin_status.html",
+            {"message": "All books with candidate phrases have been judged"},
+        )
+    return await run_phrase_judgment(request, engine, config, source_ids=source_ids)
+
+
+async def run_phrase_judgment(
+    request: Request,
+    engine: AppEngine,
+    config: AppConfig,
+    *,
+    source_ids: list[int] | None,
+) -> HTMLResponse:
+    """Run LLM judging for candidate phrases and render the outcome as an admin status partial.
+
+    Args:
+        request (Request): Current request, for template rendering.
+        engine (AppEngine): Engine used to open per-book judging sessions.
+        config (AppConfig): Runtime phrase-tuning settings.
+        source_ids (list[int] | None): Books to judge; ``None`` judges every indexed book.
+
+    Returns:
+        HTMLResponse: Status partial describing the judging outcome.
+    """
+    try:
+        result = await judge_candidate_phrases_for_books(engine, config, source_ids=source_ids)
+    except Exception as error:
+        logger.exception("ebook_admin_judge_phrases_failed")
+        return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
+
+    logger.info(
+        "ebook_admin_judge_phrases_complete books_seen=%s books_judged=%s books_failed=%s candidates_judged=%s "
+        "protected=%s mentions=%s",
+        result.books_seen,
+        result.books_judged,
+        result.books_failed,
+        result.candidates_judged,
+        result.protected_phrases,
+        result.phrase_mentions,
+    )
+    return templates.TemplateResponse(
+        request,
+        "partials/admin_status.html",
+        {
+            "message": (
+                f"Judged {result.candidates_judged} candidates across {result.books_judged} of "
+                f"{result.books_seen} books; {result.protected_phrases} protected phrases, "
+                f"{result.phrase_mentions} mentions"
+                + (f"; {result.books_failed} books failed" if result.books_failed else "")
+            )
+        },
+    )
+
+
 @router.post("/embed-missing", response_class=HTMLResponse)
-def embed_missing(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
+async def embed_missing(
+    request: Request,
+    config: AppConfig,
+    session: AsyncDbSession,
+    client: AppHttpClient,
+) -> HTMLResponse:
     """Embed chunks missing vectors for the configured model."""
     try:
-        count = embed_missing_chunks(session, config)
+        count = await embed_missing_chunks(session, client, config)
-        session.commit()
+        await session.commit()
     except Exception as error:
         logger.exception("ebook_admin_embed_missing_failed")
         return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
@@ -64,16 +219,21 @@ def embed_missing(request: Request, config: AppConfig, session: DbSession) -> HT
 
 
 @router.post("/embed-all", response_class=HTMLResponse)
-def embed_all(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
+async def embed_all(
+    request: Request,
+    config: AppConfig,
+    session: AsyncDbSession,
+    client: AppHttpClient,
+) -> HTMLResponse:
     """Embed all chunks missing vectors in fixed-size batches."""
     total = 0
     batches = 0
     try:
         while True:
-            count = embed_missing_chunks(session, config)
+            count = await embed_missing_chunks(session, client, config)
             if count == 0:
                 break
-            session.commit()
+            await session.commit()
             total += count
             batches += 1
             logger.info(

python/ebook_search/api/routes/health.py

@@ -11,15 +11,17 @@ from fastapi.responses import JSONResponse
 from sqlalchemy import literal, select
 from sqlalchemy.exc import SQLAlchemyError
 
-from python.ebook_search.api.dependencies import (
+from python.ebook_search.api.dependencies import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
-    AppConfig,  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+    AppConfig,
+    AppHttpClient,
 )
 from python.ebook_search.bm25_corpus import bm25_index_exists, bm25_index_path, read_bm25_manifest
 from python.ebook_search.llm_interface import check_chat_endpoint, check_embedding_endpoint
-from python.fastapi_tools import DbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+from python.fastapi_tools import AsyncDbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
 
 if TYPE_CHECKING:
-    from sqlalchemy.orm import Session
+    import httpx
+    from sqlalchemy.ext.asyncio import AsyncSession
 
     from python.ebook_search.config import EbookSearchConfig
 
@@ -29,17 +31,17 @@ router = APIRouter()
 
 
 @router.get("/health")
-def health() -> dict[str, str]:
+async def health() -> dict[str, str]:
     """Liveness probe that returns ok without touching dependencies."""
     return {"status": "ok"}
 
 
 @router.get("/ready")
-def ready(config: AppConfig, session: DbSession) -> JSONResponse:
+async def ready(config: AppConfig, session: AsyncDbSession, client: AppHttpClient) -> JSONResponse:
     """Readiness probe reporting database, embedding endpoint, and BM25 index status."""
-    database_ok = check_database(session)
+    database_ok = await check_database(session)
-    embedding_ok = check_embedding_endpoint(config)
+    embedding_ok = await check_embedding_endpoint(client, config)
-    chat_status = chat_endpoint_status(config)
+    chat_status = await chat_endpoint_status(client, config)
     bm25_status = check_bm25_status(config)
 
     checks = {
@@ -69,17 +71,17 @@ def ready(config: AppConfig, session: DbSession) -> JSONResponse:
     return JSONResponse(content={"status": status, "checks": checks}, status_code=status_code)
 
 
-def chat_endpoint_status(config: EbookSearchConfig) -> str:
+async def chat_endpoint_status(client: httpx.AsyncClient, config: EbookSearchConfig) -> str:
     """Return the answering chat endpoint status, or disabled when answers are off."""
     if not config.answer_enabled:
         return "disabled"
-    return "ok" if check_chat_endpoint(config) else "fail"
+    return "ok" if await check_chat_endpoint(client, config) else "fail"
 
 
-def check_database(session: Session) -> bool:
+async def check_database(session: AsyncSession) -> bool:
     """Return whether the database answers a trivial query."""
     try:
-        session.execute(select(literal(1)))
+        await session.execute(select(literal(1)))
     except SQLAlchemyError as error:
         logger.warning("ebook_ready_database_unavailable error=%s", error)
         return False

python/ebook_search/api/routes/page.py

@@ -3,17 +3,23 @@
 from __future__ import annotations
 
 import logging
+from typing import TYPE_CHECKING
 
-from fastapi import APIRouter, Request
+from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
-from fastapi.responses import HTMLResponse
+from fastapi.responses import HTMLResponse, RedirectResponse
-from sqlalchemy import select
+from sqlalchemy import func, select
 
 from python.ebook_search.api.dependencies import (
     AppConfig,  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
 )
+from python.ebook_search.api.judge_tasks import is_judging_book, pop_book_judgment_outcome, start_book_phrase_judgment
 from python.ebook_search.api.web import templates
-from python.fastapi_tools import DbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+from python.ebook_search.protected_phrases.generate_ngrams import recalculate_candidate_phrases_for_book
-from python.orm.richie import EbookSource
+from python.fastapi_tools import AsyncDbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+from python.orm.richie import EbookCandidatePhrase, EbookChapter, EbookChunk, EbookProtectedPhrase, EbookSource
+
+if TYPE_CHECKING:
+    from sqlalchemy.ext.asyncio import AsyncSession
 
 logger = logging.getLogger(__name__)
 
@@ -21,38 +27,176 @@ router = APIRouter()
 
 
 @router.get("/", response_class=HTMLResponse)
-def index(request: Request, config: AppConfig) -> HTMLResponse:
+async def index(request: Request, config: AppConfig) -> HTMLResponse:
     """Render the search page."""
     return templates.TemplateResponse(request, "search.html", {"config": config})
 
 
 @router.get("/books", response_class=HTMLResponse)
-def books(request: Request, session: DbSession) -> HTMLResponse:
+async def books(request: Request, session: AsyncDbSession) -> HTMLResponse:
     """Render the indexed books page."""
-    sources = list(session.scalars(select(EbookSource).order_by(EbookSource.title)).all())
+    sources = list((await session.scalars(select(EbookSource).order_by(EbookSource.title))).all())
     logger.info("ebook_books_page_loaded count=%s", len(sources))
     return templates.TemplateResponse(request, "books.html", {"sources": sources})
 
 
+async def get_chapter_count(session: AsyncSession, book_id: int) -> int:
+    """Return the number of indexed chapters for one book."""
+    return await session.scalar(select(func.count(EbookChapter.id)).where(EbookChapter.source_id == book_id)) or 0
+
+
+async def get_chunk_count(session: AsyncSession, book_id: int) -> int:
+    """Return the number of indexed chunks for one book."""
+    return await session.scalar(select(func.count(EbookChunk.id)).where(EbookChunk.source_id == book_id)) or 0
+
+
+async def get_candidate_count(session: AsyncSession, book_id: int) -> int:
+    """Return the number of indexed candidates for one book."""
+    return (
+        await session.scalar(select(func.count(EbookCandidatePhrase.id)).where(EbookCandidatePhrase.book_id == book_id))
+        or 0
+    )
+
+
+async def get_judged_candidate_count(session: AsyncSession, book_id: int) -> int:
+    """Return the number of judged candidates for one book."""
+    return (
+        await session.scalar(
+            select(func.count(EbookCandidatePhrase.id)).where(
+                EbookCandidatePhrase.book_id == book_id,
+                EbookCandidatePhrase.llm_judged.is_(True),
+            )
+        )
+        or 0
+    )
+
+
+async def get_protected_count(session: AsyncSession, book_id: int) -> int:
+    """Return the number of protected phrases for one book."""
+    return (
+        await session.scalar(select(func.count(EbookProtectedPhrase.id)).where(EbookProtectedPhrase.book_id == book_id))
+        or 0
+    )
+
+
+async def get_candidates(session: AsyncSession, book_id: int) -> list[EbookCandidatePhrase]:
+    """Return the indexed candidates for one book."""
+    return list(
+        await session.scalars(
+            select(EbookCandidatePhrase)
+            .where(EbookCandidatePhrase.book_id == book_id)
+            .order_by(EbookCandidatePhrase.candidate_score.desc())
+            .limit(100)
+        )
+    )
+
+
+async def get_protected_phrases(session: AsyncSession, book_id: int) -> list[EbookProtectedPhrase]:
+    """Return the protected phrases for one book."""
+    return list(
+        await session.scalars(
+            select(EbookProtectedPhrase)
+            .where(EbookProtectedPhrase.book_id == book_id)
+            .order_by(EbookProtectedPhrase.importance.desc())
+            .limit(100)
+        )
+    )
+
+
 @router.get("/books/{source_id}", response_class=HTMLResponse)
-def book_detail(source_id: int, request: Request, session: DbSession) -> HTMLResponse:
+async def book_detail(source_id: int, request: Request, session: AsyncDbSession) -> HTMLResponse:
     """Render details for one indexed book."""
-    source = session.get(EbookSource, source_id)
+    source = await session.get(EbookSource, source_id)
+    phrase_status_message = None
+    recalculated = request.query_params.get("phrases_recalculated")
+    if recalculated is not None:
+        phrase_status_message = f"Recalculated phrases; {recalculated} candidates generated"
+    judgment_outcome = pop_book_judgment_outcome(request.app, source_id)
+    if judgment_outcome is not None:
+        phrase_status_message = judgment_outcome
+    judging_in_progress = is_judging_book(request.app, source_id)
+    if judging_in_progress:
+        phrase_status_message = "Judging candidate phrases in the background; refresh to see progress"
     if source is not None:
-        chapter_count = len(source.chapters)
+        chapter_count = await get_chapter_count(session, source.id)
-        chunk_count = len(source.chunks)
+        chunk_count = await get_chunk_count(session, source.id)
+        candidate_count = await get_candidate_count(session, source.id)
+        judged_candidate_count = await get_judged_candidate_count(session, source.id)
+        protected_count = await get_protected_count(session, source.id)
+        candidates = await get_candidates(session, source.id)
+        protected_phrases = await get_protected_phrases(session, source.id)
     else:
         chapter_count = 0
         chunk_count = 0
+        candidate_count = 0
+        judged_candidate_count = 0
+        protected_count = 0
+        candidates = []
+        protected_phrases = []
     logger.info(
-        "ebook_book_detail_loaded source_id=%s found=%s chapters=%s chunks=%s",
+        "ebook_book_detail_loaded source_id=%s found=%s chapters=%s chunks=%s candidates=%s judged=%s protected=%s",
         source_id,
         source is not None,
         chapter_count,
         chunk_count,
+        candidate_count,
+        judged_candidate_count,
+        protected_count,
     )
     return templates.TemplateResponse(
         request,
         "book_detail.html",
-        {"chapter_count": chapter_count, "chunk_count": chunk_count, "source": source},
+        {
+            "candidate_count": candidate_count,
+            "candidates": candidates,
+            "chapter_count": chapter_count,
+            "chunk_count": chunk_count,
+            "judged_candidate_count": judged_candidate_count,
+            "judging_in_progress": judging_in_progress,
+            "protected_count": protected_count,
+            "protected_phrases": protected_phrases,
+            "phrase_status_message": phrase_status_message,
+            "source": source,
+        },
     )
+
+
+@router.post("/books/{source_id}/recalculate-phrases")
+async def recalculate_book_phrases(source_id: int, config: AppConfig, session: AsyncDbSession) -> RedirectResponse:
+    """Clear and regenerate candidate phrases for one indexed book."""
+    source = await session.get(EbookSource, source_id)
+    if source is None:
+        raise HTTPException(status_code=404, detail="Book not found")
+
+    result = await recalculate_candidate_phrases_for_book(session, source, config, use_process_pool=True)
+    logger.info(
+        "ebook_book_phrase_recalculation_complete source_id=%s candidates=%s deleted_candidates=%s "
+        "deleted_protected=%s deleted_aliases=%s deleted_mentions=%s",
+        source_id,
+        result.candidate_phrases,
+        result.deleted_candidates,
+        result.deleted_protected_phrases,
+        result.deleted_aliases,
+        result.deleted_mentions,
+    )
+    return RedirectResponse(
+        url=f"/books/{source_id}?phrases_recalculated={result.candidate_phrases}",
+        status_code=303,
+    )
+
+
+@router.post("/books/{source_id}/judge-phrases")
+async def judge_book_phrases(
+    source_id: int,
+    request: Request,
+    background_tasks: BackgroundTasks,
+    session: AsyncDbSession,
+) -> RedirectResponse:
+    """Queue background judging of one book's candidate phrases and return immediately."""
+    source = await session.get(EbookSource, source_id)
+    if source is None:
+        raise HTTPException(status_code=404, detail="Book not found")
+
+    started = start_book_phrase_judgment(request.app, background_tasks, source.id)
+    logger.info("ebook_book_phrase_judgment_requested source_id=%s started=%s", source_id, started)
+    return RedirectResponse(url=f"/books/{source_id}", status_code=303)

python/ebook_search/api/routes/search.py

@@ -14,6 +14,7 @@ from python.ebook_search.answer import answer_query
 from python.ebook_search.api.dependencies import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
     AppConfig,
     AppEngine,
+    AppHttpClient,
 )
 from python.ebook_search.api.web import templates
 from python.ebook_search.guardrails import (
@@ -26,6 +27,8 @@ from python.ebook_search.search import SearchResponse, search_ebooks
 from python.ebook_search.timing import runtime_step_from_start
 
 if TYPE_CHECKING:
+    import httpx
+
     from python.ebook_search.config import EbookSearchConfig
 
 logger = logging.getLogger(__name__)
@@ -33,7 +36,8 @@ logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
-def build_answer(
+async def build_answer(
+    client: httpx.AsyncClient,
     query: str,
     response: SearchResponse,
     config: EbookSearchConfig,
@@ -56,7 +60,7 @@ def build_answer(
         return answer, True, None
 
     try:
-        answer = answer_query(query, response.results, config)
+        answer = await answer_query(client, query, response.results, config)
     except RuntimeError as error:
         logger.warning("ebook_answer_request_failed_falling_back error=%s", error)
         return "Answer generation failed. Source chunks are still shown below.", False, None
@@ -74,22 +78,31 @@ def build_answer(
 
 
 @router.post("/search", response_class=HTMLResponse)
-def search(
+async def search(
     request: Request,
     config: AppConfig,
     engine: AppEngine,
+    client: AppHttpClient,
     query: Annotated[str, Form()],
     rerank: Annotated[str | None, Form()] = None,
+    phrase_matching: Annotated[str | None, Form()] = None,
 ) -> HTMLResponse:
     """Run a search and render HTMX results."""
     try:
-        response = search_ebooks(engine, query, config, rerank=rerank == "true")
+        response = await search_ebooks(
+            engine,
+            client,
+            query,
+            config,
+            rerank=rerank == "true",
+            phrase_matching=phrase_matching == "true",
+        )
     except Exception as error:
         logger.exception("ebook_search_request_failed")
         return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
 
     answer_start = perf_counter()
-    answer, low_confidence, citation_report = build_answer(query, response, config)
+    answer, low_confidence, citation_report = await build_answer(client, query, response, config)
     answer_step_name = "Answer generation" if config.answer_enabled else "Answer skipped"
     response = replace(
         response,

python/ebook_search/api/static/style.css

@@ -181,6 +181,12 @@ textarea:focus {
     flex-wrap: wrap;
 }
 
+.search-toggles {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 14px;
+}
+
 button {
     padding: 10px 20px;
     font: inherit;
@@ -212,6 +218,11 @@ button:hover {
     margin-bottom: 24px;
 }
 
+.actions-grid {
+    display: grid;
+    grid-template-columns: repeat(2, max-content);
+}
+
 /* Answer + results */
 #results {
     display: block;
@@ -303,6 +314,28 @@ button:hover {
     font-variant-numeric: tabular-nums;
 }
 
+.phrase-matches {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 8px;
+    align-items: baseline;
+    margin: 10px 0 0;
+    font-size: 0.78rem;
+}
+
+.phrase-matches-label {
+    color: var(--muted);
+    font-weight: 600;
+}
+
+.phrase-match {
+    padding: 3px 10px;
+    background: var(--bg);
+    border: 1px solid var(--border);
+    border-radius: 999px;
+    color: var(--accent);
+}
+
 /* Runtime — developer diagnostics, hidden unless dev mode is on */
 .runtime {
     display: none;

python/ebook_search/api/templates/admin.html

@@ -1,45 +1,110 @@
-{% extends "base.html" %}
+{% extends "base.html" %} {% block title %}EPUB Admin{% endblock %} {% block
-
+head %}
-{% block title %}EPUB Admin{% endblock %}
+<script src="https://unpkg.com/htmx.org@2.0.4"></script>
-{% block head %}<script src="https://unpkg.com/htmx.org@2.0.4"></script>{% endblock %}
+{% endblock %} {% block content %}
-
+<h1>Admin</h1>
-{% block content %}
+<section id="admin-status"></section>
-    <h1>Admin</h1>
+<section class="actions">
-    <section id="admin-status"></section>
+  <form hx-post="/admin/scan" hx-target="#admin-status" hx-swap="innerHTML">
-    <section class="actions">
+    <button type="submit">Scan</button>
-        <form hx-post="/admin/scan" hx-target="#admin-status" hx-swap="innerHTML">
+  </form>
-            <button type="submit">Scan</button>
+</section>
-        </form>
+<section>
-        <form hx-post="/admin/embed-missing" hx-target="#admin-status" hx-swap="innerHTML">
+  <h2>Embeddings</h2>
-            <button type="submit">Embed</button>
+  <section class="actions">
-        </form>
+    <form
-        <form hx-post="/admin/embed-all" hx-target="#admin-status" hx-swap="innerHTML">
+      hx-post="/admin/embed-missing"
-            <button type="submit">Embed all</button>
+      hx-target="#admin-status"
-        </form>
+      hx-swap="innerHTML"
-    </section>
+    >
-    <section>
+      <button type="submit">Embed</button>
-        <h2>Embeddings</h2>
+    </form>
-        <table>
+    <form
-            <thead>
+      hx-post="/admin/embed-all"
-                <tr>
+      hx-target="#admin-status"
-                    <th>Model</th>
+      hx-swap="innerHTML"
-                    <th>Dimensions</th>
+    >
-                    <th>Embedded</th>
+      <button type="submit">Embed all</button>
-                    <th>Missing</th>
+    </form>
-                    <th>Total chunks</th>
+  </section>
-                </tr>
+  <table>
-            </thead>
+    <thead>
-            <tbody>
+      <tr>
-                {% for item in stats %}
+        <th>Model</th>
-                    <tr>
+        <th>Dimensions</th>
-                        <td>{{ item.model_name }}</td>
+        <th>Embedded</th>
-                        <td>{{ item.dimension }}</td>
+        <th>Missing</th>
-                        <td>{{ item.embedded_chunks }}</td>
+        <th>Total chunks</th>
-                        <td>{{ item.missing_chunks }}</td>
+      </tr>
-                        <td>{{ item.total_chunks }}</td>
+    </thead>
-                    </tr>
+    <tbody>
-                {% endfor %}
+      {% for item in stats %}
-            </tbody>
+      <tr>
-        </table>
+        <td>{{ item.model_name }}</td>
-    </section>
+        <td>{{ item.dimension }}</td>
+        <td>{{ item.embedded_chunks }}</td>
+        <td>{{ item.missing_chunks }}</td>
+        <td>{{ item.total_chunks }}</td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+</section>
+<section>
+  <h2>Protected phrases</h2>
+  <section class="actions actions-grid">
+    <form
+      hx-post="/admin/phrases/generate-all"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Regenerate all phrases</button>
+    </form>
+    <form
+      hx-post="/admin/phrases/generate-missing"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Add missing phrases</button>
+    </form>
+    <form
+      hx-post="/admin/phrases/judge-all"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Judge all phrases</button>
+    </form>
+    <form
+      hx-post="/admin/phrases/judge-missing"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Judge missing phrases</button>
+    </form>
+  </section>
+  <table>
+    <thead>
+      <tr>
+        <th>Candidates</th>
+        <th>Judged</th>
+        <th>Unjudged</th>
+        <th>Protected</th>
+        <th>Books indexed</th>
+        <th>Books generated</th>
+        <th>Books fully judged</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+        <td>{{ phrase_stats.candidate_phrases }}</td>
+        <td>{{ phrase_stats.judged_candidates }}</td>
+        <td>{{ phrase_stats.unjudged_candidates }}</td>
+        <td>{{ phrase_stats.protected_phrases }}</td>
+        <td>{{ phrase_stats.total_books }}</td>
+        <td>{{ phrase_stats.books_with_candidates }}</td>
+        <td>{{ phrase_stats.books_fully_judged }}</td>
+      </tr>
+    </tbody>
+  </table>
+</section>
 {% endblock %}

python/ebook_search/api/templates/book_detail.html

@@ -6,6 +6,9 @@
     {% if source %}
         <h1>{{ source.title }}</h1>
         <p class="meta">{{ source.author or "Unknown author" }}</p>
+        {% if phrase_status_message %}
+            <p class="status">{{ phrase_status_message }}</p>
+        {% endif %}
         <dl class="card">
             <dt>File</dt>
             <dd>{{ source.file_path }}</dd>
@@ -13,7 +16,93 @@
             <dd>{{ chapter_count }}</dd>
             <dt>Chunks</dt>
             <dd>{{ chunk_count }}</dd>
+            <dt>Candidates</dt>
+            <dd>{{ candidate_count }}</dd>
+            <dt>Judged</dt>
+            <dd>{{ judged_candidate_count }}</dd>
+            <dt>Protected</dt>
+            <dd>{{ protected_count }}</dd>
         </dl>
+        <form
+            method="post"
+            action="/books/{{ source.id }}/recalculate-phrases"
+            onsubmit="return confirm('Remove old phrases for this book and generate new candidates?');"
+        >
+            <button type="submit">Recalculate phrases</button>
+        </form>
+        <form
+            method="post"
+            action="/books/{{ source.id }}/judge-phrases"
+            onsubmit="return confirm('Judge candidate phrases for this book with the LLM?');"
+        >
+            <button type="submit"{% if judging_in_progress %} disabled{% endif %}>
+                {% if judging_in_progress %}Judging&hellip;{% else %}Judge phrases{% endif %}
+            </button>
+        </form>
+
+        <section>
+            <h2>Candidate n-grams</h2>
+            {% if candidates %}
+                <table>
+                    <thead>
+                        <tr>
+                            <th>Phrase</th>
+                            <th>Status</th>
+                            <th>Score</th>
+                            <th>Count</th>
+                            <th>Chapters</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for candidate in candidates %}
+                            <tr>
+                                <td>{{ candidate.phrase_text }}</td>
+                                <td>
+                                    {% if candidate.llm_judged %}
+                                        {% if candidate.llm_keep %}Kept{% else %}Rejected{% endif %}
+                                    {% else %}
+                                        Candidate
+                                    {% endif %}
+                                </td>
+                                <td>{{ "%.2f"|format(candidate.candidate_score) }}</td>
+                                <td>{{ candidate.raw_count }}</td>
+                                <td>{{ candidate.chapter_count }}</td>
+                            </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+            {% else %}
+                <p>No candidate n-grams.</p>
+            {% endif %}
+        </section>
+
+        <section>
+            <h2>Protected phrases</h2>
+            {% if protected_phrases %}
+                <table>
+                    <thead>
+                        <tr>
+                            <th>Phrase</th>
+                            <th>Type</th>
+                            <th>Confidence</th>
+                            <th>Importance</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for phrase in protected_phrases %}
+                            <tr>
+                                <td>{{ phrase.phrase_text }}</td>
+                                <td>{{ phrase.phrase_type or "phrase" }}</td>
+                                <td>{{ "%.2f"|format(phrase.confidence) }}</td>
+                                <td>{{ "%.2f"|format(phrase.importance) }}</td>
+                            </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+            {% else %}
+                <p>No protected phrases.</p>
+            {% endif %}
+        </section>
     {% else %}
         <h1>Book not found</h1>
     {% endif %}

python/ebook_search/api/templates/partials/results.html

@@ -82,6 +82,14 @@
                         </div>
                     {% endif %}
                 </dl>
+                {% if result.matched_phrases %}
+                    <p class="phrase-matches">
+                        <span class="phrase-matches-label">boosted by</span>
+                        {% for phrase in result.matched_phrases %}
+                            <span class="phrase-match">{{ phrase }}</span>
+                        {% endfor %}
+                    </p>
+                {% endif %}
             </li>
         {% endfor %}
     </ol>

python/ebook_search/api/templates/search.html

@@ -7,12 +7,24 @@
     <h1>Search</h1>
     <form class="card" hx-post="/search" hx-target="#results" hx-swap="innerHTML">
         <label for="query">What are you looking for?</label>
-        <textarea id="query" name="query" rows="4" placeholder="Ask a question or paste a passage…" required></textarea>
+        <textarea id="query" name="query" rows="4" placeholder="Ask a question or paste a passage…" required
+            onkeydown="if (event.key === 'Enter' && !event.shiftKey) { event.preventDefault(); this.form.requestSubmit(); }"></textarea>
         <div class="form-row">
-            <label class="check">
+            <div class="search-toggles">
-                <input type="checkbox" name="rerank" value="true" {% if config.rerank.enabled %}checked{% endif %}>
+                <label class="check">
-                Rerank
+                    <input type="checkbox" name="rerank" value="true" {% if config.rerank.enabled %}checked{% endif %}>
-            </label>
+                    Rerank
+                </label>
+                <label class="check">
+                    <input
+                        type="checkbox"
+                        name="phrase_matching"
+                        value="true"
+                        {% if config.phrase_matching_enabled %}checked{% endif %}
+                    >
+                    Phrase matching
+                </label>
+            </div>
             <button type="submit">Search</button>
         </div>
     </form>

python/ebook_search/bm25_corpus.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import json
 import logging
 import shutil
@@ -17,7 +18,7 @@ from sqlalchemy import func, select, union_all
 from python.orm.richie import EbookChapter, EbookChunk, EbookSource
 
 if TYPE_CHECKING:
-    from sqlalchemy.orm import Session
+    from sqlalchemy.ext.asyncio import AsyncSession
 
     from python.ebook_search.config import EbookSearchConfig
 
@@ -73,14 +74,14 @@ def get_current_bm25_index(index_path: Path) -> Path:
     return index_path
 
 
-def ensure_bm25_corpus(session: Session, config: EbookSearchConfig) -> None:
+async def ensure_bm25_corpus(session: AsyncSession, config: EbookSearchConfig) -> None:
     """Create or refresh the persisted BM25 corpus when it is missing or stale."""
     index_path = bm25_index_path(config)
     manifest = read_bm25_manifest(index_path)
-    db_updated_at = corpus_last_updated_at(session)
+    db_updated_at = await corpus_last_updated_at(session)
     if not bm25_index_exists(index_path, manifest):
         logger.info("ebook_bm25_index_missing path=%s", index_path)
-        refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
+        await refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
         return
     if db_updated_at is not None and manifest is not None and manifest.created_at < db_updated_at:
         logger.info(
@@ -89,7 +90,7 @@ def ensure_bm25_corpus(session: Session, config: EbookSearchConfig) -> None:
             manifest.created_at.isoformat(),
             db_updated_at.isoformat(),
         )
-        refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
+        await refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
         return
     logger.info(
         "ebook_bm25_index_current path=%s chunks=%s created_at=%s",
@@ -99,21 +100,24 @@ def ensure_bm25_corpus(session: Session, config: EbookSearchConfig) -> None:
     )
 
 
-def refresh_bm25_corpus(
+async def refresh_bm25_corpus(
-    session: Session,
+    session: AsyncSession,
     config: EbookSearchConfig,
     *,
     db_updated_at: datetime | None = None,
 ) -> BM25Manifest:
-    """Rebuild and persist the BM25 corpus from the current database chunks."""
+    """Rebuild and persist the BM25 corpus from the current database chunks.
+
+    The index build is CPU and disk work, so it runs in a worker thread.
+    """
     index_path = bm25_index_path(config)
-    records, texts = fetch_bm25_corpus_records(session)
+    records, texts = await fetch_bm25_corpus_records(session)
     manifest = BM25Manifest(
         created_at=datetime.now(tz=UTC),
-        db_updated_at=db_updated_at if db_updated_at is not None else corpus_last_updated_at(session),
+        db_updated_at=db_updated_at if db_updated_at is not None else await corpus_last_updated_at(session),
         chunk_count=len(records),
     )
-    write_bm25_corpus(index_path, records, texts, manifest)
+    await asyncio.to_thread(write_bm25_corpus, index_path, records, texts, manifest)
     logger.info(
         "ebook_bm25_index_refreshed path=%s chunks=%s created_at=%s",
         index_path,
@@ -164,7 +168,7 @@ def score_bm25_corpus(query: str, corpus: BM25Corpus, *, limit: int) -> list[tup
     return results
 
 
-def fetch_bm25_corpus_records(session: Session) -> tuple[list[dict[str, object]], list[str]]:
+async def fetch_bm25_corpus_records(session: AsyncSession) -> tuple[list[dict[str, object]], list[str]]:
     """Fetch persistable BM25 corpus records and their matching index texts from the database.
 
     search_text is only needed to build the index, so it is returned separately instead of
@@ -174,6 +178,7 @@ def fetch_bm25_corpus_records(session: Session) -> tuple[list[dict[str, object]]
         select(
             EbookChunk.id.label("chunk_id"),
             EbookChunk.text.label("text"),
+            EbookSource.id.label("source_id"),
             EbookSource.title.label("source_title"),
             EbookSource.author.label("source_author"),
             EbookChapter.title.label("chapter_title"),
@@ -187,21 +192,21 @@ def fetch_bm25_corpus_records(session: Session) -> tuple[list[dict[str, object]]
     )
     records: list[dict[str, object]] = []
     texts: list[str] = []
-    for row in session.execute(statement).mappings():
+    for row in (await session.execute(statement)).mappings():
         record = dict(row)
         texts.append(str(record.pop("bm25_text")))
         records.append(record)
     return records, texts
 
 
-def corpus_last_updated_at(session: Session) -> datetime | None:
+async def corpus_last_updated_at(session: AsyncSession) -> datetime | None:
     """Return the latest source/chapter/chunk update timestamp relevant to BM25 text."""
     update_times = union_all(
         select(func.max(EbookSource.updated).label("updated")),
         select(func.max(EbookChapter.updated).label("updated")),
         select(func.max(EbookChunk.updated).label("updated")),
     ).subquery()
-    return session.scalar(select(func.max(update_times.c.updated)))
+    return await session.scalar(select(func.max(update_times.c.updated)))
 
 
 def write_bm25_corpus(

python/ebook_search/config.py

@@ -86,6 +86,25 @@ class EbookSearchConfig(BaseSettings):
     validate_citations_enabled: bool = True
     bm25_index_dir: str = ".ebook_search_bm25"
     bm25_refresh_delay_seconds: int = 60
+    protected_phrase_max_candidates_per_book: int = 5000
+    protected_phrase_llm_candidates_per_book: int = 500
+    protected_phrase_extraction_workers: int = 16
+    phrase_judge_book_workers: int = 20
+    phrase_judge_phrase_workers: int = 100
+    protected_phrase_confidence_threshold: float = 0.80
+    phrase_matching_enabled: bool = True
+    phrase_hit_boost: float = 0.25
+    phrase_min_tokens: int = 2
+    phrase_max_tokens: int = 5
+    phrase_max_entity_tokens: int = 8
+    phrase_raw_ngram_min_count: int = 2
+    phrase_raw_count_score_threshold: int = 3
+    phrase_raw_count_high_score_threshold: int = 10
+    phrase_chapter_count_score_threshold: int = 2
+    phrase_chapter_count_high_score_threshold: int = 5
+    phrase_target_protected_per_book: int = 100
+    phrase_default_allow_nested: bool = False
+    phrase_default_suppress_children: bool = True
 
     @field_validator("library_paths", mode="before")
     @classmethod

python/ebook_search/docker/Dockerfile

@@ -0,0 +1,49 @@
+FROM python:3.14-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    APP_DIR=/home/richie/dotfiles \
+    EBOOK_SEARCH_HOST=0.0.0.0 \
+    EBOOK_SEARCH_PORT=8070 \
+    EBOOK_SEARCH_BM25_INDEX_DIR=/data/bm25
+
+WORKDIR ${APP_DIR}
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends build-essential curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY pyproject.toml README.md LICENSE ./
+COPY python ./python
+
+RUN python -m pip install --upgrade pip \
+    && python -m pip install \
+        "alembic" \
+        "beautifulsoup4" \
+        "bm25s" \
+        "ebooklib" \
+        "fastapi" \
+        "httpx" \
+        "jinja2" \
+        "pgvector" \
+        "psycopg[binary]" \
+        "pydantic" \
+        "pydantic-settings" \
+        "python-multipart" \
+        "sqlalchemy[asyncio]" \
+        "tiktoken" \
+        "typer" \
+        "uvicorn[standard]" \
+        "yake" \
+    && python -m pip install --no-deps --editable "${APP_DIR}"
+
+RUN useradd --create-home --uid 10001 app \
+    && mkdir -p /data \
+    && chown -R app:app /home/richie /data
+
+USER app
+
+EXPOSE 8070
+
+CMD ["sh", "-c", "exec python -m python.ebook_search.api.main --host \"${EBOOK_SEARCH_HOST}\" --port \"${EBOOK_SEARCH_PORT}\" --log-level \"${EBOOK_SEARCH_LOG_LEVEL:-INFO}\""]

python/ebook_search/docker/README.md

@@ -0,0 +1,40 @@
+# Ebook Search Docker
+
+Run the EPUB search app against the existing Postgres database on `jeeves`:
+
+```sh
+ebook-search-containers start --library-path /path/to/epubs --build
+```
+
+All ebook-search Docker files live in this directory:
+
+- `Dockerfile`
+- `docker-compose.yml`
+- `containers.py`
+- `container.py`
+
+The app listens on `http://localhost:8070`.
+
+Useful lifecycle commands:
+
+```sh
+ebook-search-containers build
+ebook-search-containers start --library-path /path/to/epubs
+ebook-search-containers logs
+ebook-search-containers ps
+ebook-search-containers stop
+```
+
+Direct compose usage from the repo root:
+
+```sh
+docker compose -f python/ebook_search/docker/docker-compose.yml ps
+```
+
+The compose service also loads the repo root `.env` into the container via `env_file`.
+
+Mount your EPUB directory by setting `EBOOK_LIBRARY_HOST_PATH` in an env file or on the command line. The container sees it as `/library`, and `EBOOK_SEARCH_LIBRARY_PATHS` is set to `/library` inside the container.
+
+Database connection settings are controlled by `RICHIE_DB`, `RICHIE_HOST`, `RICHIE_PORT`, `RICHIE_USER`, and `RICHIE_PASSWORD`. The default host is `jeeves`.
+
+Startup runs the Richie Alembic migrations automatically after creating the `main` schema and `vector` extension.

python/ebook_search/docker/__init__.py

@@ -0,0 +1 @@
+"""Docker packaging and lifecycle tooling for ebook search."""

python/ebook_search/docker/containers.py

@@ -0,0 +1,229 @@
+"""Docker container lifecycle management for ebook search."""
+
+from __future__ import annotations
+
+import logging
+import os
+import subprocess
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from python.common import configure_logger, get_repo_dir
+
+logger = logging.getLogger(__name__)
+
+
+def get_compose_file() -> Path:
+    """Return the path to the docker-compose.yml file."""
+    return Path(__file__).resolve().with_name("docker-compose.yml")
+
+
+def compose_base_args() -> list[str]:
+    """Return the common docker compose arguments for the ebook search stack."""
+    return ["compose", "-f", str(get_compose_file())]
+
+
+def docker_run(
+    arguments: list[str],
+    *,
+    env: dict[str, str] | None = None,
+    capture_output: bool = False,
+) -> subprocess.CompletedProcess[str]:
+    """Run docker with repo-root cwd and consistent error handling."""
+    logger.info("docker %s", " ".join(arguments))
+    return subprocess.run(
+        ["docker", *arguments],
+        cwd=get_repo_dir(),
+        env=env,
+        text=True,
+        check=False,
+        capture_output=capture_output,
+    )
+
+
+def compose_env(*, library_path: Path | None = None, port: int | None = None) -> dict[str, str]:
+    """Return environment variables passed to docker compose."""
+    env = os.environ.copy()
+    if library_path is not None:
+        resolved_library = library_path.expanduser().resolve()
+        if not resolved_library.exists():
+            msg = f"EPUB library path does not exist: {resolved_library}"
+            raise FileNotFoundError(msg)
+        env["EBOOK_LIBRARY_HOST_PATH"] = str(resolved_library)
+    if port is not None:
+        env["EBOOK_SEARCH_PORT"] = str(port)
+    return env
+
+
+def ensure_compose_file() -> None:
+    """Raise if the ebook search compose file is missing."""
+    if not get_compose_file().is_file():
+        msg = f"Compose file not found: {get_compose_file()}"
+        raise FileNotFoundError(msg)
+
+
+def build_image() -> None:
+    """Build the ebook search app image."""
+    ensure_compose_file()
+    result = docker_run([*compose_base_args(), "build"])
+    if result.returncode != 0:
+        msg = "Failed to build ebook search image"
+        raise RuntimeError(msg)
+
+
+def start_stack(
+    *,
+    library_path: Path | None = None,
+    port: int | None = None,
+    build: bool = False,
+) -> None:
+    """Start the ebook search Docker compose stack."""
+    ensure_compose_file()
+    env = compose_env(library_path=library_path, port=port)
+    if build:
+        build_image()
+    result = docker_run(
+        [*compose_base_args(), "up", "-d"],
+        env=env,
+    )
+    if result.returncode != 0:
+        msg = f"Ebook search stack failed to start with code {result.returncode}"
+        raise RuntimeError(msg)
+    logger.info("Ebook search started.")
+
+
+def stop_stack(
+    *,
+    volumes: bool = False,
+) -> None:
+    """Stop and remove ebook search containers."""
+    ensure_compose_file()
+    command = [*compose_base_args(), "down"]
+    if volumes:
+        command.append("-v")
+    result = docker_run(command)
+    if result.returncode != 0:
+        msg = f"Ebook search stack failed to stop with code {result.returncode}"
+        raise RuntimeError(msg)
+
+
+def logs_stack(
+    *,
+    service: str | None = None,
+    tail: int = 100,
+    follow: bool = False,
+) -> str | None:
+    """Return recent logs from the ebook search stack."""
+    ensure_compose_file()
+    command = [*compose_base_args(), "logs", "--tail", str(tail)]
+    if follow:
+        command.append("--follow")
+    if service:
+        command.append(service)
+    result = docker_run(command, capture_output=not follow)
+    if result.returncode != 0:
+        return None
+    if follow:
+        return ""
+    return result.stdout + result.stderr
+
+
+def ps_stack() -> str | None:
+    """Return docker compose ps output for the ebook search stack."""
+    ensure_compose_file()
+    result = docker_run([*compose_base_args(), "ps"], capture_output=True)
+    if result.returncode != 0:
+        return None
+    return result.stdout + result.stderr
+
+
+app = typer.Typer(help="Ebook search Docker container management.", no_args_is_help=True)
+
+
+@app.command()
+def build() -> None:
+    """Build the ebook search Docker image."""
+    build_image()
+
+
+@app.command()
+def start(
+    library_path: Annotated[Path | None, typer.Option(help="Override host path containing EPUB files.")] = None,
+    port: Annotated[int | None, typer.Option(help="Override host port for the web UI.")] = None,
+    *,
+    build: Annotated[bool, typer.Option("--build", help="Build the image before starting.")] = False,
+    log_level: Annotated[str, typer.Option(help="Log level.")] = "INFO",
+) -> None:
+    """Start the ebook search container."""
+    configure_logger(log_level)
+    start_stack(
+        library_path=library_path,
+        port=port,
+        build=build,
+    )
+
+
+@app.command()
+def stop(
+    *,
+    volumes: Annotated[bool, typer.Option("--volumes", help="Also remove ebook search data volumes.")] = False,
+    log_level: Annotated[str, typer.Option(help="Log level.")] = "INFO",
+) -> None:
+    """Stop and remove ebook search containers."""
+    configure_logger(log_level)
+    stop_stack(volumes=volumes)
+
+
+@app.command()
+def restart(
+    library_path: Annotated[Path | None, typer.Option(help="Override host path containing EPUB files.")] = None,
+    port: Annotated[int | None, typer.Option(help="Override host port for the web UI.")] = None,
+    *,
+    build: Annotated[bool, typer.Option("--build", help="Build the image before starting.")] = False,
+    log_level: Annotated[str, typer.Option(help="Log level.")] = "INFO",
+) -> None:
+    """Restart the ebook search stack."""
+    configure_logger(log_level)
+    stop_stack()
+    start_stack(
+        library_path=library_path,
+        port=port,
+        build=build,
+    )
+
+
+@app.command()
+def logs(
+    service: Annotated[str | None, typer.Option(help="Service name, or omit for all services.")] = None,
+    tail: Annotated[int, typer.Option(help="Number of recent log lines.")] = 100,
+    *,
+    follow: Annotated[bool, typer.Option("--follow", "-f", help="Follow logs.")] = False,
+) -> None:
+    """Show recent ebook search container logs."""
+    output = logs_stack(service=service, tail=tail, follow=follow)
+    if output is None:
+        typer.echo("No ebook search containers found.")
+        raise typer.Exit(code=1)
+    if output:
+        typer.echo(output)
+
+
+@app.command("ps")
+def ps() -> None:
+    """Show ebook search container status."""
+    output = ps_stack()
+    if output is None:
+        typer.echo("No ebook search containers found.")
+        raise typer.Exit(code=1)
+    typer.echo(output)
+
+
+def cli() -> None:
+    """Typer entry point."""
+    app()
+
+
+if __name__ == "__main__":
+    cli()

python/ebook_search/docker/docker-compose.yml

@@ -0,0 +1,36 @@
+name: ebook-search
+
+services:
+  ebook-search:
+    build:
+      context: ../../..
+      dockerfile: python/ebook_search/docker/Dockerfile
+    image: ebook-search:latest
+    restart: unless-stopped
+    ports:
+      - "${EBOOK_SEARCH_PORT:-8070}:8070"
+    extra_hosts:
+      - "jeeves:192.168.90.40"
+    env_file:
+      - ../../../.env
+    environment:
+      EBOOK_SEARCH_HOST: "0.0.0.0"
+      EBOOK_SEARCH_PORT: "8070"
+      EBOOK_SEARCH_LIBRARY_PATHS: "/library"
+      EBOOK_SEARCH_BM25_INDEX_DIR: "/data/bm25"
+    volumes:
+      - "${EBOOK_LIBRARY_HOST_PATH:-/home/richie/ebooks}:/library:ro"
+      - ebook-search-data:/data
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -fsS http://127.0.0.1:8070/health >/dev/null || exit 1",
+        ]
+      interval: 30s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
+
+volumes:
+  ebook-search-data:

python/ebook_search/embeddings.py

@@ -23,7 +23,8 @@ logger = logging.getLogger(__name__)
 if TYPE_CHECKING:
     from collections.abc import Sequence
 
-    from sqlalchemy.orm import Session
+    import httpx
+    from sqlalchemy.ext.asyncio import AsyncSession
 
     from python.ebook_search.config import EbookSearchConfig
 
@@ -65,7 +66,11 @@ class EmbeddingModelStats:
         return max(self.total_chunks - self.embedded_chunks, 0)
 
 
-def embed_texts(texts: Sequence[str], config: EbookSearchConfig) -> list[list[float]]:
+async def embed_texts(
+    client: httpx.AsyncClient,
+    texts: Sequence[str],
+    config: EbookSearchConfig,
+) -> list[list[float]]:
     """Embed text with the configured vLLM embedding model."""
     logger.info(
         "ebook_embed_request_start base_url=%s model=%s count=%s",
@@ -73,7 +78,7 @@ def embed_texts(texts: Sequence[str], config: EbookSearchConfig) -> list[list[fl
         config.embedding_model,
         len(texts),
     )
-    vectors = request_embeddings(texts, config)
+    vectors = await request_embeddings(client, texts, config)
     expected_dimension = MODEL_DIMENSIONS[config.embedding_model]
     for vector in vectors:
         if len(vector) != expected_dimension:
@@ -88,28 +93,28 @@ def embed_texts(texts: Sequence[str], config: EbookSearchConfig) -> list[list[fl
     return vectors
 
 
-def embed_query(query: str, config: EbookSearchConfig) -> list[float]:
+async def embed_query(client: httpx.AsyncClient, query: str, config: EbookSearchConfig) -> list[float]:
     """Embed a search query with the Qwen retrieval instruction."""
     instructed_query = f"Instruct: Retrieve relevant passages for the query.\nQuery: {query}"
-    return embed_texts([instructed_query], config)[0]
+    return (await embed_texts(client, [instructed_query], config))[0]
 
 
-def ensure_embedding_models(session: Session) -> None:
+async def ensure_embedding_models(session: AsyncSession) -> None:
     """Ensure supported embedding model rows exist."""
     for name, dimension in MODEL_DIMENSIONS.items():
-        existing = session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == name))
+        existing = await session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == name))
         if existing is None:
             session.add(EbookEmbeddingModel(name=name, dimension=dimension, is_default=name == "qwen3-embedding-0.6b"))
             logger.info("ebook_embedding_model_created model=%s dimension=%s", name, dimension)
-    session.flush()
+    await session.flush()
 
 
-def embedding_model_stats(session: Session) -> list[EmbeddingModelStats]:
+async def embedding_model_stats(session: AsyncSession) -> list[EmbeddingModelStats]:
     """Return embedding coverage counts for every supported model."""
-    total_chunks = session.scalar(select(func.count(EbookChunk.id))) or 0
+    total_chunks = await session.scalar(select(func.count(EbookChunk.id))) or 0
     models = {
         model.name: model
-        for model in session.scalars(
+        for model in await session.scalars(
             select(EbookEmbeddingModel)
             .where(EbookEmbeddingModel.name.in_(MODEL_DIMENSIONS))
             .order_by(EbookEmbeddingModel.name)
@@ -122,7 +127,7 @@ def embedding_model_stats(session: Session) -> list[EmbeddingModelStats]:
         embedded_chunks = 0
         if model is not None:
             table = get_embedding_table(dimension)
-            embedded_chunks = session.scalar(select(func.count(table.id)).where(table.model_id == model.id)) or 0
+            embedded_chunks = await session.scalar(select(func.count(table.id)).where(table.model_id == model.id)) or 0
         stats.append(
             EmbeddingModelStats(
                 model_name=model_name,
@@ -134,10 +139,10 @@ def embedding_model_stats(session: Session) -> list[EmbeddingModelStats]:
     return stats
 
 
-def embed_missing_chunks(session: Session, config: EbookSearchConfig) -> int:
+async def embed_missing_chunks(session: AsyncSession, client: httpx.AsyncClient, config: EbookSearchConfig) -> int:
     """Embed chunks missing embeddings for the configured model."""
-    ensure_embedding_models(session)
+    await ensure_embedding_models(session)
-    model = session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == config.embedding_model))
+    model = await session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == config.embedding_model))
     if model is None:
         supported_models = ", ".join(MODEL_DIMENSIONS)
         msg = f"Unknown embedding model: {config.embedding_model}. Supported models: {supported_models}"
@@ -145,7 +150,7 @@ def embed_missing_chunks(session: Session, config: EbookSearchConfig) -> int:
 
     table = get_embedding_table(model.dimension)
     chunks = list(
-        session.scalars(
+        await session.scalars(
             select(EbookChunk)
             .outerjoin(table, (table.chunk_id == EbookChunk.id) & (table.model_id == model.id))
             .where(table.id.is_(None))
@@ -158,13 +163,13 @@ def embed_missing_chunks(session: Session, config: EbookSearchConfig) -> int:
         return 0
 
     logger.info("ebook_embed_missing_batch_start model=%s count=%s", config.embedding_model, len(chunks))
-    vectors = embed_texts([chunk.text for chunk in chunks], config)
+    vectors = await embed_texts(client, [chunk.text for chunk in chunks], config)
     rows = [
         {"chunk_id": chunk.id, "model_id": model.id, "embedding": vector}
         for chunk, vector in zip(chunks, vectors, strict=True)
     ]
     statement = insert(table).values(rows).on_conflict_do_nothing(index_elements=["chunk_id", "model_id"])
-    session.execute(statement)
+    await session.execute(statement)
-    session.flush()
+    await session.flush()
     logger.info("ebook_embed_missing_batch_complete model=%s count=%s", config.embedding_model, len(rows))
     return len(rows)

python/ebook_search/ingest.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import hashlib
 import logging
 from dataclasses import dataclass
@@ -13,6 +14,7 @@ import tiktoken
 from sqlalchemy import or_, select
 
 from python.ebook_search.epub_parse import parse_epub
+from python.ebook_search.protected_phrases.matching import index_chunk_phrase_mentions_for_book
 from python.orm.richie import EbookChapter, EbookChunk, EbookSource
 
 logger = logging.getLogger(__name__)
@@ -20,7 +22,7 @@ DEFAULT_CHUNK_TOKENS = 700
 DEFAULT_CHUNK_OVERLAP = 100
 
 if TYPE_CHECKING:
-    from sqlalchemy.orm import Session
+    from sqlalchemy.ext.asyncio import AsyncSession
 
     from python.ebook_search.config import EbookSearchConfig
     from python.ebook_search.epub_parse import ParsedChapter
@@ -72,90 +74,115 @@ def chunk_text(
     return [chunk for chunk in chunks if chunk.text]
 
 
-def ingest_configured_paths(session: Session, config: EbookSearchConfig) -> int:
+def find_library_epubs(library_path: str) -> tuple[Path, list[Path] | None]:
+    """Resolve one configured library path and collect its EPUB files (blocking filesystem walk).
+
+    Returns:
+        tuple[Path, list[Path] | None]: The expanded path and its EPUB files, or ``None`` when
+        the path is neither an EPUB file nor a directory.
+    """
+    path = Path(library_path).expanduser()
+    if path.is_file() and path.suffix.lower() == ".epub":
+        return path, [path]
+    if path.is_dir():
+        return path, sorted(path.rglob("*.epub"))
+    return path, None
+
+
+async def ingest_configured_paths(session: AsyncSession, config: EbookSearchConfig) -> int:
     """Ingest every EPUB found under configured library paths."""
     count = 0
     for library_path in config.library_paths:
-        path = Path(library_path).expanduser()
+        path, epub_paths = await asyncio.to_thread(find_library_epubs, library_path)
         logger.info("ebook_ingest_path_start path=%s", path)
-        if path.is_file() and path.suffix.lower() == ".epub":
+        if epub_paths is None:
-            count += int(ingest_file(session, path, config))
-        elif path.is_dir():
-            for epub_path in sorted(path.rglob("*.epub")):
-                count += int(ingest_file(session, epub_path, config))
-        else:
             logger.warning("ebook_ingest_path_missing path=%s", path)
+            continue
+        for epub_path in epub_paths:
+            count += int(await ingest_file(session, epub_path, config))
     logger.info("ebook_ingest_paths_complete changed_files=%s configured_paths=%s", count, len(config.library_paths))
     return count
 
 
-def ingest_file(session: Session, path: Path, config: EbookSearchConfig) -> bool:
+def resolve_ingest_path(path: Path) -> Path:
+    """Expand and resolve an ingest path (blocking filesystem call)."""
+    return path.expanduser().resolve()
+
+
+async def ingest_file(session: AsyncSession, path: Path, config: EbookSearchConfig) -> bool:
     """Ingest one EPUB file. Return True when the database changed."""
-    resolved_path = path.expanduser().resolve()
+    try:
-    logger.info("ebook_ingest_file_start path=%s", resolved_path)
+        resolved_path = await asyncio.to_thread(resolve_ingest_path, path)
-    file_hash = sha256_file(resolved_path)
+        logger.info("ebook_ingest_file_start path=%s", resolved_path)
-    existing = find_existing_source(session, resolved_path, file_hash)
+        file_hash = await asyncio.to_thread(sha256_file, resolved_path)
-    if existing is not None and existing.file_sha256 == file_hash:
+        existing = await find_existing_source(session, resolved_path, file_hash)
+        if existing is not None and existing.file_sha256 == file_hash:
+            stat = resolved_path.stat()
+            existing.file_path = str(resolved_path)
+            existing.file_mtime = datetime.fromtimestamp(stat.st_mtime, tz=UTC)
+            existing.file_size = stat.st_size
+            await session.flush()
+            logger.info("ebook_ingest_file_unchanged source_id=%s path=%s", existing.id, resolved_path)
+            return False
+        if existing is not None:
+            logger.info("ebook_ingest_file_replacing source_id=%s path=%s", existing.id, resolved_path)
+            await session.delete(existing)
+            await session.flush()
+
         stat = resolved_path.stat()
-        existing.file_path = str(resolved_path)
+        parsed = await asyncio.to_thread(parse_epub, resolved_path)
-        existing.file_mtime = datetime.fromtimestamp(stat.st_mtime, tz=UTC)
+        source = EbookSource(
-        existing.file_size = stat.st_size
+            title=parsed.title,
-        session.flush()
+            author=parsed.author,
-        logger.info("ebook_ingest_file_unchanged source_id=%s path=%s", existing.id, resolved_path)
+            language=parsed.language,
-        return False
+            publisher=parsed.publisher,
-    if existing is not None:
+            identifier=parsed.identifier,
-        logger.info("ebook_ingest_file_replacing source_id=%s path=%s", existing.id, resolved_path)
+            file_path=str(resolved_path),
-        session.delete(existing)
+            file_sha256=file_hash,
-        session.flush()
+            file_mtime=datetime.fromtimestamp(stat.st_mtime, tz=UTC),
-
+            file_size=stat.st_size,
-    stat = resolved_path.stat()
-    parsed = parse_epub(resolved_path)
-    source = EbookSource(
-        title=parsed.title,
-        author=parsed.author,
-        language=parsed.language,
-        publisher=parsed.publisher,
-        identifier=parsed.identifier,
-        file_path=str(resolved_path),
-        file_sha256=file_hash,
-        file_mtime=datetime.fromtimestamp(stat.st_mtime, tz=UTC),
-        file_size=stat.st_size,
-    )
-    session.add(source)
-    session.flush()
-
-    chunk_index = 0
-    for spine_index, parsed_chapter in enumerate(parsed.chapters):
-        chapter = EbookChapter(
-            source_id=source.id,
-            spine_index=spine_index,
-            title=parsed_chapter.title,
-            href=parsed_chapter.href,
         )
-        session.add(chapter)
+        session.add(source)
-        session.flush()
+        await session.flush()
-        chunk_index = add_chapter_chunks(session, source, chapter, parsed_chapter, chunk_index, config)
 
-    session.flush()
+        chunk_index = 0
-    logger.info(
+        for spine_index, parsed_chapter in enumerate(parsed.chapters):
-        "ebook_ingest_file_complete source_id=%s path=%s chapters=%s chunks=%s",
+            chapter = EbookChapter(
-        source.id,
+                source_id=source.id,
-        resolved_path,
+                spine_index=spine_index,
-        len(parsed.chapters),
+                title=parsed_chapter.title,
-        chunk_index,
+                href=parsed_chapter.href,
-    )
+            )
-    return True
+            session.add(chapter)
+            await session.flush()
+            chunk_index = add_chapter_chunks(session, source, chapter, parsed_chapter, chunk_index, config)
+
+        await session.commit()
+        mention_count = await index_chunk_phrase_mentions_for_book(session, source.id, config)
+        logger.info(
+            "ebook_ingest_file_complete source_id=%s path=%s chapters=%s chunks=%s phrase_mentions=%s",
+            source.id,
+            resolved_path,
+            len(parsed.chapters),
+            chunk_index,
+            mention_count,
+        )
+    except Exception:
+        logger.exception(f"ebook_ingest_file_error path={path}")
+        return False
+    else:
+        return True
 
 
-def find_existing_source(session: Session, path: Path, file_hash: str) -> EbookSource | None:
+async def find_existing_source(session: AsyncSession, path: Path, file_hash: str) -> EbookSource | None:
     """Find an existing source by canonical path or file hash."""
-    return session.scalar(
+    return await session.scalar(
         select(EbookSource).where(or_(EbookSource.file_path == str(path), EbookSource.file_sha256 == file_hash))
     )
 
 
 def add_chapter_chunks(
-    session: Session,
+    session: AsyncSession,
     source: EbookSource,
     chapter: EbookChapter,
     parsed_chapter: ParsedChapter,

python/ebook_search/llm_interface.py

@@ -22,10 +22,26 @@ def auth_headers(api_key: str) -> dict[str, str]:
     return {"Authorization": f"Bearer {api_key}"}
 
 
-def request_embeddings(texts: Sequence[str], config: EbookSearchConfig) -> list[list[float]]:
+async def request_embeddings(
-    """Request embeddings from the configured OpenAI-compatible endpoint."""
+    client: httpx.AsyncClient,
+    texts: Sequence[str],
+    config: EbookSearchConfig,
+) -> list[list[float]]:
+    """Request embeddings from the configured OpenAI-compatible endpoint.
+
+    Args:
+        client (httpx.AsyncClient): Shared async client for LLM calls.
+        texts (Sequence[str]): Texts to embed.
+        config (EbookSearchConfig): Runtime settings supplying the endpoint, model, and auth.
+
+    Returns:
+        list[list[float]]: One embedding vector per input text.
+
+    Raises:
+        RuntimeError: If the request fails or the response cannot be parsed.
+    """
     try:
-        response = httpx.post(
+        response = await client.post(
             f"{config.embedding_base_url.rstrip('/')}/embeddings",
             headers=auth_headers(config.embedding_api_key),
             json={"model": config.embedding_model, "input": list(texts)},
@@ -44,10 +60,15 @@ def request_embeddings(texts: Sequence[str], config: EbookSearchConfig) -> list[
         raise RuntimeError(msg) from error
 
 
-def check_embedding_endpoint(config: EbookSearchConfig, *, timeout_seconds: float = 5.0) -> bool:
+async def check_embedding_endpoint(
+    client: httpx.AsyncClient,
+    config: EbookSearchConfig,
+    *,
+    timeout_seconds: float = 5.0,
+) -> bool:
     """Return whether the configured embedding endpoint answers a model listing."""
     try:
-        response = httpx.get(
+        response = await client.get(
             f"{config.embedding_base_url.rstrip('/')}/models",
             headers=auth_headers(config.embedding_api_key),
             timeout=timeout_seconds,
@@ -59,10 +80,15 @@ def check_embedding_endpoint(config: EbookSearchConfig, *, timeout_seconds: floa
     return True
 
 
-def check_chat_endpoint(config: EbookSearchConfig, *, timeout_seconds: float = 5.0) -> bool:
+async def check_chat_endpoint(
+    client: httpx.AsyncClient,
+    config: EbookSearchConfig,
+    *,
+    timeout_seconds: float = 5.0,
+) -> bool:
     """Return whether the configured chat (answering) endpoint answers a model listing."""
     try:
-        response = httpx.get(
+        response = await client.get(
             f"{config.vllm_base_url.rstrip('/')}/models",
             headers=auth_headers(config.vllm_api_key),
             timeout=timeout_seconds,
@@ -98,18 +124,29 @@ def embedding_vectors_from_response(body: object) -> list[list[float]]:
     return vectors
 
 
-def request_rerank(
+async def request_rerank(
+    client: httpx.AsyncClient,
     query: str,
     documents: Sequence[str],
     config: RerankConfig,
 ) -> object | None:
-    """Request rerank scores from the configured vLLM endpoint."""
+    """Request rerank scores from the configured vLLM endpoint.
+
+    Args:
+        client (httpx.AsyncClient): Shared async client for LLM calls.
+        query (str): Query the documents are scored against.
+        documents (Sequence[str]): Candidate documents to score.
+        config (RerankConfig): Rerank endpoint settings.
+
+    Returns:
+        object | None: The decoded response body, or ``None`` when it is not valid JSON.
+    """
     payload = {
         "model": config.model,
         "query": query,
         "documents": list(documents),
     }
-    response = httpx.post(
+    response = await client.post(
         f"{config.base_url.rstrip('/')}/rerank",
         json=payload,
         timeout=config.timeout_seconds,
@@ -122,13 +159,26 @@ def request_rerank(
         return None
 
 
-def request_chat_completion(
+async def request_chat_completion(
+    client: httpx.AsyncClient,
     config: EbookSearchConfig,
     messages: Sequence[dict[str, str]],
 ) -> str:
-    """Request a chat completion from the configured OpenAI-compatible endpoint."""
+    """Request a chat completion over a shared async client.
+
+    Args:
+        client (httpx.AsyncClient): Shared async client whose connection pool bounds concurrency.
+        config (EbookSearchConfig): Runtime settings supplying the endpoint, model, and auth.
+        messages (Sequence[dict[str, str]]): OpenAI-style chat messages.
+
+    Returns:
+        str: The assistant message text.
+
+    Raises:
+        RuntimeError: If the request fails or the response cannot be parsed.
+    """
     try:
-        response = httpx.post(
+        response = await client.post(
             f"{config.vllm_base_url.rstrip('/')}/chat/completions",
             headers=auth_headers(config.vllm_api_key),
             json={

python/ebook_search/protected_phrases/__init__.py

@@ -0,0 +1 @@
+"""Init."""

python/ebook_search/protected_phrases/config/__init__.py

@@ -0,0 +1,17 @@
+"""Protected phrase extraction, storage, and runtime matching."""
+
+from python.ebook_search.protected_phrases.config.lib import (
+    get_bad_ends,
+    get_bad_starts,
+    get_ignored_phrases,
+    get_junk_tokens,
+    get_most_common_words,
+)
+
+__all__ = [
+    "get_bad_ends",
+    "get_bad_starts",
+    "get_ignored_phrases",
+    "get_junk_tokens",
+    "get_most_common_words",
+]

python/ebook_search/protected_phrases/config/bad_ends.toml

@@ -0,0 +1,31 @@
+tokens = [
+  "a",
+  "an",
+  "and",
+  "any",
+  "as",
+  "at",
+  "be",
+  "because",
+  "but",
+  "by",
+  "can",
+  "could",
+  "do",
+  "for",
+  "from",
+  "have",
+  "if",
+  "of",
+  "or",
+  "some",
+  "than",
+  "the",
+  "these",
+  "this",
+  "to",
+  "will",
+  "with",
+  "would",
+  "did",
+]

python/ebook_search/protected_phrases/config/bad_starts.toml

@@ -0,0 +1,27 @@
+tokens = [
+    "a",
+    "an",
+    "did",
+    "didn't",
+    "he",
+    "here",
+    "how",
+    "i",
+    "it",
+    "she",
+    "that",
+    "the",
+    "there",
+    "they",
+    "this",
+    "we",
+    "what",
+    "when",
+    "where",
+    "which",
+    "who",
+    "whom",
+    "whose",
+    "why",
+    "you",
+]

python/ebook_search/protected_phrases/config/ignored_phrases.toml

@@ -0,0 +1,212 @@
+phrases = [
+  "a little",
+  "across the",
+  "and she",
+  "anyone in",
+  "are you",
+  "around him",
+  "around the",
+  "as much",
+  "as soon",
+  "at all",
+  "at least",
+  "before the",
+  "behind him",
+  "between the",
+  "but she",
+  "could not",
+  "did he",
+  "did i",
+  "did it",
+  "did not believe",
+  "did not care",
+  "did not even",
+  "did not know what",
+  "did not know",
+  "did not like",
+  "did not look",
+  "did not mean",
+  "did not move",
+  "did not need",
+  "did not see",
+  "did not seem",
+  "did not think",
+  "did not understand",
+  "did not want",
+  "did not",
+  "did she",
+  "did so",
+  "did that",
+  "did the",
+  "did they",
+  "did what",
+  "did you",
+  "didn't answer",
+  "didn't care",
+  "didn't even",
+  "didn't expect",
+  "didn't feel",
+  "didn't get",
+  "didn't i",
+  "didn't know",
+  "didn't like",
+  "didn't look",
+  "didn't make",
+  "didn't mean",
+  "didn't need",
+  "didn't really",
+  "didn't say",
+  "didn't see",
+  "didn't seem",
+  "didn't think",
+  "didn't want",
+  "didn't you",
+  "end up",
+  "ended up",
+  "had a",
+  "had been",
+  "have been",
+  "he asked",
+  "he concluded",
+  "he continued",
+  "he couldn't",
+  "he did",
+  "he didn't",
+  "he felt",
+  "he had",
+  "he hadn't",
+  "he knew",
+  "he noted",
+  "he pointed",
+  "he realized",
+  "he replied",
+  "he said",
+  "he saw",
+  "he tapped",
+  "he told",
+  "he was",
+  "he wasn't",
+  "his body",
+  "his chair",
+  "his feet",
+  "his hands",
+  "his head",
+  "his office",
+  "his own",
+  "his pc",
+  "his power",
+  "his shield",
+  "his sight",
+  "his voice",
+  "his wrist",
+  "how many",
+  "i am",
+  "i don't",
+  "i said",
+  "i was",
+  "i wouldn't",
+  "i'm not",
+  "if he",
+  "if they",
+  "is in",
+  "is not",
+  "is that",
+  "is the",
+  "it had",
+  "it had",
+  "it is",
+  "it was",
+  "it wasn't",
+  "it wasn't",
+  "no one",
+  "of course",
+  "of force",
+  "of it",
+  "of magic",
+  "of marines",
+  "of power",
+  "of those",
+  "old man",
+  "older man",
+  "one of",
+  "out of",
+  "set up",
+  "she admitted",
+  "she asked",
+  "she had",
+  "she replied",
+  "she said",
+  "she snapped",
+  "she told",
+  "she was",
+  "she'd been",
+  "shook his",
+  "sure he",
+  "tell you",
+  "that had",
+  "that is",
+  "that she",
+  "that was",
+  "the dark",
+  "the door",
+  "the first",
+  "the last",
+  "the man",
+  "the one",
+  "the only",
+  "the other",
+  "the rest",
+  "the room",
+  "the same",
+  "the two",
+  "the way",
+  "the world",
+  "there are",
+  "there was",
+  "there were",
+  "they are",
+  "they had",
+  "they were",
+  "they weren't",
+  "this is",
+  "this place",
+  "though he",
+  "through his",
+  "through the",
+  "to find",
+  "to get",
+  "to keep",
+  "to stay",
+  "to stop",
+  "to tell",
+  "to try",
+  "told her",
+  "told him",
+  "under his",
+  "was a",
+  "was enough",
+  "was going",
+  "was in",
+  "was no",
+  "was not",
+  "was now",
+  "was on",
+  "was one",
+  "was only",
+  "was still",
+  "was that",
+  "was the",
+  "was there",
+  "were in",
+  "what had",
+  "what happened",
+  "what was",
+  "where the",
+  "while i",
+  "you are",
+  "you can't",
+  "you don't",
+  "you know",
+  "you need",
+  "you were",
+]

python/ebook_search/protected_phrases/config/junk_tokens.toml

@@ -0,0 +1,71 @@
+tokens = [
+    "said",
+    "asked",
+    "replied",
+    "answered",
+    "looked",
+    "nodded",
+    "turned",
+    "shook",
+    "smiled",
+    "shrugged",
+    "pointed",
+    "continued",
+    "repeated",
+    "stared",
+    "agreed",
+    "glanced",
+    "walked",
+    "told",
+    "thought",
+    "knew",
+    "wanted",
+    "muttered",
+    "whispered",
+    "laughed",
+    "sighed",
+    "paused",
+    "gestured",
+    "waved",
+    "frowned",
+    "grinned",
+    "admitted",
+    "found",
+    "noted",
+    "murmured",
+    "ordered",
+    "i'm",
+    "i've",
+    "i'd",
+    "i'll",
+    "it's",
+    "that's",
+    "don't",
+    "didn't",
+    "doesn't",
+    "can't",
+    "won't",
+    "wouldn't",
+    "couldn't",
+    "shouldn't",
+    "isn't",
+    "wasn't",
+    "aren't",
+    "weren't",
+    "you're",
+    "you've",
+    "you'll",
+    "we're",
+    "we've",
+    "we'll",
+    "they're",
+    "they've",
+    "he's",
+    "she's",
+    "there's",
+    "what's",
+    "let's",
+    "who's",
+    "he'd",
+    "she'd",
+]

python/ebook_search/protected_phrases/config/lib.py

@@ -0,0 +1,60 @@
+"""Protected phrase extraction, storage, and runtime matching."""
+
+from __future__ import annotations
+
+import logging
+import tomllib
+from functools import cache
+from pathlib import Path
+
+from python.ebook_search.protected_phrases.text_normalization import normalize_text
+
+logger = logging.getLogger(__name__)
+
+
+def _load_toml_string_set(path: Path, key: str) -> frozenset[str]:
+    """Load and validate a TOML string list as a normalized immutable set."""
+    with path.open("rb") as file:
+        body = tomllib.load(file)
+
+    values = body.get(key)
+    if not isinstance(values, list) or not all(isinstance(item, str) for item in values):
+        msg = f"{path} must contain a {key!r} string list"
+        raise ValueError(msg)
+    return frozenset(normalize_text(value) for value in values if normalize_text(value))
+
+
+@cache
+def _get_phrase_config_dir() -> Path:
+    """Return the directory containing phrase configuration files."""
+    return Path(__file__).resolve().parent
+
+
+@cache
+def get_ignored_phrases() -> frozenset[str]:
+    """Return ignored phrase strings loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "ignored_phrases.toml", "phrases")
+
+
+@cache
+def get_bad_ends() -> frozenset[str]:
+    """Return bad phrase-ending tokens loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "bad_ends.toml", "tokens")
+
+
+@cache
+def get_bad_starts() -> frozenset[str]:
+    """Return bad phrase-starting tokens loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "bad_starts.toml", "tokens")
+
+
+@cache
+def get_most_common_words() -> frozenset[str]:
+    """Return the most common English words loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "most_common_words.toml", "words")
+
+
+@cache
+def get_junk_tokens() -> frozenset[str]:
+    """Return junk tokens (dialogue verbs and pronoun contractions) loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "junk_tokens.toml", "tokens")

python/ebook_search/protected_phrases/config/most_common_words.toml

@@ -0,0 +1,102 @@
+words = [
+    "the",
+    "be",
+    "to",
+    "of",
+    "and",
+    "a",
+    "in",
+    "that",
+    "have",
+    "I",
+    "it",
+    "for",
+    "not",
+    "on",
+    "with",
+    "he",
+    "as",
+    "you",
+    "do",
+    "at",
+    "this",
+    "but",
+    "his",
+    "by",
+    "from",
+    "they",
+    "we",
+    "say",
+    "her",
+    "she",
+    "or",
+    "an",
+    "will",
+    "my",
+    "one",
+    "all",
+    "would",
+    "there",
+    "their",
+    "what",
+    "so",
+    "up",
+    "out",
+    "if",
+    "about",
+    "who",
+    "get",
+    "which",
+    "go",
+    "me",
+    "when",
+    "make",
+    "can",
+    "like",
+    "time",
+    "no",
+    "just",
+    "him",
+    "know",
+    "take",
+    "people",
+    "into",
+    "year",
+    "your",
+    "good",
+    "some",
+    "could",
+    "them",
+    "see",
+    "other",
+    "than",
+    "then",
+    "now",
+    "look",
+    "only",
+    "come",
+    "its",
+    "over",
+    "think",
+    "also",
+    "back",
+    "after",
+    "use",
+    "two",
+    "how",
+    "our",
+    "work",
+    "first",
+    "well",
+    "way",
+    "even",
+    "new",
+    "want",
+    "because",
+    "any",
+    "these",
+    "give",
+    "day",
+    "most",
+    "us",
+]

python/ebook_search/protected_phrases/extraction.py

@@ -0,0 +1,853 @@
+"""Candidate phrase extraction and scoring for protected phrases."""
+
+from __future__ import annotations
+
+import logging
+import re
+from collections import Counter, defaultdict
+from functools import lru_cache
+from time import perf_counter
+from typing import TYPE_CHECKING, Protocol
+
+from yake import KeywordExtractor
+
+from python.ebook_search.protected_phrases.config import (
+    get_bad_ends,
+    get_bad_starts,
+    get_ignored_phrases,
+    get_junk_tokens,
+    get_most_common_words,
+)
+from python.ebook_search.protected_phrases.models import PhraseCandidate
+from python.ebook_search.protected_phrases.text_normalization import tokenize, tokenize_with_offsets
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Mapping, Sequence
+
+    from python.ebook_search.config import EbookSearchConfig
+
+logger = logging.getLogger(__name__)
+
+
+BAD_START_SCORE_PENALTY = 10.0
+BAD_END_SCORE_PENALTY = 10.0
+MULTI_SOURCE_SCORE_BONUS = 2.0
+MULTI_SOURCE_MIN_SOURCES = 2
+CAPITALIZED_PHRASE_RE = re.compile(r"\b(?:[A-Z][a-zA-Z']+)(?:\s+(?:of|the|and|in|on|for|[A-Z][a-zA-Z']+)){0,6}")
+
+
+class SpacySpan(Protocol):
+    """Small protocol for the spaCy span attributes used by this module."""
+
+    text: str
+
+
+class SpacyEntity(SpacySpan, Protocol):
+    """Small protocol for the spaCy entity attributes used by this module."""
+
+    label_: str
+
+
+class SpacyDoc(Protocol):
+    """Small protocol for the spaCy doc attributes used by this module."""
+
+    ents: Iterable[SpacyEntity]
+    noun_chunks: Iterable[SpacySpan]
+
+
+class SpacyLanguage(Protocol):
+    """Small protocol for a callable spaCy language pipeline."""
+
+    def __call__(self, text: str) -> SpacyDoc:
+        """Parse text into a spaCy-like doc."""
+
+
+class YakeExtractor(Protocol):
+    """Small protocol for the YAKE extractor used by this module."""
+
+    def extract_keywords(self, text: str) -> Iterable[tuple[str, float]]:
+        """Return YAKE keyword tuples."""
+
+
+class YakeExtractorFactory(Protocol):
+    """Callable constructor protocol for YAKE keyword extractors."""
+
+    def __call__(self, *, lan: str, n: int, dedupLim: float, top: int) -> YakeExtractor:  # noqa: N803
+        """Create a YAKE keyword extractor.
+
+        Args:
+            lan (str): Language code passed to YAKE.
+            n (int): Maximum n-gram size to extract.
+            dedupLim (float): Deduplication similarity threshold.
+            top (int): Maximum number of keyphrases to return.
+
+        Returns:
+            YakeExtractor: The constructed keyword extractor.
+        """
+
+
+def normalize_candidate_phrase(
+    phrase_text: str,
+    config: EbookSearchConfig,
+    *,
+    min_tokens: int | None = None,
+    max_tokens: int | None = None,
+    strip_leading_article: bool = False,
+) -> tuple[str, str, int] | None:
+    """Normalize a candidate phrase and validate token bounds.
+
+    Args:
+        phrase_text (str): Raw phrase text to normalize.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        min_tokens (int | None): Minimum token count override; defaults to ``config.phrase_min_tokens``.
+        max_tokens (int | None): Maximum token count override; defaults to ``config.phrase_max_tokens``.
+        strip_leading_article (bool): Whether to drop a single leading English article.
+
+    Returns:
+        tuple[str, str, int] | None: Display text, normalized phrase, and token count, or ``None``
+        when the phrase falls outside the token bounds or is ignored.
+    """
+    normalized_tokens = tokenize_with_offsets(phrase_text)
+    start = 0
+    if strip_leading_article and normalized_tokens and normalized_tokens[0].text in {"the", "a", "an"}:
+        start = 1
+
+    selected_tokens = normalized_tokens[start:]
+    min_count = config.phrase_min_tokens if min_tokens is None else min_tokens
+    max_count = config.phrase_max_tokens if max_tokens is None else max_tokens
+    if len(selected_tokens) < min_count or len(selected_tokens) > max_count:
+        return None
+
+    phrase_norm = " ".join(token.text for token in selected_tokens)
+    if phrase_norm in get_ignored_phrases():
+        return None
+
+    display_text = phrase_text[selected_tokens[0].start_char : selected_tokens[-1].end_char].strip()
+    return display_text or phrase_norm, phrase_norm, len(selected_tokens)
+
+
+def count_raw_ngrams(tokens: Sequence[str], config: EbookSearchConfig) -> Counter[str]:
+    """Count every n-gram window in one normalized token block.
+
+    ``tokens`` are already normalized (see :func:`tokenize`), so each window's normalized form
+    is the joined tokens directly. Counting into a plain :class:`Counter` rather than
+    :class:`PhraseCandidate` objects keeps this hot loop cheap; callers filter ignored phrases
+    and materialize candidates per unique phrase afterwards, which is far fewer operations than
+    doing either per window.
+
+    Args:
+        tokens (Sequence[str]): Normalized tokens for one text block.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        Counter[str]: Raw occurrence counts keyed by normalized phrase.
+    """
+    return Counter(
+        " ".join(tokens[start : start + ngram_size])
+        for ngram_size in range(config.phrase_min_tokens, config.phrase_max_tokens + 1)
+        for start in range(len(tokens) - ngram_size + 1)
+    )
+
+
+def extract_raw_ngrams_by_chapter(
+    chapters: Sequence[str],
+    config: EbookSearchConfig,
+) -> dict[str, PhraseCandidate]:
+    """Extract raw n-grams across chapters, tracking both raw counts and chapter spread.
+
+    Counting each chapter separately makes chapter spread fall out of dict membership: a phrase's
+    ``chapter_count`` is simply how many per-chapter count maps contain it, so no per-window seen
+    tracking is needed. This also lets the enrichment step skip re-sliding the same n-gram sizes.
+
+    Phrases below the minimum raw count are dropped here rather than materialized: most unique
+    n-grams occur once, and :func:`filter_storable_candidates` would discard them as too rare
+    anyway, so building ``PhraseCandidate`` objects for them is wasted work.
+
+    Args:
+        chapters (Sequence[str]): Chapter-like text blocks to slide n-gram windows over.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        dict[str, PhraseCandidate]: Candidates meeting the minimum raw count, keyed by normalized
+        phrase, with raw and chapter counts.
+    """
+    chapter_count_maps = [count_raw_ngrams(tokenize(chapter), config) for chapter in chapters]
+    total_counts: Counter[str] = Counter()
+    chapter_spread: Counter[str] = Counter()
+    for chapter_counts in chapter_count_maps:
+        total_counts.update(chapter_counts)
+        chapter_spread.update(chapter_counts.keys())
+    min_raw_count = minimum_candidate_raw_count(config)
+    ignored = get_ignored_phrases()
+    return {
+        phrase_norm: PhraseCandidate(
+            phrase_text=phrase_norm,
+            phrase_norm=phrase_norm,
+            token_count=phrase_norm.count(" ") + 1,
+            source_raw_ngram=True,
+            raw_count=raw_count,
+            chapter_count=chapter_spread[phrase_norm],
+        )
+        for phrase_norm, raw_count in total_counts.items()
+        if raw_count >= min_raw_count and phrase_norm not in ignored
+    }
+
+
+@lru_cache(maxsize=2)
+def get_yake_extractor(max_ngram: int, top_k: int) -> KeywordExtractor:
+    """Return a cached YAKE extractor for the given settings.
+
+    Constructing a ``KeywordExtractor`` loads the language's stopword list from disk, so it is
+    cached and reused across books rather than rebuilt on every call.
+
+    Args:
+        max_ngram (int): Maximum n-gram size to extract.
+        top_k (int): Maximum number of keyphrases to request.
+
+    Returns:
+        KeywordExtractor: A shared extractor instance for the given settings.
+    """
+    return KeywordExtractor(lan="en", n=max_ngram, dedupLim=0.85, top=top_k)
+
+
+def extract_yake_candidates(
+    book_text: str,
+    config: EbookSearchConfig,
+    top_k: int = 1000,
+) -> dict[str, PhraseCandidate]:
+    """Extract YAKE keyphrases when the optional YAKE package is installed.
+
+    Args:
+        book_text (str): Full book text to extract keyphrases from.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        top_k (int): Maximum number of YAKE keyphrases to request.
+
+    Returns:
+        dict[str, PhraseCandidate]: Candidates keyed by normalized phrase, with YAKE scores.
+    """
+    extractor = get_yake_extractor(config.phrase_max_tokens, top_k)
+    out: dict[str, PhraseCandidate] = {}
+    for phrase_text, yake_score in extractor.extract_keywords(book_text):
+        normalized = normalize_candidate_phrase(phrase_text, config)
+        if normalized is None:
+            continue
+        display_text, phrase_norm, token_count = normalized
+        out[phrase_norm] = PhraseCandidate(
+            phrase_text=display_text,
+            phrase_norm=phrase_norm,
+            token_count=token_count,
+            source_yake=True,
+            yake_score=float(yake_score),
+        )
+    return out
+
+
+def extract_spacy_candidates(
+    book_text: str,
+    nlp: SpacyLanguage,
+    config: EbookSearchConfig,
+) -> dict[str, PhraseCandidate]:
+    """Extract spaCy named entities and noun chunks from one text block.
+
+    Args:
+        book_text (str): Text block to parse with spaCy.
+        nlp (SpacyLanguage): Callable spaCy language pipeline.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        dict[str, PhraseCandidate]: Candidates keyed by normalized phrase from entities and noun chunks.
+    """
+    out: dict[str, PhraseCandidate] = {}
+    doc = nlp(book_text)
+
+    for ent in doc.ents:
+        normalized = normalize_candidate_phrase(
+            ent.text,
+            config,
+            max_tokens=config.phrase_max_entity_tokens,
+        )
+        if normalized is None:
+            continue
+        phrase_text, phrase_norm, token_count = normalized
+        out[phrase_norm] = PhraseCandidate(
+            phrase_text=phrase_text,
+            phrase_norm=phrase_norm,
+            token_count=token_count,
+            source_spacy_ner=True,
+            spacy_label=ent.label_,
+        )
+
+    for chunk in doc.noun_chunks:
+        normalized = normalize_candidate_phrase(chunk.text, config, strip_leading_article=True)
+        if normalized is None:
+            continue
+        phrase_text, phrase_norm, token_count = normalized
+        out[phrase_norm] = PhraseCandidate(
+            phrase_text=phrase_text,
+            phrase_norm=phrase_norm,
+            token_count=token_count,
+            source_spacy_noun_chunk=True,
+        )
+    return out
+
+
+def extract_capitalized_phrases(original_text: str, config: EbookSearchConfig) -> dict[str, PhraseCandidate]:
+    """Extract capitalized phrase runs that often carry fictional terms.
+
+    Args:
+        original_text (str): Original-case book text to scan for capitalized runs.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        dict[str, PhraseCandidate]: Candidates keyed by normalized phrase from capitalized runs.
+    """
+    out: dict[str, PhraseCandidate] = {}
+    for match in CAPITALIZED_PHRASE_RE.finditer(original_text):
+        phrase_text = match.group(0).strip()
+        normalized = normalize_candidate_phrase(
+            phrase_text,
+            config,
+            max_tokens=config.phrase_max_entity_tokens,
+        )
+        if normalized is None:
+            continue
+        display_text, phrase_norm, token_count = normalized
+        out[phrase_norm] = PhraseCandidate(
+            phrase_text=display_text,
+            phrase_norm=phrase_norm,
+            token_count=token_count,
+            source_capitalized=True,
+        )
+    return out
+
+
+def extract_metadata_candidates(
+    metadata: Mapping[str, object] | None,
+    config: EbookSearchConfig,
+) -> dict[str, PhraseCandidate]:
+    """Extract phrases from book metadata values such as title, author, and series.
+
+    Args:
+        metadata (Mapping[str, object] | None): Book metadata values, or ``None`` when unavailable.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        dict[str, PhraseCandidate]: Candidates keyed by normalized phrase from metadata values.
+    """
+    if metadata is None:
+        return {}
+
+    out: dict[str, PhraseCandidate] = {}
+    for value in metadata.values():
+        if value is None:
+            continue
+        phrase_text = str(value).strip()
+        normalized = normalize_candidate_phrase(
+            phrase_text,
+            config,
+            max_tokens=config.phrase_max_entity_tokens,
+        )
+        if normalized is None:
+            continue
+        display_text, phrase_norm, token_count = normalized
+        out[phrase_norm] = PhraseCandidate(
+            phrase_text=display_text,
+            phrase_norm=phrase_norm,
+            token_count=token_count,
+            source_metadata=True,
+        )
+    return out
+
+
+def merge_candidate_sources(*sources: Mapping[str, PhraseCandidate]) -> dict[str, PhraseCandidate]:
+    """Merge candidate dictionaries by normalized phrase.
+
+    Args:
+        *sources (Mapping[str, PhraseCandidate]): Candidate maps to combine, keyed by normalized phrase.
+
+    Returns:
+        dict[str, PhraseCandidate]: One merged candidate per normalized phrase.
+    """
+    merged: dict[str, PhraseCandidate] = {}
+    for source in sources:
+        for phrase_norm, item in source.items():
+            existing = merged.setdefault(
+                phrase_norm,
+                PhraseCandidate(
+                    phrase_text=item.phrase_text,
+                    phrase_norm=phrase_norm,
+                    token_count=item.token_count,
+                ),
+            )
+            merge_candidate(existing, item)
+    return merged
+
+
+def merge_candidate(existing: PhraseCandidate, item: PhraseCandidate) -> None:
+    """Merge one candidate into an existing candidate object.
+
+    Args:
+        existing (PhraseCandidate): Candidate mutated in place to absorb ``item``.
+        item (PhraseCandidate): Candidate whose sources, counts, and scores are merged in.
+    """
+    existing.source_raw_ngram = existing.source_raw_ngram or item.source_raw_ngram
+    existing.source_yake = existing.source_yake or item.source_yake
+    existing.source_spacy_ner = existing.source_spacy_ner or item.source_spacy_ner
+    existing.source_spacy_noun_chunk = existing.source_spacy_noun_chunk or item.source_spacy_noun_chunk
+    existing.source_capitalized = existing.source_capitalized or item.source_capitalized
+    existing.source_metadata = existing.source_metadata or item.source_metadata
+    existing.raw_count += item.raw_count
+    existing.chapter_count = max(existing.chapter_count, item.chapter_count)
+    if item.yake_score is not None:
+        existing.yake_score = item.yake_score
+    if item.spacy_label:
+        existing.spacy_label = item.spacy_label
+
+
+def enrich_with_frequency_and_chapter_counts(
+    candidates: Mapping[str, PhraseCandidate],
+    chapters: Sequence[str],
+    *,
+    counted_sizes: Iterable[int] = (),
+) -> dict[str, PhraseCandidate]:
+    """Add raw occurrence and chapter-spread counts to candidates.
+
+    Candidates whose ``token_count`` is in ``counted_sizes`` are left untouched: those counts
+    were already computed while sliding the chapters in :func:`extract_raw_ngrams_by_chapter`,
+    so re-sliding those n-gram sizes here would just duplicate that work.
+
+    Args:
+        candidates (Mapping[str, PhraseCandidate]): Candidates to enrich, keyed by normalized phrase.
+        chapters (Sequence[str]): Chapter-like text blocks used to count occurrences and spread.
+        counted_sizes (Iterable[int]): Token counts whose counts are already populated and should be skipped.
+
+    Returns:
+        dict[str, PhraseCandidate]: Candidates with updated ``raw_count`` and ``chapter_count`` values.
+    """
+    if not candidates:
+        return {}
+
+    already_counted = set(counted_sizes)
+    candidate_sets_by_size: dict[int, set[str]] = defaultdict(set)
+    for phrase_norm, candidate in candidates.items():
+        if candidate.token_count in already_counted:
+            continue
+        candidate_sets_by_size[candidate.token_count].add(phrase_norm)
+
+    enriched = dict(candidates)
+    if not candidate_sets_by_size:
+        return enriched
+
+    total_counts, chapter_counts = count_candidate_occurrences(candidate_sets_by_size, chapters)
+    for phrase_norm, candidate in enriched.items():
+        if candidate.token_count in already_counted:
+            continue
+        candidate.raw_count = max(candidate.raw_count, total_counts[phrase_norm])
+        candidate.chapter_count = chapter_counts[phrase_norm]
+    return enriched
+
+
+def count_candidate_occurrences(
+    candidate_sets_by_size: Mapping[int, set[str]],
+    chapters: Sequence[str],
+) -> tuple[dict[str, int], dict[str, int]]:
+    """Count total occurrences and chapter spread for candidate phrases across chapters.
+
+    Args:
+        candidate_sets_by_size (Mapping[int, set[str]]): Candidate normalized phrases grouped by token count.
+        chapters (Sequence[str]): Chapter-like text blocks to slide n-gram windows over.
+
+    Returns:
+        tuple[dict[str, int], dict[str, int]]: Total occurrence counts and chapter-spread counts,
+        each keyed by normalized phrase.
+    """
+    total_counts: defaultdict[str, int] = defaultdict(int)
+    chapter_counts: defaultdict[str, int] = defaultdict(int)
+    for chapter in chapters:
+        seen_in_chapter: set[str] = set()
+        chapter_tokens = tokenize(chapter)
+        for ngram_size, candidate_norms in candidate_sets_by_size.items():
+            for start in range(len(chapter_tokens) - ngram_size + 1):
+                phrase_norm = " ".join(chapter_tokens[start : start + ngram_size])
+                if phrase_norm not in candidate_norms:
+                    continue
+                total_counts[phrase_norm] += 1
+                seen_in_chapter.add(phrase_norm)
+        for phrase_norm in seen_in_chapter:
+            chapter_counts[phrase_norm] += 1
+    return total_counts, chapter_counts
+
+
+def filter_storable_candidates(
+    candidates: Mapping[str, PhraseCandidate],
+    config: EbookSearchConfig,
+) -> tuple[dict[str, PhraseCandidate], int, int, int, int]:
+    """Remove candidates that should not be persisted.
+
+    Args:
+        candidates (Mapping[str, PhraseCandidate]): Candidates to filter, keyed by normalized phrase.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        tuple[dict[str, PhraseCandidate], int, int, int, int]: The storable candidates followed by the
+        counts dropped for being too short, too rare, too common, and junk.
+    """
+    min_raw_count = minimum_candidate_raw_count(config)
+    filtered: dict[str, PhraseCandidate] = {}
+    too_short = 0
+    too_rare = 0
+    too_common = 0
+    junk = 0
+    for phrase_norm, candidate in candidates.items():
+        if candidate.token_count < config.phrase_min_tokens:
+            too_short += 1
+            continue
+        if candidate.raw_count < min_raw_count:
+            too_rare += 1
+            continue
+        phrase_tokens = phrase_norm.split()
+        if is_most_common_word_phrase(phrase_tokens):
+            too_common += 1
+            continue
+        if is_junk_phrase(phrase_tokens):
+            junk += 1
+            continue
+        filtered[phrase_norm] = candidate
+    return filtered, too_short, too_rare, too_common, junk
+
+
+def minimum_candidate_raw_count(config: EbookSearchConfig) -> int:
+    """Return the minimum occurrence count required before storing a candidate.
+
+    Args:
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        int: The minimum raw occurrence count, never less than 1.
+    """
+    return max(config.phrase_raw_ngram_min_count, 1)
+
+
+def is_most_common_word_phrase(phrase_tokens: list[str]) -> bool:
+    """Return whether every token in a normalized phrase is a common word.
+
+    Args:
+        phrase_tokens (list[str]): Normalized phrase tokens to inspect.
+
+    Returns:
+        bool: True when the phrase is non-empty and every token is a common word.
+    """
+    common_words = get_most_common_words()
+    return bool(phrase_tokens) and all(token in common_words for token in phrase_tokens)
+
+
+def is_junk_phrase(phrase_tokens: list[str]) -> bool:
+    """Return whether a normalized phrase is lexical junk not worth LLM judging.
+
+    Judged data shows phrases containing a dialogue/action verb or a pronoun contraction are
+    never kept, and phrases whose tokens are mostly common words almost never are. Possessives
+    of proper nouns (``chapman's death``) pass because matching is by exact token, and
+    exactly-half-common bigrams (``data feed``) pass because the common-word rule is strict.
+
+    Args:
+        phrase_tokens (list[str]): Normalized phrase tokens to inspect.
+
+    Returns:
+        bool: True when the phrase contains a junk token or is majority common words.
+    """
+    if not phrase_tokens:
+        return False
+    junk_tokens = get_junk_tokens()
+    if any(token in junk_tokens for token in phrase_tokens):
+        return True
+    common_words = get_most_common_words()
+    half_phrase_len = len(phrase_tokens) // 2
+    return sum(token in common_words for token in phrase_tokens) > half_phrase_len
+
+
+def score_candidate(candidate: PhraseCandidate, config: EbookSearchConfig) -> float:
+    """Score a phrase candidate before LLM judging.
+
+    Args:
+        candidate (PhraseCandidate): Candidate to score.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        float: Combined score from sources, frequency, and length, less any penalties.
+    """
+    score = source_score(candidate) + frequency_score(candidate, config) + token_count_score(candidate, config)
+    if non_raw_source_count(candidate) >= MULTI_SOURCE_MIN_SOURCES:
+        score += MULTI_SOURCE_SCORE_BONUS
+    if candidate.phrase_norm in get_ignored_phrases():
+        score -= 100.0
+    if has_bad_start(candidate.phrase_norm):
+        score -= BAD_START_SCORE_PENALTY
+    if has_bad_end(candidate.phrase_norm):
+        score -= BAD_END_SCORE_PENALTY
+    return score
+
+
+def non_raw_source_count(candidate: PhraseCandidate) -> int:
+    """Count the non-raw-ngram extraction sources that produced a candidate.
+
+    Args:
+        candidate (PhraseCandidate): Candidate whose enabled sources are counted.
+
+    Returns:
+        int: Number of enabled sources other than the raw n-gram slide.
+    """
+    return sum(
+        (
+            candidate.source_yake,
+            candidate.source_spacy_ner,
+            candidate.source_spacy_noun_chunk,
+            candidate.source_capitalized,
+            candidate.source_metadata,
+        )
+    )
+
+
+def has_bad_start(phrase_norm: str) -> bool:
+    """Return whether a normalized phrase starts with a bad starting token.
+
+    Args:
+        phrase_norm (str): Normalized phrase text to inspect.
+
+    Returns:
+        bool: True when the first token is a known bad starting token.
+    """
+    phrase_tokens = phrase_norm.split()
+    return bool(phrase_tokens and phrase_tokens[0] in get_bad_starts())
+
+
+def has_bad_end(phrase_norm: str) -> bool:
+    """Return whether a normalized phrase ends with a bad ending token.
+
+    Args:
+        phrase_norm (str): Normalized phrase text to inspect.
+
+    Returns:
+        bool: True when the last token is a known bad ending token.
+    """
+    phrase_tokens = phrase_norm.split()
+    return bool(phrase_tokens and phrase_tokens[-1] in get_bad_ends())
+
+
+def source_score(candidate: PhraseCandidate) -> float:
+    """Return the score contribution from extraction sources.
+
+    Args:
+        candidate (PhraseCandidate): Candidate whose enabled sources are weighted.
+
+    Returns:
+        float: Summed weight of the candidate's enabled extraction sources.
+    """
+    return sum(
+        weight
+        for enabled, weight in (
+            (candidate.source_yake, 2.0),
+            (candidate.source_spacy_ner, 2.5),
+            (candidate.source_spacy_noun_chunk, 1.5),
+            (candidate.source_capitalized, 2.0),
+            (candidate.source_metadata, 2.0),
+            (candidate.source_raw_ngram, 0.5),
+        )
+        if enabled
+    )
+
+
+def frequency_score(candidate: PhraseCandidate, config: EbookSearchConfig) -> float:
+    """Return the score contribution from frequency and chapter spread.
+
+    Args:
+        candidate (PhraseCandidate): Candidate whose counts are scored.
+        config (EbookSearchConfig): Runtime phrase-tuning settings holding score thresholds.
+
+    Returns:
+        float: Summed weight for each frequency and chapter-spread threshold the candidate meets.
+    """
+    return sum(
+        weight
+        for count, threshold, weight in (
+            (candidate.raw_count, config.phrase_raw_count_score_threshold, 0.5),
+            (candidate.raw_count, config.phrase_raw_count_high_score_threshold, 0.5),
+            (candidate.chapter_count, config.phrase_chapter_count_score_threshold, 0.5),
+            (candidate.chapter_count, config.phrase_chapter_count_high_score_threshold, 0.5),
+        )
+        if count >= threshold
+    )
+
+
+def token_count_score(candidate: PhraseCandidate, config: EbookSearchConfig) -> float:
+    """Return the score contribution from phrase length.
+
+    Args:
+        candidate (PhraseCandidate): Candidate whose token count is scored.
+        config (EbookSearchConfig): Runtime phrase-tuning settings holding the max token bound.
+
+    Returns:
+        float: Length-based score contribution, which may be negative for over- or under-length phrases.
+    """
+    if candidate.token_count == 1:
+        return -0.5
+    if candidate.token_count in {2, 3, 4}:
+        return 0.5
+    if candidate.token_count > config.phrase_max_tokens:
+        return -1.0
+    return 0.0
+
+
+def get_sample_contexts(normalized_book_text: str, phrase_norm: str, max_contexts: int = 5) -> list[str]:
+    """Return normalized context snippets containing a candidate phrase.
+
+    ``normalized_book_text`` is expected to already be ``normalize_text``-ed by the caller
+    so the whole book is not re-normalized for every phrase.
+
+    Args:
+        normalized_book_text (str): Whole book text, already normalized, to search.
+        phrase_norm (str): Normalized phrase to find contexts around.
+        max_contexts (int): Maximum number of context snippets to return.
+
+    Returns:
+        list[str]: Up to ``max_contexts`` normalized snippets surrounding the phrase.
+    """
+    contexts: list[str] = []
+    start = 0
+    while len(contexts) < max_contexts:
+        index = normalized_book_text.find(phrase_norm, start)
+        if index == -1:
+            break
+        left = max(0, index - 300)
+        right = min(len(normalized_book_text), index + len(phrase_norm) + 300)
+        contexts.append(normalized_book_text[left:right])
+        start = index + len(phrase_norm)
+    return contexts
+
+
+def candidate_source_names(candidate: PhraseCandidate) -> list[str]:
+    """Return enabled source names for an extracted candidate.
+
+    Args:
+        candidate (PhraseCandidate): Candidate whose enabled sources are listed.
+
+    Returns:
+        list[str]: Names of the extraction sources that produced the candidate.
+    """
+    names: list[str] = []
+    if candidate.source_raw_ngram:
+        names.append("raw_ngram")
+    if candidate.source_yake:
+        names.append("yake")
+    if candidate.source_spacy_ner:
+        names.append("spacy_ner")
+    if candidate.source_spacy_noun_chunk:
+        names.append("spacy_noun_chunk")
+    if candidate.source_capitalized:
+        names.append("capitalized")
+    if candidate.source_metadata:
+        names.append("metadata")
+    return names
+
+
+def extract_phrase_candidates_for_book(
+    book_text: str,
+    chapters: Sequence[str],
+    config: EbookSearchConfig,
+    *,
+    nlp: SpacyLanguage | None = None,
+    metadata: Mapping[str, object] | None = None,
+) -> list[PhraseCandidate]:
+    """Extract, score, and limit phrase candidates for one book.
+
+    Args:
+        book_text (str): Full book text used for most extraction sources.
+        chapters (Sequence[str]): Chapter-like text blocks used for spaCy and frequency counts.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        nlp (SpacyLanguage | None): Optional spaCy pipeline for entity and noun-chunk sources.
+        metadata (Mapping[str, object] | None): Optional book metadata used as a candidate source.
+
+    Returns:
+        list[PhraseCandidate]: Scored candidates sorted best-first and capped per book.
+    """
+    started_at = perf_counter()
+    logger.info(
+        "ebook_phrase_candidate_extract_start chapters=%s chars=%s min_tokens=%s max_tokens=%s max_candidates=%s",
+        len(chapters),
+        len(book_text),
+        config.phrase_min_tokens,
+        config.phrase_max_tokens,
+        config.protected_phrase_max_candidates_per_book,
+    )
+    raw_started_at = perf_counter()
+    raw = extract_raw_ngrams_by_chapter(chapters, config)
+    logger.info(
+        "ebook_phrase_candidate_extract_raw_complete candidates=%s duration_ms=%.1f",
+        len(raw),
+        (perf_counter() - raw_started_at) * 1000,
+    )
+    yake_started_at = perf_counter()
+    yake_candidates = extract_yake_candidates(book_text, config)
+    logger.info(
+        "ebook_phrase_candidate_extract_yake_complete candidates=%s duration_ms=%.1f",
+        len(yake_candidates),
+        (perf_counter() - yake_started_at) * 1000,
+    )
+    spacy_candidates: dict[str, PhraseCandidate] = {}
+    if nlp is not None:
+        spacy_started_at = perf_counter()
+        for chapter in chapters:
+            spacy_candidates = merge_candidate_sources(spacy_candidates, extract_spacy_candidates(chapter, nlp, config))
+        logger.info(
+            "ebook_phrase_candidate_extract_spacy_complete candidates=%s duration_ms=%.1f",
+            len(spacy_candidates),
+            (perf_counter() - spacy_started_at) * 1000,
+        )
+    capitalized_started_at = perf_counter()
+    capitalized = extract_capitalized_phrases(book_text, config)
+    logger.info(
+        "ebook_phrase_candidate_extract_capitalized_complete candidates=%s duration_ms=%.1f",
+        len(capitalized),
+        (perf_counter() - capitalized_started_at) * 1000,
+    )
+    metadata_candidates = extract_metadata_candidates(metadata, config)
+
+    candidates = merge_candidate_sources(raw, yake_candidates, spacy_candidates, capitalized, metadata_candidates)
+    enriched_started_at = perf_counter()
+    # Raw n-gram sizes were already counted per chapter above, so only enrich the remaining
+    # (entity-length) sizes here instead of re-sliding every size over the whole book.
+    candidates = enrich_with_frequency_and_chapter_counts(
+        candidates,
+        chapters,
+        counted_sizes=range(config.phrase_min_tokens, config.phrase_max_tokens + 1),
+    )
+    pre_filter_count = len(candidates)
+    candidates, filtered_too_short, filtered_too_rare, filtered_too_common, filtered_junk = filter_storable_candidates(
+        candidates, config
+    )
+    for candidate in candidates.values():
+        candidate.candidate_score = score_candidate(candidate, config)
+
+    limited = sorted(candidates.values(), key=lambda item: item.candidate_score, reverse=True)[
+        : config.protected_phrase_max_candidates_per_book
+    ]
+    logger.info(
+        "ebook_phrase_candidate_extract_complete raw=%s yake=%s spacy=%s capitalized=%s metadata=%s "
+        "merged=%s filtered_too_short=%s filtered_too_rare=%s filtered_too_common=%s filtered_junk=%s "
+        "min_uses=%s storable=%s limited=%s enrich_score_ms=%.1f duration_ms=%.1f",
+        len(raw),
+        len(yake_candidates),
+        len(spacy_candidates),
+        len(capitalized),
+        len(metadata_candidates),
+        pre_filter_count,
+        filtered_too_short,
+        filtered_too_rare,
+        filtered_too_common,
+        filtered_junk,
+        minimum_candidate_raw_count(config),
+        len(candidates),
+        len(limited),
+        (perf_counter() - enriched_started_at) * 1000,
+        (perf_counter() - started_at) * 1000,
+    )
+    return limited

python/ebook_search/protected_phrases/generate_ngrams.py

@@ -0,0 +1,370 @@
+"""Book-level orchestration for candidate n-gram generation and recalculation."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections import deque
+from time import perf_counter
+from typing import TYPE_CHECKING
+
+from sqlalchemy import select
+
+from python.ebook_search.protected_phrases.extraction import extract_phrase_candidates_for_book
+from python.ebook_search.protected_phrases.models import (
+    BookCandidateResult,
+    PhraseCandidateGenerationResult,
+    PhraseRecalculationResult,
+)
+from python.ebook_search.protected_phrases.pool import extract_phrase_candidates_in_pool, get_extraction_pool
+from python.ebook_search.protected_phrases.store import (
+    bulk_upsert_unjudged_candidates,
+    delete_phrase_data_for_book,
+    load_book_chapter_texts,
+    metadata_for_source,
+    new_candidate_row,
+    prune_unstorable_unjudged_candidate_phrases,
+)
+from python.orm.richie import EbookCandidatePhrase, EbookSource
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping, Sequence
+    from concurrent.futures import Future
+
+    from sqlalchemy.ext.asyncio import AsyncSession
+
+    from python.ebook_search.config import EbookSearchConfig
+    from python.ebook_search.protected_phrases.extraction import SpacyLanguage
+    from python.ebook_search.protected_phrases.models import PhraseCandidate
+
+logger = logging.getLogger(__name__)
+
+
+async def generate_candidate_phrases_for_books(
+    session: AsyncSession,
+    config: EbookSearchConfig,
+    *,
+    only_missing: bool = False,
+) -> PhraseCandidateGenerationResult:
+    """Create or refresh candidate phrases for indexed books without calling the LLM judge.
+
+    Extraction always runs concurrently in the shared process pool so a full backfill uses
+    multiple cores.
+
+    Args:
+        session (Session): Active database session.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        only_missing (bool): When True, only generate for books that have no candidate phrases
+            yet instead of refreshing every book.
+
+    Returns:
+        PhraseCandidateGenerationResult: Per-corpus counts of books seen, built, and candidates stored.
+    """
+    source_query = select(EbookSource).order_by(EbookSource.id)
+    if only_missing:
+        has_candidates = select(EbookCandidatePhrase.id).where(EbookCandidatePhrase.book_id == EbookSource.id)
+        source_query = source_query.where(~has_candidates.exists())
+    sources = (await session.scalars(source_query)).all()
+    books_seen = len(sources)
+    logger.info(
+        "ebook_candidate_phrase_generation_start books_seen=%s min_tokens=%s max_tokens=%s max_candidates_per_book=%s",
+        books_seen,
+        config.phrase_min_tokens,
+        config.phrase_max_tokens,
+        config.protected_phrase_max_candidates_per_book,
+    )
+
+    outcomes = await generate_candidates_for_sources_pooled(session, sources, config)
+
+    result = PhraseCandidateGenerationResult(
+        books_seen=books_seen,
+        books_built=sum(1 for outcome in outcomes if outcome.built),
+        candidate_phrases=sum(outcome.candidates for outcome in outcomes),
+    )
+    logger.info(
+        "ebook_candidate_phrase_generation_complete books_seen=%s books_built=%s candidate_total=%s",
+        result.books_seen,
+        result.books_built,
+        result.candidate_phrases,
+    )
+    return result
+
+
+async def generate_candidates_for_sources_pooled(
+    session: AsyncSession,
+    sources: Sequence[EbookSource],
+    config: EbookSearchConfig,
+) -> list[BookCandidateResult]:
+    """Generate candidate phrases for many books, extracting them concurrently in worker processes.
+
+    Chapter loading and row persistence stay on the caller's session (serial), while the CPU-bound
+    extraction runs in the shared process pool. A bounded window of in-flight books overlaps
+    extraction across cores without loading every book's candidates into memory at once.
+
+    Args:
+        session (Session): Active database session.
+        sources (Sequence[EbookSource]): Indexed books to generate candidates for.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        list[BookCandidateResult]: One result per book.
+    """
+    pool = get_extraction_pool(config.protected_phrase_extraction_workers)
+    max_in_flight = max(1, config.protected_phrase_extraction_workers) * 2
+    pending: deque[tuple[EbookSource, Future[list[PhraseCandidate]]]] = deque()
+    outcomes: list[BookCandidateResult] = []
+
+    async def drain_one() -> None:
+        source, future = pending.popleft()
+        extracted = await asyncio.wrap_future(future)
+        outcomes.append(await store_source_candidates(session, source, extracted, config))
+
+    try:
+        for source in sources:
+            chapters = await load_book_chapter_texts(session, source.id)
+            if not chapters:
+                logger.warning("ebook_candidate_phrase_generation_book_empty source_id=%s", source.id)
+                outcomes.append(BookCandidateResult())
+                continue
+            future = pool.submit(
+                extract_phrase_candidates_for_book,
+                "\n\n".join(chapters),
+                chapters,
+                config,
+                metadata=metadata_for_source(source),
+            )
+            pending.append((source, future))
+            if len(pending) >= max_in_flight:
+                await drain_one()
+        while pending:
+            await drain_one()
+    except Exception:
+        for _, future in pending:
+            future.cancel()
+        await session.rollback()
+        logger.exception("ebook_candidate_phrase_generation_pooled_failed")
+        raise
+    return outcomes
+
+
+async def store_source_candidates(
+    session: AsyncSession,
+    source: EbookSource,
+    limited_candidates: list[PhraseCandidate],
+    config: EbookSearchConfig,
+) -> BookCandidateResult:
+    """Persist and commit one book's already-extracted candidates.
+
+    Args:
+        session (AsyncSession): Active database session.
+        source (EbookSource): Book the candidates belong to.
+        limited_candidates (list[PhraseCandidate]): Scored candidates to persist.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        BookCandidateResult: Candidate count and that the book was committed.
+    """
+    book_started_at = perf_counter()
+    saved_count = await store_candidate_phrases_for_book(session, source.id, None, limited_candidates, config)
+    await session.commit()
+    logger.info(
+        "ebook_candidate_phrase_generation_book_committed source_id=%s candidates=%s duration_ms=%.1f",
+        source.id,
+        saved_count,
+        (perf_counter() - book_started_at) * 1000,
+    )
+    return BookCandidateResult(candidates=saved_count, built=True)
+
+
+async def recalculate_candidate_phrases_for_book(
+    session: AsyncSession,
+    source: EbookSource,
+    config: EbookSearchConfig,
+    *,
+    nlp: SpacyLanguage | None = None,
+    use_process_pool: bool = False,
+) -> PhraseRecalculationResult:
+    """Remove all book phrase data, regenerate candidates, and commit the completed book.
+
+    Args:
+        session (Session): Active database session.
+        source (EbookSource): Indexed book to recalculate.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        nlp (SpacyLanguage | None): Optional spaCy pipeline for entity and noun-chunk sources.
+        use_process_pool (bool): Run the CPU-bound extraction in a worker process so concurrent
+            recalculations do not serialize behind the GIL. Defaults to in-process for callers
+            (tests, backfills) that do not need it.
+
+    Returns:
+        PhraseRecalculationResult: Deleted-row counts and the number of candidates regenerated.
+    """
+    started_at = perf_counter()
+    logger.info(
+        "ebook_candidate_phrase_recalculation_start source_id=%s title=%r",
+        source.id,
+        source.title,
+    )
+    try:
+        deleted = await delete_phrase_data_for_book(session, source.id)
+        chapters = await load_book_chapter_texts(session, source.id)
+        if not chapters:
+            logger.warning("ebook_candidate_phrase_recalculation_book_empty source_id=%s", source.id)
+            await session.commit()
+            return PhraseRecalculationResult(
+                book_id=source.id,
+                deleted_candidates=deleted.deleted_candidates,
+                deleted_protected_phrases=deleted.deleted_protected_phrases,
+                deleted_aliases=deleted.deleted_aliases,
+                deleted_mentions=deleted.deleted_mentions,
+                candidate_phrases=0,
+            )
+
+        candidate_count = await generate_candidate_phrases_for_book(
+            session,
+            source.id,
+            series_id=None,
+            chapters=chapters,
+            config=config,
+            nlp=nlp,
+            metadata=metadata_for_source(source),
+            replace_all=True,
+            use_process_pool=use_process_pool,
+        )
+        await session.commit()
+    except Exception:
+        await session.rollback()
+        logger.exception("ebook_candidate_phrase_recalculation_failed source_id=%s", source.id)
+        raise
+
+    result = PhraseRecalculationResult(
+        book_id=source.id,
+        deleted_candidates=deleted.deleted_candidates,
+        deleted_protected_phrases=deleted.deleted_protected_phrases,
+        deleted_aliases=deleted.deleted_aliases,
+        deleted_mentions=deleted.deleted_mentions,
+        candidate_phrases=candidate_count,
+    )
+    logger.info(
+        "ebook_candidate_phrase_recalculation_complete source_id=%s deleted_candidates=%s "
+        "deleted_protected=%s deleted_aliases=%s deleted_mentions=%s candidates=%s duration_ms=%.1f",
+        source.id,
+        result.deleted_candidates,
+        result.deleted_protected_phrases,
+        result.deleted_aliases,
+        result.deleted_mentions,
+        result.candidate_phrases,
+        (perf_counter() - started_at) * 1000,
+    )
+    return result
+
+
+async def generate_candidate_phrases_for_book(
+    session: AsyncSession,
+    book_id: int,
+    series_id: int | None,
+    chapters: Sequence[str],
+    config: EbookSearchConfig,
+    *,
+    nlp: SpacyLanguage | None = None,
+    metadata: Mapping[str, object] | None = None,
+    replace_all: bool = False,
+    use_process_pool: bool = False,
+) -> int:
+    """Extract and store candidate phrases for one book without LLM judging.
+
+    Args:
+        session (Session): Active database session.
+        book_id (int): Book the candidates belong to.
+        series_id (int | None): Series scope for the stored candidates.
+        chapters (Sequence[str]): Chapter-like text blocks used for extraction and frequency counts.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        nlp (SpacyLanguage | None): Optional spaCy pipeline for entity and noun-chunk sources.
+        metadata (Mapping[str, object] | None): Optional book metadata used as a candidate source.
+        replace_all (bool): When the caller has already cleared this book's candidates (e.g. a
+            recalculation), skip the per-candidate existence lookup and bulk-insert new rows.
+        use_process_pool (bool): Run the CPU-bound extraction in a worker process to avoid
+            serializing concurrent requests behind the GIL. Ignored when ``nlp`` is set, since
+            the spaCy pipeline cannot be sent to a worker process.
+
+    Returns:
+        int: Number of candidate phrase rows stored.
+    """
+    started_at = perf_counter()
+    book_text = "\n\n".join(chapters)
+    if use_process_pool and nlp is None:
+        limited_candidates = await extract_phrase_candidates_in_pool(book_text, chapters, config, metadata=metadata)
+    else:
+        limited_candidates = extract_phrase_candidates_for_book(
+            book_text,
+            chapters,
+            config,
+            nlp=nlp,
+            metadata=metadata,
+        )
+    saved_count = await store_candidate_phrases_for_book(
+        session,
+        book_id,
+        series_id,
+        limited_candidates,
+        config,
+        replace_all=replace_all,
+    )
+    logger.info(
+        "ebook_candidate_phrase_generation_book_duration book_id=%s candidates=%s duration_ms=%.1f",
+        book_id,
+        saved_count,
+        (perf_counter() - started_at) * 1000,
+    )
+    return saved_count
+
+
+async def store_candidate_phrases_for_book(
+    session: AsyncSession,
+    book_id: int,
+    series_id: int | None,
+    limited_candidates: list[PhraseCandidate],
+    config: EbookSearchConfig,
+    *,
+    replace_all: bool = False,
+) -> int:
+    """Persist already-extracted candidate phrase rows for one book without committing.
+
+    Args:
+        session (Session): Active database session.
+        book_id (int): Book the candidates belong to.
+        series_id (int | None): Series scope for the stored candidates.
+        limited_candidates (list[PhraseCandidate]): Scored candidates to persist.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        replace_all (bool): When the caller has already cleared this book's candidates, skip the
+            per-candidate existence lookup and bulk-insert new rows.
+
+    Returns:
+        int: Number of candidate phrase rows stored.
+    """
+    save_started_at = perf_counter()
+    if replace_all:
+        rows = [new_candidate_row(book_id, series_id, candidate) for candidate in limited_candidates]
+        session.add_all(rows)
+        await session.flush()
+        saved_count = len(rows)
+        logger.info(
+            "ebook_candidate_phrase_save_start book_id=%s candidates=%s mode=bulk_insert",
+            book_id,
+            len(limited_candidates),
+        )
+    else:
+        pruned_count = await prune_unstorable_unjudged_candidate_phrases(session, book_id, config)
+        logger.info(
+            "ebook_candidate_phrase_save_start book_id=%s candidates=%s pruned_unstorable=%s",
+            book_id,
+            len(limited_candidates),
+            pruned_count,
+        )
+        saved_count = await bulk_upsert_unjudged_candidates(session, book_id, series_id, limited_candidates)
+    logger.info(
+        "ebook_candidate_phrase_save_complete book_id=%s candidates=%s save_ms=%.1f",
+        book_id,
+        saved_count,
+        (perf_counter() - save_started_at) * 1000,
+    )
+    return saved_count

python/ebook_search/protected_phrases/judge_ngrams.py

@@ -0,0 +1,511 @@
+"""Book-level orchestration for LLM judging and promotion of candidate phrases."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import re
+from time import perf_counter
+from typing import TYPE_CHECKING
+
+import httpx
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from python.ebook_search.llm_interface import request_chat_completion
+from python.ebook_search.protected_phrases.extraction import (
+    candidate_source_names,
+    get_sample_contexts,
+    is_junk_phrase,
+    is_most_common_word_phrase,
+    score_candidate,
+)
+from python.ebook_search.protected_phrases.matching import index_chunk_phrase_mentions_for_book
+from python.ebook_search.protected_phrases.models import BookJudgmentResult, LLMJudgment, PhraseJudgmentBackfillResult
+from python.ebook_search.protected_phrases.store import (
+    count_protected_phrases,
+    count_unjudged_candidates,
+    load_book_text,
+    load_candidates_for_judgment,
+    phrase_candidate_from_row,
+    save_candidate_to_db,
+    upsert_protected_phrase,
+)
+from python.ebook_search.protected_phrases.text_normalization import normalize_text
+from python.orm.richie import EbookSource
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from sqlalchemy.ext.asyncio import AsyncEngine
+
+    from python.ebook_search.config import EbookSearchConfig
+    from python.ebook_search.protected_phrases.models import PhraseCandidate
+    from python.orm.richie import EbookProtectedPhrase
+
+JSON_OBJECT_RE = re.compile(r"\{.*\}", re.DOTALL)
+
+
+logger = logging.getLogger(__name__)
+
+
+async def judge_candidate_phrases_for_books(
+    engine: AsyncEngine,
+    config: EbookSearchConfig,
+    *,
+    source_ids: Sequence[int] | None = None,
+) -> PhraseJudgmentBackfillResult:
+    """Judge candidate phrases for books, fanning LLM calls out across books and phrases.
+
+    Up to ``phrase_judge_book_workers`` books are judged at once, and within each book candidates
+    are judged in concurrent chunks of ``phrase_judge_phrase_workers``. Each book uses its own
+    short-lived sessions for reads and writes; no database connection is held while LLM calls are
+    in flight. For a pseudo-single-threaded run (solo testing, debugging), set both worker
+    settings to 1.
+
+    Args:
+        engine (AsyncEngine): Engine used to open one session per book.
+        config (EbookSearchConfig): Runtime phrase-tuning settings and chat configuration.
+        source_ids (Sequence[int] | None): Books to judge; ``None`` judges every indexed book.
+
+    Returns:
+        PhraseJudgmentBackfillResult: Per-corpus counts of books judged, failures, candidates,
+        protected phrases, and mentions.
+    """
+    if source_ids is None:
+        async with AsyncSession(engine) as session:
+            source_ids = list((await session.scalars(select(EbookSource.id).order_by(EbookSource.id))).all())
+    books_seen = len(source_ids)
+    book_workers = max(1, config.phrase_judge_book_workers)
+    phrase_workers = max(1, config.phrase_judge_phrase_workers)
+    logger.info(
+        "ebook_candidate_phrase_judgment_start books_seen=%s book_workers=%s phrase_workers=%s "
+        "confidence_threshold=%.2f",
+        books_seen,
+        book_workers,
+        phrase_workers,
+        config.protected_phrase_confidence_threshold,
+    )
+
+    book_semaphore = asyncio.Semaphore(book_workers)
+    max_connections = book_workers * phrase_workers
+    limits = httpx.Limits(max_connections=max_connections, max_keepalive_connections=max_connections)
+    async with httpx.AsyncClient(limits=limits) as client:
+        outcomes = await asyncio.gather(
+            *(judge_one_book_async(engine, source_id, config, client, book_semaphore) for source_id in source_ids)
+        )
+
+    result = PhraseJudgmentBackfillResult(
+        books_seen=books_seen,
+        books_judged=sum(1 for outcome in outcomes if outcome.committed),
+        books_failed=sum(1 for outcome in outcomes if outcome.failed),
+        candidates_judged=sum(outcome.judged for outcome in outcomes),
+        protected_phrases=sum(outcome.protected for outcome in outcomes),
+        phrase_mentions=sum(outcome.mentions for outcome in outcomes),
+    )
+    logger.info(
+        "ebook_candidate_phrase_judgment_complete books_seen=%s books_judged=%s books_failed=%s "
+        "candidates_judged=%s protected=%s mentions=%s",
+        result.books_seen,
+        result.books_judged,
+        result.books_failed,
+        result.candidates_judged,
+        result.protected_phrases,
+        result.phrase_mentions,
+    )
+    return result
+
+
+async def judge_one_book_async(
+    engine: AsyncEngine,
+    source_id: int,
+    config: EbookSearchConfig,
+    client: httpx.AsyncClient,
+    book_semaphore: asyncio.Semaphore,
+) -> BookJudgmentResult:
+    """Judge one book concurrently and persist the outcome, honoring the book-level limit.
+
+    Args:
+        engine (AsyncEngine): Engine used to open the book's read and write sessions.
+        source_id (int): Book to judge candidates for.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        client (httpx.AsyncClient): Shared async client for LLM calls.
+        book_semaphore (asyncio.Semaphore): Caps how many books judge at once.
+
+    Returns:
+        BookJudgmentResult: The book's judgment outcome.
+    """
+    async with book_semaphore:
+        try:
+            prepared = await prepare_book_judgment(engine, source_id, config)
+            if prepared is None:
+                return BookJudgmentResult()
+            work_items, target_remaining = prepared
+            judged = await judge_book_candidates_async(client, config, source_id, work_items, target_remaining)
+            if not judged:
+                return BookJudgmentResult()
+            return await persist_book_judgments(engine, source_id, config, judged)
+        except Exception:
+            logger.exception("ebook_candidate_phrase_judgment_book_failed source_id=%s", source_id)
+            return BookJudgmentResult(failed=True)
+
+
+async def prepare_book_judgment(
+    engine: AsyncEngine,
+    source_id: int,
+    config: EbookSearchConfig,
+) -> tuple[list[tuple[int, PhraseCandidate]], int | None] | None:
+    """Load one book's candidates to judge, with sample contexts, on a short-lived read session.
+
+    Args:
+        engine (AsyncEngine): Engine used to open the read session.
+        source_id (int): Book to load candidates for.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        tuple[list[tuple[int, PhraseCandidate]], int | None] | None: Candidate rows paired with
+        in-memory candidates and the remaining protected-phrase target, or ``None`` when the book
+        has nothing to judge.
+    """
+    judgment_limit = config.protected_phrase_llm_candidates_per_book
+    if judgment_limit <= 0:
+        return None
+    async with AsyncSession(engine) as session:
+        if not await count_unjudged_candidates(session, source_id, config):
+            logger.info("ebook_candidate_phrase_judgment_book_skip_no_unjudged source_id=%s", source_id)
+            return None
+        existing_protected = await count_protected_phrases(session, source_id)
+        target_remaining: int | None = None
+        if config.phrase_target_protected_per_book > 0:
+            target_remaining = max(config.phrase_target_protected_per_book - existing_protected, 0)
+            if target_remaining == 0:
+                logger.info(
+                    "ebook_candidate_phrase_judgment_skipped_target_met source_id=%s existing_protected=%s target=%s",
+                    source_id,
+                    existing_protected,
+                    config.phrase_target_protected_per_book,
+                )
+                return None
+        book_text = await load_book_text(session, source_id)
+        if not book_text:
+            logger.warning("ebook_candidate_phrase_judgment_book_empty source_id=%s", source_id)
+            return None
+        normalized_book_text = normalize_text(book_text)
+        # Stored rows may predate the current junk filters and score weights, so re-filter and
+        # rescore every unjudged row here instead of trusting the persisted candidate_score.
+        rows = await load_candidates_for_judgment(session, source_id, config)
+        scored_items: list[tuple[int, PhraseCandidate]] = []
+        skipped_junk = 0
+        for row in rows:
+            candidate = phrase_candidate_from_row(row)
+            if is_junk_phrase(candidate.phrase_norm.split()):
+                skipped_junk += 1
+                continue
+            candidate.candidate_score = score_candidate(candidate, config)
+            scored_items.append((row.id, candidate))
+        scored_items.sort(key=lambda item: item[1].candidate_score, reverse=True)
+        work_items = scored_items[:judgment_limit]
+        for _, candidate in work_items:
+            candidate.sample_contexts = candidate.sample_contexts or get_sample_contexts(
+                normalized_book_text, candidate.phrase_norm
+            )
+    logger.info(
+        "ebook_candidate_phrase_judgment_candidates_loaded source_id=%s candidates=%s skipped_junk=%s "
+        "unjudged_rows=%s existing_protected=%s target_remaining=%s judgment_limit=%s",
+        source_id,
+        len(work_items),
+        skipped_junk,
+        len(rows),
+        existing_protected,
+        target_remaining,
+        judgment_limit,
+    )
+    return work_items, target_remaining
+
+
+async def judge_book_candidates_async(
+    client: httpx.AsyncClient,
+    config: EbookSearchConfig,
+    source_id: int,
+    work_items: list[tuple[int, PhraseCandidate]],
+    target_remaining: int | None,
+) -> list[tuple[int, PhraseCandidate, LLMJudgment, bool]]:
+    """Judge a book's candidates in concurrent chunks, stopping once the target is reached.
+
+    Promotion decisions are made in memory so judging can stop early without any database writes.
+
+    Args:
+        client (httpx.AsyncClient): Shared async client for LLM calls.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        source_id (int): Book being judged, for logging.
+        work_items (list[tuple[int, PhraseCandidate]]): Candidate row ids paired with candidates,
+            in best-first score order.
+        target_remaining (int | None): Remaining protected-phrase target, or ``None`` for no cap.
+
+    Returns:
+        list[tuple[int, PhraseCandidate, LLMJudgment, bool]]: Judged rows with their judgment and
+        whether each should be promoted.
+    """
+    chunk_size = max(1, config.phrase_judge_phrase_workers)
+    judged: list[tuple[int, PhraseCandidate, LLMJudgment, bool]] = []
+    promoted = 0
+    for start in range(0, len(work_items), chunk_size):
+        chunk = work_items[start : start + chunk_size]
+        judgments = await asyncio.gather(*(judge_candidate_async(client, config, candidate) for _, candidate in chunk))
+        for (candidate_id, candidate), judgment in zip(chunk, judgments, strict=True):
+            promote = (target_remaining is None or promoted < target_remaining) and should_protect_judged_candidate(
+                candidate, judgment, source_id, config, candidate_id=candidate_id
+            )
+            if promote:
+                promoted += 1
+            judged.append((candidate_id, candidate, judgment, promote))
+        if target_remaining is not None and promoted >= target_remaining:
+            break
+    return judged
+
+
+async def judge_candidate_async(
+    client: httpx.AsyncClient,
+    config: EbookSearchConfig,
+    candidate: PhraseCandidate,
+) -> LLMJudgment:
+    """Judge one candidate with the LLM over the shared async client.
+
+    Args:
+        client (httpx.AsyncClient): Shared async client for LLM calls.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        candidate (PhraseCandidate): Candidate to judge.
+
+    Returns:
+        LLMJudgment: The parsed judgment.
+    """
+    content = await request_chat_completion(client, config, build_judge_messages(candidate))
+    return parse_llm_judgment(content, config)
+
+
+async def persist_book_judgments(
+    engine: AsyncEngine,
+    source_id: int,
+    config: EbookSearchConfig,
+    judged: list[tuple[int, PhraseCandidate, LLMJudgment, bool]],
+) -> BookJudgmentResult:
+    """Persist one book's judgments and promotions in a single committed transaction.
+
+    Args:
+        engine (AsyncEngine): Engine used to open the write session.
+        source_id (int): Book being persisted.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        judged (list[tuple[int, PhraseCandidate, LLMJudgment, bool]]): Judged candidates with their
+            judgment and promotion flag.
+
+    Returns:
+        BookJudgmentResult: The book's committed counts, or a failed result on error.
+    """
+    book_started_at = perf_counter()
+    async with AsyncSession(engine, expire_on_commit=False) as session:
+        try:
+            protected: list[EbookProtectedPhrase] = []
+            for candidate_id, candidate, judgment, promote in judged:
+                candidate_row = await save_candidate_to_db(session, source_id, None, candidate, judgment=judgment)
+                if promote:
+                    protected.append(
+                        await upsert_protected_phrase(session, source_id, None, candidate, judgment, candidate_row)
+                    )
+                logger.info(
+                    "ebook_candidate_phrase_judgment_candidate_complete source_id=%s candidate_id=%s phrase=%r "
+                    "keep=%s confidence=%.3f category=%r promoted=%s",
+                    source_id,
+                    candidate_id,
+                    candidate.phrase_norm,
+                    judgment.keep,
+                    judgment.confidence,
+                    judgment.category,
+                    promote,
+                )
+            await session.flush()
+            mentions = await index_chunk_phrase_mentions_for_book(session, source_id, config) if protected else 0
+            await session.commit()
+        except Exception:
+            await session.rollback()
+            logger.exception("ebook_candidate_phrase_judgment_book_persist_failed source_id=%s", source_id)
+            return BookJudgmentResult(failed=True)
+    logger.info(
+        "ebook_candidate_phrase_judgment_book_committed source_id=%s judged=%s protected=%s mentions=%s "
+        "duration_ms=%.1f",
+        source_id,
+        len(judged),
+        len(protected),
+        mentions,
+        (perf_counter() - book_started_at) * 1000,
+    )
+    return BookJudgmentResult(judged=len(judged), protected=len(protected), mentions=mentions, committed=True)
+
+
+def should_protect_judged_candidate(
+    candidate: PhraseCandidate,
+    judgment: LLMJudgment,
+    book_id: int,
+    config: EbookSearchConfig,
+    *,
+    candidate_id: int,
+) -> bool:
+    """Report whether a judged candidate qualifies to become a protected phrase.
+
+    Args:
+        candidate (PhraseCandidate): In-memory candidate that was judged.
+        judgment (LLMJudgment): Judge decision for the candidate.
+        book_id (int): Book the candidate belongs to, for logging.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        candidate_id (int): Stored candidate row id the judgment came from, for logging.
+
+    Returns:
+        bool: True when the judged candidate should be promoted to a protected phrase.
+    """
+    if not judgment.keep or judgment.confidence < config.protected_phrase_confidence_threshold:
+        return False
+    accepted_norm = normalize_text(judgment.canonical or candidate.phrase_text)
+    accepted_tokens = accepted_norm.split()
+    accepted_token_count = len(accepted_tokens)
+    if accepted_token_count < config.phrase_min_tokens:
+        logger.info(
+            "ebook_candidate_phrase_judgment_candidate_skip_short_canonical book_id=%s candidate_id=%s "
+            "phrase=%r canonical=%r token_count=%s min_tokens=%s",
+            book_id,
+            candidate_id,
+            candidate.phrase_norm,
+            accepted_norm,
+            accepted_token_count,
+            config.phrase_min_tokens,
+        )
+        return False
+    if is_most_common_word_phrase(accepted_tokens):
+        logger.info(
+            "ebook_candidate_phrase_judgment_candidate_skip_common_canonical book_id=%s candidate_id=%s "
+            "phrase=%r canonical=%r",
+            book_id,
+            candidate_id,
+            candidate.phrase_norm,
+            accepted_norm,
+        )
+        return False
+    return True
+
+
+def build_judge_messages(candidate: PhraseCandidate) -> list[dict[str, str]]:
+    """Build the chat messages used to judge one candidate phrase.
+
+    Args:
+        candidate (PhraseCandidate): Candidate to describe for the judge.
+
+    Returns:
+        list[dict[str, str]]: OpenAI-style system and user messages.
+    """
+    payload = {
+        "phrase": candidate.phrase_norm,
+        "token_count": candidate.token_count,
+        "sources": candidate_source_names(candidate),
+        "raw_count": candidate.raw_count,
+        "chapter_count": candidate.chapter_count,
+        "contexts": candidate.sample_contexts,
+    }
+    return [
+        {
+            "role": "system",
+            "content": (
+                "Judge whether a candidate phrase from a book should be protected for RAG retrieval. "
+                "Do not extract new phrases. Reject common grammar fragments, ordinary nonspecific phrases, "
+                "unstable fragments, and phrases kept only because they are frequent. Keep people, places, "
+                "organizations, factions, events, technologies, fictional conditions, magic systems, formal titles, "
+                "named concepts, and recurring world-specific terms. Return only a JSON object with keys: keep, "
+                "canonical, category, aliases, confidence, importance, allow_nested, suppress_children, reason."
+            ),
+        },
+        {"role": "user", "content": json.dumps(payload, ensure_ascii=True)},
+    ]
+
+
+def parse_llm_judgment(content: str, config: EbookSearchConfig) -> LLMJudgment:
+    """Parse and validate an LLM phrase-judge response.
+
+    Args:
+        content (str): Raw model response text.
+        config (EbookSearchConfig): Runtime phrase-tuning settings supplying nesting defaults.
+
+    Returns:
+        LLMJudgment: The parsed and validated judgment.
+
+    Raises:
+        TypeError: If the decoded JSON body is not an object.
+    """
+    body = json.loads(extract_json_object(content))
+    if not isinstance(body, dict):
+        msg = "LLM phrase judge response is not a JSON object"
+        raise TypeError(msg)
+
+    aliases = body.get("aliases", ())
+    if not isinstance(aliases, list | tuple):
+        aliases = ()
+    return LLMJudgment(
+        keep=bool(body.get("keep", False)),
+        canonical=optional_text(body.get("canonical")),
+        category=optional_text(body.get("category")),
+        aliases=tuple(str(alias) for alias in aliases if isinstance(alias, str) and alias.strip()),
+        confidence=clamped_float(body.get("confidence"), default=0.0),
+        importance=clamped_float(body.get("importance"), default=0.5),
+        allow_nested=bool(body.get("allow_nested", config.phrase_default_allow_nested)),
+        suppress_children=bool(body.get("suppress_children", config.phrase_default_suppress_children)),
+        reason=optional_text(body.get("reason")),
+    )
+
+
+def extract_json_object(content: str) -> str:
+    """Extract a JSON object from plain or fenced model output.
+
+    Args:
+        content (str): Raw model response text.
+
+    Returns:
+        str: The substring spanning the first JSON object.
+
+    Raises:
+        ValueError: If no JSON object is found in the response.
+    """
+    stripped = content.strip()
+    if stripped.startswith("{") and stripped.endswith("}"):
+        return stripped
+    match = JSON_OBJECT_RE.search(stripped)
+    if match is None:
+        msg = "LLM phrase judge response did not contain a JSON object"
+        raise ValueError(msg)
+    return match.group(0)
+
+
+def optional_text(value: object) -> str | None:
+    """Return stripped text for a nullable JSON value.
+
+    Args:
+        value (object): Decoded JSON value that may or may not be a string.
+
+    Returns:
+        str | None: The stripped string, or ``None`` when it is not a non-empty string.
+    """
+    if not isinstance(value, str):
+        return None
+    stripped = value.strip()
+    return stripped or None
+
+
+def clamped_float(value: object, *, default: float) -> float:
+    """Coerce a JSON number into the 0.0 to 1.0 range.
+
+    Args:
+        value (object): Decoded JSON value that may or may not be a number.
+        default (float): Fallback returned when ``value`` is not numeric.
+
+    Returns:
+        float: The value clamped to ``[0.0, 1.0]``, or ``default`` when non-numeric.
+    """
+    if not isinstance(value, int | float):
+        return default
+    return min(max(float(value), 0.0), 1.0)

python/ebook_search/protected_phrases/matching.py

@@ -0,0 +1,491 @@
+"""Runtime protected-phrase matching and chunk mention indexing."""
+
+from __future__ import annotations
+
+import logging
+from collections import defaultdict
+from typing import TYPE_CHECKING
+
+from sqlalchemy import and_, delete, func, or_, select
+
+from python.ebook_search.protected_phrases.config import get_ignored_phrases
+from python.ebook_search.protected_phrases.models import (
+    ChunkPhraseHit,
+    HydratedPhraseMatch,
+    PhraseLookup,
+    PhraseMatch,
+)
+from python.ebook_search.protected_phrases.text_normalization import tokenize_with_offsets
+from python.orm.richie import (
+    EbookChunk,
+    EbookChunkPhraseMention,
+    EbookPhraseAlias,
+    EbookProtectedPhrase,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator, Sequence
+
+    from sqlalchemy.ext.asyncio import AsyncSession
+
+    from python.ebook_search.config import EbookSearchConfig
+    from python.ebook_search.protected_phrases.text_normalization import NormalizedToken
+
+logger = logging.getLogger(__name__)
+
+
+async def load_phrase_lookup(
+    session: AsyncSession,
+    config: EbookSearchConfig,
+    *,
+    book_id: int | None = None,
+    series_id: int | None = None,
+) -> PhraseLookup:
+    """Load protected phrases and aliases into RAM lookup maps.
+
+    Args:
+        session (AsyncSession): Active database session.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        book_id (int | None): Optional book scope to restrict loaded phrases.
+        series_id (int | None): Optional series scope to restrict loaded phrases.
+
+    Returns:
+        PhraseLookup: Normalized phrase and alias maps with the token-window bounds to test.
+    """
+    norm_to_ids: defaultdict[str, list[int]] = defaultdict(list)
+    alias_to_ids: defaultdict[str, list[int]] = defaultdict(list)
+    max_tokens = config.phrase_max_tokens
+
+    phrase_statement = select(
+        EbookProtectedPhrase.id,
+        EbookProtectedPhrase.phrase_norm,
+        EbookProtectedPhrase.token_count,
+    )
+    scope_filter = protected_phrase_scope_filter(book_id=book_id, series_id=series_id)
+    if scope_filter is not None:
+        phrase_statement = phrase_statement.where(scope_filter)
+
+    for row in await session.execute(phrase_statement):
+        phrase_id = int(row.id)
+        phrase_norm = str(row.phrase_norm)
+        norm_to_ids[phrase_norm].append(phrase_id)
+        max_tokens = max(max_tokens, int(row.token_count))
+
+    alias_statement = select(
+        EbookPhraseAlias.alias_norm,
+        EbookPhraseAlias.phrase_id,
+    ).join(EbookProtectedPhrase, EbookProtectedPhrase.id == EbookPhraseAlias.phrase_id)
+    if scope_filter is not None:
+        alias_statement = alias_statement.where(scope_filter)
+
+    for row in await session.execute(alias_statement):
+        alias_norm = str(row.alias_norm)
+        alias_to_ids[alias_norm].append(int(row.phrase_id))
+        max_tokens = max(max_tokens, len(alias_norm.split()))
+
+    return PhraseLookup(
+        norm_to_phrase_ids={key: tuple(values) for key, values in norm_to_ids.items()},
+        alias_to_phrase_ids={key: tuple(values) for key, values in alias_to_ids.items()},
+        min_tokens=config.phrase_min_tokens,
+        max_tokens=max_tokens,
+    )
+
+
+def protected_phrase_scope_filter(*, book_id: int | None, series_id: int | None) -> object | None:
+    """Build a SQLAlchemy filter for optional phrase book and series scope.
+
+    Args:
+        book_id (int | None): Optional book scope to include alongside global phrases.
+        series_id (int | None): Optional series scope to include alongside global phrases.
+
+    Returns:
+        object | None: A combined SQLAlchemy filter clause, or ``None`` when no scope is given.
+    """
+    conditions = []
+    if book_id is not None:
+        conditions.append(or_(EbookProtectedPhrase.book_id.is_(None), EbookProtectedPhrase.book_id == book_id))
+    if series_id is not None:
+        conditions.append(or_(EbookProtectedPhrase.series_id.is_(None), EbookProtectedPhrase.series_id == series_id))
+    if not conditions:
+        return None
+    return and_(*conditions)
+
+
+def generate_query_ngrams(
+    tokens_: Sequence[str],
+    min_n: int,
+    max_n: int,
+) -> Iterator[tuple[str, int, int]]:
+    """Generate normalized query windows from longest to shortest.
+
+    Args:
+        tokens_ (Sequence[str]): Normalized query tokens.
+        min_n (int): Smallest window size to yield.
+        max_n (int): Largest window size to yield, capped at the token count.
+
+    Yields:
+        tuple[str, int, int]: Normalized window text with its start and end token indices.
+    """
+    capped_max_n = min(max_n, len(tokens_))
+    for ngram_size in range(capped_max_n, min_n - 1, -1):
+        for start in range(len(tokens_) - ngram_size + 1):
+            end = start + ngram_size
+            phrase_norm = " ".join(tokens_[start:end])
+            if phrase_norm in get_ignored_phrases():
+                continue
+            yield phrase_norm, start, end
+
+
+def detect_phrase_candidates(query_text: str, lookup: PhraseLookup) -> list[PhraseMatch]:
+    """Detect protected phrase windows in a user query using RAM hash lookups.
+
+    Args:
+        query_text (str): User query text to scan.
+        lookup (PhraseLookup): In-memory phrase and alias lookup maps.
+
+    Returns:
+        list[PhraseMatch]: Unhydrated phrase matches found in the query.
+    """
+    return detect_phrase_candidates_from_tokens(tokenize_with_offsets(query_text), lookup)
+
+
+def detect_phrase_candidates_in_text(text: str, lookup: PhraseLookup) -> list[PhraseMatch]:
+    """Detect protected phrase windows in arbitrary text with character offsets.
+
+    Args:
+        text (str): Arbitrary text, such as a chunk, to scan.
+        lookup (PhraseLookup): In-memory phrase and alias lookup maps.
+
+    Returns:
+        list[PhraseMatch]: Unhydrated phrase matches found in the text.
+    """
+    return detect_phrase_candidates_from_tokens(tokenize_with_offsets(text), lookup)
+
+
+def detect_phrase_candidates_from_tokens(tokens_: Sequence[NormalizedToken], lookup: PhraseLookup) -> list[PhraseMatch]:
+    """Detect protected phrase windows from already-normalized tokens.
+
+    Args:
+        tokens_ (Sequence[NormalizedToken]): Normalized tokens with character offsets.
+        lookup (PhraseLookup): In-memory phrase and alias lookup maps.
+
+    Returns:
+        list[PhraseMatch]: Deduplicated unhydrated phrase matches with token and character spans.
+    """
+    matches: list[PhraseMatch] = []
+    seen: set[tuple[int | None, str, int, int]] = set()
+    token_texts = [token.text for token in tokens_]
+    for phrase_norm, start, end in generate_query_ngrams(token_texts, min_n=lookup.min_tokens, max_n=lookup.max_tokens):
+        phrase_ids = lookup.norm_to_phrase_ids.get(phrase_norm, ())
+        alias_ids = lookup.alias_to_phrase_ids.get(phrase_norm, ())
+        for phrase_id in (*phrase_ids, *alias_ids):
+            key = (phrase_id, phrase_norm, start, end)
+            if key in seen:
+                continue
+            seen.add(key)
+            matches.append(
+                PhraseMatch(
+                    phrase_norm=phrase_norm,
+                    phrase_id=phrase_id,
+                    start_token=start,
+                    end_token=end,
+                    token_count=end - start,
+                    start_char=tokens_[start].start_char,
+                    end_char=tokens_[end - 1].end_char,
+                )
+            )
+    return matches
+
+
+async def hydrate_matches(session: AsyncSession, matches: Sequence[PhraseMatch]) -> list[HydratedPhraseMatch]:
+    """Fetch protected phrase metadata for raw phrase matches.
+
+    Args:
+        session (AsyncSession): Active database session.
+        matches (Sequence[PhraseMatch]): Unhydrated matches to enrich.
+
+    Returns:
+        list[HydratedPhraseMatch]: Matches with protected-phrase metadata attached.
+    """
+    if not matches:
+        return []
+
+    phrase_ids = sorted({match.phrase_id for match in matches if match.phrase_id is not None})
+    if not phrase_ids:
+        return []
+
+    rows = {
+        row.id: row
+        for row in await session.scalars(select(EbookProtectedPhrase).where(EbookProtectedPhrase.id.in_(phrase_ids)))
+    }
+    hydrated: list[HydratedPhraseMatch] = []
+    for match in matches:
+        if match.phrase_id is None:
+            continue
+        phrase = rows.get(match.phrase_id)
+        if phrase is None:
+            continue
+        hydrated.append(
+            HydratedPhraseMatch(
+                phrase_id=phrase.id,
+                matched_norm=match.phrase_norm,
+                phrase_text=phrase.phrase_text,
+                phrase_norm=phrase.phrase_norm,
+                canonical_id=phrase.canonical_id,
+                phrase_type=phrase.phrase_type,
+                token_count=match.token_count,
+                confidence=phrase.confidence,
+                importance=phrase.importance,
+                allow_nested=phrase.allow_nested,
+                suppress_children=phrase.suppress_children,
+                start_token=match.start_token,
+                end_token=match.end_token,
+                start_char=match.start_char,
+                end_char=match.end_char,
+                book_id=phrase.book_id,
+                series_id=phrase.series_id,
+            )
+        )
+    return hydrated
+
+
+def overlaps(first: HydratedPhraseMatch, second: HydratedPhraseMatch) -> bool:
+    """Return whether two token spans overlap.
+
+    Args:
+        first (HydratedPhraseMatch): First match to compare.
+        second (HydratedPhraseMatch): Second match to compare.
+
+    Returns:
+        bool: True when the two token spans share at least one token position.
+    """
+    return not (first.end_token <= second.start_token or first.start_token >= second.end_token)
+
+
+def is_inside(child: HydratedPhraseMatch, parent: HydratedPhraseMatch) -> bool:
+    """Return whether one token span is strictly inside another.
+
+    Args:
+        child (HydratedPhraseMatch): Candidate nested match.
+        parent (HydratedPhraseMatch): Candidate enclosing match.
+
+    Returns:
+        bool: True when ``child`` lies within ``parent`` and is not the same span.
+    """
+    return (
+        child.start_token >= parent.start_token
+        and child.end_token <= parent.end_token
+        and (child.start_token, child.end_token, child.phrase_id)
+        != (parent.start_token, parent.end_token, parent.phrase_id)
+    )
+
+
+def rank_match(match: HydratedPhraseMatch) -> tuple[float, float, int]:
+    """Rank phrase matches by importance, confidence, then token count.
+
+    Args:
+        match (HydratedPhraseMatch): Match to build a sort key for.
+
+    Returns:
+        tuple[float, float, int]: A comparable key of importance, confidence, and token count.
+    """
+    return (match.importance, match.confidence, match.token_count)
+
+
+def should_suppress(candidate: HydratedPhraseMatch, kept: HydratedPhraseMatch) -> bool:
+    """Return whether an already-kept match should suppress a candidate.
+
+    Args:
+        candidate (HydratedPhraseMatch): Match being considered for keeping.
+        kept (HydratedPhraseMatch): Match already kept that may suppress the candidate.
+
+    Returns:
+        bool: True when the candidate should be dropped in favor of the kept match.
+    """
+    if not overlaps(candidate, kept):
+        return False
+    if candidate.canonical_id == kept.canonical_id:
+        return rank_match(kept) >= rank_match(candidate)
+    if is_inside(candidate, kept) and kept.suppress_children and not candidate.allow_nested:
+        return True
+    return not candidate.allow_nested and rank_match(kept) > rank_match(candidate)
+
+
+def resolve_overlaps(matches: Sequence[HydratedPhraseMatch]) -> list[HydratedPhraseMatch]:
+    """Resolve overlapping phrase matches without relying only on longest match.
+
+    Args:
+        matches (Sequence[HydratedPhraseMatch]): Hydrated matches that may overlap.
+
+    Returns:
+        list[HydratedPhraseMatch]: The kept, non-suppressed matches.
+    """
+    sorted_matches = sorted(
+        matches,
+        key=lambda match: (match.start_token, -match.token_count, -match.importance, -match.confidence),
+    )
+    kept: list[HydratedPhraseMatch] = []
+    for candidate in sorted_matches:
+        if any(should_suppress(candidate, existing) for existing in kept):
+            continue
+        kept.append(candidate)
+    return kept
+
+
+async def detect_protected_phrases_for_query(
+    session: AsyncSession,
+    query_text: str,
+    config: EbookSearchConfig,
+    *,
+    lookup: PhraseLookup | None = None,
+    book_id: int | None = None,
+    series_id: int | None = None,
+) -> list[HydratedPhraseMatch]:
+    """Run the full online protected-phrase query-detection pipeline.
+
+    Args:
+        session (AsyncSession): Active database session.
+        query_text (str): User query text to detect phrases in.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        lookup (PhraseLookup | None): Optional preloaded lookup; loaded on demand when ``None``.
+        book_id (int | None): Optional book scope for lookup loading.
+        series_id (int | None): Optional series scope for lookup loading.
+
+    Returns:
+        list[HydratedPhraseMatch]: Hydrated, overlap-resolved phrase matches for the query.
+    """
+    active_lookup = (
+        lookup
+        if lookup is not None
+        else await load_phrase_lookup(session, config, book_id=book_id, series_id=series_id)
+    )
+    return resolve_overlaps(await hydrate_matches(session, detect_phrase_candidates(query_text, active_lookup)))
+
+
+async def index_chunk_phrase_mentions_for_book(
+    session: AsyncSession,
+    book_id: int,
+    config: EbookSearchConfig,
+    *,
+    series_id: int | None = None,
+    lookup: PhraseLookup | None = None,
+) -> int:
+    """Rebuild chunk phrase mentions for all chunks in one book.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose chunk mentions are rebuilt.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        series_id (int | None): Optional series scope for lookup loading.
+        lookup (PhraseLookup | None): Optional preloaded lookup; loaded on demand when ``None``.
+
+    Returns:
+        int: Total number of chunk phrase mentions indexed for the book.
+    """
+    active_lookup = (
+        lookup
+        if lookup is not None
+        else await load_phrase_lookup(session, config, book_id=book_id, series_id=series_id)
+    )
+    await session.execute(delete(EbookChunkPhraseMention).where(EbookChunkPhraseMention.book_id == book_id))
+    chunks = await session.scalars(select(EbookChunk).where(EbookChunk.source_id == book_id).order_by(EbookChunk.id))
+    count = 0
+    for chunk in chunks:
+        count += await index_chunk_phrase_mentions(session, chunk, lookup=active_lookup)
+    await session.flush()
+    logger.info("ebook_chunk_phrase_mentions_indexed book_id=%s mentions=%s", book_id, count)
+    return count
+
+
+async def index_chunk_phrase_mentions(session: AsyncSession, chunk: EbookChunk, *, lookup: PhraseLookup) -> int:
+    """Store protected phrase mentions for one chunk.
+
+    Args:
+        session (AsyncSession): Active database session.
+        chunk (EbookChunk): Chunk whose text is scanned for phrase mentions.
+        lookup (PhraseLookup): In-memory phrase and alias lookup maps.
+
+    Returns:
+        int: Number of phrase mentions stored for the chunk.
+    """
+    raw_matches = detect_phrase_candidates_in_text(chunk.text, lookup)
+    hydrated = resolve_overlaps(await hydrate_matches(session, raw_matches))
+    for match in hydrated:
+        session.add(
+            EbookChunkPhraseMention(
+                chunk_id=chunk.id,
+                phrase_id=match.phrase_id,
+                book_id=match.book_id if match.book_id is not None else chunk.source_id,
+                series_id=match.series_id,
+                start_char=match.start_char if match.start_char is not None else 0,
+                end_char=match.end_char,
+            )
+        )
+    return len(hydrated)
+
+
+async def phrase_hits_for_chunks(
+    session: AsyncSession,
+    *,
+    chunk_ids: Sequence[int],
+    phrase_ids: Sequence[int],
+) -> dict[int, tuple[ChunkPhraseHit, ...]]:
+    """Return matched protected phrases with mention counts by chunk id using indexed chunk mentions.
+
+    Args:
+        session (AsyncSession): Active database session.
+        chunk_ids (Sequence[int]): Chunk ids to look up mentions for.
+        phrase_ids (Sequence[int]): Protected phrase ids to restrict the results to.
+
+    Returns:
+        dict[int, tuple[ChunkPhraseHit, ...]]: Phrase hits per chunk id, ordered by mention count.
+    """
+    if not chunk_ids or not phrase_ids:
+        return {}
+
+    mention_count = func.count(EbookChunkPhraseMention.phrase_id).label("mention_count")
+    statement = (
+        select(
+            EbookChunkPhraseMention.chunk_id,
+            EbookProtectedPhrase.id.label("phrase_id"),
+            EbookProtectedPhrase.phrase_text,
+            mention_count,
+        )
+        .join(EbookProtectedPhrase, EbookProtectedPhrase.id == EbookChunkPhraseMention.phrase_id)
+        .where(
+            EbookChunkPhraseMention.chunk_id.in_(chunk_ids),
+            EbookChunkPhraseMention.phrase_id.in_(phrase_ids),
+        )
+        .group_by(EbookChunkPhraseMention.chunk_id, EbookProtectedPhrase.id, EbookProtectedPhrase.phrase_text)
+        .order_by(EbookChunkPhraseMention.chunk_id, mention_count.desc(), EbookProtectedPhrase.phrase_text)
+    )
+    hits: defaultdict[int, list[ChunkPhraseHit]] = defaultdict(list)
+    for row in await session.execute(statement):
+        hits[int(row.chunk_id)].append(
+            ChunkPhraseHit(
+                phrase_id=int(row.phrase_id),
+                phrase_text=str(row.phrase_text),
+                mention_count=int(row.mention_count),
+            )
+        )
+    return {chunk_id: tuple(chunk_hits) for chunk_id, chunk_hits in hits.items()}
+
+
+async def phrase_hit_counts_for_chunks(
+    session: AsyncSession,
+    *,
+    chunk_ids: Sequence[int],
+    phrase_ids: Sequence[int],
+) -> dict[int, int]:
+    """Return phrase-hit counts by chunk id using indexed chunk mentions.
+
+    Args:
+        session (AsyncSession): Active database session.
+        chunk_ids (Sequence[int]): Chunk ids to count mentions for.
+        phrase_ids (Sequence[int]): Protected phrase ids to restrict the counts to.
+
+    Returns:
+        dict[int, int]: Total mention count per chunk id.
+    """
+    hits = await phrase_hits_for_chunks(session, chunk_ids=chunk_ids, phrase_ids=phrase_ids)
+    return {chunk_id: sum(hit.mention_count for hit in chunk_hits) for chunk_id, chunk_hits in hits.items()}

python/ebook_search/protected_phrases/models.py

@@ -0,0 +1,285 @@
+"""Dataclasses shared by protected phrase extraction, judging, matching, and backfills."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+
+@dataclass(slots=True)
+class PhraseCandidate:
+    """A phrase candidate with merged extraction-source metadata.
+
+    Attributes:
+        phrase_text (str): Display text for the phrase.
+        phrase_norm (str): Normalized phrase used as the merge key.
+        token_count (int): Number of normalized tokens in the phrase.
+        source_raw_ngram (bool): Whether the raw n-gram extractor produced the phrase.
+        source_yake (bool): Whether YAKE keyword extraction produced the phrase.
+        source_spacy_ner (bool): Whether spaCy named-entity recognition produced the phrase.
+        source_spacy_noun_chunk (bool): Whether spaCy noun chunking produced the phrase.
+        source_capitalized (bool): Whether the capitalized-run extractor produced the phrase.
+        source_metadata (bool): Whether book metadata produced the phrase.
+        spacy_label (str | None): spaCy entity label when NER produced the phrase.
+        raw_count (int): Occurrences counted across the book text.
+        chapter_count (int): Number of chapters containing the phrase.
+        yake_score (float | None): Raw YAKE score when available; lower is better.
+        candidate_score (float): Combined pre-judging score.
+        sample_contexts (list[str]): Normalized context snippets around occurrences.
+    """
+
+    phrase_text: str
+    phrase_norm: str
+    token_count: int
+    source_raw_ngram: bool = False
+    source_yake: bool = False
+    source_spacy_ner: bool = False
+    source_spacy_noun_chunk: bool = False
+    source_capitalized: bool = False
+    source_metadata: bool = False
+    spacy_label: str | None = None
+    raw_count: int = 0
+    chapter_count: int = 0
+    yake_score: float | None = None
+    candidate_score: float = 0.0
+    sample_contexts: list[str] = field(default_factory=list)
+
+
+@dataclass(frozen=True, slots=True)
+class LLMJudgment:
+    """A structured phrase judgment returned by the LLM judge.
+
+    Attributes:
+        keep (bool): Whether the judge accepted the phrase for protection.
+        canonical (str | None): Canonical phrase text chosen by the judge.
+        category (str | None): Phrase category such as person, place, or event.
+        aliases (tuple[str, ...]): Alternate surface forms for the phrase.
+        confidence (float): Judge confidence between 0.0 and 1.0.
+        importance (float): Judge importance between 0.0 and 1.0.
+        allow_nested (bool): Whether the phrase may match inside a larger kept match.
+        suppress_children (bool): Whether the phrase suppresses matches nested inside it.
+        reason (str | None): Free-text explanation from the judge.
+    """
+
+    keep: bool
+    canonical: str | None
+    category: str | None
+    aliases: tuple[str, ...]
+    confidence: float
+    importance: float = 0.5
+    allow_nested: bool = False
+    suppress_children: bool = True
+    reason: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class PhraseLookup:
+    """In-memory lookup maps used for constant-time phrase-window checks.
+
+    Attributes:
+        norm_to_phrase_ids (Mapping[str, tuple[int, ...]]): Normalized phrase to protected phrase ids.
+        alias_to_phrase_ids (Mapping[str, tuple[int, ...]]): Normalized alias to protected phrase ids.
+        min_tokens (int): Smallest token-window size to test.
+        max_tokens (int): Largest token-window size to test.
+    """
+
+    norm_to_phrase_ids: Mapping[str, tuple[int, ...]]
+    alias_to_phrase_ids: Mapping[str, tuple[int, ...]]
+    min_tokens: int
+    max_tokens: int
+
+
+@dataclass(frozen=True, slots=True)
+class PhraseMatch:
+    """An unhydrated query or chunk phrase match.
+
+    Attributes:
+        phrase_norm (str): Normalized text of the matched window.
+        start_token (int): Index of the first matched token.
+        end_token (int): Index one past the last matched token.
+        token_count (int): Number of tokens in the match.
+        phrase_id (int | None): Matched protected phrase id when known.
+        start_char (int | None): Start character offset in the source text.
+        end_char (int | None): End character offset in the source text.
+    """
+
+    phrase_norm: str
+    start_token: int
+    end_token: int
+    token_count: int
+    phrase_id: int | None = None
+    start_char: int | None = None
+    end_char: int | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class HydratedPhraseMatch:
+    """A phrase match with protected-phrase metadata attached.
+
+    Attributes:
+        phrase_id (int): Protected phrase id.
+        matched_norm (str): Normalized window text that matched.
+        phrase_text (str): Display text of the protected phrase.
+        phrase_norm (str): Normalized text of the protected phrase.
+        canonical_id (str): Deterministic ``category:slug`` identifier.
+        phrase_type (str | None): Phrase category.
+        token_count (int): Number of tokens in the match.
+        confidence (float): Stored judge confidence.
+        importance (float): Stored judge importance.
+        allow_nested (bool): Whether the phrase may match inside a larger kept match.
+        suppress_children (bool): Whether the phrase suppresses matches nested inside it.
+        start_token (int): Index of the first matched token.
+        end_token (int): Index one past the last matched token.
+        start_char (int | None): Start character offset in the source text.
+        end_char (int | None): End character offset in the source text.
+        book_id (int | None): Book scope of the phrase.
+        series_id (int | None): Series scope of the phrase.
+    """
+
+    phrase_id: int
+    matched_norm: str
+    phrase_text: str
+    phrase_norm: str
+    canonical_id: str
+    phrase_type: str | None
+    token_count: int
+    confidence: float
+    importance: float
+    allow_nested: bool
+    suppress_children: bool
+    start_token: int
+    end_token: int
+    start_char: int | None = None
+    end_char: int | None = None
+    book_id: int | None = None
+    series_id: int | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class ChunkPhraseHit:
+    """One protected phrase with its mention count inside one retrieved chunk.
+
+    Attributes:
+        phrase_id (int): Protected phrase id.
+        phrase_text (str): Display text of the protected phrase.
+        mention_count (int): Indexed mentions of the phrase in the chunk.
+    """
+
+    phrase_id: int
+    phrase_text: str
+    mention_count: int
+
+
+@dataclass(frozen=True, slots=True)
+class PhraseCandidateGenerationResult:
+    """Summary of candidate phrase extraction for indexed books.
+
+    Attributes:
+        books_seen (int): Indexed books examined.
+        books_built (int): Books that had candidates generated and committed.
+        candidate_phrases (int): Candidate phrases stored across all books.
+    """
+
+    books_seen: int
+    books_built: int
+    candidate_phrases: int
+
+
+@dataclass(frozen=True, slots=True)
+class CorpusPhraseStats:
+    """Corpus-wide candidate and protected phrase counts for the admin page.
+
+    Attributes:
+        total_books (int): Indexed books in the corpus.
+        books_with_candidates (int): Books that have candidate phrases generated.
+        books_fully_judged (int): Books with candidates where every candidate has been judged.
+        candidate_phrases (int): Candidate phrases stored across all books.
+        judged_candidates (int): Candidate phrases that have been LLM judged.
+        unjudged_candidates (int): Candidate phrases still waiting for judgment.
+        protected_phrases (int): Protected phrases promoted across all books.
+    """
+
+    total_books: int
+    books_with_candidates: int
+    books_fully_judged: int
+    candidate_phrases: int
+    judged_candidates: int
+    unjudged_candidates: int
+    protected_phrases: int
+
+
+@dataclass(frozen=True, slots=True)
+class PhraseJudgmentBackfillResult:
+    """Summary of LLM judging for stored candidate phrases.
+
+    Attributes:
+        books_seen (int): Indexed books examined.
+        books_judged (int): Books with judgments committed.
+        books_failed (int): Books rolled back after an error.
+        candidates_judged (int): Candidate phrases sent to the LLM judge.
+        protected_phrases (int): Protected phrases promoted from candidates.
+        phrase_mentions (int): Chunk phrase mentions indexed across all books.
+    """
+
+    books_seen: int
+    books_judged: int
+    books_failed: int
+    candidates_judged: int
+    protected_phrases: int
+    phrase_mentions: int
+
+
+@dataclass(frozen=True, slots=True)
+class BookJudgmentResult:
+    """Outcome of judging one book's candidate phrases.
+
+    Attributes:
+        judged (int): Candidate phrases sent to the LLM judge.
+        protected (int): Protected phrases promoted from candidates.
+        mentions (int): Chunk phrase mentions indexed for the book.
+        committed (bool): Whether the book's judgments were committed.
+        failed (bool): Whether the book was rolled back after an error.
+    """
+
+    judged: int = 0
+    protected: int = 0
+    mentions: int = 0
+    committed: bool = False
+    failed: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class BookCandidateResult:
+    """Outcome of generating one book's candidate phrases.
+
+    Attributes:
+        candidates (int): Candidate phrases stored for the book.
+        built (bool): Whether candidate generation was committed.
+    """
+
+    candidates: int = 0
+    built: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class PhraseRecalculationResult:
+    """Summary of phrase cleanup and candidate regeneration for one book.
+
+    Attributes:
+        book_id (int): Book the recalculation ran against.
+        deleted_candidates (int): Candidate phrase rows deleted.
+        deleted_protected_phrases (int): Protected phrase rows deleted.
+        deleted_aliases (int): Phrase alias rows deleted.
+        deleted_mentions (int): Chunk phrase mention rows deleted.
+        candidate_phrases (int): Candidate phrases regenerated after cleanup.
+    """
+
+    book_id: int
+    deleted_candidates: int
+    deleted_protected_phrases: int
+    deleted_aliases: int
+    deleted_mentions: int
+    candidate_phrases: int

python/ebook_search/protected_phrases/pool.py

@@ -0,0 +1,101 @@
+"""Process pool for offloading CPU-bound phrase extraction off the request thread.
+
+Phrase extraction is pure-Python CPU work (n-gram sliding, YAKE), so running it inline in a
+sync request handler serializes concurrent recalculations behind the GIL. Submitting it to a
+``ProcessPoolExecutor`` lets concurrent extractions run in parallel across cores instead. A
+``spawn`` context is used so workers do not inherit the parent's database engine, connections,
+or server threads.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import multiprocessing
+import os
+from concurrent.futures import ProcessPoolExecutor
+from threading import Lock
+from typing import TYPE_CHECKING
+
+from python.ebook_search.protected_phrases.extraction import extract_phrase_candidates_for_book
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping, Sequence
+
+    from python.ebook_search.config import EbookSearchConfig
+    from python.ebook_search.protected_phrases.models import PhraseCandidate
+
+logger = logging.getLogger(__name__)
+
+
+class _ExtractionPool:
+    """Lazily created process-wide extraction pool and the lock guarding it."""
+
+    def __init__(self) -> None:
+        self.lock = Lock()
+        self.pool: ProcessPoolExecutor | None = None
+
+
+_extraction_pool = _ExtractionPool()
+
+
+def get_extraction_pool(max_workers: int) -> ProcessPoolExecutor:
+    """Return the shared extraction process pool, creating it on first use.
+
+    Args:
+        max_workers (int): Desired worker count; values below 1 fall back to the CPU count.
+
+    Returns:
+        ProcessPoolExecutor: The shared pool for phrase extraction.
+    """
+    with _extraction_pool.lock:
+        if _extraction_pool.pool is None:
+            workers = max_workers if max_workers > 0 else (os.cpu_count() or 1)
+            _extraction_pool.pool = ProcessPoolExecutor(
+                max_workers=workers,
+                mp_context=multiprocessing.get_context("spawn"),
+            )
+            logger.info("ebook_phrase_extraction_pool_started workers=%s", workers)
+        return _extraction_pool.pool
+
+
+def shutdown_extraction_pool() -> None:
+    """Shut down the shared extraction pool if it was started."""
+    with _extraction_pool.lock:
+        if _extraction_pool.pool is not None:
+            _extraction_pool.pool.shutdown(wait=False, cancel_futures=True)
+            _extraction_pool.pool = None
+            logger.info("ebook_phrase_extraction_pool_shutdown")
+
+
+async def extract_phrase_candidates_in_pool(
+    book_text: str,
+    chapters: Sequence[str],
+    config: EbookSearchConfig,
+    *,
+    metadata: Mapping[str, object] | None,
+) -> list[PhraseCandidate]:
+    """Run book phrase extraction in a worker process and await the result.
+
+    Only the CPU-bound extraction runs in the worker; the caller keeps all database work in the
+    request process. The spaCy pipeline is not supported here because it is not picklable, so
+    this always runs the non-spaCy extraction path.
+
+    Args:
+        book_text (str): Full book text used for extraction.
+        chapters (Sequence[str]): Chapter-like text blocks used for frequency counts.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        metadata (Mapping[str, object] | None): Optional book metadata used as a candidate source.
+
+    Returns:
+        list[PhraseCandidate]: Scored candidates sorted best-first and capped per book.
+    """
+    pool = get_extraction_pool(config.protected_phrase_extraction_workers)
+    future = pool.submit(
+        extract_phrase_candidates_for_book,
+        book_text,
+        list(chapters),
+        config,
+        metadata=dict(metadata) if metadata is not None else None,
+    )
+    return await asyncio.wrap_future(future)

python/ebook_search/protected_phrases/store.py

@@ -0,0 +1,700 @@
+"""Database persistence for candidate and protected phrase rows."""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import TYPE_CHECKING
+
+from sqlalchemy import delete, func, or_, select
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.dialects.sqlite import insert as sqlite_insert
+
+from python.ebook_search.protected_phrases.extraction import minimum_candidate_raw_count
+from python.ebook_search.protected_phrases.models import (
+    CorpusPhraseStats,
+    PhraseCandidate,
+    PhraseRecalculationResult,
+)
+from python.ebook_search.protected_phrases.text_normalization import normalize_text
+from python.orm.richie import (
+    EbookCandidatePhrase,
+    EbookChunk,
+    EbookChunkPhraseMention,
+    EbookPhraseAlias,
+    EbookProtectedPhrase,
+    EbookSource,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from sqlalchemy.dialects.postgresql.dml import Insert as PostgresInsert
+    from sqlalchemy.dialects.sqlite.dml import Insert as SqliteInsert
+    from sqlalchemy.ext.asyncio import AsyncSession
+
+    from python.ebook_search.config import EbookSearchConfig
+    from python.ebook_search.protected_phrases.models import LLMJudgment
+    from python.orm.richie.base import TableBase
+
+logger = logging.getLogger(__name__)
+
+
+def dialect_insert(session: AsyncSession, table: type[TableBase]) -> PostgresInsert | SqliteInsert:
+    """Return a dialect-specific INSERT construct that supports ``ON CONFLICT DO UPDATE``.
+
+    Production runs on PostgreSQL while tests run on SQLite; both support upserts with
+    compatible SQLAlchemy constructs, so the correct one is chosen from the bound dialect.
+
+    Args:
+        session (AsyncSession): Active database session whose bind selects the dialect.
+        table (type[TableBase]): Mapped table to insert into.
+
+    Returns:
+        PostgresInsert | SqliteInsert: A dialect insert exposing ``on_conflict_do_update``.
+    """
+    if session.get_bind().dialect.name == "sqlite":
+        return sqlite_insert(table)
+    return pg_insert(table)
+
+
+async def load_book_text(session: AsyncSession, book_id: int) -> str:
+    """Load a book's indexed chunk text as one string for phrase extraction.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose chunk text is loaded.
+
+    Returns:
+        str: The book's chunk text joined into a single string.
+    """
+    texts = await session.scalars(
+        select(EbookChunk.text).where(EbookChunk.source_id == book_id).order_by(EbookChunk.chunk_index)
+    )
+    return "\n\n".join(stripped for text in texts if (stripped := text.strip()))
+
+
+async def load_book_chapter_texts(session: AsyncSession, book_id: int) -> list[str]:
+    """Reconstruct chapter-like text blocks from indexed chunks for phrase extraction.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose chunks are grouped into chapters.
+
+    Returns:
+        list[str]: Non-empty chapter-like text blocks in chunk order.
+    """
+    rows = await session.execute(
+        select(EbookChunk.chapter_id, EbookChunk.text)
+        .where(EbookChunk.source_id == book_id)
+        .order_by(EbookChunk.chunk_index)
+    )
+    chapters: list[str] = []
+    current_chapter_id: int | None = None
+    current_parts: list[str] = []
+    have_current = False
+
+    for chapter_id, text in rows:
+        if have_current and chapter_id != current_chapter_id:
+            chapter_text = "\n\n".join(current_parts).strip()
+            if chapter_text:
+                chapters.append(chapter_text)
+            current_parts = []
+        current_chapter_id = chapter_id
+        current_parts.append(str(text))
+        have_current = True
+
+    if current_parts:
+        chapter_text = "\n\n".join(current_parts).strip()
+        if chapter_text:
+            chapters.append(chapter_text)
+    return chapters
+
+
+def metadata_for_source(source: EbookSource) -> dict[str, object | None]:
+    """Return phrase extraction metadata for one indexed source.
+
+    Args:
+        source (EbookSource): Indexed source to read metadata from.
+
+    Returns:
+        dict[str, object | None]: Title, author, language, publisher, and identifier values.
+    """
+    return {
+        "title": source.title,
+        "author": source.author,
+        "language": source.language,
+        "publisher": source.publisher,
+        "identifier": source.identifier,
+    }
+
+
+async def metadata_for_source_id(session: AsyncSession, source_id: int) -> dict[str, object | None]:
+    """Return phrase extraction metadata for one indexed source by id.
+
+    Args:
+        session (AsyncSession): Active database session.
+        source_id (int): Id of the indexed source to read metadata from.
+
+    Returns:
+        dict[str, object | None]: Title, author, language, publisher, and identifier values.
+
+    Raises:
+        ValueError: If no source exists with the given id.
+    """
+    source = await session.get(EbookSource, source_id)
+    if source is None:
+        msg = f"No indexed source with id {source_id}"
+        raise ValueError(msg)
+    return metadata_for_source(source)
+
+
+async def count_protected_phrases(session: AsyncSession, book_id: int) -> int:
+    """Count stored protected phrases for one book.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose protected phrases are counted.
+
+    Returns:
+        int: Number of protected phrases stored for the book.
+    """
+    return (
+        await session.scalars(
+            select(func.count(EbookProtectedPhrase.id)).where(EbookProtectedPhrase.book_id == book_id)
+        )
+    ).one()
+
+
+async def count_unjudged_candidates(session: AsyncSession, book_id: int, config: EbookSearchConfig) -> int:
+    """Count storable candidate rows for a book that have not yet been judged.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose unjudged candidates are counted.
+        config (EbookSearchConfig): Runtime phrase-tuning settings supplying storage thresholds.
+
+    Returns:
+        int: Number of storable, unjudged candidate rows for the book.
+    """
+    return (
+        await session.scalars(
+            select(func.count(EbookCandidatePhrase.id)).where(
+                EbookCandidatePhrase.book_id == book_id,
+                EbookCandidatePhrase.llm_judged.is_(False),
+                EbookCandidatePhrase.token_count >= config.phrase_min_tokens,
+                EbookCandidatePhrase.raw_count >= minimum_candidate_raw_count(config),
+            )
+        )
+    ).one()
+
+
+async def corpus_phrase_stats(session: AsyncSession) -> CorpusPhraseStats:
+    """Summarize candidate and protected phrase coverage across the whole corpus.
+
+    Args:
+        session (AsyncSession): Active database session.
+
+    Returns:
+        CorpusPhraseStats: Corpus-wide phrase counts and per-book coverage counts.
+    """
+    total_books = (await session.scalars(select(func.count(EbookSource.id)))).one()
+    candidate_phrases, judged_candidates, books_with_candidates, books_with_unjudged = (
+        await session.execute(
+            select(
+                func.count(EbookCandidatePhrase.id),
+                func.count(EbookCandidatePhrase.id).filter(EbookCandidatePhrase.llm_judged.is_(True)),
+                func.count(func.distinct(EbookCandidatePhrase.book_id)),
+                func.count(func.distinct(EbookCandidatePhrase.book_id)).filter(
+                    EbookCandidatePhrase.llm_judged.is_(False)
+                ),
+            )
+        )
+    ).one()
+    protected_phrases = (await session.scalars(select(func.count(EbookProtectedPhrase.id)))).one()
+    return CorpusPhraseStats(
+        total_books=total_books,
+        books_with_candidates=books_with_candidates,
+        books_fully_judged=books_with_candidates - books_with_unjudged,
+        candidate_phrases=candidate_phrases,
+        judged_candidates=judged_candidates,
+        unjudged_candidates=candidate_phrases - judged_candidates,
+        protected_phrases=protected_phrases,
+    )
+
+
+async def book_ids_pending_first_judgment(session: AsyncSession) -> list[int]:
+    """Return books that have candidate phrases but no judged candidates yet.
+
+    Args:
+        session (AsyncSession): Active database session.
+
+    Returns:
+        list[int]: Book ids with candidates where judging has never run, ordered by id.
+    """
+    judged_books = select(EbookCandidatePhrase.book_id).where(EbookCandidatePhrase.llm_judged.is_(True)).distinct()
+    return list(
+        (
+            await session.scalars(
+                select(EbookCandidatePhrase.book_id)
+                .where(EbookCandidatePhrase.book_id.not_in(judged_books))
+                .distinct()
+                .order_by(EbookCandidatePhrase.book_id)
+            )
+        ).all()
+    )
+
+
+async def load_candidates_for_judgment(
+    session: AsyncSession,
+    book_id: int,
+    config: EbookSearchConfig,
+) -> Sequence[EbookCandidatePhrase]:
+    """Load every storable unjudged candidate row for a book.
+
+    Rows may have been stored before the current junk filters and score weights existed, so
+    callers re-check :func:`is_junk_phrase` and rescore before selecting what to judge.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose candidates are loaded.
+        config (EbookSearchConfig): Runtime phrase-tuning settings supplying storage thresholds.
+
+    Returns:
+        Sequence[EbookCandidatePhrase]: Storable, unjudged candidate rows ordered by stored score.
+    """
+    query = (
+        select(EbookCandidatePhrase)
+        .where(
+            EbookCandidatePhrase.book_id == book_id,
+            EbookCandidatePhrase.llm_judged.is_(False),
+            EbookCandidatePhrase.token_count >= config.phrase_min_tokens,
+            EbookCandidatePhrase.raw_count >= minimum_candidate_raw_count(config),
+        )
+        .order_by(
+            EbookCandidatePhrase.candidate_score.desc(),
+            EbookCandidatePhrase.raw_count.desc(),
+            EbookCandidatePhrase.id,
+        )
+    )
+    return (await session.scalars(query)).all()
+
+
+def phrase_candidate_from_row(row: EbookCandidatePhrase) -> PhraseCandidate:
+    """Recreate an in-memory candidate from a persisted candidate row.
+
+    Args:
+        row (EbookCandidatePhrase): Stored candidate row to convert.
+
+    Returns:
+        PhraseCandidate: An in-memory candidate mirroring the row's fields.
+    """
+    return PhraseCandidate(
+        phrase_text=row.phrase_text,
+        phrase_norm=row.phrase_norm,
+        token_count=row.token_count,
+        source_raw_ngram=row.source_raw_ngram,
+        source_yake=row.source_yake,
+        source_spacy_ner=row.source_spacy_ner,
+        source_spacy_noun_chunk=row.source_spacy_noun_chunk,
+        source_capitalized=row.source_capitalized,
+        source_metadata=row.source_metadata,
+        spacy_label=row.spacy_label,
+        raw_count=row.raw_count,
+        chapter_count=row.chapter_count,
+        yake_score=row.yake_score,
+        candidate_score=row.candidate_score,
+        sample_contexts=row.sample_contexts or [],
+    )
+
+
+def candidate_row_values(
+    book_id: int,
+    series_id: int | None,
+    candidate: PhraseCandidate,
+    *,
+    judgment: LLMJudgment | None,
+) -> dict[str, object]:
+    """Build the column values for one candidate phrase upsert.
+
+    Args:
+        book_id (int): Book the candidate belongs to.
+        series_id (int | None): Series scope stored on the row.
+        candidate (PhraseCandidate): Candidate whose fields are written to the row.
+        judgment (LLMJudgment | None): Judgment to record, or ``None`` to leave the row unjudged.
+
+    Returns:
+        dict[str, object]: Column values keyed by column name.
+    """
+    values: dict[str, object] = {
+        "book_id": book_id,
+        "phrase_norm": candidate.phrase_norm,
+        "series_id": series_id,
+        "phrase_text": candidate.phrase_text,
+        "token_count": candidate.token_count,
+        "source_raw_ngram": candidate.source_raw_ngram,
+        "source_yake": candidate.source_yake,
+        "source_spacy_ner": candidate.source_spacy_ner,
+        "source_spacy_noun_chunk": candidate.source_spacy_noun_chunk,
+        "source_capitalized": candidate.source_capitalized,
+        "source_metadata": candidate.source_metadata,
+        "spacy_label": candidate.spacy_label,
+        "raw_count": candidate.raw_count,
+        "chapter_count": candidate.chapter_count,
+        "yake_score": candidate.yake_score,
+        "candidate_score": candidate.candidate_score,
+        "llm_judged": judgment is not None,
+    }
+    if candidate.sample_contexts:
+        values["sample_contexts"] = list(candidate.sample_contexts)
+    if judgment is not None:
+        values.update(
+            llm_keep=judgment.keep,
+            llm_confidence=judgment.confidence,
+            llm_category=judgment.category,
+            llm_reason=judgment.reason,
+        )
+    return values
+
+
+async def save_candidate_to_db(
+    session: AsyncSession,
+    book_id: int,
+    series_id: int | None,
+    candidate: PhraseCandidate,
+    *,
+    judgment: LLMJudgment | None,
+) -> EbookCandidatePhrase:
+    """Insert or update one candidate phrase row.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book the candidate belongs to.
+        series_id (int | None): Series scope stored on the row.
+        candidate (PhraseCandidate): Candidate whose fields are written to the row.
+        judgment (LLMJudgment | None): Judgment to record, or ``None`` to leave the row unjudged.
+
+    Returns:
+        EbookCandidatePhrase: The inserted or updated candidate row.
+    """
+    values = candidate_row_values(book_id, series_id, candidate, judgment=judgment)
+
+    # Preserve an existing judgment when this call is only refreshing candidate fields.
+    skip_update = {"book_id", "phrase_norm"}
+    if judgment is None:
+        skip_update.add("llm_judged")
+    insert_statement = dialect_insert(session, EbookCandidatePhrase).values(**values)
+    statement = insert_statement.on_conflict_do_update(
+        index_elements=["book_id", "phrase_norm"],
+        set_={column: insert_statement.excluded[column] for column in values if column not in skip_update},
+    ).returning(EbookCandidatePhrase)
+    return (await session.scalars(statement, execution_options={"populate_existing": True})).one()
+
+
+BULK_CANDIDATE_UPSERT_CHUNK = 1000
+
+
+async def bulk_upsert_unjudged_candidates(
+    session: AsyncSession,
+    book_id: int,
+    series_id: int | None,
+    candidates: Sequence[PhraseCandidate],
+) -> int:
+    """Insert or update many freshly extracted candidate rows in chunked multi-row upserts.
+
+    Saving one row per statement costs one database round trip per candidate, which dominated
+    generation time for full books, so candidates are written ``BULK_CANDIDATE_UPSERT_CHUNK``
+    rows per statement instead. Existing judgments and sample contexts are never overwritten:
+    fresh extractions carry no contexts, and ``llm_judged`` plus the ``llm_*`` columns are left
+    out of the conflict update. Candidates must have unique ``phrase_norm`` values, as produced
+    by extraction, since one multi-row upsert cannot touch the same row twice.
+
+    Args:
+        session (Session): Active database session.
+        book_id (int): Book the candidates belong to.
+        series_id (int | None): Series scope stored on the rows.
+        candidates (Sequence[PhraseCandidate]): Freshly extracted candidates to persist.
+
+    Returns:
+        int: Number of candidate rows written.
+    """
+    values = [
+        candidate_row_values(book_id, series_id, candidate, judgment=None)
+        for candidate in candidates
+        if not candidate.sample_contexts
+    ]
+    if len(values) != len(candidates):
+        msg = "bulk_upsert_unjudged_candidates only accepts freshly extracted candidates without sample contexts"
+        raise ValueError(msg)
+    skip_update = {"book_id", "phrase_norm", "llm_judged"}
+    for chunk_start in range(0, len(values), BULK_CANDIDATE_UPSERT_CHUNK):
+        chunk = values[chunk_start : chunk_start + BULK_CANDIDATE_UPSERT_CHUNK]
+        insert_statement = dialect_insert(session, EbookCandidatePhrase).values(chunk)
+        statement = insert_statement.on_conflict_do_update(
+            index_elements=["book_id", "phrase_norm"],
+            set_={column: insert_statement.excluded[column] for column in chunk[0] if column not in skip_update},
+        )
+        await session.execute(statement)
+    return len(values)
+
+
+def new_candidate_row(book_id: int, series_id: int | None, candidate: PhraseCandidate) -> EbookCandidatePhrase:
+    """Build a fresh unjudged candidate row without checking for an existing one.
+
+    Unlike :func:`save_candidate_to_db`, this does no lookup, so it is only safe when the caller
+    guarantees there is no existing row for ``(book_id, candidate.phrase_norm)`` — for example
+    right after :func:`delete_phrase_data_for_book` has cleared the book.
+
+    Args:
+        book_id (int): Book the candidate belongs to.
+        series_id (int | None): Series scope stored on the row.
+        candidate (PhraseCandidate): Candidate whose fields are written to the row.
+
+    Returns:
+        EbookCandidatePhrase: A new, unattached candidate row.
+    """
+    row = EbookCandidatePhrase(book_id=book_id, phrase_norm=candidate.phrase_norm)
+    row.llm_judged = False
+    row.series_id = series_id
+    row.phrase_text = candidate.phrase_text
+    row.token_count = candidate.token_count
+    row.source_raw_ngram = candidate.source_raw_ngram
+    row.source_yake = candidate.source_yake
+    row.source_spacy_ner = candidate.source_spacy_ner
+    row.source_spacy_noun_chunk = candidate.source_spacy_noun_chunk
+    row.source_capitalized = candidate.source_capitalized
+    row.source_metadata = candidate.source_metadata
+    row.spacy_label = candidate.spacy_label
+    row.raw_count = candidate.raw_count
+    row.chapter_count = candidate.chapter_count
+    row.yake_score = candidate.yake_score
+    row.candidate_score = candidate.candidate_score
+    if candidate.sample_contexts:
+        row.sample_contexts = list(candidate.sample_contexts)
+    return row
+
+
+async def upsert_protected_phrase(
+    session: AsyncSession,
+    book_id: int,
+    series_id: int | None,
+    candidate: PhraseCandidate,
+    judgment: LLMJudgment,
+    source_candidate: EbookCandidatePhrase,
+) -> EbookProtectedPhrase:
+    """Insert or update one accepted protected phrase and its aliases.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book the protected phrase belongs to.
+        series_id (int | None): Series scope stored on the phrase.
+        candidate (PhraseCandidate): Candidate the phrase was promoted from.
+        judgment (LLMJudgment): Accepted judgment supplying canonical text, category, and aliases.
+        source_candidate (EbookCandidatePhrase): Candidate row the phrase was promoted from.
+
+    Returns:
+        EbookProtectedPhrase: The inserted or updated protected phrase row.
+
+    Raises:
+        ValueError: If the chosen phrase text normalizes to empty.
+    """
+    phrase_text = judgment.canonical or candidate.phrase_text
+    phrase_norm = normalize_text(phrase_text)
+    if not phrase_norm:
+        msg = f"Protected phrase normalized to empty text: {phrase_text!r}"
+        raise ValueError(msg)
+
+    values = {
+        "book_id": book_id,
+        "phrase_norm": phrase_norm,
+        "series_id": series_id,
+        "phrase_text": phrase_text,
+        "canonical_id": make_canonical_id(judgment, phrase_norm),
+        "phrase_type": judgment.category,
+        "token_count": len(phrase_norm.split()),
+        "confidence": judgment.confidence,
+        "importance": judgment.importance,
+        "allow_nested": judgment.allow_nested,
+        "suppress_children": judgment.suppress_children,
+        "source_candidate_id": source_candidate.id,
+    }
+    insert_statement = dialect_insert(session, EbookProtectedPhrase).values(**values)
+    statement = insert_statement.on_conflict_do_update(
+        index_elements=["book_id", "phrase_norm"],
+        set_={
+            column: insert_statement.excluded[column] for column in values if column not in {"book_id", "phrase_norm"}
+        },
+    ).returning(EbookProtectedPhrase)
+    row = (await session.scalars(statement, execution_options={"populate_existing": True})).one()
+
+    for alias_text in judgment.aliases:
+        await upsert_phrase_alias(session, row, alias_text)
+    return row
+
+
+async def upsert_phrase_alias(
+    session: AsyncSession,
+    phrase: EbookProtectedPhrase,
+    alias_text: str,
+) -> EbookPhraseAlias | None:
+    """Insert or update one protected phrase alias.
+
+    Args:
+        session (AsyncSession): Active database session.
+        phrase (EbookProtectedPhrase): Protected phrase the alias points to.
+        alias_text (str): Alias surface form to store.
+
+    Returns:
+        EbookPhraseAlias | None: The alias row, or ``None`` when the alias is empty or equals the phrase.
+    """
+    alias_norm = normalize_text(alias_text)
+    if not alias_norm or alias_norm == phrase.phrase_norm:
+        return None
+
+    insert_statement = dialect_insert(session, EbookPhraseAlias).values(
+        phrase_id=phrase.id,
+        alias_norm=alias_norm,
+        alias_text=alias_text,
+        confidence=1.0,
+    )
+    statement = insert_statement.on_conflict_do_update(
+        index_elements=["phrase_id", "alias_norm"],
+        set_={
+            "alias_text": insert_statement.excluded.alias_text,
+            "confidence": insert_statement.excluded.confidence,
+        },
+    ).returning(EbookPhraseAlias)
+    return (await session.scalars(statement, execution_options={"populate_existing": True})).one()
+
+
+def make_canonical_id(judgment: LLMJudgment, phrase_norm: str) -> str:
+    """Create a deterministic canonical id from a judgment category and phrase.
+
+    Args:
+        judgment (LLMJudgment): Judgment supplying the phrase category.
+        phrase_norm (str): Normalized phrase text to slugify.
+
+    Returns:
+        str: A ``category:slug`` canonical identifier.
+    """
+    category = slugify_identifier(judgment.category or "phrase")
+    phrase_slug = slugify_identifier(phrase_norm)
+    return f"{category}:{phrase_slug}"
+
+
+def slugify_identifier(value: str) -> str:
+    """Normalize text for use inside a canonical id.
+
+    Args:
+        value (str): Text to slugify.
+
+    Returns:
+        str: A lowercase underscore slug, or ``"unknown"`` when empty.
+    """
+    slug = re.sub(r"[^a-z0-9]+", "_", normalize_text(value).replace("'", ""))
+    return slug.strip("_") or "unknown"
+
+
+async def prune_unstorable_unjudged_candidate_phrases(
+    session: AsyncSession,
+    book_id: int,
+    config: EbookSearchConfig,
+) -> int:
+    """Delete old unjudged candidate rows that no longer satisfy storage filters.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose stale candidates are pruned.
+        config (EbookSearchConfig): Runtime phrase-tuning settings supplying storage thresholds.
+
+    Returns:
+        int: Number of candidate rows deleted.
+    """
+    deleted = rowcount(
+        await session.execute(
+            delete(EbookCandidatePhrase).where(
+                EbookCandidatePhrase.book_id == book_id,
+                EbookCandidatePhrase.llm_judged.is_(False),
+                or_(
+                    EbookCandidatePhrase.token_count < config.phrase_min_tokens,
+                    EbookCandidatePhrase.raw_count < minimum_candidate_raw_count(config),
+                ),
+            )
+        )
+    )
+    if deleted:
+        logger.info(
+            "ebook_candidate_phrase_unstorable_pruned book_id=%s deleted=%s min_tokens=%s min_uses=%s",
+            book_id,
+            deleted,
+            config.phrase_min_tokens,
+            minimum_candidate_raw_count(config),
+        )
+    return deleted
+
+
+async def delete_phrase_data_for_book(session: AsyncSession, book_id: int) -> PhraseRecalculationResult:
+    """Delete all candidate, protected, alias, and mention phrase data for one book.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose phrase data is deleted.
+
+    Returns:
+        PhraseRecalculationResult: Deleted-row counts with ``candidate_phrases`` set to 0.
+    """
+    protected_ids = (
+        await session.scalars(select(EbookProtectedPhrase.id).where(EbookProtectedPhrase.book_id == book_id))
+    ).all()
+    deleted_aliases = 0
+    if protected_ids:
+        deleted_aliases = rowcount(
+            await session.execute(delete(EbookPhraseAlias).where(EbookPhraseAlias.phrase_id.in_(protected_ids)))
+        )
+
+    deleted_mentions = rowcount(
+        await session.execute(delete(EbookChunkPhraseMention).where(EbookChunkPhraseMention.book_id == book_id))
+    )
+    if protected_ids:
+        deleted_mentions += rowcount(
+            await session.execute(
+                delete(EbookChunkPhraseMention).where(EbookChunkPhraseMention.phrase_id.in_(protected_ids))
+            )
+        )
+
+    deleted_protected = rowcount(
+        await session.execute(delete(EbookProtectedPhrase).where(EbookProtectedPhrase.book_id == book_id))
+    )
+    deleted_candidates = rowcount(
+        await session.execute(delete(EbookCandidatePhrase).where(EbookCandidatePhrase.book_id == book_id))
+    )
+    await session.flush()
+    logger.info(
+        "ebook_candidate_phrase_data_deleted book_id=%s candidates=%s protected=%s aliases=%s mentions=%s",
+        book_id,
+        deleted_candidates,
+        deleted_protected,
+        deleted_aliases,
+        deleted_mentions,
+    )
+    return PhraseRecalculationResult(
+        book_id=book_id,
+        deleted_candidates=deleted_candidates,
+        deleted_protected_phrases=deleted_protected,
+        deleted_aliases=deleted_aliases,
+        deleted_mentions=deleted_mentions,
+        candidate_phrases=0,
+    )
+
+
+def rowcount(result: object) -> int:
+    """Return a safe integer rowcount from a SQLAlchemy execution result.
+
+    Args:
+        result (object): SQLAlchemy execution result that may expose ``rowcount``.
+
+    Returns:
+        int: The result's rowcount, or 0 when it is missing or negative.
+    """
+    count = getattr(result, "rowcount", 0)
+    return int(count if count is not None and count >= 0 else 0)

python/ebook_search/protected_phrases/text_normalization.py

@@ -0,0 +1,91 @@
+"""Protected phrase extraction, storage, and runtime matching."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+JSON_OBJECT_RE = re.compile(r"\{.*\}", re.DOTALL)
+
+
+@dataclass(frozen=True, slots=True)
+class NormalizedToken:
+    """A normalized token plus its source character span."""
+
+    text: str
+    start_char: int
+    end_char: int
+
+
+def normalize_text(text: str) -> str:
+    """Normalize text for phrase storage and lookup.
+
+    Args:
+        text (str): Raw text to normalize.
+
+    Returns:
+        str: Normalized tokens joined by single spaces.
+    """
+    return " ".join(token.text for token in tokenize_with_offsets(text))
+
+
+def tokenize(text: str) -> list[str]:
+    """Normalize and split text into phrase-detection tokens.
+
+    Args:
+        text (str): Raw text to tokenize.
+
+    Returns:
+        list[str]: Normalized token strings.
+    """
+    return [token.text for token in tokenize_with_offsets(text)]
+
+
+def tokenize_with_offsets(text: str) -> list[NormalizedToken]:
+    """Normalize text into tokens while preserving original character offsets.
+
+    Args:
+        text (str): Raw text to tokenize.
+
+    Returns:
+        list[NormalizedToken]: Normalized tokens with their source character spans.
+    """
+    tokens: list[NormalizedToken] = []
+    current: list[str] = []
+    start_char: int | None = None
+
+    for index, char in enumerate(text):
+        normalized = normalize_char(char)
+        if normalized == " ":
+            if current and start_char is not None:
+                tokens.append(NormalizedToken(text="".join(current), start_char=start_char, end_char=index))
+                current = []
+                start_char = None
+            continue
+        if start_char is None:
+            start_char = index
+        current.append(normalized)
+
+    if current and start_char is not None:
+        tokens.append(NormalizedToken(text="".join(current), start_char=start_char, end_char=len(text)))
+    return tokens
+
+
+def normalize_char(char: str) -> str:
+    """Normalize one character into a token character or a separator.
+
+    Args:
+        char (str): Single source character to normalize.
+
+    Returns:
+        str: The normalized token character, or a space acting as a separator.
+    """
+    if char in {"\u2019", "\u2018"}:
+        return "'"
+    if char in {"-", "\u2013", "\u2014"}:
+        return " "
+
+    lowered = char.lower()
+    if lowered in "abcdefghijklmnopqrstuvwxyz0123456789'":
+        return lowered
+    return " "

python/ebook_search/rerank.py

@@ -9,6 +9,8 @@ from typing import TYPE_CHECKING
 from python.ebook_search.llm_interface import request_rerank
 
 if TYPE_CHECKING:
+    import httpx
+
     from python.ebook_search.config import RerankConfig
     from python.ebook_search.search import SearchResult
 
@@ -23,7 +25,12 @@ class RerankResult:
     score: float
 
 
-def rerank_chunks(query: str, candidates: list[SearchResult], config: RerankConfig) -> list[SearchResult]:
+async def rerank_chunks(
+    client: httpx.AsyncClient,
+    query: str,
+    candidates: list[SearchResult],
+    config: RerankConfig,
+) -> list[SearchResult]:
     """Rerank candidates with a vLLM rerank endpoint."""
     if not candidates:
         return []
@@ -34,7 +41,7 @@ def rerank_chunks(query: str, candidates: list[SearchResult], config: RerankConf
         config.model,
         len(candidates),
     )
-    scores = score_candidates(query, candidates, config)
+    scores = await score_candidates(client, query, candidates, config)
     results = sorted(
         (
             replace(
@@ -56,13 +63,14 @@ def rerank_chunks(query: str, candidates: list[SearchResult], config: RerankConf
     return results
 
 
-def score_candidates(
+async def score_candidates(
+    client: httpx.AsyncClient,
     query: str,
     candidates: list[SearchResult],
     config: RerankConfig,
 ) -> dict[int, RerankResult]:
     """Score candidate chunks with the configured rerank API."""
-    body = request_rerank(query, [candidate.text for candidate in candidates], config)
+    body = await request_rerank(client, query, [candidate.text for candidate in candidates], config)
     if body is None:
         return zero_rerank_scores(candidates)
 

python/ebook_search/search.py

@@ -2,16 +2,17 @@
 
 from __future__ import annotations
 
+import asyncio
 import logging
 import re
 from collections import defaultdict
-from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, replace
 from typing import TYPE_CHECKING
 
 from pgvector.sqlalchemy import Vector
 from sqlalchemy import literal, select
-from sqlalchemy.orm import Session
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.ext.asyncio import AsyncSession
 
 from python.ebook_search.bm25_corpus import (
     BM25CorpusUnavailableError,
@@ -19,8 +20,12 @@ from python.ebook_search.bm25_corpus import (
     score_bm25_corpus,
 )
 from python.ebook_search.embeddings import MODEL_DIMENSIONS, embed_query, get_embedding_table
+from python.ebook_search.protected_phrases.matching import (
+    detect_protected_phrases_for_query,
+    phrase_hits_for_chunks,
+)
 from python.ebook_search.rerank import rerank_chunks
-from python.ebook_search.timing import RuntimeStep, timed_result
+from python.ebook_search.timing import RuntimeStep, async_timed_result, timed_result
 from python.orm.richie import (
     EbookChapter,
     EbookChunk,
@@ -29,11 +34,13 @@ from python.orm.richie import (
 )
 
 if TYPE_CHECKING:
-    from collections.abc import Mapping
+    from collections.abc import Mapping, Sequence
 
-    from sqlalchemy.engine import Engine
+    import httpx
+    from sqlalchemy.ext.asyncio import AsyncEngine
 
     from python.ebook_search.config import EbookSearchConfig
+    from python.ebook_search.protected_phrases.models import HydratedPhraseMatch
 
 logger = logging.getLogger(__name__)
 
@@ -45,11 +52,14 @@ class SearchResult:
     chunk_id: int
     text: str
     source_title: str
+    source_id: int | None = None
     score: float = 0.0
     vector_score: float | None = None
     bm25_score: float | None = None
     fused_score: float | None = None
     rerank_score: float | None = None
+    phrase_hit_count: int = 0
+    matched_phrases: tuple[str, ...] = ()
     source_author: str | None = None
     chapter_title: str | None = None
     page_label: str | None = None
@@ -64,6 +74,7 @@ class SearchResponse:
     results: list[SearchResult]
     rank_label: str
     timings: tuple[RuntimeStep, ...] = ()
+    phrase_matches: tuple[HydratedPhraseMatch, ...] = ()
 
     @property
     def total_runtime_ms(self) -> float:
@@ -80,26 +91,38 @@ class RetrievalResponse:
     timings: tuple[RuntimeStep, ...]
 
 
-def search_ebooks(
+async def search_ebooks(
-    engine: Engine,
+    engine: AsyncEngine,
+    client: httpx.AsyncClient,
     query: str,
     config: EbookSearchConfig,
     *,
     rerank: bool = False,
+    phrase_matching: bool | None = None,
 ) -> SearchResponse:
     """Run hybrid vector/BM25 search and optional reranking."""
     if not query.strip():
         logger.info("ebook_search_empty_query")
         return SearchResponse(query=query, results=[], rank_label="Hybrid")
 
-    logger.info("ebook_search_start query_length=%s rerank=%s", len(query), rerank)
+    phrase_matching_enabled = config.phrase_matching_enabled if phrase_matching is None else phrase_matching
+    logger.info(
+        "ebook_search_start query_length=%s rerank=%s phrase_matching=%s",
+        len(query),
+        rerank,
+        phrase_matching_enabled,
+    )
     timings: list[RuntimeStep] = []
-    retrieval, timing = timed_result(
+    if phrase_matching_enabled:
+        phrase_matches, timing = await async_timed_result(
+            "Protected phrase detection", query_phrase_matches(engine, query, config)
+        )
+    else:
+        phrase_matches, timing = timed_result("Protected phrase detection skipped", skip_phrase_matches)
+    timings.append(timing)
+    retrieval, timing = await async_timed_result(
         "Hybrid retrieval",
-        parallel_retrieval,
+        parallel_retrieval(engine, client, query, config),
-        engine,
-        query,
-        config,
     )
     timings.extend(retrieval.timings)
     timings.append(timing)
@@ -111,18 +134,28 @@ def search_ebooks(
         rank_constant=config.rrf_rank_constant,
     )
     timings.append(timing)
+    if phrase_matching_enabled:
+        fused, timing = await async_timed_result(
+            "Phrase mention boost",
+            apply_phrase_mention_boosts(engine, fused, phrase_matches, config.phrase_hit_boost),
+        )
+    else:
+        fused, timing = timed_result("Phrase mention boost skipped", skip_phrase_mention_boosts, fused)
+    timings.append(timing)
     if config.rerank.enabled and rerank:
-        response, timing = timed_result("Rerank", apply_rerank, query, fused, config)
+        response, timing = await async_timed_result("Rerank", apply_rerank(client, query, fused, config))
     else:
         response, timing = timed_result("Rerank skipped", skip_rerank, query, fused, config)
     timings.append(timing)
-    response = replace(response, timings=tuple(timings))
+    response = replace(response, timings=tuple(timings), phrase_matches=tuple(phrase_matches))
     logger.info(
         "ebook_search_complete vector_candidates=%s lexical_candidates=%s "
-        "fused_candidates=%s returned=%s rank_label=%s runtime_ms=%.1f",
+        "fused_candidates=%s phrase_matching=%s phrase_matches=%s returned=%s rank_label=%s runtime_ms=%.1f",
         len(retrieval.vector_results),
         len(retrieval.lexical_results),
         len(fused),
+        phrase_matching_enabled,
+        len(phrase_matches),
         len(response.results),
         response.rank_label,
         response.total_runtime_ms,
@@ -130,30 +163,100 @@ def search_ebooks(
     return response
 
 
-def parallel_retrieval(
+def skip_phrase_matches() -> list[HydratedPhraseMatch]:
-    engine: Engine,
+    """Return no protected phrase matches when phrase matching is disabled."""
+    logger.info("ebook_protected_phrase_detection_skipped")
+    return []
+
+
+async def query_phrase_matches(
+    engine: AsyncEngine,
+    query: str,
+    config: EbookSearchConfig,
+) -> list[HydratedPhraseMatch]:
+    """Detect protected phrases in a query without making search fail when phrase tables are unavailable."""
+    try:
+        async with AsyncSession(engine) as session:
+            return await detect_protected_phrases_for_query(session, query, config)
+    except SQLAlchemyError as error:
+        logger.warning("ebook_protected_phrase_detection_unavailable error=%s", error)
+        return []
+
+
+def skip_phrase_mention_boosts(candidates: list[SearchResult]) -> list[SearchResult]:
+    """Return candidates unchanged when phrase matching is disabled."""
+    logger.info("ebook_phrase_boost_skipped candidates=%s", len(candidates))
+    return candidates
+
+
+async def apply_phrase_mention_boosts(
+    engine: AsyncEngine,
+    candidates: list[SearchResult],
+    phrase_matches: Sequence[HydratedPhraseMatch],
+    phrase_hit_boost: float,
+) -> list[SearchResult]:
+    """Boost retrieved chunks that have indexed mentions for detected protected phrases."""
+    phrase_ids = sorted({match.phrase_id for match in phrase_matches})
+    if not candidates or not phrase_ids or phrase_hit_boost <= 0:
+        return candidates
+
+    chunk_ids = [candidate.chunk_id for candidate in candidates]
+    try:
+        async with AsyncSession(engine) as session:
+            phrase_hits = await phrase_hits_for_chunks(session, chunk_ids=chunk_ids, phrase_ids=phrase_ids)
+    except SQLAlchemyError as error:
+        logger.warning("ebook_phrase_boost_unavailable error=%s", error)
+        return candidates
+
+    if not phrase_hits:
+        return candidates
+
+    hit_counts = {
+        chunk_id: sum(hit.mention_count for hit in chunk_hits) for chunk_id, chunk_hits in phrase_hits.items()
+    }
+    boosted = [
+        replace(
+            candidate,
+            score=candidate.score + (hit_counts.get(candidate.chunk_id, 0) * phrase_hit_boost),
+            fused_score=boosted_fused_score(candidate, hit_counts.get(candidate.chunk_id, 0), phrase_hit_boost),
+            phrase_hit_count=hit_counts.get(candidate.chunk_id, 0),
+            matched_phrases=tuple(hit.phrase_text for hit in phrase_hits.get(candidate.chunk_id, ())),
+            rank_source=phrase_rank_source(candidate.rank_source, hit_counts.get(candidate.chunk_id, 0)),
+        )
+        for candidate in candidates
+    ]
+    return sorted(boosted, key=lambda candidate: candidate.score, reverse=True)
+
+
+def boosted_fused_score(candidate: SearchResult, phrase_hit_count: int, phrase_hit_boost: float) -> float | None:
+    """Return a fused score adjusted by phrase hits when a fused score exists."""
+    if candidate.fused_score is None:
+        return None
+    return candidate.fused_score + (phrase_hit_count * phrase_hit_boost)
+
+
+def phrase_rank_source(rank_source: str, phrase_hit_count: int) -> str:
+    """Append phrase evidence to a rank-source label when a chunk was boosted."""
+    if phrase_hit_count <= 0 or "phrases" in rank_source:
+        return rank_source
+    return f"{rank_source} + phrases"
+
+
+async def parallel_retrieval(
+    engine: AsyncEngine,
+    client: httpx.AsyncClient,
     query: str,
     config: EbookSearchConfig,
 ) -> RetrievalResponse:
-    """Run vector and BM25 candidate retrieval concurrently with separate database sessions."""
+    """Run vector and BM25 candidate retrieval concurrently with separate database sessions.
-    with ThreadPoolExecutor(max_workers=2, thread_name_prefix="ebook-search") as executor:
+
-        vector_future = executor.submit(
+    BM25 scoring is pure CPU work over the cached corpus, so it runs in a worker thread
-            timed_result,
+    instead of on the event loop.
-            "Embedding + vector search",
+    """
-            vector_candidates,
+    (vector_results, vector_timing), (lexical_results, lexical_timing) = await asyncio.gather(
-            engine,
+        async_timed_result("Embedding + vector search", vector_candidates(engine, client, query, config)),
-            query,
+        async_timed_result("BM25 search", asyncio.to_thread(bm25_candidates, query, config)),
-            config,
+    )
-        )
-        bm25_future = executor.submit(
-            timed_result,
-            "BM25 search",
-            bm25_candidates,
-            query,
-            config,
-        )
-        vector_results, vector_timing = vector_future.result()
-        lexical_results, lexical_timing = bm25_future.result()
 
     logger.info(
         "ebook_parallel_retrieval_complete vector_candidates=%s lexical_candidates=%s",
@@ -180,13 +283,14 @@ def skip_rerank(
     return SearchResponse(query=query, results=candidates[: config.top_k], rank_label="Hybrid")
 
 
-def apply_rerank(
+async def apply_rerank(
+    client: httpx.AsyncClient,
     query: str,
     candidates: list[SearchResult],
     config: EbookSearchConfig,
 ) -> SearchResponse:
     """Rerank already-fused hybrid candidates."""
-    reranked = rerank_chunks(query, candidates[: config.rerank.candidates], config.rerank)
+    reranked = await rerank_chunks(client, query, candidates[: config.rerank.candidates], config.rerank)
     logger.info(
         "ebook_rerank_complete input_candidates=%s returned=%s",
         min(len(candidates), config.rerank.candidates),
@@ -199,10 +303,17 @@ def apply_rerank(
     )
 
 
-def vector_candidates(engine: Engine, query: str, config: EbookSearchConfig) -> list[SearchResult]:
+async def vector_candidates(
+    engine: AsyncEngine,
+    client: httpx.AsyncClient,
+    query: str,
+    config: EbookSearchConfig,
+) -> list[SearchResult]:
     """Return pgvector cosine candidates for a natural-language query."""
-    with Session(engine) as session:
+    async with AsyncSession(engine) as session:
-        model = session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == config.embedding_model))
+        model = await session.scalar(
+            select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == config.embedding_model)
+        )
         if model is None:
             msg = f"Embedding model is not registered: {config.embedding_model}"
             raise ValueError(msg)
@@ -212,7 +323,7 @@ def vector_candidates(engine: Engine, query: str, config: EbookSearchConfig) ->
             msg = f"Model row dimension {model.dimension} does not match configured dimension {expected_dimension}"
             raise ValueError(msg)
 
-        embedding = embed_query(query, config)
+        embedding = await embed_query(client, query, config)
         limit = max(config.rerank.candidates, config.top_k) * config.vector_candidate_multiplier
         embedding_table = get_embedding_table(model.dimension)
 
@@ -223,6 +334,7 @@ def vector_candidates(engine: Engine, query: str, config: EbookSearchConfig) ->
             select(
                 EbookChunk.id.label("chunk_id"),
                 EbookChunk.text.label("text"),
+                EbookSource.id.label("source_id"),
                 EbookSource.title.label("source_title"),
                 EbookSource.author.label("source_author"),
                 EbookChapter.title.label("chapter_title"),
@@ -237,7 +349,7 @@ def vector_candidates(engine: Engine, query: str, config: EbookSearchConfig) ->
             .order_by(distance)
             .limit(limit)
         )
-        rows = session.execute(statement).mappings()
+        rows = (await session.execute(statement)).mappings()
         results = [search_result_from_row(row) for row in rows]
         logger.info(
             "ebook_vector_search_complete model=%s dimension=%s candidates=%s",
@@ -317,9 +429,11 @@ def reciprocal_rank_fusion(
 
 def search_result_from_row(row: Mapping[str, object]) -> SearchResult:
     """Convert a database row mapping into a search result."""
+    source_id = row.get("source_id")
     return SearchResult(
         chunk_id=int(row["chunk_id"]),
         text=str(row["text"]),
+        source_id=int(source_id) if source_id is not None else None,
         source_title=str(row["source_title"]),
         source_author=optional_str(row["source_author"]),
         chapter_title=optional_str(row["chapter_title"]),

python/ebook_search/timing.py

@@ -7,7 +7,7 @@ from time import perf_counter
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from collections.abc import Callable
+    from collections.abc import Awaitable, Callable
 
 
 @dataclass(frozen=True)
@@ -34,3 +34,10 @@ def timed_result[T, **P](
     start_seconds = perf_counter()
     result = operation(*args, **kwargs)
     return result, runtime_step_from_start(name, start_seconds)
+
+
+async def async_timed_result[T](name: str, awaitable: Awaitable[T]) -> tuple[T, RuntimeStep]:
+    """Await an operation and return its result plus elapsed runtime."""
+    start_seconds = perf_counter()
+    result = await awaitable
+    return result, runtime_step_from_start(name, start_seconds)

python/fastapi_tools/__init__.py

@@ -1,6 +1,6 @@
 """Reusable FastAPI tools."""
 
-from python.fastapi_tools.db import DbSession, get_db
+from python.fastapi_tools.db import AsyncDbSession, DbSession, get_async_db, get_db
 from python.fastapi_tools.zstd_middleware import ZstdMiddleware
 
-__all__ = ["DbSession", "ZstdMiddleware", "get_db"]
+__all__ = ["AsyncDbSession", "DbSession", "ZstdMiddleware", "get_async_db", "get_db"]

python/fastapi_tools/db.py

@@ -5,10 +5,11 @@ from __future__ import annotations
 from typing import TYPE_CHECKING, Annotated
 
 from fastapi import Depends, Request
+from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import Session
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator
+    from collections.abc import AsyncIterator, Iterator
 
 
 def get_db(request: Request) -> Iterator[Session]:
@@ -17,4 +18,15 @@ def get_db(request: Request) -> Iterator[Session]:
         yield session
 
 
+async def get_async_db(request: Request) -> AsyncIterator[AsyncSession]:
+    """Get an async database session from app state.
+
+    expire_on_commit=False keeps ORM attributes readable after commit without
+    triggering implicit IO, which would raise under asyncio.
+    """
+    async with AsyncSession(request.app.state.engine, expire_on_commit=False) as session:
+        yield session
+
+
 DbSession = Annotated[Session, Depends(get_db)]
+AsyncDbSession = Annotated[AsyncSession, Depends(get_async_db)]

python/installer/__main__.py

@@ -5,41 +5,60 @@ from __future__ import annotations
 import curses
 import logging
 import sys
-from os import getenv
+from argparse import ArgumentParser
+from os import environ, getenv
 from pathlib import Path
 from random import getrandbits
-from subprocess import PIPE, Popen, run
+from subprocess import run
 from time import sleep
 from typing import TYPE_CHECKING
 
-from python.common import configure_logger
 from python.installer.tui import draw_menu
+from python.logging_config import configure_logger
+from python.process import require_commands, run_output
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
 
 logger = logging.getLogger(__name__)
 
+REQUIRED_COMMANDS = (
+    "blkdiscard",
+    "cryptsetup",
+    "find",
+    "lsblk",
+    "mkfs.vfat",
+    "mount",
+    "nixos-generate-config",
+    "nixos-install",
+    "parted",
+    "readlink",
+    "zfs",
+    "zpool",
+)
 
-def bash_wrapper(command: str) -> str:
-    """Execute a bash command and capture the output.
 
-    Args:
+def configure_terminal() -> None:
-        command (str): The bash command to be executed.
+    """Force a terminal type every live system has a terminfo entry for.
 
-    Returns:
+    Terminals such as kitty advertise TERM values (xterm-kitty) that live
-        Tuple[str, int]: A tuple containing the output of the command (stdout) as a string,
+    systems have no terminfo entry for, so setupterm fails before the TUI
-        the error output (stderr) as a string (optional), and the return code as an integer.
+    can start.
     """
-    logger.debug(f"running {command=}")
+    environ["TERM"] = "xterm-256color"
-    # This is a acceptable risk
-    process = Popen(command.split(), stdout=PIPE, stderr=PIPE)
-    output, _ = process.communicate()
-    if process.returncode != 0:
-        error = f"Failed to run command {command=} return code {process.returncode=}"
-        raise RuntimeError(error)
 
-    return output.decode()
+    if getenv("TERMINFO") or getenv("TERMINFO_DIRS"):
+        return
+    terminfo_fallback_directories = (
+        Path("/run/current-system/sw/share/terminfo"),
+        Path("/usr/share/terminfo"),
+        Path("/etc/terminfo"),
+        Path("/lib/terminfo"),
+    )
+
+    existing_directories = [str(directory) for directory in terminfo_fallback_directories if directory.is_dir()]
+    if existing_directories:
+        environ["TERMINFO_DIRS"] = ":".join(existing_directories)
 
 
 def partition_disk(disk: str, swap_size: int, reserve: int = 0) -> None:
@@ -56,7 +75,7 @@ def partition_disk(disk: str, swap_size: int, reserve: int = 0) -> None:
     swap_size = max(swap_size, 1)
     reserve = max(reserve, 0)
 
-    bash_wrapper(f"blkdiscard -f {disk}")
+    run_output(("blkdiscard", "-f", disk))
 
     if reserve > 0:
         msg = f"Creating swap partition on {disk=} with size {swap_size=}GiB and reserve {reserve=}GiB"
@@ -72,14 +91,28 @@ def partition_disk(disk: str, swap_size: int, reserve: int = 0) -> None:
     logger.debug(f"{swap_partition=}")
 
     create_partitions = (
-        f"parted --script --align=optimal {disk} -- "
+        "parted",
-        "mklabel gpt "
+        "--script",
-        "mkpart EFI 1MiB 4GiB "
+        "--align=optimal",
-        f"mkpart root_pool 4GiB -{swap_start}GiB "
+        disk,
-        f"{swap_partition}"
+        "--",
-        "set 1 esp on"
+        "mklabel",
+        "gpt",
+        "mkpart",
+        "EFI",
+        "1MiB",
+        "4GiB",
+        "mkpart",
+        "root_pool",
+        "4GiB",
+        f"-{swap_start}GiB",
+        *swap_partition.split(),
+        "set",
+        "1",
+        "esp",
+        "on",
     )
-    bash_wrapper(create_partitions)
+    run_output(create_partitions)
 
     logger.info(f"{disk=} successfully partitioned")
 
@@ -95,30 +128,43 @@ def create_zfs_pool(pool_disks: Sequence[str], mnt_dir: str) -> None:
         error = "disks must be a tuple of at least length 1"
         raise ValueError(error)
 
-    zpool_create = (
+    zpool_create = [
-        "zpool create "
+        "zpool",
-        "-o ashift=12 "
+        "create",
-        "-o autotrim=on "
+        "-o",
-        f"-R {mnt_dir} "
+        "ashift=12",
-        "-O acltype=posixacl "
+        "-o",
-        "-O canmount=off "
+        "autotrim=on",
-        "-O dnodesize=auto "
+        "-R",
-        "-O normalization=formD "
+        mnt_dir,
-        "-O relatime=on "
+        "-O",
-        "-O xattr=sa "
+        "acltype=posixacl",
-        "-O mountpoint=legacy "
+        "-O",
-        "-O compression=zstd "
+        "canmount=off",
-        "-O atime=off "
+        "-O",
-        "root_pool "
+        "dnodesize=auto",
-    )
+        "-O",
+        "normalization=formD",
+        "-O",
+        "relatime=on",
+        "-O",
+        "xattr=sa",
+        "-O",
+        "mountpoint=legacy",
+        "-O",
+        "compression=zstd",
+        "-O",
+        "atime=off",
+        "root_pool",
+    ]
     if len(pool_disks) == 1:
-        zpool_create += pool_disks[0]
+        zpool_create.append(pool_disks[0])
     else:
-        zpool_create += "mirror "
+        zpool_create.append("mirror")
-        zpool_create += " ".join(pool_disks)
+        zpool_create.extend(pool_disks)
 
-    bash_wrapper(zpool_create)
+    run_output(zpool_create)
-    zpools = bash_wrapper("zpool list -o name")
+    zpools = run_output(("zpool", "list", "-o", "name"))
     if "root_pool" not in zpools.splitlines():
         logger.critical("Failed to create root_pool")
         sys.exit(1)
@@ -126,11 +172,11 @@ def create_zfs_pool(pool_disks: Sequence[str], mnt_dir: str) -> None:
 
 def create_zfs_datasets() -> None:
     """Create ZFS datasets."""
-    bash_wrapper("zfs create -o canmount=noauto -o reservation=10G root_pool/root")
+    run_output(("zfs", "create", "-o", "canmount=noauto", "-o", "reservation=10G", "root_pool/root"))
-    bash_wrapper("zfs create root_pool/home")
+    run_output(("zfs", "create", "root_pool/home"))
-    bash_wrapper("zfs create root_pool/var -o reservation=1G")
+    run_output(("zfs", "create", "-o", "reservation=1G", "root_pool/var"))
-    bash_wrapper("zfs create -o compression=zstd-9 -o reservation=10G root_pool/nix")
+    run_output(("zfs", "create", "-o", "compression=zstd-9", "-o", "reservation=10G", "root_pool/nix"))
-    datasets = bash_wrapper("zfs list -o name")
+    datasets = run_output(("zfs", "list", "-o", "name"))
 
     expected_datasets = {
         "root_pool/root",
@@ -146,7 +192,7 @@ def create_zfs_datasets() -> None:
 
 def get_cpu_manufacturer() -> str:
     """Get the CPU manufacturer."""
-    output = bash_wrapper("cat /proc/cpuinfo")
+    output = Path("/proc/cpuinfo").read_text()
 
     id_vendor = {"AuthenticAMD": "amd", "GenuineIntel": "intel"}
 
@@ -160,7 +206,7 @@ def get_cpu_manufacturer() -> str:
 
 def get_boot_drive_id(disk: str) -> str:
     """Get the boot drive ID."""
-    output = bash_wrapper(f"lsblk -o UUID {disk}-part1")
+    output = run_output(("lsblk", "-o", "UUID", f"{disk}-part1"))
     return output.splitlines()[1]
 
 
@@ -220,21 +266,28 @@ def create_nix_hardware_file(mnt_dir: str, disks: Sequence[str], encrypt: str |
 
 def install_nixos(mnt_dir: str, disks: Sequence[str], encrypt: str | None) -> None:
     """Install NixOS."""
-    bash_wrapper(f"mount -o X-mount.mkdir -t zfs root_pool/root {mnt_dir}")
+    run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/root", mnt_dir))
-    bash_wrapper(f"mount -o X-mount.mkdir -t zfs root_pool/home {mnt_dir}/home")
+    run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/home", f"{mnt_dir}/home"))
-    bash_wrapper(f"mount -o X-mount.mkdir -t zfs root_pool/var {mnt_dir}/var")
+    run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/var", f"{mnt_dir}/var"))
-    bash_wrapper(f"mount -o X-mount.mkdir -t zfs root_pool/nix {mnt_dir}/nix")
+    run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/nix", f"{mnt_dir}/nix"))
 
     for disk in disks:
-        bash_wrapper(f"mkfs.vfat -n EFI {disk}-part1")
+        run_output(("mkfs.vfat", "-n", "EFI", f"{disk}-part1"))
 
     # set up mirroring afterwards if more than one disk
-    boot_partition = (
+    run_output(
-        f"mount -t vfat -o fmask=0077,dmask=0077,iocharset=iso8859-1,X-mount.mkdir {disks[0]}-part1 {mnt_dir}/boot"
+        (
+            "mount",
+            "-t",
+            "vfat",
+            "-o",
+            "fmask=0077,dmask=0077,iocharset=iso8859-1,X-mount.mkdir",
+            f"{disks[0]}-part1",
+            f"{mnt_dir}/boot",
+        ),
     )
-    bash_wrapper(boot_partition)
 
-    bash_wrapper(f"nixos-generate-config --root {mnt_dir}")
+    run_output(("nixos-generate-config", "--root", mnt_dir))
 
     create_nix_hardware_file(mnt_dir, disks, encrypt)
 
@@ -249,21 +302,28 @@ def installer(
 ) -> None:
     """Main."""
     logger.info("Starting installation")
+    require_commands(REQUIRED_COMMANDS)
+    disks = tuple(sorted(disks))
 
     for disk in disks:
         partition_disk(disk, swap_size, reserve)
 
-        test = Popen(("printf", f"'{encrypt_key}'"), stdout=PIPE)
         if encrypt_key:
             sleep(1)
-            for command in (
+            key_input = encrypt_key.encode()
-                f"cryptsetup luksFormat --type luks2 {disk}-part2 -",
+            run(
-                f"cryptsetup luksOpen {disk}-part2 luks-root-pool-{disk.split('/')[-1]}-part2 -",
+                ("cryptsetup", "luksFormat", "--type", "luks2", f"{disk}-part2", "-"),
-            ):
+                input=key_input,
-                run(command, check=True, stdin=test.stdout)
+                check=True,
+            )
+            run(
+                ("cryptsetup", "luksOpen", f"{disk}-part2", f"luks-root-pool-{disk.split('/')[-1]}-part2", "-"),
+                input=key_input,
+                check=True,
+            )
 
-    # Fixed mount point for the new system; the installer runs as root on a fresh disk
+    # nixos-install rejects mount points under world-writable paths like /tmp
-    mnt_dir = "/tmp/nix_install"  # noqa: S108
+    mnt_dir = "/mnt"
 
     Path(mnt_dir).mkdir(parents=True, exist_ok=True)
 
@@ -281,14 +341,36 @@ def installer(
     logger.info("Installation complete")
 
 
-def main() -> None:
+def main(argv: Sequence[str] | None = None) -> None:
     """Main."""
-    configure_logger("DEBUG")
+    parser = ArgumentParser(description="Install this NixOS configuration onto a ZFS root pool.")
+    parser.add_argument(
+        "--check",
+        action="store_true",
+        help="validate that the live environment has the external installer commands and exit",
+    )
+    parser.add_argument("--log-level", default=getenv("LOG_LEVEL", "DEBUG"), help="Python log level")
+    args = parser.parse_args(argv)
 
+    configure_logger(args.log_level)
+
+    if args.check:
+        require_commands(REQUIRED_COMMANDS)
+        logger.info("installer runtime dependencies are available")
+        return
+
+    configure_terminal()
     state = curses.wrapper(draw_menu)
 
     encrypt_key = getenv("ENCRYPT_KEY")
 
+    if not encrypt_key:
+        encrypt_key = state.encryption_password
+
+    if not state.selected_device_ids:
+        logger.error("No disks selected; exiting without installing")
+        sys.exit(1)
+
     logger.info("installing_nixos")
     logger.info(f"disks: {state.selected_device_ids}")
     logger.info(f"swap_size: {state.swap_size}")

python/installer/build.py

@@ -0,0 +1,184 @@
+"""Build the one-file installer binary."""
+
+from __future__ import annotations
+
+import logging
+import os
+import shutil
+import stat
+import subprocess
+from argparse import ArgumentParser, Namespace
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+logger = logging.getLogger(__name__)
+
+BINARY_NAME = "nixos-installer"
+DEFAULT_INTERPRETER = "/lib64/ld-linux-x86-64.so.2"
+INSTALLER_SOURCE_FILES = (
+    Path("python/__init__.py"),
+    Path("python/logging_config.py"),
+    Path("python/process.py"),
+    Path("python/installer/__init__.py"),
+    Path("python/installer/__main__.py"),
+    Path("python/installer/tui.py"),
+)
+
+
+class InstallerBuildError(RuntimeError):
+    """Raised when the installer binary cannot be built."""
+
+
+class MissingSourceFileError(InstallerBuildError):
+    """Raised when a required source file is missing."""
+
+    def __init__(self, path: Path) -> None:
+        """Store the missing path."""
+        super().__init__(f"Required installer source file is missing: {path}")
+        self.path = path
+
+
+class MissingToolError(InstallerBuildError):
+    """Raised when a required build tool is missing."""
+
+    def __init__(self, tool: str) -> None:
+        """Store the missing tool name."""
+        super().__init__(f"Required build tool is missing from PATH: {tool}")
+        self.tool = tool
+
+
+def repo_root() -> Path:
+    """Return the repository root for direct script usage."""
+    return Path(__file__).resolve().parents[2]
+
+
+def require_tool(tool: str) -> str:
+    """Return the path to a tool or raise."""
+    tool_path = shutil.which(tool)
+    if tool_path is None:
+        raise MissingToolError(tool)
+    return tool_path
+
+
+def copy_installer_source(source_root: Path, destination: Path) -> None:
+    """Copy only the installer files into a minimal staging tree."""
+    if destination.exists():
+        shutil.rmtree(destination)
+    destination.mkdir(parents=True)
+
+    for relative_path in INSTALLER_SOURCE_FILES:
+        source = source_root / relative_path
+        if not source.is_file():
+            raise MissingSourceFileError(source)
+
+        target = destination / relative_path
+        target.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(source, target)
+
+
+def run_command(command: Sequence[str], *, env: dict[str, str] | None = None) -> None:
+    """Run a build command."""
+    logger.info("running command=%s", command)
+    subprocess.run(command, check=True, env=env)
+
+
+def pyinstaller_environment(staged_source: Path, build_root: Path) -> dict[str, str]:
+    """Return environment variables for PyInstaller."""
+    env = os.environ.copy()
+    home = build_root / "home"
+    home.mkdir(parents=True, exist_ok=True)
+
+    env["HOME"] = str(home)
+    if existing_pythonpath := env.get("PYTHONPATH"):
+        env["PYTHONPATH"] = f"{staged_source}{os.pathsep}{existing_pythonpath}"
+    else:
+        env["PYTHONPATH"] = str(staged_source)
+
+    return env
+
+
+def build_installer(
+    *,
+    source_root: Path,
+    build_root: Path,
+    output: Path,
+    interpreter: str,
+    patch_elf: bool,
+) -> Path:
+    """Build the one-file installer binary."""
+    pyinstaller = require_tool("pyinstaller")
+    if patch_elf:
+        patchelf = require_tool("patchelf")
+
+    source_root = source_root.resolve()
+    build_root = build_root.resolve()
+    output = output.resolve()
+
+    staged_source = build_root / "source"
+    dist_dir = build_root / "dist"
+    work_dir = build_root / "work"
+    spec_dir = build_root / "spec"
+
+    copy_installer_source(source_root, staged_source)
+    output.parent.mkdir(parents=True, exist_ok=True)
+
+    run_command(
+        (
+            pyinstaller,
+            "--clean",
+            "--onefile",
+            "--name",
+            BINARY_NAME,
+            "--paths",
+            str(staged_source),
+            "--distpath",
+            str(dist_dir),
+            "--workpath",
+            str(work_dir),
+            "--specpath",
+            str(spec_dir),
+            str(staged_source / "python/installer/__main__.py"),
+        ),
+        env=pyinstaller_environment(staged_source, build_root),
+    )
+
+    built_binary = dist_dir / BINARY_NAME
+    shutil.copy2(built_binary, output)
+    output.chmod(output.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
+    if patch_elf:
+        run_command((patchelf, "--set-interpreter", interpreter, "--remove-rpath", str(output)))
+
+    return output
+
+
+def parse_args(argv: Sequence[str] | None = None) -> Namespace:
+    """Parse command-line arguments."""
+    parser = ArgumentParser(description="Build the one-file NixOS installer binary.")
+    parser.add_argument("--source-root", type=Path, default=repo_root(), help="repo or staged source root")
+    parser.add_argument("--build-root", type=Path, default=Path("build/nixos-installer"), help="temporary build root")
+    parser.add_argument("--output", type=Path, default=Path("dist/nixos-installer"), help="output binary path")
+    parser.add_argument("--interpreter", default=DEFAULT_INTERPRETER, help="ELF interpreter path for the USB binary")
+    parser.add_argument("--skip-patchelf", action="store_true", help="do not patch the final ELF binary")
+    parser.add_argument("--log-level", default="INFO", help="Python log level")
+    return parser.parse_args(argv)
+
+
+def main(argv: Sequence[str] | None = None) -> None:
+    """Build the installer binary from the command line."""
+    args = parse_args(argv)
+    logging.basicConfig(level=args.log_level, format="%(levelname)s %(message)s")
+    build_installer(
+        source_root=args.source_root,
+        build_root=args.build_root,
+        output=args.output,
+        interpreter=args.interpreter,
+        patch_elf=not args.skip_patchelf,
+    )
+
+
+if __name__ == "__main__":
+    main()

python/installer/old_installer.py

@@ -1,739 +0,0 @@
-"""Install NixOS on a ZFS pool."""
-
-from __future__ import annotations
-
-import curses
-import logging
-import sys
-from collections import defaultdict
-from os import getenv
-from pathlib import Path
-from random import getrandbits
-from subprocess import PIPE, Popen, run
-from time import sleep
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from collections.abc import Sequence
-
-logger = logging.getLogger(__name__)
-ESCAPE_KEY = 27
-
-
-def configure_logger(level: str = "INFO") -> None:
-    """Configure the logger.
-
-    Args:
-        level (str, optional): The logging level. Defaults to "INFO".
-    """
-    logging.basicConfig(
-        level=level,
-        datefmt="%Y-%m-%dT%H:%M:%S%z",
-        format="%(asctime)s %(levelname)s %(filename)s:%(lineno)d - %(message)s",
-        handlers=[logging.StreamHandler(sys.stdout)],
-    )
-
-
-def bash_wrapper(command: str) -> str:
-    """Execute a bash command and capture the output.
-
-    Args:
-        command (str): The bash command to be executed.
-
-    Returns:
-        Tuple[str, int]: A tuple containing the output of the command (stdout) as a string,
-        the error output (stderr) as a string (optional), and the return code as an integer.
-    """
-    logger.debug(f"running {command=}")
-    # This is a acceptable risk
-    process = Popen(command.split(), stdout=PIPE, stderr=PIPE)
-    output, _ = process.communicate()
-    if process.returncode != 0:
-        error = f"Failed to run command {command=} return code {process.returncode=}"
-        raise RuntimeError(error)
-
-    return output.decode()
-
-
-def partition_disk(disk: str, swap_size: int, reserve: int = 0) -> None:
-    """Partition a disk.
-
-    Args:
-        disk (str): The disk to partition.
-        swap_size (int): The size of the swap partition in GB.
-            minimum value is 1.
-        reserve (int, optional): The size of the reserve partition in GB. Defaults to 0.
-            minimum value is 0.
-    """
-    logger.info(f"partitioning {disk=}")
-    swap_size = max(swap_size, 1)
-    reserve = max(reserve, 0)
-
-    bash_wrapper(f"blkdiscard -f {disk}")
-
-    if reserve > 0:
-        msg = f"Creating swap partition on {disk=} with size {swap_size=}GiB and reserve {reserve=}GiB"
-        logger.info(msg)
-
-        swap_start = swap_size + reserve
-        swap_partition = f"mkpart swap -{swap_start}GiB -{reserve}GiB "
-    else:
-        logger.info(f"Creating swap partition on {disk=} with size {swap_size=}GiB")
-        swap_start = swap_size
-        swap_partition = f"mkpart swap -{swap_start}GiB 100% "
-
-    logger.debug(f"{swap_partition=}")
-
-    create_partitions = (
-        f"parted --script --align=optimal {disk} -- "
-        "mklabel gpt "
-        "mkpart EFI 1MiB 4GiB "
-        f"mkpart root_pool 4GiB -{swap_start}GiB "
-        f"{swap_partition}"
-        "set 1 esp on"
-    )
-    bash_wrapper(create_partitions)
-
-    logger.info(f"{disk=} successfully partitioned")
-
-
-def create_zfs_pool(pool_disks: Sequence[str], mnt_dir: str) -> None:
-    """Create a ZFS pool.
-
-    Args:
-        pool_disks (Sequence[str]): A tuple of disks to use for the pool.
-        mnt_dir (str): The mount directory.
-    """
-    if len(pool_disks) <= 0:
-        error = "disks must be a tuple of at least length 1"
-        raise ValueError(error)
-
-    zpool_create = (
-        "zpool create "
-        "-o ashift=12 "
-        "-o autotrim=on "
-        f"-R {mnt_dir} "
-        "-O acltype=posixacl "
-        "-O canmount=off "
-        "-O dnodesize=auto "
-        "-O normalization=formD "
-        "-O relatime=on "
-        "-O xattr=sa "
-        "-O mountpoint=legacy "
-        "-O compression=zstd "
-        "-O atime=off "
-        "root_pool "
-    )
-    if len(pool_disks) == 1:
-        zpool_create += pool_disks[0]
-    else:
-        zpool_create += "mirror "
-        zpool_create += " ".join(pool_disks)
-
-    bash_wrapper(zpool_create)
-    zpools = bash_wrapper("zpool list -o name")
-    if "root_pool" not in zpools.splitlines():
-        logger.critical("Failed to create root_pool")
-        sys.exit(1)
-
-
-def create_zfs_datasets() -> None:
-    """Create ZFS datasets."""
-    bash_wrapper("zfs create -o canmount=noauto -o reservation=10G root_pool/root")
-    bash_wrapper("zfs create root_pool/home")
-    bash_wrapper("zfs create root_pool/var -o reservation=1G")
-    bash_wrapper("zfs create -o compression=zstd-9 -o reservation=10G root_pool/nix")
-    datasets = bash_wrapper("zfs list -o name")
-
-    expected_datasets = {
-        "root_pool/root",
-        "root_pool/home",
-        "root_pool/var",
-        "root_pool/nix",
-    }
-    missing_datasets = expected_datasets.difference(datasets.splitlines())
-    if missing_datasets:
-        logger.critical(f"Failed to create pools {missing_datasets}")
-        sys.exit(1)
-
-
-def get_cpu_manufacturer() -> str:
-    """Get the CPU manufacturer."""
-    output = bash_wrapper("cat /proc/cpuinfo")
-
-    id_vendor = {"AuthenticAMD": "amd", "GenuineIntel": "intel"}
-
-    for line in output.splitlines():
-        if "vendor_id" in line:
-            return id_vendor[line.split(": ")[1].strip()]
-    error = "Failed to get CPU manufacturer"
-    raise RuntimeError(error)
-
-
-def get_boot_drive_id(disk: str) -> str:
-    """Get the boot drive ID."""
-    output = bash_wrapper(f"lsblk -o UUID {disk}-part1")
-    return output.splitlines()[1]
-
-
-def create_nix_hardware_file(mnt_dir: str, disks: Sequence[str], *, encrypt: bool) -> None:
-    """Create a NixOS hardware file."""
-    cpu_manufacturer = get_cpu_manufacturer()
-
-    devices = ""
-    if encrypt:
-        disk = disks[0]
-
-        devices = (
-            f'     luks.devices."luks-root-pool-{disk.split("/")[-1]}-part2"'
-            "= {\n"
-            f'        device = "{disk}-part2";\n'
-            "        bypassWorkqueues = true;\n"
-            "        allowDiscards = true;\n"
-            "      };\n"
-        )
-
-    host_id = format(getrandbits(32), "08x")
-
-    nix_hardware = (
-        "{ config, lib, modulesPath, ... }:\n"
-        "{\n"
-        '  imports = [ (modulesPath + "/installer/scan/not-detected.nix") ];\n\n'
-        "  boot = {\n"
-        "    initrd = {\n"
-        "      availableKernelModules = [ \n"
-        '        "ahci"\n'
-        '        "ehci_pci"\n'
-        '        "nvme"\n'
-        '        "sd_mod"\n'
-        '        "usb_storage"\n'
-        '        "usbhid"\n'
-        '        "xhci_pci"\n'
-        "      ];\n"
-        "      kernelModules = [ ];\n"
-        f" {devices}"
-        "    };\n"
-        f'    kernelModules = [ "kvm-{cpu_manufacturer}" ];\n'
-        "    extraModulePackages = [ ];\n"
-        "  };\n\n"
-        "  fileSystems = {\n"
-        '    "/" = lib.mkDefault {\n      device = "root_pool/root";\n      fsType = "zfs";\n    };\n\n'
-        '    "/home" = {\n      device = "root_pool/home";\n      fsType = "zfs";\n    };\n\n'
-        '    "/var" = {\n      device = "root_pool/var";\n      fsType = "zfs";\n    };\n\n'
-        '    "/nix" = {\n      device = "root_pool/nix";\n      fsType = "zfs";\n    };\n\n'
-        '    "/boot" = {\n'
-        f'      device = "/dev/disk/by-uuid/{get_boot_drive_id(disks[0])}";\n'
-        '      fsType = "vfat";\n'
-        "      options = [\n"
-        '        "fmask=0077"\n'
-        '        "dmask=0077"\n'
-        "      ];\n"
-        "    };\n"
-        "  };\n\n"
-        "  swapDevices = [ ];\n\n"
-        "  networking.useDHCP = lib.mkDefault true;\n\n"
-        '  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";\n'
-        f"  hardware.cpu.{cpu_manufacturer}.updateMicrocode = lib.mkDefault "
-        "config.hardware.enableRedistributableFirmware;\n"
-        f'  networking.hostId = "{host_id}";\n'
-        "}\n"
-    )
-
-    Path(f"{mnt_dir}/etc/nixos/hardware-configuration.nix").write_text(nix_hardware)
-
-
-def install_nixos(mnt_dir: str, disks: Sequence[str], *, encrypt: bool) -> None:
-    """Install NixOS."""
-    bash_wrapper(f"mount -o X-mount.mkdir -t zfs root_pool/root {mnt_dir}")
-    bash_wrapper(f"mount -o X-mount.mkdir -t zfs root_pool/home {mnt_dir}/home")
-    bash_wrapper(f"mount -o X-mount.mkdir -t zfs root_pool/var {mnt_dir}/var")
-    bash_wrapper(f"mount -o X-mount.mkdir -t zfs root_pool/nix {mnt_dir}/nix")
-
-    for disk in disks:
-        bash_wrapper(f"mkfs.vfat -n EFI {disk}-part1")
-
-    # set up mirroring afterwards if more than one disk
-    boot_partition = (
-        f"mount -t vfat -o fmask=0077,dmask=0077,iocharset=iso8859-1,X-mount.mkdir {disks[0]}-part1 {mnt_dir}/boot"
-    )
-    bash_wrapper(boot_partition)
-
-    bash_wrapper(f"nixos-generate-config --root {mnt_dir}")
-
-    create_nix_hardware_file(mnt_dir, disks, encrypt=encrypt)
-
-    run(("nixos-install", "--root", mnt_dir), check=True)
-
-
-def installer(
-    disks: set[str],
-    swap_size: int,
-    reserve: int,
-    encrypt_key: str | None,
-) -> None:
-    """Main."""
-    logger.info("Starting installation")
-
-    for disk in disks:
-        partition_disk(disk, swap_size, reserve)
-
-        if encrypt_key:
-            sleep(1)
-            key_input = encrypt_key.encode()
-            run(
-                ("cryptsetup", "luksFormat", "--type", "luks2", f"{disk}-part2", "-"),
-                input=key_input,
-                check=True,
-            )
-            run(
-                (
-                    "cryptsetup",
-                    "luksOpen",
-                    f"{disk}-part2",
-                    f"luks-root-pool-{disk.split('/')[-1]}-part2",
-                    "-",
-                ),
-                input=key_input,
-                check=True,
-            )
-
-    # Fixed mount point for the new system; the installer runs as root on a fresh disk
-    mnt_dir = "/tmp/nix_install"  # noqa: S108
-
-    Path(mnt_dir).mkdir(parents=True, exist_ok=True)
-
-    if encrypt_key:
-        pool_disks = [f"/dev/mapper/luks-root-pool-{disk.split('/')[-1]}-part2" for disk in disks]
-    else:
-        pool_disks = [f"{disk}-part2" for disk in disks]
-
-    create_zfs_pool(pool_disks, mnt_dir)
-
-    create_zfs_datasets()
-
-    install_nixos(mnt_dir, disks, encrypt=bool(encrypt_key))
-
-    logger.info("Installation complete")
-
-
-class Cursor:
-    """Track cursor position and constrain movement to screen bounds."""
-
-    def __init__(self) -> None:
-        """Initialize cursor position and screen dimensions."""
-        self.x_position = 0
-        self.y_position = 0
-        self.height = 0
-        self.width = 0
-
-    def set_height(self, height: int) -> None:
-        """Set the maximum screen height."""
-        self.height = height
-
-    def set_width(self, width: int) -> None:
-        """Set the maximum screen width."""
-        self.width = width
-
-    def x_bounce_check(self, cursor: int) -> int:
-        """Clamp an x position to the screen width."""
-        cursor = max(0, cursor)
-        return min(self.width - 1, cursor)
-
-    def y_bounce_check(self, cursor: int) -> int:
-        """Clamp a y position to the screen height."""
-        cursor = max(0, cursor)
-        return min(self.height - 1, cursor)
-
-    def set_x(self, x: int) -> None:
-        """Set the cursor x position."""
-        self.x_position = self.x_bounce_check(x)
-
-    def set_y(self, y: int) -> None:
-        """Set the cursor y position."""
-        self.y_position = self.y_bounce_check(y)
-
-    def get_x(self) -> int:
-        """Get the cursor x position."""
-        return self.x_position
-
-    def get_y(self) -> int:
-        """Get the cursor y position."""
-        return self.y_position
-
-    def move_up(self) -> None:
-        """Move the cursor up one row."""
-        self.set_y(self.y_position - 1)
-
-    def move_down(self) -> None:
-        """Move the cursor down one row."""
-        self.set_y(self.y_position + 1)
-
-    def move_left(self) -> None:
-        """Move the cursor left one column."""
-        self.set_x(self.x_position - 1)
-
-    def move_right(self) -> None:
-        """Move the cursor right one column."""
-        self.set_x(self.x_position + 1)
-
-    def navigation(self, key: int) -> None:
-        """Move the cursor for a curses navigation key."""
-        action = {
-            curses.KEY_DOWN: self.move_down,
-            curses.KEY_UP: self.move_up,
-            curses.KEY_RIGHT: self.move_right,
-            curses.KEY_LEFT: self.move_left,
-        }
-
-        action.get(key, lambda: None)()
-
-
-class State:
-    """State class to store the state of the program."""
-
-    def __init__(self) -> None:
-        """Initialize installer menu state."""
-        self.key = 0
-        self.cursor = Cursor()
-
-        self.swap_size = 0
-        self.show_swap_input = False
-
-        self.reserve_size = 0
-        self.show_reserve_input = False
-
-        self.selected_device_ids = set()
-
-    def get_selected_devices(self) -> tuple[str]:
-        """Get selected devices."""
-        return tuple(self.selected_device_ids)
-
-
-def get_device(raw_device: str) -> dict[str, str]:
-    """Parse an lsblk key-value device row."""
-    raw_device_components = raw_device.split(" ")
-    return {thing.split("=")[0].lower(): thing.split("=")[1].strip('"') for thing in raw_device_components}
-
-
-def get_devices() -> list[dict[str, str]]:
-    """Get a list of devices."""
-    # --bytes
-    raw_devices = bash_wrapper("lsblk --paths --pairs").splitlines()
-    return [get_device(raw_device) for raw_device in raw_devices]
-
-
-def get_device_id_mapping() -> dict[str, set[str]]:
-    """Get a list of device ids.
-
-    Returns:
-        list[str]: the list of device ids
-    """
-    device_ids = bash_wrapper("find /dev/disk/by-id -type l").splitlines()
-
-    device_id_mapping: dict[str, set[str]] = defaultdict(set)
-
-    for device_id in device_ids:
-        device = bash_wrapper(f"readlink -f {device_id}").strip()
-        device_id_mapping[device].add(device_id)
-
-    return device_id_mapping
-
-
-def calculate_device_menu_padding(devices: list[dict[str, str]], column: str, padding: int = 0) -> int:
-    """Calculate the width needed for a device menu column."""
-    return max(len(device[column]) for device in devices) + padding
-
-
-def draw_device_ids(
-    state: State,
-    row_number: int,
-    menu_start_x: int,
-    std_screen: curses.window,
-    menu_width: list[int],
-    device_ids: set[str],
-) -> tuple[State, int]:
-    """Draw selectable device IDs for a device row."""
-    for device_id in sorted(device_ids):
-        row_number = row_number + 1
-        if row_number == state.cursor.get_y() and state.cursor.get_x() in menu_width:
-            std_screen.attron(curses.A_BOLD)
-            if state.key == ord(" "):
-                if device_id not in state.selected_device_ids:
-                    state.selected_device_ids.add(device_id)
-                else:
-                    state.selected_device_ids.remove(device_id)
-
-        if device_id in state.selected_device_ids:
-            std_screen.attron(curses.color_pair(7))
-
-        std_screen.addstr(row_number, menu_start_x, f"  {device_id}")
-
-        std_screen.attroff(curses.color_pair(7))
-        std_screen.attroff(curses.A_BOLD)
-
-    return state, row_number
-
-
-def draw_device_menu(
-    std_screen: curses.window,
-    devices: list[dict[str, str]],
-    device_id_mapping: dict[str, set[str]],
-    state: State,
-    menu_start_y: int = 0,
-    menu_start_x: int = 0,
-) -> tuple[State, int]:
-    """Draw the device menu and handle user input.
-
-    Args:
-        std_screen (curses.window): the curses window to draw on
-        devices (list[dict[str, str]]): the list of devices to draw
-        device_id_mapping (dict[str, set[str]]): the list of device ids to draw
-        state (State): the state object to update
-        menu_start_y (int, optional): the y position to start drawing the menu. Defaults to 0.
-        menu_start_x (int, optional): the x position to start drawing the menu. Defaults to 0.
-
-    Returns:
-        State: the updated state object
-    """
-    padding = 2
-
-    name_padding = calculate_device_menu_padding(devices, "name", padding)
-    size_padding = calculate_device_menu_padding(devices, "size", padding)
-    type_padding = calculate_device_menu_padding(devices, "type", padding)
-    mountpoints_padding = calculate_device_menu_padding(devices, "mountpoints", padding)
-
-    device_header = (
-        f"{'Name':{name_padding}}{'Size':{size_padding}}{'Type':{type_padding}}{'Mountpoints':{mountpoints_padding}}"
-    )
-
-    menu_width = range(menu_start_x, len(device_header) + menu_start_x)
-
-    std_screen.addstr(menu_start_y, menu_start_x, device_header, curses.color_pair(5))
-    devises_list_start = menu_start_y + 1
-
-    row_number = devises_list_start
-
-    for device in devices:
-        row_number = row_number + 1
-        device_name = device["name"]
-        device_row = (
-            f"{device_name:{name_padding}}"
-            f"{device['size']:{size_padding}}"
-            f"{device['type']:{type_padding}}"
-            f"{device['mountpoints']:{mountpoints_padding}}"
-        )
-        std_screen.addstr(row_number, menu_start_x, device_row)
-
-        state, row_number = draw_device_ids(
-            state=state,
-            row_number=row_number,
-            menu_start_x=menu_start_x,
-            std_screen=std_screen,
-            menu_width=menu_width,
-            device_ids=device_id_mapping[device_name],
-        )
-
-    return state, row_number
-
-
-def debug_menu(std_screen: curses.window, key: int) -> None:
-    """Draw debug information for the current curses screen."""
-    height, width = std_screen.getmaxyx()
-    width_height = f"Width: {width}, Height: {height}"
-    std_screen.addstr(height - 4, 0, width_height, curses.color_pair(5))
-
-    key_pressed = f"Last key pressed: {key}"[: width - 1]
-    if key == 0:
-        key_pressed = "No key press detected..."[: width - 1]
-    std_screen.addstr(height - 3, 0, key_pressed)
-
-    for i in range(8):
-        std_screen.addstr(height - 2, i * 3, f"{i}██", curses.color_pair(i))
-
-
-def status_bar(
-    std_screen: curses.window,
-    cursor: Cursor,
-    width: int,
-    height: int,
-) -> None:
-    """Draw the footer status bar."""
-    std_screen.attron(curses.A_REVERSE)
-    std_screen.attron(curses.color_pair(3))
-
-    status_bar = f"Press 'q' to exit | STATUS BAR | Pos: {cursor.get_x()}, {cursor.get_y()}"
-    std_screen.addstr(height - 1, 0, status_bar)
-    std_screen.addstr(height - 1, len(status_bar), " " * (width - len(status_bar) - 1))
-
-    std_screen.attroff(curses.color_pair(3))
-    std_screen.attroff(curses.A_REVERSE)
-
-
-def set_color() -> None:
-    """Initialize curses color pairs."""
-    curses.start_color()
-    curses.use_default_colors()
-    for i in range(curses.COLORS):
-        curses.init_pair(i + 1, i, -1)
-
-
-def get_text_input(std_screen: curses.window, prompt: str, y: int, x: int) -> str:
-    """Read text input from a curses screen."""
-    curses.echo()
-    std_screen.addstr(y, x, prompt)
-    input_str = ""
-    while True:
-        key = std_screen.getch()
-        if key == ord("\n"):
-            break
-        if key == ESCAPE_KEY:
-            input_str = ""
-            break
-        if key in (curses.KEY_BACKSPACE, ord("\b"), 127):
-            input_str = input_str[:-1]
-            std_screen.addstr(y, x + len(prompt), input_str + " ")
-        else:
-            input_str += chr(key)
-        std_screen.refresh()
-    curses.noecho()
-    return input_str
-
-
-def swap_size_input(
-    std_screen: curses.window,
-    state: State,
-    swap_offset: int,
-) -> State:
-    """Handle swap size input."""
-    swap_size_text = "Swap size (GB): "
-    std_screen.addstr(swap_offset, 0, f"{swap_size_text}{state.swap_size}")
-    if state.key == ord("\n") and state.cursor.get_y() == swap_offset:
-        state.show_swap_input = True
-
-    if state.show_swap_input:
-        swap_size_str = get_text_input(std_screen, swap_size_text, swap_offset, 0)
-        try:
-            state.swap_size = int(swap_size_str)
-            state.show_swap_input = False
-        except ValueError:
-            std_screen.addstr(swap_offset, 0, "Invalid input. Press any key to continue.")
-            std_screen.getch()
-            state.show_swap_input = False
-
-    return state
-
-
-def reserve_size_input(
-    std_screen: curses.window,
-    state: State,
-    reserve_offset: int,
-) -> State:
-    """Handle reserve size input."""
-    reserve_size_text = "reserve size (GB): "
-    std_screen.addstr(reserve_offset, 0, f"{reserve_size_text}{state.reserve_size}")
-    if state.key == ord("\n") and state.cursor.get_y() == reserve_offset:
-        state.show_reserve_input = True
-
-    if state.show_reserve_input:
-        reserve_size_str = get_text_input(std_screen, reserve_size_text, reserve_offset, 0)
-        try:
-            state.reserve_size = int(reserve_size_str)
-            state.show_reserve_input = False
-        except ValueError:
-            std_screen.addstr(reserve_offset, 0, "Invalid input. Press any key to continue.")
-            std_screen.getch()
-            state.show_reserve_input = False
-
-    return state
-
-
-def draw_menu(std_screen: curses.window) -> State:
-    """Draw the menu and handle user input.
-
-    Args:
-        std_screen (curses.window): the curses window to draw on
-
-    Returns:
-        State: the state object
-    """
-    # Clear and refresh the screen for a blank canvas
-    std_screen.clear()
-    std_screen.refresh()
-
-    set_color()
-
-    state = State()
-
-    devices = get_devices()
-
-    device_id_mapping = get_device_id_mapping()
-
-    # Loop where k is the last character pressed
-    while state.key != ord("q"):
-        std_screen.clear()
-        height, width = std_screen.getmaxyx()
-
-        state.cursor.set_height(height)
-        state.cursor.set_width(width)
-
-        state.cursor.navigation(state.key)
-
-        state, device_menu_size = draw_device_menu(
-            std_screen=std_screen,
-            state=state,
-            devices=devices,
-            device_id_mapping=device_id_mapping,
-        )
-
-        swap_offset = device_menu_size + 2
-
-        swap_size_input(
-            std_screen=std_screen,
-            state=state,
-            swap_offset=swap_offset,
-        )
-        reserve_size_input(
-            std_screen=std_screen,
-            state=state,
-            reserve_offset=swap_offset + 1,
-        )
-
-        status_bar(std_screen, state.cursor, width, height)
-
-        debug_menu(std_screen, state.key)
-
-        std_screen.move(state.cursor.get_y(), state.cursor.get_x())
-
-        std_screen.refresh()
-
-        state.key = std_screen.getch()
-
-    return state
-
-
-def main() -> None:
-    """Run the installer menu and start installation."""
-    configure_logger("DEBUG")
-
-    state = curses.wrapper(draw_menu)
-
-    encrypt_key = getenv("ENCRYPT_KEY")
-
-    logger.info("installing_nixos")
-    logger.info(f"disks: {state.selected_device_ids}")
-    logger.info(f"swap_size: {state.swap_size}")
-    logger.info(f"reserve: {state.reserve_size}")
-    logger.info(f"encrypted: {bool(encrypt_key)}")
-
-    sleep(3)
-
-    installer(
-        disks=state.get_selected_devices(),
-        swap_size=state.swap_size,
-        reserve=state.reserve_size,
-        encrypt_key=encrypt_key,
-    )
-
-
-if __name__ == "__main__":
-    main()

python/installer/package.nix

@@ -0,0 +1,49 @@
+{
+  lib,
+  stdenv,
+  patchelf,
+  python314,
+  python314Packages,
+  patchElf ? true,
+}:
+
+stdenv.mkDerivation {
+  pname = "nixos-installer";
+  version = "0.1.0";
+  src = ../../.;
+
+  dontPatchELF = true;
+  dontStrip = true;
+
+  nativeBuildInputs = [
+    patchelf
+    python314
+    python314Packages.pyinstaller
+  ];
+
+  buildPhase = ''
+    runHook preBuild
+
+    export HOME="$TMPDIR"
+    python "$src/python/installer/build.py" \
+      --source-root "$src" \
+      --build-root "$TMPDIR/nixos-installer-build" \
+      --output "$PWD/nixos-installer" ${lib.optionalString (!patchElf) "--skip-patchelf"}
+
+    runHook postBuild
+  '';
+
+  installPhase = ''
+    runHook preInstall
+
+    install -Dm755 nixos-installer $out/bin/nixos-installer
+
+    runHook postInstall
+  '';
+
+  meta.description =
+    if patchElf then
+      "One-file NixOS ZFS installer patched to run on foreign Linux live environments."
+    else
+      "One-file NixOS ZFS installer linked against the Nix store, for the custom install ISO.";
+}

python/installer/tui.py

@@ -5,30 +5,12 @@ from __future__ import annotations
 import curses
 import logging
 from collections import defaultdict
-from subprocess import PIPE, Popen
+
+from python.process import run_output
 
 logger = logging.getLogger(__name__)
 
-
+BYTE_MAX = 255
-def bash_wrapper(command: str) -> str:
-    """Execute a bash command and capture the output.
-
-    Args:
-        command (str): The bash command to be executed.
-
-    Returns:
-        Tuple[str, int]: A tuple containing the output of the command (stdout) as a string,
-        the error output (stderr) as a string (optional), and the return code as an integer.
-    """
-    logger.debug(f"running {command=}")
-    # This is a acceptable risk
-    process = Popen(command.split(), stdout=PIPE, stderr=PIPE)
-    output, _ = process.communicate()
-    if process.returncode != 0:
-        error = f"Failed to run command {command=} return code {process.returncode=}"
-        raise RuntimeError(error)
-
-    return output.decode()
 
 
 class Cursor:
@@ -121,6 +103,9 @@ class State:
         self.reserve_size = 0
         self.show_reserve_input = False
 
+        self.encryption_password = None
+        self.show_encryption_password_input = False
+
         self.selected_device_ids: set[str] = set()
 
     def get_selected_devices(self) -> tuple[str, ...]:
@@ -144,7 +129,7 @@ def get_device(raw_device: str) -> dict[str, str]:
 def get_devices() -> list[dict[str, str]]:
     """Get a list of devices."""
     # --bytes
-    raw_devices = bash_wrapper("lsblk --paths --pairs").splitlines()
+    raw_devices = run_output(("lsblk", "--paths", "--pairs")).splitlines()
     return [get_device(raw_device) for raw_device in raw_devices]
 
 
@@ -175,7 +160,22 @@ def debug_menu(std_screen: curses.window, key: int) -> None:
         std_screen.addstr(height - 2, i * 3, f"{i}██", curses.color_pair(i))
 
 
-def get_text_input(std_screen: curses.window, prompt: str, y: int, x: int) -> str:
+def draw_input_line(std_screen: curses.window, prompt: str, input_str: str, y: int, x: int, mask: str | None) -> None:
+    """Draw an input line without leaking masked values."""
+    _, width = std_screen.getmaxyx()
+    displayed_input = input_str if mask is None else mask * len(input_str)
+    line = f"{prompt}{displayed_input}"
+    available_width = max(0, width - x - 1)
+
+    std_screen.move(y, x)
+    if available_width > 0:
+        std_screen.addstr(y, x, line[:available_width])
+    std_screen.clrtoeol()
+    std_screen.move(y, min(x + len(line), width - 1))
+    std_screen.refresh()
+
+
+def get_text_input(std_screen: curses.window, prompt: str, y: int, x: int, mask: str | None = None) -> str | None:
     """Get text input.
 
     Args:
@@ -183,27 +183,27 @@ def get_text_input(std_screen: curses.window, prompt: str, y: int, x: int) -> st
         prompt (str): The prompt.
         y (int): The y position.
         x (int): The x position.
+        mask (str | None, optional): The character used to mask displayed input. Defaults to None.
 
     Returns:
-        str: The input string.
+        str | None: The input string, or None if input was cancelled.
     """
     esc_key = 27
-    curses.echo()
+    curses.noecho()
-    std_screen.addstr(y, x, prompt)
     input_str = ""
+    draw_input_line(std_screen, prompt, input_str, y, x, mask)
     while True:
         key = std_screen.getch()
-        if key == ord("\n"):
+        if key in (ord("\n"), ord("\r")):
             break
         if key == esc_key:
-            input_str = ""
+            curses.noecho()
-            break
+            return None
         if key in (curses.KEY_BACKSPACE, ord("\b"), 127):
             input_str = input_str[:-1]
-            std_screen.addstr(y, x + len(prompt), input_str + " ")
+        elif 0 <= key <= BYTE_MAX:
-        else:
             input_str += chr(key)
-        std_screen.refresh()
+        draw_input_line(std_screen, prompt, input_str, y, x, mask)
     curses.noecho()
     return input_str
 
@@ -230,6 +230,9 @@ def swap_size_input(
 
     if state.show_swap_input:
         swap_size_str = get_text_input(std_screen, swap_size_text, swap_offset, 0)
+        if swap_size_str is None:
+            state.show_swap_input = False
+            return state
         try:
             state.swap_size = int(swap_size_str)
             state.show_swap_input = False
@@ -263,6 +266,9 @@ def reserve_size_input(
 
     if state.show_reserve_input:
         reserve_size_str = get_text_input(std_screen, reserve_size_text, reserve_offset, 0)
+        if reserve_size_str is None:
+            state.show_reserve_input = False
+            return state
         try:
             state.reserve_size = int(reserve_size_str)
             state.show_reserve_input = False
@@ -274,6 +280,37 @@ def reserve_size_input(
     return state
 
 
+def encryption_password_input(
+    std_screen: curses.window,
+    state: State,
+    password_offset: int,
+) -> State:
+    """Encryption password input.
+
+    Args:
+        std_screen (curses.window): The curses window.
+        state (State): The state object.
+        password_offset (int): The password offset.
+
+    Returns:
+        State: The updated state object.
+    """
+    password_status = "set" if state.encryption_password else "unset"
+    encryption_label = "Encryption password: "
+    encryption_prompt = "Encryption password (blank disables LUKS): "
+    std_screen.addstr(password_offset, 0, f"{encryption_label}{password_status}")
+    if state.key == ord("\n") and state.cursor.get_y() == password_offset:
+        state.show_encryption_password_input = True
+
+    if state.show_encryption_password_input:
+        password = get_text_input(std_screen, encryption_prompt, password_offset, 0)
+        if password is not None:
+            state.encryption_password = password
+        state.show_encryption_password_input = False
+
+    return state
+
+
 def status_bar(
     std_screen: curses.window,
     cursor: Cursor,
@@ -305,12 +342,12 @@ def get_device_id_mapping() -> dict[str, set[str]]:
     Returns:
         list[str]: the list of device ids
     """
-    device_ids = bash_wrapper("find /dev/disk/by-id -type l").splitlines()
+    device_ids = run_output(("find", "/dev/disk/by-id", "-type", "l")).splitlines()
 
     device_id_mapping: dict[str, set[str]] = defaultdict(set)
 
     for device_id in device_ids:
-        device = bash_wrapper(f"readlink -f {device_id}").strip()
+        device = run_output(("readlink", "-f", device_id)).strip()
         device_id_mapping[device].add(device_id)
 
     return device_id_mapping
@@ -484,6 +521,11 @@ def draw_menu(std_screen: curses.window) -> State:
             state=state,
             reserve_offset=swap_offset + 1,
         )
+        encryption_password_input(
+            std_screen=std_screen,
+            state=state,
+            password_offset=swap_offset + 2,
+        )
 
         status_bar(std_screen, state.cursor, width, height)
 

python/logging_config.py

@@ -0,0 +1,16 @@
+"""Logging helpers shared by command-line tools."""
+
+from __future__ import annotations
+
+import logging
+import sys
+
+
+def configure_logger(level: str = "INFO") -> None:
+    """Configure process-wide logging."""
+    logging.basicConfig(
+        level=level,
+        datefmt="%Y-%m-%dT%H:%M:%S%z",
+        format="%(asctime)s %(levelname)s %(filename)s:%(lineno)d - %(message)s",
+        handlers=[logging.StreamHandler(sys.stdout)],
+    )

python/orm/common.py

@@ -7,6 +7,7 @@ from typing import cast
 
 from sqlalchemy import create_engine
 from sqlalchemy.engine import URL, Engine
+from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
 
 NAMING_CONVENTION = {
     "ix": "ix_%(table_name)s_%(column_0_name)s",
@@ -31,23 +32,16 @@ def get_connection_info(name: str) -> tuple[str, str, str, str, str | None]:
     return cast("tuple[str, str, str, str, str | None]", (database, host, port, username, password))
 
 
-def get_postgres_engine(
+def build_postgres_url(name: str, *, vector_engine: bool = False) -> tuple[URL, dict[str, str]]:
-    *,
+    """Build the Postgres connection URL and connect_args from environment variables.
-    name: str = "POSTGRES",
-    pool_pre_ping: bool = True,
-    vector_engine: bool = False,
-) -> Engine:
-    """Create a SQLAlchemy engine from environment variables.
 
     Args:
-        name (str, optional): The name of the environment variable prefix. Defaults to "POSTGRES".
+        name (str): The name of the environment variable prefix.
-        pool_pre_ping (bool, optional): Whether to ping the database before each connection. Defaults to True.
-        This fixes the issue of trying to use a conection that has timed out on the database side.
         vector_engine (bool, optional): Whether to use the vector search schema. Defaults to False.
-        This updates the search path the incldued the vecore types and operators.
+        This updates the search path to include the vector types and operators.
 
     Returns:
-        Engine: The SQLAlchemy engine.
+        tuple[URL, dict[str, str]]: The SQLAlchemy URL and connect_args for create_engine.
     """
     database, host, port, username, password = get_connection_info(name)
 
@@ -65,9 +59,66 @@ def get_postgres_engine(
     if vector_engine:
         connect_args["options"] = "-csearch_path=main,public"
 
+    return url, connect_args
+
+
+def get_postgres_engine(
+    *,
+    name: str = "POSTGRES",
+    pool_pre_ping: bool = True,
+    vector_engine: bool = False,
+    pool_size: int = 8,
+) -> Engine:
+    """Create a SQLAlchemy engine from environment variables.
+
+    Args:
+        name (str, optional): The name of the environment variable prefix. Defaults to "POSTGRES".
+        pool_pre_ping (bool, optional): Whether to ping the database before each connection. Defaults to True.
+        This fixes the issue of trying to use a conection that has timed out on the database side.
+        vector_engine (bool, optional): Whether to use the vector search schema. Defaults to False.
+        This updates the search path the incldued the vecore types and operators.
+        pool_size (int, optional): Number of connections to keep in the pool. Defaults to 8.
+
+    Returns:
+        Engine: The SQLAlchemy engine.
+    """
+    url, connect_args = build_postgres_url(name, vector_engine=vector_engine)
+
     return create_engine(
         url=url,
         pool_pre_ping=pool_pre_ping,
         pool_recycle=1800,
         connect_args=connect_args,
+        pool_size=pool_size,
+    )
+
+
+def get_async_postgres_engine(
+    *,
+    name: str = "POSTGRES",
+    pool_pre_ping: bool = True,
+    vector_engine: bool = False,
+    pool_size: int = 8,
+) -> AsyncEngine:
+    """Create an async SQLAlchemy engine from environment variables.
+
+    Args:
+        name (str, optional): The name of the environment variable prefix. Defaults to "POSTGRES".
+        pool_pre_ping (bool, optional): Whether to ping the database before each connection. Defaults to True.
+        This fixes the issue of trying to use a conection that has timed out on the database side.
+        vector_engine (bool, optional): Whether to use the vector search schema. Defaults to False.
+        This updates the search path the incldued the vecore types and operators.
+        pool_size (int, optional): Number of connections to keep in the pool. Defaults to 8.
+
+    Returns:
+        AsyncEngine: The async SQLAlchemy engine.
+    """
+    url, connect_args = build_postgres_url(name, vector_engine=vector_engine)
+
+    return create_async_engine(
+        url=url,
+        pool_pre_ping=pool_pre_ping,
+        pool_recycle=1800,
+        connect_args=connect_args,
+        pool_size=pool_size,
     )

python/orm/richie/__init__.py

@@ -12,12 +12,16 @@ from python.orm.richie.contact import (
     RelationshipType,
 )
 from python.orm.richie.ebook import (
+    EbookCandidatePhrase,
     EbookChapter,
     EbookChunk,
     EbookChunkEmbedding1024,
     EbookChunkEmbedding2560,
     EbookChunkEmbedding4096,
+    EbookChunkPhraseMention,
     EbookEmbeddingModel,
+    EbookPhraseAlias,
+    EbookProtectedPhrase,
     EbookSource,
 )
 
@@ -28,12 +32,16 @@ __all__ = [
     "Contact",
     "ContactNeed",
     "ContactRelationship",
+    "EbookCandidatePhrase",
     "EbookChapter",
     "EbookChunk",
     "EbookChunkEmbedding1024",
     "EbookChunkEmbedding2560",
     "EbookChunkEmbedding4096",
+    "EbookChunkPhraseMention",
     "EbookEmbeddingModel",
+    "EbookPhraseAlias",
+    "EbookProtectedPhrase",
     "EbookSource",
     "Need",
     "RelationshipType",

python/orm/richie/ebook.py

@@ -5,11 +5,23 @@ from __future__ import annotations
 from datetime import datetime
 
 from pgvector.sqlalchemy import Vector
-from sqlalchemy import BigInteger, Boolean, DateTime, ForeignKey, Index, String, UniqueConstraint
+from sqlalchemy import (
+    JSON,
+    BigInteger,
+    DateTime,
+    ForeignKey,
+    Index,
+    String,
+    Text,
+    UniqueConstraint,
+)
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from python.orm.richie.base import TableBase, TableBaseBig
 
+JSON_DOCUMENT = JSON().with_variant(JSONB, "postgresql")
+
 
 class EbookSource(TableBase):
     """One indexed EPUB file."""
@@ -94,7 +106,7 @@ class EbookEmbeddingModel(TableBase):
 
     name: Mapped[str] = mapped_column(String, unique=True)
     dimension: Mapped[int]
-    is_default: Mapped[bool] = mapped_column(Boolean, default=False)
+    is_default: Mapped[bool] = mapped_column(default=False)
 
 
 class EbookChunkEmbedding1024(TableBaseBig):
@@ -136,3 +148,97 @@ class EbookChunkEmbedding4096(TableBaseBig):
     chunk_id: Mapped[int] = mapped_column(ForeignKey("main.ebook_chunk.id", ondelete="CASCADE"))
     model_id: Mapped[int] = mapped_column(ForeignKey("main.ebook_embedding_model.id", ondelete="CASCADE"))
     embedding: Mapped[list[float]] = mapped_column(Vector(4096))
+
+
+class EbookCandidatePhrase(TableBase):
+    """A high-recall phrase candidate extracted from one book."""
+
+    __tablename__ = "candidate_phrases"
+    __table_args__ = (
+        UniqueConstraint("book_id", "phrase_norm", name="uq_candidate_phrases_book_id_phrase_norm"),
+        Index("candidate_phrases_book_score_idx", "book_id", "candidate_score"),
+        Index("candidate_phrases_book_norm_idx", "book_id", "phrase_norm"),
+    )
+
+    book_id: Mapped[int] = mapped_column(ForeignKey("main.ebook_source.id", ondelete="CASCADE"))
+    series_id: Mapped[int | None]
+    phrase_text: Mapped[str] = mapped_column(Text)
+    phrase_norm: Mapped[str] = mapped_column(Text)
+    token_count: Mapped[int]
+    source_raw_ngram: Mapped[bool] = mapped_column(default=False)
+    source_yake: Mapped[bool] = mapped_column(default=False)
+    source_spacy_ner: Mapped[bool] = mapped_column(default=False)
+    source_spacy_noun_chunk: Mapped[bool] = mapped_column(default=False)
+    source_capitalized: Mapped[bool] = mapped_column(default=False)
+    source_metadata: Mapped[bool] = mapped_column(default=False)
+    spacy_label: Mapped[str | None]
+    raw_count: Mapped[int] = mapped_column(default=0)
+    chapter_count: Mapped[int] = mapped_column(default=0)
+    yake_score: Mapped[float | None]
+    candidate_score: Mapped[float] = mapped_column(default=0.0)
+    sample_contexts: Mapped[list[str] | None] = mapped_column(JSON_DOCUMENT)
+    llm_judged: Mapped[bool] = mapped_column(default=False)
+    llm_keep: Mapped[bool | None]
+    llm_confidence: Mapped[float | None]
+    llm_category: Mapped[str | None]
+    llm_reason: Mapped[str | None] = mapped_column(Text)
+
+
+class EbookProtectedPhrase(TableBase):
+    """A phrase accepted by the LLM judge for protected query matching."""
+
+    __tablename__ = "protected_phrases"
+    __table_args__ = (
+        UniqueConstraint("book_id", "phrase_norm", name="uq_protected_phrases_book_id_phrase_norm"),
+        Index("protected_phrases_norm_idx", "phrase_norm"),
+        Index("protected_phrases_book_norm_idx", "book_id", "phrase_norm"),
+        Index("protected_phrases_series_norm_idx", "series_id", "phrase_norm"),
+    )
+
+    book_id: Mapped[int | None] = mapped_column(ForeignKey("main.ebook_source.id", ondelete="CASCADE"))
+    series_id: Mapped[int | None]
+    phrase_text: Mapped[str] = mapped_column(Text)
+    phrase_norm: Mapped[str] = mapped_column(Text)
+    canonical_id: Mapped[str]
+    phrase_type: Mapped[str | None]
+    token_count: Mapped[int]
+    confidence: Mapped[float]
+    importance: Mapped[float] = mapped_column(default=0.5)
+    allow_nested: Mapped[bool] = mapped_column(default=False)
+    suppress_children: Mapped[bool] = mapped_column(default=True)
+    source_candidate_id: Mapped[int | None] = mapped_column(
+        ForeignKey("main.candidate_phrases.id", ondelete="SET NULL")
+    )
+
+
+class EbookPhraseAlias(TableBase):
+    """A normalized alias that maps to a protected phrase."""
+
+    __tablename__ = "phrase_aliases"
+    __table_args__ = (
+        UniqueConstraint("phrase_id", "alias_norm", name="uq_phrase_aliases_phrase_id_alias_norm"),
+        Index("phrase_aliases_norm_idx", "alias_norm"),
+    )
+
+    phrase_id: Mapped[int] = mapped_column(ForeignKey("main.protected_phrases.id", ondelete="CASCADE"))
+    alias_text: Mapped[str] = mapped_column(Text)
+    alias_norm: Mapped[str] = mapped_column(Text)
+    confidence: Mapped[float] = mapped_column(default=1.0)
+
+
+class EbookChunkPhraseMention(TableBase):
+    """A precomputed occurrence of a protected phrase inside one chunk."""
+
+    __tablename__ = "chunk_phrase_mentions"
+    __table_args__ = (
+        UniqueConstraint("chunk_id", "phrase_id", "start_char", name="uq_chunk_phrase_mentions_chunk_phrase_start"),
+        Index("chunk_phrase_mentions_phrase_idx", "phrase_id"),
+        Index("chunk_phrase_mentions_chunk_idx", "chunk_id"),
+    )
+
+    chunk_id: Mapped[int] = mapped_column(ForeignKey("main.ebook_chunk.id", ondelete="CASCADE"))
+    phrase_id: Mapped[int] = mapped_column(ForeignKey("main.protected_phrases.id", ondelete="CASCADE"))
+    book_id: Mapped[int | None] = mapped_column(ForeignKey("main.ebook_source.id", ondelete="CASCADE"))
+    series_id: Mapped[int | None]
+    start_char: Mapped[int]
+    end_char: Mapped[int | None]

python/process.py

@@ -0,0 +1,52 @@
+"""Small subprocess helpers."""
+
+from __future__ import annotations
+
+import logging
+import shutil
+from subprocess import run
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Sequence
+
+logger = logging.getLogger(__name__)
+
+
+class CommandError(RuntimeError):
+    """Raised when an external command fails."""
+
+    def __init__(self, command: Sequence[str], returncode: int, stderr: str) -> None:
+        """Store command failure details."""
+        command_text = " ".join(command)
+        super().__init__(f"Failed to run command {command_text!r}: exit {returncode}\n{stderr}")
+        self.command = command
+        self.returncode = returncode
+        self.stderr = stderr
+
+
+class MissingCommandsError(RuntimeError):
+    """Raised when required external commands are not available."""
+
+    def __init__(self, commands: Sequence[str]) -> None:
+        """Store missing command details."""
+        missing = ", ".join(commands)
+        super().__init__(f"Missing required installer commands: {missing}")
+        self.commands = commands
+
+
+def require_commands(commands: Iterable[str]) -> None:
+    """Raise when one or more executables are missing from PATH."""
+    missing_commands = sorted({command for command in commands if shutil.which(command) is None})
+    if missing_commands:
+        raise MissingCommandsError(missing_commands)
+
+
+def run_output(command: Sequence[str]) -> str:
+    """Run a command and return stdout."""
+    logger.debug("running command=%s", command)
+    result = run(command, capture_output=True, text=True, check=False)
+    if result.returncode != 0:
+        raise CommandError(command, result.returncode, result.stderr)
+
+    return result.stdout

python/signal_alert.py

@@ -0,0 +1,30 @@
+"""signal_alert."""
+
+from __future__ import annotations
+
+import logging
+from os import getenv
+
+from apprise import Apprise
+
+logger = logging.getLogger(__name__)
+
+
+def signal_alert(body: str, title: str = "") -> None:
+    """Send a signal alert.
+
+    Args:
+        body (str): The body of the alert.
+        title (str, optional): The title of the alert. Defaults to "".
+    """
+    apprise_client = Apprise()
+
+    from_phone = getenv("SIGNAL_ALERT_FROM_PHONE")
+    to_phone = getenv("SIGNAL_ALERT_TO_PHONE")
+    if not from_phone or not to_phone:
+        logger.info("SIGNAL_ALERT_FROM_PHONE or SIGNAL_ALERT_TO_PHONE not set")
+        return
+
+    apprise_client.add(f"signal://localhost:8989/{from_phone}/{to_phone}")
+
+    apprise_client.notify(title=title, body=body)

python/system_tests/validate_system.py

@@ -9,7 +9,8 @@ from socket import gethostname
 
 import typer
 
-from python.common import configure_logger, signal_alert
+from python.common import configure_logger
+from python.signal_alert import signal_alert
 from python.system_tests.components import systemd_tests, zpool_tests
 
 logger = logging.getLogger(__name__)

python/tools/snapshot_manager.py

@@ -12,7 +12,8 @@ from re import search
 
 import typer
 
-from python.common import configure_logger, signal_alert, utcnow
+from python.common import configure_logger, utcnow
+from python.signal_alert import signal_alert
 from python.zfs import Dataset, get_datasets
 
 logger = logging.getLogger(__name__)

python/tools/whisper/transcribe.py

@@ -14,7 +14,7 @@ from typing import Annotated
 
 import typer
 
-from python.common import configure_logger
+from python.common import configure_logger, get_repo_dir
 
 logger = logging.getLogger(__name__)
 
@@ -24,7 +24,7 @@ class Config:
 
     image_tag = "whisper-transcribe:latest"
     model_volume = "whisper-models"
-    repo_root = Path(__file__).resolve().parents[3]
+    repo_root = get_repo_dir()
     dockerfile = Path(__file__).resolve().parent / "Dockerfile"
     huggingface_cache = "/root/.cache/huggingface"
 

systems/iso/default.nix

@@ -0,0 +1,55 @@
+{
+  outputs,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+{
+  imports = [
+    "${modulesPath}/installer/cd-dvd/installation-cd-minimal.nix"
+  ];
+
+  nixpkgs.hostPlatform = "x86_64-linux";
+
+  image.baseName = lib.mkForce "nixos-zfs-installer";
+
+  # Keep the live kernel and ZFS in sync with the deployed systems so pools
+  # created by the installer import cleanly on first boot.
+  boot = {
+    kernelPackages = pkgs.linuxPackages_6_18;
+    zfs.package = pkgs.zfs_2_4;
+    supportedFilesystems.zfs = true;
+  };
+
+  networking.hostName = "installer";
+
+  # On flake-built systems <nixpkgs> resolves through the flake registry,
+  # so nixos-install fails without these features enabled.
+  nix.settings.experimental-features = [
+    "flakes"
+    "nix-command"
+  ];
+
+  environment.systemPackages = [
+    outputs.packages.${pkgs.stdenv.hostPlatform.system}.installer-nixos
+  ];
+
+  # Live-media only: sshd is already enabled by the installation profile,
+  # but ssh logins need a non-empty password.
+  users.users = {
+    nixos = {
+      password = "nixos";
+      initialHashedPassword = lib.mkForce null;
+    };
+    root = {
+      password = "nixos";
+      initialHashedPassword = lib.mkForce null;
+    };
+  };
+
+  services.getty.helpLine = ''
+    Run "sudo nixos-installer" to install NixOS onto a ZFS root pool.
+    SSH is enabled; the "nixos" and "root" passwords are "nixos".
+  '';
+}

systems/jeeves/networking.nix

@@ -22,6 +22,11 @@
       ];
     };
     useNetworkd = true;
+    # Without this, unconfigured NICs get a catch-all DHCP network and
+    # join the LAN uninvited (how the dead atlantic card ended up
+    # ARP-poisoning 192.168.99.14). Every interface must be configured
+    # explicitly below.
+    useDHCP = false;
   };
 
   systemd.network = {
@@ -46,6 +51,20 @@
       };
     };
     networks = {
+      # Unused second port of the onboard i350; keep it down so the dead
+      # PCIe link (see dmesg "PCIe link lost") stops logging Tx errors.
+      "01-unused-igb-port" = {
+        matchConfig.Name = "enp97s0f0";
+        linkConfig.ActivationPolicy = "always-down";
+      };
+      # Aquantia 10G card (0000:21:00.0); loses PCIe link, and its
+      # unicast RX path is dead while TX still works, so when it's up it
+      # answers ARP for the host IPs and black-holes the replies. Keep
+      # it down until the card is fixed or replaced.
+      "01-unused-atlantic-port" = {
+        matchConfig.Name = "enp33s0";
+        linkConfig.ActivationPolicy = "always-down";
+      };
       "10-Primary" = {
         matchConfig.Name = "enp97s0f1";
         address = [ "192.168.99.14/24" ];

systems/jeeves/services/postgress.nix

@@ -29,6 +29,7 @@ in
       host  all  richie   ::1/128         trust
       host  all  richie   192.168.90.1/24 trust
       host  all  richie   192.168.99.1/24 trust
+      host  all  richie   172.16.0.0/12   trust
 
 
       #type database DBuser origin-address auth-method

systems/jeeves/services/systemd.nix

@@ -16,6 +16,20 @@ in
           ExecStart = "${pkgs.bash}/bin/bash ${../scripts/plex_permission.sh}";
         };
       };
+      # The unused onboard i350 port hangs ~90s after every boot ("PCIe
+      # link lost", MMIO reads return all-Fs) even with pcie_aspm=off.
+      # Remove it from the PCI bus so nothing (hwmon temperature polls,
+      # networkd) touches the dead hardware. Reversible via
+      # `echo 1 > /sys/bus/pci/rescan`.
+      remove_dead_igb_port = {
+        wantedBy = [ "multi-user.target" ];
+        description = "removes hung i350 port 0000:61:00.0 from the PCI bus";
+        unitConfig.ConditionPathExists = "/sys/bus/pci/devices/0000:61:00.0/remove";
+        serviceConfig = {
+          Type = "oneshot";
+          ExecStart = "${pkgs.bash}/bin/bash -c 'echo 1 > /sys/bus/pci/devices/0000:61:00.0/remove'";
+        };
+      };
       startup_validation = {
         requires = [ "network-online.target" ];
         after = [ "network-online.target" ];

systems/jeeves/web_services/haproxy.cfg

@@ -37,6 +37,23 @@ frontend ContentSwitching
   acl host_gitea  hdr(host) -i gitea.tmmworkshop.com
   acl host_norn_sight  hdr(host) -i www.norn-sight.com
 
+  # --- Request logging ---
+  # Capture the Host header and User-Agent so the httplog shows who is
+  # requesting what. They appear in the log's {captured|headers} field,
+  # in this order: {host|user-agent}. Client IP is already logged by httplog.
+  # Kept above the rate limiting so denied (429) requests are captured too.
+  http-request capture req.hdr(Host) len 100
+  http-request capture req.hdr(User-Agent) len 128
+
+  # --- robots.txt ---
+  # Serve a single global robots.txt for every vhost (asks crawlers to wait
+  # 10s between requests via Crawl-delay). Returned for both HTTP and HTTPS.
+  # File is deployed to /etc/haproxy/robots.txt by haproxy.nix.
+  # Kept above the rate limiting so crawlers can always read it: `return` is a
+  # terminating action, so robots.txt requests are never tracked or denied.
+  acl is_robots path /robots.txt
+  http-request return status 200 content-type "text/plain" file /etc/haproxy/robots.txt if is_robots
+
   # --- Rate limiting (Gitea only, per source IP) ---
   # Trusted devices exempt from rate limiting (add one line per IP/CIDR).
   # Internal / reserved-for-private-use ranges:
@@ -47,42 +64,25 @@ frontend ContentSwitching
   # Add specific public devices below as needed:
   # acl rate_limit_allowlist src 192.0.2.50
 
-  # Logged-in Gitea sessions bypass the rate limits. Gitea sets the
-  # `i_like_gitea` session cookie on login, and it is only sent to the Gitea
-  # vhost, so this only affects Gitea traffic. Note: this matches cookie
-  # PRESENCE, not validity, so it filters anonymous crawlers (which carry no
-  # cookie) rather than acting as a hard security boundary.
-  acl gitea_logged_in req.cook(i_like_gitea) -m found
-
   # Track HTTP request rate per client IP over a 10s sliding window. Only Gitea
   # is rate-limited; all other vhosts are left alone.
   # ipv6 table type also covers IPv4 (mapped), so it works for both binds.
   stick-table type ipv6 size 100k expire 30s store http_req_rate(10s)
-  http-request track-sc0 src if host_gitea !is_acme !rate_limit_allowlist !gitea_logged_in
+  http-request track-sc0 src if host_gitea !is_acme !rate_limit_allowlist
   # Threshold: deny (429) when a client exceeds this many requests per 10s.
-  acl over_rate_limit sc_http_req_rate(0) gt 10
+  # Kept loose (50/10s) since logged-in users are rate-limited too; a page
-  http-request deny deny_status 429 if over_rate_limit host_gitea !is_acme !rate_limit_allowlist !gitea_logged_in
+  # load can burst a few dozen asset requests.
-
+  acl over_rate_limit sc_http_req_rate(0) gt 50
-  # --- Request logging ---
+  http-request deny deny_status 429 if over_rate_limit host_gitea !is_acme !rate_limit_allowlist
-  # Capture the Host header and User-Agent so the httplog shows who is
-  # requesting what. They appear in the log's {captured|headers} field,
-  # in this order: {host|user-agent}. Client IP is already logged by httplog.
-  http-request capture req.hdr(Host) len 100
-  http-request capture req.hdr(User-Agent) len 128
-
-  # --- robots.txt ---
-  # Serve a single global robots.txt for every vhost (asks crawlers to wait
-  # 10s between requests via Crawl-delay). Returned for both HTTP and HTTPS.
-  # File is deployed to /etc/haproxy/robots.txt by haproxy.nix.
-  acl is_robots path /robots.txt
-  http-request return status 200 content-type "text/plain" file /etc/haproxy/robots.txt if is_robots
 
   # --- Per-endpoint limit: Gitea compare/diff is expensive; cap at 1 req / 5 min / IP ---
   # Tracked in a separate 5-minute table (st_compare) since a proxy has only one
   # inline stick-table. Allow-listed (internal) IPs are exempt.
-  acl is_gitea_compare path_beg /Richie/dotfiles/compare
+  # Matches /<owner>/<repo>/compare on every repo; -i because Gitea routes are
-  http-request track-sc1 src table st_compare if host_gitea is_gitea_compare !rate_limit_allowlist !gitea_logged_in
+  # case-insensitive.
-  http-request deny deny_status 429 if host_gitea is_gitea_compare !rate_limit_allowlist !gitea_logged_in { sc_http_req_rate(1,st_compare) gt 1 }
+  acl is_gitea_compare path_reg -i ^/[^/]+/[^/]+/compare
+  http-request track-sc1 src table st_compare if host_gitea is_gitea_compare !rate_limit_allowlist
+  http-request deny deny_status 429 if host_gitea is_gitea_compare !rate_limit_allowlist { sc_http_req_rate(1,st_compare) gt 1 }
 
   # Hosts allowed to serve plain HTTP (add entries to skip the HTTPS redirect)
   acl allow_http hdr(host) -i __none__
@@ -117,11 +117,9 @@ backend cache_nodes
   server server 127.0.0.1:5000
 
 backend jellyfin
-    option httpchk
+  mode http
-    option forwardfor
+  option forwardfor
-    http-check send meth GET uri /health
+  server jellyfin 127.0.0.1:8096
-    http-check expect string Healthy
-    server jellyfin 127.0.0.1:8096
 
 backend share_nodes
   mode http

tests/ebook_search/test_core.py

@@ -10,8 +10,8 @@ from types import ModuleType
 from typing import TYPE_CHECKING
 
 import pytest
-from sqlalchemy import create_engine, select
+from sqlalchemy import select
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, create_async_engine
 
 from python.ebook_search.answer import answer_query
 from python.ebook_search.bm25_corpus import (
@@ -77,10 +77,17 @@ def test_reciprocal_rank_fusion_combines_vector_and_bm25_rankings() -> None:
     assert fused[0].fused_score == fused[0].score
 
 
-def test_find_existing_source_matches_path_or_hash() -> None:
+async def build_async_engine() -> AsyncEngine:
-    engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
+    """Create an in-memory async engine with the Richie schema."""
-    RichieBase.metadata.create_all(engine)
+    engine = create_async_engine("sqlite+aiosqlite:///:memory:")
-    with sessionmaker(bind=engine, expire_on_commit=False, future=True)() as session:
+    async with engine.begin() as connection:
+        await connection.run_sync(RichieBase.metadata.create_all)
+    return engine
+
+
+async def test_find_existing_source_matches_path_or_hash() -> None:
+    engine = await build_async_engine()
+    async with AsyncSession(engine, expire_on_commit=False) as session:
         source = EbookSource(
             title="Book",
             author=None,
@@ -93,16 +100,15 @@ def test_find_existing_source_matches_path_or_hash() -> None:
             file_size=10,
         )
         session.add(source)
-        session.commit()
+        await session.commit()
 
-        assert find_existing_source(session, Path("/old/book.epub"), "b" * 64) == source
+        assert await find_existing_source(session, Path("/old/book.epub"), "b" * 64) == source
-        assert find_existing_source(session, Path("/new/book.epub"), "a" * 64) == source
+        assert await find_existing_source(session, Path("/new/book.epub"), "a" * 64) == source
 
 
-def test_bm25_corpus_uses_existing_search_text_without_duplicate_metadata() -> None:
+async def test_bm25_corpus_uses_existing_search_text_without_duplicate_metadata() -> None:
-    engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
+    engine = await build_async_engine()
-    RichieBase.metadata.create_all(engine)
+    async with AsyncSession(engine, expire_on_commit=False) as session:
-    with sessionmaker(bind=engine, expire_on_commit=False, future=True)() as session:
         source = EbookSource(
             title="Book",
             author="Author",
@@ -115,10 +121,10 @@ def test_bm25_corpus_uses_existing_search_text_without_duplicate_metadata() -> N
             file_size=10,
         )
         session.add(source)
-        session.flush()
+        await session.flush()
         chapter = EbookChapter(source_id=source.id, spine_index=0, title="Chapter", href=None)
         session.add(chapter)
-        session.flush()
+        await session.flush()
         session.add(
             EbookChunk(
                 id=1,
@@ -133,9 +139,9 @@ def test_bm25_corpus_uses_existing_search_text_without_duplicate_metadata() -> N
                 search_text="Book Author Chapter content",
             )
         )
-        session.commit()
+        await session.commit()
 
-        records, texts = fetch_bm25_corpus_records(session)
+        records, texts = await fetch_bm25_corpus_records(session)
 
     assert texts == ["Book Author Chapter content"]
     assert records[0]["chunk_id"] == 1
@@ -370,7 +376,7 @@ def test_load_bm25_corpus_raises_when_index_is_missing(mocker: MockerFixture, tm
         load_bm25_corpus.cache_clear()
 
 
-def test_ensure_bm25_corpus_refreshes_missing_index(mocker: MockerFixture) -> None:
+async def test_ensure_bm25_corpus_refreshes_missing_index(mocker: MockerFixture) -> None:
     refreshed: list[object] = []
     db_updated_at = datetime.now(tz=UTC)
 
@@ -385,12 +391,12 @@ def test_ensure_bm25_corpus_refreshes_missing_index(mocker: MockerFixture) -> No
     config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
     session = object()
 
-    ensure_bm25_corpus(session, config)
+    await ensure_bm25_corpus(session, config)
 
     assert refreshed == [(session, config, db_updated_at)]
 
 
-def test_ensure_bm25_corpus_refreshes_stale_index(mocker: MockerFixture) -> None:
+async def test_ensure_bm25_corpus_refreshes_stale_index(mocker: MockerFixture) -> None:
     refreshed: list[object] = []
     created_at = datetime(2026, 1, 1, tzinfo=UTC)
     db_updated_at = datetime(2026, 1, 2, tzinfo=UTC)
@@ -407,7 +413,7 @@ def test_ensure_bm25_corpus_refreshes_stale_index(mocker: MockerFixture) -> None
     config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
     session = object()
 
-    ensure_bm25_corpus(session, config)
+    await ensure_bm25_corpus(session, config)
 
     assert refreshed == [(session, config, db_updated_at)]
 
@@ -420,14 +426,13 @@ def test_supported_embedding_models_match_service_names() -> None:
     }
 
 
-def test_ensure_embedding_models_registers_service_names() -> None:
+async def test_ensure_embedding_models_registers_service_names() -> None:
-    engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
+    engine = await build_async_engine()
-    RichieBase.metadata.create_all(engine)
+    async with AsyncSession(engine, expire_on_commit=False) as session:
-    with sessionmaker(bind=engine, expire_on_commit=False, future=True)() as session:
+        await ensure_embedding_models(session)
-        ensure_embedding_models(session)
+        await session.commit()
-        session.commit()
 
-        models = list(session.scalars(select(EbookEmbeddingModel).order_by(EbookEmbeddingModel.name)))
+        models = list(await session.scalars(select(EbookEmbeddingModel).order_by(EbookEmbeddingModel.name)))
 
     assert [(model.name, model.dimension) for model in models] == [
         ("qwen3-embedding-0.6b", 1024),
@@ -496,10 +501,10 @@ def test_chat_api_key_falls_back_to_ollama_api_key(mocker: MockerFixture) -> Non
     assert config.vllm_api_key == "ollama-key"
 
 
-def test_answer_query_does_not_call_model_when_disabled() -> None:
+async def test_answer_query_does_not_call_model_when_disabled(mocker: MockerFixture) -> None:
     config = load_config().model_copy(update={"answer_enabled": False})
     result = SearchResult(chunk_id=1, text="source text", source_title="Book")
 
-    answer = answer_query("question", [result], config)
+    answer = await answer_query(mocker.Mock(), "question", [result], config)
 
     assert "Answer generation is disabled" in answer

tests/ebook_search/test_guardrails.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 from fastapi.testclient import TestClient
-from sqlalchemy import create_engine
+from sqlalchemy.ext.asyncio import create_async_engine
 
 from python.ebook_search.api.main import create_app
 from python.ebook_search.config import EbookSearchConfig, RerankConfig
@@ -66,8 +66,8 @@ def test_is_confident_against_threshold() -> None:
 
 def patch_app_runtime(mocker: MockerFixture):
     mocker.patch(
-        "python.ebook_search.api.main.get_postgres_engine",
+        "python.ebook_search.api.main.get_async_postgres_engine",
-        side_effect=lambda **_kwargs: create_engine("sqlite+pysqlite:///:memory:", future=True),
+        side_effect=lambda **_kwargs: create_async_engine("sqlite+aiosqlite:///:memory:"),
     )
     mocker.patch("python.ebook_search.api.main.ensure_bm25_corpus", side_effect=lambda _session, _config: None)
 
@@ -75,11 +75,12 @@ def patch_app_runtime(mocker: MockerFixture):
 def test_low_confidence_skips_answer_generation(mocker: MockerFixture) -> None:
     called = False
 
-    def fake_search_ebooks(_engine, query, _config, *, rerank=False):
+    def fake_search_ebooks(_engine, _client, query, _config, *, rerank=False, phrase_matching=False):
         del rerank
+        del phrase_matching
         return SearchResponse(query=query, rank_label="Hybrid", results=make_results(1, vector_score=0.05))
 
-    def fake_answer_query(_query, _results, _config):
+    def fake_answer_query(_client, _query, _results, _config):
         nonlocal called
         called = True
         return "answer"
@@ -104,14 +105,15 @@ def test_low_confidence_skips_answer_generation(mocker: MockerFixture) -> None:
 
 
 def test_invalid_citation_is_flagged(mocker: MockerFixture) -> None:
-    def fake_search_ebooks(_engine, query, _config, *, rerank=False):
+    def fake_search_ebooks(_engine, _client, query, _config, *, rerank=False, phrase_matching=False):
         del rerank
+        del phrase_matching
         return SearchResponse(query=query, rank_label="Hybrid", results=make_results(2, vector_score=0.9))
 
     mocker.patch("python.ebook_search.api.routes.search.search_ebooks", side_effect=fake_search_ebooks)
     mocker.patch(
         "python.ebook_search.api.routes.search.answer_query",
-        side_effect=lambda _query, _results, _config: "Per the text [9].",
+        side_effect=lambda _client, _query, _results, _config: "Per the text [9].",
     )
     patch_app_runtime(mocker)
     app = create_app()
@@ -126,14 +128,15 @@ def test_invalid_citation_is_flagged(mocker: MockerFixture) -> None:
 
 
 def test_grounded_answer_has_no_warning_badge(mocker: MockerFixture) -> None:
-    def fake_search_ebooks(_engine, query, _config, *, rerank=False):
+    def fake_search_ebooks(_engine, _client, query, _config, *, rerank=False, phrase_matching=False):
         del rerank
+        del phrase_matching
         return SearchResponse(query=query, rank_label="Hybrid", results=make_results(2, vector_score=0.9))
 
     mocker.patch("python.ebook_search.api.routes.search.search_ebooks", side_effect=fake_search_ebooks)
     mocker.patch(
         "python.ebook_search.api.routes.search.answer_query",
-        side_effect=lambda _query, _results, _config: "Grounded in [1] and [2].",
+        side_effect=lambda _client, _query, _results, _config: "Grounded in [1] and [2].",
     )
     patch_app_runtime(mocker)
     app = create_app()

tests/ebook_search/test_health.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 from fastapi.testclient import TestClient
-from sqlalchemy import create_engine
+from sqlalchemy.ext.asyncio import create_async_engine
 
 from python.ebook_search.api.main import create_app
 from python.ebook_search.config import EbookSearchConfig, RerankConfig
@@ -18,18 +18,18 @@ if TYPE_CHECKING:
 
 def fake_get_postgres_engine(**_kwargs):
     """Return an in-memory engine for route tests."""
-    return create_engine("sqlite+pysqlite:///:memory:", future=True)
+    return create_async_engine("sqlite+aiosqlite:///:memory:")
 
 
 def patch_app_runtime(mocker: MockerFixture):
-    mocker.patch("python.ebook_search.api.main.get_postgres_engine", side_effect=fake_get_postgres_engine)
+    mocker.patch("python.ebook_search.api.main.get_async_postgres_engine", side_effect=fake_get_postgres_engine)
     mocker.patch("python.ebook_search.api.main.ensure_bm25_corpus", side_effect=lambda _session, _config: None)
 
 
 def patch_dependencies(mocker: MockerFixture, *, database=True, embedding=True, chat=True, bm25="ok"):
     mocker.patch(f"{HEALTH_MODULE}.check_database", side_effect=lambda _session: database)
-    mocker.patch(f"{HEALTH_MODULE}.check_embedding_endpoint", side_effect=lambda _config: embedding)
+    mocker.patch(f"{HEALTH_MODULE}.check_embedding_endpoint", side_effect=lambda _client, _config: embedding)
-    mocker.patch(f"{HEALTH_MODULE}.check_chat_endpoint", side_effect=lambda _config: chat)
+    mocker.patch(f"{HEALTH_MODULE}.check_chat_endpoint", side_effect=lambda _client, _config: chat)
     mocker.patch(f"{HEALTH_MODULE}.check_bm25_status", side_effect=lambda _config: bm25)
 
 

tests/ebook_search/test_http.py

@@ -16,7 +16,14 @@ if TYPE_CHECKING:
     from pytest_mock import MockerFixture
 
 
-def test_answer_query_uses_httpx_chat_completions(mocker: MockerFixture) -> None:
+def make_async_client(mocker: MockerFixture, fake_post) -> httpx.AsyncClient:
+    """Build a mock async client whose post call is served by fake_post."""
+    client = mocker.MagicMock(spec=httpx.AsyncClient)
+    client.post = mocker.AsyncMock(side_effect=fake_post)
+    return client
+
+
+async def test_answer_query_uses_httpx_chat_completions(mocker: MockerFixture) -> None:
     captured: dict[str, object] = {}
 
     def fake_post(url: str, **kwargs: object) -> httpx.Response:
@@ -28,7 +35,7 @@ def test_answer_query_uses_httpx_chat_completions(mocker: MockerFixture) -> None
             request=httpx.Request("POST", url),
         )
 
-    mocker.patch.object(httpx, "post", side_effect=fake_post)
+    client = make_async_client(mocker, fake_post)
     config = EbookSearchConfig(
         rerank=RerankConfig(enabled=False),
         vllm_base_url="https://ollama.com/v1",
@@ -36,7 +43,8 @@ def test_answer_query_uses_httpx_chat_completions(mocker: MockerFixture) -> None
         chat_model="deepseek-v4-flash",
     )
 
-    answer = answer_query("question", [SearchResult(chunk_id=1, text="source", source_title="Book")], config)
+    results = [SearchResult(chunk_id=1, text="source", source_title="Book")]
+    answer = await answer_query(client, "question", results, config)
 
     assert answer == "grounded answer"
     assert captured["url"] == "https://ollama.com/v1/chat/completions"
@@ -48,7 +56,7 @@ def test_answer_query_uses_httpx_chat_completions(mocker: MockerFixture) -> None
     assert payload["model"] == "deepseek-v4-flash"
 
 
-def test_embed_texts_uses_httpx_embeddings(mocker: MockerFixture) -> None:
+async def test_embed_texts_uses_httpx_embeddings(mocker: MockerFixture) -> None:
     captured: dict[str, object] = {}
     vector = [0.0] * 1024
 
@@ -61,14 +69,14 @@ def test_embed_texts_uses_httpx_embeddings(mocker: MockerFixture) -> None:
             request=httpx.Request("POST", url),
         )
 
-    mocker.patch.object(httpx, "post", side_effect=fake_post)
+    client = make_async_client(mocker, fake_post)
     config = EbookSearchConfig(
         rerank=RerankConfig(enabled=False),
         embedding_base_url="http://bob:8000/v1",
         embedding_model="qwen3-embedding-0.6b",
     )
 
-    embeddings = embed_texts(["hello"], config)
+    embeddings = await embed_texts(client, ["hello"], config)
 
     assert embeddings == [vector]
     assert captured["url"] == "http://bob:8000/v1/embeddings"
@@ -78,12 +86,12 @@ def test_embed_texts_uses_httpx_embeddings(mocker: MockerFixture) -> None:
     assert kwargs["json"] == {"model": "qwen3-embedding-0.6b", "input": ["hello"]}
 
 
-def test_embed_texts_rejects_bad_response_shape(mocker: MockerFixture) -> None:
+async def test_embed_texts_rejects_bad_response_shape(mocker: MockerFixture) -> None:
     def fake_post(url: str, **_kwargs: object) -> httpx.Response:
         return httpx.Response(200, json={"data": [{}]}, request=httpx.Request("POST", url))
 
-    mocker.patch.object(httpx, "post", side_effect=fake_post)
+    client = make_async_client(mocker, fake_post)
     config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
 
     with pytest.raises(RuntimeError, match="Embedding request failed"):
-        embed_texts(["hello"], config)
+        await embed_texts(client, ["hello"], config)

tests/ebook_search/test_protected_phrases.py

@@ -0,0 +1,957 @@
+"""Tests for protected phrase extraction and matching."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING
+
+import pytest
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, create_async_engine
+from sqlalchemy.pool import StaticPool
+
+from python.ebook_search.config import EbookSearchConfig
+from python.ebook_search.protected_phrases.config import (
+    get_bad_ends,
+    get_most_common_words,
+)
+from python.ebook_search.protected_phrases.extraction import (
+    extract_capitalized_phrases,
+    is_junk_phrase,
+    score_candidate,
+)
+from python.ebook_search.protected_phrases.generate_ngrams import (
+    generate_candidate_phrases_for_books,
+    recalculate_candidate_phrases_for_book,
+)
+from python.ebook_search.protected_phrases.judge_ngrams import (
+    judge_candidate_phrases_for_books,
+    prepare_book_judgment,
+)
+from python.ebook_search.protected_phrases.matching import (
+    detect_protected_phrases_for_query,
+    index_chunk_phrase_mentions,
+    load_phrase_lookup,
+    phrase_hit_counts_for_chunks,
+    phrase_hits_for_chunks,
+    resolve_overlaps,
+)
+from python.ebook_search.protected_phrases.models import (
+    ChunkPhraseHit,
+    HydratedPhraseMatch,
+    LLMJudgment,
+    PhraseCandidate,
+)
+from python.ebook_search.protected_phrases.store import book_ids_pending_first_judgment, corpus_phrase_stats
+from python.ebook_search.protected_phrases.text_normalization import normalize_text, tokenize
+from python.orm.richie import (
+    EbookCandidatePhrase,
+    EbookChunk,
+    EbookChunkPhraseMention,
+    EbookPhraseAlias,
+    EbookProtectedPhrase,
+    EbookSource,
+    RichieBase,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncGenerator
+
+    from pytest_mock import MockerFixture
+
+
+@pytest.fixture
+async def engine() -> AsyncGenerator[AsyncEngine]:
+    """Create a shared in-memory async database engine for phrase tests."""
+    test_engine = create_async_engine(
+        "sqlite+aiosqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    async with test_engine.begin() as connection:
+        await connection.run_sync(RichieBase.metadata.create_all)
+    yield test_engine
+    await test_engine.dispose()
+
+
+@pytest.fixture
+async def session(engine: AsyncEngine) -> AsyncGenerator[AsyncSession]:
+    """Provide a session on the shared in-memory database."""
+    async with AsyncSession(engine, expire_on_commit=False) as test_session:
+        yield test_session
+
+
+@pytest.fixture
+def config() -> EbookSearchConfig:
+    """Provide default phrase-tuning settings for tests."""
+    return EbookSearchConfig()
+
+
+def test_normalize_text_preserves_phrase_stopwords_and_word_order() -> None:
+    """Normalization should not collapse protected phrases by removing stopwords."""
+    assert normalize_text("Haden\u2019s syndrome -- lock-in!") == "haden's syndrome lock in"
+    assert tokenize("House of the Dragon") == ["house", "of", "the", "dragon"]
+
+
+def test_capitalized_phrase_keeps_bad_end_tokens_for_downranking(config: EbookSearchConfig) -> None:
+    """Capitalized extraction should keep dangling endings so scoring can downrank them."""
+    candidates = extract_capitalized_phrases("Damion Montgomery and the left.", config)
+
+    assert "damion montgomery and the" in candidates
+    assert candidates["damion montgomery and the"].phrase_text == "Damion Montgomery and the"
+    assert candidates["damion montgomery and the"].phrase_norm.split()[-1] in get_bad_ends()
+
+
+def test_is_junk_phrase_rejects_dialogue_verbs_contractions_and_majority_common() -> None:
+    """Phrases the judge never keeps should be caught lexically before any LLM call."""
+    assert is_junk_phrase(["vann", "said"])
+    assert is_junk_phrase(["shook", "her", "head"])
+    assert is_junk_phrase(["i'm", "going"])
+    assert is_junk_phrase(["don't", "know"])
+    assert is_junk_phrase(["of", "the", "vault"])
+
+
+def test_is_junk_phrase_keeps_possessives_and_half_common_world_terms() -> None:
+    """Proper-noun possessives and phrasal world terms must survive the junk filter."""
+    assert not is_junk_phrase(["chapman's", "death"])
+    assert not is_junk_phrase(["boston", "bays'"])
+    assert not is_junk_phrase(["lock", "in"])
+    assert not is_junk_phrase(["data", "feed"])
+    assert not is_junk_phrase(["haden's", "syndrome"])
+
+
+def test_score_candidate_rewards_multiple_non_raw_sources(config: EbookSearchConfig) -> None:
+    """A second non-raw source should add its weight plus the multi-source bonus."""
+    single_source = PhraseCandidate(
+        phrase_text="lock in",
+        phrase_norm="lock in",
+        token_count=2,
+        source_capitalized=True,
+        raw_count=3,
+        chapter_count=2,
+    )
+    multi_source = PhraseCandidate(
+        phrase_text="lock in",
+        phrase_norm="lock in",
+        token_count=2,
+        source_capitalized=True,
+        source_yake=True,
+        raw_count=3,
+        chapter_count=2,
+    )
+
+    assert score_candidate(multi_source, config) == score_candidate(single_source, config) + 2.0 + 2.0
+
+
+def test_score_candidate_caps_frequency_contribution(config: EbookSearchConfig) -> None:
+    """Very frequent raw-only phrases should no longer out-score sourced entities."""
+    frequent_raw_only = PhraseCandidate(
+        phrase_text="mage king",
+        phrase_norm="mage king",
+        token_count=2,
+        source_raw_ngram=True,
+        raw_count=1000,
+        chapter_count=100,
+    )
+
+    assert score_candidate(frequent_raw_only, config) == 0.5 + 2.0 + 0.5
+
+
+def test_score_candidate_weights_metadata_source(config: EbookSearchConfig) -> None:
+    """Title and series phrases should get credit for the metadata source."""
+    metadata_only = PhraseCandidate(
+        phrase_text="lock in",
+        phrase_norm="lock in",
+        token_count=2,
+        source_metadata=True,
+    )
+
+    assert score_candidate(metadata_only, config) == 2.0 + 0.5
+
+
+async def test_detect_protected_phrases_hydrates_alias_matches(
+    session: AsyncSession,
+    config: EbookSearchConfig,
+) -> None:
+    """Query detection should use RAM aliases and hydrate phrase metadata from the DB."""
+    source = await add_source(session)
+    phrase = await add_phrase(session, source.id, phrase_text="lock in", phrase_norm="lock in")
+    session.add(EbookPhraseAlias(phrase_id=phrase.id, alias_text="locked in", alias_norm="locked in"))
+    await session.commit()
+
+    matches = await detect_protected_phrases_for_query(session, "what is locked-in", config)
+
+    assert [(match.phrase_text, match.canonical_id, match.phrase_type) for match in matches] == [
+        ("lock in", "condition:lock_in", "fictional_condition")
+    ]
+
+
+def test_resolve_overlaps_keeps_independent_nested_phrases() -> None:
+    """Overlap resolution should keep useful nested concepts when metadata permits it."""
+    child = hydrated_match(
+        phrase_id=1,
+        phrase_text="mage king",
+        canonical_id="title:mage_king",
+        start_token=3,
+        end_token=5,
+        allow_nested=True,
+    )
+    parent = hydrated_match(
+        phrase_id=2,
+        phrase_text="mage king of mars",
+        canonical_id="entity:mage_king_of_mars",
+        start_token=3,
+        end_token=7,
+        suppress_children=False,
+    )
+
+    assert [match.phrase_text for match in resolve_overlaps([child, parent])] == [
+        "mage king of mars",
+        "mage king",
+    ]
+
+
+def test_resolve_overlaps_suppresses_weaker_same_canonical_match() -> None:
+    """Same-canonical overlaps should keep the stronger evidence."""
+    weak = hydrated_match(
+        phrase_id=1,
+        phrase_text="lock",
+        canonical_id="condition:lock_in",
+        start_token=2,
+        end_token=3,
+        importance=0.2,
+    )
+    strong = hydrated_match(
+        phrase_id=2,
+        phrase_text="lock in",
+        canonical_id="condition:lock_in",
+        start_token=2,
+        end_token=4,
+        importance=0.9,
+    )
+
+    assert [match.phrase_text for match in resolve_overlaps([weak, strong])] == ["lock in"]
+
+
+async def test_index_chunk_phrase_mentions_uses_normalized_window_lookup(
+    session: AsyncSession,
+    config: EbookSearchConfig,
+) -> None:
+    """Chunk indexing should store mention rows without scanning all phrases at query time."""
+    source = await add_source(session)
+    phrase = await add_phrase(session, source.id, phrase_text="lock in", phrase_norm="lock in")
+    chunk = EbookChunk(
+        id=1,
+        source_id=source.id,
+        chapter_id=None,
+        chunk_index=0,
+        text="Victims experienced lock-in during the crisis.",
+        token_start=0,
+        token_count=7,
+        page_label=None,
+        content_sha256="b" * 64,
+        search_text="Victims experienced lock-in during the crisis.",
+    )
+    session.add(chunk)
+    await session.commit()
+    lookup = await load_phrase_lookup(session, config, book_id=source.id)
+
+    count = await index_chunk_phrase_mentions(session, chunk, lookup=lookup)
+    await session.commit()
+
+    mention = await session.scalar(select(EbookChunkPhraseMention))
+    assert count == 1
+    assert mention is not None
+    assert mention.chunk_id == chunk.id
+    assert mention.phrase_id == phrase.id
+    assert chunk.text[mention.start_char : mention.end_char] == "lock-in"
+    assert await phrase_hit_counts_for_chunks(session, chunk_ids=[chunk.id], phrase_ids=[phrase.id]) == {chunk.id: 1}
+    assert await phrase_hits_for_chunks(session, chunk_ids=[chunk.id], phrase_ids=[phrase.id]) == {
+        chunk.id: (ChunkPhraseHit(phrase_id=phrase.id, phrase_text="lock in", mention_count=1),)
+    }
+
+
+async def test_generate_candidate_phrases_for_books_stores_unjudged_candidates(
+    session: AsyncSession,
+    config: EbookSearchConfig,
+) -> None:
+    """Candidate generation should populate the per-book phrase list without calling the LLM."""
+    source = await add_source(session)
+    session.add(
+        EbookChunk(
+            id=1,
+            source_id=source.id,
+            chapter_id=None,
+            chunk_index=0,
+            text="lock in lock in lock in",
+            token_start=0,
+            token_count=6,
+            page_label=None,
+            content_sha256="d" * 64,
+            search_text="lock in lock in lock in",
+        )
+    )
+    await session.commit()
+    build_config = config.model_copy(
+        update={
+            "protected_phrase_max_candidates_per_book": 1,
+            "phrase_min_tokens": 2,
+            "phrase_max_tokens": 2,
+        }
+    )
+
+    result = await generate_candidate_phrases_for_books(session, build_config)
+    await session.commit()
+
+    candidate = await session.scalar(select(EbookCandidatePhrase))
+    assert result.books_seen == 1
+    assert result.books_built == 1
+    assert result.candidate_phrases == 1
+    assert candidate is not None
+    assert candidate.phrase_norm == "lock in"
+    assert candidate.llm_judged is False
+    assert await session.scalar(select(EbookProtectedPhrase)) is None
+
+
+async def test_generate_candidate_phrases_for_books_filters_one_token_and_one_use_candidates(
+    session: AsyncSession,
+    config: EbookSearchConfig,
+) -> None:
+    """Candidate generation should not persist one-token or single-use phrases."""
+    source = await add_source(session)
+    session.add(
+        EbookChunk(
+            id=1,
+            source_id=source.id,
+            chapter_id=None,
+            chunk_index=0,
+            text=(
+                "Damion walked away. Damion woke up. rare phrase appeared once. "
+                "and then and then. lock in lock in lock in."
+            ),
+            token_start=0,
+            token_count=16,
+            page_label=None,
+            content_sha256="i" * 64,
+            search_text=(
+                "Damion walked away. Damion woke up. rare phrase appeared once. "
+                "and then and then. lock in lock in lock in."
+            ),
+        )
+    )
+    session.add_all(
+        [
+            EbookCandidatePhrase(
+                book_id=source.id,
+                series_id=None,
+                phrase_text="Damion",
+                phrase_norm="damion",
+                token_count=1,
+                source_capitalized=True,
+                raw_count=2,
+                chapter_count=1,
+                candidate_score=10.0,
+                llm_judged=False,
+            ),
+            EbookCandidatePhrase(
+                book_id=source.id,
+                series_id=None,
+                phrase_text="rare phrase",
+                phrase_norm="rare phrase",
+                token_count=2,
+                source_raw_ngram=True,
+                raw_count=1,
+                chapter_count=1,
+                candidate_score=9.0,
+                llm_judged=False,
+            ),
+        ]
+    )
+    await session.commit()
+    build_config = config.model_copy(
+        update={
+            "protected_phrase_max_candidates_per_book": 50,
+            "phrase_min_tokens": 2,
+            "phrase_max_tokens": 2,
+            "phrase_raw_ngram_min_count": 2,
+        }
+    )
+
+    result = await generate_candidate_phrases_for_books(session, build_config)
+    await session.commit()
+
+    candidates = list(await session.scalars(select(EbookCandidatePhrase)))
+    phrase_norms = {candidate.phrase_norm for candidate in candidates}
+    common_words = get_most_common_words()
+    assert result.candidate_phrases == len(candidates)
+    assert "lock in" in phrase_norms
+    assert "damion" not in phrase_norms
+    assert "rare phrase" not in phrase_norms
+    assert "and then" not in phrase_norms
+    assert {"and", "then"}.issubset(common_words)
+    assert all(candidate.token_count >= 2 for candidate in candidates)
+    assert all(candidate.raw_count >= 2 for candidate in candidates)
+    assert all(not all(token in common_words for token in candidate.phrase_norm.split()) for candidate in candidates)
+
+
+async def test_generate_candidate_phrases_for_books_commits_after_each_book(
+    session: AsyncSession,
+    config: EbookSearchConfig,
+    mocker: MockerFixture,
+) -> None:
+    """Candidate generation should persist each completed book independently."""
+    first = await add_source(session)
+    second = await add_source(session, file_path="/library/book-2.epub", file_sha256="z" * 64)
+    session.add_all(
+        [
+            EbookChunk(
+                id=1,
+                source_id=first.id,
+                chapter_id=None,
+                chunk_index=0,
+                text="lock in lock in lock in",
+                token_start=0,
+                token_count=6,
+                page_label=None,
+                content_sha256="f" * 64,
+                search_text="lock in lock in lock in",
+            ),
+            EbookChunk(
+                id=2,
+                source_id=second.id,
+                chapter_id=None,
+                chunk_index=0,
+                text="mage king mage king mage king",
+                token_start=0,
+                token_count=6,
+                page_label=None,
+                content_sha256="g" * 64,
+                search_text="mage king mage king mage king",
+            ),
+        ]
+    )
+    await session.commit()
+    commit_spy = mocker.spy(session, "commit")
+    build_config = config.model_copy(
+        update={
+            "protected_phrase_max_candidates_per_book": 1,
+            "phrase_min_tokens": 2,
+            "phrase_max_tokens": 2,
+        }
+    )
+
+    result = await generate_candidate_phrases_for_books(session, build_config)
+
+    assert result.books_built == 2
+    assert result.candidate_phrases == 2
+    assert commit_spy.call_count == 2
+
+
+async def test_judge_candidate_phrases_for_books_promotes_stored_candidates(
+    engine: AsyncEngine,
+    session: AsyncSession,
+    config: EbookSearchConfig,
+    mocker: MockerFixture,
+) -> None:
+    """Judging should work from existing candidate rows and retain the candidate history."""
+    source = await add_source(session)
+    session.add(
+        EbookChunk(
+            id=1,
+            source_id=source.id,
+            chapter_id=None,
+            chunk_index=0,
+            text="lock in lock in lock in",
+            token_start=0,
+            token_count=6,
+            page_label=None,
+            content_sha256="e" * 64,
+            search_text="lock in lock in lock in",
+        )
+    )
+    await session.commit()
+    build_config = config.model_copy(
+        update={
+            "protected_phrase_max_candidates_per_book": 1,
+            "protected_phrase_llm_candidates_per_book": 1,
+            "phrase_min_tokens": 2,
+            "phrase_max_tokens": 2,
+            "phrase_judge_book_workers": 1,
+            "phrase_judge_phrase_workers": 1,
+        }
+    )
+    await generate_candidate_phrases_for_books(session, build_config)
+    await session.commit()
+    mocker.patch(
+        "python.ebook_search.protected_phrases.judge_ngrams.judge_candidate_async",
+        return_value=LLMJudgment(
+            keep=True,
+            canonical="lock in",
+            category="fictional_condition",
+            aliases=(),
+            confidence=0.95,
+            importance=0.9,
+        ),
+    )
+
+    result = await judge_candidate_phrases_for_books(engine, build_config)
+
+    session.expire_all()
+    candidate = await session.scalar(select(EbookCandidatePhrase))
+    phrase = await session.scalar(select(EbookProtectedPhrase))
+    mention = await session.scalar(select(EbookChunkPhraseMention))
+    assert result.books_seen == 1
+    assert result.books_judged == 1
+    assert result.candidates_judged == 1
+    assert result.protected_phrases == 1
+    assert candidate is not None
+    assert candidate.llm_judged is True
+    assert candidate.llm_keep is True
+    assert phrase is not None
+    assert phrase.source_candidate_id == candidate.id
+    assert mention is not None
+    assert mention.phrase_id == phrase.id
+
+
+async def test_judge_candidate_phrases_for_books_logs_and_continues_after_book_failure(
+    engine: AsyncEngine,
+    session: AsyncSession,
+    config: EbookSearchConfig,
+    mocker: MockerFixture,
+) -> None:
+    """A failing book should be rolled back and logged while later books are still judged and committed."""
+    first = await add_source(session)
+    second = await add_source(session, file_path="/library/book-2.epub", file_sha256="z" * 64)
+    session.add_all(
+        [
+            EbookChunk(
+                id=1,
+                source_id=first.id,
+                chapter_id=None,
+                chunk_index=0,
+                text="lock in lock in lock in",
+                token_start=0,
+                token_count=6,
+                page_label=None,
+                content_sha256="f" * 64,
+                search_text="lock in lock in lock in",
+            ),
+            EbookChunk(
+                id=2,
+                source_id=second.id,
+                chapter_id=None,
+                chunk_index=0,
+                text="mage king mage king mage king",
+                token_start=0,
+                token_count=6,
+                page_label=None,
+                content_sha256="g" * 64,
+                search_text="mage king mage king mage king",
+            ),
+        ]
+    )
+    await session.commit()
+    build_config = config.model_copy(
+        update={
+            "protected_phrase_max_candidates_per_book": 1,
+            "protected_phrase_llm_candidates_per_book": 1,
+            "phrase_min_tokens": 2,
+            "phrase_max_tokens": 2,
+            "phrase_judge_book_workers": 1,
+            "phrase_judge_phrase_workers": 1,
+        }
+    )
+    await generate_candidate_phrases_for_books(session, build_config)
+    await session.commit()
+
+    def judge_or_fail(_client: object, _config: EbookSearchConfig, candidate: PhraseCandidate) -> LLMJudgment:
+        if candidate.phrase_norm == "lock in":
+            message = "llm judge unavailable"
+            raise RuntimeError(message)
+        return LLMJudgment(
+            keep=True,
+            canonical="mage king",
+            category="title",
+            aliases=(),
+            confidence=0.95,
+            importance=0.9,
+        )
+
+    mocker.patch(
+        "python.ebook_search.protected_phrases.judge_ngrams.judge_candidate_async",
+        side_effect=judge_or_fail,
+    )
+
+    second_book_id = second.id
+
+    result = await judge_candidate_phrases_for_books(engine, build_config)
+
+    session.expire_all()
+    assert result.books_seen == 2
+    assert result.books_judged == 1
+    assert result.books_failed == 1
+    assert result.protected_phrases == 1
+    phrase = await session.scalar(select(EbookProtectedPhrase))
+    assert phrase is not None
+    assert phrase.book_id == second_book_id
+
+
+async def test_judge_candidate_phrases_for_books_skips_unstorable_existing_candidates(
+    engine: AsyncEngine,
+    session: AsyncSession,
+    config: EbookSearchConfig,
+    mocker: MockerFixture,
+) -> None:
+    """Old one-token or single-use candidate rows should not be judged or promoted."""
+    source = await add_source(session)
+    session.add(
+        EbookChunk(
+            id=1,
+            source_id=source.id,
+            chapter_id=None,
+            chunk_index=0,
+            text="Damion Damion rare phrase",
+            token_start=0,
+            token_count=4,
+            page_label=None,
+            content_sha256="j" * 64,
+            search_text="Damion Damion rare phrase",
+        )
+    )
+    session.add_all(
+        [
+            EbookCandidatePhrase(
+                book_id=source.id,
+                series_id=None,
+                phrase_text="Damion",
+                phrase_norm="damion",
+                token_count=1,
+                source_capitalized=True,
+                raw_count=2,
+                chapter_count=1,
+                candidate_score=10.0,
+                llm_judged=False,
+            ),
+            EbookCandidatePhrase(
+                book_id=source.id,
+                series_id=None,
+                phrase_text="rare phrase",
+                phrase_norm="rare phrase",
+                token_count=2,
+                source_raw_ngram=True,
+                raw_count=1,
+                chapter_count=1,
+                candidate_score=9.0,
+                llm_judged=False,
+            ),
+        ]
+    )
+    await session.commit()
+    judge_mock = mocker.patch("python.ebook_search.protected_phrases.judge_ngrams.judge_candidate_async")
+    build_config = config.model_copy(
+        update={
+            "phrase_judge_book_workers": 1,
+            "phrase_judge_phrase_workers": 1,
+        }
+    )
+
+    result = await judge_candidate_phrases_for_books(engine, build_config)
+
+    assert result.books_judged == 0
+    assert result.candidates_judged == 0
+    assert result.protected_phrases == 0
+    judge_mock.assert_not_called()
+    assert await session.scalar(select(EbookProtectedPhrase)) is None
+
+
+async def test_prepare_book_judgment_skips_stored_junk_and_rescores_stale_rows(
+    engine: AsyncEngine,
+    session: AsyncSession,
+    config: EbookSearchConfig,
+) -> None:
+    """Judgment selection should drop junk rows and rank by fresh scores, not stored ones."""
+    source = await add_source(session)
+    session.add(
+        EbookChunk(
+            id=1,
+            source_id=source.id,
+            chapter_id=None,
+            chunk_index=0,
+            text="lock in lock in lock in",
+            token_start=0,
+            token_count=6,
+            page_label=None,
+            content_sha256="k" * 64,
+            search_text="lock in lock in lock in",
+        )
+    )
+    session.add_all(
+        [
+            EbookCandidatePhrase(
+                book_id=source.id,
+                series_id=None,
+                phrase_text="vann said",
+                phrase_norm="vann said",
+                token_count=2,
+                source_raw_ngram=True,
+                raw_count=50,
+                chapter_count=10,
+                candidate_score=10.0,
+                llm_judged=False,
+            ),
+            EbookCandidatePhrase(
+                book_id=source.id,
+                series_id=None,
+                phrase_text="boring pair",
+                phrase_norm="boring pair",
+                token_count=2,
+                source_raw_ngram=True,
+                raw_count=50,
+                chapter_count=10,
+                candidate_score=9.0,
+                llm_judged=False,
+            ),
+            EbookCandidatePhrase(
+                book_id=source.id,
+                series_id=None,
+                phrase_text="lock in",
+                phrase_norm="lock in",
+                token_count=2,
+                source_capitalized=True,
+                source_yake=True,
+                raw_count=3,
+                chapter_count=2,
+                candidate_score=1.0,
+                llm_judged=False,
+            ),
+        ]
+    )
+    await session.commit()
+    build_config = config.model_copy(
+        update={
+            "protected_phrase_llm_candidates_per_book": 2,
+            "phrase_min_tokens": 2,
+            "phrase_max_tokens": 2,
+        }
+    )
+
+    prepared = await prepare_book_judgment(engine, source.id, build_config)
+
+    assert prepared is not None
+    work_items, target_remaining = prepared
+    judged_norms = [candidate.phrase_norm for _, candidate in work_items]
+    assert judged_norms == ["lock in", "boring pair"]
+    assert target_remaining == build_config.phrase_target_protected_per_book
+    fresh_scores = [candidate.candidate_score for _, candidate in work_items]
+    assert fresh_scores == sorted(fresh_scores, reverse=True)
+    assert fresh_scores[0] > 1.0
+
+
+async def test_recalculate_candidate_phrases_for_book_removes_old_phrase_data(
+    session: AsyncSession,
+    config: EbookSearchConfig,
+) -> None:
+    """Book-level recalculation should clear stale candidates, protected phrases, aliases, and mentions."""
+    source = await add_source(session)
+    session.add(
+        EbookChunk(
+            id=1,
+            source_id=source.id,
+            chapter_id=None,
+            chunk_index=0,
+            text="new phrase new phrase new phrase",
+            token_start=0,
+            token_count=6,
+            page_label=None,
+            content_sha256="h" * 64,
+            search_text="new phrase new phrase new phrase",
+        )
+    )
+    old_candidate = EbookCandidatePhrase(
+        book_id=source.id,
+        series_id=None,
+        phrase_text="old phrase",
+        phrase_norm="old phrase",
+        token_count=2,
+        source_raw_ngram=True,
+        raw_count=1,
+        chapter_count=1,
+        candidate_score=1.0,
+        llm_judged=False,
+    )
+    session.add(old_candidate)
+    await session.flush()
+    old_phrase = await add_phrase(session, source.id, phrase_text="old phrase", phrase_norm="old phrase")
+    old_phrase.source_candidate_id = old_candidate.id
+    session.add(EbookPhraseAlias(phrase_id=old_phrase.id, alias_text="old alias", alias_norm="old alias"))
+    session.add(
+        EbookChunkPhraseMention(
+            chunk_id=1,
+            phrase_id=old_phrase.id,
+            book_id=source.id,
+            series_id=None,
+            start_char=0,
+            end_char=10,
+        )
+    )
+    await session.commit()
+    build_config = config.model_copy(
+        update={
+            "protected_phrase_max_candidates_per_book": 1,
+            "phrase_min_tokens": 2,
+            "phrase_max_tokens": 2,
+        }
+    )
+
+    result = await recalculate_candidate_phrases_for_book(session, source, build_config)
+
+    candidates = list(await session.scalars(select(EbookCandidatePhrase)))
+    assert result.deleted_candidates == 1
+    assert result.deleted_protected_phrases == 1
+    assert result.deleted_aliases == 1
+    assert result.deleted_mentions == 1
+    assert result.candidate_phrases == 1
+    assert [candidate.phrase_norm for candidate in candidates] == ["new phrase"]
+    assert await session.scalar(select(EbookProtectedPhrase)) is None
+    assert await session.scalar(select(EbookPhraseAlias)) is None
+    assert await session.scalar(select(EbookChunkPhraseMention)) is None
+
+
+async def test_corpus_phrase_stats_counts_phrases_and_book_coverage(session: AsyncSession) -> None:
+    """Corpus stats should count phrases plus how many books are generated and fully judged."""
+    unjudged_book = await add_source(session)
+    judged_book = await add_source(session, file_path="/library/judged.epub", file_sha256="b" * 64)
+    mixed_book = await add_source(session, file_path="/library/mixed.epub", file_sha256="c" * 64)
+    await add_source(session, file_path="/library/empty.epub", file_sha256="d" * 64)
+    await add_candidate(session, unjudged_book.id, phrase_norm="rare phrase", llm_judged=False)
+    await add_candidate(session, judged_book.id, phrase_norm="lock in", llm_judged=True)
+    await add_candidate(session, mixed_book.id, phrase_norm="haden's syndrome", llm_judged=True)
+    await add_candidate(session, mixed_book.id, phrase_norm="boston bays", llm_judged=False)
+    await add_phrase(session, judged_book.id, phrase_text="Lock In", phrase_norm="lock in")
+
+    stats = await corpus_phrase_stats(session)
+
+    assert stats.total_books == 4
+    assert stats.books_with_candidates == 3
+    assert stats.books_fully_judged == 1
+    assert stats.candidate_phrases == 4
+    assert stats.judged_candidates == 2
+    assert stats.unjudged_candidates == 2
+    assert stats.protected_phrases == 1
+
+
+async def test_book_ids_pending_first_judgment_returns_only_never_judged_books(session: AsyncSession) -> None:
+    """Only books whose candidates are all unjudged should be pending a first judgment."""
+    unjudged_book = await add_source(session)
+    judged_book = await add_source(session, file_path="/library/judged.epub", file_sha256="b" * 64)
+    mixed_book = await add_source(session, file_path="/library/mixed.epub", file_sha256="c" * 64)
+    await add_candidate(session, unjudged_book.id, phrase_norm="rare phrase", llm_judged=False)
+    await add_candidate(session, judged_book.id, phrase_norm="lock in", llm_judged=True)
+    await add_candidate(session, mixed_book.id, phrase_norm="haden's syndrome", llm_judged=True)
+    await add_candidate(session, mixed_book.id, phrase_norm="boston bays", llm_judged=False)
+
+    assert await book_ids_pending_first_judgment(session) == [unjudged_book.id]
+
+
+async def add_candidate(
+    session: AsyncSession,
+    book_id: int,
+    *,
+    phrase_norm: str,
+    llm_judged: bool,
+) -> EbookCandidatePhrase:
+    """Add a minimal candidate phrase row."""
+    candidate = EbookCandidatePhrase(
+        book_id=book_id,
+        series_id=None,
+        phrase_text=phrase_norm,
+        phrase_norm=phrase_norm,
+        token_count=len(phrase_norm.split()),
+        source_raw_ngram=True,
+        raw_count=3,
+        chapter_count=2,
+        candidate_score=5.0,
+        llm_judged=llm_judged,
+    )
+    session.add(candidate)
+    await session.flush()
+    return candidate
+
+
+async def add_source(
+    session: AsyncSession,
+    *,
+    file_path: str = "/library/book.epub",
+    file_sha256: str = "a" * 64,
+) -> EbookSource:
+    """Add a minimal ebook source."""
+    source = EbookSource(
+        title="Book",
+        author="Author",
+        language=None,
+        publisher=None,
+        identifier=None,
+        file_path=file_path,
+        file_sha256=file_sha256,
+        file_mtime=datetime.now(tz=UTC),
+        file_size=10,
+    )
+    session.add(source)
+    await session.flush()
+    return source
+
+
+async def add_phrase(
+    session: AsyncSession,
+    book_id: int,
+    *,
+    phrase_text: str,
+    phrase_norm: str,
+) -> EbookProtectedPhrase:
+    """Add a protected phrase row."""
+    phrase = EbookProtectedPhrase(
+        book_id=book_id,
+        series_id=None,
+        phrase_text=phrase_text,
+        phrase_norm=phrase_norm,
+        canonical_id="condition:lock_in",
+        phrase_type="fictional_condition",
+        token_count=len(phrase_norm.split()),
+        confidence=0.91,
+        importance=0.85,
+        allow_nested=False,
+        suppress_children=True,
+        source_candidate_id=None,
+    )
+    session.add(phrase)
+    await session.flush()
+    return phrase
+
+
+def hydrated_match(
+    *,
+    phrase_id: int,
+    phrase_text: str,
+    canonical_id: str,
+    start_token: int,
+    end_token: int,
+    importance: float = 0.8,
+    allow_nested: bool = False,
+    suppress_children: bool = True,
+) -> HydratedPhraseMatch:
+    """Build a hydrated match for overlap tests."""
+    return HydratedPhraseMatch(
+        phrase_id=phrase_id,
+        matched_norm=phrase_text,
+        phrase_text=phrase_text,
+        phrase_norm=phrase_text,
+        canonical_id=canonical_id,
+        phrase_type=None,
+        token_count=end_token - start_token,
+        confidence=0.9,
+        importance=importance,
+        allow_nested=allow_nested,
+        suppress_children=suppress_children,
+        start_token=start_token,
+        end_token=end_token,
+    )

tests/ebook_search/test_rag_pipeline.py

@@ -2,10 +2,11 @@
 
 from __future__ import annotations
 
+import asyncio
 from threading import Event
 from typing import TYPE_CHECKING
 
-from sqlalchemy import create_engine
+from sqlalchemy.ext.asyncio import create_async_engine
 
 from python.ebook_search.config import EbookSearchConfig, RerankConfig
 from python.ebook_search.search import SearchResult, search_ebooks
@@ -14,18 +15,18 @@ if TYPE_CHECKING:
     from pytest_mock import MockerFixture
 
 
-def test_search_ebooks_runs_vector_and_bm25_in_parallel(mocker: MockerFixture) -> None:
+async def test_search_ebooks_runs_vector_and_bm25_in_parallel(mocker: MockerFixture) -> None:
-    engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
+    engine = create_async_engine("sqlite+aiosqlite:///:memory:")
     vector_started = Event()
     bm25_started = Event()
     received_engines: list[object] = []
 
-    def fake_vector_candidates(received_engine, query, _config):
+    async def fake_vector_candidates(received_engine, _client, query, _config):
         """Return vector candidates after confirming BM25 has started."""
         received_engines.append(received_engine)
         assert query == "what is parallel"
         vector_started.set()
-        assert bm25_started.wait(timeout=2)
+        assert await asyncio.to_thread(bm25_started.wait, 2)
         return [SearchResult(chunk_id=1, text="vector", source_title="Vector", vector_score=0.9)]
 
     def fake_bm25_candidates(query, _config):
@@ -39,7 +40,7 @@ def test_search_ebooks_runs_vector_and_bm25_in_parallel(mocker: MockerFixture) -
     mocker.patch("python.ebook_search.search.bm25_candidates", side_effect=fake_bm25_candidates)
     config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
 
-    response = search_ebooks(engine, "what is parallel", config)
+    response = await search_ebooks(engine, mocker.Mock(), "what is parallel", config)
 
     timings = {step.name: step for step in response.timings}
     assert [result.chunk_id for result in response.results] == [1, 2]
@@ -47,3 +48,26 @@ def test_search_ebooks_runs_vector_and_bm25_in_parallel(mocker: MockerFixture) -
     assert timings["BM25 search"].counts_toward_total is False
     assert timings["Hybrid retrieval"].counts_toward_total is True
     assert received_engines == [engine]
+
+
+async def test_search_ebooks_skips_phrase_matching_when_disabled(mocker: MockerFixture) -> None:
+    """Phrase matching can be disabled for one search request."""
+    engine = create_async_engine("sqlite+aiosqlite:///:memory:")
+    mocker.patch(
+        "python.ebook_search.search.vector_candidates",
+        return_value=[SearchResult(chunk_id=1, text="vector", source_title="Vector", vector_score=0.9)],
+    )
+    mocker.patch("python.ebook_search.search.bm25_candidates", return_value=[])
+    detect_mock = mocker.patch("python.ebook_search.search.query_phrase_matches")
+    boost_mock = mocker.patch("python.ebook_search.search.apply_phrase_mention_boosts")
+    config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
+
+    response = await search_ebooks(engine, mocker.Mock(), "what is parallel", config, phrase_matching=False)
+
+    timing_names = {step.name for step in response.timings}
+    assert [result.chunk_id for result in response.results] == [1]
+    assert response.phrase_matches == ()
+    assert "Protected phrase detection skipped" in timing_names
+    assert "Phrase mention boost skipped" in timing_names
+    detect_mock.assert_not_called()
+    boost_mock.assert_not_called()

tests/ebook_search/test_rerank.py

@@ -24,6 +24,13 @@ def candidates() -> list[SearchResult]:
     ]
 
 
+def make_async_client(mocker: MockerFixture, fake_post) -> httpx.AsyncClient:
+    """Build a mock async client whose post call is served by fake_post."""
+    client = mocker.MagicMock(spec=httpx.AsyncClient)
+    client.post = mocker.AsyncMock(side_effect=fake_post)
+    return client
+
+
 def rerank_response(payload: dict[str, object] | None = None, *, content: bytes | None = None) -> httpx.Response:
     return httpx.Response(
         200,
@@ -59,7 +66,7 @@ def test_reranking_disabled_returns_original_fused_order() -> None:
     assert [result.chunk_id for result in response.results] == [1, 2]
 
 
-def test_reranking_enabled_reorders_candidates(mocker: MockerFixture) -> None:
+async def test_reranking_enabled_reorders_candidates(mocker: MockerFixture) -> None:
     def fake_post(_url: str, *, json: dict[str, object], timeout: float) -> httpx.Response:
         assert timeout == 30
         assert json == {
@@ -77,16 +84,16 @@ def test_reranking_enabled_reorders_candidates(mocker: MockerFixture) -> None:
             }
         )
 
-    mocker.patch.object(httpx, "post", side_effect=fake_post)
+    client = make_async_client(mocker, fake_post)
 
-    results = rerank_chunks("query", candidates(), RerankConfig())
+    results = await rerank_chunks(client, "query", candidates(), RerankConfig())
 
     assert [result.chunk_id for result in results] == [2, 1, 3]
     assert [round(result.score, 3) for result in results] == [0.78, 0.37, 0.28]
     assert [result.rerank_score for result in results] == [0.9, 0.1, 0.4]
 
 
-def test_reranking_cannot_ignore_hybrid_score(mocker: MockerFixture) -> None:
+async def test_reranking_cannot_ignore_hybrid_score(mocker: MockerFixture) -> None:
     candidates = [
         SearchResult(chunk_id=1, text="strong hybrid", source_title="A", score=1.0),
         SearchResult(chunk_id=2, text="weak hybrid", source_title="B", score=0.1),
@@ -102,9 +109,9 @@ def test_reranking_cannot_ignore_hybrid_score(mocker: MockerFixture) -> None:
             }
         )
 
-    mocker.patch.object(httpx, "post", side_effect=fake_post)
+    client = make_async_client(mocker, fake_post)
 
-    results = rerank_chunks("query", candidates, RerankConfig())
+    results = await rerank_chunks(client, "query", candidates, RerankConfig())
 
     assert [result.chunk_id for result in results] == [1, 2]
     assert results[0].score == pytest.approx(0.79)
@@ -112,8 +119,9 @@ def test_reranking_cannot_ignore_hybrid_score(mocker: MockerFixture) -> None:
     assert results[1].rerank_score == 1.0
 
 
-def test_vllm_rerank_timeout_raises(mocker: MockerFixture) -> None:
+async def test_vllm_rerank_timeout_raises(mocker: MockerFixture) -> None:
     def fake_rerank_chunks(
+        _client: httpx.AsyncClient,
         _query: str,
         _candidates: list[SearchResult],
         _config: RerankConfig,
@@ -125,21 +133,21 @@ def test_vllm_rerank_timeout_raises(mocker: MockerFixture) -> None:
     config = EbookSearchConfig(rerank=RerankConfig(enabled=True), top_k=2)
 
     with pytest.raises(httpx.TimeoutException, match="timeout"):
-        apply_rerank("query", candidates(), config)
+        await apply_rerank(mocker.Mock(), "query", candidates(), config)
 
 
-def test_malformed_vllm_rerank_json_does_not_crash_search(mocker: MockerFixture) -> None:
+async def test_malformed_vllm_rerank_json_does_not_crash_search(mocker: MockerFixture) -> None:
     def fake_post(_url: str, **_kwargs: object) -> httpx.Response:
         return rerank_response(content=b"not-json")
 
-    mocker.patch.object(httpx, "post", side_effect=fake_post)
+    client = make_async_client(mocker, fake_post)
 
-    results = rerank_chunks("query", candidates()[:1], RerankConfig())
+    results = await rerank_chunks(client, "query", candidates()[:1], RerankConfig())
 
     assert results[0].score == 0.3
 
 
-def test_vllm_rerank_scores_are_clamped(mocker: MockerFixture) -> None:
+async def test_vllm_rerank_scores_are_clamped(mocker: MockerFixture) -> None:
     def fake_post(_url: str, **_kwargs: object) -> httpx.Response:
         return rerank_response(
             {
@@ -150,8 +158,8 @@ def test_vllm_rerank_scores_are_clamped(mocker: MockerFixture) -> None:
             }
         )
 
-    mocker.patch.object(httpx, "post", side_effect=fake_post)
+    client = make_async_client(mocker, fake_post)
 
-    results = rerank_chunks("query", candidates()[:2], RerankConfig())
+    results = await rerank_chunks(client, "query", candidates()[:2], RerankConfig())
 
     assert {result.chunk_id: result.rerank_score for result in results} == {1: 0.0, 2: 1.0}

tests/ebook_search/test_ui.py

@@ -2,32 +2,49 @@
 
 from __future__ import annotations
 
+import asyncio
 from compression import zstd
+from datetime import UTC, datetime
 from typing import TYPE_CHECKING
 
+from fastapi import BackgroundTasks
 from fastapi.testclient import TestClient
-from sqlalchemy import create_engine
+from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
+from sqlalchemy.pool import StaticPool
 
 from python.ebook_search.api.bm25_tasks import refresh_bm25_for_engine
+from python.ebook_search.api.judge_tasks import (
+    is_judging_book,
+    judge_book_phrases_for_app,
+    pop_book_judgment_outcome,
+    start_book_phrase_judgment,
+)
 from python.ebook_search.api.main import create_app
 from python.ebook_search.config import EbookSearchConfig, RerankConfig
 from python.ebook_search.embeddings import EmbeddingModelStats
+from python.ebook_search.protected_phrases.models import (
+    CorpusPhraseStats,
+    PhraseCandidateGenerationResult,
+    PhraseJudgmentBackfillResult,
+)
 from python.ebook_search.search import SearchResponse, SearchResult
 from python.ebook_search.timing import RuntimeStep
+from python.orm.richie import EbookSource, RichieBase
 
 if TYPE_CHECKING:
     from pytest_mock import MockerFixture
+    from sqlalchemy.ext.asyncio import AsyncEngine
 
 
 def patch_app_runtime(mocker: MockerFixture):
     """Patch app startup dependencies used by UI route tests."""
-    mocker.patch("python.ebook_search.api.main.get_postgres_engine", side_effect=fake_get_postgres_engine)
+    mocker.patch("python.ebook_search.api.main.get_async_postgres_engine", side_effect=fake_get_postgres_engine)
     mocker.patch("python.ebook_search.api.main.ensure_bm25_corpus", side_effect=lambda _session, _config: None)
 
 
 def fake_get_postgres_engine(**_kwargs):
     """Return an in-memory engine for route tests."""
-    return create_engine("sqlite+pysqlite:///:memory:", future=True)
+    return create_async_engine("sqlite+aiosqlite:///:memory:")
 
 
 def test_search_page_uses_zstd_when_requested(mocker: MockerFixture) -> None:
@@ -43,36 +60,68 @@ def test_search_page_uses_zstd_when_requested(mocker: MockerFixture) -> None:
     assert b"EPUB Search" in zstd.decompress(response.content)
 
 
-def test_ui_form_passes_rerank_flag_to_search_handler(mocker: MockerFixture) -> None:
+def test_ui_form_passes_search_toggles_to_search_handler(mocker: MockerFixture) -> None:
     captured: dict[str, object] = {}
 
-    def fake_search_ebooks(_engine, query, config, *, rerank=False):
+    def fake_search_ebooks(_engine, _client, query, config, *, rerank=False, phrase_matching=False):
         captured["query"] = query
         captured["rerank"] = rerank
+        captured["phrase_matching"] = phrase_matching
         captured["config"] = config
         return SearchResponse(query=query, results=[], rank_label="Hybrid + rerank")
 
     mocker.patch("python.ebook_search.api.routes.search.search_ebooks", side_effect=fake_search_ebooks)
     mocker.patch(
         "python.ebook_search.api.routes.search.answer_query",
-        side_effect=lambda _query, _results, _config: "answer",
+        side_effect=lambda _client, _query, _results, _config: "answer",
     )
     patch_app_runtime(mocker)
     app = create_app()
     app.state.config = EbookSearchConfig(rerank=RerankConfig(enabled=False), top_k=12, answer_enabled=True)
 
     with TestClient(app) as client:
-        response = client.post("/search", data={"query": "where is the quote?", "rerank": "true"})
+        response = client.post(
+            "/search",
+            data={"query": "where is the quote?", "rerank": "true", "phrase_matching": "true"},
+        )
 
     assert response.status_code == 200
     assert "Hybrid + rerank" in response.text
     assert captured["query"] == "where is the quote?"
     assert captured["rerank"] is True
+    assert captured["phrase_matching"] is True
+
+
+def test_ui_form_can_disable_phrase_matching(mocker: MockerFixture) -> None:
+    captured: dict[str, object] = {}
+
+    def fake_search_ebooks(_engine, _client, query, _config, *, rerank=False, phrase_matching=False):
+        del rerank
+        captured["query"] = query
+        captured["phrase_matching"] = phrase_matching
+        return SearchResponse(query=query, results=[], rank_label="Hybrid")
+
+    mocker.patch("python.ebook_search.api.routes.search.search_ebooks", side_effect=fake_search_ebooks)
+    mocker.patch(
+        "python.ebook_search.api.routes.search.answer_query",
+        side_effect=lambda _client, _query, _results, _config: "answer",
+    )
+    patch_app_runtime(mocker)
+    app = create_app()
+    app.state.config = EbookSearchConfig(rerank=RerankConfig(enabled=False), top_k=12, answer_enabled=True)
+
+    with TestClient(app) as client:
+        response = client.post("/search", data={"query": "where is the quote?"})
+
+    assert response.status_code == 200
+    assert captured["query"] == "where is the quote?"
+    assert captured["phrase_matching"] is False
 
 
 def test_ui_search_failure_returns_visible_error(mocker: MockerFixture) -> None:
-    def fake_search_ebooks(_engine, _query, _config, *, rerank=False):
+    def fake_search_ebooks(_engine, _client, _query, _config, *, rerank=False, phrase_matching=False):
         del rerank
+        del phrase_matching
         msg = "search exploded"
         raise RuntimeError(msg)
 
@@ -89,11 +138,12 @@ def test_ui_search_failure_returns_visible_error(mocker: MockerFixture) -> None:
 
 
 def test_ui_answer_failure_still_returns_sources(mocker: MockerFixture) -> None:
-    def fake_search_ebooks(_engine, query, _config, *, rerank=False):
+    def fake_search_ebooks(_engine, _client, query, _config, *, rerank=False, phrase_matching=False):
         del rerank
+        del phrase_matching
         return SearchResponse(query=query, results=[], rank_label="Hybrid")
 
-    def fake_answer_query(_query, _results, _config):
+    def fake_answer_query(_client, _query, _results, _config):
         msg = "answer exploded"
         raise RuntimeError(msg)
 
@@ -113,11 +163,12 @@ def test_ui_answer_failure_still_returns_sources(mocker: MockerFixture) -> None:
 def test_ui_skips_answer_when_disabled(mocker: MockerFixture) -> None:
     called = False
 
-    def fake_search_ebooks(_engine, query, _config, *, rerank=False):
+    def fake_search_ebooks(_engine, _client, query, _config, *, rerank=False, phrase_matching=False):
         del rerank
+        del phrase_matching
         return SearchResponse(query=query, results=[], rank_label="Hybrid")
 
-    def fake_answer_query(_query, _results, _config):
+    def fake_answer_query(_client, _query, _results, _config):
         nonlocal called
         called = True
         return "answer"
@@ -138,8 +189,9 @@ def test_ui_skips_answer_when_disabled(mocker: MockerFixture) -> None:
 
 
 def test_ui_shows_component_scores(mocker: MockerFixture) -> None:
-    def fake_search_ebooks(_engine, query, _config, *, rerank=False):
+    def fake_search_ebooks(_engine, _client, query, _config, *, rerank=False, phrase_matching=False):
         del rerank
+        del phrase_matching
         return SearchResponse(
             query=query,
             rank_label="Hybrid + rerank",
@@ -160,7 +212,7 @@ def test_ui_shows_component_scores(mocker: MockerFixture) -> None:
     mocker.patch("python.ebook_search.api.routes.search.search_ebooks", side_effect=fake_search_ebooks)
     mocker.patch(
         "python.ebook_search.api.routes.search.answer_query",
-        side_effect=lambda _query, _results, _config: "answer",
+        side_effect=lambda _client, _query, _results, _config: "answer",
     )
     patch_app_runtime(mocker)
     app = create_app()
@@ -176,9 +228,47 @@ def test_ui_shows_component_scores(mocker: MockerFixture) -> None:
     assert "RRF" in response.text
 
 
-def test_ui_shows_search_runtime_chart(mocker: MockerFixture) -> None:
+def test_ui_shows_matched_phrases_that_boosted_a_result(mocker: MockerFixture) -> None:
-    def fake_search_ebooks(_engine, query, _config, *, rerank=False):
+    def fake_search_ebooks(_engine, _client, query, _config, *, rerank=False, phrase_matching=False):
         del rerank
+        del phrase_matching
+        return SearchResponse(
+            query=query,
+            rank_label="Hybrid",
+            results=[
+                SearchResult(
+                    chunk_id=1,
+                    text="source text",
+                    source_title="Book",
+                    score=0.9,
+                    phrase_hit_count=3,
+                    matched_phrases=("lock in", "haden's syndrome"),
+                )
+            ],
+        )
+
+    mocker.patch("python.ebook_search.api.routes.search.search_ebooks", side_effect=fake_search_ebooks)
+    mocker.patch(
+        "python.ebook_search.api.routes.search.answer_query",
+        side_effect=lambda _client, _query, _results, _config: "answer",
+    )
+    patch_app_runtime(mocker)
+    app = create_app()
+    app.state.config = EbookSearchConfig(rerank=RerankConfig(enabled=False), answer_enabled=True)
+
+    with TestClient(app) as client:
+        response = client.post("/search", data={"query": "what is lock in?"})
+
+    assert response.status_code == 200
+    assert "boosted by" in response.text
+    assert "lock in" in response.text
+    assert "haden's syndrome" in response.text
+
+
+def test_ui_shows_search_runtime_chart(mocker: MockerFixture) -> None:
+    def fake_search_ebooks(_engine, _client, query, _config, *, rerank=False, phrase_matching=False):
+        del rerank
+        del phrase_matching
         return SearchResponse(
             query=query,
             rank_label="Hybrid",
@@ -192,7 +282,7 @@ def test_ui_shows_search_runtime_chart(mocker: MockerFixture) -> None:
     mocker.patch("python.ebook_search.api.routes.search.search_ebooks", side_effect=fake_search_ebooks)
     mocker.patch(
         "python.ebook_search.api.routes.search.answer_query",
-        side_effect=lambda _query, _results, _config: "answer",
+        side_effect=lambda _client, _query, _results, _config: "answer",
     )
     patch_app_runtime(mocker)
     app = create_app()
@@ -214,7 +304,7 @@ def test_ui_embed_all_batches_until_complete(mocker: MockerFixture) -> None:
     counts = iter([32, 32, 5, 0])
     batch_sizes: list[int] = []
 
-    def fake_embed_missing_chunks(_session, config):
+    def fake_embed_missing_chunks(_session, _client, config):
         batch_sizes.append(config.embedding_batch_size)
         return next(counts)
 
@@ -256,7 +346,7 @@ def test_ui_scan_schedules_bm25_refresh_after_database_change(mocker: MockerFixt
     assert scheduled is True
 
 
-def test_bm25_refresh_clears_loaded_corpus_cache(mocker: MockerFixture) -> None:
+async def test_bm25_refresh_clears_loaded_corpus_cache(mocker: MockerFixture) -> None:
     refreshed: list[object] = []
     cache_cleared = False
 
@@ -269,16 +359,146 @@ def test_bm25_refresh_clears_loaded_corpus_cache(mocker: MockerFixture) -> None:
 
     mocker.patch("python.ebook_search.api.bm25_tasks.refresh_bm25_corpus", side_effect=fake_refresh_bm25_corpus)
     mocker.patch("python.ebook_search.api.bm25_tasks.load_bm25_corpus.cache_clear", side_effect=fake_cache_clear)
-    engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
+    engine = create_async_engine("sqlite+aiosqlite:///:memory:")
     config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
 
-    refresh_bm25_for_engine(engine, config)
+    await refresh_bm25_for_engine(engine, config)
 
     assert len(refreshed) == 1
     assert refreshed[0][1] == config
     assert cache_cleared is True
 
 
+def build_engine_with_book() -> AsyncEngine:
+    """Create a shareable in-memory async engine holding one indexed book."""
+    engine = create_async_engine(
+        "sqlite+aiosqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+
+    async def seed() -> None:
+        async with engine.begin() as connection:
+            await connection.run_sync(RichieBase.metadata.create_all)
+        async with AsyncSession(engine) as session:
+            session.add(
+                EbookSource(
+                    title="Book",
+                    author="Author",
+                    language=None,
+                    publisher=None,
+                    identifier=None,
+                    file_path="/library/book.epub",
+                    file_sha256="a" * 64,
+                    file_mtime=datetime.now(tz=UTC),
+                    file_size=10,
+                )
+            )
+            await session.commit()
+
+    asyncio.run(seed())
+    return engine
+
+
+def test_ui_judge_phrases_redirects_and_judges_in_background(mocker: MockerFixture) -> None:
+    mocker.patch("python.ebook_search.api.main.get_async_postgres_engine", return_value=build_engine_with_book())
+    mocker.patch("python.ebook_search.api.main.ensure_bm25_corpus", side_effect=lambda _session, _config: None)
+    judged_source_ids: list[list[int]] = []
+
+    def fake_judge(_engine: object, _config: object, *, source_ids: list[int]) -> PhraseJudgmentBackfillResult:
+        judged_source_ids.append(source_ids)
+        return PhraseJudgmentBackfillResult(
+            books_seen=1,
+            books_judged=1,
+            books_failed=0,
+            candidates_judged=3,
+            protected_phrases=2,
+            phrase_mentions=4,
+        )
+
+    mocker.patch(
+        "python.ebook_search.api.judge_tasks.judge_candidate_phrases_for_books",
+        side_effect=fake_judge,
+    )
+    app = create_app()
+
+    with TestClient(app) as client:
+        response = client.post("/books/1/judge-phrases", follow_redirects=False)
+        detail_after = client.get("/books/1")
+        detail_again = client.get("/books/1")
+
+    assert response.status_code == 303
+    assert response.headers["location"] == "/books/1"
+    assert judged_source_ids == [[1]]
+    assert "Judged 3 candidates; 2 protected phrases promoted" in detail_after.text
+    assert "Judged 3 candidates" not in detail_again.text
+
+
+def test_ui_book_detail_shows_judging_in_progress(mocker: MockerFixture) -> None:
+    mocker.patch("python.ebook_search.api.main.get_async_postgres_engine", return_value=build_engine_with_book())
+    mocker.patch("python.ebook_search.api.main.ensure_bm25_corpus", side_effect=lambda _session, _config: None)
+    mocker.patch("python.ebook_search.api.routes.page.is_judging_book", return_value=True)
+    app = create_app()
+
+    with TestClient(app) as client:
+        response = client.get("/books/1")
+
+    assert response.status_code == 200
+    assert "Judging candidate phrases in the background" in response.text
+    assert "disabled" in response.text
+
+
+def test_book_phrase_judgment_rejects_duplicate_while_queued(mocker: MockerFixture) -> None:
+    mocker.patch(
+        "python.ebook_search.api.judge_tasks.judge_candidate_phrases_for_books",
+        return_value=PhraseJudgmentBackfillResult(
+            books_seen=1,
+            books_judged=1,
+            books_failed=0,
+            candidates_judged=3,
+            protected_phrases=2,
+            phrase_mentions=4,
+        ),
+    )
+    app = create_app()
+    app.state.engine = None
+    app.state.config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
+    background_tasks = BackgroundTasks()
+
+    assert start_book_phrase_judgment(app, background_tasks, 1) is True
+    assert is_judging_book(app, 1) is True
+    assert start_book_phrase_judgment(app, background_tasks, 1) is False
+    assert len(background_tasks.tasks) == 1
+
+    asyncio.run(judge_book_phrases_for_app(app, 1))
+
+    assert is_judging_book(app, 1) is False
+    assert pop_book_judgment_outcome(app, 1) == "Judged 3 candidates; 2 protected phrases promoted"
+    assert pop_book_judgment_outcome(app, 1) is None
+    assert start_book_phrase_judgment(app, background_tasks, 1) is True
+
+
+def test_book_phrase_judgment_records_failure_outcome(mocker: MockerFixture) -> None:
+    def fake_judge(_engine: object, _config: object, *, source_ids: list[int]) -> PhraseJudgmentBackfillResult:
+        del source_ids
+        message = "llm judge unavailable"
+        raise RuntimeError(message)
+
+    mocker.patch(
+        "python.ebook_search.api.judge_tasks.judge_candidate_phrases_for_books",
+        side_effect=fake_judge,
+    )
+    app = create_app()
+    app.state.engine = None
+    app.state.config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
+
+    start_book_phrase_judgment(app, BackgroundTasks(), 7)
+    asyncio.run(judge_book_phrases_for_app(app, 7))
+
+    assert is_judging_book(app, 7) is False
+    assert pop_book_judgment_outcome(app, 7) == "Judging failed; see server logs for details"
+
+
 def test_admin_page_shows_embedding_counts_by_model(mocker: MockerFixture) -> None:
     def fake_embedding_model_stats(_session):
         return [
@@ -297,6 +517,10 @@ def test_admin_page_shows_embedding_counts_by_model(mocker: MockerFixture) -> No
         ]
 
     mocker.patch("python.ebook_search.api.routes.admin.embedding_model_stats", side_effect=fake_embedding_model_stats)
+    mocker.patch(
+        "python.ebook_search.api.routes.admin.corpus_phrase_stats",
+        return_value=fake_corpus_phrase_stats(),
+    )
     patch_app_runtime(mocker)
     app = create_app()
 
@@ -310,3 +534,128 @@ def test_admin_page_shows_embedding_counts_by_model(mocker: MockerFixture) -> No
     assert "24" in response.text
     assert "qwen3-embedding-4b" in response.text
     assert "2560" in response.text
+
+
+def fake_corpus_phrase_stats() -> CorpusPhraseStats:
+    """Build distinctive corpus phrase stats for admin page assertions."""
+    return CorpusPhraseStats(
+        total_books=17,
+        books_with_candidates=13,
+        books_fully_judged=11,
+        candidate_phrases=901,
+        judged_candidates=703,
+        unjudged_candidates=198,
+        protected_phrases=157,
+    )
+
+
+def test_admin_page_shows_protected_phrase_stats(mocker: MockerFixture) -> None:
+    mocker.patch("python.ebook_search.api.routes.admin.embedding_model_stats", return_value=[])
+    mocker.patch(
+        "python.ebook_search.api.routes.admin.corpus_phrase_stats",
+        return_value=fake_corpus_phrase_stats(),
+    )
+    patch_app_runtime(mocker)
+    app = create_app()
+
+    with TestClient(app) as client:
+        response = client.get("/admin")
+
+    assert response.status_code == 200
+    assert "Protected phrases" in response.text
+    for value in ("17", "13", "11", "901", "703", "198", "157"):
+        assert value in response.text
+
+
+def test_ui_add_missing_phrases_generates_only_missing_books(mocker: MockerFixture) -> None:
+    captured: dict[str, object] = {}
+
+    def fake_generate(_session, _config, *, only_missing=False):
+        captured["only_missing"] = only_missing
+        return PhraseCandidateGenerationResult(books_seen=3, books_built=2, candidate_phrases=42)
+
+    mocker.patch(
+        "python.ebook_search.api.routes.admin.generate_candidate_phrases_for_books",
+        side_effect=fake_generate,
+    )
+    patch_app_runtime(mocker)
+    app = create_app()
+
+    with TestClient(app) as client:
+        response = client.post("/admin/phrases/generate-missing")
+
+    assert response.status_code == 200
+    assert captured["only_missing"] is True
+    assert "42 candidates stored" in response.text
+
+
+def test_ui_regenerate_all_phrases_generates_every_book(mocker: MockerFixture) -> None:
+    captured: dict[str, object] = {}
+
+    def fake_generate(_session, _config, *, only_missing=False):
+        captured["only_missing"] = only_missing
+        return PhraseCandidateGenerationResult(books_seen=5, books_built=5, candidate_phrases=99)
+
+    mocker.patch(
+        "python.ebook_search.api.routes.admin.generate_candidate_phrases_for_books",
+        side_effect=fake_generate,
+    )
+    patch_app_runtime(mocker)
+    app = create_app()
+
+    with TestClient(app) as client:
+        response = client.post("/admin/phrases/generate-all")
+
+    assert response.status_code == 200
+    assert captured["only_missing"] is False
+    assert "5 of 5 books" in response.text
+
+
+def test_ui_judge_missing_phrases_judges_only_pending_books(mocker: MockerFixture) -> None:
+    captured: dict[str, object] = {}
+
+    async def fake_judge(_engine, _config, *, source_ids=None):
+        captured["source_ids"] = source_ids
+        return PhraseJudgmentBackfillResult(
+            books_seen=2,
+            books_judged=2,
+            books_failed=0,
+            candidates_judged=10,
+            protected_phrases=4,
+            phrase_mentions=9,
+        )
+
+    mocker.patch(
+        "python.ebook_search.api.routes.admin.judge_candidate_phrases_for_books",
+        side_effect=fake_judge,
+    )
+    mocker.patch(
+        "python.ebook_search.api.routes.admin.book_ids_pending_first_judgment",
+        return_value=[3, 5],
+    )
+    patch_app_runtime(mocker)
+    app = create_app()
+
+    with TestClient(app) as client:
+        response = client.post("/admin/phrases/judge-missing")
+
+    assert response.status_code == 200
+    assert captured["source_ids"] == [3, 5]
+    assert "4 protected phrases" in response.text
+
+
+def test_ui_judge_missing_phrases_reports_when_nothing_is_pending(mocker: MockerFixture) -> None:
+    judge = mocker.patch("python.ebook_search.api.routes.admin.judge_candidate_phrases_for_books")
+    mocker.patch(
+        "python.ebook_search.api.routes.admin.book_ids_pending_first_judgment",
+        return_value=[],
+    )
+    patch_app_runtime(mocker)
+    app = create_app()
+
+    with TestClient(app) as client:
+        response = client.post("/admin/phrases/judge-missing")
+
+    assert response.status_code == 200
+    assert "have been judged" in response.text
+    judge.assert_not_called()

tests/test_common.py

@@ -2,15 +2,14 @@
 
 from __future__ import annotations
 
-from os import environ
+from python.common import bash_wrapper, get_repo_dir, utcnow
-from typing import TYPE_CHECKING
 
-from apprise import Apprise
 
-from python.common import bash_wrapper, signal_alert, utcnow
+def test_get_repo_dir() -> None:
-
+    """test_get_repo_dir."""
-if TYPE_CHECKING:
+    repo_dir = get_repo_dir()
-    from pytest_mock import MockerFixture
+    assert (repo_dir / "pyproject.toml").is_file()
+    assert (repo_dir / "python" / "common.py").is_file()
 
 
 def test_utcnow() -> None:
@@ -18,34 +17,6 @@ def test_utcnow() -> None:
     utcnow()
 
 
-def test_signal_alert(mocker: MockerFixture) -> None:
-    """test_signal_alert."""
-    environ["SIGNAL_ALERT_FROM_PHONE"] = "1234567890"
-    environ["SIGNAL_ALERT_TO_PHONE"] = "0987654321"
-
-    mock_logger = mocker.patch("python.common.logger")
-    mock_apprise_client = mocker.MagicMock(spec=Apprise)
-    mocker.patch("python.common.Apprise", return_value=mock_apprise_client)
-
-    signal_alert("test")
-
-    mock_logger.info.assert_not_called()
-    mock_apprise_client.add.assert_called_once_with("signal://localhost:8989/1234567890/0987654321")
-    mock_apprise_client.notify.assert_called_once_with(title="", body="test")
-
-
-def test_signal_alert_no_phones(mocker: MockerFixture) -> None:
-    """test_signal_alert_no_phones."""
-    if "SIGNAL_ALERT_FROM_PHONE" in environ:
-        del environ["SIGNAL_ALERT_FROM_PHONE"]
-    if "SIGNAL_ALERT_TO_PHONE" in environ:
-        del environ["SIGNAL_ALERT_TO_PHONE"]
-    mock_logger = mocker.patch("python.common.logger")
-    signal_alert("test")
-
-    mock_logger.info.assert_called_once_with("SIGNAL_ALERT_FROM_PHONE or SIGNAL_ALERT_TO_PHONE not set")
-
-
 def test_test_bash_wrapper() -> None:
     """test_test_bash_wrapper."""
     stdout, returncode = bash_wrapper("echo test")

tests/test_signal_alert.py

@@ -0,0 +1,41 @@
+"""test_signal_alert."""
+
+from __future__ import annotations
+
+from os import environ
+from typing import TYPE_CHECKING
+
+from apprise import Apprise
+
+from python.signal_alert import signal_alert
+
+if TYPE_CHECKING:
+    from pytest_mock import MockerFixture
+
+
+def test_signal_alert(mocker: MockerFixture) -> None:
+    """test_signal_alert."""
+    environ["SIGNAL_ALERT_FROM_PHONE"] = "1234567890"
+    environ["SIGNAL_ALERT_TO_PHONE"] = "0987654321"
+
+    mock_logger = mocker.patch("python.signal_alert.logger")
+    mock_apprise_client = mocker.MagicMock(spec=Apprise)
+    mocker.patch("python.signal_alert.Apprise", return_value=mock_apprise_client)
+
+    signal_alert("test")
+
+    mock_logger.info.assert_not_called()
+    mock_apprise_client.add.assert_called_once_with("signal://localhost:8989/1234567890/0987654321")
+    mock_apprise_client.notify.assert_called_once_with(title="", body="test")
+
+
+def test_signal_alert_no_phones(mocker: MockerFixture) -> None:
+    """test_signal_alert_no_phones."""
+    if "SIGNAL_ALERT_FROM_PHONE" in environ:
+        del environ["SIGNAL_ALERT_FROM_PHONE"]
+    if "SIGNAL_ALERT_TO_PHONE" in environ:
+        del environ["SIGNAL_ALERT_TO_PHONE"]
+    mock_logger = mocker.patch("python.signal_alert.logger")
+    signal_alert("test")
+
+    mock_logger.info.assert_called_once_with("SIGNAL_ALERT_FROM_PHONE or SIGNAL_ALERT_TO_PHONE not set")

users/richie/home/app_image_path.nix

@@ -0,0 +1,5 @@
+{
+  home.sessionPath = [
+    "/home/richie/app_images/"
+  ];
+}

users/richie/home/gui/vscode/settings.json

@@ -76,6 +76,7 @@
   "cSpell.userWords": [
     "Cahill",
     "Corvidae",
+    "dedup",
     "drivername",
     "fastapi",
     "Michal",

users/richie/home/programs.nix

@@ -61,5 +61,6 @@
     nix-tree
     nixfmt
     treefmt
+    codebase-memory-mcp
   ];
 }

users/richie/systems/rhapsody-in-green.nix

@@ -1,5 +1,6 @@
 {
   imports = [
+    ../home/app_image_path.nix
     ../home/global.nix
     ../home/gui
   ];