feat(zfs): manage jeeves datasets declaratively from nix

Add a zfs_manager module that reconciles the live datasets on jeeves
against a nix declaration, and generate the snapshot retention config
from that same declaration so the two can no longer drift apart.

systems/jeeves/datasets.nix declares every dataset on the media, storage
and scratch pools, nested the way zfs nests them and flattened into
pool/parent/child names. Values were transcribed from the live pools
rather than from scripts/zfs.sh, which had gone stale: acltype reads back
as posix, and media/secure/important, scratch/kestra and storage/nomad
were never recorded. root_pool datasets are declared for retention only,
their properties stay unmanaged for now.

python.tools.zfs_manager creates missing datasets and corrects drifted
properties, and never destroys anything. Undeclared properties are judged
by the zfs source field, so inherited and default values stay quiet while
locally set ones warn. Size values are normalised to bytes so that 16K and
16384 do not re-issue zfs set on every run.

vars.nix now derives its paths from the declared mountpoints instead of
repeating them, dropping three zfs_* keys that nothing referenced.

Replaces systems/jeeves/snapshot_config.toml, which listed a dataset that
does not exist and omitted thirteen that do.

.dockerignore

@@ -0,0 +1,28 @@
+.git
+.direnv
+.mypy_cache
+.pytest_cache
+.ruff_cache
+.venv
+**/.venv
+.env
+.cache
+.claude
+.coverage
+.vscode
+.stfolder
+.literotica_data
+esphome
+htmlcov
+data
+ebooks
+__pycache__
+**/__pycache__
+*.pyc
+*.pyo
+.ebook_search_bm25
+result
+result-*
+*.egg-info
+dist
+build

.github/workflows/test_ebook_search.yml

@@ -0,0 +1,26 @@
+name: test ebook search
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+env:
+  UV_PYTHON_DOWNLOADS: never
+  UV_CACHE_DIR: /var/cache/uv
+  UV_LINK_MODE: copy
+
+jobs:
+  test-ebook-search:
+    runs-on: self-hosted
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install dependencies
+        run: nix develop .#devShells.x86_64-linux.ebook-search -c uv sync --locked --project python/ebook_search/docker
+      - name: Run ebook search tests
+        run: nix develop .#devShells.x86_64-linux.ebook-search -c uv run --project python/ebook_search/docker --no-sync pytest tests/ebook_search --override-ini addopts="-n auto -ra"

.gitignore

@@ -173,3 +173,6 @@ frontend/node_modules/
 # data from testing llms
 data/*
 .ebook_search_bm25
+
+# gems data
+.gems

.vscode/settings.json

@@ -71,6 +71,7 @@
     "ehci",
     "emerg",
     "endlessh",
+    "ents",
     "errorlens",
     "esbenp",
     "esphome",
@@ -172,6 +173,8 @@
     "Networkd",
     "networkmanager",
     "newtabpage",
+    "ngram",
+    "ngrams",
     "nixfmt",
     "nixos",
     "nixpkgs",
@@ -298,7 +301,9 @@
     "uiprotect",
     "uitour",
     "unifi",
+    "unjudged",
     "unrar",
+    "unstorable",
     "unsubmitted",
     "uptimekuma",
     "urlbar",
@@ -326,6 +331,7 @@
     "xcursorgen",
     "xdist",
     "xhci",
+    "yake",
     "yazi",
     "yubikey",
     "yubioath",

common/optional/zfs_manager.nix

@@ -0,0 +1,168 @@
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  cfg = config.services.zfs_manager;
+
+  snapshotOptions = {
+    options = {
+      "15_min" = lib.mkOption {
+        type = lib.types.int;
+        default = 0;
+        description = "How many 15 minute snapshots to keep.";
+      };
+      hourly = lib.mkOption {
+        type = lib.types.int;
+        default = 0;
+        description = "How many hourly snapshots to keep.";
+      };
+      daily = lib.mkOption {
+        type = lib.types.int;
+        default = 0;
+        description = "How many daily snapshots to keep.";
+      };
+      monthly = lib.mkOption {
+        type = lib.types.int;
+        default = 0;
+        description = "How many monthly snapshots to keep.";
+      };
+    };
+  };
+
+  datasetOptions = {
+    options = {
+      manageProperties = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+        description = ''
+          Whether zfs_manager owns this dataset's properties. When false the
+          dataset only contributes its snapshot retention, which is how
+          root_pool datasets are declared.
+        '';
+      };
+      properties = lib.mkOption {
+        type = lib.types.attrsOf lib.types.str;
+        default = { };
+        description = ''
+          The zfs properties this dataset should have. Values are compared
+          against the live dataset and corrected when they differ.
+        '';
+      };
+      snapshots = lib.mkOption {
+        type = lib.types.submodule snapshotOptions;
+        default = { };
+        description = "Snapshot retention for this dataset.";
+      };
+    };
+  };
+
+  # snapshot_manager.py only ever walks datasets below a pool root, so pool
+  # roots are left out of the retention table. It also indexes the table
+  # directly, which is why every entry carries all four keys.
+  snapshotTable = lib.mapAttrs (_: dataset: dataset.snapshots) (
+    lib.filterAttrs (name: _: lib.hasInfix "/" name) cfg.datasets
+  );
+
+  snapshotConfig = (pkgs.formats.toml { }).generate "snapshot_config.toml" (
+    snapshotTable // { default = cfg.defaultSnapshots; }
+  );
+
+  # Every declared dataset is emitted, including the ones whose properties are
+  # not managed, so the tool can tell "deliberately hands off" apart from
+  # "nobody has written this down yet".
+  datasetConfig = (pkgs.formats.json { }).generate "zfs_datasets.json" {
+    datasets = lib.mapAttrs (_: dataset: {
+      inherit (dataset) manageProperties properties;
+    }) cfg.datasets;
+  };
+in
+{
+  options = {
+    services.zfs_manager = {
+      enable = lib.mkEnableOption "declarative ZFS dataset management";
+      datasets = lib.mkOption {
+        type = lib.types.attrsOf (lib.types.submodule datasetOptions);
+        default = { };
+        example = lib.literalExpression ''
+          {
+            "media/temp".properties = {
+              sync = "disabled";
+              redundant_metadata = "none";
+            };
+          }
+        '';
+        description = ''
+          The datasets to manage, keyed by full dataset name. Missing datasets
+          are created and drifted properties are corrected. Nothing is ever
+          destroyed, and datasets that are not declared are left alone.
+
+          A name without a "/" is a pool root filesystem. Its properties are
+          managed but it is never created, pool creation stays manual.
+        '';
+      };
+      defaultSnapshots = lib.mkOption {
+        type = lib.types.submodule snapshotOptions;
+        default = { };
+        description = ''
+          Retention for any dataset that is not declared above, emitted as the
+          "default" table of the snapshot config.
+        '';
+      };
+      dryRun = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Log every change that would be made without touching zfs. Use this to
+          validate a new or heavily edited declaration before applying it.
+        '';
+      };
+      PYTHONPATH = lib.mkOption {
+        type = lib.types.str;
+        description = ''
+          the PYTHONPATH to use for the zfs_manager service.
+        '';
+      };
+      EnvironmentFile = lib.mkOption {
+        type = lib.types.nullOr (lib.types.coercedTo lib.types.path toString lib.types.str);
+        default = null;
+        description = ''
+          Single environment file for the service (e.g. /etc/zfs-manager/env).
+          Use a leading "-" to ignore if missing (systemd feature).
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.snapshot_manager.path = snapshotConfig;
+
+    systemd = {
+      services.zfs_manager = {
+        description = "ZFS Dataset Manager";
+        requires = [ "zfs-import.target" ];
+        after = [
+          "zfs-import.target"
+          "zfs-mount.service"
+        ];
+        wantedBy = [ "multi-user.target" ];
+        path = [ pkgs.zfs ];
+        # Re-run on nixos-rebuild switch whenever the declaration changes.
+        restartTriggers = [ datasetConfig ];
+        environment = {
+          PYTHONPATH = cfg.PYTHONPATH;
+        };
+        serviceConfig = {
+          Type = "oneshot";
+          RemainAfterExit = true;
+          ExecStart = "${pkgs.my_python}/bin/python -m python.tools.zfs_manager ${lib.escapeShellArg datasetConfig}${lib.optionalString cfg.dryRun " --dry-run"}";
+        }
+        // lib.optionalAttrs (cfg.EnvironmentFile != null) {
+          EnvironmentFile = cfg.EnvironmentFile;
+        };
+      };
+    };
+  };
+}

flake.lock

@@ -8,11 +8,11 @@
       },
       "locked": {
         "dir": "pkgs/firefox-addons",
-        "lastModified": 1782964936,
-        "narHash": "sha256-wXEBDr7/dFQYhVpDwCKc9fkrYQQE4x0bdirX1bsLBGA=",
+        "lastModified": 1784963334,
+        "narHash": "sha256-Xbj8fFapX+ufz8kx0sH33RAvF0Pauzfg5oihn38SWtE=",
         "owner": "rycee",
         "repo": "nur-expressions",
-        "rev": "64feee871e0373dd6121e412c3fb12e372d1bfb5",
+        "rev": "557935d63e75f4bfbd61ca3cede3a632edbed7e5",
         "type": "gitlab"
       },
       "original": {
@@ -29,11 +29,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1783005591,
-        "narHash": "sha256-NcLHV5uBAeggDUE2wPbKszjfyaSLsoqaYt7izOphkZw=",
+        "lastModified": 1784913159,
+        "narHash": "sha256-JWq0BfjO4ktpH5USfQNQzdvHpIDT8fSKD5K7LvdMRFs=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "f469c79b955609d6a8fdd9e689be76a93b1621d7",
+        "rev": "079a3b5d1aa6a719920a51316253b7d6dd22738d",
         "type": "github"
       },
       "original": {
@@ -47,11 +47,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1782562157,
-        "narHash": "sha256-a7+T6QSeowynwZ1ZJJbP8T8ntAytvrui8kFGJmIZt2c=",
+        "lastModified": 1784723954,
+        "narHash": "sha256-1CfD8ZUjCkTgjsneLZ/lxCHhgDfqxxE7/GX0MmsgiqA=",
         "owner": "nixos",
         "repo": "nixos-hardware",
-        "rev": "a9cf7546a938c737b079e738de73934a13de9784",
+        "rev": "a017f5b72210026af5b3ac5949f08d94380a6fbd",
         "type": "github"
       },
       "original": {
@@ -76,11 +76,11 @@
     },
     "nixpkgs-master": {
       "locked": {
-        "lastModified": 1783021952,
-        "narHash": "sha256-8PghAtSGGZ0umfVI8Qbd7ZbFrfZPiH1UwtVbgLeikDA=",
+        "lastModified": 1785001027,
+        "narHash": "sha256-Cc+7WLCh3sss1d+4DDLuEma4a2v3YdXGez/N+MJoomY=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "f136374c679c54171a3ace589d15e9e79a8bd086",
+        "rev": "b2a8a65ad069640cf1a0656b355338a65c41ac12",
         "type": "github"
       },
       "original": {
@@ -108,11 +108,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1782723713,
-        "narHash": "sha256-oPXCU/SSUokcGaJREHibG1CBX3+s/W7orDWQOZDsEeQ=",
+        "lastModified": 1784796856,
+        "narHash": "sha256-wWFrV5/Qbm+lyt5x20E/bSbfJiGKMo4RCxZV8cl/WZI=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "b5aa0fbd538984f6e3d201be0005b4463d8b09f8",
+        "rev": "e2587caef70cea85dd97d7daab492899902dbf5d",
         "type": "github"
       },
       "original": {
@@ -141,11 +141,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1782165805,
-        "narHash": "sha256-478kKQBvK6SYTOdN2h9jhKJv94nbXRbFMfuL1WshErg=",
+        "lastModified": 1783174389,
+        "narHash": "sha256-aCWC8ngycU7OdJrU2+Je3qf+1a2ykuBvpPhZT/9tXMc=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "56b24064fdcaedca53553b1a6d607fd23b613a24",
+        "rev": "f1406619a3884cd5c47992a70b8b35c9c0fcb4c9",
         "type": "github"
       },
       "original": {

overlays/default.nix

@@ -17,60 +17,31 @@
 
   python-env = final: _prev: {
     my_python = final.python314.withPackages (
-      ps:
-      let
-        bm25s = ps.buildPythonPackage rec {
-          pname = "bm25s";
-          version = "0.3.9";
-          pyproject = true;
-
-          src = final.fetchPypi {
-            inherit pname version;
-            hash = "sha256-iVxnnZUrfeg1XttfPhpiCh4vKU0dQrkZvwghzOLi9Zc=";
-          };
-
-          build-system = [ ps.setuptools ];
-          dependencies = with ps; [
-            numpy
-            scipy
-          ];
-
-          pythonImportsCheck = [ "bm25s" ];
-        };
-      in
-      with ps;
-      [
+      ps: with ps; [
         alembic
         apprise
         apscheduler
-        beautifulsoup4
-        ebooklib
         fastapi
         fastapi-cli
         httpx
+        jinja2
         mypy
-        numpy
-        orjson
         pgvector
-        polars
         psycopg
         pydantic
-        pydantic-settings
         pyfakefs
         pytest
         pytest-cov
         pytest-mock
         pytest-xdist
         python-multipart
+        pydantic-settings
         ruff
-        scalene
         sqlalchemy
-        bm25s
         tenacity
-        textual
-        tiktoken
         tinytuya
         typer
+        uvicorn
         websockets
       ]
     );

pyproject.toml

@@ -12,19 +12,27 @@ dependencies = [
     "alembic",
     "apprise",
     "apscheduler",
+    "beautifulsoup4",
+    "bm25s",
+    "ebooklib",
     "fastapi",
     "fastapi-cli",
     "httpx",
+    "jinja2",
+    "pgvector",
     "polars",
     "psycopg[binary]",
     "pydantic",
     "pydantic-settings",
     "python-multipart",
-    "sqlalchemy",
+    "sqlalchemy[asyncio]",
     "tenacity",
+    "tiktoken",
     "tinytuya",
     "typer",
+    "uvicorn",
     "websockets",
+    "yake",
 ]
 
 [project.scripts]
@@ -33,8 +41,10 @@ whisper-transcribe = "python.tools.whisper.transcribe:main"
 
 [dependency-groups]
 dev = [
+    "aiosqlite",
     "mypy",
     "pyfakefs",
+    "pytest-asyncio",
     "pytest-cov",
     "pytest-mock",
     "pytest-xdist",
@@ -109,6 +119,7 @@ exclude_lines = [
 ]
 
 [tool.pytest.ini_options]
-addopts = "-n auto -ra"
+addopts = "-n auto -ra --ignore=tests/ebook_search"
+asyncio_mode = "auto"
 testpaths = ["tests"]
 # --cov=system_tools --cov-report=term-missing --cov-report=xml --cov-report=html --cov-branch

python/alembic/richie/versions/2026_06_29-adding_phrase_metadata_tables_dddee09eddcc.py

@@ -0,0 +1,206 @@
+"""adding Phrase metadata tables.
+
+Revision ID: dddee09eddcc
+Revises: 96d72c748c24
+Create Date: 2026-06-29 00:49:07.344159
+
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+from python.orm import RichieBase
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+# revision identifiers, used by Alembic.
+revision: str = "dddee09eddcc"
+down_revision: str | None = "96d72c748c24"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+schema = RichieBase.schema_name
+
+
+def upgrade() -> None:
+    """Upgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "candidate_phrases",
+        sa.Column("book_id", sa.Integer(), nullable=False),
+        sa.Column("series_id", sa.Integer(), nullable=True),
+        sa.Column("phrase_text", sa.Text(), nullable=False),
+        sa.Column("phrase_norm", sa.Text(), nullable=False),
+        sa.Column("token_count", sa.Integer(), nullable=False),
+        sa.Column("source_raw_ngram", sa.Boolean(), nullable=False),
+        sa.Column("source_yake", sa.Boolean(), nullable=False),
+        sa.Column("source_spacy_ner", sa.Boolean(), nullable=False),
+        sa.Column("source_spacy_noun_chunk", sa.Boolean(), nullable=False),
+        sa.Column("source_capitalized", sa.Boolean(), nullable=False),
+        sa.Column("source_metadata", sa.Boolean(), nullable=False),
+        sa.Column("spacy_label", sa.String(), nullable=True),
+        sa.Column("raw_count", sa.Integer(), nullable=False),
+        sa.Column("chapter_count", sa.Integer(), nullable=False),
+        sa.Column("yake_score", sa.Float(), nullable=True),
+        sa.Column("candidate_score", sa.Float(), nullable=False),
+        sa.Column(
+            "sample_contexts",
+            sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), "postgresql"),
+            nullable=True,
+        ),
+        sa.Column("llm_judged", sa.Boolean(), nullable=False),
+        sa.Column("llm_keep", sa.Boolean(), nullable=True),
+        sa.Column("llm_confidence", sa.Float(), nullable=True),
+        sa.Column("llm_category", sa.String(), nullable=True),
+        sa.Column("llm_reason", sa.Text(), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["book_id"],
+            [f"{schema}.ebook_source.id"],
+            name=op.f("fk_candidate_phrases_book_id_ebook_source"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_candidate_phrases")),
+        sa.UniqueConstraint("book_id", "phrase_norm", name="uq_candidate_phrases_book_id_phrase_norm"),
+        schema=schema,
+    )
+    op.create_index(
+        "candidate_phrases_book_norm_idx", "candidate_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
+    )
+    op.create_index(
+        "candidate_phrases_book_score_idx",
+        "candidate_phrases",
+        ["book_id", "candidate_score"],
+        unique=False,
+        schema=schema,
+    )
+    op.create_table(
+        "protected_phrases",
+        sa.Column("book_id", sa.Integer(), nullable=True),
+        sa.Column("series_id", sa.Integer(), nullable=True),
+        sa.Column("phrase_text", sa.Text(), nullable=False),
+        sa.Column("phrase_norm", sa.Text(), nullable=False),
+        sa.Column("canonical_id", sa.String(), nullable=False),
+        sa.Column("phrase_type", sa.String(), nullable=True),
+        sa.Column("token_count", sa.Integer(), nullable=False),
+        sa.Column("confidence", sa.Float(), nullable=False),
+        sa.Column("importance", sa.Float(), nullable=False),
+        sa.Column("allow_nested", sa.Boolean(), nullable=False),
+        sa.Column("suppress_children", sa.Boolean(), nullable=False),
+        sa.Column("source_candidate_id", sa.Integer(), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["book_id"],
+            [f"{schema}.ebook_source.id"],
+            name=op.f("fk_protected_phrases_book_id_ebook_source"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["source_candidate_id"],
+            [f"{schema}.candidate_phrases.id"],
+            name=op.f("fk_protected_phrases_source_candidate_id_candidate_phrases"),
+            ondelete="SET NULL",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_protected_phrases")),
+        sa.UniqueConstraint("book_id", "phrase_norm", name="uq_protected_phrases_book_id_phrase_norm"),
+        schema=schema,
+    )
+    op.create_index(
+        "protected_phrases_book_norm_idx", "protected_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
+    )
+    op.create_index("protected_phrases_norm_idx", "protected_phrases", ["phrase_norm"], unique=False, schema=schema)
+    op.create_index(
+        "protected_phrases_series_norm_idx",
+        "protected_phrases",
+        ["series_id", "phrase_norm"],
+        unique=False,
+        schema=schema,
+    )
+    op.create_table(
+        "chunk_phrase_mentions",
+        sa.Column("chunk_id", sa.BigInteger(), nullable=False),
+        sa.Column("phrase_id", sa.Integer(), nullable=False),
+        sa.Column("book_id", sa.Integer(), nullable=True),
+        sa.Column("series_id", sa.Integer(), nullable=True),
+        sa.Column("start_char", sa.Integer(), nullable=False),
+        sa.Column("end_char", sa.Integer(), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["book_id"],
+            [f"{schema}.ebook_source.id"],
+            name=op.f("fk_chunk_phrase_mentions_book_id_ebook_source"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["chunk_id"],
+            [f"{schema}.ebook_chunk.id"],
+            name=op.f("fk_chunk_phrase_mentions_chunk_id_ebook_chunk"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["phrase_id"],
+            [f"{schema}.protected_phrases.id"],
+            name=op.f("fk_chunk_phrase_mentions_phrase_id_protected_phrases"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_chunk_phrase_mentions")),
+        sa.UniqueConstraint("chunk_id", "phrase_id", "start_char", name="uq_chunk_phrase_mentions_chunk_phrase_start"),
+        schema=schema,
+    )
+    op.create_index(
+        "chunk_phrase_mentions_chunk_idx", "chunk_phrase_mentions", ["chunk_id"], unique=False, schema=schema
+    )
+    op.create_index(
+        "chunk_phrase_mentions_phrase_idx", "chunk_phrase_mentions", ["phrase_id"], unique=False, schema=schema
+    )
+    op.create_table(
+        "phrase_aliases",
+        sa.Column("phrase_id", sa.Integer(), nullable=False),
+        sa.Column("alias_text", sa.Text(), nullable=False),
+        sa.Column("alias_norm", sa.Text(), nullable=False),
+        sa.Column("confidence", sa.Float(), nullable=False),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["phrase_id"],
+            [f"{schema}.protected_phrases.id"],
+            name=op.f("fk_phrase_aliases_phrase_id_protected_phrases"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_phrase_aliases")),
+        sa.UniqueConstraint("phrase_id", "alias_norm", name="uq_phrase_aliases_phrase_id_alias_norm"),
+        schema=schema,
+    )
+    op.create_index("phrase_aliases_norm_idx", "phrase_aliases", ["alias_norm"], unique=False, schema=schema)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index("phrase_aliases_norm_idx", table_name="phrase_aliases", schema=schema)
+    op.drop_table("phrase_aliases", schema=schema)
+    op.drop_index("chunk_phrase_mentions_phrase_idx", table_name="chunk_phrase_mentions", schema=schema)
+    op.drop_index("chunk_phrase_mentions_chunk_idx", table_name="chunk_phrase_mentions", schema=schema)
+    op.drop_table("chunk_phrase_mentions", schema=schema)
+    op.drop_index("protected_phrases_series_norm_idx", table_name="protected_phrases", schema=schema)
+    op.drop_index("protected_phrases_norm_idx", table_name="protected_phrases", schema=schema)
+    op.drop_index("protected_phrases_book_norm_idx", table_name="protected_phrases", schema=schema)
+    op.drop_table("protected_phrases", schema=schema)
+    op.drop_index("candidate_phrases_book_score_idx", table_name="candidate_phrases", schema=schema)
+    op.drop_index("candidate_phrases_book_norm_idx", table_name="candidate_phrases", schema=schema)
+    op.drop_table("candidate_phrases", schema=schema)
+    # ### end Alembic commands ###

python/alembic/richie/versions/2026_07_09-remove_spacy_ner_751260fc3228.py

@@ -0,0 +1,55 @@
+"""remove spaCy-ner.
+
+Revision ID: 751260fc3228
+Revises: dddee09eddcc
+Create Date: 2026-07-09 23:03:39.554083
+
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import sqlalchemy as sa
+from alembic import op
+
+from python.orm import RichieBase
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+# revision identifiers, used by Alembic.
+revision: str = "751260fc3228"
+down_revision: str | None = "dddee09eddcc"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+schema = RichieBase.schema_name
+
+
+def upgrade() -> None:
+    """Upgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("candidate_phrases", "source_spacy_noun_chunk", schema=schema)
+    op.drop_column("candidate_phrases", "source_spacy_ner", schema=schema)
+    op.drop_column("candidate_phrases", "spacy_label", schema=schema)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "candidate_phrases", sa.Column("spacy_label", sa.VARCHAR(), autoincrement=False, nullable=True), schema=schema
+    )
+    op.add_column(
+        "candidate_phrases",
+        sa.Column("source_spacy_ner", sa.BOOLEAN(), autoincrement=False, nullable=False),
+        schema=schema,
+    )
+    op.add_column(
+        "candidate_phrases",
+        sa.Column("source_spacy_noun_chunk", sa.BOOLEAN(), autoincrement=False, nullable=False),
+        schema=schema,
+    )
+    # ### end Alembic commands ###

python/api/__init__.py

@@ -1 +0,0 @@
-"""FastAPI applications."""

python/api/main.py

@@ -1,56 +0,0 @@
-"""FastAPI interface for Contact database."""
-
-from __future__ import annotations
-
-import logging
-from contextlib import asynccontextmanager
-from typing import TYPE_CHECKING, Annotated
-
-import typer
-import uvicorn
-from fastapi import FastAPI
-
-from python.api.routers import contact_router, views_router
-from python.common import configure_logger
-from python.fastapi_tools import ZstdMiddleware
-from python.orm.common import get_postgres_engine
-
-if TYPE_CHECKING:
-    from collections.abc import AsyncIterator
-
-logger = logging.getLogger(__name__)
-
-
-def create_app() -> FastAPI:
-    """Create and configure the FastAPI application."""
-
-    @asynccontextmanager
-    async def lifespan(app: FastAPI) -> AsyncIterator[None]:
-        """Manage application lifespan."""
-        app.state.engine = get_postgres_engine()
-        yield
-        app.state.engine.dispose()
-
-    app = FastAPI(title="Contact Database API", lifespan=lifespan)
-    app.add_middleware(ZstdMiddleware)
-
-    app.include_router(contact_router)
-    app.include_router(views_router)
-
-    return app
-
-
-def serve(
-    host: Annotated[str, typer.Option("--host", "-h", help="Host to bind to")],
-    port: Annotated[int, typer.Option("--port", "-p", help="Port to bind to")] = 8000,
-    log_level: Annotated[str, typer.Option("--log-level", "-l", help="Log level")] = "INFO",
-) -> None:
-    """Start the Contact API server."""
-    configure_logger(log_level)
-
-    app = create_app()
-    uvicorn.run(app, host=host, port=port)
-
-
-if __name__ == "__main__":
-    typer.run(serve)

python/api/routers/__init__.py

@@ -1,6 +0,0 @@
-"""API routers."""
-
-from python.api.routers.contact import router as contact_router
-from python.api.routers.views import router as views_router
-
-__all__ = ["contact_router", "views_router"]

python/api/routers/contact.py

@@ -1,481 +0,0 @@
-"""Contact API router."""
-
-from pathlib import Path
-
-from fastapi import APIRouter, HTTPException, Request
-from fastapi.responses import HTMLResponse
-from fastapi.templating import Jinja2Templates
-from pydantic import BaseModel
-from sqlalchemy import select
-from sqlalchemy.orm import selectinload
-
-from python.fastapi_tools.db import DbSession  # noqa: TC001 this is a FastAPI needed at runtime
-from python.orm.richie.contact import Contact, ContactRelationship, Need, RelationshipType
-
-TEMPLATES_DIR = Path(__file__).parent.parent / "templates"
-templates = Jinja2Templates(directory=TEMPLATES_DIR)
-
-
-def _is_htmx(request: Request) -> bool:
-    """Check if the request is from HTMX."""
-    return request.headers.get("HX-Request") == "true"
-
-
-class NeedBase(BaseModel):
-    """Base schema for Need."""
-
-    name: str
-    description: str | None = None
-
-
-class NeedCreate(NeedBase):
-    """Schema for creating a Need."""
-
-
-class NeedResponse(NeedBase):
-    """Schema for Need response."""
-
-    id: int
-
-    model_config = {"from_attributes": True}
-
-
-class ContactRelationshipCreate(BaseModel):
-    """Schema for creating a contact relationship."""
-
-    related_contact_id: int
-    relationship_type: RelationshipType
-    closeness_weight: int | None = None
-
-
-class ContactRelationshipUpdate(BaseModel):
-    """Schema for updating a contact relationship."""
-
-    relationship_type: RelationshipType | None = None
-    closeness_weight: int | None = None
-
-
-class ContactRelationshipResponse(BaseModel):
-    """Schema for contact relationship response."""
-
-    contact_id: int
-    related_contact_id: int
-    relationship_type: str
-    closeness_weight: int
-
-    model_config = {"from_attributes": True}
-
-
-class RelationshipTypeInfo(BaseModel):
-    """Information about a relationship type."""
-
-    value: str
-    display_name: str
-    default_weight: int
-
-
-class GraphNode(BaseModel):
-    """Node in the relationship graph."""
-
-    id: int
-    name: str
-    current_job: str | None = None
-
-
-class GraphEdge(BaseModel):
-    """Edge in the relationship graph."""
-
-    source: int
-    target: int
-    relationship_type: str
-    closeness_weight: int
-
-
-class GraphData(BaseModel):
-    """Complete graph data for visualization."""
-
-    nodes: list[GraphNode]
-    edges: list[GraphEdge]
-
-
-class ContactBase(BaseModel):
-    """Base schema for Contact."""
-
-    name: str
-    age: int | None = None
-    bio: str | None = None
-    current_job: str | None = None
-    gender: str | None = None
-    goals: str | None = None
-    legal_name: str | None = None
-    profile_pic: str | None = None
-    safe_conversation_starters: str | None = None
-    self_sufficiency_score: int | None = None
-    social_structure_style: str | None = None
-    ssn: str | None = None
-    suffix: str | None = None
-    timezone: str | None = None
-    topics_to_avoid: str | None = None
-
-
-class ContactCreate(ContactBase):
-    """Schema for creating a Contact."""
-
-    need_ids: list[int] = []
-
-
-class ContactUpdate(BaseModel):
-    """Schema for updating a Contact."""
-
-    name: str | None = None
-    age: int | None = None
-    bio: str | None = None
-    current_job: str | None = None
-    gender: str | None = None
-    goals: str | None = None
-    legal_name: str | None = None
-    profile_pic: str | None = None
-    safe_conversation_starters: str | None = None
-    self_sufficiency_score: int | None = None
-    social_structure_style: str | None = None
-    ssn: str | None = None
-    suffix: str | None = None
-    timezone: str | None = None
-    topics_to_avoid: str | None = None
-    need_ids: list[int] | None = None
-
-
-class ContactResponse(ContactBase):
-    """Schema for Contact response with relationships."""
-
-    id: int
-    needs: list[NeedResponse] = []
-    related_to: list[ContactRelationshipResponse] = []
-    related_from: list[ContactRelationshipResponse] = []
-
-    model_config = {"from_attributes": True}
-
-
-class ContactListResponse(ContactBase):
-    """Schema for Contact list response."""
-
-    id: int
-
-    model_config = {"from_attributes": True}
-
-
-router = APIRouter(prefix="/api", tags=["contacts"])
-
-
-@router.post("/needs", response_model=NeedResponse)
-def create_need(need: NeedCreate, db: DbSession) -> Need:
-    """Create a new need."""
-    db_need = Need(name=need.name, description=need.description)
-    db.add(db_need)
-    db.commit()
-    db.refresh(db_need)
-    return db_need
-
-
-@router.get("/needs", response_model=list[NeedResponse])
-def list_needs(db: DbSession) -> list[Need]:
-    """List all needs."""
-    return list(db.scalars(select(Need)).all())
-
-
-@router.get("/needs/{need_id}", response_model=NeedResponse)
-def get_need(need_id: int, db: DbSession) -> Need:
-    """Get a need by ID."""
-    need = db.get(Need, need_id)
-    if not need:
-        raise HTTPException(status_code=404, detail="Need not found")
-    return need
-
-
-@router.delete("/needs/{need_id}", response_model=None)
-def delete_need(need_id: int, request: Request, db: DbSession) -> dict[str, bool] | HTMLResponse:
-    """Delete a need by ID."""
-    need = db.get(Need, need_id)
-    if not need:
-        raise HTTPException(status_code=404, detail="Need not found")
-    db.delete(need)
-    db.commit()
-    if _is_htmx(request):
-        return HTMLResponse("")
-    return {"deleted": True}
-
-
-@router.post("/contacts", response_model=ContactResponse)
-def create_contact(contact: ContactCreate, db: DbSession) -> Contact:
-    """Create a new contact."""
-    need_ids = contact.need_ids
-    contact_data = contact.model_dump(exclude={"need_ids"})
-    db_contact = Contact(**contact_data)
-
-    if need_ids:
-        needs = list(db.scalars(select(Need).where(Need.id.in_(need_ids))).all())
-        db_contact.needs = needs
-
-    db.add(db_contact)
-    db.commit()
-    db.refresh(db_contact)
-    return db_contact
-
-
-@router.get("/contacts", response_model=list[ContactListResponse])
-def list_contacts(
-    db: DbSession,
-    skip: int = 0,
-    limit: int = 100,
-) -> list[Contact]:
-    """List all contacts with pagination."""
-    return list(db.scalars(select(Contact).offset(skip).limit(limit)).all())
-
-
-@router.get("/contacts/{contact_id}", response_model=ContactResponse)
-def get_contact(contact_id: int, db: DbSession) -> Contact:
-    """Get a contact by ID with all relationships."""
-    contact = db.scalar(
-        select(Contact)
-        .where(Contact.id == contact_id)
-        .options(
-            selectinload(Contact.needs),
-            selectinload(Contact.related_to),
-            selectinload(Contact.related_from),
-        )
-    )
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-    return contact
-
-
-@router.patch("/contacts/{contact_id}", response_model=ContactResponse)
-def update_contact(
-    contact_id: int,
-    contact: ContactUpdate,
-    db: DbSession,
-) -> Contact:
-    """Update a contact by ID."""
-    db_contact = db.get(Contact, contact_id)
-    if not db_contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    update_data = contact.model_dump(exclude_unset=True)
-    need_ids = update_data.pop("need_ids", None)
-
-    for key, value in update_data.items():
-        setattr(db_contact, key, value)
-
-    if need_ids is not None:
-        needs = list(db.scalars(select(Need).where(Need.id.in_(need_ids))).all())
-        db_contact.needs = needs
-
-    db.commit()
-    db.refresh(db_contact)
-    return db_contact
-
-
-@router.delete("/contacts/{contact_id}", response_model=None)
-def delete_contact(contact_id: int, request: Request, db: DbSession) -> dict[str, bool] | HTMLResponse:
-    """Delete a contact by ID."""
-    contact = db.get(Contact, contact_id)
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-    db.delete(contact)
-    db.commit()
-    if _is_htmx(request):
-        return HTMLResponse("")
-    return {"deleted": True}
-
-
-@router.post("/contacts/{contact_id}/needs/{need_id}")
-def add_need_to_contact(
-    contact_id: int,
-    need_id: int,
-    db: DbSession,
-) -> dict[str, bool]:
-    """Add a need to a contact."""
-    contact = db.get(Contact, contact_id)
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    need = db.get(Need, need_id)
-    if not need:
-        raise HTTPException(status_code=404, detail="Need not found")
-
-    if need not in contact.needs:
-        contact.needs.append(need)
-        db.commit()
-
-    return {"added": True}
-
-
-@router.delete("/contacts/{contact_id}/needs/{need_id}", response_model=None)
-def remove_need_from_contact(
-    contact_id: int,
-    need_id: int,
-    request: Request,
-    db: DbSession,
-) -> dict[str, bool] | HTMLResponse:
-    """Remove a need from a contact."""
-    contact = db.get(Contact, contact_id)
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    need = db.get(Need, need_id)
-    if not need:
-        raise HTTPException(status_code=404, detail="Need not found")
-
-    if need in contact.needs:
-        contact.needs.remove(need)
-        db.commit()
-
-    if _is_htmx(request):
-        return HTMLResponse("")
-    return {"removed": True}
-
-
-@router.post(
-    "/contacts/{contact_id}/relationships",
-    response_model=ContactRelationshipResponse,
-)
-def add_contact_relationship(
-    contact_id: int,
-    relationship: ContactRelationshipCreate,
-    db: DbSession,
-) -> ContactRelationship:
-    """Add a relationship between two contacts."""
-    contact = db.get(Contact, contact_id)
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    related_contact = db.get(Contact, relationship.related_contact_id)
-    if not related_contact:
-        raise HTTPException(status_code=404, detail="Related contact not found")
-
-    if contact_id == relationship.related_contact_id:
-        raise HTTPException(status_code=400, detail="Cannot relate contact to itself")
-
-    # Use provided weight or default from relationship type
-    weight = relationship.closeness_weight
-    if weight is None:
-        weight = relationship.relationship_type.default_weight
-
-    db_relationship = ContactRelationship(
-        contact_id=contact_id,
-        related_contact_id=relationship.related_contact_id,
-        relationship_type=relationship.relationship_type.value,
-        closeness_weight=weight,
-    )
-    db.add(db_relationship)
-    db.commit()
-    db.refresh(db_relationship)
-    return db_relationship
-
-
-@router.get(
-    "/contacts/{contact_id}/relationships",
-    response_model=list[ContactRelationshipResponse],
-)
-def get_contact_relationships(
-    contact_id: int,
-    db: DbSession,
-) -> list[ContactRelationship]:
-    """Get all relationships for a contact."""
-    contact = db.get(Contact, contact_id)
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    outgoing = list(db.scalars(select(ContactRelationship).where(ContactRelationship.contact_id == contact_id)).all())
-    incoming = list(
-        db.scalars(select(ContactRelationship).where(ContactRelationship.related_contact_id == contact_id)).all()
-    )
-    return outgoing + incoming
-
-
-@router.patch(
-    "/contacts/{contact_id}/relationships/{related_contact_id}",
-    response_model=ContactRelationshipResponse,
-)
-def update_contact_relationship(
-    contact_id: int,
-    related_contact_id: int,
-    update: ContactRelationshipUpdate,
-    db: DbSession,
-) -> ContactRelationship:
-    """Update a relationship between two contacts."""
-    relationship = db.scalar(
-        select(ContactRelationship).where(
-            ContactRelationship.contact_id == contact_id,
-            ContactRelationship.related_contact_id == related_contact_id,
-        )
-    )
-    if not relationship:
-        raise HTTPException(status_code=404, detail="Relationship not found")
-
-    if update.relationship_type is not None:
-        relationship.relationship_type = update.relationship_type.value
-    if update.closeness_weight is not None:
-        relationship.closeness_weight = update.closeness_weight
-
-    db.commit()
-    db.refresh(relationship)
-    return relationship
-
-
-@router.delete("/contacts/{contact_id}/relationships/{related_contact_id}", response_model=None)
-def remove_contact_relationship(
-    contact_id: int,
-    related_contact_id: int,
-    request: Request,
-    db: DbSession,
-) -> dict[str, bool] | HTMLResponse:
-    """Remove a relationship between two contacts."""
-    relationship = db.scalar(
-        select(ContactRelationship).where(
-            ContactRelationship.contact_id == contact_id,
-            ContactRelationship.related_contact_id == related_contact_id,
-        )
-    )
-    if not relationship:
-        raise HTTPException(status_code=404, detail="Relationship not found")
-
-    db.delete(relationship)
-    db.commit()
-    if _is_htmx(request):
-        return HTMLResponse("")
-    return {"deleted": True}
-
-
-@router.get("/relationship-types")
-def list_relationship_types() -> list[RelationshipTypeInfo]:
-    """List all available relationship types with their default weights."""
-    return [
-        RelationshipTypeInfo(
-            value=rt.value,
-            display_name=rt.display_name,
-            default_weight=rt.default_weight,
-        )
-        for rt in RelationshipType
-    ]
-
-
-@router.get("/graph")
-def get_relationship_graph(db: DbSession) -> GraphData:
-    """Get all contacts and relationships as graph data for visualization."""
-    contacts = list(db.scalars(select(Contact)).all())
-    relationships = list(db.scalars(select(ContactRelationship)).all())
-
-    nodes = [GraphNode(id=c.id, name=c.name, current_job=c.current_job) for c in contacts]
-
-    edges = [
-        GraphEdge(
-            source=rel.contact_id,
-            target=rel.related_contact_id,
-            relationship_type=rel.relationship_type,
-            closeness_weight=rel.closeness_weight,
-        )
-        for rel in relationships
-    ]
-
-    return GraphData(nodes=nodes, edges=edges)

python/api/routers/views.py

@@ -1,345 +0,0 @@
-"""HTMX server-rendered view router."""
-
-from pathlib import Path
-from typing import Annotated, Any
-
-from fastapi import APIRouter, Form, HTTPException, Request
-from fastapi.responses import HTMLResponse, RedirectResponse
-from fastapi.templating import Jinja2Templates
-from sqlalchemy import select
-from sqlalchemy.orm import Session, selectinload
-
-from python.fastapi_tools.db import DbSession  # noqa: TC001 this is a FastAPI needed at runtime
-from python.orm.richie.contact import Contact, ContactRelationship, Need, RelationshipType
-
-TEMPLATES_DIR = Path(__file__).parent.parent / "templates"
-templates = Jinja2Templates(directory=TEMPLATES_DIR)
-
-router = APIRouter(tags=["views"])
-
-FAMILIAL_TYPES = {
-    "parent",
-    "child",
-    "sibling",
-    "grandparent",
-    "grandchild",
-    "aunt_uncle",
-    "niece_nephew",
-    "cousin",
-    "in_law",
-}
-FRIEND_TYPES = {"best_friend", "close_friend", "friend", "acquaintance", "neighbor"}
-PARTNER_TYPES = {"spouse", "partner"}
-PROFESSIONAL_TYPES = {"mentor", "mentee", "business_partner", "colleague", "manager", "direct_report", "client"}
-
-CONTACT_STRING_FIELDS = (
-    "name",
-    "legal_name",
-    "suffix",
-    "gender",
-    "current_job",
-    "timezone",
-    "profile_pic",
-    "bio",
-    "goals",
-    "social_structure_style",
-    "safe_conversation_starters",
-    "topics_to_avoid",
-    "ssn",
-)
-
-CONTACT_INT_FIELDS = ("age", "self_sufficiency_score")
-
-
-def _group_relationships(relationships: list[ContactRelationship]) -> dict[str, list[ContactRelationship]]:
-    """Group relationships by category."""
-    groups: dict[str, list[ContactRelationship]] = {
-        "familial": [],
-        "partners": [],
-        "friends": [],
-        "professional": [],
-        "other": [],
-    }
-    for rel in relationships:
-        if rel.relationship_type in FAMILIAL_TYPES:
-            groups["familial"].append(rel)
-        elif rel.relationship_type in PARTNER_TYPES:
-            groups["partners"].append(rel)
-        elif rel.relationship_type in FRIEND_TYPES:
-            groups["friends"].append(rel)
-        elif rel.relationship_type in PROFESSIONAL_TYPES:
-            groups["professional"].append(rel)
-        else:
-            groups["other"].append(rel)
-    return groups
-
-
-def _build_contact_name_map(database: Session, contact: Contact) -> dict[int, str]:
-    """Build a mapping of contact IDs to names for relationship display."""
-    related_ids = {rel.related_contact_id for rel in contact.related_to}
-    related_ids |= {rel.contact_id for rel in contact.related_from}
-    related_ids.discard(contact.id)
-
-    if not related_ids:
-        return {}
-
-    related_contacts = list(database.scalars(select(Contact).where(Contact.id.in_(related_ids))).all())
-    return {related.id: related.name for related in related_contacts}
-
-
-def _get_relationship_type_display() -> dict[str, str]:
-    """Build a mapping of relationship type values to display names."""
-    return {rel_type.value: rel_type.display_name for rel_type in RelationshipType}
-
-
-async def _parse_contact_form(request: Request) -> dict[str, Any]:
-    """Parse contact form data from a multipart/form request."""
-    form_data = await request.form()
-    result: dict[str, Any] = {}
-
-    for field in CONTACT_STRING_FIELDS:
-        value = form_data.get(field, "")
-        result[field] = str(value) if value else None
-
-    for field in CONTACT_INT_FIELDS:
-        value = form_data.get(field, "")
-        result[field] = int(value) if value else None
-
-    result["need_ids"] = [int(value) for value in form_data.getlist("need_ids")]
-    return result
-
-
-def _save_contact_from_form(database: Session, contact: Contact, form_result: dict[str, Any]) -> None:
-    """Apply parsed form data to a Contact and save associated needs."""
-    need_ids = form_result.pop("need_ids")
-
-    for key, value in form_result.items():
-        setattr(contact, key, value)
-
-    if need_ids:
-        contact.needs = list(database.scalars(select(Need).where(Need.id.in_(need_ids))).all())
-    else:
-        contact.needs = []
-
-
-@router.get("/", response_class=HTMLResponse)
-@router.get("/contacts", response_class=HTMLResponse)
-def contact_list_page(request: Request, database: DbSession) -> HTMLResponse:
-    """Render the contacts list page."""
-    contacts = list(database.scalars(select(Contact)).all())
-    return templates.TemplateResponse(request, "contact_list.html", {"contacts": contacts})
-
-
-@router.get("/contacts/new", response_class=HTMLResponse)
-def new_contact_page(request: Request, database: DbSession) -> HTMLResponse:
-    """Render the new contact form page."""
-    all_needs = list(database.scalars(select(Need)).all())
-    return templates.TemplateResponse(request, "contact_form.html", {"contact": None, "all_needs": all_needs})
-
-
-@router.post("/htmx/contacts/new")
-async def create_contact_form(request: Request, database: DbSession) -> RedirectResponse:
-    """Handle the create contact form submission."""
-    form_result = await _parse_contact_form(request)
-    contact = Contact()
-    _save_contact_from_form(database, contact, form_result)
-
-    database.add(contact)
-    database.commit()
-    database.refresh(contact)
-    return RedirectResponse(url=f"/contacts/{contact.id}", status_code=303)
-
-
-@router.get("/contacts/{contact_id}", response_class=HTMLResponse)
-def contact_detail_page(contact_id: int, request: Request, database: DbSession) -> HTMLResponse:
-    """Render the contact detail page."""
-    contact = database.scalar(
-        select(Contact)
-        .where(Contact.id == contact_id)
-        .options(
-            selectinload(Contact.needs),
-            selectinload(Contact.related_to),
-            selectinload(Contact.related_from),
-        )
-    )
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    contact_names = _build_contact_name_map(database, contact)
-    grouped_relationships = _group_relationships(contact.related_to)
-    all_contacts = list(database.scalars(select(Contact)).all())
-    all_needs = list(database.scalars(select(Need)).all())
-    available_needs = [need for need in all_needs if need not in contact.needs]
-
-    return templates.TemplateResponse(
-        request,
-        "contact_detail.html",
-        {
-            "contact": contact,
-            "contact_names": contact_names,
-            "grouped_relationships": grouped_relationships,
-            "all_contacts": all_contacts,
-            "available_needs": available_needs,
-            "relationship_types": list(RelationshipType),
-        },
-    )
-
-
-@router.get("/contacts/{contact_id}/edit", response_class=HTMLResponse)
-def edit_contact_page(contact_id: int, request: Request, database: DbSession) -> HTMLResponse:
-    """Render the edit contact form page."""
-    contact = database.scalar(select(Contact).where(Contact.id == contact_id).options(selectinload(Contact.needs)))
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    all_needs = list(database.scalars(select(Need)).all())
-    return templates.TemplateResponse(request, "contact_form.html", {"contact": contact, "all_needs": all_needs})
-
-
-@router.post("/htmx/contacts/{contact_id}/edit")
-async def update_contact_form(contact_id: int, request: Request, database: DbSession) -> RedirectResponse:
-    """Handle the edit contact form submission."""
-    contact = database.get(Contact, contact_id)
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    form_result = await _parse_contact_form(request)
-    _save_contact_from_form(database, contact, form_result)
-
-    database.commit()
-    return RedirectResponse(url=f"/contacts/{contact_id}", status_code=303)
-
-
-@router.post("/htmx/contacts/{contact_id}/add-need", response_class=HTMLResponse)
-def add_need_to_contact_htmx(
-    contact_id: int,
-    request: Request,
-    database: DbSession,
-    need_id: Annotated[int, Form()],
-) -> HTMLResponse:
-    """Add a need to a contact and return updated manage-needs partial."""
-    contact = database.scalar(select(Contact).where(Contact.id == contact_id).options(selectinload(Contact.needs)))
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    need = database.get(Need, need_id)
-    if not need:
-        raise HTTPException(status_code=404, detail="Need not found")
-
-    if need not in contact.needs:
-        contact.needs.append(need)
-        database.commit()
-        database.refresh(contact)
-
-    return templates.TemplateResponse(request, "partials/manage_needs.html", {"contact": contact})
-
-
-@router.post("/htmx/contacts/{contact_id}/add-relationship", response_class=HTMLResponse)
-def add_relationship_htmx(
-    contact_id: int,
-    request: Request,
-    database: DbSession,
-    related_contact_id: Annotated[int, Form()],
-    relationship_type: Annotated[str, Form()],
-) -> HTMLResponse:
-    """Add a relationship and return updated manage-relationships partial."""
-    contact = database.scalar(select(Contact).where(Contact.id == contact_id).options(selectinload(Contact.related_to)))
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    related_contact = database.get(Contact, related_contact_id)
-    if not related_contact:
-        raise HTTPException(status_code=404, detail="Related contact not found")
-
-    rel_type = RelationshipType(relationship_type)
-    weight = rel_type.default_weight
-
-    relationship = ContactRelationship(
-        contact_id=contact_id,
-        related_contact_id=related_contact_id,
-        relationship_type=relationship_type,
-        closeness_weight=weight,
-    )
-    database.add(relationship)
-    database.commit()
-    database.refresh(contact)
-
-    contact_names = _build_contact_name_map(database, contact)
-    return templates.TemplateResponse(
-        request,
-        "partials/manage_relationships.html",
-        {"contact": contact, "contact_names": contact_names},
-    )
-
-
-@router.post("/htmx/contacts/{contact_id}/relationships/{related_contact_id}/weight")
-def update_relationship_weight_htmx(
-    contact_id: int,
-    related_contact_id: int,
-    database: DbSession,
-    closeness_weight: Annotated[int, Form()],
-) -> HTMLResponse:
-    """Update a relationship's closeness weight from HTMX range input."""
-    relationship = database.scalar(
-        select(ContactRelationship).where(
-            ContactRelationship.contact_id == contact_id,
-            ContactRelationship.related_contact_id == related_contact_id,
-        )
-    )
-    if not relationship:
-        raise HTTPException(status_code=404, detail="Relationship not found")
-
-    relationship.closeness_weight = closeness_weight
-    database.commit()
-    return HTMLResponse("")
-
-
-@router.post("/htmx/needs", response_class=HTMLResponse)
-def create_need_htmx(
-    request: Request,
-    database: DbSession,
-    name: Annotated[str, Form()],
-    description: Annotated[str, Form()] = "",
-) -> HTMLResponse:
-    """Create a need via form data and return updated needs list."""
-    need = Need(name=name, description=description or None)
-    database.add(need)
-    database.commit()
-    needs = list(database.scalars(select(Need)).all())
-    return templates.TemplateResponse(request, "partials/need_items.html", {"needs": needs})
-
-
-@router.get("/needs", response_class=HTMLResponse)
-def needs_page(request: Request, database: DbSession) -> HTMLResponse:
-    """Render the needs list page."""
-    needs = list(database.scalars(select(Need)).all())
-    return templates.TemplateResponse(request, "need_list.html", {"needs": needs})
-
-
-@router.get("/graph", response_class=HTMLResponse)
-def graph_page(request: Request, database: DbSession) -> HTMLResponse:
-    """Render the relationship graph page."""
-    contacts = list(database.scalars(select(Contact)).all())
-    relationships = list(database.scalars(select(ContactRelationship)).all())
-
-    graph_data = {
-        "nodes": [{"id": contact.id, "name": contact.name, "current_job": contact.current_job} for contact in contacts],
-        "edges": [
-            {
-                "source": rel.contact_id,
-                "target": rel.related_contact_id,
-                "relationship_type": rel.relationship_type,
-                "closeness_weight": rel.closeness_weight,
-            }
-            for rel in relationships
-        ],
-    }
-
-    return templates.TemplateResponse(
-        request,
-        "graph.html",
-        {
-            "graph_data": graph_data,
-            "relationship_type_display": _get_relationship_type_display(),
-        },
-    )

python/api/templates/base.html

@@ -1,198 +0,0 @@
-<!DOCTYPE html>
-<html lang="en" data-theme="light">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>{% block title %}Contact Database{% endblock %}</title>
-    <script src="https://unpkg.com/htmx.org@2.0.4"></script>
-    <style>
-        :root {
-            --color-bg: #f5f5f5;
-            --color-bg-card: #ffffff;
-            --color-bg-hover: #f0f0f0;
-            --color-bg-muted: #f9f9f9;
-            --color-bg-error: #ffe0e0;
-            --color-text: #333333;
-            --color-text-muted: #666666;
-            --color-text-error: #cc0000;
-            --color-border: #dddddd;
-            --color-border-light: #eeeeee;
-            --color-border-lighter: #f0f0f0;
-            --color-primary: #0066cc;
-            --color-primary-hover: #0055aa;
-            --color-danger: #cc3333;
-            --color-danger-hover: #aa2222;
-            --color-tag-bg: #e0e0e0;
-            --shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
-            font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
-            line-height: 1.5;
-            color: var(--color-text);
-            background-color: var(--color-bg);
-        }
-        [data-theme="dark"] {
-            --color-bg: #1a1a1a;
-            --color-bg-card: #2d2d2d;
-            --color-bg-hover: #3d3d3d;
-            --color-bg-muted: #252525;
-            --color-bg-error: #4a2020;
-            --color-text: #e0e0e0;
-            --color-text-muted: #a0a0a0;
-            --color-text-error: #ff6b6b;
-            --color-border: #404040;
-            --color-border-light: #353535;
-            --color-border-lighter: #303030;
-            --color-primary: #4da6ff;
-            --color-primary-hover: #7dbfff;
-            --color-danger: #ff6b6b;
-            --color-danger-hover: #ff8a8a;
-            --color-tag-bg: #404040;
-            --shadow: 0 1px 3px rgba(0, 0, 0, 0.3);
-        }
-        * { box-sizing: border-box; }
-        body { margin: 0; background: var(--color-bg); color: var(--color-text); }
-        .app { max-width: 1000px; margin: 0 auto; padding: 20px; }
-        nav { display: flex; align-items: center; gap: 20px; padding: 15px 0; border-bottom: 1px solid var(--color-border); margin-bottom: 20px; }
-        nav a { color: var(--color-primary); text-decoration: none; font-weight: 500; }
-        nav a:hover { text-decoration: underline; }
-        .theme-toggle { margin-left: auto; }
-        main { background: var(--color-bg-card); padding: 20px; border-radius: 8px; box-shadow: var(--shadow); }
-        .header { display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px; }
-        .header h1 { margin: 0; }
-        a { color: var(--color-primary); }
-        a:hover { text-decoration: underline; }
-
-        .btn { display: inline-block; padding: 8px 16px; border: 1px solid var(--color-border); border-radius: 4px; background: var(--color-bg-card); color: var(--color-text); text-decoration: none; cursor: pointer; font-size: 14px; margin-left: 8px; }
-        .btn:hover { background: var(--color-bg-hover); }
-        .btn-primary { background: var(--color-primary); border-color: var(--color-primary); color: white; }
-        .btn-primary:hover { background: var(--color-primary-hover); }
-        .btn-danger { background: var(--color-danger); border-color: var(--color-danger); color: white; }
-        .btn-danger:hover { background: var(--color-danger-hover); }
-        .btn-small { padding: 4px 8px; font-size: 12px; }
-        .btn:disabled { opacity: 0.6; cursor: not-allowed; }
-
-        table { width: 100%; border-collapse: collapse; }
-        th, td { padding: 12px; text-align: left; border-bottom: 1px solid var(--color-border-light); }
-        th { font-weight: 600; background: var(--color-bg-muted); }
-        tr:hover { background: var(--color-bg-muted); }
-
-        .error { background: var(--color-bg-error); color: var(--color-text-error); padding: 10px; border-radius: 4px; margin-bottom: 20px; }
-        .tag { display: inline-block; background: var(--color-tag-bg); padding: 2px 8px; border-radius: 12px; font-size: 12px; color: var(--color-text-muted); }
-
-        .add-form { display: flex; gap: 10px; margin-top: 15px; flex-wrap: wrap; }
-        .add-form select, .add-form input { padding: 8px; border: 1px solid var(--color-border); border-radius: 4px; min-width: 200px; background: var(--color-bg-card); color: var(--color-text); }
-
-        .form-group { margin-bottom: 20px; }
-        .form-group label { display: block; font-weight: 500; margin-bottom: 5px; }
-        .form-group input, .form-group textarea, .form-group select { width: 100%; padding: 10px; border: 1px solid var(--color-border); border-radius: 4px; font-size: 14px; background: var(--color-bg-card); color: var(--color-text); }
-        .form-group textarea { resize: vertical; }
-        .form-row { display: grid; grid-template-columns: 1fr 1fr; gap: 20px; }
-        .checkbox-group { display: flex; flex-wrap: wrap; gap: 15px; }
-        .checkbox-label { display: flex; align-items: center; gap: 5px; cursor: pointer; }
-        .form-actions { display: flex; gap: 10px; margin-top: 30px; padding-top: 20px; border-top: 1px solid var(--color-border-light); }
-
-        .need-form { background: var(--color-bg-muted); padding: 20px; border-radius: 4px; margin-bottom: 20px; }
-        .need-items { list-style: none; padding: 0; }
-        .need-items li { display: flex; justify-content: space-between; align-items: flex-start; padding: 15px; border: 1px solid var(--color-border-light); border-radius: 4px; margin-bottom: 10px; }
-        .need-info p { margin: 5px 0 0; color: var(--color-text-muted); font-size: 14px; }
-
-        .graph-container { width: 100%; }
-        .graph-hint { color: var(--color-text-muted); font-size: 14px; margin-bottom: 15px; }
-        .selected-info { margin-top: 15px; padding: 15px; background: var(--color-bg-muted); border-radius: 8px; }
-        .selected-info h3 { margin: 0 0 10px; }
-        .selected-info p { margin: 5px 0; color: var(--color-text-muted); }
-        .legend { margin-top: 20px; padding: 15px; background: var(--color-bg-muted); border-radius: 8px; }
-        .legend h4 { margin: 0 0 10px; font-size: 14px; }
-        .legend-items { display: flex; flex-wrap: wrap; gap: 15px; }
-        .legend-item { display: flex; align-items: center; gap: 8px; font-size: 12px; color: var(--color-text-muted); }
-        .legend-line { width: 30px; border-radius: 2px; }
-
-        .id-card { width: 100%; }
-        .id-card-inner { background: linear-gradient(135deg, #0a0a0f 0%, #1a1a2e 50%, #0a0a0f 100%); background-image: radial-gradient(white 1px, transparent 1px), linear-gradient(135deg, #0a0a0f 0%, #1a1a2e 50%, #0a0a0f 100%); background-size: 50px 50px, 100% 100%; color: #fff; border-radius: 12px; padding: 25px; min-height: 500px; position: relative; overflow: hidden; }
-        .id-card-header { display: flex; justify-content: space-between; align-items: flex-start; margin-bottom: 15px; }
-        .id-card-header-left { flex: 1; }
-        .id-card-header-right { display: flex; flex-direction: column; align-items: flex-end; gap: 10px; }
-        .id-card-title { font-size: 2.5rem; font-weight: 700; margin: 0; color: #fff; text-shadow: 2px 2px 4px rgba(0,0,0,0.5); }
-        .id-profile-pic { width: 80px; height: 80px; border-radius: 8px; object-fit: cover; border: 2px solid rgba(255,255,255,0.3); }
-        .id-profile-placeholder { width: 80px; height: 80px; border-radius: 8px; background: linear-gradient(135deg, #4ecdc4 0%, #44a8a0 100%); display: flex; align-items: center; justify-content: center; border: 2px solid rgba(255,255,255,0.3); }
-        .id-profile-placeholder span { font-size: 2rem; font-weight: 700; color: #fff; text-shadow: 1px 1px 2px rgba(0,0,0,0.3); }
-        .id-card-actions { display: flex; gap: 8px; }
-        .id-card-actions .btn { background: rgba(255,255,255,0.1); border-color: rgba(255,255,255,0.3); color: #fff; }
-        .id-card-actions .btn:hover { background: rgba(255,255,255,0.2); }
-        .id-card-body { display: grid; grid-template-columns: 1fr 1.5fr; gap: 30px; }
-        .id-card-left { display: flex; flex-direction: column; gap: 8px; }
-        .id-field { font-size: 1rem; line-height: 1.4; }
-        .id-field-block { margin-top: 15px; font-size: 0.95rem; line-height: 1.5; }
-        .id-label { color: #4ecdc4; font-weight: 500; }
-        .id-card-right { display: flex; flex-direction: column; gap: 20px; }
-        .id-bio { font-size: 0.9rem; line-height: 1.6; color: #e0e0e0; }
-        .id-relationships { margin-top: 10px; }
-        .id-section-title { font-size: 1.5rem; margin: 0 0 15px; color: #fff; border-bottom: 1px solid rgba(255,255,255,0.2); padding-bottom: 8px; }
-        .id-rel-group { margin-bottom: 12px; font-size: 0.9rem; line-height: 1.6; }
-        .id-rel-label { color: #a0a0a0; }
-        .id-rel-group a { color: #4ecdc4; text-decoration: none; }
-        .id-rel-group a:hover { text-decoration: underline; }
-        .id-rel-type { color: #888; font-size: 0.85em; }
-        .id-card-warnings { margin-top: 30px; padding-top: 20px; border-top: 1px solid rgba(255,255,255,0.2); display: flex; flex-wrap: wrap; gap: 20px; }
-        .id-warning { display: flex; align-items: center; gap: 8px; font-size: 0.9rem; color: #ff6b6b; }
-        .warning-dot { width: 8px; height: 8px; background: #ff6b6b; border-radius: 50%; flex-shrink: 0; }
-        .warning-desc { color: #ccc; }
-
-        .id-card-manage { margin-top: 20px; background: var(--color-bg-muted); border-radius: 8px; padding: 15px; }
-        .id-card-manage summary { cursor: pointer; font-weight: 600; font-size: 1.1rem; padding: 5px 0; }
-        .id-card-manage[open] summary { margin-bottom: 15px; border-bottom: 1px solid var(--color-border-light); padding-bottom: 10px; }
-        .manage-section { margin-bottom: 25px; }
-        .manage-section h3 { margin: 0 0 15px; font-size: 1rem; }
-        .manage-relationships { display: flex; flex-direction: column; gap: 10px; margin-bottom: 15px; }
-        .manage-rel-item { display: flex; align-items: center; gap: 12px; padding: 10px; background: var(--color-bg-card); border-radius: 6px; flex-wrap: wrap; }
-        .manage-rel-item a { font-weight: 500; min-width: 120px; }
-        .weight-control { display: flex; align-items: center; gap: 8px; font-size: 12px; color: var(--color-text-muted); }
-        .weight-control input[type="range"] { width: 80px; cursor: pointer; }
-        .weight-value { min-width: 20px; text-align: center; font-weight: 600; }
-        .manage-needs-list { list-style: none; padding: 0; margin: 0 0 15px; }
-        .manage-needs-list li { display: flex; align-items: center; gap: 12px; padding: 10px; background: var(--color-bg-card); border-radius: 6px; margin-bottom: 8px; }
-        .manage-needs-list li .btn { margin-left: auto; }
-
-        .htmx-indicator { display: none; }
-        .htmx-request .htmx-indicator { display: inline; }
-        .htmx-request.htmx-indicator { display: inline; }
-
-        @media (max-width: 768px) {
-            .id-card-body { grid-template-columns: 1fr; }
-            .id-card-title { font-size: 1.8rem; }
-            .id-card-header { flex-direction: column; gap: 15px; }
-        }
-    </style>
-</head>
-<body>
-    <div class="app">
-        <nav>
-            <a href="/contacts">Contacts</a>
-            <a href="/graph">Graph</a>
-            <a href="/needs">Needs</a>
-            <button class="btn btn-small theme-toggle" onclick="toggleTheme()">
-                <span id="theme-label">Dark</span>
-            </button>
-        </nav>
-
-        <main id="main-content">
-            {% block content %}{% endblock %}
-        </main>
-    </div>
-
-    <script>
-        function toggleTheme() {
-            const html = document.documentElement;
-            const current = html.getAttribute('data-theme');
-            const next = current === 'light' ? 'dark' : 'light';
-            html.setAttribute('data-theme', next);
-            localStorage.setItem('theme', next);
-            document.getElementById('theme-label').textContent = next === 'light' ? 'Dark' : 'Light';
-        }
-        (function() {
-            const saved = localStorage.getItem('theme') || 'light';
-            document.documentElement.setAttribute('data-theme', saved);
-            document.getElementById('theme-label').textContent = saved === 'light' ? 'Dark' : 'Light';
-        })();
-    </script>
-</body>
-</html>

python/api/templates/contact_detail.html

@@ -1,204 +0,0 @@
-{% extends "base.html" %}
-{% block title %}{{ contact.name }}{% endblock %}
-{% block content %}
-<div class="id-card">
-    <div class="id-card-inner">
-        <div class="id-card-header">
-            <div class="id-card-header-left">
-                <h1 class="id-card-title">I.D.: {{ contact.name }}</h1>
-            </div>
-            <div class="id-card-header-right">
-                {% if contact.profile_pic %}
-                <img src="{{ contact.profile_pic }}" alt="{{ contact.name }}'s profile" class="id-profile-pic">
-                {% else %}
-                <div class="id-profile-placeholder">
-                    <span>{{ contact.name[0]|upper }}</span>
-                </div>
-                {% endif %}
-                <div class="id-card-actions">
-                    <a href="/contacts/{{ contact.id }}/edit" class="btn btn-small">Edit</a>
-                    <a href="/contacts" class="btn btn-small">Back</a>
-                </div>
-            </div>
-        </div>
-
-        <div class="id-card-body">
-            <div class="id-card-left">
-                {% if contact.legal_name %}
-                <div class="id-field">Legal name: {{ contact.legal_name }}</div>
-                {% endif %}
-                {% if contact.suffix %}
-                <div class="id-field">Suffix: {{ contact.suffix }}</div>
-                {% endif %}
-                {% if contact.gender %}
-                <div class="id-field">Gender: {{ contact.gender }}</div>
-                {% endif %}
-                {% if contact.age %}
-                <div class="id-field">Age: {{ contact.age }}</div>
-                {% endif %}
-                {% if contact.current_job %}
-                <div class="id-field">Job: {{ contact.current_job }}</div>
-                {% endif %}
-                {% if contact.social_structure_style %}
-                <div class="id-field">Social style: {{ contact.social_structure_style }}</div>
-                {% endif %}
-                {% if contact.self_sufficiency_score is not none %}
-                <div class="id-field">Self-Sufficiency: {{ contact.self_sufficiency_score }}</div>
-                {% endif %}
-                {% if contact.timezone %}
-                <div class="id-field">Timezone: {{ contact.timezone }}</div>
-                {% endif %}
-                {% if contact.safe_conversation_starters %}
-                <div class="id-field-block">
-                    <span class="id-label">Safe con starters:</span> {{ contact.safe_conversation_starters }}
-                </div>
-                {% endif %}
-                {% if contact.topics_to_avoid %}
-                <div class="id-field-block">
-                    <span class="id-label">Topics to avoid:</span> {{ contact.topics_to_avoid }}
-                </div>
-                {% endif %}
-                {% if contact.goals %}
-                <div class="id-field-block">
-                    <span class="id-label">Goals:</span> {{ contact.goals }}
-                </div>
-                {% endif %}
-            </div>
-
-            <div class="id-card-right">
-                {% if contact.bio %}
-                <div class="id-bio">
-                    <span class="id-label">Bio:</span> {{ contact.bio }}
-                </div>
-                {% endif %}
-
-                <div class="id-relationships">
-                    <h2 class="id-section-title">Relationships</h2>
-
-                    {% if grouped_relationships.familial %}
-                    <div class="id-rel-group">
-                        <span class="id-rel-label">Familial:</span>
-                        {% for rel in grouped_relationships.familial %}
-                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a><span class="id-rel-type">({{ rel.relationship_type|replace("_", " ")|title }})</span>{% if not loop.last %}, {% endif %}
-                        {% endfor %}
-                    </div>
-                    {% endif %}
-
-                    {% if grouped_relationships.partners %}
-                    <div class="id-rel-group">
-                        <span class="id-rel-label">Partners:</span>
-                        {% for rel in grouped_relationships.partners %}
-                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a>{% if not loop.last %}, {% endif %}
-                        {% endfor %}
-                    </div>
-                    {% endif %}
-
-                    {% if grouped_relationships.friends %}
-                    <div class="id-rel-group">
-                        <span class="id-rel-label">Friends:</span>
-                        {% for rel in grouped_relationships.friends %}
-                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a>{% if not loop.last %}, {% endif %}
-                        {% endfor %}
-                    </div>
-                    {% endif %}
-
-                    {% if grouped_relationships.professional %}
-                    <div class="id-rel-group">
-                        <span class="id-rel-label">Professional:</span>
-                        {% for rel in grouped_relationships.professional %}
-                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a><span class="id-rel-type">({{ rel.relationship_type|replace("_", " ")|title }})</span>{% if not loop.last %}, {% endif %}
-                        {% endfor %}
-                    </div>
-                    {% endif %}
-
-                    {% if grouped_relationships.other %}
-                    <div class="id-rel-group">
-                        <span class="id-rel-label">Other:</span>
-                        {% for rel in grouped_relationships.other %}
-                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a><span class="id-rel-type">({{ rel.relationship_type|replace("_", " ")|title }})</span>{% if not loop.last %}, {% endif %}
-                        {% endfor %}
-                    </div>
-                    {% endif %}
-
-                    {% if contact.related_from %}
-                    <div class="id-rel-group">
-                        <span class="id-rel-label">Known by:</span>
-                        {% for rel in contact.related_from %}
-                        <a href="/contacts/{{ rel.contact_id }}">{{ contact_names[rel.contact_id] }}</a>{% if not loop.last %}, {% endif %}
-                        {% endfor %}
-                    </div>
-                    {% endif %}
-                </div>
-            </div>
-        </div>
-
-        {% if contact.needs %}
-        <div class="id-card-warnings">
-            {% for need in contact.needs %}
-            <div class="id-warning">
-                <span class="warning-dot"></span>
-                Warning: {{ need.name }}
-                {% if need.description %}<span class="warning-desc"> - {{ need.description }}</span>{% endif %}
-            </div>
-            {% endfor %}
-        </div>
-        {% endif %}
-    </div>
-
-    <details class="id-card-manage">
-        <summary>Manage Contact</summary>
-
-        <div class="manage-section">
-            <h3>Manage Relationships</h3>
-            <div id="manage-relationships" class="manage-relationships">
-                {% include "partials/manage_relationships.html" %}
-            </div>
-
-            {% if all_contacts %}
-            <form hx-post="/htmx/contacts/{{ contact.id }}/add-relationship"
-                  hx-target="#manage-relationships"
-                  hx-swap="innerHTML"
-                  class="add-form">
-                <select name="related_contact_id" required>
-                    <option value="">Select contact...</option>
-                    {% for other in all_contacts %}
-                    {% if other.id != contact.id %}
-                    <option value="{{ other.id }}">{{ other.name }}</option>
-                    {% endif %}
-                    {% endfor %}
-                </select>
-                <select name="relationship_type" required>
-                    <option value="">Select relationship type...</option>
-                    {% for rel_type in relationship_types %}
-                    <option value="{{ rel_type.value }}">{{ rel_type.display_name }}</option>
-                    {% endfor %}
-                </select>
-                <button type="submit" class="btn btn-primary">Add Relationship</button>
-            </form>
-            {% endif %}
-        </div>
-
-        <div class="manage-section">
-            <h3>Manage Needs/Warnings</h3>
-            <div id="manage-needs">
-                {% include "partials/manage_needs.html" %}
-            </div>
-
-            {% if available_needs %}
-            <form hx-post="/htmx/contacts/{{ contact.id }}/add-need"
-                  hx-target="#manage-needs"
-                  hx-swap="innerHTML"
-                  class="add-form">
-                <select name="need_id" required>
-                    <option value="">Select a need...</option>
-                    {% for need in available_needs %}
-                    <option value="{{ need.id }}">{{ need.name }}</option>
-                    {% endfor %}
-                </select>
-                <button type="submit" class="btn btn-primary">Add Need</button>
-            </form>
-            {% endif %}
-        </div>
-    </details>
-</div>
-{% endblock %}

python/api/templates/contact_form.html

@@ -1,115 +0,0 @@
-{% extends "base.html" %}
-{% block title %}{{ "Edit " + contact.name if contact else "New Contact" }}{% endblock %}
-{% block content %}
-<div class="contact-form">
-    <h1>{{ "Edit Contact" if contact else "New Contact" }}</h1>
-
-    {% if contact %}
-    <form method="post" action="/htmx/contacts/{{ contact.id }}/edit">
-    {% else %}
-    <form method="post" action="/htmx/contacts/new">
-    {% endif %}
-
-        <div class="form-group">
-            <label for="name">Name *</label>
-            <input id="name" name="name" type="text" value="{{ contact.name if contact else '' }}" required>
-        </div>
-
-        <div class="form-row">
-            <div class="form-group">
-                <label for="legal_name">Legal Name</label>
-                <input id="legal_name" name="legal_name" type="text" value="{{ contact.legal_name or '' }}">
-            </div>
-            <div class="form-group">
-                <label for="suffix">Suffix</label>
-                <input id="suffix" name="suffix" type="text" value="{{ contact.suffix or '' }}">
-            </div>
-        </div>
-
-        <div class="form-row">
-            <div class="form-group">
-                <label for="age">Age</label>
-                <input id="age" name="age" type="number" value="{{ contact.age if contact and contact.age is not none else '' }}">
-            </div>
-            <div class="form-group">
-                <label for="gender">Gender</label>
-                <input id="gender" name="gender" type="text" value="{{ contact.gender or '' }}">
-            </div>
-        </div>
-
-        <div class="form-group">
-            <label for="current_job">Current Job</label>
-            <input id="current_job" name="current_job" type="text" value="{{ contact.current_job or '' }}">
-        </div>
-
-        <div class="form-group">
-            <label for="timezone">Timezone</label>
-            <input id="timezone" name="timezone" type="text" value="{{ contact.timezone or '' }}">
-        </div>
-
-        <div class="form-group">
-            <label for="profile_pic">Profile Picture URL</label>
-            <input id="profile_pic" name="profile_pic" type="url" placeholder="https://example.com/photo.jpg" value="{{ contact.profile_pic or '' }}">
-        </div>
-
-        <div class="form-group">
-            <label for="bio">Bio</label>
-            <textarea id="bio" name="bio" rows="3">{{ contact.bio or '' }}</textarea>
-        </div>
-
-        <div class="form-group">
-            <label for="goals">Goals</label>
-            <textarea id="goals" name="goals" rows="3">{{ contact.goals or '' }}</textarea>
-        </div>
-
-        <div class="form-group">
-            <label for="social_structure_style">Social Structure Style</label>
-            <input id="social_structure_style" name="social_structure_style" type="text" value="{{ contact.social_structure_style or '' }}">
-        </div>
-
-        <div class="form-group">
-            <label for="self_sufficiency_score">Self-Sufficiency Score (1-10)</label>
-            <input id="self_sufficiency_score" name="self_sufficiency_score" type="number" min="1" max="10" value="{{ contact.self_sufficiency_score if contact and contact.self_sufficiency_score is not none else '' }}">
-        </div>
-
-        <div class="form-group">
-            <label for="safe_conversation_starters">Safe Conversation Starters</label>
-            <textarea id="safe_conversation_starters" name="safe_conversation_starters" rows="2">{{ contact.safe_conversation_starters or '' }}</textarea>
-        </div>
-
-        <div class="form-group">
-            <label for="topics_to_avoid">Topics to Avoid</label>
-            <textarea id="topics_to_avoid" name="topics_to_avoid" rows="2">{{ contact.topics_to_avoid or '' }}</textarea>
-        </div>
-
-        <div class="form-group">
-            <label for="ssn">SSN</label>
-            <input id="ssn" name="ssn" type="text" value="{{ contact.ssn or '' }}">
-        </div>
-
-        {% if all_needs %}
-        <div class="form-group">
-            <label>Needs/Accommodations</label>
-            <div class="checkbox-group">
-                {% for need in all_needs %}
-                <label class="checkbox-label">
-                    <input type="checkbox" name="need_ids" value="{{ need.id }}"
-                           {% if contact and need in contact.needs %}checked{% endif %}>
-                    {{ need.name }}
-                </label>
-                {% endfor %}
-            </div>
-        </div>
-        {% endif %}
-
-        <div class="form-actions">
-            <button type="submit" class="btn btn-primary">Save</button>
-            {% if contact %}
-            <a href="/contacts/{{ contact.id }}" class="btn">Cancel</a>
-            {% else %}
-            <a href="/contacts" class="btn">Cancel</a>
-            {% endif %}
-        </div>
-    </form>
-</div>
-{% endblock %}

python/api/templates/contact_list.html

@@ -1,14 +0,0 @@
-{% extends "base.html" %}
-{% block title %}Contacts{% endblock %}
-{% block content %}
-<div class="contact-list">
-    <div class="header">
-        <h1>Contacts</h1>
-        <a href="/contacts/new" class="btn btn-primary">Add Contact</a>
-    </div>
-
-    <div id="contact-table">
-        {% include "partials/contact_table.html" %}
-    </div>
-</div>
-{% endblock %}

python/api/templates/graph.html

@@ -1,198 +0,0 @@
-{% extends "base.html" %}
-{% block title %}Relationship Graph{% endblock %}
-{% block content %}
-<div class="graph-container">
-    <div class="header">
-        <h1>Relationship Graph</h1>
-    </div>
-    <p class="graph-hint">Drag nodes to reposition. Closer relationships have shorter, darker edges.</p>
-    <canvas id="graph-canvas" width="900" height="600"
-            style="border: 1px solid var(--color-border); border-radius: 8px; background: var(--color-bg); cursor: grab;">
-    </canvas>
-    <div id="selected-info"></div>
-    <div class="legend">
-        <h4>Relationship Closeness (1-10)</h4>
-        <div class="legend-items">
-            <div class="legend-item">
-                <span class="legend-line" style="background: hsl(220, 70%, 40%); height: 4px; display: inline-block;"></span>
-                <span>10 - Very Close (Spouse, Partner)</span>
-            </div>
-            <div class="legend-item">
-                <span class="legend-line" style="background: hsl(220, 70%, 52%); height: 3px; display: inline-block;"></span>
-                <span>7 - Close (Family, Best Friend)</span>
-            </div>
-            <div class="legend-item">
-                <span class="legend-line" style="background: hsl(220, 70%, 64%); height: 2px; display: inline-block;"></span>
-                <span>4 - Moderate (Friend, Colleague)</span>
-            </div>
-            <div class="legend-item">
-                <span class="legend-line" style="background: hsl(220, 70%, 72%); height: 1px; display: inline-block;"></span>
-                <span>2 - Distant (Acquaintance)</span>
-            </div>
-        </div>
-    </div>
-</div>
-
-<script>
-(function() {
-    const RELATIONSHIP_DISPLAY = {{ relationship_type_display|tojson }};
-    const graphData = {{ graph_data|tojson }};
-
-    const canvas = document.getElementById('graph-canvas');
-    const ctx = canvas.getContext('2d');
-    const width = canvas.width;
-    const height = canvas.height;
-    const centerX = width / 2;
-    const centerY = height / 2;
-
-    const nodes = graphData.nodes.map(function(node) {
-        return Object.assign({}, node, {
-            x: centerX + (Math.random() - 0.5) * 300,
-            y: centerY + (Math.random() - 0.5) * 300,
-            vx: 0,
-            vy: 0
-        });
-    });
-
-    const nodeMap = new Map(nodes.map(function(node) { return [node.id, node]; }));
-
-    const edges = graphData.edges.map(function(edge) {
-        const sourceNode = nodeMap.get(edge.source);
-        const targetNode = nodeMap.get(edge.target);
-        if (!sourceNode || !targetNode) return null;
-        return Object.assign({}, edge, { sourceNode: sourceNode, targetNode: targetNode });
-    }).filter(function(edge) { return edge !== null; });
-
-    let dragNode = null;
-    let selectedNode = null;
-
-    const repulsion = 5000;
-    const springStrength = 0.05;
-    const baseSpringLength = 150;
-    const damping = 0.9;
-    const centerPull = 0.01;
-
-    function simulate() {
-        for (const node of nodes) { node.vx = 0; node.vy = 0; }
-        for (let i = 0; i < nodes.length; i++) {
-            for (let j = i + 1; j < nodes.length; j++) {
-                const dx = nodes[j].x - nodes[i].x;
-                const dy = nodes[j].y - nodes[i].y;
-                const dist = Math.sqrt(dx * dx + dy * dy) || 1;
-                const force = repulsion / (dist * dist);
-                const fx = (dx / dist) * force;
-                const fy = (dy / dist) * force;
-                nodes[i].vx -= fx; nodes[i].vy -= fy;
-                nodes[j].vx += fx; nodes[j].vy += fy;
-            }
-        }
-        for (const edge of edges) {
-            const dx = edge.targetNode.x - edge.sourceNode.x;
-            const dy = edge.targetNode.y - edge.sourceNode.y;
-            const dist = Math.sqrt(dx * dx + dy * dy) || 1;
-            const normalizedWeight = edge.closeness_weight / 10;
-            const idealLength = baseSpringLength * (1.5 - normalizedWeight);
-            const displacement = dist - idealLength;
-            const force = springStrength * displacement;
-            const fx = (dx / dist) * force;
-            const fy = (dy / dist) * force;
-            edge.sourceNode.vx += fx; edge.sourceNode.vy += fy;
-            edge.targetNode.vx -= fx; edge.targetNode.vy -= fy;
-        }
-        for (const node of nodes) {
-            node.vx += (centerX - node.x) * centerPull;
-            node.vy += (centerY - node.y) * centerPull;
-        }
-        for (const node of nodes) {
-            if (node === dragNode) continue;
-            node.x += node.vx * damping;
-            node.y += node.vy * damping;
-            node.x = Math.max(30, Math.min(width - 30, node.x));
-            node.y = Math.max(30, Math.min(height - 30, node.y));
-        }
-    }
-
-    function getEdgeColor(weight) {
-        const normalized = weight / 10;
-        return 'hsl(220, 70%, ' + (80 - normalized * 40) + '%)';
-    }
-
-    function draw() {
-        ctx.clearRect(0, 0, width, height);
-        for (const edge of edges) {
-            const lineWidth = 1 + (edge.closeness_weight / 10) * 3;
-            ctx.strokeStyle = getEdgeColor(edge.closeness_weight);
-            ctx.lineWidth = lineWidth;
-            ctx.beginPath();
-            ctx.moveTo(edge.sourceNode.x, edge.sourceNode.y);
-            ctx.lineTo(edge.targetNode.x, edge.targetNode.y);
-            ctx.stroke();
-            const midX = (edge.sourceNode.x + edge.targetNode.x) / 2;
-            const midY = (edge.sourceNode.y + edge.targetNode.y) / 2;
-            ctx.fillStyle = '#666';
-            ctx.font = '10px sans-serif';
-            ctx.textAlign = 'center';
-            const label = RELATIONSHIP_DISPLAY[edge.relationship_type] || edge.relationship_type;
-            ctx.fillText(label, midX, midY - 5);
-        }
-        for (const node of nodes) {
-            const isSelected = node === selectedNode;
-            const radius = isSelected ? 25 : 20;
-            ctx.beginPath();
-            ctx.arc(node.x, node.y, radius, 0, Math.PI * 2);
-            ctx.fillStyle = isSelected ? '#0066cc' : '#fff';
-            ctx.fill();
-            ctx.strokeStyle = '#0066cc';
-            ctx.lineWidth = 2;
-            ctx.stroke();
-            ctx.fillStyle = isSelected ? '#fff' : '#333';
-            ctx.font = '12px sans-serif';
-            ctx.textAlign = 'center';
-            ctx.textBaseline = 'middle';
-            const name = node.name.length > 10 ? node.name.slice(0, 9) + '\u2026' : node.name;
-            ctx.fillText(name, node.x, node.y);
-        }
-    }
-
-    function animate() {
-        simulate();
-        draw();
-        requestAnimationFrame(animate);
-    }
-    animate();
-
-    function getNodeAt(x, y) {
-        for (const node of nodes) {
-            const dx = x - node.x;
-            const dy = y - node.y;
-            if (dx * dx + dy * dy < 400) return node;
-        }
-        return null;
-    }
-
-    canvas.addEventListener('mousedown', function(event) {
-        const rect = canvas.getBoundingClientRect();
-        const node = getNodeAt(event.clientX - rect.left, event.clientY - rect.top);
-        if (node) {
-            dragNode = node;
-            selectedNode = node;
-            const infoDiv = document.getElementById('selected-info');
-            let html = '<div class="selected-info"><h3>' + node.name + '</h3>';
-            if (node.current_job) html += '<p>Job: ' + node.current_job + '</p>';
-            html += '<a href="/contacts/' + node.id + '">View details</a></div>';
-            infoDiv.innerHTML = html;
-        }
-    });
-
-    canvas.addEventListener('mousemove', function(event) {
-        if (!dragNode) return;
-        const rect = canvas.getBoundingClientRect();
-        dragNode.x = event.clientX - rect.left;
-        dragNode.y = event.clientY - rect.top;
-    });
-
-    canvas.addEventListener('mouseup', function() { dragNode = null; });
-    canvas.addEventListener('mouseleave', function() { dragNode = null; });
-})();
-</script>
-{% endblock %}

python/api/templates/need_list.html

@@ -1,31 +0,0 @@
-{% extends "base.html" %}
-{% block title %}Needs{% endblock %}
-{% block content %}
-<div class="need-list">
-    <div class="header">
-        <h1>Needs / Accommodations</h1>
-        <button class="btn btn-primary" onclick="document.getElementById('need-form').toggleAttribute('hidden')">Add Need</button>
-    </div>
-
-    <form id="need-form" hidden
-          hx-post="/htmx/needs"
-          hx-target="#need-items"
-          hx-swap="innerHTML"
-          hx-on::after-request="if(event.detail.successful) this.reset()"
-          class="need-form">
-        <div class="form-group">
-            <label for="name">Name *</label>
-            <input id="name" name="name" type="text" placeholder="e.g., Light Sensitive, ADHD" required>
-        </div>
-        <div class="form-group">
-            <label for="description">Description</label>
-            <textarea id="description" name="description" placeholder="Optional description..." rows="2"></textarea>
-        </div>
-        <button type="submit" class="btn btn-primary">Create</button>
-    </form>
-
-    <div id="need-items">
-        {% include "partials/need_items.html" %}
-    </div>
-</div>
-{% endblock %}

python/api/templates/partials/contact_table.html

@@ -1,33 +0,0 @@
-{% if contacts %}
-<table>
-    <thead>
-        <tr>
-            <th>Name</th>
-            <th>Job</th>
-            <th>Timezone</th>
-            <th>Actions</th>
-        </tr>
-    </thead>
-    <tbody>
-        {% for contact in contacts %}
-        <tr id="contact-row-{{ contact.id }}">
-            <td><a href="/contacts/{{ contact.id }}">{{ contact.name }}</a></td>
-            <td>{{ contact.current_job or "-" }}</td>
-            <td>{{ contact.timezone or "-" }}</td>
-            <td>
-                <a href="/contacts/{{ contact.id }}/edit" class="btn">Edit</a>
-                <button class="btn btn-danger"
-                        hx-delete="/api/contacts/{{ contact.id }}"
-                        hx-target="#contact-row-{{ contact.id }}"
-                        hx-swap="outerHTML"
-                        hx-confirm="Delete this contact?">
-                    Delete
-                </button>
-            </td>
-        </tr>
-        {% endfor %}
-    </tbody>
-</table>
-{% else %}
-<p>No contacts yet.</p>
-{% endif %}

python/api/templates/partials/manage_needs.html

@@ -1,14 +0,0 @@
-<ul class="manage-needs-list">
-    {% for need in contact.needs %}
-    <li id="contact-need-{{ need.id }}">
-        <strong>{{ need.name }}</strong>
-        {% if need.description %}<span> - {{ need.description }}</span>{% endif %}
-        <button class="btn btn-small btn-danger"
-                hx-delete="/api/contacts/{{ contact.id }}/needs/{{ need.id }}"
-                hx-target="#contact-need-{{ need.id }}"
-                hx-swap="outerHTML">
-            Remove
-        </button>
-    </li>
-    {% endfor %}
-</ul>

python/api/templates/partials/manage_relationships.html

@@ -1,23 +0,0 @@
-{% for rel in contact.related_to %}
-<div class="manage-rel-item" id="rel-{{ contact.id }}-{{ rel.related_contact_id }}">
-    <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a>
-    <span class="tag">{{ rel.relationship_type|replace("_", " ")|title }}</span>
-    <label class="weight-control">
-        <span>Closeness:</span>
-        <input type="range" min="1" max="10" value="{{ rel.closeness_weight }}"
-               hx-post="/htmx/contacts/{{ contact.id }}/relationships/{{ rel.related_contact_id }}/weight"
-               hx-trigger="change"
-               hx-include="this"
-               name="closeness_weight"
-               hx-swap="none"
-               oninput="this.nextElementSibling.textContent = this.value">
-        <span class="weight-value">{{ rel.closeness_weight }}</span>
-    </label>
-    <button class="btn btn-small btn-danger"
-            hx-delete="/api/contacts/{{ contact.id }}/relationships/{{ rel.related_contact_id }}"
-            hx-target="#rel-{{ contact.id }}-{{ rel.related_contact_id }}"
-            hx-swap="outerHTML">
-        Remove
-    </button>
-</div>
-{% endfor %}

python/api/templates/partials/need_items.html

@@ -1,21 +0,0 @@
-{% if needs %}
-<ul class="need-items">
-    {% for need in needs %}
-    <li id="need-item-{{ need.id }}">
-        <div class="need-info">
-            <strong>{{ need.name }}</strong>
-            {% if need.description %}<p>{{ need.description }}</p>{% endif %}
-        </div>
-        <button class="btn btn-danger"
-                hx-delete="/api/needs/{{ need.id }}"
-                hx-target="#need-item-{{ need.id }}"
-                hx-swap="outerHTML"
-                hx-confirm="Delete this need?">
-            Delete
-        </button>
-    </li>
-    {% endfor %}
-</ul>
-{% else %}
-<p>No needs defined yet.</p>
-{% endif %}

python/common.py

@@ -4,16 +4,19 @@ from __future__ import annotations
 
 import logging
 from datetime import UTC, datetime
-from os import getenv
+from pathlib import Path
 from subprocess import PIPE, Popen
 
-from apprise import Apprise
-
 from python.logging_config import configure_logger as _configure_logger
 
 logger = logging.getLogger(__name__)
 
 
+def get_repo_dir() -> Path:
+    """Return the repository root directory."""
+    return Path(__file__).resolve().parents[1]
+
+
 def configure_logger(level: str = "INFO") -> None:
     """Configure the logger."""
     _configure_logger(level)
@@ -39,26 +42,6 @@ def bash_wrapper(command: str) -> tuple[str, int]:
     return output.decode(), process.returncode
 
 
-def signal_alert(body: str, title: str = "") -> None:
-    """Send a signal alert.
-
-    Args:
-        body (str): The body of the alert.
-        title (str, optional): The title of the alert. Defaults to "".
-    """
-    apprise_client = Apprise()
-
-    from_phone = getenv("SIGNAL_ALERT_FROM_PHONE")
-    to_phone = getenv("SIGNAL_ALERT_TO_PHONE")
-    if not from_phone or not to_phone:
-        logger.info("SIGNAL_ALERT_FROM_PHONE or SIGNAL_ALERT_TO_PHONE not set")
-        return
-
-    apprise_client.add(f"signal://localhost:8989/{from_phone}/{to_phone}")
-
-    apprise_client.notify(title=title, body=body)
-
-
 def utcnow() -> datetime:
     """Get the current UTC time."""
     return datetime.now(tz=UTC)

python/ebook_search/answer.py

@@ -6,15 +6,23 @@ import logging
 from typing import TYPE_CHECKING
 
 from python.ebook_search.llm_interface import request_chat_completion
+from python.ebook_search.prompts import load_prompt
 
 if TYPE_CHECKING:
+    import httpx
+
     from python.ebook_search.config import EbookSearchConfig
     from python.ebook_search.search import SearchResult
 
 logger = logging.getLogger(__name__)
 
 
-def answer_query(query: str, results: list[SearchResult], config: EbookSearchConfig) -> str:
+async def answer_query(
+    client: httpx.AsyncClient,
+    query: str,
+    results: list[SearchResult],
+    config: EbookSearchConfig,
+) -> str:
     """Answer a question using only retrieved chunks."""
     if not config.answer_enabled:
         logger.info("ebook_answer_skipped_disabled")
@@ -25,33 +33,18 @@ def answer_query(query: str, results: list[SearchResult], config: EbookSearchCon
         return "No relevant sources were found."
 
     logger.info(
-        "ebook_answer_request_start base_url=%s model=%s sources=%s query_length=%s",
-        config.vllm_base_url,
-        config.chat_model,
-        len(results),
-        len(query),
+        f"ebook_answer_request_start {config.vllm_base_url=} {config.chat_model=} sources={len(results)} "
+        f"query_length={len(query)}"
     )
     context = "\n\n".join(
         f"[{index}] {result.source_title}{' - ' + result.chapter_title if result.chapter_title else ''}\n{result.text}"
         for index, result in enumerate(results, start=1)
     )
-    content = request_chat_completion(
+    content = await request_chat_completion(
+        client,
         config,
-        [
-            {
-                "role": "system",
-                "content": (
-                    "Answer only from the provided context. Cite sources with bracketed numbers like [1]. "
-                    "If the context is insufficient, say so."
-                ),
-            },
-            {"role": "user", "content": f"Question:\n{query}\n\nContext:\n{context}"},
-        ],
+        load_prompt("answer").messages(query=query, context=context),
     )
 
-    logger.info(
-        "ebook_answer_request_complete model=%s answer_length=%s",
-        config.chat_model,
-        len(content),
-    )
+    logger.info(f"ebook_answer_request_complete {config.chat_model=} answer_length={len(content)}")
     return content or "The model returned an empty answer."

python/ebook_search/api/bm25_tasks.py

@@ -1,18 +1,22 @@
-"""Background BM25 refresh tasks for the web app."""
+"""Background BM25 refresh tasks for the web app.
+
+The refresh is scheduled on the event loop instead of a thread because the async psycopg
+driver only works from the loop; a bare thread cannot open a session on the async engine.
+"""
 
 from __future__ import annotations
 
+import asyncio
 import logging
-from threading import Timer
 from typing import TYPE_CHECKING
 
-from sqlalchemy.orm import Session
+from sqlalchemy.ext.asyncio import AsyncSession
 
 from python.ebook_search.bm25_corpus import load_bm25_corpus, refresh_bm25_corpus
 
 if TYPE_CHECKING:
     from fastapi import FastAPI
-    from sqlalchemy.engine import Engine
+    from sqlalchemy.ext.asyncio import AsyncEngine
 
     from python.ebook_search.config import EbookSearchConfig
 
@@ -20,41 +24,47 @@ logger = logging.getLogger(__name__)
 
 
 def schedule_bm25_refresh(app: FastAPI) -> None:
-    """Schedule a delayed BM25 corpus refresh, replacing any pending refresh."""
-    existing_timer = getattr(app.state, "bm25_refresh_timer", None)
-    if existing_timer is not None:
-        existing_timer.cancel()
+    """Schedule a delayed BM25 corpus refresh, replacing any pending refresh.
 
-    timer = Timer(app.state.config.bm25_refresh_delay_seconds, refresh_bm25_for_app, args=(app,))
-    timer.daemon = True
-    timer.start()
-    app.state.bm25_refresh_timer = timer
-    logger.info(
-        "ebook_bm25_refresh_scheduled delay_seconds=%s",
-        app.state.config.bm25_refresh_delay_seconds,
-    )
+    Only called from route handlers, so a running event loop is guaranteed.
+    """
+    cancel_bm25_refresh(app)
+
+    loop = asyncio.get_running_loop()
+
+    def start_refresh() -> None:
+        app.state.bm25_refresh_task = loop.create_task(refresh_bm25_for_app(app))
+
+    app.state.bm25_refresh_timer = loop.call_later(app.state.config.bm25_refresh_delay_seconds, start_refresh)
+    logger.info(f"ebook_bm25_refresh_scheduled {app.state.config.bm25_refresh_delay_seconds=}")
 
 
 def cancel_bm25_refresh(app: FastAPI) -> None:
-    """Cancel any pending BM25 corpus refresh."""
+    """Cancel any pending BM25 corpus refresh timer and in-flight refresh task."""
     existing_timer = getattr(app.state, "bm25_refresh_timer", None)
     if existing_timer is not None:
         existing_timer.cancel()
         app.state.bm25_refresh_timer = None
         logger.info("ebook_bm25_refresh_cancelled")
 
+    existing_task = getattr(app.state, "bm25_refresh_task", None)
+    if existing_task is not None:
+        if not existing_task.done():
+            existing_task.cancel()
+        app.state.bm25_refresh_task = None
 
-def refresh_bm25_for_app(app: FastAPI) -> None:
+
+async def refresh_bm25_for_app(app: FastAPI) -> None:
     """Refresh the BM25 corpus using the app engine and config."""
     try:
-        refresh_bm25_for_engine(app.state.engine, app.state.config)
+        await refresh_bm25_for_engine(app.state.engine, app.state.config)
     except Exception:
         logger.exception("ebook_bm25_refresh_failed")
 
 
-def refresh_bm25_for_engine(engine: Engine, config: EbookSearchConfig) -> None:
-    """Refresh the BM25 corpus using a SQLAlchemy engine."""
-    with Session(engine) as session:
-        refresh_bm25_corpus(session, config)
+async def refresh_bm25_for_engine(engine: AsyncEngine, config: EbookSearchConfig) -> None:
+    """Refresh the BM25 corpus using an async SQLAlchemy engine."""
+    async with AsyncSession(engine) as session:
+        await refresh_bm25_corpus(session, config)
     load_bm25_corpus.cache_clear()
     logger.info("ebook_bm25_corpus_cache_cleared_after_refresh")

python/ebook_search/api/dependencies.py

@@ -4,8 +4,8 @@ from __future__ import annotations
 
 from typing import Annotated
 
+import httpx
 from fastapi import Depends, Request
-from sqlalchemy.engine import Engine
 
 from python.ebook_search.config import EbookSearchConfig
 
@@ -15,10 +15,10 @@ def get_config(request: Request) -> EbookSearchConfig:
     return request.app.state.config
 
 
-def get_engine(request: Request) -> Engine:
-    """Get the database engine from app state."""
-    return request.app.state.engine
+def get_http_client(request: Request) -> httpx.AsyncClient:
+    """Get the shared LLM HTTP client from app state."""
+    return request.app.state.http_client
 
 
 AppConfig = Annotated[EbookSearchConfig, Depends(get_config)]
-AppEngine = Annotated[Engine, Depends(get_engine)]
+AppHttpClient = Annotated[httpx.AsyncClient, Depends(get_http_client)]

python/ebook_search/api/judge_tasks.py

@@ -0,0 +1,127 @@
+"""Background phrase-judging tasks for the web app.
+
+Judging a book sends one LLM request per candidate phrase, which can take minutes, so it must
+not run inside the request where it would block the UI. Judgments run as async FastAPI
+background tasks, awaited on the event loop after the response is sent, and are tracked per
+book in app state so a second judge request for a book that is already being judged is
+rejected instead of doubling the work.
+
+State is loop-confined: every read and mutation happens on the event loop (async route
+handlers and async background tasks) and no critical section contains an ``await``, so each
+mutation is atomic per loop iteration and no locking is needed.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
+
+if TYPE_CHECKING:
+    from fastapi import BackgroundTasks, FastAPI
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class JudgeTaskState:
+    """Running book judgments and last outcome messages, keyed by book id."""
+
+    running_book_ids: set[int] = field(default_factory=set)
+    outcome_messages: dict[int, str] = field(default_factory=dict)
+
+
+def get_judge_task_state(app: FastAPI) -> JudgeTaskState:
+    """Return the app's judge task state, creating it on first use.
+
+    Args:
+        app (FastAPI): App whose state holds the judge task registry.
+
+    Returns:
+        JudgeTaskState: The shared judge task state for this app.
+    """
+    state = getattr(app.state, "judge_tasks", None)
+    if state is None:
+        state = JudgeTaskState()
+        app.state.judge_tasks = state
+    return state
+
+
+def start_book_phrase_judgment(app: FastAPI, background_tasks: BackgroundTasks, source_id: int) -> bool:
+    """Queue judging of one book's candidate phrases as a FastAPI background task.
+
+    The book is claimed before the response returns, so a repeated judge request cannot queue
+    a second run while one is pending or running.
+
+    Args:
+        app (FastAPI): App supplying the engine, config, and judge task state.
+        background_tasks (BackgroundTasks): Request's background tasks to queue the judgment on.
+        source_id (int): Book to judge candidates for.
+
+    Returns:
+        bool: True when a judgment was queued, False when one is already running for this book.
+    """
+    state = get_judge_task_state(app)
+    if source_id in state.running_book_ids:
+        logger.info(f"ebook_book_phrase_judgment_already_running {source_id=}")
+        return False
+    state.running_book_ids.add(source_id)
+    state.outcome_messages.pop(source_id, None)
+    background_tasks.add_task(judge_book_phrases_for_app, app, source_id)
+    logger.info(f"ebook_book_phrase_judgment_queued {source_id=}")
+    return True
+
+
+async def judge_book_phrases_for_app(app: FastAPI, source_id: int) -> None:
+    """Judge one book using the app engine and config, recording the outcome message.
+
+    Args:
+        app (FastAPI): App supplying the engine, config, and judge task state.
+        source_id (int): Book to judge candidates for.
+    """
+    state = get_judge_task_state(app)
+    try:
+        result = await judge_candidate_phrases_for_books(app.state.engine, app.state.config, source_ids=[source_id])
+        logger.info(
+            f"ebook_book_phrase_judgment_complete {source_id=} {result.candidates_judged=} {result.protected_phrases=} "
+            f"{result.phrase_mentions=} {result.books_failed=}"
+        )
+        if result.books_failed:
+            message = "Judging failed; see server logs for details"
+        else:
+            message = (
+                f"Judged {result.candidates_judged} candidates; {result.protected_phrases} protected phrases promoted"
+            )
+    except Exception:
+        logger.exception(f"ebook_book_phrase_judgment_task_failed {source_id=}")
+        message = "Judging failed; see server logs for details"
+    state.running_book_ids.discard(source_id)
+    state.outcome_messages[source_id] = message
+
+
+def is_judging_book(app: FastAPI, source_id: int) -> bool:
+    """Report whether a judgment is currently queued or running for one book.
+
+    Args:
+        app (FastAPI): App supplying the judge task state.
+        source_id (int): Book to check.
+
+    Returns:
+        bool: True while the book's judgment is pending or running.
+    """
+    return source_id in get_judge_task_state(app).running_book_ids
+
+
+def pop_book_judgment_outcome(app: FastAPI, source_id: int) -> str | None:
+    """Return and clear the outcome message from one book's last finished judgment.
+
+    Args:
+        app (FastAPI): App supplying the judge task state.
+        source_id (int): Book to fetch the outcome for.
+
+    Returns:
+        str | None: The outcome message, or None when there is nothing new to report.
+    """
+    return get_judge_task_state(app).outcome_messages.pop(source_id, None)

python/ebook_search/api/main.py

@@ -6,11 +6,12 @@ import logging
 from contextlib import asynccontextmanager
 from typing import TYPE_CHECKING, Annotated
 
+import httpx
 import typer
 import uvicorn
 from fastapi import FastAPI
 from fastapi.staticfiles import StaticFiles
-from sqlalchemy.orm import Session
+from sqlalchemy.ext.asyncio import AsyncSession
 
 from python.common import configure_logger
 from python.ebook_search.api.bm25_tasks import cancel_bm25_refresh
@@ -18,8 +19,9 @@ from python.ebook_search.api.routes import admin_router, health_router, page_rou
 from python.ebook_search.api.web import STATIC_DIR
 from python.ebook_search.bm25_corpus import ensure_bm25_corpus
 from python.ebook_search.config import load_config
+from python.ebook_search.protected_phrases.pool import shutdown_extraction_pool
 from python.fastapi_tools import ZstdMiddleware
-from python.orm.common import get_postgres_engine
+from python.orm.common import get_async_postgres_engine
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
@@ -35,27 +37,30 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     config = load_config()
     app.state.config = config
     logger.info(
-        "ebook_search_config_loaded top_k=%s embedding_model=%s embedding_base_url=%s vllm_base_url=%s "
-        "rerank_enabled=%s answer_enabled=%s library_paths=%s",
-        config.top_k,
-        config.embedding_model,
-        config.embedding_base_url,
-        config.vllm_base_url,
-        config.rerank.enabled,
-        config.answer_enabled,
-        len(config.library_paths),
+        f"ebook_search_config_loaded {config.top_k=} {config.embedding_model=} {config.embedding_base_url=} "
+        f"{config.vllm_base_url=} {config.rerank.enabled=} {config.phrase_matching_enabled=} {config.answer_enabled=} "
+        f"library_paths={len(config.library_paths)}"
     )
     if not config.library_paths:
         logger.warning("ebook_search_no_library_paths_configured")
-    app.state.engine = get_postgres_engine(name="RICHIE", vector_engine=True)
-    with Session(app.state.engine) as session:
-        ensure_bm25_corpus(session, config)
+    # Concurrent phrase judging opens one session per book worker on this engine, so size the pool
+    # to cover those plus headroom for ordinary web requests.
+    app.state.engine = get_async_postgres_engine(
+        name="RICHIE",
+        vector_engine=True,
+        pool_size=config.phrase_judge_book_workers + 10,
+    )
+    app.state.http_client = httpx.AsyncClient()
+    async with AsyncSession(app.state.engine, expire_on_commit=False) as session:
+        await ensure_bm25_corpus(session, config)
     try:
         yield
     finally:
         logger.info("ebook_search_shutdown")
         cancel_bm25_refresh(app)
-        app.state.engine.dispose()
+        shutdown_extraction_pool()
+        await app.state.http_client.aclose()
+        await app.state.engine.dispose()
 
 
 def create_app() -> FastAPI:

python/ebook_search/api/routes/admin.py

@@ -8,13 +8,20 @@ from fastapi import APIRouter, Request
 from fastapi.responses import HTMLResponse
 
 from python.ebook_search.api.bm25_tasks import schedule_bm25_refresh
-from python.ebook_search.api.dependencies import (
-    AppConfig,  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+from python.ebook_search.api.dependencies import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
+    AppConfig,
+    AppHttpClient,
 )
-from python.ebook_search.api.web import templates
+from python.ebook_search.api.web import error_response, templates
 from python.ebook_search.embeddings import embed_missing_chunks, embedding_model_stats
 from python.ebook_search.ingest import ingest_configured_paths
-from python.fastapi_tools import DbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+from python.ebook_search.protected_phrases.generate_ngrams import generate_candidate_phrases_for_books
+from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
+from python.ebook_search.protected_phrases.store import book_ids_pending_first_judgment, corpus_phrase_stats
+from python.fastapi_tools import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
+    AppAsyncEngine,
+    AsyncDbSession,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -22,40 +29,143 @@ router = APIRouter(prefix="/admin")
 
 
 @router.get("", response_class=HTMLResponse)
-def admin(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
+async def admin(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
     """Render the admin page."""
-    stats = embedding_model_stats(session)
-    logger.info("ebook_admin_page_loaded models=%s", len(stats))
-    return templates.TemplateResponse(request, "admin.html", {"config": config, "stats": stats})
+    stats = await embedding_model_stats(session)
+    phrase_stats = await corpus_phrase_stats(session)
+    logger.info(
+        f"ebook_admin_page_loaded models={len(stats)} {phrase_stats.candidate_phrases=} "
+        f"{phrase_stats.protected_phrases=}"
+    )
+    return templates.TemplateResponse(
+        request,
+        "admin.html",
+        {"config": config, "stats": stats, "phrase_stats": phrase_stats},
+    )
 
 
 @router.post("/scan", response_class=HTMLResponse)
-def scan_library(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
+async def scan_library(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
     """Scan configured library paths for EPUB changes."""
     try:
-        count = ingest_configured_paths(session, config)
-        session.commit()
+        count = await ingest_configured_paths(session, config)
+        await session.commit()
     except Exception as error:
         logger.exception("ebook_admin_scan_failed")
-        return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
+        return error_response(request, error)
 
-    logger.info("ebook_admin_scan_complete changed_files=%s", count)
+    logger.info(f"ebook_admin_scan_complete {count=}")
     if count > 0:
         schedule_bm25_refresh(request.app)
     return templates.TemplateResponse(request, "partials/admin_status.html", {"message": f"Indexed {count} EPUBs"})
 
 
+@router.post("/phrases/generate-all", response_class=HTMLResponse)
+async def generate_all_phrases(request: Request, config: AppConfig, engine: AppAsyncEngine) -> HTMLResponse:
+    """Regenerate candidate phrases for every indexed book without LLM judging."""
+    try:
+        result = await generate_candidate_phrases_for_books(engine, config)
+    except Exception as error:
+        logger.exception("ebook_admin_generate_phrases_failed")
+        return error_response(request, error)
+
+    logger.info(
+        f"ebook_admin_generate_phrases_complete {result.books_seen=} {result.books_built=} {result.candidate_phrases=}"
+    )
+    return templates.TemplateResponse(
+        request,
+        "partials/admin_status.html",
+        {
+            "message": (
+                f"Generated phrases for {result.books_built} of {result.books_seen} books; "
+                f"{result.candidate_phrases} candidates stored"
+            )
+        },
+    )
+
+
+@router.post("/phrases/judge-all", response_class=HTMLResponse)
+async def judge_all_phrases(request: Request, engine: AppAsyncEngine, config: AppConfig) -> HTMLResponse:
+    """Judge unjudged candidate phrases across every indexed book."""
+    return await run_phrase_judgment(request, engine, config, source_ids=None)
+
+
+@router.post("/phrases/judge-missing", response_class=HTMLResponse)
+async def judge_missing_phrases(
+    request: Request,
+    engine: AppAsyncEngine,
+    config: AppConfig,
+    session: AsyncDbSession,
+) -> HTMLResponse:
+    """Judge candidate phrases only for books where judging has never run."""
+    source_ids = await book_ids_pending_first_judgment(session)
+    if not source_ids:
+        return templates.TemplateResponse(
+            request,
+            "partials/admin_status.html",
+            {"message": "All books with candidate phrases have been judged"},
+        )
+    return await run_phrase_judgment(request, engine, config, source_ids=source_ids)
+
+
+async def run_phrase_judgment(
+    request: Request,
+    engine: AppAsyncEngine,
+    config: AppConfig,
+    *,
+    source_ids: list[int] | None,
+) -> HTMLResponse:
+    """Run LLM judging for candidate phrases and render the outcome as an admin status partial.
+
+    Args:
+        request (Request): Current request, for template rendering.
+        engine (AppAsyncEngine): Engine used to open per-book judging sessions.
+        config (AppConfig): Runtime phrase-tuning settings.
+        source_ids (list[int] | None): Books to judge; ``None`` judges every indexed book.
+
+    Returns:
+        HTMLResponse: Status partial describing the judging outcome.
+    """
+    try:
+        result = await judge_candidate_phrases_for_books(engine, config, source_ids=source_ids)
+    except Exception as error:
+        logger.exception("ebook_admin_judge_phrases_failed")
+        return error_response(request, error)
+
+    logger.info(
+        f"ebook_admin_judge_phrases_complete {result.books_seen=} {result.books_judged=} {result.books_failed=} "
+        f"{result.candidates_judged=} {result.protected_phrases=} {result.phrase_mentions=}"
+    )
+    return templates.TemplateResponse(
+        request,
+        "partials/admin_status.html",
+        {
+            "message": (
+                f"Judged {result.candidates_judged} candidates across {result.books_judged} of "
+                f"{result.books_seen} books; {result.protected_phrases} protected phrases, "
+                f"{result.phrase_mentions} mentions"
+                + (f"; {result.books_failed} books failed" if result.books_failed else "")
+            )
+        },
+    )
+
+
 @router.post("/embed-missing", response_class=HTMLResponse)
-def embed_missing(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
+async def embed_missing(
+    request: Request,
+    config: AppConfig,
+    session: AsyncDbSession,
+    client: AppHttpClient,
+) -> HTMLResponse:
     """Embed chunks missing vectors for the configured model."""
     try:
-        count = embed_missing_chunks(session, config)
-        session.commit()
+        count = await embed_missing_chunks(session, client, config)
+        await session.commit()
     except Exception as error:
         logger.exception("ebook_admin_embed_missing_failed")
-        return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
+        return error_response(request, error)
 
-    logger.info("ebook_admin_embed_missing_complete chunks=%s", count)
+    logger.info(f"ebook_admin_embed_missing_complete {count=}")
     return templates.TemplateResponse(
         request,
         "partials/admin_status.html",
@@ -64,38 +174,29 @@ def embed_missing(request: Request, config: AppConfig, session: DbSession) -> HT
 
 
 @router.post("/embed-all", response_class=HTMLResponse)
-def embed_all(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
+async def embed_all(
+    request: Request,
+    config: AppConfig,
+    session: AsyncDbSession,
+    client: AppHttpClient,
+) -> HTMLResponse:
     """Embed all chunks missing vectors in fixed-size batches."""
     total = 0
     batches = 0
     try:
         while True:
-            count = embed_missing_chunks(session, config)
+            count = await embed_missing_chunks(session, client, config)
             if count == 0:
                 break
-            session.commit()
+            await session.commit()
             total += count
             batches += 1
-            logger.info(
-                "ebook_admin_embed_all_batch_complete batch=%s chunks=%s total_chunks=%s",
-                batches,
-                count,
-                total,
-            )
+            logger.info(f"ebook_admin_embed_all_batch_complete {batches=} {count=} {total=}")
     except Exception as error:
-        logger.exception(
-            "ebook_admin_embed_all_failed batches=%s chunks=%s",
-            batches,
-            total,
-        )
-        return templates.TemplateResponse(
-            request,
-            "partials/error.html",
-            {"message": f"Embed all failed after {total} chunks in {batches} batches: {error}"},
-            status_code=500,
-        )
+        logger.exception(f"ebook_admin_embed_all_failed {batches=} {total=}")
+        return error_response(request, f"Embed all failed after {total} chunks in {batches} batches: {error}")
 
-    logger.info("ebook_admin_embed_all_complete batches=%s chunks=%s", batches, total)
+    logger.info(f"ebook_admin_embed_all_complete {batches=} {total=}")
     return templates.TemplateResponse(
         request,
         "partials/admin_status.html",

python/ebook_search/api/routes/health.py

@@ -11,15 +11,17 @@ from fastapi.responses import JSONResponse
 from sqlalchemy import literal, select
 from sqlalchemy.exc import SQLAlchemyError
 
-from python.ebook_search.api.dependencies import (
-    AppConfig,  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+from python.ebook_search.api.dependencies import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
+    AppConfig,
+    AppHttpClient,
 )
 from python.ebook_search.bm25_corpus import bm25_index_exists, bm25_index_path, read_bm25_manifest
 from python.ebook_search.llm_interface import check_chat_endpoint, check_embedding_endpoint
-from python.fastapi_tools import DbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+from python.fastapi_tools import AsyncDbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
 
 if TYPE_CHECKING:
-    from sqlalchemy.orm import Session
+    import httpx
+    from sqlalchemy.ext.asyncio import AsyncSession
 
     from python.ebook_search.config import EbookSearchConfig
 
@@ -29,17 +31,17 @@ router = APIRouter()
 
 
 @router.get("/health")
-def health() -> dict[str, str]:
+async def health() -> dict[str, str]:
     """Liveness probe that returns ok without touching dependencies."""
     return {"status": "ok"}
 
 
 @router.get("/ready")
-def ready(config: AppConfig, session: DbSession) -> JSONResponse:
+async def ready(config: AppConfig, session: AsyncDbSession, client: AppHttpClient) -> JSONResponse:
     """Readiness probe reporting database, embedding endpoint, and BM25 index status."""
-    database_ok = check_database(session)
-    embedding_ok = check_embedding_endpoint(config)
-    chat_status = chat_endpoint_status(config)
+    database_ok = await check_database(session)
+    embedding_ok = await check_embedding_endpoint(client, config)
+    chat_status = await chat_endpoint_status(client, config)
     bm25_status = check_bm25_status(config)
 
     checks = {
@@ -58,30 +60,23 @@ def ready(config: AppConfig, session: DbSession) -> JSONResponse:
         status = "ready"
         status_code = HTTPStatus.OK
 
-    logger.info(
-        "ebook_ready_check status=%s database=%s embedding=%s chat=%s bm25=%s",
-        status,
-        database_ok,
-        embedding_ok,
-        chat_status,
-        bm25_status,
-    )
+    logger.info(f"ebook_ready_check {status=} {database_ok=} {embedding_ok=} {chat_status=} {bm25_status=}")
     return JSONResponse(content={"status": status, "checks": checks}, status_code=status_code)
 
 
-def chat_endpoint_status(config: EbookSearchConfig) -> str:
+async def chat_endpoint_status(client: httpx.AsyncClient, config: EbookSearchConfig) -> str:
     """Return the answering chat endpoint status, or disabled when answers are off."""
     if not config.answer_enabled:
         return "disabled"
-    return "ok" if check_chat_endpoint(config) else "fail"
+    return "ok" if await check_chat_endpoint(client, config) else "fail"
 
 
-def check_database(session: Session) -> bool:
+async def check_database(session: AsyncSession) -> bool:
     """Return whether the database answers a trivial query."""
     try:
-        session.execute(select(literal(1)))
+        await session.execute(select(literal(1)))
     except SQLAlchemyError as error:
-        logger.warning("ebook_ready_database_unavailable error=%s", error)
+        logger.warning(f"ebook_ready_database_unavailable {error=}")
         return False
     return True
 

python/ebook_search/api/routes/page.py

@@ -3,17 +3,24 @@
 from __future__ import annotations
 
 import logging
+from typing import TYPE_CHECKING
 
-from fastapi import APIRouter, Request
-from fastapi.responses import HTMLResponse
-from sqlalchemy import select
+from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
+from fastapi.responses import HTMLResponse, RedirectResponse
+from sqlalchemy import func, select
 
 from python.ebook_search.api.dependencies import (
     AppConfig,  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
 )
+from python.ebook_search.api.judge_tasks import is_judging_book, pop_book_judgment_outcome, start_book_phrase_judgment
 from python.ebook_search.api.web import templates
-from python.fastapi_tools import DbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
-from python.orm.richie import EbookSource
+from python.ebook_search.protected_phrases.generate_ngrams import recalculate_candidate_phrases_for_book
+from python.ebook_search.protected_phrases.store import count_protected_phrases
+from python.fastapi_tools import AsyncDbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+from python.orm.richie import EbookCandidatePhrase, EbookChapter, EbookChunk, EbookProtectedPhrase, EbookSource
+
+if TYPE_CHECKING:
+    from sqlalchemy.ext.asyncio import AsyncSession
 
 logger = logging.getLogger(__name__)
 
@@ -21,38 +28,160 @@ router = APIRouter()
 
 
 @router.get("/", response_class=HTMLResponse)
-def index(request: Request, config: AppConfig) -> HTMLResponse:
+async def index(request: Request, config: AppConfig) -> HTMLResponse:
     """Render the search page."""
     return templates.TemplateResponse(request, "search.html", {"config": config})
 
 
 @router.get("/books", response_class=HTMLResponse)
-def books(request: Request, session: DbSession) -> HTMLResponse:
+async def books(request: Request, session: AsyncDbSession) -> HTMLResponse:
     """Render the indexed books page."""
-    sources = list(session.scalars(select(EbookSource).order_by(EbookSource.title)).all())
-    logger.info("ebook_books_page_loaded count=%s", len(sources))
+    sources = list((await session.scalars(select(EbookSource).order_by(EbookSource.title))).all())
+    logger.info(f"ebook_books_page_loaded count={len(sources)}")
     return templates.TemplateResponse(request, "books.html", {"sources": sources})
 
 
+async def get_chapter_count(session: AsyncSession, book_id: int) -> int:
+    """Return the number of indexed chapters for one book."""
+    return await session.scalar(select(func.count(EbookChapter.id)).where(EbookChapter.source_id == book_id)) or 0
+
+
+async def get_chunk_count(session: AsyncSession, book_id: int) -> int:
+    """Return the number of indexed chunks for one book."""
+    return await session.scalar(select(func.count(EbookChunk.id)).where(EbookChunk.source_id == book_id)) or 0
+
+
+async def get_candidate_count(session: AsyncSession, book_id: int) -> int:
+    """Return the number of indexed candidates for one book."""
+    return (
+        await session.scalar(select(func.count(EbookCandidatePhrase.id)).where(EbookCandidatePhrase.book_id == book_id))
+        or 0
+    )
+
+
+async def get_judged_candidate_count(session: AsyncSession, book_id: int) -> int:
+    """Return the number of judged candidates for one book."""
+    return (
+        await session.scalar(
+            select(func.count(EbookCandidatePhrase.id)).where(
+                EbookCandidatePhrase.book_id == book_id,
+                EbookCandidatePhrase.llm_judged.is_(True),
+            )
+        )
+        or 0
+    )
+
+
+async def get_candidates(session: AsyncSession, book_id: int) -> list[EbookCandidatePhrase]:
+    """Return the indexed candidates for one book."""
+    return list(
+        await session.scalars(
+            select(EbookCandidatePhrase)
+            .where(EbookCandidatePhrase.book_id == book_id)
+            .order_by(EbookCandidatePhrase.candidate_score.desc())
+            .limit(100)
+        )
+    )
+
+
+async def get_protected_phrases(session: AsyncSession, book_id: int) -> list[EbookProtectedPhrase]:
+    """Return the protected phrases for one book."""
+    return list(
+        await session.scalars(
+            select(EbookProtectedPhrase)
+            .where(EbookProtectedPhrase.book_id == book_id)
+            .order_by(EbookProtectedPhrase.importance.desc())
+            .limit(100)
+        )
+    )
+
+
 @router.get("/books/{source_id}", response_class=HTMLResponse)
-def book_detail(source_id: int, request: Request, session: DbSession) -> HTMLResponse:
+async def book_detail(source_id: int, request: Request, session: AsyncDbSession) -> HTMLResponse:
     """Render details for one indexed book."""
-    source = session.get(EbookSource, source_id)
+    source = await session.get(EbookSource, source_id)
+    phrase_status_message = None
+    recalculated = request.query_params.get("phrases_recalculated")
+    if recalculated is not None:
+        phrase_status_message = f"Recalculated phrases; {recalculated} candidates generated"
+    judgment_outcome = pop_book_judgment_outcome(request.app, source_id)
+    if judgment_outcome is not None:
+        phrase_status_message = judgment_outcome
+    judging_in_progress = is_judging_book(request.app, source_id)
+    if judging_in_progress:
+        phrase_status_message = "Judging candidate phrases in the background; refresh to see progress"
     if source is not None:
-        chapter_count = len(source.chapters)
-        chunk_count = len(source.chunks)
+        chapter_count = await get_chapter_count(session, source.id)
+        chunk_count = await get_chunk_count(session, source.id)
+        candidate_count = await get_candidate_count(session, source.id)
+        judged_candidate_count = await get_judged_candidate_count(session, source.id)
+        protected_count = await count_protected_phrases(session, source.id)
+        candidates = await get_candidates(session, source.id)
+        protected_phrases = await get_protected_phrases(session, source.id)
     else:
         chapter_count = 0
         chunk_count = 0
+        candidate_count = 0
+        judged_candidate_count = 0
+        protected_count = 0
+        candidates = []
+        protected_phrases = []
     logger.info(
-        "ebook_book_detail_loaded source_id=%s found=%s chapters=%s chunks=%s",
-        source_id,
-        source is not None,
-        chapter_count,
-        chunk_count,
+        f"ebook_book_detail_loaded {source_id=} found={source is not None} {chapter_count=} {chunk_count=} "
+        f"{candidate_count=} {judged_candidate_count=} {protected_count=}"
     )
     return templates.TemplateResponse(
         request,
         "book_detail.html",
-        {"chapter_count": chapter_count, "chunk_count": chunk_count, "source": source},
+        {
+            "candidate_count": candidate_count,
+            "candidates": candidates,
+            "chapter_count": chapter_count,
+            "chunk_count": chunk_count,
+            "judged_candidate_count": judged_candidate_count,
+            "judging_in_progress": judging_in_progress,
+            "protected_count": protected_count,
+            "protected_phrases": protected_phrases,
+            "phrase_status_message": phrase_status_message,
+            "source": source,
+        },
     )
+
+
+@router.post("/books/{source_id}/recalculate-phrases")
+async def recalculate_book_phrases(source_id: int, config: AppConfig, session: AsyncDbSession) -> RedirectResponse:
+    """Clear and regenerate candidate phrases for one indexed book."""
+    source = await session.get(EbookSource, source_id)
+    if source is None:
+        raise HTTPException(status_code=404, detail="Book not found")
+
+    try:
+        result = await recalculate_candidate_phrases_for_book(session, source, config)
+    except ValueError as error:
+        raise HTTPException(status_code=409, detail=str(error)) from error
+    logger.info(
+        f"ebook_book_phrase_recalculation_complete {source_id=} {result.candidate_phrases=} "
+        f"{result.deleted_candidates=} {result.deleted_protected_phrases=} {result.deleted_aliases=} "
+        f"{result.deleted_mentions=}"
+    )
+    return RedirectResponse(
+        url=f"/books/{source_id}?phrases_recalculated={result.candidate_phrases}",
+        status_code=303,
+    )
+
+
+@router.post("/books/{source_id}/judge-phrases")
+async def judge_book_phrases(
+    source_id: int,
+    request: Request,
+    background_tasks: BackgroundTasks,
+    session: AsyncDbSession,
+) -> RedirectResponse:
+    """Queue background judging of one book's candidate phrases and return immediately."""
+    source = await session.get(EbookSource, source_id)
+    if source is None:
+        raise HTTPException(status_code=404, detail="Book not found")
+
+    started = start_book_phrase_judgment(request.app, background_tasks, source.id)
+    logger.info(f"ebook_book_phrase_judgment_requested {source_id=} {started=}")
+    return RedirectResponse(url=f"/books/{source_id}", status_code=303)

python/ebook_search/api/routes/search.py

@@ -13,9 +13,9 @@ from fastapi.responses import HTMLResponse
 from python.ebook_search.answer import answer_query
 from python.ebook_search.api.dependencies import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
     AppConfig,
-    AppEngine,
+    AppHttpClient,
 )
-from python.ebook_search.api.web import templates
+from python.ebook_search.api.web import error_response, templates
 from python.ebook_search.guardrails import (
     CitationReport,
     is_confident,
@@ -24,8 +24,11 @@ from python.ebook_search.guardrails import (
 )
 from python.ebook_search.search import SearchResponse, search_ebooks
 from python.ebook_search.timing import runtime_step_from_start
+from python.fastapi_tools import AppAsyncEngine  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
 
 if TYPE_CHECKING:
+    import httpx
+
     from python.ebook_search.config import EbookSearchConfig
 
 logger = logging.getLogger(__name__)
@@ -33,7 +36,8 @@ logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
-def build_answer(
+async def build_answer(
+    client: httpx.AsyncClient,
     query: str,
     response: SearchResponse,
     config: EbookSearchConfig,
@@ -45,9 +49,8 @@ def build_answer(
 
     if not is_confident(response.results, config):
         logger.info(
-            "ebook_answer_low_confidence confidence=%.4f threshold=%.4f",
-            retrieval_confidence(response.results),
-            config.min_retrieval_confidence,
+            f"ebook_answer_low_confidence confidence={retrieval_confidence(response.results):.4f} "
+            f"{config.min_retrieval_confidence=:.4f}"
         )
         answer = (
             "Retrieval confidence is low for this query, so answer generation was skipped. "
@@ -56,40 +59,46 @@ def build_answer(
         return answer, True, None
 
     try:
-        answer = answer_query(query, response.results, config)
+        answer = await answer_query(client, query, response.results, config)
     except RuntimeError as error:
-        logger.warning("ebook_answer_request_failed_falling_back error=%s", error)
+        logger.warning(f"ebook_answer_request_failed_falling_back {error=}")
         return "Answer generation failed. Source chunks are still shown below.", False, None
 
     citation_report = None
     if config.validate_citations_enabled and response.results:
         citation_report = validate_citations(answer, len(response.results))
         if citation_report.invalid or not citation_report.grounded:
-            logger.warning(
-                "ebook_answer_citation_issue invalid=%s grounded=%s",
-                citation_report.invalid,
-                citation_report.grounded,
-            )
+            logger.warning(f"ebook_answer_citation_issue {citation_report.invalid=} {citation_report.grounded=}")
     return answer, False, citation_report
 
 
 @router.post("/search", response_class=HTMLResponse)
-def search(
+async def search(
     request: Request,
     config: AppConfig,
-    engine: AppEngine,
+    engine: AppAsyncEngine,
+    client: AppHttpClient,
     query: Annotated[str, Form()],
-    rerank: Annotated[str | None, Form()] = None,
+    *,
+    rerank: Annotated[bool, Form()] = False,
+    phrase_matching: Annotated[bool, Form()] = False,
 ) -> HTMLResponse:
     """Run a search and render HTMX results."""
     try:
-        response = search_ebooks(engine, query, config, rerank=rerank == "true")
+        response = await search_ebooks(
+            engine,
+            client,
+            query,
+            config,
+            rerank=rerank,
+            phrase_matching=phrase_matching,
+        )
     except Exception as error:
         logger.exception("ebook_search_request_failed")
-        return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
+        return error_response(request, error)
 
     answer_start = perf_counter()
-    answer, low_confidence, citation_report = build_answer(query, response, config)
+    answer, low_confidence, citation_report = await build_answer(client, query, response, config)
     answer_step_name = "Answer generation" if config.answer_enabled else "Answer skipped"
     response = replace(
         response,
@@ -97,12 +106,10 @@ def search(
     )
 
     for step in response.timings:
-        logger.info("ebook_search_timing step=%r runtime_ms=%.1f", step.name, step.duration_ms)
+        logger.info(f"ebook_search_timing {step.name=} {step.duration_ms=:.1f}")
     logger.info(
-        "ebook_search_request_complete results=%s rank_label=%s runtime_ms=%.1f",
-        len(response.results),
-        response.rank_label,
-        response.total_runtime_ms,
+        f"ebook_search_request_complete results={len(response.results)} {response.rank_label=} "
+        f"{response.total_runtime_ms=:.1f}"
     )
     return templates.TemplateResponse(
         request,

python/ebook_search/api/static/style.css

@@ -181,6 +181,12 @@ textarea:focus {
     flex-wrap: wrap;
 }
 
+.search-toggles {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 14px;
+}
+
 button {
     padding: 10px 20px;
     font: inherit;
@@ -212,6 +218,11 @@ button:hover {
     margin-bottom: 24px;
 }
 
+.actions-grid {
+    display: grid;
+    grid-template-columns: repeat(2, max-content);
+}
+
 /* Answer + results */
 #results {
     display: block;
@@ -303,6 +314,28 @@ button:hover {
     font-variant-numeric: tabular-nums;
 }
 
+.phrase-matches {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 8px;
+    align-items: baseline;
+    margin: 10px 0 0;
+    font-size: 0.78rem;
+}
+
+.phrase-matches-label {
+    color: var(--muted);
+    font-weight: 600;
+}
+
+.phrase-match {
+    padding: 3px 10px;
+    background: var(--bg);
+    border: 1px solid var(--border);
+    border-radius: 999px;
+    color: var(--accent);
+}
+
 /* Runtime — developer diagnostics, hidden unless dev mode is on */
 .runtime {
     display: none;

python/ebook_search/api/templates/admin.html

@@ -1,45 +1,103 @@
-{% extends "base.html" %}
-
-{% block title %}EPUB Admin{% endblock %}
-{% block head %}<script src="https://unpkg.com/htmx.org@2.0.4"></script>{% endblock %}
-
-{% block content %}
-    <h1>Admin</h1>
-    <section id="admin-status"></section>
-    <section class="actions">
-        <form hx-post="/admin/scan" hx-target="#admin-status" hx-swap="innerHTML">
-            <button type="submit">Scan</button>
-        </form>
-        <form hx-post="/admin/embed-missing" hx-target="#admin-status" hx-swap="innerHTML">
-            <button type="submit">Embed</button>
-        </form>
-        <form hx-post="/admin/embed-all" hx-target="#admin-status" hx-swap="innerHTML">
-            <button type="submit">Embed all</button>
-        </form>
-    </section>
-    <section>
-        <h2>Embeddings</h2>
-        <table>
-            <thead>
-                <tr>
-                    <th>Model</th>
-                    <th>Dimensions</th>
-                    <th>Embedded</th>
-                    <th>Missing</th>
-                    <th>Total chunks</th>
-                </tr>
-            </thead>
-            <tbody>
-                {% for item in stats %}
-                    <tr>
-                        <td>{{ item.model_name }}</td>
-                        <td>{{ item.dimension }}</td>
-                        <td>{{ item.embedded_chunks }}</td>
-                        <td>{{ item.missing_chunks }}</td>
-                        <td>{{ item.total_chunks }}</td>
-                    </tr>
-                {% endfor %}
-            </tbody>
-        </table>
-    </section>
+{% extends "base.html" %} {% block title %}EPUB Admin{% endblock %} {% block
+head %}
+<script src="https://unpkg.com/htmx.org@2.0.4"></script>
+{% endblock %} {% block content %}
+<h1>Admin</h1>
+<section id="admin-status"></section>
+<section class="actions">
+  <form hx-post="/admin/scan" hx-target="#admin-status" hx-swap="innerHTML">
+    <button type="submit">Scan</button>
+  </form>
+</section>
+<section>
+  <h2>Embeddings</h2>
+  <section class="actions">
+    <form
+      hx-post="/admin/embed-missing"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Embed</button>
+    </form>
+    <form
+      hx-post="/admin/embed-all"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Embed all</button>
+    </form>
+  </section>
+  <table>
+    <thead>
+      <tr>
+        <th>Model</th>
+        <th>Dimensions</th>
+        <th>Embedded</th>
+        <th>Missing</th>
+        <th>Total chunks</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for item in stats %}
+      <tr>
+        <td>{{ item.model_name }}</td>
+        <td>{{ item.dimension }}</td>
+        <td>{{ item.embedded_chunks }}</td>
+        <td>{{ item.missing_chunks }}</td>
+        <td>{{ item.total_chunks }}</td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+</section>
+<section>
+  <h2>Protected phrases</h2>
+  <section class="actions actions-grid">
+    <form
+      hx-post="/admin/phrases/generate-all"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Regenerate all phrases</button>
+    </form>
+    <form
+      hx-post="/admin/phrases/judge-all"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Judge all phrases</button>
+    </form>
+    <form
+      hx-post="/admin/phrases/judge-missing"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Judge missing phrases</button>
+    </form>
+  </section>
+  <table>
+    <thead>
+      <tr>
+        <th>Candidates</th>
+        <th>Judged</th>
+        <th>Unjudged</th>
+        <th>Protected</th>
+        <th>Books indexed</th>
+        <th>Books generated</th>
+        <th>Books fully judged</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+        <td>{{ phrase_stats.candidate_phrases }}</td>
+        <td>{{ phrase_stats.judged_candidates }}</td>
+        <td>{{ phrase_stats.unjudged_candidates }}</td>
+        <td>{{ phrase_stats.protected_phrases }}</td>
+        <td>{{ phrase_stats.total_books }}</td>
+        <td>{{ phrase_stats.books_with_candidates }}</td>
+        <td>{{ phrase_stats.books_fully_judged }}</td>
+      </tr>
+    </tbody>
+  </table>
+</section>
 {% endblock %}

python/ebook_search/api/templates/book_detail.html

@@ -6,6 +6,9 @@
     {% if source %}
         <h1>{{ source.title }}</h1>
         <p class="meta">{{ source.author or "Unknown author" }}</p>
+        {% if phrase_status_message %}
+            <p class="status">{{ phrase_status_message }}</p>
+        {% endif %}
         <dl class="card">
             <dt>File</dt>
             <dd>{{ source.file_path }}</dd>
@@ -13,7 +16,93 @@
             <dd>{{ chapter_count }}</dd>
             <dt>Chunks</dt>
             <dd>{{ chunk_count }}</dd>
+            <dt>Candidates</dt>
+            <dd>{{ candidate_count }}</dd>
+            <dt>Judged</dt>
+            <dd>{{ judged_candidate_count }}</dd>
+            <dt>Protected</dt>
+            <dd>{{ protected_count }}</dd>
         </dl>
+        <form
+            method="post"
+            action="/books/{{ source.id }}/recalculate-phrases"
+            onsubmit="return confirm('Remove old phrases for this book and generate new candidates?');"
+        >
+            <button type="submit">Recalculate phrases</button>
+        </form>
+        <form
+            method="post"
+            action="/books/{{ source.id }}/judge-phrases"
+            onsubmit="return confirm('Judge candidate phrases for this book with the LLM?');"
+        >
+            <button type="submit"{% if judging_in_progress %} disabled{% endif %}>
+                {% if judging_in_progress %}Judging&hellip;{% else %}Judge phrases{% endif %}
+            </button>
+        </form>
+
+        <section>
+            <h2>Candidate n-grams</h2>
+            {% if candidates %}
+                <table>
+                    <thead>
+                        <tr>
+                            <th>Phrase</th>
+                            <th>Status</th>
+                            <th>Score</th>
+                            <th>Count</th>
+                            <th>Chapters</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for candidate in candidates %}
+                            <tr>
+                                <td>{{ candidate.phrase_text }}</td>
+                                <td>
+                                    {% if candidate.llm_judged %}
+                                        {% if candidate.llm_keep %}Kept{% else %}Rejected{% endif %}
+                                    {% else %}
+                                        Candidate
+                                    {% endif %}
+                                </td>
+                                <td>{{ "%.2f"|format(candidate.candidate_score) }}</td>
+                                <td>{{ candidate.raw_count }}</td>
+                                <td>{{ candidate.chapter_count }}</td>
+                            </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+            {% else %}
+                <p>No candidate n-grams.</p>
+            {% endif %}
+        </section>
+
+        <section>
+            <h2>Protected phrases</h2>
+            {% if protected_phrases %}
+                <table>
+                    <thead>
+                        <tr>
+                            <th>Phrase</th>
+                            <th>Type</th>
+                            <th>Confidence</th>
+                            <th>Importance</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for phrase in protected_phrases %}
+                            <tr>
+                                <td>{{ phrase.phrase_text }}</td>
+                                <td>{{ phrase.phrase_type or "phrase" }}</td>
+                                <td>{{ "%.2f"|format(phrase.confidence) }}</td>
+                                <td>{{ "%.2f"|format(phrase.importance) }}</td>
+                            </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+            {% else %}
+                <p>No protected phrases.</p>
+            {% endif %}
+        </section>
     {% else %}
         <h1>Book not found</h1>
     {% endif %}

python/ebook_search/api/templates/partials/results.html

@@ -82,6 +82,14 @@
                         </div>
                     {% endif %}
                 </dl>
+                {% if result.matched_phrases %}
+                    <p class="phrase-matches">
+                        <span class="phrase-matches-label">boosted by</span>
+                        {% for phrase in result.matched_phrases %}
+                            <span class="phrase-match">{{ phrase }}</span>
+                        {% endfor %}
+                    </p>
+                {% endif %}
             </li>
         {% endfor %}
     </ol>

python/ebook_search/api/templates/search.html

@@ -7,12 +7,24 @@
     <h1>Search</h1>
     <form class="card" hx-post="/search" hx-target="#results" hx-swap="innerHTML">
         <label for="query">What are you looking for?</label>
-        <textarea id="query" name="query" rows="4" placeholder="Ask a question or paste a passage…" required></textarea>
+        <textarea id="query" name="query" rows="4" placeholder="Ask a question or paste a passage…" required
+            onkeydown="if (event.key === 'Enter' && !event.shiftKey) { event.preventDefault(); this.form.requestSubmit(); }"></textarea>
         <div class="form-row">
-            <label class="check">
-                <input type="checkbox" name="rerank" value="true" {% if config.rerank.enabled %}checked{% endif %}>
-                Rerank
-            </label>
+            <div class="search-toggles">
+                <label class="check">
+                    <input type="checkbox" name="rerank" value="true" {% if config.rerank.enabled %}checked{% endif %}>
+                    Rerank
+                </label>
+                <label class="check">
+                    <input
+                        type="checkbox"
+                        name="phrase_matching"
+                        value="true"
+                        {% if config.phrase_matching_enabled %}checked{% endif %}
+                    >
+                    Phrase matching
+                </label>
+            </div>
             <button type="submit">Search</button>
         </div>
     </form>

python/ebook_search/api/web.py

@@ -3,9 +3,14 @@
 from __future__ import annotations
 
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from fastapi.templating import Jinja2Templates
 
+if TYPE_CHECKING:
+    from fastapi import Request
+    from fastapi.responses import HTMLResponse
+
 PACKAGE_DIR = Path(__file__).resolve().parent
 TEMPLATE_DIR = PACKAGE_DIR / "templates"
 STATIC_DIR = PACKAGE_DIR / "static"
@@ -21,3 +26,8 @@ def static_version(filename: str) -> int:
 
 templates = Jinja2Templates(directory=TEMPLATE_DIR)
 templates.env.globals["static_version"] = static_version
+
+
+def error_response(request: Request, message: object) -> HTMLResponse:
+    """Render the shared error partial for a failed UI request."""
+    return templates.TemplateResponse(request, "partials/error.html", {"message": str(message)}, status_code=500)

python/ebook_search/bm25_corpus.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import json
 import logging
 import shutil
@@ -14,10 +15,11 @@ from typing import TYPE_CHECKING
 import bm25s
 from sqlalchemy import func, select, union_all
 
+from python.ebook_search.chunk_records import CHUNK_RECORD_COLUMNS
 from python.orm.richie import EbookChapter, EbookChunk, EbookSource
 
 if TYPE_CHECKING:
-    from sqlalchemy.orm import Session
+    from sqlalchemy.ext.asyncio import AsyncSession
 
     from python.ebook_search.config import EbookSearchConfig
 
@@ -73,52 +75,48 @@ def get_current_bm25_index(index_path: Path) -> Path:
     return index_path
 
 
-def ensure_bm25_corpus(session: Session, config: EbookSearchConfig) -> None:
+async def ensure_bm25_corpus(session: AsyncSession, config: EbookSearchConfig) -> None:
     """Create or refresh the persisted BM25 corpus when it is missing or stale."""
     index_path = bm25_index_path(config)
     manifest = read_bm25_manifest(index_path)
-    db_updated_at = corpus_last_updated_at(session)
+    db_updated_at = await corpus_last_updated_at(session)
     if not bm25_index_exists(index_path, manifest):
-        logger.info("ebook_bm25_index_missing path=%s", index_path)
-        refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
+        logger.info(f"ebook_bm25_index_missing {index_path=}")
+        await refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
         return
     if db_updated_at is not None and manifest is not None and manifest.created_at < db_updated_at:
         logger.info(
-            "ebook_bm25_index_stale path=%s created_at=%s db_updated_at=%s",
-            index_path,
-            manifest.created_at.isoformat(),
-            db_updated_at.isoformat(),
+            f"ebook_bm25_index_stale {index_path=} created_at={manifest.created_at.isoformat()} "
+            f"db_updated_at={db_updated_at.isoformat()}"
         )
-        refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
+        await refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
         return
     logger.info(
-        "ebook_bm25_index_current path=%s chunks=%s created_at=%s",
-        index_path,
-        manifest.chunk_count if manifest else 0,
-        manifest.created_at.isoformat() if manifest else None,
+        f"ebook_bm25_index_current {index_path=} chunks={manifest.chunk_count if manifest else 0} "
+        f"created_at={manifest.created_at.isoformat() if manifest else None}"
     )
 
 
-def refresh_bm25_corpus(
-    session: Session,
+async def refresh_bm25_corpus(
+    session: AsyncSession,
     config: EbookSearchConfig,
     *,
     db_updated_at: datetime | None = None,
 ) -> BM25Manifest:
-    """Rebuild and persist the BM25 corpus from the current database chunks."""
+    """Rebuild and persist the BM25 corpus from the current database chunks.
+
+    The index build is CPU and disk work, so it runs in a worker thread.
+    """
     index_path = bm25_index_path(config)
-    records, texts = fetch_bm25_corpus_records(session)
+    records, texts = await fetch_bm25_corpus_records(session)
     manifest = BM25Manifest(
         created_at=datetime.now(tz=UTC),
-        db_updated_at=db_updated_at if db_updated_at is not None else corpus_last_updated_at(session),
+        db_updated_at=db_updated_at if db_updated_at is not None else await corpus_last_updated_at(session),
         chunk_count=len(records),
     )
-    write_bm25_corpus(index_path, records, texts, manifest)
+    await asyncio.to_thread(write_bm25_corpus, index_path, records, texts, manifest)
     logger.info(
-        "ebook_bm25_index_refreshed path=%s chunks=%s created_at=%s",
-        index_path,
-        manifest.chunk_count,
-        manifest.created_at.isoformat(),
+        f"ebook_bm25_index_refreshed {index_path=} {manifest.chunk_count=} created_at={manifest.created_at.isoformat()}"
     )
     return manifest
 
@@ -131,7 +129,7 @@ def load_bm25_corpus(config: EbookSearchConfig) -> BM25Corpus:
     """
     index_path = bm25_index_path(config)
     active_index_path = get_current_bm25_index(index_path)
-    logger.info("ebook_bm25_corpus_cache_load path=%s active_path=%s", index_path, active_index_path)
+    logger.info(f"ebook_bm25_corpus_cache_load {index_path=} {active_index_path=}")
     manifest = read_bm25_manifest(index_path)
     if manifest is None or not bm25_index_exists(index_path, manifest):
         msg = f"BM25 corpus is not available: {index_path}"
@@ -164,7 +162,7 @@ def score_bm25_corpus(query: str, corpus: BM25Corpus, *, limit: int) -> list[tup
     return results
 
 
-def fetch_bm25_corpus_records(session: Session) -> tuple[list[dict[str, object]], list[str]]:
+async def fetch_bm25_corpus_records(session: AsyncSession) -> tuple[list[dict[str, object]], list[str]]:
     """Fetch persistable BM25 corpus records and their matching index texts from the database.
 
     search_text is only needed to build the index, so it is returned separately instead of
@@ -172,12 +170,7 @@ def fetch_bm25_corpus_records(session: Session) -> tuple[list[dict[str, object]]
     """
     statement = (
         select(
-            EbookChunk.id.label("chunk_id"),
-            EbookChunk.text.label("text"),
-            EbookSource.title.label("source_title"),
-            EbookSource.author.label("source_author"),
-            EbookChapter.title.label("chapter_title"),
-            EbookChunk.page_label.label("page_label"),
+            *CHUNK_RECORD_COLUMNS,
             EbookChunk.search_text.label("bm25_text"),
         )
         .select_from(EbookChunk)
@@ -187,21 +180,21 @@ def fetch_bm25_corpus_records(session: Session) -> tuple[list[dict[str, object]]
     )
     records: list[dict[str, object]] = []
     texts: list[str] = []
-    for row in session.execute(statement).mappings():
+    for row in (await session.execute(statement)).mappings():
         record = dict(row)
         texts.append(str(record.pop("bm25_text")))
         records.append(record)
     return records, texts
 
 
-def corpus_last_updated_at(session: Session) -> datetime | None:
+async def corpus_last_updated_at(session: AsyncSession) -> datetime | None:
     """Return the latest source/chapter/chunk update timestamp relevant to BM25 text."""
     update_times = union_all(
         select(func.max(EbookSource.updated).label("updated")),
         select(func.max(EbookChapter.updated).label("updated")),
         select(func.max(EbookChunk.updated).label("updated")),
     ).subquery()
-    return session.scalar(select(func.max(update_times.c.updated)))
+    return await session.scalar(select(func.max(update_times.c.updated)))
 
 
 def write_bm25_corpus(

python/ebook_search/chunk_records.py

@@ -0,0 +1,13 @@
+"""Shared database columns used to build search-result records."""
+
+from python.orm.richie import EbookChapter, EbookChunk, EbookSource
+
+CHUNK_RECORD_COLUMNS = (
+    EbookChunk.id.label("chunk_id"),
+    EbookChunk.text.label("text"),
+    EbookSource.id.label("source_id"),
+    EbookSource.title.label("source_title"),
+    EbookSource.author.label("source_author"),
+    EbookChapter.title.label("chapter_title"),
+    EbookChunk.page_label.label("page_label"),
+)

python/ebook_search/config.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-from os import getenv
 from typing import Annotated, Self
 
 from pydantic import AliasChoices, Field, field_validator, model_validator
@@ -32,11 +31,6 @@ def normalize_embedding_alias(model: str) -> str:
     return standard_model
 
 
-def normalize_embedding_model(default: str = "qwen3-embedding-0.6b") -> str:
-    """Normalize the configured embedding alias to its provider model name."""
-    return normalize_embedding_alias(getenv("EBOOK_SEARCH_EMBEDDING_MODEL", default))
-
-
 class RerankConfig(BaseSettings):
     """vLLM reranker settings."""
 
@@ -86,6 +80,31 @@ class EbookSearchConfig(BaseSettings):
     validate_citations_enabled: bool = True
     bm25_index_dir: str = ".ebook_search_bm25"
     bm25_refresh_delay_seconds: int = 60
+    protected_phrase_max_candidates_per_book: int = 5000
+    protected_phrase_llm_candidates_per_book: int = 500
+    protected_phrase_extraction_workers: int = 16
+    phrase_judge_book_workers: int = 20
+    phrase_judge_phrase_workers: int = 100
+    protected_phrase_confidence_threshold: float = 0.80
+    phrase_matching_enabled: bool = True
+    phrase_hit_boost: float = 0.25
+    phrase_min_tokens: int = 2
+    phrase_max_tokens: int = 5
+    phrase_max_entity_tokens: int = 8
+    phrase_yake_top_k: int = 1000
+    phrase_yake_dedup_limit: float = 0.85
+    phrase_raw_ngram_min_count: int = 2
+    phrase_raw_count_score_threshold: int = 3
+    phrase_raw_count_high_score_threshold: int = 10
+    phrase_chapter_count_score_threshold: int = 2
+    phrase_chapter_count_high_score_threshold: int = 5
+    phrase_target_protected_per_book: int = 100
+    phrase_default_allow_nested: bool = False
+    phrase_default_suppress_children: bool = True
+    phrase_bad_start_score_penalty: float = 10.0
+    phrase_bad_end_score_penalty: float = 10.0
+    phrase_multi_source_score_bonus: float = 2.0
+    phrase_multi_source_min_sources: int = 2
 
     @field_validator("library_paths", mode="before")
     @classmethod

python/ebook_search/docker/Dockerfile

@@ -0,0 +1,54 @@
+FROM python:3.14-slim AS base
+
+COPY --from=ghcr.io/astral-sh/uv:0.11.26 /uv /uvx /bin/
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    APP_DIR=/home/richie/dotfiles \
+    UV_PROJECT_ENVIRONMENT=/opt/venv \
+    UV_PYTHON_DOWNLOADS=never \
+    UV_NO_CACHE=1
+
+# Separate ENV instruction so ${APP_DIR} and ${PATH} from above resolve.
+ENV PYTHONPATH=${APP_DIR} \
+    PATH=/opt/venv/bin:${PATH}
+
+WORKDIR ${APP_DIR}
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends build-essential curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY python/ebook_search/docker/pyproject.toml python/ebook_search/docker/uv.lock ./
+
+RUN uv sync --locked --no-dev
+
+
+FROM base AS test
+
+RUN uv sync --locked
+
+COPY python ./python
+COPY tests/__init__.py ./tests/__init__.py
+COPY tests/ebook_search ./tests/ebook_search
+
+CMD ["pytest"]
+
+
+FROM base AS runtime
+
+ENV EBOOK_SEARCH_HOST=0.0.0.0 \
+    EBOOK_SEARCH_PORT=8070 \
+    EBOOK_SEARCH_BM25_INDEX_DIR=/data/bm25
+
+COPY python ./python
+
+RUN useradd --create-home --uid 10001 app \
+    && mkdir -p /data \
+    && chown -R app:app /home/richie /data
+
+USER app
+
+EXPOSE 8070
+
+CMD ["sh", "-c", "exec python -m python.ebook_search.api.main --host \"${EBOOK_SEARCH_HOST}\" --port \"${EBOOK_SEARCH_PORT}\" --log-level \"${EBOOK_SEARCH_LOG_LEVEL:-INFO}\""]

python/ebook_search/docker/README.md

@@ -0,0 +1,77 @@
+# Ebook Search Docker
+
+Run the EPUB search app against the existing Postgres database on `jeeves`:
+
+```sh
+python -m python.ebook_search.docker.containers start --library-path /path/to/epubs --build
+```
+
+All ebook-search Docker files live in this directory:
+
+- `Dockerfile` — multi-stage: `test` (runs pytest) and `runtime` (default target, the app image)
+- `docker-compose.yml`
+- `containers.py` — Typer lifecycle CLI
+- `pyproject.toml` / `uv.lock` — the container's uv-locked dependencies
+
+The app listens on `http://localhost:8070`.
+
+Useful lifecycle commands:
+
+```sh
+python -m python.ebook_search.docker.containers build
+python -m python.ebook_search.docker.containers start --library-path /path/to/epubs
+python -m python.ebook_search.docker.containers test
+python -m python.ebook_search.docker.containers logs
+python -m python.ebook_search.docker.containers ps
+python -m python.ebook_search.docker.containers stop
+```
+
+Direct compose usage from the repo root:
+
+```sh
+docker compose -f python/ebook_search/docker/docker-compose.yml ps
+```
+
+## Dependencies
+
+The image builds its environment with uv from `pyproject.toml` + `uv.lock` in this
+directory — this is the source of truth for the container's dependencies. To add or
+update a dependency, edit `pyproject.toml` here and regenerate the lock (uv is
+available in the `ebook-search` dev shell):
+
+```sh
+nix develop .#ebook-search -c uv lock --project python/ebook_search/docker
+```
+
+## Tests
+
+The main pytest suite excludes `tests/ebook_search` (its dependencies are no longer
+in the nix dev shell). The `test ebook search` CI workflow runs them in a uv env
+built from the lockfile in this directory — same commands work locally from the
+repo root (the `--override-ini` drops the main suite's ignore):
+
+```sh
+uv sync --locked --project python/ebook_search/docker
+uv run --project python/ebook_search/docker --no-sync pytest tests/ebook_search --override-ini addopts="-n auto -ra"
+```
+
+They can also run inside the Docker `test` image, which validates the image itself:
+
+```sh
+python -m python.ebook_search.docker.containers test
+```
+
+or the raw docker equivalent:
+
+```sh
+docker build --file python/ebook_search/docker/Dockerfile --target test --tag ebook-search:test .
+docker run --rm ebook-search:test
+```
+
+## Configuration
+
+The compose service loads the repo root `.env` into the container via `env_file`.
+
+Mount your EPUB directory by setting `EBOOK_LIBRARY_HOST_PATH` in an env file or on the command line. The container sees it as `/library`, and `EBOOK_SEARCH_LIBRARY_PATHS` is set to `/library` inside the container.
+
+Database connection settings are controlled by `RICHIE_DB`, `RICHIE_HOST`, `RICHIE_PORT`, `RICHIE_USER`, and `RICHIE_PASSWORD`. The default host is `jeeves`.

python/ebook_search/docker/__init__.py

@@ -0,0 +1 @@
+"""Docker packaging and lifecycle tooling for ebook search."""

python/ebook_search/docker/containers.py

@@ -0,0 +1,259 @@
+"""Docker container lifecycle management for ebook search."""
+
+from __future__ import annotations
+
+import logging
+import os
+import subprocess
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from python.common import configure_logger, get_repo_dir
+
+logger = logging.getLogger(__name__)
+
+
+def get_compose_file() -> Path:
+    """Return the path to the docker-compose.yml file."""
+    return Path(__file__).resolve().with_name("docker-compose.yml")
+
+
+def compose_base_args() -> list[str]:
+    """Return the common docker compose arguments for the ebook search stack."""
+    return ["compose", "-f", str(get_compose_file())]
+
+
+def docker_run(
+    arguments: list[str],
+    *,
+    env: dict[str, str] | None = None,
+    capture_output: bool = False,
+) -> subprocess.CompletedProcess[str]:
+    """Run docker with repo-root cwd and consistent error handling."""
+    logger.info(f"docker {' '.join(arguments)}")
+    return subprocess.run(
+        ["docker", *arguments],
+        cwd=get_repo_dir(),
+        env=env,
+        text=True,
+        check=False,
+        capture_output=capture_output,
+    )
+
+
+def compose_env(*, library_path: Path | None = None, port: int | None = None) -> dict[str, str]:
+    """Return environment variables passed to docker compose."""
+    env = os.environ.copy()
+    if library_path is not None:
+        resolved_library = library_path.expanduser().resolve()
+        if not resolved_library.exists():
+            msg = f"EPUB library path does not exist: {resolved_library}"
+            raise FileNotFoundError(msg)
+        env["EBOOK_LIBRARY_HOST_PATH"] = str(resolved_library)
+    if port is not None:
+        env["EBOOK_SEARCH_PORT"] = str(port)
+    return env
+
+
+def ensure_compose_file() -> None:
+    """Raise if the ebook search compose file is missing."""
+    if not get_compose_file().is_file():
+        msg = f"Compose file not found: {get_compose_file()}"
+        raise FileNotFoundError(msg)
+
+
+def build_image() -> None:
+    """Build the ebook search app image."""
+    ensure_compose_file()
+    result = docker_run([*compose_base_args(), "build"])
+    if result.returncode != 0:
+        msg = "Failed to build ebook search image"
+        raise RuntimeError(msg)
+
+
+def build_test_image() -> None:
+    """Build the ebook search test Docker image."""
+    dockerfile = Path(__file__).resolve().with_name("Dockerfile")
+    result = docker_run(["build", "--file", str(dockerfile), "--target", "test", "--tag", "ebook-search:test", "."])
+    if result.returncode != 0:
+        msg = "Failed to build ebook search test image"
+        raise RuntimeError(msg)
+
+
+def run_test_image() -> None:
+    """Run the ebook search test suite inside Docker."""
+    result = docker_run(["run", "--rm", "ebook-search:test"])
+    if result.returncode != 0:
+        msg = f"Ebook search tests failed with code {result.returncode}"
+        raise RuntimeError(msg)
+
+
+def start_stack(
+    *,
+    library_path: Path | None = None,
+    port: int | None = None,
+    build: bool = False,
+) -> None:
+    """Start the ebook search Docker compose stack."""
+    ensure_compose_file()
+    env = compose_env(library_path=library_path, port=port)
+    if build:
+        build_image()
+    result = docker_run(
+        [*compose_base_args(), "up", "-d"],
+        env=env,
+    )
+    if result.returncode != 0:
+        msg = f"Ebook search stack failed to start with code {result.returncode}"
+        raise RuntimeError(msg)
+    logger.info("Ebook search started.")
+
+
+def stop_stack(
+    *,
+    volumes: bool = False,
+) -> None:
+    """Stop and remove ebook search containers."""
+    ensure_compose_file()
+    command = [*compose_base_args(), "down"]
+    if volumes:
+        command.append("-v")
+    result = docker_run(command)
+    if result.returncode != 0:
+        msg = f"Ebook search stack failed to stop with code {result.returncode}"
+        raise RuntimeError(msg)
+
+
+def logs_stack(
+    *,
+    service: str | None = None,
+    tail: int = 100,
+    follow: bool = False,
+) -> str | None:
+    """Return recent logs from the ebook search stack."""
+    ensure_compose_file()
+    command = [*compose_base_args(), "logs", "--tail", str(tail)]
+    if follow:
+        command.append("--follow")
+    if service:
+        command.append(service)
+    result = docker_run(command, capture_output=not follow)
+    if result.returncode != 0:
+        return None
+    if follow:
+        return ""
+    return result.stdout + result.stderr
+
+
+def ps_stack() -> str | None:
+    """Return docker compose ps output for the ebook search stack."""
+    ensure_compose_file()
+    result = docker_run([*compose_base_args(), "ps"], capture_output=True)
+    if result.returncode != 0:
+        return None
+    return result.stdout + result.stderr
+
+
+app = typer.Typer(help="Ebook search Docker container management.", no_args_is_help=True)
+
+
+@app.command()
+def build() -> None:
+    """Build the ebook search Docker image."""
+    build_image()
+
+
+@app.command()
+def start(
+    library_path: Annotated[Path | None, typer.Option(help="Override host path containing EPUB files.")] = None,
+    port: Annotated[int | None, typer.Option(help="Override host port for the web UI.")] = None,
+    *,
+    build: Annotated[bool, typer.Option("--build", help="Build the image before starting.")] = False,
+    log_level: Annotated[str, typer.Option(help="Log level.")] = "INFO",
+) -> None:
+    """Start the ebook search container."""
+    configure_logger(log_level)
+    start_stack(
+        library_path=library_path,
+        port=port,
+        build=build,
+    )
+
+
+@app.command()
+def stop(
+    *,
+    volumes: Annotated[bool, typer.Option("--volumes", help="Also remove ebook search data volumes.")] = False,
+    log_level: Annotated[str, typer.Option(help="Log level.")] = "INFO",
+) -> None:
+    """Stop and remove ebook search containers."""
+    configure_logger(log_level)
+    stop_stack(volumes=volumes)
+
+
+@app.command()
+def restart(
+    library_path: Annotated[Path | None, typer.Option(help="Override host path containing EPUB files.")] = None,
+    port: Annotated[int | None, typer.Option(help="Override host port for the web UI.")] = None,
+    *,
+    build: Annotated[bool, typer.Option("--build", help="Build the image before starting.")] = False,
+    log_level: Annotated[str, typer.Option(help="Log level.")] = "INFO",
+) -> None:
+    """Restart the ebook search stack."""
+    configure_logger(log_level)
+    stop_stack()
+    start_stack(
+        library_path=library_path,
+        port=port,
+        build=build,
+    )
+
+
+@app.command()
+def logs(
+    service: Annotated[str | None, typer.Option(help="Service name, or omit for all services.")] = None,
+    tail: Annotated[int, typer.Option(help="Number of recent log lines.")] = 100,
+    *,
+    follow: Annotated[bool, typer.Option("--follow", "-f", help="Follow logs.")] = False,
+) -> None:
+    """Show recent ebook search container logs."""
+    output = logs_stack(service=service, tail=tail, follow=follow)
+    if output is None:
+        typer.echo("No ebook search containers found.")
+        raise typer.Exit(code=1)
+    if output:
+        typer.echo(output)
+
+
+@app.command("test")
+def run_tests(
+    *,
+    build: Annotated[bool, typer.Option("--build/--no-build", help="Build the test image before running.")] = True,
+    log_level: Annotated[str, typer.Option(help="Log level.")] = "INFO",
+) -> None:
+    """Run ebook search tests inside the Docker test image."""
+    configure_logger(log_level)
+    if build:
+        build_test_image()
+    run_test_image()
+
+
+@app.command("ps")
+def ps() -> None:
+    """Show ebook search container status."""
+    output = ps_stack()
+    if output is None:
+        typer.echo("No ebook search containers found.")
+        raise typer.Exit(code=1)
+    typer.echo(output)
+
+
+def cli() -> None:
+    """Typer entry point."""
+    app()
+
+
+if __name__ == "__main__":
+    cli()

python/ebook_search/docker/docker-compose.yml

@@ -0,0 +1,36 @@
+name: ebook-search
+
+services:
+  ebook-search:
+    build:
+      context: ../../..
+      dockerfile: python/ebook_search/docker/Dockerfile
+    image: ebook-search:latest
+    restart: unless-stopped
+    ports:
+      - "${EBOOK_SEARCH_PORT:-8070}:8070"
+    extra_hosts:
+      - "jeeves:192.168.90.40"
+    env_file:
+      - ../../../.env
+    environment:
+      EBOOK_SEARCH_HOST: "0.0.0.0"
+      EBOOK_SEARCH_PORT: "8070"
+      EBOOK_SEARCH_LIBRARY_PATHS: "/library"
+      EBOOK_SEARCH_BM25_INDEX_DIR: "/data/bm25"
+    volumes:
+      - "${EBOOK_LIBRARY_HOST_PATH:-/home/richie/ebooks}:/library:ro"
+      - ebook-search-data:/data
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -fsS http://127.0.0.1:8070/health >/dev/null || exit 1",
+        ]
+      interval: 30s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
+
+volumes:
+  ebook-search-data:

python/ebook_search/docker/pyproject.toml

@@ -0,0 +1,41 @@
+[project]
+name = "ebook-search"
+version = "0.1.0"
+description = "Locked runtime environment for the ebook search container."
+requires-python = "~=3.14.0"
+dependencies = [
+    "alembic",
+    "beautifulsoup4",
+    "bm25s",
+    "ebooklib",
+    "fastapi",
+    "httpx",
+    "jinja2",
+    "pgvector",
+    "psycopg[binary]",
+    "pydantic",
+    "pydantic-settings",
+    "python-multipart",
+    "sqlalchemy[asyncio]",
+    "tiktoken",
+    "typer",
+    "uvicorn[standard]",
+    "yake",
+]
+
+[dependency-groups]
+dev = [
+    "aiosqlite",
+    "pytest",
+    "pytest-asyncio",
+    "pytest-mock",
+    "pytest-xdist",
+]
+
+[tool.uv]
+package = false
+
+[tool.pytest.ini_options]
+addopts = "-n auto -ra"
+asyncio_mode = "auto"
+testpaths = ["tests/ebook_search"]

python/ebook_search/embeddings.py

@@ -23,7 +23,8 @@ logger = logging.getLogger(__name__)
 if TYPE_CHECKING:
     from collections.abc import Sequence
 
-    from sqlalchemy.orm import Session
+    import httpx
+    from sqlalchemy.ext.asyncio import AsyncSession
 
     from python.ebook_search.config import EbookSearchConfig
 
@@ -65,51 +66,45 @@ class EmbeddingModelStats:
         return max(self.total_chunks - self.embedded_chunks, 0)
 
 
-def embed_texts(texts: Sequence[str], config: EbookSearchConfig) -> list[list[float]]:
+async def embed_texts(
+    client: httpx.AsyncClient,
+    texts: Sequence[str],
+    config: EbookSearchConfig,
+) -> list[list[float]]:
     """Embed text with the configured vLLM embedding model."""
-    logger.info(
-        "ebook_embed_request_start base_url=%s model=%s count=%s",
-        config.embedding_base_url,
-        config.embedding_model,
-        len(texts),
-    )
-    vectors = request_embeddings(texts, config)
+    logger.info(f"ebook_embed_request_start {config.embedding_base_url=} {config.embedding_model=} count={len(texts)}")
+    vectors = await request_embeddings(client, texts, config)
     expected_dimension = MODEL_DIMENSIONS[config.embedding_model]
     for vector in vectors:
         if len(vector) != expected_dimension:
             msg = f"Expected {expected_dimension} dimensions, got {len(vector)}"
             raise ValueError(msg)
-    logger.info(
-        "ebook_embed_request_complete model=%s count=%s dimension=%s",
-        config.embedding_model,
-        len(vectors),
-        expected_dimension,
-    )
+    logger.info(f"ebook_embed_request_complete {config.embedding_model=} count={len(vectors)} {expected_dimension=}")
     return vectors
 
 
-def embed_query(query: str, config: EbookSearchConfig) -> list[float]:
+async def embed_query(client: httpx.AsyncClient, query: str, config: EbookSearchConfig) -> list[float]:
     """Embed a search query with the Qwen retrieval instruction."""
     instructed_query = f"Instruct: Retrieve relevant passages for the query.\nQuery: {query}"
-    return embed_texts([instructed_query], config)[0]
+    return (await embed_texts(client, [instructed_query], config))[0]
 
 
-def ensure_embedding_models(session: Session) -> None:
+async def ensure_embedding_models(session: AsyncSession) -> None:
     """Ensure supported embedding model rows exist."""
     for name, dimension in MODEL_DIMENSIONS.items():
-        existing = session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == name))
+        existing = await session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == name))
         if existing is None:
             session.add(EbookEmbeddingModel(name=name, dimension=dimension, is_default=name == "qwen3-embedding-0.6b"))
-            logger.info("ebook_embedding_model_created model=%s dimension=%s", name, dimension)
-    session.flush()
+            logger.info(f"ebook_embedding_model_created {name=} {dimension=}")
+    await session.flush()
 
 
-def embedding_model_stats(session: Session) -> list[EmbeddingModelStats]:
+async def embedding_model_stats(session: AsyncSession) -> list[EmbeddingModelStats]:
     """Return embedding coverage counts for every supported model."""
-    total_chunks = session.scalar(select(func.count(EbookChunk.id))) or 0
+    total_chunks = await session.scalar(select(func.count(EbookChunk.id))) or 0
     models = {
         model.name: model
-        for model in session.scalars(
+        for model in await session.scalars(
             select(EbookEmbeddingModel)
             .where(EbookEmbeddingModel.name.in_(MODEL_DIMENSIONS))
             .order_by(EbookEmbeddingModel.name)
@@ -122,7 +117,7 @@ def embedding_model_stats(session: Session) -> list[EmbeddingModelStats]:
         embedded_chunks = 0
         if model is not None:
             table = get_embedding_table(dimension)
-            embedded_chunks = session.scalar(select(func.count(table.id)).where(table.model_id == model.id)) or 0
+            embedded_chunks = await session.scalar(select(func.count(table.id)).where(table.model_id == model.id)) or 0
         stats.append(
             EmbeddingModelStats(
                 model_name=model_name,
@@ -134,10 +129,10 @@ def embedding_model_stats(session: Session) -> list[EmbeddingModelStats]:
     return stats
 
 
-def embed_missing_chunks(session: Session, config: EbookSearchConfig) -> int:
+async def embed_missing_chunks(session: AsyncSession, client: httpx.AsyncClient, config: EbookSearchConfig) -> int:
     """Embed chunks missing embeddings for the configured model."""
-    ensure_embedding_models(session)
-    model = session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == config.embedding_model))
+    await ensure_embedding_models(session)
+    model = await session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == config.embedding_model))
     if model is None:
         supported_models = ", ".join(MODEL_DIMENSIONS)
         msg = f"Unknown embedding model: {config.embedding_model}. Supported models: {supported_models}"
@@ -145,7 +140,7 @@ def embed_missing_chunks(session: Session, config: EbookSearchConfig) -> int:
 
     table = get_embedding_table(model.dimension)
     chunks = list(
-        session.scalars(
+        await session.scalars(
             select(EbookChunk)
             .outerjoin(table, (table.chunk_id == EbookChunk.id) & (table.model_id == model.id))
             .where(table.id.is_(None))
@@ -154,17 +149,17 @@ def embed_missing_chunks(session: Session, config: EbookSearchConfig) -> int:
         )
     )
     if not chunks:
-        logger.info("ebook_embed_missing_none model=%s", config.embedding_model)
+        logger.info(f"ebook_embed_missing_none {config.embedding_model=}")
         return 0
 
-    logger.info("ebook_embed_missing_batch_start model=%s count=%s", config.embedding_model, len(chunks))
-    vectors = embed_texts([chunk.text for chunk in chunks], config)
+    logger.info(f"ebook_embed_missing_batch_start {config.embedding_model=} count={len(chunks)}")
+    vectors = await embed_texts(client, [chunk.text for chunk in chunks], config)
     rows = [
         {"chunk_id": chunk.id, "model_id": model.id, "embedding": vector}
         for chunk, vector in zip(chunks, vectors, strict=True)
     ]
     statement = insert(table).values(rows).on_conflict_do_nothing(index_elements=["chunk_id", "model_id"])
-    session.execute(statement)
-    session.flush()
-    logger.info("ebook_embed_missing_batch_complete model=%s count=%s", config.embedding_model, len(rows))
+    await session.execute(statement)
+    await session.flush()
+    logger.info(f"ebook_embed_missing_batch_complete {config.embedding_model=} count={len(rows)}")
     return len(rows)

python/ebook_search/ingest.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import hashlib
 import logging
 from dataclasses import dataclass
@@ -10,9 +11,11 @@ from pathlib import Path
 from typing import TYPE_CHECKING
 
 import tiktoken
+from anyio import Path as AsyncPath
 from sqlalchemy import or_, select
 
 from python.ebook_search.epub_parse import parse_epub
+from python.ebook_search.protected_phrases.matching import index_chunk_phrase_mentions_for_book
 from python.orm.richie import EbookChapter, EbookChunk, EbookSource
 
 logger = logging.getLogger(__name__)
@@ -20,7 +23,7 @@ DEFAULT_CHUNK_TOKENS = 700
 DEFAULT_CHUNK_OVERLAP = 100
 
 if TYPE_CHECKING:
-    from sqlalchemy.orm import Session
+    from sqlalchemy.ext.asyncio import AsyncSession
 
     from python.ebook_search.config import EbookSearchConfig
     from python.ebook_search.epub_parse import ParsedChapter
@@ -72,90 +75,113 @@ def chunk_text(
     return [chunk for chunk in chunks if chunk.text]
 
 
-def ingest_configured_paths(session: Session, config: EbookSearchConfig) -> int:
+async def find_library_epubs(library_path: str) -> tuple[AsyncPath, list[AsyncPath] | None]:
+    """Resolve one configured library path and collect its EPUB files asynchronously.
+
+    Returns:
+        tuple[Path, list[Path] | None]: The expanded path and its EPUB files, or ``None`` when
+        the path is neither an EPUB file nor a directory.
+    """
+    path = await AsyncPath(library_path).expanduser()
+    if await path.is_file() and path.suffix.lower() == ".epub":
+        return path, [path]
+    if await path.is_dir():
+        return path, sorted([epub_path async for epub_path in path.rglob("*.epub")])
+    return path, None
+
+
+async def ingest_configured_paths(session: AsyncSession, config: EbookSearchConfig) -> int:
     """Ingest every EPUB found under configured library paths."""
     count = 0
     for library_path in config.library_paths:
-        path = Path(library_path).expanduser()
-        logger.info("ebook_ingest_path_start path=%s", path)
-        if path.is_file() and path.suffix.lower() == ".epub":
-            count += int(ingest_file(session, path, config))
-        elif path.is_dir():
-            for epub_path in sorted(path.rglob("*.epub")):
-                count += int(ingest_file(session, epub_path, config))
-        else:
-            logger.warning("ebook_ingest_path_missing path=%s", path)
-    logger.info("ebook_ingest_paths_complete changed_files=%s configured_paths=%s", count, len(config.library_paths))
+        path, epub_paths = await find_library_epubs(library_path)
+        logger.info(f"ebook_ingest_path_start {path=}")
+        if epub_paths is None:
+            logger.warning(f"ebook_ingest_path_missing {path=}")
+            continue
+        for epub_path in epub_paths:
+            count += int(await ingest_file(session, epub_path, config))
+    logger.info(f"ebook_ingest_paths_complete {count=} configured_paths={len(config.library_paths)}")
     return count
 
 
-def ingest_file(session: Session, path: Path, config: EbookSearchConfig) -> bool:
+async def resolve_ingest_path(path: Path | AsyncPath) -> AsyncPath:
+    """Expand and resolve an ingest path without blocking the event loop."""
+    expanded_path = await AsyncPath(path).expanduser()
+    return await expanded_path.resolve()
+
+
+async def ingest_file(session: AsyncSession, path: Path | AsyncPath, config: EbookSearchConfig) -> bool:
     """Ingest one EPUB file. Return True when the database changed."""
-    resolved_path = path.expanduser().resolve()
-    logger.info("ebook_ingest_file_start path=%s", resolved_path)
-    file_hash = sha256_file(resolved_path)
-    existing = find_existing_source(session, resolved_path, file_hash)
-    if existing is not None and existing.file_sha256 == file_hash:
-        stat = resolved_path.stat()
-        existing.file_path = str(resolved_path)
-        existing.file_mtime = datetime.fromtimestamp(stat.st_mtime, tz=UTC)
-        existing.file_size = stat.st_size
-        session.flush()
-        logger.info("ebook_ingest_file_unchanged source_id=%s path=%s", existing.id, resolved_path)
-        return False
-    if existing is not None:
-        logger.info("ebook_ingest_file_replacing source_id=%s path=%s", existing.id, resolved_path)
-        session.delete(existing)
-        session.flush()
+    try:
+        resolved_path = await resolve_ingest_path(path)
+        logger.info(f"ebook_ingest_file_start {resolved_path=}")
+        file_hash = await sha256_file(resolved_path)
+        existing = await find_existing_source(session, resolved_path, file_hash)
+        if existing is not None and existing.file_sha256 == file_hash:
+            stat = await resolved_path.stat()
+            existing.file_path = str(resolved_path)
+            existing.file_mtime = datetime.fromtimestamp(stat.st_mtime, tz=UTC)
+            existing.file_size = stat.st_size
+            await session.flush()
+            logger.info(f"ebook_ingest_file_unchanged {existing.id=} {resolved_path=}")
+            return False
+        if existing is not None:
+            logger.info(f"ebook_ingest_file_replacing {existing.id=} {resolved_path=}")
+            await session.delete(existing)
+            await session.flush()
 
-    stat = resolved_path.stat()
-    parsed = parse_epub(resolved_path)
-    source = EbookSource(
-        title=parsed.title,
-        author=parsed.author,
-        language=parsed.language,
-        publisher=parsed.publisher,
-        identifier=parsed.identifier,
-        file_path=str(resolved_path),
-        file_sha256=file_hash,
-        file_mtime=datetime.fromtimestamp(stat.st_mtime, tz=UTC),
-        file_size=stat.st_size,
-    )
-    session.add(source)
-    session.flush()
-
-    chunk_index = 0
-    for spine_index, parsed_chapter in enumerate(parsed.chapters):
-        chapter = EbookChapter(
-            source_id=source.id,
-            spine_index=spine_index,
-            title=parsed_chapter.title,
-            href=parsed_chapter.href,
+        stat = await resolved_path.stat()
+        parsed = await asyncio.to_thread(parse_epub, Path(resolved_path))
+        source = EbookSource(
+            title=parsed.title,
+            author=parsed.author,
+            language=parsed.language,
+            publisher=parsed.publisher,
+            identifier=parsed.identifier,
+            file_path=str(resolved_path),
+            file_sha256=file_hash,
+            file_mtime=datetime.fromtimestamp(stat.st_mtime, tz=UTC),
+            file_size=stat.st_size,
         )
-        session.add(chapter)
-        session.flush()
-        chunk_index = add_chapter_chunks(session, source, chapter, parsed_chapter, chunk_index, config)
+        session.add(source)
+        await session.flush()
 
-    session.flush()
-    logger.info(
-        "ebook_ingest_file_complete source_id=%s path=%s chapters=%s chunks=%s",
-        source.id,
-        resolved_path,
-        len(parsed.chapters),
-        chunk_index,
-    )
-    return True
+        chunk_index = 0
+        for spine_index, parsed_chapter in enumerate(parsed.chapters):
+            chapter = EbookChapter(
+                source_id=source.id,
+                spine_index=spine_index,
+                title=parsed_chapter.title,
+                href=parsed_chapter.href,
+            )
+            session.add(chapter)
+            await session.flush()
+            chunk_index = add_chapter_chunks(session, source, chapter, parsed_chapter, chunk_index, config)
+
+        mention_count = await index_chunk_phrase_mentions_for_book(session, source.id, config)
+        await session.commit()
+        logger.info(
+            f"ebook_ingest_file_complete {source.id=} {resolved_path=} chapters={len(parsed.chapters)} {chunk_index=} "
+            f"{mention_count=}"
+        )
+    except Exception:
+        await session.rollback()
+        logger.exception(f"ebook_ingest_file_error {path=}")
+        return False
+    else:
+        return True
 
 
-def find_existing_source(session: Session, path: Path, file_hash: str) -> EbookSource | None:
+async def find_existing_source(session: AsyncSession, path: Path | AsyncPath, file_hash: str) -> EbookSource | None:
     """Find an existing source by canonical path or file hash."""
-    return session.scalar(
+    return await session.scalar(
         select(EbookSource).where(or_(EbookSource.file_path == str(path), EbookSource.file_sha256 == file_hash))
     )
 
 
 def add_chapter_chunks(
-    session: Session,
+    session: AsyncSession,
     source: EbookSource,
     chapter: EbookChapter,
     parsed_chapter: ParsedChapter,
@@ -186,10 +212,10 @@ def add_chapter_chunks(
     return chunk_index
 
 
-def sha256_file(path: Path) -> str:
-    """Calculate the SHA-256 digest for a file."""
+async def sha256_file(path: AsyncPath) -> str:
+    """Calculate the SHA-256 digest for a file without blocking the event loop."""
     digest = hashlib.sha256()
-    with path.open("rb") as file:
-        for block in iter(lambda: file.read(1024 * 1024), b""):
+    async with await path.open("rb") as file:
+        while block := await file.read(1024 * 1024):
             digest.update(block)
     return digest.hexdigest()

python/ebook_search/llm_interface.py

@@ -22,10 +22,26 @@ def auth_headers(api_key: str) -> dict[str, str]:
     return {"Authorization": f"Bearer {api_key}"}
 
 
-def request_embeddings(texts: Sequence[str], config: EbookSearchConfig) -> list[list[float]]:
-    """Request embeddings from the configured OpenAI-compatible endpoint."""
+async def request_embeddings(
+    client: httpx.AsyncClient,
+    texts: Sequence[str],
+    config: EbookSearchConfig,
+) -> list[list[float]]:
+    """Request embeddings from the configured OpenAI-compatible endpoint.
+
+    Args:
+        client (httpx.AsyncClient): Shared async client for LLM calls.
+        texts (Sequence[str]): Texts to embed.
+        config (EbookSearchConfig): Runtime settings supplying the endpoint, model, and auth.
+
+    Returns:
+        list[list[float]]: One embedding vector per input text.
+
+    Raises:
+        RuntimeError: If the request fails or the response cannot be parsed.
+    """
     try:
-        response = httpx.post(
+        response = await client.post(
             f"{config.embedding_base_url.rstrip('/')}/embeddings",
             headers=auth_headers(config.embedding_api_key),
             json={"model": config.embedding_model, "input": list(texts)},
@@ -35,41 +51,62 @@ def request_embeddings(texts: Sequence[str], config: EbookSearchConfig) -> list[
         return embedding_vectors_from_response(response.json())
     except (httpx.HTTPError, ValueError, KeyError, TypeError) as error:
         logger.exception(
-            "ebook_embed_request_failed base_url=%s model=%s count=%s",
-            config.embedding_base_url,
-            config.embedding_model,
-            len(texts),
+            f"ebook_embed_request_failed {config.embedding_base_url=} {config.embedding_model=} count={len(texts)}"
         )
         msg = f"Embedding request failed. base_url={config.embedding_base_url} model={config.embedding_model}"
         raise RuntimeError(msg) from error
 
 
-def check_embedding_endpoint(config: EbookSearchConfig, *, timeout_seconds: float = 5.0) -> bool:
+async def check_embedding_endpoint(
+    client: httpx.AsyncClient,
+    config: EbookSearchConfig,
+    *,
+    timeout_seconds: float = 5.0,
+) -> bool:
     """Return whether the configured embedding endpoint answers a model listing."""
-    try:
-        response = httpx.get(
-            f"{config.embedding_base_url.rstrip('/')}/models",
-            headers=auth_headers(config.embedding_api_key),
-            timeout=timeout_seconds,
-        )
-        response.raise_for_status()
-    except httpx.HTTPError as error:
-        logger.warning("ebook_embedding_endpoint_unreachable base_url=%s error=%s", config.embedding_base_url, error)
-        return False
-    return True
+    return await _check_endpoint(
+        client,
+        base_url=config.embedding_base_url,
+        api_key=config.embedding_api_key,
+        timeout_seconds=timeout_seconds,
+        unavailable_log=f"ebook_embedding_endpoint_unreachable {config.embedding_base_url=}",
+    )
 
 
-def check_chat_endpoint(config: EbookSearchConfig, *, timeout_seconds: float = 5.0) -> bool:
+async def check_chat_endpoint(
+    client: httpx.AsyncClient,
+    config: EbookSearchConfig,
+    *,
+    timeout_seconds: float = 5.0,
+) -> bool:
     """Return whether the configured chat (answering) endpoint answers a model listing."""
+    return await _check_endpoint(
+        client,
+        base_url=config.vllm_base_url,
+        api_key=config.vllm_api_key,
+        timeout_seconds=timeout_seconds,
+        unavailable_log=f"ebook_chat_endpoint_unreachable {config.vllm_base_url=}",
+    )
+
+
+async def _check_endpoint(
+    client: httpx.AsyncClient,
+    *,
+    base_url: str,
+    api_key: str,
+    timeout_seconds: float,
+    unavailable_log: str,
+) -> bool:
+    """Return whether an OpenAI-compatible endpoint answers a model listing."""
     try:
-        response = httpx.get(
-            f"{config.vllm_base_url.rstrip('/')}/models",
-            headers=auth_headers(config.vllm_api_key),
+        response = await client.get(
+            f"{base_url.rstrip('/')}/models",
+            headers=auth_headers(api_key),
             timeout=timeout_seconds,
         )
         response.raise_for_status()
     except httpx.HTTPError as error:
-        logger.warning("ebook_chat_endpoint_unreachable base_url=%s error=%s", config.vllm_base_url, error)
+        logger.warning(f"{unavailable_log} {error=}")
         return False
     return True
 
@@ -98,18 +135,29 @@ def embedding_vectors_from_response(body: object) -> list[list[float]]:
     return vectors
 
 
-def request_rerank(
+async def request_rerank(
+    client: httpx.AsyncClient,
     query: str,
     documents: Sequence[str],
     config: RerankConfig,
 ) -> object | None:
-    """Request rerank scores from the configured vLLM endpoint."""
+    """Request rerank scores from the configured vLLM endpoint.
+
+    Args:
+        client (httpx.AsyncClient): Shared async client for LLM calls.
+        query (str): Query the documents are scored against.
+        documents (Sequence[str]): Candidate documents to score.
+        config (RerankConfig): Rerank endpoint settings.
+
+    Returns:
+        object | None: The decoded response body, or ``None`` when it is not valid JSON.
+    """
     payload = {
         "model": config.model,
         "query": query,
         "documents": list(documents),
     }
-    response = httpx.post(
+    response = await client.post(
         f"{config.base_url.rstrip('/')}/rerank",
         json=payload,
         timeout=config.timeout_seconds,
@@ -122,20 +170,33 @@ def request_rerank(
         return None
 
 
-def request_chat_completion(
+async def request_chat_completion(
+    client: httpx.AsyncClient,
     config: EbookSearchConfig,
     messages: Sequence[dict[str, str]],
+    *,
+    response_format: dict[str, object] | None = None,
 ) -> str:
-    """Request a chat completion from the configured OpenAI-compatible endpoint."""
+    """Request a chat completion over a shared async client.
+
+    Args:
+        client (httpx.AsyncClient): Shared async client whose connection pool bounds concurrency.
+        config (EbookSearchConfig): Runtime settings supplying the endpoint, model, and auth.
+        messages (Sequence[dict[str, str]]): OpenAI-style chat messages.
+        response_format (dict[str, object] | None): Optional OpenAI-compatible structured output constraint.
+
+    Returns:
+        str: The assistant message text.
+
+    Raises:
+        RuntimeError: If the request fails or the response cannot be parsed.
+    """
     try:
-        response = httpx.post(
+        response = await client.post(
             f"{config.vllm_base_url.rstrip('/')}/chat/completions",
             headers=auth_headers(config.vllm_api_key),
-            json={
-                "model": config.chat_model,
-                "messages": list(messages),
-                "temperature": 0,
-            },
+            json={"model": config.chat_model, "messages": list(messages), "temperature": 0}
+            | ({"response_format": response_format} if response_format is not None else {}),
             timeout=config.chat_timeout_seconds,
         )
         response.raise_for_status()

python/ebook_search/loadtest.py

@@ -112,7 +112,7 @@ async def send_search(client: httpx.AsyncClient, query: str, *, rerank: bool) ->
     try:
         response = await client.post("/search", data=data)
     except httpx.HTTPError as error:
-        logger.warning("ebook_loadtest_request_failed error=%s", error)
+        logger.warning(f"ebook_loadtest_request_failed {error=}")
         return RequestResult(status_code=0, latency_ms=(time.perf_counter() - start) * 1000, ok=False)
     return RequestResult(
         status_code=response.status_code,
@@ -192,14 +192,7 @@ def main(
     """Load test the search endpoint and report latency and throughput."""
     configure_logger(log_level)
     queries = load_queries(queries_file)
-    logger.info(
-        "ebook_loadtest_start base_url=%s requests=%s concurrency=%s rerank=%s queries=%s",
-        base_url,
-        request_count,
-        concurrency,
-        rerank,
-        len(queries),
-    )
+    logger.info(f"ebook_loadtest_start {base_url=} {request_count=} {concurrency=} {rerank=} queries={len(queries)}")
     summary = asyncio.run(
         run_load(
             base_url=base_url,

python/ebook_search/prompts/__init__.py

@@ -0,0 +1,8 @@
+"""LLM prompt templates for EPUB search."""
+
+from python.ebook_search.prompts.lib import (
+    Prompt,
+    load_prompt,
+)
+
+__all__ = ["Prompt", "load_prompt"]

python/ebook_search/prompts/answer.toml

@@ -0,0 +1,9 @@
+system = """\
+Answer only from the provided context. Cite sources with bracketed numbers like [1]. \
+If the context is insufficient, say so."""
+user = """\
+Question:
+{query}
+
+Context:
+{context}"""

python/ebook_search/prompts/lib.py

@@ -0,0 +1,44 @@
+"""Load and render TOML-backed LLM prompt templates."""
+
+from __future__ import annotations
+
+import tomllib
+from dataclasses import dataclass
+from functools import cache
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class Prompt:
+    """A system and user prompt pair loaded from TOML."""
+
+    system: str
+    user: str
+
+    def messages(self, **values: str) -> list[dict[str, str]]:
+        """Render this prompt as OpenAI-style chat messages."""
+        return [
+            {"role": "system", "content": self.system.format(**values)},
+            {"role": "user", "content": self.user.format(**values)},
+        ]
+
+
+@cache
+def _get_prompt_dir() -> Path:
+    """Return the directory containing prompt template files."""
+    return Path(__file__).resolve().parent
+
+
+@cache
+def load_prompt(name: str) -> Prompt:
+    """Load and validate a named system and user prompt pair from TOML."""
+    path = _get_prompt_dir() / f"{name}.toml"
+    with path.open("rb") as file:
+        body = tomllib.load(file)
+
+    system = body.get("system")
+    user = body.get("user")
+    if not isinstance(system, str) or not isinstance(user, str):
+        msg = f"{path} must define string system and user prompts"
+        raise TypeError(msg)
+    return Prompt(system=system, user=user)

python/ebook_search/prompts/phrase_judge.toml

@@ -0,0 +1,10 @@
+system = """\
+Judge whether a candidate phrase from a book should be protected for RAG retrieval. \
+Do not extract new phrases. Reject common grammar fragments, ordinary nonspecific \
+phrases, unstable fragments, and phrases kept only because they are frequent. Keep \
+people, places, organizations, factions, events, technologies, fictional conditions, \
+magic systems, formal titles, named concepts, and recurring world-specific terms. \
+Return only a JSON object with keys: keep, canonical, category, aliases, confidence, \
+importance, allow_nested, suppress_children, reason."""
+
+user = "{candidate_json}"

python/ebook_search/protected_phrases/__init__.py

@@ -0,0 +1 @@
+"""Protected phrase extraction and matching for ebook search."""

python/ebook_search/protected_phrases/config/__init__.py

@@ -0,0 +1,17 @@
+"""Protected phrase extraction, storage, and runtime matching."""
+
+from python.ebook_search.protected_phrases.config.lib import (
+    get_bad_ends,
+    get_bad_starts,
+    get_ignored_phrases,
+    get_junk_tokens,
+    get_most_common_words,
+)
+
+__all__ = [
+    "get_bad_ends",
+    "get_bad_starts",
+    "get_ignored_phrases",
+    "get_junk_tokens",
+    "get_most_common_words",
+]

python/ebook_search/protected_phrases/config/bad_ends.toml

@@ -0,0 +1,31 @@
+tokens = [
+  "a",
+  "an",
+  "and",
+  "any",
+  "as",
+  "at",
+  "be",
+  "because",
+  "but",
+  "by",
+  "can",
+  "could",
+  "do",
+  "for",
+  "from",
+  "have",
+  "if",
+  "of",
+  "or",
+  "some",
+  "than",
+  "the",
+  "these",
+  "this",
+  "to",
+  "will",
+  "with",
+  "would",
+  "did",
+]

python/ebook_search/protected_phrases/config/bad_starts.toml

@@ -0,0 +1,27 @@
+tokens = [
+    "a",
+    "an",
+    "did",
+    "didn't",
+    "he",
+    "here",
+    "how",
+    "i",
+    "it",
+    "she",
+    "that",
+    "the",
+    "there",
+    "they",
+    "this",
+    "we",
+    "what",
+    "when",
+    "where",
+    "which",
+    "who",
+    "whom",
+    "whose",
+    "why",
+    "you",
+]

python/ebook_search/protected_phrases/config/ignored_phrases.toml

@@ -0,0 +1,212 @@
+phrases = [
+  "a little",
+  "across the",
+  "and she",
+  "anyone in",
+  "are you",
+  "around him",
+  "around the",
+  "as much",
+  "as soon",
+  "at all",
+  "at least",
+  "before the",
+  "behind him",
+  "between the",
+  "but she",
+  "could not",
+  "did he",
+  "did i",
+  "did it",
+  "did not believe",
+  "did not care",
+  "did not even",
+  "did not know what",
+  "did not know",
+  "did not like",
+  "did not look",
+  "did not mean",
+  "did not move",
+  "did not need",
+  "did not see",
+  "did not seem",
+  "did not think",
+  "did not understand",
+  "did not want",
+  "did not",
+  "did she",
+  "did so",
+  "did that",
+  "did the",
+  "did they",
+  "did what",
+  "did you",
+  "didn't answer",
+  "didn't care",
+  "didn't even",
+  "didn't expect",
+  "didn't feel",
+  "didn't get",
+  "didn't i",
+  "didn't know",
+  "didn't like",
+  "didn't look",
+  "didn't make",
+  "didn't mean",
+  "didn't need",
+  "didn't really",
+  "didn't say",
+  "didn't see",
+  "didn't seem",
+  "didn't think",
+  "didn't want",
+  "didn't you",
+  "end up",
+  "ended up",
+  "had a",
+  "had been",
+  "have been",
+  "he asked",
+  "he concluded",
+  "he continued",
+  "he couldn't",
+  "he did",
+  "he didn't",
+  "he felt",
+  "he had",
+  "he hadn't",
+  "he knew",
+  "he noted",
+  "he pointed",
+  "he realized",
+  "he replied",
+  "he said",
+  "he saw",
+  "he tapped",
+  "he told",
+  "he was",
+  "he wasn't",
+  "his body",
+  "his chair",
+  "his feet",
+  "his hands",
+  "his head",
+  "his office",
+  "his own",
+  "his pc",
+  "his power",
+  "his shield",
+  "his sight",
+  "his voice",
+  "his wrist",
+  "how many",
+  "i am",
+  "i don't",
+  "i said",
+  "i was",
+  "i wouldn't",
+  "i'm not",
+  "if he",
+  "if they",
+  "is in",
+  "is not",
+  "is that",
+  "is the",
+  "it had",
+  "it had",
+  "it is",
+  "it was",
+  "it wasn't",
+  "it wasn't",
+  "no one",
+  "of course",
+  "of force",
+  "of it",
+  "of magic",
+  "of marines",
+  "of power",
+  "of those",
+  "old man",
+  "older man",
+  "one of",
+  "out of",
+  "set up",
+  "she admitted",
+  "she asked",
+  "she had",
+  "she replied",
+  "she said",
+  "she snapped",
+  "she told",
+  "she was",
+  "she'd been",
+  "shook his",
+  "sure he",
+  "tell you",
+  "that had",
+  "that is",
+  "that she",
+  "that was",
+  "the dark",
+  "the door",
+  "the first",
+  "the last",
+  "the man",
+  "the one",
+  "the only",
+  "the other",
+  "the rest",
+  "the room",
+  "the same",
+  "the two",
+  "the way",
+  "the world",
+  "there are",
+  "there was",
+  "there were",
+  "they are",
+  "they had",
+  "they were",
+  "they weren't",
+  "this is",
+  "this place",
+  "though he",
+  "through his",
+  "through the",
+  "to find",
+  "to get",
+  "to keep",
+  "to stay",
+  "to stop",
+  "to tell",
+  "to try",
+  "told her",
+  "told him",
+  "under his",
+  "was a",
+  "was enough",
+  "was going",
+  "was in",
+  "was no",
+  "was not",
+  "was now",
+  "was on",
+  "was one",
+  "was only",
+  "was still",
+  "was that",
+  "was the",
+  "was there",
+  "were in",
+  "what had",
+  "what happened",
+  "what was",
+  "where the",
+  "while i",
+  "you are",
+  "you can't",
+  "you don't",
+  "you know",
+  "you need",
+  "you were",
+]

python/ebook_search/protected_phrases/config/junk_tokens.toml

@@ -0,0 +1,71 @@
+tokens = [
+    "said",
+    "asked",
+    "replied",
+    "answered",
+    "looked",
+    "nodded",
+    "turned",
+    "shook",
+    "smiled",
+    "shrugged",
+    "pointed",
+    "continued",
+    "repeated",
+    "stared",
+    "agreed",
+    "glanced",
+    "walked",
+    "told",
+    "thought",
+    "knew",
+    "wanted",
+    "muttered",
+    "whispered",
+    "laughed",
+    "sighed",
+    "paused",
+    "gestured",
+    "waved",
+    "frowned",
+    "grinned",
+    "admitted",
+    "found",
+    "noted",
+    "murmured",
+    "ordered",
+    "i'm",
+    "i've",
+    "i'd",
+    "i'll",
+    "it's",
+    "that's",
+    "don't",
+    "didn't",
+    "doesn't",
+    "can't",
+    "won't",
+    "wouldn't",
+    "couldn't",
+    "shouldn't",
+    "isn't",
+    "wasn't",
+    "aren't",
+    "weren't",
+    "you're",
+    "you've",
+    "you'll",
+    "we're",
+    "we've",
+    "we'll",
+    "they're",
+    "they've",
+    "he's",
+    "she's",
+    "there's",
+    "what's",
+    "let's",
+    "who's",
+    "he'd",
+    "she'd",
+]

python/ebook_search/protected_phrases/config/lib.py

@@ -0,0 +1,60 @@
+"""Protected phrase extraction, storage, and runtime matching."""
+
+from __future__ import annotations
+
+import logging
+import tomllib
+from functools import cache
+from pathlib import Path
+
+from python.ebook_search.protected_phrases.text_normalization import normalize_text
+
+logger = logging.getLogger(__name__)
+
+
+def _load_toml_string_set(path: Path, key: str) -> frozenset[str]:
+    """Load and validate a TOML string list as a normalized immutable set."""
+    with path.open("rb") as file:
+        body = tomllib.load(file)
+
+    values = body.get(key)
+    if not isinstance(values, list) or not all(isinstance(item, str) for item in values):
+        msg = f"{path} must contain a {key!r} string list"
+        raise ValueError(msg)
+    return frozenset(normalize_text(value) for value in values if normalize_text(value))
+
+
+@cache
+def _get_phrase_config_dir() -> Path:
+    """Return the directory containing phrase configuration files."""
+    return Path(__file__).resolve().parent
+
+
+@cache
+def get_ignored_phrases() -> frozenset[str]:
+    """Return ignored phrase strings loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "ignored_phrases.toml", "phrases")
+
+
+@cache
+def get_bad_ends() -> frozenset[str]:
+    """Return bad phrase-ending tokens loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "bad_ends.toml", "tokens")
+
+
+@cache
+def get_bad_starts() -> frozenset[str]:
+    """Return bad phrase-starting tokens loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "bad_starts.toml", "tokens")
+
+
+@cache
+def get_most_common_words() -> frozenset[str]:
+    """Return the most common English words loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "most_common_words.toml", "words")
+
+
+@cache
+def get_junk_tokens() -> frozenset[str]:
+    """Return junk tokens (dialogue verbs and pronoun contractions) loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "junk_tokens.toml", "tokens")

python/ebook_search/protected_phrases/config/most_common_words.toml

@@ -0,0 +1,102 @@
+words = [
+    "the",
+    "be",
+    "to",
+    "of",
+    "and",
+    "a",
+    "in",
+    "that",
+    "have",
+    "I",
+    "it",
+    "for",
+    "not",
+    "on",
+    "with",
+    "he",
+    "as",
+    "you",
+    "do",
+    "at",
+    "this",
+    "but",
+    "his",
+    "by",
+    "from",
+    "they",
+    "we",
+    "say",
+    "her",
+    "she",
+    "or",
+    "an",
+    "will",
+    "my",
+    "one",
+    "all",
+    "would",
+    "there",
+    "their",
+    "what",
+    "so",
+    "up",
+    "out",
+    "if",
+    "about",
+    "who",
+    "get",
+    "which",
+    "go",
+    "me",
+    "when",
+    "make",
+    "can",
+    "like",
+    "time",
+    "no",
+    "just",
+    "him",
+    "know",
+    "take",
+    "people",
+    "into",
+    "year",
+    "your",
+    "good",
+    "some",
+    "could",
+    "them",
+    "see",
+    "other",
+    "than",
+    "then",
+    "now",
+    "look",
+    "only",
+    "come",
+    "its",
+    "over",
+    "think",
+    "also",
+    "back",
+    "after",
+    "use",
+    "two",
+    "how",
+    "our",
+    "work",
+    "first",
+    "well",
+    "way",
+    "even",
+    "new",
+    "want",
+    "because",
+    "any",
+    "these",
+    "give",
+    "day",
+    "most",
+    "us",
+]

python/ebook_search/protected_phrases/extraction.py

@@ -0,0 +1,724 @@
+"""Candidate phrase extraction and scoring for protected phrases."""
+
+from __future__ import annotations
+
+import logging
+import re
+from collections import Counter, defaultdict
+from functools import lru_cache
+from time import perf_counter
+from typing import TYPE_CHECKING, Protocol
+
+from yake import KeywordExtractor
+
+from python.ebook_search.protected_phrases.config import (
+    get_bad_ends,
+    get_bad_starts,
+    get_ignored_phrases,
+    get_junk_tokens,
+    get_most_common_words,
+)
+from python.ebook_search.protected_phrases.models import PhraseCandidate
+from python.ebook_search.protected_phrases.text_normalization import tokenize, tokenize_with_offsets
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Mapping, Sequence
+
+    from python.ebook_search.config import EbookSearchConfig
+
+logger = logging.getLogger(__name__)
+
+
+CAPITALIZED_PHRASE_RE = re.compile(r"\b(?:[A-Z][a-zA-Z']+)(?:\s+(?:of|the|and|in|on|for|[A-Z][a-zA-Z']+)){0,6}")
+
+
+class YakeExtractor(Protocol):
+    """Small protocol for the YAKE extractor used by this module."""
+
+    def extract_keywords(self, text: str) -> Iterable[tuple[str, float]]:
+        """Return YAKE keyword tuples."""
+
+
+class YakeExtractorFactory(Protocol):
+    """Callable constructor protocol for YAKE keyword extractors."""
+
+    def __call__(self, *, lan: str, n: int, dedupLim: float, top: int) -> YakeExtractor:  # noqa: N803
+        """Create a YAKE keyword extractor.
+
+        Args:
+            lan (str): Language code passed to YAKE.
+            n (int): Maximum n-gram size to extract.
+            dedupLim (float): Deduplication similarity threshold.
+            top (int): Maximum number of keyphrases to return.
+
+        Returns:
+            YakeExtractor: The constructed keyword extractor.
+        """
+
+
+def normalize_candidate_phrase(
+    phrase_text: str,
+    config: EbookSearchConfig,
+    *,
+    max_tokens: int | None = None,
+) -> tuple[str, str, int] | None:
+    """Normalize a candidate phrase and validate token bounds.
+
+    Args:
+        phrase_text (str): Raw phrase text to normalize.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        max_tokens (int | None): Maximum token count override; defaults to ``config.phrase_max_tokens``.
+
+    Returns:
+        tuple[str, str, int] | None: Display text, normalized phrase, and token count, or ``None``
+        when the phrase falls outside the token bounds or is ignored.
+    """
+    normalized_tokens = tokenize_with_offsets(phrase_text)
+    max_count = config.phrase_max_tokens if max_tokens is None else max_tokens
+    if len(normalized_tokens) < config.phrase_min_tokens or len(normalized_tokens) > max_count:
+        return None
+
+    phrase_norm = " ".join(token.text for token in normalized_tokens)
+    if phrase_norm in get_ignored_phrases():
+        return None
+
+    display_text = phrase_text[normalized_tokens[0].start_char : normalized_tokens[-1].end_char].strip()
+    return display_text or phrase_norm, phrase_norm, len(normalized_tokens)
+
+
+def count_raw_ngrams(tokens: Sequence[str], config: EbookSearchConfig) -> Counter[str]:
+    """Count every n-gram window in one normalized token block.
+
+    ``tokens`` are already normalized (see :func:`tokenize`), so each window's normalized form
+    is the joined tokens directly. Counting into a plain :class:`Counter` rather than
+    :class:`PhraseCandidate` objects keeps this hot loop cheap; callers filter ignored phrases
+    and materialize candidates per unique phrase afterwards, which is far fewer operations than
+    doing either per window.
+
+    Args:
+        tokens (Sequence[str]): Normalized tokens for one text block.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        Counter[str]: Raw occurrence counts keyed by normalized phrase.
+    """
+    return Counter(
+        " ".join(tokens[start : start + ngram_size])
+        for ngram_size in range(config.phrase_min_tokens, config.phrase_max_tokens + 1)
+        for start in range(len(tokens) - ngram_size + 1)
+    )
+
+
+def extract_raw_ngrams_by_chapter(
+    chapters: Sequence[str],
+    config: EbookSearchConfig,
+) -> dict[str, PhraseCandidate]:
+    """Extract raw n-grams across chapters, tracking both raw counts and chapter spread.
+
+    Counting each chapter separately makes chapter spread fall out of dict membership: a phrase's
+    ``chapter_count`` is simply how many per-chapter count maps contain it, so no per-window seen
+    tracking is needed. This also lets the enrichment step skip re-sliding the same n-gram sizes.
+
+    Phrases below the minimum raw count are dropped here rather than materialized: most unique
+    n-grams occur once, and :func:`filter_storable_candidates` would discard them as too rare
+    anyway, so building ``PhraseCandidate`` objects for them is wasted work.
+
+    Args:
+        chapters (Sequence[str]): Chapter-like text blocks to slide n-gram windows over.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        dict[str, PhraseCandidate]: Candidates meeting the minimum raw count, keyed by normalized
+        phrase, with raw and chapter counts.
+    """
+    chapter_count_maps = [count_raw_ngrams(tokenize(chapter), config) for chapter in chapters]
+    total_counts: Counter[str] = Counter()
+    chapter_spread: Counter[str] = Counter()
+    for chapter_counts in chapter_count_maps:
+        total_counts.update(chapter_counts)
+        chapter_spread.update(chapter_counts.keys())
+    min_raw_count = minimum_candidate_raw_count(config)
+    ignored = get_ignored_phrases()
+    return {
+        phrase_norm: PhraseCandidate(
+            phrase_text=phrase_norm,
+            phrase_norm=phrase_norm,
+            token_count=phrase_norm.count(" ") + 1,
+            source_raw_ngram=True,
+            raw_count=raw_count,
+            chapter_count=chapter_spread[phrase_norm],
+        )
+        for phrase_norm, raw_count in total_counts.items()
+        if raw_count >= min_raw_count and phrase_norm not in ignored
+    }
+
+
+@lru_cache(maxsize=2)
+def get_yake_extractor(max_ngram: int, top_k: int, dedup_limit: float) -> KeywordExtractor:
+    """Return a cached YAKE extractor for the given settings.
+
+    Constructing a ``KeywordExtractor`` loads the language's stopword list from disk, so it is
+    cached and reused across books rather than rebuilt on every call.
+
+    Args:
+        max_ngram (int): Maximum n-gram size to extract.
+        top_k (int): Maximum number of keyphrases to request.
+        dedup_limit (float): Deduplication similarity threshold.
+
+    Returns:
+        KeywordExtractor: A shared extractor instance for the given settings.
+    """
+    return KeywordExtractor(lan="en", n=max_ngram, dedupLim=dedup_limit, top=top_k)
+
+
+def extract_yake_candidates(
+    book_text: str,
+    config: EbookSearchConfig,
+) -> dict[str, PhraseCandidate]:
+    """Extract YAKE keyphrases when the optional YAKE package is installed.
+
+    Args:
+        book_text (str): Full book text to extract keyphrases from.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        dict[str, PhraseCandidate]: Candidates keyed by normalized phrase, with YAKE scores.
+    """
+    extractor = get_yake_extractor(
+        config.phrase_max_tokens,
+        config.phrase_yake_top_k,
+        config.phrase_yake_dedup_limit,
+    )
+    out: dict[str, PhraseCandidate] = {}
+    for phrase_text, yake_score in extractor.extract_keywords(book_text):
+        normalized = normalize_candidate_phrase(phrase_text, config)
+        if normalized is None:
+            continue
+        display_text, phrase_norm, token_count = normalized
+        out[phrase_norm] = PhraseCandidate(
+            phrase_text=display_text,
+            phrase_norm=phrase_norm,
+            token_count=token_count,
+            source_yake=True,
+            yake_score=float(yake_score),
+        )
+    return out
+
+
+def extract_capitalized_phrases(original_text: str, config: EbookSearchConfig) -> dict[str, PhraseCandidate]:
+    """Extract capitalized phrase runs that often carry fictional terms.
+
+    Args:
+        original_text (str): Original-case book text to scan for capitalized runs.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        dict[str, PhraseCandidate]: Candidates keyed by normalized phrase from capitalized runs.
+    """
+    out: dict[str, PhraseCandidate] = {}
+    for match in CAPITALIZED_PHRASE_RE.finditer(original_text):
+        phrase_text = match.group(0).strip()
+        normalized = normalize_candidate_phrase(
+            phrase_text,
+            config,
+            max_tokens=config.phrase_max_entity_tokens,
+        )
+        if normalized is None:
+            continue
+        display_text, phrase_norm, token_count = normalized
+        out[phrase_norm] = PhraseCandidate(
+            phrase_text=display_text,
+            phrase_norm=phrase_norm,
+            token_count=token_count,
+            source_capitalized=True,
+        )
+    return out
+
+
+def extract_metadata_candidates(
+    metadata: Mapping[str, object] | None,
+    config: EbookSearchConfig,
+) -> dict[str, PhraseCandidate]:
+    """Extract phrases from book metadata values such as title, author, and series.
+
+    Args:
+        metadata (Mapping[str, object] | None): Book metadata values, or ``None`` when unavailable.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        dict[str, PhraseCandidate]: Candidates keyed by normalized phrase from metadata values.
+    """
+    if metadata is None:
+        return {}
+
+    out: dict[str, PhraseCandidate] = {}
+    for value in metadata.values():
+        if value is None:
+            continue
+        phrase_text = str(value).strip()
+        normalized = normalize_candidate_phrase(
+            phrase_text,
+            config,
+            max_tokens=config.phrase_max_entity_tokens,
+        )
+        if normalized is None:
+            continue
+        display_text, phrase_norm, token_count = normalized
+        out[phrase_norm] = PhraseCandidate(
+            phrase_text=display_text,
+            phrase_norm=phrase_norm,
+            token_count=token_count,
+            source_metadata=True,
+        )
+    return out
+
+
+def merge_candidate_sources(*sources: Mapping[str, PhraseCandidate]) -> dict[str, PhraseCandidate]:
+    """Merge candidate dictionaries by normalized phrase.
+
+    Args:
+        *sources (Mapping[str, PhraseCandidate]): Candidate maps to combine, keyed by normalized phrase.
+
+    Returns:
+        dict[str, PhraseCandidate]: One merged candidate per normalized phrase.
+    """
+    merged: dict[str, PhraseCandidate] = {}
+    for source in sources:
+        for phrase_norm, item in source.items():
+            existing = merged.setdefault(
+                phrase_norm,
+                PhraseCandidate(
+                    phrase_text=item.phrase_text,
+                    phrase_norm=phrase_norm,
+                    token_count=item.token_count,
+                ),
+            )
+            merge_candidate(existing, item)
+    return merged
+
+
+def merge_candidate(existing: PhraseCandidate, item: PhraseCandidate) -> None:
+    """Merge one candidate into an existing candidate object.
+
+    Args:
+        existing (PhraseCandidate): Candidate mutated in place to absorb ``item``.
+        item (PhraseCandidate): Candidate whose sources, counts, and scores are merged in.
+    """
+    existing.source_raw_ngram = existing.source_raw_ngram or item.source_raw_ngram
+    existing.source_yake = existing.source_yake or item.source_yake
+    existing.source_capitalized = existing.source_capitalized or item.source_capitalized
+    existing.source_metadata = existing.source_metadata or item.source_metadata
+    existing.raw_count += item.raw_count
+    existing.chapter_count = max(existing.chapter_count, item.chapter_count)
+    if item.yake_score is not None:
+        existing.yake_score = item.yake_score
+
+
+def enrich_with_frequency_and_chapter_counts(
+    candidates: Mapping[str, PhraseCandidate],
+    chapters: Sequence[str],
+    *,
+    counted_sizes: Iterable[int] = (),
+) -> dict[str, PhraseCandidate]:
+    """Add raw occurrence and chapter-spread counts to candidates.
+
+    Candidates whose ``token_count`` is in ``counted_sizes`` are left untouched: those counts
+    were already computed while sliding the chapters in :func:`extract_raw_ngrams_by_chapter`,
+    so re-sliding those n-gram sizes here would just duplicate that work.
+
+    Args:
+        candidates (Mapping[str, PhraseCandidate]): Candidates to enrich, keyed by normalized phrase.
+        chapters (Sequence[str]): Chapter-like text blocks used to count occurrences and spread.
+        counted_sizes (Iterable[int]): Token counts whose counts are already populated and should be skipped.
+
+    Returns:
+        dict[str, PhraseCandidate]: Candidates with updated ``raw_count`` and ``chapter_count`` values.
+    """
+    if not candidates:
+        return {}
+
+    already_counted = set(counted_sizes)
+    candidate_sets_by_size: dict[int, set[str]] = defaultdict(set)
+    for phrase_norm, candidate in candidates.items():
+        if candidate.token_count in already_counted:
+            continue
+        candidate_sets_by_size[candidate.token_count].add(phrase_norm)
+
+    enriched = dict(candidates)
+    if not candidate_sets_by_size:
+        return enriched
+
+    total_counts, chapter_counts = count_candidate_occurrences(candidate_sets_by_size, chapters)
+    for phrase_norm, candidate in enriched.items():
+        if candidate.token_count in already_counted:
+            continue
+        candidate.raw_count = max(candidate.raw_count, total_counts[phrase_norm])
+        candidate.chapter_count = chapter_counts[phrase_norm]
+    return enriched
+
+
+def count_candidate_occurrences(
+    candidate_sets_by_size: Mapping[int, set[str]],
+    chapters: Sequence[str],
+) -> tuple[dict[str, int], dict[str, int]]:
+    """Count total occurrences and chapter spread for candidate phrases across chapters.
+
+    Args:
+        candidate_sets_by_size (Mapping[int, set[str]]): Candidate normalized phrases grouped by token count.
+        chapters (Sequence[str]): Chapter-like text blocks to slide n-gram windows over.
+
+    Returns:
+        tuple[dict[str, int], dict[str, int]]: Total occurrence counts and chapter-spread counts,
+        each keyed by normalized phrase.
+    """
+    total_counts: defaultdict[str, int] = defaultdict(int)
+    chapter_counts: defaultdict[str, int] = defaultdict(int)
+    for chapter in chapters:
+        seen_in_chapter: set[str] = set()
+        chapter_tokens = tokenize(chapter)
+        for ngram_size, candidate_norms in candidate_sets_by_size.items():
+            for start in range(len(chapter_tokens) - ngram_size + 1):
+                phrase_norm = " ".join(chapter_tokens[start : start + ngram_size])
+                if phrase_norm not in candidate_norms:
+                    continue
+                total_counts[phrase_norm] += 1
+                seen_in_chapter.add(phrase_norm)
+        for phrase_norm in seen_in_chapter:
+            chapter_counts[phrase_norm] += 1
+    return total_counts, chapter_counts
+
+
+def filter_storable_candidates(
+    candidates: Mapping[str, PhraseCandidate],
+    config: EbookSearchConfig,
+) -> tuple[dict[str, PhraseCandidate], int, int, int, int]:
+    """Remove candidates that should not be persisted.
+
+    Args:
+        candidates (Mapping[str, PhraseCandidate]): Candidates to filter, keyed by normalized phrase.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        tuple[dict[str, PhraseCandidate], int, int, int, int]: The storable candidates followed by the
+        counts dropped for being too short, too rare, too common, and junk.
+    """
+    min_raw_count = minimum_candidate_raw_count(config)
+    filtered: dict[str, PhraseCandidate] = {}
+    too_short = 0
+    too_rare = 0
+    too_common = 0
+    junk = 0
+    for phrase_norm, candidate in candidates.items():
+        if candidate.token_count < config.phrase_min_tokens:
+            too_short += 1
+            continue
+        if candidate.raw_count < min_raw_count:
+            too_rare += 1
+            continue
+        phrase_tokens = phrase_norm.split()
+        if is_most_common_word_phrase(phrase_tokens):
+            too_common += 1
+            continue
+        if is_junk_phrase(phrase_tokens):
+            junk += 1
+            continue
+        filtered[phrase_norm] = candidate
+    return filtered, too_short, too_rare, too_common, junk
+
+
+def minimum_candidate_raw_count(config: EbookSearchConfig) -> int:
+    """Return the minimum occurrence count required before storing a candidate.
+
+    Args:
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        int: The minimum raw occurrence count, never less than 1.
+    """
+    return max(config.phrase_raw_ngram_min_count, 1)
+
+
+def is_most_common_word_phrase(phrase_tokens: list[str]) -> bool:
+    """Return whether every token in a normalized phrase is a common word.
+
+    Args:
+        phrase_tokens (list[str]): Normalized phrase tokens to inspect.
+
+    Returns:
+        bool: True when the phrase is non-empty and every token is a common word.
+    """
+    common_words = get_most_common_words()
+    return bool(phrase_tokens) and all(token in common_words for token in phrase_tokens)
+
+
+def is_junk_phrase(phrase_tokens: list[str]) -> bool:
+    """Return whether a normalized phrase is lexical junk not worth LLM judging.
+
+    Judged data shows phrases containing a dialogue/action verb or a pronoun contraction are
+    never kept, and phrases whose tokens are mostly common words almost never are. Possessives
+    of proper nouns (``chapman's death``) pass because matching is by exact token, and
+    exactly-half-common bigrams (``data feed``) pass because the common-word rule is strict.
+
+    Args:
+        phrase_tokens (list[str]): Normalized phrase tokens to inspect.
+
+    Returns:
+        bool: True when the phrase contains a junk token or is majority common words.
+    """
+    if not phrase_tokens:
+        return False
+    junk_tokens = get_junk_tokens()
+    if any(token in junk_tokens for token in phrase_tokens):
+        return True
+    common_words = get_most_common_words()
+    half_phrase_len = len(phrase_tokens) // 2
+    return sum(token in common_words for token in phrase_tokens) > half_phrase_len
+
+
+def score_candidate(candidate: PhraseCandidate, config: EbookSearchConfig) -> float:
+    """Score a phrase candidate before LLM judging.
+
+    Args:
+        candidate (PhraseCandidate): Candidate to score.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        float: Combined score from sources, frequency, and length, less any penalties.
+    """
+    score = source_score(candidate) + frequency_score(candidate, config) + token_count_score(candidate, config)
+    if non_raw_source_count(candidate) >= config.phrase_multi_source_min_sources:
+        score += config.phrase_multi_source_score_bonus
+    if candidate.phrase_norm in get_ignored_phrases():
+        score -= 100.0
+    if has_bad_start(candidate.phrase_norm):
+        score -= config.phrase_bad_start_score_penalty
+    if has_bad_end(candidate.phrase_norm):
+        score -= config.phrase_bad_end_score_penalty
+    return score
+
+
+def non_raw_source_count(candidate: PhraseCandidate) -> int:
+    """Count the non-raw-ngram extraction sources that produced a candidate.
+
+    Args:
+        candidate (PhraseCandidate): Candidate whose enabled sources are counted.
+
+    Returns:
+        int: Number of enabled sources other than the raw n-gram slide.
+    """
+    return sum(
+        (
+            candidate.source_yake,
+            candidate.source_capitalized,
+            candidate.source_metadata,
+        )
+    )
+
+
+def has_bad_start(phrase_norm: str) -> bool:
+    """Return whether a normalized phrase starts with a bad starting token.
+
+    Args:
+        phrase_norm (str): Normalized phrase text to inspect.
+
+    Returns:
+        bool: True when the first token is a known bad starting token.
+    """
+    phrase_tokens = phrase_norm.split()
+    return bool(phrase_tokens and phrase_tokens[0] in get_bad_starts())
+
+
+def has_bad_end(phrase_norm: str) -> bool:
+    """Return whether a normalized phrase ends with a bad ending token.
+
+    Args:
+        phrase_norm (str): Normalized phrase text to inspect.
+
+    Returns:
+        bool: True when the last token is a known bad ending token.
+    """
+    phrase_tokens = phrase_norm.split()
+    return bool(phrase_tokens and phrase_tokens[-1] in get_bad_ends())
+
+
+def source_score(candidate: PhraseCandidate) -> float:
+    """Return the score contribution from extraction sources.
+
+    Args:
+        candidate (PhraseCandidate): Candidate whose enabled sources are weighted.
+
+    Returns:
+        float: Summed weight of the candidate's enabled extraction sources.
+    """
+    return sum(
+        weight
+        for enabled, weight in (
+            (candidate.source_yake, 2.0),
+            (candidate.source_capitalized, 2.0),
+            (candidate.source_metadata, 2.0),
+            (candidate.source_raw_ngram, 0.5),
+        )
+        if enabled
+    )
+
+
+def frequency_score(candidate: PhraseCandidate, config: EbookSearchConfig) -> float:
+    """Return the score contribution from frequency and chapter spread.
+
+    Args:
+        candidate (PhraseCandidate): Candidate whose counts are scored.
+        config (EbookSearchConfig): Runtime phrase-tuning settings holding score thresholds.
+
+    Returns:
+        float: Summed weight for each frequency and chapter-spread threshold the candidate meets.
+    """
+    return sum(
+        weight
+        for count, threshold, weight in (
+            (candidate.raw_count, config.phrase_raw_count_score_threshold, 0.5),
+            (candidate.raw_count, config.phrase_raw_count_high_score_threshold, 0.5),
+            (candidate.chapter_count, config.phrase_chapter_count_score_threshold, 0.5),
+            (candidate.chapter_count, config.phrase_chapter_count_high_score_threshold, 0.5),
+        )
+        if count >= threshold
+    )
+
+
+def token_count_score(candidate: PhraseCandidate, config: EbookSearchConfig) -> float:
+    """Return the score contribution from phrase length.
+
+    Args:
+        candidate (PhraseCandidate): Candidate whose token count is scored.
+        config (EbookSearchConfig): Runtime phrase-tuning settings holding the max token bound.
+
+    Returns:
+        float: Length-based score contribution, which may be negative for over- or under-length phrases.
+    """
+    if candidate.token_count == 1:
+        return -0.5
+    if candidate.token_count in {2, 3, 4}:
+        return 0.5
+    if candidate.token_count > config.phrase_max_tokens:
+        return -1.0
+    return 0.0
+
+
+def get_sample_contexts(normalized_book_text: str, phrase_norm: str, max_contexts: int = 5) -> list[str]:
+    """Return normalized context snippets containing a candidate phrase.
+
+    ``normalized_book_text`` is expected to already be ``normalize_text``-ed by the caller
+    so the whole book is not re-normalized for every phrase.
+
+    Args:
+        normalized_book_text (str): Whole book text, already normalized, to search.
+        phrase_norm (str): Normalized phrase to find contexts around.
+        max_contexts (int): Maximum number of context snippets to return.
+
+    Returns:
+        list[str]: Up to ``max_contexts`` normalized snippets surrounding the phrase.
+    """
+    contexts: list[str] = []
+    start = 0
+    while len(contexts) < max_contexts:
+        index = normalized_book_text.find(phrase_norm, start)
+        if index == -1:
+            break
+        left = max(0, index - 300)
+        right = min(len(normalized_book_text), index + len(phrase_norm) + 300)
+        contexts.append(normalized_book_text[left:right])
+        start = index + len(phrase_norm)
+    return contexts
+
+
+def candidate_source_names(candidate: PhraseCandidate) -> list[str]:
+    """Return enabled source names for an extracted candidate.
+
+    Args:
+        candidate (PhraseCandidate): Candidate whose enabled sources are listed.
+
+    Returns:
+        list[str]: Names of the extraction sources that produced the candidate.
+    """
+    names: list[str] = []
+    if candidate.source_raw_ngram:
+        names.append("raw_ngram")
+    if candidate.source_yake:
+        names.append("yake")
+    if candidate.source_capitalized:
+        names.append("capitalized")
+    if candidate.source_metadata:
+        names.append("metadata")
+    return names
+
+
+def extract_phrase_candidates_for_book(
+    book_text: str,
+    chapters: Sequence[str],
+    config: EbookSearchConfig,
+    *,
+    metadata: Mapping[str, object] | None = None,
+) -> list[PhraseCandidate]:
+    """Extract, score, and limit phrase candidates for one book.
+
+    Args:
+        book_text (str): Full book text used for most extraction sources.
+        chapters (Sequence[str]): Chapter-like text blocks used for frequency counts.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        metadata (Mapping[str, object] | None): Optional book metadata used as a candidate source.
+
+    Returns:
+        list[PhraseCandidate]: Scored candidates sorted best-first and capped per book.
+    """
+    started_at = perf_counter()
+    logger.info(
+        f"ebook_phrase_candidate_extract_start chapters={len(chapters)} chars={len(book_text)} "
+        f"{config.phrase_min_tokens=} {config.phrase_max_tokens=} {config.protected_phrase_max_candidates_per_book=}"
+    )
+    raw_started_at = perf_counter()
+    raw = extract_raw_ngrams_by_chapter(chapters, config)
+    logger.info(
+        f"ebook_phrase_candidate_extract_raw_complete candidates={len(raw)} "
+        f"duration_ms={(perf_counter() - raw_started_at) * 1000:.1f}"
+    )
+    yake_started_at = perf_counter()
+    yake_candidates = extract_yake_candidates(book_text, config)
+    logger.info(
+        f"ebook_phrase_candidate_extract_yake_complete candidates={len(yake_candidates)} "
+        f"duration_ms={(perf_counter() - yake_started_at) * 1000:.1f}"
+    )
+    capitalized_started_at = perf_counter()
+    capitalized = extract_capitalized_phrases(book_text, config)
+    logger.info(
+        f"ebook_phrase_candidate_extract_capitalized_complete candidates={len(capitalized)} "
+        f"duration_ms={(perf_counter() - capitalized_started_at) * 1000:.1f}"
+    )
+    metadata_candidates = extract_metadata_candidates(metadata, config)
+
+    candidates = merge_candidate_sources(raw, yake_candidates, capitalized, metadata_candidates)
+    enriched_started_at = perf_counter()
+    # Raw n-gram sizes were already counted per chapter above, so only enrich the remaining
+    # (entity-length) sizes here instead of re-sliding every size over the whole book.
+    candidates = enrich_with_frequency_and_chapter_counts(
+        candidates,
+        chapters,
+        counted_sizes=range(config.phrase_min_tokens, config.phrase_max_tokens + 1),
+    )
+    pre_filter_count = len(candidates)
+    candidates, filtered_too_short, filtered_too_rare, filtered_too_common, filtered_junk = filter_storable_candidates(
+        candidates, config
+    )
+    for candidate in candidates.values():
+        candidate.candidate_score = score_candidate(candidate, config)
+
+    limited = sorted(candidates.values(), key=lambda item: item.candidate_score, reverse=True)[
+        : config.protected_phrase_max_candidates_per_book
+    ]
+    logger.info(
+        f"ebook_phrase_candidate_extract_complete raw={len(raw)} yake={len(yake_candidates)} "
+        f"capitalized={len(capitalized)} metadata={len(metadata_candidates)} {pre_filter_count=} {filtered_too_short=} "
+        f"{filtered_too_rare=} {filtered_too_common=} {filtered_junk=} min_uses={minimum_candidate_raw_count(config)} "
+        f"storable={len(candidates)} limited={len(limited)} "
+        f"enrich_score_ms={(perf_counter() - enriched_started_at) * 1000:.1f} "
+        f"duration_ms={(perf_counter() - started_at) * 1000:.1f}"
+    )
+    return limited

python/ebook_search/protected_phrases/generate_ngrams.py

@@ -0,0 +1,295 @@
+"""Book-level orchestration for candidate n-gram generation and recalculation."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from time import perf_counter
+from typing import TYPE_CHECKING
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from python.ebook_search.protected_phrases.extraction import extract_phrase_candidates_for_book
+from python.ebook_search.protected_phrases.models import (
+    BookCandidateResult,
+    PhraseCandidateGenerationResult,
+    PhraseRecalculationResult,
+)
+from python.ebook_search.protected_phrases.pool import get_extraction_pool
+from python.ebook_search.protected_phrases.store import (
+    bulk_upsert_unjudged_candidates,
+    delete_phrase_data_for_book,
+    load_book_chapter_texts,
+    metadata_for_source_id,
+    new_candidate_row,
+    prune_unstorable_unjudged_candidate_phrases,
+)
+from python.orm.common import get_async_postgres_engine
+from python.orm.richie import EbookSource
+
+if TYPE_CHECKING:
+    from sqlalchemy.ext.asyncio import AsyncEngine
+
+    from python.ebook_search.config import EbookSearchConfig
+    from python.ebook_search.protected_phrases.models import PhraseCandidate
+
+logger = logging.getLogger(__name__)
+
+
+class BookHasNoChaptersError(ValueError):
+    """Raised when a book has no indexed chapter text to generate phrases from."""
+
+
+async def generate_candidate_phrases_for_books(
+    engine: AsyncEngine,
+    config: EbookSearchConfig,
+) -> PhraseCandidateGenerationResult:
+    """Create or refresh candidate phrases for indexed books without calling the LLM judge.
+
+    Every book is submitted to the shared process pool up front and runs in parallel across the
+    pool's workers; the call blocks until all books have finished. Each worker opens its own
+    database engine from environment variables, loads the book's chapters, and commits the
+    book's candidates independently.
+
+    Args:
+        engine (AsyncEngine): Engine used to read the book list in this process.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        PhraseCandidateGenerationResult: Per-corpus counts of books seen, built, and candidates stored.
+
+    Results are collected in book order while the pool keeps working. A book failure (including
+    a book with no indexed chapters) is logged and counted as not built; the remaining books
+    are unaffected.
+    """
+    async with AsyncSession(engine, expire_on_commit=False) as session:
+        source_query = select(EbookSource.id).order_by(EbookSource.id)
+        source_ids = (await session.scalars(source_query)).all()
+    books_seen = len(source_ids)
+    logger.info(
+        f"ebook_candidate_phrase_generation_start {books_seen=} {config.phrase_min_tokens=} "
+        f"{config.phrase_max_tokens=} {config.protected_phrase_max_candidates_per_book=}"
+    )
+
+    pool = get_extraction_pool(config.protected_phrase_extraction_workers)
+    wrapped_futures = [
+        (
+            source_id,
+            asyncio.wrap_future(pool.submit(generate_candidate_phrases_for_book_in_worker, source_id, None, config)),
+        )
+        for source_id in source_ids
+    ]
+    outcomes: list[BookCandidateResult] = []
+    for source_id, wrapped_future in wrapped_futures:
+        await asyncio.wait([wrapped_future])
+        exception = wrapped_future.exception()
+        if exception is not None:
+            logger.error(f"ebook_candidate_phrase_generation_book_failed {source_id=}")
+            outcomes.append(BookCandidateResult())
+            continue
+        saved_count = wrapped_future.result()
+        logger.info(f"ebook_candidate_phrase_generation_book_committed {source_id=} {saved_count=}")
+        outcomes.append(BookCandidateResult(candidates=saved_count, built=True))
+
+    result = PhraseCandidateGenerationResult(
+        books_seen=books_seen,
+        books_built=sum(1 for outcome in outcomes if outcome.built),
+        candidate_phrases=sum(outcome.candidates for outcome in outcomes),
+    )
+    logger.info(
+        f"ebook_candidate_phrase_generation_complete {result.books_seen=} {result.books_built=} "
+        f"{result.candidate_phrases=}"
+    )
+    return result
+
+
+async def recalculate_candidate_phrases_for_book(
+    session: AsyncSession,
+    source: EbookSource,
+    config: EbookSearchConfig,
+) -> PhraseRecalculationResult:
+    """Remove all book phrase data, regenerate candidates, and commit the completed book.
+
+    Args:
+        session (AsyncSession): Active database session; deletion and regeneration commit on it.
+        source (EbookSource): Indexed book to recalculate.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        PhraseRecalculationResult: Deleted-row counts and the number of candidates regenerated.
+
+    Raises:
+        BookHasNoChaptersError: If the book has no indexed chapters. The deletion is rolled
+            back, so the book's existing phrases stay intact.
+
+    The deletion and regeneration share the caller's session, so they commit together; a
+    regeneration failure rolls the deletion back.
+    """
+    started_at = perf_counter()
+    logger.info(f"ebook_candidate_phrase_recalculation_start {source.id=} {source.title=}")
+    deleted = await delete_phrase_data_for_book(session, source.id)
+    candidate_count = await generate_candidate_phrases_for_book(
+        session,
+        source.id,
+        series_id=None,
+        config=config,
+        replace_all=True,
+    )
+
+    result = PhraseRecalculationResult(
+        book_id=source.id,
+        deleted_candidates=deleted.deleted_candidates,
+        deleted_protected_phrases=deleted.deleted_protected_phrases,
+        deleted_aliases=deleted.deleted_aliases,
+        deleted_mentions=deleted.deleted_mentions,
+        candidate_phrases=candidate_count,
+    )
+    logger.info(
+        f"ebook_candidate_phrase_recalculation_complete {source.id=} {result.deleted_candidates=} "
+        f"{result.deleted_protected_phrases=} {result.deleted_aliases=} {result.deleted_mentions=} "
+        f"{result.candidate_phrases=} duration_ms={(perf_counter() - started_at) * 1000:.1f}"
+    )
+    return result
+
+
+def generate_candidate_phrases_for_book_in_worker(
+    book_id: int,
+    series_id: int | None,
+    config: EbookSearchConfig,
+) -> int:
+    """Run one book's candidate generation in a pooled worker process.
+
+    The worker has no engine or session to inherit (neither can cross process boundaries), so
+    it creates its own engine from environment variables, opens the book's session on it, and
+    disposes the engine once the book is stored.
+
+    Args:
+        book_id (int): Book the candidates belong to.
+        series_id (int | None): Series scope for the stored candidates.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        int: Number of candidate phrase rows stored.
+    """
+
+    async def generate_with_worker_engine() -> int:
+        engine = get_async_postgres_engine(name="RICHIE", vector_engine=True, pool_size=1)
+        try:
+            async with AsyncSession(engine, expire_on_commit=False) as session:
+                return await generate_candidate_phrases_for_book(
+                    session,
+                    book_id,
+                    series_id,
+                    config,
+                )
+        finally:
+            await engine.dispose()
+
+    return asyncio.run(generate_with_worker_engine())
+
+
+async def generate_candidate_phrases_for_book(
+    session: AsyncSession,
+    book_id: int,
+    series_id: int | None,
+    config: EbookSearchConfig,
+    *,
+    replace_all: bool = False,
+) -> int:
+    """Load a book's chapters and metadata, extract candidate phrases, and store them without LLM judging.
+
+    The session commits only when the whole book succeeds; any failure rolls the session back,
+    which also restores rows the caller deleted in the same transaction (e.g. a recalculation).
+
+    Args:
+        session (AsyncSession): Active database session; committed on success, rolled back on failure.
+        book_id (int): Book the candidates belong to.
+        series_id (int | None): Series scope for the stored candidates.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        replace_all (bool): When the caller has already cleared this book's candidates (e.g. a
+            recalculation), skip the per-candidate existence lookup and bulk-insert new rows.
+
+    Returns:
+        int: Number of candidate phrase rows stored.
+
+    Raises:
+        BookHasNoChaptersError: If the book has no indexed chapter text.
+    """
+    started_at = perf_counter()
+    chapters = await load_book_chapter_texts(session, book_id)
+    if not chapters:
+        await session.rollback()
+        message = f"book {book_id} has no indexed chapters"
+        raise BookHasNoChaptersError(message)
+    metadata = await metadata_for_source_id(session, book_id)
+    try:
+        book_text = "\n\n".join(chapters)
+        candidates = extract_phrase_candidates_for_book(
+            book_text,
+            chapters,
+            config,
+            metadata=metadata,
+        )
+        saved_count = await store_candidate_phrases_for_book(
+            session,
+            book_id,
+            series_id,
+            candidates,
+            config,
+            replace_all=replace_all,
+        )
+        await session.commit()
+    except Exception:
+        await session.rollback()
+        raise
+    logger.info(
+        f"ebook_candidate_phrase_generation_book_duration {book_id=} {saved_count=} "
+        f"duration_ms={(perf_counter() - started_at) * 1000:.1f}"
+    )
+    return saved_count
+
+
+async def store_candidate_phrases_for_book(
+    session: AsyncSession,
+    book_id: int,
+    series_id: int | None,
+    limited_candidates: list[PhraseCandidate],
+    config: EbookSearchConfig,
+    *,
+    replace_all: bool = False,
+) -> int:
+    """Persist already-extracted candidate phrase rows for one book without committing.
+
+    Args:
+        session (Session): Active database session.
+        book_id (int): Book the candidates belong to.
+        series_id (int | None): Series scope for the stored candidates.
+        limited_candidates (list[PhraseCandidate]): Scored candidates to persist.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        replace_all (bool): When the caller has already cleared this book's candidates, skip the
+            per-candidate existence lookup and bulk-insert new rows.
+
+    Returns:
+        int: Number of candidate phrase rows stored.
+    """
+    save_started_at = perf_counter()
+    if replace_all:
+        rows = [new_candidate_row(book_id, series_id, candidate) for candidate in limited_candidates]
+        session.add_all(rows)
+        await session.flush()
+        saved_count = len(rows)
+        logger.info(
+            f"ebook_candidate_phrase_save_start {book_id=} candidates={len(limited_candidates)} mode=bulk_insert"
+        )
+    else:
+        pruned_count = await prune_unstorable_unjudged_candidate_phrases(session, book_id, config)
+        logger.info(
+            f"ebook_candidate_phrase_save_start {book_id=} candidates={len(limited_candidates)} {pruned_count=}"
+        )
+        saved_count = await bulk_upsert_unjudged_candidates(session, book_id, series_id, limited_candidates)
+    logger.info(
+        f"ebook_candidate_phrase_save_complete {book_id=} {saved_count=} "
+        f"save_ms={(perf_counter() - save_started_at) * 1000:.1f}"
+    )
+    return saved_count

python/ebook_search/protected_phrases/judge_ngrams.py

@@ -0,0 +1,473 @@
+"""Book-level orchestration for LLM judging and promotion of candidate phrases."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from dataclasses import replace
+from time import perf_counter
+from typing import TYPE_CHECKING
+
+import httpx
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from python.ebook_search.llm_interface import request_chat_completion
+from python.ebook_search.prompts import load_prompt
+from python.ebook_search.protected_phrases.extraction import (
+    candidate_source_names,
+    get_sample_contexts,
+    is_junk_phrase,
+    is_most_common_word_phrase,
+    score_candidate,
+)
+from python.ebook_search.protected_phrases.matching import index_chunk_phrase_mentions_for_book
+from python.ebook_search.protected_phrases.models import BookJudgmentResult, LLMJudgment, PhraseJudgmentBackfillResult
+from python.ebook_search.protected_phrases.store import (
+    count_protected_phrases,
+    count_unjudged_candidates,
+    load_book_text,
+    load_candidates_for_judgment,
+    phrase_candidate_from_row,
+    save_candidate_to_db,
+    upsert_protected_phrase,
+)
+from python.ebook_search.protected_phrases.text_normalization import normalize_text
+from python.orm.richie import EbookSource
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from sqlalchemy.ext.asyncio import AsyncEngine
+
+    from python.ebook_search.config import EbookSearchConfig
+    from python.ebook_search.protected_phrases.models import PhraseCandidate
+    from python.orm.richie import EbookProtectedPhrase
+
+logger = logging.getLogger(__name__)
+
+
+async def judge_candidate_phrases_for_books(
+    engine: AsyncEngine,
+    config: EbookSearchConfig,
+    *,
+    source_ids: Sequence[int] | None = None,
+) -> PhraseJudgmentBackfillResult:
+    """Judge candidate phrases for books, fanning LLM calls out across books and phrases.
+
+    Up to ``phrase_judge_book_workers`` books are judged at once, and within each book candidates
+    are judged in concurrent chunks of ``phrase_judge_phrase_workers``. Each book uses its own
+    short-lived sessions for reads and writes; no database connection is held while LLM calls are
+    in flight. For a pseudo-single-threaded run (solo testing, debugging), set both worker
+    settings to 1.
+
+    Args:
+        engine (AsyncEngine): Engine used to open one session per book.
+        config (EbookSearchConfig): Runtime phrase-tuning settings and chat configuration.
+        source_ids (Sequence[int] | None): Books to judge; ``None`` judges every indexed book.
+
+    Returns:
+        PhraseJudgmentBackfillResult: Per-corpus counts of books judged, failures, candidates,
+        protected phrases, and mentions.
+    """
+    if source_ids is None:
+        async with AsyncSession(engine) as session:
+            source_ids = list((await session.scalars(select(EbookSource.id).order_by(EbookSource.id))).all())
+    books_seen = len(source_ids)
+    book_workers = max(1, config.phrase_judge_book_workers)
+    phrase_workers = max(1, config.phrase_judge_phrase_workers)
+    logger.info(
+        f"ebook_candidate_phrase_judgment_start {books_seen=} {book_workers=} {phrase_workers=} "
+        f"{config.protected_phrase_confidence_threshold=:.2f}"
+    )
+
+    book_semaphore = asyncio.Semaphore(book_workers)
+    max_connections = book_workers * phrase_workers
+    limits = httpx.Limits(max_connections=max_connections, max_keepalive_connections=max_connections)
+    async with httpx.AsyncClient(limits=limits) as client:
+        outcomes = await asyncio.gather(
+            *(judge_one_book_async(engine, source_id, config, client, book_semaphore) for source_id in source_ids)
+        )
+
+    result = PhraseJudgmentBackfillResult(
+        books_seen=books_seen,
+        books_judged=sum(1 for outcome in outcomes if outcome.committed),
+        books_failed=sum(1 for outcome in outcomes if outcome.failed),
+        candidates_judged=sum(outcome.judged for outcome in outcomes),
+        protected_phrases=sum(outcome.protected for outcome in outcomes),
+        phrase_mentions=sum(outcome.mentions for outcome in outcomes),
+    )
+    logger.info(
+        f"ebook_candidate_phrase_judgment_complete {result.books_seen=} {result.books_judged=} {result.books_failed=} "
+        f"{result.candidates_judged=} {result.protected_phrases=} {result.phrase_mentions=}"
+    )
+    return result
+
+
+async def judge_one_book_async(
+    engine: AsyncEngine,
+    source_id: int,
+    config: EbookSearchConfig,
+    client: httpx.AsyncClient,
+    book_semaphore: asyncio.Semaphore,
+) -> BookJudgmentResult:
+    """Judge one book concurrently and persist the outcome, honoring the book-level limit.
+
+    Args:
+        engine (AsyncEngine): Engine used to open the book's read and write sessions.
+        source_id (int): Book to judge candidates for.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        client (httpx.AsyncClient): Shared async client for LLM calls.
+        book_semaphore (asyncio.Semaphore): Caps how many books judge at once.
+
+    Returns:
+        BookJudgmentResult: The book's judgment outcome.
+    """
+    async with book_semaphore:
+        try:
+            prepared = await prepare_book_judgment(engine, source_id, config)
+            if prepared is None:
+                return BookJudgmentResult()
+            work_items, target_remaining = prepared
+            judged = await judge_book_candidates_async(client, config, source_id, work_items, target_remaining)
+            if not judged:
+                return BookJudgmentResult()
+            return await persist_book_judgments(engine, source_id, config, judged)
+        except Exception:
+            logger.exception(f"ebook_candidate_phrase_judgment_book_failed {source_id=}")
+            return BookJudgmentResult(failed=True)
+
+
+async def prepare_book_judgment(
+    engine: AsyncEngine,
+    source_id: int,
+    config: EbookSearchConfig,
+) -> tuple[list[tuple[int, PhraseCandidate]], int | None] | None:
+    """Load one book's candidates to judge, with sample contexts, on a short-lived read session.
+
+    Args:
+        engine (AsyncEngine): Engine used to open the read session.
+        source_id (int): Book to load candidates for.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        tuple[list[tuple[int, PhraseCandidate]], int | None] | None: Candidate rows paired with
+        in-memory candidates and the remaining protected-phrase target, or ``None`` when the book
+        has nothing to judge.
+    """
+    judgment_limit = config.protected_phrase_llm_candidates_per_book
+    if judgment_limit <= 0:
+        return None
+    async with AsyncSession(engine) as session:
+        if not await count_unjudged_candidates(session, source_id, config):
+            logger.info(f"ebook_candidate_phrase_judgment_book_skip_no_unjudged {source_id=}")
+            return None
+        existing_protected = await count_protected_phrases(session, source_id)
+        target_remaining: int | None = None
+        if config.phrase_target_protected_per_book > 0:
+            target_remaining = max(config.phrase_target_protected_per_book - existing_protected, 0)
+            if target_remaining == 0:
+                logger.info(
+                    f"ebook_candidate_phrase_judgment_skipped_target_met {source_id=} {existing_protected=} "
+                    f"{config.phrase_target_protected_per_book=}"
+                )
+                return None
+        book_text = await load_book_text(session, source_id)
+        if not book_text:
+            logger.warning(f"ebook_candidate_phrase_judgment_book_empty {source_id=}")
+            return None
+        normalized_book_text = normalize_text(book_text)
+        # Stored rows may predate the current junk filters and score weights, so re-filter and
+        # rescore every unjudged row here instead of trusting the persisted candidate_score.
+        rows = await load_candidates_for_judgment(session, source_id, config)
+        scored_items: list[tuple[int, PhraseCandidate]] = []
+        skipped_junk = 0
+        for row in rows:
+            candidate = phrase_candidate_from_row(row)
+            if is_junk_phrase(candidate.phrase_norm.split()):
+                skipped_junk += 1
+                continue
+            candidate.candidate_score = score_candidate(candidate, config)
+            scored_items.append((row.id, candidate))
+        scored_items.sort(key=lambda item: item[1].candidate_score, reverse=True)
+        work_items = scored_items[:judgment_limit]
+        for _, candidate in work_items:
+            candidate.sample_contexts = candidate.sample_contexts or get_sample_contexts(
+                normalized_book_text, candidate.phrase_norm
+            )
+    logger.info(
+        f"ebook_candidate_phrase_judgment_candidates_loaded {source_id=} candidates={len(work_items)} {skipped_junk=} "
+        f"unjudged_rows={len(rows)} {existing_protected=} {target_remaining=} {judgment_limit=}"
+    )
+    return work_items, target_remaining
+
+
+async def judge_book_candidates_async(
+    client: httpx.AsyncClient,
+    config: EbookSearchConfig,
+    source_id: int,
+    work_items: list[tuple[int, PhraseCandidate]],
+    target_remaining: int | None,
+) -> list[tuple[int, PhraseCandidate, LLMJudgment, bool]]:
+    """Judge a book's candidates in concurrent chunks, stopping once the target is reached.
+
+    Promotion decisions are made in memory so judging can stop early without any database writes.
+
+    Args:
+        client (httpx.AsyncClient): Shared async client for LLM calls.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        source_id (int): Book being judged, for logging.
+        work_items (list[tuple[int, PhraseCandidate]]): Candidate row ids paired with candidates,
+            in best-first score order.
+        target_remaining (int | None): Remaining protected-phrase target, or ``None`` for no cap.
+
+    Returns:
+        list[tuple[int, PhraseCandidate, LLMJudgment, bool]]: Judged rows with their judgment and
+        whether each should be promoted.
+    """
+    chunk_size = max(1, config.phrase_judge_phrase_workers)
+    judged: list[tuple[int, PhraseCandidate, LLMJudgment, bool]] = []
+    promoted = 0
+    for start in range(0, len(work_items), chunk_size):
+        chunk = work_items[start : start + chunk_size]
+        judgments = await asyncio.gather(*(judge_candidate_async(client, config, candidate) for _, candidate in chunk))
+        for (candidate_id, candidate), judgment in zip(chunk, judgments, strict=True):
+            promote = (target_remaining is None or promoted < target_remaining) and should_protect_judged_candidate(
+                candidate, judgment, source_id, config, candidate_id=candidate_id
+            )
+            if promote:
+                promoted += 1
+            judged.append((candidate_id, candidate, judgment, promote))
+        if target_remaining is not None and promoted >= target_remaining:
+            break
+    return judged
+
+
+async def judge_candidate_async(
+    client: httpx.AsyncClient,
+    config: EbookSearchConfig,
+    candidate: PhraseCandidate,
+) -> LLMJudgment:
+    """Judge one candidate with the LLM over the shared async client.
+
+    Args:
+        client (httpx.AsyncClient): Shared async client for LLM calls.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        candidate (PhraseCandidate): Candidate to judge.
+
+    Returns:
+        LLMJudgment: The parsed judgment.
+    """
+    content = await request_chat_completion(
+        client, config, build_judge_messages(candidate), response_format={"type": "json_object"}
+    )
+    return parse_llm_judgment(content, config)
+
+
+async def persist_book_judgments(
+    engine: AsyncEngine,
+    source_id: int,
+    config: EbookSearchConfig,
+    judged: list[tuple[int, PhraseCandidate, LLMJudgment, bool]],
+) -> BookJudgmentResult:
+    """Persist one book's judgments and promotions in a single committed transaction.
+
+    Args:
+        engine (AsyncEngine): Engine used to open the write session.
+        source_id (int): Book being persisted.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        judged (list[tuple[int, PhraseCandidate, LLMJudgment, bool]]): Judged candidates with their
+            judgment and promotion flag.
+
+    Returns:
+        BookJudgmentResult: The book's committed counts, or a failed result on error.
+    """
+    book_started_at = perf_counter()
+    async with AsyncSession(engine, expire_on_commit=False) as session:
+        try:
+            normalized_book_text = normalize_text(await load_book_text(session, source_id))
+            protected: list[EbookProtectedPhrase] = []
+            for candidate_id, candidate, judgment, promote in judged:
+                filtered_judgment = replace(
+                    judgment,
+                    aliases=tuple(
+                        alias for alias in judgment.aliases if alias_occurs_in_book(alias, normalized_book_text)
+                    ),
+                )
+                candidate_row = await save_candidate_to_db(
+                    session, source_id, None, candidate, judgment=filtered_judgment
+                )
+                if promote:
+                    protected.append(
+                        await upsert_protected_phrase(
+                            session, source_id, None, candidate, filtered_judgment, candidate_row
+                        )
+                    )
+                logger.info(
+                    f"ebook_candidate_phrase_judgment_candidate_complete {source_id=} {candidate_id=} "
+                    f"{candidate.phrase_norm=} {judgment.keep=} {judgment.confidence=:.3f} {judgment.category=} "
+                    f"{promote=}"
+                )
+            await session.flush()
+            mentions = await index_chunk_phrase_mentions_for_book(session, source_id, config) if protected else 0
+            await session.commit()
+        except Exception:
+            await session.rollback()
+            logger.exception(f"ebook_candidate_phrase_judgment_book_persist_failed {source_id=}")
+            return BookJudgmentResult(failed=True)
+    logger.info(
+        f"ebook_candidate_phrase_judgment_book_committed {source_id=} judged={len(judged)} protected={len(protected)} "
+        f"{mentions=} duration_ms={(perf_counter() - book_started_at) * 1000:.1f}"
+    )
+    return BookJudgmentResult(judged=len(judged), protected=len(protected), mentions=mentions, committed=True)
+
+
+def should_protect_judged_candidate(
+    candidate: PhraseCandidate,
+    judgment: LLMJudgment,
+    book_id: int,
+    config: EbookSearchConfig,
+    *,
+    candidate_id: int,
+) -> bool:
+    """Report whether a judged candidate qualifies to become a protected phrase.
+
+    Args:
+        candidate (PhraseCandidate): In-memory candidate that was judged.
+        judgment (LLMJudgment): Judge decision for the candidate.
+        book_id (int): Book the candidate belongs to, for logging.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        candidate_id (int): Stored candidate row id the judgment came from, for logging.
+
+    Returns:
+        bool: True when the judged candidate should be promoted to a protected phrase.
+    """
+    if not judgment.keep or judgment.confidence < config.protected_phrase_confidence_threshold:
+        return False
+    accepted_norm = normalize_text(judgment.canonical or candidate.phrase_text)
+    accepted_tokens = accepted_norm.split()
+    accepted_token_count = len(accepted_tokens)
+    if accepted_token_count < config.phrase_min_tokens:
+        logger.info(
+            f"ebook_candidate_phrase_judgment_candidate_skip_short_canonical {book_id=} {candidate_id=} "
+            f"{candidate.phrase_norm=} {accepted_norm=} {accepted_token_count=} {config.phrase_min_tokens=}"
+        )
+        return False
+    if is_most_common_word_phrase(accepted_tokens):
+        logger.info(
+            f"ebook_candidate_phrase_judgment_candidate_skip_common_canonical {book_id=} {candidate_id=} "
+            f"{candidate.phrase_norm=} {accepted_norm=}"
+        )
+        return False
+    return True
+
+
+def build_judge_messages(candidate: PhraseCandidate) -> list[dict[str, str]]:
+    """Build the chat messages used to judge one candidate phrase.
+
+    Args:
+        candidate (PhraseCandidate): Candidate to describe for the judge.
+
+    Returns:
+        list[dict[str, str]]: OpenAI-style system and user messages.
+    """
+    payload = {
+        "phrase": candidate.phrase_norm,
+        "token_count": candidate.token_count,
+        "sources": candidate_source_names(candidate),
+        "raw_count": candidate.raw_count,
+        "chapter_count": candidate.chapter_count,
+        "contexts": candidate.sample_contexts,
+    }
+    return load_prompt("phrase_judge").messages(candidate_json=json.dumps(payload, ensure_ascii=True))
+
+
+def parse_llm_judgment(content: str, config: EbookSearchConfig) -> LLMJudgment:
+    """Parse and validate an LLM phrase-judge response.
+
+    Args:
+        content (str): Raw model response text.
+        config (EbookSearchConfig): Runtime phrase-tuning settings supplying nesting defaults.
+
+    Returns:
+        LLMJudgment: The parsed and validated judgment.
+
+    Raises:
+        TypeError: If the decoded JSON body is not an object.
+    """
+    body = json.loads(extract_json_object(content))
+    if not isinstance(body, dict):
+        msg = "LLM phrase judge response is not a JSON object"
+        raise TypeError(msg)
+
+    aliases = body.get("aliases", ())
+    if not isinstance(aliases, list | tuple):
+        aliases = ()
+    return LLMJudgment(
+        keep=strict_bool(body.get("keep"), default=False),
+        canonical=optional_text(body.get("canonical")),
+        category=optional_text(body.get("category")),
+        aliases=tuple(str(alias) for alias in aliases if isinstance(alias, str) and alias.strip()),
+        confidence=clamped_float(body.get("confidence"), default=0.0),
+        importance=clamped_float(body.get("importance"), default=0.5),
+        allow_nested=strict_bool(body.get("allow_nested"), default=config.phrase_default_allow_nested),
+        suppress_children=strict_bool(body.get("suppress_children"), default=config.phrase_default_suppress_children),
+        reason=optional_text(body.get("reason")),
+    )
+
+
+def extract_json_object(content: str) -> str:
+    """Extract a JSON object from plain or fenced model output.
+
+    Args:
+        content (str): Raw model response text.
+
+    Returns:
+        str: The substring spanning the first JSON object.
+
+    Raises:
+        ValueError: If no JSON object is found in the response.
+    """
+    return content.strip()
+
+
+def alias_occurs_in_book(alias: str, normalized_book_text: str) -> bool:
+    """Return whether a normalized alias occurs as a complete phrase in the source book."""
+    alias_norm = normalize_text(alias)
+    return bool(alias_norm) and f" {alias_norm} " in f" {normalized_book_text} "
+
+
+def optional_text(value: object) -> str | None:
+    """Return stripped text for a nullable JSON value.
+
+    Args:
+        value (object): Decoded JSON value that may or may not be a string.
+
+    Returns:
+        str | None: The stripped string, or ``None`` when it is not a non-empty string.
+    """
+    if not isinstance(value, str):
+        return None
+    stripped = value.strip()
+    return stripped or None
+
+
+def strict_bool(value: object, *, default: bool) -> bool:
+    """Return a JSON boolean, falling back when the value has another type."""
+    return value if isinstance(value, bool) else default
+
+
+def clamped_float(value: object, *, default: float) -> float:
+    """Coerce a JSON number into the 0.0 to 1.0 range.
+
+    Args:
+        value (object): Decoded JSON value that may or may not be a number.
+        default (float): Fallback returned when ``value`` is not numeric.
+
+    Returns:
+        float: The value clamped to ``[0.0, 1.0]``, or ``default`` when non-numeric.
+    """
+    if not isinstance(value, int | float):
+        return default
+    return min(max(float(value), 0.0), 1.0)

python/ebook_search/protected_phrases/matching.py

@@ -0,0 +1,405 @@
+"""Runtime protected-phrase matching and chunk mention indexing."""
+
+from __future__ import annotations
+
+import logging
+from collections import defaultdict
+from typing import TYPE_CHECKING
+
+from sqlalchemy import and_, delete, func, or_, select, union
+
+from python.ebook_search.protected_phrases.config import get_ignored_phrases
+from python.ebook_search.protected_phrases.models import (
+    ChunkPhraseHit,
+    PhraseLookup,
+    PhraseMatch,
+)
+from python.ebook_search.protected_phrases.text_normalization import tokenize_with_offsets
+from python.orm.richie import (
+    EbookChunk,
+    EbookChunkPhraseMention,
+    EbookPhraseAlias,
+    EbookProtectedPhrase,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator, Sequence
+
+    from sqlalchemy.ext.asyncio import AsyncSession
+
+    from python.ebook_search.config import EbookSearchConfig
+
+logger = logging.getLogger(__name__)
+
+
+async def detect_protected_phrases_for_query(
+    session: AsyncSession,
+    query_text: str,
+    config: EbookSearchConfig,
+) -> list[PhraseMatch]:
+    """Find query phrases with indexed exact matches on canonical and alias norms.
+
+    Args:
+        session (AsyncSession): Active database session.
+        query_text (str): User query text to detect phrases in.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        list[PhraseMatch]: Metadata-backed, overlap-resolved phrase matches for the query.
+    """
+    tokens_ = tokenize_with_offsets(query_text)
+    windows_by_norm: defaultdict[str, list[tuple[int, int]]] = defaultdict(list)
+    token_texts = [token.text for token in tokens_]
+    max_tokens = max(config.phrase_max_tokens, config.phrase_max_entity_tokens)
+    for phrase_norm, start, end in generate_query_ngrams(
+        token_texts,
+        min_n=config.phrase_min_tokens,
+        max_n=max_tokens,
+    ):
+        windows_by_norm[phrase_norm].append((start, end))
+    if not windows_by_norm:
+        return []
+
+    query_norms = tuple(windows_by_norm)
+    matched_norms = union(
+        select(
+            EbookProtectedPhrase.id.label("phrase_id"),
+            EbookProtectedPhrase.phrase_norm.label("matched_norm"),
+        ).where(EbookProtectedPhrase.phrase_norm.in_(query_norms)),
+        select(
+            EbookPhraseAlias.phrase_id.label("phrase_id"),
+            EbookPhraseAlias.alias_norm.label("matched_norm"),
+        ).where(EbookPhraseAlias.alias_norm.in_(query_norms)),
+    ).subquery()
+    statement = select(EbookProtectedPhrase, matched_norms.c.matched_norm).join(
+        matched_norms,
+        matched_norms.c.phrase_id == EbookProtectedPhrase.id,
+    )
+
+    matches: list[PhraseMatch] = []
+    for phrase, matched_norm in await session.execute(statement):
+        for start, end in windows_by_norm[matched_norm]:
+            matches.append(
+                PhraseMatch(
+                    phrase_id=phrase.id,
+                    matched_norm=matched_norm,
+                    phrase_text=phrase.phrase_text,
+                    phrase_norm=phrase.phrase_norm,
+                    canonical_id=phrase.canonical_id,
+                    phrase_type=phrase.phrase_type,
+                    token_count=end - start,
+                    confidence=phrase.confidence,
+                    importance=phrase.importance,
+                    allow_nested=phrase.allow_nested,
+                    suppress_children=phrase.suppress_children,
+                    start_token=start,
+                    end_token=end,
+                    start_char=tokens_[start].start_char,
+                    end_char=tokens_[end - 1].end_char,
+                    book_id=phrase.book_id,
+                    series_id=phrase.series_id,
+                )
+            )
+    return resolve_overlaps(matches)
+
+
+async def index_chunk_phrase_mentions_for_book(
+    session: AsyncSession,
+    book_id: int,
+    config: EbookSearchConfig,
+) -> int:
+    """Rebuild chunk phrase mentions for all chunks in one book.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose chunk mentions are rebuilt.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        int: Total number of chunk phrase mentions indexed for the book.
+    """
+    lookup = await load_phrase_lookup(session, config, book_id=book_id)
+    await session.execute(delete(EbookChunkPhraseMention).where(EbookChunkPhraseMention.book_id == book_id))
+    chunks = await session.scalars(select(EbookChunk).where(EbookChunk.source_id == book_id).order_by(EbookChunk.id))
+    count = 0
+    for chunk in chunks:
+        count += index_chunk_phrase_mentions(session, chunk, lookup=lookup)
+    await session.flush()
+    logger.info(f"ebook_chunk_phrase_mentions_indexed {book_id=} {count=}")
+    return count
+
+
+async def load_phrase_lookup(
+    session: AsyncSession,
+    config: EbookSearchConfig,
+    *,
+    book_id: int | None = None,
+    series_id: int | None = None,
+) -> PhraseLookup:
+    """Load protected phrases and aliases into RAM lookup maps.
+
+    Args:
+        session (AsyncSession): Active database session.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        book_id (int | None): Optional book scope to restrict loaded phrases.
+        series_id (int | None): Optional series scope to restrict loaded phrases.
+
+    Returns:
+        PhraseLookup: Normalized phrase and alias maps with the token-window bounds to test.
+    """
+    phrase_ids_by_norm: defaultdict[str, set[int]] = defaultdict(set)
+    phrases_by_id: dict[int, EbookProtectedPhrase] = {}
+    max_tokens = config.phrase_max_tokens
+
+    statement = select(
+        EbookProtectedPhrase,
+        EbookPhraseAlias.alias_norm,
+    ).outerjoin(EbookPhraseAlias, EbookPhraseAlias.phrase_id == EbookProtectedPhrase.id)
+    scope_filter = protected_phrase_scope_filter(book_id=book_id, series_id=series_id)
+    if scope_filter is not None:
+        statement = statement.where(scope_filter)
+
+    for phrase, alias_norm in await session.execute(statement):
+        phrases_by_id[phrase.id] = phrase
+        phrase_ids_by_norm[phrase.phrase_norm].add(phrase.id)
+        max_tokens = max(max_tokens, phrase.token_count)
+        if alias_norm is not None:
+            phrase_ids_by_norm[alias_norm].add(phrase.id)
+            max_tokens = max(max_tokens, len(alias_norm.split()))
+
+    return PhraseLookup(
+        phrase_ids_by_norm={key: tuple(sorted(values)) for key, values in phrase_ids_by_norm.items()},
+        phrases_by_id=phrases_by_id,
+        min_tokens=config.phrase_min_tokens,
+        max_tokens=max_tokens,
+    )
+
+
+def protected_phrase_scope_filter(*, book_id: int | None, series_id: int | None) -> object | None:
+    """Build a SQLAlchemy filter for optional phrase book and series scope.
+
+    Args:
+        book_id (int | None): Optional book scope to include alongside global phrases.
+        series_id (int | None): Optional series scope to include alongside global phrases.
+
+    Returns:
+        object | None: A combined SQLAlchemy filter clause, or ``None`` when no scope is given.
+    """
+    conditions = []
+    if book_id is not None:
+        conditions.append(or_(EbookProtectedPhrase.book_id.is_(None), EbookProtectedPhrase.book_id == book_id))
+    if series_id is not None:
+        conditions.append(or_(EbookProtectedPhrase.series_id.is_(None), EbookProtectedPhrase.series_id == series_id))
+    if not conditions:
+        return None
+    return and_(*conditions)
+
+
+def is_inside(child: PhraseMatch, parent: PhraseMatch) -> bool:
+    """Return whether one token span is strictly inside another.
+
+    Args:
+        child (PhraseMatch): Candidate nested match.
+        parent (PhraseMatch): Candidate enclosing match.
+
+    Returns:
+        bool: True when ``child`` lies within ``parent`` and is not the same span.
+    """
+    return (
+        child.start_token >= parent.start_token
+        and child.end_token <= parent.end_token
+        and (child.start_token, child.end_token, child.phrase_id)
+        != (parent.start_token, parent.end_token, parent.phrase_id)
+    )
+
+
+def index_chunk_phrase_mentions(session: AsyncSession, chunk: EbookChunk, *, lookup: PhraseLookup) -> int:
+    """Store protected phrase mentions for one chunk.
+
+    Args:
+        session (AsyncSession): Active database session.
+        chunk (EbookChunk): Chunk whose text is scanned for phrase mentions.
+        lookup (PhraseLookup): In-memory phrase and alias lookup maps.
+
+    Returns:
+        int: Number of phrase mentions stored for the chunk.
+    """
+    tokens_ = tokenize_with_offsets(chunk.text)
+    token_texts = [token.text for token in tokens_]
+    raw_matches: list[PhraseMatch] = []
+    phrase_windows = generate_query_ngrams(token_texts, min_n=lookup.min_tokens, max_n=lookup.max_tokens)
+    for matched_norm, start, end in phrase_windows:
+        for phrase_id in lookup.phrase_ids_by_norm.get(matched_norm, ()):
+            phrase = lookup.phrases_by_id[phrase_id]
+            raw_matches.append(
+                PhraseMatch(
+                    phrase_id=phrase_id,
+                    matched_norm=matched_norm,
+                    phrase_text=phrase.phrase_text,
+                    phrase_norm=phrase.phrase_norm,
+                    canonical_id=phrase.canonical_id,
+                    phrase_type=phrase.phrase_type,
+                    confidence=phrase.confidence,
+                    importance=phrase.importance,
+                    allow_nested=phrase.allow_nested,
+                    suppress_children=phrase.suppress_children,
+                    start_token=start,
+                    end_token=end,
+                    token_count=end - start,
+                    start_char=tokens_[start].start_char,
+                    end_char=tokens_[end - 1].end_char,
+                    book_id=phrase.book_id,
+                    series_id=phrase.series_id,
+                )
+            )
+
+    matches = resolve_overlaps(raw_matches)
+    for match in matches:
+        session.add(
+            EbookChunkPhraseMention(
+                chunk_id=chunk.id,
+                phrase_id=match.phrase_id,
+                book_id=match.book_id if match.book_id is not None else chunk.source_id,
+                series_id=match.series_id,
+                start_char=match.start_char if match.start_char is not None else 0,
+                end_char=match.end_char,
+            )
+        )
+    return len(matches)
+
+
+def resolve_overlaps(matches: Sequence[PhraseMatch]) -> list[PhraseMatch]:
+    """Resolve overlapping phrase matches without relying only on longest match.
+
+    Args:
+        matches (Sequence[PhraseMatch]): Metadata-backed matches that may overlap.
+
+    Returns:
+        list[PhraseMatch]: The kept, non-suppressed matches.
+    """
+    sorted_matches = sorted(
+        matches,
+        key=lambda match: (match.start_token, -match.token_count, -match.importance, -match.confidence),
+    )
+    kept: list[PhraseMatch] = []
+    for candidate in sorted_matches:
+        if any(should_suppress(candidate, existing) for existing in kept):
+            continue
+        kept.append(candidate)
+    return kept
+
+
+def should_suppress(candidate: PhraseMatch, kept: PhraseMatch) -> bool:
+    """Return whether an already-kept match should suppress a candidate.
+
+    Args:
+        candidate (PhraseMatch): Match being considered for keeping.
+        kept (PhraseMatch): Match already kept that may suppress the candidate.
+
+    Returns:
+        bool: True when the candidate should be dropped in favor of the kept match.
+    """
+    if not overlaps(candidate, kept):
+        return False
+    if candidate.canonical_id == kept.canonical_id:
+        return rank_match(kept) >= rank_match(candidate)
+    if is_inside(candidate, kept) and kept.suppress_children and not candidate.allow_nested:
+        return True
+    return not candidate.allow_nested and rank_match(kept) > rank_match(candidate)
+
+
+def overlaps(first: PhraseMatch, second: PhraseMatch) -> bool:
+    """Return whether two token spans overlap.
+
+    Args:
+        first (PhraseMatch): First match to compare.
+        second (PhraseMatch): Second match to compare.
+
+    Returns:
+        bool: True when the two token spans share at least one token position.
+    """
+    return not (first.end_token <= second.start_token or first.start_token >= second.end_token)
+
+
+def rank_match(match: PhraseMatch) -> tuple[float, float, int]:
+    """Rank phrase matches by importance, confidence, then token count.
+
+    Args:
+        match (PhraseMatch): Match to build a sort key for.
+
+    Returns:
+        tuple[float, float, int]: A comparable key of importance, confidence, and token count.
+    """
+    return (match.importance, match.confidence, match.token_count)
+
+
+def generate_query_ngrams(
+    tokens_: Sequence[str],
+    min_n: int,
+    max_n: int,
+) -> Iterator[tuple[str, int, int]]:
+    """Generate normalized query windows from longest to shortest.
+
+    Args:
+        tokens_ (Sequence[str]): Normalized query tokens.
+        min_n (int): Smallest window size to yield.
+        max_n (int): Largest window size to yield, capped at the token count.
+
+    Yields:
+        tuple[str, int, int]: Normalized window text with its start and end token indices.
+    """
+    capped_max_n = min(max_n, len(tokens_))
+    for ngram_size in range(capped_max_n, min_n - 1, -1):
+        for start in range(len(tokens_) - ngram_size + 1):
+            end = start + ngram_size
+            phrase_norm = " ".join(tokens_[start:end])
+            if phrase_norm in get_ignored_phrases():
+                continue
+            yield phrase_norm, start, end
+
+
+async def phrase_hits_for_chunks(
+    session: AsyncSession,
+    *,
+    chunk_ids: Sequence[int],
+    phrase_ids: Sequence[int],
+) -> dict[int, tuple[ChunkPhraseHit, ...]]:
+    """Return matched protected phrases with mention counts by chunk id.
+
+    Args:
+        session (AsyncSession): Active database session.
+        chunk_ids (Sequence[int]): Chunk ids to look up mentions for.
+        phrase_ids (Sequence[int]): Protected phrase ids to restrict the results to.
+
+    Returns:
+        dict[int, tuple[ChunkPhraseHit, ...]]: Phrase hits per chunk id, ordered by mention count.
+    """
+    if not chunk_ids or not phrase_ids:
+        return {}
+
+    mention_count = func.count(EbookChunkPhraseMention.phrase_id).label("mention_count")
+    statement = (
+        select(
+            EbookChunkPhraseMention.chunk_id,
+            EbookProtectedPhrase.id.label("phrase_id"),
+            EbookProtectedPhrase.phrase_text,
+            mention_count,
+        )
+        .join(EbookProtectedPhrase, EbookProtectedPhrase.id == EbookChunkPhraseMention.phrase_id)
+        .where(
+            EbookChunkPhraseMention.chunk_id.in_(chunk_ids),
+            EbookChunkPhraseMention.phrase_id.in_(phrase_ids),
+        )
+        .group_by(EbookChunkPhraseMention.chunk_id, EbookProtectedPhrase.id, EbookProtectedPhrase.phrase_text)
+        .order_by(EbookChunkPhraseMention.chunk_id, mention_count.desc(), EbookProtectedPhrase.phrase_text)
+    )
+    hits: defaultdict[int, list[ChunkPhraseHit]] = defaultdict(list)
+    for row in await session.execute(statement):
+        hits[row.chunk_id].append(
+            ChunkPhraseHit(
+                phrase_id=row.phrase_id,
+                phrase_text=row.phrase_text,
+                mention_count=row.mention_count,
+            )
+        )
+    return {chunk_id: tuple(chunk_hits) for chunk_id, chunk_hits in hits.items()}

python/ebook_search/protected_phrases/models.py

@@ -0,0 +1,258 @@
+"""Dataclasses shared by protected phrase extraction, judging, matching, and backfills."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+    from python.orm.richie import EbookProtectedPhrase
+
+
+@dataclass(slots=True)
+class PhraseCandidate:
+    """A phrase candidate with merged extraction-source metadata.
+
+    Attributes:
+        phrase_text (str): Display text for the phrase.
+        phrase_norm (str): Normalized phrase used as the merge key.
+        token_count (int): Number of normalized tokens in the phrase.
+        source_raw_ngram (bool): Whether the raw n-gram extractor produced the phrase.
+        source_yake (bool): Whether YAKE keyword extraction produced the phrase.
+        source_capitalized (bool): Whether the capitalized-run extractor produced the phrase.
+        source_metadata (bool): Whether book metadata produced the phrase.
+        raw_count (int): Occurrences counted across the book text.
+        chapter_count (int): Number of chapters containing the phrase.
+        yake_score (float | None): Raw YAKE score when available; lower is better.
+        candidate_score (float): Combined pre-judging score.
+        sample_contexts (list[str]): Normalized context snippets around occurrences.
+    """
+
+    phrase_text: str
+    phrase_norm: str
+    token_count: int
+    source_raw_ngram: bool = False
+    source_yake: bool = False
+    source_capitalized: bool = False
+    source_metadata: bool = False
+    raw_count: int = 0
+    chapter_count: int = 0
+    yake_score: float | None = None
+    candidate_score: float = 0.0
+    sample_contexts: list[str] = field(default_factory=list)
+
+
+@dataclass(frozen=True, slots=True)
+class LLMJudgment:
+    """A structured phrase judgment returned by the LLM judge.
+
+    Attributes:
+        keep (bool): Whether the judge accepted the phrase for protection.
+        canonical (str | None): Canonical phrase text chosen by the judge.
+        category (str | None): Phrase category such as person, place, or event.
+        aliases (tuple[str, ...]): Alternate surface forms for the phrase.
+        confidence (float): Judge confidence between 0.0 and 1.0.
+        importance (float): Judge importance between 0.0 and 1.0.
+        allow_nested (bool): Whether the phrase may match inside a larger kept match.
+        suppress_children (bool): Whether the phrase suppresses matches nested inside it.
+        reason (str | None): Free-text explanation from the judge.
+    """
+
+    keep: bool
+    canonical: str | None
+    category: str | None
+    aliases: tuple[str, ...]
+    confidence: float
+    importance: float = 0.5
+    allow_nested: bool = False
+    suppress_children: bool = True
+    reason: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class PhraseLookup:
+    """In-memory phrase metadata used for constant-time text-window checks.
+
+    Attributes:
+        phrase_ids_by_norm (Mapping[str, tuple[int, ...]]): Canonical and alias norms to phrase ids.
+        phrases_by_id (Mapping[int, EbookProtectedPhrase]): Protected phrase metadata by id.
+        min_tokens (int): Smallest token-window size to test.
+        max_tokens (int): Largest token-window size to test.
+    """
+
+    phrase_ids_by_norm: Mapping[str, tuple[int, ...]]
+    phrases_by_id: Mapping[int, EbookProtectedPhrase]
+    min_tokens: int
+    max_tokens: int
+
+
+@dataclass(frozen=True, slots=True)
+class PhraseMatch:
+    """A detected phrase match with protected-phrase metadata attached.
+
+    Attributes:
+        phrase_id (int): Protected phrase id.
+        matched_norm (str): Normalized window text that matched.
+        phrase_text (str): Display text of the protected phrase.
+        phrase_norm (str): Normalized text of the protected phrase.
+        canonical_id (str): Deterministic ``category:slug`` identifier.
+        phrase_type (str | None): Phrase category.
+        token_count (int): Number of tokens in the match.
+        confidence (float): Stored judge confidence.
+        importance (float): Stored judge importance.
+        allow_nested (bool): Whether the phrase may match inside a larger kept match.
+        suppress_children (bool): Whether the phrase suppresses matches nested inside it.
+        start_token (int): Index of the first matched token.
+        end_token (int): Index one past the last matched token.
+        start_char (int | None): Start character offset in the source text.
+        end_char (int | None): End character offset in the source text.
+        book_id (int | None): Book scope of the phrase.
+        series_id (int | None): Series scope of the phrase.
+    """
+
+    phrase_id: int
+    matched_norm: str
+    phrase_text: str
+    phrase_norm: str
+    canonical_id: str
+    phrase_type: str | None
+    token_count: int
+    confidence: float
+    importance: float
+    allow_nested: bool
+    suppress_children: bool
+    start_token: int
+    end_token: int
+    start_char: int | None = None
+    end_char: int | None = None
+    book_id: int | None = None
+    series_id: int | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class ChunkPhraseHit:
+    """One protected phrase with its mention count inside one retrieved chunk.
+
+    Attributes:
+        phrase_id (int): Protected phrase id.
+        phrase_text (str): Display text for the phrase.
+        mention_count (int): Indexed mentions inside the chunk.
+    """
+
+    phrase_id: int
+    phrase_text: str
+    mention_count: int
+
+
+@dataclass(frozen=True, slots=True)
+class PhraseCandidateGenerationResult:
+    """Summary of candidate phrase extraction for indexed books.
+
+    Attributes:
+        books_seen (int): Indexed books examined.
+        books_built (int): Books that had candidates generated and committed.
+        candidate_phrases (int): Candidate phrases stored across all books.
+    """
+
+    books_seen: int
+    books_built: int
+    candidate_phrases: int
+
+
+@dataclass(frozen=True, slots=True)
+class CorpusPhraseStats:
+    """Corpus-wide candidate and protected phrase counts for the admin page.
+
+    Attributes:
+        total_books (int): Indexed books in the corpus.
+        books_with_candidates (int): Books that have candidate phrases generated.
+        books_fully_judged (int): Books with candidates where every candidate has been judged.
+        candidate_phrases (int): Candidate phrases stored across all books.
+        judged_candidates (int): Candidate phrases that have been LLM judged.
+        unjudged_candidates (int): Candidate phrases still waiting for judgment.
+        protected_phrases (int): Protected phrases promoted across all books.
+    """
+
+    total_books: int
+    books_with_candidates: int
+    books_fully_judged: int
+    candidate_phrases: int
+    judged_candidates: int
+    unjudged_candidates: int
+    protected_phrases: int
+
+
+@dataclass(frozen=True, slots=True)
+class PhraseJudgmentBackfillResult:
+    """Summary of LLM judging for stored candidate phrases.
+
+    Attributes:
+        books_seen (int): Indexed books examined.
+        books_judged (int): Books with judgments committed.
+        books_failed (int): Books rolled back after an error.
+        candidates_judged (int): Candidate phrases sent to the LLM judge.
+        protected_phrases (int): Protected phrases promoted from candidates.
+        phrase_mentions (int): Chunk phrase mentions indexed across all books.
+    """
+
+    books_seen: int
+    books_judged: int
+    books_failed: int
+    candidates_judged: int
+    protected_phrases: int
+    phrase_mentions: int
+
+
+@dataclass(frozen=True, slots=True)
+class BookJudgmentResult:
+    """Outcome of judging one book's candidate phrases.
+
+    Attributes:
+        judged (int): Candidate phrases sent to the LLM judge.
+        protected (int): Protected phrases promoted from candidates.
+        mentions (int): Chunk phrase mentions indexed for the book.
+        committed (bool): Whether the book's judgments were committed.
+        failed (bool): Whether the book was rolled back after an error.
+    """
+
+    judged: int = 0
+    protected: int = 0
+    mentions: int = 0
+    committed: bool = False
+    failed: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class BookCandidateResult:
+    """Outcome of generating one book's candidate phrases.
+
+    Attributes:
+        candidates (int): Candidate phrases stored for the book.
+        built (bool): Whether candidate generation was committed.
+    """
+
+    candidates: int = 0
+    built: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class PhraseRecalculationResult:
+    """Summary of phrase cleanup and candidate regeneration for one book.
+
+    Attributes:
+        book_id (int): Book the recalculation ran against.
+        deleted_candidates (int): Candidate phrase rows deleted.
+        deleted_protected_phrases (int): Protected phrase rows deleted.
+        deleted_aliases (int): Phrase alias rows deleted.
+        deleted_mentions (int): Chunk phrase mention rows deleted.
+        candidate_phrases (int): Candidate phrases regenerated after cleanup.
+    """
+
+    book_id: int
+    deleted_candidates: int
+    deleted_protected_phrases: int
+    deleted_aliases: int
+    deleted_mentions: int
+    candidate_phrases: int

python/ebook_search/protected_phrases/pool.py

@@ -0,0 +1,58 @@
+"""Process pool for offloading CPU-bound phrase extraction off the request thread.
+
+Phrase extraction is pure-Python CPU work (n-gram sliding, YAKE), so running it inline in a
+sync request handler serializes concurrent recalculations behind the GIL. Submitting it to a
+``ProcessPoolExecutor`` lets concurrent extractions run in parallel across cores instead. A
+``spawn`` context is used so workers do not inherit the parent's database engine, connections,
+or server threads.
+"""
+
+from __future__ import annotations
+
+import logging
+import multiprocessing
+import os
+from concurrent.futures import ProcessPoolExecutor
+from threading import Lock
+
+logger = logging.getLogger(__name__)
+
+
+class _ExtractionPool:
+    """Lazily created process-wide extraction pool and the lock guarding it."""
+
+    def __init__(self) -> None:
+        self.lock = Lock()
+        self.pool: ProcessPoolExecutor | None = None
+
+
+_extraction_pool = _ExtractionPool()
+
+
+def get_extraction_pool(max_workers: int) -> ProcessPoolExecutor:
+    """Return the shared extraction process pool, creating it on first use.
+
+    Args:
+        max_workers (int): Desired worker count; values below 1 fall back to the CPU count.
+
+    Returns:
+        ProcessPoolExecutor: The shared pool for phrase extraction.
+    """
+    with _extraction_pool.lock:
+        if _extraction_pool.pool is None:
+            workers = max_workers if max_workers > 0 else (os.cpu_count() or 1)
+            _extraction_pool.pool = ProcessPoolExecutor(
+                max_workers=workers,
+                mp_context=multiprocessing.get_context("spawn"),
+            )
+            logger.info(f"ebook_phrase_extraction_pool_started {workers=}")
+        return _extraction_pool.pool
+
+
+def shutdown_extraction_pool() -> None:
+    """Shut down the shared extraction pool if it was started."""
+    with _extraction_pool.lock:
+        if _extraction_pool.pool is not None:
+            _extraction_pool.pool.shutdown(wait=False, cancel_futures=True)
+            _extraction_pool.pool = None
+            logger.info("ebook_phrase_extraction_pool_shutdown")

python/ebook_search/protected_phrases/store.py

@@ -0,0 +1,662 @@
+"""Database persistence for candidate and protected phrase rows."""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import TYPE_CHECKING
+
+from sqlalchemy import delete, func, or_, select
+from sqlalchemy.dialects.postgresql import insert
+
+from python.ebook_search.protected_phrases.extraction import minimum_candidate_raw_count
+from python.ebook_search.protected_phrases.models import (
+    CorpusPhraseStats,
+    PhraseCandidate,
+    PhraseRecalculationResult,
+)
+from python.ebook_search.protected_phrases.text_normalization import normalize_text
+from python.orm.richie import (
+    EbookCandidatePhrase,
+    EbookChunk,
+    EbookChunkPhraseMention,
+    EbookPhraseAlias,
+    EbookProtectedPhrase,
+    EbookSource,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from sqlalchemy.ext.asyncio import AsyncSession
+
+    from python.ebook_search.config import EbookSearchConfig
+    from python.ebook_search.protected_phrases.models import LLMJudgment
+
+logger = logging.getLogger(__name__)
+
+
+async def load_book_text(session: AsyncSession, book_id: int) -> str:
+    """Load a book's indexed chunk text as one string for phrase extraction.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose chunk text is loaded.
+
+    Returns:
+        str: The book's chunk text joined into a single string.
+    """
+    texts = await session.scalars(
+        select(EbookChunk.text).where(EbookChunk.source_id == book_id).order_by(EbookChunk.chunk_index)
+    )
+    return "\n\n".join(stripped for text in texts if (stripped := text.strip()))
+
+
+async def load_book_chapter_texts(session: AsyncSession, book_id: int) -> list[str]:
+    """Reconstruct chapter-like text blocks from indexed chunks for phrase extraction.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose chunks are grouped into chapters.
+
+    Returns:
+        list[str]: Non-empty chapter-like text blocks in chunk order.
+    """
+    rows = await session.execute(
+        select(EbookChunk.chapter_id, EbookChunk.text)
+        .where(EbookChunk.source_id == book_id)
+        .order_by(EbookChunk.chunk_index)
+    )
+    chapters: list[str] = []
+    current_chapter_id: int | None = None
+    current_parts: list[str] = []
+    have_current = False
+
+    for chapter_id, text in rows:
+        if have_current and chapter_id != current_chapter_id:
+            chapter_text = "\n\n".join(current_parts).strip()
+            if chapter_text:
+                chapters.append(chapter_text)
+            current_parts = []
+        current_chapter_id = chapter_id
+        current_parts.append(str(text))
+        have_current = True
+
+    if current_parts:
+        chapter_text = "\n\n".join(current_parts).strip()
+        if chapter_text:
+            chapters.append(chapter_text)
+    return chapters
+
+
+def metadata_for_source(source: EbookSource) -> dict[str, object | None]:
+    """Return phrase extraction metadata for one indexed source.
+
+    Args:
+        source (EbookSource): Indexed source to read metadata from.
+
+    Returns:
+        dict[str, object | None]: Title, author, language, publisher, and identifier values.
+    """
+    return {
+        "title": source.title,
+        "author": source.author,
+        "language": source.language,
+        "publisher": source.publisher,
+        "identifier": source.identifier,
+    }
+
+
+async def metadata_for_source_id(session: AsyncSession, source_id: int) -> dict[str, object | None]:
+    """Return phrase extraction metadata for one indexed source by id.
+
+    Args:
+        session (AsyncSession): Active database session.
+        source_id (int): Id of the indexed source to read metadata from.
+
+    Returns:
+        dict[str, object | None]: Title, author, language, publisher, and identifier values.
+
+    Raises:
+        ValueError: If no source exists with the given id.
+    """
+    source = await session.get(EbookSource, source_id)
+    if source is None:
+        msg = f"No indexed source with id {source_id}"
+        raise ValueError(msg)
+    return metadata_for_source(source)
+
+
+async def count_protected_phrases(session: AsyncSession, book_id: int) -> int:
+    """Count stored protected phrases for one book.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose protected phrases are counted.
+
+    Returns:
+        int: Number of protected phrases stored for the book.
+    """
+    return (
+        await session.scalars(
+            select(func.count(EbookProtectedPhrase.id)).where(EbookProtectedPhrase.book_id == book_id)
+        )
+    ).one()
+
+
+async def count_unjudged_candidates(session: AsyncSession, book_id: int, config: EbookSearchConfig) -> int:
+    """Count storable candidate rows for a book that have not yet been judged.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose unjudged candidates are counted.
+        config (EbookSearchConfig): Runtime phrase-tuning settings supplying storage thresholds.
+
+    Returns:
+        int: Number of storable, unjudged candidate rows for the book.
+    """
+    return (
+        await session.scalars(
+            select(func.count(EbookCandidatePhrase.id)).where(
+                EbookCandidatePhrase.book_id == book_id,
+                EbookCandidatePhrase.llm_judged.is_(False),
+                EbookCandidatePhrase.token_count >= config.phrase_min_tokens,
+                EbookCandidatePhrase.raw_count >= minimum_candidate_raw_count(config),
+            )
+        )
+    ).one()
+
+
+async def corpus_phrase_stats(session: AsyncSession) -> CorpusPhraseStats:
+    """Summarize candidate and protected phrase coverage across the whole corpus.
+
+    Args:
+        session (AsyncSession): Active database session.
+
+    Returns:
+        CorpusPhraseStats: Corpus-wide phrase counts and per-book coverage counts.
+    """
+    total_books = (await session.scalars(select(func.count(EbookSource.id)))).one()
+    candidate_phrases, judged_candidates, books_with_candidates, books_with_unjudged = (
+        await session.execute(
+            select(
+                func.count(EbookCandidatePhrase.id),
+                func.count(EbookCandidatePhrase.id).filter(EbookCandidatePhrase.llm_judged.is_(True)),
+                func.count(func.distinct(EbookCandidatePhrase.book_id)),
+                func.count(func.distinct(EbookCandidatePhrase.book_id)).filter(
+                    EbookCandidatePhrase.llm_judged.is_(False)
+                ),
+            )
+        )
+    ).one()
+    protected_phrases = (await session.scalars(select(func.count(EbookProtectedPhrase.id)))).one()
+    return CorpusPhraseStats(
+        total_books=total_books,
+        books_with_candidates=books_with_candidates,
+        books_fully_judged=books_with_candidates - books_with_unjudged,
+        candidate_phrases=candidate_phrases,
+        judged_candidates=judged_candidates,
+        unjudged_candidates=candidate_phrases - judged_candidates,
+        protected_phrases=protected_phrases,
+    )
+
+
+async def book_ids_pending_first_judgment(session: AsyncSession) -> list[int]:
+    """Return books that have candidate phrases but no judged candidates yet.
+
+    Args:
+        session (AsyncSession): Active database session.
+
+    Returns:
+        list[int]: Book ids with candidates where judging has never run, ordered by id.
+    """
+    judged_books = select(EbookCandidatePhrase.book_id).where(EbookCandidatePhrase.llm_judged.is_(True)).distinct()
+    return list(
+        (
+            await session.scalars(
+                select(EbookCandidatePhrase.book_id)
+                .where(EbookCandidatePhrase.book_id.not_in(judged_books))
+                .distinct()
+                .order_by(EbookCandidatePhrase.book_id)
+            )
+        ).all()
+    )
+
+
+async def load_candidates_for_judgment(
+    session: AsyncSession,
+    book_id: int,
+    config: EbookSearchConfig,
+) -> Sequence[EbookCandidatePhrase]:
+    """Load every storable unjudged candidate row for a book.
+
+    Rows may have been stored before the current junk filters and score weights existed, so
+    callers re-check :func:`is_junk_phrase` and rescore before selecting what to judge.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose candidates are loaded.
+        config (EbookSearchConfig): Runtime phrase-tuning settings supplying storage thresholds.
+
+    Returns:
+        Sequence[EbookCandidatePhrase]: Storable, unjudged candidate rows ordered by stored score.
+    """
+    query = (
+        select(EbookCandidatePhrase)
+        .where(
+            EbookCandidatePhrase.book_id == book_id,
+            EbookCandidatePhrase.llm_judged.is_(False),
+            EbookCandidatePhrase.token_count >= config.phrase_min_tokens,
+            EbookCandidatePhrase.raw_count >= minimum_candidate_raw_count(config),
+        )
+        .order_by(
+            EbookCandidatePhrase.candidate_score.desc(),
+            EbookCandidatePhrase.raw_count.desc(),
+            EbookCandidatePhrase.id,
+        )
+    )
+    return (await session.scalars(query)).all()
+
+
+def phrase_candidate_from_row(row: EbookCandidatePhrase) -> PhraseCandidate:
+    """Recreate an in-memory candidate from a persisted candidate row.
+
+    Args:
+        row (EbookCandidatePhrase): Stored candidate row to convert.
+
+    Returns:
+        PhraseCandidate: An in-memory candidate mirroring the row's fields.
+    """
+    return PhraseCandidate(
+        phrase_text=row.phrase_text,
+        phrase_norm=row.phrase_norm,
+        token_count=row.token_count,
+        source_raw_ngram=row.source_raw_ngram,
+        source_yake=row.source_yake,
+        source_capitalized=row.source_capitalized,
+        source_metadata=row.source_metadata,
+        raw_count=row.raw_count,
+        chapter_count=row.chapter_count,
+        yake_score=row.yake_score,
+        candidate_score=row.candidate_score,
+        sample_contexts=row.sample_contexts or [],
+    )
+
+
+def candidate_row_values(
+    book_id: int,
+    series_id: int | None,
+    candidate: PhraseCandidate,
+    *,
+    judgment: LLMJudgment | None,
+) -> dict[str, object]:
+    """Build the column values for one candidate phrase upsert.
+
+    Args:
+        book_id (int): Book the candidate belongs to.
+        series_id (int | None): Series scope stored on the row.
+        candidate (PhraseCandidate): Candidate whose fields are written to the row.
+        judgment (LLMJudgment | None): Judgment to record, or ``None`` to leave the row unjudged.
+
+    Returns:
+        dict[str, object]: Column values keyed by column name.
+    """
+    values: dict[str, object] = {
+        "book_id": book_id,
+        "phrase_norm": candidate.phrase_norm,
+        "series_id": series_id,
+        "phrase_text": candidate.phrase_text,
+        "token_count": candidate.token_count,
+        "source_raw_ngram": candidate.source_raw_ngram,
+        "source_yake": candidate.source_yake,
+        "source_capitalized": candidate.source_capitalized,
+        "source_metadata": candidate.source_metadata,
+        "raw_count": candidate.raw_count,
+        "chapter_count": candidate.chapter_count,
+        "yake_score": candidate.yake_score,
+        "candidate_score": candidate.candidate_score,
+        "llm_judged": judgment is not None,
+    }
+    if candidate.sample_contexts:
+        values["sample_contexts"] = list(candidate.sample_contexts)
+    if judgment is not None:
+        values.update(
+            llm_keep=judgment.keep,
+            llm_confidence=judgment.confidence,
+            llm_category=judgment.category,
+            llm_reason=judgment.reason,
+        )
+    return values
+
+
+async def save_candidate_to_db(
+    session: AsyncSession,
+    book_id: int,
+    series_id: int | None,
+    candidate: PhraseCandidate,
+    *,
+    judgment: LLMJudgment | None,
+) -> EbookCandidatePhrase:
+    """Insert or update one candidate phrase row.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book the candidate belongs to.
+        series_id (int | None): Series scope stored on the row.
+        candidate (PhraseCandidate): Candidate whose fields are written to the row.
+        judgment (LLMJudgment | None): Judgment to record, or ``None`` to leave the row unjudged.
+
+    Returns:
+        EbookCandidatePhrase: The inserted or updated candidate row.
+    """
+    values = candidate_row_values(book_id, series_id, candidate, judgment=judgment)
+
+    # Preserve an existing judgment when this call is only refreshing candidate fields.
+    skip_update = {"book_id", "phrase_norm"}
+    if judgment is None:
+        skip_update.add("llm_judged")
+    insert_statement = insert(EbookCandidatePhrase).values(**values)
+    statement = insert_statement.on_conflict_do_update(
+        index_elements=["book_id", "phrase_norm"],
+        set_={column: insert_statement.excluded[column] for column in values if column not in skip_update},
+    ).returning(EbookCandidatePhrase)
+    return (await session.scalars(statement, execution_options={"populate_existing": True})).one()
+
+
+BULK_CANDIDATE_UPSERT_CHUNK = 1000
+
+
+async def bulk_upsert_unjudged_candidates(
+    session: AsyncSession,
+    book_id: int,
+    series_id: int | None,
+    candidates: Sequence[PhraseCandidate],
+) -> int:
+    """Insert or update many freshly extracted candidate rows in chunked multi-row upserts.
+
+    Saving one row per statement costs one database round trip per candidate, which dominated
+    generation time for full books, so candidates are written ``BULK_CANDIDATE_UPSERT_CHUNK``
+    rows per statement instead. Existing judgments and sample contexts are never overwritten:
+    fresh extractions carry no contexts, and ``llm_judged`` plus the ``llm_*`` columns are left
+    out of the conflict update. Candidates must have unique ``phrase_norm`` values, as produced
+    by extraction, since one multi-row upsert cannot touch the same row twice.
+
+    Args:
+        session (Session): Active database session.
+        book_id (int): Book the candidates belong to.
+        series_id (int | None): Series scope stored on the rows.
+        candidates (Sequence[PhraseCandidate]): Freshly extracted candidates to persist.
+
+    Returns:
+        int: Number of candidate rows written.
+    """
+    values = [
+        candidate_row_values(book_id, series_id, candidate, judgment=None)
+        for candidate in candidates
+        if not candidate.sample_contexts
+    ]
+    if len(values) != len(candidates):
+        msg = "bulk_upsert_unjudged_candidates only accepts freshly extracted candidates without sample contexts"
+        raise ValueError(msg)
+    skip_update = {"book_id", "phrase_norm", "llm_judged"}
+    for chunk_start in range(0, len(values), BULK_CANDIDATE_UPSERT_CHUNK):
+        chunk = values[chunk_start : chunk_start + BULK_CANDIDATE_UPSERT_CHUNK]
+        insert_statement = insert(EbookCandidatePhrase).values(chunk)
+        statement = insert_statement.on_conflict_do_update(
+            index_elements=["book_id", "phrase_norm"],
+            set_={column: insert_statement.excluded[column] for column in chunk[0] if column not in skip_update},
+        )
+        await session.execute(statement)
+    return len(values)
+
+
+def new_candidate_row(book_id: int, series_id: int | None, candidate: PhraseCandidate) -> EbookCandidatePhrase:
+    """Build a fresh unjudged candidate row without checking for an existing one.
+
+    Unlike :func:`save_candidate_to_db`, this does no lookup, so it is only safe when the caller
+    guarantees there is no existing row for ``(book_id, candidate.phrase_norm)`` — for example
+    right after :func:`delete_phrase_data_for_book` has cleared the book.
+
+    Args:
+        book_id (int): Book the candidate belongs to.
+        series_id (int | None): Series scope stored on the row.
+        candidate (PhraseCandidate): Candidate whose fields are written to the row.
+
+    Returns:
+        EbookCandidatePhrase: A new, unattached candidate row.
+    """
+    row = EbookCandidatePhrase(book_id=book_id, phrase_norm=candidate.phrase_norm)
+    row.llm_judged = False
+    row.series_id = series_id
+    row.phrase_text = candidate.phrase_text
+    row.token_count = candidate.token_count
+    row.source_raw_ngram = candidate.source_raw_ngram
+    row.source_yake = candidate.source_yake
+    row.source_capitalized = candidate.source_capitalized
+    row.source_metadata = candidate.source_metadata
+    row.raw_count = candidate.raw_count
+    row.chapter_count = candidate.chapter_count
+    row.yake_score = candidate.yake_score
+    row.candidate_score = candidate.candidate_score
+    if candidate.sample_contexts:
+        row.sample_contexts = list(candidate.sample_contexts)
+    return row
+
+
+async def upsert_protected_phrase(
+    session: AsyncSession,
+    book_id: int,
+    series_id: int | None,
+    candidate: PhraseCandidate,
+    judgment: LLMJudgment,
+    source_candidate: EbookCandidatePhrase,
+) -> EbookProtectedPhrase:
+    """Insert or update one accepted protected phrase and its aliases.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book the protected phrase belongs to.
+        series_id (int | None): Series scope stored on the phrase.
+        candidate (PhraseCandidate): Candidate the phrase was promoted from.
+        judgment (LLMJudgment): Accepted judgment supplying canonical text, category, and aliases.
+        source_candidate (EbookCandidatePhrase): Candidate row the phrase was promoted from.
+
+    Returns:
+        EbookProtectedPhrase: The inserted or updated protected phrase row.
+
+    Raises:
+        ValueError: If the chosen phrase text normalizes to empty.
+    """
+    phrase_text = judgment.canonical or candidate.phrase_text
+    phrase_norm = normalize_text(phrase_text)
+    if not phrase_norm:
+        msg = f"Protected phrase normalized to empty text: {phrase_text!r}"
+        raise ValueError(msg)
+
+    values = {
+        "book_id": book_id,
+        "phrase_norm": phrase_norm,
+        "series_id": series_id,
+        "phrase_text": phrase_text,
+        "canonical_id": make_canonical_id(judgment, phrase_norm),
+        "phrase_type": judgment.category,
+        "token_count": len(phrase_norm.split()),
+        "confidence": judgment.confidence,
+        "importance": judgment.importance,
+        "allow_nested": judgment.allow_nested,
+        "suppress_children": judgment.suppress_children,
+        "source_candidate_id": source_candidate.id,
+    }
+    insert_statement = insert(EbookProtectedPhrase).values(**values)
+    statement = insert_statement.on_conflict_do_update(
+        index_elements=["book_id", "phrase_norm"],
+        set_={
+            column: insert_statement.excluded[column] for column in values if column not in {"book_id", "phrase_norm"}
+        },
+    ).returning(EbookProtectedPhrase)
+    row = (await session.scalars(statement, execution_options={"populate_existing": True})).one()
+
+    for alias_text in judgment.aliases:
+        await upsert_phrase_alias(session, row, alias_text)
+    return row
+
+
+async def upsert_phrase_alias(
+    session: AsyncSession,
+    phrase: EbookProtectedPhrase,
+    alias_text: str,
+) -> EbookPhraseAlias | None:
+    """Insert or update one protected phrase alias.
+
+    Args:
+        session (AsyncSession): Active database session.
+        phrase (EbookProtectedPhrase): Protected phrase the alias points to.
+        alias_text (str): Alias surface form to store.
+
+    Returns:
+        EbookPhraseAlias | None: The alias row, or ``None`` when the alias is empty or equals the phrase.
+    """
+    alias_norm = normalize_text(alias_text)
+    if not alias_norm or alias_norm == phrase.phrase_norm:
+        return None
+
+    insert_statement = insert(EbookPhraseAlias).values(
+        phrase_id=phrase.id,
+        alias_norm=alias_norm,
+        alias_text=alias_text,
+        confidence=1.0,
+    )
+    statement = insert_statement.on_conflict_do_update(
+        index_elements=["phrase_id", "alias_norm"],
+        set_={
+            "alias_text": insert_statement.excluded.alias_text,
+            "confidence": insert_statement.excluded.confidence,
+        },
+    ).returning(EbookPhraseAlias)
+    return (await session.scalars(statement, execution_options={"populate_existing": True})).one()
+
+
+def make_canonical_id(judgment: LLMJudgment, phrase_norm: str) -> str:
+    """Create a deterministic canonical id from a judgment category and phrase.
+
+    Args:
+        judgment (LLMJudgment): Judgment supplying the phrase category.
+        phrase_norm (str): Normalized phrase text to slugify.
+
+    Returns:
+        str: A ``category:slug`` canonical identifier.
+    """
+    category = slugify_identifier(judgment.category or "phrase")
+    phrase_slug = slugify_identifier(phrase_norm)
+    return f"{category}:{phrase_slug}"
+
+
+def slugify_identifier(value: str) -> str:
+    """Normalize text for use inside a canonical id.
+
+    Args:
+        value (str): Text to slugify.
+
+    Returns:
+        str: A lowercase underscore slug, or ``"unknown"`` when empty.
+    """
+    slug = re.sub(r"[^a-z0-9]+", "_", normalize_text(value).replace("'", ""))
+    return slug.strip("_") or "unknown"
+
+
+async def prune_unstorable_unjudged_candidate_phrases(
+    session: AsyncSession,
+    book_id: int,
+    config: EbookSearchConfig,
+) -> int:
+    """Delete old unjudged candidate rows that no longer satisfy storage filters.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose stale candidates are pruned.
+        config (EbookSearchConfig): Runtime phrase-tuning settings supplying storage thresholds.
+
+    Returns:
+        int: Number of candidate rows deleted.
+    """
+    deleted = rowcount(
+        await session.execute(
+            delete(EbookCandidatePhrase).where(
+                EbookCandidatePhrase.book_id == book_id,
+                EbookCandidatePhrase.llm_judged.is_(False),
+                or_(
+                    EbookCandidatePhrase.token_count < config.phrase_min_tokens,
+                    EbookCandidatePhrase.raw_count < minimum_candidate_raw_count(config),
+                ),
+            )
+        )
+    )
+    if deleted:
+        logger.info(
+            f"ebook_candidate_phrase_unstorable_pruned {book_id=} {deleted=} {config.phrase_min_tokens=} "
+            f"min_uses={minimum_candidate_raw_count(config)}"
+        )
+    return deleted
+
+
+async def delete_phrase_data_for_book(session: AsyncSession, book_id: int) -> PhraseRecalculationResult:
+    """Delete all candidate, protected, alias, and mention phrase data for one book.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose phrase data is deleted.
+
+    Returns:
+        PhraseRecalculationResult: Deleted-row counts with ``candidate_phrases`` set to 0.
+    """
+    protected_ids = (
+        await session.scalars(select(EbookProtectedPhrase.id).where(EbookProtectedPhrase.book_id == book_id))
+    ).all()
+    deleted_aliases = 0
+    if protected_ids:
+        deleted_aliases = rowcount(
+            await session.execute(delete(EbookPhraseAlias).where(EbookPhraseAlias.phrase_id.in_(protected_ids)))
+        )
+
+    deleted_mentions = rowcount(
+        await session.execute(delete(EbookChunkPhraseMention).where(EbookChunkPhraseMention.book_id == book_id))
+    )
+    if protected_ids:
+        deleted_mentions += rowcount(
+            await session.execute(
+                delete(EbookChunkPhraseMention).where(EbookChunkPhraseMention.phrase_id.in_(protected_ids))
+            )
+        )
+
+    deleted_protected = rowcount(
+        await session.execute(delete(EbookProtectedPhrase).where(EbookProtectedPhrase.book_id == book_id))
+    )
+    deleted_candidates = rowcount(
+        await session.execute(delete(EbookCandidatePhrase).where(EbookCandidatePhrase.book_id == book_id))
+    )
+    await session.flush()
+    logger.info(
+        f"ebook_candidate_phrase_data_deleted {book_id=} {deleted_candidates=} {deleted_protected=} {deleted_aliases=} "
+        f"{deleted_mentions=}"
+    )
+    return PhraseRecalculationResult(
+        book_id=book_id,
+        deleted_candidates=deleted_candidates,
+        deleted_protected_phrases=deleted_protected,
+        deleted_aliases=deleted_aliases,
+        deleted_mentions=deleted_mentions,
+        candidate_phrases=0,
+    )
+
+
+def rowcount(result: object) -> int:
+    """Return a safe integer rowcount from a SQLAlchemy execution result.
+
+    Args:
+        result (object): SQLAlchemy execution result that may expose ``rowcount``.
+
+    Returns:
+        int: The result's rowcount, or 0 when it is missing or negative.
+    """
+    count = getattr(result, "rowcount", 0)
+    return int(count if count is not None and count >= 0 else 0)

python/ebook_search/protected_phrases/text_normalization.py

@@ -0,0 +1,91 @@
+"""Protected phrase extraction, storage, and runtime matching."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+JSON_OBJECT_RE = re.compile(r"\{.*\}", re.DOTALL)
+
+
+@dataclass(frozen=True, slots=True)
+class NormalizedToken:
+    """A normalized token plus its source character span."""
+
+    text: str
+    start_char: int
+    end_char: int
+
+
+def normalize_text(text: str) -> str:
+    """Normalize text for phrase storage and lookup.
+
+    Args:
+        text (str): Raw text to normalize.
+
+    Returns:
+        str: Normalized tokens joined by single spaces.
+    """
+    return " ".join(token.text for token in tokenize_with_offsets(text))
+
+
+def tokenize(text: str) -> list[str]:
+    """Normalize and split text into phrase-detection tokens.
+
+    Args:
+        text (str): Raw text to tokenize.
+
+    Returns:
+        list[str]: Normalized token strings.
+    """
+    return [token.text for token in tokenize_with_offsets(text)]
+
+
+def tokenize_with_offsets(text: str) -> list[NormalizedToken]:
+    """Normalize text into tokens while preserving original character offsets.
+
+    Args:
+        text (str): Raw text to tokenize.
+
+    Returns:
+        list[NormalizedToken]: Normalized tokens with their source character spans.
+    """
+    tokens: list[NormalizedToken] = []
+    current: list[str] = []
+    start_char: int | None = None
+
+    for index, char in enumerate(text):
+        normalized = normalize_char(char)
+        if normalized == " ":
+            if current and start_char is not None:
+                tokens.append(NormalizedToken(text="".join(current), start_char=start_char, end_char=index))
+                current = []
+                start_char = None
+            continue
+        if start_char is None:
+            start_char = index
+        current.append(normalized)
+
+    if current and start_char is not None:
+        tokens.append(NormalizedToken(text="".join(current), start_char=start_char, end_char=len(text)))
+    return tokens
+
+
+def normalize_char(char: str) -> str:
+    """Normalize one character into a token character or a separator.
+
+    Args:
+        char (str): Single source character to normalize.
+
+    Returns:
+        str: The normalized token character, or a space acting as a separator.
+    """
+    if char in {"\u2019", "\u2018"}:
+        return "'"
+    if char in {"-", "\u2013", "\u2014"}:
+        return " "
+
+    lowered = char.lower()
+    if lowered in "abcdefghijklmnopqrstuvwxyz0123456789'":
+        return lowered
+    return " "

python/ebook_search/rerank.py

@@ -9,6 +9,8 @@ from typing import TYPE_CHECKING
 from python.ebook_search.llm_interface import request_rerank
 
 if TYPE_CHECKING:
+    import httpx
+
     from python.ebook_search.config import RerankConfig
     from python.ebook_search.search import SearchResult
 
@@ -23,18 +25,18 @@ class RerankResult:
     score: float
 
 
-def rerank_chunks(query: str, candidates: list[SearchResult], config: RerankConfig) -> list[SearchResult]:
+async def rerank_chunks(
+    client: httpx.AsyncClient,
+    query: str,
+    candidates: list[SearchResult],
+    config: RerankConfig,
+) -> list[SearchResult]:
     """Rerank candidates with a vLLM rerank endpoint."""
     if not candidates:
         return []
 
-    logger.info(
-        "ebook_rerank_request_start base_url=%s model=%s candidates=%s",
-        config.base_url,
-        config.model,
-        len(candidates),
-    )
-    scores = score_candidates(query, candidates, config)
+    logger.info(f"ebook_rerank_request_start {config.base_url=} {config.model=} candidates={len(candidates)}")
+    scores = await score_candidates(client, query, candidates, config)
     results = sorted(
         (
             replace(
@@ -47,28 +49,24 @@ def rerank_chunks(query: str, candidates: list[SearchResult], config: RerankConf
         key=lambda result: result.score,
         reverse=True,
     )
-    logger.info(
-        "ebook_rerank_request_complete base_url=%s model=%s candidates=%s",
-        config.base_url,
-        config.model,
-        len(results),
-    )
+    logger.info(f"ebook_rerank_request_complete {config.base_url=} {config.model=} candidates={len(results)}")
     return results
 
 
-def score_candidates(
+async def score_candidates(
+    client: httpx.AsyncClient,
     query: str,
     candidates: list[SearchResult],
     config: RerankConfig,
 ) -> dict[int, RerankResult]:
     """Score candidate chunks with the configured rerank API."""
-    body = request_rerank(query, [candidate.text for candidate in candidates], config)
+    body = await request_rerank(client, query, [candidate.text for candidate in candidates], config)
     if body is None:
         return zero_rerank_scores(candidates)
 
     scores = parse_vllm_scores(body, candidates)
     for result in scores.values():
-        logger.debug("ebook_rerank_candidate_scored chunk_id=%s score=%s", result.chunk_id, result.score)
+        logger.debug(f"ebook_rerank_candidate_scored {result.chunk_id=} {result.score=}")
     return scores
 
 

python/ebook_search/search.py

@@ -2,25 +2,31 @@
 
 from __future__ import annotations
 
+import asyncio
 import logging
 import re
 from collections import defaultdict
-from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, replace
 from typing import TYPE_CHECKING
 
 from pgvector.sqlalchemy import Vector
 from sqlalchemy import literal, select
-from sqlalchemy.orm import Session
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.ext.asyncio import AsyncSession
 
 from python.ebook_search.bm25_corpus import (
     BM25CorpusUnavailableError,
     load_bm25_corpus,
     score_bm25_corpus,
 )
+from python.ebook_search.chunk_records import CHUNK_RECORD_COLUMNS
 from python.ebook_search.embeddings import MODEL_DIMENSIONS, embed_query, get_embedding_table
+from python.ebook_search.protected_phrases.matching import (
+    detect_protected_phrases_for_query,
+    phrase_hits_for_chunks,
+)
 from python.ebook_search.rerank import rerank_chunks
-from python.ebook_search.timing import RuntimeStep, timed_result
+from python.ebook_search.timing import RuntimeStep, async_timed_result, timed_result
 from python.orm.richie import (
     EbookChapter,
     EbookChunk,
@@ -29,11 +35,13 @@ from python.orm.richie import (
 )
 
 if TYPE_CHECKING:
-    from collections.abc import Mapping
+    from collections.abc import Mapping, Sequence
 
-    from sqlalchemy.engine import Engine
+    import httpx
+    from sqlalchemy.ext.asyncio import AsyncEngine
 
     from python.ebook_search.config import EbookSearchConfig
+    from python.ebook_search.protected_phrases.models import PhraseMatch
 
 logger = logging.getLogger(__name__)
 
@@ -45,11 +53,14 @@ class SearchResult:
     chunk_id: int
     text: str
     source_title: str
+    source_id: int | None = None
     score: float = 0.0
     vector_score: float | None = None
     bm25_score: float | None = None
     fused_score: float | None = None
     rerank_score: float | None = None
+    phrase_hit_count: int = 0
+    matched_phrases: tuple[str, ...] = ()
     source_author: str | None = None
     chapter_title: str | None = None
     page_label: str | None = None
@@ -64,6 +75,7 @@ class SearchResponse:
     results: list[SearchResult]
     rank_label: str
     timings: tuple[RuntimeStep, ...] = ()
+    phrase_matches: tuple[PhraseMatch, ...] = ()
 
     @property
     def total_runtime_ms(self) -> float:
@@ -73,34 +85,40 @@ class SearchResponse:
 
 @dataclass(frozen=True)
 class RetrievalResponse:
-    """Parallel retrieval output for vector and BM25 candidates."""
+    """Parallel retrieval output for vector, BM25, and protected phrase candidates."""
 
     vector_results: list[SearchResult]
     lexical_results: list[SearchResult]
+    phrase_matches: list[PhraseMatch]
     timings: tuple[RuntimeStep, ...]
 
 
-def search_ebooks(
-    engine: Engine,
+async def search_ebooks(
+    engine: AsyncEngine,
+    client: httpx.AsyncClient,
     query: str,
     config: EbookSearchConfig,
     *,
-    rerank: bool = False,
+    rerank: bool,
+    phrase_matching: bool,
 ) -> SearchResponse:
-    """Run hybrid vector/BM25 search and optional reranking."""
+    """Run hybrid vector/BM25 search and optional reranking.
+
+    Phrase matching only runs when both the request asks for it and
+    ``config.phrase_matching_enabled`` allows it.
+    """
     if not query.strip():
         logger.info("ebook_search_empty_query")
         return SearchResponse(query=query, results=[], rank_label="Hybrid")
 
-    logger.info("ebook_search_start query_length=%s rerank=%s", len(query), rerank)
+    phrase_matching = phrase_matching and config.phrase_matching_enabled
+    logger.info(f"ebook_search_start query_length={len(query)} {rerank=} {phrase_matching=}")
     timings: list[RuntimeStep] = []
-    retrieval, timing = timed_result(
+    retrieval, timing = await async_timed_result(
         "Hybrid retrieval",
-        parallel_retrieval,
-        engine,
-        query,
-        config,
+        parallel_retrieval(engine, client, query, config, phrase_matching=phrase_matching),
     )
+    phrase_matches = retrieval.phrase_matches
     timings.extend(retrieval.timings)
     timings.append(timing)
     fused, timing = timed_result(
@@ -111,86 +129,174 @@ def search_ebooks(
         rank_constant=config.rrf_rank_constant,
     )
     timings.append(timing)
-    if config.rerank.enabled and rerank:
-        response, timing = timed_result("Rerank", apply_rerank, query, fused, config)
-    else:
-        response, timing = timed_result("Rerank skipped", skip_rerank, query, fused, config)
+    phrase_boost_timing_name = "Phrase mention boost" if phrase_matching else "Phrase mention boost skipped"
+    fused, timing = await async_timed_result(
+        phrase_boost_timing_name,
+        apply_phrase_mention_boosts(
+            engine,
+            fused,
+            phrase_matches,
+            config.phrase_hit_boost,
+            phrase_matching=phrase_matching,
+        ),
+    )
     timings.append(timing)
-    response = replace(response, timings=tuple(timings))
+    rerank_enabled = config.rerank.enabled and rerank
+    rerank_timing_name = "Rerank" if rerank_enabled else "Rerank skipped"
+    response, timing = await async_timed_result(
+        rerank_timing_name,
+        apply_rerank(client, query, fused, config, rerank=rerank_enabled),
+    )
+    timings.append(timing)
+    response = replace(response, timings=tuple(timings), phrase_matches=tuple(phrase_matches))
     logger.info(
-        "ebook_search_complete vector_candidates=%s lexical_candidates=%s "
-        "fused_candidates=%s returned=%s rank_label=%s runtime_ms=%.1f",
-        len(retrieval.vector_results),
-        len(retrieval.lexical_results),
-        len(fused),
-        len(response.results),
-        response.rank_label,
-        response.total_runtime_ms,
+        f"ebook_search_complete vector_candidates={len(retrieval.vector_results)} "
+        f"lexical_candidates={len(retrieval.lexical_results)} fused_candidates={len(fused)} {phrase_matching=} "
+        f"phrase_matches={len(phrase_matches)} returned={len(response.results)} {response.rank_label=} "
+        f"{response.total_runtime_ms=:.1f}"
     )
     return response
 
 
-def parallel_retrieval(
-    engine: Engine,
+async def query_phrase_matches(
+    engine: AsyncEngine,
     query: str,
     config: EbookSearchConfig,
+    *,
+    phrase_matching: bool,
+) -> list[PhraseMatch]:
+    """Detect protected phrases in a query without making search fail when phrase tables are unavailable."""
+    if not phrase_matching:
+        logger.info("ebook_protected_phrase_detection_skipped")
+        return []
+    try:
+        async with AsyncSession(engine) as session:
+            return await detect_protected_phrases_for_query(session, query, config)
+    except SQLAlchemyError as error:
+        logger.warning(f"ebook_protected_phrase_detection_unavailable {error=}")
+        return []
+
+
+async def apply_phrase_mention_boosts(
+    engine: AsyncEngine,
+    candidates: list[SearchResult],
+    phrase_matches: Sequence[PhraseMatch],
+    phrase_hit_boost: float,
+    *,
+    phrase_matching: bool,
+) -> list[SearchResult]:
+    """Boost retrieved chunks that have indexed mentions for detected protected phrases when enabled."""
+    if not phrase_matching:
+        logger.info(f"ebook_phrase_boost_skipped candidates={len(candidates)}")
+        return candidates
+
+    phrase_ids = sorted({match.phrase_id for match in phrase_matches})
+    if not candidates or not phrase_ids or phrase_hit_boost <= 0:
+        return candidates
+
+    chunk_ids = [candidate.chunk_id for candidate in candidates]
+    try:
+        async with AsyncSession(engine) as session:
+            phrase_hits = await phrase_hits_for_chunks(session, chunk_ids=chunk_ids, phrase_ids=phrase_ids)
+    except SQLAlchemyError as error:
+        logger.warning(f"ebook_phrase_boost_unavailable {error=}")
+        return candidates
+
+    if not phrase_hits:
+        return candidates
+
+    hit_counts = {
+        chunk_id: sum(hit.mention_count for hit in chunk_hits) for chunk_id, chunk_hits in phrase_hits.items()
+    }
+    boosted = [
+        replace(
+            candidate,
+            score=candidate.score + (hit_counts.get(candidate.chunk_id, 0) * phrase_hit_boost),
+            fused_score=boosted_fused_score(candidate, hit_counts.get(candidate.chunk_id, 0), phrase_hit_boost),
+            phrase_hit_count=hit_counts.get(candidate.chunk_id, 0),
+            matched_phrases=tuple(hit.phrase_text for hit in phrase_hits.get(candidate.chunk_id, ())),
+            rank_source=phrase_rank_source(candidate.rank_source, hit_counts.get(candidate.chunk_id, 0)),
+        )
+        for candidate in candidates
+    ]
+    return sorted(boosted, key=lambda candidate: candidate.score, reverse=True)
+
+
+def boosted_fused_score(candidate: SearchResult, phrase_hit_count: int, phrase_hit_boost: float) -> float | None:
+    """Return a fused score adjusted by phrase hits when a fused score exists."""
+    if candidate.fused_score is None:
+        return None
+    return candidate.fused_score + (phrase_hit_count * phrase_hit_boost)
+
+
+def phrase_rank_source(rank_source: str, phrase_hit_count: int) -> str:
+    """Append phrase evidence to a rank-source label when a chunk was boosted."""
+    if phrase_hit_count <= 0 or "phrases" in rank_source:
+        return rank_source
+    return f"{rank_source} + phrases"
+
+
+async def parallel_retrieval(
+    engine: AsyncEngine,
+    client: httpx.AsyncClient,
+    query: str,
+    config: EbookSearchConfig,
+    *,
+    phrase_matching: bool,
 ) -> RetrievalResponse:
-    """Run vector and BM25 candidate retrieval concurrently with separate database sessions."""
-    with ThreadPoolExecutor(max_workers=2, thread_name_prefix="ebook-search") as executor:
-        vector_future = executor.submit(
-            timed_result,
-            "Embedding + vector search",
-            vector_candidates,
-            engine,
-            query,
-            config,
-        )
-        bm25_future = executor.submit(
-            timed_result,
-            "BM25 search",
-            bm25_candidates,
-            query,
-            config,
-        )
-        vector_results, vector_timing = vector_future.result()
-        lexical_results, lexical_timing = bm25_future.result()
+    """Run vector, BM25, and protected phrase retrieval concurrently with separate database sessions.
+
+    BM25 scoring is pure CPU work over the cached corpus, so it runs in a worker thread
+    instead of on the event loop. Protected phrase detection only depends on the query, so
+    it joins the gather as a third task and returns immediately when phrase matching is disabled.
+    """
+    phrase_timing_name = "Protected phrase detection" if phrase_matching else "Protected phrase detection skipped"
+    (
+        (vector_results, vector_timing),
+        (lexical_results, lexical_timing),
+        (phrase_matches, phrase_timing),
+    ) = await asyncio.gather(
+        async_timed_result("Embedding + vector search", vector_candidates(engine, client, query, config)),
+        async_timed_result("BM25 search", asyncio.to_thread(bm25_candidates, query, config)),
+        async_timed_result(
+            phrase_timing_name,
+            query_phrase_matches(engine, query, config, phrase_matching=phrase_matching),
+        ),
+    )
 
     logger.info(
-        "ebook_parallel_retrieval_complete vector_candidates=%s lexical_candidates=%s",
-        len(vector_results),
-        len(lexical_results),
+        f"ebook_parallel_retrieval_complete vector_candidates={len(vector_results)} "
+        f"lexical_candidates={len(lexical_results)} phrase_matches={len(phrase_matches)}"
     )
     return RetrievalResponse(
         vector_results=vector_results,
         lexical_results=lexical_results,
+        phrase_matches=phrase_matches,
         timings=(
             replace(vector_timing, counts_toward_total=False),
             replace(lexical_timing, counts_toward_total=False),
+            replace(phrase_timing, counts_toward_total=False),
         ),
     )
 
 
-def skip_rerank(
+async def apply_rerank(
+    client: httpx.AsyncClient,
     query: str,
     candidates: list[SearchResult],
     config: EbookSearchConfig,
+    *,
+    rerank: bool,
 ) -> SearchResponse:
-    """Return fused hybrid results without reranking."""
-    logger.info("ebook_rerank_skipped candidates=%s", len(candidates))
-    return SearchResponse(query=query, results=candidates[: config.top_k], rank_label="Hybrid")
+    """Rerank already-fused hybrid candidates when enabled for this request."""
+    if not rerank:
+        logger.info(f"ebook_rerank_skipped candidates={len(candidates)}")
+        return SearchResponse(query=query, results=candidates[: config.top_k], rank_label="Hybrid")
 
-
-def apply_rerank(
-    query: str,
-    candidates: list[SearchResult],
-    config: EbookSearchConfig,
-) -> SearchResponse:
-    """Rerank already-fused hybrid candidates."""
-    reranked = rerank_chunks(query, candidates[: config.rerank.candidates], config.rerank)
+    reranked = await rerank_chunks(client, query, candidates[: config.rerank.candidates], config.rerank)
     logger.info(
-        "ebook_rerank_complete input_candidates=%s returned=%s",
-        min(len(candidates), config.rerank.candidates),
-        len(reranked),
+        f"ebook_rerank_complete input_candidates={min(len(candidates), config.rerank.candidates)} "
+        f"returned={len(reranked)}"
     )
     return SearchResponse(
         query=query,
@@ -199,10 +305,17 @@ def apply_rerank(
     )
 
 
-def vector_candidates(engine: Engine, query: str, config: EbookSearchConfig) -> list[SearchResult]:
+async def vector_candidates(
+    engine: AsyncEngine,
+    client: httpx.AsyncClient,
+    query: str,
+    config: EbookSearchConfig,
+) -> list[SearchResult]:
     """Return pgvector cosine candidates for a natural-language query."""
-    with Session(engine) as session:
-        model = session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == config.embedding_model))
+    async with AsyncSession(engine) as session:
+        model = await session.scalar(
+            select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == config.embedding_model)
+        )
         if model is None:
             msg = f"Embedding model is not registered: {config.embedding_model}"
             raise ValueError(msg)
@@ -212,7 +325,7 @@ def vector_candidates(engine: Engine, query: str, config: EbookSearchConfig) ->
             msg = f"Model row dimension {model.dimension} does not match configured dimension {expected_dimension}"
             raise ValueError(msg)
 
-        embedding = embed_query(query, config)
+        embedding = await embed_query(client, query, config)
         limit = max(config.rerank.candidates, config.top_k) * config.vector_candidate_multiplier
         embedding_table = get_embedding_table(model.dimension)
 
@@ -221,12 +334,7 @@ def vector_candidates(engine: Engine, query: str, config: EbookSearchConfig) ->
         score = (literal(1.0) - distance).label("score")
         statement = (
             select(
-                EbookChunk.id.label("chunk_id"),
-                EbookChunk.text.label("text"),
-                EbookSource.title.label("source_title"),
-                EbookSource.author.label("source_author"),
-                EbookChapter.title.label("chapter_title"),
-                EbookChunk.page_label.label("page_label"),
+                *CHUNK_RECORD_COLUMNS,
                 score,
             )
             .select_from(embedding_table)
@@ -237,13 +345,10 @@ def vector_candidates(engine: Engine, query: str, config: EbookSearchConfig) ->
             .order_by(distance)
             .limit(limit)
         )
-        rows = session.execute(statement).mappings()
+        rows = (await session.execute(statement)).mappings()
         results = [search_result_from_row(row) for row in rows]
         logger.info(
-            "ebook_vector_search_complete model=%s dimension=%s candidates=%s",
-            config.embedding_model,
-            model.dimension,
-            len(results),
+            f"ebook_vector_search_complete {config.embedding_model=} {model.dimension=} candidates={len(results)}"
         )
         return results
 
@@ -253,7 +358,7 @@ def bm25_candidates(query: str, config: EbookSearchConfig) -> list[SearchResult]
     try:
         corpus = load_bm25_corpus(config)
     except BM25CorpusUnavailableError as error:
-        logger.warning("ebook_bm25_index_unavailable_skipping error=%s", error)
+        logger.warning(f"ebook_bm25_index_unavailable_skipping {error=}")
         return []
 
     if not corpus.records:
@@ -268,12 +373,7 @@ def bm25_candidates(query: str, config: EbookSearchConfig) -> list[SearchResult]
     ]
 
     max_score = results[0].bm25_score if results else 0.0
-    logger.info(
-        "ebook_bm25_search_complete corpus=%s candidates=%s max_score=%.6f",
-        len(corpus.records),
-        len(results),
-        max_score,
-    )
+    logger.info(f"ebook_bm25_search_complete corpus={len(corpus.records)} candidates={len(results)} {max_score=:.6f}")
     return results
 
 
@@ -317,9 +417,11 @@ def reciprocal_rank_fusion(
 
 def search_result_from_row(row: Mapping[str, object]) -> SearchResult:
     """Convert a database row mapping into a search result."""
+    source_id = row.get("source_id")
     return SearchResult(
         chunk_id=int(row["chunk_id"]),
         text=str(row["text"]),
+        source_id=int(source_id) if source_id is not None else None,
         source_title=str(row["source_title"]),
         source_author=optional_str(row["source_author"]),
         chapter_title=optional_str(row["chapter_title"]),

python/ebook_search/timing.py

@@ -7,7 +7,7 @@ from time import perf_counter
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from collections.abc import Callable
+    from collections.abc import Awaitable, Callable
 
 
 @dataclass(frozen=True)
@@ -34,3 +34,10 @@ def timed_result[T, **P](
     start_seconds = perf_counter()
     result = operation(*args, **kwargs)
     return result, runtime_step_from_start(name, start_seconds)
+
+
+async def async_timed_result[T](name: str, awaitable: Awaitable[T]) -> tuple[T, RuntimeStep]:
+    """Await an operation and return its result plus elapsed runtime."""
+    start_seconds = perf_counter()
+    result = await awaitable
+    return result, runtime_step_from_start(name, start_seconds)

python/fastapi_tools/__init__.py

@@ -1,6 +1,25 @@
 """Reusable FastAPI tools."""
 
-from python.fastapi_tools.db import DbSession, get_db
+from python.fastapi_tools.db import (
+    AppAsyncEngine,
+    AppEngine,
+    AsyncDbSession,
+    DbSession,
+    get_async_db,
+    get_async_engine,
+    get_db,
+    get_engine,
+)
 from python.fastapi_tools.zstd_middleware import ZstdMiddleware
 
-__all__ = ["DbSession", "ZstdMiddleware", "get_db"]
+__all__ = [
+    "AppAsyncEngine",
+    "AppEngine",
+    "AsyncDbSession",
+    "DbSession",
+    "ZstdMiddleware",
+    "get_async_db",
+    "get_async_engine",
+    "get_db",
+    "get_engine",
+]

python/fastapi_tools/db.py

@@ -5,10 +5,22 @@ from __future__ import annotations
 from typing import TYPE_CHECKING, Annotated
 
 from fastapi import Depends, Request
+from sqlalchemy.engine import Engine
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession
 from sqlalchemy.orm import Session
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator
+    from collections.abc import AsyncIterator, Iterator
+
+
+def get_engine(request: Request) -> Engine:
+    """Get a synchronous database engine from app state."""
+    return request.app.state.engine
+
+
+def get_async_engine(request: Request) -> AsyncEngine:
+    """Get an asynchronous database engine from app state."""
+    return request.app.state.engine
 
 
 def get_db(request: Request) -> Iterator[Session]:
@@ -17,4 +29,17 @@ def get_db(request: Request) -> Iterator[Session]:
         yield session
 
 
+async def get_async_db(request: Request) -> AsyncIterator[AsyncSession]:
+    """Get an async database session from app state.
+
+    expire_on_commit=False keeps ORM attributes readable after commit without
+    triggering implicit IO, which would raise under asyncio.
+    """
+    async with AsyncSession(request.app.state.engine, expire_on_commit=False) as session:
+        yield session
+
+
+AppEngine = Annotated[Engine, Depends(get_engine)]
+AppAsyncEngine = Annotated[AsyncEngine, Depends(get_async_engine)]
 DbSession = Annotated[Session, Depends(get_db)]
+AsyncDbSession = Annotated[AsyncSession, Depends(get_async_db)]

python/gems/README.md

@@ -0,0 +1,54 @@
+# Gems
+
+Gems is a server-rendered, turn-based resource-engine game for one to four human or AI players. It uses FastAPI,
+Jinja, HTMX, server-sent events, and SQLite.
+
+The application deliberately contains no playable card deck, patron/governor set, objective set, official artwork,
+or copied rulebook text. A room host must upload a content pack they are entitled to use before starting a game.
+
+## Run locally
+
+```shell
+uv run gems --host 127.0.0.1 --port 8082
+```
+
+The default database and installation key are created under `.gems/`. The following environment variables override
+runtime behavior:
+
+- `GEMS_DATABASE_PATH`
+- `GEMS_KEY_PATH`
+- `GEMS_PUBLIC_ORIGIN`
+- `GEMS_SECURE_COOKIES`
+- `GEMS_HOST`
+- `GEMS_PORT`
+
+## Content packs
+
+The current schema is available from a running server at `/schemas/content-pack-v1.json`. A pack defines exactly
+five normal resources, one wild resource, cards, and optional patrons, objectives, and outposts. `patrons` is the
+canonical field name; `governors` is accepted as an input alias.
+
+Cards may use only the built-in, bounded effect vocabulary:
+
+- `none`
+- `virtual_wild`
+- `copy_bonus`
+- `copy_and_claim`
+- `multi_bonus`
+- `claim_free`
+- an optional discard-cards alternate cost
+
+Unknown fields, resource references, executable expressions, HTML, artwork URLs, and files larger than 512 KiB are
+rejected. The normalized pack is private to its room and becomes immutable when play starts.
+
+## Neutral module mapping
+
+Gems calls the four optional mechanics Objectives, Outposts, Eastern Decks, and Fortifications. Lobby presets combine
+these mechanics into the familiar base, objective-race, objective-plus-outpost, eastern-plus-fortification, and
+all-module configurations. Component identities and values always come from the uploaded pack.
+
+## Jeeves
+
+The NixOS module runs one Uvicorn worker on `127.0.0.1:8002`, stores state in
+`/zfs/media/services/gems`, and publishes it through HAProxy at `https://gems.tmmworkshop.com`. The DNS record must
+point to Jeeves before ACME can issue the certificate.

python/gems/__init__.py

@@ -0,0 +1 @@
+"""Gems multiplayer card-engine game."""

python/gems/ai/__init__.py

@@ -0,0 +1,5 @@
+"""AI controllers for Gems."""
+
+from .runner import choose_ai_command
+
+__all__ = ["choose_ai_command"]

python/gems/ai/runner.py

@@ -0,0 +1,68 @@
+"""Deterministic AI command selection at three difficulty levels."""
+
+from __future__ import annotations
+
+import random
+from typing import TYPE_CHECKING
+
+from python.gems.domain.engine import RuleError, apply_command, score
+from python.gems.domain.legal_actions import legal_commands
+
+if TYPE_CHECKING:
+    from python.gems.domain.models import ContentPack, GameCommand, GameSettings, GameState
+
+
+def choose_ai_command(
+    state: GameState,
+    pack: ContentPack,
+    settings: GameSettings,
+    seat: int,
+    difficulty: str,
+) -> GameCommand:
+    """Choose from legal commands using only public/current-player information."""
+    actions = legal_commands(state, pack, settings, seat)
+    if not actions:
+        error = "AI has no legal command"
+        raise RuntimeError(error)
+    rng = random.Random(f"{state.seed}:{state.revision}:{seat}:{difficulty}")  # noqa: S311 - deterministic AI
+    if difficulty == "easy":
+        return rng.choice(actions)
+
+    ranked = sorted(actions, key=lambda item: _heuristic(item, state, pack), reverse=True)
+    if difficulty == "medium":
+        return ranked[0]
+
+    # Bounded deterministic rollout: examine at most the twelve strongest actions.
+    best = ranked[0]
+    best_value = float("-inf")
+    for candidate in ranked[:12]:
+        try:
+            future = apply_command(state, candidate, pack, settings, actor_seat=seat)
+        except RuleError:
+            continue
+        value = score(future.players[seat], pack) * 100 + _heuristic(candidate, state, pack)
+        value += rng.random() * 0.001
+        if value > best_value:
+            best, best_value = candidate, value
+    return best
+
+
+def _heuristic(command: GameCommand, state: GameState, pack: ContentPack) -> float:
+    player = state.players[state.current_seat]
+    value = 0.0
+    if command.type == "purchase":
+        card = pack.card(str(command.payload.get("card_id")))
+        value += 40 + card.points * 25 + (8 if card.bonus_resource else 0)
+        value -= sum(command.payload.get("payment", {}).values())
+    elif command.type == "take_distinct":
+        value += 10 + len(command.payload.get("resources", []))
+    elif command.type == "reserve":
+        value += 6
+    elif command.type == "take_double":
+        value += 4
+    elif command.type == "choose":
+        value += 20
+    elif command.type == "decline":
+        value -= 2
+    value += score(player, pack) * 0.01
+    return value

python/gems/config.py

@@ -0,0 +1,25 @@
+"""Runtime configuration loaded from environment variables."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class GemsConfig(BaseSettings):
+    """Configure persistence, public URLs, cookies, and the HTTP server."""
+
+    model_config = SettingsConfigDict(env_prefix="GEMS_")
+
+    database_path: Path = Path(".gems/gems.sqlite3")
+    key_path: Path = Path(".gems/instance.key")
+    public_origin: str = "http://127.0.0.1:8082"
+    secure_cookies: bool = False
+    host: str = "127.0.0.1"
+    port: int = 8082
+
+
+def load_config() -> GemsConfig:
+    """Load Gems configuration from defaults and environment variables."""
+    return GemsConfig()

python/gems/content.py

@@ -0,0 +1,55 @@
+"""Content-pack parsing, normalization, and JSON Schema publication."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from typing import Any
+
+from pydantic import ValidationError
+
+from .domain.models import ContentPack
+
+MAX_PACK_BYTES = 512 * 1024
+
+
+@dataclass(frozen=True, slots=True)
+class ParsedPack:
+    """Validated content plus its canonical representation."""
+
+    pack: ContentPack
+    canonical_json: str
+    digest: str
+
+
+class ContentPackError(ValueError):
+    """A safe, user-visible content validation error."""
+
+
+def parse_content_pack(raw: bytes | str) -> ParsedPack:
+    """Validate a content pack and return stable canonical JSON."""
+    data = raw.encode() if isinstance(raw, str) else raw
+    if len(data) > MAX_PACK_BYTES:
+        error = "Content pack exceeds the 512 KiB limit"
+        raise ContentPackError(error)
+    try:
+        decoded: Any = json.loads(data)
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        error = f"Invalid JSON: {exc}"
+        raise ContentPackError(error) from exc
+    try:
+        pack = ContentPack.model_validate(decoded)
+    except ValidationError as exc:
+        messages = []
+        for issue in exc.errors(include_url=False):
+            path = ".".join(str(part) for part in issue["loc"])
+            messages.append(f"{path or '$'}: {issue['msg']}")
+        raise ContentPackError("\n".join(messages)) from exc
+    canonical = json.dumps(pack.model_dump(mode="json", by_alias=False), sort_keys=True, separators=(",", ":"))
+    return ParsedPack(pack=pack, canonical_json=canonical, digest=hashlib.sha256(canonical.encode()).hexdigest())
+
+
+def content_pack_schema() -> dict[str, Any]:
+    """Return the authoritative version-one JSON Schema."""
+    return ContentPack.model_json_schema(by_alias=False)

python/gems/domain/__init__.py

@@ -0,0 +1 @@
+"""Domain models and rules for Gems."""

python/gems/domain/engine.py

@@ -0,0 +1,903 @@
+"""Deterministic, server-authoritative Gems rule engine."""
+
+from __future__ import annotations
+
+import random
+from typing import Any
+
+from .models import (
+    CardDefinition,
+    CardEffectKind,
+    ContentPack,
+    GameCommand,
+    GameSettings,
+    GameState,
+    OutpostDefinition,
+    OutpostPower,
+    OwnedCard,
+    PendingChoice,
+    PlayerState,
+)
+from .requirements import requirements_met
+
+MAX_PLAYERS = 4
+TWO_PLAYER_COUNT = 2
+THREE_PLAYER_COUNT = 3
+DISTINCT_TAKE_COUNT = 3
+DOUBLE_TAKE_MINIMUM = 4
+BLIND_RESERVE_DRAW_COUNT = 2
+
+
+class RuleError(ValueError):
+    """A command was not legal for the current state."""
+
+
+def deck_key(deck: str, tier: int) -> str:
+    """Build the state key for a named deck and tier."""
+    return f"{deck}:{tier}"
+
+
+def new_game(
+    room_code: str,
+    names: list[str],
+    pack: ContentPack,
+    settings: GameSettings,
+    *,
+    seed: int,
+) -> GameState:
+    """Create and deal a deterministic game."""
+    if not 1 <= len(names) <= MAX_PLAYERS:
+        error = "Games require one to four seats"
+        raise RuleError(error)
+    if settings.first_player_mode == "selected" and settings.first_player_seat >= len(names):
+        error = "The selected first player is no longer in the room"
+        raise RuleError(error)
+    _validate_startable(pack, settings)
+    rng = random.Random(seed)  # noqa: S311 - deterministic seeded shuffle
+    first_seat = settings.first_player_seat if settings.first_player_mode == "selected" else rng.randrange(len(names))
+    normal_count = 4 if len(names) <= TWO_PLAYER_COUNT else 5 if len(names) == THREE_PLAYER_COUNT else 7
+    supply = dict.fromkeys(pack.resource_ids, normal_count)
+    supply[pack.wild_resource.id] = 5
+    players = [
+        PlayerState(
+            seat=index,
+            name=name,
+            tokens=dict.fromkeys((*pack.resource_ids, pack.wild_resource.id), 0),
+            fortifications_available=settings.fortifications_per_player if settings.modules.fortifications else 0,
+        )
+        for index, name in enumerate(names)
+    ]
+    decks: dict[str, list[str]] = {}
+    markets: dict[str, list[str]] = {}
+    for source in ("base", "eastern"):
+        if source == "eastern" and not settings.modules.eastern_decks:
+            continue
+        market_size = settings.base_market_size if source == "base" else settings.eastern_market_size
+        for tier in (1, 2, 3):
+            key = deck_key(source, tier)
+            cards = [card.id for card in pack.cards if card.deck == source and card.tier == tier]
+            rng.shuffle(cards)
+            markets[key] = [cards.pop() for _ in range(min(market_size, len(cards)))]
+            decks[key] = cards
+    patrons = [patron.id for patron in pack.patrons]
+    rng.shuffle(patrons)
+    patrons = patrons[: min(len(patrons), len(players) + 1)]
+    objectives = [objective.id for objective in pack.objectives]
+    rng.shuffle(objectives)
+    objectives = objectives[: settings.objective_count] if settings.modules.objectives else []
+    return GameState(
+        room_code=room_code,
+        seed=seed,
+        players=players,
+        supply=supply,
+        decks=decks,
+        markets=markets,
+        available_patrons=[] if settings.modules.objectives else patrons,
+        available_objectives=objectives,
+        current_seat=first_seat,
+        first_seat=first_seat,
+    )
+
+
+def _validate_startable(pack: ContentPack, settings: GameSettings) -> None:
+    for tier in (1, 2, 3):
+        if not any(card.deck == "base" and card.tier == tier for card in pack.cards):
+            error = f"The base deck has no tier-{tier} cards"
+            raise RuleError(error)
+        if settings.modules.eastern_decks and not any(
+            card.deck == "eastern" and card.tier == tier for card in pack.cards
+        ):
+            error = f"The eastern deck has no tier-{tier} cards"
+            raise RuleError(error)
+    if settings.modules.objectives and not pack.objectives:
+        error = "The Objectives module needs objective definitions"
+        raise RuleError(error)
+    if settings.modules.outposts and not pack.outposts:
+        error = "The Outposts module needs outpost definitions"
+        raise RuleError(error)
+
+
+def card_map(pack: ContentPack) -> dict[str, CardDefinition]:
+    """Index all cards in a content pack by identifier."""
+    return {card.id: card for card in pack.cards}
+
+
+def bonuses(player: PlayerState, pack: ContentPack) -> dict[str, int]:
+    """Calculate effective bonuses after copies, multiples, and discards."""
+    cards = card_map(pack)
+    result = dict.fromkeys(pack.resource_ids, 0)
+    for owned in player.cards:
+        definition = cards[owned.card_id]
+        resource = owned.copied_resource or definition.bonus_resource
+        if resource is not None:
+            result[resource] += definition.effect.amount if definition.effect.kind == CardEffectKind.MULTI_BONUS else 1
+    return result
+
+
+def score(player: PlayerState, pack: ContentPack) -> int:
+    """Calculate a player's score from cards, patrons, and outposts."""
+    cards = card_map(pack)
+    patrons = {patron.id: patron for patron in pack.patrons}
+    outposts = {outpost.id: outpost for outpost in pack.outposts}
+    value = sum(cards[item.card_id].points for item in player.cards)
+    value += sum(patrons[item].points for item in player.patrons)
+    for item in player.outposts:
+        definition = outposts[item]
+        if definition.power == OutpostPower.POINTS_PER_OUTPOST:
+            value += definition.value * len(player.outposts)
+    return value
+
+
+def affordable_payments(player: PlayerState, card: CardDefinition, pack: ContentPack, *, double_wild: bool) -> bool:
+    """Report whether a player can cover a card's standard cost."""
+    discount = bonuses(player, pack)
+    shortage = sum(
+        max(0, amount - discount.get(resource, 0) - player.tokens.get(resource, 0))
+        for resource, amount in card.cost.items()
+    )
+    wild_value = 2 if double_wild else 1
+    return shortage <= player.tokens.get(pack.wild_resource.id, 0) * wild_value
+
+
+def visible_cards(state: GameState) -> set[str]:
+    """Return the identifiers of cards currently visible in markets."""
+    return {card for market in state.markets.values() for card in market}
+
+
+def apply_command(
+    state: GameState,
+    command: GameCommand,
+    pack: ContentPack,
+    settings: GameSettings,
+    *,
+    actor_seat: int,
+) -> GameState:
+    """Validate and apply one command, returning a new state snapshot."""
+    if state.finished:
+        error = "The game is finished"
+        raise RuleError(error)
+    if command.expected_revision != state.revision:
+        error = "The board changed; refresh and try again"
+        raise RuleError(error)
+    if actor_seat != state.current_seat:
+        error = "It is not your turn"
+        raise RuleError(error)
+    if state.pending and state.pending.seat != actor_seat:
+        error = "Another player must resolve the pending choice"
+        raise RuleError(error)
+
+    result = state.model_copy(deep=True)
+    if result.pending:
+        _apply_pending(result, command, pack, settings)
+    elif command.type == "take_distinct":
+        _take_distinct(result, command.payload, pack, settings)
+    elif command.type == "take_double":
+        _take_double(result, command.payload, pack, settings)
+    elif command.type == "reserve":
+        _reserve(result, command.payload, pack, settings)
+    elif command.type == "purchase":
+        _purchase(result, command.payload, pack, settings, is_conquest=False)
+    else:
+        error = "Choose a normal turn action"
+        raise RuleError(error)
+    result.revision += 1
+    return result
+
+
+def _take_distinct(state: GameState, payload: dict[str, Any], pack: ContentPack, settings: GameSettings) -> None:
+    colors = payload.get("resources")
+    if not isinstance(colors, list) or len(colors) != len(set(colors)):
+        error = "Choose distinct resources"
+        raise RuleError(error)
+    available = [resource for resource in pack.resource_ids if state.supply.get(resource, 0) > 0]
+    required = DISTINCT_TAKE_COUNT if len(available) >= DISTINCT_TAKE_COUNT else None
+    if required is not None and len(colors) != required:
+        error = "Take exactly three different resources when possible"
+        raise RuleError(error)
+    if required is None and not 1 <= len(colors) <= len(available):
+        error = "Choose one or more available resources"
+        raise RuleError(error)
+    if not set(colors) <= set(available):
+        error = "A selected resource is unavailable"
+        raise RuleError(error)
+    player = state.players[state.current_seat]
+    for resource in colors:
+        state.supply[resource] -= 1
+        player.tokens[resource] += 1
+    state.log.append(f"{player.name} took {len(colors)} different resources")
+    _after_standard_action(state, pack, settings, action="take_distinct")
+
+
+def _take_double(state: GameState, payload: dict[str, Any], pack: ContentPack, settings: GameSettings) -> None:
+    resource = payload.get("resource")
+    if resource not in pack.resource_ids or state.supply.get(resource, 0) < DOUBLE_TAKE_MINIMUM:
+        error = "That resource cannot be taken twice"
+        raise RuleError(error)
+    player = state.players[state.current_seat]
+    state.supply[resource] -= 2
+    player.tokens[resource] += 2
+    state.log.append(f"{player.name} took two {resource} resources")
+    extra = _owned_outpost(player, pack, OutpostPower.RESOURCE_AFTER_DOUBLE)
+    if settings.modules.outposts and extra:
+        options = [item for item in pack.resource_ids if item != resource and state.supply.get(item, 0) > 0]
+        if options:
+            state.pending = PendingChoice(
+                kind="resource", seat=player.seat, options=options, context={"next": "after_action"}
+            )
+            return
+    _after_standard_action(state, pack, settings, action="take_double")
+
+
+def _reserve(state: GameState, payload: dict[str, Any], pack: ContentPack, settings: GameSettings) -> None:
+    player = state.players[state.current_seat]
+    if len(player.reserved) >= settings.reserve_limit:
+        error = "Your reserve is full"
+        raise RuleError(error)
+    card_id = payload.get("card_id")
+    source = payload.get("deck")
+    drawn: list[str]
+    if card_id:
+        if card_id not in visible_cards(state):
+            error = "That card is not visible"
+            raise RuleError(error)
+        _assert_not_blocked(state, card_id, player.seat)
+        _remove_visible(state, card_id)
+        drawn = [card_id]
+    elif isinstance(source, str) and source in state.decks:
+        if not state.decks[source]:
+            error = "That deck is empty"
+            raise RuleError(error)
+        draw_count = 1
+        post = _owned_outpost(player, pack, OutpostPower.BLIND_RESERVE_TWO)
+        if post and (not source.startswith("eastern:") or settings.interactions.outpost_reserve_applies_eastern):
+            draw_count = min(BLIND_RESERVE_DRAW_COUNT, len(state.decks[source]))
+        drawn = [state.decks[source].pop() for _ in range(draw_count)]
+    else:
+        error = "Choose a visible card or deck"
+        raise RuleError(error)
+    if len(drawn) == BLIND_RESERVE_DRAW_COUNT:
+        state.pending = PendingChoice(kind="reserve_keep", seat=player.seat, options=drawn, context={"deck": source})
+        return
+    _finish_reserve(state, drawn[0], pack, settings)
+
+
+def _finish_reserve(state: GameState, card_id: str, pack: ContentPack, settings: GameSettings) -> None:
+    player = state.players[state.current_seat]
+    player.reserved.append(card_id)
+    _refill_markets(state, settings)
+    wild = pack.wild_resource.id
+    if state.supply[wild] > 0:
+        state.supply[wild] -= 1
+        player.tokens[wild] += 1
+    state.log.append(f"{player.name} reserved a card")
+    _after_standard_action(state, pack, settings, action="reserve")
+
+
+def _purchase(
+    state: GameState,
+    payload: dict[str, Any],
+    pack: ContentPack,
+    settings: GameSettings,
+    *,
+    is_conquest: bool,
+) -> None:
+    player = state.players[state.current_seat]
+    card_id = payload.get("card_id")
+    if not isinstance(card_id, str):
+        error = "Choose a card"
+        raise RuleError(error)
+    from_reserve = card_id in player.reserved
+    if not from_reserve:
+        if card_id not in visible_cards(state):
+            error = "That card is not available"
+            raise RuleError(error)
+        _assert_not_blocked(state, card_id, player.seat)
+    if is_conquest and state.fortifications.get(card_id, {}).get(player.seat, 0) < settings.fortifications_per_player:
+        error = "All your fortifications must occupy the conquest card"
+        raise RuleError(error)
+    card = pack.card(card_id)
+    _pay_for_purchase(state, player, card, payload, pack, settings)
+    if from_reserve:
+        player.reserved.remove(card_id)
+    else:
+        _remove_visible(state, card_id)
+    _return_fortifications(state, card_id)
+    owned = OwnedCard(card_id=card_id)
+    player.cards.append(owned)
+    player.purchased_card_count += 1
+    state.turn_purchase_count += 1
+    state.log.append(f"{player.name} purchased {card.label}")
+    if card.effect.kind in {CardEffectKind.COPY_BONUS, CardEffectKind.COPY_AND_CLAIM}:
+        # Do not allow the new copy card to satisfy its own target requirement.
+        prior = player.cards.pop()
+        options = sorted(resource for resource, amount in bonuses(player, pack).items() if amount)
+        player.cards.append(prior)
+        if not options:
+            error = "This card requires an existing bonus to copy"
+            raise RuleError(error)
+        state.pending = PendingChoice(
+            kind="copy_bonus",
+            seat=player.seat,
+            options=options,
+            context={"card_id": card_id, "is_conquest": is_conquest},
+        )
+        return
+    if card.effect.kind == CardEffectKind.CLAIM_FREE:
+        _queue_free_card(state, card, pack, settings, is_conquest=is_conquest)
+        return
+    _after_purchase(state, pack, settings, is_conquest=is_conquest)
+
+
+def _pay_for_purchase(
+    state: GameState,
+    player: PlayerState,
+    card: CardDefinition,
+    payload: dict[str, Any],
+    pack: ContentPack,
+    settings: GameSettings,
+) -> None:
+    if card.alternate_cost:
+        discarded = payload.get("discard_cards", [])
+        _pay_alternate(player, card, discarded, pack)
+        if not settings.interactions.retain_outposts_after_discard:
+            _reconcile_outposts(player, pack)
+    else:
+        payment = payload.get("payment", {})
+        virtual_cards = payload.get("virtual_wild_cards", [])
+        _pay_tokens(state, player, card, payment, virtual_cards, pack, settings)
+
+
+def _pay_tokens(  # noqa: C901 - payment validation mirrors the rule sequence
+    state: GameState,
+    player: PlayerState,
+    card: CardDefinition,
+    payment: object,
+    virtual_cards: object,
+    pack: ContentPack,
+    settings: GameSettings,
+) -> None:
+    if not isinstance(payment, dict) or any(not isinstance(value, int) or value < 0 for value in payment.values()):
+        error = "Provide a valid payment"
+        raise RuleError(error)
+    allowed = {*pack.resource_ids, pack.wild_resource.id}
+    if not set(payment) <= allowed:
+        error = "Payment contains an unknown resource"
+        raise RuleError(error)
+    if any(payment.get(resource, 0) > player.tokens.get(resource, 0) for resource in allowed):
+        error = "Payment uses resources you do not have"
+        raise RuleError(error)
+    discount = bonuses(player, pack)
+    if not isinstance(virtual_cards, list) or len(virtual_cards) != len(set(virtual_cards)):
+        error = "Virtual-wild selections must be unique"
+        raise RuleError(error)
+    owned_by_id = {item.card_id: item for item in player.cards}
+    virtual_value = 0
+    for card_id in virtual_cards:
+        owned = owned_by_id.get(card_id)
+        if owned is None:
+            error = "You do not own a selected virtual-wild card"
+            raise RuleError(error)
+        definition = pack.card(card_id)
+        if definition.effect.kind != CardEffectKind.VIRTUAL_WILD:
+            error = "A selected card does not grant virtual wild resources"
+            raise RuleError(error)
+        virtual_value += definition.effect.amount
+    wild = payment.get(pack.wild_resource.id, 0)
+    double = bool(_owned_outpost(player, pack, OutpostPower.DOUBLE_WILD))
+    wild_value = 2 if double else 1
+    virtual_multiplier = 2 if double and settings.interactions.virtual_wild_can_double else 1
+    remaining_wild_value = wild * wild_value + virtual_value * virtual_multiplier
+    for resource in pack.resource_ids:
+        due = max(0, card.cost.get(resource, 0) - discount.get(resource, 0))
+        normal = payment.get(resource, 0)
+        if normal > due:
+            error = "Payment overpays a normal resource"
+            raise RuleError(error)
+        remaining_wild_value -= due - normal
+    if remaining_wild_value < 0:
+        error = "Payment does not exactly cover the card cost"
+        raise RuleError(error)
+    for resource, amount in payment.items():
+        player.tokens[resource] -= amount
+        state.supply[resource] += amount
+    if virtual_cards:
+        player.cards = [item for item in player.cards if item.card_id not in virtual_cards]
+
+
+def _pay_alternate(player: PlayerState, card: CardDefinition, selected: object, pack: ContentPack) -> None:
+    if not isinstance(selected, list) or not card.alternate_cost or len(selected) != card.alternate_cost.count:
+        error = "Choose the required owned cards to discard"
+        raise RuleError(error)
+    if len(selected) != len(set(selected)):
+        error = "A card can only be discarded once"
+        raise RuleError(error)
+    owned_by_id = {item.card_id: item for item in player.cards}
+    for card_id in selected:
+        owned = owned_by_id.get(card_id)
+        if owned is None:
+            error = "You do not own a selected discard"
+            raise RuleError(error)
+        definition = pack.card(card_id)
+        effective = owned.copied_resource or definition.bonus_resource
+        if effective != card.alternate_cost.discard_resource:
+            error = "A selected discard has the wrong color"
+            raise RuleError(error)
+    # Copy cards of the effective color must be discarded first.
+    matching_copies = [
+        item.card_id for item in player.cards if item.copied_resource == card.alternate_cost.discard_resource
+    ]
+    if any(item not in selected for item in matching_copies[: min(len(matching_copies), len(selected))]):
+        error = "Copied cards of this color must be discarded first"
+        raise RuleError(error)
+    player.cards = [item for item in player.cards if item.card_id not in selected]
+
+
+def _apply_pending(  # noqa: C901, PLR0911, PLR0912, PLR0915 - explicit pending-choice state machine
+    state: GameState, command: GameCommand, pack: ContentPack, settings: GameSettings
+) -> None:
+    pending = state.pending
+    if pending is None or command.type not in {"choose", "decline", "purchase"}:
+        error = "Resolve the pending choice"
+        raise RuleError(error)
+    player = state.players[state.current_seat]
+    choice = command.payload.get("choice")
+    if pending.kind == "reserve_keep":
+        if choice not in pending.options:
+            error = "Choose one of the drawn cards"
+            raise RuleError(error)
+        other = next(item for item in pending.options if item != choice)
+        state.decks[pending.context["deck"]].insert(0, other)
+        state.pending = None
+        _finish_reserve(state, choice, pack, settings)
+        return
+    if pending.kind == "copy_bonus":
+        if choice not in pending.options:
+            error = "Choose an existing bonus"
+            raise RuleError(error)
+        card_id = pending.context["card_id"]
+        owned = next(item for item in reversed(player.cards) if item.card_id == card_id)
+        owned.copied_resource = choice
+        definition = pack.card(card_id)
+        state.pending = None
+        if definition.effect.kind == CardEffectKind.COPY_AND_CLAIM:
+            _queue_free_card(state, definition, pack, settings, is_conquest=bool(pending.context.get("is_conquest")))
+        else:
+            _after_purchase(
+                state,
+                pack,
+                settings,
+                is_conquest=bool(pending.context.get("is_conquest")),
+                chained=bool(pending.context.get("chained")),
+            )
+        return
+    if pending.kind == "free_card":
+        if choice not in pending.options:
+            error = "Choose an eligible free card"
+            raise RuleError(error)
+        if settings.interactions.fortifications_block_free_claim:
+            _assert_not_blocked(state, choice, player.seat)
+        _remove_visible(state, choice)
+        _return_fortifications(state, choice)
+        claimed = pack.card(choice)
+        player.cards.append(OwnedCard(card_id=choice))
+        state.turn_chain_free_count += 1
+        state.log.append(f"{player.name} claimed {claimed.label} without payment")
+        state.pending = None
+        if claimed.effect.kind in {CardEffectKind.COPY_BONUS, CardEffectKind.COPY_AND_CLAIM}:
+            prior = player.cards.pop()
+            options = sorted(resource for resource, amount in bonuses(player, pack).items() if amount)
+            player.cards.append(prior)
+            if options:
+                state.pending = PendingChoice(
+                    kind="copy_bonus",
+                    seat=player.seat,
+                    options=options,
+                    context={
+                        "card_id": choice,
+                        "is_conquest": pending.context.get("is_conquest", False),
+                        "chained": True,
+                    },
+                )
+                return
+        _after_purchase(state, pack, settings, is_conquest=bool(pending.context.get("is_conquest")), chained=True)
+        return
+    if pending.kind == "resource":
+        if choice not in pending.options or state.supply.get(str(choice), 0) <= 0:
+            error = "Choose an available resource"
+            raise RuleError(error)
+        state.supply[str(choice)] -= 1
+        player.tokens[str(choice)] += 1
+        context = pending.context
+        state.pending = None
+        remaining = int(context.get("remaining", 1))
+        if context.get("next") == "after_purchase" and remaining > 1:
+            options = [item for item in pack.resource_ids if state.supply.get(item, 0) > 0]
+            if options:
+                state.pending = PendingChoice(
+                    kind="resource",
+                    seat=player.seat,
+                    options=options,
+                    context={**context, "remaining": remaining - 1},
+                )
+                return
+        if context.get("next") == "after_purchase":
+            _after_purchase(state, pack, settings, is_conquest=bool(context.get("is_conquest")), skip_resource=True)
+        else:
+            _after_standard_action(state, pack, settings, action="take_double")
+        return
+    if pending.kind == "discard_tokens":
+        discard = command.payload.get("tokens", {})
+        allowed = {*pack.resource_ids, pack.wild_resource.id}
+        if not isinstance(discard, dict) or not set(discard) <= allowed or sum(discard.values()) != pending.amount:
+            error = "Discard exactly the required number of resources"
+            raise RuleError(error)
+        for resource, amount in discard.items():
+            if not isinstance(amount, int) or amount <= 0 or amount > player.tokens.get(resource, 0):
+                error = "Invalid token discard"
+                raise RuleError(error)
+        for resource, amount in discard.items():
+            player.tokens[resource] -= amount
+            state.supply[resource] += amount
+        state.pending = None
+        _end_checks(state, pack, settings)
+        return
+    if pending.kind == "patron":
+        if choice not in pending.options:
+            error = "Choose an eligible patron"
+            raise RuleError(error)
+        player.patrons.append(choice)
+        state.available_patrons.remove(choice)
+        state.pending = None
+        _check_outposts_or_objectives(state, pack, settings)
+        return
+    if pending.kind == "outpost":
+        if choice not in pending.options:
+            error = "Choose an eligible outpost"
+            raise RuleError(error)
+        player.outposts.append(choice)
+        state.pending = None
+        _check_objectives(state, pack, settings)
+        return
+    if pending.kind == "fortification":
+        target = command.payload.get("card_id")
+        mode = command.payload.get("mode")
+        if target not in visible_cards(state):
+            error = "Choose a visible card"
+            raise RuleError(error)
+        if (
+            target in state.markets.get(deck_key("eastern", pack.card(target).tier), [])
+            and not settings.interactions.fortifications_on_eastern
+        ):
+            error = "Fortifications cannot occupy eastern cards with this setting"
+            raise RuleError(error)
+        if mode == "place":
+            occupants = state.fortifications.get(target, {})
+            if any(seat != player.seat and count for seat, count in occupants.items()):
+                error = "An opponent occupies that card"
+                raise RuleError(error)
+            source = command.payload.get("from_card")
+            if source:
+                if state.fortifications.get(source, {}).get(player.seat, 0) <= 0:
+                    error = "You have no fortification there"
+                    raise RuleError(error)
+                state.fortifications[source][player.seat] -= 1
+            elif player.fortifications_available > 0:
+                player.fortifications_available -= 1
+            else:
+                error = "Move one of your placed fortifications"
+                raise RuleError(error)
+            state.fortifications.setdefault(target, {})[player.seat] = occupants.get(player.seat, 0) + 1
+        elif mode == "remove":
+            occupants = state.fortifications.get(target, {})
+            opponents = [(seat, count) for seat, count in occupants.items() if seat != player.seat and count == 1]
+            if len(opponents) != 1:
+                error = "Choose a card with exactly one opposing fortification"
+                raise RuleError(error)
+            seat, _ = opponents[0]
+            occupants[seat] = 0
+            state.players[seat].fortifications_available += 1
+        else:
+            error = "Choose place or remove"
+            raise RuleError(error)
+        remaining = int(pending.context.get("remaining", 1))
+        state.pending = None
+        if remaining > 1 and _queue_fortification(state, pack, settings, remaining=remaining - 1):
+            return
+        _after_fortification(state, pack, settings)
+        return
+    if pending.kind == "conquest":
+        if command.type == "decline":
+            state.pending = None
+            _enforce_token_limit_or_checks(state, pack, settings)
+            return
+        if command.type != "purchase":
+            error = "Purchase the conquest card or decline"
+            raise RuleError(error)
+        state.pending = None
+        _purchase(state, command.payload, pack, settings, is_conquest=True)
+        return
+    error = "Unsupported pending choice"
+    raise RuleError(error)
+
+
+def _queue_free_card(
+    state: GameState, card: CardDefinition, pack: ContentPack, settings: GameSettings, *, is_conquest: bool
+) -> None:
+    options = []
+    for card_id in visible_cards(state):
+        candidate = pack.card(card_id)
+        if candidate.tier != card.effect.target_tier:
+            continue
+        if settings.interactions.fortifications_block_free_claim:
+            occupants = state.fortifications.get(card_id, {})
+            if any(seat != state.current_seat and count for seat, count in occupants.items()):
+                continue
+        options.append(card_id)
+    if options:
+        state.pending = PendingChoice(
+            kind="free_card", seat=state.current_seat, options=sorted(options), context={"is_conquest": is_conquest}
+        )
+    else:
+        _after_purchase(state, pack, settings, is_conquest=is_conquest)
+
+
+def _after_purchase(
+    state: GameState,
+    pack: ContentPack,
+    settings: GameSettings,
+    *,
+    is_conquest: bool,
+    chained: bool = False,
+    skip_resource: bool = False,
+) -> None:
+    player = state.players[state.current_seat]
+    _refill_markets(state, settings)
+    resource_post = _owned_outpost(player, pack, OutpostPower.RESOURCE_AFTER_PURCHASE)
+    triggers = not is_conquest or settings.interactions.purchase_resource_on_conquest
+    if settings.modules.outposts and resource_post and triggers and not skip_resource:
+        options = [item for item in pack.resource_ids if state.supply.get(item, 0) > 0]
+        if options:
+            remaining = 1
+            if chained and not settings.interactions.chained_claim_is_not_purchase:
+                remaining += state.turn_chain_free_count
+            state.pending = PendingChoice(
+                kind="resource",
+                seat=player.seat,
+                options=options,
+                context={"next": "after_purchase", "is_conquest": is_conquest, "remaining": remaining},
+            )
+            return
+    if settings.modules.fortifications:
+        remaining = 1
+        if chained and not settings.interactions.one_fortification_decision_per_purchase_chain:
+            remaining += state.turn_chain_free_count
+        if _queue_fortification(state, pack, settings, remaining=remaining):
+            return
+    _after_fortification(state, pack, settings)
+
+
+def _after_fortification(state: GameState, pack: ContentPack, settings: GameSettings) -> None:
+    player = state.players[state.current_seat]
+    conquest = [
+        card_id
+        for card_id, occupants in state.fortifications.items()
+        if occupants.get(player.seat, 0) >= settings.fortifications_per_player and card_id in visible_cards(state)
+    ]
+    if settings.modules.fortifications and conquest:
+        state.turn_chain_free_count = 0
+        state.pending = PendingChoice(kind="conquest", seat=player.seat, options=conquest)
+        return
+    state.turn_chain_free_count = 0
+    _enforce_token_limit_or_checks(state, pack, settings)
+
+
+def _queue_fortification(
+    state: GameState,
+    pack: ContentPack,
+    settings: GameSettings,
+    *,
+    remaining: int,
+) -> bool:
+    player = state.players[state.current_seat]
+    options = [
+        card_id
+        for card_id in visible_cards(state)
+        if settings.interactions.fortifications_on_eastern or pack.card(card_id).deck != "eastern"
+    ]
+    can_act = False
+    for card_id in options:
+        occupants = state.fortifications.get(card_id, {})
+        can_place = not any(seat != player.seat and count for seat, count in occupants.items())
+        can_remove = any(seat != player.seat and count == 1 for seat, count in occupants.items())
+        if can_place or can_remove:
+            can_act = True
+            break
+    if not can_act:
+        return False
+    state.pending = PendingChoice(
+        kind="fortification",
+        seat=player.seat,
+        options=sorted(options),
+        context={"remaining": remaining},
+    )
+    return True
+
+
+def _after_standard_action(state: GameState, pack: ContentPack, settings: GameSettings, *, action: str) -> None:
+    del action
+    if settings.modules.fortifications and state.turn_purchase_count:
+        _after_purchase(state, pack, settings, is_conquest=False)
+    else:
+        _enforce_token_limit_or_checks(state, pack, settings)
+
+
+def _enforce_token_limit_or_checks(state: GameState, pack: ContentPack, settings: GameSettings) -> None:
+    player = state.players[state.current_seat]
+    excess = sum(player.tokens.values()) - settings.token_limit
+    if excess > 0:
+        state.pending = PendingChoice(kind="discard_tokens", seat=player.seat, amount=excess)
+        return
+    _end_checks(state, pack, settings)
+
+
+def _end_checks(state: GameState, pack: ContentPack, settings: GameSettings) -> None:
+    player = state.players[state.current_seat]
+    bonus = bonuses(player, pack)
+    eligible_patrons = [
+        patron.id
+        for patron in pack.patrons
+        if patron.id in state.available_patrons and requirements_met(patron.requirements, bonus)
+    ]
+    if eligible_patrons:
+        state.pending = PendingChoice(kind="patron", seat=player.seat, options=eligible_patrons)
+        return
+    _check_outposts_or_objectives(state, pack, settings)
+
+
+def _check_outposts_or_objectives(state: GameState, pack: ContentPack, settings: GameSettings) -> None:
+    if settings.modules.outposts and settings.interactions.outposts_before_objectives and _queue_outpost(state, pack):
+        return
+    _check_objectives(state, pack, settings)
+
+
+def _queue_outpost(state: GameState, pack: ContentPack) -> bool:
+    player = state.players[state.current_seat]
+    bonus = bonuses(player, pack)
+    choices = [
+        outpost.id
+        for outpost in pack.outposts
+        if outpost.id not in player.outposts and requirements_met(outpost.requirements, bonus)
+    ]
+    if choices:
+        state.pending = PendingChoice(kind="outpost", seat=player.seat, options=choices)
+        return True
+    return False
+
+
+def _check_objectives(state: GameState, pack: ContentPack, settings: GameSettings) -> None:
+    player = state.players[state.current_seat]
+    if settings.modules.objectives:
+        bonus = bonuses(player, pack)
+        choices = [
+            objective.id
+            for objective in pack.objectives
+            if objective.id in state.available_objectives
+            and score(player, pack) >= objective.minimum_score
+            and requirements_met(objective.requirements, bonus)
+        ]
+        if choices:
+            player.objective_met = choices[0]
+            if player.seat not in state.objective_qualifiers:
+                state.objective_qualifiers.append(player.seat)
+    if (
+        settings.modules.outposts
+        and not settings.interactions.outposts_before_objectives
+        and _queue_outpost(state, pack)
+    ):
+        return
+    _finish_turn(state, pack, settings)
+
+
+def _finish_turn(state: GameState, pack: ContentPack, settings: GameSettings) -> None:
+    player = state.players[state.current_seat]
+    has_score = score(player, pack) >= settings.target_score
+    has_objective = player.objective_met is not None
+    triggered = {
+        "score": has_score,
+        "objective": has_objective,
+        "either": has_score or has_objective,
+        "both": has_score and has_objective,
+    }[settings.victory_condition]
+    if triggered and state.finish_at_seat is None:
+        state.finish_at_seat = (state.first_seat - 1) % len(state.players)
+    if state.finish_at_seat == state.current_seat:
+        candidates = (
+            state.objective_qualifiers if settings.victory_condition == "objective" else list(range(len(state.players)))
+        )
+        if not candidates:
+            candidates = list(range(len(state.players)))
+        best_score = max(score(state.players[seat], pack) for seat in candidates)
+        candidates = [seat for seat in candidates if score(state.players[seat], pack) == best_score]
+        fewest = min(state.players[seat].purchased_card_count for seat in candidates)
+        state.winners = [seat for seat in candidates if state.players[seat].purchased_card_count == fewest]
+        state.finished = True
+        state.pending = None
+        return
+    state.current_seat = (state.current_seat + 1) % len(state.players)
+    if state.current_seat == state.first_seat:
+        state.round_number += 1
+    state.turn_purchase_count = 0
+    state.turn_chain_free_count = 0
+
+
+def _remove_visible(state: GameState, card_id: str) -> None:
+    for market in state.markets.values():
+        if card_id in market:
+            market.remove(card_id)
+            return
+    error = "Card is not visible"
+    raise RuleError(error)
+
+
+def _refill_markets(state: GameState, settings: GameSettings) -> None:
+    for key, market in state.markets.items():
+        desired = settings.eastern_market_size if key.startswith("eastern:") else settings.base_market_size
+        deck = state.decks[key]
+        while len(market) < desired and deck:
+            market.append(deck.pop())
+
+
+def _assert_not_blocked(state: GameState, card_id: str, seat: int) -> None:
+    occupants = state.fortifications.get(card_id, {})
+    if any(owner != seat and count > 0 for owner, count in occupants.items()):
+        error = "An opponent's fortification protects that card"
+        raise RuleError(error)
+
+
+def _return_fortifications(state: GameState, card_id: str) -> None:
+    for seat, count in state.fortifications.pop(card_id, {}).items():
+        state.players[seat].fortifications_available += count
+
+
+def _owned_outpost(player: PlayerState, pack: ContentPack, power: OutpostPower) -> OutpostDefinition | None:
+    definitions = {item.id: item for item in pack.outposts}
+    return next((definitions[item] for item in player.outposts if definitions[item].power == power), None)
+
+
+def _reconcile_outposts(player: PlayerState, pack: ContentPack) -> None:
+    definitions = {item.id: item for item in pack.outposts}
+    bonus = bonuses(player, pack)
+    player.outposts = [item for item in player.outposts if requirements_met(definitions[item].requirements, bonus)]
+
+
+def public_state(state: GameState, viewer_seat: int | None) -> dict[str, Any]:
+    """Serialize state while redacting other players' reserved cards and deck order."""
+    data = state.model_dump(mode="json")
+    for player in data["players"]:
+        if player["seat"] != viewer_seat:
+            player["reserved"] = [None] * len(player["reserved"])
+    data["decks"] = {key: len(value) for key, value in state.decks.items()}
+    if state.pending and state.pending.seat != viewer_seat:
+        data["pending"]["options"] = []
+        data["pending"]["context"] = {}
+    return data

python/gems/domain/legal_actions.py

@@ -0,0 +1,159 @@
+"""Legal command generation used by bots and server-rendered controls."""
+
+from __future__ import annotations
+
+import itertools
+import math
+import uuid
+
+from .engine import bonuses, visible_cards
+from .models import ContentPack, GameCommand, GameSettings, GameState, OutpostPower
+
+DISTINCT_TAKE_COUNT = 3
+DOUBLE_TAKE_MINIMUM = 4
+
+
+def command(state: GameState, type_: str, payload: dict | None = None) -> GameCommand:
+    """Create a command targeting the state's current revision."""
+    return GameCommand(
+        command_id=uuid.uuid4().hex,
+        expected_revision=state.revision,
+        type=type_,  # type: ignore[arg-type]
+        payload=payload or {},
+    )
+
+
+def legal_commands(  # noqa: C901, PLR0911, PLR0912 - mirrors pending and normal action branches
+    state: GameState, pack: ContentPack, settings: GameSettings, seat: int
+) -> list[GameCommand]:
+    """Enumerate meaningful legal commands without revealing hidden information."""
+    if state.finished or seat != state.current_seat:
+        return []
+    player = state.players[seat]
+    if state.pending:
+        pending = state.pending
+        if pending.seat != seat:
+            return []
+        if pending.kind == "discard_tokens":
+            available = [item for item, amount in player.tokens.items() for _ in range(amount)]
+            payload: dict[str, int] = {}
+            for item in available[: pending.amount]:
+                payload[item] = payload.get(item, 0) + 1
+            return [command(state, "choose", {"tokens": payload})]
+        if pending.kind == "fortification":
+            pending_results = []
+            for target in pending.options:
+                occupants = state.fortifications.get(target, {})
+                if not any(owner != seat and count for owner, count in occupants.items()):
+                    if player.fortifications_available:
+                        pending_results.append(command(state, "choose", {"card_id": target, "mode": "place"}))
+                    for source, source_occupants in state.fortifications.items():
+                        if source_occupants.get(seat, 0):
+                            pending_results.append(
+                                command(state, "choose", {"card_id": target, "from_card": source, "mode": "place"})
+                            )
+                if any(owner != seat and count == 1 for owner, count in occupants.items()):
+                    pending_results.append(command(state, "choose", {"card_id": target, "mode": "remove"}))
+            return pending_results
+        if pending.kind == "conquest":
+            conquest_results = [command(state, "decline")]
+            for card_id in pending.options:
+                payment = default_payment(state, pack, settings, card_id, seat)
+                if payment is not None:
+                    conquest_results.append(command(state, "purchase", {"card_id": card_id, **payment}))
+            return conquest_results
+        return [command(state, "choose", {"choice": item}) for item in pending.options]
+
+    results: list[GameCommand] = []
+    available = [resource for resource in pack.resource_ids if state.supply.get(resource, 0)]
+    take_size = DISTINCT_TAKE_COUNT if len(available) >= DISTINCT_TAKE_COUNT else 1
+    sizes = [take_size] if len(available) >= DISTINCT_TAKE_COUNT else list(range(1, len(available) + 1))
+    for size in sizes:
+        results.extend(
+            command(state, "take_distinct", {"resources": list(items)})
+            for items in itertools.combinations(available, size)
+        )
+    results.extend(
+        command(state, "take_double", {"resource": resource})
+        for resource in pack.resource_ids
+        if state.supply.get(resource, 0) >= DOUBLE_TAKE_MINIMUM
+    )
+    if len(player.reserved) < settings.reserve_limit:
+        results.extend(
+            command(state, "reserve", {"card_id": card_id})
+            for card_id in visible_cards(state)
+            if not _blocked(state, card_id, seat)
+        )
+        results.extend(command(state, "reserve", {"deck": key}) for key, cards in state.decks.items() if cards)
+    for card_id in [*visible_cards(state), *player.reserved]:
+        if card_id not in player.reserved and _blocked(state, card_id, seat):
+            continue
+        payment = default_payment(state, pack, settings, card_id, seat)
+        if payment is not None:
+            results.append(command(state, "purchase", {"card_id": card_id, **payment}))
+    return results
+
+
+def default_payment(  # noqa: C901, PLR0912 - ordered payment rules are intentionally explicit
+    state: GameState,
+    pack: ContentPack,
+    settings: GameSettings,
+    card_id: str,
+    seat: int,
+) -> dict | None:
+    """Return one legal colored-first payment, or None when unaffordable."""
+    player = state.players[seat]
+    card = pack.card(card_id)
+    if card.alternate_cost:
+        matching = []
+        definitions = {item.id: item for item in pack.cards}
+        for owned in player.cards:
+            resource = owned.copied_resource or definitions[owned.card_id].bonus_resource
+            if resource == card.alternate_cost.discard_resource:
+                matching.append(owned.card_id)
+        copies = [
+            item for item in matching if next(owned for owned in player.cards if owned.card_id == item).copied_resource
+        ]
+        ordered = [*copies, *(item for item in matching if item not in copies)]
+        if len(ordered) < card.alternate_cost.count:
+            return None
+        return {"discard_cards": ordered[: card.alternate_cost.count]}
+    discount = bonuses(player, pack)
+    payment: dict[str, int] = {}
+    shortage = 0
+    for resource in pack.resource_ids:
+        due = max(0, card.cost.get(resource, 0) - discount.get(resource, 0))
+        amount = min(due, player.tokens.get(resource, 0))
+        if amount:
+            payment[resource] = amount
+        shortage += due - amount
+    double_wild = any(
+        outpost.power == OutpostPower.DOUBLE_WILD and outpost.id in player.outposts for outpost in pack.outposts
+    )
+    wild_value = 2 if double_wild else 1
+    virtual_multiplier = 2 if double_wild and settings.interactions.virtual_wild_can_double else 1
+    virtual_cards: list[str] = []
+    virtual_total = 0
+    if shortage > player.tokens.get(pack.wild_resource.id, 0) * wild_value:
+        for owned in player.cards:
+            definition = pack.card(owned.card_id)
+            if definition.effect.kind.value != "virtual_wild":
+                continue
+            virtual_cards.append(owned.card_id)
+            virtual_total += definition.effect.amount * virtual_multiplier
+            if virtual_total + player.tokens.get(pack.wild_resource.id, 0) * wild_value >= shortage:
+                break
+    remaining = max(0, shortage - virtual_total)
+    wild_needed = math.ceil(remaining / wild_value)
+    if wild_needed > player.tokens.get(pack.wild_resource.id, 0) or virtual_total + wild_needed * wild_value < shortage:
+        return None
+    if wild_needed:
+        payment[pack.wild_resource.id] = wild_needed
+    result: dict[str, object] = {"payment": payment}
+    if virtual_cards:
+        result["virtual_wild_cards"] = virtual_cards
+    return result
+
+
+def _blocked(state: GameState, card_id: str, seat: int) -> bool:
+    return any(owner != seat and count for owner, count in state.fortifications.get(card_id, {}).items())

python/gems/domain/models.py

@@ -0,0 +1,394 @@
+"""Typed content, configuration, commands, and game state."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Annotated, Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class StrictModel(BaseModel):
+    """Base model that rejects misspelled fields."""
+
+    model_config = ConfigDict(extra="forbid")
+
+
+Slug = Annotated[str, Field(pattern=r"^[a-z][a-z0-9_-]{0,47}$")]
+ShortText = Annotated[str, Field(min_length=1, max_length=80)]
+TOKEN_DARK_INK_THRESHOLD = 0.82
+STANDARD_RESOURCE_SYMBOLS = {
+    "onyx": "O",
+    "sapphire": "S",
+    "emerald": "E",
+    "ruby": "R",
+    "diamond": "D",
+    "gold": "G",
+}
+
+
+class ResourceDefinition(StrictModel):
+    """A normal or wild resource rendered by the UI."""
+
+    id: Slug
+    label: ShortText
+    symbol: Annotated[str, Field(min_length=1, max_length=4)]
+    color: Annotated[str, Field(pattern=r"^#[0-9a-fA-F]{6}$")]
+
+    @model_validator(mode="after")
+    def normalize_standard_symbol(self) -> ResourceDefinition:
+        """Store the conventional symbol for a standard gem label."""
+        self.symbol = STANDARD_RESOURCE_SYMBOLS.get(self.label.casefold(), self.symbol)
+        return self
+
+    @property
+    def ink_color(self) -> str:
+        """Return readable token lettering for the configured background."""
+        red, green, blue = (int(self.color[index : index + 2], 16) for index in (1, 3, 5))
+        perceived_brightness = (299 * red + 587 * green + 114 * blue) / 255_000
+        return "#111111" if perceived_brightness >= TOKEN_DARK_INK_THRESHOLD else "#ffffff"
+
+
+class RequirementKind(StrEnum):
+    """Identify how a requirement selects resource colors."""
+
+    COLOR = "color"
+    ANY_COLOR = "any_color"
+
+
+class Requirement(StrictModel):
+    """A fixed-color or same-color bonus requirement."""
+
+    id: Slug
+    kind: RequirementKind
+    count: Annotated[int, Field(ge=1, le=99)]
+    resource: Slug | None = None
+    exclude: list[Slug] = Field(default_factory=list)
+    distinct_from: list[Slug] = Field(default_factory=list)
+
+    @model_validator(mode="after")
+    def validate_shape(self) -> Requirement:
+        """Validate fields that depend on the requirement kind."""
+        if self.kind == RequirementKind.COLOR and self.resource is None:
+            error = "color requirements need a resource"
+            raise ValueError(error)
+        if self.kind == RequirementKind.ANY_COLOR and self.resource is not None:
+            error = "any_color requirements cannot name a resource"
+            raise ValueError(error)
+        return self
+
+
+class CardEffectKind(StrEnum):
+    """Identify a supported server-side card effect."""
+
+    NONE = "none"
+    VIRTUAL_WILD = "virtual_wild"
+    COPY_BONUS = "copy_bonus"
+    COPY_AND_CLAIM = "copy_and_claim"
+    MULTI_BONUS = "multi_bonus"
+    CLAIM_FREE = "claim_free"
+
+
+class CardEffect(StrictModel):
+    """A bounded built-in card effect; uploaded code is never evaluated."""
+
+    kind: CardEffectKind = CardEffectKind.NONE
+    amount: Annotated[int, Field(ge=1, le=10)] = 1
+    target_tier: Annotated[int, Field(ge=1, le=3)] | None = None
+
+
+class AlternateCost(StrictModel):
+    """Purchase a card by discarding owned cards of one effective color."""
+
+    discard_resource: Slug
+    count: Annotated[int, Field(ge=1, le=10)]
+
+
+class CardDefinition(StrictModel):
+    """A card supplied by a user-owned content pack."""
+
+    id: Slug
+    label: ShortText
+    deck: Literal["base", "eastern"] = "base"
+    tier: Annotated[int, Field(ge=1, le=3)]
+    points: Annotated[int, Field(ge=0, le=99)] = 0
+    bonus_resource: Slug | None = None
+    cost: dict[Slug, Annotated[int, Field(ge=0, le=99)]] = Field(default_factory=dict)
+    effect: CardEffect = Field(default_factory=CardEffect)
+    alternate_cost: AlternateCost | None = None
+
+    @model_validator(mode="after")
+    def validate_effect(self) -> CardDefinition:
+        """Validate fields that depend on the selected card effect."""
+        effect = self.effect
+        if effect.kind in {CardEffectKind.CLAIM_FREE, CardEffectKind.COPY_AND_CLAIM}:
+            if effect.target_tier is None:
+                error = "claim effects require target_tier"
+                raise ValueError(error)
+            if effect.target_tier >= self.tier:
+                error = "free-card effects must target a lower tier"
+                raise ValueError(error)
+        elif effect.target_tier is not None:
+            error = "target_tier is only valid for claim effects"
+            raise ValueError(error)
+        if effect.kind == CardEffectKind.MULTI_BONUS and self.bonus_resource is None:
+            error = "multi_bonus cards require bonus_resource"
+            raise ValueError(error)
+        return self
+
+
+class PatronDefinition(StrictModel):
+    """Define a patron claimed by meeting permanent-bonus requirements."""
+
+    id: Slug
+    label: ShortText
+    points: Annotated[int, Field(ge=0, le=99)]
+    requirements: list[Requirement]
+
+
+class ObjectiveDefinition(StrictModel):
+    """Define an optional objective-based victory condition."""
+
+    id: Slug
+    label: ShortText
+    minimum_score: Annotated[int, Field(ge=0, le=999)]
+    requirements: list[Requirement]
+
+
+class OutpostPower(StrEnum):
+    """Identify a built-in outpost ability."""
+
+    RESOURCE_AFTER_PURCHASE = "resource_after_purchase"
+    RESOURCE_AFTER_DOUBLE = "resource_after_double"
+    DOUBLE_WILD = "double_wild"
+    POINTS_PER_OUTPOST = "points_per_outpost"
+    BLIND_RESERVE_TWO = "blind_reserve_two"
+
+
+class OutpostDefinition(StrictModel):
+    """Define an outpost and the bonuses required to claim it."""
+
+    id: Slug
+    label: ShortText
+    requirements: list[Requirement]
+    power: OutpostPower
+    value: Annotated[int, Field(ge=1, le=10)] = 1
+
+
+class PackMetadata(StrictModel):
+    """Describe the identity and authorship of a content pack."""
+
+    id: Slug
+    name: ShortText
+    version: Annotated[str, Field(min_length=1, max_length=32)]
+    author: Annotated[str, Field(max_length=80)] = ""
+
+
+class ContentPack(StrictModel):
+    """Versioned user-provided game content."""
+
+    schema_version: Literal[1]
+    metadata: PackMetadata
+    resources: Annotated[list[ResourceDefinition], Field(min_length=5, max_length=5)]
+    wild_resource: ResourceDefinition
+    cards: list[CardDefinition]
+    patrons: list[PatronDefinition] = Field(default_factory=list)
+    objectives: list[ObjectiveDefinition] = Field(default_factory=list)
+    outposts: list[OutpostDefinition] = Field(default_factory=list)
+
+    @model_validator(mode="before")
+    @classmethod
+    def normalize_patrons(cls, data: object) -> object:
+        """Normalize the legacy governors key to the neutral patrons key."""
+        if isinstance(data, dict):
+            if "patrons" in data and "governors" in data:
+                error = "use patrons or governors, not both"
+                raise ValueError(error)
+            if "governors" in data:
+                data = dict(data)
+                data["patrons"] = data.pop("governors")
+        return data
+
+    @model_validator(mode="after")
+    def validate_references(self) -> ContentPack:  # noqa: C901, PLR0912 - validates each reference family
+        """Validate identifiers and references across the complete pack."""
+        resources = {item.id for item in self.resources}
+        if self.wild_resource.id in resources:
+            error = "wild resource ID must be distinct"
+            raise ValueError(error)
+        id_collections = [
+            [item.id for item in self.resources],
+            [item.id for item in self.cards],
+            [item.id for item in self.patrons],
+            [item.id for item in self.objectives],
+            [item.id for item in self.outposts],
+        ]
+        for ids in id_collections:
+            if len(ids) != len(set(ids)):
+                error = "IDs must be unique within each content collection"
+                raise ValueError(error)
+        req_ids: set[str]
+        for card in self.cards:
+            if card.bonus_resource is not None and card.bonus_resource not in resources:
+                error = f"card {card.id} has an unknown bonus resource"
+                raise ValueError(error)
+            if not set(card.cost) <= resources:
+                error = f"card {card.id} has an unknown cost resource"
+                raise ValueError(error)
+            if card.alternate_cost and card.alternate_cost.discard_resource not in resources:
+                error = f"card {card.id} has an unknown alternate-cost resource"
+                raise ValueError(error)
+        requirement_owners: list[PatronDefinition | ObjectiveDefinition | OutpostDefinition] = [
+            *self.patrons,
+            *self.objectives,
+            *self.outposts,
+        ]
+        for owner in requirement_owners:
+            req_ids = {requirement.id for requirement in owner.requirements}
+            if len(req_ids) != len(owner.requirements):
+                error = f"{owner.id} has duplicate requirement IDs"
+                raise ValueError(error)
+            for requirement in owner.requirements:
+                if requirement.resource and requirement.resource not in resources:
+                    error = f"{owner.id} references an unknown resource"
+                    raise ValueError(error)
+                if not set(requirement.exclude) <= resources:
+                    error = f"{owner.id} excludes an unknown resource"
+                    raise ValueError(error)
+                if not set(requirement.distinct_from) <= req_ids:
+                    error = f"{owner.id} references an unknown requirement"
+                    raise ValueError(error)
+        return self
+
+    @property
+    def resource_ids(self) -> tuple[str, ...]:
+        """Return normal resource identifiers in display order."""
+        return tuple(resource.id for resource in self.resources)
+
+    def card(self, card_id: str) -> CardDefinition:
+        """Return a card definition by identifier."""
+        return next(card for card in self.cards if card.id == card_id)
+
+
+class Modules(StrictModel):
+    """Select optional rule modules for a game."""
+
+    objectives: bool = False
+    outposts: bool = False
+    eastern_decks: bool = False
+    fortifications: bool = False
+
+
+class Interactions(StrictModel):
+    """Configure interactions between optional rule modules."""
+
+    outpost_reserve_applies_eastern: bool = True
+    virtual_wild_can_double: bool = True
+    retain_outposts_after_discard: bool = True
+    outposts_before_objectives: bool = True
+    purchase_resource_on_conquest: bool = True
+    chained_claim_is_not_purchase: bool = True
+    fortifications_on_eastern: bool = True
+    fortifications_block_free_claim: bool = True
+    one_fortification_decision_per_purchase_chain: bool = True
+
+
+class GameSettings(StrictModel):
+    """Configure win conditions, limits, markets, and rule modules."""
+
+    target_score: Annotated[int, Field(ge=1, le=999)] = 15
+    victory_condition: Literal["score", "objective", "either", "both"] = "score"
+    first_player_mode: Literal["random", "selected"] = "random"
+    first_player_seat: Annotated[int, Field(ge=0, le=3)] = 0
+    token_limit: Annotated[int, Field(ge=1, le=99)] = 10
+    reserve_limit: Annotated[int, Field(ge=0, le=20)] = 3
+    base_market_size: Annotated[int, Field(ge=1, le=10)] = 4
+    eastern_market_size: Annotated[int, Field(ge=1, le=10)] = 2
+    objective_count: Annotated[int, Field(ge=1, le=10)] = 3
+    fortifications_per_player: Annotated[int, Field(ge=1, le=10)] = 3
+    modules: Modules = Field(default_factory=Modules)
+    interactions: Interactions = Field(default_factory=Interactions)
+
+
+class OwnedCard(StrictModel):
+    """Track an owned card and any copied resource assignment."""
+
+    card_id: Slug
+    copied_resource: Slug | None = None
+
+
+class PlayerState(StrictModel):
+    """Store the mutable state belonging to one player."""
+
+    seat: int
+    name: ShortText
+    tokens: dict[str, int]
+    cards: list[OwnedCard] = Field(default_factory=list)
+    reserved: list[Slug] = Field(default_factory=list)
+    patrons: list[Slug] = Field(default_factory=list)
+    outposts: list[Slug] = Field(default_factory=list)
+    objective_met: Slug | None = None
+    fortifications_available: int = 0
+    purchased_card_count: int = 0
+
+
+class PendingChoice(StrictModel):
+    """Describe a choice that must be resolved before play continues."""
+
+    kind: Literal[
+        "discard_tokens",
+        "copy_bonus",
+        "free_card",
+        "reserve_keep",
+        "resource",
+        "fortification",
+        "conquest",
+        "patron",
+        "outpost",
+    ]
+    seat: int
+    options: list[str] = Field(default_factory=list)
+    amount: int = 1
+    context: dict[str, Any] = Field(default_factory=dict)
+
+
+class GameState(StrictModel):
+    """Store a complete authoritative game snapshot."""
+
+    room_code: str
+    seed: int
+    revision: int = 0
+    players: list[PlayerState]
+    supply: dict[str, int]
+    decks: dict[str, list[Slug]]
+    markets: dict[str, list[Slug]]
+    available_patrons: list[Slug]
+    available_objectives: list[Slug]
+    fortifications: dict[Slug, dict[int, int]] = Field(default_factory=dict)
+    current_seat: int = 0
+    first_seat: int = 0
+    round_number: int = 1
+    pending: PendingChoice | None = None
+    finish_at_seat: int | None = None
+    objective_qualifiers: list[int] = Field(default_factory=list)
+    winners: list[int] = Field(default_factory=list)
+    finished: bool = False
+    turn_purchase_count: int = 0
+    turn_chain_free_count: int = 0
+    log: list[str] = Field(default_factory=list)
+
+
+class GameCommand(StrictModel):
+    """Represent one revision-bound action submitted to the engine."""
+
+    command_id: Annotated[str, Field(min_length=8, max_length=64)]
+    expected_revision: Annotated[int, Field(ge=0)]
+    type: Literal[
+        "take_distinct",
+        "take_double",
+        "reserve",
+        "purchase",
+        "choose",
+        "decline",
+    ]
+    payload: dict[str, Any] = Field(default_factory=dict)

python/gems/domain/requirements.py

@@ -0,0 +1,58 @@
+"""Requirement evaluation shared by patrons, objectives, outposts, and AI."""
+
+from __future__ import annotations
+
+from .models import Requirement, RequirementKind
+
+
+def requirements_met(requirements: list[Requirement], bonuses: dict[str, int]) -> bool:
+    """Return whether one assignment satisfies all fixed and wildcard clauses."""
+    chosen: dict[str, str] = {}
+
+    def visit(index: int) -> bool:
+        if index == len(requirements):
+            return True
+        requirement = requirements[index]
+        if requirement.kind == RequirementKind.COLOR:
+            if bonuses.get(requirement.resource or "", 0) < requirement.count:
+                return False
+            chosen[requirement.id] = requirement.resource or ""
+            return visit(index + 1)
+
+        forbidden = set(requirement.exclude)
+        forbidden.update(chosen[item] for item in requirement.distinct_from if item in chosen)
+        for resource, amount in bonuses.items():
+            if resource in forbidden or amount < requirement.count:
+                continue
+            chosen[requirement.id] = resource
+            if visit(index + 1):
+                return True
+            chosen.pop(requirement.id, None)
+        return False
+
+    fixed = [item for item in requirements if item.kind == RequirementKind.COLOR]
+    flexible = [item for item in requirements if item.kind == RequirementKind.ANY_COLOR]
+    return visit_ordered([*fixed, *flexible], bonuses, chosen)
+
+
+def visit_ordered(requirements: list[Requirement], bonuses: dict[str, int], chosen: dict[str, str]) -> bool:
+    """Backtracking evaluator kept separate for straightforward unit testing."""
+    if not requirements:
+        return True
+    requirement, *rest = requirements
+    if requirement.kind == RequirementKind.COLOR:
+        resource = requirement.resource or ""
+        if bonuses.get(resource, 0) < requirement.count:
+            return False
+        chosen[requirement.id] = resource
+        return visit_ordered(rest, bonuses, chosen)
+    forbidden = set(requirement.exclude)
+    forbidden.update(chosen[item] for item in requirement.distinct_from if item in chosen)
+    for resource, amount in bonuses.items():
+        if resource in forbidden or amount < requirement.count:
+            continue
+        chosen[requirement.id] = resource
+        if visit_ordered(rest, bonuses, chosen):
+            return True
+        chosen.pop(requirement.id, None)
+    return False

python/gems/events.py

@@ -0,0 +1,35 @@
+"""In-process revision notifications for SSE clients."""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from collections import defaultdict
+
+
+class EventBroker:
+    """Fan out room revisions; clients always reload the latest snapshot."""
+
+    def __init__(self) -> None:
+        """Initialize an empty subscription registry."""
+        self._queues: dict[str, set[asyncio.Queue[int]]] = defaultdict(set)
+
+    def subscribe(self, room_code: str) -> asyncio.Queue[int]:
+        """Subscribe a bounded notification queue to a room."""
+        queue: asyncio.Queue[int] = asyncio.Queue(maxsize=1)
+        self._queues[room_code].add(queue)
+        return queue
+
+    def unsubscribe(self, room_code: str, queue: asyncio.Queue[int]) -> None:
+        """Remove a room notification queue from the registry."""
+        self._queues[room_code].discard(queue)
+        if not self._queues[room_code]:
+            self._queues.pop(room_code, None)
+
+    def publish(self, room_code: str, revision: int) -> None:
+        """Publish the newest room revision to every subscriber."""
+        for queue in tuple(self._queues.get(room_code, ())):
+            if queue.full():
+                with contextlib.suppress(asyncio.QueueEmpty):
+                    queue.get_nowait()
+            queue.put_nowait(revision)

python/gems/main.py

@@ -0,0 +1,91 @@
+"""FastAPI entry point for Gems."""
+
+from __future__ import annotations
+
+import asyncio
+from contextlib import asynccontextmanager, suppress
+from typing import TYPE_CHECKING, Annotated
+
+import typer
+import uvicorn
+from fastapi import FastAPI, Request, Response
+from fastapi.staticfiles import StaticFiles
+
+from .config import load_config
+from .events import EventBroker
+from .persistence import Repository
+from .rooms import RoomService
+from .routes import router
+from .security import CredentialService
+from .web import STATIC_DIR
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
+    from starlette.middleware.base import RequestResponseEndpoint
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI) -> AsyncIterator[None]:
+    """Initialize and close application-scoped services."""
+    config = app.state.config
+    repository = Repository(config.database_path)
+    app.state.repository = repository
+    app.state.rooms = RoomService(repository, CredentialService(config.key_path), EventBroker())
+    repository.cleanup()
+    cleanup_task = asyncio.create_task(_cleanup_rooms(repository))
+    try:
+        yield
+    finally:
+        cleanup_task.cancel()
+        with suppress(asyncio.CancelledError):
+            await cleanup_task
+        repository.close()
+
+
+async def _cleanup_rooms(repository: Repository) -> None:
+    while True:
+        await asyncio.sleep(3600)
+        repository.cleanup()
+
+
+def create_app() -> FastAPI:
+    """Create an isolated application instance."""
+    app = FastAPI(title="Gems", docs_url=None, redoc_url=None, lifespan=lifespan)
+    app.state.config = load_config()
+    app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
+    app.include_router(router)
+
+    @app.middleware("http")
+    async def security_headers(request: Request, call_next: RequestResponseEndpoint) -> Response:
+        response = await call_next(request)
+        response.headers.setdefault("Referrer-Policy", "no-referrer")
+        response.headers.setdefault("X-Content-Type-Options", "nosniff")
+        response.headers.setdefault("X-Frame-Options", "DENY")
+        response.headers.setdefault("X-Robots-Tag", "noindex, nofollow")
+        response.headers.setdefault(
+            "Content-Security-Policy",
+            "default-src 'self'; script-src 'self'; style-src 'self'; style-src-attr 'unsafe-inline'; "
+            "img-src 'self' data:; connect-src 'self'",
+        )
+        if request.url.path.startswith("/rooms/"):
+            response.headers.setdefault("Cache-Control", "no-store")
+        return response
+
+    return app
+
+
+app = create_app()
+
+
+def serve(
+    host: Annotated[str | None, typer.Option()] = None,
+    port: Annotated[int | None, typer.Option()] = None,
+) -> None:
+    """Run the Gems ASGI application with Uvicorn."""
+    config = load_config()
+    uvicorn.run("python.gems.main:app", host=host or config.host, port=port or config.port, workers=1)
+
+
+if __name__ == "__main__":
+    typer.run(serve)

python/gems/persistence.py

@@ -0,0 +1,255 @@
+"""SQLite persistence for rooms, memberships, snapshots, and events."""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import threading
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from typing import TYPE_CHECKING
+
+from .domain.models import ContentPack, GameSettings, GameState
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+@dataclass(slots=True)
+class Member:
+    """Represent a human or AI seat persisted for a room."""
+
+    room_code: str
+    seat: int
+    name: str
+    controller: str
+    difficulty: str | None
+    credential_hash: str | None
+    is_host: bool
+    ready: bool
+    last_seen: str
+
+
+@dataclass(slots=True)
+class Room:
+    """Represent persisted room metadata and its current snapshot."""
+
+    code: str
+    status: str
+    settings: GameSettings
+    pack: ContentPack | None
+    pack_digest: str | None
+    state: GameState | None
+    revision: int
+    created_at: str
+    updated_at: str
+
+
+class Repository:
+    """Small transactional repository designed for one application worker."""
+
+    def __init__(self, path: Path) -> None:
+        """Open the SQLite database and initialize its schema."""
+        path.parent.mkdir(parents=True, exist_ok=True)
+        self._connection = sqlite3.connect(path, check_same_thread=False)
+        self._connection.row_factory = sqlite3.Row
+        self._lock = threading.RLock()
+        with self._connection:
+            self._connection.execute("PRAGMA journal_mode=WAL")
+            self._connection.execute("PRAGMA foreign_keys=ON")
+            self._connection.execute("PRAGMA busy_timeout=5000")
+            self._connection.executescript(
+                """
+                CREATE TABLE IF NOT EXISTS schema_version(version INTEGER NOT NULL);
+                INSERT INTO schema_version(version) SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM schema_version);
+                CREATE TABLE IF NOT EXISTS rooms(
+                    code TEXT PRIMARY KEY, status TEXT NOT NULL, settings_json TEXT NOT NULL,
+                    pack_json TEXT, pack_digest TEXT, state_json TEXT, revision INTEGER NOT NULL,
+                    created_at TEXT NOT NULL, updated_at TEXT NOT NULL
+                );
+                CREATE TABLE IF NOT EXISTS members(
+                    room_code TEXT NOT NULL REFERENCES rooms(code) ON DELETE CASCADE,
+                    seat INTEGER NOT NULL, name TEXT NOT NULL, controller TEXT NOT NULL,
+                    difficulty TEXT, credential_hash TEXT, is_host INTEGER NOT NULL,
+                    ready INTEGER NOT NULL, last_seen TEXT NOT NULL,
+                    PRIMARY KEY(room_code, seat), UNIQUE(room_code, credential_hash)
+                );
+                CREATE TABLE IF NOT EXISTS room_events(
+                    room_code TEXT NOT NULL REFERENCES rooms(code) ON DELETE CASCADE,
+                    revision INTEGER NOT NULL, command_id TEXT NOT NULL, actor_seat INTEGER,
+                    event_type TEXT NOT NULL, payload_json TEXT NOT NULL, created_at TEXT NOT NULL,
+                    PRIMARY KEY(room_code, command_id)
+                );
+                """
+            )
+
+    def room_exists(self, code: str) -> bool:
+        """Report whether a room code exists."""
+        return self._connection.execute("SELECT 1 FROM rooms WHERE code=?", (code,)).fetchone() is not None
+
+    def create_room(self, room: Room, host: Member) -> None:
+        """Persist a new room and its host in one transaction."""
+        with self._lock, self._connection:
+            self._connection.execute(
+                "INSERT INTO rooms VALUES(?,?,?,?,?,?,?,?,?)",
+                (
+                    room.code,
+                    room.status,
+                    room.settings.model_dump_json(),
+                    None,
+                    None,
+                    None,
+                    room.revision,
+                    room.created_at,
+                    room.updated_at,
+                ),
+            )
+            self._insert_member(host)
+
+    def _insert_member(self, member: Member) -> None:
+        self._connection.execute(
+            "INSERT INTO members VALUES(?,?,?,?,?,?,?,?,?)",
+            (
+                member.room_code,
+                member.seat,
+                member.name,
+                member.controller,
+                member.difficulty,
+                member.credential_hash,
+                int(member.is_host),
+                int(member.ready),
+                member.last_seen,
+            ),
+        )
+
+    def add_member(self, member: Member) -> None:
+        """Add a member and refresh the room's activity timestamp."""
+        with self._lock, self._connection:
+            self._insert_member(member)
+            self.touch(member.room_code)
+
+    def update_member(self, member: Member) -> None:
+        """Persist all mutable fields for an existing member."""
+        with self._lock, self._connection:
+            self._connection.execute(
+                """UPDATE members SET name=?,controller=?,difficulty=?,credential_hash=?,is_host=?,ready=?,last_seen=?
+                   WHERE room_code=? AND seat=?""",
+                (
+                    member.name,
+                    member.controller,
+                    member.difficulty,
+                    member.credential_hash,
+                    int(member.is_host),
+                    int(member.ready),
+                    member.last_seen,
+                    member.room_code,
+                    member.seat,
+                ),
+            )
+
+    def remove_member(self, code: str, seat: int) -> None:
+        """Remove a member from a room seat."""
+        with self._lock, self._connection:
+            self._connection.execute("DELETE FROM members WHERE room_code=? AND seat=?", (code, seat))
+
+    def get_room(self, code: str) -> Room | None:
+        """Load a room by code, including its typed JSON fields."""
+        row = self._connection.execute("SELECT * FROM rooms WHERE code=?", (code.upper(),)).fetchone()
+        if row is None:
+            return None
+        return Room(
+            code=row["code"],
+            status=row["status"],
+            settings=GameSettings.model_validate_json(row["settings_json"]),
+            pack=ContentPack.model_validate_json(row["pack_json"]) if row["pack_json"] else None,
+            pack_digest=row["pack_digest"],
+            state=GameState.model_validate_json(row["state_json"]) if row["state_json"] else None,
+            revision=row["revision"],
+            created_at=row["created_at"],
+            updated_at=row["updated_at"],
+        )
+
+    def members(self, code: str) -> list[Member]:
+        """Load all room members ordered by seat."""
+        rows = self._connection.execute("SELECT * FROM members WHERE room_code=? ORDER BY seat", (code,)).fetchall()
+        return [
+            Member(
+                room_code=row["room_code"],
+                seat=row["seat"],
+                name=row["name"],
+                controller=row["controller"],
+                difficulty=row["difficulty"],
+                credential_hash=row["credential_hash"],
+                is_host=bool(row["is_host"]),
+                ready=bool(row["ready"]),
+                last_seen=row["last_seen"],
+            )
+            for row in rows
+        ]
+
+    def member_for_credential(self, code: str, credential_hash: str) -> Member | None:
+        """Find the room member associated with a credential digest."""
+        return next((member for member in self.members(code) if member.credential_hash == credential_hash), None)
+
+    def save_lobby(self, room: Room) -> None:
+        """Persist mutable lobby configuration and revision data."""
+        with self._lock, self._connection:
+            self._connection.execute(
+                """UPDATE rooms SET settings_json=?,pack_json=?,pack_digest=?,revision=?,updated_at=? WHERE code=?""",
+                (
+                    room.settings.model_dump_json(),
+                    room.pack.model_dump_json() if room.pack else None,
+                    room.pack_digest,
+                    room.revision,
+                    room.updated_at,
+                    room.code,
+                ),
+            )
+
+    def save_state(self, room: Room, command_id: str, actor_seat: int | None, payload: dict) -> bool:
+        """Persist a game snapshot and its idempotent command event."""
+        now = utc_now()
+        with self._lock, self._connection:
+            existing = self._connection.execute(
+                "SELECT 1 FROM room_events WHERE room_code=? AND command_id=?", (room.code, command_id)
+            ).fetchone()
+            if existing:
+                return False
+            self._connection.execute(
+                "UPDATE rooms SET status=?,state_json=?,revision=?,updated_at=? WHERE code=?",
+                (room.status, room.state.model_dump_json() if room.state else None, room.revision, now, room.code),
+            )
+            self._connection.execute(
+                "INSERT INTO room_events VALUES(?,?,?,?,?,?,?)",
+                (room.code, room.revision, command_id, actor_seat, "command", json.dumps(payload), now),
+            )
+        return True
+
+    def has_command(self, code: str, command_id: str) -> bool:
+        """Report whether a command was already persisted for a room."""
+        row = self._connection.execute(
+            "SELECT 1 FROM room_events WHERE room_code=? AND command_id=?",
+            (code, command_id),
+        ).fetchone()
+        return row is not None
+
+    def touch(self, code: str) -> None:
+        """Refresh a room's activity timestamp."""
+        with self._connection:
+            self._connection.execute("UPDATE rooms SET updated_at=? WHERE code=?", (utc_now(), code))
+
+    def cleanup(self, days: int = 30) -> int:
+        """Delete rooms inactive for the requested number of days."""
+        cutoff = (datetime.now(UTC) - timedelta(days=days)).isoformat()
+        with self._lock, self._connection:
+            cursor = self._connection.execute("DELETE FROM rooms WHERE updated_at < ?", (cutoff,))
+        return cursor.rowcount
+
+    def close(self) -> None:
+        """Close the underlying SQLite connection."""
+        self._connection.close()
+
+
+def utc_now() -> str:
+    """Return the current UTC time as an ISO 8601 string."""
+    return datetime.now(UTC).isoformat()

python/gems/rooms.py

@@ -0,0 +1,375 @@
+"""Application service for anonymous rooms and AI turns."""
+
+from __future__ import annotations
+
+import asyncio
+import secrets
+from collections import defaultdict
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING
+
+from .ai import choose_ai_command
+from .domain.engine import RuleError, apply_command, new_game
+from .domain.models import GameCommand, GameSettings
+from .persistence import Member, Repository, Room, utc_now
+
+if TYPE_CHECKING:
+    from .content import ParsedPack
+    from .events import EventBroker
+    from .security import CredentialService
+
+ROOM_ALPHABET = "23456789ABCDEFGHJKMNPQRSTUVWXYZ"
+MAX_PLAYERS = 4
+OFFLINE_AFTER_SECONDS = 30
+DISPLAY_NAME_MAX_LENGTH = 32
+CONTROL_CHARACTER_LIMIT = 32
+
+
+class RoomError(ValueError):
+    """Safe user-facing room operation error."""
+
+
+class RoomService:
+    """Coordinates persistence, authorization, per-room locking, and bots."""
+
+    def __init__(self, repository: Repository, credentials: CredentialService, broker: EventBroker) -> None:
+        """Initialize the service with persistence, security, and events."""
+        self.repository = repository
+        self.credentials = credentials
+        self.broker = broker
+        self._locks: dict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
+        self._ai_tasks: dict[str, asyncio.Task[None]] = {}
+
+    def create(self, host_name: str, credential: str | None = None) -> tuple[Room, str]:
+        """Create a lobby and assign its first human member as host."""
+        name = validate_name(host_name)
+        code = self._new_code()
+        credential = credential or self.credentials.issue()
+        now = utc_now()
+        room = Room(
+            code=code,
+            status="lobby",
+            settings=GameSettings(),
+            pack=None,
+            pack_digest=None,
+            state=None,
+            revision=0,
+            created_at=now,
+            updated_at=now,
+        )
+        host = Member(
+            room_code=code,
+            seat=0,
+            name=name,
+            controller="human",
+            difficulty=None,
+            credential_hash=self.credentials.digest(credential),
+            is_host=True,
+            ready=False,
+            last_seen=now,
+        )
+        self.repository.create_room(room, host)
+        return room, credential
+
+    def join(self, code: str, name: str, credential: str | None = None) -> tuple[Room, str]:
+        """Join an available lobby or restore an existing membership."""
+        room = self.require_room(code)
+        credential = credential or self.credentials.issue()
+        existing = self.repository.member_for_credential(room.code, self.credentials.digest(credential))
+        if existing is not None:
+            return room, credential
+        if room.status != "lobby":
+            error = "This room is no longer accepting players"
+            raise RoomError(error)
+        members = self.repository.members(room.code)
+        if len(members) >= MAX_PLAYERS:
+            error = "This room is full"
+            raise RoomError(error)
+        clean_name = validate_name(name)
+        if any(member.name.casefold() == clean_name.casefold() for member in members):
+            error = "That display name is already in use"
+            raise RoomError(error)
+        seat = next(item for item in range(MAX_PLAYERS) if item not in {member.seat for member in members})
+        member = Member(
+            room_code=room.code,
+            seat=seat,
+            name=clean_name,
+            controller="human",
+            difficulty=None,
+            credential_hash=self.credentials.digest(credential),
+            is_host=False,
+            ready=False,
+            last_seen=utc_now(),
+        )
+        self.repository.add_member(member)
+        self._bump_lobby(room)
+        return room, credential
+
+    def authenticate(self, code: str, credential: str | None) -> tuple[Room, Member]:
+        """Authenticate a room member and refresh their presence timestamp."""
+        room = self.require_room(code)
+        if not credential:
+            error = "Join the room to continue"
+            raise RoomError(error)
+        member = self.repository.member_for_credential(room.code, self.credentials.digest(credential))
+        if member is None:
+            error = "Join the room to continue"
+            raise RoomError(error)
+        member.last_seen = utc_now()
+        self.repository.update_member(member)
+        return room, member
+
+    def require_room(self, code: str) -> Room:
+        """Load a room or raise a safe user-facing error."""
+        room = self.repository.get_room(code.upper())
+        if room is None:
+            error = "Room not found"
+            raise RoomError(error)
+        return room
+
+    def set_pack(self, room: Room, actor: Member, parsed: ParsedPack) -> Room:
+        """Assign a validated content pack to a lobby."""
+        self._require_host_lobby(room, actor)
+        room.pack = parsed.pack
+        room.pack_digest = parsed.digest
+        self._bump_lobby(room)
+        return room
+
+    def set_settings(self, room: Room, actor: Member, settings: GameSettings) -> Room:
+        """Replace the editable rules for a lobby."""
+        self._require_host_lobby(room, actor)
+        room.settings = settings
+        self._bump_lobby(room)
+        return room
+
+    def set_ready(self, room: Room, actor: Member, *, ready: bool) -> None:
+        """Update a human lobby member's readiness."""
+        if room.status != "lobby" or actor.controller != "human":
+            error = "Readiness can only change in the lobby"
+            raise RoomError(error)
+        actor.ready = ready
+        actor.last_seen = utc_now()
+        self.repository.update_member(actor)
+        self._bump_lobby(room)
+
+    def add_ai(self, room: Room, actor: Member, difficulty: str) -> None:
+        """Add an AI-controlled seat to a lobby."""
+        self._require_host_lobby(room, actor)
+        if difficulty not in {"easy", "medium", "hard"}:
+            error = "Choose easy, medium, or hard AI"
+            raise RoomError(error)
+        members = self.repository.members(room.code)
+        if len(members) >= MAX_PLAYERS:
+            error = "This room is full"
+            raise RoomError(error)
+        seat = next(item for item in range(MAX_PLAYERS) if item not in {member.seat for member in members})
+        number = 1 + sum(member.controller == "ai" for member in members)
+        self.repository.add_member(
+            Member(
+                room_code=room.code,
+                seat=seat,
+                name=f"Bot {number}",
+                controller="ai",
+                difficulty=difficulty,
+                credential_hash=None,
+                is_host=False,
+                ready=True,
+                last_seen=utc_now(),
+            )
+        )
+        self._bump_lobby(room)
+
+    def remove_seat(self, room: Room, actor: Member, seat: int) -> None:
+        """Remove a non-host seat from a lobby."""
+        self._require_host_lobby(room, actor)
+        target = next((member for member in self.repository.members(room.code) if member.seat == seat), None)
+        if target is None or target.is_host:
+            error = "That seat cannot be removed"
+            raise RoomError(error)
+        self.repository.remove_member(room.code, seat)
+        self._bump_lobby(room)
+
+    def transfer_host(self, room: Room, actor: Member, seat: int) -> None:
+        """Transfer lobby ownership to another human player."""
+        self._require_host_lobby(room, actor)
+        target = next((member for member in self.repository.members(room.code) if member.seat == seat), None)
+        if target is None or target.controller != "human" or target.seat == actor.seat:
+            error = "Choose another human player"
+            raise RoomError(error)
+        actor.is_host = False
+        target.is_host = True
+        self.repository.update_member(actor)
+        self.repository.update_member(target)
+        self._bump_lobby(room)
+
+    async def start(self, room: Room, actor: Member) -> Room:
+        """Validate the lobby and start its first game."""
+        self._require_host_lobby(room, actor)
+        if room.pack is None:
+            error = "Upload a valid content pack first"
+            raise RoomError(error)
+        members = self.repository.members(room.code)
+        humans = [member for member in members if member.controller == "human"]
+        if not humans or any(not member.ready for member in humans):
+            error = "Every human player must be ready"
+            raise RoomError(error)
+        ordered = sorted(members, key=lambda member: member.seat)
+        # Compress lobby seat gaps so engine seats always index the players list.
+        for new_seat, member in enumerate(ordered):
+            if member.seat != new_seat:
+                self.repository.remove_member(room.code, member.seat)
+                member.seat = new_seat
+                self.repository.add_member(member)
+        try:
+            state = new_game(
+                room.code, [member.name for member in ordered], room.pack, room.settings, seed=secrets.randbits(63)
+            )
+        except RuleError as exc:
+            raise RoomError(str(exc)) from exc
+        room.state = state
+        room.status = "playing"
+        room.revision = state.revision
+        self.repository.save_state(room, f"start-{secrets.token_hex(8)}", actor.seat, {"type": "start"})
+        self.broker.publish(room.code, room.revision)
+        self.schedule_ai(room.code)
+        return room
+
+    async def play_again(self, room: Room, actor: Member) -> Room:
+        """Start a fresh game in a finished room while preserving its table setup."""
+        if not actor.is_host or room.status != "finished":
+            error = "Only the host can replay a finished game"
+            raise RoomError(error)
+        async with self._locks[room.code]:
+            room = self.require_room(room.code)
+            if room.status != "finished" or room.pack is None:
+                error = "This game is not ready for a replay"
+                raise RoomError(error)
+            members = sorted(self.repository.members(room.code), key=lambda member: member.seat)
+            try:
+                room.state = new_game(
+                    room.code,
+                    [member.name for member in members],
+                    room.pack,
+                    room.settings,
+                    seed=secrets.randbits(63),
+                )
+            except RuleError as exc:
+                raise RoomError(str(exc)) from exc
+            room.status = "playing"
+            room.revision = room.state.revision
+            self.repository.save_state(
+                room,
+                f"replay-{secrets.token_hex(8)}",
+                actor.seat,
+                {"type": "play_again"},
+            )
+        self.broker.publish(room.code, room.revision)
+        self.schedule_ai(room.code)
+        return room
+
+    async def submit(self, room: Room, actor: Member, command: GameCommand) -> Room:
+        """Apply and persist one idempotent human game command."""
+        if room.status != "playing" or room.state is None or room.pack is None:
+            error = "This game is not active"
+            raise RoomError(error)
+        async with self._locks[room.code]:
+            room = self.require_room(room.code)
+            if self.repository.has_command(room.code, command.command_id):
+                return room
+            if room.state is None or room.pack is None:
+                error = "This game is not active"
+                raise RoomError(error)
+            try:
+                room.state = apply_command(room.state, command, room.pack, room.settings, actor_seat=actor.seat)
+            except RuleError as exc:
+                raise RoomError(str(exc)) from exc
+            room.revision = room.state.revision
+            room.status = "finished" if room.state.finished else "playing"
+            room.state.log = room.state.log[-200:]
+            self.repository.save_state(room, command.command_id, actor.seat, command.model_dump(mode="json"))
+        self.broker.publish(room.code, room.revision)
+        self.schedule_ai(room.code)
+        return room
+
+    def replace_with_ai(self, room: Room, actor: Member, seat: int, difficulty: str) -> None:
+        """Replace a disconnected human player with an AI controller."""
+        if not actor.is_host or room.status != "playing":
+            error = "Only the host can replace a player during a game"
+            raise RoomError(error)
+        target = next((item for item in self.repository.members(room.code) if item.seat == seat), None)
+        if target is None or target.controller != "human" or target.is_host:
+            error = "That player cannot be replaced"
+            raise RoomError(error)
+        last_seen = datetime.fromisoformat(target.last_seen)
+        if (datetime.now(UTC) - last_seen).total_seconds() < OFFLINE_AFTER_SECONDS:
+            error = "That player is still connected"
+            raise RoomError(error)
+        target.controller = "ai"
+        target.difficulty = difficulty
+        target.credential_hash = None
+        target.ready = True
+        self.repository.update_member(target)
+        self.broker.publish(room.code, room.revision)
+        self.schedule_ai(room.code)
+
+    def schedule_ai(self, code: str) -> None:
+        """Schedule an AI runner when a room does not already have one."""
+        task = self._ai_tasks.get(code)
+        if task is None or task.done():
+            self._ai_tasks[code] = asyncio.create_task(self._run_ai(code))
+
+    async def _run_ai(self, code: str) -> None:
+        while True:
+            room = self.require_room(code)
+            if room.status != "playing" or room.state is None or room.pack is None:
+                return
+            member = next(
+                (item for item in self.repository.members(code) if item.seat == room.state.current_seat), None
+            )
+            if member is None or member.controller != "ai":
+                return
+            await asyncio.sleep(0.35)
+            async with self._locks[code]:
+                room = self.require_room(code)
+                if room.state is None or room.pack is None or room.state.current_seat != member.seat:
+                    continue
+                chosen = choose_ai_command(
+                    room.state, room.pack, room.settings, member.seat, member.difficulty or "medium"
+                )
+                try:
+                    room.state = apply_command(room.state, chosen, room.pack, room.settings, actor_seat=member.seat)
+                except RuleError:
+                    return
+                room.revision = room.state.revision
+                room.status = "finished" if room.state.finished else "playing"
+                room.state.log = room.state.log[-200:]
+                self.repository.save_state(room, chosen.command_id, member.seat, chosen.model_dump(mode="json"))
+            self.broker.publish(code, room.revision)
+
+    def _new_code(self) -> str:
+        while True:
+            code = "".join(secrets.choice(ROOM_ALPHABET) for _ in range(8))
+            if not self.repository.room_exists(code):
+                return code
+
+    def _require_host_lobby(self, room: Room, actor: Member) -> None:
+        if not actor.is_host or room.status != "lobby":
+            error = "Only the host can change this lobby"
+            raise RoomError(error)
+
+    def _bump_lobby(self, room: Room) -> None:
+        room.revision += 1
+        room.updated_at = utc_now()
+        self.repository.save_lobby(room)
+        self.broker.publish(room.code, room.revision)
+
+
+def validate_name(name: str) -> str:
+    """Normalize and validate a player display name."""
+    clean = " ".join(name.split())
+    if not 1 <= len(clean) <= DISPLAY_NAME_MAX_LENGTH or any(
+        ord(character) < CONTROL_CHARACTER_LIMIT for character in clean
+    ):
+        error = "Display names must be 1-32 plain-text characters"
+        raise RoomError(error)
+    return clean

python/gems/routes/__init__.py

@@ -0,0 +1,5 @@
+"""HTTP routes for Gems."""
+
+from .app import router
+
+__all__ = ["router"]

python/gems/routes/app.py

@@ -0,0 +1,626 @@
+"""FastAPI pages, HTMX mutations, and SSE stream."""
+
+from __future__ import annotations
+
+import asyncio
+import html
+from collections.abc import AsyncIterator, Callable, Mapping
+from datetime import UTC, datetime
+from typing import Annotated, Literal
+
+from fastapi import APIRouter, Form, Request, Response, UploadFile
+from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, StreamingResponse
+from pydantic import ValidationError
+
+from python.gems.content import ContentPackError, content_pack_schema, parse_content_pack
+from python.gems.domain.engine import bonuses, public_state, score
+from python.gems.domain.legal_actions import command as build_command
+from python.gems.domain.legal_actions import legal_commands
+from python.gems.domain.models import GameCommand, GameSettings, Interactions, Modules
+from python.gems.persistence import Member, Room
+from python.gems.rooms import RoomError, RoomService
+from python.gems.web import templates
+
+router = APIRouter()
+COOKIE_NAME = "gems_credential"
+DISPLAY_NAME_COOKIE = "gems_display_name"
+DISPLAY_NAME_MAX_AGE = 365 * 24 * 60 * 60
+DISPLAY_NAME_MAX_LENGTH = 32
+OFFLINE_AFTER_SECONDS = 30
+RoomOperation = Callable[[Room, Member], None]
+
+
+def service(request: Request) -> RoomService:
+    """Return the application-scoped room service."""
+    return request.app.state.rooms
+
+
+def _request_origin(request: Request) -> str:
+    return f"{request.url.scheme}://{request.url.netloc}".rstrip("/")
+
+
+def _public_origin(request: Request) -> str:
+    """Prefer the current address when the untouched local default is configured."""
+    configured = request.app.state.config.public_origin.rstrip("/")
+    if configured == "http://127.0.0.1:8082":
+        return _request_origin(request)
+    return configured
+
+
+def _context(request: Request, room: Room, member: Member, *, error: str | None = None) -> dict[str, object]:
+    members = service(request).repository.members(room.code)
+    humans = [item for item in members if item.controller == "human"]
+    all_humans_ready = bool(humans) and all(item.ready for item in humans)
+    csrf = service(request).credentials.csrf(request.cookies.get(COOKIE_NAME, ""), room.code)
+    context: dict[str, object] = {
+        "request": request,
+        "room": room,
+        "member": member,
+        "members": members,
+        "csrf": csrf,
+        "error": error,
+        "public_origin": _public_origin(request),
+        "all_humans_ready": all_humans_ready,
+        "can_start": room.pack is not None and all_humans_ready,
+        "offline_seats": {
+            item.seat
+            for item in members
+            if item.controller == "human"
+            and (datetime.now(UTC) - datetime.fromisoformat(item.last_seen)).total_seconds() >= OFFLINE_AFTER_SECONDS
+        },
+    }
+    if room.state and room.pack:
+        context["state"] = public_state(room.state, member.seat)
+        context["pack"] = room.pack
+        context["cards"] = {card.id: card for card in room.pack.cards}
+        context["resources"] = {resource.id: resource for resource in [*room.pack.resources, room.pack.wild_resource]}
+        context["normal_resource_ids"] = room.pack.resource_ids
+        context["all_resource_ids"] = (*room.pack.resource_ids, room.pack.wild_resource.id)
+        context["member_by_seat"] = {item.seat: item for item in members}
+        context["bonus_counts"] = {player.seat: bonuses(player, room.pack) for player in room.state.players}
+        context["patrons"] = {item.id: item for item in room.pack.patrons}
+        context["objectives"] = {item.id: item for item in room.pack.objectives}
+        context["outposts"] = {item.id: item for item in room.pack.outposts}
+        commands = (
+            legal_commands(room.state, room.pack, room.settings, member.seat) if member.controller == "human" else []
+        )
+        context["actions"] = [(action_label(item, room), item.model_dump_json()) for item in commands]
+        context["double_resource_ids"] = {
+            str(item.payload["resource"])
+            for item in commands
+            if item.type == "take_double" and item.payload.get("resource")
+        }
+        card_actions: dict[str, dict[str, str]] = {}
+        deck_reserve_actions: dict[str, str] = {}
+        for item in commands:
+            card_id = item.payload.get("card_id")
+            if item.type in {"purchase", "reserve"} and card_id:
+                card_actions.setdefault(str(card_id), {})[item.type] = item.model_dump_json()
+            elif item.type == "reserve" and item.payload.get("deck"):
+                deck_reserve_actions[str(item.payload["deck"])] = item.model_dump_json()
+        context["card_actions"] = card_actions
+        context["deck_reserve_actions"] = deck_reserve_actions
+        context["scores"] = {player.seat: score(player, room.pack) for player in room.state.players}
+    return context
+
+
+def action_label(command: GameCommand, room: Room) -> str:
+    """Build a human-readable label for a legal game command."""
+    payload = command.payload
+    if command.type == "take_distinct":
+        label = "Take " + ", ".join(payload["resources"])
+    elif command.type == "take_double":
+        label = f"Take two {payload['resource']}"
+    elif command.type == "reserve":
+        if payload.get("card_id") and room.pack:
+            label = f"Reserve {room.pack.card(payload['card_id']).label}"
+        else:
+            label = f"Reserve from {payload.get('deck', 'deck')}"
+    elif command.type == "purchase" and room.pack:
+        label = f"Purchase {room.pack.card(payload['card_id']).label}"
+    elif command.type == "choose" and isinstance(payload.get("tokens"), dict) and room.pack:
+        resources = {item.id: item for item in [*room.pack.resources, room.pack.wild_resource]}
+        discarded = payload["tokens"]
+        label = "Discard " + ", ".join(
+            f"{amount} {resources[str(resource)].label}" for resource, amount in discarded.items()
+        )
+    elif command.type == "decline":
+        label = "Decline"
+    else:
+        choice = payload.get("choice") or payload.get("card_id") or "selection"
+        label = f"Choose {choice}"
+    return label
+
+
+def _selected_gem_command(room: Room, seat: int, selected: list[str]) -> GameCommand:
+    if room.state is None or room.pack is None:
+        error = "The game has not started"
+        raise RoomError(error)
+    if not selected:
+        error = "Select gem piles before taking gems"
+        raise RoomError(error)
+    if len(selected) != len(set(selected)) or not set(selected) <= set(room.pack.resource_ids):
+        error = "Choose each normal gem pile at most once"
+        raise RoomError(error)
+    commands = legal_commands(room.state, room.pack, room.settings, seat)
+    chosen = next(
+        (
+            item
+            for item in commands
+            if item.type == "take_double" and len(selected) == 1 and item.payload.get("resource") == selected[0]
+        ),
+        None,
+    )
+    if chosen is None:
+        chosen = next(
+            (
+                item
+                for item in commands
+                if item.type == "take_distinct"
+                and set(item.payload.get("resources", [])) == set(selected)
+                and len(item.payload.get("resources", [])) == len(selected)
+            ),
+            None,
+        )
+    if chosen is None:
+        error = "Choose three different available gems, or one pile with at least four gems to take a pair"
+        raise RoomError(error)
+    return chosen
+
+
+def _double_gem_command(room: Room, seat: int, resource: str) -> GameCommand:
+    if room.state is None or room.pack is None:
+        error = "The game has not started"
+        raise RoomError(error)
+    chosen = next(
+        (
+            item
+            for item in legal_commands(room.state, room.pack, room.settings, seat)
+            if item.type == "take_double" and item.payload.get("resource") == resource
+        ),
+        None,
+    )
+    if chosen is None:
+        error = "That gem pair is unavailable; a pile needs at least four gems"
+        raise RoomError(error)
+    return chosen
+
+
+def _discard_token_command(room: Room, seat: int, submitted: Mapping[str, object]) -> GameCommand:
+    if room.state is None or room.pack is None:
+        error = "The game has not started"
+        raise RoomError(error)
+    pending = room.state.pending
+    if pending is None or pending.kind != "discard_tokens" or pending.seat != seat:
+        error = "There are no excess tokens to discard"
+        raise RoomError(error)
+    discard: dict[str, int] = {}
+    for resource in (*room.pack.resource_ids, room.pack.wild_resource.id):
+        value = submitted.get(resource, "0")
+        try:
+            amount = int(value) if isinstance(value, str) else 0
+        except ValueError as exc:
+            error = "Token discard amounts must be whole numbers"
+            raise RoomError(error) from exc
+        if amount < 0:
+            error = "Token discard amounts cannot be negative"
+            raise RoomError(error)
+        if amount:
+            discard[resource] = amount
+    return build_command(room.state, "choose", {"tokens": discard})
+
+
+def _authenticate(request: Request, code: str) -> tuple[Room, Member]:
+    return service(request).authenticate(code, request.cookies.get(COOKIE_NAME))
+
+
+def _saved_display_name(request: Request) -> str:
+    name = request.cookies.get(DISPLAY_NAME_COOKIE, "").strip()
+    return name if len(name) <= DISPLAY_NAME_MAX_LENGTH else ""
+
+
+def _set_identity_cookies(response: Response, request: Request, credential: str, name: str) -> None:
+    cookie_options = {"httponly": True, "secure": request.app.state.config.secure_cookies, "samesite": "lax"}
+    response.set_cookie(COOKIE_NAME, credential, **cookie_options)
+    response.set_cookie(DISPLAY_NAME_COOKIE, name.strip(), max_age=DISPLAY_NAME_MAX_AGE, **cookie_options)
+
+
+def _check_csrf(request: Request, room_code: str, csrf: str) -> None:
+    credential = request.cookies.get(COOKIE_NAME, "")
+    if not service(request).credentials.valid_csrf(credential, room_code, csrf):
+        error = "Your form expired; reload and try again"
+        raise RoomError(error)
+    origin = request.headers.get("origin")
+    allowed_origins = {
+        request.app.state.config.public_origin.rstrip("/"),
+        _request_origin(request),
+    }
+    if origin and origin.rstrip("/") not in allowed_origins:
+        error = "Request origin was rejected"
+        raise RoomError(error)
+
+
+@router.get("/", response_class=HTMLResponse)
+def home(request: Request) -> Response:
+    """Render the room creation and join page."""
+    return templates.TemplateResponse(request, "home.html", {"display_name": _saved_display_name(request)})
+
+
+@router.post("/rooms")
+def create_room(request: Request, name: Annotated[str, Form()]) -> Response:
+    """Create a room and redirect its host to the lobby."""
+    try:
+        room, credential = service(request).create(name, request.cookies.get(COOKIE_NAME))
+    except RoomError as exc:
+        return templates.TemplateResponse(
+            request, "home.html", {"error": str(exc), "display_name": name}, status_code=422
+        )
+    response = RedirectResponse(f"/rooms/{room.code}", status_code=303)
+    _set_identity_cookies(response, request, credential, name)
+    return response
+
+
+@router.post("/join")
+def join_code(code: Annotated[str, Form()]) -> Response:
+    """Normalize a room code and redirect to its join page."""
+    return RedirectResponse(f"/join/{code.strip().upper()}", status_code=303)
+
+
+@router.get("/join/{code}", response_class=HTMLResponse)
+def join_page(request: Request, code: str) -> Response:
+    """Render the display-name form for an existing room."""
+    try:
+        room = service(request).require_room(code)
+    except RoomError:
+        return templates.TemplateResponse(
+            request,
+            "join.html",
+            {"code": code, "error": "Room not found", "display_name": _saved_display_name(request)},
+            status_code=404,
+        )
+    return templates.TemplateResponse(
+        request, "join.html", {"code": room.code, "display_name": _saved_display_name(request)}
+    )
+
+
+@router.post("/join/{code}")
+def join_room(request: Request, code: str, name: Annotated[str, Form()]) -> Response:
+    """Join a room and persist the member's browser identity."""
+    try:
+        room, credential = service(request).join(code, name, request.cookies.get(COOKIE_NAME))
+    except RoomError as exc:
+        return templates.TemplateResponse(
+            request, "join.html", {"code": code, "error": str(exc), "display_name": name}, status_code=422
+        )
+    response = RedirectResponse(f"/rooms/{room.code}", status_code=303)
+    _set_identity_cookies(response, request, credential, name)
+    return response
+
+
+@router.get("/rooms/{code}", response_class=HTMLResponse)
+def room_page(request: Request, code: str) -> Response:
+    """Render the authenticated lobby or game table."""
+    try:
+        room, member = _authenticate(request, code)
+    except RoomError:
+        return RedirectResponse(f"/join/{code}", status_code=303)
+    return templates.TemplateResponse(request, "room.html", _context(request, room, member))
+
+
+def _partial(request: Request, code: str, operation: RoomOperation, *, error_status: int = 422) -> Response:
+    try:
+        room, member = _authenticate(request, code)
+        operation(room, member)
+        room = service(request).require_room(code)
+        return templates.TemplateResponse(request, "partials/room_state.html", _context(request, room, member))
+    except (RoomError, ValidationError, ContentPackError) as exc:
+        try:
+            room, member = _authenticate(request, code)
+            return templates.TemplateResponse(
+                request,
+                "partials/room_state.html",
+                _context(request, room, member, error=str(exc)),
+                status_code=error_status,
+            )
+        except RoomError:
+            return HTMLResponse(html.escape(str(exc)), status_code=error_status)
+
+
+def _csrf_partial(request: Request, code: str, csrf: str, operation: RoomOperation) -> Response:
+    def authorized(room: Room, member: Member) -> None:
+        _check_csrf(request, room.code, csrf)
+        operation(room, member)
+
+    return _partial(request, code, authorized)
+
+
+@router.post("/rooms/{code}/pack", response_class=HTMLResponse)
+async def upload_pack(request: Request, code: str, csrf: Annotated[str, Form()], pack_file: UploadFile) -> Response:
+    """Validate and activate an uploaded JSON content pack."""
+    raw = await pack_file.read(512 * 1024 + 1)
+
+    def operation(room: Room, member: Member) -> None:
+        _check_csrf(request, room.code, csrf)
+        service(request).set_pack(room, member, parse_content_pack(raw))
+
+    return _partial(request, code, operation)
+
+
+@router.post("/rooms/{code}/settings/preset", response_class=HTMLResponse)
+def apply_preset(
+    request: Request,
+    code: str,
+    csrf: Annotated[str, Form()],
+    preset: Annotated[str, Form()],
+) -> Response:
+    """Apply a built-in rules preset to a lobby."""
+    presets: dict[str, tuple[Modules, Literal["score", "objective", "either", "both"]]] = {
+        "classic": (Modules(), "score"),
+        "objectives": (Modules(objectives=True), "objective"),
+        "objectives_outposts": (Modules(objectives=True, outposts=True), "objective"),
+        "eastern_fortifications": (Modules(eastern_decks=True, fortifications=True), "score"),
+        "all": (Modules(objectives=True, outposts=True, eastern_decks=True, fortifications=True), "objective"),
+    }
+
+    def operation(room: Room, member: Member) -> None:
+        _check_csrf(request, room.code, csrf)
+        if preset not in presets:
+            error = "Unknown rules preset"
+            raise RoomError(error)
+        modules, victory = presets[preset]
+        settings = room.settings.model_copy(deep=True)
+        settings.modules = modules
+        settings.victory_condition = victory
+        settings.interactions = Interactions()
+        service(request).set_settings(room, member, settings)
+
+    return _partial(request, code, operation)
+
+
+@router.get("/rooms/{code}/pack")
+def download_pack(request: Request, code: str) -> Response:
+    """Download the room's active content pack as JSON."""
+    room, _ = _authenticate(request, code)
+    if room.pack is None:
+        return JSONResponse({"detail": "No pack uploaded"}, status_code=404)
+    return JSONResponse(
+        room.pack.model_dump(mode="json"),
+        headers={"Content-Disposition": f'attachment; filename="{room.pack.metadata.id}.json"'},
+    )
+
+
+@router.post("/rooms/{code}/settings", response_class=HTMLResponse)
+async def update_settings(request: Request, code: str) -> Response:
+    """Validate and save custom lobby rules."""
+    form = await request.form()
+
+    def checked(name: str) -> bool:
+        return name in form
+
+    def value(name: str, default: str) -> str:
+        submitted = form.get(name)
+        return submitted if isinstance(submitted, str) else default
+
+    def operation(room: Room, member: Member) -> None:
+        _check_csrf(request, room.code, value("csrf", ""))
+        settings = GameSettings.model_validate(
+            {
+                "target_score": value("target_score", "15"),
+                "victory_condition": value("victory_condition", "score"),
+                "first_player_mode": value("first_player_mode", "random"),
+                "first_player_seat": value("first_player_seat", "0"),
+                "token_limit": value("token_limit", "10"),
+                "reserve_limit": value("reserve_limit", "3"),
+                "base_market_size": value("base_market_size", "4"),
+                "eastern_market_size": value("eastern_market_size", "2"),
+                "objective_count": value("objective_count", "3"),
+                "fortifications_per_player": value("fortifications_per_player", "3"),
+                "modules": Modules(
+                    objectives=checked("objectives"),
+                    outposts=checked("outposts"),
+                    eastern_decks=checked("eastern_decks"),
+                    fortifications=checked("fortifications"),
+                ),
+                "interactions": Interactions(**{field: checked(field) for field in Interactions.model_fields}),
+            }
+        )
+        service(request).set_settings(room, member, settings)
+
+    return _partial(request, code, operation)
+
+
+@router.post("/rooms/{code}/ready", response_class=HTMLResponse)
+def ready(
+    request: Request,
+    code: str,
+    csrf: Annotated[str, Form()],
+    ready_value: Annotated[bool, Form(alias="ready")],
+) -> Response:
+    """Update the authenticated human player's ready state."""
+    return _csrf_partial(
+        request, code, csrf, lambda room, member: service(request).set_ready(room, member, ready=ready_value)
+    )
+
+
+@router.post("/rooms/{code}/seats/ai", response_class=HTMLResponse)
+def add_ai(request: Request, code: str, csrf: Annotated[str, Form()], difficulty: Annotated[str, Form()]) -> Response:
+    """Add an AI-controlled player to a lobby."""
+    return _csrf_partial(request, code, csrf, lambda room, member: service(request).add_ai(room, member, difficulty))
+
+
+@router.post("/rooms/{code}/seats/{seat}/remove", response_class=HTMLResponse)
+def remove_seat(request: Request, code: str, seat: int, csrf: Annotated[str, Form()]) -> Response:
+    """Remove a non-host player from a lobby."""
+    return _csrf_partial(request, code, csrf, lambda room, member: service(request).remove_seat(room, member, seat))
+
+
+@router.post("/rooms/{code}/seats/{seat}/make-host", response_class=HTMLResponse)
+def transfer_host(request: Request, code: str, seat: int, csrf: Annotated[str, Form()]) -> Response:
+    """Transfer lobby ownership to another human player."""
+    return _csrf_partial(request, code, csrf, lambda room, member: service(request).transfer_host(room, member, seat))
+
+
+@router.post("/rooms/{code}/seats/{seat}/replace-with-ai", response_class=HTMLResponse)
+def replace_with_ai(
+    request: Request,
+    code: str,
+    seat: int,
+    csrf: Annotated[str, Form()],
+    difficulty: Annotated[str, Form()] = "medium",
+) -> Response:
+    """Replace a disconnected player with an AI controller."""
+    return _csrf_partial(
+        request,
+        code,
+        csrf,
+        lambda room, member: service(request).replace_with_ai(room, member, seat, difficulty),
+    )
+
+
+@router.post("/rooms/{code}/start", response_class=HTMLResponse)
+async def start_game(request: Request, code: str, csrf: Annotated[str, Form()]) -> Response:
+    """Start a game after validating the lobby's readiness."""
+    try:
+        room, member = _authenticate(request, code)
+        _check_csrf(request, room.code, csrf)
+        await service(request).start(room, member)
+        room = service(request).require_room(code)
+        return templates.TemplateResponse(request, "partials/room_state.html", _context(request, room, member))
+    except RoomError as exc:
+        room, member = _authenticate(request, code)
+        return templates.TemplateResponse(
+            request, "partials/room_state.html", _context(request, room, member, error=str(exc)), status_code=422
+        )
+
+
+@router.post("/rooms/{code}/play-again", response_class=HTMLResponse)
+async def play_again(request: Request, code: str, csrf: Annotated[str, Form()]) -> Response:
+    """Start a fresh game with the finished room's current setup."""
+    try:
+        room, member = _authenticate(request, code)
+        _check_csrf(request, room.code, csrf)
+        room = await service(request).play_again(room, member)
+        return templates.TemplateResponse(request, "partials/room_state.html", _context(request, room, member))
+    except RoomError as exc:
+        room, member = _authenticate(request, code)
+        return templates.TemplateResponse(
+            request, "partials/room_state.html", _context(request, room, member, error=str(exc)), status_code=422
+        )
+
+
+@router.post("/rooms/{code}/commands", response_class=HTMLResponse)
+async def game_command(
+    request: Request, code: str, csrf: Annotated[str, Form()], command_json: Annotated[str, Form()]
+) -> Response:
+    """Validate and submit one serialized game command."""
+    try:
+        room, member = _authenticate(request, code)
+        _check_csrf(request, room.code, csrf)
+        command = GameCommand.model_validate_json(command_json)
+        room = await service(request).submit(room, member, command)
+        return templates.TemplateResponse(request, "partials/room_state.html", _context(request, room, member))
+    except (RoomError, ValidationError) as exc:
+        room, member = _authenticate(request, code)
+        return templates.TemplateResponse(
+            request, "partials/room_state.html", _context(request, room, member, error=str(exc)), status_code=422
+        )
+
+
+@router.post("/rooms/{code}/take-gems", response_class=HTMLResponse)
+async def take_gems(request: Request, code: str) -> Response:
+    """Translate the visual gem selection into a legal command."""
+    form = await request.form()
+    try:
+        room, member = _authenticate(request, code)
+        _check_csrf(request, room.code, str(form.get("csrf", "")))
+        pair_resource = form.get("pair_resource")
+        if pair_resource is not None:
+            chosen = _double_gem_command(room, member.seat, str(pair_resource))
+        else:
+            selected = [str(item) for item in form.getlist("resources")]
+            chosen = _selected_gem_command(room, member.seat, selected)
+        room = await service(request).submit(room, member, chosen)
+        return templates.TemplateResponse(request, "partials/room_state.html", _context(request, room, member))
+    except RoomError as exc:
+        room, member = _authenticate(request, code)
+        return templates.TemplateResponse(
+            request, "partials/room_state.html", _context(request, room, member, error=str(exc)), status_code=422
+        )
+
+
+@router.post("/rooms/{code}/discard-tokens", response_class=HTMLResponse)
+async def discard_tokens(request: Request, code: str) -> Response:
+    """Return a player's selected excess tokens to the supply."""
+    form = await request.form()
+    try:
+        room, member = _authenticate(request, code)
+        _check_csrf(request, room.code, str(form.get("csrf", "")))
+        submitted = {
+            resource: form.get(f"token_{resource}", "0")
+            for resource in ((*room.pack.resource_ids, room.pack.wild_resource.id) if room.pack else ())
+        }
+        chosen = _discard_token_command(room, member.seat, submitted)
+        room = await service(request).submit(room, member, chosen)
+        return templates.TemplateResponse(request, "partials/room_state.html", _context(request, room, member))
+    except RoomError as exc:
+        room, member = _authenticate(request, code)
+        return templates.TemplateResponse(
+            request, "partials/room_state.html", _context(request, room, member, error=str(exc)), status_code=422
+        )
+
+
+@router.get("/rooms/{code}/events")
+async def room_events(request: Request, code: str) -> Response:
+    """Stream room revisions to an authenticated browser over SSE."""
+    try:
+        room, member = _authenticate(request, code)
+    except RoomError:
+        return HTMLResponse("Unauthorized", status_code=401)
+    queue = service(request).broker.subscribe(room.code)
+    stream_member = member
+
+    async def stream() -> AsyncIterator[str]:
+        current_member = stream_member
+        try:
+            # Comments keep the SSE connection alive without replacing in-progress form input.
+            yield ": connected\n\n"
+            while True:
+                try:
+                    await asyncio.wait_for(queue.get(), timeout=15)
+                    latest = service(request).require_room(code)
+                    yield _sse_fragment(request, latest, current_member)
+                except TimeoutError:
+                    _, current_member = _authenticate(request, code)
+                    yield ": keepalive\n\n"
+                if await request.is_disconnected():
+                    break
+        finally:
+            service(request).broker.unsubscribe(room.code, queue)
+
+    return StreamingResponse(
+        stream(), media_type="text/event-stream", headers={"Cache-Control": "no-store", "X-Accel-Buffering": "no"}
+    )
+
+
+def _sse_fragment(request: Request, room: Room, member: Member) -> str:
+    rendered = templates.get_template("partials/room_state.html").render(_context(request, room, member))
+    return "event: room\n" + "\n".join(f"data: {line}" for line in rendered.splitlines()) + "\n\n"
+
+
+@router.get("/schemas/content-pack-v1.json")
+def schema() -> Response:
+    """Return the JSON Schema for content-pack version one."""
+    return JSONResponse(content_pack_schema())
+
+
+@router.get("/healthz")
+def health() -> dict[str, str]:
+    """Report that the HTTP process is running."""
+    return {"status": "ok"}
+
+
+@router.get("/readyz")
+def readyz(request: Request) -> dict[str, str]:
+    """Report readiness after checking the database connection."""
+    request.app.state.repository._connection.execute("SELECT 1").fetchone()  # noqa: SLF001
+    return {"status": "ready"}

python/gems/security.py

@@ -0,0 +1,40 @@
+"""Opaque browser credentials and CSRF protection."""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import secrets
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+class CredentialService:
+    """Hash browser credentials with a stable per-installation key."""
+
+    def __init__(self, key_path: Path) -> None:
+        """Load or create the installation's credential-signing key."""
+        key_path.parent.mkdir(parents=True, exist_ok=True)
+        if not key_path.exists():
+            key_path.write_bytes(secrets.token_bytes(32))
+            key_path.chmod(0o600)
+        self._key = key_path.read_bytes()
+
+    @staticmethod
+    def issue() -> str:
+        """Issue a cryptographically random browser credential."""
+        return secrets.token_urlsafe(32)
+
+    def digest(self, credential: str) -> str:
+        """Create the persistent keyed digest of a browser credential."""
+        return hmac.new(self._key, credential.encode(), hashlib.sha256).hexdigest()
+
+    def csrf(self, credential: str, room_code: str) -> str:
+        """Create a room-scoped CSRF token for a browser credential."""
+        return hmac.new(self._key, f"csrf:{credential}:{room_code}".encode(), hashlib.sha256).hexdigest()
+
+    def valid_csrf(self, credential: str, room_code: str, candidate: str) -> bool:
+        """Validate a candidate CSRF token using constant-time comparison."""
+        return hmac.compare_digest(self.csrf(credential, room_code), candidate)

python/gems/static/overrides.css

@@ -0,0 +1,18 @@
+.preset-row{display:flex;gap:.5rem;flex-wrap:wrap;margin-bottom:1rem}.invite-link{margin:0;min-width:min(430px,55vw)}.danger{color:var(--danger)}.pack-status{border-left:3px solid var(--danger);padding:.65rem .8rem;background:#4a201d55;border-radius:0 9px 9px 0}button:disabled,button:disabled:hover{background:#252b2f;border-color:#3b4348;color:#7f898f;cursor:not-allowed;opacity:.7}
+.room-topbar{height:auto;min-height:68px;gap:1.25rem}.room-identity{display:flex;align-items:baseline;gap:.65rem;margin-right:auto}.room-identity .eyebrow{margin:0}.room-identity strong{font-size:1.2rem;letter-spacing:.14em}.room-invite{display:flex;align-items:center;grid-template-columns:auto minmax(260px,430px);gap:.65rem}.room-invite input{padding:.5rem .65rem}
+@media(max-width:700px){.room-topbar{flex-wrap:wrap;padding-block:.7rem}.room-invite{order:3;width:100%;grid-template-columns:auto 1fr}.invite-link{min-width:0}.room-identity strong{font-size:1rem}}
+
+.game-layout{grid-template-columns:330px minmax(0,1fr) 280px;grid-template-areas:"players market supply";align-items:start}.board-sidebar{display:contents}.players-stack{grid-area:players;display:grid;gap:.55rem}.market{grid-area:market}.right-supply{grid-area:supply;position:sticky;top:1rem}.sidebar-player{padding:.75rem}.sidebar-player.active{border-color:var(--gold);box-shadow:0 0 0 1px var(--gold),0 12px 28px #0003}.sidebar-player-summary{list-style:none;cursor:pointer}.sidebar-player-summary::-webkit-details-marker{display:none}.sidebar-player-heading{display:flex;justify-content:space-between;align-items:center;gap:.5rem}.sidebar-player-heading>span{color:var(--gold);font-size:.82rem}.count-legend{display:flex;justify-content:space-between;color:var(--muted);font-size:.68rem;margin:.45rem 0 .25rem}.player-color-counts{display:grid;grid-template-columns:repeat(3,1fr);gap:.3rem}.player-color-count{--gem-color:#697780;display:flex;align-items:center;justify-content:space-between;gap:.25rem;padding:.3rem .4rem;border:1px solid color-mix(in srgb,var(--gem-color) 55%,var(--line));border-radius:8px;background:color-mix(in srgb,var(--gem-color) 10%,#0e1418);font-size:.75rem}.player-color-count b{color:color-mix(in srgb,var(--gem-color) 70%,white);text-shadow:0 1px 2px #000}.player-detail-hint{display:block;margin-top:.45rem}.sidebar-player-details{border-top:1px solid var(--line);margin-top:.65rem;padding-top:.65rem}.owned-card-grid{display:grid;gap:.4rem;max-height:18rem;overflow-y:auto}.owned-card-inspector{--gem-color:#697780;border:1px solid color-mix(in srgb,var(--gem-color) 55%,var(--line));border-radius:9px;overflow:hidden;background:color-mix(in srgb,var(--gem-color) 9%,#0e1418)}.owned-card-inspector>summary{display:grid;grid-template-columns:1.8rem 1fr auto;align-items:center;gap:.45rem;padding:.5rem;cursor:pointer}.owned-card-bonus{display:grid;place-items:center;width:1.8rem;height:1.8rem;border-radius:50%;background:var(--gem-color);color:var(--gem-ink,#fff);font-weight:900;text-shadow:0 1px 2px #000}.owned-card-detail{padding:.5rem;border-top:1px solid var(--line);font-size:.78rem}.owned-card-detail p{margin:.25rem 0}.owned-card-detail .resource-cost{display:inline-block;margin:.15rem}
+.gem-picker{display:grid;gap:.55rem}.gem-picker .fine-print{margin:.1rem 0 .35rem}.gem-pile,.wild-pile{--gem-color:#697780;display:grid;grid-template-columns:auto 1fr auto;align-items:center;gap:.55rem;margin:0;padding:.55rem;border:1px solid color-mix(in srgb,var(--gem-color) 65%,var(--line));border-radius:12px;background:#0e1418;color:var(--text)}.gem-pile input{width:1.1rem;height:1.1rem;margin:0;accent-color:var(--gem-color)}.gem-choice{display:grid;grid-template-columns:2.6rem 1fr;align-items:center;gap:.55rem;margin:0;cursor:pointer}.gem-pile:has(input:checked){border-color:var(--gem-color);box-shadow:0 0 0 1px var(--gem-color),0 0 14px color-mix(in srgb,var(--gem-color) 35%,transparent);background:color-mix(in srgb,var(--gem-color) 18%,#0e1418)}.gem-pile.empty{opacity:.45}.take-pair{padding:.45rem .55rem;white-space:nowrap}.gem-disc{display:grid;place-items:center;width:2.5rem;height:2.5rem;border-radius:50%;border:3px solid color-mix(in srgb,var(--gem-color) 55%,white);background:radial-gradient(circle at 35% 30%,color-mix(in srgb,var(--gem-color) 45%,white),var(--gem-color) 65%);color:var(--gem-ink,#fff);text-shadow:0 1px 2px #000,0 0 3px #000;font-size:1.15rem;font-weight:900;box-shadow:0 4px 8px #0008,0 0 8px color-mix(in srgb,var(--gem-color) 35%,transparent)}.card-bonus-token{width:2rem;height:2rem;border-width:2px;font-size:.9rem;line-height:1;flex:none}.gem-pile strong,.wild-pile strong{color:color-mix(in srgb,var(--gem-color) 68%,white)}.gem-pile small,.wild-pile small{display:block}.gem-picker.readonly .gem-pile{grid-template-columns:2.6rem 1fr;cursor:default}.wild-pile{grid-template-columns:2.6rem 1fr;margin-top:.7rem}.resource-cost{border-color:color-mix(in srgb,var(--gem-color) 65%,var(--line))!important;box-shadow:inset 3px 0 0 var(--gem-color)}
+.gem-disc.light-gem{border-color:#090c0e;background:radial-gradient(circle at 32% 25%,#fff 0 18%,#f8f7f2 48%,#d9dde0 100%);color:#090c0e;font-weight:800;text-shadow:none;-webkit-font-smoothing:antialiased;text-rendering:geometricPrecision;box-shadow:0 4px 9px #0009,0 0 0 1px #596168,inset 0 0 0 2px #fff9}.hand-gem.light-gem>b{border:2px solid #090c0e;background:radial-gradient(circle at 32% 25%,#fff 0 18%,#f8f7f2 48%,#d9dde0 100%);color:#090c0e;font-weight:800;text-shadow:none;-webkit-font-smoothing:antialiased;text-rendering:geometricPrecision;box-shadow:0 2px 5px #0008,0 0 0 1px #596168,inset 0 0 0 1px #fff9}
+.card-bonus-token.light-gem{border-color:#090c0e;background:radial-gradient(circle at 32% 25%,#fff 0 16%,#f7f6f1 42%,#d9dde0 100%);color:#090c0e;text-shadow:none;box-shadow:0 3px 7px #000b,0 0 0 1px #596168,inset 0 0 0 2px #fff9}
+.tier-label{align-items:center}.compact-button{padding:.35rem .55rem}.game-card{display:flex;flex-direction:column;padding:0;overflow:hidden}.card-face{min-height:170px;padding:.9rem;flex:1}.game-card.actionable{border-color:#46615d}.card-actions{display:flex;gap:.45rem;padding:.75rem;border-top:1px solid var(--line);background:#0e1418}.card-actions form,.card-actions>button{flex:1}.card-actions button{width:100%;padding:.5rem}.board-help{grid-template-columns:170px 1fr}.board-help p{margin:0;color:var(--muted)}
+.reserved-cards{margin-top:1rem}.reserved-cards h3{margin-bottom:.5rem}.reserved-card{border:1px solid var(--line);border-radius:10px;margin-top:.4rem;overflow:hidden}.reserved-card-face{display:flex;justify-content:space-between;gap:.5rem;padding:.6rem}.reserved-card.actionable{border-color:var(--teal)}.reserved-card-face span{color:var(--muted);font-size:.8rem}
+@media(max-width:900px){.game-layout{grid-template-columns:1fr;grid-template-areas:"players" "market" "supply"}.right-supply{position:static}.gem-picker{grid-template-columns:repeat(2,minmax(0,1fr))}.gem-picker .fine-print,.gem-picker>.take-checked{grid-column:1/-1}}
+@media(max-width:540px){.gem-picker{grid-template-columns:1fr}.gem-picker .fine-print,.gem-picker>.take-checked{grid-column:auto}.board-help{display:flex}}
+.player-gem-dock{position:fixed;left:50%;bottom:.55rem;transform:translateX(-50%);z-index:15;display:flex;align-items:center;gap:.65rem;max-width:calc(100vw - 1rem);padding:.4rem .65rem;border:1px solid var(--gold);border-radius:14px;background:#151c21ee;backdrop-filter:blur(16px);box-shadow:0 12px 35px #0009}.player-gem-dock>strong{white-space:nowrap;color:var(--gold)}.hand-total{display:grid;place-items:center;min-width:3.8rem;padding:.25rem .45rem;border:1px solid var(--gold);border-radius:10px;background:#2a2417}.hand-total strong{color:var(--gold);font-size:.9rem}.hand-total small{font-size:.6rem;text-transform:uppercase;letter-spacing:.08em}.hand-legend{white-space:nowrap;color:var(--muted);font-size:.72rem}.player-gem-hand{display:flex;gap:.4rem}.hand-gem{--gem-color:#697780;display:grid;grid-template-columns:1.8rem auto;align-items:center;gap:.35rem;padding:.25rem .5rem .25rem .25rem;border:1px solid color-mix(in srgb,var(--gem-color) 65%,var(--line));border-radius:999px;background:color-mix(in srgb,var(--gem-color) 13%,#0e1418)}.hand-gem>b{display:grid;place-items:center;width:1.8rem;height:1.8rem;border-radius:50%;background:var(--gem-color);color:var(--gem-ink,#fff);text-shadow:0 1px 2px #000}.hand-gem>span{display:flex;align-items:center;gap:.2rem}.hand-gem>span strong{font-size:.95rem}.hand-gem i{color:var(--muted);font-style:normal}.action-dock{bottom:4.7rem}
+.bottom-reserved{display:flex;align-items:center;gap:.35rem;padding-left:.6rem;border-left:1px solid var(--line)}.bottom-reserved-card{display:flex;align-items:center;gap:.35rem;padding:.25rem;border:1px solid var(--line);border-radius:9px;background:#0e1418}.bottom-reserved-card>span{max-width:8rem}.bottom-reserved-card strong,.bottom-reserved-card small{display:block;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.bottom-reserved-card strong{font-size:.72rem}.bottom-reserved-card small{color:var(--muted);font-size:.62rem}.bottom-reserved-card button{padding:.35rem .45rem;font-size:.7rem}
+.action-dock>div:first-child small{display:block;margin-top:.25rem}.discard-picker{display:flex;align-items:center;gap:.65rem;min-width:0}.discard-colors{display:flex;gap:.4rem;overflow-x:auto;padding:.15rem;min-width:0}.discard-color{display:grid;grid-template-columns:2rem minmax(4.5rem,auto) 3.5rem;align-items:center;gap:.4rem;flex:none;margin:0;padding:.35rem;border:1px solid color-mix(in srgb,var(--gem-color) 65%,var(--line));border-radius:10px;background:color-mix(in srgb,var(--gem-color) 10%,#0e1418)}.discard-color .gem-disc{width:2rem;height:2rem;font-size:.9rem}.discard-color>span:nth-child(2){font-size:.75rem;font-weight:700;color:color-mix(in srgb,var(--gem-color) 70%,white)}.discard-color small{display:block;font-size:.6rem}.discard-color input{width:3.5rem;margin:0;padding:.4rem}.discard-picker>button{flex:none;white-space:nowrap}
+@media(max-width:700px){.player-gem-dock{width:calc(100vw - 1rem);overflow-x:auto;justify-content:flex-start}.player-gem-hand{flex:none}.player-gem-dock>strong{font-size:.8rem}.hand-gem{grid-template-columns:1.5rem auto}.hand-gem b{width:1.5rem;height:1.5rem}}
+
+@media(min-width:901px){.board-main{max-width:1900px;padding:clamp(.55rem,1.2vh,1rem) 1rem 7rem;scroll-padding-bottom:7rem}.board-main .status-strip{padding:.5rem 0;margin-bottom:.55rem}.board-main .game-layout{grid-template-columns:300px minmax(0,1fr) 270px;gap:.55rem}.board-main .players-stack{gap:.35rem}.board-main .sidebar-player{padding:.5rem}.board-main .count-legend{margin:.25rem 0 .15rem}.board-main .player-color-counts{gap:.2rem}.board-main .player-color-count{padding:.2rem .3rem}.board-main .player-detail-hint{margin-top:.25rem}.board-main .market{gap:.45rem}.board-main .market-row{padding:.55rem}.board-main .tier-label{margin-bottom:.4rem}.board-main .card-row{gap:.45rem}.board-main .card-face{min-height:108px;padding:.6rem}.board-main .game-card h3{font-size:1rem;line-height:1.15;margin:.55rem 0 .25rem}.board-main .game-card p{margin:.25rem 0}.board-main .card-actions{padding:.4rem;gap:.3rem}.board-main .card-actions button{padding:.4rem .25rem}.board-main .right-supply{padding:.65rem;top:.5rem}.board-main .right-supply h2{font-size:1.1rem;margin-bottom:.35rem}.board-main .right-supply .section-heading{margin-bottom:.35rem}.board-main .right-supply .gem-picker{gap:.3rem}.board-main .right-supply .gem-pile,.board-main .right-supply .wild-pile{padding:.3rem;gap:.35rem}.board-main .right-supply .gem-choice{grid-template-columns:2rem 1fr;gap:.35rem}.board-main .right-supply .gem-disc{width:2rem;height:2rem;font-size:.9rem}.board-main .right-supply .take-pair{padding:.3rem}.board-main .right-supply .fine-print{font-size:.7rem}.history-inline{position:relative}.history-inline summary{cursor:pointer;color:var(--teal)}.history-inline>div{position:absolute;right:0;top:1.8rem;width:min(360px,70vw);max-height:50vh;overflow:auto;z-index:20;padding:.7rem;border:1px solid var(--line);border-radius:10px;background:var(--panel);box-shadow:0 16px 40px #0008}.history-inline p{margin:.3rem 0;font-size:.8rem}.patrons-section{padding-bottom:.5rem;margin-bottom:.55rem;border-bottom:1px solid var(--line)}.patron-list{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:.3rem}.patron-card{display:grid;grid-template-columns:auto 1fr;align-items:start;gap:.3rem;padding:.35rem;border:1px solid var(--line);border-radius:8px;background:#0e1418;font-size:.72rem}.patron-card>strong{color:var(--gold)}.patron-card b,.patron-card small{display:block}.requirement-chip{display:inline-block;margin:.15rem .1rem 0;padding:.1rem .25rem;border:1px solid var(--line);border-radius:999px}.claimed-patrons{display:grid;gap:.25rem;margin-top:.5rem}.claimed-patrons span{font-size:.78rem;color:var(--gold)}}