feat(van-weather): port weather service to Rust

Add a Rust implementation with privacy-masked locations, HTTP retries,
and Home Assistant sensor publishing. Package it with Nix and switch the
systemd service from Python to the compiled binary.

.gitignore

@@ -173,3 +173,6 @@ frontend/node_modules/
 # data from testing llms
 data/*
 .ebook_search_bm25
+
+# gems data
+.gems

.vscode/settings.json

@@ -71,6 +71,7 @@
     "ehci",
     "emerg",
     "endlessh",
+    "ents",
     "errorlens",
     "esbenp",
     "esphome",
@@ -172,6 +173,8 @@
     "Networkd",
     "networkmanager",
     "newtabpage",
+    "ngram",
+    "ngrams",
     "nixfmt",
     "nixos",
     "nixpkgs",
@@ -298,7 +301,9 @@
     "uiprotect",
     "uitour",
     "unifi",
+    "unjudged",
     "unrar",
+    "unstorable",
     "unsubmitted",
     "uptimekuma",
     "urlbar",
@@ -326,6 +331,7 @@
     "xcursorgen",
     "xdist",
     "xhci",
+    "yake",
     "yazi",
     "yubikey",
     "yubioath",

Cargo.toml

@@ -0,0 +1,4 @@
+[workspace]
+resolver = "2"
+
+members = ["rust/*"]

flake.lock

@@ -8,11 +8,11 @@
       },
       "locked": {
         "dir": "pkgs/firefox-addons",
-        "lastModified": 1783828963,
-        "narHash": "sha256-eTytzcUJCaDUZ3/9EF0+V3fvlikQMQBwiX1Sx4Gy+No=",
+        "lastModified": 1785729742,
+        "narHash": "sha256-PBavY37OTsIM7VJMUYPv2Rz/gSpbtJGxAGl9iXCaMU4=",
         "owner": "rycee",
         "repo": "nur-expressions",
-        "rev": "8d61e9afde605cd6c22dab68b83d7a71f0a6c5b2",
+        "rev": "1529ecae5978cd2ac18a8edbf27967350bf4b80e",
         "type": "gitlab"
       },
       "original": {
@@ -29,11 +29,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1783823409,
-        "narHash": "sha256-OI4IkRjRXa1e7hYmCGJDPDq5H/kPwhsyoS80cNUF9fI=",
+        "lastModified": 1785531816,
+        "narHash": "sha256-vkMnV0JIyw+g/NmcfoajlGaAO+9a0ezia+FZohQJrik=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "7566825d4652a1b885bd4ce65bd9e8def432fec9",
+        "rev": "bf9ce9fec78f95f374e8dd3b503863a3ec128ebe",
         "type": "github"
       },
       "original": {
@@ -47,11 +47,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1783792734,
-        "narHash": "sha256-50rvY9GdFvpYDcMLcD/4cWSi0hVxArT5wsGlVsHy8eY=",
+        "lastModified": 1785232496,
+        "narHash": "sha256-65EQYIRRpTdpH8lUiB6Mvo5uBkG60aBIzAJuALfx+O0=",
         "owner": "nixos",
         "repo": "nixos-hardware",
-        "rev": "8efb4337e857949f4cfac86d12ef1066f417f31f",
+        "rev": "2e790b0a6be8ec2b76174ac0931b8ff11919ec98",
         "type": "github"
       },
       "original": {
@@ -76,11 +76,11 @@
     },
     "nixpkgs-master": {
       "locked": {
-        "lastModified": 1783874024,
-        "narHash": "sha256-Fd8rPvyBv6JjcO/nZxZiFQan6Fww/jAF4TYj0Th/Yfo=",
+        "lastModified": 1785777863,
+        "narHash": "sha256-BPLjbZgQ7hud+zRbLDqyFAW4qTKD2MBLR+TRI7oOg/w=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "0b4f03c64b236e4ba4252414274e92796c300124",
+        "rev": "0adbd05f9f410c2b26d550d344ac019efa6b7224",
         "type": "github"
       },
       "original": {
@@ -108,11 +108,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1783776592,
-        "narHash": "sha256-UgCQzxeWI75XM8G+hPrPh+MKzEPjG3SpAj7dtqSbksA=",
+        "lastModified": 1785692966,
+        "narHash": "sha256-vUfIeBEfpbAfZ5zjgIkYk7eHBeVfCYVjLbWnMkseYnk=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "e7a3ca8092b61ff85b6a45bf863ea2b2d6a661b3",
+        "rev": "643809054d65fdd466a63e3155b8c498cb483c04",
         "type": "github"
       },
       "original": {

overlays/default.nix

@@ -24,6 +24,7 @@
         fastapi
         fastapi-cli
         httpx
+        jinja2
         mypy
         pgvector
         psycopg
@@ -34,11 +35,13 @@
         pytest-mock
         pytest-xdist
         python-multipart
+        pydantic-settings
         ruff
         sqlalchemy
         tenacity
         tinytuya
         typer
+        uvicorn
         websockets
       ]
     );

pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
     "pydantic",
     "pydantic-settings",
     "python-multipart",
-    "sqlalchemy",
+    "sqlalchemy[asyncio]",
     "tenacity",
     "tiktoken",
     "tinytuya",
@@ -41,8 +41,10 @@ whisper-transcribe = "python.tools.whisper.transcribe:main"
 
 [dependency-groups]
 dev = [
+    "aiosqlite",
     "mypy",
     "pyfakefs",
+    "pytest-asyncio",
     "pytest-cov",
     "pytest-mock",
     "pytest-xdist",
@@ -63,6 +65,7 @@ lint.ignore = [
     "ISC001", # (TEMP) conflicts when used with the formatter
     "S603",   # (PERM) This is known to cause a false positive
     "S607",   # (PERM) This is becoming a consistent annoyance
+    "CPY001",  # (PERM) I don't include the license in every file
 ]
 
 [tool.ruff.lint.per-file-ignores]
@@ -118,5 +121,6 @@ exclude_lines = [
 
 [tool.pytest.ini_options]
 addopts = "-n auto -ra --ignore=tests/ebook_search"
+asyncio_mode = "auto"
 testpaths = ["tests"]
 # --cov=system_tools --cov-report=term-missing --cov-report=xml --cov-report=html --cov-branch

python/alembic/richie/versions/2026_06_29-adding_phrase_metadata_tables_dddee09eddcc.py

@@ -0,0 +1,206 @@
+"""adding Phrase metadata tables.
+
+Revision ID: dddee09eddcc
+Revises: 96d72c748c24
+Create Date: 2026-06-29 00:49:07.344159
+
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+from python.orm import RichieBase
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+# revision identifiers, used by Alembic.
+revision: str = "dddee09eddcc"
+down_revision: str | None = "96d72c748c24"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+schema = RichieBase.schema_name
+
+
+def upgrade() -> None:
+    """Upgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "candidate_phrases",
+        sa.Column("book_id", sa.Integer(), nullable=False),
+        sa.Column("series_id", sa.Integer(), nullable=True),
+        sa.Column("phrase_text", sa.Text(), nullable=False),
+        sa.Column("phrase_norm", sa.Text(), nullable=False),
+        sa.Column("token_count", sa.Integer(), nullable=False),
+        sa.Column("source_raw_ngram", sa.Boolean(), nullable=False),
+        sa.Column("source_yake", sa.Boolean(), nullable=False),
+        sa.Column("source_spacy_ner", sa.Boolean(), nullable=False),
+        sa.Column("source_spacy_noun_chunk", sa.Boolean(), nullable=False),
+        sa.Column("source_capitalized", sa.Boolean(), nullable=False),
+        sa.Column("source_metadata", sa.Boolean(), nullable=False),
+        sa.Column("spacy_label", sa.String(), nullable=True),
+        sa.Column("raw_count", sa.Integer(), nullable=False),
+        sa.Column("chapter_count", sa.Integer(), nullable=False),
+        sa.Column("yake_score", sa.Float(), nullable=True),
+        sa.Column("candidate_score", sa.Float(), nullable=False),
+        sa.Column(
+            "sample_contexts",
+            sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), "postgresql"),
+            nullable=True,
+        ),
+        sa.Column("llm_judged", sa.Boolean(), nullable=False),
+        sa.Column("llm_keep", sa.Boolean(), nullable=True),
+        sa.Column("llm_confidence", sa.Float(), nullable=True),
+        sa.Column("llm_category", sa.String(), nullable=True),
+        sa.Column("llm_reason", sa.Text(), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["book_id"],
+            [f"{schema}.ebook_source.id"],
+            name=op.f("fk_candidate_phrases_book_id_ebook_source"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_candidate_phrases")),
+        sa.UniqueConstraint("book_id", "phrase_norm", name="uq_candidate_phrases_book_id_phrase_norm"),
+        schema=schema,
+    )
+    op.create_index(
+        "candidate_phrases_book_norm_idx", "candidate_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
+    )
+    op.create_index(
+        "candidate_phrases_book_score_idx",
+        "candidate_phrases",
+        ["book_id", "candidate_score"],
+        unique=False,
+        schema=schema,
+    )
+    op.create_table(
+        "protected_phrases",
+        sa.Column("book_id", sa.Integer(), nullable=True),
+        sa.Column("series_id", sa.Integer(), nullable=True),
+        sa.Column("phrase_text", sa.Text(), nullable=False),
+        sa.Column("phrase_norm", sa.Text(), nullable=False),
+        sa.Column("canonical_id", sa.String(), nullable=False),
+        sa.Column("phrase_type", sa.String(), nullable=True),
+        sa.Column("token_count", sa.Integer(), nullable=False),
+        sa.Column("confidence", sa.Float(), nullable=False),
+        sa.Column("importance", sa.Float(), nullable=False),
+        sa.Column("allow_nested", sa.Boolean(), nullable=False),
+        sa.Column("suppress_children", sa.Boolean(), nullable=False),
+        sa.Column("source_candidate_id", sa.Integer(), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["book_id"],
+            [f"{schema}.ebook_source.id"],
+            name=op.f("fk_protected_phrases_book_id_ebook_source"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["source_candidate_id"],
+            [f"{schema}.candidate_phrases.id"],
+            name=op.f("fk_protected_phrases_source_candidate_id_candidate_phrases"),
+            ondelete="SET NULL",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_protected_phrases")),
+        sa.UniqueConstraint("book_id", "phrase_norm", name="uq_protected_phrases_book_id_phrase_norm"),
+        schema=schema,
+    )
+    op.create_index(
+        "protected_phrases_book_norm_idx", "protected_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
+    )
+    op.create_index("protected_phrases_norm_idx", "protected_phrases", ["phrase_norm"], unique=False, schema=schema)
+    op.create_index(
+        "protected_phrases_series_norm_idx",
+        "protected_phrases",
+        ["series_id", "phrase_norm"],
+        unique=False,
+        schema=schema,
+    )
+    op.create_table(
+        "chunk_phrase_mentions",
+        sa.Column("chunk_id", sa.BigInteger(), nullable=False),
+        sa.Column("phrase_id", sa.Integer(), nullable=False),
+        sa.Column("book_id", sa.Integer(), nullable=True),
+        sa.Column("series_id", sa.Integer(), nullable=True),
+        sa.Column("start_char", sa.Integer(), nullable=False),
+        sa.Column("end_char", sa.Integer(), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["book_id"],
+            [f"{schema}.ebook_source.id"],
+            name=op.f("fk_chunk_phrase_mentions_book_id_ebook_source"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["chunk_id"],
+            [f"{schema}.ebook_chunk.id"],
+            name=op.f("fk_chunk_phrase_mentions_chunk_id_ebook_chunk"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["phrase_id"],
+            [f"{schema}.protected_phrases.id"],
+            name=op.f("fk_chunk_phrase_mentions_phrase_id_protected_phrases"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_chunk_phrase_mentions")),
+        sa.UniqueConstraint("chunk_id", "phrase_id", "start_char", name="uq_chunk_phrase_mentions_chunk_phrase_start"),
+        schema=schema,
+    )
+    op.create_index(
+        "chunk_phrase_mentions_chunk_idx", "chunk_phrase_mentions", ["chunk_id"], unique=False, schema=schema
+    )
+    op.create_index(
+        "chunk_phrase_mentions_phrase_idx", "chunk_phrase_mentions", ["phrase_id"], unique=False, schema=schema
+    )
+    op.create_table(
+        "phrase_aliases",
+        sa.Column("phrase_id", sa.Integer(), nullable=False),
+        sa.Column("alias_text", sa.Text(), nullable=False),
+        sa.Column("alias_norm", sa.Text(), nullable=False),
+        sa.Column("confidence", sa.Float(), nullable=False),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["phrase_id"],
+            [f"{schema}.protected_phrases.id"],
+            name=op.f("fk_phrase_aliases_phrase_id_protected_phrases"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_phrase_aliases")),
+        sa.UniqueConstraint("phrase_id", "alias_norm", name="uq_phrase_aliases_phrase_id_alias_norm"),
+        schema=schema,
+    )
+    op.create_index("phrase_aliases_norm_idx", "phrase_aliases", ["alias_norm"], unique=False, schema=schema)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index("phrase_aliases_norm_idx", table_name="phrase_aliases", schema=schema)
+    op.drop_table("phrase_aliases", schema=schema)
+    op.drop_index("chunk_phrase_mentions_phrase_idx", table_name="chunk_phrase_mentions", schema=schema)
+    op.drop_index("chunk_phrase_mentions_chunk_idx", table_name="chunk_phrase_mentions", schema=schema)
+    op.drop_table("chunk_phrase_mentions", schema=schema)
+    op.drop_index("protected_phrases_series_norm_idx", table_name="protected_phrases", schema=schema)
+    op.drop_index("protected_phrases_norm_idx", table_name="protected_phrases", schema=schema)
+    op.drop_index("protected_phrases_book_norm_idx", table_name="protected_phrases", schema=schema)
+    op.drop_table("protected_phrases", schema=schema)
+    op.drop_index("candidate_phrases_book_score_idx", table_name="candidate_phrases", schema=schema)
+    op.drop_index("candidate_phrases_book_norm_idx", table_name="candidate_phrases", schema=schema)
+    op.drop_table("candidate_phrases", schema=schema)
+    # ### end Alembic commands ###

python/alembic/richie/versions/2026_07_09-remove_spacy_ner_751260fc3228.py

@@ -0,0 +1,55 @@
+"""remove spaCy-ner.
+
+Revision ID: 751260fc3228
+Revises: dddee09eddcc
+Create Date: 2026-07-09 23:03:39.554083
+
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import sqlalchemy as sa
+from alembic import op
+
+from python.orm import RichieBase
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+# revision identifiers, used by Alembic.
+revision: str = "751260fc3228"
+down_revision: str | None = "dddee09eddcc"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+schema = RichieBase.schema_name
+
+
+def upgrade() -> None:
+    """Upgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("candidate_phrases", "source_spacy_noun_chunk", schema=schema)
+    op.drop_column("candidate_phrases", "source_spacy_ner", schema=schema)
+    op.drop_column("candidate_phrases", "spacy_label", schema=schema)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "candidate_phrases", sa.Column("spacy_label", sa.VARCHAR(), autoincrement=False, nullable=True), schema=schema
+    )
+    op.add_column(
+        "candidate_phrases",
+        sa.Column("source_spacy_ner", sa.BOOLEAN(), autoincrement=False, nullable=False),
+        schema=schema,
+    )
+    op.add_column(
+        "candidate_phrases",
+        sa.Column("source_spacy_noun_chunk", sa.BOOLEAN(), autoincrement=False, nullable=False),
+        schema=schema,
+    )
+    # ### end Alembic commands ###

python/ebook_search/answer.py

@@ -6,15 +6,23 @@ import logging
 from typing import TYPE_CHECKING
 
 from python.ebook_search.llm_interface import request_chat_completion
+from python.ebook_search.prompts import load_prompt
 
 if TYPE_CHECKING:
+    import httpx
+
     from python.ebook_search.config import EbookSearchConfig
     from python.ebook_search.search import SearchResult
 
 logger = logging.getLogger(__name__)
 
 
-def answer_query(query: str, results: list[SearchResult], config: EbookSearchConfig) -> str:
+async def answer_query(
+    client: httpx.AsyncClient,
+    query: str,
+    results: list[SearchResult],
+    config: EbookSearchConfig,
+) -> str:
     """Answer a question using only retrieved chunks."""
     if not config.answer_enabled:
         logger.info("ebook_answer_skipped_disabled")
@@ -25,33 +33,18 @@ def answer_query(query: str, results: list[SearchResult], config: EbookSearchCon
         return "No relevant sources were found."
 
     logger.info(
-        "ebook_answer_request_start base_url=%s model=%s sources=%s query_length=%s",
-        config.vllm_base_url,
-        config.chat_model,
-        len(results),
-        len(query),
+        f"ebook_answer_request_start {config.vllm_base_url=} {config.chat_model=} sources={len(results)} "
+        f"query_length={len(query)}"
     )
     context = "\n\n".join(
         f"[{index}] {result.source_title}{' - ' + result.chapter_title if result.chapter_title else ''}\n{result.text}"
         for index, result in enumerate(results, start=1)
     )
-    content = request_chat_completion(
+    content = await request_chat_completion(
+        client,
         config,
-        [
-            {
-                "role": "system",
-                "content": (
-                    "Answer only from the provided context. Cite sources with bracketed numbers like [1]. "
-                    "If the context is insufficient, say so."
-                ),
-            },
-            {"role": "user", "content": f"Question:\n{query}\n\nContext:\n{context}"},
-        ],
+        load_prompt("answer").messages(query=query, context=context),
     )
 
-    logger.info(
-        "ebook_answer_request_complete model=%s answer_length=%s",
-        config.chat_model,
-        len(content),
-    )
+    logger.info(f"ebook_answer_request_complete {config.chat_model=} answer_length={len(content)}")
     return content or "The model returned an empty answer."

python/ebook_search/api/bm25_tasks.py

@@ -1,18 +1,22 @@
-"""Background BM25 refresh tasks for the web app."""
+"""Background BM25 refresh tasks for the web app.
+
+The refresh is scheduled on the event loop instead of a thread because the async psycopg
+driver only works from the loop; a bare thread cannot open a session on the async engine.
+"""
 
 from __future__ import annotations
 
+import asyncio
 import logging
-from threading import Timer
 from typing import TYPE_CHECKING
 
-from sqlalchemy.orm import Session
+from sqlalchemy.ext.asyncio import AsyncSession
 
 from python.ebook_search.bm25_corpus import load_bm25_corpus, refresh_bm25_corpus
 
 if TYPE_CHECKING:
     from fastapi import FastAPI
-    from sqlalchemy.engine import Engine
+    from sqlalchemy.ext.asyncio import AsyncEngine
 
     from python.ebook_search.config import EbookSearchConfig
 
@@ -20,41 +24,47 @@ logger = logging.getLogger(__name__)
 
 
 def schedule_bm25_refresh(app: FastAPI) -> None:
-    """Schedule a delayed BM25 corpus refresh, replacing any pending refresh."""
-    existing_timer = getattr(app.state, "bm25_refresh_timer", None)
-    if existing_timer is not None:
-        existing_timer.cancel()
+    """Schedule a delayed BM25 corpus refresh, replacing any pending refresh.
 
-    timer = Timer(app.state.config.bm25_refresh_delay_seconds, refresh_bm25_for_app, args=(app,))
-    timer.daemon = True
-    timer.start()
-    app.state.bm25_refresh_timer = timer
-    logger.info(
-        "ebook_bm25_refresh_scheduled delay_seconds=%s",
-        app.state.config.bm25_refresh_delay_seconds,
-    )
+    Only called from route handlers, so a running event loop is guaranteed.
+    """
+    cancel_bm25_refresh(app)
+
+    loop = asyncio.get_running_loop()
+
+    def start_refresh() -> None:
+        app.state.bm25_refresh_task = loop.create_task(refresh_bm25_for_app(app))
+
+    app.state.bm25_refresh_timer = loop.call_later(app.state.config.bm25_refresh_delay_seconds, start_refresh)
+    logger.info(f"ebook_bm25_refresh_scheduled {app.state.config.bm25_refresh_delay_seconds=}")
 
 
 def cancel_bm25_refresh(app: FastAPI) -> None:
-    """Cancel any pending BM25 corpus refresh."""
+    """Cancel any pending BM25 corpus refresh timer and in-flight refresh task."""
     existing_timer = getattr(app.state, "bm25_refresh_timer", None)
     if existing_timer is not None:
         existing_timer.cancel()
         app.state.bm25_refresh_timer = None
         logger.info("ebook_bm25_refresh_cancelled")
 
+    existing_task = getattr(app.state, "bm25_refresh_task", None)
+    if existing_task is not None:
+        if not existing_task.done():
+            existing_task.cancel()
+        app.state.bm25_refresh_task = None
 
-def refresh_bm25_for_app(app: FastAPI) -> None:
+
+async def refresh_bm25_for_app(app: FastAPI) -> None:
     """Refresh the BM25 corpus using the app engine and config."""
     try:
-        refresh_bm25_for_engine(app.state.engine, app.state.config)
+        await refresh_bm25_for_engine(app.state.engine, app.state.config)
     except Exception:
         logger.exception("ebook_bm25_refresh_failed")
 
 
-def refresh_bm25_for_engine(engine: Engine, config: EbookSearchConfig) -> None:
-    """Refresh the BM25 corpus using a SQLAlchemy engine."""
-    with Session(engine) as session:
-        refresh_bm25_corpus(session, config)
+async def refresh_bm25_for_engine(engine: AsyncEngine, config: EbookSearchConfig) -> None:
+    """Refresh the BM25 corpus using an async SQLAlchemy engine."""
+    async with AsyncSession(engine) as session:
+        await refresh_bm25_corpus(session, config)
     load_bm25_corpus.cache_clear()
     logger.info("ebook_bm25_corpus_cache_cleared_after_refresh")

python/ebook_search/api/dependencies.py

@@ -4,8 +4,8 @@ from __future__ import annotations
 
 from typing import Annotated
 
+import httpx
 from fastapi import Depends, Request
-from sqlalchemy.engine import Engine
 
 from python.ebook_search.config import EbookSearchConfig
 
@@ -15,10 +15,10 @@ def get_config(request: Request) -> EbookSearchConfig:
     return request.app.state.config
 
 
-def get_engine(request: Request) -> Engine:
-    """Get the database engine from app state."""
-    return request.app.state.engine
+def get_http_client(request: Request) -> httpx.AsyncClient:
+    """Get the shared LLM HTTP client from app state."""
+    return request.app.state.http_client
 
 
 AppConfig = Annotated[EbookSearchConfig, Depends(get_config)]
-AppEngine = Annotated[Engine, Depends(get_engine)]
+AppHttpClient = Annotated[httpx.AsyncClient, Depends(get_http_client)]

python/ebook_search/api/judge_tasks.py

@@ -0,0 +1,127 @@
+"""Background phrase-judging tasks for the web app.
+
+Judging a book sends one LLM request per candidate phrase, which can take minutes, so it must
+not run inside the request where it would block the UI. Judgments run as async FastAPI
+background tasks, awaited on the event loop after the response is sent, and are tracked per
+book in app state so a second judge request for a book that is already being judged is
+rejected instead of doubling the work.
+
+State is loop-confined: every read and mutation happens on the event loop (async route
+handlers and async background tasks) and no critical section contains an ``await``, so each
+mutation is atomic per loop iteration and no locking is needed.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
+
+if TYPE_CHECKING:
+    from fastapi import BackgroundTasks, FastAPI
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class JudgeTaskState:
+    """Running book judgments and last outcome messages, keyed by book id."""
+
+    running_book_ids: set[int] = field(default_factory=set)
+    outcome_messages: dict[int, str] = field(default_factory=dict)
+
+
+def get_judge_task_state(app: FastAPI) -> JudgeTaskState:
+    """Return the app's judge task state, creating it on first use.
+
+    Args:
+        app (FastAPI): App whose state holds the judge task registry.
+
+    Returns:
+        JudgeTaskState: The shared judge task state for this app.
+    """
+    state = getattr(app.state, "judge_tasks", None)
+    if state is None:
+        state = JudgeTaskState()
+        app.state.judge_tasks = state
+    return state
+
+
+def start_book_phrase_judgment(app: FastAPI, background_tasks: BackgroundTasks, source_id: int) -> bool:
+    """Queue judging of one book's candidate phrases as a FastAPI background task.
+
+    The book is claimed before the response returns, so a repeated judge request cannot queue
+    a second run while one is pending or running.
+
+    Args:
+        app (FastAPI): App supplying the engine, config, and judge task state.
+        background_tasks (BackgroundTasks): Request's background tasks to queue the judgment on.
+        source_id (int): Book to judge candidates for.
+
+    Returns:
+        bool: True when a judgment was queued, False when one is already running for this book.
+    """
+    state = get_judge_task_state(app)
+    if source_id in state.running_book_ids:
+        logger.info(f"ebook_book_phrase_judgment_already_running {source_id=}")
+        return False
+    state.running_book_ids.add(source_id)
+    state.outcome_messages.pop(source_id, None)
+    background_tasks.add_task(judge_book_phrases_for_app, app, source_id)
+    logger.info(f"ebook_book_phrase_judgment_queued {source_id=}")
+    return True
+
+
+async def judge_book_phrases_for_app(app: FastAPI, source_id: int) -> None:
+    """Judge one book using the app engine and config, recording the outcome message.
+
+    Args:
+        app (FastAPI): App supplying the engine, config, and judge task state.
+        source_id (int): Book to judge candidates for.
+    """
+    state = get_judge_task_state(app)
+    try:
+        result = await judge_candidate_phrases_for_books(app.state.engine, app.state.config, source_ids=[source_id])
+        logger.info(
+            f"ebook_book_phrase_judgment_complete {source_id=} {result.candidates_judged=} {result.protected_phrases=} "
+            f"{result.phrase_mentions=} {result.books_failed=}"
+        )
+        if result.books_failed:
+            message = "Judging failed; see server logs for details"
+        else:
+            message = (
+                f"Judged {result.candidates_judged} candidates; {result.protected_phrases} protected phrases promoted"
+            )
+    except Exception:
+        logger.exception(f"ebook_book_phrase_judgment_task_failed {source_id=}")
+        message = "Judging failed; see server logs for details"
+    state.running_book_ids.discard(source_id)
+    state.outcome_messages[source_id] = message
+
+
+def is_judging_book(app: FastAPI, source_id: int) -> bool:
+    """Report whether a judgment is currently queued or running for one book.
+
+    Args:
+        app (FastAPI): App supplying the judge task state.
+        source_id (int): Book to check.
+
+    Returns:
+        bool: True while the book's judgment is pending or running.
+    """
+    return source_id in get_judge_task_state(app).running_book_ids
+
+
+def pop_book_judgment_outcome(app: FastAPI, source_id: int) -> str | None:
+    """Return and clear the outcome message from one book's last finished judgment.
+
+    Args:
+        app (FastAPI): App supplying the judge task state.
+        source_id (int): Book to fetch the outcome for.
+
+    Returns:
+        str | None: The outcome message, or None when there is nothing new to report.
+    """
+    return get_judge_task_state(app).outcome_messages.pop(source_id, None)

python/ebook_search/api/main.py

@@ -6,11 +6,12 @@ import logging
 from contextlib import asynccontextmanager
 from typing import TYPE_CHECKING, Annotated
 
+import httpx
 import typer
 import uvicorn
 from fastapi import FastAPI
 from fastapi.staticfiles import StaticFiles
-from sqlalchemy.orm import Session
+from sqlalchemy.ext.asyncio import AsyncSession
 
 from python.common import configure_logger
 from python.ebook_search.api.bm25_tasks import cancel_bm25_refresh
@@ -18,8 +19,9 @@ from python.ebook_search.api.routes import admin_router, health_router, page_rou
 from python.ebook_search.api.web import STATIC_DIR
 from python.ebook_search.bm25_corpus import ensure_bm25_corpus
 from python.ebook_search.config import load_config
+from python.ebook_search.protected_phrases.pool import shutdown_extraction_pool
 from python.fastapi_tools import ZstdMiddleware
-from python.orm.common import get_postgres_engine
+from python.orm.common import get_async_postgres_engine
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
@@ -35,27 +37,30 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     config = load_config()
     app.state.config = config
     logger.info(
-        "ebook_search_config_loaded top_k=%s embedding_model=%s embedding_base_url=%s vllm_base_url=%s "
-        "rerank_enabled=%s answer_enabled=%s library_paths=%s",
-        config.top_k,
-        config.embedding_model,
-        config.embedding_base_url,
-        config.vllm_base_url,
-        config.rerank.enabled,
-        config.answer_enabled,
-        len(config.library_paths),
+        f"ebook_search_config_loaded {config.top_k=} {config.embedding_model=} {config.embedding_base_url=} "
+        f"{config.vllm_base_url=} {config.rerank.enabled=} {config.phrase_matching_enabled=} {config.answer_enabled=} "
+        f"library_paths={len(config.library_paths)}"
     )
     if not config.library_paths:
         logger.warning("ebook_search_no_library_paths_configured")
-    app.state.engine = get_postgres_engine(name="RICHIE", vector_engine=True)
-    with Session(app.state.engine) as session:
-        ensure_bm25_corpus(session, config)
+    # Concurrent phrase judging opens one session per book worker on this engine, so size the pool
+    # to cover those plus headroom for ordinary web requests.
+    app.state.engine = get_async_postgres_engine(
+        name="RICHIE",
+        vector_engine=True,
+        pool_size=config.phrase_judge_book_workers + 10,
+    )
+    app.state.http_client = httpx.AsyncClient()
+    async with AsyncSession(app.state.engine, expire_on_commit=False) as session:
+        await ensure_bm25_corpus(session, config)
     try:
         yield
     finally:
         logger.info("ebook_search_shutdown")
         cancel_bm25_refresh(app)
-        app.state.engine.dispose()
+        shutdown_extraction_pool()
+        await app.state.http_client.aclose()
+        await app.state.engine.dispose()
 
 
 def create_app() -> FastAPI:

python/ebook_search/api/routes/admin.py

@@ -8,13 +8,20 @@ from fastapi import APIRouter, Request
 from fastapi.responses import HTMLResponse
 
 from python.ebook_search.api.bm25_tasks import schedule_bm25_refresh
-from python.ebook_search.api.dependencies import (
-    AppConfig,  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+from python.ebook_search.api.dependencies import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
+    AppConfig,
+    AppHttpClient,
 )
-from python.ebook_search.api.web import templates
+from python.ebook_search.api.web import error_response, templates
 from python.ebook_search.embeddings import embed_missing_chunks, embedding_model_stats
 from python.ebook_search.ingest import ingest_configured_paths
-from python.fastapi_tools import DbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+from python.ebook_search.protected_phrases.generate_ngrams import generate_candidate_phrases_for_books
+from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
+from python.ebook_search.protected_phrases.store import book_ids_pending_first_judgment, corpus_phrase_stats
+from python.fastapi_tools import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
+    AppAsyncEngine,
+    AsyncDbSession,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -22,40 +29,143 @@ router = APIRouter(prefix="/admin")
 
 
 @router.get("", response_class=HTMLResponse)
-def admin(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
+async def admin(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
     """Render the admin page."""
-    stats = embedding_model_stats(session)
-    logger.info("ebook_admin_page_loaded models=%s", len(stats))
-    return templates.TemplateResponse(request, "admin.html", {"config": config, "stats": stats})
+    stats = await embedding_model_stats(session)
+    phrase_stats = await corpus_phrase_stats(session)
+    logger.info(
+        f"ebook_admin_page_loaded models={len(stats)} {phrase_stats.candidate_phrases=} "
+        f"{phrase_stats.protected_phrases=}"
+    )
+    return templates.TemplateResponse(
+        request,
+        "admin.html",
+        {"config": config, "stats": stats, "phrase_stats": phrase_stats},
+    )
 
 
 @router.post("/scan", response_class=HTMLResponse)
-def scan_library(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
+async def scan_library(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
     """Scan configured library paths for EPUB changes."""
     try:
-        count = ingest_configured_paths(session, config)
-        session.commit()
+        count = await ingest_configured_paths(session, config)
+        await session.commit()
     except Exception as error:
         logger.exception("ebook_admin_scan_failed")
-        return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
+        return error_response(request, error)
 
-    logger.info("ebook_admin_scan_complete changed_files=%s", count)
+    logger.info(f"ebook_admin_scan_complete {count=}")
     if count > 0:
         schedule_bm25_refresh(request.app)
     return templates.TemplateResponse(request, "partials/admin_status.html", {"message": f"Indexed {count} EPUBs"})
 
 
+@router.post("/phrases/generate-all", response_class=HTMLResponse)
+async def generate_all_phrases(request: Request, config: AppConfig, engine: AppAsyncEngine) -> HTMLResponse:
+    """Regenerate candidate phrases for every indexed book without LLM judging."""
+    try:
+        result = await generate_candidate_phrases_for_books(engine, config)
+    except Exception as error:
+        logger.exception("ebook_admin_generate_phrases_failed")
+        return error_response(request, error)
+
+    logger.info(
+        f"ebook_admin_generate_phrases_complete {result.books_seen=} {result.books_built=} {result.candidate_phrases=}"
+    )
+    return templates.TemplateResponse(
+        request,
+        "partials/admin_status.html",
+        {
+            "message": (
+                f"Generated phrases for {result.books_built} of {result.books_seen} books; "
+                f"{result.candidate_phrases} candidates stored"
+            )
+        },
+    )
+
+
+@router.post("/phrases/judge-all", response_class=HTMLResponse)
+async def judge_all_phrases(request: Request, engine: AppAsyncEngine, config: AppConfig) -> HTMLResponse:
+    """Judge unjudged candidate phrases across every indexed book."""
+    return await run_phrase_judgment(request, engine, config, source_ids=None)
+
+
+@router.post("/phrases/judge-missing", response_class=HTMLResponse)
+async def judge_missing_phrases(
+    request: Request,
+    engine: AppAsyncEngine,
+    config: AppConfig,
+    session: AsyncDbSession,
+) -> HTMLResponse:
+    """Judge candidate phrases only for books where judging has never run."""
+    source_ids = await book_ids_pending_first_judgment(session)
+    if not source_ids:
+        return templates.TemplateResponse(
+            request,
+            "partials/admin_status.html",
+            {"message": "All books with candidate phrases have been judged"},
+        )
+    return await run_phrase_judgment(request, engine, config, source_ids=source_ids)
+
+
+async def run_phrase_judgment(
+    request: Request,
+    engine: AppAsyncEngine,
+    config: AppConfig,
+    *,
+    source_ids: list[int] | None,
+) -> HTMLResponse:
+    """Run LLM judging for candidate phrases and render the outcome as an admin status partial.
+
+    Args:
+        request (Request): Current request, for template rendering.
+        engine (AppAsyncEngine): Engine used to open per-book judging sessions.
+        config (AppConfig): Runtime phrase-tuning settings.
+        source_ids (list[int] | None): Books to judge; ``None`` judges every indexed book.
+
+    Returns:
+        HTMLResponse: Status partial describing the judging outcome.
+    """
+    try:
+        result = await judge_candidate_phrases_for_books(engine, config, source_ids=source_ids)
+    except Exception as error:
+        logger.exception("ebook_admin_judge_phrases_failed")
+        return error_response(request, error)
+
+    logger.info(
+        f"ebook_admin_judge_phrases_complete {result.books_seen=} {result.books_judged=} {result.books_failed=} "
+        f"{result.candidates_judged=} {result.protected_phrases=} {result.phrase_mentions=}"
+    )
+    return templates.TemplateResponse(
+        request,
+        "partials/admin_status.html",
+        {
+            "message": (
+                f"Judged {result.candidates_judged} candidates across {result.books_judged} of "
+                f"{result.books_seen} books; {result.protected_phrases} protected phrases, "
+                f"{result.phrase_mentions} mentions"
+                + (f"; {result.books_failed} books failed" if result.books_failed else "")
+            )
+        },
+    )
+
+
 @router.post("/embed-missing", response_class=HTMLResponse)
-def embed_missing(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
+async def embed_missing(
+    request: Request,
+    config: AppConfig,
+    session: AsyncDbSession,
+    client: AppHttpClient,
+) -> HTMLResponse:
     """Embed chunks missing vectors for the configured model."""
     try:
-        count = embed_missing_chunks(session, config)
-        session.commit()
+        count = await embed_missing_chunks(session, client, config)
+        await session.commit()
     except Exception as error:
         logger.exception("ebook_admin_embed_missing_failed")
-        return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
+        return error_response(request, error)
 
-    logger.info("ebook_admin_embed_missing_complete chunks=%s", count)
+    logger.info(f"ebook_admin_embed_missing_complete {count=}")
     return templates.TemplateResponse(
         request,
         "partials/admin_status.html",
@@ -64,38 +174,29 @@ def embed_missing(request: Request, config: AppConfig, session: DbSession) -> HT
 
 
 @router.post("/embed-all", response_class=HTMLResponse)
-def embed_all(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
+async def embed_all(
+    request: Request,
+    config: AppConfig,
+    session: AsyncDbSession,
+    client: AppHttpClient,
+) -> HTMLResponse:
     """Embed all chunks missing vectors in fixed-size batches."""
     total = 0
     batches = 0
     try:
         while True:
-            count = embed_missing_chunks(session, config)
+            count = await embed_missing_chunks(session, client, config)
             if count == 0:
                 break
-            session.commit()
+            await session.commit()
             total += count
             batches += 1
-            logger.info(
-                "ebook_admin_embed_all_batch_complete batch=%s chunks=%s total_chunks=%s",
-                batches,
-                count,
-                total,
-            )
+            logger.info(f"ebook_admin_embed_all_batch_complete {batches=} {count=} {total=}")
     except Exception as error:
-        logger.exception(
-            "ebook_admin_embed_all_failed batches=%s chunks=%s",
-            batches,
-            total,
-        )
-        return templates.TemplateResponse(
-            request,
-            "partials/error.html",
-            {"message": f"Embed all failed after {total} chunks in {batches} batches: {error}"},
-            status_code=500,
-        )
+        logger.exception(f"ebook_admin_embed_all_failed {batches=} {total=}")
+        return error_response(request, f"Embed all failed after {total} chunks in {batches} batches: {error}")
 
-    logger.info("ebook_admin_embed_all_complete batches=%s chunks=%s", batches, total)
+    logger.info(f"ebook_admin_embed_all_complete {batches=} {total=}")
     return templates.TemplateResponse(
         request,
         "partials/admin_status.html",

python/ebook_search/api/routes/health.py

@@ -11,15 +11,17 @@ from fastapi.responses import JSONResponse
 from sqlalchemy import literal, select
 from sqlalchemy.exc import SQLAlchemyError
 
-from python.ebook_search.api.dependencies import (
-    AppConfig,  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+from python.ebook_search.api.dependencies import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
+    AppConfig,
+    AppHttpClient,
 )
 from python.ebook_search.bm25_corpus import bm25_index_exists, bm25_index_path, read_bm25_manifest
 from python.ebook_search.llm_interface import check_chat_endpoint, check_embedding_endpoint
-from python.fastapi_tools import DbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+from python.fastapi_tools import AsyncDbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
 
 if TYPE_CHECKING:
-    from sqlalchemy.orm import Session
+    import httpx
+    from sqlalchemy.ext.asyncio import AsyncSession
 
     from python.ebook_search.config import EbookSearchConfig
 
@@ -29,17 +31,17 @@ router = APIRouter()
 
 
 @router.get("/health")
-def health() -> dict[str, str]:
+async def health() -> dict[str, str]:
     """Liveness probe that returns ok without touching dependencies."""
     return {"status": "ok"}
 
 
 @router.get("/ready")
-def ready(config: AppConfig, session: DbSession) -> JSONResponse:
+async def ready(config: AppConfig, session: AsyncDbSession, client: AppHttpClient) -> JSONResponse:
     """Readiness probe reporting database, embedding endpoint, and BM25 index status."""
-    database_ok = check_database(session)
-    embedding_ok = check_embedding_endpoint(config)
-    chat_status = chat_endpoint_status(config)
+    database_ok = await check_database(session)
+    embedding_ok = await check_embedding_endpoint(client, config)
+    chat_status = await chat_endpoint_status(client, config)
     bm25_status = check_bm25_status(config)
 
     checks = {
@@ -58,30 +60,23 @@ def ready(config: AppConfig, session: DbSession) -> JSONResponse:
         status = "ready"
         status_code = HTTPStatus.OK
 
-    logger.info(
-        "ebook_ready_check status=%s database=%s embedding=%s chat=%s bm25=%s",
-        status,
-        database_ok,
-        embedding_ok,
-        chat_status,
-        bm25_status,
-    )
+    logger.info(f"ebook_ready_check {status=} {database_ok=} {embedding_ok=} {chat_status=} {bm25_status=}")
     return JSONResponse(content={"status": status, "checks": checks}, status_code=status_code)
 
 
-def chat_endpoint_status(config: EbookSearchConfig) -> str:
+async def chat_endpoint_status(client: httpx.AsyncClient, config: EbookSearchConfig) -> str:
     """Return the answering chat endpoint status, or disabled when answers are off."""
     if not config.answer_enabled:
         return "disabled"
-    return "ok" if check_chat_endpoint(config) else "fail"
+    return "ok" if await check_chat_endpoint(client, config) else "fail"
 
 
-def check_database(session: Session) -> bool:
+async def check_database(session: AsyncSession) -> bool:
     """Return whether the database answers a trivial query."""
     try:
-        session.execute(select(literal(1)))
+        await session.execute(select(literal(1)))
     except SQLAlchemyError as error:
-        logger.warning("ebook_ready_database_unavailable error=%s", error)
+        logger.warning(f"ebook_ready_database_unavailable {error=}")
         return False
     return True
 

python/ebook_search/api/routes/page.py

@@ -3,17 +3,24 @@
 from __future__ import annotations
 
 import logging
+from typing import TYPE_CHECKING
 
-from fastapi import APIRouter, Request
-from fastapi.responses import HTMLResponse
-from sqlalchemy import select
+from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
+from fastapi.responses import HTMLResponse, RedirectResponse
+from sqlalchemy import func, select
 
 from python.ebook_search.api.dependencies import (
     AppConfig,  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
 )
+from python.ebook_search.api.judge_tasks import is_judging_book, pop_book_judgment_outcome, start_book_phrase_judgment
 from python.ebook_search.api.web import templates
-from python.fastapi_tools import DbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
-from python.orm.richie import EbookSource
+from python.ebook_search.protected_phrases.generate_ngrams import recalculate_candidate_phrases_for_book
+from python.ebook_search.protected_phrases.store import count_protected_phrases
+from python.fastapi_tools import AsyncDbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+from python.orm.richie import EbookCandidatePhrase, EbookChapter, EbookChunk, EbookProtectedPhrase, EbookSource
+
+if TYPE_CHECKING:
+    from sqlalchemy.ext.asyncio import AsyncSession
 
 logger = logging.getLogger(__name__)
 
@@ -21,38 +28,160 @@ router = APIRouter()
 
 
 @router.get("/", response_class=HTMLResponse)
-def index(request: Request, config: AppConfig) -> HTMLResponse:
+async def index(request: Request, config: AppConfig) -> HTMLResponse:
     """Render the search page."""
     return templates.TemplateResponse(request, "search.html", {"config": config})
 
 
 @router.get("/books", response_class=HTMLResponse)
-def books(request: Request, session: DbSession) -> HTMLResponse:
+async def books(request: Request, session: AsyncDbSession) -> HTMLResponse:
     """Render the indexed books page."""
-    sources = list(session.scalars(select(EbookSource).order_by(EbookSource.title)).all())
-    logger.info("ebook_books_page_loaded count=%s", len(sources))
+    sources = list((await session.scalars(select(EbookSource).order_by(EbookSource.title))).all())
+    logger.info(f"ebook_books_page_loaded count={len(sources)}")
     return templates.TemplateResponse(request, "books.html", {"sources": sources})
 
 
+async def get_chapter_count(session: AsyncSession, book_id: int) -> int:
+    """Return the number of indexed chapters for one book."""
+    return await session.scalar(select(func.count(EbookChapter.id)).where(EbookChapter.source_id == book_id)) or 0
+
+
+async def get_chunk_count(session: AsyncSession, book_id: int) -> int:
+    """Return the number of indexed chunks for one book."""
+    return await session.scalar(select(func.count(EbookChunk.id)).where(EbookChunk.source_id == book_id)) or 0
+
+
+async def get_candidate_count(session: AsyncSession, book_id: int) -> int:
+    """Return the number of indexed candidates for one book."""
+    return (
+        await session.scalar(select(func.count(EbookCandidatePhrase.id)).where(EbookCandidatePhrase.book_id == book_id))
+        or 0
+    )
+
+
+async def get_judged_candidate_count(session: AsyncSession, book_id: int) -> int:
+    """Return the number of judged candidates for one book."""
+    return (
+        await session.scalar(
+            select(func.count(EbookCandidatePhrase.id)).where(
+                EbookCandidatePhrase.book_id == book_id,
+                EbookCandidatePhrase.llm_judged.is_(True),
+            )
+        )
+        or 0
+    )
+
+
+async def get_candidates(session: AsyncSession, book_id: int) -> list[EbookCandidatePhrase]:
+    """Return the indexed candidates for one book."""
+    return list(
+        await session.scalars(
+            select(EbookCandidatePhrase)
+            .where(EbookCandidatePhrase.book_id == book_id)
+            .order_by(EbookCandidatePhrase.candidate_score.desc())
+            .limit(100)
+        )
+    )
+
+
+async def get_protected_phrases(session: AsyncSession, book_id: int) -> list[EbookProtectedPhrase]:
+    """Return the protected phrases for one book."""
+    return list(
+        await session.scalars(
+            select(EbookProtectedPhrase)
+            .where(EbookProtectedPhrase.book_id == book_id)
+            .order_by(EbookProtectedPhrase.importance.desc())
+            .limit(100)
+        )
+    )
+
+
 @router.get("/books/{source_id}", response_class=HTMLResponse)
-def book_detail(source_id: int, request: Request, session: DbSession) -> HTMLResponse:
+async def book_detail(source_id: int, request: Request, session: AsyncDbSession) -> HTMLResponse:
     """Render details for one indexed book."""
-    source = session.get(EbookSource, source_id)
+    source = await session.get(EbookSource, source_id)
+    phrase_status_message = None
+    recalculated = request.query_params.get("phrases_recalculated")
+    if recalculated is not None:
+        phrase_status_message = f"Recalculated phrases; {recalculated} candidates generated"
+    judgment_outcome = pop_book_judgment_outcome(request.app, source_id)
+    if judgment_outcome is not None:
+        phrase_status_message = judgment_outcome
+    judging_in_progress = is_judging_book(request.app, source_id)
+    if judging_in_progress:
+        phrase_status_message = "Judging candidate phrases in the background; refresh to see progress"
     if source is not None:
-        chapter_count = len(source.chapters)
-        chunk_count = len(source.chunks)
+        chapter_count = await get_chapter_count(session, source.id)
+        chunk_count = await get_chunk_count(session, source.id)
+        candidate_count = await get_candidate_count(session, source.id)
+        judged_candidate_count = await get_judged_candidate_count(session, source.id)
+        protected_count = await count_protected_phrases(session, source.id)
+        candidates = await get_candidates(session, source.id)
+        protected_phrases = await get_protected_phrases(session, source.id)
     else:
         chapter_count = 0
         chunk_count = 0
+        candidate_count = 0
+        judged_candidate_count = 0
+        protected_count = 0
+        candidates = []
+        protected_phrases = []
     logger.info(
-        "ebook_book_detail_loaded source_id=%s found=%s chapters=%s chunks=%s",
-        source_id,
-        source is not None,
-        chapter_count,
-        chunk_count,
+        f"ebook_book_detail_loaded {source_id=} found={source is not None} {chapter_count=} {chunk_count=} "
+        f"{candidate_count=} {judged_candidate_count=} {protected_count=}"
     )
     return templates.TemplateResponse(
         request,
         "book_detail.html",
-        {"chapter_count": chapter_count, "chunk_count": chunk_count, "source": source},
+        {
+            "candidate_count": candidate_count,
+            "candidates": candidates,
+            "chapter_count": chapter_count,
+            "chunk_count": chunk_count,
+            "judged_candidate_count": judged_candidate_count,
+            "judging_in_progress": judging_in_progress,
+            "protected_count": protected_count,
+            "protected_phrases": protected_phrases,
+            "phrase_status_message": phrase_status_message,
+            "source": source,
+        },
     )
+
+
+@router.post("/books/{source_id}/recalculate-phrases")
+async def recalculate_book_phrases(source_id: int, config: AppConfig, session: AsyncDbSession) -> RedirectResponse:
+    """Clear and regenerate candidate phrases for one indexed book."""
+    source = await session.get(EbookSource, source_id)
+    if source is None:
+        raise HTTPException(status_code=404, detail="Book not found")
+
+    try:
+        result = await recalculate_candidate_phrases_for_book(session, source, config)
+    except ValueError as error:
+        raise HTTPException(status_code=409, detail=str(error)) from error
+    logger.info(
+        f"ebook_book_phrase_recalculation_complete {source_id=} {result.candidate_phrases=} "
+        f"{result.deleted_candidates=} {result.deleted_protected_phrases=} {result.deleted_aliases=} "
+        f"{result.deleted_mentions=}"
+    )
+    return RedirectResponse(
+        url=f"/books/{source_id}?phrases_recalculated={result.candidate_phrases}",
+        status_code=303,
+    )
+
+
+@router.post("/books/{source_id}/judge-phrases")
+async def judge_book_phrases(
+    source_id: int,
+    request: Request,
+    background_tasks: BackgroundTasks,
+    session: AsyncDbSession,
+) -> RedirectResponse:
+    """Queue background judging of one book's candidate phrases and return immediately."""
+    source = await session.get(EbookSource, source_id)
+    if source is None:
+        raise HTTPException(status_code=404, detail="Book not found")
+
+    started = start_book_phrase_judgment(request.app, background_tasks, source.id)
+    logger.info(f"ebook_book_phrase_judgment_requested {source_id=} {started=}")
+    return RedirectResponse(url=f"/books/{source_id}", status_code=303)

python/ebook_search/api/routes/search.py

@@ -13,9 +13,9 @@ from fastapi.responses import HTMLResponse
 from python.ebook_search.answer import answer_query
 from python.ebook_search.api.dependencies import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
     AppConfig,
-    AppEngine,
+    AppHttpClient,
 )
-from python.ebook_search.api.web import templates
+from python.ebook_search.api.web import error_response, templates
 from python.ebook_search.guardrails import (
     CitationReport,
     is_confident,
@@ -24,8 +24,11 @@ from python.ebook_search.guardrails import (
 )
 from python.ebook_search.search import SearchResponse, search_ebooks
 from python.ebook_search.timing import runtime_step_from_start
+from python.fastapi_tools import AppAsyncEngine  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
 
 if TYPE_CHECKING:
+    import httpx
+
     from python.ebook_search.config import EbookSearchConfig
 
 logger = logging.getLogger(__name__)
@@ -33,7 +36,8 @@ logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
-def build_answer(
+async def build_answer(
+    client: httpx.AsyncClient,
     query: str,
     response: SearchResponse,
     config: EbookSearchConfig,
@@ -45,9 +49,8 @@ def build_answer(
 
     if not is_confident(response.results, config):
         logger.info(
-            "ebook_answer_low_confidence confidence=%.4f threshold=%.4f",
-            retrieval_confidence(response.results),
-            config.min_retrieval_confidence,
+            f"ebook_answer_low_confidence confidence={retrieval_confidence(response.results):.4f} "
+            f"{config.min_retrieval_confidence=:.4f}"
         )
         answer = (
             "Retrieval confidence is low for this query, so answer generation was skipped. "
@@ -56,40 +59,46 @@ def build_answer(
         return answer, True, None
 
     try:
-        answer = answer_query(query, response.results, config)
+        answer = await answer_query(client, query, response.results, config)
     except RuntimeError as error:
-        logger.warning("ebook_answer_request_failed_falling_back error=%s", error)
+        logger.warning(f"ebook_answer_request_failed_falling_back {error=}")
         return "Answer generation failed. Source chunks are still shown below.", False, None
 
     citation_report = None
     if config.validate_citations_enabled and response.results:
         citation_report = validate_citations(answer, len(response.results))
         if citation_report.invalid or not citation_report.grounded:
-            logger.warning(
-                "ebook_answer_citation_issue invalid=%s grounded=%s",
-                citation_report.invalid,
-                citation_report.grounded,
-            )
+            logger.warning(f"ebook_answer_citation_issue {citation_report.invalid=} {citation_report.grounded=}")
     return answer, False, citation_report
 
 
 @router.post("/search", response_class=HTMLResponse)
-def search(
+async def search(
     request: Request,
     config: AppConfig,
-    engine: AppEngine,
+    engine: AppAsyncEngine,
+    client: AppHttpClient,
     query: Annotated[str, Form()],
-    rerank: Annotated[str | None, Form()] = None,
+    *,
+    rerank: Annotated[bool, Form()] = False,
+    phrase_matching: Annotated[bool, Form()] = False,
 ) -> HTMLResponse:
     """Run a search and render HTMX results."""
     try:
-        response = search_ebooks(engine, query, config, rerank=rerank == "true")
+        response = await search_ebooks(
+            engine,
+            client,
+            query,
+            config,
+            rerank=rerank,
+            phrase_matching=phrase_matching,
+        )
     except Exception as error:
         logger.exception("ebook_search_request_failed")
-        return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
+        return error_response(request, error)
 
     answer_start = perf_counter()
-    answer, low_confidence, citation_report = build_answer(query, response, config)
+    answer, low_confidence, citation_report = await build_answer(client, query, response, config)
     answer_step_name = "Answer generation" if config.answer_enabled else "Answer skipped"
     response = replace(
         response,
@@ -97,12 +106,10 @@ def search(
     )
 
     for step in response.timings:
-        logger.info("ebook_search_timing step=%r runtime_ms=%.1f", step.name, step.duration_ms)
+        logger.info(f"ebook_search_timing {step.name=} {step.duration_ms=:.1f}")
     logger.info(
-        "ebook_search_request_complete results=%s rank_label=%s runtime_ms=%.1f",
-        len(response.results),
-        response.rank_label,
-        response.total_runtime_ms,
+        f"ebook_search_request_complete results={len(response.results)} {response.rank_label=} "
+        f"{response.total_runtime_ms=:.1f}"
     )
     return templates.TemplateResponse(
         request,

python/ebook_search/api/static/style.css

@@ -181,6 +181,12 @@ textarea:focus {
     flex-wrap: wrap;
 }
 
+.search-toggles {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 14px;
+}
+
 button {
     padding: 10px 20px;
     font: inherit;
@@ -212,6 +218,11 @@ button:hover {
     margin-bottom: 24px;
 }
 
+.actions-grid {
+    display: grid;
+    grid-template-columns: repeat(2, max-content);
+}
+
 /* Answer + results */
 #results {
     display: block;
@@ -303,6 +314,28 @@ button:hover {
     font-variant-numeric: tabular-nums;
 }
 
+.phrase-matches {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 8px;
+    align-items: baseline;
+    margin: 10px 0 0;
+    font-size: 0.78rem;
+}
+
+.phrase-matches-label {
+    color: var(--muted);
+    font-weight: 600;
+}
+
+.phrase-match {
+    padding: 3px 10px;
+    background: var(--bg);
+    border: 1px solid var(--border);
+    border-radius: 999px;
+    color: var(--accent);
+}
+
 /* Runtime — developer diagnostics, hidden unless dev mode is on */
 .runtime {
     display: none;

python/ebook_search/api/templates/admin.html

@@ -1,45 +1,103 @@
-{% extends "base.html" %}
-
-{% block title %}EPUB Admin{% endblock %}
-{% block head %}<script src="https://unpkg.com/htmx.org@2.0.4"></script>{% endblock %}
-
-{% block content %}
-    <h1>Admin</h1>
-    <section id="admin-status"></section>
-    <section class="actions">
-        <form hx-post="/admin/scan" hx-target="#admin-status" hx-swap="innerHTML">
-            <button type="submit">Scan</button>
-        </form>
-        <form hx-post="/admin/embed-missing" hx-target="#admin-status" hx-swap="innerHTML">
-            <button type="submit">Embed</button>
-        </form>
-        <form hx-post="/admin/embed-all" hx-target="#admin-status" hx-swap="innerHTML">
-            <button type="submit">Embed all</button>
-        </form>
-    </section>
-    <section>
-        <h2>Embeddings</h2>
-        <table>
-            <thead>
-                <tr>
-                    <th>Model</th>
-                    <th>Dimensions</th>
-                    <th>Embedded</th>
-                    <th>Missing</th>
-                    <th>Total chunks</th>
-                </tr>
-            </thead>
-            <tbody>
-                {% for item in stats %}
-                    <tr>
-                        <td>{{ item.model_name }}</td>
-                        <td>{{ item.dimension }}</td>
-                        <td>{{ item.embedded_chunks }}</td>
-                        <td>{{ item.missing_chunks }}</td>
-                        <td>{{ item.total_chunks }}</td>
-                    </tr>
-                {% endfor %}
-            </tbody>
-        </table>
-    </section>
+{% extends "base.html" %} {% block title %}EPUB Admin{% endblock %} {% block
+head %}
+<script src="https://unpkg.com/htmx.org@2.0.4"></script>
+{% endblock %} {% block content %}
+<h1>Admin</h1>
+<section id="admin-status"></section>
+<section class="actions">
+  <form hx-post="/admin/scan" hx-target="#admin-status" hx-swap="innerHTML">
+    <button type="submit">Scan</button>
+  </form>
+</section>
+<section>
+  <h2>Embeddings</h2>
+  <section class="actions">
+    <form
+      hx-post="/admin/embed-missing"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Embed</button>
+    </form>
+    <form
+      hx-post="/admin/embed-all"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Embed all</button>
+    </form>
+  </section>
+  <table>
+    <thead>
+      <tr>
+        <th>Model</th>
+        <th>Dimensions</th>
+        <th>Embedded</th>
+        <th>Missing</th>
+        <th>Total chunks</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for item in stats %}
+      <tr>
+        <td>{{ item.model_name }}</td>
+        <td>{{ item.dimension }}</td>
+        <td>{{ item.embedded_chunks }}</td>
+        <td>{{ item.missing_chunks }}</td>
+        <td>{{ item.total_chunks }}</td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+</section>
+<section>
+  <h2>Protected phrases</h2>
+  <section class="actions actions-grid">
+    <form
+      hx-post="/admin/phrases/generate-all"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Regenerate all phrases</button>
+    </form>
+    <form
+      hx-post="/admin/phrases/judge-all"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Judge all phrases</button>
+    </form>
+    <form
+      hx-post="/admin/phrases/judge-missing"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Judge missing phrases</button>
+    </form>
+  </section>
+  <table>
+    <thead>
+      <tr>
+        <th>Candidates</th>
+        <th>Judged</th>
+        <th>Unjudged</th>
+        <th>Protected</th>
+        <th>Books indexed</th>
+        <th>Books generated</th>
+        <th>Books fully judged</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+        <td>{{ phrase_stats.candidate_phrases }}</td>
+        <td>{{ phrase_stats.judged_candidates }}</td>
+        <td>{{ phrase_stats.unjudged_candidates }}</td>
+        <td>{{ phrase_stats.protected_phrases }}</td>
+        <td>{{ phrase_stats.total_books }}</td>
+        <td>{{ phrase_stats.books_with_candidates }}</td>
+        <td>{{ phrase_stats.books_fully_judged }}</td>
+      </tr>
+    </tbody>
+  </table>
+</section>
 {% endblock %}

python/ebook_search/api/templates/book_detail.html

@@ -6,6 +6,9 @@
     {% if source %}
         <h1>{{ source.title }}</h1>
         <p class="meta">{{ source.author or "Unknown author" }}</p>
+        {% if phrase_status_message %}
+            <p class="status">{{ phrase_status_message }}</p>
+        {% endif %}
         <dl class="card">
             <dt>File</dt>
             <dd>{{ source.file_path }}</dd>
@@ -13,7 +16,93 @@
             <dd>{{ chapter_count }}</dd>
             <dt>Chunks</dt>
             <dd>{{ chunk_count }}</dd>
+            <dt>Candidates</dt>
+            <dd>{{ candidate_count }}</dd>
+            <dt>Judged</dt>
+            <dd>{{ judged_candidate_count }}</dd>
+            <dt>Protected</dt>
+            <dd>{{ protected_count }}</dd>
         </dl>
+        <form
+            method="post"
+            action="/books/{{ source.id }}/recalculate-phrases"
+            onsubmit="return confirm('Remove old phrases for this book and generate new candidates?');"
+        >
+            <button type="submit">Recalculate phrases</button>
+        </form>
+        <form
+            method="post"
+            action="/books/{{ source.id }}/judge-phrases"
+            onsubmit="return confirm('Judge candidate phrases for this book with the LLM?');"
+        >
+            <button type="submit"{% if judging_in_progress %} disabled{% endif %}>
+                {% if judging_in_progress %}Judging&hellip;{% else %}Judge phrases{% endif %}
+            </button>
+        </form>
+
+        <section>
+            <h2>Candidate n-grams</h2>
+            {% if candidates %}
+                <table>
+                    <thead>
+                        <tr>
+                            <th>Phrase</th>
+                            <th>Status</th>
+                            <th>Score</th>
+                            <th>Count</th>
+                            <th>Chapters</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for candidate in candidates %}
+                            <tr>
+                                <td>{{ candidate.phrase_text }}</td>
+                                <td>
+                                    {% if candidate.llm_judged %}
+                                        {% if candidate.llm_keep %}Kept{% else %}Rejected{% endif %}
+                                    {% else %}
+                                        Candidate
+                                    {% endif %}
+                                </td>
+                                <td>{{ "%.2f"|format(candidate.candidate_score) }}</td>
+                                <td>{{ candidate.raw_count }}</td>
+                                <td>{{ candidate.chapter_count }}</td>
+                            </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+            {% else %}
+                <p>No candidate n-grams.</p>
+            {% endif %}
+        </section>
+
+        <section>
+            <h2>Protected phrases</h2>
+            {% if protected_phrases %}
+                <table>
+                    <thead>
+                        <tr>
+                            <th>Phrase</th>
+                            <th>Type</th>
+                            <th>Confidence</th>
+                            <th>Importance</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for phrase in protected_phrases %}
+                            <tr>
+                                <td>{{ phrase.phrase_text }}</td>
+                                <td>{{ phrase.phrase_type or "phrase" }}</td>
+                                <td>{{ "%.2f"|format(phrase.confidence) }}</td>
+                                <td>{{ "%.2f"|format(phrase.importance) }}</td>
+                            </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+            {% else %}
+                <p>No protected phrases.</p>
+            {% endif %}
+        </section>
     {% else %}
         <h1>Book not found</h1>
     {% endif %}

python/ebook_search/api/templates/partials/results.html

@@ -82,6 +82,14 @@
                         </div>
                     {% endif %}
                 </dl>
+                {% if result.matched_phrases %}
+                    <p class="phrase-matches">
+                        <span class="phrase-matches-label">boosted by</span>
+                        {% for phrase in result.matched_phrases %}
+                            <span class="phrase-match">{{ phrase }}</span>
+                        {% endfor %}
+                    </p>
+                {% endif %}
             </li>
         {% endfor %}
     </ol>

python/ebook_search/api/templates/search.html

@@ -7,12 +7,24 @@
     <h1>Search</h1>
     <form class="card" hx-post="/search" hx-target="#results" hx-swap="innerHTML">
         <label for="query">What are you looking for?</label>
-        <textarea id="query" name="query" rows="4" placeholder="Ask a question or paste a passage…" required></textarea>
+        <textarea id="query" name="query" rows="4" placeholder="Ask a question or paste a passage…" required
+            onkeydown="if (event.key === 'Enter' && !event.shiftKey) { event.preventDefault(); this.form.requestSubmit(); }"></textarea>
         <div class="form-row">
-            <label class="check">
-                <input type="checkbox" name="rerank" value="true" {% if config.rerank.enabled %}checked{% endif %}>
-                Rerank
-            </label>
+            <div class="search-toggles">
+                <label class="check">
+                    <input type="checkbox" name="rerank" value="true" {% if config.rerank.enabled %}checked{% endif %}>
+                    Rerank
+                </label>
+                <label class="check">
+                    <input
+                        type="checkbox"
+                        name="phrase_matching"
+                        value="true"
+                        {% if config.phrase_matching_enabled %}checked{% endif %}
+                    >
+                    Phrase matching
+                </label>
+            </div>
             <button type="submit">Search</button>
         </div>
     </form>

python/ebook_search/api/web.py

@@ -3,9 +3,14 @@
 from __future__ import annotations
 
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from fastapi.templating import Jinja2Templates
 
+if TYPE_CHECKING:
+    from fastapi import Request
+    from fastapi.responses import HTMLResponse
+
 PACKAGE_DIR = Path(__file__).resolve().parent
 TEMPLATE_DIR = PACKAGE_DIR / "templates"
 STATIC_DIR = PACKAGE_DIR / "static"
@@ -21,3 +26,8 @@ def static_version(filename: str) -> int:
 
 templates = Jinja2Templates(directory=TEMPLATE_DIR)
 templates.env.globals["static_version"] = static_version
+
+
+def error_response(request: Request, message: object) -> HTMLResponse:
+    """Render the shared error partial for a failed UI request."""
+    return templates.TemplateResponse(request, "partials/error.html", {"message": str(message)}, status_code=500)

python/ebook_search/bm25_corpus.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import json
 import logging
 import shutil
@@ -14,10 +15,11 @@ from typing import TYPE_CHECKING
 import bm25s
 from sqlalchemy import func, select, union_all
 
+from python.ebook_search.chunk_records import CHUNK_RECORD_COLUMNS
 from python.orm.richie import EbookChapter, EbookChunk, EbookSource
 
 if TYPE_CHECKING:
-    from sqlalchemy.orm import Session
+    from sqlalchemy.ext.asyncio import AsyncSession
 
     from python.ebook_search.config import EbookSearchConfig
 
@@ -73,52 +75,48 @@ def get_current_bm25_index(index_path: Path) -> Path:
     return index_path
 
 
-def ensure_bm25_corpus(session: Session, config: EbookSearchConfig) -> None:
+async def ensure_bm25_corpus(session: AsyncSession, config: EbookSearchConfig) -> None:
     """Create or refresh the persisted BM25 corpus when it is missing or stale."""
     index_path = bm25_index_path(config)
     manifest = read_bm25_manifest(index_path)
-    db_updated_at = corpus_last_updated_at(session)
+    db_updated_at = await corpus_last_updated_at(session)
     if not bm25_index_exists(index_path, manifest):
-        logger.info("ebook_bm25_index_missing path=%s", index_path)
-        refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
+        logger.info(f"ebook_bm25_index_missing {index_path=}")
+        await refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
         return
     if db_updated_at is not None and manifest is not None and manifest.created_at < db_updated_at:
         logger.info(
-            "ebook_bm25_index_stale path=%s created_at=%s db_updated_at=%s",
-            index_path,
-            manifest.created_at.isoformat(),
-            db_updated_at.isoformat(),
+            f"ebook_bm25_index_stale {index_path=} created_at={manifest.created_at.isoformat()} "
+            f"db_updated_at={db_updated_at.isoformat()}"
         )
-        refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
+        await refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
         return
     logger.info(
-        "ebook_bm25_index_current path=%s chunks=%s created_at=%s",
-        index_path,
-        manifest.chunk_count if manifest else 0,
-        manifest.created_at.isoformat() if manifest else None,
+        f"ebook_bm25_index_current {index_path=} chunks={manifest.chunk_count if manifest else 0} "
+        f"created_at={manifest.created_at.isoformat() if manifest else None}"
     )
 
 
-def refresh_bm25_corpus(
-    session: Session,
+async def refresh_bm25_corpus(
+    session: AsyncSession,
     config: EbookSearchConfig,
     *,
     db_updated_at: datetime | None = None,
 ) -> BM25Manifest:
-    """Rebuild and persist the BM25 corpus from the current database chunks."""
+    """Rebuild and persist the BM25 corpus from the current database chunks.
+
+    The index build is CPU and disk work, so it runs in a worker thread.
+    """
     index_path = bm25_index_path(config)
-    records, texts = fetch_bm25_corpus_records(session)
+    records, texts = await fetch_bm25_corpus_records(session)
     manifest = BM25Manifest(
         created_at=datetime.now(tz=UTC),
-        db_updated_at=db_updated_at if db_updated_at is not None else corpus_last_updated_at(session),
+        db_updated_at=db_updated_at if db_updated_at is not None else await corpus_last_updated_at(session),
         chunk_count=len(records),
     )
-    write_bm25_corpus(index_path, records, texts, manifest)
+    await asyncio.to_thread(write_bm25_corpus, index_path, records, texts, manifest)
     logger.info(
-        "ebook_bm25_index_refreshed path=%s chunks=%s created_at=%s",
-        index_path,
-        manifest.chunk_count,
-        manifest.created_at.isoformat(),
+        f"ebook_bm25_index_refreshed {index_path=} {manifest.chunk_count=} created_at={manifest.created_at.isoformat()}"
     )
     return manifest
 
@@ -131,7 +129,7 @@ def load_bm25_corpus(config: EbookSearchConfig) -> BM25Corpus:
     """
     index_path = bm25_index_path(config)
     active_index_path = get_current_bm25_index(index_path)
-    logger.info("ebook_bm25_corpus_cache_load path=%s active_path=%s", index_path, active_index_path)
+    logger.info(f"ebook_bm25_corpus_cache_load {index_path=} {active_index_path=}")
     manifest = read_bm25_manifest(index_path)
     if manifest is None or not bm25_index_exists(index_path, manifest):
         msg = f"BM25 corpus is not available: {index_path}"
@@ -164,7 +162,7 @@ def score_bm25_corpus(query: str, corpus: BM25Corpus, *, limit: int) -> list[tup
     return results
 
 
-def fetch_bm25_corpus_records(session: Session) -> tuple[list[dict[str, object]], list[str]]:
+async def fetch_bm25_corpus_records(session: AsyncSession) -> tuple[list[dict[str, object]], list[str]]:
     """Fetch persistable BM25 corpus records and their matching index texts from the database.
 
     search_text is only needed to build the index, so it is returned separately instead of
@@ -172,12 +170,7 @@ def fetch_bm25_corpus_records(session: Session) -> tuple[list[dict[str, object]]
     """
     statement = (
         select(
-            EbookChunk.id.label("chunk_id"),
-            EbookChunk.text.label("text"),
-            EbookSource.title.label("source_title"),
-            EbookSource.author.label("source_author"),
-            EbookChapter.title.label("chapter_title"),
-            EbookChunk.page_label.label("page_label"),
+            *CHUNK_RECORD_COLUMNS,
             EbookChunk.search_text.label("bm25_text"),
         )
         .select_from(EbookChunk)
@@ -187,21 +180,21 @@ def fetch_bm25_corpus_records(session: Session) -> tuple[list[dict[str, object]]
     )
     records: list[dict[str, object]] = []
     texts: list[str] = []
-    for row in session.execute(statement).mappings():
+    for row in (await session.execute(statement)).mappings():
         record = dict(row)
         texts.append(str(record.pop("bm25_text")))
         records.append(record)
     return records, texts
 
 
-def corpus_last_updated_at(session: Session) -> datetime | None:
+async def corpus_last_updated_at(session: AsyncSession) -> datetime | None:
     """Return the latest source/chapter/chunk update timestamp relevant to BM25 text."""
     update_times = union_all(
         select(func.max(EbookSource.updated).label("updated")),
         select(func.max(EbookChapter.updated).label("updated")),
         select(func.max(EbookChunk.updated).label("updated")),
     ).subquery()
-    return session.scalar(select(func.max(update_times.c.updated)))
+    return await session.scalar(select(func.max(update_times.c.updated)))
 
 
 def write_bm25_corpus(

python/ebook_search/chunk_records.py

@@ -0,0 +1,13 @@
+"""Shared database columns used to build search-result records."""
+
+from python.orm.richie import EbookChapter, EbookChunk, EbookSource
+
+CHUNK_RECORD_COLUMNS = (
+    EbookChunk.id.label("chunk_id"),
+    EbookChunk.text.label("text"),
+    EbookSource.id.label("source_id"),
+    EbookSource.title.label("source_title"),
+    EbookSource.author.label("source_author"),
+    EbookChapter.title.label("chapter_title"),
+    EbookChunk.page_label.label("page_label"),
+)

python/ebook_search/config.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-from os import getenv
 from typing import Annotated, Self
 
 from pydantic import AliasChoices, Field, field_validator, model_validator
@@ -32,11 +31,6 @@ def normalize_embedding_alias(model: str) -> str:
     return standard_model
 
 
-def normalize_embedding_model(default: str = "qwen3-embedding-0.6b") -> str:
-    """Normalize the configured embedding alias to its provider model name."""
-    return normalize_embedding_alias(getenv("EBOOK_SEARCH_EMBEDDING_MODEL", default))
-
-
 class RerankConfig(BaseSettings):
     """vLLM reranker settings."""
 
@@ -86,6 +80,31 @@ class EbookSearchConfig(BaseSettings):
     validate_citations_enabled: bool = True
     bm25_index_dir: str = ".ebook_search_bm25"
     bm25_refresh_delay_seconds: int = 60
+    protected_phrase_max_candidates_per_book: int = 5000
+    protected_phrase_llm_candidates_per_book: int = 500
+    protected_phrase_extraction_workers: int = 16
+    phrase_judge_book_workers: int = 20
+    phrase_judge_phrase_workers: int = 100
+    protected_phrase_confidence_threshold: float = 0.80
+    phrase_matching_enabled: bool = True
+    phrase_hit_boost: float = 0.25
+    phrase_min_tokens: int = 2
+    phrase_max_tokens: int = 5
+    phrase_max_entity_tokens: int = 8
+    phrase_yake_top_k: int = 1000
+    phrase_yake_dedup_limit: float = 0.85
+    phrase_raw_ngram_min_count: int = 2
+    phrase_raw_count_score_threshold: int = 3
+    phrase_raw_count_high_score_threshold: int = 10
+    phrase_chapter_count_score_threshold: int = 2
+    phrase_chapter_count_high_score_threshold: int = 5
+    phrase_target_protected_per_book: int = 100
+    phrase_default_allow_nested: bool = False
+    phrase_default_suppress_children: bool = True
+    phrase_bad_start_score_penalty: float = 10.0
+    phrase_bad_end_score_penalty: float = 10.0
+    phrase_multi_source_score_bonus: float = 2.0
+    phrase_multi_source_min_sources: int = 2
 
     @field_validator("library_paths", mode="before")
     @classmethod

python/ebook_search/docker/containers.py

@@ -32,7 +32,7 @@ def docker_run(
     capture_output: bool = False,
 ) -> subprocess.CompletedProcess[str]:
     """Run docker with repo-root cwd and consistent error handling."""
-    logger.info("docker %s", " ".join(arguments))
+    logger.info(f"docker {' '.join(arguments)}")
     return subprocess.run(
         ["docker", *arguments],
         cwd=get_repo_dir(),

python/ebook_search/docker/pyproject.toml

@@ -16,7 +16,7 @@ dependencies = [
     "pydantic",
     "pydantic-settings",
     "python-multipart",
-    "sqlalchemy",
+    "sqlalchemy[asyncio]",
     "tiktoken",
     "typer",
     "uvicorn[standard]",
@@ -25,7 +25,9 @@ dependencies = [
 
 [dependency-groups]
 dev = [
+    "aiosqlite",
     "pytest",
+    "pytest-asyncio",
     "pytest-mock",
     "pytest-xdist",
 ]
@@ -35,4 +37,5 @@ package = false
 
 [tool.pytest.ini_options]
 addopts = "-n auto -ra"
+asyncio_mode = "auto"
 testpaths = ["tests/ebook_search"]

python/ebook_search/docker/uv.lock

@@ -2,6 +2,15 @@ version = 1
 revision = 3
 requires-python = "==3.14.*"
 
+[[package]]
+name = "aiosqlite"
+version = "0.22.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/4e/8a/64761f4005f17809769d23e518d915db74e6310474e733e3593cfc854ef1/aiosqlite-0.22.1.tar.gz", hash = "sha256:043e0bd78d32888c0a9ca90fc788b38796843360c855a7262a532813133a0650", size = 14821, upload-time = "2025-12-23T19:25:43.997Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/00/b7/e3bf5133d697a08128598c8d0abc5e16377b51465a33756de24fa7dee953/aiosqlite-0.22.1-py3-none-any.whl", hash = "sha256:21c002eb13823fad740196c5a2e9d8e62f6243bd9e7e4a1f87fb5e44ecb4fceb", size = 17405, upload-time = "2025-12-23T19:25:42.139Z" },
+]
+
 [[package]]
 name = "alembic"
 version = "1.18.5"
@@ -153,7 +162,7 @@ dependencies = [
     { name = "pydantic" },
     { name = "pydantic-settings" },
     { name = "python-multipart" },
-    { name = "sqlalchemy" },
+    { name = "sqlalchemy", extra = ["asyncio"] },
     { name = "tiktoken" },
     { name = "typer" },
     { name = "uvicorn", extra = ["standard"] },
@@ -162,7 +171,9 @@ dependencies = [
 
 [package.dev-dependencies]
 dev = [
+    { name = "aiosqlite" },
     { name = "pytest" },
+    { name = "pytest-asyncio" },
     { name = "pytest-mock" },
     { name = "pytest-xdist" },
 ]
@@ -181,7 +192,7 @@ requires-dist = [
     { name = "pydantic" },
     { name = "pydantic-settings" },
     { name = "python-multipart" },
-    { name = "sqlalchemy" },
+    { name = "sqlalchemy", extras = ["asyncio"] },
     { name = "tiktoken" },
     { name = "typer" },
     { name = "uvicorn", extras = ["standard"] },
@@ -190,7 +201,9 @@ requires-dist = [
 
 [package.metadata.requires-dev]
 dev = [
+    { name = "aiosqlite" },
     { name = "pytest" },
+    { name = "pytest-asyncio" },
     { name = "pytest-mock" },
     { name = "pytest-xdist" },
 ]
@@ -242,7 +255,9 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c3/93/43e116ee114b28737ba7e12952a0d4e2f55944d0f84e42bc91ba7192a3c9/greenlet-3.5.3-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:fd2e02fa07485778536a036222d616ab957b1d533f36b3ed98ce725d9c9d3117", size = 288202, upload-time = "2026-06-26T18:23:49.604Z" },
     { url = "https://files.pythonhosted.org/packages/82/2f/146d218299046a43d1f029fd544b3d110d0f175a09c715c7e8da4a4a345d/greenlet-3.5.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df0a0628d1597eb0897b62f55d1343f772405fd25f3b2a796c76874b0c2e22e8", size = 654096, upload-time = "2026-06-26T19:07:12.71Z" },
     { url = "https://files.pythonhosted.org/packages/a0/cc/04738cafb3f45fa991ea44f9de94c47dcec964f5a972300988a6751f49d9/greenlet-3.5.3-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ebd933a6adabc298bab47731a130fe6bfb888bd934eee37810f151159544540d", size = 666304, upload-time = "2026-06-26T19:10:09.503Z" },
+    { url = "https://files.pythonhosted.org/packages/86/a9/73fa62893d5b84b4205544e6b673c654cc43aa5b9899bac00f04d64af73d/greenlet-3.5.3-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8d19fe6c39ebff9259f07bcc685d3290f8fa4ea2278e51dd0008e4d6b0f2d814", size = 670657, upload-time = "2026-06-26T19:24:19.967Z" },
     { url = "https://files.pythonhosted.org/packages/ce/aa/4e0dad5e605c270c784ab911c43da6adb136ccd4d81180f763ca429a723d/greenlet-3.5.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b9d501b40e80b70e32323c799dd9b420a5577a9601469d362ae1ffb690f3a7c", size = 663635, upload-time = "2026-06-26T18:32:20.802Z" },
+    { url = "https://files.pythonhosted.org/packages/29/7e/2ffce64929fb3cab7b65d5a0b20aaf9764e227681d731b041077fc9a525a/greenlet-3.5.3-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:962c5df2db8cb446da51edf1ca5296c389d93b99c9d8aa2ee4c7d0d8f1218260", size = 473497, upload-time = "2026-06-26T19:25:39.421Z" },
     { url = "https://files.pythonhosted.org/packages/d1/50/13efdbea246fe3d3b735e191fec08fb50809f53cd2383ebe123d0809e44b/greenlet-3.5.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a1fad1d11e7d6aab184107baa8e4ece11ccba3ec9599cd7efa5ff4d70d43256a", size = 1621252, upload-time = "2026-06-26T19:09:05.647Z" },
     { url = "https://files.pythonhosted.org/packages/f7/22/c0a336ae4a1410fd5f5121098e5bfbf1865f64c5ef80b4b5412886c4a332/greenlet-3.5.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:fad5aec764399f1b5cc347ad250a59660f20c8f8888ea6bae1f93b769cce1154", size = 1684824, upload-time = "2026-06-26T18:31:47.738Z" },
     { url = "https://files.pythonhosted.org/packages/7a/94/91aec0030bea75c4b3244251d0de60a1f3432d1ecb53ab6c437fb5c3ba61/greenlet-3.5.3-cp314-cp314-win_amd64.whl", hash = "sha256:7669aa24cf2a1041d6f7899575b494a3ab4cf68bfcc8609b1dc0be7272db835e", size = 240754, upload-time = "2026-06-26T18:22:15.669Z" },
@@ -250,7 +265,9 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/91/95/3e161213d7f1d378d15aa9e792093e9bfe01844680d04b7fd6e0107c9098/greenlet-3.5.3-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:271a8ea7c1024e8a0d7dd2be66dd66dda8a07193f41a17b9e924f7600f5b62be", size = 296389, upload-time = "2026-06-26T18:22:20.657Z" },
     { url = "https://files.pythonhosted.org/packages/00/92/715c44721abe2b4d1ae9abde4179411868a5bff312479f54e105d372f131/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19131729ae0ddc3c2e1ef85e650169b5e37ee32e400f215f78b94d7b0d567310", size = 653382, upload-time = "2026-06-26T19:07:14.209Z" },
     { url = "https://files.pythonhosted.org/packages/a0/83/37a10372a1090a6624cca8e74c12df1a36c2dc36429ed0255b7fb1aeee23/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1540dd8e5fc2a5aec40fbb98ef8e149fa47c89a4b4a1cf2575a14d3d1869d7a8", size = 659401, upload-time = "2026-06-26T19:10:10.876Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/73/8faec206b851c22b1733545fda900829a1f3f5b1c78ae7e0fb3dba57d9f4/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b897d97759425953f69a9c0fac67f8fe333ec0ce7377ef186fb2b0c3ad5e354d", size = 659582, upload-time = "2026-06-26T19:24:21.357Z" },
     { url = "https://files.pythonhosted.org/packages/db/e2/d1509cad4207da559cc42986ecdd8fc67ad0d1bba2bf03023c467fd5e0f3/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e81fa194a1d20967877bdf9c7794db2bc99063e5be36aee710c08f04c5bb087f", size = 656969, upload-time = "2026-06-26T18:32:22.272Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/55/50c19e49f8045834ada71ef12f8ad048eba8517c6aa41161bed676328fae/greenlet-3.5.3-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:3236754d423955ea08e9bb5f6c04a7895f9e22c290b66aa7653fcb922d839eb0", size = 491037, upload-time = "2026-06-26T19:25:40.672Z" },
     { url = "https://files.pythonhosted.org/packages/86/7d/eaf70de20aadca3a5884aec58362861c64ce45e7b277f47ed026926a3b89/greenlet-3.5.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:55cf4d777485d43110e47133cbba6d74a8885a87ec1227ef0267f9ee80c5aa21", size = 1617822, upload-time = "2026-06-26T19:09:06.893Z" },
     { url = "https://files.pythonhosted.org/packages/8a/f9/414d38fc400ae4350d4185eaad1827676f7cf5287b9136e0ed1cbbe20a7f/greenlet-3.5.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:12a248ba75f6a9a236375f52296c498c89ff1d8badf32deb9eca7abd5853f7da", size = 1677983, upload-time = "2026-06-26T18:31:49.396Z" },
     { url = "https://files.pythonhosted.org/packages/e4/15/7edb977e08f9bff702fe42d6c902702786ff6b9694058b4e6a2a6ac90e57/greenlet-3.5.3-cp314-cp314t-win_amd64.whl", hash = "sha256:efc6bd60ea02e085862c74a3ef64b147ffc6f1a5ea7d9f26e7a939943f68c1e3", size = 243626, upload-time = "2026-06-26T18:24:41.485Z" },
@@ -665,6 +682,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" },
 ]
 
+[[package]]
+name = "pytest-asyncio"
+version = "1.4.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "pytest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" },
+]
+
 [[package]]
 name = "pytest-mock"
 version = "3.15.1"
@@ -868,6 +897,11 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e2/22/dbf013a12ec759e54a34a119e9e217435b3f71b2dd5c61a7ade0a25dae87/sqlalchemy-2.0.51-py3-none-any.whl", hash = "sha256:bb024d8b621d0be75f4f44ecc7c950450026e76d66dc8f791bb5331d7fed59d5", size = 1944334, upload-time = "2026-06-15T16:09:22.418Z" },
 ]
 
+[package.optional-dependencies]
+asyncio = [
+    { name = "greenlet" },
+]
+
 [[package]]
 name = "starlette"
 version = "1.3.1"

python/ebook_search/embeddings.py

@@ -23,7 +23,8 @@ logger = logging.getLogger(__name__)
 if TYPE_CHECKING:
     from collections.abc import Sequence
 
-    from sqlalchemy.orm import Session
+    import httpx
+    from sqlalchemy.ext.asyncio import AsyncSession
 
     from python.ebook_search.config import EbookSearchConfig
 
@@ -65,51 +66,45 @@ class EmbeddingModelStats:
         return max(self.total_chunks - self.embedded_chunks, 0)
 
 
-def embed_texts(texts: Sequence[str], config: EbookSearchConfig) -> list[list[float]]:
+async def embed_texts(
+    client: httpx.AsyncClient,
+    texts: Sequence[str],
+    config: EbookSearchConfig,
+) -> list[list[float]]:
     """Embed text with the configured vLLM embedding model."""
-    logger.info(
-        "ebook_embed_request_start base_url=%s model=%s count=%s",
-        config.embedding_base_url,
-        config.embedding_model,
-        len(texts),
-    )
-    vectors = request_embeddings(texts, config)
+    logger.info(f"ebook_embed_request_start {config.embedding_base_url=} {config.embedding_model=} count={len(texts)}")
+    vectors = await request_embeddings(client, texts, config)
     expected_dimension = MODEL_DIMENSIONS[config.embedding_model]
     for vector in vectors:
         if len(vector) != expected_dimension:
             msg = f"Expected {expected_dimension} dimensions, got {len(vector)}"
             raise ValueError(msg)
-    logger.info(
-        "ebook_embed_request_complete model=%s count=%s dimension=%s",
-        config.embedding_model,
-        len(vectors),
-        expected_dimension,
-    )
+    logger.info(f"ebook_embed_request_complete {config.embedding_model=} count={len(vectors)} {expected_dimension=}")
     return vectors
 
 
-def embed_query(query: str, config: EbookSearchConfig) -> list[float]:
+async def embed_query(client: httpx.AsyncClient, query: str, config: EbookSearchConfig) -> list[float]:
     """Embed a search query with the Qwen retrieval instruction."""
     instructed_query = f"Instruct: Retrieve relevant passages for the query.\nQuery: {query}"
-    return embed_texts([instructed_query], config)[0]
+    return (await embed_texts(client, [instructed_query], config))[0]
 
 
-def ensure_embedding_models(session: Session) -> None:
+async def ensure_embedding_models(session: AsyncSession) -> None:
     """Ensure supported embedding model rows exist."""
     for name, dimension in MODEL_DIMENSIONS.items():
-        existing = session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == name))
+        existing = await session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == name))
         if existing is None:
             session.add(EbookEmbeddingModel(name=name, dimension=dimension, is_default=name == "qwen3-embedding-0.6b"))
-            logger.info("ebook_embedding_model_created model=%s dimension=%s", name, dimension)
-    session.flush()
+            logger.info(f"ebook_embedding_model_created {name=} {dimension=}")
+    await session.flush()
 
 
-def embedding_model_stats(session: Session) -> list[EmbeddingModelStats]:
+async def embedding_model_stats(session: AsyncSession) -> list[EmbeddingModelStats]:
     """Return embedding coverage counts for every supported model."""
-    total_chunks = session.scalar(select(func.count(EbookChunk.id))) or 0
+    total_chunks = await session.scalar(select(func.count(EbookChunk.id))) or 0
     models = {
         model.name: model
-        for model in session.scalars(
+        for model in await session.scalars(
             select(EbookEmbeddingModel)
             .where(EbookEmbeddingModel.name.in_(MODEL_DIMENSIONS))
             .order_by(EbookEmbeddingModel.name)
@@ -122,7 +117,7 @@ def embedding_model_stats(session: Session) -> list[EmbeddingModelStats]:
         embedded_chunks = 0
         if model is not None:
             table = get_embedding_table(dimension)
-            embedded_chunks = session.scalar(select(func.count(table.id)).where(table.model_id == model.id)) or 0
+            embedded_chunks = await session.scalar(select(func.count(table.id)).where(table.model_id == model.id)) or 0
         stats.append(
             EmbeddingModelStats(
                 model_name=model_name,
@@ -134,10 +129,10 @@ def embedding_model_stats(session: Session) -> list[EmbeddingModelStats]:
     return stats
 
 
-def embed_missing_chunks(session: Session, config: EbookSearchConfig) -> int:
+async def embed_missing_chunks(session: AsyncSession, client: httpx.AsyncClient, config: EbookSearchConfig) -> int:
     """Embed chunks missing embeddings for the configured model."""
-    ensure_embedding_models(session)
-    model = session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == config.embedding_model))
+    await ensure_embedding_models(session)
+    model = await session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == config.embedding_model))
     if model is None:
         supported_models = ", ".join(MODEL_DIMENSIONS)
         msg = f"Unknown embedding model: {config.embedding_model}. Supported models: {supported_models}"
@@ -145,7 +140,7 @@ def embed_missing_chunks(session: Session, config: EbookSearchConfig) -> int:
 
     table = get_embedding_table(model.dimension)
     chunks = list(
-        session.scalars(
+        await session.scalars(
             select(EbookChunk)
             .outerjoin(table, (table.chunk_id == EbookChunk.id) & (table.model_id == model.id))
             .where(table.id.is_(None))
@@ -154,17 +149,17 @@ def embed_missing_chunks(session: Session, config: EbookSearchConfig) -> int:
         )
     )
     if not chunks:
-        logger.info("ebook_embed_missing_none model=%s", config.embedding_model)
+        logger.info(f"ebook_embed_missing_none {config.embedding_model=}")
         return 0
 
-    logger.info("ebook_embed_missing_batch_start model=%s count=%s", config.embedding_model, len(chunks))
-    vectors = embed_texts([chunk.text for chunk in chunks], config)
+    logger.info(f"ebook_embed_missing_batch_start {config.embedding_model=} count={len(chunks)}")
+    vectors = await embed_texts(client, [chunk.text for chunk in chunks], config)
     rows = [
         {"chunk_id": chunk.id, "model_id": model.id, "embedding": vector}
         for chunk, vector in zip(chunks, vectors, strict=True)
     ]
     statement = insert(table).values(rows).on_conflict_do_nothing(index_elements=["chunk_id", "model_id"])
-    session.execute(statement)
-    session.flush()
-    logger.info("ebook_embed_missing_batch_complete model=%s count=%s", config.embedding_model, len(rows))
+    await session.execute(statement)
+    await session.flush()
+    logger.info(f"ebook_embed_missing_batch_complete {config.embedding_model=} count={len(rows)}")
     return len(rows)

python/ebook_search/ingest.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import hashlib
 import logging
 from dataclasses import dataclass
@@ -10,9 +11,11 @@ from pathlib import Path
 from typing import TYPE_CHECKING
 
 import tiktoken
+from anyio import Path as AsyncPath
 from sqlalchemy import or_, select
 
 from python.ebook_search.epub_parse import parse_epub
+from python.ebook_search.protected_phrases.matching import index_chunk_phrase_mentions_for_book
 from python.orm.richie import EbookChapter, EbookChunk, EbookSource
 
 logger = logging.getLogger(__name__)
@@ -20,7 +23,7 @@ DEFAULT_CHUNK_TOKENS = 700
 DEFAULT_CHUNK_OVERLAP = 100
 
 if TYPE_CHECKING:
-    from sqlalchemy.orm import Session
+    from sqlalchemy.ext.asyncio import AsyncSession
 
     from python.ebook_search.config import EbookSearchConfig
     from python.ebook_search.epub_parse import ParsedChapter
@@ -72,90 +75,113 @@ def chunk_text(
     return [chunk for chunk in chunks if chunk.text]
 
 
-def ingest_configured_paths(session: Session, config: EbookSearchConfig) -> int:
+async def find_library_epubs(library_path: str) -> tuple[AsyncPath, list[AsyncPath] | None]:
+    """Resolve one configured library path and collect its EPUB files asynchronously.
+
+    Returns:
+        tuple[Path, list[Path] | None]: The expanded path and its EPUB files, or ``None`` when
+        the path is neither an EPUB file nor a directory.
+    """
+    path = await AsyncPath(library_path).expanduser()
+    if await path.is_file() and path.suffix.lower() == ".epub":
+        return path, [path]
+    if await path.is_dir():
+        return path, sorted([epub_path async for epub_path in path.rglob("*.epub")])
+    return path, None
+
+
+async def ingest_configured_paths(session: AsyncSession, config: EbookSearchConfig) -> int:
     """Ingest every EPUB found under configured library paths."""
     count = 0
     for library_path in config.library_paths:
-        path = Path(library_path).expanduser()
-        logger.info("ebook_ingest_path_start path=%s", path)
-        if path.is_file() and path.suffix.lower() == ".epub":
-            count += int(ingest_file(session, path, config))
-        elif path.is_dir():
-            for epub_path in sorted(path.rglob("*.epub")):
-                count += int(ingest_file(session, epub_path, config))
-        else:
-            logger.warning("ebook_ingest_path_missing path=%s", path)
-    logger.info("ebook_ingest_paths_complete changed_files=%s configured_paths=%s", count, len(config.library_paths))
+        path, epub_paths = await find_library_epubs(library_path)
+        logger.info(f"ebook_ingest_path_start {path=}")
+        if epub_paths is None:
+            logger.warning(f"ebook_ingest_path_missing {path=}")
+            continue
+        for epub_path in epub_paths:
+            count += int(await ingest_file(session, epub_path, config))
+    logger.info(f"ebook_ingest_paths_complete {count=} configured_paths={len(config.library_paths)}")
     return count
 
 
-def ingest_file(session: Session, path: Path, config: EbookSearchConfig) -> bool:
+async def resolve_ingest_path(path: Path | AsyncPath) -> AsyncPath:
+    """Expand and resolve an ingest path without blocking the event loop."""
+    expanded_path = await AsyncPath(path).expanduser()
+    return await expanded_path.resolve()
+
+
+async def ingest_file(session: AsyncSession, path: Path | AsyncPath, config: EbookSearchConfig) -> bool:
     """Ingest one EPUB file. Return True when the database changed."""
-    resolved_path = path.expanduser().resolve()
-    logger.info("ebook_ingest_file_start path=%s", resolved_path)
-    file_hash = sha256_file(resolved_path)
-    existing = find_existing_source(session, resolved_path, file_hash)
-    if existing is not None and existing.file_sha256 == file_hash:
-        stat = resolved_path.stat()
-        existing.file_path = str(resolved_path)
-        existing.file_mtime = datetime.fromtimestamp(stat.st_mtime, tz=UTC)
-        existing.file_size = stat.st_size
-        session.flush()
-        logger.info("ebook_ingest_file_unchanged source_id=%s path=%s", existing.id, resolved_path)
-        return False
-    if existing is not None:
-        logger.info("ebook_ingest_file_replacing source_id=%s path=%s", existing.id, resolved_path)
-        session.delete(existing)
-        session.flush()
+    try:
+        resolved_path = await resolve_ingest_path(path)
+        logger.info(f"ebook_ingest_file_start {resolved_path=}")
+        file_hash = await sha256_file(resolved_path)
+        existing = await find_existing_source(session, resolved_path, file_hash)
+        if existing is not None and existing.file_sha256 == file_hash:
+            stat = await resolved_path.stat()
+            existing.file_path = str(resolved_path)
+            existing.file_mtime = datetime.fromtimestamp(stat.st_mtime, tz=UTC)
+            existing.file_size = stat.st_size
+            await session.flush()
+            logger.info(f"ebook_ingest_file_unchanged {existing.id=} {resolved_path=}")
+            return False
+        if existing is not None:
+            logger.info(f"ebook_ingest_file_replacing {existing.id=} {resolved_path=}")
+            await session.delete(existing)
+            await session.flush()
 
-    stat = resolved_path.stat()
-    parsed = parse_epub(resolved_path)
-    source = EbookSource(
-        title=parsed.title,
-        author=parsed.author,
-        language=parsed.language,
-        publisher=parsed.publisher,
-        identifier=parsed.identifier,
-        file_path=str(resolved_path),
-        file_sha256=file_hash,
-        file_mtime=datetime.fromtimestamp(stat.st_mtime, tz=UTC),
-        file_size=stat.st_size,
-    )
-    session.add(source)
-    session.flush()
-
-    chunk_index = 0
-    for spine_index, parsed_chapter in enumerate(parsed.chapters):
-        chapter = EbookChapter(
-            source_id=source.id,
-            spine_index=spine_index,
-            title=parsed_chapter.title,
-            href=parsed_chapter.href,
+        stat = await resolved_path.stat()
+        parsed = await asyncio.to_thread(parse_epub, Path(resolved_path))
+        source = EbookSource(
+            title=parsed.title,
+            author=parsed.author,
+            language=parsed.language,
+            publisher=parsed.publisher,
+            identifier=parsed.identifier,
+            file_path=str(resolved_path),
+            file_sha256=file_hash,
+            file_mtime=datetime.fromtimestamp(stat.st_mtime, tz=UTC),
+            file_size=stat.st_size,
         )
-        session.add(chapter)
-        session.flush()
-        chunk_index = add_chapter_chunks(session, source, chapter, parsed_chapter, chunk_index, config)
+        session.add(source)
+        await session.flush()
 
-    session.flush()
-    logger.info(
-        "ebook_ingest_file_complete source_id=%s path=%s chapters=%s chunks=%s",
-        source.id,
-        resolved_path,
-        len(parsed.chapters),
-        chunk_index,
-    )
-    return True
+        chunk_index = 0
+        for spine_index, parsed_chapter in enumerate(parsed.chapters):
+            chapter = EbookChapter(
+                source_id=source.id,
+                spine_index=spine_index,
+                title=parsed_chapter.title,
+                href=parsed_chapter.href,
+            )
+            session.add(chapter)
+            await session.flush()
+            chunk_index = add_chapter_chunks(session, source, chapter, parsed_chapter, chunk_index, config)
+
+        mention_count = await index_chunk_phrase_mentions_for_book(session, source.id, config)
+        await session.commit()
+        logger.info(
+            f"ebook_ingest_file_complete {source.id=} {resolved_path=} chapters={len(parsed.chapters)} {chunk_index=} "
+            f"{mention_count=}"
+        )
+    except Exception:
+        await session.rollback()
+        logger.exception(f"ebook_ingest_file_error {path=}")
+        return False
+    else:
+        return True
 
 
-def find_existing_source(session: Session, path: Path, file_hash: str) -> EbookSource | None:
+async def find_existing_source(session: AsyncSession, path: Path | AsyncPath, file_hash: str) -> EbookSource | None:
     """Find an existing source by canonical path or file hash."""
-    return session.scalar(
+    return await session.scalar(
         select(EbookSource).where(or_(EbookSource.file_path == str(path), EbookSource.file_sha256 == file_hash))
     )
 
 
 def add_chapter_chunks(
-    session: Session,
+    session: AsyncSession,
     source: EbookSource,
     chapter: EbookChapter,
     parsed_chapter: ParsedChapter,
@@ -186,10 +212,10 @@ def add_chapter_chunks(
     return chunk_index
 
 
-def sha256_file(path: Path) -> str:
-    """Calculate the SHA-256 digest for a file."""
+async def sha256_file(path: AsyncPath) -> str:
+    """Calculate the SHA-256 digest for a file without blocking the event loop."""
     digest = hashlib.sha256()
-    with path.open("rb") as file:
-        for block in iter(lambda: file.read(1024 * 1024), b""):
+    async with await path.open("rb") as file:
+        while block := await file.read(1024 * 1024):
             digest.update(block)
     return digest.hexdigest()

python/ebook_search/llm_interface.py

@@ -22,10 +22,26 @@ def auth_headers(api_key: str) -> dict[str, str]:
     return {"Authorization": f"Bearer {api_key}"}
 
 
-def request_embeddings(texts: Sequence[str], config: EbookSearchConfig) -> list[list[float]]:
-    """Request embeddings from the configured OpenAI-compatible endpoint."""
+async def request_embeddings(
+    client: httpx.AsyncClient,
+    texts: Sequence[str],
+    config: EbookSearchConfig,
+) -> list[list[float]]:
+    """Request embeddings from the configured OpenAI-compatible endpoint.
+
+    Args:
+        client (httpx.AsyncClient): Shared async client for LLM calls.
+        texts (Sequence[str]): Texts to embed.
+        config (EbookSearchConfig): Runtime settings supplying the endpoint, model, and auth.
+
+    Returns:
+        list[list[float]]: One embedding vector per input text.
+
+    Raises:
+        RuntimeError: If the request fails or the response cannot be parsed.
+    """
     try:
-        response = httpx.post(
+        response = await client.post(
             f"{config.embedding_base_url.rstrip('/')}/embeddings",
             headers=auth_headers(config.embedding_api_key),
             json={"model": config.embedding_model, "input": list(texts)},
@@ -35,41 +51,62 @@ def request_embeddings(texts: Sequence[str], config: EbookSearchConfig) -> list[
         return embedding_vectors_from_response(response.json())
     except (httpx.HTTPError, ValueError, KeyError, TypeError) as error:
         logger.exception(
-            "ebook_embed_request_failed base_url=%s model=%s count=%s",
-            config.embedding_base_url,
-            config.embedding_model,
-            len(texts),
+            f"ebook_embed_request_failed {config.embedding_base_url=} {config.embedding_model=} count={len(texts)}"
         )
         msg = f"Embedding request failed. base_url={config.embedding_base_url} model={config.embedding_model}"
         raise RuntimeError(msg) from error
 
 
-def check_embedding_endpoint(config: EbookSearchConfig, *, timeout_seconds: float = 5.0) -> bool:
+async def check_embedding_endpoint(
+    client: httpx.AsyncClient,
+    config: EbookSearchConfig,
+    *,
+    timeout_seconds: float = 5.0,
+) -> bool:
     """Return whether the configured embedding endpoint answers a model listing."""
-    try:
-        response = httpx.get(
-            f"{config.embedding_base_url.rstrip('/')}/models",
-            headers=auth_headers(config.embedding_api_key),
-            timeout=timeout_seconds,
-        )
-        response.raise_for_status()
-    except httpx.HTTPError as error:
-        logger.warning("ebook_embedding_endpoint_unreachable base_url=%s error=%s", config.embedding_base_url, error)
-        return False
-    return True
+    return await _check_endpoint(
+        client,
+        base_url=config.embedding_base_url,
+        api_key=config.embedding_api_key,
+        timeout_seconds=timeout_seconds,
+        unavailable_log=f"ebook_embedding_endpoint_unreachable {config.embedding_base_url=}",
+    )
 
 
-def check_chat_endpoint(config: EbookSearchConfig, *, timeout_seconds: float = 5.0) -> bool:
+async def check_chat_endpoint(
+    client: httpx.AsyncClient,
+    config: EbookSearchConfig,
+    *,
+    timeout_seconds: float = 5.0,
+) -> bool:
     """Return whether the configured chat (answering) endpoint answers a model listing."""
+    return await _check_endpoint(
+        client,
+        base_url=config.vllm_base_url,
+        api_key=config.vllm_api_key,
+        timeout_seconds=timeout_seconds,
+        unavailable_log=f"ebook_chat_endpoint_unreachable {config.vllm_base_url=}",
+    )
+
+
+async def _check_endpoint(
+    client: httpx.AsyncClient,
+    *,
+    base_url: str,
+    api_key: str,
+    timeout_seconds: float,
+    unavailable_log: str,
+) -> bool:
+    """Return whether an OpenAI-compatible endpoint answers a model listing."""
     try:
-        response = httpx.get(
-            f"{config.vllm_base_url.rstrip('/')}/models",
-            headers=auth_headers(config.vllm_api_key),
+        response = await client.get(
+            f"{base_url.rstrip('/')}/models",
+            headers=auth_headers(api_key),
             timeout=timeout_seconds,
         )
         response.raise_for_status()
     except httpx.HTTPError as error:
-        logger.warning("ebook_chat_endpoint_unreachable base_url=%s error=%s", config.vllm_base_url, error)
+        logger.warning(f"{unavailable_log} {error=}")
         return False
     return True
 
@@ -98,18 +135,29 @@ def embedding_vectors_from_response(body: object) -> list[list[float]]:
     return vectors
 
 
-def request_rerank(
+async def request_rerank(
+    client: httpx.AsyncClient,
     query: str,
     documents: Sequence[str],
     config: RerankConfig,
 ) -> object | None:
-    """Request rerank scores from the configured vLLM endpoint."""
+    """Request rerank scores from the configured vLLM endpoint.
+
+    Args:
+        client (httpx.AsyncClient): Shared async client for LLM calls.
+        query (str): Query the documents are scored against.
+        documents (Sequence[str]): Candidate documents to score.
+        config (RerankConfig): Rerank endpoint settings.
+
+    Returns:
+        object | None: The decoded response body, or ``None`` when it is not valid JSON.
+    """
     payload = {
         "model": config.model,
         "query": query,
         "documents": list(documents),
     }
-    response = httpx.post(
+    response = await client.post(
         f"{config.base_url.rstrip('/')}/rerank",
         json=payload,
         timeout=config.timeout_seconds,
@@ -122,20 +170,33 @@ def request_rerank(
         return None
 
 
-def request_chat_completion(
+async def request_chat_completion(
+    client: httpx.AsyncClient,
     config: EbookSearchConfig,
     messages: Sequence[dict[str, str]],
+    *,
+    response_format: dict[str, object] | None = None,
 ) -> str:
-    """Request a chat completion from the configured OpenAI-compatible endpoint."""
+    """Request a chat completion over a shared async client.
+
+    Args:
+        client (httpx.AsyncClient): Shared async client whose connection pool bounds concurrency.
+        config (EbookSearchConfig): Runtime settings supplying the endpoint, model, and auth.
+        messages (Sequence[dict[str, str]]): OpenAI-style chat messages.
+        response_format (dict[str, object] | None): Optional OpenAI-compatible structured output constraint.
+
+    Returns:
+        str: The assistant message text.
+
+    Raises:
+        RuntimeError: If the request fails or the response cannot be parsed.
+    """
     try:
-        response = httpx.post(
+        response = await client.post(
             f"{config.vllm_base_url.rstrip('/')}/chat/completions",
             headers=auth_headers(config.vllm_api_key),
-            json={
-                "model": config.chat_model,
-                "messages": list(messages),
-                "temperature": 0,
-            },
+            json={"model": config.chat_model, "messages": list(messages), "temperature": 0}
+            | ({"response_format": response_format} if response_format is not None else {}),
             timeout=config.chat_timeout_seconds,
         )
         response.raise_for_status()

python/ebook_search/loadtest.py

@@ -112,7 +112,7 @@ async def send_search(client: httpx.AsyncClient, query: str, *, rerank: bool) ->
     try:
         response = await client.post("/search", data=data)
     except httpx.HTTPError as error:
-        logger.warning("ebook_loadtest_request_failed error=%s", error)
+        logger.warning(f"ebook_loadtest_request_failed {error=}")
         return RequestResult(status_code=0, latency_ms=(time.perf_counter() - start) * 1000, ok=False)
     return RequestResult(
         status_code=response.status_code,
@@ -192,14 +192,7 @@ def main(
     """Load test the search endpoint and report latency and throughput."""
     configure_logger(log_level)
     queries = load_queries(queries_file)
-    logger.info(
-        "ebook_loadtest_start base_url=%s requests=%s concurrency=%s rerank=%s queries=%s",
-        base_url,
-        request_count,
-        concurrency,
-        rerank,
-        len(queries),
-    )
+    logger.info(f"ebook_loadtest_start {base_url=} {request_count=} {concurrency=} {rerank=} queries={len(queries)}")
     summary = asyncio.run(
         run_load(
             base_url=base_url,

python/ebook_search/prompts/__init__.py

@@ -0,0 +1,8 @@
+"""LLM prompt templates for EPUB search."""
+
+from python.ebook_search.prompts.lib import (
+    Prompt,
+    load_prompt,
+)
+
+__all__ = ["Prompt", "load_prompt"]

python/ebook_search/prompts/answer.toml

@@ -0,0 +1,9 @@
+system = """\
+Answer only from the provided context. Cite sources with bracketed numbers like [1]. \
+If the context is insufficient, say so."""
+user = """\
+Question:
+{query}
+
+Context:
+{context}"""

python/ebook_search/prompts/lib.py

@@ -0,0 +1,44 @@
+"""Load and render TOML-backed LLM prompt templates."""
+
+from __future__ import annotations
+
+import tomllib
+from dataclasses import dataclass
+from functools import cache
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class Prompt:
+    """A system and user prompt pair loaded from TOML."""
+
+    system: str
+    user: str
+
+    def messages(self, **values: str) -> list[dict[str, str]]:
+        """Render this prompt as OpenAI-style chat messages."""
+        return [
+            {"role": "system", "content": self.system.format(**values)},
+            {"role": "user", "content": self.user.format(**values)},
+        ]
+
+
+@cache
+def _get_prompt_dir() -> Path:
+    """Return the directory containing prompt template files."""
+    return Path(__file__).resolve().parent
+
+
+@cache
+def load_prompt(name: str) -> Prompt:
+    """Load and validate a named system and user prompt pair from TOML."""
+    path = _get_prompt_dir() / f"{name}.toml"
+    with path.open("rb") as file:
+        body = tomllib.load(file)
+
+    system = body.get("system")
+    user = body.get("user")
+    if not isinstance(system, str) or not isinstance(user, str):
+        msg = f"{path} must define string system and user prompts"
+        raise TypeError(msg)
+    return Prompt(system=system, user=user)

python/ebook_search/prompts/phrase_judge.toml

@@ -0,0 +1,10 @@
+system = """\
+Judge whether a candidate phrase from a book should be protected for RAG retrieval. \
+Do not extract new phrases. Reject common grammar fragments, ordinary nonspecific \
+phrases, unstable fragments, and phrases kept only because they are frequent. Keep \
+people, places, organizations, factions, events, technologies, fictional conditions, \
+magic systems, formal titles, named concepts, and recurring world-specific terms. \
+Return only a JSON object with keys: keep, canonical, category, aliases, confidence, \
+importance, allow_nested, suppress_children, reason."""
+
+user = "{candidate_json}"

python/ebook_search/protected_phrases/__init__.py

@@ -0,0 +1 @@
+"""Protected phrase extraction and matching for ebook search."""

python/ebook_search/protected_phrases/config/__init__.py

@@ -0,0 +1,17 @@
+"""Protected phrase extraction, storage, and runtime matching."""
+
+from python.ebook_search.protected_phrases.config.lib import (
+    get_bad_ends,
+    get_bad_starts,
+    get_ignored_phrases,
+    get_junk_tokens,
+    get_most_common_words,
+)
+
+__all__ = [
+    "get_bad_ends",
+    "get_bad_starts",
+    "get_ignored_phrases",
+    "get_junk_tokens",
+    "get_most_common_words",
+]

python/ebook_search/protected_phrases/config/bad_ends.toml

@@ -0,0 +1,31 @@
+tokens = [
+  "a",
+  "an",
+  "and",
+  "any",
+  "as",
+  "at",
+  "be",
+  "because",
+  "but",
+  "by",
+  "can",
+  "could",
+  "do",
+  "for",
+  "from",
+  "have",
+  "if",
+  "of",
+  "or",
+  "some",
+  "than",
+  "the",
+  "these",
+  "this",
+  "to",
+  "will",
+  "with",
+  "would",
+  "did",
+]

python/ebook_search/protected_phrases/config/bad_starts.toml

@@ -0,0 +1,27 @@
+tokens = [
+    "a",
+    "an",
+    "did",
+    "didn't",
+    "he",
+    "here",
+    "how",
+    "i",
+    "it",
+    "she",
+    "that",
+    "the",
+    "there",
+    "they",
+    "this",
+    "we",
+    "what",
+    "when",
+    "where",
+    "which",
+    "who",
+    "whom",
+    "whose",
+    "why",
+    "you",
+]

python/ebook_search/protected_phrases/config/ignored_phrases.toml

@@ -0,0 +1,212 @@
+phrases = [
+  "a little",
+  "across the",
+  "and she",
+  "anyone in",
+  "are you",
+  "around him",
+  "around the",
+  "as much",
+  "as soon",
+  "at all",
+  "at least",
+  "before the",
+  "behind him",
+  "between the",
+  "but she",
+  "could not",
+  "did he",
+  "did i",
+  "did it",
+  "did not believe",
+  "did not care",
+  "did not even",
+  "did not know what",
+  "did not know",
+  "did not like",
+  "did not look",
+  "did not mean",
+  "did not move",
+  "did not need",
+  "did not see",
+  "did not seem",
+  "did not think",
+  "did not understand",
+  "did not want",
+  "did not",
+  "did she",
+  "did so",
+  "did that",
+  "did the",
+  "did they",
+  "did what",
+  "did you",
+  "didn't answer",
+  "didn't care",
+  "didn't even",
+  "didn't expect",
+  "didn't feel",
+  "didn't get",
+  "didn't i",
+  "didn't know",
+  "didn't like",
+  "didn't look",
+  "didn't make",
+  "didn't mean",
+  "didn't need",
+  "didn't really",
+  "didn't say",
+  "didn't see",
+  "didn't seem",
+  "didn't think",
+  "didn't want",
+  "didn't you",
+  "end up",
+  "ended up",
+  "had a",
+  "had been",
+  "have been",
+  "he asked",
+  "he concluded",
+  "he continued",
+  "he couldn't",
+  "he did",
+  "he didn't",
+  "he felt",
+  "he had",
+  "he hadn't",
+  "he knew",
+  "he noted",
+  "he pointed",
+  "he realized",
+  "he replied",
+  "he said",
+  "he saw",
+  "he tapped",
+  "he told",
+  "he was",
+  "he wasn't",
+  "his body",
+  "his chair",
+  "his feet",
+  "his hands",
+  "his head",
+  "his office",
+  "his own",
+  "his pc",
+  "his power",
+  "his shield",
+  "his sight",
+  "his voice",
+  "his wrist",
+  "how many",
+  "i am",
+  "i don't",
+  "i said",
+  "i was",
+  "i wouldn't",
+  "i'm not",
+  "if he",
+  "if they",
+  "is in",
+  "is not",
+  "is that",
+  "is the",
+  "it had",
+  "it had",
+  "it is",
+  "it was",
+  "it wasn't",
+  "it wasn't",
+  "no one",
+  "of course",
+  "of force",
+  "of it",
+  "of magic",
+  "of marines",
+  "of power",
+  "of those",
+  "old man",
+  "older man",
+  "one of",
+  "out of",
+  "set up",
+  "she admitted",
+  "she asked",
+  "she had",
+  "she replied",
+  "she said",
+  "she snapped",
+  "she told",
+  "she was",
+  "she'd been",
+  "shook his",
+  "sure he",
+  "tell you",
+  "that had",
+  "that is",
+  "that she",
+  "that was",
+  "the dark",
+  "the door",
+  "the first",
+  "the last",
+  "the man",
+  "the one",
+  "the only",
+  "the other",
+  "the rest",
+  "the room",
+  "the same",
+  "the two",
+  "the way",
+  "the world",
+  "there are",
+  "there was",
+  "there were",
+  "they are",
+  "they had",
+  "they were",
+  "they weren't",
+  "this is",
+  "this place",
+  "though he",
+  "through his",
+  "through the",
+  "to find",
+  "to get",
+  "to keep",
+  "to stay",
+  "to stop",
+  "to tell",
+  "to try",
+  "told her",
+  "told him",
+  "under his",
+  "was a",
+  "was enough",
+  "was going",
+  "was in",
+  "was no",
+  "was not",
+  "was now",
+  "was on",
+  "was one",
+  "was only",
+  "was still",
+  "was that",
+  "was the",
+  "was there",
+  "were in",
+  "what had",
+  "what happened",
+  "what was",
+  "where the",
+  "while i",
+  "you are",
+  "you can't",
+  "you don't",
+  "you know",
+  "you need",
+  "you were",
+]

python/ebook_search/protected_phrases/config/junk_tokens.toml

@@ -0,0 +1,71 @@
+tokens = [
+    "said",
+    "asked",
+    "replied",
+    "answered",
+    "looked",
+    "nodded",
+    "turned",
+    "shook",
+    "smiled",
+    "shrugged",
+    "pointed",
+    "continued",
+    "repeated",
+    "stared",
+    "agreed",
+    "glanced",
+    "walked",
+    "told",
+    "thought",
+    "knew",
+    "wanted",
+    "muttered",
+    "whispered",
+    "laughed",
+    "sighed",
+    "paused",
+    "gestured",
+    "waved",
+    "frowned",
+    "grinned",
+    "admitted",
+    "found",
+    "noted",
+    "murmured",
+    "ordered",
+    "i'm",
+    "i've",
+    "i'd",
+    "i'll",
+    "it's",
+    "that's",
+    "don't",
+    "didn't",
+    "doesn't",
+    "can't",
+    "won't",
+    "wouldn't",
+    "couldn't",
+    "shouldn't",
+    "isn't",
+    "wasn't",
+    "aren't",
+    "weren't",
+    "you're",
+    "you've",
+    "you'll",
+    "we're",
+    "we've",
+    "we'll",
+    "they're",
+    "they've",
+    "he's",
+    "she's",
+    "there's",
+    "what's",
+    "let's",
+    "who's",
+    "he'd",
+    "she'd",
+]

python/ebook_search/protected_phrases/config/lib.py

@@ -0,0 +1,60 @@
+"""Protected phrase extraction, storage, and runtime matching."""
+
+from __future__ import annotations
+
+import logging
+import tomllib
+from functools import cache
+from pathlib import Path
+
+from python.ebook_search.protected_phrases.text_normalization import normalize_text
+
+logger = logging.getLogger(__name__)
+
+
+def _load_toml_string_set(path: Path, key: str) -> frozenset[str]:
+    """Load and validate a TOML string list as a normalized immutable set."""
+    with path.open("rb") as file:
+        body = tomllib.load(file)
+
+    values = body.get(key)
+    if not isinstance(values, list) or not all(isinstance(item, str) for item in values):
+        msg = f"{path} must contain a {key!r} string list"
+        raise ValueError(msg)
+    return frozenset(normalize_text(value) for value in values if normalize_text(value))
+
+
+@cache
+def _get_phrase_config_dir() -> Path:
+    """Return the directory containing phrase configuration files."""
+    return Path(__file__).resolve().parent
+
+
+@cache
+def get_ignored_phrases() -> frozenset[str]:
+    """Return ignored phrase strings loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "ignored_phrases.toml", "phrases")
+
+
+@cache
+def get_bad_ends() -> frozenset[str]:
+    """Return bad phrase-ending tokens loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "bad_ends.toml", "tokens")
+
+
+@cache
+def get_bad_starts() -> frozenset[str]:
+    """Return bad phrase-starting tokens loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "bad_starts.toml", "tokens")
+
+
+@cache
+def get_most_common_words() -> frozenset[str]:
+    """Return the most common English words loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "most_common_words.toml", "words")
+
+
+@cache
+def get_junk_tokens() -> frozenset[str]:
+    """Return junk tokens (dialogue verbs and pronoun contractions) loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "junk_tokens.toml", "tokens")

python/ebook_search/protected_phrases/config/most_common_words.toml

@@ -0,0 +1,102 @@
+words = [
+    "the",
+    "be",
+    "to",
+    "of",
+    "and",
+    "a",
+    "in",
+    "that",
+    "have",
+    "I",
+    "it",
+    "for",
+    "not",
+    "on",
+    "with",
+    "he",
+    "as",
+    "you",
+    "do",
+    "at",
+    "this",
+    "but",
+    "his",
+    "by",
+    "from",
+    "they",
+    "we",
+    "say",
+    "her",
+    "she",
+    "or",
+    "an",
+    "will",
+    "my",
+    "one",
+    "all",
+    "would",
+    "there",
+    "their",
+    "what",
+    "so",
+    "up",
+    "out",
+    "if",
+    "about",
+    "who",
+    "get",
+    "which",
+    "go",
+    "me",
+    "when",
+    "make",
+    "can",
+    "like",
+    "time",
+    "no",
+    "just",
+    "him",
+    "know",
+    "take",
+    "people",
+    "into",
+    "year",
+    "your",
+    "good",
+    "some",
+    "could",
+    "them",
+    "see",
+    "other",
+    "than",
+    "then",
+    "now",
+    "look",
+    "only",
+    "come",
+    "its",
+    "over",
+    "think",
+    "also",
+    "back",
+    "after",
+    "use",
+    "two",
+    "how",
+    "our",
+    "work",
+    "first",
+    "well",
+    "way",
+    "even",
+    "new",
+    "want",
+    "because",
+    "any",
+    "these",
+    "give",
+    "day",
+    "most",
+    "us",
+]

python/ebook_search/protected_phrases/extraction.py

@@ -0,0 +1,724 @@
+"""Candidate phrase extraction and scoring for protected phrases."""
+
+from __future__ import annotations
+
+import logging
+import re
+from collections import Counter, defaultdict
+from functools import lru_cache
+from time import perf_counter
+from typing import TYPE_CHECKING, Protocol
+
+from yake import KeywordExtractor
+
+from python.ebook_search.protected_phrases.config import (
+    get_bad_ends,
+    get_bad_starts,
+    get_ignored_phrases,
+    get_junk_tokens,
+    get_most_common_words,
+)
+from python.ebook_search.protected_phrases.models import PhraseCandidate
+from python.ebook_search.protected_phrases.text_normalization import tokenize, tokenize_with_offsets
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Mapping, Sequence
+
+    from python.ebook_search.config import EbookSearchConfig
+
+logger = logging.getLogger(__name__)
+
+
+CAPITALIZED_PHRASE_RE = re.compile(r"\b(?:[A-Z][a-zA-Z']+)(?:\s+(?:of|the|and|in|on|for|[A-Z][a-zA-Z']+)){0,6}")
+
+
+class YakeExtractor(Protocol):
+    """Small protocol for the YAKE extractor used by this module."""
+
+    def extract_keywords(self, text: str) -> Iterable[tuple[str, float]]:
+        """Return YAKE keyword tuples."""
+
+
+class YakeExtractorFactory(Protocol):
+    """Callable constructor protocol for YAKE keyword extractors."""
+
+    def __call__(self, *, lan: str, n: int, dedupLim: float, top: int) -> YakeExtractor:  # noqa: N803
+        """Create a YAKE keyword extractor.
+
+        Args:
+            lan (str): Language code passed to YAKE.
+            n (int): Maximum n-gram size to extract.
+            dedupLim (float): Deduplication similarity threshold.
+            top (int): Maximum number of keyphrases to return.
+
+        Returns:
+            YakeExtractor: The constructed keyword extractor.
+        """
+
+
+def normalize_candidate_phrase(
+    phrase_text: str,
+    config: EbookSearchConfig,
+    *,
+    max_tokens: int | None = None,
+) -> tuple[str, str, int] | None:
+    """Normalize a candidate phrase and validate token bounds.
+
+    Args:
+        phrase_text (str): Raw phrase text to normalize.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        max_tokens (int | None): Maximum token count override; defaults to ``config.phrase_max_tokens``.
+
+    Returns:
+        tuple[str, str, int] | None: Display text, normalized phrase, and token count, or ``None``
+        when the phrase falls outside the token bounds or is ignored.
+    """
+    normalized_tokens = tokenize_with_offsets(phrase_text)
+    max_count = config.phrase_max_tokens if max_tokens is None else max_tokens
+    if len(normalized_tokens) < config.phrase_min_tokens or len(normalized_tokens) > max_count:
+        return None
+
+    phrase_norm = " ".join(token.text for token in normalized_tokens)
+    if phrase_norm in get_ignored_phrases():
+        return None
+
+    display_text = phrase_text[normalized_tokens[0].start_char : normalized_tokens[-1].end_char].strip()
+    return display_text or phrase_norm, phrase_norm, len(normalized_tokens)
+
+
+def count_raw_ngrams(tokens: Sequence[str], config: EbookSearchConfig) -> Counter[str]:
+    """Count every n-gram window in one normalized token block.
+
+    ``tokens`` are already normalized (see :func:`tokenize`), so each window's normalized form
+    is the joined tokens directly. Counting into a plain :class:`Counter` rather than
+    :class:`PhraseCandidate` objects keeps this hot loop cheap; callers filter ignored phrases
+    and materialize candidates per unique phrase afterwards, which is far fewer operations than
+    doing either per window.
+
+    Args:
+        tokens (Sequence[str]): Normalized tokens for one text block.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        Counter[str]: Raw occurrence counts keyed by normalized phrase.
+    """
+    return Counter(
+        " ".join(tokens[start : start + ngram_size])
+        for ngram_size in range(config.phrase_min_tokens, config.phrase_max_tokens + 1)
+        for start in range(len(tokens) - ngram_size + 1)
+    )
+
+
+def extract_raw_ngrams_by_chapter(
+    chapters: Sequence[str],
+    config: EbookSearchConfig,
+) -> dict[str, PhraseCandidate]:
+    """Extract raw n-grams across chapters, tracking both raw counts and chapter spread.
+
+    Counting each chapter separately makes chapter spread fall out of dict membership: a phrase's
+    ``chapter_count`` is simply how many per-chapter count maps contain it, so no per-window seen
+    tracking is needed. This also lets the enrichment step skip re-sliding the same n-gram sizes.
+
+    Phrases below the minimum raw count are dropped here rather than materialized: most unique
+    n-grams occur once, and :func:`filter_storable_candidates` would discard them as too rare
+    anyway, so building ``PhraseCandidate`` objects for them is wasted work.
+
+    Args:
+        chapters (Sequence[str]): Chapter-like text blocks to slide n-gram windows over.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        dict[str, PhraseCandidate]: Candidates meeting the minimum raw count, keyed by normalized
+        phrase, with raw and chapter counts.
+    """
+    chapter_count_maps = [count_raw_ngrams(tokenize(chapter), config) for chapter in chapters]
+    total_counts: Counter[str] = Counter()
+    chapter_spread: Counter[str] = Counter()
+    for chapter_counts in chapter_count_maps:
+        total_counts.update(chapter_counts)
+        chapter_spread.update(chapter_counts.keys())
+    min_raw_count = minimum_candidate_raw_count(config)
+    ignored = get_ignored_phrases()
+    return {
+        phrase_norm: PhraseCandidate(
+            phrase_text=phrase_norm,
+            phrase_norm=phrase_norm,
+            token_count=phrase_norm.count(" ") + 1,
+            source_raw_ngram=True,
+            raw_count=raw_count,
+            chapter_count=chapter_spread[phrase_norm],
+        )
+        for phrase_norm, raw_count in total_counts.items()
+        if raw_count >= min_raw_count and phrase_norm not in ignored
+    }
+
+
+@lru_cache(maxsize=2)
+def get_yake_extractor(max_ngram: int, top_k: int, dedup_limit: float) -> KeywordExtractor:
+    """Return a cached YAKE extractor for the given settings.
+
+    Constructing a ``KeywordExtractor`` loads the language's stopword list from disk, so it is
+    cached and reused across books rather than rebuilt on every call.
+
+    Args:
+        max_ngram (int): Maximum n-gram size to extract.
+        top_k (int): Maximum number of keyphrases to request.
+        dedup_limit (float): Deduplication similarity threshold.
+
+    Returns:
+        KeywordExtractor: A shared extractor instance for the given settings.
+    """
+    return KeywordExtractor(lan="en", n=max_ngram, dedupLim=dedup_limit, top=top_k)
+
+
+def extract_yake_candidates(
+    book_text: str,
+    config: EbookSearchConfig,
+) -> dict[str, PhraseCandidate]:
+    """Extract YAKE keyphrases when the optional YAKE package is installed.
+
+    Args:
+        book_text (str): Full book text to extract keyphrases from.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        dict[str, PhraseCandidate]: Candidates keyed by normalized phrase, with YAKE scores.
+    """
+    extractor = get_yake_extractor(
+        config.phrase_max_tokens,
+        config.phrase_yake_top_k,
+        config.phrase_yake_dedup_limit,
+    )
+    out: dict[str, PhraseCandidate] = {}
+    for phrase_text, yake_score in extractor.extract_keywords(book_text):
+        normalized = normalize_candidate_phrase(phrase_text, config)
+        if normalized is None:
+            continue
+        display_text, phrase_norm, token_count = normalized
+        out[phrase_norm] = PhraseCandidate(
+            phrase_text=display_text,
+            phrase_norm=phrase_norm,
+            token_count=token_count,
+            source_yake=True,
+            yake_score=float(yake_score),
+        )
+    return out
+
+
+def extract_capitalized_phrases(original_text: str, config: EbookSearchConfig) -> dict[str, PhraseCandidate]:
+    """Extract capitalized phrase runs that often carry fictional terms.
+
+    Args:
+        original_text (str): Original-case book text to scan for capitalized runs.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        dict[str, PhraseCandidate]: Candidates keyed by normalized phrase from capitalized runs.
+    """
+    out: dict[str, PhraseCandidate] = {}
+    for match in CAPITALIZED_PHRASE_RE.finditer(original_text):
+        phrase_text = match.group(0).strip()
+        normalized = normalize_candidate_phrase(
+            phrase_text,
+            config,
+            max_tokens=config.phrase_max_entity_tokens,
+        )
+        if normalized is None:
+            continue
+        display_text, phrase_norm, token_count = normalized
+        out[phrase_norm] = PhraseCandidate(
+            phrase_text=display_text,
+            phrase_norm=phrase_norm,
+            token_count=token_count,
+            source_capitalized=True,
+        )
+    return out
+
+
+def extract_metadata_candidates(
+    metadata: Mapping[str, object] | None,
+    config: EbookSearchConfig,
+) -> dict[str, PhraseCandidate]:
+    """Extract phrases from book metadata values such as title, author, and series.
+
+    Args:
+        metadata (Mapping[str, object] | None): Book metadata values, or ``None`` when unavailable.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        dict[str, PhraseCandidate]: Candidates keyed by normalized phrase from metadata values.
+    """
+    if metadata is None:
+        return {}
+
+    out: dict[str, PhraseCandidate] = {}
+    for value in metadata.values():
+        if value is None:
+            continue
+        phrase_text = str(value).strip()
+        normalized = normalize_candidate_phrase(
+            phrase_text,
+            config,
+            max_tokens=config.phrase_max_entity_tokens,
+        )
+        if normalized is None:
+            continue
+        display_text, phrase_norm, token_count = normalized
+        out[phrase_norm] = PhraseCandidate(
+            phrase_text=display_text,
+            phrase_norm=phrase_norm,
+            token_count=token_count,
+            source_metadata=True,
+        )
+    return out
+
+
+def merge_candidate_sources(*sources: Mapping[str, PhraseCandidate]) -> dict[str, PhraseCandidate]:
+    """Merge candidate dictionaries by normalized phrase.
+
+    Args:
+        *sources (Mapping[str, PhraseCandidate]): Candidate maps to combine, keyed by normalized phrase.
+
+    Returns:
+        dict[str, PhraseCandidate]: One merged candidate per normalized phrase.
+    """
+    merged: dict[str, PhraseCandidate] = {}
+    for source in sources:
+        for phrase_norm, item in source.items():
+            existing = merged.setdefault(
+                phrase_norm,
+                PhraseCandidate(
+                    phrase_text=item.phrase_text,
+                    phrase_norm=phrase_norm,
+                    token_count=item.token_count,
+                ),
+            )
+            merge_candidate(existing, item)
+    return merged
+
+
+def merge_candidate(existing: PhraseCandidate, item: PhraseCandidate) -> None:
+    """Merge one candidate into an existing candidate object.
+
+    Args:
+        existing (PhraseCandidate): Candidate mutated in place to absorb ``item``.
+        item (PhraseCandidate): Candidate whose sources, counts, and scores are merged in.
+    """
+    existing.source_raw_ngram = existing.source_raw_ngram or item.source_raw_ngram
+    existing.source_yake = existing.source_yake or item.source_yake
+    existing.source_capitalized = existing.source_capitalized or item.source_capitalized
+    existing.source_metadata = existing.source_metadata or item.source_metadata
+    existing.raw_count += item.raw_count
+    existing.chapter_count = max(existing.chapter_count, item.chapter_count)
+    if item.yake_score is not None:
+        existing.yake_score = item.yake_score
+
+
+def enrich_with_frequency_and_chapter_counts(
+    candidates: Mapping[str, PhraseCandidate],
+    chapters: Sequence[str],
+    *,
+    counted_sizes: Iterable[int] = (),
+) -> dict[str, PhraseCandidate]:
+    """Add raw occurrence and chapter-spread counts to candidates.
+
+    Candidates whose ``token_count`` is in ``counted_sizes`` are left untouched: those counts
+    were already computed while sliding the chapters in :func:`extract_raw_ngrams_by_chapter`,
+    so re-sliding those n-gram sizes here would just duplicate that work.
+
+    Args:
+        candidates (Mapping[str, PhraseCandidate]): Candidates to enrich, keyed by normalized phrase.
+        chapters (Sequence[str]): Chapter-like text blocks used to count occurrences and spread.
+        counted_sizes (Iterable[int]): Token counts whose counts are already populated and should be skipped.
+
+    Returns:
+        dict[str, PhraseCandidate]: Candidates with updated ``raw_count`` and ``chapter_count`` values.
+    """
+    if not candidates:
+        return {}
+
+    already_counted = set(counted_sizes)
+    candidate_sets_by_size: dict[int, set[str]] = defaultdict(set)
+    for phrase_norm, candidate in candidates.items():
+        if candidate.token_count in already_counted:
+            continue
+        candidate_sets_by_size[candidate.token_count].add(phrase_norm)
+
+    enriched = dict(candidates)
+    if not candidate_sets_by_size:
+        return enriched
+
+    total_counts, chapter_counts = count_candidate_occurrences(candidate_sets_by_size, chapters)
+    for phrase_norm, candidate in enriched.items():
+        if candidate.token_count in already_counted:
+            continue
+        candidate.raw_count = max(candidate.raw_count, total_counts[phrase_norm])
+        candidate.chapter_count = chapter_counts[phrase_norm]
+    return enriched
+
+
+def count_candidate_occurrences(
+    candidate_sets_by_size: Mapping[int, set[str]],
+    chapters: Sequence[str],
+) -> tuple[dict[str, int], dict[str, int]]:
+    """Count total occurrences and chapter spread for candidate phrases across chapters.
+
+    Args:
+        candidate_sets_by_size (Mapping[int, set[str]]): Candidate normalized phrases grouped by token count.
+        chapters (Sequence[str]): Chapter-like text blocks to slide n-gram windows over.
+
+    Returns:
+        tuple[dict[str, int], dict[str, int]]: Total occurrence counts and chapter-spread counts,
+        each keyed by normalized phrase.
+    """
+    total_counts: defaultdict[str, int] = defaultdict(int)
+    chapter_counts: defaultdict[str, int] = defaultdict(int)
+    for chapter in chapters:
+        seen_in_chapter: set[str] = set()
+        chapter_tokens = tokenize(chapter)
+        for ngram_size, candidate_norms in candidate_sets_by_size.items():
+            for start in range(len(chapter_tokens) - ngram_size + 1):
+                phrase_norm = " ".join(chapter_tokens[start : start + ngram_size])
+                if phrase_norm not in candidate_norms:
+                    continue
+                total_counts[phrase_norm] += 1
+                seen_in_chapter.add(phrase_norm)
+        for phrase_norm in seen_in_chapter:
+            chapter_counts[phrase_norm] += 1
+    return total_counts, chapter_counts
+
+
+def filter_storable_candidates(
+    candidates: Mapping[str, PhraseCandidate],
+    config: EbookSearchConfig,
+) -> tuple[dict[str, PhraseCandidate], int, int, int, int]:
+    """Remove candidates that should not be persisted.
+
+    Args:
+        candidates (Mapping[str, PhraseCandidate]): Candidates to filter, keyed by normalized phrase.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        tuple[dict[str, PhraseCandidate], int, int, int, int]: The storable candidates followed by the
+        counts dropped for being too short, too rare, too common, and junk.
+    """
+    min_raw_count = minimum_candidate_raw_count(config)
+    filtered: dict[str, PhraseCandidate] = {}
+    too_short = 0
+    too_rare = 0
+    too_common = 0
+    junk = 0
+    for phrase_norm, candidate in candidates.items():
+        if candidate.token_count < config.phrase_min_tokens:
+            too_short += 1
+            continue
+        if candidate.raw_count < min_raw_count:
+            too_rare += 1
+            continue
+        phrase_tokens = phrase_norm.split()
+        if is_most_common_word_phrase(phrase_tokens):
+            too_common += 1
+            continue
+        if is_junk_phrase(phrase_tokens):
+            junk += 1
+            continue
+        filtered[phrase_norm] = candidate
+    return filtered, too_short, too_rare, too_common, junk
+
+
+def minimum_candidate_raw_count(config: EbookSearchConfig) -> int:
+    """Return the minimum occurrence count required before storing a candidate.
+
+    Args:
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        int: The minimum raw occurrence count, never less than 1.
+    """
+    return max(config.phrase_raw_ngram_min_count, 1)
+
+
+def is_most_common_word_phrase(phrase_tokens: list[str]) -> bool:
+    """Return whether every token in a normalized phrase is a common word.
+
+    Args:
+        phrase_tokens (list[str]): Normalized phrase tokens to inspect.
+
+    Returns:
+        bool: True when the phrase is non-empty and every token is a common word.
+    """
+    common_words = get_most_common_words()
+    return bool(phrase_tokens) and all(token in common_words for token in phrase_tokens)
+
+
+def is_junk_phrase(phrase_tokens: list[str]) -> bool:
+    """Return whether a normalized phrase is lexical junk not worth LLM judging.
+
+    Judged data shows phrases containing a dialogue/action verb or a pronoun contraction are
+    never kept, and phrases whose tokens are mostly common words almost never are. Possessives
+    of proper nouns (``chapman's death``) pass because matching is by exact token, and
+    exactly-half-common bigrams (``data feed``) pass because the common-word rule is strict.
+
+    Args:
+        phrase_tokens (list[str]): Normalized phrase tokens to inspect.
+
+    Returns:
+        bool: True when the phrase contains a junk token or is majority common words.
+    """
+    if not phrase_tokens:
+        return False
+    junk_tokens = get_junk_tokens()
+    if any(token in junk_tokens for token in phrase_tokens):
+        return True
+    common_words = get_most_common_words()
+    half_phrase_len = len(phrase_tokens) // 2
+    return sum(token in common_words for token in phrase_tokens) > half_phrase_len
+
+
+def score_candidate(candidate: PhraseCandidate, config: EbookSearchConfig) -> float:
+    """Score a phrase candidate before LLM judging.
+
+    Args:
+        candidate (PhraseCandidate): Candidate to score.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        float: Combined score from sources, frequency, and length, less any penalties.
+    """
+    score = source_score(candidate) + frequency_score(candidate, config) + token_count_score(candidate, config)
+    if non_raw_source_count(candidate) >= config.phrase_multi_source_min_sources:
+        score += config.phrase_multi_source_score_bonus
+    if candidate.phrase_norm in get_ignored_phrases():
+        score -= 100.0
+    if has_bad_start(candidate.phrase_norm):
+        score -= config.phrase_bad_start_score_penalty
+    if has_bad_end(candidate.phrase_norm):
+        score -= config.phrase_bad_end_score_penalty
+    return score
+
+
+def non_raw_source_count(candidate: PhraseCandidate) -> int:
+    """Count the non-raw-ngram extraction sources that produced a candidate.
+
+    Args:
+        candidate (PhraseCandidate): Candidate whose enabled sources are counted.
+
+    Returns:
+        int: Number of enabled sources other than the raw n-gram slide.
+    """
+    return sum(
+        (
+            candidate.source_yake,
+            candidate.source_capitalized,
+            candidate.source_metadata,
+        )
+    )
+
+
+def has_bad_start(phrase_norm: str) -> bool:
+    """Return whether a normalized phrase starts with a bad starting token.
+
+    Args:
+        phrase_norm (str): Normalized phrase text to inspect.
+
+    Returns:
+        bool: True when the first token is a known bad starting token.
+    """
+    phrase_tokens = phrase_norm.split()
+    return bool(phrase_tokens and phrase_tokens[0] in get_bad_starts())
+
+
+def has_bad_end(phrase_norm: str) -> bool:
+    """Return whether a normalized phrase ends with a bad ending token.
+
+    Args:
+        phrase_norm (str): Normalized phrase text to inspect.
+
+    Returns:
+        bool: True when the last token is a known bad ending token.
+    """
+    phrase_tokens = phrase_norm.split()
+    return bool(phrase_tokens and phrase_tokens[-1] in get_bad_ends())
+
+
+def source_score(candidate: PhraseCandidate) -> float:
+    """Return the score contribution from extraction sources.
+
+    Args:
+        candidate (PhraseCandidate): Candidate whose enabled sources are weighted.
+
+    Returns:
+        float: Summed weight of the candidate's enabled extraction sources.
+    """
+    return sum(
+        weight
+        for enabled, weight in (
+            (candidate.source_yake, 2.0),
+            (candidate.source_capitalized, 2.0),
+            (candidate.source_metadata, 2.0),
+            (candidate.source_raw_ngram, 0.5),
+        )
+        if enabled
+    )
+
+
+def frequency_score(candidate: PhraseCandidate, config: EbookSearchConfig) -> float:
+    """Return the score contribution from frequency and chapter spread.
+
+    Args:
+        candidate (PhraseCandidate): Candidate whose counts are scored.
+        config (EbookSearchConfig): Runtime phrase-tuning settings holding score thresholds.
+
+    Returns:
+        float: Summed weight for each frequency and chapter-spread threshold the candidate meets.
+    """
+    return sum(
+        weight
+        for count, threshold, weight in (
+            (candidate.raw_count, config.phrase_raw_count_score_threshold, 0.5),
+            (candidate.raw_count, config.phrase_raw_count_high_score_threshold, 0.5),
+            (candidate.chapter_count, config.phrase_chapter_count_score_threshold, 0.5),
+            (candidate.chapter_count, config.phrase_chapter_count_high_score_threshold, 0.5),
+        )
+        if count >= threshold
+    )
+
+
+def token_count_score(candidate: PhraseCandidate, config: EbookSearchConfig) -> float:
+    """Return the score contribution from phrase length.
+
+    Args:
+        candidate (PhraseCandidate): Candidate whose token count is scored.
+        config (EbookSearchConfig): Runtime phrase-tuning settings holding the max token bound.
+
+    Returns:
+        float: Length-based score contribution, which may be negative for over- or under-length phrases.
+    """
+    if candidate.token_count == 1:
+        return -0.5
+    if candidate.token_count in {2, 3, 4}:
+        return 0.5
+    if candidate.token_count > config.phrase_max_tokens:
+        return -1.0
+    return 0.0
+
+
+def get_sample_contexts(normalized_book_text: str, phrase_norm: str, max_contexts: int = 5) -> list[str]:
+    """Return normalized context snippets containing a candidate phrase.
+
+    ``normalized_book_text`` is expected to already be ``normalize_text``-ed by the caller
+    so the whole book is not re-normalized for every phrase.
+
+    Args:
+        normalized_book_text (str): Whole book text, already normalized, to search.
+        phrase_norm (str): Normalized phrase to find contexts around.
+        max_contexts (int): Maximum number of context snippets to return.
+
+    Returns:
+        list[str]: Up to ``max_contexts`` normalized snippets surrounding the phrase.
+    """
+    contexts: list[str] = []
+    start = 0
+    while len(contexts) < max_contexts:
+        index = normalized_book_text.find(phrase_norm, start)
+        if index == -1:
+            break
+        left = max(0, index - 300)
+        right = min(len(normalized_book_text), index + len(phrase_norm) + 300)
+        contexts.append(normalized_book_text[left:right])
+        start = index + len(phrase_norm)
+    return contexts
+
+
+def candidate_source_names(candidate: PhraseCandidate) -> list[str]:
+    """Return enabled source names for an extracted candidate.
+
+    Args:
+        candidate (PhraseCandidate): Candidate whose enabled sources are listed.
+
+    Returns:
+        list[str]: Names of the extraction sources that produced the candidate.
+    """
+    names: list[str] = []
+    if candidate.source_raw_ngram:
+        names.append("raw_ngram")
+    if candidate.source_yake:
+        names.append("yake")
+    if candidate.source_capitalized:
+        names.append("capitalized")
+    if candidate.source_metadata:
+        names.append("metadata")
+    return names
+
+
+def extract_phrase_candidates_for_book(
+    book_text: str,
+    chapters: Sequence[str],
+    config: EbookSearchConfig,
+    *,
+    metadata: Mapping[str, object] | None = None,
+) -> list[PhraseCandidate]:
+    """Extract, score, and limit phrase candidates for one book.
+
+    Args:
+        book_text (str): Full book text used for most extraction sources.
+        chapters (Sequence[str]): Chapter-like text blocks used for frequency counts.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        metadata (Mapping[str, object] | None): Optional book metadata used as a candidate source.
+
+    Returns:
+        list[PhraseCandidate]: Scored candidates sorted best-first and capped per book.
+    """
+    started_at = perf_counter()
+    logger.info(
+        f"ebook_phrase_candidate_extract_start chapters={len(chapters)} chars={len(book_text)} "
+        f"{config.phrase_min_tokens=} {config.phrase_max_tokens=} {config.protected_phrase_max_candidates_per_book=}"
+    )
+    raw_started_at = perf_counter()
+    raw = extract_raw_ngrams_by_chapter(chapters, config)
+    logger.info(
+        f"ebook_phrase_candidate_extract_raw_complete candidates={len(raw)} "
+        f"duration_ms={(perf_counter() - raw_started_at) * 1000:.1f}"
+    )
+    yake_started_at = perf_counter()
+    yake_candidates = extract_yake_candidates(book_text, config)
+    logger.info(
+        f"ebook_phrase_candidate_extract_yake_complete candidates={len(yake_candidates)} "
+        f"duration_ms={(perf_counter() - yake_started_at) * 1000:.1f}"
+    )
+    capitalized_started_at = perf_counter()
+    capitalized = extract_capitalized_phrases(book_text, config)
+    logger.info(
+        f"ebook_phrase_candidate_extract_capitalized_complete candidates={len(capitalized)} "
+        f"duration_ms={(perf_counter() - capitalized_started_at) * 1000:.1f}"
+    )
+    metadata_candidates = extract_metadata_candidates(metadata, config)
+
+    candidates = merge_candidate_sources(raw, yake_candidates, capitalized, metadata_candidates)
+    enriched_started_at = perf_counter()
+    # Raw n-gram sizes were already counted per chapter above, so only enrich the remaining
+    # (entity-length) sizes here instead of re-sliding every size over the whole book.
+    candidates = enrich_with_frequency_and_chapter_counts(
+        candidates,
+        chapters,
+        counted_sizes=range(config.phrase_min_tokens, config.phrase_max_tokens + 1),
+    )
+    pre_filter_count = len(candidates)
+    candidates, filtered_too_short, filtered_too_rare, filtered_too_common, filtered_junk = filter_storable_candidates(
+        candidates, config
+    )
+    for candidate in candidates.values():
+        candidate.candidate_score = score_candidate(candidate, config)
+
+    limited = sorted(candidates.values(), key=lambda item: item.candidate_score, reverse=True)[
+        : config.protected_phrase_max_candidates_per_book
+    ]
+    logger.info(
+        f"ebook_phrase_candidate_extract_complete raw={len(raw)} yake={len(yake_candidates)} "
+        f"capitalized={len(capitalized)} metadata={len(metadata_candidates)} {pre_filter_count=} {filtered_too_short=} "
+        f"{filtered_too_rare=} {filtered_too_common=} {filtered_junk=} min_uses={minimum_candidate_raw_count(config)} "
+        f"storable={len(candidates)} limited={len(limited)} "
+        f"enrich_score_ms={(perf_counter() - enriched_started_at) * 1000:.1f} "
+        f"duration_ms={(perf_counter() - started_at) * 1000:.1f}"
+    )
+    return limited

python/ebook_search/protected_phrases/generate_ngrams.py

@@ -0,0 +1,295 @@
+"""Book-level orchestration for candidate n-gram generation and recalculation."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from time import perf_counter
+from typing import TYPE_CHECKING
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from python.ebook_search.protected_phrases.extraction import extract_phrase_candidates_for_book
+from python.ebook_search.protected_phrases.models import (
+    BookCandidateResult,
+    PhraseCandidateGenerationResult,
+    PhraseRecalculationResult,
+)
+from python.ebook_search.protected_phrases.pool import get_extraction_pool
+from python.ebook_search.protected_phrases.store import (
+    bulk_upsert_unjudged_candidates,
+    delete_phrase_data_for_book,
+    load_book_chapter_texts,
+    metadata_for_source_id,
+    new_candidate_row,
+    prune_unstorable_unjudged_candidate_phrases,
+)
+from python.orm.common import get_async_postgres_engine
+from python.orm.richie import EbookSource
+
+if TYPE_CHECKING:
+    from sqlalchemy.ext.asyncio import AsyncEngine
+
+    from python.ebook_search.config import EbookSearchConfig
+    from python.ebook_search.protected_phrases.models import PhraseCandidate
+
+logger = logging.getLogger(__name__)
+
+
+class BookHasNoChaptersError(ValueError):
+    """Raised when a book has no indexed chapter text to generate phrases from."""
+
+
+async def generate_candidate_phrases_for_books(
+    engine: AsyncEngine,
+    config: EbookSearchConfig,
+) -> PhraseCandidateGenerationResult:
+    """Create or refresh candidate phrases for indexed books without calling the LLM judge.
+
+    Every book is submitted to the shared process pool up front and runs in parallel across the
+    pool's workers; the call blocks until all books have finished. Each worker opens its own
+    database engine from environment variables, loads the book's chapters, and commits the
+    book's candidates independently.
+
+    Args:
+        engine (AsyncEngine): Engine used to read the book list in this process.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        PhraseCandidateGenerationResult: Per-corpus counts of books seen, built, and candidates stored.
+
+    Results are collected in book order while the pool keeps working. A book failure (including
+    a book with no indexed chapters) is logged and counted as not built; the remaining books
+    are unaffected.
+    """
+    async with AsyncSession(engine, expire_on_commit=False) as session:
+        source_query = select(EbookSource.id).order_by(EbookSource.id)
+        source_ids = (await session.scalars(source_query)).all()
+    books_seen = len(source_ids)
+    logger.info(
+        f"ebook_candidate_phrase_generation_start {books_seen=} {config.phrase_min_tokens=} "
+        f"{config.phrase_max_tokens=} {config.protected_phrase_max_candidates_per_book=}"
+    )
+
+    pool = get_extraction_pool(config.protected_phrase_extraction_workers)
+    wrapped_futures = [
+        (
+            source_id,
+            asyncio.wrap_future(pool.submit(generate_candidate_phrases_for_book_in_worker, source_id, None, config)),
+        )
+        for source_id in source_ids
+    ]
+    outcomes: list[BookCandidateResult] = []
+    for source_id, wrapped_future in wrapped_futures:
+        await asyncio.wait([wrapped_future])
+        exception = wrapped_future.exception()
+        if exception is not None:
+            logger.error(f"ebook_candidate_phrase_generation_book_failed {source_id=}")
+            outcomes.append(BookCandidateResult())
+            continue
+        saved_count = wrapped_future.result()
+        logger.info(f"ebook_candidate_phrase_generation_book_committed {source_id=} {saved_count=}")
+        outcomes.append(BookCandidateResult(candidates=saved_count, built=True))
+
+    result = PhraseCandidateGenerationResult(
+        books_seen=books_seen,
+        books_built=sum(1 for outcome in outcomes if outcome.built),
+        candidate_phrases=sum(outcome.candidates for outcome in outcomes),
+    )
+    logger.info(
+        f"ebook_candidate_phrase_generation_complete {result.books_seen=} {result.books_built=} "
+        f"{result.candidate_phrases=}"
+    )
+    return result
+
+
+async def recalculate_candidate_phrases_for_book(
+    session: AsyncSession,
+    source: EbookSource,
+    config: EbookSearchConfig,
+) -> PhraseRecalculationResult:
+    """Remove all book phrase data, regenerate candidates, and commit the completed book.
+
+    Args:
+        session (AsyncSession): Active database session; deletion and regeneration commit on it.
+        source (EbookSource): Indexed book to recalculate.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        PhraseRecalculationResult: Deleted-row counts and the number of candidates regenerated.
+
+    Raises:
+        BookHasNoChaptersError: If the book has no indexed chapters. The deletion is rolled
+            back, so the book's existing phrases stay intact.
+
+    The deletion and regeneration share the caller's session, so they commit together; a
+    regeneration failure rolls the deletion back.
+    """
+    started_at = perf_counter()
+    logger.info(f"ebook_candidate_phrase_recalculation_start {source.id=} {source.title=}")
+    deleted = await delete_phrase_data_for_book(session, source.id)
+    candidate_count = await generate_candidate_phrases_for_book(
+        session,
+        source.id,
+        series_id=None,
+        config=config,
+        replace_all=True,
+    )
+
+    result = PhraseRecalculationResult(
+        book_id=source.id,
+        deleted_candidates=deleted.deleted_candidates,
+        deleted_protected_phrases=deleted.deleted_protected_phrases,
+        deleted_aliases=deleted.deleted_aliases,
+        deleted_mentions=deleted.deleted_mentions,
+        candidate_phrases=candidate_count,
+    )
+    logger.info(
+        f"ebook_candidate_phrase_recalculation_complete {source.id=} {result.deleted_candidates=} "
+        f"{result.deleted_protected_phrases=} {result.deleted_aliases=} {result.deleted_mentions=} "
+        f"{result.candidate_phrases=} duration_ms={(perf_counter() - started_at) * 1000:.1f}"
+    )
+    return result
+
+
+def generate_candidate_phrases_for_book_in_worker(
+    book_id: int,
+    series_id: int | None,
+    config: EbookSearchConfig,
+) -> int:
+    """Run one book's candidate generation in a pooled worker process.
+
+    The worker has no engine or session to inherit (neither can cross process boundaries), so
+    it creates its own engine from environment variables, opens the book's session on it, and
+    disposes the engine once the book is stored.
+
+    Args:
+        book_id (int): Book the candidates belong to.
+        series_id (int | None): Series scope for the stored candidates.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        int: Number of candidate phrase rows stored.
+    """
+
+    async def generate_with_worker_engine() -> int:
+        engine = get_async_postgres_engine(name="RICHIE", vector_engine=True, pool_size=1)
+        try:
+            async with AsyncSession(engine, expire_on_commit=False) as session:
+                return await generate_candidate_phrases_for_book(
+                    session,
+                    book_id,
+                    series_id,
+                    config,
+                )
+        finally:
+            await engine.dispose()
+
+    return asyncio.run(generate_with_worker_engine())
+
+
+async def generate_candidate_phrases_for_book(
+    session: AsyncSession,
+    book_id: int,
+    series_id: int | None,
+    config: EbookSearchConfig,
+    *,
+    replace_all: bool = False,
+) -> int:
+    """Load a book's chapters and metadata, extract candidate phrases, and store them without LLM judging.
+
+    The session commits only when the whole book succeeds; any failure rolls the session back,
+    which also restores rows the caller deleted in the same transaction (e.g. a recalculation).
+
+    Args:
+        session (AsyncSession): Active database session; committed on success, rolled back on failure.
+        book_id (int): Book the candidates belong to.
+        series_id (int | None): Series scope for the stored candidates.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        replace_all (bool): When the caller has already cleared this book's candidates (e.g. a
+            recalculation), skip the per-candidate existence lookup and bulk-insert new rows.
+
+    Returns:
+        int: Number of candidate phrase rows stored.
+
+    Raises:
+        BookHasNoChaptersError: If the book has no indexed chapter text.
+    """
+    started_at = perf_counter()
+    chapters = await load_book_chapter_texts(session, book_id)
+    if not chapters:
+        await session.rollback()
+        message = f"book {book_id} has no indexed chapters"
+        raise BookHasNoChaptersError(message)
+    metadata = await metadata_for_source_id(session, book_id)
+    try:
+        book_text = "\n\n".join(chapters)
+        candidates = extract_phrase_candidates_for_book(
+            book_text,
+            chapters,
+            config,
+            metadata=metadata,
+        )
+        saved_count = await store_candidate_phrases_for_book(
+            session,
+            book_id,
+            series_id,
+            candidates,
+            config,
+            replace_all=replace_all,
+        )
+        await session.commit()
+    except Exception:
+        await session.rollback()
+        raise
+    logger.info(
+        f"ebook_candidate_phrase_generation_book_duration {book_id=} {saved_count=} "
+        f"duration_ms={(perf_counter() - started_at) * 1000:.1f}"
+    )
+    return saved_count
+
+
+async def store_candidate_phrases_for_book(
+    session: AsyncSession,
+    book_id: int,
+    series_id: int | None,
+    limited_candidates: list[PhraseCandidate],
+    config: EbookSearchConfig,
+    *,
+    replace_all: bool = False,
+) -> int:
+    """Persist already-extracted candidate phrase rows for one book without committing.
+
+    Args:
+        session (Session): Active database session.
+        book_id (int): Book the candidates belong to.
+        series_id (int | None): Series scope for the stored candidates.
+        limited_candidates (list[PhraseCandidate]): Scored candidates to persist.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        replace_all (bool): When the caller has already cleared this book's candidates, skip the
+            per-candidate existence lookup and bulk-insert new rows.
+
+    Returns:
+        int: Number of candidate phrase rows stored.
+    """
+    save_started_at = perf_counter()
+    if replace_all:
+        rows = [new_candidate_row(book_id, series_id, candidate) for candidate in limited_candidates]
+        session.add_all(rows)
+        await session.flush()
+        saved_count = len(rows)
+        logger.info(
+            f"ebook_candidate_phrase_save_start {book_id=} candidates={len(limited_candidates)} mode=bulk_insert"
+        )
+    else:
+        pruned_count = await prune_unstorable_unjudged_candidate_phrases(session, book_id, config)
+        logger.info(
+            f"ebook_candidate_phrase_save_start {book_id=} candidates={len(limited_candidates)} {pruned_count=}"
+        )
+        saved_count = await bulk_upsert_unjudged_candidates(session, book_id, series_id, limited_candidates)
+    logger.info(
+        f"ebook_candidate_phrase_save_complete {book_id=} {saved_count=} "
+        f"save_ms={(perf_counter() - save_started_at) * 1000:.1f}"
+    )
+    return saved_count

python/ebook_search/protected_phrases/judge_ngrams.py

@@ -0,0 +1,473 @@
+"""Book-level orchestration for LLM judging and promotion of candidate phrases."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from dataclasses import replace
+from time import perf_counter
+from typing import TYPE_CHECKING
+
+import httpx
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from python.ebook_search.llm_interface import request_chat_completion
+from python.ebook_search.prompts import load_prompt
+from python.ebook_search.protected_phrases.extraction import (
+    candidate_source_names,
+    get_sample_contexts,
+    is_junk_phrase,
+    is_most_common_word_phrase,
+    score_candidate,
+)
+from python.ebook_search.protected_phrases.matching import index_chunk_phrase_mentions_for_book
+from python.ebook_search.protected_phrases.models import BookJudgmentResult, LLMJudgment, PhraseJudgmentBackfillResult
+from python.ebook_search.protected_phrases.store import (
+    count_protected_phrases,
+    count_unjudged_candidates,
+    load_book_text,
+    load_candidates_for_judgment,
+    phrase_candidate_from_row,
+    save_candidate_to_db,
+    upsert_protected_phrase,
+)
+from python.ebook_search.protected_phrases.text_normalization import normalize_text
+from python.orm.richie import EbookSource
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from sqlalchemy.ext.asyncio import AsyncEngine
+
+    from python.ebook_search.config import EbookSearchConfig
+    from python.ebook_search.protected_phrases.models import PhraseCandidate
+    from python.orm.richie import EbookProtectedPhrase
+
+logger = logging.getLogger(__name__)
+
+
+async def judge_candidate_phrases_for_books(
+    engine: AsyncEngine,
+    config: EbookSearchConfig,
+    *,
+    source_ids: Sequence[int] | None = None,
+) -> PhraseJudgmentBackfillResult:
+    """Judge candidate phrases for books, fanning LLM calls out across books and phrases.
+
+    Up to ``phrase_judge_book_workers`` books are judged at once, and within each book candidates
+    are judged in concurrent chunks of ``phrase_judge_phrase_workers``. Each book uses its own
+    short-lived sessions for reads and writes; no database connection is held while LLM calls are
+    in flight. For a pseudo-single-threaded run (solo testing, debugging), set both worker
+    settings to 1.
+
+    Args:
+        engine (AsyncEngine): Engine used to open one session per book.
+        config (EbookSearchConfig): Runtime phrase-tuning settings and chat configuration.
+        source_ids (Sequence[int] | None): Books to judge; ``None`` judges every indexed book.
+
+    Returns:
+        PhraseJudgmentBackfillResult: Per-corpus counts of books judged, failures, candidates,
+        protected phrases, and mentions.
+    """
+    if source_ids is None:
+        async with AsyncSession(engine) as session:
+            source_ids = list((await session.scalars(select(EbookSource.id).order_by(EbookSource.id))).all())
+    books_seen = len(source_ids)
+    book_workers = max(1, config.phrase_judge_book_workers)
+    phrase_workers = max(1, config.phrase_judge_phrase_workers)
+    logger.info(
+        f"ebook_candidate_phrase_judgment_start {books_seen=} {book_workers=} {phrase_workers=} "
+        f"{config.protected_phrase_confidence_threshold=:.2f}"
+    )
+
+    book_semaphore = asyncio.Semaphore(book_workers)
+    max_connections = book_workers * phrase_workers
+    limits = httpx.Limits(max_connections=max_connections, max_keepalive_connections=max_connections)
+    async with httpx.AsyncClient(limits=limits) as client:
+        outcomes = await asyncio.gather(
+            *(judge_one_book_async(engine, source_id, config, client, book_semaphore) for source_id in source_ids)
+        )
+
+    result = PhraseJudgmentBackfillResult(
+        books_seen=books_seen,
+        books_judged=sum(1 for outcome in outcomes if outcome.committed),
+        books_failed=sum(1 for outcome in outcomes if outcome.failed),
+        candidates_judged=sum(outcome.judged for outcome in outcomes),
+        protected_phrases=sum(outcome.protected for outcome in outcomes),
+        phrase_mentions=sum(outcome.mentions for outcome in outcomes),
+    )
+    logger.info(
+        f"ebook_candidate_phrase_judgment_complete {result.books_seen=} {result.books_judged=} {result.books_failed=} "
+        f"{result.candidates_judged=} {result.protected_phrases=} {result.phrase_mentions=}"
+    )
+    return result
+
+
+async def judge_one_book_async(
+    engine: AsyncEngine,
+    source_id: int,
+    config: EbookSearchConfig,
+    client: httpx.AsyncClient,
+    book_semaphore: asyncio.Semaphore,
+) -> BookJudgmentResult:
+    """Judge one book concurrently and persist the outcome, honoring the book-level limit.
+
+    Args:
+        engine (AsyncEngine): Engine used to open the book's read and write sessions.
+        source_id (int): Book to judge candidates for.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        client (httpx.AsyncClient): Shared async client for LLM calls.
+        book_semaphore (asyncio.Semaphore): Caps how many books judge at once.
+
+    Returns:
+        BookJudgmentResult: The book's judgment outcome.
+    """
+    async with book_semaphore:
+        try:
+            prepared = await prepare_book_judgment(engine, source_id, config)
+            if prepared is None:
+                return BookJudgmentResult()
+            work_items, target_remaining = prepared
+            judged = await judge_book_candidates_async(client, config, source_id, work_items, target_remaining)
+            if not judged:
+                return BookJudgmentResult()
+            return await persist_book_judgments(engine, source_id, config, judged)
+        except Exception:
+            logger.exception(f"ebook_candidate_phrase_judgment_book_failed {source_id=}")
+            return BookJudgmentResult(failed=True)
+
+
+async def prepare_book_judgment(
+    engine: AsyncEngine,
+    source_id: int,
+    config: EbookSearchConfig,
+) -> tuple[list[tuple[int, PhraseCandidate]], int | None] | None:
+    """Load one book's candidates to judge, with sample contexts, on a short-lived read session.
+
+    Args:
+        engine (AsyncEngine): Engine used to open the read session.
+        source_id (int): Book to load candidates for.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        tuple[list[tuple[int, PhraseCandidate]], int | None] | None: Candidate rows paired with
+        in-memory candidates and the remaining protected-phrase target, or ``None`` when the book
+        has nothing to judge.
+    """
+    judgment_limit = config.protected_phrase_llm_candidates_per_book
+    if judgment_limit <= 0:
+        return None
+    async with AsyncSession(engine) as session:
+        if not await count_unjudged_candidates(session, source_id, config):
+            logger.info(f"ebook_candidate_phrase_judgment_book_skip_no_unjudged {source_id=}")
+            return None
+        existing_protected = await count_protected_phrases(session, source_id)
+        target_remaining: int | None = None
+        if config.phrase_target_protected_per_book > 0:
+            target_remaining = max(config.phrase_target_protected_per_book - existing_protected, 0)
+            if target_remaining == 0:
+                logger.info(
+                    f"ebook_candidate_phrase_judgment_skipped_target_met {source_id=} {existing_protected=} "
+                    f"{config.phrase_target_protected_per_book=}"
+                )
+                return None
+        book_text = await load_book_text(session, source_id)
+        if not book_text:
+            logger.warning(f"ebook_candidate_phrase_judgment_book_empty {source_id=}")
+            return None
+        normalized_book_text = normalize_text(book_text)
+        # Stored rows may predate the current junk filters and score weights, so re-filter and
+        # rescore every unjudged row here instead of trusting the persisted candidate_score.
+        rows = await load_candidates_for_judgment(session, source_id, config)
+        scored_items: list[tuple[int, PhraseCandidate]] = []
+        skipped_junk = 0
+        for row in rows:
+            candidate = phrase_candidate_from_row(row)
+            if is_junk_phrase(candidate.phrase_norm.split()):
+                skipped_junk += 1
+                continue
+            candidate.candidate_score = score_candidate(candidate, config)
+            scored_items.append((row.id, candidate))
+        scored_items.sort(key=lambda item: item[1].candidate_score, reverse=True)
+        work_items = scored_items[:judgment_limit]
+        for _, candidate in work_items:
+            candidate.sample_contexts = candidate.sample_contexts or get_sample_contexts(
+                normalized_book_text, candidate.phrase_norm
+            )
+    logger.info(
+        f"ebook_candidate_phrase_judgment_candidates_loaded {source_id=} candidates={len(work_items)} {skipped_junk=} "
+        f"unjudged_rows={len(rows)} {existing_protected=} {target_remaining=} {judgment_limit=}"
+    )
+    return work_items, target_remaining
+
+
+async def judge_book_candidates_async(
+    client: httpx.AsyncClient,
+    config: EbookSearchConfig,
+    source_id: int,
+    work_items: list[tuple[int, PhraseCandidate]],
+    target_remaining: int | None,
+) -> list[tuple[int, PhraseCandidate, LLMJudgment, bool]]:
+    """Judge a book's candidates in concurrent chunks, stopping once the target is reached.
+
+    Promotion decisions are made in memory so judging can stop early without any database writes.
+
+    Args:
+        client (httpx.AsyncClient): Shared async client for LLM calls.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        source_id (int): Book being judged, for logging.
+        work_items (list[tuple[int, PhraseCandidate]]): Candidate row ids paired with candidates,
+            in best-first score order.
+        target_remaining (int | None): Remaining protected-phrase target, or ``None`` for no cap.
+
+    Returns:
+        list[tuple[int, PhraseCandidate, LLMJudgment, bool]]: Judged rows with their judgment and
+        whether each should be promoted.
+    """
+    chunk_size = max(1, config.phrase_judge_phrase_workers)
+    judged: list[tuple[int, PhraseCandidate, LLMJudgment, bool]] = []
+    promoted = 0
+    for start in range(0, len(work_items), chunk_size):
+        chunk = work_items[start : start + chunk_size]
+        judgments = await asyncio.gather(*(judge_candidate_async(client, config, candidate) for _, candidate in chunk))
+        for (candidate_id, candidate), judgment in zip(chunk, judgments, strict=True):
+            promote = (target_remaining is None or promoted < target_remaining) and should_protect_judged_candidate(
+                candidate, judgment, source_id, config, candidate_id=candidate_id
+            )
+            if promote:
+                promoted += 1
+            judged.append((candidate_id, candidate, judgment, promote))
+        if target_remaining is not None and promoted >= target_remaining:
+            break
+    return judged
+
+
+async def judge_candidate_async(
+    client: httpx.AsyncClient,
+    config: EbookSearchConfig,
+    candidate: PhraseCandidate,
+) -> LLMJudgment:
+    """Judge one candidate with the LLM over the shared async client.
+
+    Args:
+        client (httpx.AsyncClient): Shared async client for LLM calls.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        candidate (PhraseCandidate): Candidate to judge.
+
+    Returns:
+        LLMJudgment: The parsed judgment.
+    """
+    content = await request_chat_completion(
+        client, config, build_judge_messages(candidate), response_format={"type": "json_object"}
+    )
+    return parse_llm_judgment(content, config)
+
+
+async def persist_book_judgments(
+    engine: AsyncEngine,
+    source_id: int,
+    config: EbookSearchConfig,
+    judged: list[tuple[int, PhraseCandidate, LLMJudgment, bool]],
+) -> BookJudgmentResult:
+    """Persist one book's judgments and promotions in a single committed transaction.
+
+    Args:
+        engine (AsyncEngine): Engine used to open the write session.
+        source_id (int): Book being persisted.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        judged (list[tuple[int, PhraseCandidate, LLMJudgment, bool]]): Judged candidates with their
+            judgment and promotion flag.
+
+    Returns:
+        BookJudgmentResult: The book's committed counts, or a failed result on error.
+    """
+    book_started_at = perf_counter()
+    async with AsyncSession(engine, expire_on_commit=False) as session:
+        try:
+            normalized_book_text = normalize_text(await load_book_text(session, source_id))
+            protected: list[EbookProtectedPhrase] = []
+            for candidate_id, candidate, judgment, promote in judged:
+                filtered_judgment = replace(
+                    judgment,
+                    aliases=tuple(
+                        alias for alias in judgment.aliases if alias_occurs_in_book(alias, normalized_book_text)
+                    ),
+                )
+                candidate_row = await save_candidate_to_db(
+                    session, source_id, None, candidate, judgment=filtered_judgment
+                )
+                if promote:
+                    protected.append(
+                        await upsert_protected_phrase(
+                            session, source_id, None, candidate, filtered_judgment, candidate_row
+                        )
+                    )
+                logger.info(
+                    f"ebook_candidate_phrase_judgment_candidate_complete {source_id=} {candidate_id=} "
+                    f"{candidate.phrase_norm=} {judgment.keep=} {judgment.confidence=:.3f} {judgment.category=} "
+                    f"{promote=}"
+                )
+            await session.flush()
+            mentions = await index_chunk_phrase_mentions_for_book(session, source_id, config) if protected else 0
+            await session.commit()
+        except Exception:
+            await session.rollback()
+            logger.exception(f"ebook_candidate_phrase_judgment_book_persist_failed {source_id=}")
+            return BookJudgmentResult(failed=True)
+    logger.info(
+        f"ebook_candidate_phrase_judgment_book_committed {source_id=} judged={len(judged)} protected={len(protected)} "
+        f"{mentions=} duration_ms={(perf_counter() - book_started_at) * 1000:.1f}"
+    )
+    return BookJudgmentResult(judged=len(judged), protected=len(protected), mentions=mentions, committed=True)
+
+
+def should_protect_judged_candidate(
+    candidate: PhraseCandidate,
+    judgment: LLMJudgment,
+    book_id: int,
+    config: EbookSearchConfig,
+    *,
+    candidate_id: int,
+) -> bool:
+    """Report whether a judged candidate qualifies to become a protected phrase.
+
+    Args:
+        candidate (PhraseCandidate): In-memory candidate that was judged.
+        judgment (LLMJudgment): Judge decision for the candidate.
+        book_id (int): Book the candidate belongs to, for logging.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        candidate_id (int): Stored candidate row id the judgment came from, for logging.
+
+    Returns:
+        bool: True when the judged candidate should be promoted to a protected phrase.
+    """
+    if not judgment.keep or judgment.confidence < config.protected_phrase_confidence_threshold:
+        return False
+    accepted_norm = normalize_text(judgment.canonical or candidate.phrase_text)
+    accepted_tokens = accepted_norm.split()
+    accepted_token_count = len(accepted_tokens)
+    if accepted_token_count < config.phrase_min_tokens:
+        logger.info(
+            f"ebook_candidate_phrase_judgment_candidate_skip_short_canonical {book_id=} {candidate_id=} "
+            f"{candidate.phrase_norm=} {accepted_norm=} {accepted_token_count=} {config.phrase_min_tokens=}"
+        )
+        return False
+    if is_most_common_word_phrase(accepted_tokens):
+        logger.info(
+            f"ebook_candidate_phrase_judgment_candidate_skip_common_canonical {book_id=} {candidate_id=} "
+            f"{candidate.phrase_norm=} {accepted_norm=}"
+        )
+        return False
+    return True
+
+
+def build_judge_messages(candidate: PhraseCandidate) -> list[dict[str, str]]:
+    """Build the chat messages used to judge one candidate phrase.
+
+    Args:
+        candidate (PhraseCandidate): Candidate to describe for the judge.
+
+    Returns:
+        list[dict[str, str]]: OpenAI-style system and user messages.
+    """
+    payload = {
+        "phrase": candidate.phrase_norm,
+        "token_count": candidate.token_count,
+        "sources": candidate_source_names(candidate),
+        "raw_count": candidate.raw_count,
+        "chapter_count": candidate.chapter_count,
+        "contexts": candidate.sample_contexts,
+    }
+    return load_prompt("phrase_judge").messages(candidate_json=json.dumps(payload, ensure_ascii=True))
+
+
+def parse_llm_judgment(content: str, config: EbookSearchConfig) -> LLMJudgment:
+    """Parse and validate an LLM phrase-judge response.
+
+    Args:
+        content (str): Raw model response text.
+        config (EbookSearchConfig): Runtime phrase-tuning settings supplying nesting defaults.
+
+    Returns:
+        LLMJudgment: The parsed and validated judgment.
+
+    Raises:
+        TypeError: If the decoded JSON body is not an object.
+    """
+    body = json.loads(extract_json_object(content))
+    if not isinstance(body, dict):
+        msg = "LLM phrase judge response is not a JSON object"
+        raise TypeError(msg)
+
+    aliases = body.get("aliases", ())
+    if not isinstance(aliases, list | tuple):
+        aliases = ()
+    return LLMJudgment(
+        keep=strict_bool(body.get("keep"), default=False),
+        canonical=optional_text(body.get("canonical")),
+        category=optional_text(body.get("category")),
+        aliases=tuple(str(alias) for alias in aliases if isinstance(alias, str) and alias.strip()),
+        confidence=clamped_float(body.get("confidence"), default=0.0),
+        importance=clamped_float(body.get("importance"), default=0.5),
+        allow_nested=strict_bool(body.get("allow_nested"), default=config.phrase_default_allow_nested),
+        suppress_children=strict_bool(body.get("suppress_children"), default=config.phrase_default_suppress_children),
+        reason=optional_text(body.get("reason")),
+    )
+
+
+def extract_json_object(content: str) -> str:
+    """Extract a JSON object from plain or fenced model output.
+
+    Args:
+        content (str): Raw model response text.
+
+    Returns:
+        str: The substring spanning the first JSON object.
+
+    Raises:
+        ValueError: If no JSON object is found in the response.
+    """
+    return content.strip()
+
+
+def alias_occurs_in_book(alias: str, normalized_book_text: str) -> bool:
+    """Return whether a normalized alias occurs as a complete phrase in the source book."""
+    alias_norm = normalize_text(alias)
+    return bool(alias_norm) and f" {alias_norm} " in f" {normalized_book_text} "
+
+
+def optional_text(value: object) -> str | None:
+    """Return stripped text for a nullable JSON value.
+
+    Args:
+        value (object): Decoded JSON value that may or may not be a string.
+
+    Returns:
+        str | None: The stripped string, or ``None`` when it is not a non-empty string.
+    """
+    if not isinstance(value, str):
+        return None
+    stripped = value.strip()
+    return stripped or None
+
+
+def strict_bool(value: object, *, default: bool) -> bool:
+    """Return a JSON boolean, falling back when the value has another type."""
+    return value if isinstance(value, bool) else default
+
+
+def clamped_float(value: object, *, default: float) -> float:
+    """Coerce a JSON number into the 0.0 to 1.0 range.
+
+    Args:
+        value (object): Decoded JSON value that may or may not be a number.
+        default (float): Fallback returned when ``value`` is not numeric.
+
+    Returns:
+        float: The value clamped to ``[0.0, 1.0]``, or ``default`` when non-numeric.
+    """
+    if not isinstance(value, int | float):
+        return default
+    return min(max(float(value), 0.0), 1.0)

python/ebook_search/protected_phrases/matching.py

@@ -0,0 +1,405 @@
+"""Runtime protected-phrase matching and chunk mention indexing."""
+
+from __future__ import annotations
+
+import logging
+from collections import defaultdict
+from typing import TYPE_CHECKING
+
+from sqlalchemy import and_, delete, func, or_, select, union
+
+from python.ebook_search.protected_phrases.config import get_ignored_phrases
+from python.ebook_search.protected_phrases.models import (
+    ChunkPhraseHit,
+    PhraseLookup,
+    PhraseMatch,
+)
+from python.ebook_search.protected_phrases.text_normalization import tokenize_with_offsets
+from python.orm.richie import (
+    EbookChunk,
+    EbookChunkPhraseMention,
+    EbookPhraseAlias,
+    EbookProtectedPhrase,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator, Sequence
+
+    from sqlalchemy.ext.asyncio import AsyncSession
+
+    from python.ebook_search.config import EbookSearchConfig
+
+logger = logging.getLogger(__name__)
+
+
+async def detect_protected_phrases_for_query(
+    session: AsyncSession,
+    query_text: str,
+    config: EbookSearchConfig,
+) -> list[PhraseMatch]:
+    """Find query phrases with indexed exact matches on canonical and alias norms.
+
+    Args:
+        session (AsyncSession): Active database session.
+        query_text (str): User query text to detect phrases in.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        list[PhraseMatch]: Metadata-backed, overlap-resolved phrase matches for the query.
+    """
+    tokens_ = tokenize_with_offsets(query_text)
+    windows_by_norm: defaultdict[str, list[tuple[int, int]]] = defaultdict(list)
+    token_texts = [token.text for token in tokens_]
+    max_tokens = max(config.phrase_max_tokens, config.phrase_max_entity_tokens)
+    for phrase_norm, start, end in generate_query_ngrams(
+        token_texts,
+        min_n=config.phrase_min_tokens,
+        max_n=max_tokens,
+    ):
+        windows_by_norm[phrase_norm].append((start, end))
+    if not windows_by_norm:
+        return []
+
+    query_norms = tuple(windows_by_norm)
+    matched_norms = union(
+        select(
+            EbookProtectedPhrase.id.label("phrase_id"),
+            EbookProtectedPhrase.phrase_norm.label("matched_norm"),
+        ).where(EbookProtectedPhrase.phrase_norm.in_(query_norms)),
+        select(
+            EbookPhraseAlias.phrase_id.label("phrase_id"),
+            EbookPhraseAlias.alias_norm.label("matched_norm"),
+        ).where(EbookPhraseAlias.alias_norm.in_(query_norms)),
+    ).subquery()
+    statement = select(EbookProtectedPhrase, matched_norms.c.matched_norm).join(
+        matched_norms,
+        matched_norms.c.phrase_id == EbookProtectedPhrase.id,
+    )
+
+    matches: list[PhraseMatch] = []
+    for phrase, matched_norm in await session.execute(statement):
+        for start, end in windows_by_norm[matched_norm]:
+            matches.append(
+                PhraseMatch(
+                    phrase_id=phrase.id,
+                    matched_norm=matched_norm,
+                    phrase_text=phrase.phrase_text,
+                    phrase_norm=phrase.phrase_norm,
+                    canonical_id=phrase.canonical_id,
+                    phrase_type=phrase.phrase_type,
+                    token_count=end - start,
+                    confidence=phrase.confidence,
+                    importance=phrase.importance,
+                    allow_nested=phrase.allow_nested,
+                    suppress_children=phrase.suppress_children,
+                    start_token=start,
+                    end_token=end,
+                    start_char=tokens_[start].start_char,
+                    end_char=tokens_[end - 1].end_char,
+                    book_id=phrase.book_id,
+                    series_id=phrase.series_id,
+                )
+            )
+    return resolve_overlaps(matches)
+
+
+async def index_chunk_phrase_mentions_for_book(
+    session: AsyncSession,
+    book_id: int,
+    config: EbookSearchConfig,
+) -> int:
+    """Rebuild chunk phrase mentions for all chunks in one book.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose chunk mentions are rebuilt.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        int: Total number of chunk phrase mentions indexed for the book.
+    """
+    lookup = await load_phrase_lookup(session, config, book_id=book_id)
+    await session.execute(delete(EbookChunkPhraseMention).where(EbookChunkPhraseMention.book_id == book_id))
+    chunks = await session.scalars(select(EbookChunk).where(EbookChunk.source_id == book_id).order_by(EbookChunk.id))
+    count = 0
+    for chunk in chunks:
+        count += index_chunk_phrase_mentions(session, chunk, lookup=lookup)
+    await session.flush()
+    logger.info(f"ebook_chunk_phrase_mentions_indexed {book_id=} {count=}")
+    return count
+
+
+async def load_phrase_lookup(
+    session: AsyncSession,
+    config: EbookSearchConfig,
+    *,
+    book_id: int | None = None,
+    series_id: int | None = None,
+) -> PhraseLookup:
+    """Load protected phrases and aliases into RAM lookup maps.
+
+    Args:
+        session (AsyncSession): Active database session.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        book_id (int | None): Optional book scope to restrict loaded phrases.
+        series_id (int | None): Optional series scope to restrict loaded phrases.
+
+    Returns:
+        PhraseLookup: Normalized phrase and alias maps with the token-window bounds to test.
+    """
+    phrase_ids_by_norm: defaultdict[str, set[int]] = defaultdict(set)
+    phrases_by_id: dict[int, EbookProtectedPhrase] = {}
+    max_tokens = config.phrase_max_tokens
+
+    statement = select(
+        EbookProtectedPhrase,
+        EbookPhraseAlias.alias_norm,
+    ).outerjoin(EbookPhraseAlias, EbookPhraseAlias.phrase_id == EbookProtectedPhrase.id)
+    scope_filter = protected_phrase_scope_filter(book_id=book_id, series_id=series_id)
+    if scope_filter is not None:
+        statement = statement.where(scope_filter)
+
+    for phrase, alias_norm in await session.execute(statement):
+        phrases_by_id[phrase.id] = phrase
+        phrase_ids_by_norm[phrase.phrase_norm].add(phrase.id)
+        max_tokens = max(max_tokens, phrase.token_count)
+        if alias_norm is not None:
+            phrase_ids_by_norm[alias_norm].add(phrase.id)
+            max_tokens = max(max_tokens, len(alias_norm.split()))
+
+    return PhraseLookup(
+        phrase_ids_by_norm={key: tuple(sorted(values)) for key, values in phrase_ids_by_norm.items()},
+        phrases_by_id=phrases_by_id,
+        min_tokens=config.phrase_min_tokens,
+        max_tokens=max_tokens,
+    )
+
+
+def protected_phrase_scope_filter(*, book_id: int | None, series_id: int | None) -> object | None:
+    """Build a SQLAlchemy filter for optional phrase book and series scope.
+
+    Args:
+        book_id (int | None): Optional book scope to include alongside global phrases.
+        series_id (int | None): Optional series scope to include alongside global phrases.
+
+    Returns:
+        object | None: A combined SQLAlchemy filter clause, or ``None`` when no scope is given.
+    """
+    conditions = []
+    if book_id is not None:
+        conditions.append(or_(EbookProtectedPhrase.book_id.is_(None), EbookProtectedPhrase.book_id == book_id))
+    if series_id is not None:
+        conditions.append(or_(EbookProtectedPhrase.series_id.is_(None), EbookProtectedPhrase.series_id == series_id))
+    if not conditions:
+        return None
+    return and_(*conditions)
+
+
+def is_inside(child: PhraseMatch, parent: PhraseMatch) -> bool:
+    """Return whether one token span is strictly inside another.
+
+    Args:
+        child (PhraseMatch): Candidate nested match.
+        parent (PhraseMatch): Candidate enclosing match.
+
+    Returns:
+        bool: True when ``child`` lies within ``parent`` and is not the same span.
+    """
+    return (
+        child.start_token >= parent.start_token
+        and child.end_token <= parent.end_token
+        and (child.start_token, child.end_token, child.phrase_id)
+        != (parent.start_token, parent.end_token, parent.phrase_id)
+    )
+
+
+def index_chunk_phrase_mentions(session: AsyncSession, chunk: EbookChunk, *, lookup: PhraseLookup) -> int:
+    """Store protected phrase mentions for one chunk.
+
+    Args:
+        session (AsyncSession): Active database session.
+        chunk (EbookChunk): Chunk whose text is scanned for phrase mentions.
+        lookup (PhraseLookup): In-memory phrase and alias lookup maps.
+
+    Returns:
+        int: Number of phrase mentions stored for the chunk.
+    """
+    tokens_ = tokenize_with_offsets(chunk.text)
+    token_texts = [token.text for token in tokens_]
+    raw_matches: list[PhraseMatch] = []
+    phrase_windows = generate_query_ngrams(token_texts, min_n=lookup.min_tokens, max_n=lookup.max_tokens)
+    for matched_norm, start, end in phrase_windows:
+        for phrase_id in lookup.phrase_ids_by_norm.get(matched_norm, ()):
+            phrase = lookup.phrases_by_id[phrase_id]
+            raw_matches.append(
+                PhraseMatch(
+                    phrase_id=phrase_id,
+                    matched_norm=matched_norm,
+                    phrase_text=phrase.phrase_text,
+                    phrase_norm=phrase.phrase_norm,
+                    canonical_id=phrase.canonical_id,
+                    phrase_type=phrase.phrase_type,
+                    confidence=phrase.confidence,
+                    importance=phrase.importance,
+                    allow_nested=phrase.allow_nested,
+                    suppress_children=phrase.suppress_children,
+                    start_token=start,
+                    end_token=end,
+                    token_count=end - start,
+                    start_char=tokens_[start].start_char,
+                    end_char=tokens_[end - 1].end_char,
+                    book_id=phrase.book_id,
+                    series_id=phrase.series_id,
+                )
+            )
+
+    matches = resolve_overlaps(raw_matches)
+    for match in matches:
+        session.add(
+            EbookChunkPhraseMention(
+                chunk_id=chunk.id,
+                phrase_id=match.phrase_id,
+                book_id=match.book_id if match.book_id is not None else chunk.source_id,
+                series_id=match.series_id,
+                start_char=match.start_char if match.start_char is not None else 0,
+                end_char=match.end_char,
+            )
+        )
+    return len(matches)
+
+
+def resolve_overlaps(matches: Sequence[PhraseMatch]) -> list[PhraseMatch]:
+    """Resolve overlapping phrase matches without relying only on longest match.
+
+    Args:
+        matches (Sequence[PhraseMatch]): Metadata-backed matches that may overlap.
+
+    Returns:
+        list[PhraseMatch]: The kept, non-suppressed matches.
+    """
+    sorted_matches = sorted(
+        matches,
+        key=lambda match: (match.start_token, -match.token_count, -match.importance, -match.confidence),
+    )
+    kept: list[PhraseMatch] = []
+    for candidate in sorted_matches:
+        if any(should_suppress(candidate, existing) for existing in kept):
+            continue
+        kept.append(candidate)
+    return kept
+
+
+def should_suppress(candidate: PhraseMatch, kept: PhraseMatch) -> bool:
+    """Return whether an already-kept match should suppress a candidate.
+
+    Args:
+        candidate (PhraseMatch): Match being considered for keeping.
+        kept (PhraseMatch): Match already kept that may suppress the candidate.
+
+    Returns:
+        bool: True when the candidate should be dropped in favor of the kept match.
+    """
+    if not overlaps(candidate, kept):
+        return False
+    if candidate.canonical_id == kept.canonical_id:
+        return rank_match(kept) >= rank_match(candidate)
+    if is_inside(candidate, kept) and kept.suppress_children and not candidate.allow_nested:
+        return True
+    return not candidate.allow_nested and rank_match(kept) > rank_match(candidate)
+
+
+def overlaps(first: PhraseMatch, second: PhraseMatch) -> bool:
+    """Return whether two token spans overlap.
+
+    Args:
+        first (PhraseMatch): First match to compare.
+        second (PhraseMatch): Second match to compare.
+
+    Returns:
+        bool: True when the two token spans share at least one token position.
+    """
+    return not (first.end_token <= second.start_token or first.start_token >= second.end_token)
+
+
+def rank_match(match: PhraseMatch) -> tuple[float, float, int]:
+    """Rank phrase matches by importance, confidence, then token count.
+
+    Args:
+        match (PhraseMatch): Match to build a sort key for.
+
+    Returns:
+        tuple[float, float, int]: A comparable key of importance, confidence, and token count.
+    """
+    return (match.importance, match.confidence, match.token_count)
+
+
+def generate_query_ngrams(
+    tokens_: Sequence[str],
+    min_n: int,
+    max_n: int,
+) -> Iterator[tuple[str, int, int]]:
+    """Generate normalized query windows from longest to shortest.
+
+    Args:
+        tokens_ (Sequence[str]): Normalized query tokens.
+        min_n (int): Smallest window size to yield.
+        max_n (int): Largest window size to yield, capped at the token count.
+
+    Yields:
+        tuple[str, int, int]: Normalized window text with its start and end token indices.
+    """
+    capped_max_n = min(max_n, len(tokens_))
+    for ngram_size in range(capped_max_n, min_n - 1, -1):
+        for start in range(len(tokens_) - ngram_size + 1):
+            end = start + ngram_size
+            phrase_norm = " ".join(tokens_[start:end])
+            if phrase_norm in get_ignored_phrases():
+                continue
+            yield phrase_norm, start, end
+
+
+async def phrase_hits_for_chunks(
+    session: AsyncSession,
+    *,
+    chunk_ids: Sequence[int],
+    phrase_ids: Sequence[int],
+) -> dict[int, tuple[ChunkPhraseHit, ...]]:
+    """Return matched protected phrases with mention counts by chunk id.
+
+    Args:
+        session (AsyncSession): Active database session.
+        chunk_ids (Sequence[int]): Chunk ids to look up mentions for.
+        phrase_ids (Sequence[int]): Protected phrase ids to restrict the results to.
+
+    Returns:
+        dict[int, tuple[ChunkPhraseHit, ...]]: Phrase hits per chunk id, ordered by mention count.
+    """
+    if not chunk_ids or not phrase_ids:
+        return {}
+
+    mention_count = func.count(EbookChunkPhraseMention.phrase_id).label("mention_count")
+    statement = (
+        select(
+            EbookChunkPhraseMention.chunk_id,
+            EbookProtectedPhrase.id.label("phrase_id"),
+            EbookProtectedPhrase.phrase_text,
+            mention_count,
+        )
+        .join(EbookProtectedPhrase, EbookProtectedPhrase.id == EbookChunkPhraseMention.phrase_id)
+        .where(
+            EbookChunkPhraseMention.chunk_id.in_(chunk_ids),
+            EbookChunkPhraseMention.phrase_id.in_(phrase_ids),
+        )
+        .group_by(EbookChunkPhraseMention.chunk_id, EbookProtectedPhrase.id, EbookProtectedPhrase.phrase_text)
+        .order_by(EbookChunkPhraseMention.chunk_id, mention_count.desc(), EbookProtectedPhrase.phrase_text)
+    )
+    hits: defaultdict[int, list[ChunkPhraseHit]] = defaultdict(list)
+    for row in await session.execute(statement):
+        hits[row.chunk_id].append(
+            ChunkPhraseHit(
+                phrase_id=row.phrase_id,
+                phrase_text=row.phrase_text,
+                mention_count=row.mention_count,
+            )
+        )
+    return {chunk_id: tuple(chunk_hits) for chunk_id, chunk_hits in hits.items()}

python/ebook_search/protected_phrases/models.py

@@ -0,0 +1,258 @@
+"""Dataclasses shared by protected phrase extraction, judging, matching, and backfills."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+    from python.orm.richie import EbookProtectedPhrase
+
+
+@dataclass(slots=True)
+class PhraseCandidate:
+    """A phrase candidate with merged extraction-source metadata.
+
+    Attributes:
+        phrase_text (str): Display text for the phrase.
+        phrase_norm (str): Normalized phrase used as the merge key.
+        token_count (int): Number of normalized tokens in the phrase.
+        source_raw_ngram (bool): Whether the raw n-gram extractor produced the phrase.
+        source_yake (bool): Whether YAKE keyword extraction produced the phrase.
+        source_capitalized (bool): Whether the capitalized-run extractor produced the phrase.
+        source_metadata (bool): Whether book metadata produced the phrase.
+        raw_count (int): Occurrences counted across the book text.
+        chapter_count (int): Number of chapters containing the phrase.
+        yake_score (float | None): Raw YAKE score when available; lower is better.
+        candidate_score (float): Combined pre-judging score.
+        sample_contexts (list[str]): Normalized context snippets around occurrences.
+    """
+
+    phrase_text: str
+    phrase_norm: str
+    token_count: int
+    source_raw_ngram: bool = False
+    source_yake: bool = False
+    source_capitalized: bool = False
+    source_metadata: bool = False
+    raw_count: int = 0
+    chapter_count: int = 0
+    yake_score: float | None = None
+    candidate_score: float = 0.0
+    sample_contexts: list[str] = field(default_factory=list)
+
+
+@dataclass(frozen=True, slots=True)
+class LLMJudgment:
+    """A structured phrase judgment returned by the LLM judge.
+
+    Attributes:
+        keep (bool): Whether the judge accepted the phrase for protection.
+        canonical (str | None): Canonical phrase text chosen by the judge.
+        category (str | None): Phrase category such as person, place, or event.
+        aliases (tuple[str, ...]): Alternate surface forms for the phrase.
+        confidence (float): Judge confidence between 0.0 and 1.0.
+        importance (float): Judge importance between 0.0 and 1.0.
+        allow_nested (bool): Whether the phrase may match inside a larger kept match.
+        suppress_children (bool): Whether the phrase suppresses matches nested inside it.
+        reason (str | None): Free-text explanation from the judge.
+    """
+
+    keep: bool
+    canonical: str | None
+    category: str | None
+    aliases: tuple[str, ...]
+    confidence: float
+    importance: float = 0.5
+    allow_nested: bool = False
+    suppress_children: bool = True
+    reason: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class PhraseLookup:
+    """In-memory phrase metadata used for constant-time text-window checks.
+
+    Attributes:
+        phrase_ids_by_norm (Mapping[str, tuple[int, ...]]): Canonical and alias norms to phrase ids.
+        phrases_by_id (Mapping[int, EbookProtectedPhrase]): Protected phrase metadata by id.
+        min_tokens (int): Smallest token-window size to test.
+        max_tokens (int): Largest token-window size to test.
+    """
+
+    phrase_ids_by_norm: Mapping[str, tuple[int, ...]]
+    phrases_by_id: Mapping[int, EbookProtectedPhrase]
+    min_tokens: int
+    max_tokens: int
+
+
+@dataclass(frozen=True, slots=True)
+class PhraseMatch:
+    """A detected phrase match with protected-phrase metadata attached.
+
+    Attributes:
+        phrase_id (int): Protected phrase id.
+        matched_norm (str): Normalized window text that matched.
+        phrase_text (str): Display text of the protected phrase.
+        phrase_norm (str): Normalized text of the protected phrase.
+        canonical_id (str): Deterministic ``category:slug`` identifier.
+        phrase_type (str | None): Phrase category.
+        token_count (int): Number of tokens in the match.
+        confidence (float): Stored judge confidence.
+        importance (float): Stored judge importance.
+        allow_nested (bool): Whether the phrase may match inside a larger kept match.
+        suppress_children (bool): Whether the phrase suppresses matches nested inside it.
+        start_token (int): Index of the first matched token.
+        end_token (int): Index one past the last matched token.
+        start_char (int | None): Start character offset in the source text.
+        end_char (int | None): End character offset in the source text.
+        book_id (int | None): Book scope of the phrase.
+        series_id (int | None): Series scope of the phrase.
+    """
+
+    phrase_id: int
+    matched_norm: str
+    phrase_text: str
+    phrase_norm: str
+    canonical_id: str
+    phrase_type: str | None
+    token_count: int
+    confidence: float
+    importance: float
+    allow_nested: bool
+    suppress_children: bool
+    start_token: int
+    end_token: int
+    start_char: int | None = None
+    end_char: int | None = None
+    book_id: int | None = None
+    series_id: int | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class ChunkPhraseHit:
+    """One protected phrase with its mention count inside one retrieved chunk.
+
+    Attributes:
+        phrase_id (int): Protected phrase id.
+        phrase_text (str): Display text for the phrase.
+        mention_count (int): Indexed mentions inside the chunk.
+    """
+
+    phrase_id: int
+    phrase_text: str
+    mention_count: int
+
+
+@dataclass(frozen=True, slots=True)
+class PhraseCandidateGenerationResult:
+    """Summary of candidate phrase extraction for indexed books.
+
+    Attributes:
+        books_seen (int): Indexed books examined.
+        books_built (int): Books that had candidates generated and committed.
+        candidate_phrases (int): Candidate phrases stored across all books.
+    """
+
+    books_seen: int
+    books_built: int
+    candidate_phrases: int
+
+
+@dataclass(frozen=True, slots=True)
+class CorpusPhraseStats:
+    """Corpus-wide candidate and protected phrase counts for the admin page.
+
+    Attributes:
+        total_books (int): Indexed books in the corpus.
+        books_with_candidates (int): Books that have candidate phrases generated.
+        books_fully_judged (int): Books with candidates where every candidate has been judged.
+        candidate_phrases (int): Candidate phrases stored across all books.
+        judged_candidates (int): Candidate phrases that have been LLM judged.
+        unjudged_candidates (int): Candidate phrases still waiting for judgment.
+        protected_phrases (int): Protected phrases promoted across all books.
+    """
+
+    total_books: int
+    books_with_candidates: int
+    books_fully_judged: int
+    candidate_phrases: int
+    judged_candidates: int
+    unjudged_candidates: int
+    protected_phrases: int
+
+
+@dataclass(frozen=True, slots=True)
+class PhraseJudgmentBackfillResult:
+    """Summary of LLM judging for stored candidate phrases.
+
+    Attributes:
+        books_seen (int): Indexed books examined.
+        books_judged (int): Books with judgments committed.
+        books_failed (int): Books rolled back after an error.
+        candidates_judged (int): Candidate phrases sent to the LLM judge.
+        protected_phrases (int): Protected phrases promoted from candidates.
+        phrase_mentions (int): Chunk phrase mentions indexed across all books.
+    """
+
+    books_seen: int
+    books_judged: int
+    books_failed: int
+    candidates_judged: int
+    protected_phrases: int
+    phrase_mentions: int
+
+
+@dataclass(frozen=True, slots=True)
+class BookJudgmentResult:
+    """Outcome of judging one book's candidate phrases.
+
+    Attributes:
+        judged (int): Candidate phrases sent to the LLM judge.
+        protected (int): Protected phrases promoted from candidates.
+        mentions (int): Chunk phrase mentions indexed for the book.
+        committed (bool): Whether the book's judgments were committed.
+        failed (bool): Whether the book was rolled back after an error.
+    """
+
+    judged: int = 0
+    protected: int = 0
+    mentions: int = 0
+    committed: bool = False
+    failed: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class BookCandidateResult:
+    """Outcome of generating one book's candidate phrases.
+
+    Attributes:
+        candidates (int): Candidate phrases stored for the book.
+        built (bool): Whether candidate generation was committed.
+    """
+
+    candidates: int = 0
+    built: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class PhraseRecalculationResult:
+    """Summary of phrase cleanup and candidate regeneration for one book.
+
+    Attributes:
+        book_id (int): Book the recalculation ran against.
+        deleted_candidates (int): Candidate phrase rows deleted.
+        deleted_protected_phrases (int): Protected phrase rows deleted.
+        deleted_aliases (int): Phrase alias rows deleted.
+        deleted_mentions (int): Chunk phrase mention rows deleted.
+        candidate_phrases (int): Candidate phrases regenerated after cleanup.
+    """
+
+    book_id: int
+    deleted_candidates: int
+    deleted_protected_phrases: int
+    deleted_aliases: int
+    deleted_mentions: int
+    candidate_phrases: int

python/ebook_search/protected_phrases/pool.py

@@ -0,0 +1,58 @@
+"""Process pool for offloading CPU-bound phrase extraction off the request thread.
+
+Phrase extraction is pure-Python CPU work (n-gram sliding, YAKE), so running it inline in a
+sync request handler serializes concurrent recalculations behind the GIL. Submitting it to a
+``ProcessPoolExecutor`` lets concurrent extractions run in parallel across cores instead. A
+``spawn`` context is used so workers do not inherit the parent's database engine, connections,
+or server threads.
+"""
+
+from __future__ import annotations
+
+import logging
+import multiprocessing
+import os
+from concurrent.futures import ProcessPoolExecutor
+from threading import Lock
+
+logger = logging.getLogger(__name__)
+
+
+class _ExtractionPool:
+    """Lazily created process-wide extraction pool and the lock guarding it."""
+
+    def __init__(self) -> None:
+        self.lock = Lock()
+        self.pool: ProcessPoolExecutor | None = None
+
+
+_extraction_pool = _ExtractionPool()
+
+
+def get_extraction_pool(max_workers: int) -> ProcessPoolExecutor:
+    """Return the shared extraction process pool, creating it on first use.
+
+    Args:
+        max_workers (int): Desired worker count; values below 1 fall back to the CPU count.
+
+    Returns:
+        ProcessPoolExecutor: The shared pool for phrase extraction.
+    """
+    with _extraction_pool.lock:
+        if _extraction_pool.pool is None:
+            workers = max_workers if max_workers > 0 else (os.cpu_count() or 1)
+            _extraction_pool.pool = ProcessPoolExecutor(
+                max_workers=workers,
+                mp_context=multiprocessing.get_context("spawn"),
+            )
+            logger.info(f"ebook_phrase_extraction_pool_started {workers=}")
+        return _extraction_pool.pool
+
+
+def shutdown_extraction_pool() -> None:
+    """Shut down the shared extraction pool if it was started."""
+    with _extraction_pool.lock:
+        if _extraction_pool.pool is not None:
+            _extraction_pool.pool.shutdown(wait=False, cancel_futures=True)
+            _extraction_pool.pool = None
+            logger.info("ebook_phrase_extraction_pool_shutdown")

python/ebook_search/protected_phrases/store.py

@@ -0,0 +1,662 @@
+"""Database persistence for candidate and protected phrase rows."""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import TYPE_CHECKING
+
+from sqlalchemy import delete, func, or_, select
+from sqlalchemy.dialects.postgresql import insert
+
+from python.ebook_search.protected_phrases.extraction import minimum_candidate_raw_count
+from python.ebook_search.protected_phrases.models import (
+    CorpusPhraseStats,
+    PhraseCandidate,
+    PhraseRecalculationResult,
+)
+from python.ebook_search.protected_phrases.text_normalization import normalize_text
+from python.orm.richie import (
+    EbookCandidatePhrase,
+    EbookChunk,
+    EbookChunkPhraseMention,
+    EbookPhraseAlias,
+    EbookProtectedPhrase,
+    EbookSource,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from sqlalchemy.ext.asyncio import AsyncSession
+
+    from python.ebook_search.config import EbookSearchConfig
+    from python.ebook_search.protected_phrases.models import LLMJudgment
+
+logger = logging.getLogger(__name__)
+
+
+async def load_book_text(session: AsyncSession, book_id: int) -> str:
+    """Load a book's indexed chunk text as one string for phrase extraction.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose chunk text is loaded.
+
+    Returns:
+        str: The book's chunk text joined into a single string.
+    """
+    texts = await session.scalars(
+        select(EbookChunk.text).where(EbookChunk.source_id == book_id).order_by(EbookChunk.chunk_index)
+    )
+    return "\n\n".join(stripped for text in texts if (stripped := text.strip()))
+
+
+async def load_book_chapter_texts(session: AsyncSession, book_id: int) -> list[str]:
+    """Reconstruct chapter-like text blocks from indexed chunks for phrase extraction.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose chunks are grouped into chapters.
+
+    Returns:
+        list[str]: Non-empty chapter-like text blocks in chunk order.
+    """
+    rows = await session.execute(
+        select(EbookChunk.chapter_id, EbookChunk.text)
+        .where(EbookChunk.source_id == book_id)
+        .order_by(EbookChunk.chunk_index)
+    )
+    chapters: list[str] = []
+    current_chapter_id: int | None = None
+    current_parts: list[str] = []
+    have_current = False
+
+    for chapter_id, text in rows:
+        if have_current and chapter_id != current_chapter_id:
+            chapter_text = "\n\n".join(current_parts).strip()
+            if chapter_text:
+                chapters.append(chapter_text)
+            current_parts = []
+        current_chapter_id = chapter_id
+        current_parts.append(str(text))
+        have_current = True
+
+    if current_parts:
+        chapter_text = "\n\n".join(current_parts).strip()
+        if chapter_text:
+            chapters.append(chapter_text)
+    return chapters
+
+
+def metadata_for_source(source: EbookSource) -> dict[str, object | None]:
+    """Return phrase extraction metadata for one indexed source.
+
+    Args:
+        source (EbookSource): Indexed source to read metadata from.
+
+    Returns:
+        dict[str, object | None]: Title, author, language, publisher, and identifier values.
+    """
+    return {
+        "title": source.title,
+        "author": source.author,
+        "language": source.language,
+        "publisher": source.publisher,
+        "identifier": source.identifier,
+    }
+
+
+async def metadata_for_source_id(session: AsyncSession, source_id: int) -> dict[str, object | None]:
+    """Return phrase extraction metadata for one indexed source by id.
+
+    Args:
+        session (AsyncSession): Active database session.
+        source_id (int): Id of the indexed source to read metadata from.
+
+    Returns:
+        dict[str, object | None]: Title, author, language, publisher, and identifier values.
+
+    Raises:
+        ValueError: If no source exists with the given id.
+    """
+    source = await session.get(EbookSource, source_id)
+    if source is None:
+        msg = f"No indexed source with id {source_id}"
+        raise ValueError(msg)
+    return metadata_for_source(source)
+
+
+async def count_protected_phrases(session: AsyncSession, book_id: int) -> int:
+    """Count stored protected phrases for one book.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose protected phrases are counted.
+
+    Returns:
+        int: Number of protected phrases stored for the book.
+    """
+    return (
+        await session.scalars(
+            select(func.count(EbookProtectedPhrase.id)).where(EbookProtectedPhrase.book_id == book_id)
+        )
+    ).one()
+
+
+async def count_unjudged_candidates(session: AsyncSession, book_id: int, config: EbookSearchConfig) -> int:
+    """Count storable candidate rows for a book that have not yet been judged.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose unjudged candidates are counted.
+        config (EbookSearchConfig): Runtime phrase-tuning settings supplying storage thresholds.
+
+    Returns:
+        int: Number of storable, unjudged candidate rows for the book.
+    """
+    return (
+        await session.scalars(
+            select(func.count(EbookCandidatePhrase.id)).where(
+                EbookCandidatePhrase.book_id == book_id,
+                EbookCandidatePhrase.llm_judged.is_(False),
+                EbookCandidatePhrase.token_count >= config.phrase_min_tokens,
+                EbookCandidatePhrase.raw_count >= minimum_candidate_raw_count(config),
+            )
+        )
+    ).one()
+
+
+async def corpus_phrase_stats(session: AsyncSession) -> CorpusPhraseStats:
+    """Summarize candidate and protected phrase coverage across the whole corpus.
+
+    Args:
+        session (AsyncSession): Active database session.
+
+    Returns:
+        CorpusPhraseStats: Corpus-wide phrase counts and per-book coverage counts.
+    """
+    total_books = (await session.scalars(select(func.count(EbookSource.id)))).one()
+    candidate_phrases, judged_candidates, books_with_candidates, books_with_unjudged = (
+        await session.execute(
+            select(
+                func.count(EbookCandidatePhrase.id),
+                func.count(EbookCandidatePhrase.id).filter(EbookCandidatePhrase.llm_judged.is_(True)),
+                func.count(func.distinct(EbookCandidatePhrase.book_id)),
+                func.count(func.distinct(EbookCandidatePhrase.book_id)).filter(
+                    EbookCandidatePhrase.llm_judged.is_(False)
+                ),
+            )
+        )
+    ).one()
+    protected_phrases = (await session.scalars(select(func.count(EbookProtectedPhrase.id)))).one()
+    return CorpusPhraseStats(
+        total_books=total_books,
+        books_with_candidates=books_with_candidates,
+        books_fully_judged=books_with_candidates - books_with_unjudged,
+        candidate_phrases=candidate_phrases,
+        judged_candidates=judged_candidates,
+        unjudged_candidates=candidate_phrases - judged_candidates,
+        protected_phrases=protected_phrases,
+    )
+
+
+async def book_ids_pending_first_judgment(session: AsyncSession) -> list[int]:
+    """Return books that have candidate phrases but no judged candidates yet.
+
+    Args:
+        session (AsyncSession): Active database session.
+
+    Returns:
+        list[int]: Book ids with candidates where judging has never run, ordered by id.
+    """
+    judged_books = select(EbookCandidatePhrase.book_id).where(EbookCandidatePhrase.llm_judged.is_(True)).distinct()
+    return list(
+        (
+            await session.scalars(
+                select(EbookCandidatePhrase.book_id)
+                .where(EbookCandidatePhrase.book_id.not_in(judged_books))
+                .distinct()
+                .order_by(EbookCandidatePhrase.book_id)
+            )
+        ).all()
+    )
+
+
+async def load_candidates_for_judgment(
+    session: AsyncSession,
+    book_id: int,
+    config: EbookSearchConfig,
+) -> Sequence[EbookCandidatePhrase]:
+    """Load every storable unjudged candidate row for a book.
+
+    Rows may have been stored before the current junk filters and score weights existed, so
+    callers re-check :func:`is_junk_phrase` and rescore before selecting what to judge.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose candidates are loaded.
+        config (EbookSearchConfig): Runtime phrase-tuning settings supplying storage thresholds.
+
+    Returns:
+        Sequence[EbookCandidatePhrase]: Storable, unjudged candidate rows ordered by stored score.
+    """
+    query = (
+        select(EbookCandidatePhrase)
+        .where(
+            EbookCandidatePhrase.book_id == book_id,
+            EbookCandidatePhrase.llm_judged.is_(False),
+            EbookCandidatePhrase.token_count >= config.phrase_min_tokens,
+            EbookCandidatePhrase.raw_count >= minimum_candidate_raw_count(config),
+        )
+        .order_by(
+            EbookCandidatePhrase.candidate_score.desc(),
+            EbookCandidatePhrase.raw_count.desc(),
+            EbookCandidatePhrase.id,
+        )
+    )
+    return (await session.scalars(query)).all()
+
+
+def phrase_candidate_from_row(row: EbookCandidatePhrase) -> PhraseCandidate:
+    """Recreate an in-memory candidate from a persisted candidate row.
+
+    Args:
+        row (EbookCandidatePhrase): Stored candidate row to convert.
+
+    Returns:
+        PhraseCandidate: An in-memory candidate mirroring the row's fields.
+    """
+    return PhraseCandidate(
+        phrase_text=row.phrase_text,
+        phrase_norm=row.phrase_norm,
+        token_count=row.token_count,
+        source_raw_ngram=row.source_raw_ngram,
+        source_yake=row.source_yake,
+        source_capitalized=row.source_capitalized,
+        source_metadata=row.source_metadata,
+        raw_count=row.raw_count,
+        chapter_count=row.chapter_count,
+        yake_score=row.yake_score,
+        candidate_score=row.candidate_score,
+        sample_contexts=row.sample_contexts or [],
+    )
+
+
+def candidate_row_values(
+    book_id: int,
+    series_id: int | None,
+    candidate: PhraseCandidate,
+    *,
+    judgment: LLMJudgment | None,
+) -> dict[str, object]:
+    """Build the column values for one candidate phrase upsert.
+
+    Args:
+        book_id (int): Book the candidate belongs to.
+        series_id (int | None): Series scope stored on the row.
+        candidate (PhraseCandidate): Candidate whose fields are written to the row.
+        judgment (LLMJudgment | None): Judgment to record, or ``None`` to leave the row unjudged.
+
+    Returns:
+        dict[str, object]: Column values keyed by column name.
+    """
+    values: dict[str, object] = {
+        "book_id": book_id,
+        "phrase_norm": candidate.phrase_norm,
+        "series_id": series_id,
+        "phrase_text": candidate.phrase_text,
+        "token_count": candidate.token_count,
+        "source_raw_ngram": candidate.source_raw_ngram,
+        "source_yake": candidate.source_yake,
+        "source_capitalized": candidate.source_capitalized,
+        "source_metadata": candidate.source_metadata,
+        "raw_count": candidate.raw_count,
+        "chapter_count": candidate.chapter_count,
+        "yake_score": candidate.yake_score,
+        "candidate_score": candidate.candidate_score,
+        "llm_judged": judgment is not None,
+    }
+    if candidate.sample_contexts:
+        values["sample_contexts"] = list(candidate.sample_contexts)
+    if judgment is not None:
+        values.update(
+            llm_keep=judgment.keep,
+            llm_confidence=judgment.confidence,
+            llm_category=judgment.category,
+            llm_reason=judgment.reason,
+        )
+    return values
+
+
+async def save_candidate_to_db(
+    session: AsyncSession,
+    book_id: int,
+    series_id: int | None,
+    candidate: PhraseCandidate,
+    *,
+    judgment: LLMJudgment | None,
+) -> EbookCandidatePhrase:
+    """Insert or update one candidate phrase row.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book the candidate belongs to.
+        series_id (int | None): Series scope stored on the row.
+        candidate (PhraseCandidate): Candidate whose fields are written to the row.
+        judgment (LLMJudgment | None): Judgment to record, or ``None`` to leave the row unjudged.
+
+    Returns:
+        EbookCandidatePhrase: The inserted or updated candidate row.
+    """
+    values = candidate_row_values(book_id, series_id, candidate, judgment=judgment)
+
+    # Preserve an existing judgment when this call is only refreshing candidate fields.
+    skip_update = {"book_id", "phrase_norm"}
+    if judgment is None:
+        skip_update.add("llm_judged")
+    insert_statement = insert(EbookCandidatePhrase).values(**values)
+    statement = insert_statement.on_conflict_do_update(
+        index_elements=["book_id", "phrase_norm"],
+        set_={column: insert_statement.excluded[column] for column in values if column not in skip_update},
+    ).returning(EbookCandidatePhrase)
+    return (await session.scalars(statement, execution_options={"populate_existing": True})).one()
+
+
+BULK_CANDIDATE_UPSERT_CHUNK = 1000
+
+
+async def bulk_upsert_unjudged_candidates(
+    session: AsyncSession,
+    book_id: int,
+    series_id: int | None,
+    candidates: Sequence[PhraseCandidate],
+) -> int:
+    """Insert or update many freshly extracted candidate rows in chunked multi-row upserts.
+
+    Saving one row per statement costs one database round trip per candidate, which dominated
+    generation time for full books, so candidates are written ``BULK_CANDIDATE_UPSERT_CHUNK``
+    rows per statement instead. Existing judgments and sample contexts are never overwritten:
+    fresh extractions carry no contexts, and ``llm_judged`` plus the ``llm_*`` columns are left
+    out of the conflict update. Candidates must have unique ``phrase_norm`` values, as produced
+    by extraction, since one multi-row upsert cannot touch the same row twice.
+
+    Args:
+        session (Session): Active database session.
+        book_id (int): Book the candidates belong to.
+        series_id (int | None): Series scope stored on the rows.
+        candidates (Sequence[PhraseCandidate]): Freshly extracted candidates to persist.
+
+    Returns:
+        int: Number of candidate rows written.
+    """
+    values = [
+        candidate_row_values(book_id, series_id, candidate, judgment=None)
+        for candidate in candidates
+        if not candidate.sample_contexts
+    ]
+    if len(values) != len(candidates):
+        msg = "bulk_upsert_unjudged_candidates only accepts freshly extracted candidates without sample contexts"
+        raise ValueError(msg)
+    skip_update = {"book_id", "phrase_norm", "llm_judged"}
+    for chunk_start in range(0, len(values), BULK_CANDIDATE_UPSERT_CHUNK):
+        chunk = values[chunk_start : chunk_start + BULK_CANDIDATE_UPSERT_CHUNK]
+        insert_statement = insert(EbookCandidatePhrase).values(chunk)
+        statement = insert_statement.on_conflict_do_update(
+            index_elements=["book_id", "phrase_norm"],
+            set_={column: insert_statement.excluded[column] for column in chunk[0] if column not in skip_update},
+        )
+        await session.execute(statement)
+    return len(values)
+
+
+def new_candidate_row(book_id: int, series_id: int | None, candidate: PhraseCandidate) -> EbookCandidatePhrase:
+    """Build a fresh unjudged candidate row without checking for an existing one.
+
+    Unlike :func:`save_candidate_to_db`, this does no lookup, so it is only safe when the caller
+    guarantees there is no existing row for ``(book_id, candidate.phrase_norm)`` — for example
+    right after :func:`delete_phrase_data_for_book` has cleared the book.
+
+    Args:
+        book_id (int): Book the candidate belongs to.
+        series_id (int | None): Series scope stored on the row.
+        candidate (PhraseCandidate): Candidate whose fields are written to the row.
+
+    Returns:
+        EbookCandidatePhrase: A new, unattached candidate row.
+    """
+    row = EbookCandidatePhrase(book_id=book_id, phrase_norm=candidate.phrase_norm)
+    row.llm_judged = False
+    row.series_id = series_id
+    row.phrase_text = candidate.phrase_text
+    row.token_count = candidate.token_count
+    row.source_raw_ngram = candidate.source_raw_ngram
+    row.source_yake = candidate.source_yake
+    row.source_capitalized = candidate.source_capitalized
+    row.source_metadata = candidate.source_metadata
+    row.raw_count = candidate.raw_count
+    row.chapter_count = candidate.chapter_count
+    row.yake_score = candidate.yake_score
+    row.candidate_score = candidate.candidate_score
+    if candidate.sample_contexts:
+        row.sample_contexts = list(candidate.sample_contexts)
+    return row
+
+
+async def upsert_protected_phrase(
+    session: AsyncSession,
+    book_id: int,
+    series_id: int | None,
+    candidate: PhraseCandidate,
+    judgment: LLMJudgment,
+    source_candidate: EbookCandidatePhrase,
+) -> EbookProtectedPhrase:
+    """Insert or update one accepted protected phrase and its aliases.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book the protected phrase belongs to.
+        series_id (int | None): Series scope stored on the phrase.
+        candidate (PhraseCandidate): Candidate the phrase was promoted from.
+        judgment (LLMJudgment): Accepted judgment supplying canonical text, category, and aliases.
+        source_candidate (EbookCandidatePhrase): Candidate row the phrase was promoted from.
+
+    Returns:
+        EbookProtectedPhrase: The inserted or updated protected phrase row.
+
+    Raises:
+        ValueError: If the chosen phrase text normalizes to empty.
+    """
+    phrase_text = judgment.canonical or candidate.phrase_text
+    phrase_norm = normalize_text(phrase_text)
+    if not phrase_norm:
+        msg = f"Protected phrase normalized to empty text: {phrase_text!r}"
+        raise ValueError(msg)
+
+    values = {
+        "book_id": book_id,
+        "phrase_norm": phrase_norm,
+        "series_id": series_id,
+        "phrase_text": phrase_text,
+        "canonical_id": make_canonical_id(judgment, phrase_norm),
+        "phrase_type": judgment.category,
+        "token_count": len(phrase_norm.split()),
+        "confidence": judgment.confidence,
+        "importance": judgment.importance,
+        "allow_nested": judgment.allow_nested,
+        "suppress_children": judgment.suppress_children,
+        "source_candidate_id": source_candidate.id,
+    }
+    insert_statement = insert(EbookProtectedPhrase).values(**values)
+    statement = insert_statement.on_conflict_do_update(
+        index_elements=["book_id", "phrase_norm"],
+        set_={
+            column: insert_statement.excluded[column] for column in values if column not in {"book_id", "phrase_norm"}
+        },
+    ).returning(EbookProtectedPhrase)
+    row = (await session.scalars(statement, execution_options={"populate_existing": True})).one()
+
+    for alias_text in judgment.aliases:
+        await upsert_phrase_alias(session, row, alias_text)
+    return row
+
+
+async def upsert_phrase_alias(
+    session: AsyncSession,
+    phrase: EbookProtectedPhrase,
+    alias_text: str,
+) -> EbookPhraseAlias | None:
+    """Insert or update one protected phrase alias.
+
+    Args:
+        session (AsyncSession): Active database session.
+        phrase (EbookProtectedPhrase): Protected phrase the alias points to.
+        alias_text (str): Alias surface form to store.
+
+    Returns:
+        EbookPhraseAlias | None: The alias row, or ``None`` when the alias is empty or equals the phrase.
+    """
+    alias_norm = normalize_text(alias_text)
+    if not alias_norm or alias_norm == phrase.phrase_norm:
+        return None
+
+    insert_statement = insert(EbookPhraseAlias).values(
+        phrase_id=phrase.id,
+        alias_norm=alias_norm,
+        alias_text=alias_text,
+        confidence=1.0,
+    )
+    statement = insert_statement.on_conflict_do_update(
+        index_elements=["phrase_id", "alias_norm"],
+        set_={
+            "alias_text": insert_statement.excluded.alias_text,
+            "confidence": insert_statement.excluded.confidence,
+        },
+    ).returning(EbookPhraseAlias)
+    return (await session.scalars(statement, execution_options={"populate_existing": True})).one()
+
+
+def make_canonical_id(judgment: LLMJudgment, phrase_norm: str) -> str:
+    """Create a deterministic canonical id from a judgment category and phrase.
+
+    Args:
+        judgment (LLMJudgment): Judgment supplying the phrase category.
+        phrase_norm (str): Normalized phrase text to slugify.
+
+    Returns:
+        str: A ``category:slug`` canonical identifier.
+    """
+    category = slugify_identifier(judgment.category or "phrase")
+    phrase_slug = slugify_identifier(phrase_norm)
+    return f"{category}:{phrase_slug}"
+
+
+def slugify_identifier(value: str) -> str:
+    """Normalize text for use inside a canonical id.
+
+    Args:
+        value (str): Text to slugify.
+
+    Returns:
+        str: A lowercase underscore slug, or ``"unknown"`` when empty.
+    """
+    slug = re.sub(r"[^a-z0-9]+", "_", normalize_text(value).replace("'", ""))
+    return slug.strip("_") or "unknown"
+
+
+async def prune_unstorable_unjudged_candidate_phrases(
+    session: AsyncSession,
+    book_id: int,
+    config: EbookSearchConfig,
+) -> int:
+    """Delete old unjudged candidate rows that no longer satisfy storage filters.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose stale candidates are pruned.
+        config (EbookSearchConfig): Runtime phrase-tuning settings supplying storage thresholds.
+
+    Returns:
+        int: Number of candidate rows deleted.
+    """
+    deleted = rowcount(
+        await session.execute(
+            delete(EbookCandidatePhrase).where(
+                EbookCandidatePhrase.book_id == book_id,
+                EbookCandidatePhrase.llm_judged.is_(False),
+                or_(
+                    EbookCandidatePhrase.token_count < config.phrase_min_tokens,
+                    EbookCandidatePhrase.raw_count < minimum_candidate_raw_count(config),
+                ),
+            )
+        )
+    )
+    if deleted:
+        logger.info(
+            f"ebook_candidate_phrase_unstorable_pruned {book_id=} {deleted=} {config.phrase_min_tokens=} "
+            f"min_uses={minimum_candidate_raw_count(config)}"
+        )
+    return deleted
+
+
+async def delete_phrase_data_for_book(session: AsyncSession, book_id: int) -> PhraseRecalculationResult:
+    """Delete all candidate, protected, alias, and mention phrase data for one book.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose phrase data is deleted.
+
+    Returns:
+        PhraseRecalculationResult: Deleted-row counts with ``candidate_phrases`` set to 0.
+    """
+    protected_ids = (
+        await session.scalars(select(EbookProtectedPhrase.id).where(EbookProtectedPhrase.book_id == book_id))
+    ).all()
+    deleted_aliases = 0
+    if protected_ids:
+        deleted_aliases = rowcount(
+            await session.execute(delete(EbookPhraseAlias).where(EbookPhraseAlias.phrase_id.in_(protected_ids)))
+        )
+
+    deleted_mentions = rowcount(
+        await session.execute(delete(EbookChunkPhraseMention).where(EbookChunkPhraseMention.book_id == book_id))
+    )
+    if protected_ids:
+        deleted_mentions += rowcount(
+            await session.execute(
+                delete(EbookChunkPhraseMention).where(EbookChunkPhraseMention.phrase_id.in_(protected_ids))
+            )
+        )
+
+    deleted_protected = rowcount(
+        await session.execute(delete(EbookProtectedPhrase).where(EbookProtectedPhrase.book_id == book_id))
+    )
+    deleted_candidates = rowcount(
+        await session.execute(delete(EbookCandidatePhrase).where(EbookCandidatePhrase.book_id == book_id))
+    )
+    await session.flush()
+    logger.info(
+        f"ebook_candidate_phrase_data_deleted {book_id=} {deleted_candidates=} {deleted_protected=} {deleted_aliases=} "
+        f"{deleted_mentions=}"
+    )
+    return PhraseRecalculationResult(
+        book_id=book_id,
+        deleted_candidates=deleted_candidates,
+        deleted_protected_phrases=deleted_protected,
+        deleted_aliases=deleted_aliases,
+        deleted_mentions=deleted_mentions,
+        candidate_phrases=0,
+    )
+
+
+def rowcount(result: object) -> int:
+    """Return a safe integer rowcount from a SQLAlchemy execution result.
+
+    Args:
+        result (object): SQLAlchemy execution result that may expose ``rowcount``.
+
+    Returns:
+        int: The result's rowcount, or 0 when it is missing or negative.
+    """
+    count = getattr(result, "rowcount", 0)
+    return int(count if count is not None and count >= 0 else 0)

python/ebook_search/protected_phrases/text_normalization.py

@@ -0,0 +1,91 @@
+"""Protected phrase extraction, storage, and runtime matching."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+JSON_OBJECT_RE = re.compile(r"\{.*\}", re.DOTALL)
+
+
+@dataclass(frozen=True, slots=True)
+class NormalizedToken:
+    """A normalized token plus its source character span."""
+
+    text: str
+    start_char: int
+    end_char: int
+
+
+def normalize_text(text: str) -> str:
+    """Normalize text for phrase storage and lookup.
+
+    Args:
+        text (str): Raw text to normalize.
+
+    Returns:
+        str: Normalized tokens joined by single spaces.
+    """
+    return " ".join(token.text for token in tokenize_with_offsets(text))
+
+
+def tokenize(text: str) -> list[str]:
+    """Normalize and split text into phrase-detection tokens.
+
+    Args:
+        text (str): Raw text to tokenize.
+
+    Returns:
+        list[str]: Normalized token strings.
+    """
+    return [token.text for token in tokenize_with_offsets(text)]
+
+
+def tokenize_with_offsets(text: str) -> list[NormalizedToken]:
+    """Normalize text into tokens while preserving original character offsets.
+
+    Args:
+        text (str): Raw text to tokenize.
+
+    Returns:
+        list[NormalizedToken]: Normalized tokens with their source character spans.
+    """
+    tokens: list[NormalizedToken] = []
+    current: list[str] = []
+    start_char: int | None = None
+
+    for index, char in enumerate(text):
+        normalized = normalize_char(char)
+        if normalized == " ":
+            if current and start_char is not None:
+                tokens.append(NormalizedToken(text="".join(current), start_char=start_char, end_char=index))
+                current = []
+                start_char = None
+            continue
+        if start_char is None:
+            start_char = index
+        current.append(normalized)
+
+    if current and start_char is not None:
+        tokens.append(NormalizedToken(text="".join(current), start_char=start_char, end_char=len(text)))
+    return tokens
+
+
+def normalize_char(char: str) -> str:
+    """Normalize one character into a token character or a separator.
+
+    Args:
+        char (str): Single source character to normalize.
+
+    Returns:
+        str: The normalized token character, or a space acting as a separator.
+    """
+    if char in {"\u2019", "\u2018"}:
+        return "'"
+    if char in {"-", "\u2013", "\u2014"}:
+        return " "
+
+    lowered = char.lower()
+    if lowered in "abcdefghijklmnopqrstuvwxyz0123456789'":
+        return lowered
+    return " "

python/ebook_search/rerank.py

@@ -9,6 +9,8 @@ from typing import TYPE_CHECKING
 from python.ebook_search.llm_interface import request_rerank
 
 if TYPE_CHECKING:
+    import httpx
+
     from python.ebook_search.config import RerankConfig
     from python.ebook_search.search import SearchResult
 
@@ -23,18 +25,18 @@ class RerankResult:
     score: float
 
 
-def rerank_chunks(query: str, candidates: list[SearchResult], config: RerankConfig) -> list[SearchResult]:
+async def rerank_chunks(
+    client: httpx.AsyncClient,
+    query: str,
+    candidates: list[SearchResult],
+    config: RerankConfig,
+) -> list[SearchResult]:
     """Rerank candidates with a vLLM rerank endpoint."""
     if not candidates:
         return []
 
-    logger.info(
-        "ebook_rerank_request_start base_url=%s model=%s candidates=%s",
-        config.base_url,
-        config.model,
-        len(candidates),
-    )
-    scores = score_candidates(query, candidates, config)
+    logger.info(f"ebook_rerank_request_start {config.base_url=} {config.model=} candidates={len(candidates)}")
+    scores = await score_candidates(client, query, candidates, config)
     results = sorted(
         (
             replace(
@@ -47,28 +49,24 @@ def rerank_chunks(query: str, candidates: list[SearchResult], config: RerankConf
         key=lambda result: result.score,
         reverse=True,
     )
-    logger.info(
-        "ebook_rerank_request_complete base_url=%s model=%s candidates=%s",
-        config.base_url,
-        config.model,
-        len(results),
-    )
+    logger.info(f"ebook_rerank_request_complete {config.base_url=} {config.model=} candidates={len(results)}")
     return results
 
 
-def score_candidates(
+async def score_candidates(
+    client: httpx.AsyncClient,
     query: str,
     candidates: list[SearchResult],
     config: RerankConfig,
 ) -> dict[int, RerankResult]:
     """Score candidate chunks with the configured rerank API."""
-    body = request_rerank(query, [candidate.text for candidate in candidates], config)
+    body = await request_rerank(client, query, [candidate.text for candidate in candidates], config)
     if body is None:
         return zero_rerank_scores(candidates)
 
     scores = parse_vllm_scores(body, candidates)
     for result in scores.values():
-        logger.debug("ebook_rerank_candidate_scored chunk_id=%s score=%s", result.chunk_id, result.score)
+        logger.debug(f"ebook_rerank_candidate_scored {result.chunk_id=} {result.score=}")
     return scores
 
 

python/ebook_search/search.py

@@ -2,25 +2,31 @@
 
 from __future__ import annotations
 
+import asyncio
 import logging
 import re
 from collections import defaultdict
-from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, replace
 from typing import TYPE_CHECKING
 
 from pgvector.sqlalchemy import Vector
 from sqlalchemy import literal, select
-from sqlalchemy.orm import Session
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.ext.asyncio import AsyncSession
 
 from python.ebook_search.bm25_corpus import (
     BM25CorpusUnavailableError,
     load_bm25_corpus,
     score_bm25_corpus,
 )
+from python.ebook_search.chunk_records import CHUNK_RECORD_COLUMNS
 from python.ebook_search.embeddings import MODEL_DIMENSIONS, embed_query, get_embedding_table
+from python.ebook_search.protected_phrases.matching import (
+    detect_protected_phrases_for_query,
+    phrase_hits_for_chunks,
+)
 from python.ebook_search.rerank import rerank_chunks
-from python.ebook_search.timing import RuntimeStep, timed_result
+from python.ebook_search.timing import RuntimeStep, async_timed_result, timed_result
 from python.orm.richie import (
     EbookChapter,
     EbookChunk,
@@ -29,11 +35,13 @@ from python.orm.richie import (
 )
 
 if TYPE_CHECKING:
-    from collections.abc import Mapping
+    from collections.abc import Mapping, Sequence
 
-    from sqlalchemy.engine import Engine
+    import httpx
+    from sqlalchemy.ext.asyncio import AsyncEngine
 
     from python.ebook_search.config import EbookSearchConfig
+    from python.ebook_search.protected_phrases.models import PhraseMatch
 
 logger = logging.getLogger(__name__)
 
@@ -45,11 +53,14 @@ class SearchResult:
     chunk_id: int
     text: str
     source_title: str
+    source_id: int | None = None
     score: float = 0.0
     vector_score: float | None = None
     bm25_score: float | None = None
     fused_score: float | None = None
     rerank_score: float | None = None
+    phrase_hit_count: int = 0
+    matched_phrases: tuple[str, ...] = ()
     source_author: str | None = None
     chapter_title: str | None = None
     page_label: str | None = None
@@ -64,6 +75,7 @@ class SearchResponse:
     results: list[SearchResult]
     rank_label: str
     timings: tuple[RuntimeStep, ...] = ()
+    phrase_matches: tuple[PhraseMatch, ...] = ()
 
     @property
     def total_runtime_ms(self) -> float:
@@ -73,34 +85,40 @@ class SearchResponse:
 
 @dataclass(frozen=True)
 class RetrievalResponse:
-    """Parallel retrieval output for vector and BM25 candidates."""
+    """Parallel retrieval output for vector, BM25, and protected phrase candidates."""
 
     vector_results: list[SearchResult]
     lexical_results: list[SearchResult]
+    phrase_matches: list[PhraseMatch]
     timings: tuple[RuntimeStep, ...]
 
 
-def search_ebooks(
-    engine: Engine,
+async def search_ebooks(
+    engine: AsyncEngine,
+    client: httpx.AsyncClient,
     query: str,
     config: EbookSearchConfig,
     *,
-    rerank: bool = False,
+    rerank: bool,
+    phrase_matching: bool,
 ) -> SearchResponse:
-    """Run hybrid vector/BM25 search and optional reranking."""
+    """Run hybrid vector/BM25 search and optional reranking.
+
+    Phrase matching only runs when both the request asks for it and
+    ``config.phrase_matching_enabled`` allows it.
+    """
     if not query.strip():
         logger.info("ebook_search_empty_query")
         return SearchResponse(query=query, results=[], rank_label="Hybrid")
 
-    logger.info("ebook_search_start query_length=%s rerank=%s", len(query), rerank)
+    phrase_matching = phrase_matching and config.phrase_matching_enabled
+    logger.info(f"ebook_search_start query_length={len(query)} {rerank=} {phrase_matching=}")
     timings: list[RuntimeStep] = []
-    retrieval, timing = timed_result(
+    retrieval, timing = await async_timed_result(
         "Hybrid retrieval",
-        parallel_retrieval,
-        engine,
-        query,
-        config,
+        parallel_retrieval(engine, client, query, config, phrase_matching=phrase_matching),
     )
+    phrase_matches = retrieval.phrase_matches
     timings.extend(retrieval.timings)
     timings.append(timing)
     fused, timing = timed_result(
@@ -111,86 +129,174 @@ def search_ebooks(
         rank_constant=config.rrf_rank_constant,
     )
     timings.append(timing)
-    if config.rerank.enabled and rerank:
-        response, timing = timed_result("Rerank", apply_rerank, query, fused, config)
-    else:
-        response, timing = timed_result("Rerank skipped", skip_rerank, query, fused, config)
+    phrase_boost_timing_name = "Phrase mention boost" if phrase_matching else "Phrase mention boost skipped"
+    fused, timing = await async_timed_result(
+        phrase_boost_timing_name,
+        apply_phrase_mention_boosts(
+            engine,
+            fused,
+            phrase_matches,
+            config.phrase_hit_boost,
+            phrase_matching=phrase_matching,
+        ),
+    )
     timings.append(timing)
-    response = replace(response, timings=tuple(timings))
+    rerank_enabled = config.rerank.enabled and rerank
+    rerank_timing_name = "Rerank" if rerank_enabled else "Rerank skipped"
+    response, timing = await async_timed_result(
+        rerank_timing_name,
+        apply_rerank(client, query, fused, config, rerank=rerank_enabled),
+    )
+    timings.append(timing)
+    response = replace(response, timings=tuple(timings), phrase_matches=tuple(phrase_matches))
     logger.info(
-        "ebook_search_complete vector_candidates=%s lexical_candidates=%s "
-        "fused_candidates=%s returned=%s rank_label=%s runtime_ms=%.1f",
-        len(retrieval.vector_results),
-        len(retrieval.lexical_results),
-        len(fused),
-        len(response.results),
-        response.rank_label,
-        response.total_runtime_ms,
+        f"ebook_search_complete vector_candidates={len(retrieval.vector_results)} "
+        f"lexical_candidates={len(retrieval.lexical_results)} fused_candidates={len(fused)} {phrase_matching=} "
+        f"phrase_matches={len(phrase_matches)} returned={len(response.results)} {response.rank_label=} "
+        f"{response.total_runtime_ms=:.1f}"
     )
     return response
 
 
-def parallel_retrieval(
-    engine: Engine,
+async def query_phrase_matches(
+    engine: AsyncEngine,
     query: str,
     config: EbookSearchConfig,
+    *,
+    phrase_matching: bool,
+) -> list[PhraseMatch]:
+    """Detect protected phrases in a query without making search fail when phrase tables are unavailable."""
+    if not phrase_matching:
+        logger.info("ebook_protected_phrase_detection_skipped")
+        return []
+    try:
+        async with AsyncSession(engine) as session:
+            return await detect_protected_phrases_for_query(session, query, config)
+    except SQLAlchemyError as error:
+        logger.warning(f"ebook_protected_phrase_detection_unavailable {error=}")
+        return []
+
+
+async def apply_phrase_mention_boosts(
+    engine: AsyncEngine,
+    candidates: list[SearchResult],
+    phrase_matches: Sequence[PhraseMatch],
+    phrase_hit_boost: float,
+    *,
+    phrase_matching: bool,
+) -> list[SearchResult]:
+    """Boost retrieved chunks that have indexed mentions for detected protected phrases when enabled."""
+    if not phrase_matching:
+        logger.info(f"ebook_phrase_boost_skipped candidates={len(candidates)}")
+        return candidates
+
+    phrase_ids = sorted({match.phrase_id for match in phrase_matches})
+    if not candidates or not phrase_ids or phrase_hit_boost <= 0:
+        return candidates
+
+    chunk_ids = [candidate.chunk_id for candidate in candidates]
+    try:
+        async with AsyncSession(engine) as session:
+            phrase_hits = await phrase_hits_for_chunks(session, chunk_ids=chunk_ids, phrase_ids=phrase_ids)
+    except SQLAlchemyError as error:
+        logger.warning(f"ebook_phrase_boost_unavailable {error=}")
+        return candidates
+
+    if not phrase_hits:
+        return candidates
+
+    hit_counts = {
+        chunk_id: sum(hit.mention_count for hit in chunk_hits) for chunk_id, chunk_hits in phrase_hits.items()
+    }
+    boosted = [
+        replace(
+            candidate,
+            score=candidate.score + (hit_counts.get(candidate.chunk_id, 0) * phrase_hit_boost),
+            fused_score=boosted_fused_score(candidate, hit_counts.get(candidate.chunk_id, 0), phrase_hit_boost),
+            phrase_hit_count=hit_counts.get(candidate.chunk_id, 0),
+            matched_phrases=tuple(hit.phrase_text for hit in phrase_hits.get(candidate.chunk_id, ())),
+            rank_source=phrase_rank_source(candidate.rank_source, hit_counts.get(candidate.chunk_id, 0)),
+        )
+        for candidate in candidates
+    ]
+    return sorted(boosted, key=lambda candidate: candidate.score, reverse=True)
+
+
+def boosted_fused_score(candidate: SearchResult, phrase_hit_count: int, phrase_hit_boost: float) -> float | None:
+    """Return a fused score adjusted by phrase hits when a fused score exists."""
+    if candidate.fused_score is None:
+        return None
+    return candidate.fused_score + (phrase_hit_count * phrase_hit_boost)
+
+
+def phrase_rank_source(rank_source: str, phrase_hit_count: int) -> str:
+    """Append phrase evidence to a rank-source label when a chunk was boosted."""
+    if phrase_hit_count <= 0 or "phrases" in rank_source:
+        return rank_source
+    return f"{rank_source} + phrases"
+
+
+async def parallel_retrieval(
+    engine: AsyncEngine,
+    client: httpx.AsyncClient,
+    query: str,
+    config: EbookSearchConfig,
+    *,
+    phrase_matching: bool,
 ) -> RetrievalResponse:
-    """Run vector and BM25 candidate retrieval concurrently with separate database sessions."""
-    with ThreadPoolExecutor(max_workers=2, thread_name_prefix="ebook-search") as executor:
-        vector_future = executor.submit(
-            timed_result,
-            "Embedding + vector search",
-            vector_candidates,
-            engine,
-            query,
-            config,
-        )
-        bm25_future = executor.submit(
-            timed_result,
-            "BM25 search",
-            bm25_candidates,
-            query,
-            config,
-        )
-        vector_results, vector_timing = vector_future.result()
-        lexical_results, lexical_timing = bm25_future.result()
+    """Run vector, BM25, and protected phrase retrieval concurrently with separate database sessions.
+
+    BM25 scoring is pure CPU work over the cached corpus, so it runs in a worker thread
+    instead of on the event loop. Protected phrase detection only depends on the query, so
+    it joins the gather as a third task and returns immediately when phrase matching is disabled.
+    """
+    phrase_timing_name = "Protected phrase detection" if phrase_matching else "Protected phrase detection skipped"
+    (
+        (vector_results, vector_timing),
+        (lexical_results, lexical_timing),
+        (phrase_matches, phrase_timing),
+    ) = await asyncio.gather(
+        async_timed_result("Embedding + vector search", vector_candidates(engine, client, query, config)),
+        async_timed_result("BM25 search", asyncio.to_thread(bm25_candidates, query, config)),
+        async_timed_result(
+            phrase_timing_name,
+            query_phrase_matches(engine, query, config, phrase_matching=phrase_matching),
+        ),
+    )
 
     logger.info(
-        "ebook_parallel_retrieval_complete vector_candidates=%s lexical_candidates=%s",
-        len(vector_results),
-        len(lexical_results),
+        f"ebook_parallel_retrieval_complete vector_candidates={len(vector_results)} "
+        f"lexical_candidates={len(lexical_results)} phrase_matches={len(phrase_matches)}"
     )
     return RetrievalResponse(
         vector_results=vector_results,
         lexical_results=lexical_results,
+        phrase_matches=phrase_matches,
         timings=(
             replace(vector_timing, counts_toward_total=False),
             replace(lexical_timing, counts_toward_total=False),
+            replace(phrase_timing, counts_toward_total=False),
         ),
     )
 
 
-def skip_rerank(
+async def apply_rerank(
+    client: httpx.AsyncClient,
     query: str,
     candidates: list[SearchResult],
     config: EbookSearchConfig,
+    *,
+    rerank: bool,
 ) -> SearchResponse:
-    """Return fused hybrid results without reranking."""
-    logger.info("ebook_rerank_skipped candidates=%s", len(candidates))
-    return SearchResponse(query=query, results=candidates[: config.top_k], rank_label="Hybrid")
+    """Rerank already-fused hybrid candidates when enabled for this request."""
+    if not rerank:
+        logger.info(f"ebook_rerank_skipped candidates={len(candidates)}")
+        return SearchResponse(query=query, results=candidates[: config.top_k], rank_label="Hybrid")
 
-
-def apply_rerank(
-    query: str,
-    candidates: list[SearchResult],
-    config: EbookSearchConfig,
-) -> SearchResponse:
-    """Rerank already-fused hybrid candidates."""
-    reranked = rerank_chunks(query, candidates[: config.rerank.candidates], config.rerank)
+    reranked = await rerank_chunks(client, query, candidates[: config.rerank.candidates], config.rerank)
     logger.info(
-        "ebook_rerank_complete input_candidates=%s returned=%s",
-        min(len(candidates), config.rerank.candidates),
-        len(reranked),
+        f"ebook_rerank_complete input_candidates={min(len(candidates), config.rerank.candidates)} "
+        f"returned={len(reranked)}"
     )
     return SearchResponse(
         query=query,
@@ -199,10 +305,17 @@ def apply_rerank(
     )
 
 
-def vector_candidates(engine: Engine, query: str, config: EbookSearchConfig) -> list[SearchResult]:
+async def vector_candidates(
+    engine: AsyncEngine,
+    client: httpx.AsyncClient,
+    query: str,
+    config: EbookSearchConfig,
+) -> list[SearchResult]:
     """Return pgvector cosine candidates for a natural-language query."""
-    with Session(engine) as session:
-        model = session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == config.embedding_model))
+    async with AsyncSession(engine) as session:
+        model = await session.scalar(
+            select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == config.embedding_model)
+        )
         if model is None:
             msg = f"Embedding model is not registered: {config.embedding_model}"
             raise ValueError(msg)
@@ -212,7 +325,7 @@ def vector_candidates(engine: Engine, query: str, config: EbookSearchConfig) ->
             msg = f"Model row dimension {model.dimension} does not match configured dimension {expected_dimension}"
             raise ValueError(msg)
 
-        embedding = embed_query(query, config)
+        embedding = await embed_query(client, query, config)
         limit = max(config.rerank.candidates, config.top_k) * config.vector_candidate_multiplier
         embedding_table = get_embedding_table(model.dimension)
 
@@ -221,12 +334,7 @@ def vector_candidates(engine: Engine, query: str, config: EbookSearchConfig) ->
         score = (literal(1.0) - distance).label("score")
         statement = (
             select(
-                EbookChunk.id.label("chunk_id"),
-                EbookChunk.text.label("text"),
-                EbookSource.title.label("source_title"),
-                EbookSource.author.label("source_author"),
-                EbookChapter.title.label("chapter_title"),
-                EbookChunk.page_label.label("page_label"),
+                *CHUNK_RECORD_COLUMNS,
                 score,
             )
             .select_from(embedding_table)
@@ -237,13 +345,10 @@ def vector_candidates(engine: Engine, query: str, config: EbookSearchConfig) ->
             .order_by(distance)
             .limit(limit)
         )
-        rows = session.execute(statement).mappings()
+        rows = (await session.execute(statement)).mappings()
         results = [search_result_from_row(row) for row in rows]
         logger.info(
-            "ebook_vector_search_complete model=%s dimension=%s candidates=%s",
-            config.embedding_model,
-            model.dimension,
-            len(results),
+            f"ebook_vector_search_complete {config.embedding_model=} {model.dimension=} candidates={len(results)}"
         )
         return results
 
@@ -253,7 +358,7 @@ def bm25_candidates(query: str, config: EbookSearchConfig) -> list[SearchResult]
     try:
         corpus = load_bm25_corpus(config)
     except BM25CorpusUnavailableError as error:
-        logger.warning("ebook_bm25_index_unavailable_skipping error=%s", error)
+        logger.warning(f"ebook_bm25_index_unavailable_skipping {error=}")
         return []
 
     if not corpus.records:
@@ -268,12 +373,7 @@ def bm25_candidates(query: str, config: EbookSearchConfig) -> list[SearchResult]
     ]
 
     max_score = results[0].bm25_score if results else 0.0
-    logger.info(
-        "ebook_bm25_search_complete corpus=%s candidates=%s max_score=%.6f",
-        len(corpus.records),
-        len(results),
-        max_score,
-    )
+    logger.info(f"ebook_bm25_search_complete corpus={len(corpus.records)} candidates={len(results)} {max_score=:.6f}")
     return results
 
 
@@ -317,9 +417,11 @@ def reciprocal_rank_fusion(
 
 def search_result_from_row(row: Mapping[str, object]) -> SearchResult:
     """Convert a database row mapping into a search result."""
+    source_id = row.get("source_id")
     return SearchResult(
         chunk_id=int(row["chunk_id"]),
         text=str(row["text"]),
+        source_id=int(source_id) if source_id is not None else None,
         source_title=str(row["source_title"]),
         source_author=optional_str(row["source_author"]),
         chapter_title=optional_str(row["chapter_title"]),

python/ebook_search/timing.py

@@ -7,7 +7,7 @@ from time import perf_counter
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from collections.abc import Callable
+    from collections.abc import Awaitable, Callable
 
 
 @dataclass(frozen=True)
@@ -34,3 +34,10 @@ def timed_result[T, **P](
     start_seconds = perf_counter()
     result = operation(*args, **kwargs)
     return result, runtime_step_from_start(name, start_seconds)
+
+
+async def async_timed_result[T](name: str, awaitable: Awaitable[T]) -> tuple[T, RuntimeStep]:
+    """Await an operation and return its result plus elapsed runtime."""
+    start_seconds = perf_counter()
+    result = await awaitable
+    return result, runtime_step_from_start(name, start_seconds)

python/fastapi_tools/__init__.py

@@ -1,6 +1,25 @@
 """Reusable FastAPI tools."""
 
-from python.fastapi_tools.db import DbSession, get_db
+from python.fastapi_tools.db import (
+    AppAsyncEngine,
+    AppEngine,
+    AsyncDbSession,
+    DbSession,
+    get_async_db,
+    get_async_engine,
+    get_db,
+    get_engine,
+)
 from python.fastapi_tools.zstd_middleware import ZstdMiddleware
 
-__all__ = ["DbSession", "ZstdMiddleware", "get_db"]
+__all__ = [
+    "AppAsyncEngine",
+    "AppEngine",
+    "AsyncDbSession",
+    "DbSession",
+    "ZstdMiddleware",
+    "get_async_db",
+    "get_async_engine",
+    "get_db",
+    "get_engine",
+]

python/fastapi_tools/db.py

@@ -5,10 +5,22 @@ from __future__ import annotations
 from typing import TYPE_CHECKING, Annotated
 
 from fastapi import Depends, Request
+from sqlalchemy.engine import Engine
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession
 from sqlalchemy.orm import Session
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator
+    from collections.abc import AsyncIterator, Iterator
+
+
+def get_engine(request: Request) -> Engine:
+    """Get a synchronous database engine from app state."""
+    return request.app.state.engine
+
+
+def get_async_engine(request: Request) -> AsyncEngine:
+    """Get an asynchronous database engine from app state."""
+    return request.app.state.engine
 
 
 def get_db(request: Request) -> Iterator[Session]:
@@ -17,4 +29,17 @@ def get_db(request: Request) -> Iterator[Session]:
         yield session
 
 
+async def get_async_db(request: Request) -> AsyncIterator[AsyncSession]:
+    """Get an async database session from app state.
+
+    expire_on_commit=False keeps ORM attributes readable after commit without
+    triggering implicit IO, which would raise under asyncio.
+    """
+    async with AsyncSession(request.app.state.engine, expire_on_commit=False) as session:
+        yield session
+
+
+AppEngine = Annotated[Engine, Depends(get_engine)]
+AppAsyncEngine = Annotated[AsyncEngine, Depends(get_async_engine)]
 DbSession = Annotated[Session, Depends(get_db)]
+AsyncDbSession = Annotated[AsyncSession, Depends(get_async_db)]

python/gems/README.md

@@ -0,0 +1,54 @@
+# Gems
+
+Gems is a server-rendered, turn-based resource-engine game for one to four human or AI players. It uses FastAPI,
+Jinja, HTMX, server-sent events, and SQLite.
+
+The application deliberately contains no playable card deck, patron/governor set, objective set, official artwork,
+or copied rulebook text. A room host must upload a content pack they are entitled to use before starting a game.
+
+## Run locally
+
+```shell
+uv run gems --host 127.0.0.1 --port 8082
+```
+
+The default database and installation key are created under `.gems/`. The following environment variables override
+runtime behavior:
+
+- `GEMS_DATABASE_PATH`
+- `GEMS_KEY_PATH`
+- `GEMS_PUBLIC_ORIGIN`
+- `GEMS_SECURE_COOKIES`
+- `GEMS_HOST`
+- `GEMS_PORT`
+
+## Content packs
+
+The current schema is available from a running server at `/schemas/content-pack-v1.json`. A pack defines exactly
+five normal resources, one wild resource, cards, and optional patrons, objectives, and outposts. `patrons` is the
+canonical field name; `governors` is accepted as an input alias.
+
+Cards may use only the built-in, bounded effect vocabulary:
+
+- `none`
+- `virtual_wild`
+- `copy_bonus`
+- `copy_and_claim`
+- `multi_bonus`
+- `claim_free`
+- an optional discard-cards alternate cost
+
+Unknown fields, resource references, executable expressions, HTML, artwork URLs, and files larger than 512 KiB are
+rejected. The normalized pack is private to its room and becomes immutable when play starts.
+
+## Neutral module mapping
+
+Gems calls the four optional mechanics Objectives, Outposts, Eastern Decks, and Fortifications. Lobby presets combine
+these mechanics into the familiar base, objective-race, objective-plus-outpost, eastern-plus-fortification, and
+all-module configurations. Component identities and values always come from the uploaded pack.
+
+## Jeeves
+
+The NixOS module runs one Uvicorn worker on `127.0.0.1:8002`, stores state in
+`/zfs/media/services/gems`, and publishes it through HAProxy at `https://gems.tmmworkshop.com`. The DNS record must
+point to Jeeves before ACME can issue the certificate.

python/gems/__init__.py

@@ -0,0 +1 @@
+"""Gems multiplayer card-engine game."""

python/gems/ai/__init__.py

@@ -0,0 +1,5 @@
+"""AI controllers for Gems."""
+
+from .runner import choose_ai_command
+
+__all__ = ["choose_ai_command"]

python/gems/ai/runner.py

@@ -0,0 +1,68 @@
+"""Deterministic AI command selection at three difficulty levels."""
+
+from __future__ import annotations
+
+import random
+from typing import TYPE_CHECKING
+
+from python.gems.domain.engine import RuleError, apply_command, score
+from python.gems.domain.legal_actions import legal_commands
+
+if TYPE_CHECKING:
+    from python.gems.domain.models import ContentPack, GameCommand, GameSettings, GameState
+
+
+def choose_ai_command(
+    state: GameState,
+    pack: ContentPack,
+    settings: GameSettings,
+    seat: int,
+    difficulty: str,
+) -> GameCommand:
+    """Choose from legal commands using only public/current-player information."""
+    actions = legal_commands(state, pack, settings, seat)
+    if not actions:
+        error = "AI has no legal command"
+        raise RuntimeError(error)
+    rng = random.Random(f"{state.seed}:{state.revision}:{seat}:{difficulty}")  # noqa: S311 - deterministic AI
+    if difficulty == "easy":
+        return rng.choice(actions)
+
+    ranked = sorted(actions, key=lambda item: _heuristic(item, state, pack), reverse=True)
+    if difficulty == "medium":
+        return ranked[0]
+
+    # Bounded deterministic rollout: examine at most the twelve strongest actions.
+    best = ranked[0]
+    best_value = float("-inf")
+    for candidate in ranked[:12]:
+        try:
+            future = apply_command(state, candidate, pack, settings, actor_seat=seat)
+        except RuleError:
+            continue
+        value = score(future.players[seat], pack) * 100 + _heuristic(candidate, state, pack)
+        value += rng.random() * 0.001
+        if value > best_value:
+            best, best_value = candidate, value
+    return best
+
+
+def _heuristic(command: GameCommand, state: GameState, pack: ContentPack) -> float:
+    player = state.players[state.current_seat]
+    value = 0.0
+    if command.type == "purchase":
+        card = pack.card(str(command.payload.get("card_id")))
+        value += 40 + card.points * 25 + (8 if card.bonus_resource else 0)
+        value -= sum(command.payload.get("payment", {}).values())
+    elif command.type == "take_distinct":
+        value += 10 + len(command.payload.get("resources", []))
+    elif command.type == "reserve":
+        value += 6
+    elif command.type == "take_double":
+        value += 4
+    elif command.type == "choose":
+        value += 20
+    elif command.type == "decline":
+        value -= 2
+    value += score(player, pack) * 0.01
+    return value

python/gems/config.py

@@ -0,0 +1,25 @@
+"""Runtime configuration loaded from environment variables."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class GemsConfig(BaseSettings):
+    """Configure persistence, public URLs, cookies, and the HTTP server."""
+
+    model_config = SettingsConfigDict(env_prefix="GEMS_")
+
+    database_path: Path = Path(".gems/gems.sqlite3")
+    key_path: Path = Path(".gems/instance.key")
+    public_origin: str = "http://127.0.0.1:8082"
+    secure_cookies: bool = False
+    host: str = "127.0.0.1"
+    port: int = 8082
+
+
+def load_config() -> GemsConfig:
+    """Load Gems configuration from defaults and environment variables."""
+    return GemsConfig()

python/gems/content.py

@@ -0,0 +1,55 @@
+"""Content-pack parsing, normalization, and JSON Schema publication."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from typing import Any
+
+from pydantic import ValidationError
+
+from .domain.models import ContentPack
+
+MAX_PACK_BYTES = 512 * 1024
+
+
+@dataclass(frozen=True, slots=True)
+class ParsedPack:
+    """Validated content plus its canonical representation."""
+
+    pack: ContentPack
+    canonical_json: str
+    digest: str
+
+
+class ContentPackError(ValueError):
+    """A safe, user-visible content validation error."""
+
+
+def parse_content_pack(raw: bytes | str) -> ParsedPack:
+    """Validate a content pack and return stable canonical JSON."""
+    data = raw.encode() if isinstance(raw, str) else raw
+    if len(data) > MAX_PACK_BYTES:
+        error = "Content pack exceeds the 512 KiB limit"
+        raise ContentPackError(error)
+    try:
+        decoded: Any = json.loads(data)
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        error = f"Invalid JSON: {exc}"
+        raise ContentPackError(error) from exc
+    try:
+        pack = ContentPack.model_validate(decoded)
+    except ValidationError as exc:
+        messages = []
+        for issue in exc.errors(include_url=False):
+            path = ".".join(str(part) for part in issue["loc"])
+            messages.append(f"{path or '$'}: {issue['msg']}")
+        raise ContentPackError("\n".join(messages)) from exc
+    canonical = json.dumps(pack.model_dump(mode="json", by_alias=False), sort_keys=True, separators=(",", ":"))
+    return ParsedPack(pack=pack, canonical_json=canonical, digest=hashlib.sha256(canonical.encode()).hexdigest())
+
+
+def content_pack_schema() -> dict[str, Any]:
+    """Return the authoritative version-one JSON Schema."""
+    return ContentPack.model_json_schema(by_alias=False)

python/gems/domain/__init__.py

@@ -0,0 +1 @@
+"""Domain models and rules for Gems."""

python/gems/domain/engine.py

@@ -0,0 +1,903 @@
+"""Deterministic, server-authoritative Gems rule engine."""
+
+from __future__ import annotations
+
+import random
+from typing import Any
+
+from .models import (
+    CardDefinition,
+    CardEffectKind,
+    ContentPack,
+    GameCommand,
+    GameSettings,
+    GameState,
+    OutpostDefinition,
+    OutpostPower,
+    OwnedCard,
+    PendingChoice,
+    PlayerState,
+)
+from .requirements import requirements_met
+
+MAX_PLAYERS = 4
+TWO_PLAYER_COUNT = 2
+THREE_PLAYER_COUNT = 3
+DISTINCT_TAKE_COUNT = 3
+DOUBLE_TAKE_MINIMUM = 4
+BLIND_RESERVE_DRAW_COUNT = 2
+
+
+class RuleError(ValueError):
+    """A command was not legal for the current state."""
+
+
+def deck_key(deck: str, tier: int) -> str:
+    """Build the state key for a named deck and tier."""
+    return f"{deck}:{tier}"
+
+
+def new_game(
+    room_code: str,
+    names: list[str],
+    pack: ContentPack,
+    settings: GameSettings,
+    *,
+    seed: int,
+) -> GameState:
+    """Create and deal a deterministic game."""
+    if not 1 <= len(names) <= MAX_PLAYERS:
+        error = "Games require one to four seats"
+        raise RuleError(error)
+    if settings.first_player_mode == "selected" and settings.first_player_seat >= len(names):
+        error = "The selected first player is no longer in the room"
+        raise RuleError(error)
+    _validate_startable(pack, settings)
+    rng = random.Random(seed)  # noqa: S311 - deterministic seeded shuffle
+    first_seat = settings.first_player_seat if settings.first_player_mode == "selected" else rng.randrange(len(names))
+    normal_count = 4 if len(names) <= TWO_PLAYER_COUNT else 5 if len(names) == THREE_PLAYER_COUNT else 7
+    supply = dict.fromkeys(pack.resource_ids, normal_count)
+    supply[pack.wild_resource.id] = 5
+    players = [
+        PlayerState(
+            seat=index,
+            name=name,
+            tokens=dict.fromkeys((*pack.resource_ids, pack.wild_resource.id), 0),
+            fortifications_available=settings.fortifications_per_player if settings.modules.fortifications else 0,
+        )
+        for index, name in enumerate(names)
+    ]
+    decks: dict[str, list[str]] = {}
+    markets: dict[str, list[str]] = {}
+    for source in ("base", "eastern"):
+        if source == "eastern" and not settings.modules.eastern_decks:
+            continue
+        market_size = settings.base_market_size if source == "base" else settings.eastern_market_size
+        for tier in (1, 2, 3):
+            key = deck_key(source, tier)
+            cards = [card.id for card in pack.cards if card.deck == source and card.tier == tier]
+            rng.shuffle(cards)
+            markets[key] = [cards.pop() for _ in range(min(market_size, len(cards)))]
+            decks[key] = cards
+    patrons = [patron.id for patron in pack.patrons]
+    rng.shuffle(patrons)
+    patrons = patrons[: min(len(patrons), len(players) + 1)]
+    objectives = [objective.id for objective in pack.objectives]
+    rng.shuffle(objectives)
+    objectives = objectives[: settings.objective_count] if settings.modules.objectives else []
+    return GameState(
+        room_code=room_code,
+        seed=seed,
+        players=players,
+        supply=supply,
+        decks=decks,
+        markets=markets,
+        available_patrons=[] if settings.modules.objectives else patrons,
+        available_objectives=objectives,
+        current_seat=first_seat,
+        first_seat=first_seat,
+    )
+
+
+def _validate_startable(pack: ContentPack, settings: GameSettings) -> None:
+    for tier in (1, 2, 3):
+        if not any(card.deck == "base" and card.tier == tier for card in pack.cards):
+            error = f"The base deck has no tier-{tier} cards"
+            raise RuleError(error)
+        if settings.modules.eastern_decks and not any(
+            card.deck == "eastern" and card.tier == tier for card in pack.cards
+        ):
+            error = f"The eastern deck has no tier-{tier} cards"
+            raise RuleError(error)
+    if settings.modules.objectives and not pack.objectives:
+        error = "The Objectives module needs objective definitions"
+        raise RuleError(error)
+    if settings.modules.outposts and not pack.outposts:
+        error = "The Outposts module needs outpost definitions"
+        raise RuleError(error)
+
+
+def card_map(pack: ContentPack) -> dict[str, CardDefinition]:
+    """Index all cards in a content pack by identifier."""
+    return {card.id: card for card in pack.cards}
+
+
+def bonuses(player: PlayerState, pack: ContentPack) -> dict[str, int]:
+    """Calculate effective bonuses after copies, multiples, and discards."""
+    cards = card_map(pack)
+    result = dict.fromkeys(pack.resource_ids, 0)
+    for owned in player.cards:
+        definition = cards[owned.card_id]
+        resource = owned.copied_resource or definition.bonus_resource
+        if resource is not None:
+            result[resource] += definition.effect.amount if definition.effect.kind == CardEffectKind.MULTI_BONUS else 1
+    return result
+
+
+def score(player: PlayerState, pack: ContentPack) -> int:
+    """Calculate a player's score from cards, patrons, and outposts."""
+    cards = card_map(pack)
+    patrons = {patron.id: patron for patron in pack.patrons}
+    outposts = {outpost.id: outpost for outpost in pack.outposts}
+    value = sum(cards[item.card_id].points for item in player.cards)
+    value += sum(patrons[item].points for item in player.patrons)
+    for item in player.outposts:
+        definition = outposts[item]
+        if definition.power == OutpostPower.POINTS_PER_OUTPOST:
+            value += definition.value * len(player.outposts)
+    return value
+
+
+def affordable_payments(player: PlayerState, card: CardDefinition, pack: ContentPack, *, double_wild: bool) -> bool:
+    """Report whether a player can cover a card's standard cost."""
+    discount = bonuses(player, pack)
+    shortage = sum(
+        max(0, amount - discount.get(resource, 0) - player.tokens.get(resource, 0))
+        for resource, amount in card.cost.items()
+    )
+    wild_value = 2 if double_wild else 1
+    return shortage <= player.tokens.get(pack.wild_resource.id, 0) * wild_value
+
+
+def visible_cards(state: GameState) -> set[str]:
+    """Return the identifiers of cards currently visible in markets."""
+    return {card for market in state.markets.values() for card in market}
+
+
+def apply_command(
+    state: GameState,
+    command: GameCommand,
+    pack: ContentPack,
+    settings: GameSettings,
+    *,
+    actor_seat: int,
+) -> GameState:
+    """Validate and apply one command, returning a new state snapshot."""
+    if state.finished:
+        error = "The game is finished"
+        raise RuleError(error)
+    if command.expected_revision != state.revision:
+        error = "The board changed; refresh and try again"
+        raise RuleError(error)
+    if actor_seat != state.current_seat:
+        error = "It is not your turn"
+        raise RuleError(error)
+    if state.pending and state.pending.seat != actor_seat:
+        error = "Another player must resolve the pending choice"
+        raise RuleError(error)
+
+    result = state.model_copy(deep=True)
+    if result.pending:
+        _apply_pending(result, command, pack, settings)
+    elif command.type == "take_distinct":
+        _take_distinct(result, command.payload, pack, settings)
+    elif command.type == "take_double":
+        _take_double(result, command.payload, pack, settings)
+    elif command.type == "reserve":
+        _reserve(result, command.payload, pack, settings)
+    elif command.type == "purchase":
+        _purchase(result, command.payload, pack, settings, is_conquest=False)
+    else:
+        error = "Choose a normal turn action"
+        raise RuleError(error)
+    result.revision += 1
+    return result
+
+
+def _take_distinct(state: GameState, payload: dict[str, Any], pack: ContentPack, settings: GameSettings) -> None:
+    colors = payload.get("resources")
+    if not isinstance(colors, list) or len(colors) != len(set(colors)):
+        error = "Choose distinct resources"
+        raise RuleError(error)
+    available = [resource for resource in pack.resource_ids if state.supply.get(resource, 0) > 0]
+    required = DISTINCT_TAKE_COUNT if len(available) >= DISTINCT_TAKE_COUNT else None
+    if required is not None and len(colors) != required:
+        error = "Take exactly three different resources when possible"
+        raise RuleError(error)
+    if required is None and not 1 <= len(colors) <= len(available):
+        error = "Choose one or more available resources"
+        raise RuleError(error)
+    if not set(colors) <= set(available):
+        error = "A selected resource is unavailable"
+        raise RuleError(error)
+    player = state.players[state.current_seat]
+    for resource in colors:
+        state.supply[resource] -= 1
+        player.tokens[resource] += 1
+    state.log.append(f"{player.name} took {len(colors)} different resources")
+    _after_standard_action(state, pack, settings, action="take_distinct")
+
+
+def _take_double(state: GameState, payload: dict[str, Any], pack: ContentPack, settings: GameSettings) -> None:
+    resource = payload.get("resource")
+    if resource not in pack.resource_ids or state.supply.get(resource, 0) < DOUBLE_TAKE_MINIMUM:
+        error = "That resource cannot be taken twice"
+        raise RuleError(error)
+    player = state.players[state.current_seat]
+    state.supply[resource] -= 2
+    player.tokens[resource] += 2
+    state.log.append(f"{player.name} took two {resource} resources")
+    extra = _owned_outpost(player, pack, OutpostPower.RESOURCE_AFTER_DOUBLE)
+    if settings.modules.outposts and extra:
+        options = [item for item in pack.resource_ids if item != resource and state.supply.get(item, 0) > 0]
+        if options:
+            state.pending = PendingChoice(
+                kind="resource", seat=player.seat, options=options, context={"next": "after_action"}
+            )
+            return
+    _after_standard_action(state, pack, settings, action="take_double")
+
+
+def _reserve(state: GameState, payload: dict[str, Any], pack: ContentPack, settings: GameSettings) -> None:
+    player = state.players[state.current_seat]
+    if len(player.reserved) >= settings.reserve_limit:
+        error = "Your reserve is full"
+        raise RuleError(error)
+    card_id = payload.get("card_id")
+    source = payload.get("deck")
+    drawn: list[str]
+    if card_id:
+        if card_id not in visible_cards(state):
+            error = "That card is not visible"
+            raise RuleError(error)
+        _assert_not_blocked(state, card_id, player.seat)
+        _remove_visible(state, card_id)
+        drawn = [card_id]
+    elif isinstance(source, str) and source in state.decks:
+        if not state.decks[source]:
+            error = "That deck is empty"
+            raise RuleError(error)
+        draw_count = 1
+        post = _owned_outpost(player, pack, OutpostPower.BLIND_RESERVE_TWO)
+        if post and (not source.startswith("eastern:") or settings.interactions.outpost_reserve_applies_eastern):
+            draw_count = min(BLIND_RESERVE_DRAW_COUNT, len(state.decks[source]))
+        drawn = [state.decks[source].pop() for _ in range(draw_count)]
+    else:
+        error = "Choose a visible card or deck"
+        raise RuleError(error)
+    if len(drawn) == BLIND_RESERVE_DRAW_COUNT:
+        state.pending = PendingChoice(kind="reserve_keep", seat=player.seat, options=drawn, context={"deck": source})
+        return
+    _finish_reserve(state, drawn[0], pack, settings)
+
+
+def _finish_reserve(state: GameState, card_id: str, pack: ContentPack, settings: GameSettings) -> None:
+    player = state.players[state.current_seat]
+    player.reserved.append(card_id)
+    _refill_markets(state, settings)
+    wild = pack.wild_resource.id
+    if state.supply[wild] > 0:
+        state.supply[wild] -= 1
+        player.tokens[wild] += 1
+    state.log.append(f"{player.name} reserved a card")
+    _after_standard_action(state, pack, settings, action="reserve")
+
+
+def _purchase(
+    state: GameState,
+    payload: dict[str, Any],
+    pack: ContentPack,
+    settings: GameSettings,
+    *,
+    is_conquest: bool,
+) -> None:
+    player = state.players[state.current_seat]
+    card_id = payload.get("card_id")
+    if not isinstance(card_id, str):
+        error = "Choose a card"
+        raise RuleError(error)
+    from_reserve = card_id in player.reserved
+    if not from_reserve:
+        if card_id not in visible_cards(state):
+            error = "That card is not available"
+            raise RuleError(error)
+        _assert_not_blocked(state, card_id, player.seat)
+    if is_conquest and state.fortifications.get(card_id, {}).get(player.seat, 0) < settings.fortifications_per_player:
+        error = "All your fortifications must occupy the conquest card"
+        raise RuleError(error)
+    card = pack.card(card_id)
+    _pay_for_purchase(state, player, card, payload, pack, settings)
+    if from_reserve:
+        player.reserved.remove(card_id)
+    else:
+        _remove_visible(state, card_id)
+    _return_fortifications(state, card_id)
+    owned = OwnedCard(card_id=card_id)
+    player.cards.append(owned)
+    player.purchased_card_count += 1
+    state.turn_purchase_count += 1
+    state.log.append(f"{player.name} purchased {card.label}")
+    if card.effect.kind in {CardEffectKind.COPY_BONUS, CardEffectKind.COPY_AND_CLAIM}:
+        # Do not allow the new copy card to satisfy its own target requirement.
+        prior = player.cards.pop()
+        options = sorted(resource for resource, amount in bonuses(player, pack).items() if amount)
+        player.cards.append(prior)
+        if not options:
+            error = "This card requires an existing bonus to copy"
+            raise RuleError(error)
+        state.pending = PendingChoice(
+            kind="copy_bonus",
+            seat=player.seat,
+            options=options,
+            context={"card_id": card_id, "is_conquest": is_conquest},
+        )
+        return
+    if card.effect.kind == CardEffectKind.CLAIM_FREE:
+        _queue_free_card(state, card, pack, settings, is_conquest=is_conquest)
+        return
+    _after_purchase(state, pack, settings, is_conquest=is_conquest)
+
+
+def _pay_for_purchase(
+    state: GameState,
+    player: PlayerState,
+    card: CardDefinition,
+    payload: dict[str, Any],
+    pack: ContentPack,
+    settings: GameSettings,
+) -> None:
+    if card.alternate_cost:
+        discarded = payload.get("discard_cards", [])
+        _pay_alternate(player, card, discarded, pack)
+        if not settings.interactions.retain_outposts_after_discard:
+            _reconcile_outposts(player, pack)
+    else:
+        payment = payload.get("payment", {})
+        virtual_cards = payload.get("virtual_wild_cards", [])
+        _pay_tokens(state, player, card, payment, virtual_cards, pack, settings)
+
+
+def _pay_tokens(  # noqa: C901 - payment validation mirrors the rule sequence
+    state: GameState,
+    player: PlayerState,
+    card: CardDefinition,
+    payment: object,
+    virtual_cards: object,
+    pack: ContentPack,
+    settings: GameSettings,
+) -> None:
+    if not isinstance(payment, dict) or any(not isinstance(value, int) or value < 0 for value in payment.values()):
+        error = "Provide a valid payment"
+        raise RuleError(error)
+    allowed = {*pack.resource_ids, pack.wild_resource.id}
+    if not set(payment) <= allowed:
+        error = "Payment contains an unknown resource"
+        raise RuleError(error)
+    if any(payment.get(resource, 0) > player.tokens.get(resource, 0) for resource in allowed):
+        error = "Payment uses resources you do not have"
+        raise RuleError(error)
+    discount = bonuses(player, pack)
+    if not isinstance(virtual_cards, list) or len(virtual_cards) != len(set(virtual_cards)):
+        error = "Virtual-wild selections must be unique"
+        raise RuleError(error)
+    owned_by_id = {item.card_id: item for item in player.cards}
+    virtual_value = 0
+    for card_id in virtual_cards:
+        owned = owned_by_id.get(card_id)
+        if owned is None:
+            error = "You do not own a selected virtual-wild card"
+            raise RuleError(error)
+        definition = pack.card(card_id)
+        if definition.effect.kind != CardEffectKind.VIRTUAL_WILD:
+            error = "A selected card does not grant virtual wild resources"
+            raise RuleError(error)
+        virtual_value += definition.effect.amount
+    wild = payment.get(pack.wild_resource.id, 0)
+    double = bool(_owned_outpost(player, pack, OutpostPower.DOUBLE_WILD))
+    wild_value = 2 if double else 1
+    virtual_multiplier = 2 if double and settings.interactions.virtual_wild_can_double else 1
+    remaining_wild_value = wild * wild_value + virtual_value * virtual_multiplier
+    for resource in pack.resource_ids:
+        due = max(0, card.cost.get(resource, 0) - discount.get(resource, 0))
+        normal = payment.get(resource, 0)
+        if normal > due:
+            error = "Payment overpays a normal resource"
+            raise RuleError(error)
+        remaining_wild_value -= due - normal
+    if remaining_wild_value < 0:
+        error = "Payment does not exactly cover the card cost"
+        raise RuleError(error)
+    for resource, amount in payment.items():
+        player.tokens[resource] -= amount
+        state.supply[resource] += amount
+    if virtual_cards:
+        player.cards = [item for item in player.cards if item.card_id not in virtual_cards]
+
+
+def _pay_alternate(player: PlayerState, card: CardDefinition, selected: object, pack: ContentPack) -> None:
+    if not isinstance(selected, list) or not card.alternate_cost or len(selected) != card.alternate_cost.count:
+        error = "Choose the required owned cards to discard"
+        raise RuleError(error)
+    if len(selected) != len(set(selected)):
+        error = "A card can only be discarded once"
+        raise RuleError(error)
+    owned_by_id = {item.card_id: item for item in player.cards}
+    for card_id in selected:
+        owned = owned_by_id.get(card_id)
+        if owned is None:
+            error = "You do not own a selected discard"
+            raise RuleError(error)
+        definition = pack.card(card_id)
+        effective = owned.copied_resource or definition.bonus_resource
+        if effective != card.alternate_cost.discard_resource:
+            error = "A selected discard has the wrong color"
+            raise RuleError(error)
+    # Copy cards of the effective color must be discarded first.
+    matching_copies = [
+        item.card_id for item in player.cards if item.copied_resource == card.alternate_cost.discard_resource
+    ]
+    if any(item not in selected for item in matching_copies[: min(len(matching_copies), len(selected))]):
+        error = "Copied cards of this color must be discarded first"
+        raise RuleError(error)
+    player.cards = [item for item in player.cards if item.card_id not in selected]
+
+
+def _apply_pending(  # noqa: C901, PLR0911, PLR0912, PLR0915 - explicit pending-choice state machine
+    state: GameState, command: GameCommand, pack: ContentPack, settings: GameSettings
+) -> None:
+    pending = state.pending
+    if pending is None or command.type not in {"choose", "decline", "purchase"}:
+        error = "Resolve the pending choice"
+        raise RuleError(error)
+    player = state.players[state.current_seat]
+    choice = command.payload.get("choice")
+    if pending.kind == "reserve_keep":
+        if choice not in pending.options:
+            error = "Choose one of the drawn cards"
+            raise RuleError(error)
+        other = next(item for item in pending.options if item != choice)
+        state.decks[pending.context["deck"]].insert(0, other)
+        state.pending = None
+        _finish_reserve(state, choice, pack, settings)
+        return
+    if pending.kind == "copy_bonus":
+        if choice not in pending.options:
+            error = "Choose an existing bonus"
+            raise RuleError(error)
+        card_id = pending.context["card_id"]
+        owned = next(item for item in reversed(player.cards) if item.card_id == card_id)
+        owned.copied_resource = choice
+        definition = pack.card(card_id)
+        state.pending = None
+        if definition.effect.kind == CardEffectKind.COPY_AND_CLAIM:
+            _queue_free_card(state, definition, pack, settings, is_conquest=bool(pending.context.get("is_conquest")))
+        else:
+            _after_purchase(
+                state,
+                pack,
+                settings,
+                is_conquest=bool(pending.context.get("is_conquest")),
+                chained=bool(pending.context.get("chained")),
+            )
+        return
+    if pending.kind == "free_card":
+        if choice not in pending.options:
+            error = "Choose an eligible free card"
+            raise RuleError(error)
+        if settings.interactions.fortifications_block_free_claim:
+            _assert_not_blocked(state, choice, player.seat)
+        _remove_visible(state, choice)
+        _return_fortifications(state, choice)
+        claimed = pack.card(choice)
+        player.cards.append(OwnedCard(card_id=choice))
+        state.turn_chain_free_count += 1
+        state.log.append(f"{player.name} claimed {claimed.label} without payment")
+        state.pending = None
+        if claimed.effect.kind in {CardEffectKind.COPY_BONUS, CardEffectKind.COPY_AND_CLAIM}:
+            prior = player.cards.pop()
+            options = sorted(resource for resource, amount in bonuses(player, pack).items() if amount)
+            player.cards.append(prior)
+            if options:
+                state.pending = PendingChoice(
+                    kind="copy_bonus",
+                    seat=player.seat,
+                    options=options,
+                    context={
+                        "card_id": choice,
+                        "is_conquest": pending.context.get("is_conquest", False),
+                        "chained": True,
+                    },
+                )
+                return
+        _after_purchase(state, pack, settings, is_conquest=bool(pending.context.get("is_conquest")), chained=True)
+        return
+    if pending.kind == "resource":
+        if choice not in pending.options or state.supply.get(str(choice), 0) <= 0:
+            error = "Choose an available resource"
+            raise RuleError(error)
+        state.supply[str(choice)] -= 1
+        player.tokens[str(choice)] += 1
+        context = pending.context
+        state.pending = None
+        remaining = int(context.get("remaining", 1))
+        if context.get("next") == "after_purchase" and remaining > 1:
+            options = [item for item in pack.resource_ids if state.supply.get(item, 0) > 0]
+            if options:
+                state.pending = PendingChoice(
+                    kind="resource",
+                    seat=player.seat,
+                    options=options,
+                    context={**context, "remaining": remaining - 1},
+                )
+                return
+        if context.get("next") == "after_purchase":
+            _after_purchase(state, pack, settings, is_conquest=bool(context.get("is_conquest")), skip_resource=True)
+        else:
+            _after_standard_action(state, pack, settings, action="take_double")
+        return
+    if pending.kind == "discard_tokens":
+        discard = command.payload.get("tokens", {})
+        allowed = {*pack.resource_ids, pack.wild_resource.id}
+        if not isinstance(discard, dict) or not set(discard) <= allowed or sum(discard.values()) != pending.amount:
+            error = "Discard exactly the required number of resources"
+            raise RuleError(error)
+        for resource, amount in discard.items():
+            if not isinstance(amount, int) or amount <= 0 or amount > player.tokens.get(resource, 0):
+                error = "Invalid token discard"
+                raise RuleError(error)
+        for resource, amount in discard.items():
+            player.tokens[resource] -= amount
+            state.supply[resource] += amount
+        state.pending = None
+        _end_checks(state, pack, settings)
+        return
+    if pending.kind == "patron":
+        if choice not in pending.options:
+            error = "Choose an eligible patron"
+            raise RuleError(error)
+        player.patrons.append(choice)
+        state.available_patrons.remove(choice)
+        state.pending = None
+        _check_outposts_or_objectives(state, pack, settings)
+        return
+    if pending.kind == "outpost":
+        if choice not in pending.options:
+            error = "Choose an eligible outpost"
+            raise RuleError(error)
+        player.outposts.append(choice)
+        state.pending = None
+        _check_objectives(state, pack, settings)
+        return
+    if pending.kind == "fortification":
+        target = command.payload.get("card_id")
+        mode = command.payload.get("mode")
+        if target not in visible_cards(state):
+            error = "Choose a visible card"
+            raise RuleError(error)
+        if (
+            target in state.markets.get(deck_key("eastern", pack.card(target).tier), [])
+            and not settings.interactions.fortifications_on_eastern
+        ):
+            error = "Fortifications cannot occupy eastern cards with this setting"
+            raise RuleError(error)
+        if mode == "place":
+            occupants = state.fortifications.get(target, {})
+            if any(seat != player.seat and count for seat, count in occupants.items()):
+                error = "An opponent occupies that card"
+                raise RuleError(error)
+            source = command.payload.get("from_card")
+            if source:
+                if state.fortifications.get(source, {}).get(player.seat, 0) <= 0:
+                    error = "You have no fortification there"
+                    raise RuleError(error)
+                state.fortifications[source][player.seat] -= 1
+            elif player.fortifications_available > 0:
+                player.fortifications_available -= 1
+            else:
+                error = "Move one of your placed fortifications"
+                raise RuleError(error)
+            state.fortifications.setdefault(target, {})[player.seat] = occupants.get(player.seat, 0) + 1
+        elif mode == "remove":
+            occupants = state.fortifications.get(target, {})
+            opponents = [(seat, count) for seat, count in occupants.items() if seat != player.seat and count == 1]
+            if len(opponents) != 1:
+                error = "Choose a card with exactly one opposing fortification"
+                raise RuleError(error)
+            seat, _ = opponents[0]
+            occupants[seat] = 0
+            state.players[seat].fortifications_available += 1
+        else:
+            error = "Choose place or remove"
+            raise RuleError(error)
+        remaining = int(pending.context.get("remaining", 1))
+        state.pending = None
+        if remaining > 1 and _queue_fortification(state, pack, settings, remaining=remaining - 1):
+            return
+        _after_fortification(state, pack, settings)
+        return
+    if pending.kind == "conquest":
+        if command.type == "decline":
+            state.pending = None
+            _enforce_token_limit_or_checks(state, pack, settings)
+            return
+        if command.type != "purchase":
+            error = "Purchase the conquest card or decline"
+            raise RuleError(error)
+        state.pending = None
+        _purchase(state, command.payload, pack, settings, is_conquest=True)
+        return
+    error = "Unsupported pending choice"
+    raise RuleError(error)
+
+
+def _queue_free_card(
+    state: GameState, card: CardDefinition, pack: ContentPack, settings: GameSettings, *, is_conquest: bool
+) -> None:
+    options = []
+    for card_id in visible_cards(state):
+        candidate = pack.card(card_id)
+        if candidate.tier != card.effect.target_tier:
+            continue
+        if settings.interactions.fortifications_block_free_claim:
+            occupants = state.fortifications.get(card_id, {})
+            if any(seat != state.current_seat and count for seat, count in occupants.items()):
+                continue
+        options.append(card_id)
+    if options:
+        state.pending = PendingChoice(
+            kind="free_card", seat=state.current_seat, options=sorted(options), context={"is_conquest": is_conquest}
+        )
+    else:
+        _after_purchase(state, pack, settings, is_conquest=is_conquest)
+
+
+def _after_purchase(
+    state: GameState,
+    pack: ContentPack,
+    settings: GameSettings,
+    *,
+    is_conquest: bool,
+    chained: bool = False,
+    skip_resource: bool = False,
+) -> None:
+    player = state.players[state.current_seat]
+    _refill_markets(state, settings)
+    resource_post = _owned_outpost(player, pack, OutpostPower.RESOURCE_AFTER_PURCHASE)
+    triggers = not is_conquest or settings.interactions.purchase_resource_on_conquest
+    if settings.modules.outposts and resource_post and triggers and not skip_resource:
+        options = [item for item in pack.resource_ids if state.supply.get(item, 0) > 0]
+        if options:
+            remaining = 1
+            if chained and not settings.interactions.chained_claim_is_not_purchase:
+                remaining += state.turn_chain_free_count
+            state.pending = PendingChoice(
+                kind="resource",
+                seat=player.seat,
+                options=options,
+                context={"next": "after_purchase", "is_conquest": is_conquest, "remaining": remaining},
+            )
+            return
+    if settings.modules.fortifications:
+        remaining = 1
+        if chained and not settings.interactions.one_fortification_decision_per_purchase_chain:
+            remaining += state.turn_chain_free_count
+        if _queue_fortification(state, pack, settings, remaining=remaining):
+            return
+    _after_fortification(state, pack, settings)
+
+
+def _after_fortification(state: GameState, pack: ContentPack, settings: GameSettings) -> None:
+    player = state.players[state.current_seat]
+    conquest = [
+        card_id
+        for card_id, occupants in state.fortifications.items()
+        if occupants.get(player.seat, 0) >= settings.fortifications_per_player and card_id in visible_cards(state)
+    ]
+    if settings.modules.fortifications and conquest:
+        state.turn_chain_free_count = 0
+        state.pending = PendingChoice(kind="conquest", seat=player.seat, options=conquest)
+        return
+    state.turn_chain_free_count = 0
+    _enforce_token_limit_or_checks(state, pack, settings)
+
+
+def _queue_fortification(
+    state: GameState,
+    pack: ContentPack,
+    settings: GameSettings,
+    *,
+    remaining: int,
+) -> bool:
+    player = state.players[state.current_seat]
+    options = [
+        card_id
+        for card_id in visible_cards(state)
+        if settings.interactions.fortifications_on_eastern or pack.card(card_id).deck != "eastern"
+    ]
+    can_act = False
+    for card_id in options:
+        occupants = state.fortifications.get(card_id, {})
+        can_place = not any(seat != player.seat and count for seat, count in occupants.items())
+        can_remove = any(seat != player.seat and count == 1 for seat, count in occupants.items())
+        if can_place or can_remove:
+            can_act = True
+            break
+    if not can_act:
+        return False
+    state.pending = PendingChoice(
+        kind="fortification",
+        seat=player.seat,
+        options=sorted(options),
+        context={"remaining": remaining},
+    )
+    return True
+
+
+def _after_standard_action(state: GameState, pack: ContentPack, settings: GameSettings, *, action: str) -> None:
+    del action
+    if settings.modules.fortifications and state.turn_purchase_count:
+        _after_purchase(state, pack, settings, is_conquest=False)
+    else:
+        _enforce_token_limit_or_checks(state, pack, settings)
+
+
+def _enforce_token_limit_or_checks(state: GameState, pack: ContentPack, settings: GameSettings) -> None:
+    player = state.players[state.current_seat]
+    excess = sum(player.tokens.values()) - settings.token_limit
+    if excess > 0:
+        state.pending = PendingChoice(kind="discard_tokens", seat=player.seat, amount=excess)
+        return
+    _end_checks(state, pack, settings)
+
+
+def _end_checks(state: GameState, pack: ContentPack, settings: GameSettings) -> None:
+    player = state.players[state.current_seat]
+    bonus = bonuses(player, pack)
+    eligible_patrons = [
+        patron.id
+        for patron in pack.patrons
+        if patron.id in state.available_patrons and requirements_met(patron.requirements, bonus)
+    ]
+    if eligible_patrons:
+        state.pending = PendingChoice(kind="patron", seat=player.seat, options=eligible_patrons)
+        return
+    _check_outposts_or_objectives(state, pack, settings)
+
+
+def _check_outposts_or_objectives(state: GameState, pack: ContentPack, settings: GameSettings) -> None:
+    if settings.modules.outposts and settings.interactions.outposts_before_objectives and _queue_outpost(state, pack):
+        return
+    _check_objectives(state, pack, settings)
+
+
+def _queue_outpost(state: GameState, pack: ContentPack) -> bool:
+    player = state.players[state.current_seat]
+    bonus = bonuses(player, pack)
+    choices = [
+        outpost.id
+        for outpost in pack.outposts
+        if outpost.id not in player.outposts and requirements_met(outpost.requirements, bonus)
+    ]
+    if choices:
+        state.pending = PendingChoice(kind="outpost", seat=player.seat, options=choices)
+        return True
+    return False
+
+
+def _check_objectives(state: GameState, pack: ContentPack, settings: GameSettings) -> None:
+    player = state.players[state.current_seat]
+    if settings.modules.objectives:
+        bonus = bonuses(player, pack)
+        choices = [
+            objective.id
+            for objective in pack.objectives
+            if objective.id in state.available_objectives
+            and score(player, pack) >= objective.minimum_score
+            and requirements_met(objective.requirements, bonus)
+        ]
+        if choices:
+            player.objective_met = choices[0]
+            if player.seat not in state.objective_qualifiers:
+                state.objective_qualifiers.append(player.seat)
+    if (
+        settings.modules.outposts
+        and not settings.interactions.outposts_before_objectives
+        and _queue_outpost(state, pack)
+    ):
+        return
+    _finish_turn(state, pack, settings)
+
+
+def _finish_turn(state: GameState, pack: ContentPack, settings: GameSettings) -> None:
+    player = state.players[state.current_seat]
+    has_score = score(player, pack) >= settings.target_score
+    has_objective = player.objective_met is not None
+    triggered = {
+        "score": has_score,
+        "objective": has_objective,
+        "either": has_score or has_objective,
+        "both": has_score and has_objective,
+    }[settings.victory_condition]
+    if triggered and state.finish_at_seat is None:
+        state.finish_at_seat = (state.first_seat - 1) % len(state.players)
+    if state.finish_at_seat == state.current_seat:
+        candidates = (
+            state.objective_qualifiers if settings.victory_condition == "objective" else list(range(len(state.players)))
+        )
+        if not candidates:
+            candidates = list(range(len(state.players)))
+        best_score = max(score(state.players[seat], pack) for seat in candidates)
+        candidates = [seat for seat in candidates if score(state.players[seat], pack) == best_score]
+        fewest = min(state.players[seat].purchased_card_count for seat in candidates)
+        state.winners = [seat for seat in candidates if state.players[seat].purchased_card_count == fewest]
+        state.finished = True
+        state.pending = None
+        return
+    state.current_seat = (state.current_seat + 1) % len(state.players)
+    if state.current_seat == state.first_seat:
+        state.round_number += 1
+    state.turn_purchase_count = 0
+    state.turn_chain_free_count = 0
+
+
+def _remove_visible(state: GameState, card_id: str) -> None:
+    for market in state.markets.values():
+        if card_id in market:
+            market.remove(card_id)
+            return
+    error = "Card is not visible"
+    raise RuleError(error)
+
+
+def _refill_markets(state: GameState, settings: GameSettings) -> None:
+    for key, market in state.markets.items():
+        desired = settings.eastern_market_size if key.startswith("eastern:") else settings.base_market_size
+        deck = state.decks[key]
+        while len(market) < desired and deck:
+            market.append(deck.pop())
+
+
+def _assert_not_blocked(state: GameState, card_id: str, seat: int) -> None:
+    occupants = state.fortifications.get(card_id, {})
+    if any(owner != seat and count > 0 for owner, count in occupants.items()):
+        error = "An opponent's fortification protects that card"
+        raise RuleError(error)
+
+
+def _return_fortifications(state: GameState, card_id: str) -> None:
+    for seat, count in state.fortifications.pop(card_id, {}).items():
+        state.players[seat].fortifications_available += count
+
+
+def _owned_outpost(player: PlayerState, pack: ContentPack, power: OutpostPower) -> OutpostDefinition | None:
+    definitions = {item.id: item for item in pack.outposts}
+    return next((definitions[item] for item in player.outposts if definitions[item].power == power), None)
+
+
+def _reconcile_outposts(player: PlayerState, pack: ContentPack) -> None:
+    definitions = {item.id: item for item in pack.outposts}
+    bonus = bonuses(player, pack)
+    player.outposts = [item for item in player.outposts if requirements_met(definitions[item].requirements, bonus)]
+
+
+def public_state(state: GameState, viewer_seat: int | None) -> dict[str, Any]:
+    """Serialize state while redacting other players' reserved cards and deck order."""
+    data = state.model_dump(mode="json")
+    for player in data["players"]:
+        if player["seat"] != viewer_seat:
+            player["reserved"] = [None] * len(player["reserved"])
+    data["decks"] = {key: len(value) for key, value in state.decks.items()}
+    if state.pending and state.pending.seat != viewer_seat:
+        data["pending"]["options"] = []
+        data["pending"]["context"] = {}
+    return data

python/gems/domain/legal_actions.py

@@ -0,0 +1,159 @@
+"""Legal command generation used by bots and server-rendered controls."""
+
+from __future__ import annotations
+
+import itertools
+import math
+import uuid
+
+from .engine import bonuses, visible_cards
+from .models import ContentPack, GameCommand, GameSettings, GameState, OutpostPower
+
+DISTINCT_TAKE_COUNT = 3
+DOUBLE_TAKE_MINIMUM = 4
+
+
+def command(state: GameState, type_: str, payload: dict | None = None) -> GameCommand:
+    """Create a command targeting the state's current revision."""
+    return GameCommand(
+        command_id=uuid.uuid4().hex,
+        expected_revision=state.revision,
+        type=type_,  # type: ignore[arg-type]
+        payload=payload or {},
+    )
+
+
+def legal_commands(  # noqa: C901, PLR0911, PLR0912 - mirrors pending and normal action branches
+    state: GameState, pack: ContentPack, settings: GameSettings, seat: int
+) -> list[GameCommand]:
+    """Enumerate meaningful legal commands without revealing hidden information."""
+    if state.finished or seat != state.current_seat:
+        return []
+    player = state.players[seat]
+    if state.pending:
+        pending = state.pending
+        if pending.seat != seat:
+            return []
+        if pending.kind == "discard_tokens":
+            available = [item for item, amount in player.tokens.items() for _ in range(amount)]
+            payload: dict[str, int] = {}
+            for item in available[: pending.amount]:
+                payload[item] = payload.get(item, 0) + 1
+            return [command(state, "choose", {"tokens": payload})]
+        if pending.kind == "fortification":
+            pending_results = []
+            for target in pending.options:
+                occupants = state.fortifications.get(target, {})
+                if not any(owner != seat and count for owner, count in occupants.items()):
+                    if player.fortifications_available:
+                        pending_results.append(command(state, "choose", {"card_id": target, "mode": "place"}))
+                    for source, source_occupants in state.fortifications.items():
+                        if source_occupants.get(seat, 0):
+                            pending_results.append(
+                                command(state, "choose", {"card_id": target, "from_card": source, "mode": "place"})
+                            )
+                if any(owner != seat and count == 1 for owner, count in occupants.items()):
+                    pending_results.append(command(state, "choose", {"card_id": target, "mode": "remove"}))
+            return pending_results
+        if pending.kind == "conquest":
+            conquest_results = [command(state, "decline")]
+            for card_id in pending.options:
+                payment = default_payment(state, pack, settings, card_id, seat)
+                if payment is not None:
+                    conquest_results.append(command(state, "purchase", {"card_id": card_id, **payment}))
+            return conquest_results
+        return [command(state, "choose", {"choice": item}) for item in pending.options]
+
+    results: list[GameCommand] = []
+    available = [resource for resource in pack.resource_ids if state.supply.get(resource, 0)]
+    take_size = DISTINCT_TAKE_COUNT if len(available) >= DISTINCT_TAKE_COUNT else 1
+    sizes = [take_size] if len(available) >= DISTINCT_TAKE_COUNT else list(range(1, len(available) + 1))
+    for size in sizes:
+        results.extend(
+            command(state, "take_distinct", {"resources": list(items)})
+            for items in itertools.combinations(available, size)
+        )
+    results.extend(
+        command(state, "take_double", {"resource": resource})
+        for resource in pack.resource_ids
+        if state.supply.get(resource, 0) >= DOUBLE_TAKE_MINIMUM
+    )
+    if len(player.reserved) < settings.reserve_limit:
+        results.extend(
+            command(state, "reserve", {"card_id": card_id})
+            for card_id in visible_cards(state)
+            if not _blocked(state, card_id, seat)
+        )
+        results.extend(command(state, "reserve", {"deck": key}) for key, cards in state.decks.items() if cards)
+    for card_id in [*visible_cards(state), *player.reserved]:
+        if card_id not in player.reserved and _blocked(state, card_id, seat):
+            continue
+        payment = default_payment(state, pack, settings, card_id, seat)
+        if payment is not None:
+            results.append(command(state, "purchase", {"card_id": card_id, **payment}))
+    return results
+
+
+def default_payment(  # noqa: C901, PLR0912 - ordered payment rules are intentionally explicit
+    state: GameState,
+    pack: ContentPack,
+    settings: GameSettings,
+    card_id: str,
+    seat: int,
+) -> dict | None:
+    """Return one legal colored-first payment, or None when unaffordable."""
+    player = state.players[seat]
+    card = pack.card(card_id)
+    if card.alternate_cost:
+        matching = []
+        definitions = {item.id: item for item in pack.cards}
+        for owned in player.cards:
+            resource = owned.copied_resource or definitions[owned.card_id].bonus_resource
+            if resource == card.alternate_cost.discard_resource:
+                matching.append(owned.card_id)
+        copies = [
+            item for item in matching if next(owned for owned in player.cards if owned.card_id == item).copied_resource
+        ]
+        ordered = [*copies, *(item for item in matching if item not in copies)]
+        if len(ordered) < card.alternate_cost.count:
+            return None
+        return {"discard_cards": ordered[: card.alternate_cost.count]}
+    discount = bonuses(player, pack)
+    payment: dict[str, int] = {}
+    shortage = 0
+    for resource in pack.resource_ids:
+        due = max(0, card.cost.get(resource, 0) - discount.get(resource, 0))
+        amount = min(due, player.tokens.get(resource, 0))
+        if amount:
+            payment[resource] = amount
+        shortage += due - amount
+    double_wild = any(
+        outpost.power == OutpostPower.DOUBLE_WILD and outpost.id in player.outposts for outpost in pack.outposts
+    )
+    wild_value = 2 if double_wild else 1
+    virtual_multiplier = 2 if double_wild and settings.interactions.virtual_wild_can_double else 1
+    virtual_cards: list[str] = []
+    virtual_total = 0
+    if shortage > player.tokens.get(pack.wild_resource.id, 0) * wild_value:
+        for owned in player.cards:
+            definition = pack.card(owned.card_id)
+            if definition.effect.kind.value != "virtual_wild":
+                continue
+            virtual_cards.append(owned.card_id)
+            virtual_total += definition.effect.amount * virtual_multiplier
+            if virtual_total + player.tokens.get(pack.wild_resource.id, 0) * wild_value >= shortage:
+                break
+    remaining = max(0, shortage - virtual_total)
+    wild_needed = math.ceil(remaining / wild_value)
+    if wild_needed > player.tokens.get(pack.wild_resource.id, 0) or virtual_total + wild_needed * wild_value < shortage:
+        return None
+    if wild_needed:
+        payment[pack.wild_resource.id] = wild_needed
+    result: dict[str, object] = {"payment": payment}
+    if virtual_cards:
+        result["virtual_wild_cards"] = virtual_cards
+    return result
+
+
+def _blocked(state: GameState, card_id: str, seat: int) -> bool:
+    return any(owner != seat and count for owner, count in state.fortifications.get(card_id, {}).items())

python/gems/domain/models.py

@@ -0,0 +1,394 @@
+"""Typed content, configuration, commands, and game state."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Annotated, Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class StrictModel(BaseModel):
+    """Base model that rejects misspelled fields."""
+
+    model_config = ConfigDict(extra="forbid")
+
+
+Slug = Annotated[str, Field(pattern=r"^[a-z][a-z0-9_-]{0,47}$")]
+ShortText = Annotated[str, Field(min_length=1, max_length=80)]
+TOKEN_DARK_INK_THRESHOLD = 0.82
+STANDARD_RESOURCE_SYMBOLS = {
+    "onyx": "O",
+    "sapphire": "S",
+    "emerald": "E",
+    "ruby": "R",
+    "diamond": "D",
+    "gold": "G",
+}
+
+
+class ResourceDefinition(StrictModel):
+    """A normal or wild resource rendered by the UI."""
+
+    id: Slug
+    label: ShortText
+    symbol: Annotated[str, Field(min_length=1, max_length=4)]
+    color: Annotated[str, Field(pattern=r"^#[0-9a-fA-F]{6}$")]
+
+    @model_validator(mode="after")
+    def normalize_standard_symbol(self) -> ResourceDefinition:
+        """Store the conventional symbol for a standard gem label."""
+        self.symbol = STANDARD_RESOURCE_SYMBOLS.get(self.label.casefold(), self.symbol)
+        return self
+
+    @property
+    def ink_color(self) -> str:
+        """Return readable token lettering for the configured background."""
+        red, green, blue = (int(self.color[index : index + 2], 16) for index in (1, 3, 5))
+        perceived_brightness = (299 * red + 587 * green + 114 * blue) / 255_000
+        return "#111111" if perceived_brightness >= TOKEN_DARK_INK_THRESHOLD else "#ffffff"
+
+
+class RequirementKind(StrEnum):
+    """Identify how a requirement selects resource colors."""
+
+    COLOR = "color"
+    ANY_COLOR = "any_color"
+
+
+class Requirement(StrictModel):
+    """A fixed-color or same-color bonus requirement."""
+
+    id: Slug
+    kind: RequirementKind
+    count: Annotated[int, Field(ge=1, le=99)]
+    resource: Slug | None = None
+    exclude: list[Slug] = Field(default_factory=list)
+    distinct_from: list[Slug] = Field(default_factory=list)
+
+    @model_validator(mode="after")
+    def validate_shape(self) -> Requirement:
+        """Validate fields that depend on the requirement kind."""
+        if self.kind == RequirementKind.COLOR and self.resource is None:
+            error = "color requirements need a resource"
+            raise ValueError(error)
+        if self.kind == RequirementKind.ANY_COLOR and self.resource is not None:
+            error = "any_color requirements cannot name a resource"
+            raise ValueError(error)
+        return self
+
+
+class CardEffectKind(StrEnum):
+    """Identify a supported server-side card effect."""
+
+    NONE = "none"
+    VIRTUAL_WILD = "virtual_wild"
+    COPY_BONUS = "copy_bonus"
+    COPY_AND_CLAIM = "copy_and_claim"
+    MULTI_BONUS = "multi_bonus"
+    CLAIM_FREE = "claim_free"
+
+
+class CardEffect(StrictModel):
+    """A bounded built-in card effect; uploaded code is never evaluated."""
+
+    kind: CardEffectKind = CardEffectKind.NONE
+    amount: Annotated[int, Field(ge=1, le=10)] = 1
+    target_tier: Annotated[int, Field(ge=1, le=3)] | None = None
+
+
+class AlternateCost(StrictModel):
+    """Purchase a card by discarding owned cards of one effective color."""
+
+    discard_resource: Slug
+    count: Annotated[int, Field(ge=1, le=10)]
+
+
+class CardDefinition(StrictModel):
+    """A card supplied by a user-owned content pack."""
+
+    id: Slug
+    label: ShortText
+    deck: Literal["base", "eastern"] = "base"
+    tier: Annotated[int, Field(ge=1, le=3)]
+    points: Annotated[int, Field(ge=0, le=99)] = 0
+    bonus_resource: Slug | None = None
+    cost: dict[Slug, Annotated[int, Field(ge=0, le=99)]] = Field(default_factory=dict)
+    effect: CardEffect = Field(default_factory=CardEffect)
+    alternate_cost: AlternateCost | None = None
+
+    @model_validator(mode="after")
+    def validate_effect(self) -> CardDefinition:
+        """Validate fields that depend on the selected card effect."""
+        effect = self.effect
+        if effect.kind in {CardEffectKind.CLAIM_FREE, CardEffectKind.COPY_AND_CLAIM}:
+            if effect.target_tier is None:
+                error = "claim effects require target_tier"
+                raise ValueError(error)
+            if effect.target_tier >= self.tier:
+                error = "free-card effects must target a lower tier"
+                raise ValueError(error)
+        elif effect.target_tier is not None:
+            error = "target_tier is only valid for claim effects"
+            raise ValueError(error)
+        if effect.kind == CardEffectKind.MULTI_BONUS and self.bonus_resource is None:
+            error = "multi_bonus cards require bonus_resource"
+            raise ValueError(error)
+        return self
+
+
+class PatronDefinition(StrictModel):
+    """Define a patron claimed by meeting permanent-bonus requirements."""
+
+    id: Slug
+    label: ShortText
+    points: Annotated[int, Field(ge=0, le=99)]
+    requirements: list[Requirement]
+
+
+class ObjectiveDefinition(StrictModel):
+    """Define an optional objective-based victory condition."""
+
+    id: Slug
+    label: ShortText
+    minimum_score: Annotated[int, Field(ge=0, le=999)]
+    requirements: list[Requirement]
+
+
+class OutpostPower(StrEnum):
+    """Identify a built-in outpost ability."""
+
+    RESOURCE_AFTER_PURCHASE = "resource_after_purchase"
+    RESOURCE_AFTER_DOUBLE = "resource_after_double"
+    DOUBLE_WILD = "double_wild"
+    POINTS_PER_OUTPOST = "points_per_outpost"
+    BLIND_RESERVE_TWO = "blind_reserve_two"
+
+
+class OutpostDefinition(StrictModel):
+    """Define an outpost and the bonuses required to claim it."""
+
+    id: Slug
+    label: ShortText
+    requirements: list[Requirement]
+    power: OutpostPower
+    value: Annotated[int, Field(ge=1, le=10)] = 1
+
+
+class PackMetadata(StrictModel):
+    """Describe the identity and authorship of a content pack."""
+
+    id: Slug
+    name: ShortText
+    version: Annotated[str, Field(min_length=1, max_length=32)]
+    author: Annotated[str, Field(max_length=80)] = ""
+
+
+class ContentPack(StrictModel):
+    """Versioned user-provided game content."""
+
+    schema_version: Literal[1]
+    metadata: PackMetadata
+    resources: Annotated[list[ResourceDefinition], Field(min_length=5, max_length=5)]
+    wild_resource: ResourceDefinition
+    cards: list[CardDefinition]
+    patrons: list[PatronDefinition] = Field(default_factory=list)
+    objectives: list[ObjectiveDefinition] = Field(default_factory=list)
+    outposts: list[OutpostDefinition] = Field(default_factory=list)
+
+    @model_validator(mode="before")
+    @classmethod
+    def normalize_patrons(cls, data: object) -> object:
+        """Normalize the legacy governors key to the neutral patrons key."""
+        if isinstance(data, dict):
+            if "patrons" in data and "governors" in data:
+                error = "use patrons or governors, not both"
+                raise ValueError(error)
+            if "governors" in data:
+                data = dict(data)
+                data["patrons"] = data.pop("governors")
+        return data
+
+    @model_validator(mode="after")
+    def validate_references(self) -> ContentPack:  # noqa: C901, PLR0912 - validates each reference family
+        """Validate identifiers and references across the complete pack."""
+        resources = {item.id for item in self.resources}
+        if self.wild_resource.id in resources:
+            error = "wild resource ID must be distinct"
+            raise ValueError(error)
+        id_collections = [
+            [item.id for item in self.resources],
+            [item.id for item in self.cards],
+            [item.id for item in self.patrons],
+            [item.id for item in self.objectives],
+            [item.id for item in self.outposts],
+        ]
+        for ids in id_collections:
+            if len(ids) != len(set(ids)):
+                error = "IDs must be unique within each content collection"
+                raise ValueError(error)
+        req_ids: set[str]
+        for card in self.cards:
+            if card.bonus_resource is not None and card.bonus_resource not in resources:
+                error = f"card {card.id} has an unknown bonus resource"
+                raise ValueError(error)
+            if not set(card.cost) <= resources:
+                error = f"card {card.id} has an unknown cost resource"
+                raise ValueError(error)
+            if card.alternate_cost and card.alternate_cost.discard_resource not in resources:
+                error = f"card {card.id} has an unknown alternate-cost resource"
+                raise ValueError(error)
+        requirement_owners: list[PatronDefinition | ObjectiveDefinition | OutpostDefinition] = [
+            *self.patrons,
+            *self.objectives,
+            *self.outposts,
+        ]
+        for owner in requirement_owners:
+            req_ids = {requirement.id for requirement in owner.requirements}
+            if len(req_ids) != len(owner.requirements):
+                error = f"{owner.id} has duplicate requirement IDs"
+                raise ValueError(error)
+            for requirement in owner.requirements:
+                if requirement.resource and requirement.resource not in resources:
+                    error = f"{owner.id} references an unknown resource"
+                    raise ValueError(error)
+                if not set(requirement.exclude) <= resources:
+                    error = f"{owner.id} excludes an unknown resource"
+                    raise ValueError(error)
+                if not set(requirement.distinct_from) <= req_ids:
+                    error = f"{owner.id} references an unknown requirement"
+                    raise ValueError(error)
+        return self
+
+    @property
+    def resource_ids(self) -> tuple[str, ...]:
+        """Return normal resource identifiers in display order."""
+        return tuple(resource.id for resource in self.resources)
+
+    def card(self, card_id: str) -> CardDefinition:
+        """Return a card definition by identifier."""
+        return next(card for card in self.cards if card.id == card_id)
+
+
+class Modules(StrictModel):
+    """Select optional rule modules for a game."""
+
+    objectives: bool = False
+    outposts: bool = False
+    eastern_decks: bool = False
+    fortifications: bool = False
+
+
+class Interactions(StrictModel):
+    """Configure interactions between optional rule modules."""
+
+    outpost_reserve_applies_eastern: bool = True
+    virtual_wild_can_double: bool = True
+    retain_outposts_after_discard: bool = True
+    outposts_before_objectives: bool = True
+    purchase_resource_on_conquest: bool = True
+    chained_claim_is_not_purchase: bool = True
+    fortifications_on_eastern: bool = True
+    fortifications_block_free_claim: bool = True
+    one_fortification_decision_per_purchase_chain: bool = True
+
+
+class GameSettings(StrictModel):
+    """Configure win conditions, limits, markets, and rule modules."""
+
+    target_score: Annotated[int, Field(ge=1, le=999)] = 15
+    victory_condition: Literal["score", "objective", "either", "both"] = "score"
+    first_player_mode: Literal["random", "selected"] = "random"
+    first_player_seat: Annotated[int, Field(ge=0, le=3)] = 0
+    token_limit: Annotated[int, Field(ge=1, le=99)] = 10
+    reserve_limit: Annotated[int, Field(ge=0, le=20)] = 3
+    base_market_size: Annotated[int, Field(ge=1, le=10)] = 4
+    eastern_market_size: Annotated[int, Field(ge=1, le=10)] = 2
+    objective_count: Annotated[int, Field(ge=1, le=10)] = 3
+    fortifications_per_player: Annotated[int, Field(ge=1, le=10)] = 3
+    modules: Modules = Field(default_factory=Modules)
+    interactions: Interactions = Field(default_factory=Interactions)
+
+
+class OwnedCard(StrictModel):
+    """Track an owned card and any copied resource assignment."""
+
+    card_id: Slug
+    copied_resource: Slug | None = None
+
+
+class PlayerState(StrictModel):
+    """Store the mutable state belonging to one player."""
+
+    seat: int
+    name: ShortText
+    tokens: dict[str, int]
+    cards: list[OwnedCard] = Field(default_factory=list)
+    reserved: list[Slug] = Field(default_factory=list)
+    patrons: list[Slug] = Field(default_factory=list)
+    outposts: list[Slug] = Field(default_factory=list)
+    objective_met: Slug | None = None
+    fortifications_available: int = 0
+    purchased_card_count: int = 0
+
+
+class PendingChoice(StrictModel):
+    """Describe a choice that must be resolved before play continues."""
+
+    kind: Literal[
+        "discard_tokens",
+        "copy_bonus",
+        "free_card",
+        "reserve_keep",
+        "resource",
+        "fortification",
+        "conquest",
+        "patron",
+        "outpost",
+    ]
+    seat: int
+    options: list[str] = Field(default_factory=list)
+    amount: int = 1
+    context: dict[str, Any] = Field(default_factory=dict)
+
+
+class GameState(StrictModel):
+    """Store a complete authoritative game snapshot."""
+
+    room_code: str
+    seed: int
+    revision: int = 0
+    players: list[PlayerState]
+    supply: dict[str, int]
+    decks: dict[str, list[Slug]]
+    markets: dict[str, list[Slug]]
+    available_patrons: list[Slug]
+    available_objectives: list[Slug]
+    fortifications: dict[Slug, dict[int, int]] = Field(default_factory=dict)
+    current_seat: int = 0
+    first_seat: int = 0
+    round_number: int = 1
+    pending: PendingChoice | None = None
+    finish_at_seat: int | None = None
+    objective_qualifiers: list[int] = Field(default_factory=list)
+    winners: list[int] = Field(default_factory=list)
+    finished: bool = False
+    turn_purchase_count: int = 0
+    turn_chain_free_count: int = 0
+    log: list[str] = Field(default_factory=list)
+
+
+class GameCommand(StrictModel):
+    """Represent one revision-bound action submitted to the engine."""
+
+    command_id: Annotated[str, Field(min_length=8, max_length=64)]
+    expected_revision: Annotated[int, Field(ge=0)]
+    type: Literal[
+        "take_distinct",
+        "take_double",
+        "reserve",
+        "purchase",
+        "choose",
+        "decline",
+    ]
+    payload: dict[str, Any] = Field(default_factory=dict)

python/gems/domain/requirements.py

@@ -0,0 +1,58 @@
+"""Requirement evaluation shared by patrons, objectives, outposts, and AI."""
+
+from __future__ import annotations
+
+from .models import Requirement, RequirementKind
+
+
+def requirements_met(requirements: list[Requirement], bonuses: dict[str, int]) -> bool:
+    """Return whether one assignment satisfies all fixed and wildcard clauses."""
+    chosen: dict[str, str] = {}
+
+    def visit(index: int) -> bool:
+        if index == len(requirements):
+            return True
+        requirement = requirements[index]
+        if requirement.kind == RequirementKind.COLOR:
+            if bonuses.get(requirement.resource or "", 0) < requirement.count:
+                return False
+            chosen[requirement.id] = requirement.resource or ""
+            return visit(index + 1)
+
+        forbidden = set(requirement.exclude)
+        forbidden.update(chosen[item] for item in requirement.distinct_from if item in chosen)
+        for resource, amount in bonuses.items():
+            if resource in forbidden or amount < requirement.count:
+                continue
+            chosen[requirement.id] = resource
+            if visit(index + 1):
+                return True
+            chosen.pop(requirement.id, None)
+        return False
+
+    fixed = [item for item in requirements if item.kind == RequirementKind.COLOR]
+    flexible = [item for item in requirements if item.kind == RequirementKind.ANY_COLOR]
+    return visit_ordered([*fixed, *flexible], bonuses, chosen)
+
+
+def visit_ordered(requirements: list[Requirement], bonuses: dict[str, int], chosen: dict[str, str]) -> bool:
+    """Backtracking evaluator kept separate for straightforward unit testing."""
+    if not requirements:
+        return True
+    requirement, *rest = requirements
+    if requirement.kind == RequirementKind.COLOR:
+        resource = requirement.resource or ""
+        if bonuses.get(resource, 0) < requirement.count:
+            return False
+        chosen[requirement.id] = resource
+        return visit_ordered(rest, bonuses, chosen)
+    forbidden = set(requirement.exclude)
+    forbidden.update(chosen[item] for item in requirement.distinct_from if item in chosen)
+    for resource, amount in bonuses.items():
+        if resource in forbidden or amount < requirement.count:
+            continue
+        chosen[requirement.id] = resource
+        if visit_ordered(rest, bonuses, chosen):
+            return True
+        chosen.pop(requirement.id, None)
+    return False

python/gems/events.py

@@ -0,0 +1,35 @@
+"""In-process revision notifications for SSE clients."""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from collections import defaultdict
+
+
+class EventBroker:
+    """Fan out room revisions; clients always reload the latest snapshot."""
+
+    def __init__(self) -> None:
+        """Initialize an empty subscription registry."""
+        self._queues: dict[str, set[asyncio.Queue[int]]] = defaultdict(set)
+
+    def subscribe(self, room_code: str) -> asyncio.Queue[int]:
+        """Subscribe a bounded notification queue to a room."""
+        queue: asyncio.Queue[int] = asyncio.Queue(maxsize=1)
+        self._queues[room_code].add(queue)
+        return queue
+
+    def unsubscribe(self, room_code: str, queue: asyncio.Queue[int]) -> None:
+        """Remove a room notification queue from the registry."""
+        self._queues[room_code].discard(queue)
+        if not self._queues[room_code]:
+            self._queues.pop(room_code, None)
+
+    def publish(self, room_code: str, revision: int) -> None:
+        """Publish the newest room revision to every subscriber."""
+        for queue in tuple(self._queues.get(room_code, ())):
+            if queue.full():
+                with contextlib.suppress(asyncio.QueueEmpty):
+                    queue.get_nowait()
+            queue.put_nowait(revision)

python/gems/main.py

@@ -0,0 +1,91 @@
+"""FastAPI entry point for Gems."""
+
+from __future__ import annotations
+
+import asyncio
+from contextlib import asynccontextmanager, suppress
+from typing import TYPE_CHECKING, Annotated
+
+import typer
+import uvicorn
+from fastapi import FastAPI, Request, Response
+from fastapi.staticfiles import StaticFiles
+
+from .config import load_config
+from .events import EventBroker
+from .persistence import Repository
+from .rooms import RoomService
+from .routes import router
+from .security import CredentialService
+from .web import STATIC_DIR
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
+    from starlette.middleware.base import RequestResponseEndpoint
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI) -> AsyncIterator[None]:
+    """Initialize and close application-scoped services."""
+    config = app.state.config
+    repository = Repository(config.database_path)
+    app.state.repository = repository
+    app.state.rooms = RoomService(repository, CredentialService(config.key_path), EventBroker())
+    repository.cleanup()
+    cleanup_task = asyncio.create_task(_cleanup_rooms(repository))
+    try:
+        yield
+    finally:
+        cleanup_task.cancel()
+        with suppress(asyncio.CancelledError):
+            await cleanup_task
+        repository.close()
+
+
+async def _cleanup_rooms(repository: Repository) -> None:
+    while True:
+        await asyncio.sleep(3600)
+        repository.cleanup()
+
+
+def create_app() -> FastAPI:
+    """Create an isolated application instance."""
+    app = FastAPI(title="Gems", docs_url=None, redoc_url=None, lifespan=lifespan)
+    app.state.config = load_config()
+    app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
+    app.include_router(router)
+
+    @app.middleware("http")
+    async def security_headers(request: Request, call_next: RequestResponseEndpoint) -> Response:
+        response = await call_next(request)
+        response.headers.setdefault("Referrer-Policy", "no-referrer")
+        response.headers.setdefault("X-Content-Type-Options", "nosniff")
+        response.headers.setdefault("X-Frame-Options", "DENY")
+        response.headers.setdefault("X-Robots-Tag", "noindex, nofollow")
+        response.headers.setdefault(
+            "Content-Security-Policy",
+            "default-src 'self'; script-src 'self'; style-src 'self'; style-src-attr 'unsafe-inline'; "
+            "img-src 'self' data:; connect-src 'self'",
+        )
+        if request.url.path.startswith("/rooms/"):
+            response.headers.setdefault("Cache-Control", "no-store")
+        return response
+
+    return app
+
+
+app = create_app()
+
+
+def serve(
+    host: Annotated[str | None, typer.Option()] = None,
+    port: Annotated[int | None, typer.Option()] = None,
+) -> None:
+    """Run the Gems ASGI application with Uvicorn."""
+    config = load_config()
+    uvicorn.run("python.gems.main:app", host=host or config.host, port=port or config.port, workers=1)
+
+
+if __name__ == "__main__":
+    typer.run(serve)

python/gems/persistence.py

@@ -0,0 +1,255 @@
+"""SQLite persistence for rooms, memberships, snapshots, and events."""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import threading
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from typing import TYPE_CHECKING
+
+from .domain.models import ContentPack, GameSettings, GameState
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+@dataclass(slots=True)
+class Member:
+    """Represent a human or AI seat persisted for a room."""
+
+    room_code: str
+    seat: int
+    name: str
+    controller: str
+    difficulty: str | None
+    credential_hash: str | None
+    is_host: bool
+    ready: bool
+    last_seen: str
+
+
+@dataclass(slots=True)
+class Room:
+    """Represent persisted room metadata and its current snapshot."""
+
+    code: str
+    status: str
+    settings: GameSettings
+    pack: ContentPack | None
+    pack_digest: str | None
+    state: GameState | None
+    revision: int
+    created_at: str
+    updated_at: str
+
+
+class Repository:
+    """Small transactional repository designed for one application worker."""
+
+    def __init__(self, path: Path) -> None:
+        """Open the SQLite database and initialize its schema."""
+        path.parent.mkdir(parents=True, exist_ok=True)
+        self._connection = sqlite3.connect(path, check_same_thread=False)
+        self._connection.row_factory = sqlite3.Row
+        self._lock = threading.RLock()
+        with self._connection:
+            self._connection.execute("PRAGMA journal_mode=WAL")
+            self._connection.execute("PRAGMA foreign_keys=ON")
+            self._connection.execute("PRAGMA busy_timeout=5000")
+            self._connection.executescript(
+                """
+                CREATE TABLE IF NOT EXISTS schema_version(version INTEGER NOT NULL);
+                INSERT INTO schema_version(version) SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM schema_version);
+                CREATE TABLE IF NOT EXISTS rooms(
+                    code TEXT PRIMARY KEY, status TEXT NOT NULL, settings_json TEXT NOT NULL,
+                    pack_json TEXT, pack_digest TEXT, state_json TEXT, revision INTEGER NOT NULL,
+                    created_at TEXT NOT NULL, updated_at TEXT NOT NULL
+                );
+                CREATE TABLE IF NOT EXISTS members(
+                    room_code TEXT NOT NULL REFERENCES rooms(code) ON DELETE CASCADE,
+                    seat INTEGER NOT NULL, name TEXT NOT NULL, controller TEXT NOT NULL,
+                    difficulty TEXT, credential_hash TEXT, is_host INTEGER NOT NULL,
+                    ready INTEGER NOT NULL, last_seen TEXT NOT NULL,
+                    PRIMARY KEY(room_code, seat), UNIQUE(room_code, credential_hash)
+                );
+                CREATE TABLE IF NOT EXISTS room_events(
+                    room_code TEXT NOT NULL REFERENCES rooms(code) ON DELETE CASCADE,
+                    revision INTEGER NOT NULL, command_id TEXT NOT NULL, actor_seat INTEGER,
+                    event_type TEXT NOT NULL, payload_json TEXT NOT NULL, created_at TEXT NOT NULL,
+                    PRIMARY KEY(room_code, command_id)
+                );
+                """
+            )
+
+    def room_exists(self, code: str) -> bool:
+        """Report whether a room code exists."""
+        return self._connection.execute("SELECT 1 FROM rooms WHERE code=?", (code,)).fetchone() is not None
+
+    def create_room(self, room: Room, host: Member) -> None:
+        """Persist a new room and its host in one transaction."""
+        with self._lock, self._connection:
+            self._connection.execute(
+                "INSERT INTO rooms VALUES(?,?,?,?,?,?,?,?,?)",
+                (
+                    room.code,
+                    room.status,
+                    room.settings.model_dump_json(),
+                    None,
+                    None,
+                    None,
+                    room.revision,
+                    room.created_at,
+                    room.updated_at,
+                ),
+            )
+            self._insert_member(host)
+
+    def _insert_member(self, member: Member) -> None:
+        self._connection.execute(
+            "INSERT INTO members VALUES(?,?,?,?,?,?,?,?,?)",
+            (
+                member.room_code,
+                member.seat,
+                member.name,
+                member.controller,
+                member.difficulty,
+                member.credential_hash,
+                int(member.is_host),
+                int(member.ready),
+                member.last_seen,
+            ),
+        )
+
+    def add_member(self, member: Member) -> None:
+        """Add a member and refresh the room's activity timestamp."""
+        with self._lock, self._connection:
+            self._insert_member(member)
+            self.touch(member.room_code)
+
+    def update_member(self, member: Member) -> None:
+        """Persist all mutable fields for an existing member."""
+        with self._lock, self._connection:
+            self._connection.execute(
+                """UPDATE members SET name=?,controller=?,difficulty=?,credential_hash=?,is_host=?,ready=?,last_seen=?
+                   WHERE room_code=? AND seat=?""",
+                (
+                    member.name,
+                    member.controller,
+                    member.difficulty,
+                    member.credential_hash,
+                    int(member.is_host),
+                    int(member.ready),
+                    member.last_seen,
+                    member.room_code,
+                    member.seat,
+                ),
+            )
+
+    def remove_member(self, code: str, seat: int) -> None:
+        """Remove a member from a room seat."""
+        with self._lock, self._connection:
+            self._connection.execute("DELETE FROM members WHERE room_code=? AND seat=?", (code, seat))
+
+    def get_room(self, code: str) -> Room | None:
+        """Load a room by code, including its typed JSON fields."""
+        row = self._connection.execute("SELECT * FROM rooms WHERE code=?", (code.upper(),)).fetchone()
+        if row is None:
+            return None
+        return Room(
+            code=row["code"],
+            status=row["status"],
+            settings=GameSettings.model_validate_json(row["settings_json"]),
+            pack=ContentPack.model_validate_json(row["pack_json"]) if row["pack_json"] else None,
+            pack_digest=row["pack_digest"],
+            state=GameState.model_validate_json(row["state_json"]) if row["state_json"] else None,
+            revision=row["revision"],
+            created_at=row["created_at"],
+            updated_at=row["updated_at"],
+        )
+
+    def members(self, code: str) -> list[Member]:
+        """Load all room members ordered by seat."""
+        rows = self._connection.execute("SELECT * FROM members WHERE room_code=? ORDER BY seat", (code,)).fetchall()
+        return [
+            Member(
+                room_code=row["room_code"],
+                seat=row["seat"],
+                name=row["name"],
+                controller=row["controller"],
+                difficulty=row["difficulty"],
+                credential_hash=row["credential_hash"],
+                is_host=bool(row["is_host"]),
+                ready=bool(row["ready"]),
+                last_seen=row["last_seen"],
+            )
+            for row in rows
+        ]
+
+    def member_for_credential(self, code: str, credential_hash: str) -> Member | None:
+        """Find the room member associated with a credential digest."""
+        return next((member for member in self.members(code) if member.credential_hash == credential_hash), None)
+
+    def save_lobby(self, room: Room) -> None:
+        """Persist mutable lobby configuration and revision data."""
+        with self._lock, self._connection:
+            self._connection.execute(
+                """UPDATE rooms SET settings_json=?,pack_json=?,pack_digest=?,revision=?,updated_at=? WHERE code=?""",
+                (
+                    room.settings.model_dump_json(),
+                    room.pack.model_dump_json() if room.pack else None,
+                    room.pack_digest,
+                    room.revision,
+                    room.updated_at,
+                    room.code,
+                ),
+            )
+
+    def save_state(self, room: Room, command_id: str, actor_seat: int | None, payload: dict) -> bool:
+        """Persist a game snapshot and its idempotent command event."""
+        now = utc_now()
+        with self._lock, self._connection:
+            existing = self._connection.execute(
+                "SELECT 1 FROM room_events WHERE room_code=? AND command_id=?", (room.code, command_id)
+            ).fetchone()
+            if existing:
+                return False
+            self._connection.execute(
+                "UPDATE rooms SET status=?,state_json=?,revision=?,updated_at=? WHERE code=?",
+                (room.status, room.state.model_dump_json() if room.state else None, room.revision, now, room.code),
+            )
+            self._connection.execute(
+                "INSERT INTO room_events VALUES(?,?,?,?,?,?,?)",
+                (room.code, room.revision, command_id, actor_seat, "command", json.dumps(payload), now),
+            )
+        return True
+
+    def has_command(self, code: str, command_id: str) -> bool:
+        """Report whether a command was already persisted for a room."""
+        row = self._connection.execute(
+            "SELECT 1 FROM room_events WHERE room_code=? AND command_id=?",
+            (code, command_id),
+        ).fetchone()
+        return row is not None
+
+    def touch(self, code: str) -> None:
+        """Refresh a room's activity timestamp."""
+        with self._connection:
+            self._connection.execute("UPDATE rooms SET updated_at=? WHERE code=?", (utc_now(), code))
+
+    def cleanup(self, days: int = 30) -> int:
+        """Delete rooms inactive for the requested number of days."""
+        cutoff = (datetime.now(UTC) - timedelta(days=days)).isoformat()
+        with self._lock, self._connection:
+            cursor = self._connection.execute("DELETE FROM rooms WHERE updated_at < ?", (cutoff,))
+        return cursor.rowcount
+
+    def close(self) -> None:
+        """Close the underlying SQLite connection."""
+        self._connection.close()
+
+
+def utc_now() -> str:
+    """Return the current UTC time as an ISO 8601 string."""
+    return datetime.now(UTC).isoformat()

python/gems/rooms.py

@@ -0,0 +1,375 @@
+"""Application service for anonymous rooms and AI turns."""
+
+from __future__ import annotations
+
+import asyncio
+import secrets
+from collections import defaultdict
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING
+
+from .ai import choose_ai_command
+from .domain.engine import RuleError, apply_command, new_game
+from .domain.models import GameCommand, GameSettings
+from .persistence import Member, Repository, Room, utc_now
+
+if TYPE_CHECKING:
+    from .content import ParsedPack
+    from .events import EventBroker
+    from .security import CredentialService
+
+ROOM_ALPHABET = "23456789ABCDEFGHJKMNPQRSTUVWXYZ"
+MAX_PLAYERS = 4
+OFFLINE_AFTER_SECONDS = 30
+DISPLAY_NAME_MAX_LENGTH = 32
+CONTROL_CHARACTER_LIMIT = 32
+
+
+class RoomError(ValueError):
+    """Safe user-facing room operation error."""
+
+
+class RoomService:
+    """Coordinates persistence, authorization, per-room locking, and bots."""
+
+    def __init__(self, repository: Repository, credentials: CredentialService, broker: EventBroker) -> None:
+        """Initialize the service with persistence, security, and events."""
+        self.repository = repository
+        self.credentials = credentials
+        self.broker = broker
+        self._locks: dict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
+        self._ai_tasks: dict[str, asyncio.Task[None]] = {}
+
+    def create(self, host_name: str, credential: str | None = None) -> tuple[Room, str]:
+        """Create a lobby and assign its first human member as host."""
+        name = validate_name(host_name)
+        code = self._new_code()
+        credential = credential or self.credentials.issue()
+        now = utc_now()
+        room = Room(
+            code=code,
+            status="lobby",
+            settings=GameSettings(),
+            pack=None,
+            pack_digest=None,
+            state=None,
+            revision=0,
+            created_at=now,
+            updated_at=now,
+        )
+        host = Member(
+            room_code=code,
+            seat=0,
+            name=name,
+            controller="human",
+            difficulty=None,
+            credential_hash=self.credentials.digest(credential),
+            is_host=True,
+            ready=False,
+            last_seen=now,
+        )
+        self.repository.create_room(room, host)
+        return room, credential
+
+    def join(self, code: str, name: str, credential: str | None = None) -> tuple[Room, str]:
+        """Join an available lobby or restore an existing membership."""
+        room = self.require_room(code)
+        credential = credential or self.credentials.issue()
+        existing = self.repository.member_for_credential(room.code, self.credentials.digest(credential))
+        if existing is not None:
+            return room, credential
+        if room.status != "lobby":
+            error = "This room is no longer accepting players"
+            raise RoomError(error)
+        members = self.repository.members(room.code)
+        if len(members) >= MAX_PLAYERS:
+            error = "This room is full"
+            raise RoomError(error)
+        clean_name = validate_name(name)
+        if any(member.name.casefold() == clean_name.casefold() for member in members):
+            error = "That display name is already in use"
+            raise RoomError(error)
+        seat = next(item for item in range(MAX_PLAYERS) if item not in {member.seat for member in members})
+        member = Member(
+            room_code=room.code,
+            seat=seat,
+            name=clean_name,
+            controller="human",
+            difficulty=None,
+            credential_hash=self.credentials.digest(credential),
+            is_host=False,
+            ready=False,
+            last_seen=utc_now(),
+        )
+        self.repository.add_member(member)
+        self._bump_lobby(room)
+        return room, credential
+
+    def authenticate(self, code: str, credential: str | None) -> tuple[Room, Member]:
+        """Authenticate a room member and refresh their presence timestamp."""
+        room = self.require_room(code)
+        if not credential:
+            error = "Join the room to continue"
+            raise RoomError(error)
+        member = self.repository.member_for_credential(room.code, self.credentials.digest(credential))
+        if member is None:
+            error = "Join the room to continue"
+            raise RoomError(error)
+        member.last_seen = utc_now()
+        self.repository.update_member(member)
+        return room, member
+
+    def require_room(self, code: str) -> Room:
+        """Load a room or raise a safe user-facing error."""
+        room = self.repository.get_room(code.upper())
+        if room is None:
+            error = "Room not found"
+            raise RoomError(error)
+        return room
+
+    def set_pack(self, room: Room, actor: Member, parsed: ParsedPack) -> Room:
+        """Assign a validated content pack to a lobby."""
+        self._require_host_lobby(room, actor)
+        room.pack = parsed.pack
+        room.pack_digest = parsed.digest
+        self._bump_lobby(room)
+        return room
+
+    def set_settings(self, room: Room, actor: Member, settings: GameSettings) -> Room:
+        """Replace the editable rules for a lobby."""
+        self._require_host_lobby(room, actor)
+        room.settings = settings
+        self._bump_lobby(room)
+        return room
+
+    def set_ready(self, room: Room, actor: Member, *, ready: bool) -> None:
+        """Update a human lobby member's readiness."""
+        if room.status != "lobby" or actor.controller != "human":
+            error = "Readiness can only change in the lobby"
+            raise RoomError(error)
+        actor.ready = ready
+        actor.last_seen = utc_now()
+        self.repository.update_member(actor)
+        self._bump_lobby(room)
+
+    def add_ai(self, room: Room, actor: Member, difficulty: str) -> None:
+        """Add an AI-controlled seat to a lobby."""
+        self._require_host_lobby(room, actor)
+        if difficulty not in {"easy", "medium", "hard"}:
+            error = "Choose easy, medium, or hard AI"
+            raise RoomError(error)
+        members = self.repository.members(room.code)
+        if len(members) >= MAX_PLAYERS:
+            error = "This room is full"
+            raise RoomError(error)
+        seat = next(item for item in range(MAX_PLAYERS) if item not in {member.seat for member in members})
+        number = 1 + sum(member.controller == "ai" for member in members)
+        self.repository.add_member(
+            Member(
+                room_code=room.code,
+                seat=seat,
+                name=f"Bot {number}",
+                controller="ai",
+                difficulty=difficulty,
+                credential_hash=None,
+                is_host=False,
+                ready=True,
+                last_seen=utc_now(),
+            )
+        )
+        self._bump_lobby(room)
+
+    def remove_seat(self, room: Room, actor: Member, seat: int) -> None:
+        """Remove a non-host seat from a lobby."""
+        self._require_host_lobby(room, actor)
+        target = next((member for member in self.repository.members(room.code) if member.seat == seat), None)
+        if target is None or target.is_host:
+            error = "That seat cannot be removed"
+            raise RoomError(error)
+        self.repository.remove_member(room.code, seat)
+        self._bump_lobby(room)
+
+    def transfer_host(self, room: Room, actor: Member, seat: int) -> None:
+        """Transfer lobby ownership to another human player."""
+        self._require_host_lobby(room, actor)
+        target = next((member for member in self.repository.members(room.code) if member.seat == seat), None)
+        if target is None or target.controller != "human" or target.seat == actor.seat:
+            error = "Choose another human player"
+            raise RoomError(error)
+        actor.is_host = False
+        target.is_host = True
+        self.repository.update_member(actor)
+        self.repository.update_member(target)
+        self._bump_lobby(room)
+
+    async def start(self, room: Room, actor: Member) -> Room:
+        """Validate the lobby and start its first game."""
+        self._require_host_lobby(room, actor)
+        if room.pack is None:
+            error = "Upload a valid content pack first"
+            raise RoomError(error)
+        members = self.repository.members(room.code)
+        humans = [member for member in members if member.controller == "human"]
+        if not humans or any(not member.ready for member in humans):
+            error = "Every human player must be ready"
+            raise RoomError(error)
+        ordered = sorted(members, key=lambda member: member.seat)
+        # Compress lobby seat gaps so engine seats always index the players list.
+        for new_seat, member in enumerate(ordered):
+            if member.seat != new_seat:
+                self.repository.remove_member(room.code, member.seat)
+                member.seat = new_seat
+                self.repository.add_member(member)
+        try:
+            state = new_game(
+                room.code, [member.name for member in ordered], room.pack, room.settings, seed=secrets.randbits(63)
+            )
+        except RuleError as exc:
+            raise RoomError(str(exc)) from exc
+        room.state = state
+        room.status = "playing"
+        room.revision = state.revision
+        self.repository.save_state(room, f"start-{secrets.token_hex(8)}", actor.seat, {"type": "start"})
+        self.broker.publish(room.code, room.revision)
+        self.schedule_ai(room.code)
+        return room
+
+    async def play_again(self, room: Room, actor: Member) -> Room:
+        """Start a fresh game in a finished room while preserving its table setup."""
+        if not actor.is_host or room.status != "finished":
+            error = "Only the host can replay a finished game"
+            raise RoomError(error)
+        async with self._locks[room.code]:
+            room = self.require_room(room.code)
+            if room.status != "finished" or room.pack is None:
+                error = "This game is not ready for a replay"
+                raise RoomError(error)
+            members = sorted(self.repository.members(room.code), key=lambda member: member.seat)
+            try:
+                room.state = new_game(
+                    room.code,
+                    [member.name for member in members],
+                    room.pack,
+                    room.settings,
+                    seed=secrets.randbits(63),
+                )
+            except RuleError as exc:
+                raise RoomError(str(exc)) from exc
+            room.status = "playing"
+            room.revision = room.state.revision
+            self.repository.save_state(
+                room,
+                f"replay-{secrets.token_hex(8)}",
+                actor.seat,
+                {"type": "play_again"},
+            )
+        self.broker.publish(room.code, room.revision)
+        self.schedule_ai(room.code)
+        return room
+
+    async def submit(self, room: Room, actor: Member, command: GameCommand) -> Room:
+        """Apply and persist one idempotent human game command."""
+        if room.status != "playing" or room.state is None or room.pack is None:
+            error = "This game is not active"
+            raise RoomError(error)
+        async with self._locks[room.code]:
+            room = self.require_room(room.code)
+            if self.repository.has_command(room.code, command.command_id):
+                return room
+            if room.state is None or room.pack is None:
+                error = "This game is not active"
+                raise RoomError(error)
+            try:
+                room.state = apply_command(room.state, command, room.pack, room.settings, actor_seat=actor.seat)
+            except RuleError as exc:
+                raise RoomError(str(exc)) from exc
+            room.revision = room.state.revision
+            room.status = "finished" if room.state.finished else "playing"
+            room.state.log = room.state.log[-200:]
+            self.repository.save_state(room, command.command_id, actor.seat, command.model_dump(mode="json"))
+        self.broker.publish(room.code, room.revision)
+        self.schedule_ai(room.code)
+        return room
+
+    def replace_with_ai(self, room: Room, actor: Member, seat: int, difficulty: str) -> None:
+        """Replace a disconnected human player with an AI controller."""
+        if not actor.is_host or room.status != "playing":
+            error = "Only the host can replace a player during a game"
+            raise RoomError(error)
+        target = next((item for item in self.repository.members(room.code) if item.seat == seat), None)
+        if target is None or target.controller != "human" or target.is_host:
+            error = "That player cannot be replaced"
+            raise RoomError(error)
+        last_seen = datetime.fromisoformat(target.last_seen)
+        if (datetime.now(UTC) - last_seen).total_seconds() < OFFLINE_AFTER_SECONDS:
+            error = "That player is still connected"
+            raise RoomError(error)
+        target.controller = "ai"
+        target.difficulty = difficulty
+        target.credential_hash = None
+        target.ready = True
+        self.repository.update_member(target)
+        self.broker.publish(room.code, room.revision)
+        self.schedule_ai(room.code)
+
+    def schedule_ai(self, code: str) -> None:
+        """Schedule an AI runner when a room does not already have one."""
+        task = self._ai_tasks.get(code)
+        if task is None or task.done():
+            self._ai_tasks[code] = asyncio.create_task(self._run_ai(code))
+
+    async def _run_ai(self, code: str) -> None:
+        while True:
+            room = self.require_room(code)
+            if room.status != "playing" or room.state is None or room.pack is None:
+                return
+            member = next(
+                (item for item in self.repository.members(code) if item.seat == room.state.current_seat), None
+            )
+            if member is None or member.controller != "ai":
+                return
+            await asyncio.sleep(0.35)
+            async with self._locks[code]:
+                room = self.require_room(code)
+                if room.state is None or room.pack is None or room.state.current_seat != member.seat:
+                    continue
+                chosen = choose_ai_command(
+                    room.state, room.pack, room.settings, member.seat, member.difficulty or "medium"
+                )
+                try:
+                    room.state = apply_command(room.state, chosen, room.pack, room.settings, actor_seat=member.seat)
+                except RuleError:
+                    return
+                room.revision = room.state.revision
+                room.status = "finished" if room.state.finished else "playing"
+                room.state.log = room.state.log[-200:]
+                self.repository.save_state(room, chosen.command_id, member.seat, chosen.model_dump(mode="json"))
+            self.broker.publish(code, room.revision)
+
+    def _new_code(self) -> str:
+        while True:
+            code = "".join(secrets.choice(ROOM_ALPHABET) for _ in range(8))
+            if not self.repository.room_exists(code):
+                return code
+
+    def _require_host_lobby(self, room: Room, actor: Member) -> None:
+        if not actor.is_host or room.status != "lobby":
+            error = "Only the host can change this lobby"
+            raise RoomError(error)
+
+    def _bump_lobby(self, room: Room) -> None:
+        room.revision += 1
+        room.updated_at = utc_now()
+        self.repository.save_lobby(room)
+        self.broker.publish(room.code, room.revision)
+
+
+def validate_name(name: str) -> str:
+    """Normalize and validate a player display name."""
+    clean = " ".join(name.split())
+    if not 1 <= len(clean) <= DISPLAY_NAME_MAX_LENGTH or any(
+        ord(character) < CONTROL_CHARACTER_LIMIT for character in clean
+    ):
+        error = "Display names must be 1-32 plain-text characters"
+        raise RoomError(error)
+    return clean

python/gems/routes/__init__.py

@@ -0,0 +1,5 @@
+"""HTTP routes for Gems."""
+
+from .app import router
+
+__all__ = ["router"]

python/gems/routes/app.py

@@ -0,0 +1,626 @@
+"""FastAPI pages, HTMX mutations, and SSE stream."""
+
+from __future__ import annotations
+
+import asyncio
+import html
+from collections.abc import AsyncIterator, Callable, Mapping
+from datetime import UTC, datetime
+from typing import Annotated, Literal
+
+from fastapi import APIRouter, Form, Request, Response, UploadFile
+from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, StreamingResponse
+from pydantic import ValidationError
+
+from python.gems.content import ContentPackError, content_pack_schema, parse_content_pack
+from python.gems.domain.engine import bonuses, public_state, score
+from python.gems.domain.legal_actions import command as build_command
+from python.gems.domain.legal_actions import legal_commands
+from python.gems.domain.models import GameCommand, GameSettings, Interactions, Modules
+from python.gems.persistence import Member, Room
+from python.gems.rooms import RoomError, RoomService
+from python.gems.web import templates
+
+router = APIRouter()
+COOKIE_NAME = "gems_credential"
+DISPLAY_NAME_COOKIE = "gems_display_name"
+DISPLAY_NAME_MAX_AGE = 365 * 24 * 60 * 60
+DISPLAY_NAME_MAX_LENGTH = 32
+OFFLINE_AFTER_SECONDS = 30
+RoomOperation = Callable[[Room, Member], None]
+
+
+def service(request: Request) -> RoomService:
+    """Return the application-scoped room service."""
+    return request.app.state.rooms
+
+
+def _request_origin(request: Request) -> str:
+    return f"{request.url.scheme}://{request.url.netloc}".rstrip("/")
+
+
+def _public_origin(request: Request) -> str:
+    """Prefer the current address when the untouched local default is configured."""
+    configured = request.app.state.config.public_origin.rstrip("/")
+    if configured == "http://127.0.0.1:8082":
+        return _request_origin(request)
+    return configured
+
+
+def _context(request: Request, room: Room, member: Member, *, error: str | None = None) -> dict[str, object]:
+    members = service(request).repository.members(room.code)
+    humans = [item for item in members if item.controller == "human"]
+    all_humans_ready = bool(humans) and all(item.ready for item in humans)
+    csrf = service(request).credentials.csrf(request.cookies.get(COOKIE_NAME, ""), room.code)
+    context: dict[str, object] = {
+        "request": request,
+        "room": room,
+        "member": member,
+        "members": members,
+        "csrf": csrf,
+        "error": error,
+        "public_origin": _public_origin(request),
+        "all_humans_ready": all_humans_ready,
+        "can_start": room.pack is not None and all_humans_ready,
+        "offline_seats": {
+            item.seat
+            for item in members
+            if item.controller == "human"
+            and (datetime.now(UTC) - datetime.fromisoformat(item.last_seen)).total_seconds() >= OFFLINE_AFTER_SECONDS
+        },
+    }
+    if room.state and room.pack:
+        context["state"] = public_state(room.state, member.seat)
+        context["pack"] = room.pack
+        context["cards"] = {card.id: card for card in room.pack.cards}
+        context["resources"] = {resource.id: resource for resource in [*room.pack.resources, room.pack.wild_resource]}
+        context["normal_resource_ids"] = room.pack.resource_ids
+        context["all_resource_ids"] = (*room.pack.resource_ids, room.pack.wild_resource.id)
+        context["member_by_seat"] = {item.seat: item for item in members}
+        context["bonus_counts"] = {player.seat: bonuses(player, room.pack) for player in room.state.players}
+        context["patrons"] = {item.id: item for item in room.pack.patrons}
+        context["objectives"] = {item.id: item for item in room.pack.objectives}
+        context["outposts"] = {item.id: item for item in room.pack.outposts}
+        commands = (
+            legal_commands(room.state, room.pack, room.settings, member.seat) if member.controller == "human" else []
+        )
+        context["actions"] = [(action_label(item, room), item.model_dump_json()) for item in commands]
+        context["double_resource_ids"] = {
+            str(item.payload["resource"])
+            for item in commands
+            if item.type == "take_double" and item.payload.get("resource")
+        }
+        card_actions: dict[str, dict[str, str]] = {}
+        deck_reserve_actions: dict[str, str] = {}
+        for item in commands:
+            card_id = item.payload.get("card_id")
+            if item.type in {"purchase", "reserve"} and card_id:
+                card_actions.setdefault(str(card_id), {})[item.type] = item.model_dump_json()
+            elif item.type == "reserve" and item.payload.get("deck"):
+                deck_reserve_actions[str(item.payload["deck"])] = item.model_dump_json()
+        context["card_actions"] = card_actions
+        context["deck_reserve_actions"] = deck_reserve_actions
+        context["scores"] = {player.seat: score(player, room.pack) for player in room.state.players}
+    return context
+
+
+def action_label(command: GameCommand, room: Room) -> str:
+    """Build a human-readable label for a legal game command."""
+    payload = command.payload
+    if command.type == "take_distinct":
+        label = "Take " + ", ".join(payload["resources"])
+    elif command.type == "take_double":
+        label = f"Take two {payload['resource']}"
+    elif command.type == "reserve":
+        if payload.get("card_id") and room.pack:
+            label = f"Reserve {room.pack.card(payload['card_id']).label}"
+        else:
+            label = f"Reserve from {payload.get('deck', 'deck')}"
+    elif command.type == "purchase" and room.pack:
+        label = f"Purchase {room.pack.card(payload['card_id']).label}"
+    elif command.type == "choose" and isinstance(payload.get("tokens"), dict) and room.pack:
+        resources = {item.id: item for item in [*room.pack.resources, room.pack.wild_resource]}
+        discarded = payload["tokens"]
+        label = "Discard " + ", ".join(
+            f"{amount} {resources[str(resource)].label}" for resource, amount in discarded.items()
+        )
+    elif command.type == "decline":
+        label = "Decline"
+    else:
+        choice = payload.get("choice") or payload.get("card_id") or "selection"
+        label = f"Choose {choice}"
+    return label
+
+
+def _selected_gem_command(room: Room, seat: int, selected: list[str]) -> GameCommand:
+    if room.state is None or room.pack is None:
+        error = "The game has not started"
+        raise RoomError(error)
+    if not selected:
+        error = "Select gem piles before taking gems"
+        raise RoomError(error)
+    if len(selected) != len(set(selected)) or not set(selected) <= set(room.pack.resource_ids):
+        error = "Choose each normal gem pile at most once"
+        raise RoomError(error)
+    commands = legal_commands(room.state, room.pack, room.settings, seat)
+    chosen = next(
+        (
+            item
+            for item in commands
+            if item.type == "take_double" and len(selected) == 1 and item.payload.get("resource") == selected[0]
+        ),
+        None,
+    )
+    if chosen is None:
+        chosen = next(
+            (
+                item
+                for item in commands
+                if item.type == "take_distinct"
+                and set(item.payload.get("resources", [])) == set(selected)
+                and len(item.payload.get("resources", [])) == len(selected)
+            ),
+            None,
+        )
+    if chosen is None:
+        error = "Choose three different available gems, or one pile with at least four gems to take a pair"
+        raise RoomError(error)
+    return chosen
+
+
+def _double_gem_command(room: Room, seat: int, resource: str) -> GameCommand:
+    if room.state is None or room.pack is None:
+        error = "The game has not started"
+        raise RoomError(error)
+    chosen = next(
+        (
+            item
+            for item in legal_commands(room.state, room.pack, room.settings, seat)
+            if item.type == "take_double" and item.payload.get("resource") == resource
+        ),
+        None,
+    )
+    if chosen is None:
+        error = "That gem pair is unavailable; a pile needs at least four gems"
+        raise RoomError(error)
+    return chosen
+
+
+def _discard_token_command(room: Room, seat: int, submitted: Mapping[str, object]) -> GameCommand:
+    if room.state is None or room.pack is None:
+        error = "The game has not started"
+        raise RoomError(error)
+    pending = room.state.pending
+    if pending is None or pending.kind != "discard_tokens" or pending.seat != seat:
+        error = "There are no excess tokens to discard"
+        raise RoomError(error)
+    discard: dict[str, int] = {}
+    for resource in (*room.pack.resource_ids, room.pack.wild_resource.id):
+        value = submitted.get(resource, "0")
+        try:
+            amount = int(value) if isinstance(value, str) else 0
+        except ValueError as exc:
+            error = "Token discard amounts must be whole numbers"
+            raise RoomError(error) from exc
+        if amount < 0:
+            error = "Token discard amounts cannot be negative"
+            raise RoomError(error)
+        if amount:
+            discard[resource] = amount
+    return build_command(room.state, "choose", {"tokens": discard})
+
+
+def _authenticate(request: Request, code: str) -> tuple[Room, Member]:
+    return service(request).authenticate(code, request.cookies.get(COOKIE_NAME))
+
+
+def _saved_display_name(request: Request) -> str:
+    name = request.cookies.get(DISPLAY_NAME_COOKIE, "").strip()
+    return name if len(name) <= DISPLAY_NAME_MAX_LENGTH else ""
+
+
+def _set_identity_cookies(response: Response, request: Request, credential: str, name: str) -> None:
+    cookie_options = {"httponly": True, "secure": request.app.state.config.secure_cookies, "samesite": "lax"}
+    response.set_cookie(COOKIE_NAME, credential, **cookie_options)
+    response.set_cookie(DISPLAY_NAME_COOKIE, name.strip(), max_age=DISPLAY_NAME_MAX_AGE, **cookie_options)
+
+
+def _check_csrf(request: Request, room_code: str, csrf: str) -> None:
+    credential = request.cookies.get(COOKIE_NAME, "")
+    if not service(request).credentials.valid_csrf(credential, room_code, csrf):
+        error = "Your form expired; reload and try again"
+        raise RoomError(error)
+    origin = request.headers.get("origin")
+    allowed_origins = {
+        request.app.state.config.public_origin.rstrip("/"),
+        _request_origin(request),
+    }
+    if origin and origin.rstrip("/") not in allowed_origins:
+        error = "Request origin was rejected"
+        raise RoomError(error)
+
+
+@router.get("/", response_class=HTMLResponse)
+def home(request: Request) -> Response:
+    """Render the room creation and join page."""
+    return templates.TemplateResponse(request, "home.html", {"display_name": _saved_display_name(request)})
+
+
+@router.post("/rooms")
+def create_room(request: Request, name: Annotated[str, Form()]) -> Response:
+    """Create a room and redirect its host to the lobby."""
+    try:
+        room, credential = service(request).create(name, request.cookies.get(COOKIE_NAME))
+    except RoomError as exc:
+        return templates.TemplateResponse(
+            request, "home.html", {"error": str(exc), "display_name": name}, status_code=422
+        )
+    response = RedirectResponse(f"/rooms/{room.code}", status_code=303)
+    _set_identity_cookies(response, request, credential, name)
+    return response
+
+
+@router.post("/join")
+def join_code(code: Annotated[str, Form()]) -> Response:
+    """Normalize a room code and redirect to its join page."""
+    return RedirectResponse(f"/join/{code.strip().upper()}", status_code=303)
+
+
+@router.get("/join/{code}", response_class=HTMLResponse)
+def join_page(request: Request, code: str) -> Response:
+    """Render the display-name form for an existing room."""
+    try:
+        room = service(request).require_room(code)
+    except RoomError:
+        return templates.TemplateResponse(
+            request,
+            "join.html",
+            {"code": code, "error": "Room not found", "display_name": _saved_display_name(request)},
+            status_code=404,
+        )
+    return templates.TemplateResponse(
+        request, "join.html", {"code": room.code, "display_name": _saved_display_name(request)}
+    )
+
+
+@router.post("/join/{code}")
+def join_room(request: Request, code: str, name: Annotated[str, Form()]) -> Response:
+    """Join a room and persist the member's browser identity."""
+    try:
+        room, credential = service(request).join(code, name, request.cookies.get(COOKIE_NAME))
+    except RoomError as exc:
+        return templates.TemplateResponse(
+            request, "join.html", {"code": code, "error": str(exc), "display_name": name}, status_code=422
+        )
+    response = RedirectResponse(f"/rooms/{room.code}", status_code=303)
+    _set_identity_cookies(response, request, credential, name)
+    return response
+
+
+@router.get("/rooms/{code}", response_class=HTMLResponse)
+def room_page(request: Request, code: str) -> Response:
+    """Render the authenticated lobby or game table."""
+    try:
+        room, member = _authenticate(request, code)
+    except RoomError:
+        return RedirectResponse(f"/join/{code}", status_code=303)
+    return templates.TemplateResponse(request, "room.html", _context(request, room, member))
+
+
+def _partial(request: Request, code: str, operation: RoomOperation, *, error_status: int = 422) -> Response:
+    try:
+        room, member = _authenticate(request, code)
+        operation(room, member)
+        room = service(request).require_room(code)
+        return templates.TemplateResponse(request, "partials/room_state.html", _context(request, room, member))
+    except (RoomError, ValidationError, ContentPackError) as exc:
+        try:
+            room, member = _authenticate(request, code)
+            return templates.TemplateResponse(
+                request,
+                "partials/room_state.html",
+                _context(request, room, member, error=str(exc)),
+                status_code=error_status,
+            )
+        except RoomError:
+            return HTMLResponse(html.escape(str(exc)), status_code=error_status)
+
+
+def _csrf_partial(request: Request, code: str, csrf: str, operation: RoomOperation) -> Response:
+    def authorized(room: Room, member: Member) -> None:
+        _check_csrf(request, room.code, csrf)
+        operation(room, member)
+
+    return _partial(request, code, authorized)
+
+
+@router.post("/rooms/{code}/pack", response_class=HTMLResponse)
+async def upload_pack(request: Request, code: str, csrf: Annotated[str, Form()], pack_file: UploadFile) -> Response:
+    """Validate and activate an uploaded JSON content pack."""
+    raw = await pack_file.read(512 * 1024 + 1)
+
+    def operation(room: Room, member: Member) -> None:
+        _check_csrf(request, room.code, csrf)
+        service(request).set_pack(room, member, parse_content_pack(raw))
+
+    return _partial(request, code, operation)
+
+
+@router.post("/rooms/{code}/settings/preset", response_class=HTMLResponse)
+def apply_preset(
+    request: Request,
+    code: str,
+    csrf: Annotated[str, Form()],
+    preset: Annotated[str, Form()],
+) -> Response:
+    """Apply a built-in rules preset to a lobby."""
+    presets: dict[str, tuple[Modules, Literal["score", "objective", "either", "both"]]] = {
+        "classic": (Modules(), "score"),
+        "objectives": (Modules(objectives=True), "objective"),
+        "objectives_outposts": (Modules(objectives=True, outposts=True), "objective"),
+        "eastern_fortifications": (Modules(eastern_decks=True, fortifications=True), "score"),
+        "all": (Modules(objectives=True, outposts=True, eastern_decks=True, fortifications=True), "objective"),
+    }
+
+    def operation(room: Room, member: Member) -> None:
+        _check_csrf(request, room.code, csrf)
+        if preset not in presets:
+            error = "Unknown rules preset"
+            raise RoomError(error)
+        modules, victory = presets[preset]
+        settings = room.settings.model_copy(deep=True)
+        settings.modules = modules
+        settings.victory_condition = victory
+        settings.interactions = Interactions()
+        service(request).set_settings(room, member, settings)
+
+    return _partial(request, code, operation)
+
+
+@router.get("/rooms/{code}/pack")
+def download_pack(request: Request, code: str) -> Response:
+    """Download the room's active content pack as JSON."""
+    room, _ = _authenticate(request, code)
+    if room.pack is None:
+        return JSONResponse({"detail": "No pack uploaded"}, status_code=404)
+    return JSONResponse(
+        room.pack.model_dump(mode="json"),
+        headers={"Content-Disposition": f'attachment; filename="{room.pack.metadata.id}.json"'},
+    )
+
+
+@router.post("/rooms/{code}/settings", response_class=HTMLResponse)
+async def update_settings(request: Request, code: str) -> Response:
+    """Validate and save custom lobby rules."""
+    form = await request.form()
+
+    def checked(name: str) -> bool:
+        return name in form
+
+    def value(name: str, default: str) -> str:
+        submitted = form.get(name)
+        return submitted if isinstance(submitted, str) else default
+
+    def operation(room: Room, member: Member) -> None:
+        _check_csrf(request, room.code, value("csrf", ""))
+        settings = GameSettings.model_validate(
+            {
+                "target_score": value("target_score", "15"),
+                "victory_condition": value("victory_condition", "score"),
+                "first_player_mode": value("first_player_mode", "random"),
+                "first_player_seat": value("first_player_seat", "0"),
+                "token_limit": value("token_limit", "10"),
+                "reserve_limit": value("reserve_limit", "3"),
+                "base_market_size": value("base_market_size", "4"),
+                "eastern_market_size": value("eastern_market_size", "2"),
+                "objective_count": value("objective_count", "3"),
+                "fortifications_per_player": value("fortifications_per_player", "3"),
+                "modules": Modules(
+                    objectives=checked("objectives"),
+                    outposts=checked("outposts"),
+                    eastern_decks=checked("eastern_decks"),
+                    fortifications=checked("fortifications"),
+                ),
+                "interactions": Interactions(**{field: checked(field) for field in Interactions.model_fields}),
+            }
+        )
+        service(request).set_settings(room, member, settings)
+
+    return _partial(request, code, operation)
+
+
+@router.post("/rooms/{code}/ready", response_class=HTMLResponse)
+def ready(
+    request: Request,
+    code: str,
+    csrf: Annotated[str, Form()],
+    ready_value: Annotated[bool, Form(alias="ready")],
+) -> Response:
+    """Update the authenticated human player's ready state."""
+    return _csrf_partial(
+        request, code, csrf, lambda room, member: service(request).set_ready(room, member, ready=ready_value)
+    )
+
+
+@router.post("/rooms/{code}/seats/ai", response_class=HTMLResponse)
+def add_ai(request: Request, code: str, csrf: Annotated[str, Form()], difficulty: Annotated[str, Form()]) -> Response:
+    """Add an AI-controlled player to a lobby."""
+    return _csrf_partial(request, code, csrf, lambda room, member: service(request).add_ai(room, member, difficulty))
+
+
+@router.post("/rooms/{code}/seats/{seat}/remove", response_class=HTMLResponse)
+def remove_seat(request: Request, code: str, seat: int, csrf: Annotated[str, Form()]) -> Response:
+    """Remove a non-host player from a lobby."""
+    return _csrf_partial(request, code, csrf, lambda room, member: service(request).remove_seat(room, member, seat))
+
+
+@router.post("/rooms/{code}/seats/{seat}/make-host", response_class=HTMLResponse)
+def transfer_host(request: Request, code: str, seat: int, csrf: Annotated[str, Form()]) -> Response:
+    """Transfer lobby ownership to another human player."""
+    return _csrf_partial(request, code, csrf, lambda room, member: service(request).transfer_host(room, member, seat))
+
+
+@router.post("/rooms/{code}/seats/{seat}/replace-with-ai", response_class=HTMLResponse)
+def replace_with_ai(
+    request: Request,
+    code: str,
+    seat: int,
+    csrf: Annotated[str, Form()],
+    difficulty: Annotated[str, Form()] = "medium",
+) -> Response:
+    """Replace a disconnected player with an AI controller."""
+    return _csrf_partial(
+        request,
+        code,
+        csrf,
+        lambda room, member: service(request).replace_with_ai(room, member, seat, difficulty),
+    )
+
+
+@router.post("/rooms/{code}/start", response_class=HTMLResponse)
+async def start_game(request: Request, code: str, csrf: Annotated[str, Form()]) -> Response:
+    """Start a game after validating the lobby's readiness."""
+    try:
+        room, member = _authenticate(request, code)
+        _check_csrf(request, room.code, csrf)
+        await service(request).start(room, member)
+        room = service(request).require_room(code)
+        return templates.TemplateResponse(request, "partials/room_state.html", _context(request, room, member))
+    except RoomError as exc:
+        room, member = _authenticate(request, code)
+        return templates.TemplateResponse(
+            request, "partials/room_state.html", _context(request, room, member, error=str(exc)), status_code=422
+        )
+
+
+@router.post("/rooms/{code}/play-again", response_class=HTMLResponse)
+async def play_again(request: Request, code: str, csrf: Annotated[str, Form()]) -> Response:
+    """Start a fresh game with the finished room's current setup."""
+    try:
+        room, member = _authenticate(request, code)
+        _check_csrf(request, room.code, csrf)
+        room = await service(request).play_again(room, member)
+        return templates.TemplateResponse(request, "partials/room_state.html", _context(request, room, member))
+    except RoomError as exc:
+        room, member = _authenticate(request, code)
+        return templates.TemplateResponse(
+            request, "partials/room_state.html", _context(request, room, member, error=str(exc)), status_code=422
+        )
+
+
+@router.post("/rooms/{code}/commands", response_class=HTMLResponse)
+async def game_command(
+    request: Request, code: str, csrf: Annotated[str, Form()], command_json: Annotated[str, Form()]
+) -> Response:
+    """Validate and submit one serialized game command."""
+    try:
+        room, member = _authenticate(request, code)
+        _check_csrf(request, room.code, csrf)
+        command = GameCommand.model_validate_json(command_json)
+        room = await service(request).submit(room, member, command)
+        return templates.TemplateResponse(request, "partials/room_state.html", _context(request, room, member))
+    except (RoomError, ValidationError) as exc:
+        room, member = _authenticate(request, code)
+        return templates.TemplateResponse(
+            request, "partials/room_state.html", _context(request, room, member, error=str(exc)), status_code=422
+        )
+
+
+@router.post("/rooms/{code}/take-gems", response_class=HTMLResponse)
+async def take_gems(request: Request, code: str) -> Response:
+    """Translate the visual gem selection into a legal command."""
+    form = await request.form()
+    try:
+        room, member = _authenticate(request, code)
+        _check_csrf(request, room.code, str(form.get("csrf", "")))
+        pair_resource = form.get("pair_resource")
+        if pair_resource is not None:
+            chosen = _double_gem_command(room, member.seat, str(pair_resource))
+        else:
+            selected = [str(item) for item in form.getlist("resources")]
+            chosen = _selected_gem_command(room, member.seat, selected)
+        room = await service(request).submit(room, member, chosen)
+        return templates.TemplateResponse(request, "partials/room_state.html", _context(request, room, member))
+    except RoomError as exc:
+        room, member = _authenticate(request, code)
+        return templates.TemplateResponse(
+            request, "partials/room_state.html", _context(request, room, member, error=str(exc)), status_code=422
+        )
+
+
+@router.post("/rooms/{code}/discard-tokens", response_class=HTMLResponse)
+async def discard_tokens(request: Request, code: str) -> Response:
+    """Return a player's selected excess tokens to the supply."""
+    form = await request.form()
+    try:
+        room, member = _authenticate(request, code)
+        _check_csrf(request, room.code, str(form.get("csrf", "")))
+        submitted = {
+            resource: form.get(f"token_{resource}", "0")
+            for resource in ((*room.pack.resource_ids, room.pack.wild_resource.id) if room.pack else ())
+        }
+        chosen = _discard_token_command(room, member.seat, submitted)
+        room = await service(request).submit(room, member, chosen)
+        return templates.TemplateResponse(request, "partials/room_state.html", _context(request, room, member))
+    except RoomError as exc:
+        room, member = _authenticate(request, code)
+        return templates.TemplateResponse(
+            request, "partials/room_state.html", _context(request, room, member, error=str(exc)), status_code=422
+        )
+
+
+@router.get("/rooms/{code}/events")
+async def room_events(request: Request, code: str) -> Response:
+    """Stream room revisions to an authenticated browser over SSE."""
+    try:
+        room, member = _authenticate(request, code)
+    except RoomError:
+        return HTMLResponse("Unauthorized", status_code=401)
+    queue = service(request).broker.subscribe(room.code)
+    stream_member = member
+
+    async def stream() -> AsyncIterator[str]:
+        current_member = stream_member
+        try:
+            # Comments keep the SSE connection alive without replacing in-progress form input.
+            yield ": connected\n\n"
+            while True:
+                try:
+                    await asyncio.wait_for(queue.get(), timeout=15)
+                    latest = service(request).require_room(code)
+                    yield _sse_fragment(request, latest, current_member)
+                except TimeoutError:
+                    _, current_member = _authenticate(request, code)
+                    yield ": keepalive\n\n"
+                if await request.is_disconnected():
+                    break
+        finally:
+            service(request).broker.unsubscribe(room.code, queue)
+
+    return StreamingResponse(
+        stream(), media_type="text/event-stream", headers={"Cache-Control": "no-store", "X-Accel-Buffering": "no"}
+    )
+
+
+def _sse_fragment(request: Request, room: Room, member: Member) -> str:
+    rendered = templates.get_template("partials/room_state.html").render(_context(request, room, member))
+    return "event: room\n" + "\n".join(f"data: {line}" for line in rendered.splitlines()) + "\n\n"
+
+
+@router.get("/schemas/content-pack-v1.json")
+def schema() -> Response:
+    """Return the JSON Schema for content-pack version one."""
+    return JSONResponse(content_pack_schema())
+
+
+@router.get("/healthz")
+def health() -> dict[str, str]:
+    """Report that the HTTP process is running."""
+    return {"status": "ok"}
+
+
+@router.get("/readyz")
+def readyz(request: Request) -> dict[str, str]:
+    """Report readiness after checking the database connection."""
+    request.app.state.repository._connection.execute("SELECT 1").fetchone()  # noqa: SLF001
+    return {"status": "ready"}

python/gems/security.py

@@ -0,0 +1,40 @@
+"""Opaque browser credentials and CSRF protection."""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import secrets
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+class CredentialService:
+    """Hash browser credentials with a stable per-installation key."""
+
+    def __init__(self, key_path: Path) -> None:
+        """Load or create the installation's credential-signing key."""
+        key_path.parent.mkdir(parents=True, exist_ok=True)
+        if not key_path.exists():
+            key_path.write_bytes(secrets.token_bytes(32))
+            key_path.chmod(0o600)
+        self._key = key_path.read_bytes()
+
+    @staticmethod
+    def issue() -> str:
+        """Issue a cryptographically random browser credential."""
+        return secrets.token_urlsafe(32)
+
+    def digest(self, credential: str) -> str:
+        """Create the persistent keyed digest of a browser credential."""
+        return hmac.new(self._key, credential.encode(), hashlib.sha256).hexdigest()
+
+    def csrf(self, credential: str, room_code: str) -> str:
+        """Create a room-scoped CSRF token for a browser credential."""
+        return hmac.new(self._key, f"csrf:{credential}:{room_code}".encode(), hashlib.sha256).hexdigest()
+
+    def valid_csrf(self, credential: str, room_code: str, candidate: str) -> bool:
+        """Validate a candidate CSRF token using constant-time comparison."""
+        return hmac.compare_digest(self.csrf(credential, room_code), candidate)

python/gems/static/overrides.css

@@ -0,0 +1,18 @@
+.preset-row{display:flex;gap:.5rem;flex-wrap:wrap;margin-bottom:1rem}.invite-link{margin:0;min-width:min(430px,55vw)}.danger{color:var(--danger)}.pack-status{border-left:3px solid var(--danger);padding:.65rem .8rem;background:#4a201d55;border-radius:0 9px 9px 0}button:disabled,button:disabled:hover{background:#252b2f;border-color:#3b4348;color:#7f898f;cursor:not-allowed;opacity:.7}
+.room-topbar{height:auto;min-height:68px;gap:1.25rem}.room-identity{display:flex;align-items:baseline;gap:.65rem;margin-right:auto}.room-identity .eyebrow{margin:0}.room-identity strong{font-size:1.2rem;letter-spacing:.14em}.room-invite{display:flex;align-items:center;grid-template-columns:auto minmax(260px,430px);gap:.65rem}.room-invite input{padding:.5rem .65rem}
+@media(max-width:700px){.room-topbar{flex-wrap:wrap;padding-block:.7rem}.room-invite{order:3;width:100%;grid-template-columns:auto 1fr}.invite-link{min-width:0}.room-identity strong{font-size:1rem}}
+
+.game-layout{grid-template-columns:330px minmax(0,1fr) 280px;grid-template-areas:"players market supply";align-items:start}.board-sidebar{display:contents}.players-stack{grid-area:players;display:grid;gap:.55rem}.market{grid-area:market}.right-supply{grid-area:supply;position:sticky;top:1rem}.sidebar-player{padding:.75rem}.sidebar-player.active{border-color:var(--gold);box-shadow:0 0 0 1px var(--gold),0 12px 28px #0003}.sidebar-player-summary{list-style:none;cursor:pointer}.sidebar-player-summary::-webkit-details-marker{display:none}.sidebar-player-heading{display:flex;justify-content:space-between;align-items:center;gap:.5rem}.sidebar-player-heading>span{color:var(--gold);font-size:.82rem}.count-legend{display:flex;justify-content:space-between;color:var(--muted);font-size:.68rem;margin:.45rem 0 .25rem}.player-color-counts{display:grid;grid-template-columns:repeat(3,1fr);gap:.3rem}.player-color-count{--gem-color:#697780;display:flex;align-items:center;justify-content:space-between;gap:.25rem;padding:.3rem .4rem;border:1px solid color-mix(in srgb,var(--gem-color) 55%,var(--line));border-radius:8px;background:color-mix(in srgb,var(--gem-color) 10%,#0e1418);font-size:.75rem}.player-color-count b{color:color-mix(in srgb,var(--gem-color) 70%,white);text-shadow:0 1px 2px #000}.player-detail-hint{display:block;margin-top:.45rem}.sidebar-player-details{border-top:1px solid var(--line);margin-top:.65rem;padding-top:.65rem}.owned-card-grid{display:grid;gap:.4rem;max-height:18rem;overflow-y:auto}.owned-card-inspector{--gem-color:#697780;border:1px solid color-mix(in srgb,var(--gem-color) 55%,var(--line));border-radius:9px;overflow:hidden;background:color-mix(in srgb,var(--gem-color) 9%,#0e1418)}.owned-card-inspector>summary{display:grid;grid-template-columns:1.8rem 1fr auto;align-items:center;gap:.45rem;padding:.5rem;cursor:pointer}.owned-card-bonus{display:grid;place-items:center;width:1.8rem;height:1.8rem;border-radius:50%;background:var(--gem-color);color:var(--gem-ink,#fff);font-weight:900;text-shadow:0 1px 2px #000}.owned-card-detail{padding:.5rem;border-top:1px solid var(--line);font-size:.78rem}.owned-card-detail p{margin:.25rem 0}.owned-card-detail .resource-cost{display:inline-block;margin:.15rem}
+.gem-picker{display:grid;gap:.55rem}.gem-picker .fine-print{margin:.1rem 0 .35rem}.gem-pile,.wild-pile{--gem-color:#697780;display:grid;grid-template-columns:auto 1fr auto;align-items:center;gap:.55rem;margin:0;padding:.55rem;border:1px solid color-mix(in srgb,var(--gem-color) 65%,var(--line));border-radius:12px;background:#0e1418;color:var(--text)}.gem-pile input{width:1.1rem;height:1.1rem;margin:0;accent-color:var(--gem-color)}.gem-choice{display:grid;grid-template-columns:2.6rem 1fr;align-items:center;gap:.55rem;margin:0;cursor:pointer}.gem-pile:has(input:checked){border-color:var(--gem-color);box-shadow:0 0 0 1px var(--gem-color),0 0 14px color-mix(in srgb,var(--gem-color) 35%,transparent);background:color-mix(in srgb,var(--gem-color) 18%,#0e1418)}.gem-pile.empty{opacity:.45}.take-pair{padding:.45rem .55rem;white-space:nowrap}.gem-disc{display:grid;place-items:center;width:2.5rem;height:2.5rem;border-radius:50%;border:3px solid color-mix(in srgb,var(--gem-color) 55%,white);background:radial-gradient(circle at 35% 30%,color-mix(in srgb,var(--gem-color) 45%,white),var(--gem-color) 65%);color:var(--gem-ink,#fff);text-shadow:0 1px 2px #000,0 0 3px #000;font-size:1.15rem;font-weight:900;box-shadow:0 4px 8px #0008,0 0 8px color-mix(in srgb,var(--gem-color) 35%,transparent)}.card-bonus-token{width:2rem;height:2rem;border-width:2px;font-size:.9rem;line-height:1;flex:none}.gem-pile strong,.wild-pile strong{color:color-mix(in srgb,var(--gem-color) 68%,white)}.gem-pile small,.wild-pile small{display:block}.gem-picker.readonly .gem-pile{grid-template-columns:2.6rem 1fr;cursor:default}.wild-pile{grid-template-columns:2.6rem 1fr;margin-top:.7rem}.resource-cost{border-color:color-mix(in srgb,var(--gem-color) 65%,var(--line))!important;box-shadow:inset 3px 0 0 var(--gem-color)}
+.gem-disc.light-gem{border-color:#090c0e;background:radial-gradient(circle at 32% 25%,#fff 0 18%,#f8f7f2 48%,#d9dde0 100%);color:#090c0e;font-weight:800;text-shadow:none;-webkit-font-smoothing:antialiased;text-rendering:geometricPrecision;box-shadow:0 4px 9px #0009,0 0 0 1px #596168,inset 0 0 0 2px #fff9}.hand-gem.light-gem>b{border:2px solid #090c0e;background:radial-gradient(circle at 32% 25%,#fff 0 18%,#f8f7f2 48%,#d9dde0 100%);color:#090c0e;font-weight:800;text-shadow:none;-webkit-font-smoothing:antialiased;text-rendering:geometricPrecision;box-shadow:0 2px 5px #0008,0 0 0 1px #596168,inset 0 0 0 1px #fff9}
+.card-bonus-token.light-gem{border-color:#090c0e;background:radial-gradient(circle at 32% 25%,#fff 0 16%,#f7f6f1 42%,#d9dde0 100%);color:#090c0e;text-shadow:none;box-shadow:0 3px 7px #000b,0 0 0 1px #596168,inset 0 0 0 2px #fff9}
+.tier-label{align-items:center}.compact-button{padding:.35rem .55rem}.game-card{display:flex;flex-direction:column;padding:0;overflow:hidden}.card-face{min-height:170px;padding:.9rem;flex:1}.game-card.actionable{border-color:#46615d}.card-actions{display:flex;gap:.45rem;padding:.75rem;border-top:1px solid var(--line);background:#0e1418}.card-actions form,.card-actions>button{flex:1}.card-actions button{width:100%;padding:.5rem}.board-help{grid-template-columns:170px 1fr}.board-help p{margin:0;color:var(--muted)}
+.reserved-cards{margin-top:1rem}.reserved-cards h3{margin-bottom:.5rem}.reserved-card{border:1px solid var(--line);border-radius:10px;margin-top:.4rem;overflow:hidden}.reserved-card-face{display:flex;justify-content:space-between;gap:.5rem;padding:.6rem}.reserved-card.actionable{border-color:var(--teal)}.reserved-card-face span{color:var(--muted);font-size:.8rem}
+@media(max-width:900px){.game-layout{grid-template-columns:1fr;grid-template-areas:"players" "market" "supply"}.right-supply{position:static}.gem-picker{grid-template-columns:repeat(2,minmax(0,1fr))}.gem-picker .fine-print,.gem-picker>.take-checked{grid-column:1/-1}}
+@media(max-width:540px){.gem-picker{grid-template-columns:1fr}.gem-picker .fine-print,.gem-picker>.take-checked{grid-column:auto}.board-help{display:flex}}
+.player-gem-dock{position:fixed;left:50%;bottom:.55rem;transform:translateX(-50%);z-index:15;display:flex;align-items:center;gap:.65rem;max-width:calc(100vw - 1rem);padding:.4rem .65rem;border:1px solid var(--gold);border-radius:14px;background:#151c21ee;backdrop-filter:blur(16px);box-shadow:0 12px 35px #0009}.player-gem-dock>strong{white-space:nowrap;color:var(--gold)}.hand-total{display:grid;place-items:center;min-width:3.8rem;padding:.25rem .45rem;border:1px solid var(--gold);border-radius:10px;background:#2a2417}.hand-total strong{color:var(--gold);font-size:.9rem}.hand-total small{font-size:.6rem;text-transform:uppercase;letter-spacing:.08em}.hand-legend{white-space:nowrap;color:var(--muted);font-size:.72rem}.player-gem-hand{display:flex;gap:.4rem}.hand-gem{--gem-color:#697780;display:grid;grid-template-columns:1.8rem auto;align-items:center;gap:.35rem;padding:.25rem .5rem .25rem .25rem;border:1px solid color-mix(in srgb,var(--gem-color) 65%,var(--line));border-radius:999px;background:color-mix(in srgb,var(--gem-color) 13%,#0e1418)}.hand-gem>b{display:grid;place-items:center;width:1.8rem;height:1.8rem;border-radius:50%;background:var(--gem-color);color:var(--gem-ink,#fff);text-shadow:0 1px 2px #000}.hand-gem>span{display:flex;align-items:center;gap:.2rem}.hand-gem>span strong{font-size:.95rem}.hand-gem i{color:var(--muted);font-style:normal}.action-dock{bottom:4.7rem}
+.bottom-reserved{display:flex;align-items:center;gap:.35rem;padding-left:.6rem;border-left:1px solid var(--line)}.bottom-reserved-card{display:flex;align-items:center;gap:.35rem;padding:.25rem;border:1px solid var(--line);border-radius:9px;background:#0e1418}.bottom-reserved-card>span{max-width:8rem}.bottom-reserved-card strong,.bottom-reserved-card small{display:block;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.bottom-reserved-card strong{font-size:.72rem}.bottom-reserved-card small{color:var(--muted);font-size:.62rem}.bottom-reserved-card button{padding:.35rem .45rem;font-size:.7rem}
+.action-dock>div:first-child small{display:block;margin-top:.25rem}.discard-picker{display:flex;align-items:center;gap:.65rem;min-width:0}.discard-colors{display:flex;gap:.4rem;overflow-x:auto;padding:.15rem;min-width:0}.discard-color{display:grid;grid-template-columns:2rem minmax(4.5rem,auto) 3.5rem;align-items:center;gap:.4rem;flex:none;margin:0;padding:.35rem;border:1px solid color-mix(in srgb,var(--gem-color) 65%,var(--line));border-radius:10px;background:color-mix(in srgb,var(--gem-color) 10%,#0e1418)}.discard-color .gem-disc{width:2rem;height:2rem;font-size:.9rem}.discard-color>span:nth-child(2){font-size:.75rem;font-weight:700;color:color-mix(in srgb,var(--gem-color) 70%,white)}.discard-color small{display:block;font-size:.6rem}.discard-color input{width:3.5rem;margin:0;padding:.4rem}.discard-picker>button{flex:none;white-space:nowrap}
+@media(max-width:700px){.player-gem-dock{width:calc(100vw - 1rem);overflow-x:auto;justify-content:flex-start}.player-gem-hand{flex:none}.player-gem-dock>strong{font-size:.8rem}.hand-gem{grid-template-columns:1.5rem auto}.hand-gem b{width:1.5rem;height:1.5rem}}
+
+@media(min-width:901px){.board-main{max-width:1900px;padding:clamp(.55rem,1.2vh,1rem) 1rem 7rem;scroll-padding-bottom:7rem}.board-main .status-strip{padding:.5rem 0;margin-bottom:.55rem}.board-main .game-layout{grid-template-columns:300px minmax(0,1fr) 270px;gap:.55rem}.board-main .players-stack{gap:.35rem}.board-main .sidebar-player{padding:.5rem}.board-main .count-legend{margin:.25rem 0 .15rem}.board-main .player-color-counts{gap:.2rem}.board-main .player-color-count{padding:.2rem .3rem}.board-main .player-detail-hint{margin-top:.25rem}.board-main .market{gap:.45rem}.board-main .market-row{padding:.55rem}.board-main .tier-label{margin-bottom:.4rem}.board-main .card-row{gap:.45rem}.board-main .card-face{min-height:108px;padding:.6rem}.board-main .game-card h3{font-size:1rem;line-height:1.15;margin:.55rem 0 .25rem}.board-main .game-card p{margin:.25rem 0}.board-main .card-actions{padding:.4rem;gap:.3rem}.board-main .card-actions button{padding:.4rem .25rem}.board-main .right-supply{padding:.65rem;top:.5rem}.board-main .right-supply h2{font-size:1.1rem;margin-bottom:.35rem}.board-main .right-supply .section-heading{margin-bottom:.35rem}.board-main .right-supply .gem-picker{gap:.3rem}.board-main .right-supply .gem-pile,.board-main .right-supply .wild-pile{padding:.3rem;gap:.35rem}.board-main .right-supply .gem-choice{grid-template-columns:2rem 1fr;gap:.35rem}.board-main .right-supply .gem-disc{width:2rem;height:2rem;font-size:.9rem}.board-main .right-supply .take-pair{padding:.3rem}.board-main .right-supply .fine-print{font-size:.7rem}.history-inline{position:relative}.history-inline summary{cursor:pointer;color:var(--teal)}.history-inline>div{position:absolute;right:0;top:1.8rem;width:min(360px,70vw);max-height:50vh;overflow:auto;z-index:20;padding:.7rem;border:1px solid var(--line);border-radius:10px;background:var(--panel);box-shadow:0 16px 40px #0008}.history-inline p{margin:.3rem 0;font-size:.8rem}.patrons-section{padding-bottom:.5rem;margin-bottom:.55rem;border-bottom:1px solid var(--line)}.patron-list{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:.3rem}.patron-card{display:grid;grid-template-columns:auto 1fr;align-items:start;gap:.3rem;padding:.35rem;border:1px solid var(--line);border-radius:8px;background:#0e1418;font-size:.72rem}.patron-card>strong{color:var(--gold)}.patron-card b,.patron-card small{display:block}.requirement-chip{display:inline-block;margin:.15rem .1rem 0;padding:.1rem .25rem;border:1px solid var(--line);border-radius:999px}.claimed-patrons{display:grid;gap:.25rem;margin-top:.5rem}.claimed-patrons span{font-size:.78rem;color:var(--gold)}}

python/gems/static/vendor/HTMX-LICENSE

@@ -0,0 +1,13 @@
+Zero-Clause BSD
+=============
+
+Permission to use, copy, modify, and/or distribute this software for
+any purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED “AS IS” AND THE AUTHOR DISCLAIMS ALL
+WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE
+FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY
+DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

python/gems/static/vendor/SSE-LICENSE

@@ -0,0 +1,14 @@
+BSD Zero Clause License
+
+Copyright (c) 2023, Alexander Petros
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

python/gems/static/vendor/sse.min.js

@@ -0,0 +1 @@
+(function(){var g;htmx.defineExtension("sse",{init:function(e){g=e;if(htmx.createEventSource==undefined){htmx.createEventSource=t}},getSelectors:function(){return["[sse-connect]","[data-sse-connect]","[sse-swap]","[data-sse-swap]"]},onEvent:function(e,t){var r=t.target||t.detail.elt;switch(e){case"htmx:beforeCleanupElement":var n=g.getInternalData(r);var s=n.sseEventSource;if(s){g.triggerEvent(r,"htmx:sseClose",{source:s,type:"nodeReplaced"});n.sseEventSource.close()}return;case"htmx:afterProcessNode":i(r)}}});function t(e){return new EventSource(e,{withCredentials:true})}function a(n){if(g.getAttributeValue(n,"sse-swap")){var s=g.getClosestMatch(n,v);if(s==null){return null}var e=g.getInternalData(s);var a=e.sseEventSource;var t=g.getAttributeValue(n,"sse-swap");var r=t.split(",");for(var i=0;i<r.length;i++){const u=r[i].trim();const c=function(e){if(l(s)){return}if(!g.bodyContains(n)){a.removeEventListener(u,c);return}if(!g.triggerEvent(n,"htmx:sseBeforeMessage",e)){return}f(n,e.data);g.triggerEvent(n,"htmx:sseMessage",e)};g.getInternalData(n).sseEventListener=c;a.addEventListener(u,c)}}if(g.getAttributeValue(n,"hx-trigger")){var s=g.getClosestMatch(n,v);if(s==null){return null}var e=g.getInternalData(s);var a=e.sseEventSource;var o=g.getTriggerSpecs(n);o.forEach(function(t){if(t.trigger.slice(0,4)!=="sse:"){return}var r=function(e){if(l(s)){return}if(!g.bodyContains(n)){a.removeEventListener(t.trigger.slice(4),r)}htmx.trigger(n,t.trigger,e);htmx.trigger(n,"htmx:sseMessage",e)};g.getInternalData(n).sseEventListener=r;a.addEventListener(t.trigger.slice(4),r)})}}function i(e,t){if(e==null){return null}if(g.getAttributeValue(e,"sse-connect")){var r=g.getAttributeValue(e,"sse-connect");if(r==null){return}n(e,r,t)}a(e)}function n(r,e,n){var s=htmx.createEventSource(e);s.onerror=function(e){g.triggerErrorEvent(r,"htmx:sseError",{error:e,source:s});if(l(r)){return}if(s.readyState===EventSource.CLOSED){n=n||0;n=Math.max(Math.min(n*2,128),1);var t=n*500;window.setTimeout(function(){i(r,n)},t)}};s.onopen=function(e){g.triggerEvent(r,"htmx:sseOpen",{source:s});if(n&&n>0){const t=r.querySelectorAll("[sse-swap], [data-sse-swap], [hx-trigger], [data-hx-trigger]");for(let e=0;e<t.length;e++){a(t[e])}n=0}};g.getInternalData(r).sseEventSource=s;var t=g.getAttributeValue(r,"sse-close");if(t){s.addEventListener(t,function(){g.triggerEvent(r,"htmx:sseClose",{source:s,type:"message"});s.close()})}}function l(e){if(!g.bodyContains(e)){var t=g.getInternalData(e).sseEventSource;if(t!=undefined){g.triggerEvent(e,"htmx:sseClose",{source:t,type:"nodeMissing"});t.close();return true}}return false}function f(t,r){g.withExtensions(t,function(e){r=e.transformResponse(r,null,t)});var e=g.getSwapSpecification(t);var n=g.getTarget(t);g.swap(n,r,e)}function v(e){return g.getInternalData(e).sseEventSource!=null}})();

python/gems/templates/base.html

@@ -0,0 +1,18 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <meta name="theme-color" content="#101418">
+  <meta name="htmx-config" content='{"responseHandling":[{"code":"204","swap":false},{"code":"[23]..","swap":true},{"code":"422","swap":true,"error":false},{"code":"[45]..","swap":false,"error":true}]}'>
+  <title>{% block title %}Gems{% endblock %}</title>
+  <link rel="stylesheet" href="/static/gems.css">
+  <link rel="stylesheet" href="/static/overrides.css">
+  <script src="/static/vendor/htmx.min.js" defer></script>
+  <script src="/static/vendor/sse.min.js" defer></script>
+</head>
+<body>
+  {% block topbar %}<header class="topbar"><a href="/" class="brand"><span class="brand-mark">◆</span> Gems</a></header>{% endblock %}
+  <main class="{% block main_class %}{% endblock %}">{% block content %}{% endblock %}</main>
+</body>
+</html>

python/gems/templates/home.html

@@ -0,0 +1,22 @@
+{% extends "base.html" %}
+{% block content %}
+<section class="hero">
+  <p class="eyebrow">A private strategy table</p>
+  <h1>Build an engine.<br>Race for prestige.</h1>
+  <p class="lede">Bring your own content pack, invite up to three friends or bots, and shape the rules before the first turn.</p>
+  {% if error %}<p class="alert" role="alert">{{ error }}</p>{% endif %}
+  <div class="home-grid">
+    <form class="panel" method="post" action="/rooms">
+      <h2>Create a table</h2>
+      <label>Display name<input name="name" maxlength="32" value="{{ display_name }}" required autocomplete="nickname"></label>
+      <button class="primary">Create room</button>
+    </form>
+    <form class="panel" method="post" action="/join">
+      <h2>Join a table</h2>
+      <label>Invite code<input name="code" minlength="8" maxlength="8" required autocapitalize="characters"></label>
+      <button>Continue</button>
+    </form>
+  </div>
+  <p class="fine-print">No cards, artwork, or playable content are bundled with Gems.</p>
+</section>
+{% endblock %}

python/gems/templates/join.html

@@ -0,0 +1,14 @@
+{% extends "base.html" %}
+{% block content %}
+<section class="narrow">
+  <p class="eyebrow">Invitation {{ code }}</p>
+  <div class="panel">
+    <h1>Take a seat</h1>
+    {% if error %}<p class="alert" role="alert">{{ error }}</p>{% endif %}
+    <form method="post" action="/join/{{ code }}">
+      <label>Display name<input name="name" maxlength="32" value="{{ display_name }}" required autofocus autocomplete="nickname"></label>
+      <button class="primary">Join room</button>
+    </form>
+  </div>
+</section>
+{% endblock %}

python/gems/templates/partials/room_state.html

@@ -0,0 +1,211 @@
+{% if error %}<p class="alert floating" role="alert">{{ error }}</p>{% endif %}
+
+{% if room.status == "lobby" %}
+<div class="lobby-grid">
+  <section class="panel">
+    <div class="section-heading"><h2>Seats</h2><span>{{ members|length }}/4</span></div>
+    <ol class="seat-list">
+      {% for seat in members %}
+      <li>
+        <span class="seat-number">{{ seat.seat + 1 }}</span>
+        <span><strong>{{ seat.name }}</strong><small>{{ seat.difficulty|capitalize if seat.controller == 'ai' else ('Ready' if seat.ready else 'Not ready') }}{% if seat.is_host %} · Host{% endif %}</small></span>
+        {% if member.is_host and not seat.is_host %}
+        <form hx-post="/rooms/{{ room.code }}/seats/{{ seat.seat }}/make-host" hx-target="#room-state" hx-swap="innerHTML">
+          <input type="hidden" name="csrf" value="{{ csrf }}"><button class="quiet">Make host</button>
+        </form>
+        <form hx-post="/rooms/{{ room.code }}/seats/{{ seat.seat }}/remove" hx-target="#room-state" hx-swap="innerHTML">
+          <input type="hidden" name="csrf" value="{{ csrf }}"><button class="icon-button" aria-label="Remove {{ seat.name }}">×</button>
+        </form>
+        {% endif %}
+      </li>
+      {% endfor %}
+    </ol>
+    {% if member.is_host and members|length < 4 %}
+    <form class="inline-form" hx-post="/rooms/{{ room.code }}/seats/ai" hx-target="#room-state" hx-swap="innerHTML">
+      <input type="hidden" name="csrf" value="{{ csrf }}">
+      <select name="difficulty" aria-label="AI difficulty"><option value="easy">Easy AI</option><option value="medium" selected>Medium AI</option><option value="hard">Hard AI</option></select>
+      <button>Add bot</button>
+    </form>
+    {% endif %}
+    {% if member.controller == 'human' %}
+    <form hx-post="/rooms/{{ room.code }}/ready" hx-target="#room-state" hx-swap="innerHTML">
+      <input type="hidden" name="csrf" value="{{ csrf }}"><input type="hidden" name="ready" value="{{ 'false' if member.ready else 'true' }}">
+      <button class="{{ 'quiet' if member.ready else 'primary' }}">{{ 'Not ready' if member.ready else 'Ready up' }}</button>
+    </form>
+    {% endif %}
+  </section>
+
+  <section class="panel">
+    <div class="section-heading"><h2>Content pack</h2>{% if room.pack %}<span class="success">Loaded and validated</span>{% else %}<span class="danger">Not loaded</span>{% endif %}</div>
+    {% if room.pack %}<p><strong>{{ room.pack.metadata.name }}</strong> <span class="muted">v{{ room.pack.metadata.version }}</span></p>{% else %}<p class="pack-status" role="status">No content pack is loaded. The game cannot start until the host uploads a valid JSON pack.</p>{% endif %}
+    {% if member.is_host %}
+    <form hx-post="/rooms/{{ room.code }}/pack" hx-target="#room-state" hx-swap="innerHTML" hx-encoding="multipart/form-data">
+      <input type="hidden" name="csrf" value="{{ csrf }}"><label>JSON file<input type="file" name="pack_file" accept="application/json,.json" required></label>
+      <button>Validate and use pack</button>
+    </form>
+    {% endif %}
+    <p class="fine-print"><a href="/schemas/content-pack-v1.json">JSON Schema</a>{% if room.pack %} · <a href="/rooms/{{ room.code }}/pack">Download active pack</a>{% endif %}</p>
+  </section>
+
+  <section class="panel rules-panel">
+    <div class="section-heading"><h2>Table rules</h2><span>Frozen on start</span></div>
+    {% if member.is_host %}<div class="preset-row">
+      {% for key, label in [('classic','Classic'),('objectives','Objective race'),('objectives_outposts','Objective + outpost'),('eastern_fortifications','Eastern + fortification'),('all','All modules')] %}
+      <form hx-post="/rooms/{{ room.code }}/settings/preset" hx-target="#room-state" hx-swap="innerHTML"><input type="hidden" name="csrf" value="{{ csrf }}"><input type="hidden" name="preset" value="{{ key }}"><button class="quiet">{{ label }}</button></form>
+      {% endfor %}
+    </div>{% endif %}
+    <form hx-post="/rooms/{{ room.code }}/settings" hx-target="#room-state" hx-swap="innerHTML">
+      <input type="hidden" name="csrf" value="{{ csrf }}">
+      <fieldset {% if not member.is_host %}disabled{% endif %}>
+        <div class="settings-grid">
+          <label>Target score<input type="number" name="target_score" min="1" max="999" value="{{ room.settings.target_score }}"></label>
+          <label>Victory<select name="victory_condition"><option value="score" {% if room.settings.victory_condition == 'score' %}selected{% endif %}>Score</option><option value="objective" {% if room.settings.victory_condition == 'objective' %}selected{% endif %}>Objective</option><option value="either" {% if room.settings.victory_condition == 'either' %}selected{% endif %}>Either</option><option value="both" {% if room.settings.victory_condition == 'both' %}selected{% endif %}>Both</option></select></label>
+          <label>First player<select name="first_player_mode"><option value="random" {% if room.settings.first_player_mode == 'random' %}selected{% endif %}>Random</option><option value="selected" {% if room.settings.first_player_mode == 'selected' %}selected{% endif %}>Choose player</option></select></label>
+          <label>Chosen player<select name="first_player_seat" aria-label="Chosen first player">{% for seat in members %}<option value="{{ loop.index0 }}" {% if room.settings.first_player_seat == loop.index0 %}selected{% endif %}>{{ seat.name }}</option>{% endfor %}</select><small>Used when “Choose player” is selected.</small></label>
+          <label>Token limit<input type="number" name="token_limit" min="1" max="99" value="{{ room.settings.token_limit }}"></label>
+          <label>Reserve limit<input type="number" name="reserve_limit" min="0" max="20" value="{{ room.settings.reserve_limit }}"></label>
+          <label>Base market/tier<input type="number" name="base_market_size" min="1" max="10" value="{{ room.settings.base_market_size }}"></label>
+          <label>Eastern market/tier<input type="number" name="eastern_market_size" min="1" max="10" value="{{ room.settings.eastern_market_size }}"></label>
+          <label>Objectives shown<input type="number" name="objective_count" min="1" max="10" value="{{ room.settings.objective_count }}"></label>
+          <label>Fortifications/player<input type="number" name="fortifications_per_player" min="1" max="10" value="{{ room.settings.fortifications_per_player }}"></label>
+        </div>
+        <h3>Modules</h3>
+        <div class="toggle-grid">
+          {% for key, label in [('objectives','Objectives'),('outposts','Outposts'),('eastern_decks','Eastern decks'),('fortifications','Fortifications')] %}
+          <label class="toggle"><input type="checkbox" name="{{ key }}" {% if room.settings.modules[key] %}checked{% endif %}><span>{{ label }}</span></label>
+          {% endfor %}
+        </div>
+        <details><summary>Advanced module interactions</summary><div class="toggle-grid advanced">
+          {% for key, value in room.settings.interactions %}<label class="toggle"><input type="checkbox" name="{{ key }}" {% if value %}checked{% endif %}><span>{{ key|replace('_',' ')|capitalize }}</span></label>{% endfor %}
+        </div></details>
+      </fieldset>
+      {% if member.is_host %}<button>Save rules</button>{% endif %}
+    </form>
+  </section>
+</div>
+{% if member.is_host %}
+<form class="start-bar" hx-post="/rooms/{{ room.code }}/start" hx-target="#room-state" hx-swap="innerHTML"><input type="hidden" name="csrf" value="{{ csrf }}"><span>{{ 'Pack loaded' if room.pack else 'Pack required' }} · {{ 'Players ready' if all_humans_ready else 'Waiting for every human to ready up' }} · Rules set</span><button class="primary"{% if not can_start %} disabled{% endif %}>Start game</button></form>
+{% endif %}
+
+{% else %}
+<section class="status-strip">
+  <span>Round {{ state.round_number }}</span><strong>{% if room.status == 'finished' %}Game complete · {% for seat in state.winners %}{{ state.players[seat].name }}{% if not loop.last %}, {% endif %}{% endfor %}{% else %}{{ state.players[state.current_seat].name }}’s turn{% endif %}</strong><span>Target {{ room.settings.target_score }}</span><details class="history-inline"><summary>History</summary><div>{% for entry in state.log[-12:]|reverse %}<p>{{ entry }}</p>{% endfor %}</div></details>{% if room.status == 'finished' %}{% if member.is_host %}<form hx-post="/rooms/{{ room.code }}/play-again" hx-target="#room-state" hx-swap="innerHTML"><input type="hidden" name="csrf" value="{{ csrf }}"><button class="primary">Play again</button></form>{% else %}<span class="muted">Waiting for the host</span>{% endif %}{% endif %}
+</section>
+
+<div class="game-layout">
+  <aside class="board-sidebar">
+    <section class="players-stack" aria-label="Players">
+    {% for player in state.players %}
+      <details class="sidebar-player panel compact{% if player.seat == state.current_seat %} active{% endif %}">
+        <summary class="sidebar-player-summary">
+          <div class="sidebar-player-heading"><strong>{{ player.name }}</strong><span>{% if member_by_seat[player.seat].controller == 'ai' %}Bot · {% endif %}{{ scores[player.seat] }}◆</span></div>
+          <div class="count-legend"><span>Color</span><span>Gems / Cards</span></div>
+          <div class="player-color-counts">
+            {% for resource_id in normal_resource_ids %}{% set resource = resources[resource_id] %}
+            <span class="player-color-count" style="--gem-color:{{ resource.color }}" title="{{ resource.label }}: {{ player.tokens[resource_id] }} gems, {{ bonus_counts[player.seat].get(resource_id, 0) }} cards"><b>{{ resource.symbol }}</b><span>{{ player.tokens[resource_id] }} / {{ bonus_counts[player.seat].get(resource_id, 0) }}</span></span>
+            {% endfor %}
+            {% set wild = pack.wild_resource %}<span class="player-color-count" style="--gem-color:{{ wild.color }}" title="{{ wild.label }}: {{ player.tokens[wild.id] }} gems"><b>{{ wild.symbol }}</b><span>{{ player.tokens[wild.id] }} / 0</span></span>
+          </div>
+          <small class="player-detail-hint">Click for card details · {{ player.cards|length }} owned · {{ player.reserved|length }} reserved</small>
+        </summary>
+        <div class="sidebar-player-details">
+          {% if player.cards %}<div class="owned-card-grid">
+            {% for owned in player.cards %}{% set card = cards[owned.card_id] %}{% set bonus_id = owned.copied_resource or card.bonus_resource %}
+            <details class="owned-card-inspector" {% if bonus_id %}style="--gem-color:{{ resources[bonus_id].color }};--gem-ink:{{ resources[bonus_id].ink_color }}"{% endif %}>
+              <summary><span class="owned-card-bonus">{{ resources[bonus_id].symbol if bonus_id else '◇' }}</span><strong>{{ card.label }}</strong><span>{{ card.points }}◆</span></summary>
+              <div class="owned-card-detail"><p>Cost {% for resource_id, value in card.cost.items() %}<span class="resource-cost" style="--gem-color:{{ resources[resource_id].color }}">{{ resources[resource_id].symbol }} {{ value }}</span>{% else %}<span>Free</span>{% endfor %}</p><p>Effect: {{ card.effect.kind|replace('_',' ') }}{% if owned.copied_resource %} · copied {{ resources[owned.copied_resource].label }}{% endif %}</p></div>
+            </details>
+            {% endfor %}
+          </div>{% else %}<p class="muted">No owned cards yet.</p>{% endif %}
+          {% if player.patrons %}<div class="claimed-patrons"><strong>Claimed patrons</strong>{% for patron_id in player.patrons %}<span>{{ patrons[patron_id].label }} · {{ patrons[patron_id].points }}◆</span>{% endfor %}</div>{% endif %}
+          {% if player.seat == member.seat and player.reserved %}
+          <div class="reserved-cards"><h3>Your reserved cards</h3>
+            {% for card_id in player.reserved %}{% set card = cards[card_id] %}{% set purchase = card_actions.get(card_id, {}).get('purchase') %}
+            <div class="reserved-card{% if purchase %} actionable{% endif %}"><div class="reserved-card-face"><strong>{{ card.label }}</strong><span>{{ card.points }}◆ · {% for resource_id, value in card.cost.items() %}{{ resources[resource_id].symbol }} {{ value }} {% endfor %}</span></div><div class="card-actions">{% if purchase %}<form hx-post="/rooms/{{ room.code }}/commands" hx-target="#room-state" hx-swap="innerHTML"><input type="hidden" name="csrf" value="{{ csrf }}"><input type="hidden" name="command_json" value="{{ purchase|e }}"><button class="primary">Purchase</button></form>{% else %}<button class="primary" disabled>Purchase</button>{% endif %}<button disabled>Reserved</button></div></div>
+            {% endfor %}
+          </div>
+          {% endif %}
+          {% if member.is_host and player.seat in offline_seats and player.seat != member.seat and room.status == 'playing' %}<form hx-post="/rooms/{{ room.code }}/seats/{{ player.seat }}/replace-with-ai" hx-target="#room-state" hx-swap="innerHTML"><input type="hidden" name="csrf" value="{{ csrf }}"><input type="hidden" name="difficulty" value="medium"><button>Replace with AI</button></form>{% endif %}
+        </div>
+      </details>
+    {% endfor %}
+    </section>
+    <section class="supply right-supply panel compact">
+    <div class="patrons-section">
+      <div class="section-heading"><h2>Patrons</h2><span>{{ state.available_patrons|length }} available</span></div>
+      <div class="patron-list">
+      {% for patron_id in state.available_patrons %}{% set patron = patrons[patron_id] %}
+        <article class="patron-card"><strong>{{ patron.points }}◆</strong><span><b>{{ patron.label }}</b><small>{% for requirement in patron.requirements %}{% if requirement.resource %}<span class="requirement-chip resource-cost" style="--gem-color:{{ resources[requirement.resource].color }}">{{ resources[requirement.resource].symbol }} {{ requirement.count }}</span>{% else %}<span class="requirement-chip">Any {{ requirement.count }}</span>{% endif %}{% endfor %}</small></span></article>
+      {% else %}<p class="muted">No unclaimed patrons.</p>
+      {% endfor %}
+      </div>
+    </div>
+    <div class="section-heading"><h2>Gem piles</h2><span>Supply</span></div>
+    {% if member.seat == state.current_seat and not state.pending and room.status == 'playing' %}
+    <form class="gem-picker" hx-post="/rooms/{{ room.code }}/take-gems" hx-target="#room-state" hx-swap="innerHTML">
+      <input type="hidden" name="csrf" value="{{ csrf }}">
+      <p class="fine-print">Check 3 different gems, or use Take 2 beside one pile.</p>
+      {% for resource_id in normal_resource_ids %}{% set resource = resources[resource_id] %}{% set amount = state.supply[resource_id] %}
+      <div class="gem-pile{% if not amount %} empty{% endif %}" style="--gem-color:{{ resource.color }};--gem-ink:{{ resource.ink_color }}">
+        <input id="gem-{{ resource_id }}" type="checkbox" name="resources" value="{{ resource_id }}" {% if not amount %}disabled{% endif %}>
+        <label class="gem-choice" for="gem-{{ resource_id }}"><span class="gem-disc{% if resource.ink_color == '#111111' %} light-gem{% endif %}">{{ resource.symbol }}</span><span><strong>{{ resource.label }}</strong><small>{{ amount }} available</small></span></label>
+        <button class="take-pair" type="submit" name="pair_resource" value="{{ resource_id }}" {% if resource_id not in double_resource_ids %}disabled{% endif %}>Take 2</button>
+      </div>
+      {% endfor %}
+      <button class="primary take-checked">Take checked gems</button>
+    </form>
+    {% else %}
+    <div class="gem-picker readonly">
+      {% for resource_id in normal_resource_ids %}{% set resource = resources[resource_id] %}
+      <div class="gem-pile" style="--gem-color:{{ resource.color }};--gem-ink:{{ resource.ink_color }}"><span class="gem-disc{% if resource.ink_color == '#111111' %} light-gem{% endif %}">{{ resource.symbol }}</span><span><strong>{{ resource.label }}</strong><small>{{ state.supply[resource_id] }} available</small></span></div>
+      {% endfor %}
+    </div>
+    {% endif %}
+    {% set wild = pack.wild_resource %}
+    <div class="wild-pile" style="--gem-color:{{ wild.color }};--gem-ink:{{ wild.ink_color }}"><span class="gem-disc{% if wild.ink_color == '#111111' %} light-gem{% endif %}">{{ wild.symbol }}</span><span><strong>{{ wild.label }}</strong><small>{{ state.supply[wild.id] }} available · gained by reserving</small></span></div>
+    </section>
+  </aside>
+  <section class="market">
+    {% for key, market in state.markets.items() %}
+    <div class="market-row">
+      <div class="tier-label"><strong>{{ key|replace(':',' · Tier ') }}</strong><span>{{ state.decks[key] }} left</span>{% if deck_reserve_actions.get(key) %}<form hx-post="/rooms/{{ room.code }}/commands" hx-target="#room-state" hx-swap="innerHTML"><input type="hidden" name="csrf" value="{{ csrf }}"><input type="hidden" name="command_json" value="{{ deck_reserve_actions[key]|e }}"><button class="quiet compact-button">Reserve top card</button></form>{% else %}<button class="quiet compact-button" disabled>Reserve top card</button>{% endif %}</div>
+      <div class="card-row">
+      {% for card_id in market %}{% set card = cards[card_id] %}{% set available = card_actions.get(card_id, {}) %}
+        <article class="game-card{% if available %} actionable{% endif %}">
+          <div class="card-face"><div class="card-top"><span>{{ card.points }}◆</span>{% if card.bonus_resource %}<span class="gem-disc card-bonus-token{% if resources[card.bonus_resource].ink_color == '#111111' %} light-gem{% endif %}" style="--gem-color:{{ resources[card.bonus_resource].color }};--gem-ink:{{ resources[card.bonus_resource].ink_color }}" role="img" aria-label="{{ resources[card.bonus_resource].label }} bonus" title="{{ resources[card.bonus_resource].label }} bonus">{{ resources[card.bonus_resource].symbol }}</span>{% else %}<span>◇</span>{% endif %}</div><h3>{{ card.label }}</h3><p>Tier {{ card.tier }} · {{ card.effect.kind|replace('_',' ') }}</p><div class="cost">{% for resource_id, value in card.cost.items() %}<span class="resource-cost" style="--gem-color:{{ resources[resource_id].color }}">{{ resources[resource_id].symbol }} {{ value }}</span>{% endfor %}</div>{% if state.fortifications.get(card_id) %}<small>Fortified</small>{% endif %}</div>
+          <div class="card-actions">
+            {% if available.get('purchase') %}<form hx-post="/rooms/{{ room.code }}/commands" hx-target="#room-state" hx-swap="innerHTML"><input type="hidden" name="csrf" value="{{ csrf }}"><input type="hidden" name="command_json" value="{{ available['purchase']|e }}"><button class="primary">Purchase</button></form>{% else %}<button class="primary" disabled>Purchase</button>{% endif %}
+            {% if available.get('reserve') %}<form hx-post="/rooms/{{ room.code }}/commands" hx-target="#room-state" hx-swap="innerHTML"><input type="hidden" name="csrf" value="{{ csrf }}"><input type="hidden" name="command_json" value="{{ available['reserve']|e }}"><button>Reserve</button></form>{% else %}<button disabled>Reserve</button>{% endif %}
+          </div>
+        </article>
+      {% endfor %}
+      </div>
+    </div>
+    {% endfor %}
+  </section>
+</div>
+
+{% set viewing_player = state.players[member.seat] %}
+<section class="player-gem-dock" aria-label="{{ viewing_player.name }} gems and cards">
+  <strong>{{ viewing_player.name }}</strong><span class="hand-total"><strong>{{ viewing_player.tokens.values()|sum }} / {{ room.settings.token_limit }}</strong><small>Gems</small></span><span class="hand-legend">Gems / Cards</span>
+  <div class="player-gem-hand">
+    {% for resource_id in normal_resource_ids %}{% set resource = resources[resource_id] %}<span class="hand-gem{% if resource.ink_color == '#111111' %} light-gem{% endif %}" style="--gem-color:{{ resource.color }};--gem-ink:{{ resource.ink_color }}" title="{{ resource.label }}: {{ viewing_player.tokens[resource_id] }} gems, {{ bonus_counts[viewing_player.seat].get(resource_id, 0) }} cards"><b>{{ resource.symbol }}</b><span><strong>{{ viewing_player.tokens[resource_id] }}</strong><i>/</i><strong>{{ bonus_counts[viewing_player.seat].get(resource_id, 0) }}</strong></span></span>{% endfor %}
+    {% set wild = pack.wild_resource %}<span class="hand-gem wild{% if wild.ink_color == '#111111' %} light-gem{% endif %}" style="--gem-color:{{ wild.color }};--gem-ink:{{ wild.ink_color }}" title="{{ wild.label }}: {{ viewing_player.tokens[wild.id] }} gems, 0 cards"><b>{{ wild.symbol }}</b><span><strong>{{ viewing_player.tokens[wild.id] }}</strong><i>/</i><strong>0</strong></span></span>
+  </div>
+  {% if viewing_player.reserved %}<div class="bottom-reserved"><span class="hand-legend">Reserved</span>{% for card_id in viewing_player.reserved %}{% set card = cards[card_id] %}{% set purchase = card_actions.get(card_id, {}).get('purchase') %}<div class="bottom-reserved-card"><span><strong>{{ card.label }}</strong><small>{{ card.points }}◆ · {% for resource_id, value in card.cost.items() %}{{ resources[resource_id].symbol }}{{ value }} {% endfor %}</small></span>{% if purchase %}<form hx-post="/rooms/{{ room.code }}/commands" hx-target="#room-state" hx-swap="innerHTML"><input type="hidden" name="csrf" value="{{ csrf }}"><input type="hidden" name="command_json" value="{{ purchase|e }}"><button class="primary">Purchase</button></form>{% else %}<button class="primary" disabled>Purchase</button>{% endif %}</div>{% endfor %}</div>{% endif %}
+</section>
+
+{% if room.status != 'finished' and member.seat == state.current_seat and state.pending %}
+<section class="action-dock"><div><p class="eyebrow">Your move</p><strong>Resolve: {{ state.pending.kind|replace('_',' ') }}</strong>{% if state.pending.kind == 'discard_tokens' %}<small>Return exactly {{ state.pending.amount }} excess gem{{ '' if state.pending.amount == 1 else 's' }}.</small>{% endif %}</div>
+  {% if state.pending.kind == 'discard_tokens' %}
+  <form class="discard-picker" hx-post="/rooms/{{ room.code }}/discard-tokens" hx-target="#room-state" hx-swap="innerHTML">
+    <input type="hidden" name="csrf" value="{{ csrf }}">
+    <div class="discard-colors">{% for resource_id in all_resource_ids %}{% set resource = resources[resource_id] %}<label class="discard-color" style="--gem-color:{{ resource.color }};--gem-ink:{{ resource.ink_color }}"><span class="gem-disc{% if resource.ink_color == '#111111' %} light-gem{% endif %}">{{ resource.symbol }}</span><span>{{ resource.label }}<small>{{ viewing_player.tokens[resource_id] }} held</small></span><input type="number" name="token_{{ resource_id }}" min="0" max="{{ viewing_player.tokens[resource_id] }}" value="0" aria-label="{{ resource.label }} gems to return"></label>{% endfor %}</div>
+    <button class="primary">Return {{ state.pending.amount }} gem{{ '' if state.pending.amount == 1 else 's' }}</button>
+  </form>
+  {% else %}
+  <div class="actions">{% for label, command_json in actions %}<form hx-post="/rooms/{{ room.code }}/commands" hx-target="#room-state" hx-swap="innerHTML"><input type="hidden" name="csrf" value="{{ csrf }}"><input type="hidden" name="command_json" value="{{ command_json|e }}"><button>{{ label }}</button></form>{% else %}<span class="muted">No legal action available.</span>{% endfor %}</div>
+  {% endif %}
+</section>
+{% endif %}
+{% endif %}

python/gems/templates/room.html

@@ -0,0 +1,17 @@
+{% extends "base.html" %}
+{% block title %}Room {{ room.code }} · Gems{% endblock %}
+{% block main_class %}board-main{% endblock %}
+{% block topbar %}
+<header class="topbar room-topbar">
+  <a href="/" class="brand"><span class="brand-mark">◆</span> Gems</a>
+  <div class="room-identity"><span class="eyebrow">Room</span><strong>{{ room.code }}</strong></div>
+  <label class="invite-link room-invite">Invite link<input readonly value="{{ public_origin }}/join/{{ room.code }}" aria-label="Invite link"></label>
+</header>
+{% endblock %}
+{% block content %}
+<div class="room-shell">
+  <div id="room-state" hx-ext="sse" sse-connect="/rooms/{{ room.code }}/events" sse-swap="room">
+    {% include "partials/room_state.html" %}
+  </div>
+</div>
+{% endblock %}

python/gems/web.py

@@ -0,0 +1,11 @@
+"""Template and static resource configuration."""
+
+from pathlib import Path
+
+from fastapi.templating import Jinja2Templates
+
+PACKAGE_DIR = Path(__file__).resolve().parent
+TEMPLATE_DIR = PACKAGE_DIR / "templates"
+STATIC_DIR = PACKAGE_DIR / "static"
+
+templates = Jinja2Templates(directory=TEMPLATE_DIR)

python/orm/common.py

@@ -7,6 +7,7 @@ from typing import cast
 
 from sqlalchemy import create_engine
 from sqlalchemy.engine import URL, Engine
+from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
 
 NAMING_CONVENTION = {
     "ix": "ix_%(table_name)s_%(column_0_name)s",
@@ -31,23 +32,16 @@ def get_connection_info(name: str) -> tuple[str, str, str, str, str | None]:
     return cast("tuple[str, str, str, str, str | None]", (database, host, port, username, password))
 
 
-def get_postgres_engine(
-    *,
-    name: str = "POSTGRES",
-    pool_pre_ping: bool = True,
-    vector_engine: bool = False,
-) -> Engine:
-    """Create a SQLAlchemy engine from environment variables.
+def build_postgres_url(name: str, *, vector_engine: bool = False) -> tuple[URL, dict[str, str]]:
+    """Build the Postgres connection URL and connect_args from environment variables.
 
     Args:
-        name (str, optional): The name of the environment variable prefix. Defaults to "POSTGRES".
-        pool_pre_ping (bool, optional): Whether to ping the database before each connection. Defaults to True.
-        This fixes the issue of trying to use a conection that has timed out on the database side.
+        name (str): The name of the environment variable prefix.
         vector_engine (bool, optional): Whether to use the vector search schema. Defaults to False.
-        This updates the search path the incldued the vecore types and operators.
+        This updates the search path to include the vector types and operators.
 
     Returns:
-        Engine: The SQLAlchemy engine.
+        tuple[URL, dict[str, str]]: The SQLAlchemy URL and connect_args for create_engine.
     """
     database, host, port, username, password = get_connection_info(name)
 
@@ -65,9 +59,66 @@ def get_postgres_engine(
     if vector_engine:
         connect_args["options"] = "-csearch_path=main,public"
 
+    return url, connect_args
+
+
+def get_postgres_engine(
+    *,
+    name: str = "POSTGRES",
+    pool_pre_ping: bool = True,
+    vector_engine: bool = False,
+    pool_size: int = 8,
+) -> Engine:
+    """Create a SQLAlchemy engine from environment variables.
+
+    Args:
+        name (str, optional): The name of the environment variable prefix. Defaults to "POSTGRES".
+        pool_pre_ping (bool, optional): Whether to ping the database before each connection. Defaults to True.
+        This fixes the issue of trying to use a conection that has timed out on the database side.
+        vector_engine (bool, optional): Whether to use the vector search schema. Defaults to False.
+        This updates the search path the incldued the vecore types and operators.
+        pool_size (int, optional): Number of connections to keep in the pool. Defaults to 8.
+
+    Returns:
+        Engine: The SQLAlchemy engine.
+    """
+    url, connect_args = build_postgres_url(name, vector_engine=vector_engine)
+
     return create_engine(
         url=url,
         pool_pre_ping=pool_pre_ping,
         pool_recycle=1800,
         connect_args=connect_args,
+        pool_size=pool_size,
+    )
+
+
+def get_async_postgres_engine(
+    *,
+    name: str = "POSTGRES",
+    pool_pre_ping: bool = True,
+    vector_engine: bool = False,
+    pool_size: int = 8,
+) -> AsyncEngine:
+    """Create an async SQLAlchemy engine from environment variables.
+
+    Args:
+        name (str, optional): The name of the environment variable prefix. Defaults to "POSTGRES".
+        pool_pre_ping (bool, optional): Whether to ping the database before each connection. Defaults to True.
+        This fixes the issue of trying to use a conection that has timed out on the database side.
+        vector_engine (bool, optional): Whether to use the vector search schema. Defaults to False.
+        This updates the search path the incldued the vecore types and operators.
+        pool_size (int, optional): Number of connections to keep in the pool. Defaults to 8.
+
+    Returns:
+        AsyncEngine: The async SQLAlchemy engine.
+    """
+    url, connect_args = build_postgres_url(name, vector_engine=vector_engine)
+
+    return create_async_engine(
+        url=url,
+        pool_pre_ping=pool_pre_ping,
+        pool_recycle=1800,
+        connect_args=connect_args,
+        pool_size=pool_size,
     )

python/orm/richie/__init__.py

@@ -12,12 +12,16 @@ from python.orm.richie.contact import (
     RelationshipType,
 )
 from python.orm.richie.ebook import (
+    EbookCandidatePhrase,
     EbookChapter,
     EbookChunk,
     EbookChunkEmbedding1024,
     EbookChunkEmbedding2560,
     EbookChunkEmbedding4096,
+    EbookChunkPhraseMention,
     EbookEmbeddingModel,
+    EbookPhraseAlias,
+    EbookProtectedPhrase,
     EbookSource,
 )
 
@@ -28,12 +32,16 @@ __all__ = [
     "Contact",
     "ContactNeed",
     "ContactRelationship",
+    "EbookCandidatePhrase",
     "EbookChapter",
     "EbookChunk",
     "EbookChunkEmbedding1024",
     "EbookChunkEmbedding2560",
     "EbookChunkEmbedding4096",
+    "EbookChunkPhraseMention",
     "EbookEmbeddingModel",
+    "EbookPhraseAlias",
+    "EbookProtectedPhrase",
     "EbookSource",
     "Need",
     "RelationshipType",

python/orm/richie/ebook.py

@@ -5,11 +5,23 @@ from __future__ import annotations
 from datetime import datetime
 
 from pgvector.sqlalchemy import Vector
-from sqlalchemy import BigInteger, Boolean, DateTime, ForeignKey, Index, String, UniqueConstraint
+from sqlalchemy import (
+    JSON,
+    BigInteger,
+    DateTime,
+    ForeignKey,
+    Index,
+    String,
+    Text,
+    UniqueConstraint,
+)
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from python.orm.richie.base import TableBase, TableBaseBig
 
+JSON_DOCUMENT = JSON().with_variant(JSONB, "postgresql")
+
 
 class EbookSource(TableBase):
     """One indexed EPUB file."""
@@ -94,7 +106,7 @@ class EbookEmbeddingModel(TableBase):
 
     name: Mapped[str] = mapped_column(String, unique=True)
     dimension: Mapped[int]
-    is_default: Mapped[bool] = mapped_column(Boolean, default=False)
+    is_default: Mapped[bool] = mapped_column(default=False)
 
 
 class EbookChunkEmbedding1024(TableBaseBig):
@@ -136,3 +148,94 @@ class EbookChunkEmbedding4096(TableBaseBig):
     chunk_id: Mapped[int] = mapped_column(ForeignKey("main.ebook_chunk.id", ondelete="CASCADE"))
     model_id: Mapped[int] = mapped_column(ForeignKey("main.ebook_embedding_model.id", ondelete="CASCADE"))
     embedding: Mapped[list[float]] = mapped_column(Vector(4096))
+
+
+class EbookCandidatePhrase(TableBase):
+    """A high-recall phrase candidate extracted from one book."""
+
+    __tablename__ = "candidate_phrases"
+    __table_args__ = (
+        UniqueConstraint("book_id", "phrase_norm", name="uq_candidate_phrases_book_id_phrase_norm"),
+        Index("candidate_phrases_book_score_idx", "book_id", "candidate_score"),
+        Index("candidate_phrases_book_norm_idx", "book_id", "phrase_norm"),
+    )
+
+    book_id: Mapped[int] = mapped_column(ForeignKey("main.ebook_source.id", ondelete="CASCADE"))
+    series_id: Mapped[int | None]
+    phrase_text: Mapped[str] = mapped_column(Text)
+    phrase_norm: Mapped[str] = mapped_column(Text)
+    token_count: Mapped[int]
+    source_raw_ngram: Mapped[bool] = mapped_column(default=False)
+    source_yake: Mapped[bool] = mapped_column(default=False)
+    source_capitalized: Mapped[bool] = mapped_column(default=False)
+    source_metadata: Mapped[bool] = mapped_column(default=False)
+    raw_count: Mapped[int] = mapped_column(default=0)
+    chapter_count: Mapped[int] = mapped_column(default=0)
+    yake_score: Mapped[float | None]
+    candidate_score: Mapped[float] = mapped_column(default=0.0)
+    sample_contexts: Mapped[list[str] | None] = mapped_column(JSON_DOCUMENT)
+    llm_judged: Mapped[bool] = mapped_column(default=False)
+    llm_keep: Mapped[bool | None]
+    llm_confidence: Mapped[float | None]
+    llm_category: Mapped[str | None]
+    llm_reason: Mapped[str | None] = mapped_column(Text)
+
+
+class EbookProtectedPhrase(TableBase):
+    """A phrase accepted by the LLM judge for protected query matching."""
+
+    __tablename__ = "protected_phrases"
+    __table_args__ = (
+        UniqueConstraint("book_id", "phrase_norm", name="uq_protected_phrases_book_id_phrase_norm"),
+        Index("protected_phrases_norm_idx", "phrase_norm"),
+        Index("protected_phrases_book_norm_idx", "book_id", "phrase_norm"),
+        Index("protected_phrases_series_norm_idx", "series_id", "phrase_norm"),
+    )
+
+    book_id: Mapped[int | None] = mapped_column(ForeignKey("main.ebook_source.id", ondelete="CASCADE"))
+    series_id: Mapped[int | None]
+    phrase_text: Mapped[str] = mapped_column(Text)
+    phrase_norm: Mapped[str] = mapped_column(Text)
+    canonical_id: Mapped[str]
+    phrase_type: Mapped[str | None]
+    token_count: Mapped[int]
+    confidence: Mapped[float]
+    importance: Mapped[float] = mapped_column(default=0.5)
+    allow_nested: Mapped[bool] = mapped_column(default=False)
+    suppress_children: Mapped[bool] = mapped_column(default=True)
+    source_candidate_id: Mapped[int | None] = mapped_column(
+        ForeignKey("main.candidate_phrases.id", ondelete="SET NULL")
+    )
+
+
+class EbookPhraseAlias(TableBase):
+    """A normalized alias that maps to a protected phrase."""
+
+    __tablename__ = "phrase_aliases"
+    __table_args__ = (
+        UniqueConstraint("phrase_id", "alias_norm", name="uq_phrase_aliases_phrase_id_alias_norm"),
+        Index("phrase_aliases_norm_idx", "alias_norm"),
+    )
+
+    phrase_id: Mapped[int] = mapped_column(ForeignKey("main.protected_phrases.id", ondelete="CASCADE"))
+    alias_text: Mapped[str] = mapped_column(Text)
+    alias_norm: Mapped[str] = mapped_column(Text)
+    confidence: Mapped[float] = mapped_column(default=1.0)
+
+
+class EbookChunkPhraseMention(TableBase):
+    """A precomputed occurrence of a protected phrase inside one chunk."""
+
+    __tablename__ = "chunk_phrase_mentions"
+    __table_args__ = (
+        UniqueConstraint("chunk_id", "phrase_id", "start_char", name="uq_chunk_phrase_mentions_chunk_phrase_start"),
+        Index("chunk_phrase_mentions_phrase_idx", "phrase_id"),
+        Index("chunk_phrase_mentions_chunk_idx", "chunk_id"),
+    )
+
+    chunk_id: Mapped[int] = mapped_column(ForeignKey("main.ebook_chunk.id", ondelete="CASCADE"))
+    phrase_id: Mapped[int] = mapped_column(ForeignKey("main.protected_phrases.id", ondelete="CASCADE"))
+    book_id: Mapped[int | None] = mapped_column(ForeignKey("main.ebook_source.id", ondelete="CASCADE"))
+    series_id: Mapped[int | None]
+    start_char: Mapped[int]
+    end_char: Mapped[int | None]

python/zfs/__init__.py

@@ -1,11 +1,30 @@
 """init."""
 
-from python.zfs.dataset import Dataset, Snapshot, get_datasets
+# run_command is deliberately not re-exported here. It is available from
+# python.zfs.command when something genuinely needs another binary, but the
+# wrappers are what callers should reach for by default.
+from python.zfs.command import CommandResult, run_zfs, run_zpool
+from python.zfs.dataset import (
+    Dataset,
+    Snapshot,
+    create_dataset,
+    get_datasets,
+    get_properties,
+    list_dataset_names,
+    set_property,
+)
 from python.zfs.zpool import Zpool
 
 __all__ = [
+    "CommandResult",
     "Dataset",
     "Snapshot",
     "Zpool",
+    "create_dataset",
     "get_datasets",
+    "get_properties",
+    "list_dataset_names",
+    "run_zfs",
+    "run_zpool",
+    "set_property",
 ]

python/zfs/command.py

@@ -0,0 +1,90 @@
+"""Running zfs and zpool commands.
+
+One implementation shared by both, so the zpool side gets the same handling the
+zfs side does: arguments passed as a list, streams kept apart, and failures
+returned as data rather than guessed at by the caller.
+"""
+
+from __future__ import annotations
+
+import logging
+import subprocess
+from dataclasses import dataclass
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class CommandResult:
+    """The outcome of a single zfs or zpool invocation."""
+
+    args: tuple[str, ...]
+    stdout: str
+    stderr: str
+    return_code: int
+
+    @property
+    def ok(self) -> bool:
+        """Whether the command reported success."""
+        return self.return_code == 0
+
+    @property
+    def message(self) -> str:
+        """The most useful description of what went wrong."""
+        return (self.stderr or self.stdout).strip()
+
+
+def run_command(*args: str) -> CommandResult:
+    """Run a command, passing arguments as a list rather than a shell string.
+
+    Two things this buys over bash_wrapper. Arguments are never split on
+    whitespace, so a value containing a space arrives intact. And stdout stays
+    separate from stderr, so a warning on a successful command is never
+    mistaken for output, which bash_wrapper does whenever stderr is non-empty
+    regardless of the return code.
+
+    The encoding is pinned rather than using text=True, which would decode with
+    the locale encoding. These run from systemd units, where LANG is often
+    unset.
+
+    Args:
+        *args: The command and its arguments.
+
+    Returns:
+        CommandResult: The streams and return code, never raising on failure.
+    """
+    completed = subprocess.run(list(args), capture_output=True, encoding="utf-8", check=False)
+
+    if completed.returncode != 0:
+        logger.debug(f"{' '.join(args)} exited {completed.returncode}: {completed.stderr.strip()}")
+
+    return CommandResult(
+        args=tuple(args),
+        stdout=completed.stdout,
+        stderr=completed.stderr,
+        return_code=completed.returncode,
+    )
+
+
+def run_zfs(*args: str) -> CommandResult:
+    """Run a zfs command.
+
+    Args:
+        *args: The arguments to pass to zfs.
+
+    Returns:
+        CommandResult: The streams and return code.
+    """
+    return run_command("zfs", *args)
+
+
+def run_zpool(*args: str) -> CommandResult:
+    """Run a zpool command.
+
+    Args:
+        *args: The arguments to pass to zpool.
+
+    Returns:
+        CommandResult: The streams and return code.
+    """
+    return run_command("zpool", *args)

python/zfs/dataset.py

@@ -8,6 +8,7 @@ from datetime import UTC, datetime
 from typing import Any
 
 from python.common import bash_wrapper
+from python.zfs.command import run_zfs
 
 logger = logging.getLogger(__name__)
 
@@ -207,8 +208,91 @@ def get_datasets() -> list[Dataset]:
     """
     logger.info("Getting zfs list")
 
-    dataset_names, _ = bash_wrapper("zfs list -Hp -t filesystem -o name")
+    return [Dataset(dataset_name) for dataset_name in list_dataset_names() if "/" in dataset_name]
 
-    cleaned_datasets = dataset_names.strip().split("\n")
 
-    return [Dataset(dataset_name) for dataset_name in cleaned_datasets if "/" in dataset_name]
+def get_properties(name: str) -> dict[str, tuple[str, str]]:
+    """Get every property of a dataset along with where its value came from.
+
+    The source is what distinguishes a property that was deliberately set on
+    this dataset from one that is merely inherited or left at its default.
+
+    Args:
+        name (str): The name of the dataset.
+
+    Returns:
+        dict[str, tuple[str, str]]: A mapping of property name to (value, source).
+    """
+    result = run_zfs("get", "-Hp", "-o", "property,value,source", "all", name)
+    if not result.ok:
+        error = f"Failed to get properties for {name}: {result.message}"
+        raise RuntimeError(error)
+
+    properties = {}
+    for line in result.stdout.strip().splitlines():
+        if not line:
+            continue
+        prop, value, source = line.split("\t")
+        properties[prop] = (value, source)
+
+    return properties
+
+
+def create_dataset(name: str, properties: dict[str, str]) -> str | None:
+    """Create a dataset with the given properties.
+
+    Args:
+        name (str): The name of the dataset.
+        properties (dict[str, str]): The properties to create the dataset with.
+
+    Returns:
+        str | None: An error message on failure, None on success.
+    """
+    options = [argument for key, value in sorted(properties.items()) for argument in ("-o", f"{key}={value}")]
+    logger.debug(f"creating {name} with {properties}")
+
+    result = run_zfs("create", *options, name)
+    if not result.ok:
+        return f"Failed to create {name}: {result.message}"
+    return None
+
+
+def set_property(name: str, key: str, value: str) -> str | None:
+    """Set a single property on a dataset.
+
+    Args:
+        name (str): The name of the dataset.
+        key (str): The property to set.
+        value (str): The value to set the property to.
+
+    Returns:
+        str | None: An error message on failure, None on success.
+    """
+    logger.debug(f"setting {key}={value} on {name}")
+
+    result = run_zfs("set", f"{key}={value}", name)
+    if not result.ok:
+        return f"Failed to set {key}={value} on {name}: {result.message}"
+    return None
+
+
+def list_dataset_names() -> list[str]:
+    """List every zfs filesystem name, including pool root filesystems.
+
+    Unlike get_datasets this does not build Dataset objects and does not filter
+    out pool roots, which makes it usable for existence checks.
+
+    Returns:
+        list[str]: The names of every zfs filesystem.
+
+    Raises:
+        RuntimeError: If zfs list fails. Never returns a partial or error
+            derived list, since treating stderr as dataset names would make
+            the reconciler think every dataset is missing.
+    """
+    result = run_zfs("list", "-Hp", "-t", "filesystem", "-o", "name")
+    if not result.ok:
+        error = f"Failed to list ZFS datasets: {result.message}"
+        raise RuntimeError(error)
+
+    return [name for name in result.stdout.strip().splitlines() if name]

python/zfs/zpool.py

@@ -1,28 +1,42 @@
-"""test."""
+"""zpool."""
 
 from __future__ import annotations
 
 import json
 from typing import Any
 
-from python.common import bash_wrapper
+from python.zfs.command import run_zpool
 
 
-def _zpool_list(zfs_list: str) -> dict[str, Any]:
-    """Check the version of zfs."""
-    raw_zfs_list_data, _ = bash_wrapper(zfs_list)
+def _zpool_list(*args: str) -> dict[str, Any]:
+    """Run a zpool list and check the output is a format we understand.
 
-    zfs_list_data = json.loads(raw_zfs_list_data)
+    Args:
+        *args: The arguments to pass to zpool.
 
-    vers_major = zfs_list_data["output_version"]["vers_major"]
-    vers_minor = zfs_list_data["output_version"]["vers_minor"]
-    command = zfs_list_data["output_version"]["command"]
+    Returns:
+        dict[str, Any]: The decoded output.
+
+    Raises:
+        RuntimeError: If zpool fails, or reports a format this does not parse.
+            Never decodes a partial or error derived payload.
+    """
+    result = run_zpool(*args)
+    if not result.ok:
+        error = f"Failed to run zpool {' '.join(args)}: {result.message}"
+        raise RuntimeError(error)
+
+    zpool_data = json.loads(result.stdout)
+
+    vers_major = zpool_data["output_version"]["vers_major"]
+    vers_minor = zpool_data["output_version"]["vers_minor"]
+    command = zpool_data["output_version"]["command"]
 
     if vers_major != 0 or vers_minor != 1 or command != "zpool list":
         error = f"Datasets are not in the correct format {vers_major=} {vers_minor=} {command=}"
         raise RuntimeError(error)
 
-    return zfs_list_data
+    return zpool_data
 
 
 class Zpool:
@@ -33,7 +47,7 @@ class Zpool:
         name: str,
     ) -> None:
         """__init__."""
-        zpool_data = _zpool_list(f"zpool list {name} -pHj -o all")
+        zpool_data = _zpool_list("list", name, "-pHj", "-o", "all")
 
         properties = zpool_data["pools"][name]["properties"]
 

rust/van_weather/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "van-weather"
+version = "0.1.0"
+edition = "2024"
+description = "Fetch privacy-masked weather for a van and publish it to Home Assistant"
+license = "MIT"
+
+[dependencies]
+anyhow = "1.0"
+chrono = "0.4"
+clap = { version = "4.5", features = ["derive", "env"] }
+env_logger = "0.11"
+log = "0.4"
+reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+
+[lints.clippy]
+all = "deny"
+pedantic = "deny"
+

rust/van_weather/package.nix

@@ -0,0 +1,13 @@
+{ rustPlatform }:
+rustPlatform.buildRustPackage {
+  pname = "van-weather";
+  version = "0.1.0";
+  src = ./.;
+
+  cargoLock.lockFile = ./Cargo.lock;
+
+  meta = {
+    description = "Privacy-masked van weather publisher for Home Assistant";
+    mainProgram = "van-weather";
+  };
+}

rust/van_weather/src/main.rs

@@ -0,0 +1,565 @@
+use std::{thread, time::Duration};
+
+use anyhow::{Context, Result, bail};
+use chrono::{DateTime, Utc};
+use clap::Parser;
+use log::{error, info};
+use reqwest::{
+    StatusCode, Url,
+    blocking::{Client, ClientBuilder},
+    header::{AUTHORIZATION, HeaderMap, HeaderValue},
+    retry,
+};
+use serde::Deserialize;
+use serde_json::{Value, json};
+
+const LAT_ENTITY: &str = "sensor.van_last_known_latitude";
+const LON_ENTITY: &str = "sensor.van_last_known_longitude";
+const PIRATE_WEATHER_HOST: &str = "api.pirateweather.net";
+const MASK_DECIMALS: u32 = 1;
+const MASK_FACTOR: f64 = decimal_factor(MASK_DECIMALS);
+const RETRIES_PER_REQUEST: u32 = 2;
+
+const fn decimal_factor(decimals: u32) -> f64 {
+    let mut factor = 1.0;
+    let mut remaining = decimals;
+    while remaining > 0 {
+        factor *= 10.0;
+        remaining -= 1;
+    }
+    factor
+}
+
+#[derive(Debug, Parser)]
+#[command(about, version)]
+struct Args {
+    #[arg(long, env = "HA_URL")]
+    ha_url: String,
+
+    #[arg(long, env = "HA_TOKEN", hide_env_values = true)]
+    ha_token: String,
+
+    #[arg(long, env = "PIRATE_WEATHER_API_KEY", hide_env_values = true)]
+    pirate_weather_api_key: String,
+
+    #[arg(
+        long,
+        default_value_t = 900,
+        value_parser = clap::value_parser!(u64).range(1..)
+    )]
+    interval: u64,
+
+    #[arg(long, default_value = "info", env = "RUST_LOG")]
+    log_level: String,
+}
+
+#[derive(Debug, Deserialize)]
+struct HaState {
+    state: String,
+}
+
+#[derive(Debug)]
+struct HttpClients {
+    home_assistant: Client,
+    pirate_weather: Client,
+}
+
+#[derive(Debug, Default, Deserialize)]
+struct ApiResponse {
+    #[serde(default)]
+    currently: CurrentWeather,
+    #[serde(default)]
+    daily: ForecastBlock<DailyApiForecast>,
+    #[serde(default)]
+    hourly: ForecastBlock<HourlyApiForecast>,
+}
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct CurrentWeather {
+    temperature: Option<f64>,
+    apparent_temperature: Option<f64>,
+    humidity: Option<f64>,
+    wind_speed: Option<f64>,
+    wind_bearing: Option<f64>,
+    icon: Option<String>,
+    pressure: Option<f64>,
+    visibility: Option<f64>,
+    uv_index: Option<f64>,
+    ozone: Option<f64>,
+    nearest_storm_distance: Option<f64>,
+    nearest_storm_bearing: Option<f64>,
+    precip_probability: Option<f64>,
+    cloud_cover: Option<f64>,
+}
+
+#[derive(Debug, Deserialize)]
+struct ForecastBlock<T> {
+    #[serde(default)]
+    data: Vec<T>,
+}
+
+impl<T> Default for ForecastBlock<T> {
+    fn default() -> Self {
+        Self { data: Vec::new() }
+    }
+}
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct DailyApiForecast {
+    time: Option<i64>,
+    icon: Option<String>,
+    temperature_high: Option<f64>,
+    temperature_low: Option<f64>,
+    precip_probability: Option<f64>,
+}
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct HourlyApiForecast {
+    time: Option<i64>,
+    icon: Option<String>,
+    temperature: Option<f64>,
+    precip_probability: Option<f64>,
+}
+
+#[derive(Debug)]
+struct Weather {
+    current: CurrentWeather,
+    daily: Vec<DailyForecast>,
+    hourly: Vec<HourlyForecast>,
+}
+
+#[derive(Debug)]
+struct DailyForecast {
+    datetime: DateTime<Utc>,
+    condition: &'static str,
+    temperature: Option<f64>,
+    templow: Option<f64>,
+    precipitation_probability: Option<f64>,
+}
+
+#[derive(Debug)]
+struct HourlyForecast {
+    datetime: DateTime<Utc>,
+    condition: &'static str,
+    temperature: Option<f64>,
+    precipitation_probability: Option<f64>,
+}
+
+fn main() -> Result<()> {
+    let Args {
+        ha_url,
+        ha_token,
+        pirate_weather_api_key,
+        interval,
+        log_level,
+    } = Args::parse();
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or(log_level.as_str()))
+        .init();
+
+    let ha_url = ha_url.trim_end_matches('/').to_owned();
+    let ha_host = Url::parse(&ha_url)
+        .context("HA_URL is not a valid URL")?
+        .host_str()
+        .context("HA_URL has no host")?
+        .to_owned();
+    let clients = HttpClients {
+        home_assistant: build_client(&ha_host, Some(&ha_token))?,
+        pirate_weather: build_client(PIRATE_WEATHER_HOST, None)?,
+    };
+
+    info!("Starting van weather service, polling every {interval}s");
+    loop {
+        if let Err(err) = update_weather(&clients, &ha_url, &pirate_weather_api_key) {
+            error!("Weather update failed: {err:#}");
+        }
+        thread::sleep(Duration::from_secs(interval));
+    }
+}
+
+fn build_client(host: &str, bearer_token: Option<&str>) -> Result<Client> {
+    let policy = retry::for_host(host.to_owned())
+        .max_retries_per_request(RETRIES_PER_REQUEST)
+        .classify_fn(|request| {
+            let retryable = request.error().is_some()
+                || request.status().is_some_and(|status| {
+                    status == StatusCode::REQUEST_TIMEOUT
+                        || status == StatusCode::TOO_MANY_REQUESTS
+                        || status.is_server_error()
+                });
+            if retryable {
+                request.retryable()
+            } else {
+                request.success()
+            }
+        });
+
+    let mut builder = ClientBuilder::new()
+        .timeout(Duration::from_secs(30))
+        .retry(policy);
+
+    if let Some(token) = bearer_token {
+        let mut authorization = HeaderValue::from_str(&format!("Bearer {token}"))
+            .context("HA_TOKEN contains invalid header characters")?;
+        authorization.set_sensitive(true);
+        let mut headers = HeaderMap::new();
+        headers.insert(AUTHORIZATION, authorization);
+        builder = builder.default_headers(headers);
+    }
+
+    builder
+        .build()
+        .with_context(|| format!("failed to create HTTP client for {host}"))
+}
+
+fn update_weather(clients: &HttpClients, ha_url: &str, api_key: &str) -> Result<()> {
+    let lat = get_ha_state(&clients.home_assistant, ha_url, LAT_ENTITY)?;
+    let lon = get_ha_state(&clients.home_assistant, ha_url, LON_ENTITY)?;
+    let masked_lat = mask_coordinate(lat);
+    let masked_lon = mask_coordinate(lon);
+    info!("Masked location: {masked_lat}, {masked_lon}");
+
+    let weather = fetch_weather(&clients.pirate_weather, api_key, masked_lat, masked_lon)?;
+    info!(
+        "Weather: {}°F, {}",
+        weather
+            .current
+            .temperature
+            .map_or_else(|| "unknown".to_owned(), |value| value.to_string()),
+        condition(weather.current.icon.as_deref())
+    );
+    post_to_ha(&clients.home_assistant, ha_url, &weather)?;
+    info!("Posted weather to Home Assistant");
+    Ok(())
+}
+
+fn mask_coordinate(value: f64) -> f64 {
+    (value * MASK_FACTOR).round() / MASK_FACTOR
+}
+
+fn get_ha_state(client: &Client, ha_url: &str, entity_id: &str) -> Result<f64> {
+    let HaState { state } = client
+        .get(format!("{ha_url}/api/states/{entity_id}"))
+        .send()
+        .with_context(|| format!("request for {entity_id} failed"))?
+        .error_for_status()
+        .with_context(|| format!("Home Assistant rejected {entity_id} request"))?
+        .json()
+        .context("Home Assistant returned invalid JSON")?;
+
+    if matches!(state.as_str(), "unavailable" | "unknown") {
+        bail!("{entity_id} is {state}");
+    }
+
+    state
+        .parse::<f64>()
+        .with_context(|| format!("{entity_id} state is not numeric: {state}"))
+}
+
+fn fetch_weather(client: &Client, api_key: &str, lat: f64, lon: f64) -> Result<Weather> {
+    let response = client
+        .get(format!(
+            "https://{PIRATE_WEATHER_HOST}/forecast/{api_key}/{lat},{lon}"
+        ))
+        .query(&[("units", "us")])
+        .send()
+        .context("Pirate Weather request failed")?
+        .error_for_status()
+        .context("Pirate Weather rejected request")?;
+    let data = response
+        .json::<ApiResponse>()
+        .context("Pirate Weather returned invalid JSON")?;
+    Ok(parse_weather(data))
+}
+
+fn parse_weather(data: ApiResponse) -> Weather {
+    let daily = data
+        .daily
+        .data
+        .into_iter()
+        .take(8)
+        .filter_map(|day| {
+            timestamp(day.time).map(|datetime| DailyForecast {
+                datetime,
+                condition: condition(day.icon.as_deref()),
+                temperature: day.temperature_high,
+                templow: day.temperature_low,
+                precipitation_probability: day.precip_probability,
+            })
+        })
+        .collect();
+    let hourly = data
+        .hourly
+        .data
+        .into_iter()
+        .take(48)
+        .filter_map(|hour| {
+            timestamp(hour.time).map(|datetime| HourlyForecast {
+                datetime,
+                condition: condition(hour.icon.as_deref()),
+                temperature: hour.temperature,
+                precipitation_probability: hour.precip_probability,
+            })
+        })
+        .collect();
+    Weather {
+        current: data.currently,
+        daily,
+        hourly,
+    }
+}
+
+fn timestamp(value: Option<i64>) -> Option<DateTime<Utc>> {
+    value
+        .filter(|value| *value != 0)
+        .and_then(DateTime::from_timestamp_secs)
+}
+
+fn condition(icon: Option<&str>) -> &'static str {
+    match icon.unwrap_or_default() {
+        "clear-day" => "sunny",
+        "clear-night" => "clear-night",
+        "rain" => "rainy",
+        "snow" => "snowy",
+        "sleet" => "snowy-rainy",
+        "wind" => "windy",
+        "fog" => "fog",
+        "partly-cloudy-day" | "partly-cloudy-night" => "partlycloudy",
+        _ => "cloudy",
+    }
+}
+
+fn post_to_ha(client: &Client, ha_url: &str, weather: &Weather) -> Result<()> {
+    for (entity_id, payload) in weather_updates(weather) {
+        let response = client
+            .post(format!("{ha_url}/api/states/{entity_id}"))
+            .json(&payload)
+            .send()
+            .with_context(|| format!("failed to post {entity_id}"))?;
+        ensure_success(response.status(), &entity_id)?;
+    }
+    Ok(())
+}
+
+fn ensure_success(status: StatusCode, entity_id: &str) -> Result<()> {
+    if status.is_success() {
+        Ok(())
+    } else {
+        bail!("Home Assistant rejected {entity_id} update with {status}")
+    }
+}
+
+#[allow(clippy::too_many_lines)]
+fn weather_updates(weather: &Weather) -> Vec<(String, Value)> {
+    let current = &weather.current;
+    let mut updates = vec![
+        sensor(
+            "sensor.van_weather_condition",
+            Some(json!(condition(current.icon.as_deref()))),
+            json!({"friendly_name": "Van Weather Condition"}),
+        ),
+        sensor(
+            "sensor.van_weather_temperature",
+            current.temperature.map(|value| json!(value)),
+            json!({"unit_of_measurement": "°F", "device_class": "temperature"}),
+        ),
+        sensor(
+            "sensor.van_weather_apparent_temperature",
+            current.apparent_temperature.map(|value| json!(value)),
+            json!({"unit_of_measurement": "°F", "device_class": "temperature"}),
+        ),
+        sensor(
+            "sensor.van_weather_humidity",
+            Some(json!(percent(current.humidity))),
+            json!({"unit_of_measurement": "%", "device_class": "humidity"}),
+        ),
+        sensor(
+            "sensor.van_weather_pressure",
+            current.pressure.map(|value| json!(value)),
+            json!({"unit_of_measurement": "mbar", "device_class": "pressure"}),
+        ),
+        sensor(
+            "sensor.van_weather_wind_speed",
+            current.wind_speed.map(|value| json!(value)),
+            json!({"unit_of_measurement": "mph", "device_class": "wind_speed"}),
+        ),
+        sensor(
+            "sensor.van_weather_wind_bearing",
+            current.wind_bearing.map(|value| json!(value)),
+            json!({"unit_of_measurement": "°"}),
+        ),
+        sensor(
+            "sensor.van_weather_visibility",
+            current.visibility.map(|value| json!(value)),
+            json!({"unit_of_measurement": "mi"}),
+        ),
+        sensor(
+            "sensor.van_weather_uv_index",
+            current.uv_index.map(|value| json!(value)),
+            json!({"friendly_name": "Van Weather UV Index", "icon": "mdi:sun-wireless"}),
+        ),
+        sensor(
+            "sensor.van_weather_ozone",
+            current.ozone.map(|value| json!(value)),
+            json!({"unit_of_measurement": "DU", "icon": "mdi:earth"}),
+        ),
+        sensor(
+            "sensor.van_weather_nearest_storm_distance",
+            current.nearest_storm_distance.map(|value| json!(value)),
+            json!({"unit_of_measurement": "mi", "icon": "mdi:weather-lightning"}),
+        ),
+        sensor(
+            "sensor.van_weather_nearest_storm_bearing",
+            current.nearest_storm_bearing.map(|value| json!(value)),
+            json!({"unit_of_measurement": "°", "icon": "mdi:weather-lightning"}),
+        ),
+        sensor(
+            "sensor.van_weather_precip_probability",
+            Some(json!(percent(current.precip_probability))),
+            json!({"unit_of_measurement": "%", "icon": "mdi:weather-rainy"}),
+        ),
+        sensor(
+            "sensor.van_weather_cloud_cover",
+            Some(json!(percent(current.cloud_cover))),
+            json!({"unit_of_measurement": "%", "icon": "mdi:weather-cloudy"}),
+        ),
+    ]
+    .into_iter()
+    .flatten()
+    .collect::<Vec<_>>();
+
+    let daily = weather
+        .daily
+        .iter()
+        .map(|forecast| {
+            json!({
+                "datetime": forecast.datetime.to_rfc3339(),
+                "condition": forecast.condition,
+                "temperature": forecast.temperature,
+                "templow": forecast.templow,
+                "precipitation_probability": percent(forecast.precipitation_probability),
+            })
+        })
+        .collect::<Vec<_>>();
+    updates.push((
+        "sensor.van_weather_forecast_daily".to_owned(),
+        json!({"state": daily.len(), "attributes": {"forecast": daily}}),
+    ));
+
+    let hourly = weather
+        .hourly
+        .iter()
+        .map(|forecast| {
+            json!({
+                "datetime": forecast.datetime.to_rfc3339(),
+                "condition": forecast.condition,
+                "temperature": forecast.temperature,
+                "precipitation_probability": percent(forecast.precipitation_probability),
+            })
+        })
+        .collect::<Vec<_>>();
+    updates.push((
+        "sensor.van_weather_forecast_hourly".to_owned(),
+        json!({"state": hourly.len(), "attributes": {"forecast": hourly}}),
+    ));
+    updates
+}
+
+fn sensor(entity_id: &str, state: Option<Value>, attributes: Value) -> Option<(String, Value)> {
+    state.map(|state| {
+        let mut payload = serde_json::Map::new();
+        payload.insert("state".to_owned(), state);
+        payload.insert("attributes".to_owned(), attributes);
+        (entity_id.to_owned(), Value::Object(payload))
+    })
+}
+
+#[allow(clippy::cast_possible_truncation)]
+fn percent(value: Option<f64>) -> i64 {
+    // Preserve Python's int(probability * 100) behavior for Home Assistant.
+    (value.unwrap_or_default() * 100.0) as i64
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn maps_conditions_for_home_assistant() {
+        assert_eq!(condition(Some("clear-day")), "sunny");
+        assert_eq!(condition(Some("sleet")), "snowy-rainy");
+        assert_eq!(condition(Some("partly-cloudy-night")), "partlycloudy");
+        assert_eq!(condition(Some("unexpected")), "cloudy");
+        assert_eq!(condition(None), "cloudy");
+    }
+
+    #[test]
+    fn parses_api_response_and_builds_compatible_payloads() {
+        let response: ApiResponse = serde_json::from_value(json!({
+            "currently": {
+                "temperature": 72.5,
+                "humidity": 0.67,
+                "icon": "clear-day",
+                "precipProbability": 0.129,
+                "cloudCover": 0.4,
+                "summary": "Fine"
+            },
+            "daily": {"data": [{
+                "time": 1_700_000_000,
+                "icon": "rain",
+                "temperatureHigh": 75.0,
+                "temperatureLow": 52.0,
+                "precipProbability": 0.8
+            }]},
+            "hourly": {"data": [{
+                "time": 1_700_000_000,
+                "icon": "fog",
+                "temperature": 61.0,
+                "precipProbability": 0.05
+            }]}
+        }))
+        .unwrap();
+
+        let weather = parse_weather(response);
+        let updates = weather_updates(&weather);
+        let find = |id: &str| updates.iter().find(|(entity, _)| entity == id).unwrap();
+
+        assert_eq!(find("sensor.van_weather_condition").1["state"], "sunny");
+        assert_eq!(find("sensor.van_weather_humidity").1["state"], 67);
+        assert_eq!(find("sensor.van_weather_precip_probability").1["state"], 12);
+        assert_eq!(find("sensor.van_weather_forecast_daily").1["state"], 1);
+        assert_eq!(
+            find("sensor.van_weather_forecast_daily").1["attributes"]["forecast"][0]["condition"],
+            "rainy"
+        );
+        assert_eq!(find("sensor.van_weather_forecast_hourly").1["state"], 1);
+    }
+
+    #[test]
+    fn omits_missing_optional_current_sensors_but_keeps_percentage_sensors() {
+        let weather = parse_weather(ApiResponse::default());
+        let updates = weather_updates(&weather);
+
+        assert!(
+            !updates
+                .iter()
+                .any(|(id, _)| id == "sensor.van_weather_temperature")
+        );
+        assert_eq!(
+            updates
+                .iter()
+                .find(|(id, _)| id == "sensor.van_weather_humidity")
+                .unwrap()
+                .1["state"],
+            0
+        );
+    }
+
+    #[test]
+    fn masks_coordinates_to_about_eleven_kilometres() {
+        assert!((mask_coordinate(37.7749) - 37.8).abs() < f64::EPSILON);
+        assert!((mask_coordinate(-122.4194) - (-122.4)).abs() < f64::EPSILON);
+    }
+}