feat(richie user env): adding app_image_path configuration

Add test_protected_phrases.py covering phrase-matching behavior in the
RAG engine, and update the existing ebook_search tests to use the async
SQLAlchemy engine/session (create_async_engine, AsyncSession) and async
HTTP paths.

.dockerignore

@@ -3,19 +3,6 @@
 .mypy_cache
 .pytest_cache
 .ruff_cache
-.venv
-**/.venv
-.env
-.cache
-.claude
-.coverage
-.vscode
-.stfolder
-.literotica_data
-esphome
-htmlcov
-data
-ebooks
 __pycache__
 **/__pycache__
 *.pyc

.github/workflows/test_ebook_search.yml

@@ -1,26 +0,0 @@
-name: test ebook search
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
-
-env:
-  UV_PYTHON_DOWNLOADS: never
-  UV_CACHE_DIR: /var/cache/uv
-  UV_LINK_MODE: copy
-
-jobs:
-  test-ebook-search:
-    runs-on: self-hosted
-
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install dependencies
-        run: nix develop .#devShells.x86_64-linux.ebook-search -c uv sync --locked --project python/ebook_search/docker
-      - name: Run ebook search tests
-        run: nix develop .#devShells.x86_64-linux.ebook-search -c uv run --project python/ebook_search/docker --no-sync pytest tests/ebook_search --override-ini addopts="-n auto -ra"

flake.lock

@@ -8,11 +8,11 @@
       },
       "locked": {
         "dir": "pkgs/firefox-addons",
-        "lastModified": 1783828963,
-        "narHash": "sha256-eTytzcUJCaDUZ3/9EF0+V3fvlikQMQBwiX1Sx4Gy+No=",
+        "lastModified": 1782964936,
+        "narHash": "sha256-wXEBDr7/dFQYhVpDwCKc9fkrYQQE4x0bdirX1bsLBGA=",
         "owner": "rycee",
         "repo": "nur-expressions",
-        "rev": "8d61e9afde605cd6c22dab68b83d7a71f0a6c5b2",
+        "rev": "64feee871e0373dd6121e412c3fb12e372d1bfb5",
         "type": "gitlab"
       },
       "original": {
@@ -29,11 +29,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1783823409,
-        "narHash": "sha256-OI4IkRjRXa1e7hYmCGJDPDq5H/kPwhsyoS80cNUF9fI=",
+        "lastModified": 1783005591,
+        "narHash": "sha256-NcLHV5uBAeggDUE2wPbKszjfyaSLsoqaYt7izOphkZw=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "7566825d4652a1b885bd4ce65bd9e8def432fec9",
+        "rev": "f469c79b955609d6a8fdd9e689be76a93b1621d7",
         "type": "github"
       },
       "original": {
@@ -47,11 +47,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1783792734,
-        "narHash": "sha256-50rvY9GdFvpYDcMLcD/4cWSi0hVxArT5wsGlVsHy8eY=",
+        "lastModified": 1782562157,
+        "narHash": "sha256-a7+T6QSeowynwZ1ZJJbP8T8ntAytvrui8kFGJmIZt2c=",
         "owner": "nixos",
         "repo": "nixos-hardware",
-        "rev": "8efb4337e857949f4cfac86d12ef1066f417f31f",
+        "rev": "a9cf7546a938c737b079e738de73934a13de9784",
         "type": "github"
       },
       "original": {
@@ -76,11 +76,11 @@
     },
     "nixpkgs-master": {
       "locked": {
-        "lastModified": 1783874024,
-        "narHash": "sha256-Fd8rPvyBv6JjcO/nZxZiFQan6Fww/jAF4TYj0Th/Yfo=",
+        "lastModified": 1783021952,
+        "narHash": "sha256-8PghAtSGGZ0umfVI8Qbd7ZbFrfZPiH1UwtVbgLeikDA=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "0b4f03c64b236e4ba4252414274e92796c300124",
+        "rev": "f136374c679c54171a3ace589d15e9e79a8bd086",
         "type": "github"
       },
       "original": {
@@ -108,11 +108,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1783776592,
-        "narHash": "sha256-UgCQzxeWI75XM8G+hPrPh+MKzEPjG3SpAj7dtqSbksA=",
+        "lastModified": 1782723713,
+        "narHash": "sha256-oPXCU/SSUokcGaJREHibG1CBX3+s/W7orDWQOZDsEeQ=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "e7a3ca8092b61ff85b6a45bf863ea2b2d6a661b3",
+        "rev": "b5aa0fbd538984f6e3d201be0005b4463d8b09f8",
         "type": "github"
       },
       "original": {
@@ -141,11 +141,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1783174389,
-        "narHash": "sha256-aCWC8ngycU7OdJrU2+Je3qf+1a2ykuBvpPhZT/9tXMc=",
+        "lastModified": 1782165805,
+        "narHash": "sha256-478kKQBvK6SYTOdN2h9jhKJv94nbXRbFMfuL1WshErg=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "f1406619a3884cd5c47992a70b8b35c9c0fcb4c9",
+        "rev": "56b24064fdcaedca53553b1a6d607fd23b613a24",
         "type": "github"
       },
       "original": {

overlays/default.nix

@@ -17,7 +17,9 @@
 
   python-env = final: _prev: {
     my_python = final.python314.withPackages (
-      ps: with ps; [
+      ps:
+      with ps;
+      [
         alembic
         apprise
         apscheduler

pyproject.toml

@@ -119,7 +119,7 @@ exclude_lines = [
 ]
 
 [tool.pytest.ini_options]
-addopts = "-n auto -ra --ignore=tests/ebook_search"
+addopts = "-n auto -ra"
 asyncio_mode = "auto"
 testpaths = ["tests"]
 # --cov=system_tools --cov-report=term-missing --cov-report=xml --cov-report=html --cov-branch

python/alembic/richie/versions/2026_07_09-remove_spacy_ner_751260fc3228.py

@@ -1,55 +0,0 @@
-"""remove spaCy-ner.
-
-Revision ID: 751260fc3228
-Revises: dddee09eddcc
-Create Date: 2026-07-09 23:03:39.554083
-
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import sqlalchemy as sa
-from alembic import op
-
-from python.orm import RichieBase
-
-if TYPE_CHECKING:
-    from collections.abc import Sequence
-
-# revision identifiers, used by Alembic.
-revision: str = "751260fc3228"
-down_revision: str | None = "dddee09eddcc"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-schema = RichieBase.schema_name
-
-
-def upgrade() -> None:
-    """Upgrade."""
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_column("candidate_phrases", "source_spacy_noun_chunk", schema=schema)
-    op.drop_column("candidate_phrases", "source_spacy_ner", schema=schema)
-    op.drop_column("candidate_phrases", "spacy_label", schema=schema)
-    # ### end Alembic commands ###
-
-
-def downgrade() -> None:
-    """Downgrade."""
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.add_column(
-        "candidate_phrases", sa.Column("spacy_label", sa.VARCHAR(), autoincrement=False, nullable=True), schema=schema
-    )
-    op.add_column(
-        "candidate_phrases",
-        sa.Column("source_spacy_ner", sa.BOOLEAN(), autoincrement=False, nullable=False),
-        schema=schema,
-    )
-    op.add_column(
-        "candidate_phrases",
-        sa.Column("source_spacy_noun_chunk", sa.BOOLEAN(), autoincrement=False, nullable=False),
-        schema=schema,
-    )
-    # ### end Alembic commands ###

python/api/__init__.py

@@ -0,0 +1 @@
+"""FastAPI applications."""

python/api/main.py

@@ -0,0 +1,56 @@
+"""FastAPI interface for Contact database."""
+
+from __future__ import annotations
+
+import logging
+from contextlib import asynccontextmanager
+from typing import TYPE_CHECKING, Annotated
+
+import typer
+import uvicorn
+from fastapi import FastAPI
+
+from python.api.routers import contact_router, views_router
+from python.common import configure_logger
+from python.fastapi_tools import ZstdMiddleware
+from python.orm.common import get_postgres_engine
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
+logger = logging.getLogger(__name__)
+
+
+def create_app() -> FastAPI:
+    """Create and configure the FastAPI application."""
+
+    @asynccontextmanager
+    async def lifespan(app: FastAPI) -> AsyncIterator[None]:
+        """Manage application lifespan."""
+        app.state.engine = get_postgres_engine()
+        yield
+        app.state.engine.dispose()
+
+    app = FastAPI(title="Contact Database API", lifespan=lifespan)
+    app.add_middleware(ZstdMiddleware)
+
+    app.include_router(contact_router)
+    app.include_router(views_router)
+
+    return app
+
+
+def serve(
+    host: Annotated[str, typer.Option("--host", "-h", help="Host to bind to")],
+    port: Annotated[int, typer.Option("--port", "-p", help="Port to bind to")] = 8000,
+    log_level: Annotated[str, typer.Option("--log-level", "-l", help="Log level")] = "INFO",
+) -> None:
+    """Start the Contact API server."""
+    configure_logger(log_level)
+
+    app = create_app()
+    uvicorn.run(app, host=host, port=port)
+
+
+if __name__ == "__main__":
+    typer.run(serve)

python/api/routers/__init__.py

@@ -0,0 +1,6 @@
+"""API routers."""
+
+from python.api.routers.contact import router as contact_router
+from python.api.routers.views import router as views_router
+
+__all__ = ["contact_router", "views_router"]

python/api/routers/contact.py

@@ -0,0 +1,481 @@
+"""Contact API router."""
+
+from pathlib import Path
+
+from fastapi import APIRouter, HTTPException, Request
+from fastapi.responses import HTMLResponse
+from fastapi.templating import Jinja2Templates
+from pydantic import BaseModel
+from sqlalchemy import select
+from sqlalchemy.orm import selectinload
+
+from python.fastapi_tools.db import DbSession  # noqa: TC001 this is a FastAPI needed at runtime
+from python.orm.richie.contact import Contact, ContactRelationship, Need, RelationshipType
+
+TEMPLATES_DIR = Path(__file__).parent.parent / "templates"
+templates = Jinja2Templates(directory=TEMPLATES_DIR)
+
+
+def _is_htmx(request: Request) -> bool:
+    """Check if the request is from HTMX."""
+    return request.headers.get("HX-Request") == "true"
+
+
+class NeedBase(BaseModel):
+    """Base schema for Need."""
+
+    name: str
+    description: str | None = None
+
+
+class NeedCreate(NeedBase):
+    """Schema for creating a Need."""
+
+
+class NeedResponse(NeedBase):
+    """Schema for Need response."""
+
+    id: int
+
+    model_config = {"from_attributes": True}
+
+
+class ContactRelationshipCreate(BaseModel):
+    """Schema for creating a contact relationship."""
+
+    related_contact_id: int
+    relationship_type: RelationshipType
+    closeness_weight: int | None = None
+
+
+class ContactRelationshipUpdate(BaseModel):
+    """Schema for updating a contact relationship."""
+
+    relationship_type: RelationshipType | None = None
+    closeness_weight: int | None = None
+
+
+class ContactRelationshipResponse(BaseModel):
+    """Schema for contact relationship response."""
+
+    contact_id: int
+    related_contact_id: int
+    relationship_type: str
+    closeness_weight: int
+
+    model_config = {"from_attributes": True}
+
+
+class RelationshipTypeInfo(BaseModel):
+    """Information about a relationship type."""
+
+    value: str
+    display_name: str
+    default_weight: int
+
+
+class GraphNode(BaseModel):
+    """Node in the relationship graph."""
+
+    id: int
+    name: str
+    current_job: str | None = None
+
+
+class GraphEdge(BaseModel):
+    """Edge in the relationship graph."""
+
+    source: int
+    target: int
+    relationship_type: str
+    closeness_weight: int
+
+
+class GraphData(BaseModel):
+    """Complete graph data for visualization."""
+
+    nodes: list[GraphNode]
+    edges: list[GraphEdge]
+
+
+class ContactBase(BaseModel):
+    """Base schema for Contact."""
+
+    name: str
+    age: int | None = None
+    bio: str | None = None
+    current_job: str | None = None
+    gender: str | None = None
+    goals: str | None = None
+    legal_name: str | None = None
+    profile_pic: str | None = None
+    safe_conversation_starters: str | None = None
+    self_sufficiency_score: int | None = None
+    social_structure_style: str | None = None
+    ssn: str | None = None
+    suffix: str | None = None
+    timezone: str | None = None
+    topics_to_avoid: str | None = None
+
+
+class ContactCreate(ContactBase):
+    """Schema for creating a Contact."""
+
+    need_ids: list[int] = []
+
+
+class ContactUpdate(BaseModel):
+    """Schema for updating a Contact."""
+
+    name: str | None = None
+    age: int | None = None
+    bio: str | None = None
+    current_job: str | None = None
+    gender: str | None = None
+    goals: str | None = None
+    legal_name: str | None = None
+    profile_pic: str | None = None
+    safe_conversation_starters: str | None = None
+    self_sufficiency_score: int | None = None
+    social_structure_style: str | None = None
+    ssn: str | None = None
+    suffix: str | None = None
+    timezone: str | None = None
+    topics_to_avoid: str | None = None
+    need_ids: list[int] | None = None
+
+
+class ContactResponse(ContactBase):
+    """Schema for Contact response with relationships."""
+
+    id: int
+    needs: list[NeedResponse] = []
+    related_to: list[ContactRelationshipResponse] = []
+    related_from: list[ContactRelationshipResponse] = []
+
+    model_config = {"from_attributes": True}
+
+
+class ContactListResponse(ContactBase):
+    """Schema for Contact list response."""
+
+    id: int
+
+    model_config = {"from_attributes": True}
+
+
+router = APIRouter(prefix="/api", tags=["contacts"])
+
+
+@router.post("/needs", response_model=NeedResponse)
+def create_need(need: NeedCreate, db: DbSession) -> Need:
+    """Create a new need."""
+    db_need = Need(name=need.name, description=need.description)
+    db.add(db_need)
+    db.commit()
+    db.refresh(db_need)
+    return db_need
+
+
+@router.get("/needs", response_model=list[NeedResponse])
+def list_needs(db: DbSession) -> list[Need]:
+    """List all needs."""
+    return list(db.scalars(select(Need)).all())
+
+
+@router.get("/needs/{need_id}", response_model=NeedResponse)
+def get_need(need_id: int, db: DbSession) -> Need:
+    """Get a need by ID."""
+    need = db.get(Need, need_id)
+    if not need:
+        raise HTTPException(status_code=404, detail="Need not found")
+    return need
+
+
+@router.delete("/needs/{need_id}", response_model=None)
+def delete_need(need_id: int, request: Request, db: DbSession) -> dict[str, bool] | HTMLResponse:
+    """Delete a need by ID."""
+    need = db.get(Need, need_id)
+    if not need:
+        raise HTTPException(status_code=404, detail="Need not found")
+    db.delete(need)
+    db.commit()
+    if _is_htmx(request):
+        return HTMLResponse("")
+    return {"deleted": True}
+
+
+@router.post("/contacts", response_model=ContactResponse)
+def create_contact(contact: ContactCreate, db: DbSession) -> Contact:
+    """Create a new contact."""
+    need_ids = contact.need_ids
+    contact_data = contact.model_dump(exclude={"need_ids"})
+    db_contact = Contact(**contact_data)
+
+    if need_ids:
+        needs = list(db.scalars(select(Need).where(Need.id.in_(need_ids))).all())
+        db_contact.needs = needs
+
+    db.add(db_contact)
+    db.commit()
+    db.refresh(db_contact)
+    return db_contact
+
+
+@router.get("/contacts", response_model=list[ContactListResponse])
+def list_contacts(
+    db: DbSession,
+    skip: int = 0,
+    limit: int = 100,
+) -> list[Contact]:
+    """List all contacts with pagination."""
+    return list(db.scalars(select(Contact).offset(skip).limit(limit)).all())
+
+
+@router.get("/contacts/{contact_id}", response_model=ContactResponse)
+def get_contact(contact_id: int, db: DbSession) -> Contact:
+    """Get a contact by ID with all relationships."""
+    contact = db.scalar(
+        select(Contact)
+        .where(Contact.id == contact_id)
+        .options(
+            selectinload(Contact.needs),
+            selectinload(Contact.related_to),
+            selectinload(Contact.related_from),
+        )
+    )
+    if not contact:
+        raise HTTPException(status_code=404, detail="Contact not found")
+    return contact
+
+
+@router.patch("/contacts/{contact_id}", response_model=ContactResponse)
+def update_contact(
+    contact_id: int,
+    contact: ContactUpdate,
+    db: DbSession,
+) -> Contact:
+    """Update a contact by ID."""
+    db_contact = db.get(Contact, contact_id)
+    if not db_contact:
+        raise HTTPException(status_code=404, detail="Contact not found")
+
+    update_data = contact.model_dump(exclude_unset=True)
+    need_ids = update_data.pop("need_ids", None)
+
+    for key, value in update_data.items():
+        setattr(db_contact, key, value)
+
+    if need_ids is not None:
+        needs = list(db.scalars(select(Need).where(Need.id.in_(need_ids))).all())
+        db_contact.needs = needs
+
+    db.commit()
+    db.refresh(db_contact)
+    return db_contact
+
+
+@router.delete("/contacts/{contact_id}", response_model=None)
+def delete_contact(contact_id: int, request: Request, db: DbSession) -> dict[str, bool] | HTMLResponse:
+    """Delete a contact by ID."""
+    contact = db.get(Contact, contact_id)
+    if not contact:
+        raise HTTPException(status_code=404, detail="Contact not found")
+    db.delete(contact)
+    db.commit()
+    if _is_htmx(request):
+        return HTMLResponse("")
+    return {"deleted": True}
+
+
+@router.post("/contacts/{contact_id}/needs/{need_id}")
+def add_need_to_contact(
+    contact_id: int,
+    need_id: int,
+    db: DbSession,
+) -> dict[str, bool]:
+    """Add a need to a contact."""
+    contact = db.get(Contact, contact_id)
+    if not contact:
+        raise HTTPException(status_code=404, detail="Contact not found")
+
+    need = db.get(Need, need_id)
+    if not need:
+        raise HTTPException(status_code=404, detail="Need not found")
+
+    if need not in contact.needs:
+        contact.needs.append(need)
+        db.commit()
+
+    return {"added": True}
+
+
+@router.delete("/contacts/{contact_id}/needs/{need_id}", response_model=None)
+def remove_need_from_contact(
+    contact_id: int,
+    need_id: int,
+    request: Request,
+    db: DbSession,
+) -> dict[str, bool] | HTMLResponse:
+    """Remove a need from a contact."""
+    contact = db.get(Contact, contact_id)
+    if not contact:
+        raise HTTPException(status_code=404, detail="Contact not found")
+
+    need = db.get(Need, need_id)
+    if not need:
+        raise HTTPException(status_code=404, detail="Need not found")
+
+    if need in contact.needs:
+        contact.needs.remove(need)
+        db.commit()
+
+    if _is_htmx(request):
+        return HTMLResponse("")
+    return {"removed": True}
+
+
+@router.post(
+    "/contacts/{contact_id}/relationships",
+    response_model=ContactRelationshipResponse,
+)
+def add_contact_relationship(
+    contact_id: int,
+    relationship: ContactRelationshipCreate,
+    db: DbSession,
+) -> ContactRelationship:
+    """Add a relationship between two contacts."""
+    contact = db.get(Contact, contact_id)
+    if not contact:
+        raise HTTPException(status_code=404, detail="Contact not found")
+
+    related_contact = db.get(Contact, relationship.related_contact_id)
+    if not related_contact:
+        raise HTTPException(status_code=404, detail="Related contact not found")
+
+    if contact_id == relationship.related_contact_id:
+        raise HTTPException(status_code=400, detail="Cannot relate contact to itself")
+
+    # Use provided weight or default from relationship type
+    weight = relationship.closeness_weight
+    if weight is None:
+        weight = relationship.relationship_type.default_weight
+
+    db_relationship = ContactRelationship(
+        contact_id=contact_id,
+        related_contact_id=relationship.related_contact_id,
+        relationship_type=relationship.relationship_type.value,
+        closeness_weight=weight,
+    )
+    db.add(db_relationship)
+    db.commit()
+    db.refresh(db_relationship)
+    return db_relationship
+
+
+@router.get(
+    "/contacts/{contact_id}/relationships",
+    response_model=list[ContactRelationshipResponse],
+)
+def get_contact_relationships(
+    contact_id: int,
+    db: DbSession,
+) -> list[ContactRelationship]:
+    """Get all relationships for a contact."""
+    contact = db.get(Contact, contact_id)
+    if not contact:
+        raise HTTPException(status_code=404, detail="Contact not found")
+
+    outgoing = list(db.scalars(select(ContactRelationship).where(ContactRelationship.contact_id == contact_id)).all())
+    incoming = list(
+        db.scalars(select(ContactRelationship).where(ContactRelationship.related_contact_id == contact_id)).all()
+    )
+    return outgoing + incoming
+
+
+@router.patch(
+    "/contacts/{contact_id}/relationships/{related_contact_id}",
+    response_model=ContactRelationshipResponse,
+)
+def update_contact_relationship(
+    contact_id: int,
+    related_contact_id: int,
+    update: ContactRelationshipUpdate,
+    db: DbSession,
+) -> ContactRelationship:
+    """Update a relationship between two contacts."""
+    relationship = db.scalar(
+        select(ContactRelationship).where(
+            ContactRelationship.contact_id == contact_id,
+            ContactRelationship.related_contact_id == related_contact_id,
+        )
+    )
+    if not relationship:
+        raise HTTPException(status_code=404, detail="Relationship not found")
+
+    if update.relationship_type is not None:
+        relationship.relationship_type = update.relationship_type.value
+    if update.closeness_weight is not None:
+        relationship.closeness_weight = update.closeness_weight
+
+    db.commit()
+    db.refresh(relationship)
+    return relationship
+
+
+@router.delete("/contacts/{contact_id}/relationships/{related_contact_id}", response_model=None)
+def remove_contact_relationship(
+    contact_id: int,
+    related_contact_id: int,
+    request: Request,
+    db: DbSession,
+) -> dict[str, bool] | HTMLResponse:
+    """Remove a relationship between two contacts."""
+    relationship = db.scalar(
+        select(ContactRelationship).where(
+            ContactRelationship.contact_id == contact_id,
+            ContactRelationship.related_contact_id == related_contact_id,
+        )
+    )
+    if not relationship:
+        raise HTTPException(status_code=404, detail="Relationship not found")
+
+    db.delete(relationship)
+    db.commit()
+    if _is_htmx(request):
+        return HTMLResponse("")
+    return {"deleted": True}
+
+
+@router.get("/relationship-types")
+def list_relationship_types() -> list[RelationshipTypeInfo]:
+    """List all available relationship types with their default weights."""
+    return [
+        RelationshipTypeInfo(
+            value=rt.value,
+            display_name=rt.display_name,
+            default_weight=rt.default_weight,
+        )
+        for rt in RelationshipType
+    ]
+
+
+@router.get("/graph")
+def get_relationship_graph(db: DbSession) -> GraphData:
+    """Get all contacts and relationships as graph data for visualization."""
+    contacts = list(db.scalars(select(Contact)).all())
+    relationships = list(db.scalars(select(ContactRelationship)).all())
+
+    nodes = [GraphNode(id=c.id, name=c.name, current_job=c.current_job) for c in contacts]
+
+    edges = [
+        GraphEdge(
+            source=rel.contact_id,
+            target=rel.related_contact_id,
+            relationship_type=rel.relationship_type,
+            closeness_weight=rel.closeness_weight,
+        )
+        for rel in relationships
+    ]
+
+    return GraphData(nodes=nodes, edges=edges)

python/api/routers/views.py

@@ -0,0 +1,345 @@
+"""HTMX server-rendered view router."""
+
+from pathlib import Path
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Form, HTTPException, Request
+from fastapi.responses import HTMLResponse, RedirectResponse
+from fastapi.templating import Jinja2Templates
+from sqlalchemy import select
+from sqlalchemy.orm import Session, selectinload
+
+from python.fastapi_tools.db import DbSession  # noqa: TC001 this is a FastAPI needed at runtime
+from python.orm.richie.contact import Contact, ContactRelationship, Need, RelationshipType
+
+TEMPLATES_DIR = Path(__file__).parent.parent / "templates"
+templates = Jinja2Templates(directory=TEMPLATES_DIR)
+
+router = APIRouter(tags=["views"])
+
+FAMILIAL_TYPES = {
+    "parent",
+    "child",
+    "sibling",
+    "grandparent",
+    "grandchild",
+    "aunt_uncle",
+    "niece_nephew",
+    "cousin",
+    "in_law",
+}
+FRIEND_TYPES = {"best_friend", "close_friend", "friend", "acquaintance", "neighbor"}
+PARTNER_TYPES = {"spouse", "partner"}
+PROFESSIONAL_TYPES = {"mentor", "mentee", "business_partner", "colleague", "manager", "direct_report", "client"}
+
+CONTACT_STRING_FIELDS = (
+    "name",
+    "legal_name",
+    "suffix",
+    "gender",
+    "current_job",
+    "timezone",
+    "profile_pic",
+    "bio",
+    "goals",
+    "social_structure_style",
+    "safe_conversation_starters",
+    "topics_to_avoid",
+    "ssn",
+)
+
+CONTACT_INT_FIELDS = ("age", "self_sufficiency_score")
+
+
+def _group_relationships(relationships: list[ContactRelationship]) -> dict[str, list[ContactRelationship]]:
+    """Group relationships by category."""
+    groups: dict[str, list[ContactRelationship]] = {
+        "familial": [],
+        "partners": [],
+        "friends": [],
+        "professional": [],
+        "other": [],
+    }
+    for rel in relationships:
+        if rel.relationship_type in FAMILIAL_TYPES:
+            groups["familial"].append(rel)
+        elif rel.relationship_type in PARTNER_TYPES:
+            groups["partners"].append(rel)
+        elif rel.relationship_type in FRIEND_TYPES:
+            groups["friends"].append(rel)
+        elif rel.relationship_type in PROFESSIONAL_TYPES:
+            groups["professional"].append(rel)
+        else:
+            groups["other"].append(rel)
+    return groups
+
+
+def _build_contact_name_map(database: Session, contact: Contact) -> dict[int, str]:
+    """Build a mapping of contact IDs to names for relationship display."""
+    related_ids = {rel.related_contact_id for rel in contact.related_to}
+    related_ids |= {rel.contact_id for rel in contact.related_from}
+    related_ids.discard(contact.id)
+
+    if not related_ids:
+        return {}
+
+    related_contacts = list(database.scalars(select(Contact).where(Contact.id.in_(related_ids))).all())
+    return {related.id: related.name for related in related_contacts}
+
+
+def _get_relationship_type_display() -> dict[str, str]:
+    """Build a mapping of relationship type values to display names."""
+    return {rel_type.value: rel_type.display_name for rel_type in RelationshipType}
+
+
+async def _parse_contact_form(request: Request) -> dict[str, Any]:
+    """Parse contact form data from a multipart/form request."""
+    form_data = await request.form()
+    result: dict[str, Any] = {}
+
+    for field in CONTACT_STRING_FIELDS:
+        value = form_data.get(field, "")
+        result[field] = str(value) if value else None
+
+    for field in CONTACT_INT_FIELDS:
+        value = form_data.get(field, "")
+        result[field] = int(value) if value else None
+
+    result["need_ids"] = [int(value) for value in form_data.getlist("need_ids")]
+    return result
+
+
+def _save_contact_from_form(database: Session, contact: Contact, form_result: dict[str, Any]) -> None:
+    """Apply parsed form data to a Contact and save associated needs."""
+    need_ids = form_result.pop("need_ids")
+
+    for key, value in form_result.items():
+        setattr(contact, key, value)
+
+    if need_ids:
+        contact.needs = list(database.scalars(select(Need).where(Need.id.in_(need_ids))).all())
+    else:
+        contact.needs = []
+
+
+@router.get("/", response_class=HTMLResponse)
+@router.get("/contacts", response_class=HTMLResponse)
+def contact_list_page(request: Request, database: DbSession) -> HTMLResponse:
+    """Render the contacts list page."""
+    contacts = list(database.scalars(select(Contact)).all())
+    return templates.TemplateResponse(request, "contact_list.html", {"contacts": contacts})
+
+
+@router.get("/contacts/new", response_class=HTMLResponse)
+def new_contact_page(request: Request, database: DbSession) -> HTMLResponse:
+    """Render the new contact form page."""
+    all_needs = list(database.scalars(select(Need)).all())
+    return templates.TemplateResponse(request, "contact_form.html", {"contact": None, "all_needs": all_needs})
+
+
+@router.post("/htmx/contacts/new")
+async def create_contact_form(request: Request, database: DbSession) -> RedirectResponse:
+    """Handle the create contact form submission."""
+    form_result = await _parse_contact_form(request)
+    contact = Contact()
+    _save_contact_from_form(database, contact, form_result)
+
+    database.add(contact)
+    database.commit()
+    database.refresh(contact)
+    return RedirectResponse(url=f"/contacts/{contact.id}", status_code=303)
+
+
+@router.get("/contacts/{contact_id}", response_class=HTMLResponse)
+def contact_detail_page(contact_id: int, request: Request, database: DbSession) -> HTMLResponse:
+    """Render the contact detail page."""
+    contact = database.scalar(
+        select(Contact)
+        .where(Contact.id == contact_id)
+        .options(
+            selectinload(Contact.needs),
+            selectinload(Contact.related_to),
+            selectinload(Contact.related_from),
+        )
+    )
+    if not contact:
+        raise HTTPException(status_code=404, detail="Contact not found")
+
+    contact_names = _build_contact_name_map(database, contact)
+    grouped_relationships = _group_relationships(contact.related_to)
+    all_contacts = list(database.scalars(select(Contact)).all())
+    all_needs = list(database.scalars(select(Need)).all())
+    available_needs = [need for need in all_needs if need not in contact.needs]
+
+    return templates.TemplateResponse(
+        request,
+        "contact_detail.html",
+        {
+            "contact": contact,
+            "contact_names": contact_names,
+            "grouped_relationships": grouped_relationships,
+            "all_contacts": all_contacts,
+            "available_needs": available_needs,
+            "relationship_types": list(RelationshipType),
+        },
+    )
+
+
+@router.get("/contacts/{contact_id}/edit", response_class=HTMLResponse)
+def edit_contact_page(contact_id: int, request: Request, database: DbSession) -> HTMLResponse:
+    """Render the edit contact form page."""
+    contact = database.scalar(select(Contact).where(Contact.id == contact_id).options(selectinload(Contact.needs)))
+    if not contact:
+        raise HTTPException(status_code=404, detail="Contact not found")
+
+    all_needs = list(database.scalars(select(Need)).all())
+    return templates.TemplateResponse(request, "contact_form.html", {"contact": contact, "all_needs": all_needs})
+
+
+@router.post("/htmx/contacts/{contact_id}/edit")
+async def update_contact_form(contact_id: int, request: Request, database: DbSession) -> RedirectResponse:
+    """Handle the edit contact form submission."""
+    contact = database.get(Contact, contact_id)
+    if not contact:
+        raise HTTPException(status_code=404, detail="Contact not found")
+
+    form_result = await _parse_contact_form(request)
+    _save_contact_from_form(database, contact, form_result)
+
+    database.commit()
+    return RedirectResponse(url=f"/contacts/{contact_id}", status_code=303)
+
+
+@router.post("/htmx/contacts/{contact_id}/add-need", response_class=HTMLResponse)
+def add_need_to_contact_htmx(
+    contact_id: int,
+    request: Request,
+    database: DbSession,
+    need_id: Annotated[int, Form()],
+) -> HTMLResponse:
+    """Add a need to a contact and return updated manage-needs partial."""
+    contact = database.scalar(select(Contact).where(Contact.id == contact_id).options(selectinload(Contact.needs)))
+    if not contact:
+        raise HTTPException(status_code=404, detail="Contact not found")
+
+    need = database.get(Need, need_id)
+    if not need:
+        raise HTTPException(status_code=404, detail="Need not found")
+
+    if need not in contact.needs:
+        contact.needs.append(need)
+        database.commit()
+        database.refresh(contact)
+
+    return templates.TemplateResponse(request, "partials/manage_needs.html", {"contact": contact})
+
+
+@router.post("/htmx/contacts/{contact_id}/add-relationship", response_class=HTMLResponse)
+def add_relationship_htmx(
+    contact_id: int,
+    request: Request,
+    database: DbSession,
+    related_contact_id: Annotated[int, Form()],
+    relationship_type: Annotated[str, Form()],
+) -> HTMLResponse:
+    """Add a relationship and return updated manage-relationships partial."""
+    contact = database.scalar(select(Contact).where(Contact.id == contact_id).options(selectinload(Contact.related_to)))
+    if not contact:
+        raise HTTPException(status_code=404, detail="Contact not found")
+
+    related_contact = database.get(Contact, related_contact_id)
+    if not related_contact:
+        raise HTTPException(status_code=404, detail="Related contact not found")
+
+    rel_type = RelationshipType(relationship_type)
+    weight = rel_type.default_weight
+
+    relationship = ContactRelationship(
+        contact_id=contact_id,
+        related_contact_id=related_contact_id,
+        relationship_type=relationship_type,
+        closeness_weight=weight,
+    )
+    database.add(relationship)
+    database.commit()
+    database.refresh(contact)
+
+    contact_names = _build_contact_name_map(database, contact)
+    return templates.TemplateResponse(
+        request,
+        "partials/manage_relationships.html",
+        {"contact": contact, "contact_names": contact_names},
+    )
+
+
+@router.post("/htmx/contacts/{contact_id}/relationships/{related_contact_id}/weight")
+def update_relationship_weight_htmx(
+    contact_id: int,
+    related_contact_id: int,
+    database: DbSession,
+    closeness_weight: Annotated[int, Form()],
+) -> HTMLResponse:
+    """Update a relationship's closeness weight from HTMX range input."""
+    relationship = database.scalar(
+        select(ContactRelationship).where(
+            ContactRelationship.contact_id == contact_id,
+            ContactRelationship.related_contact_id == related_contact_id,
+        )
+    )
+    if not relationship:
+        raise HTTPException(status_code=404, detail="Relationship not found")
+
+    relationship.closeness_weight = closeness_weight
+    database.commit()
+    return HTMLResponse("")
+
+
+@router.post("/htmx/needs", response_class=HTMLResponse)
+def create_need_htmx(
+    request: Request,
+    database: DbSession,
+    name: Annotated[str, Form()],
+    description: Annotated[str, Form()] = "",
+) -> HTMLResponse:
+    """Create a need via form data and return updated needs list."""
+    need = Need(name=name, description=description or None)
+    database.add(need)
+    database.commit()
+    needs = list(database.scalars(select(Need)).all())
+    return templates.TemplateResponse(request, "partials/need_items.html", {"needs": needs})
+
+
+@router.get("/needs", response_class=HTMLResponse)
+def needs_page(request: Request, database: DbSession) -> HTMLResponse:
+    """Render the needs list page."""
+    needs = list(database.scalars(select(Need)).all())
+    return templates.TemplateResponse(request, "need_list.html", {"needs": needs})
+
+
+@router.get("/graph", response_class=HTMLResponse)
+def graph_page(request: Request, database: DbSession) -> HTMLResponse:
+    """Render the relationship graph page."""
+    contacts = list(database.scalars(select(Contact)).all())
+    relationships = list(database.scalars(select(ContactRelationship)).all())
+
+    graph_data = {
+        "nodes": [{"id": contact.id, "name": contact.name, "current_job": contact.current_job} for contact in contacts],
+        "edges": [
+            {
+                "source": rel.contact_id,
+                "target": rel.related_contact_id,
+                "relationship_type": rel.relationship_type,
+                "closeness_weight": rel.closeness_weight,
+            }
+            for rel in relationships
+        ],
+    }
+
+    return templates.TemplateResponse(
+        request,
+        "graph.html",
+        {
+            "graph_data": graph_data,
+            "relationship_type_display": _get_relationship_type_display(),
+        },
+    )

python/api/templates/base.html

@@ -0,0 +1,198 @@
+<!DOCTYPE html>
+<html lang="en" data-theme="light">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>{% block title %}Contact Database{% endblock %}</title>
+    <script src="https://unpkg.com/htmx.org@2.0.4"></script>
+    <style>
+        :root {
+            --color-bg: #f5f5f5;
+            --color-bg-card: #ffffff;
+            --color-bg-hover: #f0f0f0;
+            --color-bg-muted: #f9f9f9;
+            --color-bg-error: #ffe0e0;
+            --color-text: #333333;
+            --color-text-muted: #666666;
+            --color-text-error: #cc0000;
+            --color-border: #dddddd;
+            --color-border-light: #eeeeee;
+            --color-border-lighter: #f0f0f0;
+            --color-primary: #0066cc;
+            --color-primary-hover: #0055aa;
+            --color-danger: #cc3333;
+            --color-danger-hover: #aa2222;
+            --color-tag-bg: #e0e0e0;
+            --shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
+            font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
+            line-height: 1.5;
+            color: var(--color-text);
+            background-color: var(--color-bg);
+        }
+        [data-theme="dark"] {
+            --color-bg: #1a1a1a;
+            --color-bg-card: #2d2d2d;
+            --color-bg-hover: #3d3d3d;
+            --color-bg-muted: #252525;
+            --color-bg-error: #4a2020;
+            --color-text: #e0e0e0;
+            --color-text-muted: #a0a0a0;
+            --color-text-error: #ff6b6b;
+            --color-border: #404040;
+            --color-border-light: #353535;
+            --color-border-lighter: #303030;
+            --color-primary: #4da6ff;
+            --color-primary-hover: #7dbfff;
+            --color-danger: #ff6b6b;
+            --color-danger-hover: #ff8a8a;
+            --color-tag-bg: #404040;
+            --shadow: 0 1px 3px rgba(0, 0, 0, 0.3);
+        }
+        * { box-sizing: border-box; }
+        body { margin: 0; background: var(--color-bg); color: var(--color-text); }
+        .app { max-width: 1000px; margin: 0 auto; padding: 20px; }
+        nav { display: flex; align-items: center; gap: 20px; padding: 15px 0; border-bottom: 1px solid var(--color-border); margin-bottom: 20px; }
+        nav a { color: var(--color-primary); text-decoration: none; font-weight: 500; }
+        nav a:hover { text-decoration: underline; }
+        .theme-toggle { margin-left: auto; }
+        main { background: var(--color-bg-card); padding: 20px; border-radius: 8px; box-shadow: var(--shadow); }
+        .header { display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px; }
+        .header h1 { margin: 0; }
+        a { color: var(--color-primary); }
+        a:hover { text-decoration: underline; }
+
+        .btn { display: inline-block; padding: 8px 16px; border: 1px solid var(--color-border); border-radius: 4px; background: var(--color-bg-card); color: var(--color-text); text-decoration: none; cursor: pointer; font-size: 14px; margin-left: 8px; }
+        .btn:hover { background: var(--color-bg-hover); }
+        .btn-primary { background: var(--color-primary); border-color: var(--color-primary); color: white; }
+        .btn-primary:hover { background: var(--color-primary-hover); }
+        .btn-danger { background: var(--color-danger); border-color: var(--color-danger); color: white; }
+        .btn-danger:hover { background: var(--color-danger-hover); }
+        .btn-small { padding: 4px 8px; font-size: 12px; }
+        .btn:disabled { opacity: 0.6; cursor: not-allowed; }
+
+        table { width: 100%; border-collapse: collapse; }
+        th, td { padding: 12px; text-align: left; border-bottom: 1px solid var(--color-border-light); }
+        th { font-weight: 600; background: var(--color-bg-muted); }
+        tr:hover { background: var(--color-bg-muted); }
+
+        .error { background: var(--color-bg-error); color: var(--color-text-error); padding: 10px; border-radius: 4px; margin-bottom: 20px; }
+        .tag { display: inline-block; background: var(--color-tag-bg); padding: 2px 8px; border-radius: 12px; font-size: 12px; color: var(--color-text-muted); }
+
+        .add-form { display: flex; gap: 10px; margin-top: 15px; flex-wrap: wrap; }
+        .add-form select, .add-form input { padding: 8px; border: 1px solid var(--color-border); border-radius: 4px; min-width: 200px; background: var(--color-bg-card); color: var(--color-text); }
+
+        .form-group { margin-bottom: 20px; }
+        .form-group label { display: block; font-weight: 500; margin-bottom: 5px; }
+        .form-group input, .form-group textarea, .form-group select { width: 100%; padding: 10px; border: 1px solid var(--color-border); border-radius: 4px; font-size: 14px; background: var(--color-bg-card); color: var(--color-text); }
+        .form-group textarea { resize: vertical; }
+        .form-row { display: grid; grid-template-columns: 1fr 1fr; gap: 20px; }
+        .checkbox-group { display: flex; flex-wrap: wrap; gap: 15px; }
+        .checkbox-label { display: flex; align-items: center; gap: 5px; cursor: pointer; }
+        .form-actions { display: flex; gap: 10px; margin-top: 30px; padding-top: 20px; border-top: 1px solid var(--color-border-light); }
+
+        .need-form { background: var(--color-bg-muted); padding: 20px; border-radius: 4px; margin-bottom: 20px; }
+        .need-items { list-style: none; padding: 0; }
+        .need-items li { display: flex; justify-content: space-between; align-items: flex-start; padding: 15px; border: 1px solid var(--color-border-light); border-radius: 4px; margin-bottom: 10px; }
+        .need-info p { margin: 5px 0 0; color: var(--color-text-muted); font-size: 14px; }
+
+        .graph-container { width: 100%; }
+        .graph-hint { color: var(--color-text-muted); font-size: 14px; margin-bottom: 15px; }
+        .selected-info { margin-top: 15px; padding: 15px; background: var(--color-bg-muted); border-radius: 8px; }
+        .selected-info h3 { margin: 0 0 10px; }
+        .selected-info p { margin: 5px 0; color: var(--color-text-muted); }
+        .legend { margin-top: 20px; padding: 15px; background: var(--color-bg-muted); border-radius: 8px; }
+        .legend h4 { margin: 0 0 10px; font-size: 14px; }
+        .legend-items { display: flex; flex-wrap: wrap; gap: 15px; }
+        .legend-item { display: flex; align-items: center; gap: 8px; font-size: 12px; color: var(--color-text-muted); }
+        .legend-line { width: 30px; border-radius: 2px; }
+
+        .id-card { width: 100%; }
+        .id-card-inner { background: linear-gradient(135deg, #0a0a0f 0%, #1a1a2e 50%, #0a0a0f 100%); background-image: radial-gradient(white 1px, transparent 1px), linear-gradient(135deg, #0a0a0f 0%, #1a1a2e 50%, #0a0a0f 100%); background-size: 50px 50px, 100% 100%; color: #fff; border-radius: 12px; padding: 25px; min-height: 500px; position: relative; overflow: hidden; }
+        .id-card-header { display: flex; justify-content: space-between; align-items: flex-start; margin-bottom: 15px; }
+        .id-card-header-left { flex: 1; }
+        .id-card-header-right { display: flex; flex-direction: column; align-items: flex-end; gap: 10px; }
+        .id-card-title { font-size: 2.5rem; font-weight: 700; margin: 0; color: #fff; text-shadow: 2px 2px 4px rgba(0,0,0,0.5); }
+        .id-profile-pic { width: 80px; height: 80px; border-radius: 8px; object-fit: cover; border: 2px solid rgba(255,255,255,0.3); }
+        .id-profile-placeholder { width: 80px; height: 80px; border-radius: 8px; background: linear-gradient(135deg, #4ecdc4 0%, #44a8a0 100%); display: flex; align-items: center; justify-content: center; border: 2px solid rgba(255,255,255,0.3); }
+        .id-profile-placeholder span { font-size: 2rem; font-weight: 700; color: #fff; text-shadow: 1px 1px 2px rgba(0,0,0,0.3); }
+        .id-card-actions { display: flex; gap: 8px; }
+        .id-card-actions .btn { background: rgba(255,255,255,0.1); border-color: rgba(255,255,255,0.3); color: #fff; }
+        .id-card-actions .btn:hover { background: rgba(255,255,255,0.2); }
+        .id-card-body { display: grid; grid-template-columns: 1fr 1.5fr; gap: 30px; }
+        .id-card-left { display: flex; flex-direction: column; gap: 8px; }
+        .id-field { font-size: 1rem; line-height: 1.4; }
+        .id-field-block { margin-top: 15px; font-size: 0.95rem; line-height: 1.5; }
+        .id-label { color: #4ecdc4; font-weight: 500; }
+        .id-card-right { display: flex; flex-direction: column; gap: 20px; }
+        .id-bio { font-size: 0.9rem; line-height: 1.6; color: #e0e0e0; }
+        .id-relationships { margin-top: 10px; }
+        .id-section-title { font-size: 1.5rem; margin: 0 0 15px; color: #fff; border-bottom: 1px solid rgba(255,255,255,0.2); padding-bottom: 8px; }
+        .id-rel-group { margin-bottom: 12px; font-size: 0.9rem; line-height: 1.6; }
+        .id-rel-label { color: #a0a0a0; }
+        .id-rel-group a { color: #4ecdc4; text-decoration: none; }
+        .id-rel-group a:hover { text-decoration: underline; }
+        .id-rel-type { color: #888; font-size: 0.85em; }
+        .id-card-warnings { margin-top: 30px; padding-top: 20px; border-top: 1px solid rgba(255,255,255,0.2); display: flex; flex-wrap: wrap; gap: 20px; }
+        .id-warning { display: flex; align-items: center; gap: 8px; font-size: 0.9rem; color: #ff6b6b; }
+        .warning-dot { width: 8px; height: 8px; background: #ff6b6b; border-radius: 50%; flex-shrink: 0; }
+        .warning-desc { color: #ccc; }
+
+        .id-card-manage { margin-top: 20px; background: var(--color-bg-muted); border-radius: 8px; padding: 15px; }
+        .id-card-manage summary { cursor: pointer; font-weight: 600; font-size: 1.1rem; padding: 5px 0; }
+        .id-card-manage[open] summary { margin-bottom: 15px; border-bottom: 1px solid var(--color-border-light); padding-bottom: 10px; }
+        .manage-section { margin-bottom: 25px; }
+        .manage-section h3 { margin: 0 0 15px; font-size: 1rem; }
+        .manage-relationships { display: flex; flex-direction: column; gap: 10px; margin-bottom: 15px; }
+        .manage-rel-item { display: flex; align-items: center; gap: 12px; padding: 10px; background: var(--color-bg-card); border-radius: 6px; flex-wrap: wrap; }
+        .manage-rel-item a { font-weight: 500; min-width: 120px; }
+        .weight-control { display: flex; align-items: center; gap: 8px; font-size: 12px; color: var(--color-text-muted); }
+        .weight-control input[type="range"] { width: 80px; cursor: pointer; }
+        .weight-value { min-width: 20px; text-align: center; font-weight: 600; }
+        .manage-needs-list { list-style: none; padding: 0; margin: 0 0 15px; }
+        .manage-needs-list li { display: flex; align-items: center; gap: 12px; padding: 10px; background: var(--color-bg-card); border-radius: 6px; margin-bottom: 8px; }
+        .manage-needs-list li .btn { margin-left: auto; }
+
+        .htmx-indicator { display: none; }
+        .htmx-request .htmx-indicator { display: inline; }
+        .htmx-request.htmx-indicator { display: inline; }
+
+        @media (max-width: 768px) {
+            .id-card-body { grid-template-columns: 1fr; }
+            .id-card-title { font-size: 1.8rem; }
+            .id-card-header { flex-direction: column; gap: 15px; }
+        }
+    </style>
+</head>
+<body>
+    <div class="app">
+        <nav>
+            <a href="/contacts">Contacts</a>
+            <a href="/graph">Graph</a>
+            <a href="/needs">Needs</a>
+            <button class="btn btn-small theme-toggle" onclick="toggleTheme()">
+                <span id="theme-label">Dark</span>
+            </button>
+        </nav>
+
+        <main id="main-content">
+            {% block content %}{% endblock %}
+        </main>
+    </div>
+
+    <script>
+        function toggleTheme() {
+            const html = document.documentElement;
+            const current = html.getAttribute('data-theme');
+            const next = current === 'light' ? 'dark' : 'light';
+            html.setAttribute('data-theme', next);
+            localStorage.setItem('theme', next);
+            document.getElementById('theme-label').textContent = next === 'light' ? 'Dark' : 'Light';
+        }
+        (function() {
+            const saved = localStorage.getItem('theme') || 'light';
+            document.documentElement.setAttribute('data-theme', saved);
+            document.getElementById('theme-label').textContent = saved === 'light' ? 'Dark' : 'Light';
+        })();
+    </script>
+</body>
+</html>

python/api/templates/contact_detail.html

@@ -0,0 +1,204 @@
+{% extends "base.html" %}
+{% block title %}{{ contact.name }}{% endblock %}
+{% block content %}
+<div class="id-card">
+    <div class="id-card-inner">
+        <div class="id-card-header">
+            <div class="id-card-header-left">
+                <h1 class="id-card-title">I.D.: {{ contact.name }}</h1>
+            </div>
+            <div class="id-card-header-right">
+                {% if contact.profile_pic %}
+                <img src="{{ contact.profile_pic }}" alt="{{ contact.name }}'s profile" class="id-profile-pic">
+                {% else %}
+                <div class="id-profile-placeholder">
+                    <span>{{ contact.name[0]|upper }}</span>
+                </div>
+                {% endif %}
+                <div class="id-card-actions">
+                    <a href="/contacts/{{ contact.id }}/edit" class="btn btn-small">Edit</a>
+                    <a href="/contacts" class="btn btn-small">Back</a>
+                </div>
+            </div>
+        </div>
+
+        <div class="id-card-body">
+            <div class="id-card-left">
+                {% if contact.legal_name %}
+                <div class="id-field">Legal name: {{ contact.legal_name }}</div>
+                {% endif %}
+                {% if contact.suffix %}
+                <div class="id-field">Suffix: {{ contact.suffix }}</div>
+                {% endif %}
+                {% if contact.gender %}
+                <div class="id-field">Gender: {{ contact.gender }}</div>
+                {% endif %}
+                {% if contact.age %}
+                <div class="id-field">Age: {{ contact.age }}</div>
+                {% endif %}
+                {% if contact.current_job %}
+                <div class="id-field">Job: {{ contact.current_job }}</div>
+                {% endif %}
+                {% if contact.social_structure_style %}
+                <div class="id-field">Social style: {{ contact.social_structure_style }}</div>
+                {% endif %}
+                {% if contact.self_sufficiency_score is not none %}
+                <div class="id-field">Self-Sufficiency: {{ contact.self_sufficiency_score }}</div>
+                {% endif %}
+                {% if contact.timezone %}
+                <div class="id-field">Timezone: {{ contact.timezone }}</div>
+                {% endif %}
+                {% if contact.safe_conversation_starters %}
+                <div class="id-field-block">
+                    <span class="id-label">Safe con starters:</span> {{ contact.safe_conversation_starters }}
+                </div>
+                {% endif %}
+                {% if contact.topics_to_avoid %}
+                <div class="id-field-block">
+                    <span class="id-label">Topics to avoid:</span> {{ contact.topics_to_avoid }}
+                </div>
+                {% endif %}
+                {% if contact.goals %}
+                <div class="id-field-block">
+                    <span class="id-label">Goals:</span> {{ contact.goals }}
+                </div>
+                {% endif %}
+            </div>
+
+            <div class="id-card-right">
+                {% if contact.bio %}
+                <div class="id-bio">
+                    <span class="id-label">Bio:</span> {{ contact.bio }}
+                </div>
+                {% endif %}
+
+                <div class="id-relationships">
+                    <h2 class="id-section-title">Relationships</h2>
+
+                    {% if grouped_relationships.familial %}
+                    <div class="id-rel-group">
+                        <span class="id-rel-label">Familial:</span>
+                        {% for rel in grouped_relationships.familial %}
+                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a><span class="id-rel-type">({{ rel.relationship_type|replace("_", " ")|title }})</span>{% if not loop.last %}, {% endif %}
+                        {% endfor %}
+                    </div>
+                    {% endif %}
+
+                    {% if grouped_relationships.partners %}
+                    <div class="id-rel-group">
+                        <span class="id-rel-label">Partners:</span>
+                        {% for rel in grouped_relationships.partners %}
+                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a>{% if not loop.last %}, {% endif %}
+                        {% endfor %}
+                    </div>
+                    {% endif %}
+
+                    {% if grouped_relationships.friends %}
+                    <div class="id-rel-group">
+                        <span class="id-rel-label">Friends:</span>
+                        {% for rel in grouped_relationships.friends %}
+                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a>{% if not loop.last %}, {% endif %}
+                        {% endfor %}
+                    </div>
+                    {% endif %}
+
+                    {% if grouped_relationships.professional %}
+                    <div class="id-rel-group">
+                        <span class="id-rel-label">Professional:</span>
+                        {% for rel in grouped_relationships.professional %}
+                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a><span class="id-rel-type">({{ rel.relationship_type|replace("_", " ")|title }})</span>{% if not loop.last %}, {% endif %}
+                        {% endfor %}
+                    </div>
+                    {% endif %}
+
+                    {% if grouped_relationships.other %}
+                    <div class="id-rel-group">
+                        <span class="id-rel-label">Other:</span>
+                        {% for rel in grouped_relationships.other %}
+                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a><span class="id-rel-type">({{ rel.relationship_type|replace("_", " ")|title }})</span>{% if not loop.last %}, {% endif %}
+                        {% endfor %}
+                    </div>
+                    {% endif %}
+
+                    {% if contact.related_from %}
+                    <div class="id-rel-group">
+                        <span class="id-rel-label">Known by:</span>
+                        {% for rel in contact.related_from %}
+                        <a href="/contacts/{{ rel.contact_id }}">{{ contact_names[rel.contact_id] }}</a>{% if not loop.last %}, {% endif %}
+                        {% endfor %}
+                    </div>
+                    {% endif %}
+                </div>
+            </div>
+        </div>
+
+        {% if contact.needs %}
+        <div class="id-card-warnings">
+            {% for need in contact.needs %}
+            <div class="id-warning">
+                <span class="warning-dot"></span>
+                Warning: {{ need.name }}
+                {% if need.description %}<span class="warning-desc"> - {{ need.description }}</span>{% endif %}
+            </div>
+            {% endfor %}
+        </div>
+        {% endif %}
+    </div>
+
+    <details class="id-card-manage">
+        <summary>Manage Contact</summary>
+
+        <div class="manage-section">
+            <h3>Manage Relationships</h3>
+            <div id="manage-relationships" class="manage-relationships">
+                {% include "partials/manage_relationships.html" %}
+            </div>
+
+            {% if all_contacts %}
+            <form hx-post="/htmx/contacts/{{ contact.id }}/add-relationship"
+                  hx-target="#manage-relationships"
+                  hx-swap="innerHTML"
+                  class="add-form">
+                <select name="related_contact_id" required>
+                    <option value="">Select contact...</option>
+                    {% for other in all_contacts %}
+                    {% if other.id != contact.id %}
+                    <option value="{{ other.id }}">{{ other.name }}</option>
+                    {% endif %}
+                    {% endfor %}
+                </select>
+                <select name="relationship_type" required>
+                    <option value="">Select relationship type...</option>
+                    {% for rel_type in relationship_types %}
+                    <option value="{{ rel_type.value }}">{{ rel_type.display_name }}</option>
+                    {% endfor %}
+                </select>
+                <button type="submit" class="btn btn-primary">Add Relationship</button>
+            </form>
+            {% endif %}
+        </div>
+
+        <div class="manage-section">
+            <h3>Manage Needs/Warnings</h3>
+            <div id="manage-needs">
+                {% include "partials/manage_needs.html" %}
+            </div>
+
+            {% if available_needs %}
+            <form hx-post="/htmx/contacts/{{ contact.id }}/add-need"
+                  hx-target="#manage-needs"
+                  hx-swap="innerHTML"
+                  class="add-form">
+                <select name="need_id" required>
+                    <option value="">Select a need...</option>
+                    {% for need in available_needs %}
+                    <option value="{{ need.id }}">{{ need.name }}</option>
+                    {% endfor %}
+                </select>
+                <button type="submit" class="btn btn-primary">Add Need</button>
+            </form>
+            {% endif %}
+        </div>
+    </details>
+</div>
+{% endblock %}

python/api/templates/contact_form.html

@@ -0,0 +1,115 @@
+{% extends "base.html" %}
+{% block title %}{{ "Edit " + contact.name if contact else "New Contact" }}{% endblock %}
+{% block content %}
+<div class="contact-form">
+    <h1>{{ "Edit Contact" if contact else "New Contact" }}</h1>
+
+    {% if contact %}
+    <form method="post" action="/htmx/contacts/{{ contact.id }}/edit">
+    {% else %}
+    <form method="post" action="/htmx/contacts/new">
+    {% endif %}
+
+        <div class="form-group">
+            <label for="name">Name *</label>
+            <input id="name" name="name" type="text" value="{{ contact.name if contact else '' }}" required>
+        </div>
+
+        <div class="form-row">
+            <div class="form-group">
+                <label for="legal_name">Legal Name</label>
+                <input id="legal_name" name="legal_name" type="text" value="{{ contact.legal_name or '' }}">
+            </div>
+            <div class="form-group">
+                <label for="suffix">Suffix</label>
+                <input id="suffix" name="suffix" type="text" value="{{ contact.suffix or '' }}">
+            </div>
+        </div>
+
+        <div class="form-row">
+            <div class="form-group">
+                <label for="age">Age</label>
+                <input id="age" name="age" type="number" value="{{ contact.age if contact and contact.age is not none else '' }}">
+            </div>
+            <div class="form-group">
+                <label for="gender">Gender</label>
+                <input id="gender" name="gender" type="text" value="{{ contact.gender or '' }}">
+            </div>
+        </div>
+
+        <div class="form-group">
+            <label for="current_job">Current Job</label>
+            <input id="current_job" name="current_job" type="text" value="{{ contact.current_job or '' }}">
+        </div>
+
+        <div class="form-group">
+            <label for="timezone">Timezone</label>
+            <input id="timezone" name="timezone" type="text" value="{{ contact.timezone or '' }}">
+        </div>
+
+        <div class="form-group">
+            <label for="profile_pic">Profile Picture URL</label>
+            <input id="profile_pic" name="profile_pic" type="url" placeholder="https://example.com/photo.jpg" value="{{ contact.profile_pic or '' }}">
+        </div>
+
+        <div class="form-group">
+            <label for="bio">Bio</label>
+            <textarea id="bio" name="bio" rows="3">{{ contact.bio or '' }}</textarea>
+        </div>
+
+        <div class="form-group">
+            <label for="goals">Goals</label>
+            <textarea id="goals" name="goals" rows="3">{{ contact.goals or '' }}</textarea>
+        </div>
+
+        <div class="form-group">
+            <label for="social_structure_style">Social Structure Style</label>
+            <input id="social_structure_style" name="social_structure_style" type="text" value="{{ contact.social_structure_style or '' }}">
+        </div>
+
+        <div class="form-group">
+            <label for="self_sufficiency_score">Self-Sufficiency Score (1-10)</label>
+            <input id="self_sufficiency_score" name="self_sufficiency_score" type="number" min="1" max="10" value="{{ contact.self_sufficiency_score if contact and contact.self_sufficiency_score is not none else '' }}">
+        </div>
+
+        <div class="form-group">
+            <label for="safe_conversation_starters">Safe Conversation Starters</label>
+            <textarea id="safe_conversation_starters" name="safe_conversation_starters" rows="2">{{ contact.safe_conversation_starters or '' }}</textarea>
+        </div>
+
+        <div class="form-group">
+            <label for="topics_to_avoid">Topics to Avoid</label>
+            <textarea id="topics_to_avoid" name="topics_to_avoid" rows="2">{{ contact.topics_to_avoid or '' }}</textarea>
+        </div>
+
+        <div class="form-group">
+            <label for="ssn">SSN</label>
+            <input id="ssn" name="ssn" type="text" value="{{ contact.ssn or '' }}">
+        </div>
+
+        {% if all_needs %}
+        <div class="form-group">
+            <label>Needs/Accommodations</label>
+            <div class="checkbox-group">
+                {% for need in all_needs %}
+                <label class="checkbox-label">
+                    <input type="checkbox" name="need_ids" value="{{ need.id }}"
+                           {% if contact and need in contact.needs %}checked{% endif %}>
+                    {{ need.name }}
+                </label>
+                {% endfor %}
+            </div>
+        </div>
+        {% endif %}
+
+        <div class="form-actions">
+            <button type="submit" class="btn btn-primary">Save</button>
+            {% if contact %}
+            <a href="/contacts/{{ contact.id }}" class="btn">Cancel</a>
+            {% else %}
+            <a href="/contacts" class="btn">Cancel</a>
+            {% endif %}
+        </div>
+    </form>
+</div>
+{% endblock %}

python/api/templates/contact_list.html

@@ -0,0 +1,14 @@
+{% extends "base.html" %}
+{% block title %}Contacts{% endblock %}
+{% block content %}
+<div class="contact-list">
+    <div class="header">
+        <h1>Contacts</h1>
+        <a href="/contacts/new" class="btn btn-primary">Add Contact</a>
+    </div>
+
+    <div id="contact-table">
+        {% include "partials/contact_table.html" %}
+    </div>
+</div>
+{% endblock %}

python/api/templates/graph.html

@@ -0,0 +1,198 @@
+{% extends "base.html" %}
+{% block title %}Relationship Graph{% endblock %}
+{% block content %}
+<div class="graph-container">
+    <div class="header">
+        <h1>Relationship Graph</h1>
+    </div>
+    <p class="graph-hint">Drag nodes to reposition. Closer relationships have shorter, darker edges.</p>
+    <canvas id="graph-canvas" width="900" height="600"
+            style="border: 1px solid var(--color-border); border-radius: 8px; background: var(--color-bg); cursor: grab;">
+    </canvas>
+    <div id="selected-info"></div>
+    <div class="legend">
+        <h4>Relationship Closeness (1-10)</h4>
+        <div class="legend-items">
+            <div class="legend-item">
+                <span class="legend-line" style="background: hsl(220, 70%, 40%); height: 4px; display: inline-block;"></span>
+                <span>10 - Very Close (Spouse, Partner)</span>
+            </div>
+            <div class="legend-item">
+                <span class="legend-line" style="background: hsl(220, 70%, 52%); height: 3px; display: inline-block;"></span>
+                <span>7 - Close (Family, Best Friend)</span>
+            </div>
+            <div class="legend-item">
+                <span class="legend-line" style="background: hsl(220, 70%, 64%); height: 2px; display: inline-block;"></span>
+                <span>4 - Moderate (Friend, Colleague)</span>
+            </div>
+            <div class="legend-item">
+                <span class="legend-line" style="background: hsl(220, 70%, 72%); height: 1px; display: inline-block;"></span>
+                <span>2 - Distant (Acquaintance)</span>
+            </div>
+        </div>
+    </div>
+</div>
+
+<script>
+(function() {
+    const RELATIONSHIP_DISPLAY = {{ relationship_type_display|tojson }};
+    const graphData = {{ graph_data|tojson }};
+
+    const canvas = document.getElementById('graph-canvas');
+    const ctx = canvas.getContext('2d');
+    const width = canvas.width;
+    const height = canvas.height;
+    const centerX = width / 2;
+    const centerY = height / 2;
+
+    const nodes = graphData.nodes.map(function(node) {
+        return Object.assign({}, node, {
+            x: centerX + (Math.random() - 0.5) * 300,
+            y: centerY + (Math.random() - 0.5) * 300,
+            vx: 0,
+            vy: 0
+        });
+    });
+
+    const nodeMap = new Map(nodes.map(function(node) { return [node.id, node]; }));
+
+    const edges = graphData.edges.map(function(edge) {
+        const sourceNode = nodeMap.get(edge.source);
+        const targetNode = nodeMap.get(edge.target);
+        if (!sourceNode || !targetNode) return null;
+        return Object.assign({}, edge, { sourceNode: sourceNode, targetNode: targetNode });
+    }).filter(function(edge) { return edge !== null; });
+
+    let dragNode = null;
+    let selectedNode = null;
+
+    const repulsion = 5000;
+    const springStrength = 0.05;
+    const baseSpringLength = 150;
+    const damping = 0.9;
+    const centerPull = 0.01;
+
+    function simulate() {
+        for (const node of nodes) { node.vx = 0; node.vy = 0; }
+        for (let i = 0; i < nodes.length; i++) {
+            for (let j = i + 1; j < nodes.length; j++) {
+                const dx = nodes[j].x - nodes[i].x;
+                const dy = nodes[j].y - nodes[i].y;
+                const dist = Math.sqrt(dx * dx + dy * dy) || 1;
+                const force = repulsion / (dist * dist);
+                const fx = (dx / dist) * force;
+                const fy = (dy / dist) * force;
+                nodes[i].vx -= fx; nodes[i].vy -= fy;
+                nodes[j].vx += fx; nodes[j].vy += fy;
+            }
+        }
+        for (const edge of edges) {
+            const dx = edge.targetNode.x - edge.sourceNode.x;
+            const dy = edge.targetNode.y - edge.sourceNode.y;
+            const dist = Math.sqrt(dx * dx + dy * dy) || 1;
+            const normalizedWeight = edge.closeness_weight / 10;
+            const idealLength = baseSpringLength * (1.5 - normalizedWeight);
+            const displacement = dist - idealLength;
+            const force = springStrength * displacement;
+            const fx = (dx / dist) * force;
+            const fy = (dy / dist) * force;
+            edge.sourceNode.vx += fx; edge.sourceNode.vy += fy;
+            edge.targetNode.vx -= fx; edge.targetNode.vy -= fy;
+        }
+        for (const node of nodes) {
+            node.vx += (centerX - node.x) * centerPull;
+            node.vy += (centerY - node.y) * centerPull;
+        }
+        for (const node of nodes) {
+            if (node === dragNode) continue;
+            node.x += node.vx * damping;
+            node.y += node.vy * damping;
+            node.x = Math.max(30, Math.min(width - 30, node.x));
+            node.y = Math.max(30, Math.min(height - 30, node.y));
+        }
+    }
+
+    function getEdgeColor(weight) {
+        const normalized = weight / 10;
+        return 'hsl(220, 70%, ' + (80 - normalized * 40) + '%)';
+    }
+
+    function draw() {
+        ctx.clearRect(0, 0, width, height);
+        for (const edge of edges) {
+            const lineWidth = 1 + (edge.closeness_weight / 10) * 3;
+            ctx.strokeStyle = getEdgeColor(edge.closeness_weight);
+            ctx.lineWidth = lineWidth;
+            ctx.beginPath();
+            ctx.moveTo(edge.sourceNode.x, edge.sourceNode.y);
+            ctx.lineTo(edge.targetNode.x, edge.targetNode.y);
+            ctx.stroke();
+            const midX = (edge.sourceNode.x + edge.targetNode.x) / 2;
+            const midY = (edge.sourceNode.y + edge.targetNode.y) / 2;
+            ctx.fillStyle = '#666';
+            ctx.font = '10px sans-serif';
+            ctx.textAlign = 'center';
+            const label = RELATIONSHIP_DISPLAY[edge.relationship_type] || edge.relationship_type;
+            ctx.fillText(label, midX, midY - 5);
+        }
+        for (const node of nodes) {
+            const isSelected = node === selectedNode;
+            const radius = isSelected ? 25 : 20;
+            ctx.beginPath();
+            ctx.arc(node.x, node.y, radius, 0, Math.PI * 2);
+            ctx.fillStyle = isSelected ? '#0066cc' : '#fff';
+            ctx.fill();
+            ctx.strokeStyle = '#0066cc';
+            ctx.lineWidth = 2;
+            ctx.stroke();
+            ctx.fillStyle = isSelected ? '#fff' : '#333';
+            ctx.font = '12px sans-serif';
+            ctx.textAlign = 'center';
+            ctx.textBaseline = 'middle';
+            const name = node.name.length > 10 ? node.name.slice(0, 9) + '\u2026' : node.name;
+            ctx.fillText(name, node.x, node.y);
+        }
+    }
+
+    function animate() {
+        simulate();
+        draw();
+        requestAnimationFrame(animate);
+    }
+    animate();
+
+    function getNodeAt(x, y) {
+        for (const node of nodes) {
+            const dx = x - node.x;
+            const dy = y - node.y;
+            if (dx * dx + dy * dy < 400) return node;
+        }
+        return null;
+    }
+
+    canvas.addEventListener('mousedown', function(event) {
+        const rect = canvas.getBoundingClientRect();
+        const node = getNodeAt(event.clientX - rect.left, event.clientY - rect.top);
+        if (node) {
+            dragNode = node;
+            selectedNode = node;
+            const infoDiv = document.getElementById('selected-info');
+            let html = '<div class="selected-info"><h3>' + node.name + '</h3>';
+            if (node.current_job) html += '<p>Job: ' + node.current_job + '</p>';
+            html += '<a href="/contacts/' + node.id + '">View details</a></div>';
+            infoDiv.innerHTML = html;
+        }
+    });
+
+    canvas.addEventListener('mousemove', function(event) {
+        if (!dragNode) return;
+        const rect = canvas.getBoundingClientRect();
+        dragNode.x = event.clientX - rect.left;
+        dragNode.y = event.clientY - rect.top;
+    });
+
+    canvas.addEventListener('mouseup', function() { dragNode = null; });
+    canvas.addEventListener('mouseleave', function() { dragNode = null; });
+})();
+</script>
+{% endblock %}

python/api/templates/need_list.html

@@ -0,0 +1,31 @@
+{% extends "base.html" %}
+{% block title %}Needs{% endblock %}
+{% block content %}
+<div class="need-list">
+    <div class="header">
+        <h1>Needs / Accommodations</h1>
+        <button class="btn btn-primary" onclick="document.getElementById('need-form').toggleAttribute('hidden')">Add Need</button>
+    </div>
+
+    <form id="need-form" hidden
+          hx-post="/htmx/needs"
+          hx-target="#need-items"
+          hx-swap="innerHTML"
+          hx-on::after-request="if(event.detail.successful) this.reset()"
+          class="need-form">
+        <div class="form-group">
+            <label for="name">Name *</label>
+            <input id="name" name="name" type="text" placeholder="e.g., Light Sensitive, ADHD" required>
+        </div>
+        <div class="form-group">
+            <label for="description">Description</label>
+            <textarea id="description" name="description" placeholder="Optional description..." rows="2"></textarea>
+        </div>
+        <button type="submit" class="btn btn-primary">Create</button>
+    </form>
+
+    <div id="need-items">
+        {% include "partials/need_items.html" %}
+    </div>
+</div>
+{% endblock %}

python/api/templates/partials/contact_table.html

@@ -0,0 +1,33 @@
+{% if contacts %}
+<table>
+    <thead>
+        <tr>
+            <th>Name</th>
+            <th>Job</th>
+            <th>Timezone</th>
+            <th>Actions</th>
+        </tr>
+    </thead>
+    <tbody>
+        {% for contact in contacts %}
+        <tr id="contact-row-{{ contact.id }}">
+            <td><a href="/contacts/{{ contact.id }}">{{ contact.name }}</a></td>
+            <td>{{ contact.current_job or "-" }}</td>
+            <td>{{ contact.timezone or "-" }}</td>
+            <td>
+                <a href="/contacts/{{ contact.id }}/edit" class="btn">Edit</a>
+                <button class="btn btn-danger"
+                        hx-delete="/api/contacts/{{ contact.id }}"
+                        hx-target="#contact-row-{{ contact.id }}"
+                        hx-swap="outerHTML"
+                        hx-confirm="Delete this contact?">
+                    Delete
+                </button>
+            </td>
+        </tr>
+        {% endfor %}
+    </tbody>
+</table>
+{% else %}
+<p>No contacts yet.</p>
+{% endif %}

python/api/templates/partials/manage_needs.html

@@ -0,0 +1,14 @@
+<ul class="manage-needs-list">
+    {% for need in contact.needs %}
+    <li id="contact-need-{{ need.id }}">
+        <strong>{{ need.name }}</strong>
+        {% if need.description %}<span> - {{ need.description }}</span>{% endif %}
+        <button class="btn btn-small btn-danger"
+                hx-delete="/api/contacts/{{ contact.id }}/needs/{{ need.id }}"
+                hx-target="#contact-need-{{ need.id }}"
+                hx-swap="outerHTML">
+            Remove
+        </button>
+    </li>
+    {% endfor %}
+</ul>

python/api/templates/partials/manage_relationships.html

@@ -0,0 +1,23 @@
+{% for rel in contact.related_to %}
+<div class="manage-rel-item" id="rel-{{ contact.id }}-{{ rel.related_contact_id }}">
+    <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a>
+    <span class="tag">{{ rel.relationship_type|replace("_", " ")|title }}</span>
+    <label class="weight-control">
+        <span>Closeness:</span>
+        <input type="range" min="1" max="10" value="{{ rel.closeness_weight }}"
+               hx-post="/htmx/contacts/{{ contact.id }}/relationships/{{ rel.related_contact_id }}/weight"
+               hx-trigger="change"
+               hx-include="this"
+               name="closeness_weight"
+               hx-swap="none"
+               oninput="this.nextElementSibling.textContent = this.value">
+        <span class="weight-value">{{ rel.closeness_weight }}</span>
+    </label>
+    <button class="btn btn-small btn-danger"
+            hx-delete="/api/contacts/{{ contact.id }}/relationships/{{ rel.related_contact_id }}"
+            hx-target="#rel-{{ contact.id }}-{{ rel.related_contact_id }}"
+            hx-swap="outerHTML">
+        Remove
+    </button>
+</div>
+{% endfor %}

python/api/templates/partials/need_items.html

@@ -0,0 +1,21 @@
+{% if needs %}
+<ul class="need-items">
+    {% for need in needs %}
+    <li id="need-item-{{ need.id }}">
+        <div class="need-info">
+            <strong>{{ need.name }}</strong>
+            {% if need.description %}<p>{{ need.description }}</p>{% endif %}
+        </div>
+        <button class="btn btn-danger"
+                hx-delete="/api/needs/{{ need.id }}"
+                hx-target="#need-item-{{ need.id }}"
+                hx-swap="outerHTML"
+                hx-confirm="Delete this need?">
+            Delete
+        </button>
+    </li>
+    {% endfor %}
+</ul>
+{% else %}
+<p>No needs defined yet.</p>
+{% endif %}

python/ebook_search/answer.py

@@ -32,8 +32,11 @@ async def answer_query(
         return "No relevant sources were found."
 
     logger.info(
-        f"ebook_answer_request_start {config.vllm_base_url=} {config.chat_model=} sources={len(results)} "
-        f"query_length={len(query)}"
+        "ebook_answer_request_start base_url=%s model=%s sources=%s query_length=%s",
+        config.vllm_base_url,
+        config.chat_model,
+        len(results),
+        len(query),
     )
     context = "\n\n".join(
         f"[{index}] {result.source_title}{' - ' + result.chapter_title if result.chapter_title else ''}\n{result.text}"
@@ -54,5 +57,9 @@ async def answer_query(
         ],
     )
 
-    logger.info(f"ebook_answer_request_complete {config.chat_model=} answer_length={len(content)}")
+    logger.info(
+        "ebook_answer_request_complete model=%s answer_length=%s",
+        config.chat_model,
+        len(content),
+    )
     return content or "The model returned an empty answer."

python/ebook_search/api/bm25_tasks.py

@@ -36,7 +36,10 @@ def schedule_bm25_refresh(app: FastAPI) -> None:
         app.state.bm25_refresh_task = loop.create_task(refresh_bm25_for_app(app))
 
     app.state.bm25_refresh_timer = loop.call_later(app.state.config.bm25_refresh_delay_seconds, start_refresh)
-    logger.info(f"ebook_bm25_refresh_scheduled {app.state.config.bm25_refresh_delay_seconds=}")
+    logger.info(
+        "ebook_bm25_refresh_scheduled delay_seconds=%s",
+        app.state.config.bm25_refresh_delay_seconds,
+    )
 
 
 def cancel_bm25_refresh(app: FastAPI) -> None:

python/ebook_search/api/judge_tasks.py

@@ -65,12 +65,12 @@ def start_book_phrase_judgment(app: FastAPI, background_tasks: BackgroundTasks,
     """
     state = get_judge_task_state(app)
     if source_id in state.running_book_ids:
-        logger.info(f"ebook_book_phrase_judgment_already_running {source_id=}")
+        logger.info("ebook_book_phrase_judgment_already_running source_id=%s", source_id)
         return False
     state.running_book_ids.add(source_id)
     state.outcome_messages.pop(source_id, None)
     background_tasks.add_task(judge_book_phrases_for_app, app, source_id)
-    logger.info(f"ebook_book_phrase_judgment_queued {source_id=}")
+    logger.info("ebook_book_phrase_judgment_queued source_id=%s", source_id)
     return True
 
 
@@ -85,8 +85,12 @@ async def judge_book_phrases_for_app(app: FastAPI, source_id: int) -> None:
     try:
         result = await judge_candidate_phrases_for_books(app.state.engine, app.state.config, source_ids=[source_id])
         logger.info(
-            f"ebook_book_phrase_judgment_complete {source_id=} {result.candidates_judged=} {result.protected_phrases=} "
-            f"{result.phrase_mentions=} {result.books_failed=}"
+            "ebook_book_phrase_judgment_complete source_id=%s judged=%s protected=%s mentions=%s failed=%s",
+            source_id,
+            result.candidates_judged,
+            result.protected_phrases,
+            result.phrase_mentions,
+            result.books_failed,
         )
         if result.books_failed:
             message = "Judging failed; see server logs for details"
@@ -95,7 +99,7 @@ async def judge_book_phrases_for_app(app: FastAPI, source_id: int) -> None:
                 f"Judged {result.candidates_judged} candidates; {result.protected_phrases} protected phrases promoted"
             )
     except Exception:
-        logger.exception(f"ebook_book_phrase_judgment_task_failed {source_id=}")
+        logger.exception("ebook_book_phrase_judgment_task_failed source_id=%s", source_id)
         message = "Judging failed; see server logs for details"
     state.running_book_ids.discard(source_id)
     state.outcome_messages[source_id] = message

python/ebook_search/api/main.py

@@ -37,9 +37,16 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     config = load_config()
     app.state.config = config
     logger.info(
-        f"ebook_search_config_loaded {config.top_k=} {config.embedding_model=} {config.embedding_base_url=} "
-        f"{config.vllm_base_url=} {config.rerank.enabled=} {config.phrase_matching_enabled=} {config.answer_enabled=} "
-        f"library_paths={len(config.library_paths)}"
+        "ebook_search_config_loaded top_k=%s embedding_model=%s embedding_base_url=%s vllm_base_url=%s "
+        "rerank_enabled=%s phrase_matching_enabled=%s answer_enabled=%s library_paths=%s",
+        config.top_k,
+        config.embedding_model,
+        config.embedding_base_url,
+        config.vllm_base_url,
+        config.rerank.enabled,
+        config.phrase_matching_enabled,
+        config.answer_enabled,
+        len(config.library_paths),
     )
     if not config.library_paths:
         logger.warning("ebook_search_no_library_paths_configured")

python/ebook_search/api/routes/admin.py

@@ -13,7 +13,7 @@ from python.ebook_search.api.dependencies import (  # noqa: TC001 FastAPI resolv
     AppEngine,
     AppHttpClient,
 )
-from python.ebook_search.api.web import error_response, templates
+from python.ebook_search.api.web import templates
 from python.ebook_search.embeddings import embed_missing_chunks, embedding_model_stats
 from python.ebook_search.ingest import ingest_configured_paths
 from python.ebook_search.protected_phrases.generate_ngrams import generate_candidate_phrases_for_books
@@ -32,8 +32,10 @@ async def admin(request: Request, config: AppConfig, session: AsyncDbSession) ->
     stats = await embedding_model_stats(session)
     phrase_stats = await corpus_phrase_stats(session)
     logger.info(
-        f"ebook_admin_page_loaded models={len(stats)} {phrase_stats.candidate_phrases=} "
-        f"{phrase_stats.protected_phrases=}"
+        "ebook_admin_page_loaded models=%s candidate_phrases=%s protected_phrases=%s",
+        len(stats),
+        phrase_stats.candidate_phrases,
+        phrase_stats.protected_phrases,
     )
     return templates.TemplateResponse(
         request,
@@ -50,26 +52,65 @@ async def scan_library(request: Request, config: AppConfig, session: AsyncDbSess
         await session.commit()
     except Exception as error:
         logger.exception("ebook_admin_scan_failed")
-        return error_response(request, error)
+        return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
 
-    logger.info(f"ebook_admin_scan_complete {count=}")
+    logger.info("ebook_admin_scan_complete changed_files=%s", count)
     if count > 0:
         schedule_bm25_refresh(request.app)
     return templates.TemplateResponse(request, "partials/admin_status.html", {"message": f"Indexed {count} EPUBs"})
 
 
 @router.post("/phrases/generate-all", response_class=HTMLResponse)
-async def generate_all_phrases(request: Request, config: AppConfig, engine: AppEngine) -> HTMLResponse:
+async def generate_all_phrases(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
     """Regenerate candidate phrases for every indexed book without LLM judging."""
+    return await run_phrase_generation(request, config, session, only_missing=False)
+
+
+@router.post("/phrases/generate-missing", response_class=HTMLResponse)
+async def generate_missing_phrases(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
+    """Generate candidate phrases only for books that have none yet."""
+    return await run_phrase_generation(request, config, session, only_missing=True)
+
+
+async def run_phrase_generation(
+    request: Request,
+    config: AppConfig,
+    session: AsyncDbSession,
+    *,
+    only_missing: bool,
+) -> HTMLResponse:
+    """Run candidate phrase generation and render the outcome as an admin status partial.
+
+    Args:
+        request (Request): Current request, for template rendering.
+        config (AppConfig): Runtime phrase-tuning settings.
+        session (AsyncDbSession): Active database session.
+        only_missing (bool): Only generate for books without candidates instead of every book.
+
+    Returns:
+        HTMLResponse: Status partial describing the generation outcome.
+    """
     try:
-        result = await generate_candidate_phrases_for_books(engine, config)
+        result = await generate_candidate_phrases_for_books(session, config, only_missing=only_missing)
+        await session.commit()
     except Exception as error:
-        logger.exception("ebook_admin_generate_phrases_failed")
-        return error_response(request, error)
+        await session.rollback()
+        logger.exception("ebook_admin_generate_phrases_failed only_missing=%s", only_missing)
+        return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
 
     logger.info(
-        f"ebook_admin_generate_phrases_complete {result.books_seen=} {result.books_built=} {result.candidate_phrases=}"
+        "ebook_admin_generate_phrases_complete only_missing=%s books_seen=%s books_built=%s candidates=%s",
+        only_missing,
+        result.books_seen,
+        result.books_built,
+        result.candidate_phrases,
     )
+    if only_missing and result.books_seen == 0:
+        return templates.TemplateResponse(
+            request,
+            "partials/admin_status.html",
+            {"message": "All books already have candidate phrases"},
+        )
     return templates.TemplateResponse(
         request,
         "partials/admin_status.html",
@@ -128,11 +169,17 @@ async def run_phrase_judgment(
         result = await judge_candidate_phrases_for_books(engine, config, source_ids=source_ids)
     except Exception as error:
         logger.exception("ebook_admin_judge_phrases_failed")
-        return error_response(request, error)
+        return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
 
     logger.info(
-        f"ebook_admin_judge_phrases_complete {result.books_seen=} {result.books_judged=} {result.books_failed=} "
-        f"{result.candidates_judged=} {result.protected_phrases=} {result.phrase_mentions=}"
+        "ebook_admin_judge_phrases_complete books_seen=%s books_judged=%s books_failed=%s candidates_judged=%s "
+        "protected=%s mentions=%s",
+        result.books_seen,
+        result.books_judged,
+        result.books_failed,
+        result.candidates_judged,
+        result.protected_phrases,
+        result.phrase_mentions,
     )
     return templates.TemplateResponse(
         request,
@@ -161,9 +208,9 @@ async def embed_missing(
         await session.commit()
     except Exception as error:
         logger.exception("ebook_admin_embed_missing_failed")
-        return error_response(request, error)
+        return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
 
-    logger.info(f"ebook_admin_embed_missing_complete {count=}")
+    logger.info("ebook_admin_embed_missing_complete chunks=%s", count)
     return templates.TemplateResponse(
         request,
         "partials/admin_status.html",
@@ -189,12 +236,26 @@ async def embed_all(
             await session.commit()
             total += count
             batches += 1
-            logger.info(f"ebook_admin_embed_all_batch_complete {batches=} {count=} {total=}")
+            logger.info(
+                "ebook_admin_embed_all_batch_complete batch=%s chunks=%s total_chunks=%s",
+                batches,
+                count,
+                total,
+            )
     except Exception as error:
-        logger.exception(f"ebook_admin_embed_all_failed {batches=} {total=}")
-        return error_response(request, f"Embed all failed after {total} chunks in {batches} batches: {error}")
+        logger.exception(
+            "ebook_admin_embed_all_failed batches=%s chunks=%s",
+            batches,
+            total,
+        )
+        return templates.TemplateResponse(
+            request,
+            "partials/error.html",
+            {"message": f"Embed all failed after {total} chunks in {batches} batches: {error}"},
+            status_code=500,
+        )
 
-    logger.info(f"ebook_admin_embed_all_complete {batches=} {total=}")
+    logger.info("ebook_admin_embed_all_complete batches=%s chunks=%s", batches, total)
     return templates.TemplateResponse(
         request,
         "partials/admin_status.html",

python/ebook_search/api/routes/health.py

@@ -60,7 +60,14 @@ async def ready(config: AppConfig, session: AsyncDbSession, client: AppHttpClien
         status = "ready"
         status_code = HTTPStatus.OK
 
-    logger.info(f"ebook_ready_check {status=} {database_ok=} {embedding_ok=} {chat_status=} {bm25_status=}")
+    logger.info(
+        "ebook_ready_check status=%s database=%s embedding=%s chat=%s bm25=%s",
+        status,
+        database_ok,
+        embedding_ok,
+        chat_status,
+        bm25_status,
+    )
     return JSONResponse(content={"status": status, "checks": checks}, status_code=status_code)
 
 
@@ -76,7 +83,7 @@ async def check_database(session: AsyncSession) -> bool:
     try:
         await session.execute(select(literal(1)))
     except SQLAlchemyError as error:
-        logger.warning(f"ebook_ready_database_unavailable {error=}")
+        logger.warning("ebook_ready_database_unavailable error=%s", error)
         return False
     return True
 

python/ebook_search/api/routes/page.py

@@ -15,7 +15,6 @@ from python.ebook_search.api.dependencies import (
 from python.ebook_search.api.judge_tasks import is_judging_book, pop_book_judgment_outcome, start_book_phrase_judgment
 from python.ebook_search.api.web import templates
 from python.ebook_search.protected_phrases.generate_ngrams import recalculate_candidate_phrases_for_book
-from python.ebook_search.protected_phrases.store import count_protected_phrases
 from python.fastapi_tools import AsyncDbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
 from python.orm.richie import EbookCandidatePhrase, EbookChapter, EbookChunk, EbookProtectedPhrase, EbookSource
 
@@ -37,7 +36,7 @@ async def index(request: Request, config: AppConfig) -> HTMLResponse:
 async def books(request: Request, session: AsyncDbSession) -> HTMLResponse:
     """Render the indexed books page."""
     sources = list((await session.scalars(select(EbookSource).order_by(EbookSource.title))).all())
-    logger.info(f"ebook_books_page_loaded count={len(sources)}")
+    logger.info("ebook_books_page_loaded count=%s", len(sources))
     return templates.TemplateResponse(request, "books.html", {"sources": sources})
 
 
@@ -72,6 +71,14 @@ async def get_judged_candidate_count(session: AsyncSession, book_id: int) -> int
     )
 
 
+async def get_protected_count(session: AsyncSession, book_id: int) -> int:
+    """Return the number of protected phrases for one book."""
+    return (
+        await session.scalar(select(func.count(EbookProtectedPhrase.id)).where(EbookProtectedPhrase.book_id == book_id))
+        or 0
+    )
+
+
 async def get_candidates(session: AsyncSession, book_id: int) -> list[EbookCandidatePhrase]:
     """Return the indexed candidates for one book."""
     return list(
@@ -115,7 +122,7 @@ async def book_detail(source_id: int, request: Request, session: AsyncDbSession)
         chunk_count = await get_chunk_count(session, source.id)
         candidate_count = await get_candidate_count(session, source.id)
         judged_candidate_count = await get_judged_candidate_count(session, source.id)
-        protected_count = await count_protected_phrases(session, source.id)
+        protected_count = await get_protected_count(session, source.id)
         candidates = await get_candidates(session, source.id)
         protected_phrases = await get_protected_phrases(session, source.id)
     else:
@@ -127,8 +134,14 @@ async def book_detail(source_id: int, request: Request, session: AsyncDbSession)
         candidates = []
         protected_phrases = []
     logger.info(
-        f"ebook_book_detail_loaded {source_id=} found={source is not None} {chapter_count=} {chunk_count=} "
-        f"{candidate_count=} {judged_candidate_count=} {protected_count=}"
+        "ebook_book_detail_loaded source_id=%s found=%s chapters=%s chunks=%s candidates=%s judged=%s protected=%s",
+        source_id,
+        source is not None,
+        chapter_count,
+        chunk_count,
+        candidate_count,
+        judged_candidate_count,
+        protected_count,
     )
     return templates.TemplateResponse(
         request,
@@ -155,14 +168,16 @@ async def recalculate_book_phrases(source_id: int, config: AppConfig, session: A
     if source is None:
         raise HTTPException(status_code=404, detail="Book not found")
 
-    try:
-        result = await recalculate_candidate_phrases_for_book(session, source, config)
-    except ValueError as error:
-        raise HTTPException(status_code=409, detail=str(error)) from error
+    result = await recalculate_candidate_phrases_for_book(session, source, config, use_process_pool=True)
     logger.info(
-        f"ebook_book_phrase_recalculation_complete {source_id=} {result.candidate_phrases=} "
-        f"{result.deleted_candidates=} {result.deleted_protected_phrases=} {result.deleted_aliases=} "
-        f"{result.deleted_mentions=}"
+        "ebook_book_phrase_recalculation_complete source_id=%s candidates=%s deleted_candidates=%s "
+        "deleted_protected=%s deleted_aliases=%s deleted_mentions=%s",
+        source_id,
+        result.candidate_phrases,
+        result.deleted_candidates,
+        result.deleted_protected_phrases,
+        result.deleted_aliases,
+        result.deleted_mentions,
     )
     return RedirectResponse(
         url=f"/books/{source_id}?phrases_recalculated={result.candidate_phrases}",
@@ -183,5 +198,5 @@ async def judge_book_phrases(
         raise HTTPException(status_code=404, detail="Book not found")
 
     started = start_book_phrase_judgment(request.app, background_tasks, source.id)
-    logger.info(f"ebook_book_phrase_judgment_requested {source_id=} {started=}")
+    logger.info("ebook_book_phrase_judgment_requested source_id=%s started=%s", source_id, started)
     return RedirectResponse(url=f"/books/{source_id}", status_code=303)

python/ebook_search/api/routes/search.py

@@ -16,7 +16,7 @@ from python.ebook_search.api.dependencies import (  # noqa: TC001 FastAPI resolv
     AppEngine,
     AppHttpClient,
 )
-from python.ebook_search.api.web import error_response, templates
+from python.ebook_search.api.web import templates
 from python.ebook_search.guardrails import (
     CitationReport,
     is_confident,
@@ -49,8 +49,9 @@ async def build_answer(
 
     if not is_confident(response.results, config):
         logger.info(
-            f"ebook_answer_low_confidence confidence={retrieval_confidence(response.results):.4f} "
-            f"{config.min_retrieval_confidence=:.4f}"
+            "ebook_answer_low_confidence confidence=%.4f threshold=%.4f",
+            retrieval_confidence(response.results),
+            config.min_retrieval_confidence,
         )
         answer = (
             "Retrieval confidence is low for this query, so answer generation was skipped. "
@@ -61,14 +62,18 @@ async def build_answer(
     try:
         answer = await answer_query(client, query, response.results, config)
     except RuntimeError as error:
-        logger.warning(f"ebook_answer_request_failed_falling_back {error=}")
+        logger.warning("ebook_answer_request_failed_falling_back error=%s", error)
         return "Answer generation failed. Source chunks are still shown below.", False, None
 
     citation_report = None
     if config.validate_citations_enabled and response.results:
         citation_report = validate_citations(answer, len(response.results))
         if citation_report.invalid or not citation_report.grounded:
-            logger.warning(f"ebook_answer_citation_issue {citation_report.invalid=} {citation_report.grounded=}")
+            logger.warning(
+                "ebook_answer_citation_issue invalid=%s grounded=%s",
+                citation_report.invalid,
+                citation_report.grounded,
+            )
     return answer, False, citation_report
 
 
@@ -79,9 +84,8 @@ async def search(
     engine: AppEngine,
     client: AppHttpClient,
     query: Annotated[str, Form()],
-    *,
-    rerank: Annotated[bool, Form()] = False,
-    phrase_matching: Annotated[bool, Form()] = False,
+    rerank: Annotated[str | None, Form()] = None,
+    phrase_matching: Annotated[str | None, Form()] = None,
 ) -> HTMLResponse:
     """Run a search and render HTMX results."""
     try:
@@ -90,12 +94,12 @@ async def search(
             client,
             query,
             config,
-            rerank=rerank,
-            phrase_matching=phrase_matching,
+            rerank=rerank == "true",
+            phrase_matching=phrase_matching == "true",
         )
     except Exception as error:
         logger.exception("ebook_search_request_failed")
-        return error_response(request, error)
+        return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
 
     answer_start = perf_counter()
     answer, low_confidence, citation_report = await build_answer(client, query, response, config)
@@ -106,10 +110,12 @@ async def search(
     )
 
     for step in response.timings:
-        logger.info(f"ebook_search_timing {step.name=} {step.duration_ms=:.1f}")
+        logger.info("ebook_search_timing step=%r runtime_ms=%.1f", step.name, step.duration_ms)
     logger.info(
-        f"ebook_search_request_complete results={len(response.results)} {response.rank_label=} "
-        f"{response.total_runtime_ms=:.1f}"
+        "ebook_search_request_complete results=%s rank_label=%s runtime_ms=%.1f",
+        len(response.results),
+        response.rank_label,
+        response.total_runtime_ms,
     )
     return templates.TemplateResponse(
         request,

python/ebook_search/api/templates/admin.html

@@ -60,6 +60,13 @@ head %}
     >
       <button type="submit">Regenerate all phrases</button>
     </form>
+    <form
+      hx-post="/admin/phrases/generate-missing"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Add missing phrases</button>
+    </form>
     <form
       hx-post="/admin/phrases/judge-all"
       hx-target="#admin-status"

python/ebook_search/api/web.py

@@ -3,14 +3,9 @@
 from __future__ import annotations
 
 from pathlib import Path
-from typing import TYPE_CHECKING
 
 from fastapi.templating import Jinja2Templates
 
-if TYPE_CHECKING:
-    from fastapi import Request
-    from fastapi.responses import HTMLResponse
-
 PACKAGE_DIR = Path(__file__).resolve().parent
 TEMPLATE_DIR = PACKAGE_DIR / "templates"
 STATIC_DIR = PACKAGE_DIR / "static"
@@ -26,8 +21,3 @@ def static_version(filename: str) -> int:
 
 templates = Jinja2Templates(directory=TEMPLATE_DIR)
 templates.env.globals["static_version"] = static_version
-
-
-def error_response(request: Request, message: object) -> HTMLResponse:
-    """Render the shared error partial for a failed UI request."""
-    return templates.TemplateResponse(request, "partials/error.html", {"message": str(message)}, status_code=500)

python/ebook_search/bm25_corpus.py

@@ -15,7 +15,6 @@ from typing import TYPE_CHECKING
 import bm25s
 from sqlalchemy import func, select, union_all
 
-from python.ebook_search.chunk_records import CHUNK_RECORD_COLUMNS
 from python.orm.richie import EbookChapter, EbookChunk, EbookSource
 
 if TYPE_CHECKING:
@@ -81,19 +80,23 @@ async def ensure_bm25_corpus(session: AsyncSession, config: EbookSearchConfig) -
     manifest = read_bm25_manifest(index_path)
     db_updated_at = await corpus_last_updated_at(session)
     if not bm25_index_exists(index_path, manifest):
-        logger.info(f"ebook_bm25_index_missing {index_path=}")
+        logger.info("ebook_bm25_index_missing path=%s", index_path)
         await refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
         return
     if db_updated_at is not None and manifest is not None and manifest.created_at < db_updated_at:
         logger.info(
-            f"ebook_bm25_index_stale {index_path=} created_at={manifest.created_at.isoformat()} "
-            f"db_updated_at={db_updated_at.isoformat()}"
+            "ebook_bm25_index_stale path=%s created_at=%s db_updated_at=%s",
+            index_path,
+            manifest.created_at.isoformat(),
+            db_updated_at.isoformat(),
         )
         await refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
         return
     logger.info(
-        f"ebook_bm25_index_current {index_path=} chunks={manifest.chunk_count if manifest else 0} "
-        f"created_at={manifest.created_at.isoformat() if manifest else None}"
+        "ebook_bm25_index_current path=%s chunks=%s created_at=%s",
+        index_path,
+        manifest.chunk_count if manifest else 0,
+        manifest.created_at.isoformat() if manifest else None,
     )
 
 
@@ -116,7 +119,10 @@ async def refresh_bm25_corpus(
     )
     await asyncio.to_thread(write_bm25_corpus, index_path, records, texts, manifest)
     logger.info(
-        f"ebook_bm25_index_refreshed {index_path=} {manifest.chunk_count=} created_at={manifest.created_at.isoformat()}"
+        "ebook_bm25_index_refreshed path=%s chunks=%s created_at=%s",
+        index_path,
+        manifest.chunk_count,
+        manifest.created_at.isoformat(),
     )
     return manifest
 
@@ -129,7 +135,7 @@ def load_bm25_corpus(config: EbookSearchConfig) -> BM25Corpus:
     """
     index_path = bm25_index_path(config)
     active_index_path = get_current_bm25_index(index_path)
-    logger.info(f"ebook_bm25_corpus_cache_load {index_path=} {active_index_path=}")
+    logger.info("ebook_bm25_corpus_cache_load path=%s active_path=%s", index_path, active_index_path)
     manifest = read_bm25_manifest(index_path)
     if manifest is None or not bm25_index_exists(index_path, manifest):
         msg = f"BM25 corpus is not available: {index_path}"
@@ -170,7 +176,13 @@ async def fetch_bm25_corpus_records(session: AsyncSession) -> tuple[list[dict[st
     """
     statement = (
         select(
-            *CHUNK_RECORD_COLUMNS,
+            EbookChunk.id.label("chunk_id"),
+            EbookChunk.text.label("text"),
+            EbookSource.id.label("source_id"),
+            EbookSource.title.label("source_title"),
+            EbookSource.author.label("source_author"),
+            EbookChapter.title.label("chapter_title"),
+            EbookChunk.page_label.label("page_label"),
             EbookChunk.search_text.label("bm25_text"),
         )
         .select_from(EbookChunk)

python/ebook_search/chunk_records.py

@@ -1,13 +0,0 @@
-"""Shared database columns used to build search-result records."""
-
-from python.orm.richie import EbookChapter, EbookChunk, EbookSource
-
-CHUNK_RECORD_COLUMNS = (
-    EbookChunk.id.label("chunk_id"),
-    EbookChunk.text.label("text"),
-    EbookSource.id.label("source_id"),
-    EbookSource.title.label("source_title"),
-    EbookSource.author.label("source_author"),
-    EbookChapter.title.label("chapter_title"),
-    EbookChunk.page_label.label("page_label"),
-)

python/ebook_search/config.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from os import getenv
 from typing import Annotated, Self
 
 from pydantic import AliasChoices, Field, field_validator, model_validator
@@ -31,6 +32,11 @@ def normalize_embedding_alias(model: str) -> str:
     return standard_model
 
 
+def normalize_embedding_model(default: str = "qwen3-embedding-0.6b") -> str:
+    """Normalize the configured embedding alias to its provider model name."""
+    return normalize_embedding_alias(getenv("EBOOK_SEARCH_EMBEDDING_MODEL", default))
+
+
 class RerankConfig(BaseSettings):
     """vLLM reranker settings."""
 

python/ebook_search/docker/Dockerfile

@@ -1,17 +1,12 @@
-FROM python:3.14-slim AS base
-
-COPY --from=ghcr.io/astral-sh/uv:0.11.26 /uv /uvx /bin/
+FROM python:3.14-slim
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
     APP_DIR=/home/richie/dotfiles \
-    UV_PROJECT_ENVIRONMENT=/opt/venv \
-    UV_PYTHON_DOWNLOADS=never \
-    UV_NO_CACHE=1
-
-# Separate ENV instruction so ${APP_DIR} and ${PATH} from above resolve.
-ENV PYTHONPATH=${APP_DIR} \
-    PATH=/opt/venv/bin:${PATH}
+    EBOOK_SEARCH_HOST=0.0.0.0 \
+    EBOOK_SEARCH_PORT=8070 \
+    EBOOK_SEARCH_BM25_INDEX_DIR=/data/bm25
 
 WORKDIR ${APP_DIR}
 
@@ -19,29 +14,29 @@ RUN apt-get update \
     && apt-get install -y --no-install-recommends build-essential curl \
     && rm -rf /var/lib/apt/lists/*
 
-COPY python/ebook_search/docker/pyproject.toml python/ebook_search/docker/uv.lock ./
-
-RUN uv sync --locked --no-dev
-
-
-FROM base AS test
-
-RUN uv sync --locked
-
+COPY pyproject.toml README.md LICENSE ./
 COPY python ./python
-COPY tests/__init__.py ./tests/__init__.py
-COPY tests/ebook_search ./tests/ebook_search
 
-CMD ["pytest"]
-
-
-FROM base AS runtime
-
-ENV EBOOK_SEARCH_HOST=0.0.0.0 \
-    EBOOK_SEARCH_PORT=8070 \
-    EBOOK_SEARCH_BM25_INDEX_DIR=/data/bm25
-
-COPY python ./python
+RUN python -m pip install --upgrade pip \
+    && python -m pip install \
+        "alembic" \
+        "beautifulsoup4" \
+        "bm25s" \
+        "ebooklib" \
+        "fastapi" \
+        "httpx" \
+        "jinja2" \
+        "pgvector" \
+        "psycopg[binary]" \
+        "pydantic" \
+        "pydantic-settings" \
+        "python-multipart" \
+        "sqlalchemy[asyncio]" \
+        "tiktoken" \
+        "typer" \
+        "uvicorn[standard]" \
+        "yake" \
+    && python -m pip install --no-deps --editable "${APP_DIR}"
 
 RUN useradd --create-home --uid 10001 app \
     && mkdir -p /data \

python/ebook_search/docker/README.md

@@ -3,27 +3,26 @@
 Run the EPUB search app against the existing Postgres database on `jeeves`:
 
 ```sh
-python -m python.ebook_search.docker.containers start --library-path /path/to/epubs --build
+ebook-search-containers start --library-path /path/to/epubs --build
 ```
 
 All ebook-search Docker files live in this directory:
 
-- `Dockerfile` — multi-stage: `test` (runs pytest) and `runtime` (default target, the app image)
+- `Dockerfile`
 - `docker-compose.yml`
-- `containers.py` — Typer lifecycle CLI
-- `pyproject.toml` / `uv.lock` — the container's uv-locked dependencies
+- `containers.py`
+- `container.py`
 
 The app listens on `http://localhost:8070`.
 
 Useful lifecycle commands:
 
 ```sh
-python -m python.ebook_search.docker.containers build
-python -m python.ebook_search.docker.containers start --library-path /path/to/epubs
-python -m python.ebook_search.docker.containers test
-python -m python.ebook_search.docker.containers logs
-python -m python.ebook_search.docker.containers ps
-python -m python.ebook_search.docker.containers stop
+ebook-search-containers build
+ebook-search-containers start --library-path /path/to/epubs
+ebook-search-containers logs
+ebook-search-containers ps
+ebook-search-containers stop
 ```
 
 Direct compose usage from the repo root:
@@ -32,46 +31,10 @@ Direct compose usage from the repo root:
 docker compose -f python/ebook_search/docker/docker-compose.yml ps
 ```
 
-## Dependencies
-
-The image builds its environment with uv from `pyproject.toml` + `uv.lock` in this
-directory — this is the source of truth for the container's dependencies. To add or
-update a dependency, edit `pyproject.toml` here and regenerate the lock (uv is
-available in the `ebook-search` dev shell):
-
-```sh
-nix develop .#ebook-search -c uv lock --project python/ebook_search/docker
-```
-
-## Tests
-
-The main pytest suite excludes `tests/ebook_search` (its dependencies are no longer
-in the nix dev shell). The `test ebook search` CI workflow runs them in a uv env
-built from the lockfile in this directory — same commands work locally from the
-repo root (the `--override-ini` drops the main suite's ignore):
-
-```sh
-uv sync --locked --project python/ebook_search/docker
-uv run --project python/ebook_search/docker --no-sync pytest tests/ebook_search --override-ini addopts="-n auto -ra"
-```
-
-They can also run inside the Docker `test` image, which validates the image itself:
-
-```sh
-python -m python.ebook_search.docker.containers test
-```
-
-or the raw docker equivalent:
-
-```sh
-docker build --file python/ebook_search/docker/Dockerfile --target test --tag ebook-search:test .
-docker run --rm ebook-search:test
-```
-
-## Configuration
-
-The compose service loads the repo root `.env` into the container via `env_file`.
+The compose service also loads the repo root `.env` into the container via `env_file`.
 
 Mount your EPUB directory by setting `EBOOK_LIBRARY_HOST_PATH` in an env file or on the command line. The container sees it as `/library`, and `EBOOK_SEARCH_LIBRARY_PATHS` is set to `/library` inside the container.
 
 Database connection settings are controlled by `RICHIE_DB`, `RICHIE_HOST`, `RICHIE_PORT`, `RICHIE_USER`, and `RICHIE_PASSWORD`. The default host is `jeeves`.
+
+Startup runs the Richie Alembic migrations automatically after creating the `main` schema and `vector` extension.

python/ebook_search/docker/containers.py

@@ -32,7 +32,7 @@ def docker_run(
     capture_output: bool = False,
 ) -> subprocess.CompletedProcess[str]:
     """Run docker with repo-root cwd and consistent error handling."""
-    logger.info(f"docker {' '.join(arguments)}")
+    logger.info("docker %s", " ".join(arguments))
     return subprocess.run(
         ["docker", *arguments],
         cwd=get_repo_dir(),
@@ -73,23 +73,6 @@ def build_image() -> None:
         raise RuntimeError(msg)
 
 
-def build_test_image() -> None:
-    """Build the ebook search test Docker image."""
-    dockerfile = Path(__file__).resolve().with_name("Dockerfile")
-    result = docker_run(["build", "--file", str(dockerfile), "--target", "test", "--tag", "ebook-search:test", "."])
-    if result.returncode != 0:
-        msg = "Failed to build ebook search test image"
-        raise RuntimeError(msg)
-
-
-def run_test_image() -> None:
-    """Run the ebook search test suite inside Docker."""
-    result = docker_run(["run", "--rm", "ebook-search:test"])
-    if result.returncode != 0:
-        msg = f"Ebook search tests failed with code {result.returncode}"
-        raise RuntimeError(msg)
-
-
 def start_stack(
     *,
     library_path: Path | None = None,
@@ -227,19 +210,6 @@ def logs(
         typer.echo(output)
 
 
-@app.command("test")
-def run_tests(
-    *,
-    build: Annotated[bool, typer.Option("--build/--no-build", help="Build the test image before running.")] = True,
-    log_level: Annotated[str, typer.Option(help="Log level.")] = "INFO",
-) -> None:
-    """Run ebook search tests inside the Docker test image."""
-    configure_logger(log_level)
-    if build:
-        build_test_image()
-    run_test_image()
-
-
 @app.command("ps")
 def ps() -> None:
     """Show ebook search container status."""

python/ebook_search/docker/pyproject.toml

@@ -1,41 +0,0 @@
-[project]
-name = "ebook-search"
-version = "0.1.0"
-description = "Locked runtime environment for the ebook search container."
-requires-python = "~=3.14.0"
-dependencies = [
-    "alembic",
-    "beautifulsoup4",
-    "bm25s",
-    "ebooklib",
-    "fastapi",
-    "httpx",
-    "jinja2",
-    "pgvector",
-    "psycopg[binary]",
-    "pydantic",
-    "pydantic-settings",
-    "python-multipart",
-    "sqlalchemy[asyncio]",
-    "tiktoken",
-    "typer",
-    "uvicorn[standard]",
-    "yake",
-]
-
-[dependency-groups]
-dev = [
-    "aiosqlite",
-    "pytest",
-    "pytest-asyncio",
-    "pytest-mock",
-    "pytest-xdist",
-]
-
-[tool.uv]
-package = false
-
-[tool.pytest.ini_options]
-addopts = "-n auto -ra"
-asyncio_mode = "auto"
-testpaths = ["tests/ebook_search"]

python/ebook_search/embeddings.py

@@ -72,14 +72,24 @@ async def embed_texts(
     config: EbookSearchConfig,
 ) -> list[list[float]]:
     """Embed text with the configured vLLM embedding model."""
-    logger.info(f"ebook_embed_request_start {config.embedding_base_url=} {config.embedding_model=} count={len(texts)}")
+    logger.info(
+        "ebook_embed_request_start base_url=%s model=%s count=%s",
+        config.embedding_base_url,
+        config.embedding_model,
+        len(texts),
+    )
     vectors = await request_embeddings(client, texts, config)
     expected_dimension = MODEL_DIMENSIONS[config.embedding_model]
     for vector in vectors:
         if len(vector) != expected_dimension:
             msg = f"Expected {expected_dimension} dimensions, got {len(vector)}"
             raise ValueError(msg)
-    logger.info(f"ebook_embed_request_complete {config.embedding_model=} count={len(vectors)} {expected_dimension=}")
+    logger.info(
+        "ebook_embed_request_complete model=%s count=%s dimension=%s",
+        config.embedding_model,
+        len(vectors),
+        expected_dimension,
+    )
     return vectors
 
 
@@ -95,7 +105,7 @@ async def ensure_embedding_models(session: AsyncSession) -> None:
         existing = await session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == name))
         if existing is None:
             session.add(EbookEmbeddingModel(name=name, dimension=dimension, is_default=name == "qwen3-embedding-0.6b"))
-            logger.info(f"ebook_embedding_model_created {name=} {dimension=}")
+            logger.info("ebook_embedding_model_created model=%s dimension=%s", name, dimension)
     await session.flush()
 
 
@@ -149,10 +159,10 @@ async def embed_missing_chunks(session: AsyncSession, client: httpx.AsyncClient,
         )
     )
     if not chunks:
-        logger.info(f"ebook_embed_missing_none {config.embedding_model=}")
+        logger.info("ebook_embed_missing_none model=%s", config.embedding_model)
         return 0
 
-    logger.info(f"ebook_embed_missing_batch_start {config.embedding_model=} count={len(chunks)}")
+    logger.info("ebook_embed_missing_batch_start model=%s count=%s", config.embedding_model, len(chunks))
     vectors = await embed_texts(client, [chunk.text for chunk in chunks], config)
     rows = [
         {"chunk_id": chunk.id, "model_id": model.id, "embedding": vector}
@@ -161,5 +171,5 @@ async def embed_missing_chunks(session: AsyncSession, client: httpx.AsyncClient,
     statement = insert(table).values(rows).on_conflict_do_nothing(index_elements=["chunk_id", "model_id"])
     await session.execute(statement)
     await session.flush()
-    logger.info(f"ebook_embed_missing_batch_complete {config.embedding_model=} count={len(rows)}")
+    logger.info("ebook_embed_missing_batch_complete model=%s count=%s", config.embedding_model, len(rows))
     return len(rows)

python/ebook_search/ingest.py

@@ -94,13 +94,13 @@ async def ingest_configured_paths(session: AsyncSession, config: EbookSearchConf
     count = 0
     for library_path in config.library_paths:
         path, epub_paths = await asyncio.to_thread(find_library_epubs, library_path)
-        logger.info(f"ebook_ingest_path_start {path=}")
+        logger.info("ebook_ingest_path_start path=%s", path)
         if epub_paths is None:
-            logger.warning(f"ebook_ingest_path_missing {path=}")
+            logger.warning("ebook_ingest_path_missing path=%s", path)
             continue
         for epub_path in epub_paths:
             count += int(await ingest_file(session, epub_path, config))
-    logger.info(f"ebook_ingest_paths_complete {count=} configured_paths={len(config.library_paths)}")
+    logger.info("ebook_ingest_paths_complete changed_files=%s configured_paths=%s", count, len(config.library_paths))
     return count
 
 
@@ -113,7 +113,7 @@ async def ingest_file(session: AsyncSession, path: Path, config: EbookSearchConf
     """Ingest one EPUB file. Return True when the database changed."""
     try:
         resolved_path = await asyncio.to_thread(resolve_ingest_path, path)
-        logger.info(f"ebook_ingest_file_start {resolved_path=}")
+        logger.info("ebook_ingest_file_start path=%s", resolved_path)
         file_hash = await asyncio.to_thread(sha256_file, resolved_path)
         existing = await find_existing_source(session, resolved_path, file_hash)
         if existing is not None and existing.file_sha256 == file_hash:
@@ -122,10 +122,10 @@ async def ingest_file(session: AsyncSession, path: Path, config: EbookSearchConf
             existing.file_mtime = datetime.fromtimestamp(stat.st_mtime, tz=UTC)
             existing.file_size = stat.st_size
             await session.flush()
-            logger.info(f"ebook_ingest_file_unchanged {existing.id=} {resolved_path=}")
+            logger.info("ebook_ingest_file_unchanged source_id=%s path=%s", existing.id, resolved_path)
             return False
         if existing is not None:
-            logger.info(f"ebook_ingest_file_replacing {existing.id=} {resolved_path=}")
+            logger.info("ebook_ingest_file_replacing source_id=%s path=%s", existing.id, resolved_path)
             await session.delete(existing)
             await session.flush()
 
@@ -160,11 +160,15 @@ async def ingest_file(session: AsyncSession, path: Path, config: EbookSearchConf
         await session.commit()
         mention_count = await index_chunk_phrase_mentions_for_book(session, source.id, config)
         logger.info(
-            f"ebook_ingest_file_complete {source.id=} {resolved_path=} chapters={len(parsed.chapters)} {chunk_index=} "
-            f"{mention_count=}"
+            "ebook_ingest_file_complete source_id=%s path=%s chapters=%s chunks=%s phrase_mentions=%s",
+            source.id,
+            resolved_path,
+            len(parsed.chapters),
+            chunk_index,
+            mention_count,
         )
     except Exception:
-        logger.exception(f"ebook_ingest_file_error {path=}")
+        logger.exception(f"ebook_ingest_file_error path={path}")
         return False
     else:
         return True

python/ebook_search/llm_interface.py

@@ -51,7 +51,10 @@ async def request_embeddings(
         return embedding_vectors_from_response(response.json())
     except (httpx.HTTPError, ValueError, KeyError, TypeError) as error:
         logger.exception(
-            f"ebook_embed_request_failed {config.embedding_base_url=} {config.embedding_model=} count={len(texts)}"
+            "ebook_embed_request_failed base_url=%s model=%s count=%s",
+            config.embedding_base_url,
+            config.embedding_model,
+            len(texts),
         )
         msg = f"Embedding request failed. base_url={config.embedding_base_url} model={config.embedding_model}"
         raise RuntimeError(msg) from error
@@ -64,13 +67,17 @@ async def check_embedding_endpoint(
     timeout_seconds: float = 5.0,
 ) -> bool:
     """Return whether the configured embedding endpoint answers a model listing."""
-    return await _check_endpoint(
-        client,
-        base_url=config.embedding_base_url,
-        api_key=config.embedding_api_key,
-        timeout_seconds=timeout_seconds,
-        unavailable_log=f"ebook_embedding_endpoint_unreachable {config.embedding_base_url=}",
-    )
+    try:
+        response = await client.get(
+            f"{config.embedding_base_url.rstrip('/')}/models",
+            headers=auth_headers(config.embedding_api_key),
+            timeout=timeout_seconds,
+        )
+        response.raise_for_status()
+    except httpx.HTTPError as error:
+        logger.warning("ebook_embedding_endpoint_unreachable base_url=%s error=%s", config.embedding_base_url, error)
+        return False
+    return True
 
 
 async def check_chat_endpoint(
@@ -80,33 +87,15 @@ async def check_chat_endpoint(
     timeout_seconds: float = 5.0,
 ) -> bool:
     """Return whether the configured chat (answering) endpoint answers a model listing."""
-    return await _check_endpoint(
-        client,
-        base_url=config.vllm_base_url,
-        api_key=config.vllm_api_key,
-        timeout_seconds=timeout_seconds,
-        unavailable_log=f"ebook_chat_endpoint_unreachable {config.vllm_base_url=}",
-    )
-
-
-async def _check_endpoint(
-    client: httpx.AsyncClient,
-    *,
-    base_url: str,
-    api_key: str,
-    timeout_seconds: float,
-    unavailable_log: str,
-) -> bool:
-    """Return whether an OpenAI-compatible endpoint answers a model listing."""
     try:
         response = await client.get(
-            f"{base_url.rstrip('/')}/models",
-            headers=auth_headers(api_key),
+            f"{config.vllm_base_url.rstrip('/')}/models",
+            headers=auth_headers(config.vllm_api_key),
             timeout=timeout_seconds,
         )
         response.raise_for_status()
     except httpx.HTTPError as error:
-        logger.warning(f"{unavailable_log} {error=}")
+        logger.warning("ebook_chat_endpoint_unreachable base_url=%s error=%s", config.vllm_base_url, error)
         return False
     return True
 

python/ebook_search/loadtest.py

@@ -112,7 +112,7 @@ async def send_search(client: httpx.AsyncClient, query: str, *, rerank: bool) ->
     try:
         response = await client.post("/search", data=data)
     except httpx.HTTPError as error:
-        logger.warning(f"ebook_loadtest_request_failed {error=}")
+        logger.warning("ebook_loadtest_request_failed error=%s", error)
         return RequestResult(status_code=0, latency_ms=(time.perf_counter() - start) * 1000, ok=False)
     return RequestResult(
         status_code=response.status_code,
@@ -192,7 +192,14 @@ def main(
     """Load test the search endpoint and report latency and throughput."""
     configure_logger(log_level)
     queries = load_queries(queries_file)
-    logger.info(f"ebook_loadtest_start {base_url=} {request_count=} {concurrency=} {rerank=} queries={len(queries)}")
+    logger.info(
+        "ebook_loadtest_start base_url=%s requests=%s concurrency=%s rerank=%s queries=%s",
+        base_url,
+        request_count,
+        concurrency,
+        rerank,
+        len(queries),
+    )
     summary = asyncio.run(
         run_load(
             base_url=base_url,

python/ebook_search/protected_phrases/__init__.py

@@ -1 +1 @@
-"""Protected phrase extraction and matching for ebook search."""
+"""Init."""

python/ebook_search/protected_phrases/extraction.py

@@ -36,6 +36,32 @@ MULTI_SOURCE_MIN_SOURCES = 2
 CAPITALIZED_PHRASE_RE = re.compile(r"\b(?:[A-Z][a-zA-Z']+)(?:\s+(?:of|the|and|in|on|for|[A-Z][a-zA-Z']+)){0,6}")
 
 
+class SpacySpan(Protocol):
+    """Small protocol for the spaCy span attributes used by this module."""
+
+    text: str
+
+
+class SpacyEntity(SpacySpan, Protocol):
+    """Small protocol for the spaCy entity attributes used by this module."""
+
+    label_: str
+
+
+class SpacyDoc(Protocol):
+    """Small protocol for the spaCy doc attributes used by this module."""
+
+    ents: Iterable[SpacyEntity]
+    noun_chunks: Iterable[SpacySpan]
+
+
+class SpacyLanguage(Protocol):
+    """Small protocol for a callable spaCy language pipeline."""
+
+    def __call__(self, text: str) -> SpacyDoc:
+        """Parse text into a spaCy-like doc."""
+
+
 class YakeExtractor(Protocol):
     """Small protocol for the YAKE extractor used by this module."""
 
@@ -64,30 +90,40 @@ def normalize_candidate_phrase(
     phrase_text: str,
     config: EbookSearchConfig,
     *,
+    min_tokens: int | None = None,
     max_tokens: int | None = None,
+    strip_leading_article: bool = False,
 ) -> tuple[str, str, int] | None:
     """Normalize a candidate phrase and validate token bounds.
 
     Args:
         phrase_text (str): Raw phrase text to normalize.
         config (EbookSearchConfig): Runtime phrase-tuning settings.
+        min_tokens (int | None): Minimum token count override; defaults to ``config.phrase_min_tokens``.
         max_tokens (int | None): Maximum token count override; defaults to ``config.phrase_max_tokens``.
+        strip_leading_article (bool): Whether to drop a single leading English article.
 
     Returns:
         tuple[str, str, int] | None: Display text, normalized phrase, and token count, or ``None``
         when the phrase falls outside the token bounds or is ignored.
     """
     normalized_tokens = tokenize_with_offsets(phrase_text)
+    start = 0
+    if strip_leading_article and normalized_tokens and normalized_tokens[0].text in {"the", "a", "an"}:
+        start = 1
+
+    selected_tokens = normalized_tokens[start:]
+    min_count = config.phrase_min_tokens if min_tokens is None else min_tokens
     max_count = config.phrase_max_tokens if max_tokens is None else max_tokens
-    if len(normalized_tokens) < config.phrase_min_tokens or len(normalized_tokens) > max_count:
+    if len(selected_tokens) < min_count or len(selected_tokens) > max_count:
         return None
 
-    phrase_norm = " ".join(token.text for token in normalized_tokens)
+    phrase_norm = " ".join(token.text for token in selected_tokens)
     if phrase_norm in get_ignored_phrases():
         return None
 
-    display_text = phrase_text[normalized_tokens[0].start_char : normalized_tokens[-1].end_char].strip()
-    return display_text or phrase_norm, phrase_norm, len(normalized_tokens)
+    display_text = phrase_text[selected_tokens[0].start_char : selected_tokens[-1].end_char].strip()
+    return display_text or phrase_norm, phrase_norm, len(selected_tokens)
 
 
 def count_raw_ngrams(tokens: Sequence[str], config: EbookSearchConfig) -> Counter[str]:
@@ -206,6 +242,55 @@ def extract_yake_candidates(
     return out
 
 
+def extract_spacy_candidates(
+    book_text: str,
+    nlp: SpacyLanguage,
+    config: EbookSearchConfig,
+) -> dict[str, PhraseCandidate]:
+    """Extract spaCy named entities and noun chunks from one text block.
+
+    Args:
+        book_text (str): Text block to parse with spaCy.
+        nlp (SpacyLanguage): Callable spaCy language pipeline.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        dict[str, PhraseCandidate]: Candidates keyed by normalized phrase from entities and noun chunks.
+    """
+    out: dict[str, PhraseCandidate] = {}
+    doc = nlp(book_text)
+
+    for ent in doc.ents:
+        normalized = normalize_candidate_phrase(
+            ent.text,
+            config,
+            max_tokens=config.phrase_max_entity_tokens,
+        )
+        if normalized is None:
+            continue
+        phrase_text, phrase_norm, token_count = normalized
+        out[phrase_norm] = PhraseCandidate(
+            phrase_text=phrase_text,
+            phrase_norm=phrase_norm,
+            token_count=token_count,
+            source_spacy_ner=True,
+            spacy_label=ent.label_,
+        )
+
+    for chunk in doc.noun_chunks:
+        normalized = normalize_candidate_phrase(chunk.text, config, strip_leading_article=True)
+        if normalized is None:
+            continue
+        phrase_text, phrase_norm, token_count = normalized
+        out[phrase_norm] = PhraseCandidate(
+            phrase_text=phrase_text,
+            phrase_norm=phrase_norm,
+            token_count=token_count,
+            source_spacy_noun_chunk=True,
+        )
+    return out
+
+
 def extract_capitalized_phrases(original_text: str, config: EbookSearchConfig) -> dict[str, PhraseCandidate]:
     """Extract capitalized phrase runs that often carry fictional terms.
 
@@ -307,12 +392,16 @@ def merge_candidate(existing: PhraseCandidate, item: PhraseCandidate) -> None:
     """
     existing.source_raw_ngram = existing.source_raw_ngram or item.source_raw_ngram
     existing.source_yake = existing.source_yake or item.source_yake
+    existing.source_spacy_ner = existing.source_spacy_ner or item.source_spacy_ner
+    existing.source_spacy_noun_chunk = existing.source_spacy_noun_chunk or item.source_spacy_noun_chunk
     existing.source_capitalized = existing.source_capitalized or item.source_capitalized
     existing.source_metadata = existing.source_metadata or item.source_metadata
     existing.raw_count += item.raw_count
     existing.chapter_count = max(existing.chapter_count, item.chapter_count)
     if item.yake_score is not None:
         existing.yake_score = item.yake_score
+    if item.spacy_label:
+        existing.spacy_label = item.spacy_label
 
 
 def enrich_with_frequency_and_chapter_counts(
@@ -510,6 +599,8 @@ def non_raw_source_count(candidate: PhraseCandidate) -> int:
     return sum(
         (
             candidate.source_yake,
+            candidate.source_spacy_ner,
+            candidate.source_spacy_noun_chunk,
             candidate.source_capitalized,
             candidate.source_metadata,
         )
@@ -555,6 +646,8 @@ def source_score(candidate: PhraseCandidate) -> float:
         weight
         for enabled, weight in (
             (candidate.source_yake, 2.0),
+            (candidate.source_spacy_ner, 2.5),
+            (candidate.source_spacy_noun_chunk, 1.5),
             (candidate.source_capitalized, 2.0),
             (candidate.source_metadata, 2.0),
             (candidate.source_raw_ngram, 0.5),
@@ -645,6 +738,10 @@ def candidate_source_names(candidate: PhraseCandidate) -> list[str]:
         names.append("raw_ngram")
     if candidate.source_yake:
         names.append("yake")
+    if candidate.source_spacy_ner:
+        names.append("spacy_ner")
+    if candidate.source_spacy_noun_chunk:
+        names.append("spacy_noun_chunk")
     if candidate.source_capitalized:
         names.append("capitalized")
     if candidate.source_metadata:
@@ -657,14 +754,16 @@ def extract_phrase_candidates_for_book(
     chapters: Sequence[str],
     config: EbookSearchConfig,
     *,
+    nlp: SpacyLanguage | None = None,
     metadata: Mapping[str, object] | None = None,
 ) -> list[PhraseCandidate]:
     """Extract, score, and limit phrase candidates for one book.
 
     Args:
         book_text (str): Full book text used for most extraction sources.
-        chapters (Sequence[str]): Chapter-like text blocks used for frequency counts.
+        chapters (Sequence[str]): Chapter-like text blocks used for spaCy and frequency counts.
         config (EbookSearchConfig): Runtime phrase-tuning settings.
+        nlp (SpacyLanguage | None): Optional spaCy pipeline for entity and noun-chunk sources.
         metadata (Mapping[str, object] | None): Optional book metadata used as a candidate source.
 
     Returns:
@@ -672,30 +771,47 @@ def extract_phrase_candidates_for_book(
     """
     started_at = perf_counter()
     logger.info(
-        f"ebook_phrase_candidate_extract_start chapters={len(chapters)} chars={len(book_text)} "
-        f"{config.phrase_min_tokens=} {config.phrase_max_tokens=} {config.protected_phrase_max_candidates_per_book=}"
+        "ebook_phrase_candidate_extract_start chapters=%s chars=%s min_tokens=%s max_tokens=%s max_candidates=%s",
+        len(chapters),
+        len(book_text),
+        config.phrase_min_tokens,
+        config.phrase_max_tokens,
+        config.protected_phrase_max_candidates_per_book,
     )
     raw_started_at = perf_counter()
     raw = extract_raw_ngrams_by_chapter(chapters, config)
     logger.info(
-        f"ebook_phrase_candidate_extract_raw_complete candidates={len(raw)} "
-        f"duration_ms={(perf_counter() - raw_started_at) * 1000:.1f}"
+        "ebook_phrase_candidate_extract_raw_complete candidates=%s duration_ms=%.1f",
+        len(raw),
+        (perf_counter() - raw_started_at) * 1000,
     )
     yake_started_at = perf_counter()
     yake_candidates = extract_yake_candidates(book_text, config)
     logger.info(
-        f"ebook_phrase_candidate_extract_yake_complete candidates={len(yake_candidates)} "
-        f"duration_ms={(perf_counter() - yake_started_at) * 1000:.1f}"
+        "ebook_phrase_candidate_extract_yake_complete candidates=%s duration_ms=%.1f",
+        len(yake_candidates),
+        (perf_counter() - yake_started_at) * 1000,
     )
+    spacy_candidates: dict[str, PhraseCandidate] = {}
+    if nlp is not None:
+        spacy_started_at = perf_counter()
+        for chapter in chapters:
+            spacy_candidates = merge_candidate_sources(spacy_candidates, extract_spacy_candidates(chapter, nlp, config))
+        logger.info(
+            "ebook_phrase_candidate_extract_spacy_complete candidates=%s duration_ms=%.1f",
+            len(spacy_candidates),
+            (perf_counter() - spacy_started_at) * 1000,
+        )
     capitalized_started_at = perf_counter()
     capitalized = extract_capitalized_phrases(book_text, config)
     logger.info(
-        f"ebook_phrase_candidate_extract_capitalized_complete candidates={len(capitalized)} "
-        f"duration_ms={(perf_counter() - capitalized_started_at) * 1000:.1f}"
+        "ebook_phrase_candidate_extract_capitalized_complete candidates=%s duration_ms=%.1f",
+        len(capitalized),
+        (perf_counter() - capitalized_started_at) * 1000,
     )
     metadata_candidates = extract_metadata_candidates(metadata, config)
 
-    candidates = merge_candidate_sources(raw, yake_candidates, capitalized, metadata_candidates)
+    candidates = merge_candidate_sources(raw, yake_candidates, spacy_candidates, capitalized, metadata_candidates)
     enriched_started_at = perf_counter()
     # Raw n-gram sizes were already counted per chapter above, so only enrich the remaining
     # (entity-length) sizes here instead of re-sliding every size over the whole book.
@@ -715,11 +831,23 @@ def extract_phrase_candidates_for_book(
         : config.protected_phrase_max_candidates_per_book
     ]
     logger.info(
-        f"ebook_phrase_candidate_extract_complete raw={len(raw)} yake={len(yake_candidates)} "
-        f"capitalized={len(capitalized)} metadata={len(metadata_candidates)} {pre_filter_count=} {filtered_too_short=} "
-        f"{filtered_too_rare=} {filtered_too_common=} {filtered_junk=} min_uses={minimum_candidate_raw_count(config)} "
-        f"storable={len(candidates)} limited={len(limited)} "
-        f"enrich_score_ms={(perf_counter() - enriched_started_at) * 1000:.1f} "
-        f"duration_ms={(perf_counter() - started_at) * 1000:.1f}"
+        "ebook_phrase_candidate_extract_complete raw=%s yake=%s spacy=%s capitalized=%s metadata=%s "
+        "merged=%s filtered_too_short=%s filtered_too_rare=%s filtered_too_common=%s filtered_junk=%s "
+        "min_uses=%s storable=%s limited=%s enrich_score_ms=%.1f duration_ms=%.1f",
+        len(raw),
+        len(yake_candidates),
+        len(spacy_candidates),
+        len(capitalized),
+        len(metadata_candidates),
+        pre_filter_count,
+        filtered_too_short,
+        filtered_too_rare,
+        filtered_too_common,
+        filtered_junk,
+        minimum_candidate_raw_count(config),
+        len(candidates),
+        len(limited),
+        (perf_counter() - enriched_started_at) * 1000,
+        (perf_counter() - started_at) * 1000,
     )
     return limited

python/ebook_search/protected_phrases/generate_ngrams.py

@@ -4,11 +4,11 @@ from __future__ import annotations
 
 import asyncio
 import logging
+from collections import deque
 from time import perf_counter
 from typing import TYPE_CHECKING
 
 from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession
 
 from python.ebook_search.protected_phrases.extraction import extract_phrase_candidates_for_book
 from python.ebook_search.protected_phrases.models import (
@@ -16,81 +16,65 @@ from python.ebook_search.protected_phrases.models import (
     PhraseCandidateGenerationResult,
     PhraseRecalculationResult,
 )
-from python.ebook_search.protected_phrases.pool import get_extraction_pool
+from python.ebook_search.protected_phrases.pool import extract_phrase_candidates_in_pool, get_extraction_pool
 from python.ebook_search.protected_phrases.store import (
     bulk_upsert_unjudged_candidates,
     delete_phrase_data_for_book,
     load_book_chapter_texts,
-    metadata_for_source_id,
+    metadata_for_source,
     new_candidate_row,
     prune_unstorable_unjudged_candidate_phrases,
 )
-from python.orm.common import get_async_postgres_engine
-from python.orm.richie import EbookSource
+from python.orm.richie import EbookCandidatePhrase, EbookSource
 
 if TYPE_CHECKING:
-    from sqlalchemy.ext.asyncio import AsyncEngine
+    from collections.abc import Mapping, Sequence
+    from concurrent.futures import Future
+
+    from sqlalchemy.ext.asyncio import AsyncSession
 
     from python.ebook_search.config import EbookSearchConfig
+    from python.ebook_search.protected_phrases.extraction import SpacyLanguage
     from python.ebook_search.protected_phrases.models import PhraseCandidate
 
 logger = logging.getLogger(__name__)
 
 
-class BookHasNoChaptersError(ValueError):
-    """Raised when a book has no indexed chapter text to generate phrases from."""
-
-
 async def generate_candidate_phrases_for_books(
-    engine: AsyncEngine,
+    session: AsyncSession,
     config: EbookSearchConfig,
+    *,
+    only_missing: bool = False,
 ) -> PhraseCandidateGenerationResult:
     """Create or refresh candidate phrases for indexed books without calling the LLM judge.
 
-    Every book is submitted to the shared process pool up front and runs in parallel across the
-    pool's workers; the call blocks until all books have finished. Each worker opens its own
-    database engine from environment variables, loads the book's chapters, and commits the
-    book's candidates independently.
+    Extraction always runs concurrently in the shared process pool so a full backfill uses
+    multiple cores.
 
     Args:
-        engine (AsyncEngine): Engine used to read the book list in this process.
+        session (Session): Active database session.
         config (EbookSearchConfig): Runtime phrase-tuning settings.
+        only_missing (bool): When True, only generate for books that have no candidate phrases
+            yet instead of refreshing every book.
 
     Returns:
         PhraseCandidateGenerationResult: Per-corpus counts of books seen, built, and candidates stored.
-
-    Results are collected in book order while the pool keeps working. A book failure (including
-    a book with no indexed chapters) is logged and counted as not built; the remaining books
-    are unaffected.
     """
-    async with AsyncSession(engine, expire_on_commit=False) as session:
-        source_query = select(EbookSource.id).order_by(EbookSource.id)
-        source_ids = (await session.scalars(source_query)).all()
-    books_seen = len(source_ids)
+    source_query = select(EbookSource).order_by(EbookSource.id)
+    if only_missing:
+        has_candidates = select(EbookCandidatePhrase.id).where(EbookCandidatePhrase.book_id == EbookSource.id)
+        source_query = source_query.where(~has_candidates.exists())
+    sources = (await session.scalars(source_query)).all()
+    books_seen = len(sources)
     logger.info(
-        f"ebook_candidate_phrase_generation_start {books_seen=} {config.phrase_min_tokens=} "
-        f"{config.phrase_max_tokens=} {config.protected_phrase_max_candidates_per_book=}"
+        "ebook_candidate_phrase_generation_start books_seen=%s min_tokens=%s max_tokens=%s max_candidates_per_book=%s",
+        books_seen,
+        config.phrase_min_tokens,
+        config.phrase_max_tokens,
+        config.protected_phrase_max_candidates_per_book,
     )
 
-    pool = get_extraction_pool(config.protected_phrase_extraction_workers)
-    wrapped_futures = [
-        (
-            source_id,
-            asyncio.wrap_future(pool.submit(generate_candidate_phrases_for_book_in_worker, source_id, None, config)),
-        )
-        for source_id in source_ids
-    ]
-    outcomes: list[BookCandidateResult] = []
-    for source_id, wrapped_future in wrapped_futures:
-        await asyncio.wait([wrapped_future])
-        exception = wrapped_future.exception()
-        if exception is not None:
-            logger.error(f"ebook_candidate_phrase_generation_book_failed {source_id=}")
-            outcomes.append(BookCandidateResult())
-            continue
-        saved_count = wrapped_future.result()
-        logger.info(f"ebook_candidate_phrase_generation_book_committed {source_id=} {saved_count=}")
-        outcomes.append(BookCandidateResult(candidates=saved_count, built=True))
+    outcomes = await generate_candidates_for_sources_pooled(session, sources, config)
 
     result = PhraseCandidateGenerationResult(
         books_seen=books_seen,
@@ -98,44 +82,159 @@ async def generate_candidate_phrases_for_books(
         candidate_phrases=sum(outcome.candidates for outcome in outcomes),
     )
     logger.info(
-        f"ebook_candidate_phrase_generation_complete {result.books_seen=} {result.books_built=} "
-        f"{result.candidate_phrases=}"
+        "ebook_candidate_phrase_generation_complete books_seen=%s books_built=%s candidate_total=%s",
+        result.books_seen,
+        result.books_built,
+        result.candidate_phrases,
     )
     return result
 
 
+async def generate_candidates_for_sources_pooled(
+    session: AsyncSession,
+    sources: Sequence[EbookSource],
+    config: EbookSearchConfig,
+) -> list[BookCandidateResult]:
+    """Generate candidate phrases for many books, extracting them concurrently in worker processes.
+
+    Chapter loading and row persistence stay on the caller's session (serial), while the CPU-bound
+    extraction runs in the shared process pool. A bounded window of in-flight books overlaps
+    extraction across cores without loading every book's candidates into memory at once.
+
+    Args:
+        session (Session): Active database session.
+        sources (Sequence[EbookSource]): Indexed books to generate candidates for.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        list[BookCandidateResult]: One result per book.
+    """
+    pool = get_extraction_pool(config.protected_phrase_extraction_workers)
+    max_in_flight = max(1, config.protected_phrase_extraction_workers) * 2
+    pending: deque[tuple[EbookSource, Future[list[PhraseCandidate]]]] = deque()
+    outcomes: list[BookCandidateResult] = []
+
+    async def drain_one() -> None:
+        source, future = pending.popleft()
+        extracted = await asyncio.wrap_future(future)
+        outcomes.append(await store_source_candidates(session, source, extracted, config))
+
+    try:
+        for source in sources:
+            chapters = await load_book_chapter_texts(session, source.id)
+            if not chapters:
+                logger.warning("ebook_candidate_phrase_generation_book_empty source_id=%s", source.id)
+                outcomes.append(BookCandidateResult())
+                continue
+            future = pool.submit(
+                extract_phrase_candidates_for_book,
+                "\n\n".join(chapters),
+                chapters,
+                config,
+                metadata=metadata_for_source(source),
+            )
+            pending.append((source, future))
+            if len(pending) >= max_in_flight:
+                await drain_one()
+        while pending:
+            await drain_one()
+    except Exception:
+        for _, future in pending:
+            future.cancel()
+        await session.rollback()
+        logger.exception("ebook_candidate_phrase_generation_pooled_failed")
+        raise
+    return outcomes
+
+
+async def store_source_candidates(
+    session: AsyncSession,
+    source: EbookSource,
+    limited_candidates: list[PhraseCandidate],
+    config: EbookSearchConfig,
+) -> BookCandidateResult:
+    """Persist and commit one book's already-extracted candidates.
+
+    Args:
+        session (AsyncSession): Active database session.
+        source (EbookSource): Book the candidates belong to.
+        limited_candidates (list[PhraseCandidate]): Scored candidates to persist.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+
+    Returns:
+        BookCandidateResult: Candidate count and that the book was committed.
+    """
+    book_started_at = perf_counter()
+    saved_count = await store_candidate_phrases_for_book(session, source.id, None, limited_candidates, config)
+    await session.commit()
+    logger.info(
+        "ebook_candidate_phrase_generation_book_committed source_id=%s candidates=%s duration_ms=%.1f",
+        source.id,
+        saved_count,
+        (perf_counter() - book_started_at) * 1000,
+    )
+    return BookCandidateResult(candidates=saved_count, built=True)
+
+
 async def recalculate_candidate_phrases_for_book(
     session: AsyncSession,
     source: EbookSource,
     config: EbookSearchConfig,
+    *,
+    nlp: SpacyLanguage | None = None,
+    use_process_pool: bool = False,
 ) -> PhraseRecalculationResult:
     """Remove all book phrase data, regenerate candidates, and commit the completed book.
 
     Args:
-        session (AsyncSession): Active database session; deletion and regeneration commit on it.
+        session (Session): Active database session.
         source (EbookSource): Indexed book to recalculate.
         config (EbookSearchConfig): Runtime phrase-tuning settings.
+        nlp (SpacyLanguage | None): Optional spaCy pipeline for entity and noun-chunk sources.
+        use_process_pool (bool): Run the CPU-bound extraction in a worker process so concurrent
+            recalculations do not serialize behind the GIL. Defaults to in-process for callers
+            (tests, backfills) that do not need it.
 
     Returns:
         PhraseRecalculationResult: Deleted-row counts and the number of candidates regenerated.
-
-    Raises:
-        BookHasNoChaptersError: If the book has no indexed chapters. The deletion is rolled
-            back, so the book's existing phrases stay intact.
-
-    The deletion and regeneration share the caller's session, so they commit together; a
-    regeneration failure rolls the deletion back.
     """
     started_at = perf_counter()
-    logger.info(f"ebook_candidate_phrase_recalculation_start {source.id=} {source.title=}")
-    deleted = await delete_phrase_data_for_book(session, source.id)
-    candidate_count = await generate_candidate_phrases_for_book(
-        session,
+    logger.info(
+        "ebook_candidate_phrase_recalculation_start source_id=%s title=%r",
         source.id,
-        series_id=None,
-        config=config,
-        replace_all=True,
+        source.title,
     )
+    try:
+        deleted = await delete_phrase_data_for_book(session, source.id)
+        chapters = await load_book_chapter_texts(session, source.id)
+        if not chapters:
+            logger.warning("ebook_candidate_phrase_recalculation_book_empty source_id=%s", source.id)
+            await session.commit()
+            return PhraseRecalculationResult(
+                book_id=source.id,
+                deleted_candidates=deleted.deleted_candidates,
+                deleted_protected_phrases=deleted.deleted_protected_phrases,
+                deleted_aliases=deleted.deleted_aliases,
+                deleted_mentions=deleted.deleted_mentions,
+                candidate_phrases=0,
+            )
+
+        candidate_count = await generate_candidate_phrases_for_book(
+            session,
+            source.id,
+            series_id=None,
+            chapters=chapters,
+            config=config,
+            nlp=nlp,
+            metadata=metadata_for_source(source),
+            replace_all=True,
+            use_process_pool=use_process_pool,
+        )
+        await session.commit()
+    except Exception:
+        await session.rollback()
+        logger.exception("ebook_candidate_phrase_recalculation_failed source_id=%s", source.id)
+        raise
 
     result = PhraseRecalculationResult(
         book_id=source.id,
@@ -146,106 +245,75 @@ async def recalculate_candidate_phrases_for_book(
         candidate_phrases=candidate_count,
     )
     logger.info(
-        f"ebook_candidate_phrase_recalculation_complete {source.id=} {result.deleted_candidates=} "
-        f"{result.deleted_protected_phrases=} {result.deleted_aliases=} {result.deleted_mentions=} "
-        f"{result.candidate_phrases=} duration_ms={(perf_counter() - started_at) * 1000:.1f}"
+        "ebook_candidate_phrase_recalculation_complete source_id=%s deleted_candidates=%s "
+        "deleted_protected=%s deleted_aliases=%s deleted_mentions=%s candidates=%s duration_ms=%.1f",
+        source.id,
+        result.deleted_candidates,
+        result.deleted_protected_phrases,
+        result.deleted_aliases,
+        result.deleted_mentions,
+        result.candidate_phrases,
+        (perf_counter() - started_at) * 1000,
     )
     return result
 
 
-def generate_candidate_phrases_for_book_in_worker(
-    book_id: int,
-    series_id: int | None,
-    config: EbookSearchConfig,
-) -> int:
-    """Run one book's candidate generation in a pooled worker process.
-
-    The worker has no engine or session to inherit (neither can cross process boundaries), so
-    it creates its own engine from environment variables, opens the book's session on it, and
-    disposes the engine once the book is stored.
-
-    Args:
-        book_id (int): Book the candidates belong to.
-        series_id (int | None): Series scope for the stored candidates.
-        config (EbookSearchConfig): Runtime phrase-tuning settings.
-
-    Returns:
-        int: Number of candidate phrase rows stored.
-    """
-
-    async def generate_with_worker_engine() -> int:
-        engine = get_async_postgres_engine(name="RICHIE", vector_engine=True, pool_size=1)
-        try:
-            async with AsyncSession(engine, expire_on_commit=False) as session:
-                return await generate_candidate_phrases_for_book(
-                    session,
-                    book_id,
-                    series_id,
-                    config,
-                )
-        finally:
-            await engine.dispose()
-
-    return asyncio.run(generate_with_worker_engine())
-
-
 async def generate_candidate_phrases_for_book(
     session: AsyncSession,
     book_id: int,
     series_id: int | None,
+    chapters: Sequence[str],
     config: EbookSearchConfig,
     *,
+    nlp: SpacyLanguage | None = None,
+    metadata: Mapping[str, object] | None = None,
     replace_all: bool = False,
+    use_process_pool: bool = False,
 ) -> int:
-    """Load a book's chapters and metadata, extract candidate phrases, and store them without LLM judging.
-
-    The session commits only when the whole book succeeds; any failure rolls the session back,
-    which also restores rows the caller deleted in the same transaction (e.g. a recalculation).
+    """Extract and store candidate phrases for one book without LLM judging.
 
     Args:
-        session (AsyncSession): Active database session; committed on success, rolled back on failure.
+        session (Session): Active database session.
         book_id (int): Book the candidates belong to.
         series_id (int | None): Series scope for the stored candidates.
+        chapters (Sequence[str]): Chapter-like text blocks used for extraction and frequency counts.
         config (EbookSearchConfig): Runtime phrase-tuning settings.
+        nlp (SpacyLanguage | None): Optional spaCy pipeline for entity and noun-chunk sources.
+        metadata (Mapping[str, object] | None): Optional book metadata used as a candidate source.
         replace_all (bool): When the caller has already cleared this book's candidates (e.g. a
             recalculation), skip the per-candidate existence lookup and bulk-insert new rows.
+        use_process_pool (bool): Run the CPU-bound extraction in a worker process to avoid
+            serializing concurrent requests behind the GIL. Ignored when ``nlp`` is set, since
+            the spaCy pipeline cannot be sent to a worker process.
 
     Returns:
         int: Number of candidate phrase rows stored.
-
-    Raises:
-        BookHasNoChaptersError: If the book has no indexed chapter text.
     """
     started_at = perf_counter()
-    chapters = await load_book_chapter_texts(session, book_id)
-    if not chapters:
-        await session.rollback()
-        message = f"book {book_id} has no indexed chapters"
-        raise BookHasNoChaptersError(message)
-    metadata = await metadata_for_source_id(session, book_id)
-    try:
-        book_text = "\n\n".join(chapters)
-        candidates = extract_phrase_candidates_for_book(
+    book_text = "\n\n".join(chapters)
+    if use_process_pool and nlp is None:
+        limited_candidates = await extract_phrase_candidates_in_pool(book_text, chapters, config, metadata=metadata)
+    else:
+        limited_candidates = extract_phrase_candidates_for_book(
             book_text,
             chapters,
             config,
+            nlp=nlp,
             metadata=metadata,
         )
-        saved_count = await store_candidate_phrases_for_book(
-            session,
-            book_id,
-            series_id,
-            candidates,
-            config,
-            replace_all=replace_all,
-        )
-        await session.commit()
-    except Exception:
-        await session.rollback()
-        raise
+    saved_count = await store_candidate_phrases_for_book(
+        session,
+        book_id,
+        series_id,
+        limited_candidates,
+        config,
+        replace_all=replace_all,
+    )
     logger.info(
-        f"ebook_candidate_phrase_generation_book_duration {book_id=} {saved_count=} "
-        f"duration_ms={(perf_counter() - started_at) * 1000:.1f}"
+        "ebook_candidate_phrase_generation_book_duration book_id=%s candidates=%s duration_ms=%.1f",
+        book_id,
+        saved_count,
+        (perf_counter() - started_at) * 1000,
     )
     return saved_count
 
@@ -280,16 +348,23 @@ async def store_candidate_phrases_for_book(
         await session.flush()
         saved_count = len(rows)
         logger.info(
-            f"ebook_candidate_phrase_save_start {book_id=} candidates={len(limited_candidates)} mode=bulk_insert"
+            "ebook_candidate_phrase_save_start book_id=%s candidates=%s mode=bulk_insert",
+            book_id,
+            len(limited_candidates),
         )
     else:
         pruned_count = await prune_unstorable_unjudged_candidate_phrases(session, book_id, config)
         logger.info(
-            f"ebook_candidate_phrase_save_start {book_id=} candidates={len(limited_candidates)} {pruned_count=}"
+            "ebook_candidate_phrase_save_start book_id=%s candidates=%s pruned_unstorable=%s",
+            book_id,
+            len(limited_candidates),
+            pruned_count,
         )
         saved_count = await bulk_upsert_unjudged_candidates(session, book_id, series_id, limited_candidates)
     logger.info(
-        f"ebook_candidate_phrase_save_complete {book_id=} {saved_count=} "
-        f"save_ms={(perf_counter() - save_started_at) * 1000:.1f}"
+        "ebook_candidate_phrase_save_complete book_id=%s candidates=%s save_ms=%.1f",
+        book_id,
+        saved_count,
+        (perf_counter() - save_started_at) * 1000,
     )
     return saved_count

python/ebook_search/protected_phrases/judge_ngrams.py

@@ -80,8 +80,12 @@ async def judge_candidate_phrases_for_books(
     book_workers = max(1, config.phrase_judge_book_workers)
     phrase_workers = max(1, config.phrase_judge_phrase_workers)
     logger.info(
-        f"ebook_candidate_phrase_judgment_start {books_seen=} {book_workers=} {phrase_workers=} "
-        f"{config.protected_phrase_confidence_threshold=:.2f}"
+        "ebook_candidate_phrase_judgment_start books_seen=%s book_workers=%s phrase_workers=%s "
+        "confidence_threshold=%.2f",
+        books_seen,
+        book_workers,
+        phrase_workers,
+        config.protected_phrase_confidence_threshold,
     )
 
     book_semaphore = asyncio.Semaphore(book_workers)
@@ -101,8 +105,14 @@ async def judge_candidate_phrases_for_books(
         phrase_mentions=sum(outcome.mentions for outcome in outcomes),
     )
     logger.info(
-        f"ebook_candidate_phrase_judgment_complete {result.books_seen=} {result.books_judged=} {result.books_failed=} "
-        f"{result.candidates_judged=} {result.protected_phrases=} {result.phrase_mentions=}"
+        "ebook_candidate_phrase_judgment_complete books_seen=%s books_judged=%s books_failed=%s "
+        "candidates_judged=%s protected=%s mentions=%s",
+        result.books_seen,
+        result.books_judged,
+        result.books_failed,
+        result.candidates_judged,
+        result.protected_phrases,
+        result.phrase_mentions,
     )
     return result
 
@@ -137,7 +147,7 @@ async def judge_one_book_async(
                 return BookJudgmentResult()
             return await persist_book_judgments(engine, source_id, config, judged)
         except Exception:
-            logger.exception(f"ebook_candidate_phrase_judgment_book_failed {source_id=}")
+            logger.exception("ebook_candidate_phrase_judgment_book_failed source_id=%s", source_id)
             return BookJudgmentResult(failed=True)
 
 
@@ -163,7 +173,7 @@ async def prepare_book_judgment(
         return None
     async with AsyncSession(engine) as session:
         if not await count_unjudged_candidates(session, source_id, config):
-            logger.info(f"ebook_candidate_phrase_judgment_book_skip_no_unjudged {source_id=}")
+            logger.info("ebook_candidate_phrase_judgment_book_skip_no_unjudged source_id=%s", source_id)
             return None
         existing_protected = await count_protected_phrases(session, source_id)
         target_remaining: int | None = None
@@ -171,13 +181,15 @@ async def prepare_book_judgment(
             target_remaining = max(config.phrase_target_protected_per_book - existing_protected, 0)
             if target_remaining == 0:
                 logger.info(
-                    f"ebook_candidate_phrase_judgment_skipped_target_met {source_id=} {existing_protected=} "
-                    f"{config.phrase_target_protected_per_book=}"
+                    "ebook_candidate_phrase_judgment_skipped_target_met source_id=%s existing_protected=%s target=%s",
+                    source_id,
+                    existing_protected,
+                    config.phrase_target_protected_per_book,
                 )
                 return None
         book_text = await load_book_text(session, source_id)
         if not book_text:
-            logger.warning(f"ebook_candidate_phrase_judgment_book_empty {source_id=}")
+            logger.warning("ebook_candidate_phrase_judgment_book_empty source_id=%s", source_id)
             return None
         normalized_book_text = normalize_text(book_text)
         # Stored rows may predate the current junk filters and score weights, so re-filter and
@@ -199,8 +211,15 @@ async def prepare_book_judgment(
                 normalized_book_text, candidate.phrase_norm
             )
     logger.info(
-        f"ebook_candidate_phrase_judgment_candidates_loaded {source_id=} candidates={len(work_items)} {skipped_junk=} "
-        f"unjudged_rows={len(rows)} {existing_protected=} {target_remaining=} {judgment_limit=}"
+        "ebook_candidate_phrase_judgment_candidates_loaded source_id=%s candidates=%s skipped_junk=%s "
+        "unjudged_rows=%s existing_protected=%s target_remaining=%s judgment_limit=%s",
+        source_id,
+        len(work_items),
+        skipped_junk,
+        len(rows),
+        existing_protected,
+        target_remaining,
+        judgment_limit,
     )
     return work_items, target_remaining
 
@@ -294,20 +313,31 @@ async def persist_book_judgments(
                         await upsert_protected_phrase(session, source_id, None, candidate, judgment, candidate_row)
                     )
                 logger.info(
-                    f"ebook_candidate_phrase_judgment_candidate_complete {source_id=} {candidate_id=} "
-                    f"{candidate.phrase_norm=} {judgment.keep=} {judgment.confidence=:.3f} {judgment.category=} "
-                    f"{promote=}"
+                    "ebook_candidate_phrase_judgment_candidate_complete source_id=%s candidate_id=%s phrase=%r "
+                    "keep=%s confidence=%.3f category=%r promoted=%s",
+                    source_id,
+                    candidate_id,
+                    candidate.phrase_norm,
+                    judgment.keep,
+                    judgment.confidence,
+                    judgment.category,
+                    promote,
                 )
             await session.flush()
             mentions = await index_chunk_phrase_mentions_for_book(session, source_id, config) if protected else 0
             await session.commit()
         except Exception:
             await session.rollback()
-            logger.exception(f"ebook_candidate_phrase_judgment_book_persist_failed {source_id=}")
+            logger.exception("ebook_candidate_phrase_judgment_book_persist_failed source_id=%s", source_id)
             return BookJudgmentResult(failed=True)
     logger.info(
-        f"ebook_candidate_phrase_judgment_book_committed {source_id=} judged={len(judged)} protected={len(protected)} "
-        f"{mentions=} duration_ms={(perf_counter() - book_started_at) * 1000:.1f}"
+        "ebook_candidate_phrase_judgment_book_committed source_id=%s judged=%s protected=%s mentions=%s "
+        "duration_ms=%.1f",
+        source_id,
+        len(judged),
+        len(protected),
+        mentions,
+        (perf_counter() - book_started_at) * 1000,
     )
     return BookJudgmentResult(judged=len(judged), protected=len(protected), mentions=mentions, committed=True)
 
@@ -339,14 +369,24 @@ def should_protect_judged_candidate(
     accepted_token_count = len(accepted_tokens)
     if accepted_token_count < config.phrase_min_tokens:
         logger.info(
-            f"ebook_candidate_phrase_judgment_candidate_skip_short_canonical {book_id=} {candidate_id=} "
-            f"{candidate.phrase_norm=} {accepted_norm=} {accepted_token_count=} {config.phrase_min_tokens=}"
+            "ebook_candidate_phrase_judgment_candidate_skip_short_canonical book_id=%s candidate_id=%s "
+            "phrase=%r canonical=%r token_count=%s min_tokens=%s",
+            book_id,
+            candidate_id,
+            candidate.phrase_norm,
+            accepted_norm,
+            accepted_token_count,
+            config.phrase_min_tokens,
         )
         return False
     if is_most_common_word_phrase(accepted_tokens):
         logger.info(
-            f"ebook_candidate_phrase_judgment_candidate_skip_common_canonical {book_id=} {candidate_id=} "
-            f"{candidate.phrase_norm=} {accepted_norm=}"
+            "ebook_candidate_phrase_judgment_candidate_skip_common_canonical book_id=%s candidate_id=%s "
+            "phrase=%r canonical=%r",
+            book_id,
+            candidate_id,
+            candidate.phrase_norm,
+            accepted_norm,
         )
         return False
     return True

python/ebook_search/protected_phrases/matching.py

@@ -6,10 +6,12 @@ import logging
 from collections import defaultdict
 from typing import TYPE_CHECKING
 
-from sqlalchemy import and_, delete, or_, select, union
+from sqlalchemy import and_, delete, func, or_, select
 
 from python.ebook_search.protected_phrases.config import get_ignored_phrases
 from python.ebook_search.protected_phrases.models import (
+    ChunkPhraseHit,
+    HydratedPhraseMatch,
     PhraseLookup,
     PhraseMatch,
 )
@@ -27,107 +29,11 @@ if TYPE_CHECKING:
     from sqlalchemy.ext.asyncio import AsyncSession
 
     from python.ebook_search.config import EbookSearchConfig
+    from python.ebook_search.protected_phrases.text_normalization import NormalizedToken
 
 logger = logging.getLogger(__name__)
 
 
-async def detect_protected_phrases_for_query(
-    session: AsyncSession,
-    query_text: str,
-    config: EbookSearchConfig,
-) -> list[PhraseMatch]:
-    """Find query phrases with indexed exact matches on canonical and alias norms.
-
-    Args:
-        session (AsyncSession): Active database session.
-        query_text (str): User query text to detect phrases in.
-        config (EbookSearchConfig): Runtime phrase-tuning settings.
-
-    Returns:
-        list[PhraseMatch]: Metadata-backed, overlap-resolved phrase matches for the query.
-    """
-    tokens_ = tokenize_with_offsets(query_text)
-    windows_by_norm: defaultdict[str, list[tuple[int, int]]] = defaultdict(list)
-    token_texts = [token.text for token in tokens_]
-    max_tokens = max(config.phrase_max_tokens, config.phrase_max_entity_tokens)
-    for phrase_norm, start, end in generate_query_ngrams(
-        token_texts,
-        min_n=config.phrase_min_tokens,
-        max_n=max_tokens,
-    ):
-        windows_by_norm[phrase_norm].append((start, end))
-    if not windows_by_norm:
-        return []
-
-    query_norms = tuple(windows_by_norm)
-    matched_norms = union(
-        select(
-            EbookProtectedPhrase.id.label("phrase_id"),
-            EbookProtectedPhrase.phrase_norm.label("matched_norm"),
-        ).where(EbookProtectedPhrase.phrase_norm.in_(query_norms)),
-        select(
-            EbookPhraseAlias.phrase_id.label("phrase_id"),
-            EbookPhraseAlias.alias_norm.label("matched_norm"),
-        ).where(EbookPhraseAlias.alias_norm.in_(query_norms)),
-    ).subquery()
-    statement = select(EbookProtectedPhrase, matched_norms.c.matched_norm).join(
-        matched_norms,
-        matched_norms.c.phrase_id == EbookProtectedPhrase.id,
-    )
-
-    matches: list[PhraseMatch] = []
-    for phrase, matched_norm in await session.execute(statement):
-        for start, end in windows_by_norm[matched_norm]:
-            matches.append(
-                PhraseMatch(
-                    phrase_id=phrase.id,
-                    matched_norm=matched_norm,
-                    phrase_text=phrase.phrase_text,
-                    phrase_norm=phrase.phrase_norm,
-                    canonical_id=phrase.canonical_id,
-                    phrase_type=phrase.phrase_type,
-                    token_count=end - start,
-                    confidence=phrase.confidence,
-                    importance=phrase.importance,
-                    allow_nested=phrase.allow_nested,
-                    suppress_children=phrase.suppress_children,
-                    start_token=start,
-                    end_token=end,
-                    start_char=tokens_[start].start_char,
-                    end_char=tokens_[end - 1].end_char,
-                    book_id=phrase.book_id,
-                    series_id=phrase.series_id,
-                )
-            )
-    return resolve_overlaps(matches)
-
-
-async def index_chunk_phrase_mentions_for_book(
-    session: AsyncSession,
-    book_id: int,
-    config: EbookSearchConfig,
-) -> int:
-    """Rebuild chunk phrase mentions for all chunks in one book.
-
-    Args:
-        session (AsyncSession): Active database session.
-        book_id (int): Book whose chunk mentions are rebuilt.
-        config (EbookSearchConfig): Runtime phrase-tuning settings.
-
-    Returns:
-        int: Total number of chunk phrase mentions indexed for the book.
-    """
-    lookup = await load_phrase_lookup(session, config, book_id=book_id)
-    await session.execute(delete(EbookChunkPhraseMention).where(EbookChunkPhraseMention.book_id == book_id))
-    chunks = await session.scalars(select(EbookChunk).where(EbookChunk.source_id == book_id).order_by(EbookChunk.id))
-    count = 0
-    for chunk in chunks:
-        count += index_chunk_phrase_mentions(session, chunk, lookup=lookup)
-    await session.flush()
-    logger.info(f"ebook_chunk_phrase_mentions_indexed {book_id=} {count=}")
-    return count
-
-
 async def load_phrase_lookup(
     session: AsyncSession,
     config: EbookSearchConfig,
@@ -146,29 +52,40 @@ async def load_phrase_lookup(
     Returns:
         PhraseLookup: Normalized phrase and alias maps with the token-window bounds to test.
     """
-    phrase_ids_by_norm: defaultdict[str, set[int]] = defaultdict(set)
-    phrases_by_id: dict[int, EbookProtectedPhrase] = {}
+    norm_to_ids: defaultdict[str, list[int]] = defaultdict(list)
+    alias_to_ids: defaultdict[str, list[int]] = defaultdict(list)
     max_tokens = config.phrase_max_tokens
 
-    statement = select(
-        EbookProtectedPhrase,
-        EbookPhraseAlias.alias_norm,
-    ).outerjoin(EbookPhraseAlias, EbookPhraseAlias.phrase_id == EbookProtectedPhrase.id)
+    phrase_statement = select(
+        EbookProtectedPhrase.id,
+        EbookProtectedPhrase.phrase_norm,
+        EbookProtectedPhrase.token_count,
+    )
     scope_filter = protected_phrase_scope_filter(book_id=book_id, series_id=series_id)
     if scope_filter is not None:
-        statement = statement.where(scope_filter)
+        phrase_statement = phrase_statement.where(scope_filter)
 
-    for phrase, alias_norm in await session.execute(statement):
-        phrases_by_id[phrase.id] = phrase
-        phrase_ids_by_norm[phrase.phrase_norm].add(phrase.id)
-        max_tokens = max(max_tokens, phrase.token_count)
-        if alias_norm is not None:
-            phrase_ids_by_norm[alias_norm].add(phrase.id)
-            max_tokens = max(max_tokens, len(alias_norm.split()))
+    for row in await session.execute(phrase_statement):
+        phrase_id = int(row.id)
+        phrase_norm = str(row.phrase_norm)
+        norm_to_ids[phrase_norm].append(phrase_id)
+        max_tokens = max(max_tokens, int(row.token_count))
+
+    alias_statement = select(
+        EbookPhraseAlias.alias_norm,
+        EbookPhraseAlias.phrase_id,
+    ).join(EbookProtectedPhrase, EbookProtectedPhrase.id == EbookPhraseAlias.phrase_id)
+    if scope_filter is not None:
+        alias_statement = alias_statement.where(scope_filter)
+
+    for row in await session.execute(alias_statement):
+        alias_norm = str(row.alias_norm)
+        alias_to_ids[alias_norm].append(int(row.phrase_id))
+        max_tokens = max(max_tokens, len(alias_norm.split()))
 
     return PhraseLookup(
-        phrase_ids_by_norm={key: tuple(sorted(values)) for key, values in phrase_ids_by_norm.items()},
-        phrases_by_id=phrases_by_id,
+        norm_to_phrase_ids={key: tuple(values) for key, values in norm_to_ids.items()},
+        alias_to_phrase_ids={key: tuple(values) for key, values in alias_to_ids.items()},
         min_tokens=config.phrase_min_tokens,
         max_tokens=max_tokens,
     )
@@ -194,144 +111,6 @@ def protected_phrase_scope_filter(*, book_id: int | None, series_id: int | None)
     return and_(*conditions)
 
 
-def is_inside(child: PhraseMatch, parent: PhraseMatch) -> bool:
-    """Return whether one token span is strictly inside another.
-
-    Args:
-        child (PhraseMatch): Candidate nested match.
-        parent (PhraseMatch): Candidate enclosing match.
-
-    Returns:
-        bool: True when ``child`` lies within ``parent`` and is not the same span.
-    """
-    return (
-        child.start_token >= parent.start_token
-        and child.end_token <= parent.end_token
-        and (child.start_token, child.end_token, child.phrase_id)
-        != (parent.start_token, parent.end_token, parent.phrase_id)
-    )
-
-
-def index_chunk_phrase_mentions(session: AsyncSession, chunk: EbookChunk, *, lookup: PhraseLookup) -> int:
-    """Store protected phrase mentions for one chunk.
-
-    Args:
-        session (AsyncSession): Active database session.
-        chunk (EbookChunk): Chunk whose text is scanned for phrase mentions.
-        lookup (PhraseLookup): In-memory phrase and alias lookup maps.
-
-    Returns:
-        int: Number of phrase mentions stored for the chunk.
-    """
-    tokens_ = tokenize_with_offsets(chunk.text)
-    token_texts = [token.text for token in tokens_]
-    raw_matches: list[PhraseMatch] = []
-    phrase_windows = generate_query_ngrams(token_texts, min_n=lookup.min_tokens, max_n=lookup.max_tokens)
-    for matched_norm, start, end in phrase_windows:
-        for phrase_id in lookup.phrase_ids_by_norm.get(matched_norm, ()):
-            phrase = lookup.phrases_by_id[phrase_id]
-            raw_matches.append(
-                PhraseMatch(
-                    phrase_id=phrase_id,
-                    matched_norm=matched_norm,
-                    phrase_text=phrase.phrase_text,
-                    phrase_norm=phrase.phrase_norm,
-                    canonical_id=phrase.canonical_id,
-                    phrase_type=phrase.phrase_type,
-                    confidence=phrase.confidence,
-                    importance=phrase.importance,
-                    allow_nested=phrase.allow_nested,
-                    suppress_children=phrase.suppress_children,
-                    start_token=start,
-                    end_token=end,
-                    token_count=end - start,
-                    start_char=tokens_[start].start_char,
-                    end_char=tokens_[end - 1].end_char,
-                    book_id=phrase.book_id,
-                    series_id=phrase.series_id,
-                )
-            )
-
-    matches = resolve_overlaps(raw_matches)
-    for match in matches:
-        session.add(
-            EbookChunkPhraseMention(
-                chunk_id=chunk.id,
-                phrase_id=match.phrase_id,
-                book_id=match.book_id if match.book_id is not None else chunk.source_id,
-                series_id=match.series_id,
-                start_char=match.start_char if match.start_char is not None else 0,
-                end_char=match.end_char,
-            )
-        )
-    return len(matches)
-
-
-def resolve_overlaps(matches: Sequence[PhraseMatch]) -> list[PhraseMatch]:
-    """Resolve overlapping phrase matches without relying only on longest match.
-
-    Args:
-        matches (Sequence[PhraseMatch]): Metadata-backed matches that may overlap.
-
-    Returns:
-        list[PhraseMatch]: The kept, non-suppressed matches.
-    """
-    sorted_matches = sorted(
-        matches,
-        key=lambda match: (match.start_token, -match.token_count, -match.importance, -match.confidence),
-    )
-    kept: list[PhraseMatch] = []
-    for candidate in sorted_matches:
-        if any(should_suppress(candidate, existing) for existing in kept):
-            continue
-        kept.append(candidate)
-    return kept
-
-
-def should_suppress(candidate: PhraseMatch, kept: PhraseMatch) -> bool:
-    """Return whether an already-kept match should suppress a candidate.
-
-    Args:
-        candidate (PhraseMatch): Match being considered for keeping.
-        kept (PhraseMatch): Match already kept that may suppress the candidate.
-
-    Returns:
-        bool: True when the candidate should be dropped in favor of the kept match.
-    """
-    if not overlaps(candidate, kept):
-        return False
-    if candidate.canonical_id == kept.canonical_id:
-        return rank_match(kept) >= rank_match(candidate)
-    if is_inside(candidate, kept) and kept.suppress_children and not candidate.allow_nested:
-        return True
-    return not candidate.allow_nested and rank_match(kept) > rank_match(candidate)
-
-
-def overlaps(first: PhraseMatch, second: PhraseMatch) -> bool:
-    """Return whether two token spans overlap.
-
-    Args:
-        first (PhraseMatch): First match to compare.
-        second (PhraseMatch): Second match to compare.
-
-    Returns:
-        bool: True when the two token spans share at least one token position.
-    """
-    return not (first.end_token <= second.start_token or first.start_token >= second.end_token)
-
-
-def rank_match(match: PhraseMatch) -> tuple[float, float, int]:
-    """Rank phrase matches by importance, confidence, then token count.
-
-    Args:
-        match (PhraseMatch): Match to build a sort key for.
-
-    Returns:
-        tuple[float, float, int]: A comparable key of importance, confidence, and token count.
-    """
-    return (match.importance, match.confidence, match.token_count)
-
-
 def generate_query_ngrams(
     tokens_: Sequence[str],
     min_n: int,
@@ -355,3 +134,358 @@ def generate_query_ngrams(
             if phrase_norm in get_ignored_phrases():
                 continue
             yield phrase_norm, start, end
+
+
+def detect_phrase_candidates(query_text: str, lookup: PhraseLookup) -> list[PhraseMatch]:
+    """Detect protected phrase windows in a user query using RAM hash lookups.
+
+    Args:
+        query_text (str): User query text to scan.
+        lookup (PhraseLookup): In-memory phrase and alias lookup maps.
+
+    Returns:
+        list[PhraseMatch]: Unhydrated phrase matches found in the query.
+    """
+    return detect_phrase_candidates_from_tokens(tokenize_with_offsets(query_text), lookup)
+
+
+def detect_phrase_candidates_in_text(text: str, lookup: PhraseLookup) -> list[PhraseMatch]:
+    """Detect protected phrase windows in arbitrary text with character offsets.
+
+    Args:
+        text (str): Arbitrary text, such as a chunk, to scan.
+        lookup (PhraseLookup): In-memory phrase and alias lookup maps.
+
+    Returns:
+        list[PhraseMatch]: Unhydrated phrase matches found in the text.
+    """
+    return detect_phrase_candidates_from_tokens(tokenize_with_offsets(text), lookup)
+
+
+def detect_phrase_candidates_from_tokens(tokens_: Sequence[NormalizedToken], lookup: PhraseLookup) -> list[PhraseMatch]:
+    """Detect protected phrase windows from already-normalized tokens.
+
+    Args:
+        tokens_ (Sequence[NormalizedToken]): Normalized tokens with character offsets.
+        lookup (PhraseLookup): In-memory phrase and alias lookup maps.
+
+    Returns:
+        list[PhraseMatch]: Deduplicated unhydrated phrase matches with token and character spans.
+    """
+    matches: list[PhraseMatch] = []
+    seen: set[tuple[int | None, str, int, int]] = set()
+    token_texts = [token.text for token in tokens_]
+    for phrase_norm, start, end in generate_query_ngrams(token_texts, min_n=lookup.min_tokens, max_n=lookup.max_tokens):
+        phrase_ids = lookup.norm_to_phrase_ids.get(phrase_norm, ())
+        alias_ids = lookup.alias_to_phrase_ids.get(phrase_norm, ())
+        for phrase_id in (*phrase_ids, *alias_ids):
+            key = (phrase_id, phrase_norm, start, end)
+            if key in seen:
+                continue
+            seen.add(key)
+            matches.append(
+                PhraseMatch(
+                    phrase_norm=phrase_norm,
+                    phrase_id=phrase_id,
+                    start_token=start,
+                    end_token=end,
+                    token_count=end - start,
+                    start_char=tokens_[start].start_char,
+                    end_char=tokens_[end - 1].end_char,
+                )
+            )
+    return matches
+
+
+async def hydrate_matches(session: AsyncSession, matches: Sequence[PhraseMatch]) -> list[HydratedPhraseMatch]:
+    """Fetch protected phrase metadata for raw phrase matches.
+
+    Args:
+        session (AsyncSession): Active database session.
+        matches (Sequence[PhraseMatch]): Unhydrated matches to enrich.
+
+    Returns:
+        list[HydratedPhraseMatch]: Matches with protected-phrase metadata attached.
+    """
+    if not matches:
+        return []
+
+    phrase_ids = sorted({match.phrase_id for match in matches if match.phrase_id is not None})
+    if not phrase_ids:
+        return []
+
+    rows = {
+        row.id: row
+        for row in await session.scalars(select(EbookProtectedPhrase).where(EbookProtectedPhrase.id.in_(phrase_ids)))
+    }
+    hydrated: list[HydratedPhraseMatch] = []
+    for match in matches:
+        if match.phrase_id is None:
+            continue
+        phrase = rows.get(match.phrase_id)
+        if phrase is None:
+            continue
+        hydrated.append(
+            HydratedPhraseMatch(
+                phrase_id=phrase.id,
+                matched_norm=match.phrase_norm,
+                phrase_text=phrase.phrase_text,
+                phrase_norm=phrase.phrase_norm,
+                canonical_id=phrase.canonical_id,
+                phrase_type=phrase.phrase_type,
+                token_count=match.token_count,
+                confidence=phrase.confidence,
+                importance=phrase.importance,
+                allow_nested=phrase.allow_nested,
+                suppress_children=phrase.suppress_children,
+                start_token=match.start_token,
+                end_token=match.end_token,
+                start_char=match.start_char,
+                end_char=match.end_char,
+                book_id=phrase.book_id,
+                series_id=phrase.series_id,
+            )
+        )
+    return hydrated
+
+
+def overlaps(first: HydratedPhraseMatch, second: HydratedPhraseMatch) -> bool:
+    """Return whether two token spans overlap.
+
+    Args:
+        first (HydratedPhraseMatch): First match to compare.
+        second (HydratedPhraseMatch): Second match to compare.
+
+    Returns:
+        bool: True when the two token spans share at least one token position.
+    """
+    return not (first.end_token <= second.start_token or first.start_token >= second.end_token)
+
+
+def is_inside(child: HydratedPhraseMatch, parent: HydratedPhraseMatch) -> bool:
+    """Return whether one token span is strictly inside another.
+
+    Args:
+        child (HydratedPhraseMatch): Candidate nested match.
+        parent (HydratedPhraseMatch): Candidate enclosing match.
+
+    Returns:
+        bool: True when ``child`` lies within ``parent`` and is not the same span.
+    """
+    return (
+        child.start_token >= parent.start_token
+        and child.end_token <= parent.end_token
+        and (child.start_token, child.end_token, child.phrase_id)
+        != (parent.start_token, parent.end_token, parent.phrase_id)
+    )
+
+
+def rank_match(match: HydratedPhraseMatch) -> tuple[float, float, int]:
+    """Rank phrase matches by importance, confidence, then token count.
+
+    Args:
+        match (HydratedPhraseMatch): Match to build a sort key for.
+
+    Returns:
+        tuple[float, float, int]: A comparable key of importance, confidence, and token count.
+    """
+    return (match.importance, match.confidence, match.token_count)
+
+
+def should_suppress(candidate: HydratedPhraseMatch, kept: HydratedPhraseMatch) -> bool:
+    """Return whether an already-kept match should suppress a candidate.
+
+    Args:
+        candidate (HydratedPhraseMatch): Match being considered for keeping.
+        kept (HydratedPhraseMatch): Match already kept that may suppress the candidate.
+
+    Returns:
+        bool: True when the candidate should be dropped in favor of the kept match.
+    """
+    if not overlaps(candidate, kept):
+        return False
+    if candidate.canonical_id == kept.canonical_id:
+        return rank_match(kept) >= rank_match(candidate)
+    if is_inside(candidate, kept) and kept.suppress_children and not candidate.allow_nested:
+        return True
+    return not candidate.allow_nested and rank_match(kept) > rank_match(candidate)
+
+
+def resolve_overlaps(matches: Sequence[HydratedPhraseMatch]) -> list[HydratedPhraseMatch]:
+    """Resolve overlapping phrase matches without relying only on longest match.
+
+    Args:
+        matches (Sequence[HydratedPhraseMatch]): Hydrated matches that may overlap.
+
+    Returns:
+        list[HydratedPhraseMatch]: The kept, non-suppressed matches.
+    """
+    sorted_matches = sorted(
+        matches,
+        key=lambda match: (match.start_token, -match.token_count, -match.importance, -match.confidence),
+    )
+    kept: list[HydratedPhraseMatch] = []
+    for candidate in sorted_matches:
+        if any(should_suppress(candidate, existing) for existing in kept):
+            continue
+        kept.append(candidate)
+    return kept
+
+
+async def detect_protected_phrases_for_query(
+    session: AsyncSession,
+    query_text: str,
+    config: EbookSearchConfig,
+    *,
+    lookup: PhraseLookup | None = None,
+    book_id: int | None = None,
+    series_id: int | None = None,
+) -> list[HydratedPhraseMatch]:
+    """Run the full online protected-phrase query-detection pipeline.
+
+    Args:
+        session (AsyncSession): Active database session.
+        query_text (str): User query text to detect phrases in.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        lookup (PhraseLookup | None): Optional preloaded lookup; loaded on demand when ``None``.
+        book_id (int | None): Optional book scope for lookup loading.
+        series_id (int | None): Optional series scope for lookup loading.
+
+    Returns:
+        list[HydratedPhraseMatch]: Hydrated, overlap-resolved phrase matches for the query.
+    """
+    active_lookup = (
+        lookup
+        if lookup is not None
+        else await load_phrase_lookup(session, config, book_id=book_id, series_id=series_id)
+    )
+    return resolve_overlaps(await hydrate_matches(session, detect_phrase_candidates(query_text, active_lookup)))
+
+
+async def index_chunk_phrase_mentions_for_book(
+    session: AsyncSession,
+    book_id: int,
+    config: EbookSearchConfig,
+    *,
+    series_id: int | None = None,
+    lookup: PhraseLookup | None = None,
+) -> int:
+    """Rebuild chunk phrase mentions for all chunks in one book.
+
+    Args:
+        session (AsyncSession): Active database session.
+        book_id (int): Book whose chunk mentions are rebuilt.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        series_id (int | None): Optional series scope for lookup loading.
+        lookup (PhraseLookup | None): Optional preloaded lookup; loaded on demand when ``None``.
+
+    Returns:
+        int: Total number of chunk phrase mentions indexed for the book.
+    """
+    active_lookup = (
+        lookup
+        if lookup is not None
+        else await load_phrase_lookup(session, config, book_id=book_id, series_id=series_id)
+    )
+    await session.execute(delete(EbookChunkPhraseMention).where(EbookChunkPhraseMention.book_id == book_id))
+    chunks = await session.scalars(select(EbookChunk).where(EbookChunk.source_id == book_id).order_by(EbookChunk.id))
+    count = 0
+    for chunk in chunks:
+        count += await index_chunk_phrase_mentions(session, chunk, lookup=active_lookup)
+    await session.flush()
+    logger.info("ebook_chunk_phrase_mentions_indexed book_id=%s mentions=%s", book_id, count)
+    return count
+
+
+async def index_chunk_phrase_mentions(session: AsyncSession, chunk: EbookChunk, *, lookup: PhraseLookup) -> int:
+    """Store protected phrase mentions for one chunk.
+
+    Args:
+        session (AsyncSession): Active database session.
+        chunk (EbookChunk): Chunk whose text is scanned for phrase mentions.
+        lookup (PhraseLookup): In-memory phrase and alias lookup maps.
+
+    Returns:
+        int: Number of phrase mentions stored for the chunk.
+    """
+    raw_matches = detect_phrase_candidates_in_text(chunk.text, lookup)
+    hydrated = resolve_overlaps(await hydrate_matches(session, raw_matches))
+    for match in hydrated:
+        session.add(
+            EbookChunkPhraseMention(
+                chunk_id=chunk.id,
+                phrase_id=match.phrase_id,
+                book_id=match.book_id if match.book_id is not None else chunk.source_id,
+                series_id=match.series_id,
+                start_char=match.start_char if match.start_char is not None else 0,
+                end_char=match.end_char,
+            )
+        )
+    return len(hydrated)
+
+
+async def phrase_hits_for_chunks(
+    session: AsyncSession,
+    *,
+    chunk_ids: Sequence[int],
+    phrase_ids: Sequence[int],
+) -> dict[int, tuple[ChunkPhraseHit, ...]]:
+    """Return matched protected phrases with mention counts by chunk id using indexed chunk mentions.
+
+    Args:
+        session (AsyncSession): Active database session.
+        chunk_ids (Sequence[int]): Chunk ids to look up mentions for.
+        phrase_ids (Sequence[int]): Protected phrase ids to restrict the results to.
+
+    Returns:
+        dict[int, tuple[ChunkPhraseHit, ...]]: Phrase hits per chunk id, ordered by mention count.
+    """
+    if not chunk_ids or not phrase_ids:
+        return {}
+
+    mention_count = func.count(EbookChunkPhraseMention.phrase_id).label("mention_count")
+    statement = (
+        select(
+            EbookChunkPhraseMention.chunk_id,
+            EbookProtectedPhrase.id.label("phrase_id"),
+            EbookProtectedPhrase.phrase_text,
+            mention_count,
+        )
+        .join(EbookProtectedPhrase, EbookProtectedPhrase.id == EbookChunkPhraseMention.phrase_id)
+        .where(
+            EbookChunkPhraseMention.chunk_id.in_(chunk_ids),
+            EbookChunkPhraseMention.phrase_id.in_(phrase_ids),
+        )
+        .group_by(EbookChunkPhraseMention.chunk_id, EbookProtectedPhrase.id, EbookProtectedPhrase.phrase_text)
+        .order_by(EbookChunkPhraseMention.chunk_id, mention_count.desc(), EbookProtectedPhrase.phrase_text)
+    )
+    hits: defaultdict[int, list[ChunkPhraseHit]] = defaultdict(list)
+    for row in await session.execute(statement):
+        hits[int(row.chunk_id)].append(
+            ChunkPhraseHit(
+                phrase_id=int(row.phrase_id),
+                phrase_text=str(row.phrase_text),
+                mention_count=int(row.mention_count),
+            )
+        )
+    return {chunk_id: tuple(chunk_hits) for chunk_id, chunk_hits in hits.items()}
+
+
+async def phrase_hit_counts_for_chunks(
+    session: AsyncSession,
+    *,
+    chunk_ids: Sequence[int],
+    phrase_ids: Sequence[int],
+) -> dict[int, int]:
+    """Return phrase-hit counts by chunk id using indexed chunk mentions.
+
+    Args:
+        session (AsyncSession): Active database session.
+        chunk_ids (Sequence[int]): Chunk ids to count mentions for.
+        phrase_ids (Sequence[int]): Protected phrase ids to restrict the counts to.
+
+    Returns:
+        dict[int, int]: Total mention count per chunk id.
+    """
+    hits = await phrase_hits_for_chunks(session, chunk_ids=chunk_ids, phrase_ids=phrase_ids)
+    return {chunk_id: sum(hit.mention_count for hit in chunk_hits) for chunk_id, chunk_hits in hits.items()}

python/ebook_search/protected_phrases/models.py

@@ -8,8 +8,6 @@ from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from collections.abc import Mapping
 
-    from python.orm.richie import EbookProtectedPhrase
-
 
 @dataclass(slots=True)
 class PhraseCandidate:
@@ -21,8 +19,11 @@ class PhraseCandidate:
         token_count (int): Number of normalized tokens in the phrase.
         source_raw_ngram (bool): Whether the raw n-gram extractor produced the phrase.
         source_yake (bool): Whether YAKE keyword extraction produced the phrase.
+        source_spacy_ner (bool): Whether spaCy named-entity recognition produced the phrase.
+        source_spacy_noun_chunk (bool): Whether spaCy noun chunking produced the phrase.
         source_capitalized (bool): Whether the capitalized-run extractor produced the phrase.
         source_metadata (bool): Whether book metadata produced the phrase.
+        spacy_label (str | None): spaCy entity label when NER produced the phrase.
         raw_count (int): Occurrences counted across the book text.
         chapter_count (int): Number of chapters containing the phrase.
         yake_score (float | None): Raw YAKE score when available; lower is better.
@@ -35,8 +36,11 @@ class PhraseCandidate:
     token_count: int
     source_raw_ngram: bool = False
     source_yake: bool = False
+    source_spacy_ner: bool = False
+    source_spacy_noun_chunk: bool = False
     source_capitalized: bool = False
     source_metadata: bool = False
+    spacy_label: str | None = None
     raw_count: int = 0
     chapter_count: int = 0
     yake_score: float | None = None
@@ -73,24 +77,47 @@ class LLMJudgment:
 
 @dataclass(frozen=True, slots=True)
 class PhraseLookup:
-    """In-memory phrase metadata used for constant-time text-window checks.
+    """In-memory lookup maps used for constant-time phrase-window checks.
 
     Attributes:
-        phrase_ids_by_norm (Mapping[str, tuple[int, ...]]): Canonical and alias norms to phrase ids.
-        phrases_by_id (Mapping[int, EbookProtectedPhrase]): Protected phrase metadata by id.
+        norm_to_phrase_ids (Mapping[str, tuple[int, ...]]): Normalized phrase to protected phrase ids.
+        alias_to_phrase_ids (Mapping[str, tuple[int, ...]]): Normalized alias to protected phrase ids.
         min_tokens (int): Smallest token-window size to test.
         max_tokens (int): Largest token-window size to test.
     """
 
-    phrase_ids_by_norm: Mapping[str, tuple[int, ...]]
-    phrases_by_id: Mapping[int, EbookProtectedPhrase]
+    norm_to_phrase_ids: Mapping[str, tuple[int, ...]]
+    alias_to_phrase_ids: Mapping[str, tuple[int, ...]]
     min_tokens: int
     max_tokens: int
 
 
 @dataclass(frozen=True, slots=True)
 class PhraseMatch:
-    """A detected phrase match with protected-phrase metadata attached.
+    """An unhydrated query or chunk phrase match.
+
+    Attributes:
+        phrase_norm (str): Normalized text of the matched window.
+        start_token (int): Index of the first matched token.
+        end_token (int): Index one past the last matched token.
+        token_count (int): Number of tokens in the match.
+        phrase_id (int | None): Matched protected phrase id when known.
+        start_char (int | None): Start character offset in the source text.
+        end_char (int | None): End character offset in the source text.
+    """
+
+    phrase_norm: str
+    start_token: int
+    end_token: int
+    token_count: int
+    phrase_id: int | None = None
+    start_char: int | None = None
+    end_char: int | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class HydratedPhraseMatch:
+    """A phrase match with protected-phrase metadata attached.
 
     Attributes:
         phrase_id (int): Protected phrase id.
@@ -131,6 +158,21 @@ class PhraseMatch:
     series_id: int | None = None
 
 
+@dataclass(frozen=True, slots=True)
+class ChunkPhraseHit:
+    """One protected phrase with its mention count inside one retrieved chunk.
+
+    Attributes:
+        phrase_id (int): Protected phrase id.
+        phrase_text (str): Display text of the protected phrase.
+        mention_count (int): Indexed mentions of the phrase in the chunk.
+    """
+
+    phrase_id: int
+    phrase_text: str
+    mention_count: int
+
+
 @dataclass(frozen=True, slots=True)
 class PhraseCandidateGenerationResult:
     """Summary of candidate phrase extraction for indexed books.

python/ebook_search/protected_phrases/pool.py

@@ -9,11 +9,21 @@ or server threads.
 
 from __future__ import annotations
 
+import asyncio
 import logging
 import multiprocessing
 import os
 from concurrent.futures import ProcessPoolExecutor
 from threading import Lock
+from typing import TYPE_CHECKING
+
+from python.ebook_search.protected_phrases.extraction import extract_phrase_candidates_for_book
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping, Sequence
+
+    from python.ebook_search.config import EbookSearchConfig
+    from python.ebook_search.protected_phrases.models import PhraseCandidate
 
 logger = logging.getLogger(__name__)
 
@@ -45,7 +55,7 @@ def get_extraction_pool(max_workers: int) -> ProcessPoolExecutor:
                 max_workers=workers,
                 mp_context=multiprocessing.get_context("spawn"),
             )
-            logger.info(f"ebook_phrase_extraction_pool_started {workers=}")
+            logger.info("ebook_phrase_extraction_pool_started workers=%s", workers)
         return _extraction_pool.pool
 
 
@@ -56,3 +66,36 @@ def shutdown_extraction_pool() -> None:
             _extraction_pool.pool.shutdown(wait=False, cancel_futures=True)
             _extraction_pool.pool = None
             logger.info("ebook_phrase_extraction_pool_shutdown")
+
+
+async def extract_phrase_candidates_in_pool(
+    book_text: str,
+    chapters: Sequence[str],
+    config: EbookSearchConfig,
+    *,
+    metadata: Mapping[str, object] | None,
+) -> list[PhraseCandidate]:
+    """Run book phrase extraction in a worker process and await the result.
+
+    Only the CPU-bound extraction runs in the worker; the caller keeps all database work in the
+    request process. The spaCy pipeline is not supported here because it is not picklable, so
+    this always runs the non-spaCy extraction path.
+
+    Args:
+        book_text (str): Full book text used for extraction.
+        chapters (Sequence[str]): Chapter-like text blocks used for frequency counts.
+        config (EbookSearchConfig): Runtime phrase-tuning settings.
+        metadata (Mapping[str, object] | None): Optional book metadata used as a candidate source.
+
+    Returns:
+        list[PhraseCandidate]: Scored candidates sorted best-first and capped per book.
+    """
+    pool = get_extraction_pool(config.protected_phrase_extraction_workers)
+    future = pool.submit(
+        extract_phrase_candidates_for_book,
+        book_text,
+        list(chapters),
+        config,
+        metadata=dict(metadata) if metadata is not None else None,
+    )
+    return await asyncio.wrap_future(future)

python/ebook_search/protected_phrases/store.py

@@ -295,8 +295,11 @@ def phrase_candidate_from_row(row: EbookCandidatePhrase) -> PhraseCandidate:
         token_count=row.token_count,
         source_raw_ngram=row.source_raw_ngram,
         source_yake=row.source_yake,
+        source_spacy_ner=row.source_spacy_ner,
+        source_spacy_noun_chunk=row.source_spacy_noun_chunk,
         source_capitalized=row.source_capitalized,
         source_metadata=row.source_metadata,
+        spacy_label=row.spacy_label,
         raw_count=row.raw_count,
         chapter_count=row.chapter_count,
         yake_score=row.yake_score,
@@ -331,8 +334,11 @@ def candidate_row_values(
         "token_count": candidate.token_count,
         "source_raw_ngram": candidate.source_raw_ngram,
         "source_yake": candidate.source_yake,
+        "source_spacy_ner": candidate.source_spacy_ner,
+        "source_spacy_noun_chunk": candidate.source_spacy_noun_chunk,
         "source_capitalized": candidate.source_capitalized,
         "source_metadata": candidate.source_metadata,
+        "spacy_label": candidate.spacy_label,
         "raw_count": candidate.raw_count,
         "chapter_count": candidate.chapter_count,
         "yake_score": candidate.yake_score,
@@ -454,8 +460,11 @@ def new_candidate_row(book_id: int, series_id: int | None, candidate: PhraseCand
     row.token_count = candidate.token_count
     row.source_raw_ngram = candidate.source_raw_ngram
     row.source_yake = candidate.source_yake
+    row.source_spacy_ner = candidate.source_spacy_ner
+    row.source_spacy_noun_chunk = candidate.source_spacy_noun_chunk
     row.source_capitalized = candidate.source_capitalized
     row.source_metadata = candidate.source_metadata
+    row.spacy_label = candidate.spacy_label
     row.raw_count = candidate.raw_count
     row.chapter_count = candidate.chapter_count
     row.yake_score = candidate.yake_score
@@ -615,8 +624,11 @@ async def prune_unstorable_unjudged_candidate_phrases(
     )
     if deleted:
         logger.info(
-            f"ebook_candidate_phrase_unstorable_pruned {book_id=} {deleted=} {config.phrase_min_tokens=} "
-            f"min_uses={minimum_candidate_raw_count(config)}"
+            "ebook_candidate_phrase_unstorable_pruned book_id=%s deleted=%s min_tokens=%s min_uses=%s",
+            book_id,
+            deleted,
+            config.phrase_min_tokens,
+            minimum_candidate_raw_count(config),
         )
     return deleted
 
@@ -658,8 +670,12 @@ async def delete_phrase_data_for_book(session: AsyncSession, book_id: int) -> Ph
     )
     await session.flush()
     logger.info(
-        f"ebook_candidate_phrase_data_deleted {book_id=} {deleted_candidates=} {deleted_protected=} {deleted_aliases=} "
-        f"{deleted_mentions=}"
+        "ebook_candidate_phrase_data_deleted book_id=%s candidates=%s protected=%s aliases=%s mentions=%s",
+        book_id,
+        deleted_candidates,
+        deleted_protected,
+        deleted_aliases,
+        deleted_mentions,
     )
     return PhraseRecalculationResult(
         book_id=book_id,

python/ebook_search/rerank.py

@@ -35,7 +35,12 @@ async def rerank_chunks(
     if not candidates:
         return []
 
-    logger.info(f"ebook_rerank_request_start {config.base_url=} {config.model=} candidates={len(candidates)}")
+    logger.info(
+        "ebook_rerank_request_start base_url=%s model=%s candidates=%s",
+        config.base_url,
+        config.model,
+        len(candidates),
+    )
     scores = await score_candidates(client, query, candidates, config)
     results = sorted(
         (
@@ -49,7 +54,12 @@ async def rerank_chunks(
         key=lambda result: result.score,
         reverse=True,
     )
-    logger.info(f"ebook_rerank_request_complete {config.base_url=} {config.model=} candidates={len(results)}")
+    logger.info(
+        "ebook_rerank_request_complete base_url=%s model=%s candidates=%s",
+        config.base_url,
+        config.model,
+        len(results),
+    )
     return results
 
 
@@ -66,7 +76,7 @@ async def score_candidates(
 
     scores = parse_vllm_scores(body, candidates)
     for result in scores.values():
-        logger.debug(f"ebook_rerank_candidate_scored {result.chunk_id=} {result.score=}")
+        logger.debug("ebook_rerank_candidate_scored chunk_id=%s score=%s", result.chunk_id, result.score)
     return scores
 
 

python/ebook_search/search.py

@@ -19,7 +19,6 @@ from python.ebook_search.bm25_corpus import (
     load_bm25_corpus,
     score_bm25_corpus,
 )
-from python.ebook_search.chunk_records import CHUNK_RECORD_COLUMNS
 from python.ebook_search.embeddings import MODEL_DIMENSIONS, embed_query, get_embedding_table
 from python.ebook_search.protected_phrases.matching import (
     detect_protected_phrases_for_query,
@@ -41,7 +40,7 @@ if TYPE_CHECKING:
     from sqlalchemy.ext.asyncio import AsyncEngine
 
     from python.ebook_search.config import EbookSearchConfig
-    from python.ebook_search.protected_phrases.models import PhraseMatch
+    from python.ebook_search.protected_phrases.models import HydratedPhraseMatch
 
 logger = logging.getLogger(__name__)
 
@@ -75,7 +74,7 @@ class SearchResponse:
     results: list[SearchResult]
     rank_label: str
     timings: tuple[RuntimeStep, ...] = ()
-    phrase_matches: tuple[PhraseMatch, ...] = ()
+    phrase_matches: tuple[HydratedPhraseMatch, ...] = ()
 
     @property
     def total_runtime_ms(self) -> float:
@@ -85,11 +84,10 @@ class SearchResponse:
 
 @dataclass(frozen=True)
 class RetrievalResponse:
-    """Parallel retrieval output for vector, BM25, and protected phrase candidates."""
+    """Parallel retrieval output for vector and BM25 candidates."""
 
     vector_results: list[SearchResult]
     lexical_results: list[SearchResult]
-    phrase_matches: list[PhraseMatch]
     timings: tuple[RuntimeStep, ...]
 
 
@@ -99,26 +97,33 @@ async def search_ebooks(
     query: str,
     config: EbookSearchConfig,
     *,
-    rerank: bool,
-    phrase_matching: bool,
+    rerank: bool = False,
+    phrase_matching: bool | None = None,
 ) -> SearchResponse:
-    """Run hybrid vector/BM25 search and optional reranking.
-
-    Phrase matching only runs when both the request asks for it and
-    ``config.phrase_matching_enabled`` allows it.
-    """
+    """Run hybrid vector/BM25 search and optional reranking."""
     if not query.strip():
         logger.info("ebook_search_empty_query")
         return SearchResponse(query=query, results=[], rank_label="Hybrid")
 
-    phrase_matching = phrase_matching and config.phrase_matching_enabled
-    logger.info(f"ebook_search_start query_length={len(query)} {rerank=} {phrase_matching=}")
+    phrase_matching_enabled = config.phrase_matching_enabled if phrase_matching is None else phrase_matching
+    logger.info(
+        "ebook_search_start query_length=%s rerank=%s phrase_matching=%s",
+        len(query),
+        rerank,
+        phrase_matching_enabled,
+    )
     timings: list[RuntimeStep] = []
+    if phrase_matching_enabled:
+        phrase_matches, timing = await async_timed_result(
+            "Protected phrase detection", query_phrase_matches(engine, query, config)
+        )
+    else:
+        phrase_matches, timing = timed_result("Protected phrase detection skipped", skip_phrase_matches)
+    timings.append(timing)
     retrieval, timing = await async_timed_result(
         "Hybrid retrieval",
-        parallel_retrieval(engine, client, query, config, phrase_matching=phrase_matching),
+        parallel_retrieval(engine, client, query, config),
     )
-    phrase_matches = retrieval.phrase_matches
     timings.extend(retrieval.timings)
     timings.append(timing)
     fused, timing = timed_result(
@@ -129,7 +134,7 @@ async def search_ebooks(
         rank_constant=config.rrf_rank_constant,
     )
     timings.append(timing)
-    if phrase_matching:
+    if phrase_matching_enabled:
         fused, timing = await async_timed_result(
             "Phrase mention boost",
             apply_phrase_mention_boosts(engine, fused, phrase_matches, config.phrase_hit_boost),
@@ -144,43 +149,50 @@ async def search_ebooks(
     timings.append(timing)
     response = replace(response, timings=tuple(timings), phrase_matches=tuple(phrase_matches))
     logger.info(
-        f"ebook_search_complete vector_candidates={len(retrieval.vector_results)} "
-        f"lexical_candidates={len(retrieval.lexical_results)} fused_candidates={len(fused)} {phrase_matching=} "
-        f"phrase_matches={len(phrase_matches)} returned={len(response.results)} {response.rank_label=} "
-        f"{response.total_runtime_ms=:.1f}"
+        "ebook_search_complete vector_candidates=%s lexical_candidates=%s "
+        "fused_candidates=%s phrase_matching=%s phrase_matches=%s returned=%s rank_label=%s runtime_ms=%.1f",
+        len(retrieval.vector_results),
+        len(retrieval.lexical_results),
+        len(fused),
+        phrase_matching_enabled,
+        len(phrase_matches),
+        len(response.results),
+        response.rank_label,
+        response.total_runtime_ms,
     )
     return response
 
 
+def skip_phrase_matches() -> list[HydratedPhraseMatch]:
+    """Return no protected phrase matches when phrase matching is disabled."""
+    logger.info("ebook_protected_phrase_detection_skipped")
+    return []
+
+
 async def query_phrase_matches(
     engine: AsyncEngine,
     query: str,
     config: EbookSearchConfig,
-    *,
-    phrase_matching: bool,
-) -> list[PhraseMatch]:
+) -> list[HydratedPhraseMatch]:
     """Detect protected phrases in a query without making search fail when phrase tables are unavailable."""
-    if not phrase_matching:
-        logger.info("ebook_protected_phrase_detection_skipped")
-        return []
     try:
         async with AsyncSession(engine) as session:
             return await detect_protected_phrases_for_query(session, query, config)
     except SQLAlchemyError as error:
-        logger.warning(f"ebook_protected_phrase_detection_unavailable {error=}")
+        logger.warning("ebook_protected_phrase_detection_unavailable error=%s", error)
         return []
 
 
 def skip_phrase_mention_boosts(candidates: list[SearchResult]) -> list[SearchResult]:
     """Return candidates unchanged when phrase matching is disabled."""
-    logger.info(f"ebook_phrase_boost_skipped candidates={len(candidates)}")
+    logger.info("ebook_phrase_boost_skipped candidates=%s", len(candidates))
     return candidates
 
 
 async def apply_phrase_mention_boosts(
     engine: AsyncEngine,
     candidates: list[SearchResult],
-    phrase_matches: Sequence[PhraseMatch],
+    phrase_matches: Sequence[HydratedPhraseMatch],
     phrase_hit_boost: float,
 ) -> list[SearchResult]:
     """Boost retrieved chunks that have indexed mentions for detected protected phrases."""
@@ -193,7 +205,7 @@ async def apply_phrase_mention_boosts(
         async with AsyncSession(engine) as session:
             phrase_hits = await phrase_hits_for_chunks(session, chunk_ids=chunk_ids, phrase_ids=phrase_ids)
     except SQLAlchemyError as error:
-        logger.warning(f"ebook_phrase_boost_unavailable {error=}")
+        logger.warning("ebook_phrase_boost_unavailable error=%s", error)
         return candidates
 
     if not phrase_hits:
@@ -235,41 +247,28 @@ async def parallel_retrieval(
     client: httpx.AsyncClient,
     query: str,
     config: EbookSearchConfig,
-    *,
-    phrase_matching: bool,
 ) -> RetrievalResponse:
-    """Run vector, BM25, and protected phrase retrieval concurrently with separate database sessions.
+    """Run vector and BM25 candidate retrieval concurrently with separate database sessions.
 
     BM25 scoring is pure CPU work over the cached corpus, so it runs in a worker thread
-    instead of on the event loop. Protected phrase detection only depends on the query, so
-    it joins the gather as a third task and returns immediately when phrase matching is disabled.
+    instead of on the event loop.
     """
-    phrase_timing_name = "Protected phrase detection" if phrase_matching else "Protected phrase detection skipped"
-    (
-        (vector_results, vector_timing),
-        (lexical_results, lexical_timing),
-        (phrase_matches, phrase_timing),
-    ) = await asyncio.gather(
+    (vector_results, vector_timing), (lexical_results, lexical_timing) = await asyncio.gather(
         async_timed_result("Embedding + vector search", vector_candidates(engine, client, query, config)),
         async_timed_result("BM25 search", asyncio.to_thread(bm25_candidates, query, config)),
-        async_timed_result(
-            phrase_timing_name,
-            query_phrase_matches(engine, query, config, phrase_matching=phrase_matching),
-        ),
     )
 
     logger.info(
-        f"ebook_parallel_retrieval_complete vector_candidates={len(vector_results)} "
-        f"lexical_candidates={len(lexical_results)} phrase_matches={len(phrase_matches)}"
+        "ebook_parallel_retrieval_complete vector_candidates=%s lexical_candidates=%s",
+        len(vector_results),
+        len(lexical_results),
     )
     return RetrievalResponse(
         vector_results=vector_results,
         lexical_results=lexical_results,
-        phrase_matches=phrase_matches,
         timings=(
             replace(vector_timing, counts_toward_total=False),
             replace(lexical_timing, counts_toward_total=False),
-            replace(phrase_timing, counts_toward_total=False),
         ),
     )
 
@@ -280,7 +279,7 @@ def skip_rerank(
     config: EbookSearchConfig,
 ) -> SearchResponse:
     """Return fused hybrid results without reranking."""
-    logger.info(f"ebook_rerank_skipped candidates={len(candidates)}")
+    logger.info("ebook_rerank_skipped candidates=%s", len(candidates))
     return SearchResponse(query=query, results=candidates[: config.top_k], rank_label="Hybrid")
 
 
@@ -293,8 +292,9 @@ async def apply_rerank(
     """Rerank already-fused hybrid candidates."""
     reranked = await rerank_chunks(client, query, candidates[: config.rerank.candidates], config.rerank)
     logger.info(
-        f"ebook_rerank_complete input_candidates={min(len(candidates), config.rerank.candidates)} "
-        f"returned={len(reranked)}"
+        "ebook_rerank_complete input_candidates=%s returned=%s",
+        min(len(candidates), config.rerank.candidates),
+        len(reranked),
     )
     return SearchResponse(
         query=query,
@@ -332,7 +332,13 @@ async def vector_candidates(
         score = (literal(1.0) - distance).label("score")
         statement = (
             select(
-                *CHUNK_RECORD_COLUMNS,
+                EbookChunk.id.label("chunk_id"),
+                EbookChunk.text.label("text"),
+                EbookSource.id.label("source_id"),
+                EbookSource.title.label("source_title"),
+                EbookSource.author.label("source_author"),
+                EbookChapter.title.label("chapter_title"),
+                EbookChunk.page_label.label("page_label"),
                 score,
             )
             .select_from(embedding_table)
@@ -346,7 +352,10 @@ async def vector_candidates(
         rows = (await session.execute(statement)).mappings()
         results = [search_result_from_row(row) for row in rows]
         logger.info(
-            f"ebook_vector_search_complete {config.embedding_model=} {model.dimension=} candidates={len(results)}"
+            "ebook_vector_search_complete model=%s dimension=%s candidates=%s",
+            config.embedding_model,
+            model.dimension,
+            len(results),
         )
         return results
 
@@ -356,7 +365,7 @@ def bm25_candidates(query: str, config: EbookSearchConfig) -> list[SearchResult]
     try:
         corpus = load_bm25_corpus(config)
     except BM25CorpusUnavailableError as error:
-        logger.warning(f"ebook_bm25_index_unavailable_skipping {error=}")
+        logger.warning("ebook_bm25_index_unavailable_skipping error=%s", error)
         return []
 
     if not corpus.records:
@@ -371,7 +380,12 @@ def bm25_candidates(query: str, config: EbookSearchConfig) -> list[SearchResult]
     ]
 
     max_score = results[0].bm25_score if results else 0.0
-    logger.info(f"ebook_bm25_search_complete corpus={len(corpus.records)} candidates={len(results)} {max_score=:.6f}")
+    logger.info(
+        "ebook_bm25_search_complete corpus=%s candidates=%s max_score=%.6f",
+        len(corpus.records),
+        len(results),
+        max_score,
+    )
     return results
 
 

python/orm/richie/ebook.py

@@ -167,8 +167,11 @@ class EbookCandidatePhrase(TableBase):
     token_count: Mapped[int]
     source_raw_ngram: Mapped[bool] = mapped_column(default=False)
     source_yake: Mapped[bool] = mapped_column(default=False)
+    source_spacy_ner: Mapped[bool] = mapped_column(default=False)
+    source_spacy_noun_chunk: Mapped[bool] = mapped_column(default=False)
     source_capitalized: Mapped[bool] = mapped_column(default=False)
     source_metadata: Mapped[bool] = mapped_column(default=False)
+    spacy_label: Mapped[str | None]
     raw_count: Mapped[int] = mapped_column(default=0)
     chapter_count: Mapped[int] = mapped_column(default=0)
     yake_score: Mapped[float | None]

shell.nix

@@ -3,20 +3,6 @@
   ...
 }:
 {
-  # For running the uv-managed ebook-search venv outside the container:
-  # PyPI manylinux wheels (numpy via bm25s) expect libstdc++.so.6 on the
-  # loader path, which NixOS does not provide globally.
-  ebook-search = pkgs.mkShell {
-    nativeBuildInputs = with pkgs; [
-      my_python
-      uv
-    ];
-    LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
-      pkgs.stdenv.cc.cc.lib
-      pkgs.zlib
-    ];
-  };
-
   default = pkgs.mkShell {
     NIX_CONFIG = "extra-experimental-features = nix-command flakes ca-derivations";
     nativeBuildInputs = with pkgs; [

systems/jeeves/runners/default.nix

@@ -1,9 +1,6 @@
 { ... }:
 {
-  imports = [
-    ./nix_builder.nix
-    ./uv_cache_prune.nix
-  ];
+  imports = [ ./nix_builder.nix ];
 
   services.nix_builder.containers = {
     nix-builder-00.enable = true;

systems/jeeves/runners/nix_builder.nix

@@ -62,11 +62,6 @@ in
             mountPoint = "/run/secrets/gitea-runners";
             isReadOnly = true;
           };
-          uv-cache = {
-            hostPath = vars.uv_cache;
-            mountPoint = "/var/cache/uv";
-            isReadOnly = false;
-          };
         };
         config =
           {
@@ -158,20 +153,14 @@ in
       }
     ) cfg.containers;
 
-    systemd = {
-      services = builtins.listToAttrs (
-        map (name: {
-          name = "container@${name}";
-          value = {
-            requires = [ "gitea.service" ];
-            after = [ "gitea.service" ];
-          };
-        }) (builtins.attrNames (filterAttrs (_: c: c.enable) cfg.containers))
-      );
-
-      tmpfiles.rules = [
-        "d ${vars.uv_cache} 0755 ${runnerUsername} ${runnerUsername} - -"
-      ];
-    };
+    systemd.services = builtins.listToAttrs (
+      map (name: {
+        name = "container@${name}";
+        value = {
+          requires = [ "gitea.service" ];
+          after = [ "gitea.service" ];
+        };
+      }) (builtins.attrNames (filterAttrs (_: c: c.enable) cfg.containers))
+    );
   };
 }

systems/jeeves/runners/uv_cache_prune.nix

@@ -1,28 +0,0 @@
-{ pkgs, ... }:
-let
-  vars = import ../vars.nix;
-  runnerUsername = "gitea-runner";
-in
-{
-  systemd = {
-    services.uv-cache-prune = {
-      description = "Prune the shared gitea runner uv cache";
-      environment.UV_CACHE_DIR = vars.uv_cache;
-      serviceConfig = {
-        Type = "oneshot";
-        User = runnerUsername;
-        Group = runnerUsername;
-        ExecStart = "${pkgs.uv}/bin/uv cache prune";
-      };
-    };
-
-    timers.uv-cache-prune = {
-      description = "Monthly prune of the shared gitea runner uv cache";
-      wantedBy = [ "timers.target" ];
-      timerConfig = {
-        OnCalendar = "*-*-01 04:00:00";
-        Persistent = true;
-      };
-    };
-  };
-}

systems/jeeves/scripts/zfs.sh

@@ -30,7 +30,6 @@ sudo zfs create media/secure/share -o mountpoint=/zfs/media/share -o exec=off
 # scratch datasets
 sudo zfs create scratch/kafka -o mountpoint=/zfs/scratch/kafka -o recordsize=1M
 sudo zfs create scratch/transmission -o mountpoint=/zfs/scratch/transmission -o recordsize=16k -o sync=disabled -o redundant_metadata=none
-sudo zfs create scratch/uv_cache -o mountpoint=/zfs/scratch/uv_cache
 
 # storage datasets
 sudo zfs create storage/ollama -o recordsize=1M -o compression=zstd-19 -o sync=disabled

systems/jeeves/services/contact_api.nix

@@ -0,0 +1,43 @@
+{
+  pkgs,
+  inputs,
+  ...
+}:
+{
+  networking.firewall.allowedTCPPorts = [
+    8069
+  ];
+  systemd.services.contact-api = {
+    description = "Contact Database API";
+    after = [
+      "postgresql.service"
+      "network.target"
+    ];
+    requires = [ "postgresql.service" ];
+    wantedBy = [ "multi-user.target" ];
+
+    environment = {
+      PYTHONPATH = "${inputs.self}";
+      POSTGRES_DB = "richie";
+      POSTGRES_HOST = "/run/postgresql";
+      POSTGRES_USER = "richie";
+      POSTGRES_PORT = "5432";
+    };
+
+    serviceConfig = {
+      Type = "simple";
+      ExecStart = "${pkgs.my_python}/bin/python -m python.api.main --host 192.168.90.40 --port 8069";
+      Restart = "on-failure";
+      RestartSec = "5s";
+      StandardOutput = "journal";
+      StandardError = "journal";
+      NoNewPrivileges = true;
+      ProtectSystem = "strict";
+      ProtectHome = "read-only";
+      PrivateTmp = true;
+      ReadOnlyPaths = [
+        "${inputs.self}"
+      ];
+    };
+  };
+}

systems/jeeves/snapshot_config.toml

@@ -121,9 +121,3 @@ monthly = 2
 hourly = 0
 daily = 0
 monthly = 0
-
-["scratch/uv_cache"]
-15_min = 2
-hourly = 0
-daily = 0
-monthly = 0

systems/jeeves/vars.nix

@@ -17,6 +17,5 @@ in
   transmission = "${zfs_storage}/transmission";
   ollama = "${zfs_storage}/ollama";
   transmission_scratch = "${zfs_scratch}/transmission";
-  uv_cache = "${zfs_scratch}/uv_cache";
   kafka = "${zfs_scratch}/kafka";
 }

tests/ebook_search/test_core.py

@@ -25,7 +25,7 @@ from python.ebook_search.bm25_corpus import (
     score_bm25_corpus,
     write_bm25_corpus,
 )
-from python.ebook_search.config import EbookSearchConfig, RerankConfig, load_config
+from python.ebook_search.config import EbookSearchConfig, RerankConfig, load_config, normalize_embedding_model
 from python.ebook_search.embeddings import MODEL_DIMENSIONS, ensure_embedding_models
 from python.ebook_search.ingest import chunk_text, find_existing_source
 from python.ebook_search.search import (
@@ -452,25 +452,24 @@ def test_1024_embedding_table_has_cosine_hnsw_index() -> None:
 
 def test_embedding_model_aliases_normalize_to_provider_names(mocker: MockerFixture) -> None:
     mocker.patch.dict(environ, {}, clear=False)
-    environ.pop("EBOOK_SEARCH_EMBEDDING_MODEL", None)
 
-    assert load_config().embedding_model == "qwen3-embedding-0.6b"
+    assert normalize_embedding_model() == "qwen3-embedding-0.6b"
 
     environ["EBOOK_SEARCH_EMBEDDING_MODEL"] = "qwen3-embedding-0.6b"
-    assert load_config().embedding_model == "qwen3-embedding-0.6b"
+    assert normalize_embedding_model() == "qwen3-embedding-0.6b"
 
     environ["EBOOK_SEARCH_EMBEDDING_MODEL"] = "Qwen3-Embedding-0.6B"
-    assert load_config().embedding_model == "qwen3-embedding-0.6b"
+    assert normalize_embedding_model() == "qwen3-embedding-0.6b"
 
     environ["EBOOK_SEARCH_EMBEDDING_MODEL"] = "Qwen/Qwen3-Embedding-4B"
 
-    assert load_config().embedding_model == "qwen3-embedding-4b"
+    assert normalize_embedding_model() == "qwen3-embedding-4b"
 
     environ["EBOOK_SEARCH_EMBEDDING_MODEL"] = "qwen3-embedding:8b"
-    assert load_config().embedding_model == "qwen3-embedding-8b"
+    assert normalize_embedding_model() == "qwen3-embedding-8b"
 
     environ["EBOOK_SEARCH_EMBEDDING_MODEL"] = "qwen3-embedding-8b"
-    assert load_config().embedding_model == "qwen3-embedding-8b"
+    assert normalize_embedding_model() == "qwen3-embedding-8b"
 
 
 def test_answer_generation_is_enabled_by_default(mocker: MockerFixture) -> None:

tests/ebook_search/test_http.py

@@ -10,7 +10,6 @@ import pytest
 from python.ebook_search.answer import answer_query
 from python.ebook_search.config import EbookSearchConfig, RerankConfig
 from python.ebook_search.embeddings import embed_texts
-from python.ebook_search.llm_interface import check_chat_endpoint, check_embedding_endpoint
 from python.ebook_search.search import SearchResult
 
 if TYPE_CHECKING:
@@ -24,44 +23,6 @@ def make_async_client(mocker: MockerFixture, fake_post) -> httpx.AsyncClient:
     return client
 
 
-async def test_model_endpoint_checks_share_http_probe(mocker: MockerFixture) -> None:
-    client = mocker.MagicMock(spec=httpx.AsyncClient)
-    response = mocker.MagicMock(spec=httpx.Response)
-    client.get = mocker.AsyncMock(return_value=response)
-    config = EbookSearchConfig(
-        rerank=RerankConfig(enabled=False),
-        embedding_base_url="https://embedding.example/v1/",
-        vllm_base_url="https://chat.example/v1/",
-        vllm_api_key="secret",
-    )
-
-    assert await check_embedding_endpoint(client, config, timeout_seconds=2.0)
-    assert await check_chat_endpoint(client, config, timeout_seconds=3.0)
-    assert client.get.await_args_list == [
-        mocker.call("https://embedding.example/v1/models", headers={}, timeout=2.0),
-        mocker.call(
-            "https://chat.example/v1/models",
-            headers={"Authorization": "Bearer secret"},
-            timeout=3.0,
-        ),
-    ]
-    assert response.raise_for_status.call_count == 2
-
-
-async def test_model_endpoint_checks_report_http_failures(mocker: MockerFixture) -> None:
-    client = mocker.MagicMock(spec=httpx.AsyncClient)
-    client.get = mocker.AsyncMock(
-        side_effect=[
-            httpx.ConnectError("embedding offline"),
-            httpx.ConnectError("chat offline"),
-        ]
-    )
-    config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
-
-    assert not await check_embedding_endpoint(client, config)
-    assert not await check_chat_endpoint(client, config)
-
-
 async def test_answer_query_uses_httpx_chat_completions(mocker: MockerFixture) -> None:
     captured: dict[str, object] = {}
 

tests/ebook_search/test_protected_phrases.py

@@ -2,16 +2,15 @@
 
 from __future__ import annotations
 
-from concurrent.futures import ThreadPoolExecutor
 from datetime import UTC, datetime
 from typing import TYPE_CHECKING
 
 import pytest
-from sqlalchemy import event, select
+from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, create_async_engine
+from sqlalchemy.pool import StaticPool
 
 from python.ebook_search.config import EbookSearchConfig
-from python.ebook_search.protected_phrases import generate_ngrams
 from python.ebook_search.protected_phrases.config import (
     get_bad_ends,
     get_most_common_words,
@@ -33,12 +32,15 @@ from python.ebook_search.protected_phrases.matching import (
     detect_protected_phrases_for_query,
     index_chunk_phrase_mentions,
     load_phrase_lookup,
+    phrase_hit_counts_for_chunks,
+    phrase_hits_for_chunks,
     resolve_overlaps,
 )
 from python.ebook_search.protected_phrases.models import (
+    ChunkPhraseHit,
+    HydratedPhraseMatch,
     LLMJudgment,
     PhraseCandidate,
-    PhraseMatch,
 )
 from python.ebook_search.protected_phrases.store import book_ids_pending_first_judgment, corpus_phrase_stats
 from python.ebook_search.protected_phrases.text_normalization import normalize_text, tokenize
@@ -53,42 +55,25 @@ from python.orm.richie import (
 )
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncGenerator, Generator
-    from pathlib import Path
+    from collections.abc import AsyncGenerator
 
     from pytest_mock import MockerFixture
 
 
 @pytest.fixture
-async def engine(tmp_path: Path) -> AsyncGenerator[AsyncEngine]:
-    """Create a file-backed async database engine that worker threads can also reach."""
-    test_engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'phrases.db'}")
+async def engine() -> AsyncGenerator[AsyncEngine]:
+    """Create a shared in-memory async database engine for phrase tests."""
+    test_engine = create_async_engine(
+        "sqlite+aiosqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
     async with test_engine.begin() as connection:
         await connection.run_sync(RichieBase.metadata.create_all)
     yield test_engine
     await test_engine.dispose()
 
 
-@pytest.fixture
-def worker_pool(engine: AsyncEngine, mocker: MockerFixture) -> Generator[ThreadPoolExecutor]:
-    """Run pooled candidate generation in threads against the test database.
-
-    Spawned worker processes can see neither the test database nor test patches, so the shared
-    extraction pool is replaced with a thread pool and worker engines are built for the test
-    database instead of from Postgres environment variables.
-    """
-    thread_pool = ThreadPoolExecutor(max_workers=1)
-    database_url = engine.url.render_as_string(hide_password=False)
-    mocker.patch.object(generate_ngrams, "get_extraction_pool", return_value=thread_pool)
-    mocker.patch.object(
-        generate_ngrams,
-        "get_async_postgres_engine",
-        side_effect=lambda **_kwargs: create_async_engine(database_url),
-    )
-    yield thread_pool
-    thread_pool.shutdown(wait=True)
-
-
 @pytest.fixture
 async def session(engine: AsyncEngine) -> AsyncGenerator[AsyncSession]:
     """Provide a session on the shared in-memory database."""
@@ -184,89 +169,26 @@ def test_score_candidate_weights_metadata_source(config: EbookSearchConfig) -> N
     assert score_candidate(metadata_only, config) == 2.0 + 0.5
 
 
-async def test_load_phrase_lookup_loads_phrases_and_aliases_with_one_query(
-    engine: AsyncEngine,
-    session: AsyncSession,
-) -> None:
-    """One joined query should load phrases with zero, one, or many aliases."""
-    source = await add_source(session)
-    aliased_phrase = await add_phrase(session, source.id, phrase_text="lock in", phrase_norm="lock in")
-    plain_phrase = await add_phrase(session, source.id, phrase_text="mage king", phrase_norm="mage king")
-    session.add_all(
-        [
-            EbookPhraseAlias(phrase_id=aliased_phrase.id, alias_text="locked in", alias_norm="locked in"),
-            EbookPhraseAlias(
-                phrase_id=aliased_phrase.id,
-                alias_text="locked completely in",
-                alias_norm="locked completely in",
-            ),
-        ]
-    )
-    await session.commit()
-
-    statements: list[str] = []
-
-    def record_statement(*args: object) -> None:
-        statements.append(str(args[2]))
-
-    event.listen(engine.sync_engine, "before_cursor_execute", record_statement)
-    try:
-        lookup = await load_phrase_lookup(session, EbookSearchConfig(phrase_max_tokens=2), book_id=source.id)
-    finally:
-        event.remove(engine.sync_engine, "before_cursor_execute", record_statement)
-
-    assert lookup.phrase_ids_by_norm == {
-        "lock in": (aliased_phrase.id,),
-        "locked completely in": (aliased_phrase.id,),
-        "locked in": (aliased_phrase.id,),
-        "mage king": (plain_phrase.id,),
-    }
-    assert lookup.phrases_by_id == {aliased_phrase.id: aliased_phrase, plain_phrase.id: plain_phrase}
-    assert lookup.max_tokens == 3
-    assert len(statements) == 1
-
-
-async def test_detect_protected_phrases_queries_canonical_and_alias_matches_once(
-    engine: AsyncEngine,
+async def test_detect_protected_phrases_hydrates_alias_matches(
     session: AsyncSession,
     config: EbookSearchConfig,
 ) -> None:
-    """Query detection should hydrate repeated canonical and alias matches with one statement."""
+    """Query detection should use RAM aliases and hydrate phrase metadata from the DB."""
     source = await add_source(session)
     phrase = await add_phrase(session, source.id, phrase_text="lock in", phrase_norm="lock in")
-    await add_phrase(session, source.id, phrase_text="mage king", phrase_norm="mage king")
     session.add(EbookPhraseAlias(phrase_id=phrase.id, alias_text="locked in", alias_norm="locked in"))
     await session.commit()
 
-    statements: list[str] = []
+    matches = await detect_protected_phrases_for_query(session, "what is locked-in", config)
 
-    def record_statement(*args: object) -> None:
-        statements.append(str(args[2]))
-
-    event.listen(engine.sync_engine, "before_cursor_execute", record_statement)
-    try:
-        matches = await detect_protected_phrases_for_query(
-            session,
-            "lock-in then locked-in and lock-in",
-            config,
-        )
-        empty_matches = await detect_protected_phrases_for_query(session, "", config)
-    finally:
-        event.remove(engine.sync_engine, "before_cursor_execute", record_statement)
-
-    assert [(match.phrase_text, match.matched_norm, match.start_token, match.end_token) for match in matches] == [
-        ("lock in", "lock in", 0, 2),
-        ("lock in", "locked in", 3, 5),
-        ("lock in", "lock in", 6, 8),
+    assert [(match.phrase_text, match.canonical_id, match.phrase_type) for match in matches] == [
+        ("lock in", "condition:lock_in", "fictional_condition")
     ]
-    assert empty_matches == []
-    assert len(statements) == 1
-    assert " UNION " in statements[0]
 
 
 def test_resolve_overlaps_keeps_independent_nested_phrases() -> None:
     """Overlap resolution should keep useful nested concepts when metadata permits it."""
-    child = phrase_match(
+    child = hydrated_match(
         phrase_id=1,
         phrase_text="mage king",
         canonical_id="title:mage_king",
@@ -274,7 +196,7 @@ def test_resolve_overlaps_keeps_independent_nested_phrases() -> None:
         end_token=5,
         allow_nested=True,
     )
-    parent = phrase_match(
+    parent = hydrated_match(
         phrase_id=2,
         phrase_text="mage king of mars",
         canonical_id="entity:mage_king_of_mars",
@@ -291,7 +213,7 @@ def test_resolve_overlaps_keeps_independent_nested_phrases() -> None:
 
 def test_resolve_overlaps_suppresses_weaker_same_canonical_match() -> None:
     """Same-canonical overlaps should keep the stronger evidence."""
-    weak = phrase_match(
+    weak = hydrated_match(
         phrase_id=1,
         phrase_text="lock",
         canonical_id="condition:lock_in",
@@ -299,7 +221,7 @@ def test_resolve_overlaps_suppresses_weaker_same_canonical_match() -> None:
         end_token=3,
         importance=0.2,
     )
-    strong = phrase_match(
+    strong = hydrated_match(
         phrase_id=2,
         phrase_text="lock in",
         canonical_id="condition:lock_in",
@@ -334,30 +256,22 @@ async def test_index_chunk_phrase_mentions_uses_normalized_window_lookup(
     await session.commit()
     lookup = await load_phrase_lookup(session, config, book_id=source.id)
 
-    statements: list[str] = []
-
-    def record_statement(*args: object) -> None:
-        statements.append(str(args[2]))
-
-    event.listen(session.bind.sync_engine, "before_cursor_execute", record_statement)
-    try:
-        count = index_chunk_phrase_mentions(session, chunk, lookup=lookup)
-    finally:
-        event.remove(session.bind.sync_engine, "before_cursor_execute", record_statement)
+    count = await index_chunk_phrase_mentions(session, chunk, lookup=lookup)
     await session.commit()
 
-    assert statements == []
     mention = await session.scalar(select(EbookChunkPhraseMention))
     assert count == 1
     assert mention is not None
     assert mention.chunk_id == chunk.id
     assert mention.phrase_id == phrase.id
     assert chunk.text[mention.start_char : mention.end_char] == "lock-in"
+    assert await phrase_hit_counts_for_chunks(session, chunk_ids=[chunk.id], phrase_ids=[phrase.id]) == {chunk.id: 1}
+    assert await phrase_hits_for_chunks(session, chunk_ids=[chunk.id], phrase_ids=[phrase.id]) == {
+        chunk.id: (ChunkPhraseHit(phrase_id=phrase.id, phrase_text="lock in", mention_count=1),)
+    }
 
 
-@pytest.mark.usefixtures("worker_pool")
 async def test_generate_candidate_phrases_for_books_stores_unjudged_candidates(
-    engine: AsyncEngine,
     session: AsyncSession,
     config: EbookSearchConfig,
 ) -> None:
@@ -386,7 +300,8 @@ async def test_generate_candidate_phrases_for_books_stores_unjudged_candidates(
         }
     )
 
-    result = await generate_candidate_phrases_for_books(engine, build_config)
+    result = await generate_candidate_phrases_for_books(session, build_config)
+    await session.commit()
 
     candidate = await session.scalar(select(EbookCandidatePhrase))
     assert result.books_seen == 1
@@ -398,9 +313,7 @@ async def test_generate_candidate_phrases_for_books_stores_unjudged_candidates(
     assert await session.scalar(select(EbookProtectedPhrase)) is None
 
 
-@pytest.mark.usefixtures("worker_pool")
 async def test_generate_candidate_phrases_for_books_filters_one_token_and_one_use_candidates(
-    engine: AsyncEngine,
     session: AsyncSession,
     config: EbookSearchConfig,
 ) -> None:
@@ -464,7 +377,8 @@ async def test_generate_candidate_phrases_for_books_filters_one_token_and_one_us
         }
     )
 
-    result = await generate_candidate_phrases_for_books(engine, build_config)
+    result = await generate_candidate_phrases_for_books(session, build_config)
+    await session.commit()
 
     candidates = list(await session.scalars(select(EbookCandidatePhrase)))
     phrase_norms = {candidate.phrase_norm for candidate in candidates}
@@ -480,9 +394,7 @@ async def test_generate_candidate_phrases_for_books_filters_one_token_and_one_us
     assert all(not all(token in common_words for token in candidate.phrase_norm.split()) for candidate in candidates)
 
 
-@pytest.mark.usefixtures("worker_pool")
 async def test_generate_candidate_phrases_for_books_commits_after_each_book(
-    engine: AsyncEngine,
     session: AsyncSession,
     config: EbookSearchConfig,
     mocker: MockerFixture,
@@ -519,7 +431,7 @@ async def test_generate_candidate_phrases_for_books_commits_after_each_book(
         ]
     )
     await session.commit()
-    commit_spy = mocker.spy(AsyncSession, "commit")
+    commit_spy = mocker.spy(session, "commit")
     build_config = config.model_copy(
         update={
             "protected_phrase_max_candidates_per_book": 1,
@@ -528,91 +440,13 @@ async def test_generate_candidate_phrases_for_books_commits_after_each_book(
         }
     )
 
-    result = await generate_candidate_phrases_for_books(engine, build_config)
+    result = await generate_candidate_phrases_for_books(session, build_config)
 
     assert result.books_built == 2
     assert result.candidate_phrases == 2
     assert commit_spy.call_count == 2
 
 
-@pytest.mark.usefixtures("worker_pool")
-async def test_generate_candidate_phrases_for_books_failure_keeps_committed_books(
-    engine: AsyncEngine,
-    session: AsyncSession,
-    config: EbookSearchConfig,
-    mocker: MockerFixture,
-) -> None:
-    """A failing book should be logged and skipped while the other books stay committed."""
-    first = await add_source(session)
-    second = await add_source(session, file_path="/library/book-2.epub", file_sha256="z" * 64)
-    session.add_all(
-        [
-            EbookChunk(
-                id=1,
-                source_id=first.id,
-                chapter_id=None,
-                chunk_index=0,
-                text="lock in lock in lock in",
-                token_start=0,
-                token_count=6,
-                page_label=None,
-                content_sha256="f" * 64,
-                search_text="lock in lock in lock in",
-            ),
-            EbookChunk(
-                id=2,
-                source_id=second.id,
-                chapter_id=None,
-                chunk_index=0,
-                text="mage king mage king mage king",
-                token_start=0,
-                token_count=6,
-                page_label=None,
-                content_sha256="g" * 64,
-                search_text="mage king mage king mage king",
-            ),
-        ]
-    )
-    await session.commit()
-    build_config = config.model_copy(
-        update={
-            "protected_phrase_max_candidates_per_book": 1,
-            "phrase_min_tokens": 2,
-            "phrase_max_tokens": 2,
-        }
-    )
-    real_store = generate_ngrams.store_candidate_phrases_for_book
-    first_book_id = first.id
-    second_book_id = second.id
-
-    async def store_failing_second_book(
-        store_session: AsyncSession,
-        book_id: int,
-        series_id: int | None,
-        limited_candidates: list[PhraseCandidate],
-        store_config: EbookSearchConfig,
-        *,
-        replace_all: bool = False,
-    ) -> int:
-        if book_id == second_book_id:
-            message = "storage exploded"
-            raise RuntimeError(message)
-        return await real_store(
-            store_session, book_id, series_id, limited_candidates, store_config, replace_all=replace_all
-        )
-
-    mocker.patch.object(generate_ngrams, "store_candidate_phrases_for_book", side_effect=store_failing_second_book)
-
-    result = await generate_candidate_phrases_for_books(engine, build_config)
-
-    stored_book_ids = set((await session.scalars(select(EbookCandidatePhrase.book_id))).all())
-    assert stored_book_ids == {first_book_id}
-    assert result.books_seen == 2
-    assert result.books_built == 1
-    assert result.candidate_phrases == 1
-
-
-@pytest.mark.usefixtures("worker_pool")
 async def test_judge_candidate_phrases_for_books_promotes_stored_candidates(
     engine: AsyncEngine,
     session: AsyncSession,
@@ -646,7 +480,8 @@ async def test_judge_candidate_phrases_for_books_promotes_stored_candidates(
             "phrase_judge_phrase_workers": 1,
         }
     )
-    await generate_candidate_phrases_for_books(engine, build_config)
+    await generate_candidate_phrases_for_books(session, build_config)
+    await session.commit()
     mocker.patch(
         "python.ebook_search.protected_phrases.judge_ngrams.judge_candidate_async",
         return_value=LLMJudgment(
@@ -678,7 +513,6 @@ async def test_judge_candidate_phrases_for_books_promotes_stored_candidates(
     assert mention.phrase_id == phrase.id
 
 
-@pytest.mark.usefixtures("worker_pool")
 async def test_judge_candidate_phrases_for_books_logs_and_continues_after_book_failure(
     engine: AsyncEngine,
     session: AsyncSession,
@@ -727,7 +561,8 @@ async def test_judge_candidate_phrases_for_books_logs_and_continues_after_book_f
             "phrase_judge_phrase_workers": 1,
         }
     )
-    await generate_candidate_phrases_for_books(engine, build_config)
+    await generate_candidate_phrases_for_books(session, build_config)
+    await session.commit()
 
     def judge_or_fail(_client: object, _config: EbookSearchConfig, candidate: PhraseCandidate) -> LLMJudgment:
         if candidate.phrase_norm == "lock in":
@@ -970,7 +805,6 @@ async def test_recalculate_candidate_phrases_for_book_removes_old_phrase_data(
 
     result = await recalculate_candidate_phrases_for_book(session, source, build_config)
 
-    session.expire_all()
     candidates = list(await session.scalars(select(EbookCandidatePhrase)))
     assert result.deleted_candidates == 1
     assert result.deleted_protected_phrases == 1
@@ -983,38 +817,6 @@ async def test_recalculate_candidate_phrases_for_book_removes_old_phrase_data(
     assert await session.scalar(select(EbookChunkPhraseMention)) is None
 
 
-async def test_recalculate_candidate_phrases_for_book_aborts_without_chapters(
-    session: AsyncSession,
-    config: EbookSearchConfig,
-) -> None:
-    """Recalculating a book with no indexed chapters should raise and leave phrase data intact."""
-    source = await add_source(session)
-    existing_candidate = EbookCandidatePhrase(
-        book_id=source.id,
-        series_id=None,
-        phrase_text="old phrase",
-        phrase_norm="old phrase",
-        token_count=2,
-        source_raw_ngram=True,
-        raw_count=1,
-        chapter_count=1,
-        candidate_score=1.0,
-        llm_judged=False,
-    )
-    session.add(existing_candidate)
-    await session.flush()
-    existing_phrase = await add_phrase(session, source.id, phrase_text="old phrase", phrase_norm="old phrase")
-    await session.commit()
-    existing_candidate_id = existing_candidate.id
-    existing_phrase_id = existing_phrase.id
-
-    with pytest.raises(ValueError, match="no indexed chapters"):
-        await recalculate_candidate_phrases_for_book(session, source, config)
-
-    assert await session.scalar(select(EbookCandidatePhrase.id)) == existing_candidate_id
-    assert await session.scalar(select(EbookProtectedPhrase.id)) == existing_phrase_id
-
-
 async def test_corpus_phrase_stats_counts_phrases_and_book_coverage(session: AsyncSession) -> None:
     """Corpus stats should count phrases plus how many books are generated and fully judged."""
     unjudged_book = await add_source(session)
@@ -1126,7 +928,7 @@ async def add_phrase(
     return phrase
 
 
-def phrase_match(
+def hydrated_match(
     *,
     phrase_id: int,
     phrase_text: str,
@@ -1136,9 +938,9 @@ def phrase_match(
     importance: float = 0.8,
     allow_nested: bool = False,
     suppress_children: bool = True,
-) -> PhraseMatch:
-    """Build a metadata-backed phrase match for overlap tests."""
-    return PhraseMatch(
+) -> HydratedPhraseMatch:
+    """Build a hydrated match for overlap tests."""
+    return HydratedPhraseMatch(
         phrase_id=phrase_id,
         matched_norm=phrase_text,
         phrase_text=phrase_text,

tests/ebook_search/test_rag_pipeline.py

@@ -40,9 +40,7 @@ async def test_search_ebooks_runs_vector_and_bm25_in_parallel(mocker: MockerFixt
     mocker.patch("python.ebook_search.search.bm25_candidates", side_effect=fake_bm25_candidates)
     config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
 
-    response = await search_ebooks(
-        engine, mocker.Mock(), "what is parallel", config, rerank=False, phrase_matching=False
-    )
+    response = await search_ebooks(engine, mocker.Mock(), "what is parallel", config)
 
     timings = {step.name: step for step in response.timings}
     assert [result.chunk_id for result in response.results] == [1, 2]
@@ -52,37 +50,6 @@ async def test_search_ebooks_runs_vector_and_bm25_in_parallel(mocker: MockerFixt
     assert received_engines == [engine]
 
 
-async def test_search_ebooks_runs_phrase_detection_in_parallel_with_retrieval(mocker: MockerFixture) -> None:
-    """Phrase detection joins the retrieval gather instead of running before it."""
-    engine = create_async_engine("sqlite+aiosqlite:///:memory:")
-    phrase_started = Event()
-
-    async def fake_vector_candidates(_engine, _client, _query, _config):
-        """Return vector candidates only once phrase detection has started."""
-        assert await asyncio.to_thread(phrase_started.wait, 2)
-        return [SearchResult(chunk_id=1, text="vector", source_title="Vector", vector_score=0.9)]
-
-    async def fake_query_phrase_matches(_engine, _query, _config, *, phrase_matching):
-        """Record that phrase detection started and return no matches."""
-        assert phrase_matching is True
-        phrase_started.set()
-        return []
-
-    mocker.patch("python.ebook_search.search.vector_candidates", side_effect=fake_vector_candidates)
-    mocker.patch("python.ebook_search.search.bm25_candidates", return_value=[])
-    mocker.patch("python.ebook_search.search.query_phrase_matches", side_effect=fake_query_phrase_matches)
-    config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
-
-    response = await search_ebooks(
-        engine, mocker.Mock(), "what is parallel", config, rerank=False, phrase_matching=True
-    )
-
-    timings = {step.name: step for step in response.timings}
-    assert [result.chunk_id for result in response.results] == [1]
-    assert timings["Protected phrase detection"].counts_toward_total is False
-    assert timings["Hybrid retrieval"].counts_toward_total is True
-
-
 async def test_search_ebooks_skips_phrase_matching_when_disabled(mocker: MockerFixture) -> None:
     """Phrase matching can be disabled for one search request."""
     engine = create_async_engine("sqlite+aiosqlite:///:memory:")
@@ -91,38 +58,11 @@ async def test_search_ebooks_skips_phrase_matching_when_disabled(mocker: MockerF
         return_value=[SearchResult(chunk_id=1, text="vector", source_title="Vector", vector_score=0.9)],
     )
     mocker.patch("python.ebook_search.search.bm25_candidates", return_value=[])
-    detect_mock = mocker.patch("python.ebook_search.search.detect_protected_phrases_for_query")
+    detect_mock = mocker.patch("python.ebook_search.search.query_phrase_matches")
     boost_mock = mocker.patch("python.ebook_search.search.apply_phrase_mention_boosts")
     config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
 
-    response = await search_ebooks(
-        engine, mocker.Mock(), "what is parallel", config, rerank=False, phrase_matching=False
-    )
-
-    timing_names = {step.name for step in response.timings}
-    assert [result.chunk_id for result in response.results] == [1]
-    assert response.phrase_matches == ()
-    assert "Protected phrase detection skipped" in timing_names
-    assert "Phrase mention boost skipped" in timing_names
-    detect_mock.assert_not_called()
-    boost_mock.assert_not_called()
-
-
-async def test_search_ebooks_ignores_phrase_matching_when_config_disabled(mocker: MockerFixture) -> None:
-    """The config kill switch overrides a request that asks for phrase matching."""
-    engine = create_async_engine("sqlite+aiosqlite:///:memory:")
-    mocker.patch(
-        "python.ebook_search.search.vector_candidates",
-        return_value=[SearchResult(chunk_id=1, text="vector", source_title="Vector", vector_score=0.9)],
-    )
-    mocker.patch("python.ebook_search.search.bm25_candidates", return_value=[])
-    detect_mock = mocker.patch("python.ebook_search.search.detect_protected_phrases_for_query")
-    boost_mock = mocker.patch("python.ebook_search.search.apply_phrase_mention_boosts")
-    config = EbookSearchConfig(rerank=RerankConfig(enabled=False), phrase_matching_enabled=False)
-
-    response = await search_ebooks(
-        engine, mocker.Mock(), "what is parallel", config, rerank=False, phrase_matching=True
-    )
+    response = await search_ebooks(engine, mocker.Mock(), "what is parallel", config, phrase_matching=False)
 
     timing_names = {step.name for step in response.timings}
     assert [result.chunk_id for result in response.results] == [1]

tests/ebook_search/test_ui.py

@@ -567,8 +567,33 @@ def test_admin_page_shows_protected_phrase_stats(mocker: MockerFixture) -> None:
         assert value in response.text
 
 
+def test_ui_add_missing_phrases_generates_only_missing_books(mocker: MockerFixture) -> None:
+    captured: dict[str, object] = {}
+
+    def fake_generate(_session, _config, *, only_missing=False):
+        captured["only_missing"] = only_missing
+        return PhraseCandidateGenerationResult(books_seen=3, books_built=2, candidate_phrases=42)
+
+    mocker.patch(
+        "python.ebook_search.api.routes.admin.generate_candidate_phrases_for_books",
+        side_effect=fake_generate,
+    )
+    patch_app_runtime(mocker)
+    app = create_app()
+
+    with TestClient(app) as client:
+        response = client.post("/admin/phrases/generate-missing")
+
+    assert response.status_code == 200
+    assert captured["only_missing"] is True
+    assert "42 candidates stored" in response.text
+
+
 def test_ui_regenerate_all_phrases_generates_every_book(mocker: MockerFixture) -> None:
-    def fake_generate(_session, _config):
+    captured: dict[str, object] = {}
+
+    def fake_generate(_session, _config, *, only_missing=False):
+        captured["only_missing"] = only_missing
         return PhraseCandidateGenerationResult(books_seen=5, books_built=5, candidate_phrases=99)
 
     mocker.patch(
@@ -582,6 +607,7 @@ def test_ui_regenerate_all_phrases_generates_every_book(mocker: MockerFixture) -
         response = client.post("/admin/phrases/generate-all")
 
     assert response.status_code == 200
+    assert captured["only_missing"] is False
     assert "5 of 5 books" in response.text
 
 

users/richie/home/app_image_path.nix

@@ -0,0 +1,5 @@
+{
+  home.sessionPath = [
+    "/home/richie/app_images/"
+  ];
+}

users/richie/systems/rhapsody-in-green.nix

@@ -1,5 +1,6 @@
 {
   imports = [
+    ../home/app_image_path.nix
     ../home/global.nix
     ../home/gui
   ];