feat(ebook): migrate to async DB/HTTP and parallelize phrase pipeline

Convert the ebook-search web app to async end to end and add concurrency
to the protected-phrase extraction and judging pipeline so large books no
longer block the event loop or the UI.

ORM / infra:
- Add get_async_postgres_engine and factor shared URL/connect_args building
  into build_postgres_url (reused by the sync and async engine builders)
- Add async FastAPI session helpers (get_async_db, AsyncDbSession) with
  expire_on_commit=False to avoid implicit IO under asyncio

App:
- Use AsyncEngine/AsyncSession throughout routes, search, ingest, embeddings,
  answer, rerank and LLM calls; convert handlers to async
- Share a single httpx.AsyncClient in app state for LLM requests; size the
  connection pool for concurrent phrase-judging workers
- Add judge_tasks: run per-book judging as tracked background tasks so a
  book already being judged isn't double-queued

Protected phrases:
- Add a process pool (pool.py) and worker-count config
  (extraction/judge book/phrase workers) to parallelize candidate generation
  and judging
- Split admin actions into all/missing variants for generation and judging

Config:
- Add protected_phrase_extraction_workers, phrase_judge_book_workers,
  phrase_judge_phrase_workers
This commit is contained in:
2026-07-16 13:09:34 -04:00
parent 9bc0d67962
commit 1f5e527cbc
29 changed files with 1824 additions and 769 deletions
+43 -24
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import asyncio
import hashlib
import logging
from dataclasses import dataclass
@@ -21,7 +22,7 @@ DEFAULT_CHUNK_TOKENS = 700
DEFAULT_CHUNK_OVERLAP = 100
if TYPE_CHECKING:
from sqlalchemy.orm import Session
from sqlalchemy.ext.asyncio import AsyncSession
from python.ebook_search.config import EbookSearchConfig
from python.ebook_search.epub_parse import ParsedChapter
@@ -73,45 +74,63 @@ def chunk_text(
return [chunk for chunk in chunks if chunk.text]
def ingest_configured_paths(session: Session, config: EbookSearchConfig) -> int:
def find_library_epubs(library_path: str) -> tuple[Path, list[Path] | None]:
"""Resolve one configured library path and collect its EPUB files (blocking filesystem walk).
Returns:
tuple[Path, list[Path] | None]: The expanded path and its EPUB files, or ``None`` when
the path is neither an EPUB file nor a directory.
"""
path = Path(library_path).expanduser()
if path.is_file() and path.suffix.lower() == ".epub":
return path, [path]
if path.is_dir():
return path, sorted(path.rglob("*.epub"))
return path, None
async def ingest_configured_paths(session: AsyncSession, config: EbookSearchConfig) -> int:
"""Ingest every EPUB found under configured library paths."""
count = 0
for library_path in config.library_paths:
path = Path(library_path).expanduser()
path, epub_paths = await asyncio.to_thread(find_library_epubs, library_path)
logger.info("ebook_ingest_path_start path=%s", path)
if path.is_file() and path.suffix.lower() == ".epub":
count += int(ingest_file(session, path, config))
elif path.is_dir():
for epub_path in sorted(path.rglob("*.epub")):
count += int(ingest_file(session, epub_path, config))
else:
if epub_paths is None:
logger.warning("ebook_ingest_path_missing path=%s", path)
continue
for epub_path in epub_paths:
count += int(await ingest_file(session, epub_path, config))
logger.info("ebook_ingest_paths_complete changed_files=%s configured_paths=%s", count, len(config.library_paths))
return count
def ingest_file(session: Session, path: Path, config: EbookSearchConfig) -> bool:
def resolve_ingest_path(path: Path) -> Path:
"""Expand and resolve an ingest path (blocking filesystem call)."""
return path.expanduser().resolve()
async def ingest_file(session: AsyncSession, path: Path, config: EbookSearchConfig) -> bool:
"""Ingest one EPUB file. Return True when the database changed."""
try:
resolved_path = path.expanduser().resolve()
resolved_path = await asyncio.to_thread(resolve_ingest_path, path)
logger.info("ebook_ingest_file_start path=%s", resolved_path)
file_hash = sha256_file(resolved_path)
existing = find_existing_source(session, resolved_path, file_hash)
file_hash = await asyncio.to_thread(sha256_file, resolved_path)
existing = await find_existing_source(session, resolved_path, file_hash)
if existing is not None and existing.file_sha256 == file_hash:
stat = resolved_path.stat()
existing.file_path = str(resolved_path)
existing.file_mtime = datetime.fromtimestamp(stat.st_mtime, tz=UTC)
existing.file_size = stat.st_size
session.flush()
await session.flush()
logger.info("ebook_ingest_file_unchanged source_id=%s path=%s", existing.id, resolved_path)
return False
if existing is not None:
logger.info("ebook_ingest_file_replacing source_id=%s path=%s", existing.id, resolved_path)
session.delete(existing)
session.flush()
await session.delete(existing)
await session.flush()
stat = resolved_path.stat()
parsed = parse_epub(resolved_path)
parsed = await asyncio.to_thread(parse_epub, resolved_path)
source = EbookSource(
title=parsed.title,
author=parsed.author,
@@ -124,7 +143,7 @@ def ingest_file(session: Session, path: Path, config: EbookSearchConfig) -> bool
file_size=stat.st_size,
)
session.add(source)
session.flush()
await session.flush()
chunk_index = 0
for spine_index, parsed_chapter in enumerate(parsed.chapters):
@@ -135,11 +154,11 @@ def ingest_file(session: Session, path: Path, config: EbookSearchConfig) -> bool
href=parsed_chapter.href,
)
session.add(chapter)
session.flush()
await session.flush()
chunk_index = add_chapter_chunks(session, source, chapter, parsed_chapter, chunk_index, config)
session.commit()
mention_count = index_chunk_phrase_mentions_for_book(session, source.id, config)
await session.commit()
mention_count = await index_chunk_phrase_mentions_for_book(session, source.id, config)
logger.info(
"ebook_ingest_file_complete source_id=%s path=%s chapters=%s chunks=%s phrase_mentions=%s",
source.id,
@@ -155,15 +174,15 @@ def ingest_file(session: Session, path: Path, config: EbookSearchConfig) -> bool
return True
def find_existing_source(session: Session, path: Path, file_hash: str) -> EbookSource | None:
async def find_existing_source(session: AsyncSession, path: Path, file_hash: str) -> EbookSource | None:
"""Find an existing source by canonical path or file hash."""
return session.scalar(
return await session.scalar(
select(EbookSource).where(or_(EbookSource.file_path == str(path), EbookSource.file_sha256 == file_hash))
)
def add_chapter_chunks(
session: Session,
session: AsyncSession,
source: EbookSource,
chapter: EbookChapter,
parsed_chapter: ParsedChapter,