feat(ebook): migrate to async DB/HTTP and parallelize phrase pipeline
Convert the ebook-search web app to async end to end and add concurrency to the protected-phrase extraction and judging pipeline so large books no longer block the event loop or the UI. ORM / infra: - Add get_async_postgres_engine and factor shared URL/connect_args building into build_postgres_url (reused by the sync and async engine builders) - Add async FastAPI session helpers (get_async_db, AsyncDbSession) with expire_on_commit=False to avoid implicit IO under asyncio App: - Use AsyncEngine/AsyncSession throughout routes, search, ingest, embeddings, answer, rerank and LLM calls; convert handlers to async - Share a single httpx.AsyncClient in app state for LLM requests; size the connection pool for concurrent phrase-judging workers - Add judge_tasks: run per-book judging as tracked background tasks so a book already being judged isn't double-queued Protected phrases: - Add a process pool (pool.py) and worker-count config (extraction/judge book/phrase workers) to parallelize candidate generation and judging - Split admin actions into all/missing variants for generation and judging Config: - Add protected_phrase_extraction_workers, phrase_judge_book_workers, phrase_judge_phrase_workers
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
@@ -17,7 +18,7 @@ from sqlalchemy import func, select, union_all
|
||||
from python.orm.richie import EbookChapter, EbookChunk, EbookSource
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from python.ebook_search.config import EbookSearchConfig
|
||||
|
||||
@@ -73,14 +74,14 @@ def get_current_bm25_index(index_path: Path) -> Path:
|
||||
return index_path
|
||||
|
||||
|
||||
def ensure_bm25_corpus(session: Session, config: EbookSearchConfig) -> None:
|
||||
async def ensure_bm25_corpus(session: AsyncSession, config: EbookSearchConfig) -> None:
|
||||
"""Create or refresh the persisted BM25 corpus when it is missing or stale."""
|
||||
index_path = bm25_index_path(config)
|
||||
manifest = read_bm25_manifest(index_path)
|
||||
db_updated_at = corpus_last_updated_at(session)
|
||||
db_updated_at = await corpus_last_updated_at(session)
|
||||
if not bm25_index_exists(index_path, manifest):
|
||||
logger.info("ebook_bm25_index_missing path=%s", index_path)
|
||||
refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
|
||||
await refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
|
||||
return
|
||||
if db_updated_at is not None and manifest is not None and manifest.created_at < db_updated_at:
|
||||
logger.info(
|
||||
@@ -89,7 +90,7 @@ def ensure_bm25_corpus(session: Session, config: EbookSearchConfig) -> None:
|
||||
manifest.created_at.isoformat(),
|
||||
db_updated_at.isoformat(),
|
||||
)
|
||||
refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
|
||||
await refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
|
||||
return
|
||||
logger.info(
|
||||
"ebook_bm25_index_current path=%s chunks=%s created_at=%s",
|
||||
@@ -99,21 +100,24 @@ def ensure_bm25_corpus(session: Session, config: EbookSearchConfig) -> None:
|
||||
)
|
||||
|
||||
|
||||
def refresh_bm25_corpus(
|
||||
session: Session,
|
||||
async def refresh_bm25_corpus(
|
||||
session: AsyncSession,
|
||||
config: EbookSearchConfig,
|
||||
*,
|
||||
db_updated_at: datetime | None = None,
|
||||
) -> BM25Manifest:
|
||||
"""Rebuild and persist the BM25 corpus from the current database chunks."""
|
||||
"""Rebuild and persist the BM25 corpus from the current database chunks.
|
||||
|
||||
The index build is CPU and disk work, so it runs in a worker thread.
|
||||
"""
|
||||
index_path = bm25_index_path(config)
|
||||
records, texts = fetch_bm25_corpus_records(session)
|
||||
records, texts = await fetch_bm25_corpus_records(session)
|
||||
manifest = BM25Manifest(
|
||||
created_at=datetime.now(tz=UTC),
|
||||
db_updated_at=db_updated_at if db_updated_at is not None else corpus_last_updated_at(session),
|
||||
db_updated_at=db_updated_at if db_updated_at is not None else await corpus_last_updated_at(session),
|
||||
chunk_count=len(records),
|
||||
)
|
||||
write_bm25_corpus(index_path, records, texts, manifest)
|
||||
await asyncio.to_thread(write_bm25_corpus, index_path, records, texts, manifest)
|
||||
logger.info(
|
||||
"ebook_bm25_index_refreshed path=%s chunks=%s created_at=%s",
|
||||
index_path,
|
||||
@@ -164,7 +168,7 @@ def score_bm25_corpus(query: str, corpus: BM25Corpus, *, limit: int) -> list[tup
|
||||
return results
|
||||
|
||||
|
||||
def fetch_bm25_corpus_records(session: Session) -> tuple[list[dict[str, object]], list[str]]:
|
||||
async def fetch_bm25_corpus_records(session: AsyncSession) -> tuple[list[dict[str, object]], list[str]]:
|
||||
"""Fetch persistable BM25 corpus records and their matching index texts from the database.
|
||||
|
||||
search_text is only needed to build the index, so it is returned separately instead of
|
||||
@@ -188,21 +192,21 @@ def fetch_bm25_corpus_records(session: Session) -> tuple[list[dict[str, object]]
|
||||
)
|
||||
records: list[dict[str, object]] = []
|
||||
texts: list[str] = []
|
||||
for row in session.execute(statement).mappings():
|
||||
for row in (await session.execute(statement)).mappings():
|
||||
record = dict(row)
|
||||
texts.append(str(record.pop("bm25_text")))
|
||||
records.append(record)
|
||||
return records, texts
|
||||
|
||||
|
||||
def corpus_last_updated_at(session: Session) -> datetime | None:
|
||||
async def corpus_last_updated_at(session: AsyncSession) -> datetime | None:
|
||||
"""Return the latest source/chapter/chunk update timestamp relevant to BM25 text."""
|
||||
update_times = union_all(
|
||||
select(func.max(EbookSource.updated).label("updated")),
|
||||
select(func.max(EbookChapter.updated).label("updated")),
|
||||
select(func.max(EbookChunk.updated).label("updated")),
|
||||
).subquery()
|
||||
return session.scalar(select(func.max(update_times.c.updated)))
|
||||
return await session.scalar(select(func.max(update_times.c.updated)))
|
||||
|
||||
|
||||
def write_bm25_corpus(
|
||||
|
||||
Reference in New Issue
Block a user