feat(ebook): migrate to async DB/HTTP and parallelize phrase pipeline
Convert the ebook-search web app to async end to end and add concurrency to the protected-phrase extraction and judging pipeline so large books no longer block the event loop or the UI. ORM / infra: - Add get_async_postgres_engine and factor shared URL/connect_args building into build_postgres_url (reused by the sync and async engine builders) - Add async FastAPI session helpers (get_async_db, AsyncDbSession) with expire_on_commit=False to avoid implicit IO under asyncio App: - Use AsyncEngine/AsyncSession throughout routes, search, ingest, embeddings, answer, rerank and LLM calls; convert handlers to async - Share a single httpx.AsyncClient in app state for LLM requests; size the connection pool for concurrent phrase-judging workers - Add judge_tasks: run per-book judging as tracked background tasks so a book already being judged isn't double-queued Protected phrases: - Add a process pool (pool.py) and worker-count config (extraction/judge book/phrase workers) to parallelize candidate generation and judging - Split admin actions into all/missing variants for generation and judging Config: - Add protected_phrase_extraction_workers, phrase_judge_book_workers, phrase_judge_phrase_workers
This commit is contained in:
@@ -9,6 +9,8 @@ from typing import TYPE_CHECKING
|
||||
from python.ebook_search.llm_interface import request_rerank
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import httpx
|
||||
|
||||
from python.ebook_search.config import RerankConfig
|
||||
from python.ebook_search.search import SearchResult
|
||||
|
||||
@@ -23,7 +25,12 @@ class RerankResult:
|
||||
score: float
|
||||
|
||||
|
||||
def rerank_chunks(query: str, candidates: list[SearchResult], config: RerankConfig) -> list[SearchResult]:
|
||||
async def rerank_chunks(
|
||||
client: httpx.AsyncClient,
|
||||
query: str,
|
||||
candidates: list[SearchResult],
|
||||
config: RerankConfig,
|
||||
) -> list[SearchResult]:
|
||||
"""Rerank candidates with a vLLM rerank endpoint."""
|
||||
if not candidates:
|
||||
return []
|
||||
@@ -34,7 +41,7 @@ def rerank_chunks(query: str, candidates: list[SearchResult], config: RerankConf
|
||||
config.model,
|
||||
len(candidates),
|
||||
)
|
||||
scores = score_candidates(query, candidates, config)
|
||||
scores = await score_candidates(client, query, candidates, config)
|
||||
results = sorted(
|
||||
(
|
||||
replace(
|
||||
@@ -56,13 +63,14 @@ def rerank_chunks(query: str, candidates: list[SearchResult], config: RerankConf
|
||||
return results
|
||||
|
||||
|
||||
def score_candidates(
|
||||
async def score_candidates(
|
||||
client: httpx.AsyncClient,
|
||||
query: str,
|
||||
candidates: list[SearchResult],
|
||||
config: RerankConfig,
|
||||
) -> dict[int, RerankResult]:
|
||||
"""Score candidate chunks with the configured rerank API."""
|
||||
body = request_rerank(query, [candidate.text for candidate in candidates], config)
|
||||
body = await request_rerank(client, query, [candidate.text for candidate in candidates], config)
|
||||
if body is None:
|
||||
return zero_rerank_scores(candidates)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user