Richie/dotfiles
1
0
Fork 0
Code Issues 1 Pull Requests 3 Actions Packages Projects Releases Wiki Activity

feat(ebook): migrate to async DB/HTTP and parallelize phrase pipeline

Browse Source 
Convert the ebook-search web app to async end to end and add concurrency
to the protected-phrase extraction and judging pipeline so large books no
longer block the event loop or the UI.

ORM / infra:
- Add get_async_postgres_engine and factor shared URL/connect_args building
  into build_postgres_url (reused by the sync and async engine builders)
- Add async FastAPI session helpers (get_async_db, AsyncDbSession) with
  expire_on_commit=False to avoid implicit IO under asyncio

App:
- Use AsyncEngine/AsyncSession throughout routes, search, ingest, embeddings,
  answer, rerank and LLM calls; convert handlers to async
- Share a single httpx.AsyncClient in app state for LLM requests; size the
  connection pool for concurrent phrase-judging workers
- Add judge_tasks: run per-book judging as tracked background tasks so a
  book already being judged isn't double-queued

Protected phrases:
- Add a process pool (pool.py) and worker-count config
  (extraction/judge book/phrase workers) to parallelize candidate generation
  and judging
- Split admin actions into all/missing variants for generation and judging

Config:
- Add protected_phrase_extraction_workers, phrase_judge_book_workers,
  phrase_judge_phrase_workers
This commit is contained in:
Richie
2026-07-24 11:38:50 -04:00
parent 38c01ec121
commit 2706c4417d
29 changed files with 1824 additions and 769 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ebook_search/api/routes/admin.py
+127 -33
View File
@@ -8,15 +8,18 @@ from fastapi import APIRouter, Request
from fastapi.responses import HTMLResponse
from python.ebook_search.api.bm25_tasks import schedule_bm25_refresh
from python.ebook_search.api.dependencies import (
AppConfig, # noqa: TC001 FastAPI resolves this annotated dependency at runtime
from python.ebook_search.api.dependencies import ( # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
AppConfig,
AppEngine,
AppHttpClient,
)
from python.ebook_search.api.web import templates
from python.ebook_search.embeddings import embed_missing_chunks, embedding_model_stats
from python.ebook_search.ingest import ingest_configured_paths
from python.ebook_search.protected_phrases.generate_ngrams import generate_candidate_phrases_for_books
from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
from python.fastapi_tools import DbSession # noqa: TC001 FastAPI resolves this annotated dependency at runtime
from python.ebook_search.protected_phrases.store import book_ids_pending_first_judgment, corpus_phrase_stats
from python.fastapi_tools import AsyncDbSession # noqa: TC001 FastAPI resolves this annotated dependency at runtime
logger = logging.getLogger(__name__)
@@ -24,19 +27,29 @@ router = APIRouter(prefix="/admin")
@router.get("", response_class=HTMLResponse)
def admin(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
async def admin(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
"""Render the admin page."""
stats = embedding_model_stats(session)
logger.info("ebook_admin_page_loaded models=%s", len(stats))
return templates.TemplateResponse(request, "admin.html", {"config": config, "stats": stats})
stats = await embedding_model_stats(session)
phrase_stats = await corpus_phrase_stats(session)
logger.info(
"ebook_admin_page_loaded models=%s candidate_phrases=%s protected_phrases=%s",
len(stats),
phrase_stats.candidate_phrases,
phrase_stats.protected_phrases,
)
return templates.TemplateResponse(
request,
"admin.html",
{"config": config, "stats": stats, "phrase_stats": phrase_stats},
)
@router.post("/scan", response_class=HTMLResponse)
def scan_library(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
async def scan_library(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
"""Scan configured library paths for EPUB changes."""
try:
count = ingest_configured_paths(session, config)
session.commit()
count = await ingest_configured_paths(session, config)
await session.commit()
except Exception as error:
logger.exception("ebook_admin_scan_failed")
return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
@@ -47,48 +60,119 @@ def scan_library(request: Request, config: AppConfig, session: DbSession) -> HTM
return templates.TemplateResponse(request, "partials/admin_status.html", {"message": f"Indexed {count} EPUBs"})
@router.post("/generate-ngrams", response_class=HTMLResponse)
def generate_ngrams(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
"""Generate candidate n-grams for indexed books without LLM judging."""
@router.post("/phrases/generate-all", response_class=HTMLResponse)
async def generate_all_phrases(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
"""Regenerate candidate phrases for every indexed book without LLM judging."""
return await run_phrase_generation(request, config, session, only_missing=False)
@router.post("/phrases/generate-missing", response_class=HTMLResponse)
async def generate_missing_phrases(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
"""Generate candidate phrases only for books that have none yet."""
return await run_phrase_generation(request, config, session, only_missing=True)
async def run_phrase_generation(
request: Request,
config: AppConfig,
session: AsyncDbSession,
*,
only_missing: bool,
) -> HTMLResponse:
"""Run candidate phrase generation and render the outcome as an admin status partial.
Args:
request (Request): Current request, for template rendering.
config (AppConfig): Runtime phrase-tuning settings.
session (AsyncDbSession): Active database session.
only_missing (bool): Only generate for books without candidates instead of every book.
Returns:
HTMLResponse: Status partial describing the generation outcome.
"""
try:
result = generate_candidate_phrases_for_books(session, config)
session.commit()
result = await generate_candidate_phrases_for_books(session, config, only_missing=only_missing)
await session.commit()
except Exception as error:
session.rollback()
logger.exception("ebook_admin_generate_ngrams_failed")
await session.rollback()
logger.exception("ebook_admin_generate_phrases_failed only_missing=%s", only_missing)
return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
logger.info(
"ebook_admin_generate_ngrams_complete books_seen=%s books_built=%s candidates=%s",
"ebook_admin_generate_phrases_complete only_missing=%s books_seen=%s books_built=%s candidates=%s",
only_missing,
result.books_seen,
result.books_built,
result.candidate_phrases,
)
if only_missing and result.books_seen == 0:
return templates.TemplateResponse(
request,
"partials/admin_status.html",
{"message": "All books already have candidate phrases"},
)
return templates.TemplateResponse(
request,
"partials/admin_status.html",
{
"message": (
f"Generated n-grams for {result.books_built} of {result.books_seen} books; "
f"Generated phrases for {result.books_built} of {result.books_seen} books; "
f"{result.candidate_phrases} candidates stored"
)
},
)
@router.post("/judge-ngrams", response_class=HTMLResponse)
def judge_ngrams(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
"""Judge stored candidate n-grams and promote accepted protected phrases."""
@router.post("/phrases/judge-all", response_class=HTMLResponse)
async def judge_all_phrases(request: Request, engine: AppEngine, config: AppConfig) -> HTMLResponse:
"""Judge unjudged candidate phrases across every indexed book."""
return await run_phrase_judgment(request, engine, config, source_ids=None)
@router.post("/phrases/judge-missing", response_class=HTMLResponse)
async def judge_missing_phrases(
request: Request,
engine: AppEngine,
config: AppConfig,
session: AsyncDbSession,
) -> HTMLResponse:
"""Judge candidate phrases only for books where judging has never run."""
source_ids = await book_ids_pending_first_judgment(session)
if not source_ids:
return templates.TemplateResponse(
request,
"partials/admin_status.html",
{"message": "All books with candidate phrases have been judged"},
)
return await run_phrase_judgment(request, engine, config, source_ids=source_ids)
async def run_phrase_judgment(
request: Request,
engine: AppEngine,
config: AppConfig,
*,
source_ids: list[int] | None,
) -> HTMLResponse:
"""Run LLM judging for candidate phrases and render the outcome as an admin status partial.
Args:
request (Request): Current request, for template rendering.
engine (AppEngine): Engine used to open per-book judging sessions.
config (AppConfig): Runtime phrase-tuning settings.
source_ids (list[int] | None): Books to judge; ``None`` judges every indexed book.
Returns:
HTMLResponse: Status partial describing the judging outcome.
"""
try:
result = judge_candidate_phrases_for_books(session, config)
session.commit()
result = await judge_candidate_phrases_for_books(engine, config, source_ids=source_ids)
except Exception as error:
session.rollback()
logger.exception("ebook_admin_judge_ngrams_failed")
logger.exception("ebook_admin_judge_phrases_failed")
return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
logger.info(
"ebook_admin_judge_ngrams_complete books_seen=%s books_judged=%s books_failed=%s candidates_judged=%s "
"ebook_admin_judge_phrases_complete books_seen=%s books_judged=%s books_failed=%s candidates_judged=%s "
"protected=%s mentions=%s",
result.books_seen,
result.books_judged,
@@ -112,11 +196,16 @@ def judge_ngrams(request: Request, config: AppConfig, session: DbSession) -> HTM
@router.post("/embed-missing", response_class=HTMLResponse)
def embed_missing(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
async def embed_missing(
request: Request,
config: AppConfig,
session: AsyncDbSession,
client: AppHttpClient,
) -> HTMLResponse:
"""Embed chunks missing vectors for the configured model."""
try:
count = embed_missing_chunks(session, config)
session.commit()
count = await embed_missing_chunks(session, client, config)
await session.commit()
except Exception as error:
logger.exception("ebook_admin_embed_missing_failed")
return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
@@ -130,16 +219,21 @@ def embed_missing(request: Request, config: AppConfig, session: DbSession) -> HT
@router.post("/embed-all", response_class=HTMLResponse)
def embed_all(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
async def embed_all(
request: Request,
config: AppConfig,
session: AsyncDbSession,
client: AppHttpClient,
) -> HTMLResponse:
"""Embed all chunks missing vectors in fixed-size batches."""
total = 0
batches = 0
try:
while True:
count = embed_missing_chunks(session, config)
count = await embed_missing_chunks(session, client, config)
if count == 0:
break
session.commit()
await session.commit()
total += count
batches += 1
logger.info(