Richie/dotfiles
1
0
Fork 0
Code Issues 1 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

feat(ebook): migrate to async DB/HTTP and parallelize phrase pipeline

Browse Source 
Convert the ebook-search web app to async end to end and add concurrency
to the protected-phrase extraction and judging pipeline so large books no
longer block the event loop or the UI.

ORM / infra:
- Add get_async_postgres_engine and factor shared URL/connect_args building
  into build_postgres_url (reused by the sync and async engine builders)
- Add async FastAPI session helpers (get_async_db, AsyncDbSession) with
  expire_on_commit=False to avoid implicit IO under asyncio

App:
- Use AsyncEngine/AsyncSession throughout routes, search, ingest, embeddings,
  answer, rerank and LLM calls; convert handlers to async
- Share a single httpx.AsyncClient in app state for LLM requests; size the
  connection pool for concurrent phrase-judging workers
- Add judge_tasks: run per-book judging as tracked background tasks so a
  book already being judged isn't double-queued

Protected phrases:
- Add a process pool (pool.py) and worker-count config
  (extraction/judge book/phrase workers) to parallelize candidate generation
  and judging
- Split admin actions into all/missing variants for generation and judging

Config:
- Add protected_phrase_extraction_workers, phrase_judge_book_workers,
  phrase_judge_phrase_workers
This commit is contained in:
Richie
2026-07-24 11:38:50 -04:00
parent 38c01ec121
commit 2706c4417d
29 changed files with 1824 additions and 769 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ebook_search/api/routes/page.py
+69 -28
View File
@@ -3,18 +3,23 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
from fastapi import APIRouter, HTTPException, Request
from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
from fastapi.responses import HTMLResponse, RedirectResponse
from sqlalchemy import func, select
from python.ebook_search.api.dependencies import (
AppConfig, # noqa: TC001 FastAPI resolves this annotated dependency at runtime
)
from python.ebook_search.api.judge_tasks import is_judging_book, pop_book_judgment_outcome, start_book_phrase_judgment
from python.ebook_search.api.web import templates
from python.ebook_search.protected_phrases.generate_ngrams import recalculate_candidate_phrases_for_book
from python.fastapi_tools import DbSession # noqa: TC001 FastAPI resolves this annotated dependency at runtime
from python.orm.richie import EbookCandidatePhrase, EbookProtectedPhrase, EbookSource
from python.fastapi_tools import AsyncDbSession # noqa: TC001 FastAPI resolves this annotated dependency at runtime
from python.orm.richie import EbookCandidatePhrase, EbookChapter, EbookChunk, EbookProtectedPhrase, EbookSource
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession
logger = logging.getLogger(__name__)
@@ -22,30 +27,41 @@ router = APIRouter()
@router.get("/", response_class=HTMLResponse)
def index(request: Request, config: AppConfig) -> HTMLResponse:
async def index(request: Request, config: AppConfig) -> HTMLResponse:
"""Render the search page."""
return templates.TemplateResponse(request, "search.html", {"config": config})
@router.get("/books", response_class=HTMLResponse)
def books(request: Request, session: DbSession) -> HTMLResponse:
async def books(request: Request, session: AsyncDbSession) -> HTMLResponse:
"""Render the indexed books page."""
sources = list(session.scalars(select(EbookSource).order_by(EbookSource.title)).all())
sources = list((await session.scalars(select(EbookSource).order_by(EbookSource.title))).all())
logger.info("ebook_books_page_loaded count=%s", len(sources))
return templates.TemplateResponse(request, "books.html", {"sources": sources})
def get_candidate_count(session: DbSession, book_id: int) -> int:
async def get_chapter_count(session: AsyncSession, book_id: int) -> int:
"""Return the number of indexed chapters for one book."""
return await session.scalar(select(func.count(EbookChapter.id)).where(EbookChapter.source_id == book_id)) or 0
async def get_chunk_count(session: AsyncSession, book_id: int) -> int:
"""Return the number of indexed chunks for one book."""
return await session.scalar(select(func.count(EbookChunk.id)).where(EbookChunk.source_id == book_id)) or 0
async def get_candidate_count(session: AsyncSession, book_id: int) -> int:
"""Return the number of indexed candidates for one book."""
return (
session.scalar(select(func.count(EbookCandidatePhrase.id)).where(EbookCandidatePhrase.book_id == book_id)) or 0
await session.scalar(select(func.count(EbookCandidatePhrase.id)).where(EbookCandidatePhrase.book_id == book_id))
or 0
)
def get_judged_candidate_count(session: DbSession, book_id: int) -> int:
async def get_judged_candidate_count(session: AsyncSession, book_id: int) -> int:
"""Return the number of judged candidates for one book."""
return (
session.scalar(
await session.scalar(
select(func.count(EbookCandidatePhrase.id)).where(
EbookCandidatePhrase.book_id == book_id,
EbookCandidatePhrase.llm_judged.is_(True),
@@ -55,17 +71,18 @@ def get_judged_candidate_count(session: DbSession, book_id: int) -> int:
)
def get_protected_count(session: DbSession, book_id: int) -> int:
async def get_protected_count(session: AsyncSession, book_id: int) -> int:
"""Return the number of protected phrases for one book."""
return (
session.scalar(select(func.count(EbookProtectedPhrase.id)).where(EbookProtectedPhrase.book_id == book_id)) or 0
await session.scalar(select(func.count(EbookProtectedPhrase.id)).where(EbookProtectedPhrase.book_id == book_id))
or 0
)
def get_candidates(session: DbSession, book_id: int) -> list[EbookCandidatePhrase]:
async def get_candidates(session: AsyncSession, book_id: int) -> list[EbookCandidatePhrase]:
"""Return the indexed candidates for one book."""
return list(
session.scalars(
await session.scalars(
select(EbookCandidatePhrase)
.where(EbookCandidatePhrase.book_id == book_id)
.order_by(EbookCandidatePhrase.candidate_score.desc())
@@ -74,10 +91,10 @@ def get_candidates(session: DbSession, book_id: int) -> list[EbookCandidatePhras
)
def get_protected_phrases(session: DbSession, book_id: int) -> list[EbookProtectedPhrase]:
async def get_protected_phrases(session: AsyncSession, book_id: int) -> list[EbookProtectedPhrase]:
"""Return the protected phrases for one book."""
return list(
session.scalars(
await session.scalars(
select(EbookProtectedPhrase)
.where(EbookProtectedPhrase.book_id == book_id)
.order_by(EbookProtectedPhrase.importance.desc())
@@ -87,21 +104,27 @@ def get_protected_phrases(session: DbSession, book_id: int) -> list[EbookProtect
@router.get("/books/{source_id}", response_class=HTMLResponse)
def book_detail(source_id: int, request: Request, session: DbSession) -> HTMLResponse:
async def book_detail(source_id: int, request: Request, session: AsyncDbSession) -> HTMLResponse:
"""Render details for one indexed book."""
source = session.get(EbookSource, source_id)
source = await session.get(EbookSource, source_id)
phrase_status_message = None
recalculated = request.query_params.get("phrases_recalculated")
if recalculated is not None:
phrase_status_message = f"Recalculated phrases; {recalculated} candidates generated"
judgment_outcome = pop_book_judgment_outcome(request.app, source_id)
if judgment_outcome is not None:
phrase_status_message = judgment_outcome
judging_in_progress = is_judging_book(request.app, source_id)
if judging_in_progress:
phrase_status_message = "Judging candidate phrases in the background; refresh to see progress"
if source is not None:
chapter_count = len(source.chapters)
chunk_count = len(source.chunks)
candidate_count = get_candidate_count(session, source.id)
judged_candidate_count = get_judged_candidate_count(session, source.id)
protected_count = get_protected_count(session, source.id)
candidates = get_candidates(session, source.id)
protected_phrases = get_protected_phrases(session, source.id)
chapter_count = await get_chapter_count(session, source.id)
chunk_count = await get_chunk_count(session, source.id)
candidate_count = await get_candidate_count(session, source.id)
judged_candidate_count = await get_judged_candidate_count(session, source.id)
protected_count = await get_protected_count(session, source.id)
candidates = await get_candidates(session, source.id)
protected_phrases = await get_protected_phrases(session, source.id)
else:
chapter_count = 0
chunk_count = 0
@@ -129,6 +152,7 @@ def book_detail(source_id: int, request: Request, session: DbSession) -> HTMLRes
"chapter_count": chapter_count,
"chunk_count": chunk_count,
"judged_candidate_count": judged_candidate_count,
"judging_in_progress": judging_in_progress,
"protected_count": protected_count,
"protected_phrases": protected_phrases,
"phrase_status_message": phrase_status_message,
@@ -138,13 +162,13 @@ def book_detail(source_id: int, request: Request, session: DbSession) -> HTMLRes
@router.post("/books/{source_id}/recalculate-phrases")
def recalculate_book_phrases(source_id: int, config: AppConfig, session: DbSession) -> RedirectResponse:
async def recalculate_book_phrases(source_id: int, config: AppConfig, session: AsyncDbSession) -> RedirectResponse:
"""Clear and regenerate candidate phrases for one indexed book."""
source = session.get(EbookSource, source_id)
source = await session.get(EbookSource, source_id)
if source is None:
raise HTTPException(status_code=404, detail="Book not found")
result = recalculate_candidate_phrases_for_book(session, source, config)
result = await recalculate_candidate_phrases_for_book(session, source, config, use_process_pool=True)
logger.info(
"ebook_book_phrase_recalculation_complete source_id=%s candidates=%s deleted_candidates=%s "
"deleted_protected=%s deleted_aliases=%s deleted_mentions=%s",
@@ -159,3 +183,20 @@ def recalculate_book_phrases(source_id: int, config: AppConfig, session: DbSessi
url=f"/books/{source_id}?phrases_recalculated={result.candidate_phrases}",
status_code=303,
)
@router.post("/books/{source_id}/judge-phrases")
async def judge_book_phrases(
source_id: int,
request: Request,
background_tasks: BackgroundTasks,
session: AsyncDbSession,
) -> RedirectResponse:
"""Queue background judging of one book's candidate phrases and return immediately."""
source = await session.get(EbookSource, source_id)
if source is None:
raise HTTPException(status_code=404, detail="Book not found")
started = start_book_phrase_judgment(request.app, background_tasks, source.id)
logger.info("ebook_book_phrase_judgment_requested source_id=%s started=%s", source_id, started)
return RedirectResponse(url=f"/books/{source_id}", status_code=303)