feat(ebook): migrate to async DB/HTTP and parallelize phrase pipeline
Convert the ebook-search web app to async end to end and add concurrency to the protected-phrase extraction and judging pipeline so large books no longer block the event loop or the UI. ORM / infra: - Add get_async_postgres_engine and factor shared URL/connect_args building into build_postgres_url (reused by the sync and async engine builders) - Add async FastAPI session helpers (get_async_db, AsyncDbSession) with expire_on_commit=False to avoid implicit IO under asyncio App: - Use AsyncEngine/AsyncSession throughout routes, search, ingest, embeddings, answer, rerank and LLM calls; convert handlers to async - Share a single httpx.AsyncClient in app state for LLM requests; size the connection pool for concurrent phrase-judging workers - Add judge_tasks: run per-book judging as tracked background tasks so a book already being judged isn't double-queued Protected phrases: - Add a process pool (pool.py) and worker-count config (extraction/judge book/phrase workers) to parallelize candidate generation and judging - Split admin actions into all/missing variants for generation and judging Config: - Add protected_phrase_extraction_workers, phrase_judge_book_workers, phrase_judge_phrase_workers
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
@@ -21,7 +22,7 @@ DEFAULT_CHUNK_TOKENS = 700
|
||||
DEFAULT_CHUNK_OVERLAP = 100
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from python.ebook_search.config import EbookSearchConfig
|
||||
from python.ebook_search.epub_parse import ParsedChapter
|
||||
@@ -73,45 +74,63 @@ def chunk_text(
|
||||
return [chunk for chunk in chunks if chunk.text]
|
||||
|
||||
|
||||
def ingest_configured_paths(session: Session, config: EbookSearchConfig) -> int:
|
||||
def find_library_epubs(library_path: str) -> tuple[Path, list[Path] | None]:
|
||||
"""Resolve one configured library path and collect its EPUB files (blocking filesystem walk).
|
||||
|
||||
Returns:
|
||||
tuple[Path, list[Path] | None]: The expanded path and its EPUB files, or ``None`` when
|
||||
the path is neither an EPUB file nor a directory.
|
||||
"""
|
||||
path = Path(library_path).expanduser()
|
||||
if path.is_file() and path.suffix.lower() == ".epub":
|
||||
return path, [path]
|
||||
if path.is_dir():
|
||||
return path, sorted(path.rglob("*.epub"))
|
||||
return path, None
|
||||
|
||||
|
||||
async def ingest_configured_paths(session: AsyncSession, config: EbookSearchConfig) -> int:
|
||||
"""Ingest every EPUB found under configured library paths."""
|
||||
count = 0
|
||||
for library_path in config.library_paths:
|
||||
path = Path(library_path).expanduser()
|
||||
path, epub_paths = await asyncio.to_thread(find_library_epubs, library_path)
|
||||
logger.info("ebook_ingest_path_start path=%s", path)
|
||||
if path.is_file() and path.suffix.lower() == ".epub":
|
||||
count += int(ingest_file(session, path, config))
|
||||
elif path.is_dir():
|
||||
for epub_path in sorted(path.rglob("*.epub")):
|
||||
count += int(ingest_file(session, epub_path, config))
|
||||
else:
|
||||
if epub_paths is None:
|
||||
logger.warning("ebook_ingest_path_missing path=%s", path)
|
||||
continue
|
||||
for epub_path in epub_paths:
|
||||
count += int(await ingest_file(session, epub_path, config))
|
||||
logger.info("ebook_ingest_paths_complete changed_files=%s configured_paths=%s", count, len(config.library_paths))
|
||||
return count
|
||||
|
||||
|
||||
def ingest_file(session: Session, path: Path, config: EbookSearchConfig) -> bool:
|
||||
def resolve_ingest_path(path: Path) -> Path:
|
||||
"""Expand and resolve an ingest path (blocking filesystem call)."""
|
||||
return path.expanduser().resolve()
|
||||
|
||||
|
||||
async def ingest_file(session: AsyncSession, path: Path, config: EbookSearchConfig) -> bool:
|
||||
"""Ingest one EPUB file. Return True when the database changed."""
|
||||
try:
|
||||
resolved_path = path.expanduser().resolve()
|
||||
resolved_path = await asyncio.to_thread(resolve_ingest_path, path)
|
||||
logger.info("ebook_ingest_file_start path=%s", resolved_path)
|
||||
file_hash = sha256_file(resolved_path)
|
||||
existing = find_existing_source(session, resolved_path, file_hash)
|
||||
file_hash = await asyncio.to_thread(sha256_file, resolved_path)
|
||||
existing = await find_existing_source(session, resolved_path, file_hash)
|
||||
if existing is not None and existing.file_sha256 == file_hash:
|
||||
stat = resolved_path.stat()
|
||||
existing.file_path = str(resolved_path)
|
||||
existing.file_mtime = datetime.fromtimestamp(stat.st_mtime, tz=UTC)
|
||||
existing.file_size = stat.st_size
|
||||
session.flush()
|
||||
await session.flush()
|
||||
logger.info("ebook_ingest_file_unchanged source_id=%s path=%s", existing.id, resolved_path)
|
||||
return False
|
||||
if existing is not None:
|
||||
logger.info("ebook_ingest_file_replacing source_id=%s path=%s", existing.id, resolved_path)
|
||||
session.delete(existing)
|
||||
session.flush()
|
||||
await session.delete(existing)
|
||||
await session.flush()
|
||||
|
||||
stat = resolved_path.stat()
|
||||
parsed = parse_epub(resolved_path)
|
||||
parsed = await asyncio.to_thread(parse_epub, resolved_path)
|
||||
source = EbookSource(
|
||||
title=parsed.title,
|
||||
author=parsed.author,
|
||||
@@ -124,7 +143,7 @@ def ingest_file(session: Session, path: Path, config: EbookSearchConfig) -> bool
|
||||
file_size=stat.st_size,
|
||||
)
|
||||
session.add(source)
|
||||
session.flush()
|
||||
await session.flush()
|
||||
|
||||
chunk_index = 0
|
||||
for spine_index, parsed_chapter in enumerate(parsed.chapters):
|
||||
@@ -135,11 +154,11 @@ def ingest_file(session: Session, path: Path, config: EbookSearchConfig) -> bool
|
||||
href=parsed_chapter.href,
|
||||
)
|
||||
session.add(chapter)
|
||||
session.flush()
|
||||
await session.flush()
|
||||
chunk_index = add_chapter_chunks(session, source, chapter, parsed_chapter, chunk_index, config)
|
||||
|
||||
session.commit()
|
||||
mention_count = index_chunk_phrase_mentions_for_book(session, source.id, config)
|
||||
await session.commit()
|
||||
mention_count = await index_chunk_phrase_mentions_for_book(session, source.id, config)
|
||||
logger.info(
|
||||
"ebook_ingest_file_complete source_id=%s path=%s chapters=%s chunks=%s phrase_mentions=%s",
|
||||
source.id,
|
||||
@@ -155,15 +174,15 @@ def ingest_file(session: Session, path: Path, config: EbookSearchConfig) -> bool
|
||||
return True
|
||||
|
||||
|
||||
def find_existing_source(session: Session, path: Path, file_hash: str) -> EbookSource | None:
|
||||
async def find_existing_source(session: AsyncSession, path: Path, file_hash: str) -> EbookSource | None:
|
||||
"""Find an existing source by canonical path or file hash."""
|
||||
return session.scalar(
|
||||
return await session.scalar(
|
||||
select(EbookSource).where(or_(EbookSource.file_path == str(path), EbookSource.file_sha256 == file_hash))
|
||||
)
|
||||
|
||||
|
||||
def add_chapter_chunks(
|
||||
session: Session,
|
||||
session: AsyncSession,
|
||||
source: EbookSource,
|
||||
chapter: EbookChapter,
|
||||
parsed_chapter: ParsedChapter,
|
||||
|
||||
Reference in New Issue
Block a user