feat(ebook-search): implement async file path resolution and EPUB discovery
This commit is contained in:
@@ -11,6 +11,7 @@ from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import tiktoken
|
||||
from anyio import Path as AsyncPath
|
||||
from sqlalchemy import or_, select
|
||||
|
||||
from python.ebook_search.epub_parse import parse_epub
|
||||
@@ -74,18 +75,18 @@ def chunk_text(
|
||||
return [chunk for chunk in chunks if chunk.text]
|
||||
|
||||
|
||||
def find_library_epubs(library_path: str) -> tuple[Path, list[Path] | None]:
|
||||
"""Resolve one configured library path and collect its EPUB files (blocking filesystem walk).
|
||||
async def find_library_epubs(library_path: str) -> tuple[AsyncPath, list[AsyncPath] | None]:
|
||||
"""Resolve one configured library path and collect its EPUB files asynchronously.
|
||||
|
||||
Returns:
|
||||
tuple[Path, list[Path] | None]: The expanded path and its EPUB files, or ``None`` when
|
||||
the path is neither an EPUB file nor a directory.
|
||||
"""
|
||||
path = Path(library_path).expanduser()
|
||||
if path.is_file() and path.suffix.lower() == ".epub":
|
||||
path = await AsyncPath(library_path).expanduser()
|
||||
if await path.is_file() and path.suffix.lower() == ".epub":
|
||||
return path, [path]
|
||||
if path.is_dir():
|
||||
return path, sorted(path.rglob("*.epub"))
|
||||
if await path.is_dir():
|
||||
return path, sorted([epub_path async for epub_path in path.rglob("*.epub")])
|
||||
return path, None
|
||||
|
||||
|
||||
@@ -93,7 +94,7 @@ async def ingest_configured_paths(session: AsyncSession, config: EbookSearchConf
|
||||
"""Ingest every EPUB found under configured library paths."""
|
||||
count = 0
|
||||
for library_path in config.library_paths:
|
||||
path, epub_paths = await asyncio.to_thread(find_library_epubs, library_path)
|
||||
path, epub_paths = await find_library_epubs(library_path)
|
||||
logger.info(f"ebook_ingest_path_start {path=}")
|
||||
if epub_paths is None:
|
||||
logger.warning(f"ebook_ingest_path_missing {path=}")
|
||||
@@ -104,20 +105,21 @@ async def ingest_configured_paths(session: AsyncSession, config: EbookSearchConf
|
||||
return count
|
||||
|
||||
|
||||
def resolve_ingest_path(path: Path) -> Path:
|
||||
"""Expand and resolve an ingest path (blocking filesystem call)."""
|
||||
return path.expanduser().resolve()
|
||||
async def resolve_ingest_path(path: Path | AsyncPath) -> AsyncPath:
|
||||
"""Expand and resolve an ingest path without blocking the event loop."""
|
||||
expanded_path = await AsyncPath(path).expanduser()
|
||||
return await expanded_path.resolve()
|
||||
|
||||
|
||||
async def ingest_file(session: AsyncSession, path: Path, config: EbookSearchConfig) -> bool:
|
||||
async def ingest_file(session: AsyncSession, path: Path | AsyncPath, config: EbookSearchConfig) -> bool:
|
||||
"""Ingest one EPUB file. Return True when the database changed."""
|
||||
try:
|
||||
resolved_path = await asyncio.to_thread(resolve_ingest_path, path)
|
||||
resolved_path = await resolve_ingest_path(path)
|
||||
logger.info(f"ebook_ingest_file_start {resolved_path=}")
|
||||
file_hash = await asyncio.to_thread(sha256_file, resolved_path)
|
||||
file_hash = await sha256_file(resolved_path)
|
||||
existing = await find_existing_source(session, resolved_path, file_hash)
|
||||
if existing is not None and existing.file_sha256 == file_hash:
|
||||
stat = resolved_path.stat()
|
||||
stat = await resolved_path.stat()
|
||||
existing.file_path = str(resolved_path)
|
||||
existing.file_mtime = datetime.fromtimestamp(stat.st_mtime, tz=UTC)
|
||||
existing.file_size = stat.st_size
|
||||
@@ -129,8 +131,8 @@ async def ingest_file(session: AsyncSession, path: Path, config: EbookSearchConf
|
||||
await session.delete(existing)
|
||||
await session.flush()
|
||||
|
||||
stat = resolved_path.stat()
|
||||
parsed = await asyncio.to_thread(parse_epub, resolved_path)
|
||||
stat = await resolved_path.stat()
|
||||
parsed = await asyncio.to_thread(parse_epub, Path(resolved_path))
|
||||
source = EbookSource(
|
||||
title=parsed.title,
|
||||
author=parsed.author,
|
||||
@@ -170,7 +172,7 @@ async def ingest_file(session: AsyncSession, path: Path, config: EbookSearchConf
|
||||
return True
|
||||
|
||||
|
||||
async def find_existing_source(session: AsyncSession, path: Path, file_hash: str) -> EbookSource | None:
|
||||
async def find_existing_source(session: AsyncSession, path: Path | AsyncPath, file_hash: str) -> EbookSource | None:
|
||||
"""Find an existing source by canonical path or file hash."""
|
||||
return await session.scalar(
|
||||
select(EbookSource).where(or_(EbookSource.file_path == str(path), EbookSource.file_sha256 == file_hash))
|
||||
@@ -209,10 +211,10 @@ def add_chapter_chunks(
|
||||
return chunk_index
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
"""Calculate the SHA-256 digest for a file."""
|
||||
async def sha256_file(path: AsyncPath) -> str:
|
||||
"""Calculate the SHA-256 digest for a file without blocking the event loop."""
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as file:
|
||||
for block in iter(lambda: file.read(1024 * 1024), b""):
|
||||
async with await path.open("rb") as file:
|
||||
while block := await file.read(1024 * 1024):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
Reference in New Issue
Block a user