fix(ebook): enhance EPUB ingestion with error handling and incrmental commits
This commit is contained in:
@@ -91,6 +91,7 @@ def ingest_configured_paths(session: Session, config: EbookSearchConfig) -> int:
|
|||||||
|
|
||||||
def ingest_file(session: Session, path: Path, config: EbookSearchConfig) -> bool:
|
def ingest_file(session: Session, path: Path, config: EbookSearchConfig) -> bool:
|
||||||
"""Ingest one EPUB file. Return True when the database changed."""
|
"""Ingest one EPUB file. Return True when the database changed."""
|
||||||
|
try:
|
||||||
resolved_path = path.expanduser().resolve()
|
resolved_path = path.expanduser().resolve()
|
||||||
logger.info("ebook_ingest_file_start path=%s", resolved_path)
|
logger.info("ebook_ingest_file_start path=%s", resolved_path)
|
||||||
file_hash = sha256_file(resolved_path)
|
file_hash = sha256_file(resolved_path)
|
||||||
@@ -136,7 +137,7 @@ def ingest_file(session: Session, path: Path, config: EbookSearchConfig) -> bool
|
|||||||
session.flush()
|
session.flush()
|
||||||
chunk_index = add_chapter_chunks(session, source, chapter, parsed_chapter, chunk_index, config)
|
chunk_index = add_chapter_chunks(session, source, chapter, parsed_chapter, chunk_index, config)
|
||||||
|
|
||||||
session.flush()
|
session.commit()
|
||||||
logger.info(
|
logger.info(
|
||||||
"ebook_ingest_file_complete source_id=%s path=%s chapters=%s chunks=%s",
|
"ebook_ingest_file_complete source_id=%s path=%s chapters=%s chunks=%s",
|
||||||
source.id,
|
source.id,
|
||||||
@@ -144,6 +145,10 @@ def ingest_file(session: Session, path: Path, config: EbookSearchConfig) -> bool
|
|||||||
len(parsed.chapters),
|
len(parsed.chapters),
|
||||||
chunk_index,
|
chunk_index,
|
||||||
)
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception(f"ebook_ingest_file_error path={path}")
|
||||||
|
return False
|
||||||
|
else:
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user