Richie/dotfiles
1
0
Fork 0
Code Issues 1 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

feat(ebook-search): implement async phrase matching for chunks and add ChunkPhraseHit model

Browse Source
This commit is contained in:
Richie
2026-07-24 11:38:51 -04:00
parent e010756e09
commit 9f126fedc7
4 changed files with 74 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ebook_search/protected_phrases/store.py
+5 -27
View File
@@ -7,8 +7,7 @@ import re
from typing import TYPE_CHECKING
from sqlalchemy import delete, func, or_, select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
from sqlalchemy.dialects.postgresql import insert
from python.ebook_search.protected_phrases.extraction import minimum_candidate_raw_count
from python.ebook_search.protected_phrases.models import (
@@ -29,35 +28,14 @@ from python.orm.richie import (
if TYPE_CHECKING:
from collections.abc import Sequence
from sqlalchemy.dialects.postgresql.dml import Insert as PostgresInsert
from sqlalchemy.dialects.sqlite.dml import Insert as SqliteInsert
from sqlalchemy.ext.asyncio import AsyncSession
from python.ebook_search.config import EbookSearchConfig
from python.ebook_search.protected_phrases.models import LLMJudgment
from python.orm.richie.base import TableBase
logger = logging.getLogger(__name__)
def dialect_insert(session: AsyncSession, table: type[TableBase]) -> PostgresInsert | SqliteInsert:
"""Return a dialect-specific INSERT construct that supports ``ON CONFLICT DO UPDATE``.
Production runs on PostgreSQL while tests run on SQLite; both support upserts with
compatible SQLAlchemy constructs, so the correct one is chosen from the bound dialect.
Args:
session (AsyncSession): Active database session whose bind selects the dialect.
table (type[TableBase]): Mapped table to insert into.
Returns:
PostgresInsert | SqliteInsert: A dialect insert exposing ``on_conflict_do_update``.
"""
if session.get_bind().dialect.name == "sqlite":
return sqlite_insert(table)
return pg_insert(table)
async def load_book_text(session: AsyncSession, book_id: int) -> str:
"""Load a book's indexed chunk text as one string for phrase extraction.
@@ -377,7 +355,7 @@ async def save_candidate_to_db(
skip_update = {"book_id", "phrase_norm"}
if judgment is None:
skip_update.add("llm_judged")
insert_statement = dialect_insert(session, EbookCandidatePhrase).values(**values)
insert_statement = insert(EbookCandidatePhrase).values(**values)
statement = insert_statement.on_conflict_do_update(
index_elements=["book_id", "phrase_norm"],
set_={column: insert_statement.excluded[column] for column in values if column not in skip_update},
@@ -423,7 +401,7 @@ async def bulk_upsert_unjudged_candidates(
skip_update = {"book_id", "phrase_norm", "llm_judged"}
for chunk_start in range(0, len(values), BULK_CANDIDATE_UPSERT_CHUNK):
chunk = values[chunk_start : chunk_start + BULK_CANDIDATE_UPSERT_CHUNK]
insert_statement = dialect_insert(session, EbookCandidatePhrase).values(chunk)
insert_statement = insert(EbookCandidatePhrase).values(chunk)
statement = insert_statement.on_conflict_do_update(
index_elements=["book_id", "phrase_norm"],
set_={column: insert_statement.excluded[column] for column in chunk[0] if column not in skip_update},
@@ -509,7 +487,7 @@ async def upsert_protected_phrase(
"suppress_children": judgment.suppress_children,
"source_candidate_id": source_candidate.id,
}
insert_statement = dialect_insert(session, EbookProtectedPhrase).values(**values)
insert_statement = insert(EbookProtectedPhrase).values(**values)
statement = insert_statement.on_conflict_do_update(
index_elements=["book_id", "phrase_norm"],
set_={
@@ -542,7 +520,7 @@ async def upsert_phrase_alias(
if not alias_norm or alias_norm == phrase.phrase_norm:
return None
insert_statement = dialect_insert(session, EbookPhraseAlias).values(
insert_statement = insert(EbookPhraseAlias).values(
phrase_id=phrase.id,
alias_norm=alias_norm,
alias_text=alias_text,