Add models and database persistence for protected phrase extraction

- Introduced dataclasses for phrase candidates, judgments, and matches in `models.py`.
- Implemented database operations for candidate and protected phrases in `store.py`, including loading, saving, and deleting phrases.
- Enhanced text normalization functions in `text_normalization.py` with detailed docstrings.
- Refactored search functionality to utilize new models and methods for detecting protected phrases.
This commit is contained in:
2026-07-09 11:04:59 -04:00
parent e34ed6c597
commit dab18c1385
12 changed files with 2803 additions and 1978 deletions
+2 -35
View File
@@ -14,11 +14,8 @@ from python.ebook_search.api.dependencies import (
from python.ebook_search.api.web import templates
from python.ebook_search.embeddings import embed_missing_chunks, embedding_model_stats
from python.ebook_search.ingest import ingest_configured_paths
from python.ebook_search.protected_phrases.lib import (
build_missing_protected_phrases,
generate_candidate_phrases_for_books,
judge_candidate_phrases_for_books,
)
from python.ebook_search.protected_phrases.generate_ngrams import generate_candidate_phrases_for_books
from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
from python.fastapi_tools import DbSession # noqa: TC001 FastAPI resolves this annotated dependency at runtime
logger = logging.getLogger(__name__)
@@ -50,36 +47,6 @@ def scan_library(request: Request, config: AppConfig, session: DbSession) -> HTM
return templates.TemplateResponse(request, "partials/admin_status.html", {"message": f"Indexed {count} EPUBs"})
@router.post("/build-phrases", response_class=HTMLResponse)
def build_phrases(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
"""Build protected phrases for indexed books that are missing them."""
try:
result = build_missing_protected_phrases(session, config)
session.commit()
except Exception as error:
session.rollback()
logger.exception("ebook_admin_build_phrases_failed")
return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
logger.info(
"ebook_admin_build_phrases_complete books_seen=%s books_built=%s protected=%s mentions=%s",
result.books_seen,
result.books_built,
result.protected_phrases,
result.phrase_mentions,
)
return templates.TemplateResponse(
request,
"partials/admin_status.html",
{
"message": (
f"Built phrases for {result.books_built} of {result.books_seen} books; "
f"{result.protected_phrases} protected phrases, {result.phrase_mentions} mentions"
)
},
)
@router.post("/generate-ngrams", response_class=HTMLResponse)
def generate_ngrams(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
"""Generate candidate n-grams for indexed books without LLM judging."""