Add models and database persistence for protected phrase extraction
- Introduced dataclasses for phrase candidates, judgments, and matches in `models.py`. - Implemented database operations for candidate and protected phrases in `store.py`, including loading, saving, and deleting phrases. - Enhanced text normalization functions in `text_normalization.py` with detailed docstrings. - Refactored search functionality to utilize new models and methods for detecting protected phrases.
This commit is contained in:
@@ -14,11 +14,8 @@ from python.ebook_search.api.dependencies import (
|
||||
from python.ebook_search.api.web import templates
|
||||
from python.ebook_search.embeddings import embed_missing_chunks, embedding_model_stats
|
||||
from python.ebook_search.ingest import ingest_configured_paths
|
||||
from python.ebook_search.protected_phrases.lib import (
|
||||
build_missing_protected_phrases,
|
||||
generate_candidate_phrases_for_books,
|
||||
judge_candidate_phrases_for_books,
|
||||
)
|
||||
from python.ebook_search.protected_phrases.generate_ngrams import generate_candidate_phrases_for_books
|
||||
from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
|
||||
from python.fastapi_tools import DbSession # noqa: TC001 FastAPI resolves this annotated dependency at runtime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -50,36 +47,6 @@ def scan_library(request: Request, config: AppConfig, session: DbSession) -> HTM
|
||||
return templates.TemplateResponse(request, "partials/admin_status.html", {"message": f"Indexed {count} EPUBs"})
|
||||
|
||||
|
||||
@router.post("/build-phrases", response_class=HTMLResponse)
|
||||
def build_phrases(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
|
||||
"""Build protected phrases for indexed books that are missing them."""
|
||||
try:
|
||||
result = build_missing_protected_phrases(session, config)
|
||||
session.commit()
|
||||
except Exception as error:
|
||||
session.rollback()
|
||||
logger.exception("ebook_admin_build_phrases_failed")
|
||||
return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
|
||||
|
||||
logger.info(
|
||||
"ebook_admin_build_phrases_complete books_seen=%s books_built=%s protected=%s mentions=%s",
|
||||
result.books_seen,
|
||||
result.books_built,
|
||||
result.protected_phrases,
|
||||
result.phrase_mentions,
|
||||
)
|
||||
return templates.TemplateResponse(
|
||||
request,
|
||||
"partials/admin_status.html",
|
||||
{
|
||||
"message": (
|
||||
f"Built phrases for {result.books_built} of {result.books_seen} books; "
|
||||
f"{result.protected_phrases} protected phrases, {result.phrase_mentions} mentions"
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/generate-ngrams", response_class=HTMLResponse)
|
||||
def generate_ngrams(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse:
|
||||
"""Generate candidate n-grams for indexed books without LLM judging."""
|
||||
|
||||
Reference in New Issue
Block a user