diff --git a/python/ebook_search/api/routes/admin.py b/python/ebook_search/api/routes/admin.py index 2e1fcd1..a441572 100644 --- a/python/ebook_search/api/routes/admin.py +++ b/python/ebook_search/api/routes/admin.py @@ -14,6 +14,11 @@ from python.ebook_search.api.dependencies import ( from python.ebook_search.api.web import templates from python.ebook_search.embeddings import embed_missing_chunks, embedding_model_stats from python.ebook_search.ingest import ingest_configured_paths +from python.ebook_search.protected_phrases.lib import ( + build_missing_protected_phrases, + generate_candidate_phrases_for_books, + judge_candidate_phrases_for_books, +) from python.fastapi_tools import DbSession # noqa: TC001 FastAPI resolves this annotated dependency at runtime logger = logging.getLogger(__name__) @@ -45,6 +50,97 @@ def scan_library(request: Request, config: AppConfig, session: DbSession) -> HTM return templates.TemplateResponse(request, "partials/admin_status.html", {"message": f"Indexed {count} EPUBs"}) +@router.post("/build-phrases", response_class=HTMLResponse) +def build_phrases(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse: + """Build protected phrases for indexed books that are missing them.""" + try: + result = build_missing_protected_phrases(session, config) + session.commit() + except Exception as error: + session.rollback() + logger.exception("ebook_admin_build_phrases_failed") + return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500) + + logger.info( + "ebook_admin_build_phrases_complete books_seen=%s books_built=%s protected=%s mentions=%s", + result.books_seen, + result.books_built, + result.protected_phrases, + result.phrase_mentions, + ) + return templates.TemplateResponse( + request, + "partials/admin_status.html", + { + "message": ( + f"Built phrases for {result.books_built} of {result.books_seen} books; " + f"{result.protected_phrases} protected phrases, {result.phrase_mentions} mentions" + ) + }, + ) + + +@router.post("/generate-ngrams", response_class=HTMLResponse) +def generate_ngrams(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse: + """Generate candidate n-grams for indexed books without LLM judging.""" + try: + result = generate_candidate_phrases_for_books(session, config) + session.commit() + except Exception as error: + session.rollback() + logger.exception("ebook_admin_generate_ngrams_failed") + return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500) + + logger.info( + "ebook_admin_generate_ngrams_complete books_seen=%s books_built=%s candidates=%s", + result.books_seen, + result.books_built, + result.candidate_phrases, + ) + return templates.TemplateResponse( + request, + "partials/admin_status.html", + { + "message": ( + f"Generated n-grams for {result.books_built} of {result.books_seen} books; " + f"{result.candidate_phrases} candidates stored" + ) + }, + ) + + +@router.post("/judge-ngrams", response_class=HTMLResponse) +def judge_ngrams(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse: + """Judge stored candidate n-grams and promote accepted protected phrases.""" + try: + result = judge_candidate_phrases_for_books(session, config) + session.commit() + except Exception as error: + session.rollback() + logger.exception("ebook_admin_judge_ngrams_failed") + return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500) + + logger.info( + "ebook_admin_judge_ngrams_complete books_seen=%s books_judged=%s candidates_judged=%s protected=%s mentions=%s", + result.books_seen, + result.books_judged, + result.candidates_judged, + result.protected_phrases, + result.phrase_mentions, + ) + return templates.TemplateResponse( + request, + "partials/admin_status.html", + { + "message": ( + f"Judged {result.candidates_judged} candidates across {result.books_judged} of " + f"{result.books_seen} books; {result.protected_phrases} protected phrases, " + f"{result.phrase_mentions} mentions" + ) + }, + ) + + @router.post("/embed-missing", response_class=HTMLResponse) def embed_missing(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse: """Embed chunks missing vectors for the configured model.""" diff --git a/python/ebook_search/api/routes/page.py b/python/ebook_search/api/routes/page.py index 0cd592e..0ba67af 100644 --- a/python/ebook_search/api/routes/page.py +++ b/python/ebook_search/api/routes/page.py @@ -4,16 +4,17 @@ from __future__ import annotations import logging -from fastapi import APIRouter, Request -from fastapi.responses import HTMLResponse -from sqlalchemy import select +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import HTMLResponse, RedirectResponse +from sqlalchemy import func, select from python.ebook_search.api.dependencies import ( AppConfig, # noqa: TC001 FastAPI resolves this annotated dependency at runtime ) from python.ebook_search.api.web import templates +from python.ebook_search.protected_phrases.lib import recalculate_candidate_phrases_for_book from python.fastapi_tools import DbSession # noqa: TC001 FastAPI resolves this annotated dependency at runtime -from python.orm.richie import EbookSource +from python.orm.richie import EbookCandidatePhrase, EbookProtectedPhrase, EbookSource logger = logging.getLogger(__name__) @@ -34,25 +35,127 @@ def books(request: Request, session: DbSession) -> HTMLResponse: return templates.TemplateResponse(request, "books.html", {"sources": sources}) +def get_candidate_count(session: DbSession, book_id: int) -> int: + """Return the number of indexed candidates for one book.""" + return ( + session.scalar(select(func.count(EbookCandidatePhrase.id)).where(EbookCandidatePhrase.book_id == book_id)) or 0 + ) + + +def get_judged_candidate_count(session: DbSession, book_id: int) -> int: + """Return the number of judged candidates for one book.""" + return ( + session.scalar( + select(func.count(EbookCandidatePhrase.id)).where( + EbookCandidatePhrase.book_id == book_id, + EbookCandidatePhrase.llm_judged.is_(True), + ) + ) + or 0 + ) + + +def get_protected_count(session: DbSession, book_id: int) -> int: + """Return the number of protected phrases for one book.""" + return ( + session.scalar(select(func.count(EbookProtectedPhrase.id)).where(EbookProtectedPhrase.book_id == book_id)) or 0 + ) + + +def get_candidates(session: DbSession, book_id: int) -> list[EbookCandidatePhrase]: + """Return the indexed candidates for one book.""" + return list( + session.scalars( + select(EbookCandidatePhrase) + .where(EbookCandidatePhrase.book_id == book_id) + .order_by(EbookCandidatePhrase.candidate_score.desc()) + .limit(100) + ) + ) + + +def get_protected_phrases(session: DbSession, book_id: int) -> list[EbookProtectedPhrase]: + """Return the protected phrases for one book.""" + return list( + session.scalars( + select(EbookProtectedPhrase) + .where(EbookProtectedPhrase.book_id == book_id) + .order_by(EbookProtectedPhrase.importance.desc()) + .limit(100) + ) + ) + + @router.get("/books/{source_id}", response_class=HTMLResponse) def book_detail(source_id: int, request: Request, session: DbSession) -> HTMLResponse: """Render details for one indexed book.""" source = session.get(EbookSource, source_id) + phrase_status_message = None + recalculated = request.query_params.get("phrases_recalculated") + if recalculated is not None: + phrase_status_message = f"Recalculated phrases; {recalculated} candidates generated" if source is not None: chapter_count = len(source.chapters) chunk_count = len(source.chunks) + candidate_count = get_candidate_count(session, source.id) + judged_candidate_count = get_judged_candidate_count(session, source.id) + protected_count = get_protected_count(session, source.id) + candidates = get_candidates(session, source.id) + protected_phrases = get_protected_phrases(session, source.id) else: chapter_count = 0 chunk_count = 0 + candidate_count = 0 + judged_candidate_count = 0 + protected_count = 0 + candidates = [] + protected_phrases = [] logger.info( - "ebook_book_detail_loaded source_id=%s found=%s chapters=%s chunks=%s", + "ebook_book_detail_loaded source_id=%s found=%s chapters=%s chunks=%s candidates=%s judged=%s protected=%s", source_id, source is not None, chapter_count, chunk_count, + candidate_count, + judged_candidate_count, + protected_count, ) return templates.TemplateResponse( request, "book_detail.html", - {"chapter_count": chapter_count, "chunk_count": chunk_count, "source": source}, + { + "candidate_count": candidate_count, + "candidates": candidates, + "chapter_count": chapter_count, + "chunk_count": chunk_count, + "judged_candidate_count": judged_candidate_count, + "protected_count": protected_count, + "protected_phrases": protected_phrases, + "phrase_status_message": phrase_status_message, + "source": source, + }, + ) + + +@router.post("/books/{source_id}/recalculate-phrases") +def recalculate_book_phrases(source_id: int, config: AppConfig, session: DbSession) -> RedirectResponse: + """Clear and regenerate candidate phrases for one indexed book.""" + source = session.get(EbookSource, source_id) + if source is None: + raise HTTPException(status_code=404, detail="Book not found") + + result = recalculate_candidate_phrases_for_book(session, source, config) + logger.info( + "ebook_book_phrase_recalculation_complete source_id=%s candidates=%s deleted_candidates=%s " + "deleted_protected=%s deleted_aliases=%s deleted_mentions=%s", + source_id, + result.candidate_phrases, + result.deleted_candidates, + result.deleted_protected_phrases, + result.deleted_aliases, + result.deleted_mentions, + ) + return RedirectResponse( + url=f"/books/{source_id}?phrases_recalculated={result.candidate_phrases}", + status_code=303, ) diff --git a/python/ebook_search/api/templates/admin.html b/python/ebook_search/api/templates/admin.html index b2656ee..cd2dacc 100644 --- a/python/ebook_search/api/templates/admin.html +++ b/python/ebook_search/api/templates/admin.html @@ -1,45 +1,65 @@ -{% extends "base.html" %} - -{% block title %}EPUB Admin{% endblock %} -{% block head %}{% endblock %} - -{% block content %} -
| Model | -Dimensions | -Embedded | -Missing | -Total chunks | -
|---|---|---|---|---|
| {{ item.model_name }} | -{{ item.dimension }} | -{{ item.embedded_chunks }} | -{{ item.missing_chunks }} | -{{ item.total_chunks }} | -
| Model | +Dimensions | +Embedded | +Missing | +Total chunks | +
|---|---|---|---|---|
| {{ item.model_name }} | +{{ item.dimension }} | +{{ item.embedded_chunks }} | +{{ item.missing_chunks }} | +{{ item.total_chunks }} | +
{{ phrase_status_message }}
+ {% endif %}| Phrase | +Status | +Score | +Count | +Chapters | +
|---|---|---|---|---|
| {{ candidate.phrase_text }} | ++ {% if candidate.llm_judged %} + {% if candidate.llm_keep %}Kept{% else %}Rejected{% endif %} + {% else %} + Candidate + {% endif %} + | +{{ "%.2f"|format(candidate.candidate_score) }} | +{{ candidate.raw_count }} | +{{ candidate.chapter_count }} | +
No candidate n-grams.
+ {% endif %} +| Phrase | +Type | +Confidence | +Importance | +
|---|---|---|---|
| {{ phrase.phrase_text }} | +{{ phrase.phrase_type or "phrase" }} | +{{ "%.2f"|format(phrase.confidence) }} | +{{ "%.2f"|format(phrase.importance) }} | +
No protected phrases.
+ {% endif %} +