diff --git a/python/ebook_search/api/routes/admin.py b/python/ebook_search/api/routes/admin.py index 2e1fcd1..a441572 100644 --- a/python/ebook_search/api/routes/admin.py +++ b/python/ebook_search/api/routes/admin.py @@ -14,6 +14,11 @@ from python.ebook_search.api.dependencies import ( from python.ebook_search.api.web import templates from python.ebook_search.embeddings import embed_missing_chunks, embedding_model_stats from python.ebook_search.ingest import ingest_configured_paths +from python.ebook_search.protected_phrases.lib import ( + build_missing_protected_phrases, + generate_candidate_phrases_for_books, + judge_candidate_phrases_for_books, +) from python.fastapi_tools import DbSession # noqa: TC001 FastAPI resolves this annotated dependency at runtime logger = logging.getLogger(__name__) @@ -45,6 +50,97 @@ def scan_library(request: Request, config: AppConfig, session: DbSession) -> HTM return templates.TemplateResponse(request, "partials/admin_status.html", {"message": f"Indexed {count} EPUBs"}) +@router.post("/build-phrases", response_class=HTMLResponse) +def build_phrases(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse: + """Build protected phrases for indexed books that are missing them.""" + try: + result = build_missing_protected_phrases(session, config) + session.commit() + except Exception as error: + session.rollback() + logger.exception("ebook_admin_build_phrases_failed") + return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500) + + logger.info( + "ebook_admin_build_phrases_complete books_seen=%s books_built=%s protected=%s mentions=%s", + result.books_seen, + result.books_built, + result.protected_phrases, + result.phrase_mentions, + ) + return templates.TemplateResponse( + request, + "partials/admin_status.html", + { + "message": ( + f"Built phrases for {result.books_built} of {result.books_seen} books; " + f"{result.protected_phrases} protected phrases, {result.phrase_mentions} mentions" + ) + }, + ) + + +@router.post("/generate-ngrams", response_class=HTMLResponse) +def generate_ngrams(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse: + """Generate candidate n-grams for indexed books without LLM judging.""" + try: + result = generate_candidate_phrases_for_books(session, config) + session.commit() + except Exception as error: + session.rollback() + logger.exception("ebook_admin_generate_ngrams_failed") + return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500) + + logger.info( + "ebook_admin_generate_ngrams_complete books_seen=%s books_built=%s candidates=%s", + result.books_seen, + result.books_built, + result.candidate_phrases, + ) + return templates.TemplateResponse( + request, + "partials/admin_status.html", + { + "message": ( + f"Generated n-grams for {result.books_built} of {result.books_seen} books; " + f"{result.candidate_phrases} candidates stored" + ) + }, + ) + + +@router.post("/judge-ngrams", response_class=HTMLResponse) +def judge_ngrams(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse: + """Judge stored candidate n-grams and promote accepted protected phrases.""" + try: + result = judge_candidate_phrases_for_books(session, config) + session.commit() + except Exception as error: + session.rollback() + logger.exception("ebook_admin_judge_ngrams_failed") + return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500) + + logger.info( + "ebook_admin_judge_ngrams_complete books_seen=%s books_judged=%s candidates_judged=%s protected=%s mentions=%s", + result.books_seen, + result.books_judged, + result.candidates_judged, + result.protected_phrases, + result.phrase_mentions, + ) + return templates.TemplateResponse( + request, + "partials/admin_status.html", + { + "message": ( + f"Judged {result.candidates_judged} candidates across {result.books_judged} of " + f"{result.books_seen} books; {result.protected_phrases} protected phrases, " + f"{result.phrase_mentions} mentions" + ) + }, + ) + + @router.post("/embed-missing", response_class=HTMLResponse) def embed_missing(request: Request, config: AppConfig, session: DbSession) -> HTMLResponse: """Embed chunks missing vectors for the configured model.""" diff --git a/python/ebook_search/api/routes/page.py b/python/ebook_search/api/routes/page.py index 0cd592e..0ba67af 100644 --- a/python/ebook_search/api/routes/page.py +++ b/python/ebook_search/api/routes/page.py @@ -4,16 +4,17 @@ from __future__ import annotations import logging -from fastapi import APIRouter, Request -from fastapi.responses import HTMLResponse -from sqlalchemy import select +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import HTMLResponse, RedirectResponse +from sqlalchemy import func, select from python.ebook_search.api.dependencies import ( AppConfig, # noqa: TC001 FastAPI resolves this annotated dependency at runtime ) from python.ebook_search.api.web import templates +from python.ebook_search.protected_phrases.lib import recalculate_candidate_phrases_for_book from python.fastapi_tools import DbSession # noqa: TC001 FastAPI resolves this annotated dependency at runtime -from python.orm.richie import EbookSource +from python.orm.richie import EbookCandidatePhrase, EbookProtectedPhrase, EbookSource logger = logging.getLogger(__name__) @@ -34,25 +35,127 @@ def books(request: Request, session: DbSession) -> HTMLResponse: return templates.TemplateResponse(request, "books.html", {"sources": sources}) +def get_candidate_count(session: DbSession, book_id: int) -> int: + """Return the number of indexed candidates for one book.""" + return ( + session.scalar(select(func.count(EbookCandidatePhrase.id)).where(EbookCandidatePhrase.book_id == book_id)) or 0 + ) + + +def get_judged_candidate_count(session: DbSession, book_id: int) -> int: + """Return the number of judged candidates for one book.""" + return ( + session.scalar( + select(func.count(EbookCandidatePhrase.id)).where( + EbookCandidatePhrase.book_id == book_id, + EbookCandidatePhrase.llm_judged.is_(True), + ) + ) + or 0 + ) + + +def get_protected_count(session: DbSession, book_id: int) -> int: + """Return the number of protected phrases for one book.""" + return ( + session.scalar(select(func.count(EbookProtectedPhrase.id)).where(EbookProtectedPhrase.book_id == book_id)) or 0 + ) + + +def get_candidates(session: DbSession, book_id: int) -> list[EbookCandidatePhrase]: + """Return the indexed candidates for one book.""" + return list( + session.scalars( + select(EbookCandidatePhrase) + .where(EbookCandidatePhrase.book_id == book_id) + .order_by(EbookCandidatePhrase.candidate_score.desc()) + .limit(100) + ) + ) + + +def get_protected_phrases(session: DbSession, book_id: int) -> list[EbookProtectedPhrase]: + """Return the protected phrases for one book.""" + return list( + session.scalars( + select(EbookProtectedPhrase) + .where(EbookProtectedPhrase.book_id == book_id) + .order_by(EbookProtectedPhrase.importance.desc()) + .limit(100) + ) + ) + + @router.get("/books/{source_id}", response_class=HTMLResponse) def book_detail(source_id: int, request: Request, session: DbSession) -> HTMLResponse: """Render details for one indexed book.""" source = session.get(EbookSource, source_id) + phrase_status_message = None + recalculated = request.query_params.get("phrases_recalculated") + if recalculated is not None: + phrase_status_message = f"Recalculated phrases; {recalculated} candidates generated" if source is not None: chapter_count = len(source.chapters) chunk_count = len(source.chunks) + candidate_count = get_candidate_count(session, source.id) + judged_candidate_count = get_judged_candidate_count(session, source.id) + protected_count = get_protected_count(session, source.id) + candidates = get_candidates(session, source.id) + protected_phrases = get_protected_phrases(session, source.id) else: chapter_count = 0 chunk_count = 0 + candidate_count = 0 + judged_candidate_count = 0 + protected_count = 0 + candidates = [] + protected_phrases = [] logger.info( - "ebook_book_detail_loaded source_id=%s found=%s chapters=%s chunks=%s", + "ebook_book_detail_loaded source_id=%s found=%s chapters=%s chunks=%s candidates=%s judged=%s protected=%s", source_id, source is not None, chapter_count, chunk_count, + candidate_count, + judged_candidate_count, + protected_count, ) return templates.TemplateResponse( request, "book_detail.html", - {"chapter_count": chapter_count, "chunk_count": chunk_count, "source": source}, + { + "candidate_count": candidate_count, + "candidates": candidates, + "chapter_count": chapter_count, + "chunk_count": chunk_count, + "judged_candidate_count": judged_candidate_count, + "protected_count": protected_count, + "protected_phrases": protected_phrases, + "phrase_status_message": phrase_status_message, + "source": source, + }, + ) + + +@router.post("/books/{source_id}/recalculate-phrases") +def recalculate_book_phrases(source_id: int, config: AppConfig, session: DbSession) -> RedirectResponse: + """Clear and regenerate candidate phrases for one indexed book.""" + source = session.get(EbookSource, source_id) + if source is None: + raise HTTPException(status_code=404, detail="Book not found") + + result = recalculate_candidate_phrases_for_book(session, source, config) + logger.info( + "ebook_book_phrase_recalculation_complete source_id=%s candidates=%s deleted_candidates=%s " + "deleted_protected=%s deleted_aliases=%s deleted_mentions=%s", + source_id, + result.candidate_phrases, + result.deleted_candidates, + result.deleted_protected_phrases, + result.deleted_aliases, + result.deleted_mentions, + ) + return RedirectResponse( + url=f"/books/{source_id}?phrases_recalculated={result.candidate_phrases}", + status_code=303, ) diff --git a/python/ebook_search/api/templates/admin.html b/python/ebook_search/api/templates/admin.html index b2656ee..cd2dacc 100644 --- a/python/ebook_search/api/templates/admin.html +++ b/python/ebook_search/api/templates/admin.html @@ -1,45 +1,65 @@ -{% extends "base.html" %} - -{% block title %}EPUB Admin{% endblock %} -{% block head %}{% endblock %} - -{% block content %} -

Admin

-
-
-
- -
-
- -
-
- -
-
-
-

Embeddings

- - - - - - - - - - - - {% for item in stats %} - - - - - - - - {% endfor %} - -
ModelDimensionsEmbeddedMissingTotal chunks
{{ item.model_name }}{{ item.dimension }}{{ item.embedded_chunks }}{{ item.missing_chunks }}{{ item.total_chunks }}
-
+{% extends "base.html" %} {% block title %}EPUB Admin{% endblock %} {% block +head %} + +{% endblock %} {% block content %} +

Admin

+
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+

Embeddings

+ + + + + + + + + + + + {% for item in stats %} + + + + + + + + {% endfor %} + +
ModelDimensionsEmbeddedMissingTotal chunks
{{ item.model_name }}{{ item.dimension }}{{ item.embedded_chunks }}{{ item.missing_chunks }}{{ item.total_chunks }}
+
{% endblock %} diff --git a/python/ebook_search/api/templates/book_detail.html b/python/ebook_search/api/templates/book_detail.html index 0da0fd6..1102bec 100644 --- a/python/ebook_search/api/templates/book_detail.html +++ b/python/ebook_search/api/templates/book_detail.html @@ -6,6 +6,9 @@ {% if source %}

{{ source.title }}

{{ source.author or "Unknown author" }}

+ {% if phrase_status_message %} +

{{ phrase_status_message }}

+ {% endif %}
File
{{ source.file_path }}
@@ -13,7 +16,84 @@
{{ chapter_count }}
Chunks
{{ chunk_count }}
+
Candidates
+
{{ candidate_count }}
+
Judged
+
{{ judged_candidate_count }}
+
Protected
+
{{ protected_count }}
+
+ +
+ +
+

Candidate n-grams

+ {% if candidates %} + + + + + + + + + + + + {% for candidate in candidates %} + + + + + + + + {% endfor %} + +
PhraseStatusScoreCountChapters
{{ candidate.phrase_text }} + {% if candidate.llm_judged %} + {% if candidate.llm_keep %}Kept{% else %}Rejected{% endif %} + {% else %} + Candidate + {% endif %} + {{ "%.2f"|format(candidate.candidate_score) }}{{ candidate.raw_count }}{{ candidate.chapter_count }}
+ {% else %} +

No candidate n-grams.

+ {% endif %} +
+ +
+

Protected phrases

+ {% if protected_phrases %} + + + + + + + + + + + {% for phrase in protected_phrases %} + + + + + + + {% endfor %} + +
PhraseTypeConfidenceImportance
{{ phrase.phrase_text }}{{ phrase.phrase_type or "phrase" }}{{ "%.2f"|format(phrase.confidence) }}{{ "%.2f"|format(phrase.importance) }}
+ {% else %} +

No protected phrases.

+ {% endif %} +
{% else %}

Book not found

{% endif %}