205 lines
7.7 KiB
Python
205 lines
7.7 KiB
Python
"""Admin routes for the EPUB search web UI."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
|
|
from fastapi import APIRouter, Request
|
|
from fastapi.responses import HTMLResponse
|
|
|
|
from python.ebook_search.api.bm25_tasks import schedule_bm25_refresh
|
|
from python.ebook_search.api.dependencies import ( # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
|
|
AppConfig,
|
|
AppHttpClient,
|
|
)
|
|
from python.ebook_search.api.web import error_response, templates
|
|
from python.ebook_search.embeddings import embed_missing_chunks, embedding_model_stats
|
|
from python.ebook_search.ingest import ingest_configured_paths
|
|
from python.ebook_search.protected_phrases.generate_ngrams import generate_candidate_phrases_for_books
|
|
from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
|
|
from python.ebook_search.protected_phrases.store import book_ids_pending_first_judgment, corpus_phrase_stats
|
|
from python.fastapi_tools import ( # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
|
|
AppAsyncEngine,
|
|
AsyncDbSession,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
router = APIRouter(prefix="/admin")
|
|
|
|
|
|
@router.get("", response_class=HTMLResponse)
|
|
async def admin(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
|
|
"""Render the admin page."""
|
|
stats = await embedding_model_stats(session)
|
|
phrase_stats = await corpus_phrase_stats(session)
|
|
logger.info(
|
|
f"ebook_admin_page_loaded models={len(stats)} {phrase_stats.candidate_phrases=} "
|
|
f"{phrase_stats.protected_phrases=}"
|
|
)
|
|
return templates.TemplateResponse(
|
|
request,
|
|
"admin.html",
|
|
{"config": config, "stats": stats, "phrase_stats": phrase_stats},
|
|
)
|
|
|
|
|
|
@router.post("/scan", response_class=HTMLResponse)
|
|
async def scan_library(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
|
|
"""Scan configured library paths for EPUB changes."""
|
|
try:
|
|
count = await ingest_configured_paths(session, config)
|
|
await session.commit()
|
|
except Exception as error:
|
|
logger.exception("ebook_admin_scan_failed")
|
|
return error_response(request, error)
|
|
|
|
logger.info(f"ebook_admin_scan_complete {count=}")
|
|
if count > 0:
|
|
schedule_bm25_refresh(request.app)
|
|
return templates.TemplateResponse(request, "partials/admin_status.html", {"message": f"Indexed {count} EPUBs"})
|
|
|
|
|
|
@router.post("/phrases/generate-all", response_class=HTMLResponse)
|
|
async def generate_all_phrases(request: Request, config: AppConfig, engine: AppAsyncEngine) -> HTMLResponse:
|
|
"""Regenerate candidate phrases for every indexed book without LLM judging."""
|
|
try:
|
|
result = await generate_candidate_phrases_for_books(engine, config)
|
|
except Exception as error:
|
|
logger.exception("ebook_admin_generate_phrases_failed")
|
|
return error_response(request, error)
|
|
|
|
logger.info(
|
|
f"ebook_admin_generate_phrases_complete {result.books_seen=} {result.books_built=} {result.candidate_phrases=}"
|
|
)
|
|
return templates.TemplateResponse(
|
|
request,
|
|
"partials/admin_status.html",
|
|
{
|
|
"message": (
|
|
f"Generated phrases for {result.books_built} of {result.books_seen} books; "
|
|
f"{result.candidate_phrases} candidates stored"
|
|
)
|
|
},
|
|
)
|
|
|
|
|
|
@router.post("/phrases/judge-all", response_class=HTMLResponse)
|
|
async def judge_all_phrases(request: Request, engine: AppAsyncEngine, config: AppConfig) -> HTMLResponse:
|
|
"""Judge unjudged candidate phrases across every indexed book."""
|
|
return await run_phrase_judgment(request, engine, config, source_ids=None)
|
|
|
|
|
|
@router.post("/phrases/judge-missing", response_class=HTMLResponse)
|
|
async def judge_missing_phrases(
|
|
request: Request,
|
|
engine: AppAsyncEngine,
|
|
config: AppConfig,
|
|
session: AsyncDbSession,
|
|
) -> HTMLResponse:
|
|
"""Judge candidate phrases only for books where judging has never run."""
|
|
source_ids = await book_ids_pending_first_judgment(session)
|
|
if not source_ids:
|
|
return templates.TemplateResponse(
|
|
request,
|
|
"partials/admin_status.html",
|
|
{"message": "All books with candidate phrases have been judged"},
|
|
)
|
|
return await run_phrase_judgment(request, engine, config, source_ids=source_ids)
|
|
|
|
|
|
async def run_phrase_judgment(
|
|
request: Request,
|
|
engine: AppAsyncEngine,
|
|
config: AppConfig,
|
|
*,
|
|
source_ids: list[int] | None,
|
|
) -> HTMLResponse:
|
|
"""Run LLM judging for candidate phrases and render the outcome as an admin status partial.
|
|
|
|
Args:
|
|
request (Request): Current request, for template rendering.
|
|
engine (AppAsyncEngine): Engine used to open per-book judging sessions.
|
|
config (AppConfig): Runtime phrase-tuning settings.
|
|
source_ids (list[int] | None): Books to judge; ``None`` judges every indexed book.
|
|
|
|
Returns:
|
|
HTMLResponse: Status partial describing the judging outcome.
|
|
"""
|
|
try:
|
|
result = await judge_candidate_phrases_for_books(engine, config, source_ids=source_ids)
|
|
except Exception as error:
|
|
logger.exception("ebook_admin_judge_phrases_failed")
|
|
return error_response(request, error)
|
|
|
|
logger.info(
|
|
f"ebook_admin_judge_phrases_complete {result.books_seen=} {result.books_judged=} {result.books_failed=} "
|
|
f"{result.candidates_judged=} {result.protected_phrases=} {result.phrase_mentions=}"
|
|
)
|
|
return templates.TemplateResponse(
|
|
request,
|
|
"partials/admin_status.html",
|
|
{
|
|
"message": (
|
|
f"Judged {result.candidates_judged} candidates across {result.books_judged} of "
|
|
f"{result.books_seen} books; {result.protected_phrases} protected phrases, "
|
|
f"{result.phrase_mentions} mentions"
|
|
+ (f"; {result.books_failed} books failed" if result.books_failed else "")
|
|
)
|
|
},
|
|
)
|
|
|
|
|
|
@router.post("/embed-missing", response_class=HTMLResponse)
|
|
async def embed_missing(
|
|
request: Request,
|
|
config: AppConfig,
|
|
session: AsyncDbSession,
|
|
client: AppHttpClient,
|
|
) -> HTMLResponse:
|
|
"""Embed chunks missing vectors for the configured model."""
|
|
try:
|
|
count = await embed_missing_chunks(session, client, config)
|
|
await session.commit()
|
|
except Exception as error:
|
|
logger.exception("ebook_admin_embed_missing_failed")
|
|
return error_response(request, error)
|
|
|
|
logger.info(f"ebook_admin_embed_missing_complete {count=}")
|
|
return templates.TemplateResponse(
|
|
request,
|
|
"partials/admin_status.html",
|
|
{"message": f"Embedded {count} chunks"},
|
|
)
|
|
|
|
|
|
@router.post("/embed-all", response_class=HTMLResponse)
|
|
async def embed_all(
|
|
request: Request,
|
|
config: AppConfig,
|
|
session: AsyncDbSession,
|
|
client: AppHttpClient,
|
|
) -> HTMLResponse:
|
|
"""Embed all chunks missing vectors in fixed-size batches."""
|
|
total = 0
|
|
batches = 0
|
|
try:
|
|
while True:
|
|
count = await embed_missing_chunks(session, client, config)
|
|
if count == 0:
|
|
break
|
|
await session.commit()
|
|
total += count
|
|
batches += 1
|
|
logger.info(f"ebook_admin_embed_all_batch_complete {batches=} {count=} {total=}")
|
|
except Exception as error:
|
|
logger.exception(f"ebook_admin_embed_all_failed {batches=} {total=}")
|
|
return error_response(request, f"Embed all failed after {total} chunks in {batches} batches: {error}")
|
|
|
|
logger.info(f"ebook_admin_embed_all_complete {batches=} {total=}")
|
|
return templates.TemplateResponse(
|
|
request,
|
|
"partials/admin_status.html",
|
|
{"message": f"Embedded {total} chunks in {batches} batches of {config.embedding_batch_size}"},
|
|
)
|