test(ebook-search): organize tests under dedicated package

Move ebook search tests into tests/ebook_search and standardize mocking on pytest-mock.
This commit is contained in:
2026-06-16 21:47:40 -04:00
parent a9daa60c17
commit dbc6b5b53b
9 changed files with 330 additions and 276 deletions
+1
View File
@@ -0,0 +1 @@
"""Focused ebook search tests."""
+505
View File
@@ -0,0 +1,505 @@
"""Tests for EPUB search core helpers."""
from __future__ import annotations
import logging
from datetime import UTC, datetime
from os import environ
from pathlib import Path
from types import ModuleType
from typing import TYPE_CHECKING
import pytest
from sqlalchemy import create_engine, select
from sqlalchemy.orm import sessionmaker
from python.ebook_search.answer import answer_query
from python.ebook_search.bm25_corpus import (
BM25Corpus,
BM25CorpusUnavailableError,
BM25Manifest,
ensure_bm25_corpus,
fetch_bm25_corpus_records,
load_bm25_corpus,
read_bm25_manifest,
score_bm25_corpus,
write_bm25_corpus,
)
from python.ebook_search.config import EbookSearchConfig, RerankConfig, load_config, normalize_embedding_model
from python.ebook_search.embeddings import MODEL_DIMENSIONS, ensure_embedding_models
from python.ebook_search.ingest import chunk_text, find_existing_source
from python.ebook_search.search import (
SearchResponse,
SearchResult,
bm25_candidates,
reciprocal_rank_fusion,
retrieval_query_from_text,
)
from python.ebook_search.timing import RuntimeStep
from python.orm.richie import (
EbookChapter,
EbookChunk,
EbookChunkEmbedding1024,
EbookEmbeddingModel,
EbookSource,
RichieBase,
)
if TYPE_CHECKING:
from pytest_mock import MockerFixture
def test_chunk_text_uses_overlap() -> None:
chunks = chunk_text(" ".join(str(index) for index in range(100)), chunk_tokens=20, overlap_tokens=5)
assert len(chunks) > 1
assert chunks[0].token_start == 0
assert chunks[1].token_start == 15
assert all(chunk.token_count <= 20 for chunk in chunks)
def test_reciprocal_rank_fusion_combines_vector_and_bm25_rankings() -> None:
vector_results = [
SearchResult(chunk_id=1, text="a", source_title="A", score=0.9, vector_score=0.9),
SearchResult(chunk_id=2, text="b", source_title="B", score=0.8, vector_score=0.8),
]
lexical_results = [
SearchResult(chunk_id=2, text="b", source_title="B", score=4.2, bm25_score=4.2),
SearchResult(chunk_id=3, text="c", source_title="C", score=2.1, bm25_score=2.1),
]
fused = reciprocal_rank_fusion(vector_results, lexical_results)
assert [result.chunk_id for result in fused] == [2, 1, 3]
assert fused[0].rank_source == "Hybrid"
assert fused[0].vector_score == 0.8
assert fused[0].bm25_score == 4.2
assert fused[0].fused_score == fused[0].score
def test_find_existing_source_matches_path_or_hash() -> None:
engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
RichieBase.metadata.create_all(engine)
with sessionmaker(bind=engine, expire_on_commit=False, future=True)() as session:
source = EbookSource(
title="Book",
author=None,
language=None,
publisher=None,
identifier=None,
file_path="/old/book.epub",
file_sha256="a" * 64,
file_mtime=datetime.now(tz=UTC),
file_size=10,
)
session.add(source)
session.commit()
assert find_existing_source(session, Path("/old/book.epub"), "b" * 64) == source
assert find_existing_source(session, Path("/new/book.epub"), "a" * 64) == source
def test_bm25_corpus_uses_existing_search_text_without_duplicate_metadata() -> None:
engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
RichieBase.metadata.create_all(engine)
with sessionmaker(bind=engine, expire_on_commit=False, future=True)() as session:
source = EbookSource(
title="Book",
author="Author",
language=None,
publisher=None,
identifier=None,
file_path="/book.epub",
file_sha256="a" * 64,
file_mtime=datetime.now(tz=UTC),
file_size=10,
)
session.add(source)
session.flush()
chapter = EbookChapter(source_id=source.id, spine_index=0, title="Chapter", href=None)
session.add(chapter)
session.flush()
session.add(
EbookChunk(
id=1,
source_id=source.id,
chapter_id=chapter.id,
chunk_index=0,
text="content",
token_start=0,
token_count=1,
page_label=None,
content_sha256="b" * 64,
search_text="Book Author Chapter content",
)
)
session.commit()
records, texts = fetch_bm25_corpus_records(session)
assert texts == ["Book Author Chapter content"]
assert records[0]["chunk_id"] == 1
assert "bm25_text" not in records[0]
def test_reciprocal_rank_fusion_marks_hybrid_source() -> None:
vector_results = [SearchResult(chunk_id=1, text="a", source_title="A")]
lexical_results = [SearchResult(chunk_id=2, text="b", source_title="B")]
fused = reciprocal_rank_fusion(vector_results, lexical_results)
assert {result.rank_source for result in fused} == {"Hybrid"}
def test_search_response_sums_runtime_steps() -> None:
response = SearchResponse(
query="query",
results=[],
rank_label="Hybrid",
timings=(
RuntimeStep(name="A", duration_ms=1.25),
RuntimeStep(name="B", duration_ms=2.75),
RuntimeStep(name="Parallel detail", duration_ms=10.0, counts_toward_total=False),
),
)
assert response.total_runtime_ms == 4.0
def test_retrieval_query_keeps_entity_and_series_terms() -> None:
assert retrieval_query_from_text("what does Damien Montgomery stand for in starship mage") == (
"damien montgomery stand starship mage"
)
def test_bm25_candidates_scores_whole_corpus(mocker: MockerFixture) -> None:
record = {
"chunk_id": 2,
"text": "high",
"source_title": "B",
"source_author": None,
"chapter_title": None,
"page_label": None,
"bm25_text": "high",
}
manifest = BM25Manifest(created_at=datetime.now(tz=UTC), db_updated_at=None, chunk_count=1)
corpus = BM25Corpus(retriever=object(), records=(record,), manifest=manifest)
captured: dict[str, object] = {}
def fake_score_bm25_corpus(query, saved_corpus, *, limit):
captured["query"] = query
captured["corpus"] = saved_corpus
captured["limit"] = limit
return [(record, 1.5)]
mocker.patch("python.ebook_search.search.load_bm25_corpus", side_effect=lambda _config: corpus)
mocker.patch("python.ebook_search.search.score_bm25_corpus", side_effect=fake_score_bm25_corpus)
config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
results = bm25_candidates("high", config)
assert captured["query"] == "high"
assert captured["corpus"] == corpus
assert captured["limit"] == 120
assert [result.chunk_id for result in results] == [2]
assert [result.bm25_score for result in results] == [1.5]
def test_bm25_candidates_returns_empty_when_corpus_is_unavailable(mocker: MockerFixture, caplog) -> None:
def fake_load_bm25_corpus(_config):
raise BM25CorpusUnavailableError
mocker.patch("python.ebook_search.search.load_bm25_corpus", side_effect=fake_load_bm25_corpus)
config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
with caplog.at_level(logging.WARNING):
results = bm25_candidates("high", config)
assert results == []
assert "ebook_bm25_index_unavailable_skipping" in caplog.text
def test_write_bm25_corpus_publishes_dated_generation(tmp_path) -> None:
index_path = tmp_path / "bm25"
index_path.mkdir()
generations_path = index_path / "generations"
generations_path.mkdir()
old_generation = generations_path / "20260101T000000.000000Z"
old_generation.mkdir()
(old_generation / "sentinel").write_text("old", encoding="utf-8")
(index_path / "current").symlink_to(Path("generations") / old_generation.name, target_is_directory=True)
manifest = BM25Manifest(
created_at=datetime(2026, 6, 12, 1, 2, 3, 456789, tzinfo=UTC),
db_updated_at=None,
chunk_count=0,
)
write_bm25_corpus(index_path, [], [], manifest)
current_path = index_path / "current"
assert current_path.is_symlink()
assert current_path.readlink() == generations_path / "20260612T010203.456789Z"
assert old_generation.is_dir()
assert (old_generation / "sentinel").read_text(encoding="utf-8") == "old"
assert (generations_path / "20260612T010203.456789Z").is_dir()
assert read_bm25_manifest(index_path) == manifest
def test_write_bm25_corpus_keeps_current_generation_when_publish_fails(mocker: MockerFixture, tmp_path) -> None:
index_path = tmp_path / "bm25"
index_path.mkdir()
generations_path = index_path / "generations"
generations_path.mkdir()
old_generation = generations_path / "20260101T000000.000000Z"
old_generation.mkdir()
(old_generation / "sentinel").write_text("old", encoding="utf-8")
current_path = index_path / "current"
current_path.symlink_to(Path("generations") / old_generation.name, target_is_directory=True)
original_replace = Path.replace
def fail_current_replace(self, target):
if self.parent == index_path and self.name.startswith(".current.") and target == current_path:
msg = "current publish failed"
raise OSError(msg)
return original_replace(self, target)
mocker.patch.object(Path, "replace", fail_current_replace)
manifest = BM25Manifest(
created_at=datetime(2026, 6, 12, 1, 2, 3, 456789, tzinfo=UTC),
db_updated_at=None,
chunk_count=0,
)
with pytest.raises(OSError, match="current publish failed"):
write_bm25_corpus(index_path, [], [], manifest)
assert current_path.readlink() == Path("generations") / old_generation.name
assert (old_generation / "sentinel").read_text(encoding="utf-8") == "old"
assert not (generations_path / "20260612T010203.456789Z").exists()
def test_load_bm25_corpus_uses_current_generation(tmp_path) -> None:
load_bm25_corpus.cache_clear()
index_path = tmp_path / "bm25"
manifest = BM25Manifest(
created_at=datetime(2026, 6, 12, 1, 2, 3, 456789, tzinfo=UTC),
db_updated_at=None,
chunk_count=1,
)
record = {
"chunk_id": 2,
"text": "cached",
"source_title": "B",
"source_author": None,
"chapter_title": None,
"page_label": None,
}
write_bm25_corpus(index_path, [record], ["cached phrase"], manifest)
config = EbookSearchConfig(rerank=RerankConfig(enabled=False), bm25_index_dir=str(index_path))
try:
corpus = load_bm25_corpus(config)
finally:
load_bm25_corpus.cache_clear()
assert corpus.manifest == manifest
assert corpus.records[0]["chunk_id"] == 2
assert score_bm25_corpus("cached", corpus, limit=10)
def test_load_bm25_corpus_caches_disk_load(mocker: MockerFixture, tmp_path) -> None:
load_bm25_corpus.cache_clear()
manifest = BM25Manifest(created_at=datetime.now(tz=UTC), db_updated_at=None, chunk_count=1)
record = {
"chunk_id": 2,
"text": "cached",
"source_title": "B",
"source_author": None,
"chapter_title": None,
"page_label": None,
"bm25_text": "cached",
}
load_count = 0
class FakeRetriever:
"""Fake persisted BM25 retriever."""
corpus = (record,)
class FakeBM25:
"""Fake BM25 class with observable load count."""
@staticmethod
def load(index_path, *, load_corpus, mmap):
nonlocal load_count
load_count += 1
assert index_path == tmp_path
assert load_corpus is True
assert mmap is True
return FakeRetriever()
fake_bm25s = ModuleType("bm25s")
fake_bm25s.BM25 = FakeBM25
mocker.patch("python.ebook_search.bm25_corpus.read_bm25_manifest", side_effect=lambda _path: manifest)
mocker.patch("python.ebook_search.bm25_corpus.bm25_index_exists", side_effect=lambda _path, _manifest: True)
mocker.patch("python.ebook_search.bm25_corpus.bm25s", fake_bm25s)
config = EbookSearchConfig(rerank=RerankConfig(enabled=False), bm25_index_dir=str(tmp_path))
try:
first = load_bm25_corpus(config)
second = load_bm25_corpus(config)
finally:
load_bm25_corpus.cache_clear()
assert first is second
assert first is not None
assert first.records == (record,)
assert load_count == 1
def test_load_bm25_corpus_raises_when_index_is_missing(mocker: MockerFixture, tmp_path) -> None:
load_bm25_corpus.cache_clear()
mocker.patch("python.ebook_search.bm25_corpus.read_bm25_manifest", side_effect=lambda _path: None)
mocker.patch("python.ebook_search.bm25_corpus.bm25_index_exists", side_effect=lambda _path, _manifest: False)
config = EbookSearchConfig(rerank=RerankConfig(enabled=False), bm25_index_dir=str(tmp_path))
try:
with pytest.raises(BM25CorpusUnavailableError, match="BM25 corpus is not available"):
load_bm25_corpus(config)
finally:
load_bm25_corpus.cache_clear()
def test_ensure_bm25_corpus_refreshes_missing_index(mocker: MockerFixture) -> None:
refreshed: list[object] = []
db_updated_at = datetime.now(tz=UTC)
mocker.patch("python.ebook_search.bm25_corpus.read_bm25_manifest", side_effect=lambda _path: None)
mocker.patch("python.ebook_search.bm25_corpus.bm25_index_exists", side_effect=lambda _path, _manifest: False)
mocker.patch("python.ebook_search.bm25_corpus.corpus_last_updated_at", side_effect=lambda _session: db_updated_at)
mocker.patch(
"python.ebook_search.bm25_corpus.refresh_bm25_corpus",
side_effect=lambda session, config, *, db_updated_at: refreshed.append((session, config, db_updated_at)),
)
config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
session = object()
ensure_bm25_corpus(session, config)
assert refreshed == [(session, config, db_updated_at)]
def test_ensure_bm25_corpus_refreshes_stale_index(mocker: MockerFixture) -> None:
refreshed: list[object] = []
created_at = datetime(2026, 1, 1, tzinfo=UTC)
db_updated_at = datetime(2026, 1, 2, tzinfo=UTC)
manifest = BM25Manifest(created_at=created_at, db_updated_at=created_at, chunk_count=10)
mocker.patch("python.ebook_search.bm25_corpus.read_bm25_manifest", side_effect=lambda _path: manifest)
mocker.patch("python.ebook_search.bm25_corpus.bm25_index_exists", side_effect=lambda _path, _manifest: True)
mocker.patch("python.ebook_search.bm25_corpus.corpus_last_updated_at", side_effect=lambda _session: db_updated_at)
mocker.patch(
"python.ebook_search.bm25_corpus.refresh_bm25_corpus",
side_effect=lambda session, config, *, db_updated_at: refreshed.append((session, config, db_updated_at)),
)
config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
session = object()
ensure_bm25_corpus(session, config)
assert refreshed == [(session, config, db_updated_at)]
def test_supported_embedding_models_match_service_names() -> None:
assert MODEL_DIMENSIONS == {
"qwen3-embedding-0.6b": 1024,
"qwen3-embedding-4b": 2560,
"qwen3-embedding-8b": 4096,
}
def test_ensure_embedding_models_registers_service_names() -> None:
engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
RichieBase.metadata.create_all(engine)
with sessionmaker(bind=engine, expire_on_commit=False, future=True)() as session:
ensure_embedding_models(session)
session.commit()
models = list(session.scalars(select(EbookEmbeddingModel).order_by(EbookEmbeddingModel.name)))
assert [(model.name, model.dimension) for model in models] == [
("qwen3-embedding-0.6b", 1024),
("qwen3-embedding-4b", 2560),
("qwen3-embedding-8b", 4096),
]
def test_1024_embedding_table_has_cosine_hnsw_index() -> None:
indexes = {index.name: index for index in EbookChunkEmbedding1024.__table__.indexes}
index = indexes["ix_ebook_chunk_embedding_1024_embedding_cosine"]
assert [column.name for column in index.columns] == ["embedding"]
assert index.dialect_options["postgresql"]["using"] == "hnsw"
assert index.dialect_options["postgresql"]["ops"] == {"embedding": "vector_cosine_ops"}
def test_embedding_model_aliases_normalize_to_provider_names(mocker: MockerFixture) -> None:
mocker.patch.dict(environ, {}, clear=False)
assert normalize_embedding_model() == "qwen3-embedding-0.6b"
environ["EBOOK_SEARCH_EMBEDDING_MODEL"] = "qwen3-embedding-0.6b"
assert normalize_embedding_model() == "qwen3-embedding-0.6b"
environ["EBOOK_SEARCH_EMBEDDING_MODEL"] = "Qwen3-Embedding-0.6B"
assert normalize_embedding_model() == "qwen3-embedding-0.6b"
environ["EBOOK_SEARCH_EMBEDDING_MODEL"] = "Qwen/Qwen3-Embedding-4B"
assert normalize_embedding_model() == "qwen3-embedding-4b"
environ["EBOOK_SEARCH_EMBEDDING_MODEL"] = "qwen3-embedding:8b"
assert normalize_embedding_model() == "qwen3-embedding-8b"
environ["EBOOK_SEARCH_EMBEDDING_MODEL"] = "qwen3-embedding-8b"
assert normalize_embedding_model() == "qwen3-embedding-8b"
def test_answer_generation_is_enabled_by_default(mocker: MockerFixture) -> None:
mocker.patch.dict(environ, {}, clear=False)
environ.pop("EBOOK_SEARCH_ANSWER_ENABLED", None)
config = load_config()
assert config.answer_enabled is True
def test_chat_defaults_use_ollama_cloud(mocker: MockerFixture) -> None:
mocker.patch.dict(environ, {}, clear=False)
environ.pop("EBOOK_SEARCH_VLLM_BASE_URL", None)
environ.pop("EBOOK_SEARCH_CHAT_MODEL", None)
config = load_config()
assert config.vllm_base_url == "https://ollama.com/v1"
assert config.chat_model == "deepseek-v4-flash"
def test_chat_api_key_falls_back_to_ollama_api_key(mocker: MockerFixture) -> None:
mocker.patch.dict(environ, {"OLLAMA_API_KEY": "ollama-key"}, clear=False)
environ.pop("EBOOK_SEARCH_VLLM_API_KEY", None)
config = load_config()
assert config.vllm_api_key == "ollama-key"
def test_answer_query_does_not_call_model_when_disabled() -> None:
config = load_config().model_copy(update={"answer_enabled": False})
result = SearchResult(chunk_id=1, text="source text", source_title="Book")
answer = answer_query("question", [result], config)
assert "Answer generation is disabled" in answer
+147
View File
@@ -0,0 +1,147 @@
"""Tests for serve-time output guardrails."""
from __future__ import annotations
from typing import TYPE_CHECKING
from fastapi.testclient import TestClient
from sqlalchemy import create_engine
from python.ebook_search.api.main import create_app
from python.ebook_search.config import EbookSearchConfig, RerankConfig
from python.ebook_search.guardrails import is_confident, retrieval_confidence, validate_citations
from python.ebook_search.search import SearchResponse, SearchResult
if TYPE_CHECKING:
from pytest_mock import MockerFixture
def make_results(count, *, vector_score=0.8):
return [
SearchResult(
chunk_id=index,
text=f"source text {index}",
source_title="Book",
score=vector_score,
vector_score=vector_score,
)
for index in range(1, count + 1)
]
def test_validate_citations_partitions_markers() -> None:
report = validate_citations("Supported by [1] and [2].", result_count=3)
assert report.cited == (1, 2)
assert report.invalid == ()
assert report.grounded is True
def test_validate_citations_flags_out_of_range_marker() -> None:
report = validate_citations("As shown in [5].", result_count=2)
assert report.cited == ()
assert report.invalid == (5,)
assert report.grounded is False
def test_validate_citations_uncited_answer_is_not_grounded() -> None:
report = validate_citations("No citations at all.", result_count=2)
assert report.cited == ()
assert report.invalid == ()
assert report.grounded is False
def test_retrieval_confidence_prefers_rerank_then_vector() -> None:
assert retrieval_confidence([]) == 0.0
rerank_top = [SearchResult(chunk_id=1, text="t", source_title="B", rerank_score=0.7, vector_score=0.2)]
assert retrieval_confidence(rerank_top) == 0.7
vector_top = [SearchResult(chunk_id=1, text="t", source_title="B", vector_score=0.5)]
assert retrieval_confidence(vector_top) == 0.5
def test_is_confident_against_threshold() -> None:
config = EbookSearchConfig(rerank=RerankConfig(enabled=False), min_retrieval_confidence=0.5)
assert is_confident(make_results(1, vector_score=0.6), config) is True
assert is_confident(make_results(1, vector_score=0.4), config) is False
def patch_app_runtime(mocker: MockerFixture):
mocker.patch(
"python.ebook_search.api.main.get_postgres_engine",
side_effect=lambda **_kwargs: create_engine("sqlite+pysqlite:///:memory:", future=True),
)
mocker.patch("python.ebook_search.api.main.ensure_bm25_corpus", side_effect=lambda _session, _config: None)
def test_low_confidence_skips_answer_generation(mocker: MockerFixture) -> None:
called = False
def fake_search_ebooks(_engine, query, _config, *, rerank=False):
del rerank
return SearchResponse(query=query, rank_label="Hybrid", results=make_results(1, vector_score=0.05))
def fake_answer_query(_query, _results, _config):
nonlocal called
called = True
return "answer"
config = EbookSearchConfig(
rerank=RerankConfig(enabled=False),
answer_enabled=True,
min_retrieval_confidence=0.5,
)
mocker.patch("python.ebook_search.api.routes.search.search_ebooks", side_effect=fake_search_ebooks)
mocker.patch("python.ebook_search.api.routes.search.answer_query", side_effect=fake_answer_query)
mocker.patch("python.ebook_search.api.main.load_config", side_effect=lambda: config)
patch_app_runtime(mocker)
app = create_app()
with TestClient(app) as client:
response = client.post("/search", data={"query": "q"})
assert response.status_code == 200
assert called is False
assert "Low retrieval confidence" in response.text
def test_invalid_citation_is_flagged(mocker: MockerFixture) -> None:
def fake_search_ebooks(_engine, query, _config, *, rerank=False):
del rerank
return SearchResponse(query=query, rank_label="Hybrid", results=make_results(2, vector_score=0.9))
mocker.patch("python.ebook_search.api.routes.search.search_ebooks", side_effect=fake_search_ebooks)
mocker.patch(
"python.ebook_search.api.routes.search.answer_query",
side_effect=lambda _query, _results, _config: "Per the text [9].",
)
patch_app_runtime(mocker)
app = create_app()
app.state.config = EbookSearchConfig(rerank=RerankConfig(enabled=False), answer_enabled=True)
with TestClient(app) as client:
response = client.post("/search", data={"query": "q"})
assert response.status_code == 200
assert "Invalid citations" in response.text
assert "9" in response.text
def test_grounded_answer_has_no_warning_badge(mocker: MockerFixture) -> None:
def fake_search_ebooks(_engine, query, _config, *, rerank=False):
del rerank
return SearchResponse(query=query, rank_label="Hybrid", results=make_results(2, vector_score=0.9))
mocker.patch("python.ebook_search.api.routes.search.search_ebooks", side_effect=fake_search_ebooks)
mocker.patch(
"python.ebook_search.api.routes.search.answer_query",
side_effect=lambda _query, _results, _config: "Grounded in [1] and [2].",
)
patch_app_runtime(mocker)
app = create_app()
app.state.config = EbookSearchConfig(rerank=RerankConfig(enabled=False), answer_enabled=True)
with TestClient(app) as client:
response = client.post("/search", data={"query": "q"})
assert response.status_code == 200
assert "Unverified" not in response.text
assert "Invalid citations" not in response.text
+122
View File
@@ -0,0 +1,122 @@
"""Tests for EPUB search health and readiness routes."""
from __future__ import annotations
from typing import TYPE_CHECKING
from fastapi.testclient import TestClient
from sqlalchemy import create_engine
from python.ebook_search.api.main import create_app
from python.ebook_search.config import EbookSearchConfig, RerankConfig
HEALTH_MODULE = "python.ebook_search.api.routes.health"
if TYPE_CHECKING:
from pytest_mock import MockerFixture
def fake_get_postgres_engine(**_kwargs):
"""Return an in-memory engine for route tests."""
return create_engine("sqlite+pysqlite:///:memory:", future=True)
def patch_app_runtime(mocker: MockerFixture):
mocker.patch("python.ebook_search.api.main.get_postgres_engine", side_effect=fake_get_postgres_engine)
mocker.patch("python.ebook_search.api.main.ensure_bm25_corpus", side_effect=lambda _session, _config: None)
def patch_dependencies(mocker: MockerFixture, *, database=True, embedding=True, chat=True, bm25="ok"):
mocker.patch(f"{HEALTH_MODULE}.check_database", side_effect=lambda _session: database)
mocker.patch(f"{HEALTH_MODULE}.check_embedding_endpoint", side_effect=lambda _config: embedding)
mocker.patch(f"{HEALTH_MODULE}.check_chat_endpoint", side_effect=lambda _config: chat)
mocker.patch(f"{HEALTH_MODULE}.check_bm25_status", side_effect=lambda _config: bm25)
def build_client(mocker: MockerFixture, config=None):
resolved = config or EbookSearchConfig(rerank=RerankConfig(enabled=False))
mocker.patch("python.ebook_search.api.main.load_config", side_effect=lambda: resolved)
patch_app_runtime(mocker)
app = create_app()
return TestClient(app)
def test_health_returns_ok(mocker: MockerFixture) -> None:
with build_client(mocker) as client:
response = client.get("/health")
assert response.status_code == 200
assert response.json() == {"status": "ok"}
def test_ready_all_dependencies_ok(mocker: MockerFixture) -> None:
patch_dependencies(mocker)
with build_client(mocker) as client:
response = client.get("/ready")
assert response.status_code == 200
body = response.json()
assert body["status"] == "ready"
assert body["checks"] == {"database": "ok", "embedding": "ok", "chat": "ok", "bm25": "ok"}
def test_ready_embedding_down_is_degraded(mocker: MockerFixture) -> None:
patch_dependencies(mocker, embedding=False)
with build_client(mocker) as client:
response = client.get("/ready")
assert response.status_code == 200
body = response.json()
assert body["status"] == "degraded"
assert body["checks"]["embedding"] == "fail"
def test_ready_chat_down_is_degraded(mocker: MockerFixture) -> None:
patch_dependencies(mocker, chat=False)
with build_client(mocker) as client:
response = client.get("/ready")
assert response.status_code == 200
body = response.json()
assert body["status"] == "degraded"
assert body["checks"]["chat"] == "fail"
def test_ready_chat_disabled_when_answers_off(mocker: MockerFixture) -> None:
patch_dependencies(mocker)
config = EbookSearchConfig(rerank=RerankConfig(enabled=False), answer_enabled=False)
with build_client(mocker, config) as client:
response = client.get("/ready")
assert response.status_code == 200
body = response.json()
assert body["status"] == "ready"
assert body["checks"]["chat"] == "disabled"
def test_ready_database_down_is_unavailable(mocker: MockerFixture) -> None:
patch_dependencies(mocker, database=False)
with build_client(mocker) as client:
response = client.get("/ready")
assert response.status_code == 503
body = response.json()
assert body["status"] == "unavailable"
assert body["checks"]["database"] == "fail"
def test_ready_bm25_missing_is_degraded(mocker: MockerFixture) -> None:
patch_dependencies(mocker, bm25="missing")
with build_client(mocker) as client:
response = client.get("/ready")
assert response.status_code == 200
body = response.json()
assert body["status"] == "degraded"
assert body["checks"]["bm25"] == "missing"
+89
View File
@@ -0,0 +1,89 @@
"""Tests for EPUB search HTTP model adapters."""
from __future__ import annotations
from typing import TYPE_CHECKING
import httpx
import pytest
from python.ebook_search.answer import answer_query
from python.ebook_search.config import EbookSearchConfig, RerankConfig
from python.ebook_search.embeddings import embed_texts
from python.ebook_search.search import SearchResult
if TYPE_CHECKING:
from pytest_mock import MockerFixture
def test_answer_query_uses_httpx_chat_completions(mocker: MockerFixture) -> None:
captured: dict[str, object] = {}
def fake_post(url: str, **kwargs: object) -> httpx.Response:
captured["url"] = url
captured["kwargs"] = kwargs
return httpx.Response(
200,
json={"choices": [{"message": {"content": "grounded answer"}}]},
request=httpx.Request("POST", url),
)
mocker.patch.object(httpx, "post", side_effect=fake_post)
config = EbookSearchConfig(
rerank=RerankConfig(enabled=False),
vllm_base_url="https://ollama.com/v1",
vllm_api_key="secret",
chat_model="deepseek-v4-flash",
)
answer = answer_query("question", [SearchResult(chunk_id=1, text="source", source_title="Book")], config)
assert answer == "grounded answer"
assert captured["url"] == "https://ollama.com/v1/chat/completions"
kwargs = captured["kwargs"]
assert isinstance(kwargs, dict)
assert kwargs["headers"] == {"Authorization": "Bearer secret"}
payload = kwargs["json"]
assert isinstance(payload, dict)
assert payload["model"] == "deepseek-v4-flash"
def test_embed_texts_uses_httpx_embeddings(mocker: MockerFixture) -> None:
captured: dict[str, object] = {}
vector = [0.0] * 1024
def fake_post(url: str, **kwargs: object) -> httpx.Response:
captured["url"] = url
captured["kwargs"] = kwargs
return httpx.Response(
200,
json={"data": [{"embedding": vector}]},
request=httpx.Request("POST", url),
)
mocker.patch.object(httpx, "post", side_effect=fake_post)
config = EbookSearchConfig(
rerank=RerankConfig(enabled=False),
embedding_base_url="http://bob:8000/v1",
embedding_model="qwen3-embedding-0.6b",
)
embeddings = embed_texts(["hello"], config)
assert embeddings == [vector]
assert captured["url"] == "http://bob:8000/v1/embeddings"
kwargs = captured["kwargs"]
assert isinstance(kwargs, dict)
assert kwargs["headers"] == {}
assert kwargs["json"] == {"model": "qwen3-embedding-0.6b", "input": ["hello"]}
def test_embed_texts_rejects_bad_response_shape(mocker: MockerFixture) -> None:
def fake_post(url: str, **_kwargs: object) -> httpx.Response:
return httpx.Response(200, json={"data": [{}]}, request=httpx.Request("POST", url))
mocker.patch.object(httpx, "post", side_effect=fake_post)
config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
with pytest.raises(RuntimeError, match="Embedding request failed"):
embed_texts(["hello"], config)
+50
View File
@@ -0,0 +1,50 @@
"""Tests for the ebook search RAG pipeline orchestration."""
from __future__ import annotations
from threading import Event
from typing import TYPE_CHECKING
from sqlalchemy import create_engine
from python.ebook_search.config import EbookSearchConfig, RerankConfig
from python.ebook_search.search import SearchResult, search_ebooks
if TYPE_CHECKING:
from pytest_mock import MockerFixture
def test_search_ebooks_runs_vector_and_bm25_in_parallel(mocker: MockerFixture) -> None:
engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
vector_started = Event()
bm25_started = Event()
received_engines: list[object] = []
def fake_vector_candidates(received_engine, query, _config):
"""Return vector candidates after confirming BM25 has started."""
received_engines.append(received_engine)
assert query == "what is parallel"
vector_started.set()
assert bm25_started.wait(timeout=2)
return [SearchResult(chunk_id=1, text="vector", source_title="Vector", vector_score=0.9)]
def fake_bm25_candidates(query, _config):
"""Return BM25 candidates after confirming vector search has started."""
assert query == "parallel"
bm25_started.set()
assert vector_started.wait(timeout=2)
return [SearchResult(chunk_id=2, text="bm25", source_title="BM25", bm25_score=2.0)]
mocker.patch("python.ebook_search.search.vector_candidates", side_effect=fake_vector_candidates)
mocker.patch("python.ebook_search.search.bm25_candidates", side_effect=fake_bm25_candidates)
config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
response = search_ebooks(engine, "what is parallel", config)
timings = {step.name: step for step in response.timings}
assert [result.chunk_id for result in response.results] == [1, 2]
assert timings["Embedding + vector search"].counts_toward_total is False
assert timings["BM25 search"].counts_toward_total is False
assert timings["Hybrid retrieval"].counts_toward_total is True
assert timings["BM25 query preparation"].counts_toward_total is True
assert received_engines == [engine]
+157
View File
@@ -0,0 +1,157 @@
"""Tests for EPUB search reranking."""
from __future__ import annotations
from os import environ
from typing import TYPE_CHECKING
import httpx
import pytest
from python.ebook_search.config import EbookSearchConfig, RerankConfig, load_rerank_config
from python.ebook_search.rerank import rerank_chunks
from python.ebook_search.search import SearchResult, apply_rerank, skip_rerank
if TYPE_CHECKING:
from pytest_mock import MockerFixture
def candidates() -> list[SearchResult]:
return [
SearchResult(chunk_id=1, text="alpha", source_title="A", score=0.9),
SearchResult(chunk_id=2, text="beta", source_title="B", score=0.8),
SearchResult(chunk_id=3, text="gamma", source_title="C", score=0.7),
]
def rerank_response(payload: dict[str, object] | None = None, *, content: bytes | None = None) -> httpx.Response:
return httpx.Response(
200,
content=content,
json=payload,
request=httpx.Request("POST", "http://rerank.test/rerank"),
)
def test_config_defaults_keep_reranking_optional(mocker: MockerFixture) -> None:
mocker.patch.dict(environ, {}, clear=False)
environ.pop("EBOOK_SEARCH_RERANK_ENABLED", None)
environ.pop("EBOOK_SEARCH_RERANK_BASE_URL", None)
environ.pop("EBOOK_SEARCH_RERANK_MODEL", None)
environ.pop("EBOOK_SEARCH_RERANK_CANDIDATES", None)
environ.pop("EBOOK_SEARCH_RERANK_TIMEOUT_SECONDS", None)
config = load_rerank_config()
assert config.enabled is False
assert config.base_url == "http://192.168.90.25:8001"
assert config.model == "qwen3-reranker-06b"
assert config.candidates == 24
assert config.timeout_seconds == 30
def test_reranking_disabled_returns_original_fused_order() -> None:
config = EbookSearchConfig(rerank=RerankConfig(enabled=False), top_k=2)
response = skip_rerank("query", candidates(), config)
assert response.rank_label == "Hybrid"
assert [result.chunk_id for result in response.results] == [1, 2]
def test_reranking_enabled_reorders_candidates(mocker: MockerFixture) -> None:
def fake_post(_url: str, *, json: dict[str, object], timeout: float) -> httpx.Response:
assert timeout == 30
assert json == {
"model": "qwen3-reranker-06b",
"query": "query",
"documents": ["alpha", "beta", "gamma"],
}
return rerank_response(
{
"results": [
{"index": 0, "relevance_score": 0.1},
{"index": 1, "relevance_score": 0.9},
{"index": 2, "relevance_score": 0.4},
]
}
)
mocker.patch.object(httpx, "post", side_effect=fake_post)
results = rerank_chunks("query", candidates(), RerankConfig())
assert [result.chunk_id for result in results] == [2, 1, 3]
assert [round(result.score, 3) for result in results] == [0.78, 0.37, 0.28]
assert [result.rerank_score for result in results] == [0.9, 0.1, 0.4]
def test_reranking_cannot_ignore_hybrid_score(mocker: MockerFixture) -> None:
candidates = [
SearchResult(chunk_id=1, text="strong hybrid", source_title="A", score=1.0),
SearchResult(chunk_id=2, text="weak hybrid", source_title="B", score=0.1),
]
def fake_post(_url: str, **_kwargs: object) -> httpx.Response:
return rerank_response(
{
"results": [
{"index": 0, "relevance_score": 0.7},
{"index": 1, "relevance_score": 1.0},
]
}
)
mocker.patch.object(httpx, "post", side_effect=fake_post)
results = rerank_chunks("query", candidates, RerankConfig())
assert [result.chunk_id for result in results] == [1, 2]
assert results[0].score == pytest.approx(0.79)
assert results[1].score == 0.7
assert results[1].rerank_score == 1.0
def test_vllm_rerank_timeout_raises(mocker: MockerFixture) -> None:
def fake_rerank_chunks(
_query: str,
_candidates: list[SearchResult],
_config: RerankConfig,
) -> list[SearchResult]:
message = "timeout"
raise httpx.TimeoutException(message)
mocker.patch("python.ebook_search.search.rerank_chunks", side_effect=fake_rerank_chunks)
config = EbookSearchConfig(rerank=RerankConfig(enabled=True), top_k=2)
with pytest.raises(httpx.TimeoutException, match="timeout"):
apply_rerank("query", candidates(), config)
def test_malformed_vllm_rerank_json_does_not_crash_search(mocker: MockerFixture) -> None:
def fake_post(_url: str, **_kwargs: object) -> httpx.Response:
return rerank_response(content=b"not-json")
mocker.patch.object(httpx, "post", side_effect=fake_post)
results = rerank_chunks("query", candidates()[:1], RerankConfig())
assert results[0].score == 0.3
def test_vllm_rerank_scores_are_clamped(mocker: MockerFixture) -> None:
def fake_post(_url: str, **_kwargs: object) -> httpx.Response:
return rerank_response(
{
"results": [
{"index": 0, "relevance_score": -1},
{"index": 1, "relevance_score": 2},
]
}
)
mocker.patch.object(httpx, "post", side_effect=fake_post)
results = rerank_chunks("query", candidates()[:2], RerankConfig())
assert {result.chunk_id: result.rerank_score for result in results} == {1: 0.0, 2: 1.0}
+312
View File
@@ -0,0 +1,312 @@
"""Tests for EPUB search HTMX routes."""
from __future__ import annotations
from compression import zstd
from typing import TYPE_CHECKING
from fastapi.testclient import TestClient
from sqlalchemy import create_engine
from python.ebook_search.api.bm25_tasks import refresh_bm25_for_engine
from python.ebook_search.api.main import create_app
from python.ebook_search.config import EbookSearchConfig, RerankConfig
from python.ebook_search.embeddings import EmbeddingModelStats
from python.ebook_search.search import SearchResponse, SearchResult
from python.ebook_search.timing import RuntimeStep
if TYPE_CHECKING:
from pytest_mock import MockerFixture
def patch_app_runtime(mocker: MockerFixture):
"""Patch app startup dependencies used by UI route tests."""
mocker.patch("python.ebook_search.api.main.get_postgres_engine", side_effect=fake_get_postgres_engine)
mocker.patch("python.ebook_search.api.main.ensure_bm25_corpus", side_effect=lambda _session, _config: None)
def fake_get_postgres_engine(**_kwargs):
"""Return an in-memory engine for route tests."""
return create_engine("sqlite+pysqlite:///:memory:", future=True)
def test_search_page_uses_zstd_when_requested(mocker: MockerFixture) -> None:
patch_app_runtime(mocker)
app = create_app()
app.state.config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
with TestClient(app) as client:
response = client.get("/", headers={"accept-encoding": "zstd"})
assert response.status_code == 200
assert response.headers["content-encoding"] == "zstd"
assert b"EPUB Search" in zstd.decompress(response.content)
def test_ui_form_passes_rerank_flag_to_search_handler(mocker: MockerFixture) -> None:
captured: dict[str, object] = {}
def fake_search_ebooks(_engine, query, config, *, rerank=False):
captured["query"] = query
captured["rerank"] = rerank
captured["config"] = config
return SearchResponse(query=query, results=[], rank_label="Hybrid + rerank")
mocker.patch("python.ebook_search.api.routes.search.search_ebooks", side_effect=fake_search_ebooks)
mocker.patch(
"python.ebook_search.api.routes.search.answer_query",
side_effect=lambda _query, _results, _config: "answer",
)
patch_app_runtime(mocker)
app = create_app()
app.state.config = EbookSearchConfig(rerank=RerankConfig(enabled=False), top_k=12, answer_enabled=True)
with TestClient(app) as client:
response = client.post("/search", data={"query": "where is the quote?", "rerank": "true"})
assert response.status_code == 200
assert "Hybrid + rerank" in response.text
assert captured["query"] == "where is the quote?"
assert captured["rerank"] is True
def test_ui_search_failure_returns_visible_error(mocker: MockerFixture) -> None:
def fake_search_ebooks(_engine, _query, _config, *, rerank=False):
del rerank
msg = "search exploded"
raise RuntimeError(msg)
mocker.patch("python.ebook_search.api.routes.search.search_ebooks", side_effect=fake_search_ebooks)
patch_app_runtime(mocker)
app = create_app()
app.state.config = EbookSearchConfig(rerank=RerankConfig(enabled=False), top_k=12)
with TestClient(app) as client:
response = client.post("/search", data={"query": "where is the quote?"})
assert response.status_code == 500
assert "search exploded" in response.text
def test_ui_answer_failure_still_returns_sources(mocker: MockerFixture) -> None:
def fake_search_ebooks(_engine, query, _config, *, rerank=False):
del rerank
return SearchResponse(query=query, results=[], rank_label="Hybrid")
def fake_answer_query(_query, _results, _config):
msg = "answer exploded"
raise RuntimeError(msg)
mocker.patch("python.ebook_search.api.routes.search.search_ebooks", side_effect=fake_search_ebooks)
mocker.patch("python.ebook_search.api.routes.search.answer_query", side_effect=fake_answer_query)
patch_app_runtime(mocker)
app = create_app()
app.state.config = EbookSearchConfig(rerank=RerankConfig(enabled=False), top_k=12, answer_enabled=True)
with TestClient(app) as client:
response = client.post("/search", data={"query": "where is the quote?"})
assert response.status_code == 200
assert "Answer generation failed" in response.text
def test_ui_skips_answer_when_disabled(mocker: MockerFixture) -> None:
called = False
def fake_search_ebooks(_engine, query, _config, *, rerank=False):
del rerank
return SearchResponse(query=query, results=[], rank_label="Hybrid")
def fake_answer_query(_query, _results, _config):
nonlocal called
called = True
return "answer"
config = EbookSearchConfig(rerank=RerankConfig(enabled=False), answer_enabled=False)
mocker.patch("python.ebook_search.api.routes.search.search_ebooks", side_effect=fake_search_ebooks)
mocker.patch("python.ebook_search.api.routes.search.answer_query", side_effect=fake_answer_query)
mocker.patch("python.ebook_search.api.main.load_config", side_effect=lambda: config)
patch_app_runtime(mocker)
app = create_app()
with TestClient(app) as client:
response = client.post("/search", data={"query": "where is the quote?"})
assert response.status_code == 200
assert called is False
assert "Answer generation is disabled" in response.text
def test_ui_shows_component_scores(mocker: MockerFixture) -> None:
def fake_search_ebooks(_engine, query, _config, *, rerank=False):
del rerank
return SearchResponse(
query=query,
rank_label="Hybrid + rerank",
results=[
SearchResult(
chunk_id=1,
text="source text",
source_title="Book",
score=0.9,
rerank_score=0.9,
vector_score=0.8,
bm25_score=2.5,
fused_score=0.03,
)
],
)
mocker.patch("python.ebook_search.api.routes.search.search_ebooks", side_effect=fake_search_ebooks)
mocker.patch(
"python.ebook_search.api.routes.search.answer_query",
side_effect=lambda _query, _results, _config: "answer",
)
patch_app_runtime(mocker)
app = create_app()
app.state.config = EbookSearchConfig(rerank=RerankConfig(enabled=False), answer_enabled=True)
with TestClient(app) as client:
response = client.post("/search", data={"query": "where is the quote?"})
assert response.status_code == 200
assert "rerank" in response.text
assert "vector cosine" in response.text
assert "BM25" in response.text
assert "RRF" in response.text
def test_ui_shows_search_runtime_chart(mocker: MockerFixture) -> None:
def fake_search_ebooks(_engine, query, _config, *, rerank=False):
del rerank
return SearchResponse(
query=query,
rank_label="Hybrid",
results=[],
timings=(
RuntimeStep(name="Embedding + vector search", duration_ms=12.5),
RuntimeStep(name="BM25 search", duration_ms=4.0),
),
)
mocker.patch("python.ebook_search.api.routes.search.search_ebooks", side_effect=fake_search_ebooks)
mocker.patch(
"python.ebook_search.api.routes.search.answer_query",
side_effect=lambda _query, _results, _config: "answer",
)
patch_app_runtime(mocker)
app = create_app()
app.state.config = EbookSearchConfig(rerank=RerankConfig(enabled=False), answer_enabled=True)
with TestClient(app) as client:
response = client.post("/search", data={"query": "where is the quote?"})
assert response.status_code == 200
assert "Runtime" in response.text
assert "Total" in response.text
assert "Embedding + vector search" in response.text
assert "BM25 search" in response.text
assert "Answer generation" in response.text
assert "ms left" in response.text
def test_ui_embed_all_batches_until_complete(mocker: MockerFixture) -> None:
counts = iter([32, 32, 5, 0])
batch_sizes: list[int] = []
def fake_embed_missing_chunks(_session, config):
batch_sizes.append(config.embedding_batch_size)
return next(counts)
mocker.patch("python.ebook_search.api.routes.admin.embed_missing_chunks", side_effect=fake_embed_missing_chunks)
patch_app_runtime(mocker)
app = create_app()
with TestClient(app) as client:
response = client.post("/admin/embed-all")
assert response.status_code == 200
assert "Embedded 69 chunks in 3 batches of 32" in response.text
assert batch_sizes == [32, 32, 32, 32]
def test_ui_scan_schedules_bm25_refresh_after_database_change(mocker: MockerFixture) -> None:
scheduled = False
def fake_ingest_configured_paths(_session, _config):
return 1
def fake_schedule_bm25_refresh(_app):
nonlocal scheduled
scheduled = True
mocker.patch(
"python.ebook_search.api.routes.admin.ingest_configured_paths",
side_effect=fake_ingest_configured_paths,
)
mocker.patch("python.ebook_search.api.routes.admin.schedule_bm25_refresh", side_effect=fake_schedule_bm25_refresh)
patch_app_runtime(mocker)
app = create_app()
with TestClient(app) as client:
response = client.post("/admin/scan")
assert response.status_code == 200
assert "Indexed 1 EPUBs" in response.text
assert scheduled is True
def test_bm25_refresh_clears_loaded_corpus_cache(mocker: MockerFixture) -> None:
refreshed: list[object] = []
cache_cleared = False
def fake_refresh_bm25_corpus(session, config):
refreshed.append((session, config))
def fake_cache_clear():
nonlocal cache_cleared
cache_cleared = True
mocker.patch("python.ebook_search.api.bm25_tasks.refresh_bm25_corpus", side_effect=fake_refresh_bm25_corpus)
mocker.patch("python.ebook_search.api.bm25_tasks.load_bm25_corpus.cache_clear", side_effect=fake_cache_clear)
engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
config = EbookSearchConfig(rerank=RerankConfig(enabled=False))
refresh_bm25_for_engine(engine, config)
assert len(refreshed) == 1
assert refreshed[0][1] == config
assert cache_cleared is True
def test_admin_page_shows_embedding_counts_by_model(mocker: MockerFixture) -> None:
def fake_embedding_model_stats(_session):
return [
EmbeddingModelStats(
model_name="qwen3-embedding-0.6b",
dimension=1024,
embedded_chunks=40,
total_chunks=64,
),
EmbeddingModelStats(
model_name="qwen3-embedding-4b",
dimension=2560,
embedded_chunks=8,
total_chunks=64,
),
]
mocker.patch("python.ebook_search.api.routes.admin.embedding_model_stats", side_effect=fake_embedding_model_stats)
patch_app_runtime(mocker)
app = create_app()
with TestClient(app) as client:
response = client.get("/admin")
assert response.status_code == 200
assert "qwen3-embedding-0.6b" in response.text
assert "1024" in response.text
assert "40" in response.text
assert "24" in response.text
assert "qwen3-embedding-4b" in response.text
assert "2560" in response.text