dotfiles/python/ebook_search/protected_phrases/extraction.py

"""Candidate phrase extraction and scoring for protected phrases."""

from __future__ import annotations

import logging
import re
from collections import Counter, defaultdict
from functools import lru_cache
from time import perf_counter
from typing import TYPE_CHECKING, Protocol

from yake import KeywordExtractor

from python.ebook_search.protected_phrases.config import (
    get_bad_ends,
    get_bad_starts,
    get_ignored_phrases,
    get_junk_tokens,
    get_most_common_words,
)
from python.ebook_search.protected_phrases.models import PhraseCandidate
from python.ebook_search.protected_phrases.text_normalization import tokenize, tokenize_with_offsets

if TYPE_CHECKING:
    from collections.abc import Iterable, Mapping, Sequence

    from python.ebook_search.config import EbookSearchConfig

logger = logging.getLogger(__name__)


BAD_START_SCORE_PENALTY = 10.0
BAD_END_SCORE_PENALTY = 10.0
MULTI_SOURCE_SCORE_BONUS = 2.0
MULTI_SOURCE_MIN_SOURCES = 2
CAPITALIZED_PHRASE_RE = re.compile(r"\b(?:[A-Z][a-zA-Z']+)(?:\s+(?:of|the|and|in|on|for|[A-Z][a-zA-Z']+)){0,6}")


class YakeExtractor(Protocol):
    """Small protocol for the YAKE extractor used by this module."""

    def extract_keywords(self, text: str) -> Iterable[tuple[str, float]]:
        """Return YAKE keyword tuples."""


class YakeExtractorFactory(Protocol):
    """Callable constructor protocol for YAKE keyword extractors."""

    def __call__(self, *, lan: str, n: int, dedupLim: float, top: int) -> YakeExtractor:  # noqa: N803
        """Create a YAKE keyword extractor.

        Args:
            lan (str): Language code passed to YAKE.
            n (int): Maximum n-gram size to extract.
            dedupLim (float): Deduplication similarity threshold.
            top (int): Maximum number of keyphrases to return.

        Returns:
            YakeExtractor: The constructed keyword extractor.
        """


def normalize_candidate_phrase(
    phrase_text: str,
    config: EbookSearchConfig,
    *,
    min_tokens: int | None = None,
    max_tokens: int | None = None,
    strip_leading_article: bool = False,
) -> tuple[str, str, int] | None:
    """Normalize a candidate phrase and validate token bounds.

    Args:
        phrase_text (str): Raw phrase text to normalize.
        config (EbookSearchConfig): Runtime phrase-tuning settings.
        min_tokens (int | None): Minimum token count override; defaults to ``config.phrase_min_tokens``.
        max_tokens (int | None): Maximum token count override; defaults to ``config.phrase_max_tokens``.
        strip_leading_article (bool): Whether to drop a single leading English article.

    Returns:
        tuple[str, str, int] | None: Display text, normalized phrase, and token count, or ``None``
        when the phrase falls outside the token bounds or is ignored.
    """
    normalized_tokens = tokenize_with_offsets(phrase_text)
    start = 0
    if strip_leading_article and normalized_tokens and normalized_tokens[0].text in {"the", "a", "an"}:
        start = 1

    selected_tokens = normalized_tokens[start:]
    min_count = config.phrase_min_tokens if min_tokens is None else min_tokens
    max_count = config.phrase_max_tokens if max_tokens is None else max_tokens
    if len(selected_tokens) < min_count or len(selected_tokens) > max_count:
        return None

    phrase_norm = " ".join(token.text for token in selected_tokens)
    if phrase_norm in get_ignored_phrases():
        return None

    display_text = phrase_text[selected_tokens[0].start_char : selected_tokens[-1].end_char].strip()
    return display_text or phrase_norm, phrase_norm, len(selected_tokens)


def count_raw_ngrams(tokens: Sequence[str], config: EbookSearchConfig) -> Counter[str]:
    """Count every n-gram window in one normalized token block.

    ``tokens`` are already normalized (see :func:`tokenize`), so each window's normalized form
    is the joined tokens directly. Counting into a plain :class:`Counter` rather than
    :class:`PhraseCandidate` objects keeps this hot loop cheap; callers filter ignored phrases
    and materialize candidates per unique phrase afterwards, which is far fewer operations than
    doing either per window.

    Args:
        tokens (Sequence[str]): Normalized tokens for one text block.
        config (EbookSearchConfig): Runtime phrase-tuning settings.

    Returns:
        Counter[str]: Raw occurrence counts keyed by normalized phrase.
    """
    return Counter(
        " ".join(tokens[start : start + ngram_size])
        for ngram_size in range(config.phrase_min_tokens, config.phrase_max_tokens + 1)
        for start in range(len(tokens) - ngram_size + 1)
    )


def extract_raw_ngrams_by_chapter(
    chapters: Sequence[str],
    config: EbookSearchConfig,
) -> dict[str, PhraseCandidate]:
    """Extract raw n-grams across chapters, tracking both raw counts and chapter spread.

    Counting each chapter separately makes chapter spread fall out of dict membership: a phrase's
    ``chapter_count`` is simply how many per-chapter count maps contain it, so no per-window seen
    tracking is needed. This also lets the enrichment step skip re-sliding the same n-gram sizes.

    Phrases below the minimum raw count are dropped here rather than materialized: most unique
    n-grams occur once, and :func:`filter_storable_candidates` would discard them as too rare
    anyway, so building ``PhraseCandidate`` objects for them is wasted work.

    Args:
        chapters (Sequence[str]): Chapter-like text blocks to slide n-gram windows over.
        config (EbookSearchConfig): Runtime phrase-tuning settings.

    Returns:
        dict[str, PhraseCandidate]: Candidates meeting the minimum raw count, keyed by normalized
        phrase, with raw and chapter counts.
    """
    chapter_count_maps = [count_raw_ngrams(tokenize(chapter), config) for chapter in chapters]
    total_counts: Counter[str] = Counter()
    chapter_spread: Counter[str] = Counter()
    for chapter_counts in chapter_count_maps:
        total_counts.update(chapter_counts)
        chapter_spread.update(chapter_counts.keys())
    min_raw_count = minimum_candidate_raw_count(config)
    ignored = get_ignored_phrases()
    return {
        phrase_norm: PhraseCandidate(
            phrase_text=phrase_norm,
            phrase_norm=phrase_norm,
            token_count=phrase_norm.count(" ") + 1,
            source_raw_ngram=True,
            raw_count=raw_count,
            chapter_count=chapter_spread[phrase_norm],
        )
        for phrase_norm, raw_count in total_counts.items()
        if raw_count >= min_raw_count and phrase_norm not in ignored
    }


@lru_cache(maxsize=2)
def get_yake_extractor(max_ngram: int, top_k: int) -> KeywordExtractor:
    """Return a cached YAKE extractor for the given settings.

    Constructing a ``KeywordExtractor`` loads the language's stopword list from disk, so it is
    cached and reused across books rather than rebuilt on every call.

    Args:
        max_ngram (int): Maximum n-gram size to extract.
        top_k (int): Maximum number of keyphrases to request.

    Returns:
        KeywordExtractor: A shared extractor instance for the given settings.
    """
    return KeywordExtractor(lan="en", n=max_ngram, dedupLim=0.85, top=top_k)


def extract_yake_candidates(
    book_text: str,
    config: EbookSearchConfig,
    top_k: int = 1000,
) -> dict[str, PhraseCandidate]:
    """Extract YAKE keyphrases when the optional YAKE package is installed.

    Args:
        book_text (str): Full book text to extract keyphrases from.
        config (EbookSearchConfig): Runtime phrase-tuning settings.
        top_k (int): Maximum number of YAKE keyphrases to request.

    Returns:
        dict[str, PhraseCandidate]: Candidates keyed by normalized phrase, with YAKE scores.
    """
    extractor = get_yake_extractor(config.phrase_max_tokens, top_k)
    out: dict[str, PhraseCandidate] = {}
    for phrase_text, yake_score in extractor.extract_keywords(book_text):
        normalized = normalize_candidate_phrase(phrase_text, config)
        if normalized is None:
            continue
        display_text, phrase_norm, token_count = normalized
        out[phrase_norm] = PhraseCandidate(
            phrase_text=display_text,
            phrase_norm=phrase_norm,
            token_count=token_count,
            source_yake=True,
            yake_score=float(yake_score),
        )
    return out


def extract_capitalized_phrases(original_text: str, config: EbookSearchConfig) -> dict[str, PhraseCandidate]:
    """Extract capitalized phrase runs that often carry fictional terms.

    Args:
        original_text (str): Original-case book text to scan for capitalized runs.
        config (EbookSearchConfig): Runtime phrase-tuning settings.

    Returns:
        dict[str, PhraseCandidate]: Candidates keyed by normalized phrase from capitalized runs.
    """
    out: dict[str, PhraseCandidate] = {}
    for match in CAPITALIZED_PHRASE_RE.finditer(original_text):
        phrase_text = match.group(0).strip()
        normalized = normalize_candidate_phrase(
            phrase_text,
            config,
            max_tokens=config.phrase_max_entity_tokens,
        )
        if normalized is None:
            continue
        display_text, phrase_norm, token_count = normalized
        out[phrase_norm] = PhraseCandidate(
            phrase_text=display_text,
            phrase_norm=phrase_norm,
            token_count=token_count,
            source_capitalized=True,
        )
    return out


def extract_metadata_candidates(
    metadata: Mapping[str, object] | None,
    config: EbookSearchConfig,
) -> dict[str, PhraseCandidate]:
    """Extract phrases from book metadata values such as title, author, and series.

    Args:
        metadata (Mapping[str, object] | None): Book metadata values, or ``None`` when unavailable.
        config (EbookSearchConfig): Runtime phrase-tuning settings.

    Returns:
        dict[str, PhraseCandidate]: Candidates keyed by normalized phrase from metadata values.
    """
    if metadata is None:
        return {}

    out: dict[str, PhraseCandidate] = {}
    for value in metadata.values():
        if value is None:
            continue
        phrase_text = str(value).strip()
        normalized = normalize_candidate_phrase(
            phrase_text,
            config,
            max_tokens=config.phrase_max_entity_tokens,
        )
        if normalized is None:
            continue
        display_text, phrase_norm, token_count = normalized
        out[phrase_norm] = PhraseCandidate(
            phrase_text=display_text,
            phrase_norm=phrase_norm,
            token_count=token_count,
            source_metadata=True,
        )
    return out


def merge_candidate_sources(*sources: Mapping[str, PhraseCandidate]) -> dict[str, PhraseCandidate]:
    """Merge candidate dictionaries by normalized phrase.

    Args:
        *sources (Mapping[str, PhraseCandidate]): Candidate maps to combine, keyed by normalized phrase.

    Returns:
        dict[str, PhraseCandidate]: One merged candidate per normalized phrase.
    """
    merged: dict[str, PhraseCandidate] = {}
    for source in sources:
        for phrase_norm, item in source.items():
            existing = merged.setdefault(
                phrase_norm,
                PhraseCandidate(
                    phrase_text=item.phrase_text,
                    phrase_norm=phrase_norm,
                    token_count=item.token_count,
                ),
            )
            merge_candidate(existing, item)
    return merged


def merge_candidate(existing: PhraseCandidate, item: PhraseCandidate) -> None:
    """Merge one candidate into an existing candidate object.

    Args:
        existing (PhraseCandidate): Candidate mutated in place to absorb ``item``.
        item (PhraseCandidate): Candidate whose sources, counts, and scores are merged in.
    """
    existing.source_raw_ngram = existing.source_raw_ngram or item.source_raw_ngram
    existing.source_yake = existing.source_yake or item.source_yake
    existing.source_capitalized = existing.source_capitalized or item.source_capitalized
    existing.source_metadata = existing.source_metadata or item.source_metadata
    existing.raw_count += item.raw_count
    existing.chapter_count = max(existing.chapter_count, item.chapter_count)
    if item.yake_score is not None:
        existing.yake_score = item.yake_score


def enrich_with_frequency_and_chapter_counts(
    candidates: Mapping[str, PhraseCandidate],
    chapters: Sequence[str],
    *,
    counted_sizes: Iterable[int] = (),
) -> dict[str, PhraseCandidate]:
    """Add raw occurrence and chapter-spread counts to candidates.

    Candidates whose ``token_count`` is in ``counted_sizes`` are left untouched: those counts
    were already computed while sliding the chapters in :func:`extract_raw_ngrams_by_chapter`,
    so re-sliding those n-gram sizes here would just duplicate that work.

    Args:
        candidates (Mapping[str, PhraseCandidate]): Candidates to enrich, keyed by normalized phrase.
        chapters (Sequence[str]): Chapter-like text blocks used to count occurrences and spread.
        counted_sizes (Iterable[int]): Token counts whose counts are already populated and should be skipped.

    Returns:
        dict[str, PhraseCandidate]: Candidates with updated ``raw_count`` and ``chapter_count`` values.
    """
    if not candidates:
        return {}

    already_counted = set(counted_sizes)
    candidate_sets_by_size: dict[int, set[str]] = defaultdict(set)
    for phrase_norm, candidate in candidates.items():
        if candidate.token_count in already_counted:
            continue
        candidate_sets_by_size[candidate.token_count].add(phrase_norm)

    enriched = dict(candidates)
    if not candidate_sets_by_size:
        return enriched

    total_counts, chapter_counts = count_candidate_occurrences(candidate_sets_by_size, chapters)
    for phrase_norm, candidate in enriched.items():
        if candidate.token_count in already_counted:
            continue
        candidate.raw_count = max(candidate.raw_count, total_counts[phrase_norm])
        candidate.chapter_count = chapter_counts[phrase_norm]
    return enriched


def count_candidate_occurrences(
    candidate_sets_by_size: Mapping[int, set[str]],
    chapters: Sequence[str],
) -> tuple[dict[str, int], dict[str, int]]:
    """Count total occurrences and chapter spread for candidate phrases across chapters.

    Args:
        candidate_sets_by_size (Mapping[int, set[str]]): Candidate normalized phrases grouped by token count.
        chapters (Sequence[str]): Chapter-like text blocks to slide n-gram windows over.

    Returns:
        tuple[dict[str, int], dict[str, int]]: Total occurrence counts and chapter-spread counts,
        each keyed by normalized phrase.
    """
    total_counts: defaultdict[str, int] = defaultdict(int)
    chapter_counts: defaultdict[str, int] = defaultdict(int)
    for chapter in chapters:
        seen_in_chapter: set[str] = set()
        chapter_tokens = tokenize(chapter)
        for ngram_size, candidate_norms in candidate_sets_by_size.items():
            for start in range(len(chapter_tokens) - ngram_size + 1):
                phrase_norm = " ".join(chapter_tokens[start : start + ngram_size])
                if phrase_norm not in candidate_norms:
                    continue
                total_counts[phrase_norm] += 1
                seen_in_chapter.add(phrase_norm)
        for phrase_norm in seen_in_chapter:
            chapter_counts[phrase_norm] += 1
    return total_counts, chapter_counts


def filter_storable_candidates(
    candidates: Mapping[str, PhraseCandidate],
    config: EbookSearchConfig,
) -> tuple[dict[str, PhraseCandidate], int, int, int, int]:
    """Remove candidates that should not be persisted.

    Args:
        candidates (Mapping[str, PhraseCandidate]): Candidates to filter, keyed by normalized phrase.
        config (EbookSearchConfig): Runtime phrase-tuning settings.

    Returns:
        tuple[dict[str, PhraseCandidate], int, int, int, int]: The storable candidates followed by the
        counts dropped for being too short, too rare, too common, and junk.
    """
    min_raw_count = minimum_candidate_raw_count(config)
    filtered: dict[str, PhraseCandidate] = {}
    too_short = 0
    too_rare = 0
    too_common = 0
    junk = 0
    for phrase_norm, candidate in candidates.items():
        if candidate.token_count < config.phrase_min_tokens:
            too_short += 1
            continue
        if candidate.raw_count < min_raw_count:
            too_rare += 1
            continue
        phrase_tokens = phrase_norm.split()
        if is_most_common_word_phrase(phrase_tokens):
            too_common += 1
            continue
        if is_junk_phrase(phrase_tokens):
            junk += 1
            continue
        filtered[phrase_norm] = candidate
    return filtered, too_short, too_rare, too_common, junk


def minimum_candidate_raw_count(config: EbookSearchConfig) -> int:
    """Return the minimum occurrence count required before storing a candidate.

    Args:
        config (EbookSearchConfig): Runtime phrase-tuning settings.

    Returns:
        int: The minimum raw occurrence count, never less than 1.
    """
    return max(config.phrase_raw_ngram_min_count, 1)


def is_most_common_word_phrase(phrase_tokens: list[str]) -> bool:
    """Return whether every token in a normalized phrase is a common word.

    Args:
        phrase_tokens (list[str]): Normalized phrase tokens to inspect.

    Returns:
        bool: True when the phrase is non-empty and every token is a common word.
    """
    common_words = get_most_common_words()
    return bool(phrase_tokens) and all(token in common_words for token in phrase_tokens)


def is_junk_phrase(phrase_tokens: list[str]) -> bool:
    """Return whether a normalized phrase is lexical junk not worth LLM judging.

    Judged data shows phrases containing a dialogue/action verb or a pronoun contraction are
    never kept, and phrases whose tokens are mostly common words almost never are. Possessives
    of proper nouns (``chapman's death``) pass because matching is by exact token, and
    exactly-half-common bigrams (``data feed``) pass because the common-word rule is strict.

    Args:
        phrase_tokens (list[str]): Normalized phrase tokens to inspect.

    Returns:
        bool: True when the phrase contains a junk token or is majority common words.
    """
    if not phrase_tokens:
        return False
    junk_tokens = get_junk_tokens()
    if any(token in junk_tokens for token in phrase_tokens):
        return True
    common_words = get_most_common_words()
    half_phrase_len = len(phrase_tokens) // 2
    return sum(token in common_words for token in phrase_tokens) > half_phrase_len


def score_candidate(candidate: PhraseCandidate, config: EbookSearchConfig) -> float:
    """Score a phrase candidate before LLM judging.

    Args:
        candidate (PhraseCandidate): Candidate to score.
        config (EbookSearchConfig): Runtime phrase-tuning settings.

    Returns:
        float: Combined score from sources, frequency, and length, less any penalties.
    """
    score = source_score(candidate) + frequency_score(candidate, config) + token_count_score(candidate, config)
    if non_raw_source_count(candidate) >= MULTI_SOURCE_MIN_SOURCES:
        score += MULTI_SOURCE_SCORE_BONUS
    if candidate.phrase_norm in get_ignored_phrases():
        score -= 100.0
    if has_bad_start(candidate.phrase_norm):
        score -= BAD_START_SCORE_PENALTY
    if has_bad_end(candidate.phrase_norm):
        score -= BAD_END_SCORE_PENALTY
    return score


def non_raw_source_count(candidate: PhraseCandidate) -> int:
    """Count the non-raw-ngram extraction sources that produced a candidate.

    Args:
        candidate (PhraseCandidate): Candidate whose enabled sources are counted.

    Returns:
        int: Number of enabled sources other than the raw n-gram slide.
    """
    return sum(
        (
            candidate.source_yake,
            candidate.source_capitalized,
            candidate.source_metadata,
        )
    )


def has_bad_start(phrase_norm: str) -> bool:
    """Return whether a normalized phrase starts with a bad starting token.

    Args:
        phrase_norm (str): Normalized phrase text to inspect.

    Returns:
        bool: True when the first token is a known bad starting token.
    """
    phrase_tokens = phrase_norm.split()
    return bool(phrase_tokens and phrase_tokens[0] in get_bad_starts())


def has_bad_end(phrase_norm: str) -> bool:
    """Return whether a normalized phrase ends with a bad ending token.

    Args:
        phrase_norm (str): Normalized phrase text to inspect.

    Returns:
        bool: True when the last token is a known bad ending token.
    """
    phrase_tokens = phrase_norm.split()
    return bool(phrase_tokens and phrase_tokens[-1] in get_bad_ends())


def source_score(candidate: PhraseCandidate) -> float:
    """Return the score contribution from extraction sources.

    Args:
        candidate (PhraseCandidate): Candidate whose enabled sources are weighted.

    Returns:
        float: Summed weight of the candidate's enabled extraction sources.
    """
    return sum(
        weight
        for enabled, weight in (
            (candidate.source_yake, 2.0),
            (candidate.source_capitalized, 2.0),
            (candidate.source_metadata, 2.0),
            (candidate.source_raw_ngram, 0.5),
        )
        if enabled
    )


def frequency_score(candidate: PhraseCandidate, config: EbookSearchConfig) -> float:
    """Return the score contribution from frequency and chapter spread.

    Args:
        candidate (PhraseCandidate): Candidate whose counts are scored.
        config (EbookSearchConfig): Runtime phrase-tuning settings holding score thresholds.

    Returns:
        float: Summed weight for each frequency and chapter-spread threshold the candidate meets.
    """
    return sum(
        weight
        for count, threshold, weight in (
            (candidate.raw_count, config.phrase_raw_count_score_threshold, 0.5),
            (candidate.raw_count, config.phrase_raw_count_high_score_threshold, 0.5),
            (candidate.chapter_count, config.phrase_chapter_count_score_threshold, 0.5),
            (candidate.chapter_count, config.phrase_chapter_count_high_score_threshold, 0.5),
        )
        if count >= threshold
    )


def token_count_score(candidate: PhraseCandidate, config: EbookSearchConfig) -> float:
    """Return the score contribution from phrase length.

    Args:
        candidate (PhraseCandidate): Candidate whose token count is scored.
        config (EbookSearchConfig): Runtime phrase-tuning settings holding the max token bound.

    Returns:
        float: Length-based score contribution, which may be negative for over- or under-length phrases.
    """
    if candidate.token_count == 1:
        return -0.5
    if candidate.token_count in {2, 3, 4}:
        return 0.5
    if candidate.token_count > config.phrase_max_tokens:
        return -1.0
    return 0.0


def get_sample_contexts(normalized_book_text: str, phrase_norm: str, max_contexts: int = 5) -> list[str]:
    """Return normalized context snippets containing a candidate phrase.

    ``normalized_book_text`` is expected to already be ``normalize_text``-ed by the caller
    so the whole book is not re-normalized for every phrase.

    Args:
        normalized_book_text (str): Whole book text, already normalized, to search.
        phrase_norm (str): Normalized phrase to find contexts around.
        max_contexts (int): Maximum number of context snippets to return.

    Returns:
        list[str]: Up to ``max_contexts`` normalized snippets surrounding the phrase.
    """
    contexts: list[str] = []
    start = 0
    while len(contexts) < max_contexts:
        index = normalized_book_text.find(phrase_norm, start)
        if index == -1:
            break
        left = max(0, index - 300)
        right = min(len(normalized_book_text), index + len(phrase_norm) + 300)
        contexts.append(normalized_book_text[left:right])
        start = index + len(phrase_norm)
    return contexts


def candidate_source_names(candidate: PhraseCandidate) -> list[str]:
    """Return enabled source names for an extracted candidate.

    Args:
        candidate (PhraseCandidate): Candidate whose enabled sources are listed.

    Returns:
        list[str]: Names of the extraction sources that produced the candidate.
    """
    names: list[str] = []
    if candidate.source_raw_ngram:
        names.append("raw_ngram")
    if candidate.source_yake:
        names.append("yake")
    if candidate.source_capitalized:
        names.append("capitalized")
    if candidate.source_metadata:
        names.append("metadata")
    return names


def extract_phrase_candidates_for_book(
    book_text: str,
    chapters: Sequence[str],
    config: EbookSearchConfig,
    *,
    metadata: Mapping[str, object] | None = None,
) -> list[PhraseCandidate]:
    """Extract, score, and limit phrase candidates for one book.

    Args:
        book_text (str): Full book text used for most extraction sources.
        chapters (Sequence[str]): Chapter-like text blocks used for frequency counts.
        config (EbookSearchConfig): Runtime phrase-tuning settings.
        metadata (Mapping[str, object] | None): Optional book metadata used as a candidate source.

    Returns:
        list[PhraseCandidate]: Scored candidates sorted best-first and capped per book.
    """
    started_at = perf_counter()
    logger.info(
        f"ebook_phrase_candidate_extract_start chapters={len(chapters)} chars={len(book_text)} "
        f"{config.phrase_min_tokens=} {config.phrase_max_tokens=} {config.protected_phrase_max_candidates_per_book=}"
    )
    raw_started_at = perf_counter()
    raw = extract_raw_ngrams_by_chapter(chapters, config)
    logger.info(
        f"ebook_phrase_candidate_extract_raw_complete candidates={len(raw)} "
        f"duration_ms={(perf_counter() - raw_started_at) * 1000:.1f}"
    )
    yake_started_at = perf_counter()
    yake_candidates = extract_yake_candidates(book_text, config)
    logger.info(
        f"ebook_phrase_candidate_extract_yake_complete candidates={len(yake_candidates)} "
        f"duration_ms={(perf_counter() - yake_started_at) * 1000:.1f}"
    )
    capitalized_started_at = perf_counter()
    capitalized = extract_capitalized_phrases(book_text, config)
    logger.info(
        f"ebook_phrase_candidate_extract_capitalized_complete candidates={len(capitalized)} "
        f"duration_ms={(perf_counter() - capitalized_started_at) * 1000:.1f}"
    )
    metadata_candidates = extract_metadata_candidates(metadata, config)

    candidates = merge_candidate_sources(raw, yake_candidates, capitalized, metadata_candidates)
    enriched_started_at = perf_counter()
    # Raw n-gram sizes were already counted per chapter above, so only enrich the remaining
    # (entity-length) sizes here instead of re-sliding every size over the whole book.
    candidates = enrich_with_frequency_and_chapter_counts(
        candidates,
        chapters,
        counted_sizes=range(config.phrase_min_tokens, config.phrase_max_tokens + 1),
    )
    pre_filter_count = len(candidates)
    candidates, filtered_too_short, filtered_too_rare, filtered_too_common, filtered_junk = filter_storable_candidates(
        candidates, config
    )
    for candidate in candidates.values():
        candidate.candidate_score = score_candidate(candidate, config)

    limited = sorted(candidates.values(), key=lambda item: item.candidate_score, reverse=True)[
        : config.protected_phrase_max_candidates_per_book
    ]
    logger.info(
        f"ebook_phrase_candidate_extract_complete raw={len(raw)} yake={len(yake_candidates)} "
        f"capitalized={len(capitalized)} metadata={len(metadata_candidates)} {pre_filter_count=} {filtered_too_short=} "
        f"{filtered_too_rare=} {filtered_too_common=} {filtered_junk=} min_uses={minimum_candidate_raw_count(config)} "
        f"storable={len(candidates)} limited={len(limited)} "
        f"enrich_score_ms={(perf_counter() - enriched_started_at) * 1000:.1f} "
        f"duration_ms={(perf_counter() - started_at) * 1000:.1f}"
    )
    return limited