"""Candidate phrase extraction and scoring for protected phrases.""" from __future__ import annotations import logging import re from collections import Counter, defaultdict from functools import lru_cache from time import perf_counter from typing import TYPE_CHECKING, Protocol from yake import KeywordExtractor from python.ebook_search.protected_phrases.config import ( get_bad_ends, get_bad_starts, get_ignored_phrases, get_junk_tokens, get_most_common_words, ) from python.ebook_search.protected_phrases.models import PhraseCandidate from python.ebook_search.protected_phrases.text_normalization import tokenize, tokenize_with_offsets if TYPE_CHECKING: from collections.abc import Iterable, Mapping, Sequence from python.ebook_search.config import EbookSearchConfig logger = logging.getLogger(__name__) BAD_START_SCORE_PENALTY = 10.0 BAD_END_SCORE_PENALTY = 10.0 MULTI_SOURCE_SCORE_BONUS = 2.0 MULTI_SOURCE_MIN_SOURCES = 2 CAPITALIZED_PHRASE_RE = re.compile(r"\b(?:[A-Z][a-zA-Z']+)(?:\s+(?:of|the|and|in|on|for|[A-Z][a-zA-Z']+)){0,6}") class SpacySpan(Protocol): """Small protocol for the spaCy span attributes used by this module.""" text: str class SpacyEntity(SpacySpan, Protocol): """Small protocol for the spaCy entity attributes used by this module.""" label_: str class SpacyDoc(Protocol): """Small protocol for the spaCy doc attributes used by this module.""" ents: Iterable[SpacyEntity] noun_chunks: Iterable[SpacySpan] class SpacyLanguage(Protocol): """Small protocol for a callable spaCy language pipeline.""" def __call__(self, text: str) -> SpacyDoc: """Parse text into a spaCy-like doc.""" class YakeExtractor(Protocol): """Small protocol for the YAKE extractor used by this module.""" def extract_keywords(self, text: str) -> Iterable[tuple[str, float]]: """Return YAKE keyword tuples.""" class YakeExtractorFactory(Protocol): """Callable constructor protocol for YAKE keyword extractors.""" def __call__(self, *, lan: str, n: int, dedupLim: float, top: int) -> YakeExtractor: # noqa: N803 """Create a YAKE keyword extractor. Args: lan (str): Language code passed to YAKE. n (int): Maximum n-gram size to extract. dedupLim (float): Deduplication similarity threshold. top (int): Maximum number of keyphrases to return. Returns: YakeExtractor: The constructed keyword extractor. """ def normalize_candidate_phrase( phrase_text: str, config: EbookSearchConfig, *, min_tokens: int | None = None, max_tokens: int | None = None, strip_leading_article: bool = False, ) -> tuple[str, str, int] | None: """Normalize a candidate phrase and validate token bounds. Args: phrase_text (str): Raw phrase text to normalize. config (EbookSearchConfig): Runtime phrase-tuning settings. min_tokens (int | None): Minimum token count override; defaults to ``config.phrase_min_tokens``. max_tokens (int | None): Maximum token count override; defaults to ``config.phrase_max_tokens``. strip_leading_article (bool): Whether to drop a single leading English article. Returns: tuple[str, str, int] | None: Display text, normalized phrase, and token count, or ``None`` when the phrase falls outside the token bounds or is ignored. """ normalized_tokens = tokenize_with_offsets(phrase_text) start = 0 if strip_leading_article and normalized_tokens and normalized_tokens[0].text in {"the", "a", "an"}: start = 1 selected_tokens = normalized_tokens[start:] min_count = config.phrase_min_tokens if min_tokens is None else min_tokens max_count = config.phrase_max_tokens if max_tokens is None else max_tokens if len(selected_tokens) < min_count or len(selected_tokens) > max_count: return None phrase_norm = " ".join(token.text for token in selected_tokens) if phrase_norm in get_ignored_phrases(): return None display_text = phrase_text[selected_tokens[0].start_char : selected_tokens[-1].end_char].strip() return display_text or phrase_norm, phrase_norm, len(selected_tokens) def count_raw_ngrams(tokens: Sequence[str], config: EbookSearchConfig) -> Counter[str]: """Count every n-gram window in one normalized token block. ``tokens`` are already normalized (see :func:`tokenize`), so each window's normalized form is the joined tokens directly. Counting into a plain :class:`Counter` rather than :class:`PhraseCandidate` objects keeps this hot loop cheap; callers filter ignored phrases and materialize candidates per unique phrase afterwards, which is far fewer operations than doing either per window. Args: tokens (Sequence[str]): Normalized tokens for one text block. config (EbookSearchConfig): Runtime phrase-tuning settings. Returns: Counter[str]: Raw occurrence counts keyed by normalized phrase. """ return Counter( " ".join(tokens[start : start + ngram_size]) for ngram_size in range(config.phrase_min_tokens, config.phrase_max_tokens + 1) for start in range(len(tokens) - ngram_size + 1) ) def extract_raw_ngrams_by_chapter( chapters: Sequence[str], config: EbookSearchConfig, ) -> dict[str, PhraseCandidate]: """Extract raw n-grams across chapters, tracking both raw counts and chapter spread. Counting each chapter separately makes chapter spread fall out of dict membership: a phrase's ``chapter_count`` is simply how many per-chapter count maps contain it, so no per-window seen tracking is needed. This also lets the enrichment step skip re-sliding the same n-gram sizes. Phrases below the minimum raw count are dropped here rather than materialized: most unique n-grams occur once, and :func:`filter_storable_candidates` would discard them as too rare anyway, so building ``PhraseCandidate`` objects for them is wasted work. Args: chapters (Sequence[str]): Chapter-like text blocks to slide n-gram windows over. config (EbookSearchConfig): Runtime phrase-tuning settings. Returns: dict[str, PhraseCandidate]: Candidates meeting the minimum raw count, keyed by normalized phrase, with raw and chapter counts. """ chapter_count_maps = [count_raw_ngrams(tokenize(chapter), config) for chapter in chapters] total_counts: Counter[str] = Counter() chapter_spread: Counter[str] = Counter() for chapter_counts in chapter_count_maps: total_counts.update(chapter_counts) chapter_spread.update(chapter_counts.keys()) min_raw_count = minimum_candidate_raw_count(config) ignored = get_ignored_phrases() return { phrase_norm: PhraseCandidate( phrase_text=phrase_norm, phrase_norm=phrase_norm, token_count=phrase_norm.count(" ") + 1, source_raw_ngram=True, raw_count=raw_count, chapter_count=chapter_spread[phrase_norm], ) for phrase_norm, raw_count in total_counts.items() if raw_count >= min_raw_count and phrase_norm not in ignored } @lru_cache(maxsize=2) def get_yake_extractor(max_ngram: int, top_k: int) -> KeywordExtractor: """Return a cached YAKE extractor for the given settings. Constructing a ``KeywordExtractor`` loads the language's stopword list from disk, so it is cached and reused across books rather than rebuilt on every call. Args: max_ngram (int): Maximum n-gram size to extract. top_k (int): Maximum number of keyphrases to request. Returns: KeywordExtractor: A shared extractor instance for the given settings. """ return KeywordExtractor(lan="en", n=max_ngram, dedupLim=0.85, top=top_k) def extract_yake_candidates( book_text: str, config: EbookSearchConfig, top_k: int = 1000, ) -> dict[str, PhraseCandidate]: """Extract YAKE keyphrases when the optional YAKE package is installed. Args: book_text (str): Full book text to extract keyphrases from. config (EbookSearchConfig): Runtime phrase-tuning settings. top_k (int): Maximum number of YAKE keyphrases to request. Returns: dict[str, PhraseCandidate]: Candidates keyed by normalized phrase, with YAKE scores. """ extractor = get_yake_extractor(config.phrase_max_tokens, top_k) out: dict[str, PhraseCandidate] = {} for phrase_text, yake_score in extractor.extract_keywords(book_text): normalized = normalize_candidate_phrase(phrase_text, config) if normalized is None: continue display_text, phrase_norm, token_count = normalized out[phrase_norm] = PhraseCandidate( phrase_text=display_text, phrase_norm=phrase_norm, token_count=token_count, source_yake=True, yake_score=float(yake_score), ) return out def extract_spacy_candidates( book_text: str, nlp: SpacyLanguage, config: EbookSearchConfig, ) -> dict[str, PhraseCandidate]: """Extract spaCy named entities and noun chunks from one text block. Args: book_text (str): Text block to parse with spaCy. nlp (SpacyLanguage): Callable spaCy language pipeline. config (EbookSearchConfig): Runtime phrase-tuning settings. Returns: dict[str, PhraseCandidate]: Candidates keyed by normalized phrase from entities and noun chunks. """ out: dict[str, PhraseCandidate] = {} doc = nlp(book_text) for ent in doc.ents: normalized = normalize_candidate_phrase( ent.text, config, max_tokens=config.phrase_max_entity_tokens, ) if normalized is None: continue phrase_text, phrase_norm, token_count = normalized out[phrase_norm] = PhraseCandidate( phrase_text=phrase_text, phrase_norm=phrase_norm, token_count=token_count, source_spacy_ner=True, spacy_label=ent.label_, ) for chunk in doc.noun_chunks: normalized = normalize_candidate_phrase(chunk.text, config, strip_leading_article=True) if normalized is None: continue phrase_text, phrase_norm, token_count = normalized out[phrase_norm] = PhraseCandidate( phrase_text=phrase_text, phrase_norm=phrase_norm, token_count=token_count, source_spacy_noun_chunk=True, ) return out def extract_capitalized_phrases(original_text: str, config: EbookSearchConfig) -> dict[str, PhraseCandidate]: """Extract capitalized phrase runs that often carry fictional terms. Args: original_text (str): Original-case book text to scan for capitalized runs. config (EbookSearchConfig): Runtime phrase-tuning settings. Returns: dict[str, PhraseCandidate]: Candidates keyed by normalized phrase from capitalized runs. """ out: dict[str, PhraseCandidate] = {} for match in CAPITALIZED_PHRASE_RE.finditer(original_text): phrase_text = match.group(0).strip() normalized = normalize_candidate_phrase( phrase_text, config, max_tokens=config.phrase_max_entity_tokens, ) if normalized is None: continue display_text, phrase_norm, token_count = normalized out[phrase_norm] = PhraseCandidate( phrase_text=display_text, phrase_norm=phrase_norm, token_count=token_count, source_capitalized=True, ) return out def extract_metadata_candidates( metadata: Mapping[str, object] | None, config: EbookSearchConfig, ) -> dict[str, PhraseCandidate]: """Extract phrases from book metadata values such as title, author, and series. Args: metadata (Mapping[str, object] | None): Book metadata values, or ``None`` when unavailable. config (EbookSearchConfig): Runtime phrase-tuning settings. Returns: dict[str, PhraseCandidate]: Candidates keyed by normalized phrase from metadata values. """ if metadata is None: return {} out: dict[str, PhraseCandidate] = {} for value in metadata.values(): if value is None: continue phrase_text = str(value).strip() normalized = normalize_candidate_phrase( phrase_text, config, max_tokens=config.phrase_max_entity_tokens, ) if normalized is None: continue display_text, phrase_norm, token_count = normalized out[phrase_norm] = PhraseCandidate( phrase_text=display_text, phrase_norm=phrase_norm, token_count=token_count, source_metadata=True, ) return out def merge_candidate_sources(*sources: Mapping[str, PhraseCandidate]) -> dict[str, PhraseCandidate]: """Merge candidate dictionaries by normalized phrase. Args: *sources (Mapping[str, PhraseCandidate]): Candidate maps to combine, keyed by normalized phrase. Returns: dict[str, PhraseCandidate]: One merged candidate per normalized phrase. """ merged: dict[str, PhraseCandidate] = {} for source in sources: for phrase_norm, item in source.items(): existing = merged.setdefault( phrase_norm, PhraseCandidate( phrase_text=item.phrase_text, phrase_norm=phrase_norm, token_count=item.token_count, ), ) merge_candidate(existing, item) return merged def merge_candidate(existing: PhraseCandidate, item: PhraseCandidate) -> None: """Merge one candidate into an existing candidate object. Args: existing (PhraseCandidate): Candidate mutated in place to absorb ``item``. item (PhraseCandidate): Candidate whose sources, counts, and scores are merged in. """ existing.source_raw_ngram = existing.source_raw_ngram or item.source_raw_ngram existing.source_yake = existing.source_yake or item.source_yake existing.source_spacy_ner = existing.source_spacy_ner or item.source_spacy_ner existing.source_spacy_noun_chunk = existing.source_spacy_noun_chunk or item.source_spacy_noun_chunk existing.source_capitalized = existing.source_capitalized or item.source_capitalized existing.source_metadata = existing.source_metadata or item.source_metadata existing.raw_count += item.raw_count existing.chapter_count = max(existing.chapter_count, item.chapter_count) if item.yake_score is not None: existing.yake_score = item.yake_score if item.spacy_label: existing.spacy_label = item.spacy_label def enrich_with_frequency_and_chapter_counts( candidates: Mapping[str, PhraseCandidate], chapters: Sequence[str], *, counted_sizes: Iterable[int] = (), ) -> dict[str, PhraseCandidate]: """Add raw occurrence and chapter-spread counts to candidates. Candidates whose ``token_count`` is in ``counted_sizes`` are left untouched: those counts were already computed while sliding the chapters in :func:`extract_raw_ngrams_by_chapter`, so re-sliding those n-gram sizes here would just duplicate that work. Args: candidates (Mapping[str, PhraseCandidate]): Candidates to enrich, keyed by normalized phrase. chapters (Sequence[str]): Chapter-like text blocks used to count occurrences and spread. counted_sizes (Iterable[int]): Token counts whose counts are already populated and should be skipped. Returns: dict[str, PhraseCandidate]: Candidates with updated ``raw_count`` and ``chapter_count`` values. """ if not candidates: return {} already_counted = set(counted_sizes) candidate_sets_by_size: dict[int, set[str]] = defaultdict(set) for phrase_norm, candidate in candidates.items(): if candidate.token_count in already_counted: continue candidate_sets_by_size[candidate.token_count].add(phrase_norm) enriched = dict(candidates) if not candidate_sets_by_size: return enriched total_counts, chapter_counts = count_candidate_occurrences(candidate_sets_by_size, chapters) for phrase_norm, candidate in enriched.items(): if candidate.token_count in already_counted: continue candidate.raw_count = max(candidate.raw_count, total_counts[phrase_norm]) candidate.chapter_count = chapter_counts[phrase_norm] return enriched def count_candidate_occurrences( candidate_sets_by_size: Mapping[int, set[str]], chapters: Sequence[str], ) -> tuple[dict[str, int], dict[str, int]]: """Count total occurrences and chapter spread for candidate phrases across chapters. Args: candidate_sets_by_size (Mapping[int, set[str]]): Candidate normalized phrases grouped by token count. chapters (Sequence[str]): Chapter-like text blocks to slide n-gram windows over. Returns: tuple[dict[str, int], dict[str, int]]: Total occurrence counts and chapter-spread counts, each keyed by normalized phrase. """ total_counts: defaultdict[str, int] = defaultdict(int) chapter_counts: defaultdict[str, int] = defaultdict(int) for chapter in chapters: seen_in_chapter: set[str] = set() chapter_tokens = tokenize(chapter) for ngram_size, candidate_norms in candidate_sets_by_size.items(): for start in range(len(chapter_tokens) - ngram_size + 1): phrase_norm = " ".join(chapter_tokens[start : start + ngram_size]) if phrase_norm not in candidate_norms: continue total_counts[phrase_norm] += 1 seen_in_chapter.add(phrase_norm) for phrase_norm in seen_in_chapter: chapter_counts[phrase_norm] += 1 return total_counts, chapter_counts def filter_storable_candidates( candidates: Mapping[str, PhraseCandidate], config: EbookSearchConfig, ) -> tuple[dict[str, PhraseCandidate], int, int, int, int]: """Remove candidates that should not be persisted. Args: candidates (Mapping[str, PhraseCandidate]): Candidates to filter, keyed by normalized phrase. config (EbookSearchConfig): Runtime phrase-tuning settings. Returns: tuple[dict[str, PhraseCandidate], int, int, int, int]: The storable candidates followed by the counts dropped for being too short, too rare, too common, and junk. """ min_raw_count = minimum_candidate_raw_count(config) filtered: dict[str, PhraseCandidate] = {} too_short = 0 too_rare = 0 too_common = 0 junk = 0 for phrase_norm, candidate in candidates.items(): if candidate.token_count < config.phrase_min_tokens: too_short += 1 continue if candidate.raw_count < min_raw_count: too_rare += 1 continue phrase_tokens = phrase_norm.split() if is_most_common_word_phrase(phrase_tokens): too_common += 1 continue if is_junk_phrase(phrase_tokens): junk += 1 continue filtered[phrase_norm] = candidate return filtered, too_short, too_rare, too_common, junk def minimum_candidate_raw_count(config: EbookSearchConfig) -> int: """Return the minimum occurrence count required before storing a candidate. Args: config (EbookSearchConfig): Runtime phrase-tuning settings. Returns: int: The minimum raw occurrence count, never less than 1. """ return max(config.phrase_raw_ngram_min_count, 1) def is_most_common_word_phrase(phrase_tokens: list[str]) -> bool: """Return whether every token in a normalized phrase is a common word. Args: phrase_tokens (list[str]): Normalized phrase tokens to inspect. Returns: bool: True when the phrase is non-empty and every token is a common word. """ common_words = get_most_common_words() return bool(phrase_tokens) and all(token in common_words for token in phrase_tokens) def is_junk_phrase(phrase_tokens: list[str]) -> bool: """Return whether a normalized phrase is lexical junk not worth LLM judging. Judged data shows phrases containing a dialogue/action verb or a pronoun contraction are never kept, and phrases whose tokens are mostly common words almost never are. Possessives of proper nouns (``chapman's death``) pass because matching is by exact token, and exactly-half-common bigrams (``data feed``) pass because the common-word rule is strict. Args: phrase_tokens (list[str]): Normalized phrase tokens to inspect. Returns: bool: True when the phrase contains a junk token or is majority common words. """ if not phrase_tokens: return False junk_tokens = get_junk_tokens() if any(token in junk_tokens for token in phrase_tokens): return True common_words = get_most_common_words() half_phrase_len = len(phrase_tokens) // 2 return sum(token in common_words for token in phrase_tokens) > half_phrase_len def score_candidate(candidate: PhraseCandidate, config: EbookSearchConfig) -> float: """Score a phrase candidate before LLM judging. Args: candidate (PhraseCandidate): Candidate to score. config (EbookSearchConfig): Runtime phrase-tuning settings. Returns: float: Combined score from sources, frequency, and length, less any penalties. """ score = source_score(candidate) + frequency_score(candidate, config) + token_count_score(candidate, config) if non_raw_source_count(candidate) >= MULTI_SOURCE_MIN_SOURCES: score += MULTI_SOURCE_SCORE_BONUS if candidate.phrase_norm in get_ignored_phrases(): score -= 100.0 if has_bad_start(candidate.phrase_norm): score -= BAD_START_SCORE_PENALTY if has_bad_end(candidate.phrase_norm): score -= BAD_END_SCORE_PENALTY return score def non_raw_source_count(candidate: PhraseCandidate) -> int: """Count the non-raw-ngram extraction sources that produced a candidate. Args: candidate (PhraseCandidate): Candidate whose enabled sources are counted. Returns: int: Number of enabled sources other than the raw n-gram slide. """ return sum( ( candidate.source_yake, candidate.source_spacy_ner, candidate.source_spacy_noun_chunk, candidate.source_capitalized, candidate.source_metadata, ) ) def has_bad_start(phrase_norm: str) -> bool: """Return whether a normalized phrase starts with a bad starting token. Args: phrase_norm (str): Normalized phrase text to inspect. Returns: bool: True when the first token is a known bad starting token. """ phrase_tokens = phrase_norm.split() return bool(phrase_tokens and phrase_tokens[0] in get_bad_starts()) def has_bad_end(phrase_norm: str) -> bool: """Return whether a normalized phrase ends with a bad ending token. Args: phrase_norm (str): Normalized phrase text to inspect. Returns: bool: True when the last token is a known bad ending token. """ phrase_tokens = phrase_norm.split() return bool(phrase_tokens and phrase_tokens[-1] in get_bad_ends()) def source_score(candidate: PhraseCandidate) -> float: """Return the score contribution from extraction sources. Args: candidate (PhraseCandidate): Candidate whose enabled sources are weighted. Returns: float: Summed weight of the candidate's enabled extraction sources. """ return sum( weight for enabled, weight in ( (candidate.source_yake, 2.0), (candidate.source_spacy_ner, 2.5), (candidate.source_spacy_noun_chunk, 1.5), (candidate.source_capitalized, 2.0), (candidate.source_metadata, 2.0), (candidate.source_raw_ngram, 0.5), ) if enabled ) def frequency_score(candidate: PhraseCandidate, config: EbookSearchConfig) -> float: """Return the score contribution from frequency and chapter spread. Args: candidate (PhraseCandidate): Candidate whose counts are scored. config (EbookSearchConfig): Runtime phrase-tuning settings holding score thresholds. Returns: float: Summed weight for each frequency and chapter-spread threshold the candidate meets. """ return sum( weight for count, threshold, weight in ( (candidate.raw_count, config.phrase_raw_count_score_threshold, 0.5), (candidate.raw_count, config.phrase_raw_count_high_score_threshold, 0.5), (candidate.chapter_count, config.phrase_chapter_count_score_threshold, 0.5), (candidate.chapter_count, config.phrase_chapter_count_high_score_threshold, 0.5), ) if count >= threshold ) def token_count_score(candidate: PhraseCandidate, config: EbookSearchConfig) -> float: """Return the score contribution from phrase length. Args: candidate (PhraseCandidate): Candidate whose token count is scored. config (EbookSearchConfig): Runtime phrase-tuning settings holding the max token bound. Returns: float: Length-based score contribution, which may be negative for over- or under-length phrases. """ if candidate.token_count == 1: return -0.5 if candidate.token_count in {2, 3, 4}: return 0.5 if candidate.token_count > config.phrase_max_tokens: return -1.0 return 0.0 def get_sample_contexts(normalized_book_text: str, phrase_norm: str, max_contexts: int = 5) -> list[str]: """Return normalized context snippets containing a candidate phrase. ``normalized_book_text`` is expected to already be ``normalize_text``-ed by the caller so the whole book is not re-normalized for every phrase. Args: normalized_book_text (str): Whole book text, already normalized, to search. phrase_norm (str): Normalized phrase to find contexts around. max_contexts (int): Maximum number of context snippets to return. Returns: list[str]: Up to ``max_contexts`` normalized snippets surrounding the phrase. """ contexts: list[str] = [] start = 0 while len(contexts) < max_contexts: index = normalized_book_text.find(phrase_norm, start) if index == -1: break left = max(0, index - 300) right = min(len(normalized_book_text), index + len(phrase_norm) + 300) contexts.append(normalized_book_text[left:right]) start = index + len(phrase_norm) return contexts def candidate_source_names(candidate: PhraseCandidate) -> list[str]: """Return enabled source names for an extracted candidate. Args: candidate (PhraseCandidate): Candidate whose enabled sources are listed. Returns: list[str]: Names of the extraction sources that produced the candidate. """ names: list[str] = [] if candidate.source_raw_ngram: names.append("raw_ngram") if candidate.source_yake: names.append("yake") if candidate.source_spacy_ner: names.append("spacy_ner") if candidate.source_spacy_noun_chunk: names.append("spacy_noun_chunk") if candidate.source_capitalized: names.append("capitalized") if candidate.source_metadata: names.append("metadata") return names def extract_phrase_candidates_for_book( book_text: str, chapters: Sequence[str], config: EbookSearchConfig, *, nlp: SpacyLanguage | None = None, metadata: Mapping[str, object] | None = None, ) -> list[PhraseCandidate]: """Extract, score, and limit phrase candidates for one book. Args: book_text (str): Full book text used for most extraction sources. chapters (Sequence[str]): Chapter-like text blocks used for spaCy and frequency counts. config (EbookSearchConfig): Runtime phrase-tuning settings. nlp (SpacyLanguage | None): Optional spaCy pipeline for entity and noun-chunk sources. metadata (Mapping[str, object] | None): Optional book metadata used as a candidate source. Returns: list[PhraseCandidate]: Scored candidates sorted best-first and capped per book. """ started_at = perf_counter() logger.info( "ebook_phrase_candidate_extract_start chapters=%s chars=%s min_tokens=%s max_tokens=%s max_candidates=%s", len(chapters), len(book_text), config.phrase_min_tokens, config.phrase_max_tokens, config.protected_phrase_max_candidates_per_book, ) raw_started_at = perf_counter() raw = extract_raw_ngrams_by_chapter(chapters, config) logger.info( "ebook_phrase_candidate_extract_raw_complete candidates=%s duration_ms=%.1f", len(raw), (perf_counter() - raw_started_at) * 1000, ) yake_started_at = perf_counter() yake_candidates = extract_yake_candidates(book_text, config) logger.info( "ebook_phrase_candidate_extract_yake_complete candidates=%s duration_ms=%.1f", len(yake_candidates), (perf_counter() - yake_started_at) * 1000, ) spacy_candidates: dict[str, PhraseCandidate] = {} if nlp is not None: spacy_started_at = perf_counter() for chapter in chapters: spacy_candidates = merge_candidate_sources(spacy_candidates, extract_spacy_candidates(chapter, nlp, config)) logger.info( "ebook_phrase_candidate_extract_spacy_complete candidates=%s duration_ms=%.1f", len(spacy_candidates), (perf_counter() - spacy_started_at) * 1000, ) capitalized_started_at = perf_counter() capitalized = extract_capitalized_phrases(book_text, config) logger.info( "ebook_phrase_candidate_extract_capitalized_complete candidates=%s duration_ms=%.1f", len(capitalized), (perf_counter() - capitalized_started_at) * 1000, ) metadata_candidates = extract_metadata_candidates(metadata, config) candidates = merge_candidate_sources(raw, yake_candidates, spacy_candidates, capitalized, metadata_candidates) enriched_started_at = perf_counter() # Raw n-gram sizes were already counted per chapter above, so only enrich the remaining # (entity-length) sizes here instead of re-sliding every size over the whole book. candidates = enrich_with_frequency_and_chapter_counts( candidates, chapters, counted_sizes=range(config.phrase_min_tokens, config.phrase_max_tokens + 1), ) pre_filter_count = len(candidates) candidates, filtered_too_short, filtered_too_rare, filtered_too_common, filtered_junk = filter_storable_candidates( candidates, config ) for candidate in candidates.values(): candidate.candidate_score = score_candidate(candidate, config) limited = sorted(candidates.values(), key=lambda item: item.candidate_score, reverse=True)[ : config.protected_phrase_max_candidates_per_book ] logger.info( "ebook_phrase_candidate_extract_complete raw=%s yake=%s spacy=%s capitalized=%s metadata=%s " "merged=%s filtered_too_short=%s filtered_too_rare=%s filtered_too_common=%s filtered_junk=%s " "min_uses=%s storable=%s limited=%s enrich_score_ms=%.1f duration_ms=%.1f", len(raw), len(yake_candidates), len(spacy_candidates), len(capitalized), len(metadata_candidates), pre_filter_count, filtered_too_short, filtered_too_rare, filtered_too_common, filtered_junk, minimum_candidate_raw_count(config), len(candidates), len(limited), (perf_counter() - enriched_started_at) * 1000, (perf_counter() - started_at) * 1000, ) return limited