feat(extraction): add cached YAKE extractor for improved performance

This commit is contained in:
2026-07-24 11:38:50 -04:00
parent a9311a2f9e
commit 38c01ec121
@@ -5,6 +5,7 @@ from __future__ import annotations
import logging
import re
from collections import defaultdict
from functools import lru_cache
from time import perf_counter
from typing import TYPE_CHECKING, Protocol
@@ -176,6 +177,21 @@ def extract_raw_ngrams(text: str, config: EbookSearchConfig) -> dict[str, Phrase
return out
@lru_cache(maxsize=2)
def get_yake_extractor(max_ngram: int, top_k: int) -> KeywordExtractor:
"""Return a cached YAKE extractor for the given settings.
Constructing a ``KeywordExtractor`` loads the language's stopword list from disk, so it is
cached and reused across books rather than rebuilt on every call.
Args:
max_ngram (int): Maximum n-gram size to extract.
top_k (int): Maximum number of keyphrases to request.
Returns:
KeywordExtractor: A shared extractor instance for the given settings.
"""
return KeywordExtractor(lan="en", n=max_ngram, dedupLim=0.85, top=top_k)
def extract_yake_candidates(
@@ -193,7 +209,7 @@ def extract_yake_candidates(
Returns:
dict[str, PhraseCandidate]: Candidates keyed by normalized phrase, with YAKE scores.
"""
extractor = KeywordExtractor(lan="en", n=config.phrase_max_tokens, dedupLim=0.85, top=top_k)
extractor = get_yake_extractor(config.phrase_max_tokens, top_k)
out: dict[str, PhraseCandidate] = {}
for phrase_text, yake_score in extractor.extract_keywords(book_text):
normalized = normalize_candidate_phrase(phrase_text, config)