feat(extraction): add deduplication limit parameter to YAKE extractor

python/ebook_search/config.py

@@ -92,6 +92,7 @@ class EbookSearchConfig(BaseSettings):
     phrase_max_tokens: int = 5
     phrase_max_entity_tokens: int = 8
     phrase_yake_top_k: int = 1000
+    phrase_yake_dedup_limit: float = 0.85
     phrase_raw_ngram_min_count: int = 2
     phrase_raw_count_score_threshold: int = 3
     phrase_raw_count_high_score_threshold: int = 10

python/ebook_search/protected_phrases/extraction.py

@@ -154,7 +154,7 @@ def extract_raw_ngrams_by_chapter(
 
 
 @lru_cache(maxsize=2)
-def get_yake_extractor(max_ngram: int, top_k: int) -> KeywordExtractor:
+def get_yake_extractor(max_ngram: int, top_k: int, dedup_limit: float) -> KeywordExtractor:
     """Return a cached YAKE extractor for the given settings.
 
     Constructing a ``KeywordExtractor`` loads the language's stopword list from disk, so it is
@@ -163,11 +163,12 @@ def get_yake_extractor(max_ngram: int, top_k: int) -> KeywordExtractor:
     Args:
         max_ngram (int): Maximum n-gram size to extract.
         top_k (int): Maximum number of keyphrases to request.
+        dedup_limit (float): Deduplication similarity threshold.
 
     Returns:
         KeywordExtractor: A shared extractor instance for the given settings.
     """
-    return KeywordExtractor(lan="en", n=max_ngram, dedupLim=0.85, top=top_k)
+    return KeywordExtractor(lan="en", n=max_ngram, dedupLim=dedup_limit, top=top_k)
 
 
 def extract_yake_candidates(
@@ -183,7 +184,11 @@ def extract_yake_candidates(
     Returns:
         dict[str, PhraseCandidate]: Candidates keyed by normalized phrase, with YAKE scores.
     """
-    extractor = get_yake_extractor(config.phrase_max_tokens, config.phrase_yake_top_k)
+    extractor = get_yake_extractor(
+        config.phrase_max_tokens,
+        config.phrase_yake_top_k,
+        config.phrase_yake_dedup_limit,
+    )
     out: dict[str, PhraseCandidate] = {}
     for phrase_text, yake_score in extractor.extract_keywords(book_text):
         normalized = normalize_candidate_phrase(phrase_text, config)