From 31ecad881fbf7a4720ecc4079087152390738e5f Mon Sep 17 00:00:00 2001 From: Richie Cahill Date: Thu, 16 Jul 2026 13:35:15 -0400 Subject: [PATCH] feat(extraction): add deduplication limit parameter to YAKE extractor --- python/ebook_search/config.py | 1 + python/ebook_search/protected_phrases/extraction.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/python/ebook_search/config.py b/python/ebook_search/config.py index b7003f2..8b7d4dd 100644 --- a/python/ebook_search/config.py +++ b/python/ebook_search/config.py @@ -92,6 +92,7 @@ class EbookSearchConfig(BaseSettings): phrase_max_tokens: int = 5 phrase_max_entity_tokens: int = 8 phrase_yake_top_k: int = 1000 + phrase_yake_dedup_limit: float = 0.85 phrase_raw_ngram_min_count: int = 2 phrase_raw_count_score_threshold: int = 3 phrase_raw_count_high_score_threshold: int = 10 diff --git a/python/ebook_search/protected_phrases/extraction.py b/python/ebook_search/protected_phrases/extraction.py index f1d43d7..f7ce822 100644 --- a/python/ebook_search/protected_phrases/extraction.py +++ b/python/ebook_search/protected_phrases/extraction.py @@ -154,7 +154,7 @@ def extract_raw_ngrams_by_chapter( @lru_cache(maxsize=2) -def get_yake_extractor(max_ngram: int, top_k: int) -> KeywordExtractor: +def get_yake_extractor(max_ngram: int, top_k: int, dedup_limit: float) -> KeywordExtractor: """Return a cached YAKE extractor for the given settings. Constructing a ``KeywordExtractor`` loads the language's stopword list from disk, so it is @@ -163,11 +163,12 @@ def get_yake_extractor(max_ngram: int, top_k: int) -> KeywordExtractor: Args: max_ngram (int): Maximum n-gram size to extract. top_k (int): Maximum number of keyphrases to request. + dedup_limit (float): Deduplication similarity threshold. Returns: KeywordExtractor: A shared extractor instance for the given settings. """ - return KeywordExtractor(lan="en", n=max_ngram, dedupLim=0.85, top=top_k) + return KeywordExtractor(lan="en", n=max_ngram, dedupLim=dedup_limit, top=top_k) def extract_yake_candidates( @@ -183,7 +184,11 @@ def extract_yake_candidates( Returns: dict[str, PhraseCandidate]: Candidates keyed by normalized phrase, with YAKE scores. """ - extractor = get_yake_extractor(config.phrase_max_tokens, config.phrase_yake_top_k) + extractor = get_yake_extractor( + config.phrase_max_tokens, + config.phrase_yake_top_k, + config.phrase_yake_dedup_limit, + ) out: dict[str, PhraseCandidate] = {} for phrase_text, yake_score in extractor.extract_keywords(book_text): normalized = normalize_candidate_phrase(phrase_text, config)