feat(extraction): add cached YAKE extractor for improved performance
This commit is contained in:
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from functools import lru_cache
|
||||
from time import perf_counter
|
||||
from typing import TYPE_CHECKING, Protocol
|
||||
|
||||
@@ -176,6 +177,21 @@ def extract_raw_ngrams(text: str, config: EbookSearchConfig) -> dict[str, Phrase
|
||||
return out
|
||||
|
||||
|
||||
@lru_cache(maxsize=2)
|
||||
def get_yake_extractor(max_ngram: int, top_k: int) -> KeywordExtractor:
|
||||
"""Return a cached YAKE extractor for the given settings.
|
||||
|
||||
Constructing a ``KeywordExtractor`` loads the language's stopword list from disk, so it is
|
||||
cached and reused across books rather than rebuilt on every call.
|
||||
|
||||
Args:
|
||||
max_ngram (int): Maximum n-gram size to extract.
|
||||
top_k (int): Maximum number of keyphrases to request.
|
||||
|
||||
Returns:
|
||||
KeywordExtractor: A shared extractor instance for the given settings.
|
||||
"""
|
||||
return KeywordExtractor(lan="en", n=max_ngram, dedupLim=0.85, top=top_k)
|
||||
|
||||
|
||||
def extract_yake_candidates(
|
||||
@@ -193,7 +209,7 @@ def extract_yake_candidates(
|
||||
Returns:
|
||||
dict[str, PhraseCandidate]: Candidates keyed by normalized phrase, with YAKE scores.
|
||||
"""
|
||||
extractor = KeywordExtractor(lan="en", n=config.phrase_max_tokens, dedupLim=0.85, top=top_k)
|
||||
extractor = get_yake_extractor(config.phrase_max_tokens, top_k)
|
||||
out: dict[str, PhraseCandidate] = {}
|
||||
for phrase_text, yake_score in extractor.extract_keywords(book_text):
|
||||
normalized = normalize_candidate_phrase(phrase_text, config)
|
||||
|
||||
Reference in New Issue
Block a user