feat(extraction): add cached YAKE extractor for improved performance
This commit is contained in:
@@ -5,6 +5,7 @@ from __future__ import annotations
|
|||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
|
from functools import lru_cache
|
||||||
from time import perf_counter
|
from time import perf_counter
|
||||||
from typing import TYPE_CHECKING, Protocol
|
from typing import TYPE_CHECKING, Protocol
|
||||||
|
|
||||||
@@ -176,6 +177,21 @@ def extract_raw_ngrams(text: str, config: EbookSearchConfig) -> dict[str, Phrase
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=2)
|
||||||
|
def get_yake_extractor(max_ngram: int, top_k: int) -> KeywordExtractor:
|
||||||
|
"""Return a cached YAKE extractor for the given settings.
|
||||||
|
|
||||||
|
Constructing a ``KeywordExtractor`` loads the language's stopword list from disk, so it is
|
||||||
|
cached and reused across books rather than rebuilt on every call.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
max_ngram (int): Maximum n-gram size to extract.
|
||||||
|
top_k (int): Maximum number of keyphrases to request.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
KeywordExtractor: A shared extractor instance for the given settings.
|
||||||
|
"""
|
||||||
|
return KeywordExtractor(lan="en", n=max_ngram, dedupLim=0.85, top=top_k)
|
||||||
|
|
||||||
|
|
||||||
def extract_yake_candidates(
|
def extract_yake_candidates(
|
||||||
@@ -193,7 +209,7 @@ def extract_yake_candidates(
|
|||||||
Returns:
|
Returns:
|
||||||
dict[str, PhraseCandidate]: Candidates keyed by normalized phrase, with YAKE scores.
|
dict[str, PhraseCandidate]: Candidates keyed by normalized phrase, with YAKE scores.
|
||||||
"""
|
"""
|
||||||
extractor = KeywordExtractor(lan="en", n=config.phrase_max_tokens, dedupLim=0.85, top=top_k)
|
extractor = get_yake_extractor(config.phrase_max_tokens, top_k)
|
||||||
out: dict[str, PhraseCandidate] = {}
|
out: dict[str, PhraseCandidate] = {}
|
||||||
for phrase_text, yake_score in extractor.extract_keywords(book_text):
|
for phrase_text, yake_score in extractor.extract_keywords(book_text):
|
||||||
normalized = normalize_candidate_phrase(phrase_text, config)
|
normalized = normalize_candidate_phrase(phrase_text, config)
|
||||||
|
|||||||
Reference in New Issue
Block a user