diff --git a/python/ebook_search/config.py b/python/ebook_search/config.py index 4e50265..b7003f2 100644 --- a/python/ebook_search/config.py +++ b/python/ebook_search/config.py @@ -91,6 +91,7 @@ class EbookSearchConfig(BaseSettings): phrase_min_tokens: int = 2 phrase_max_tokens: int = 5 phrase_max_entity_tokens: int = 8 + phrase_yake_top_k: int = 1000 phrase_raw_ngram_min_count: int = 2 phrase_raw_count_score_threshold: int = 3 phrase_raw_count_high_score_threshold: int = 10 @@ -99,6 +100,10 @@ class EbookSearchConfig(BaseSettings): phrase_target_protected_per_book: int = 100 phrase_default_allow_nested: bool = False phrase_default_suppress_children: bool = True + phrase_bad_start_score_penalty: float = 10.0 + phrase_bad_end_score_penalty: float = 10.0 + phrase_multi_source_score_bonus: float = 2.0 + phrase_multi_source_min_sources: int = 2 @field_validator("library_paths", mode="before") @classmethod diff --git a/python/ebook_search/protected_phrases/extraction.py b/python/ebook_search/protected_phrases/extraction.py index d929ee9..f1d43d7 100644 --- a/python/ebook_search/protected_phrases/extraction.py +++ b/python/ebook_search/protected_phrases/extraction.py @@ -29,10 +29,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -BAD_START_SCORE_PENALTY = 10.0 -BAD_END_SCORE_PENALTY = 10.0 -MULTI_SOURCE_SCORE_BONUS = 2.0 -MULTI_SOURCE_MIN_SOURCES = 2 CAPITALIZED_PHRASE_RE = re.compile(r"\b(?:[A-Z][a-zA-Z']+)(?:\s+(?:of|the|and|in|on|for|[A-Z][a-zA-Z']+)){0,6}") @@ -177,19 +173,17 @@ def get_yake_extractor(max_ngram: int, top_k: int) -> KeywordExtractor: def extract_yake_candidates( book_text: str, config: EbookSearchConfig, - top_k: int = 1000, ) -> dict[str, PhraseCandidate]: """Extract YAKE keyphrases when the optional YAKE package is installed. Args: book_text (str): Full book text to extract keyphrases from. config (EbookSearchConfig): Runtime phrase-tuning settings. - top_k (int): Maximum number of YAKE keyphrases to request. Returns: dict[str, PhraseCandidate]: Candidates keyed by normalized phrase, with YAKE scores. """ - extractor = get_yake_extractor(config.phrase_max_tokens, top_k) + extractor = get_yake_extractor(config.phrase_max_tokens, config.phrase_yake_top_k) out: dict[str, PhraseCandidate] = {} for phrase_text, yake_score in extractor.extract_keywords(book_text): normalized = normalize_candidate_phrase(phrase_text, config) @@ -487,14 +481,14 @@ def score_candidate(candidate: PhraseCandidate, config: EbookSearchConfig) -> fl float: Combined score from sources, frequency, and length, less any penalties. """ score = source_score(candidate) + frequency_score(candidate, config) + token_count_score(candidate, config) - if non_raw_source_count(candidate) >= MULTI_SOURCE_MIN_SOURCES: - score += MULTI_SOURCE_SCORE_BONUS + if non_raw_source_count(candidate) >= config.phrase_multi_source_min_sources: + score += config.phrase_multi_source_score_bonus if candidate.phrase_norm in get_ignored_phrases(): score -= 100.0 if has_bad_start(candidate.phrase_norm): - score -= BAD_START_SCORE_PENALTY + score -= config.phrase_bad_start_score_penalty if has_bad_end(candidate.phrase_norm): - score -= BAD_END_SCORE_PENALTY + score -= config.phrase_bad_end_score_penalty return score diff --git a/tests/ebook_search/test_protected_phrases.py b/tests/ebook_search/test_protected_phrases.py index 57e18bf..dabeb9e 100644 --- a/tests/ebook_search/test_protected_phrases.py +++ b/tests/ebook_search/test_protected_phrases.py @@ -159,6 +159,11 @@ def test_score_candidate_rewards_multiple_non_raw_sources(config: EbookSearchCon assert score_candidate(multi_source, config) == score_candidate(single_source, config) + 2.0 + 2.0 + tuned_config = config.model_copy( + update={"phrase_multi_source_min_sources": 2, "phrase_multi_source_score_bonus": 3.0} + ) + assert score_candidate(multi_source, tuned_config) == score_candidate(single_source, tuned_config) + 2.0 + 3.0 + def test_score_candidate_caps_frequency_contribution(config: EbookSearchConfig) -> None: """Very frequent raw-only phrases should no longer out-score sourced entities."""