refactor(ebook-search): simplify search and phrase matching
treefmt / nix fmt (pull_request) Successful in 5s
pytest / pytest (pull_request) Successful in 28s
test ebook search / test-ebook-search (pull_request) Failing after 35s
build_systems / build-bob (pull_request) Successful in 51s
build_systems / build-brain (pull_request) Successful in 50s
build_systems / build-rhapsody-in-green (pull_request) Successful in 1m3s
build_systems / build-jeeves (pull_request) Successful in 2m20s
treefmt / nix fmt (pull_request) Successful in 5s
pytest / pytest (pull_request) Successful in 28s
test ebook search / test-ebook-search (pull_request) Failing after 35s
build_systems / build-bob (pull_request) Successful in 51s
build_systems / build-brain (pull_request) Successful in 50s
build_systems / build-rhapsody-in-green (pull_request) Successful in 1m3s
build_systems / build-jeeves (pull_request) Successful in 2m20s
This commit is contained in:
@@ -64,40 +64,30 @@ def normalize_candidate_phrase(
|
||||
phrase_text: str,
|
||||
config: EbookSearchConfig,
|
||||
*,
|
||||
min_tokens: int | None = None,
|
||||
max_tokens: int | None = None,
|
||||
strip_leading_article: bool = False,
|
||||
) -> tuple[str, str, int] | None:
|
||||
"""Normalize a candidate phrase and validate token bounds.
|
||||
|
||||
Args:
|
||||
phrase_text (str): Raw phrase text to normalize.
|
||||
config (EbookSearchConfig): Runtime phrase-tuning settings.
|
||||
min_tokens (int | None): Minimum token count override; defaults to ``config.phrase_min_tokens``.
|
||||
max_tokens (int | None): Maximum token count override; defaults to ``config.phrase_max_tokens``.
|
||||
strip_leading_article (bool): Whether to drop a single leading English article.
|
||||
|
||||
Returns:
|
||||
tuple[str, str, int] | None: Display text, normalized phrase, and token count, or ``None``
|
||||
when the phrase falls outside the token bounds or is ignored.
|
||||
"""
|
||||
normalized_tokens = tokenize_with_offsets(phrase_text)
|
||||
start = 0
|
||||
if strip_leading_article and normalized_tokens and normalized_tokens[0].text in {"the", "a", "an"}:
|
||||
start = 1
|
||||
|
||||
selected_tokens = normalized_tokens[start:]
|
||||
min_count = config.phrase_min_tokens if min_tokens is None else min_tokens
|
||||
max_count = config.phrase_max_tokens if max_tokens is None else max_tokens
|
||||
if len(selected_tokens) < min_count or len(selected_tokens) > max_count:
|
||||
if len(normalized_tokens) < config.phrase_min_tokens or len(normalized_tokens) > max_count:
|
||||
return None
|
||||
|
||||
phrase_norm = " ".join(token.text for token in selected_tokens)
|
||||
phrase_norm = " ".join(token.text for token in normalized_tokens)
|
||||
if phrase_norm in get_ignored_phrases():
|
||||
return None
|
||||
|
||||
display_text = phrase_text[selected_tokens[0].start_char : selected_tokens[-1].end_char].strip()
|
||||
return display_text or phrase_norm, phrase_norm, len(selected_tokens)
|
||||
display_text = phrase_text[normalized_tokens[0].start_char : normalized_tokens[-1].end_char].strip()
|
||||
return display_text or phrase_norm, phrase_norm, len(normalized_tokens)
|
||||
|
||||
|
||||
def count_raw_ngrams(tokens: Sequence[str], config: EbookSearchConfig) -> Counter[str]:
|
||||
|
||||
Reference in New Issue
Block a user