feat(ebook): add junk tokens for improved phrase matching

2026-07-16 13:09:34 -04:00
parent 6c5fcbd0fc
commit d6d90489db
3 changed files with 74 additions and 0 deletions
@@ -4,6 +4,7 @@ from python.ebook_search.protected_phrases.config.lib import (
    get_bad_ends,
    get_bad_starts,
    get_ignored_phrases,
+    get_junk_tokens,
    get_most_common_words,
 )

@@ -11,5 +12,6 @@ __all__ = [
    "get_bad_ends",
    "get_bad_starts",
    "get_ignored_phrases",
+    "get_junk_tokens",
    "get_most_common_words",
 ]
@@ -0,0 +1,66 @@
+tokens = [
+    "said",
+    "asked",
+    "replied",
+    "answered",
+    "looked",
+    "nodded",
+    "turned",
+    "shook",
+    "smiled",
+    "shrugged",
+    "pointed",
+    "continued",
+    "repeated",
+    "stared",
+    "agreed",
+    "glanced",
+    "walked",
+    "told",
+    "thought",
+    "knew",
+    "wanted",
+    "muttered",
+    "whispered",
+    "laughed",
+    "sighed",
+    "paused",
+    "gestured",
+    "waved",
+    "frowned",
+    "grinned",
+    "i'm",
+    "i've",
+    "i'd",
+    "i'll",
+    "it's",
+    "that's",
+    "don't",
+    "didn't",
+    "doesn't",
+    "can't",
+    "won't",
+    "wouldn't",
+    "couldn't",
+    "shouldn't",
+    "isn't",
+    "wasn't",
+    "aren't",
+    "weren't",
+    "you're",
+    "you've",
+    "you'll",
+    "we're",
+    "we've",
+    "we'll",
+    "they're",
+    "they've",
+    "he's",
+    "she's",
+    "there's",
+    "what's",
+    "let's",
+    "who's",
+    "he'd",
+    "she'd",
+]
@@ -52,3 +52,9 @@ def get_bad_starts() -> frozenset[str]:
 def get_most_common_words() -> frozenset[str]:
    """Return the most common English words loaded from TOML."""
    return _load_toml_string_set(_get_phrase_config_dir() / "most_common_words.toml", "words")
+
+
+@cache
+def get_junk_tokens() -> frozenset[str]:
+    """Return junk tokens (dialogue verbs and pronoun contractions) loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "junk_tokens.toml", "tokens")