feat(ebook): add junk tokens for improved phrase matching

python/ebook_search/protected_phrases/config/__init__.py

@@ -4,6 +4,7 @@ from python.ebook_search.protected_phrases.config.lib import (
     get_bad_ends,
     get_bad_starts,
     get_ignored_phrases,
+    get_junk_tokens,
     get_most_common_words,
 )
 
@@ -11,5 +12,6 @@ __all__ = [
     "get_bad_ends",
     "get_bad_starts",
     "get_ignored_phrases",
+    "get_junk_tokens",
     "get_most_common_words",
 ]

python/ebook_search/protected_phrases/config/junk_tokens.toml

@@ -0,0 +1,66 @@
+tokens = [
+    "said",
+    "asked",
+    "replied",
+    "answered",
+    "looked",
+    "nodded",
+    "turned",
+    "shook",
+    "smiled",
+    "shrugged",
+    "pointed",
+    "continued",
+    "repeated",
+    "stared",
+    "agreed",
+    "glanced",
+    "walked",
+    "told",
+    "thought",
+    "knew",
+    "wanted",
+    "muttered",
+    "whispered",
+    "laughed",
+    "sighed",
+    "paused",
+    "gestured",
+    "waved",
+    "frowned",
+    "grinned",
+    "i'm",
+    "i've",
+    "i'd",
+    "i'll",
+    "it's",
+    "that's",
+    "don't",
+    "didn't",
+    "doesn't",
+    "can't",
+    "won't",
+    "wouldn't",
+    "couldn't",
+    "shouldn't",
+    "isn't",
+    "wasn't",
+    "aren't",
+    "weren't",
+    "you're",
+    "you've",
+    "you'll",
+    "we're",
+    "we've",
+    "we'll",
+    "they're",
+    "they've",
+    "he's",
+    "she's",
+    "there's",
+    "what's",
+    "let's",
+    "who's",
+    "he'd",
+    "she'd",
+]

python/ebook_search/protected_phrases/config/lib.py

@@ -52,3 +52,9 @@ def get_bad_starts() -> frozenset[str]:
 def get_most_common_words() -> frozenset[str]:
     """Return the most common English words loaded from TOML."""
     return _load_toml_string_set(_get_phrase_config_dir() / "most_common_words.toml", "words")
+
+
+@cache
+def get_junk_tokens() -> frozenset[str]:
+    """Return junk tokens (dialogue verbs and pronoun contractions) loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "junk_tokens.toml", "tokens")