feat(ebook): add junk tokens for improved phrase matching

2026-07-24 11:38:50 -04:00
parent 584b209dfa
commit a9311a2f9e
3 changed files with 74 additions and 0 deletions
@@ -52,3 +52,9 @@ def get_bad_starts() -> frozenset[str]:
 def get_most_common_words() -> frozenset[str]:
    """Return the most common English words loaded from TOML."""
    return _load_toml_string_set(_get_phrase_config_dir() / "most_common_words.toml", "words")
+
+
+@cache
+def get_junk_tokens() -> frozenset[str]:
+    """Return junk tokens (dialogue verbs and pronoun contractions) loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "junk_tokens.toml", "tokens")