feat(ebook): add junk tokens for improved phrase matching

This commit is contained in:
Richie committed 2026-07-09 11:04:59 -04:00
1 parent c11895f144
commit 086224a4c9
3 files changed
+74

No files matched your search

@@ -52,3 +52,9 @@ def get_bad_starts() -> frozenset[str]:
def get_most_common_words() -> frozenset[str]:
"""Return the most common English words loaded from TOML."""
return _load_toml_string_set(_get_phrase_config_dir() / "most_common_words.toml", "words")
@cache
def get_junk_tokens() -> frozenset[str]:
"""Return junk tokens (dialogue verbs and pronoun contractions) loaded from TOML."""
return _load_toml_string_set(_get_phrase_config_dir() / "junk_tokens.toml", "tokens")