diff --git a/python/ebook_search/protected_phrases/config/__init__.py b/python/ebook_search/protected_phrases/config/__init__.py index 15bbd34..2127efe 100644 --- a/python/ebook_search/protected_phrases/config/__init__.py +++ b/python/ebook_search/protected_phrases/config/__init__.py @@ -4,6 +4,7 @@ from python.ebook_search.protected_phrases.config.lib import ( get_bad_ends, get_bad_starts, get_ignored_phrases, + get_junk_tokens, get_most_common_words, ) @@ -11,5 +12,6 @@ __all__ = [ "get_bad_ends", "get_bad_starts", "get_ignored_phrases", + "get_junk_tokens", "get_most_common_words", ] diff --git a/python/ebook_search/protected_phrases/config/junk_tokens.toml b/python/ebook_search/protected_phrases/config/junk_tokens.toml new file mode 100644 index 0000000..6dac2d5 --- /dev/null +++ b/python/ebook_search/protected_phrases/config/junk_tokens.toml @@ -0,0 +1,66 @@ +tokens = [ + "said", + "asked", + "replied", + "answered", + "looked", + "nodded", + "turned", + "shook", + "smiled", + "shrugged", + "pointed", + "continued", + "repeated", + "stared", + "agreed", + "glanced", + "walked", + "told", + "thought", + "knew", + "wanted", + "muttered", + "whispered", + "laughed", + "sighed", + "paused", + "gestured", + "waved", + "frowned", + "grinned", + "i'm", + "i've", + "i'd", + "i'll", + "it's", + "that's", + "don't", + "didn't", + "doesn't", + "can't", + "won't", + "wouldn't", + "couldn't", + "shouldn't", + "isn't", + "wasn't", + "aren't", + "weren't", + "you're", + "you've", + "you'll", + "we're", + "we've", + "we'll", + "they're", + "they've", + "he's", + "she's", + "there's", + "what's", + "let's", + "who's", + "he'd", + "she'd", +] diff --git a/python/ebook_search/protected_phrases/config/lib.py b/python/ebook_search/protected_phrases/config/lib.py index be8acd8..67f1332 100644 --- a/python/ebook_search/protected_phrases/config/lib.py +++ b/python/ebook_search/protected_phrases/config/lib.py @@ -52,3 +52,9 @@ def get_bad_starts() -> frozenset[str]: def get_most_common_words() -> frozenset[str]: """Return the most common English words loaded from TOML.""" return _load_toml_string_set(_get_phrase_config_dir() / "most_common_words.toml", "words") + + +@cache +def get_junk_tokens() -> frozenset[str]: + """Return junk tokens (dialogue verbs and pronoun contractions) loaded from TOML.""" + return _load_toml_string_set(_get_phrase_config_dir() / "junk_tokens.toml", "tokens")