refactor(protected-phrases): extract config and text normalization helpers

python/ebook_search/protected_phrases/config/__init__.py

@@ -0,0 +1,15 @@
+"""Protected phrase extraction, storage, and runtime matching."""
+
+from python.ebook_search.protected_phrases.config.lib import (
+    get_bad_ends,
+    get_bad_starts,
+    get_ignored_phrases,
+    get_most_common_words,
+)
+
+__all__ = [
+    "get_bad_ends",
+    "get_bad_starts",
+    "get_ignored_phrases",
+    "get_most_common_words",
+]

python/ebook_search/protected_phrases/config/bad_ends.toml

@@ -0,0 +1,31 @@
+tokens = [
+  "a",
+  "an",
+  "and",
+  "any",
+  "as",
+  "at",
+  "be",
+  "because",
+  "but",
+  "by",
+  "can",
+  "could",
+  "do",
+  "for",
+  "from",
+  "have",
+  "if",
+  "of",
+  "or",
+  "some",
+  "than",
+  "the",
+  "these",
+  "this",
+  "to",
+  "will",
+  "with",
+  "would",
+  "did",
+]

python/ebook_search/protected_phrases/config/bad_starts.toml

@@ -0,0 +1,27 @@
+tokens = [
+    "a",
+    "an",
+    "did",
+    "didn't",
+    "he",
+    "here",
+    "how",
+    "i",
+    "it",
+    "she",
+    "that",
+    "the",
+    "there",
+    "they",
+    "this",
+    "we",
+    "what",
+    "when",
+    "where",
+    "which",
+    "who",
+    "whom",
+    "whose",
+    "why",
+    "you",
+]

python/ebook_search/protected_phrases/config/ignored_phrases.toml

@@ -0,0 +1,212 @@
+phrases = [
+  "a little",
+  "across the",
+  "and she",
+  "anyone in",
+  "are you",
+  "around him",
+  "around the",
+  "as much",
+  "as soon",
+  "at all",
+  "at least",
+  "before the",
+  "behind him",
+  "between the",
+  "but she",
+  "could not",
+  "did he",
+  "did i",
+  "did it",
+  "did not believe",
+  "did not care",
+  "did not even",
+  "did not know what",
+  "did not know",
+  "did not like",
+  "did not look",
+  "did not mean",
+  "did not move",
+  "did not need",
+  "did not see",
+  "did not seem",
+  "did not think",
+  "did not understand",
+  "did not want",
+  "did not",
+  "did she",
+  "did so",
+  "did that",
+  "did the",
+  "did they",
+  "did what",
+  "did you",
+  "didn't answer",
+  "didn't care",
+  "didn't even",
+  "didn't expect",
+  "didn't feel",
+  "didn't get",
+  "didn't i",
+  "didn't know",
+  "didn't like",
+  "didn't look",
+  "didn't make",
+  "didn't mean",
+  "didn't need",
+  "didn't really",
+  "didn't say",
+  "didn't see",
+  "didn't seem",
+  "didn't think",
+  "didn't want",
+  "didn't you",
+  "end up",
+  "ended up",
+  "had a",
+  "had been",
+  "have been",
+  "he asked",
+  "he concluded",
+  "he continued",
+  "he couldn't",
+  "he did",
+  "he didn't",
+  "he felt",
+  "he had",
+  "he hadn't",
+  "he knew",
+  "he noted",
+  "he pointed",
+  "he realized",
+  "he replied",
+  "he said",
+  "he saw",
+  "he tapped",
+  "he told",
+  "he was",
+  "he wasn't",
+  "his body",
+  "his chair",
+  "his feet",
+  "his hands",
+  "his head",
+  "his office",
+  "his own",
+  "his pc",
+  "his power",
+  "his shield",
+  "his sight",
+  "his voice",
+  "his wrist",
+  "how many",
+  "i am",
+  "i don't",
+  "i said",
+  "i was",
+  "i wouldn't",
+  "i'm not",
+  "if he",
+  "if they",
+  "is in",
+  "is not",
+  "is that",
+  "is the",
+  "it had",
+  "it had",
+  "it is",
+  "it was",
+  "it wasn't",
+  "it wasn't",
+  "no one",
+  "of course",
+  "of force",
+  "of it",
+  "of magic",
+  "of marines",
+  "of power",
+  "of those",
+  "old man",
+  "older man",
+  "one of",
+  "out of",
+  "set up",
+  "she admitted",
+  "she asked",
+  "she had",
+  "she replied",
+  "she said",
+  "she snapped",
+  "she told",
+  "she was",
+  "she'd been",
+  "shook his",
+  "sure he",
+  "tell you",
+  "that had",
+  "that is",
+  "that she",
+  "that was",
+  "the dark",
+  "the door",
+  "the first",
+  "the last",
+  "the man",
+  "the one",
+  "the only",
+  "the other",
+  "the rest",
+  "the room",
+  "the same",
+  "the two",
+  "the way",
+  "the world",
+  "there are",
+  "there was",
+  "there were",
+  "they are",
+  "they had",
+  "they were",
+  "they weren't",
+  "this is",
+  "this place",
+  "though he",
+  "through his",
+  "through the",
+  "to find",
+  "to get",
+  "to keep",
+  "to stay",
+  "to stop",
+  "to tell",
+  "to try",
+  "told her",
+  "told him",
+  "under his",
+  "was a",
+  "was enough",
+  "was going",
+  "was in",
+  "was no",
+  "was not",
+  "was now",
+  "was on",
+  "was one",
+  "was only",
+  "was still",
+  "was that",
+  "was the",
+  "was there",
+  "were in",
+  "what had",
+  "what happened",
+  "what was",
+  "where the",
+  "while i",
+  "you are",
+  "you can't",
+  "you don't",
+  "you know",
+  "you need",
+  "you were",
+]

python/ebook_search/protected_phrases/config/lib.py

@@ -0,0 +1,54 @@
+"""Protected phrase extraction, storage, and runtime matching."""
+
+from __future__ import annotations
+
+import logging
+import tomllib
+from functools import cache
+from pathlib import Path
+
+from python.ebook_search.protected_phrases.text_normalization import normalize_text
+
+logger = logging.getLogger(__name__)
+
+
+def _load_toml_string_set(path: Path, key: str) -> frozenset[str]:
+    """Load and validate a TOML string list as a normalized immutable set."""
+    with path.open("rb") as file:
+        body = tomllib.load(file)
+
+    values = body.get(key)
+    if not isinstance(values, list) or not all(isinstance(item, str) for item in values):
+        msg = f"{path} must contain a {key!r} string list"
+        raise ValueError(msg)
+    return frozenset(normalize_text(value) for value in values if normalize_text(value))
+
+
+@cache
+def _get_phrase_config_dir() -> Path:
+    """Return the directory containing phrase configuration files."""
+    return Path(__file__).resolve().parent
+
+
+@cache
+def get_ignored_phrases() -> frozenset[str]:
+    """Return ignored phrase strings loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "ignored_phrases.toml", "phrases")
+
+
+@cache
+def get_bad_ends() -> frozenset[str]:
+    """Return bad phrase-ending tokens loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "bad_ends.toml", "tokens")
+
+
+@cache
+def get_bad_starts() -> frozenset[str]:
+    """Return bad phrase-starting tokens loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "bad_starts.toml", "tokens")
+
+
+@cache
+def get_most_common_words() -> frozenset[str]:
+    """Return the most common English words loaded from TOML."""
+    return _load_toml_string_set(_get_phrase_config_dir() / "most_common_words.toml", "words")

python/ebook_search/protected_phrases/config/most_common_words.toml

@@ -0,0 +1,102 @@
+words = [
+    "the",
+    "be",
+    "to",
+    "of",
+    "and",
+    "a",
+    "in",
+    "that",
+    "have",
+    "I",
+    "it",
+    "for",
+    "not",
+    "on",
+    "with",
+    "he",
+    "as",
+    "you",
+    "do",
+    "at",
+    "this",
+    "but",
+    "his",
+    "by",
+    "from",
+    "they",
+    "we",
+    "say",
+    "her",
+    "she",
+    "or",
+    "an",
+    "will",
+    "my",
+    "one",
+    "all",
+    "would",
+    "there",
+    "their",
+    "what",
+    "so",
+    "up",
+    "out",
+    "if",
+    "about",
+    "who",
+    "get",
+    "which",
+    "go",
+    "me",
+    "when",
+    "make",
+    "can",
+    "like",
+    "time",
+    "no",
+    "just",
+    "him",
+    "know",
+    "take",
+    "people",
+    "into",
+    "year",
+    "your",
+    "good",
+    "some",
+    "could",
+    "them",
+    "see",
+    "other",
+    "than",
+    "then",
+    "now",
+    "look",
+    "only",
+    "come",
+    "its",
+    "over",
+    "think",
+    "also",
+    "back",
+    "after",
+    "use",
+    "two",
+    "how",
+    "our",
+    "work",
+    "first",
+    "well",
+    "way",
+    "even",
+    "new",
+    "want",
+    "because",
+    "any",
+    "these",
+    "give",
+    "day",
+    "most",
+    "us",
+]