refactor(protected-phrases): extract config and text normalization helpers

python/ebook_search/protected_phrases/text_normalization.py

@@ -0,0 +1,63 @@
+"""Protected phrase extraction, storage, and runtime matching."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True, slots=True)
+class NormalizedToken:
+    """A normalized token plus its source character span."""
+
+    text: str
+    start_char: int
+    end_char: int
+
+
+def normalize_text(text: str) -> str:
+    """Normalize text for phrase storage and lookup."""
+    return " ".join(token.text for token in tokenize_with_offsets(text))
+
+
+def tokenize(text: str) -> list[str]:
+    """Normalize and split text into phrase-detection tokens."""
+    return [token.text for token in tokenize_with_offsets(text)]
+
+
+def tokenize_with_offsets(text: str) -> list[NormalizedToken]:
+    """Normalize text into tokens while preserving original character offsets."""
+    tokens: list[NormalizedToken] = []
+    current: list[str] = []
+    start_char: int | None = None
+
+    for index, char in enumerate(text):
+        normalized = normalize_char(char)
+        if normalized == " ":
+            if current and start_char is not None:
+                tokens.append(NormalizedToken(text="".join(current), start_char=start_char, end_char=index))
+                current = []
+                start_char = None
+            continue
+        if start_char is None:
+            start_char = index
+        current.append(normalized)
+
+    if current and start_char is not None:
+        tokens.append(NormalizedToken(text="".join(current), start_char=start_char, end_char=len(text)))
+    return tokens
+
+
+def normalize_char(char: str) -> str:
+    """Normalize one character into a token character or a separator."""
+    if char in {"\u2019", "\u2018"}:
+        return "'"
+    if char in {"-", "\u2013", "\u2014"}:
+        return " "
+
+    lowered = char.lower()
+    if lowered in "abcdefghijklmnopqrstuvwxyz0123456789'":
+        return lowered
+    return " "