64 lines
1.9 KiB
Python
64 lines
1.9 KiB
Python
"""Protected phrase extraction, storage, and runtime matching."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from dataclasses import dataclass
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class NormalizedToken:
|
|
"""A normalized token plus its source character span."""
|
|
|
|
text: str
|
|
start_char: int
|
|
end_char: int
|
|
|
|
|
|
def normalize_text(text: str) -> str:
|
|
"""Normalize text for phrase storage and lookup."""
|
|
return " ".join(token.text for token in tokenize_with_offsets(text))
|
|
|
|
|
|
def tokenize(text: str) -> list[str]:
|
|
"""Normalize and split text into phrase-detection tokens."""
|
|
return [token.text for token in tokenize_with_offsets(text)]
|
|
|
|
|
|
def tokenize_with_offsets(text: str) -> list[NormalizedToken]:
|
|
"""Normalize text into tokens while preserving original character offsets."""
|
|
tokens: list[NormalizedToken] = []
|
|
current: list[str] = []
|
|
start_char: int | None = None
|
|
|
|
for index, char in enumerate(text):
|
|
normalized = normalize_char(char)
|
|
if normalized == " ":
|
|
if current and start_char is not None:
|
|
tokens.append(NormalizedToken(text="".join(current), start_char=start_char, end_char=index))
|
|
current = []
|
|
start_char = None
|
|
continue
|
|
if start_char is None:
|
|
start_char = index
|
|
current.append(normalized)
|
|
|
|
if current and start_char is not None:
|
|
tokens.append(NormalizedToken(text="".join(current), start_char=start_char, end_char=len(text)))
|
|
return tokens
|
|
|
|
|
|
def normalize_char(char: str) -> str:
|
|
"""Normalize one character into a token character or a separator."""
|
|
if char in {"\u2019", "\u2018"}:
|
|
return "'"
|
|
if char in {"-", "\u2013", "\u2014"}:
|
|
return " "
|
|
|
|
lowered = char.lower()
|
|
if lowered in "abcdefghijklmnopqrstuvwxyz0123456789'":
|
|
return lowered
|
|
return " "
|