Richie
dab18c1385
- Introduced dataclasses for phrase candidates, judgments, and matches in `models.py`. - Implemented database operations for candidate and protected phrases in `store.py`, including loading, saving, and deleting phrases. - Enhanced text normalization functions in `text_normalization.py` with detailed docstrings. - Refactored search functionality to utilize new models and methods for detecting protected phrases.
92 lines
2.4 KiB
Python
92 lines
2.4 KiB
Python
"""Protected phrase extraction, storage, and runtime matching."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from dataclasses import dataclass
|
|
|
|
JSON_OBJECT_RE = re.compile(r"\{.*\}", re.DOTALL)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class NormalizedToken:
|
|
"""A normalized token plus its source character span."""
|
|
|
|
text: str
|
|
start_char: int
|
|
end_char: int
|
|
|
|
|
|
def normalize_text(text: str) -> str:
|
|
"""Normalize text for phrase storage and lookup.
|
|
|
|
Args:
|
|
text (str): Raw text to normalize.
|
|
|
|
Returns:
|
|
str: Normalized tokens joined by single spaces.
|
|
"""
|
|
return " ".join(token.text for token in tokenize_with_offsets(text))
|
|
|
|
|
|
def tokenize(text: str) -> list[str]:
|
|
"""Normalize and split text into phrase-detection tokens.
|
|
|
|
Args:
|
|
text (str): Raw text to tokenize.
|
|
|
|
Returns:
|
|
list[str]: Normalized token strings.
|
|
"""
|
|
return [token.text for token in tokenize_with_offsets(text)]
|
|
|
|
|
|
def tokenize_with_offsets(text: str) -> list[NormalizedToken]:
|
|
"""Normalize text into tokens while preserving original character offsets.
|
|
|
|
Args:
|
|
text (str): Raw text to tokenize.
|
|
|
|
Returns:
|
|
list[NormalizedToken]: Normalized tokens with their source character spans.
|
|
"""
|
|
tokens: list[NormalizedToken] = []
|
|
current: list[str] = []
|
|
start_char: int | None = None
|
|
|
|
for index, char in enumerate(text):
|
|
normalized = normalize_char(char)
|
|
if normalized == " ":
|
|
if current and start_char is not None:
|
|
tokens.append(NormalizedToken(text="".join(current), start_char=start_char, end_char=index))
|
|
current = []
|
|
start_char = None
|
|
continue
|
|
if start_char is None:
|
|
start_char = index
|
|
current.append(normalized)
|
|
|
|
if current and start_char is not None:
|
|
tokens.append(NormalizedToken(text="".join(current), start_char=start_char, end_char=len(text)))
|
|
return tokens
|
|
|
|
|
|
def normalize_char(char: str) -> str:
|
|
"""Normalize one character into a token character or a separator.
|
|
|
|
Args:
|
|
char (str): Single source character to normalize.
|
|
|
|
Returns:
|
|
str: The normalized token character, or a space acting as a separator.
|
|
"""
|
|
if char in {"\u2019", "\u2018"}:
|
|
return "'"
|
|
if char in {"-", "\u2013", "\u2014"}:
|
|
return " "
|
|
|
|
lowered = char.lower()
|
|
if lowered in "abcdefghijklmnopqrstuvwxyz0123456789'":
|
|
return lowered
|
|
return " "
|