Add models and database persistence for protected phrase extraction

- Introduced dataclasses for phrase candidates, judgments, and matches in `models.py`.
- Implemented database operations for candidate and protected phrases in `store.py`, including loading, saving, and deleting phrases.
- Enhanced text normalization functions in `text_normalization.py` with detailed docstrings.
- Refactored search functionality to utilize new models and methods for detecting protected phrases.
This commit is contained in:
2026-07-12 17:49:52 -04:00
parent 222ae5755e
commit bcb8b7b169
12 changed files with 2803 additions and 1978 deletions
@@ -2,10 +2,10 @@
from __future__ import annotations
import logging
import re
from dataclasses import dataclass
logger = logging.getLogger(__name__)
JSON_OBJECT_RE = re.compile(r"\{.*\}", re.DOTALL)
@dataclass(frozen=True, slots=True)
@@ -18,17 +18,38 @@ class NormalizedToken:
def normalize_text(text: str) -> str:
"""Normalize text for phrase storage and lookup."""
"""Normalize text for phrase storage and lookup.
Args:
text (str): Raw text to normalize.
Returns:
str: Normalized tokens joined by single spaces.
"""
return " ".join(token.text for token in tokenize_with_offsets(text))
def tokenize(text: str) -> list[str]:
"""Normalize and split text into phrase-detection tokens."""
"""Normalize and split text into phrase-detection tokens.
Args:
text (str): Raw text to tokenize.
Returns:
list[str]: Normalized token strings.
"""
return [token.text for token in tokenize_with_offsets(text)]
def tokenize_with_offsets(text: str) -> list[NormalizedToken]:
"""Normalize text into tokens while preserving original character offsets."""
"""Normalize text into tokens while preserving original character offsets.
Args:
text (str): Raw text to tokenize.
Returns:
list[NormalizedToken]: Normalized tokens with their source character spans.
"""
tokens: list[NormalizedToken] = []
current: list[str] = []
start_char: int | None = None
@@ -51,7 +72,14 @@ def tokenize_with_offsets(text: str) -> list[NormalizedToken]:
def normalize_char(char: str) -> str:
"""Normalize one character into a token character or a separator."""
"""Normalize one character into a token character or a separator.
Args:
char (str): Single source character to normalize.
Returns:
str: The normalized token character, or a space acting as a separator.
"""
if char in {"\u2019", "\u2018"}:
return "'"
if char in {"-", "\u2013", "\u2014"}: