Add models and database persistence for protected phrase extraction
- Introduced dataclasses for phrase candidates, judgments, and matches in `models.py`. - Implemented database operations for candidate and protected phrases in `store.py`, including loading, saving, and deleting phrases. - Enhanced text normalization functions in `text_normalization.py` with detailed docstrings. - Refactored search functionality to utilize new models and methods for detecting protected phrases.
This commit is contained in:
@@ -2,10 +2,10 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
JSON_OBJECT_RE = re.compile(r"\{.*\}", re.DOTALL)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -18,17 +18,38 @@ class NormalizedToken:
|
||||
|
||||
|
||||
def normalize_text(text: str) -> str:
|
||||
"""Normalize text for phrase storage and lookup."""
|
||||
"""Normalize text for phrase storage and lookup.
|
||||
|
||||
Args:
|
||||
text (str): Raw text to normalize.
|
||||
|
||||
Returns:
|
||||
str: Normalized tokens joined by single spaces.
|
||||
"""
|
||||
return " ".join(token.text for token in tokenize_with_offsets(text))
|
||||
|
||||
|
||||
def tokenize(text: str) -> list[str]:
|
||||
"""Normalize and split text into phrase-detection tokens."""
|
||||
"""Normalize and split text into phrase-detection tokens.
|
||||
|
||||
Args:
|
||||
text (str): Raw text to tokenize.
|
||||
|
||||
Returns:
|
||||
list[str]: Normalized token strings.
|
||||
"""
|
||||
return [token.text for token in tokenize_with_offsets(text)]
|
||||
|
||||
|
||||
def tokenize_with_offsets(text: str) -> list[NormalizedToken]:
|
||||
"""Normalize text into tokens while preserving original character offsets."""
|
||||
"""Normalize text into tokens while preserving original character offsets.
|
||||
|
||||
Args:
|
||||
text (str): Raw text to tokenize.
|
||||
|
||||
Returns:
|
||||
list[NormalizedToken]: Normalized tokens with their source character spans.
|
||||
"""
|
||||
tokens: list[NormalizedToken] = []
|
||||
current: list[str] = []
|
||||
start_char: int | None = None
|
||||
@@ -51,7 +72,14 @@ def tokenize_with_offsets(text: str) -> list[NormalizedToken]:
|
||||
|
||||
|
||||
def normalize_char(char: str) -> str:
|
||||
"""Normalize one character into a token character or a separator."""
|
||||
"""Normalize one character into a token character or a separator.
|
||||
|
||||
Args:
|
||||
char (str): Single source character to normalize.
|
||||
|
||||
Returns:
|
||||
str: The normalized token character, or a space acting as a separator.
|
||||
"""
|
||||
if char in {"\u2019", "\u2018"}:
|
||||
return "'"
|
||||
if char in {"-", "\u2013", "\u2014"}:
|
||||
|
||||
Reference in New Issue
Block a user