Add models and database persistence for protected phrase extraction
- Introduced dataclasses for phrase candidates, judgments, and matches in `models.py`. - Implemented database operations for candidate and protected phrases in `store.py`, including loading, saving, and deleting phrases. - Enhanced text normalization functions in `text_normalization.py` with detailed docstrings. - Refactored search functionality to utilize new models and methods for detecting protected phrases.
This commit is contained in:
@@ -0,0 +1,262 @@
|
||||
"""Dataclasses shared by protected phrase extraction, judging, matching, and backfills."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Mapping
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class PhraseCandidate:
|
||||
"""A phrase candidate with merged extraction-source metadata.
|
||||
|
||||
Attributes:
|
||||
phrase_text (str): Display text for the phrase.
|
||||
phrase_norm (str): Normalized phrase used as the merge key.
|
||||
token_count (int): Number of normalized tokens in the phrase.
|
||||
source_raw_ngram (bool): Whether the raw n-gram extractor produced the phrase.
|
||||
source_yake (bool): Whether YAKE keyword extraction produced the phrase.
|
||||
source_spacy_ner (bool): Whether spaCy named-entity recognition produced the phrase.
|
||||
source_spacy_noun_chunk (bool): Whether spaCy noun chunking produced the phrase.
|
||||
source_capitalized (bool): Whether the capitalized-run extractor produced the phrase.
|
||||
source_metadata (bool): Whether book metadata produced the phrase.
|
||||
spacy_label (str | None): spaCy entity label when NER produced the phrase.
|
||||
raw_count (int): Occurrences counted across the book text.
|
||||
chapter_count (int): Number of chapters containing the phrase.
|
||||
yake_score (float | None): Raw YAKE score when available; lower is better.
|
||||
candidate_score (float): Combined pre-judging score.
|
||||
sample_contexts (list[str]): Normalized context snippets around occurrences.
|
||||
"""
|
||||
|
||||
phrase_text: str
|
||||
phrase_norm: str
|
||||
token_count: int
|
||||
source_raw_ngram: bool = False
|
||||
source_yake: bool = False
|
||||
source_spacy_ner: bool = False
|
||||
source_spacy_noun_chunk: bool = False
|
||||
source_capitalized: bool = False
|
||||
source_metadata: bool = False
|
||||
spacy_label: str | None = None
|
||||
raw_count: int = 0
|
||||
chapter_count: int = 0
|
||||
yake_score: float | None = None
|
||||
candidate_score: float = 0.0
|
||||
sample_contexts: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LLMJudgment:
|
||||
"""A structured phrase judgment returned by the LLM judge.
|
||||
|
||||
Attributes:
|
||||
keep (bool): Whether the judge accepted the phrase for protection.
|
||||
canonical (str | None): Canonical phrase text chosen by the judge.
|
||||
category (str | None): Phrase category such as person, place, or event.
|
||||
aliases (tuple[str, ...]): Alternate surface forms for the phrase.
|
||||
confidence (float): Judge confidence between 0.0 and 1.0.
|
||||
importance (float): Judge importance between 0.0 and 1.0.
|
||||
allow_nested (bool): Whether the phrase may match inside a larger kept match.
|
||||
suppress_children (bool): Whether the phrase suppresses matches nested inside it.
|
||||
reason (str | None): Free-text explanation from the judge.
|
||||
"""
|
||||
|
||||
keep: bool
|
||||
canonical: str | None
|
||||
category: str | None
|
||||
aliases: tuple[str, ...]
|
||||
confidence: float
|
||||
importance: float = 0.5
|
||||
allow_nested: bool = False
|
||||
suppress_children: bool = True
|
||||
reason: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PhraseLookup:
|
||||
"""In-memory lookup maps used for constant-time phrase-window checks.
|
||||
|
||||
Attributes:
|
||||
norm_to_phrase_ids (Mapping[str, tuple[int, ...]]): Normalized phrase to protected phrase ids.
|
||||
alias_to_phrase_ids (Mapping[str, tuple[int, ...]]): Normalized alias to protected phrase ids.
|
||||
min_tokens (int): Smallest token-window size to test.
|
||||
max_tokens (int): Largest token-window size to test.
|
||||
"""
|
||||
|
||||
norm_to_phrase_ids: Mapping[str, tuple[int, ...]]
|
||||
alias_to_phrase_ids: Mapping[str, tuple[int, ...]]
|
||||
min_tokens: int
|
||||
max_tokens: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PhraseMatch:
|
||||
"""An unhydrated query or chunk phrase match.
|
||||
|
||||
Attributes:
|
||||
phrase_norm (str): Normalized text of the matched window.
|
||||
start_token (int): Index of the first matched token.
|
||||
end_token (int): Index one past the last matched token.
|
||||
token_count (int): Number of tokens in the match.
|
||||
phrase_id (int | None): Matched protected phrase id when known.
|
||||
start_char (int | None): Start character offset in the source text.
|
||||
end_char (int | None): End character offset in the source text.
|
||||
"""
|
||||
|
||||
phrase_norm: str
|
||||
start_token: int
|
||||
end_token: int
|
||||
token_count: int
|
||||
phrase_id: int | None = None
|
||||
start_char: int | None = None
|
||||
end_char: int | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class HydratedPhraseMatch:
|
||||
"""A phrase match with protected-phrase metadata attached.
|
||||
|
||||
Attributes:
|
||||
phrase_id (int): Protected phrase id.
|
||||
matched_norm (str): Normalized window text that matched.
|
||||
phrase_text (str): Display text of the protected phrase.
|
||||
phrase_norm (str): Normalized text of the protected phrase.
|
||||
canonical_id (str): Deterministic ``category:slug`` identifier.
|
||||
phrase_type (str | None): Phrase category.
|
||||
token_count (int): Number of tokens in the match.
|
||||
confidence (float): Stored judge confidence.
|
||||
importance (float): Stored judge importance.
|
||||
allow_nested (bool): Whether the phrase may match inside a larger kept match.
|
||||
suppress_children (bool): Whether the phrase suppresses matches nested inside it.
|
||||
start_token (int): Index of the first matched token.
|
||||
end_token (int): Index one past the last matched token.
|
||||
start_char (int | None): Start character offset in the source text.
|
||||
end_char (int | None): End character offset in the source text.
|
||||
book_id (int | None): Book scope of the phrase.
|
||||
series_id (int | None): Series scope of the phrase.
|
||||
"""
|
||||
|
||||
phrase_id: int
|
||||
matched_norm: str
|
||||
phrase_text: str
|
||||
phrase_norm: str
|
||||
canonical_id: str
|
||||
phrase_type: str | None
|
||||
token_count: int
|
||||
confidence: float
|
||||
importance: float
|
||||
allow_nested: bool
|
||||
suppress_children: bool
|
||||
start_token: int
|
||||
end_token: int
|
||||
start_char: int | None = None
|
||||
end_char: int | None = None
|
||||
book_id: int | None = None
|
||||
series_id: int | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ChunkPhraseHit:
|
||||
"""One protected phrase with its mention count inside one retrieved chunk.
|
||||
|
||||
Attributes:
|
||||
phrase_id (int): Protected phrase id.
|
||||
phrase_text (str): Display text of the protected phrase.
|
||||
mention_count (int): Indexed mentions of the phrase in the chunk.
|
||||
"""
|
||||
|
||||
phrase_id: int
|
||||
phrase_text: str
|
||||
mention_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PhraseCandidateGenerationResult:
|
||||
"""Summary of candidate phrase extraction for indexed books.
|
||||
|
||||
Attributes:
|
||||
books_seen (int): Indexed books examined.
|
||||
books_built (int): Books that had candidates generated and committed.
|
||||
candidate_phrases (int): Candidate phrases stored across all books.
|
||||
"""
|
||||
|
||||
books_seen: int
|
||||
books_built: int
|
||||
candidate_phrases: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PhraseJudgmentBackfillResult:
|
||||
"""Summary of LLM judging for stored candidate phrases.
|
||||
|
||||
Attributes:
|
||||
books_seen (int): Indexed books examined.
|
||||
books_judged (int): Books with judgments committed.
|
||||
books_failed (int): Books rolled back after an error.
|
||||
candidates_judged (int): Candidate phrases sent to the LLM judge.
|
||||
protected_phrases (int): Protected phrases promoted from candidates.
|
||||
phrase_mentions (int): Chunk phrase mentions indexed across all books.
|
||||
"""
|
||||
|
||||
books_seen: int
|
||||
books_judged: int
|
||||
books_failed: int
|
||||
candidates_judged: int
|
||||
protected_phrases: int
|
||||
phrase_mentions: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BookJudgmentResult:
|
||||
"""Outcome of judging one book's candidate phrases.
|
||||
|
||||
Attributes:
|
||||
judged (int): Candidate phrases sent to the LLM judge.
|
||||
protected (int): Protected phrases promoted from candidates.
|
||||
mentions (int): Chunk phrase mentions indexed for the book.
|
||||
committed (bool): Whether the book's judgments were committed.
|
||||
failed (bool): Whether the book was rolled back after an error.
|
||||
"""
|
||||
|
||||
judged: int = 0
|
||||
protected: int = 0
|
||||
mentions: int = 0
|
||||
committed: bool = False
|
||||
failed: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BookCandidateResult:
|
||||
"""Outcome of generating one book's candidate phrases.
|
||||
|
||||
Attributes:
|
||||
candidates (int): Candidate phrases stored for the book.
|
||||
built (bool): Whether candidate generation was committed.
|
||||
"""
|
||||
|
||||
candidates: int = 0
|
||||
built: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PhraseRecalculationResult:
|
||||
"""Summary of phrase cleanup and candidate regeneration for one book.
|
||||
|
||||
Attributes:
|
||||
book_id (int): Book the recalculation ran against.
|
||||
deleted_candidates (int): Candidate phrase rows deleted.
|
||||
deleted_protected_phrases (int): Protected phrase rows deleted.
|
||||
deleted_aliases (int): Phrase alias rows deleted.
|
||||
deleted_mentions (int): Chunk phrase mention rows deleted.
|
||||
candidate_phrases (int): Candidate phrases regenerated after cleanup.
|
||||
"""
|
||||
|
||||
book_id: int
|
||||
deleted_candidates: int
|
||||
deleted_protected_phrases: int
|
||||
deleted_aliases: int
|
||||
deleted_mentions: int
|
||||
candidate_phrases: int
|
||||
Reference in New Issue
Block a user