Richie/dotfiles
1
0
Fork 0
Code Issues 1 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

refactor(ebook): remove spaCy-ner attributes from PhraseCandidate and related functions

Browse Source
This commit is contained in:
Richie
2026-07-12 17:51:38 -04:00
parent a30f601000
commit c4993f5a53
5 changed files with 58 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ebook_search/protected_phrases/extraction.py
+3 -103
View File
@@ -36,32 +36,6 @@ MULTI_SOURCE_MIN_SOURCES = 2
CAPITALIZED_PHRASE_RE = re.compile(r"\b(?:[A-Z][a-zA-Z']+)(?:\s+(?:of|the|and|in|on|for|[A-Z][a-zA-Z']+)){0,6}")
class SpacySpan(Protocol):
"""Small protocol for the spaCy span attributes used by this module."""
text: str
class SpacyEntity(SpacySpan, Protocol):
"""Small protocol for the spaCy entity attributes used by this module."""
label_: str
class SpacyDoc(Protocol):
"""Small protocol for the spaCy doc attributes used by this module."""
ents: Iterable[SpacyEntity]
noun_chunks: Iterable[SpacySpan]
class SpacyLanguage(Protocol):
"""Small protocol for a callable spaCy language pipeline."""
def __call__(self, text: str) -> SpacyDoc:
"""Parse text into a spaCy-like doc."""
class YakeExtractor(Protocol):
"""Small protocol for the YAKE extractor used by this module."""
@@ -242,55 +216,6 @@ def extract_yake_candidates(
return out
def extract_spacy_candidates(
book_text: str,
nlp: SpacyLanguage,
config: EbookSearchConfig,
) -> dict[str, PhraseCandidate]:
"""Extract spaCy named entities and noun chunks from one text block.
Args:
book_text (str): Text block to parse with spaCy.
nlp (SpacyLanguage): Callable spaCy language pipeline.
config (EbookSearchConfig): Runtime phrase-tuning settings.
Returns:
dict[str, PhraseCandidate]: Candidates keyed by normalized phrase from entities and noun chunks.
"""
out: dict[str, PhraseCandidate] = {}
doc = nlp(book_text)
for ent in doc.ents:
normalized = normalize_candidate_phrase(
ent.text,
config,
max_tokens=config.phrase_max_entity_tokens,
)
if normalized is None:
continue
phrase_text, phrase_norm, token_count = normalized
out[phrase_norm] = PhraseCandidate(
phrase_text=phrase_text,
phrase_norm=phrase_norm,
token_count=token_count,
source_spacy_ner=True,
spacy_label=ent.label_,
)
for chunk in doc.noun_chunks:
normalized = normalize_candidate_phrase(chunk.text, config, strip_leading_article=True)
if normalized is None:
continue
phrase_text, phrase_norm, token_count = normalized
out[phrase_norm] = PhraseCandidate(
phrase_text=phrase_text,
phrase_norm=phrase_norm,
token_count=token_count,
source_spacy_noun_chunk=True,
)
return out
def extract_capitalized_phrases(original_text: str, config: EbookSearchConfig) -> dict[str, PhraseCandidate]:
"""Extract capitalized phrase runs that often carry fictional terms.
@@ -392,16 +317,12 @@ def merge_candidate(existing: PhraseCandidate, item: PhraseCandidate) -> None:
"""
existing.source_raw_ngram = existing.source_raw_ngram or item.source_raw_ngram
existing.source_yake = existing.source_yake or item.source_yake
existing.source_spacy_ner = existing.source_spacy_ner or item.source_spacy_ner
existing.source_spacy_noun_chunk = existing.source_spacy_noun_chunk or item.source_spacy_noun_chunk
existing.source_capitalized = existing.source_capitalized or item.source_capitalized
existing.source_metadata = existing.source_metadata or item.source_metadata
existing.raw_count += item.raw_count
existing.chapter_count = max(existing.chapter_count, item.chapter_count)
if item.yake_score is not None:
existing.yake_score = item.yake_score
if item.spacy_label:
existing.spacy_label = item.spacy_label
def enrich_with_frequency_and_chapter_counts(
@@ -599,8 +520,6 @@ def non_raw_source_count(candidate: PhraseCandidate) -> int:
return sum(
(
candidate.source_yake,
candidate.source_spacy_ner,
candidate.source_spacy_noun_chunk,
candidate.source_capitalized,
candidate.source_metadata,
)
@@ -646,8 +565,6 @@ def source_score(candidate: PhraseCandidate) -> float:
weight
for enabled, weight in (
(candidate.source_yake, 2.0),
(candidate.source_spacy_ner, 2.5),
(candidate.source_spacy_noun_chunk, 1.5),
(candidate.source_capitalized, 2.0),
(candidate.source_metadata, 2.0),
(candidate.source_raw_ngram, 0.5),
@@ -738,10 +655,6 @@ def candidate_source_names(candidate: PhraseCandidate) -> list[str]:
names.append("raw_ngram")
if candidate.source_yake:
names.append("yake")
if candidate.source_spacy_ner:
names.append("spacy_ner")
if candidate.source_spacy_noun_chunk:
names.append("spacy_noun_chunk")
if candidate.source_capitalized:
names.append("capitalized")
if candidate.source_metadata:
@@ -754,16 +667,14 @@ def extract_phrase_candidates_for_book(
chapters: Sequence[str],
config: EbookSearchConfig,
*,
nlp: SpacyLanguage | None = None,
metadata: Mapping[str, object] | None = None,
) -> list[PhraseCandidate]:
"""Extract, score, and limit phrase candidates for one book.
Args:
book_text (str): Full book text used for most extraction sources.
chapters (Sequence[str]): Chapter-like text blocks used for spaCy and frequency counts.
chapters (Sequence[str]): Chapter-like text blocks used for frequency counts.
config (EbookSearchConfig): Runtime phrase-tuning settings.
nlp (SpacyLanguage | None): Optional spaCy pipeline for entity and noun-chunk sources.
metadata (Mapping[str, object] | None): Optional book metadata used as a candidate source.
Returns:
@@ -792,16 +703,6 @@ def extract_phrase_candidates_for_book(
len(yake_candidates),
(perf_counter() - yake_started_at) * 1000,
)
spacy_candidates: dict[str, PhraseCandidate] = {}
if nlp is not None:
spacy_started_at = perf_counter()
for chapter in chapters:
spacy_candidates = merge_candidate_sources(spacy_candidates, extract_spacy_candidates(chapter, nlp, config))
logger.info(
"ebook_phrase_candidate_extract_spacy_complete candidates=%s duration_ms=%.1f",
len(spacy_candidates),
(perf_counter() - spacy_started_at) * 1000,
)
capitalized_started_at = perf_counter()
capitalized = extract_capitalized_phrases(book_text, config)
logger.info(
@@ -811,7 +712,7 @@ def extract_phrase_candidates_for_book(
)
metadata_candidates = extract_metadata_candidates(metadata, config)
candidates = merge_candidate_sources(raw, yake_candidates, spacy_candidates, capitalized, metadata_candidates)
candidates = merge_candidate_sources(raw, yake_candidates, capitalized, metadata_candidates)
enriched_started_at = perf_counter()
# Raw n-gram sizes were already counted per chapter above, so only enrich the remaining
# (entity-length) sizes here instead of re-sliding every size over the whole book.
@@ -831,12 +732,11 @@ def extract_phrase_candidates_for_book(
: config.protected_phrase_max_candidates_per_book
]
logger.info(
"ebook_phrase_candidate_extract_complete raw=%s yake=%s spacy=%s capitalized=%s metadata=%s "
"ebook_phrase_candidate_extract_complete raw=%s yake=%s capitalized=%s metadata=%s "
"merged=%s filtered_too_short=%s filtered_too_rare=%s filtered_too_common=%s filtered_junk=%s "
"min_uses=%s storable=%s limited=%s enrich_score_ms=%.1f duration_ms=%.1f",
len(raw),
len(yake_candidates),
len(spacy_candidates),
len(capitalized),
len(metadata_candidates),
pre_filter_count,