refactor(ebook): remove spaCy-ner attributes from PhraseCandidate and related functions

python/alembic/richie/versions/2026_07_09-remove_spacy_ner_751260fc3228.py

@@ -0,0 +1,55 @@
+"""remove spaCy-ner.
+
+Revision ID: 751260fc3228
+Revises: dddee09eddcc
+Create Date: 2026-07-09 23:03:39.554083
+
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import sqlalchemy as sa
+from alembic import op
+
+from python.orm import RichieBase
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+# revision identifiers, used by Alembic.
+revision: str = "751260fc3228"
+down_revision: str | None = "dddee09eddcc"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+schema = RichieBase.schema_name
+
+
+def upgrade() -> None:
+    """Upgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("candidate_phrases", "source_spacy_noun_chunk", schema=schema)
+    op.drop_column("candidate_phrases", "source_spacy_ner", schema=schema)
+    op.drop_column("candidate_phrases", "spacy_label", schema=schema)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "candidate_phrases", sa.Column("spacy_label", sa.VARCHAR(), autoincrement=False, nullable=True), schema=schema
+    )
+    op.add_column(
+        "candidate_phrases",
+        sa.Column("source_spacy_ner", sa.BOOLEAN(), autoincrement=False, nullable=False),
+        schema=schema,
+    )
+    op.add_column(
+        "candidate_phrases",
+        sa.Column("source_spacy_noun_chunk", sa.BOOLEAN(), autoincrement=False, nullable=False),
+        schema=schema,
+    )
+    # ### end Alembic commands ###

python/ebook_search/protected_phrases/extraction.py

@@ -36,32 +36,6 @@ MULTI_SOURCE_MIN_SOURCES = 2
 CAPITALIZED_PHRASE_RE = re.compile(r"\b(?:[A-Z][a-zA-Z']+)(?:\s+(?:of|the|and|in|on|for|[A-Z][a-zA-Z']+)){0,6}")
 
 
-class SpacySpan(Protocol):
-    """Small protocol for the spaCy span attributes used by this module."""
-
-    text: str
-
-
-class SpacyEntity(SpacySpan, Protocol):
-    """Small protocol for the spaCy entity attributes used by this module."""
-
-    label_: str
-
-
-class SpacyDoc(Protocol):
-    """Small protocol for the spaCy doc attributes used by this module."""
-
-    ents: Iterable[SpacyEntity]
-    noun_chunks: Iterable[SpacySpan]
-
-
-class SpacyLanguage(Protocol):
-    """Small protocol for a callable spaCy language pipeline."""
-
-    def __call__(self, text: str) -> SpacyDoc:
-        """Parse text into a spaCy-like doc."""
-
-
 class YakeExtractor(Protocol):
     """Small protocol for the YAKE extractor used by this module."""
 
@@ -242,55 +216,6 @@ def extract_yake_candidates(
     return out
 
 
-def extract_spacy_candidates(
-    book_text: str,
-    nlp: SpacyLanguage,
-    config: EbookSearchConfig,
-) -> dict[str, PhraseCandidate]:
-    """Extract spaCy named entities and noun chunks from one text block.
-
-    Args:
-        book_text (str): Text block to parse with spaCy.
-        nlp (SpacyLanguage): Callable spaCy language pipeline.
-        config (EbookSearchConfig): Runtime phrase-tuning settings.
-
-    Returns:
-        dict[str, PhraseCandidate]: Candidates keyed by normalized phrase from entities and noun chunks.
-    """
-    out: dict[str, PhraseCandidate] = {}
-    doc = nlp(book_text)
-
-    for ent in doc.ents:
-        normalized = normalize_candidate_phrase(
-            ent.text,
-            config,
-            max_tokens=config.phrase_max_entity_tokens,
-        )
-        if normalized is None:
-            continue
-        phrase_text, phrase_norm, token_count = normalized
-        out[phrase_norm] = PhraseCandidate(
-            phrase_text=phrase_text,
-            phrase_norm=phrase_norm,
-            token_count=token_count,
-            source_spacy_ner=True,
-            spacy_label=ent.label_,
-        )
-
-    for chunk in doc.noun_chunks:
-        normalized = normalize_candidate_phrase(chunk.text, config, strip_leading_article=True)
-        if normalized is None:
-            continue
-        phrase_text, phrase_norm, token_count = normalized
-        out[phrase_norm] = PhraseCandidate(
-            phrase_text=phrase_text,
-            phrase_norm=phrase_norm,
-            token_count=token_count,
-            source_spacy_noun_chunk=True,
-        )
-    return out
-
-
 def extract_capitalized_phrases(original_text: str, config: EbookSearchConfig) -> dict[str, PhraseCandidate]:
     """Extract capitalized phrase runs that often carry fictional terms.
 
@@ -392,16 +317,12 @@ def merge_candidate(existing: PhraseCandidate, item: PhraseCandidate) -> None:
     """
     existing.source_raw_ngram = existing.source_raw_ngram or item.source_raw_ngram
     existing.source_yake = existing.source_yake or item.source_yake
-    existing.source_spacy_ner = existing.source_spacy_ner or item.source_spacy_ner
-    existing.source_spacy_noun_chunk = existing.source_spacy_noun_chunk or item.source_spacy_noun_chunk
     existing.source_capitalized = existing.source_capitalized or item.source_capitalized
     existing.source_metadata = existing.source_metadata or item.source_metadata
     existing.raw_count += item.raw_count
     existing.chapter_count = max(existing.chapter_count, item.chapter_count)
     if item.yake_score is not None:
         existing.yake_score = item.yake_score
-    if item.spacy_label:
-        existing.spacy_label = item.spacy_label
 
 
 def enrich_with_frequency_and_chapter_counts(
@@ -599,8 +520,6 @@ def non_raw_source_count(candidate: PhraseCandidate) -> int:
     return sum(
         (
             candidate.source_yake,
-            candidate.source_spacy_ner,
-            candidate.source_spacy_noun_chunk,
             candidate.source_capitalized,
             candidate.source_metadata,
         )
@@ -646,8 +565,6 @@ def source_score(candidate: PhraseCandidate) -> float:
         weight
         for enabled, weight in (
             (candidate.source_yake, 2.0),
-            (candidate.source_spacy_ner, 2.5),
-            (candidate.source_spacy_noun_chunk, 1.5),
             (candidate.source_capitalized, 2.0),
             (candidate.source_metadata, 2.0),
             (candidate.source_raw_ngram, 0.5),
@@ -738,10 +655,6 @@ def candidate_source_names(candidate: PhraseCandidate) -> list[str]:
         names.append("raw_ngram")
     if candidate.source_yake:
         names.append("yake")
-    if candidate.source_spacy_ner:
-        names.append("spacy_ner")
-    if candidate.source_spacy_noun_chunk:
-        names.append("spacy_noun_chunk")
     if candidate.source_capitalized:
         names.append("capitalized")
     if candidate.source_metadata:
@@ -754,16 +667,14 @@ def extract_phrase_candidates_for_book(
     chapters: Sequence[str],
     config: EbookSearchConfig,
     *,
-    nlp: SpacyLanguage | None = None,
     metadata: Mapping[str, object] | None = None,
 ) -> list[PhraseCandidate]:
     """Extract, score, and limit phrase candidates for one book.
 
     Args:
         book_text (str): Full book text used for most extraction sources.
-        chapters (Sequence[str]): Chapter-like text blocks used for spaCy and frequency counts.
+        chapters (Sequence[str]): Chapter-like text blocks used for frequency counts.
         config (EbookSearchConfig): Runtime phrase-tuning settings.
-        nlp (SpacyLanguage | None): Optional spaCy pipeline for entity and noun-chunk sources.
         metadata (Mapping[str, object] | None): Optional book metadata used as a candidate source.
 
     Returns:
@@ -792,16 +703,6 @@ def extract_phrase_candidates_for_book(
         len(yake_candidates),
         (perf_counter() - yake_started_at) * 1000,
     )
-    spacy_candidates: dict[str, PhraseCandidate] = {}
-    if nlp is not None:
-        spacy_started_at = perf_counter()
-        for chapter in chapters:
-            spacy_candidates = merge_candidate_sources(spacy_candidates, extract_spacy_candidates(chapter, nlp, config))
-        logger.info(
-            "ebook_phrase_candidate_extract_spacy_complete candidates=%s duration_ms=%.1f",
-            len(spacy_candidates),
-            (perf_counter() - spacy_started_at) * 1000,
-        )
     capitalized_started_at = perf_counter()
     capitalized = extract_capitalized_phrases(book_text, config)
     logger.info(
@@ -811,7 +712,7 @@ def extract_phrase_candidates_for_book(
     )
     metadata_candidates = extract_metadata_candidates(metadata, config)
 
-    candidates = merge_candidate_sources(raw, yake_candidates, spacy_candidates, capitalized, metadata_candidates)
+    candidates = merge_candidate_sources(raw, yake_candidates, capitalized, metadata_candidates)
     enriched_started_at = perf_counter()
     # Raw n-gram sizes were already counted per chapter above, so only enrich the remaining
     # (entity-length) sizes here instead of re-sliding every size over the whole book.
@@ -831,12 +732,11 @@ def extract_phrase_candidates_for_book(
         : config.protected_phrase_max_candidates_per_book
     ]
     logger.info(
-        "ebook_phrase_candidate_extract_complete raw=%s yake=%s spacy=%s capitalized=%s metadata=%s "
+        "ebook_phrase_candidate_extract_complete raw=%s yake=%s capitalized=%s metadata=%s "
         "merged=%s filtered_too_short=%s filtered_too_rare=%s filtered_too_common=%s filtered_junk=%s "
         "min_uses=%s storable=%s limited=%s enrich_score_ms=%.1f duration_ms=%.1f",
         len(raw),
         len(yake_candidates),
-        len(spacy_candidates),
         len(capitalized),
         len(metadata_candidates),
         pre_filter_count,

python/ebook_search/protected_phrases/models.py

@@ -19,11 +19,8 @@ class PhraseCandidate:
         token_count (int): Number of normalized tokens in the phrase.
         source_raw_ngram (bool): Whether the raw n-gram extractor produced the phrase.
         source_yake (bool): Whether YAKE keyword extraction produced the phrase.
-        source_spacy_ner (bool): Whether spaCy named-entity recognition produced the phrase.
-        source_spacy_noun_chunk (bool): Whether spaCy noun chunking produced the phrase.
         source_capitalized (bool): Whether the capitalized-run extractor produced the phrase.
         source_metadata (bool): Whether book metadata produced the phrase.
-        spacy_label (str | None): spaCy entity label when NER produced the phrase.
         raw_count (int): Occurrences counted across the book text.
         chapter_count (int): Number of chapters containing the phrase.
         yake_score (float | None): Raw YAKE score when available; lower is better.
@@ -36,11 +33,8 @@ class PhraseCandidate:
     token_count: int
     source_raw_ngram: bool = False
     source_yake: bool = False
-    source_spacy_ner: bool = False
-    source_spacy_noun_chunk: bool = False
     source_capitalized: bool = False
     source_metadata: bool = False
-    spacy_label: str | None = None
     raw_count: int = 0
     chapter_count: int = 0
     yake_score: float | None = None

python/ebook_search/protected_phrases/store.py

@@ -295,11 +295,8 @@ def phrase_candidate_from_row(row: EbookCandidatePhrase) -> PhraseCandidate:
         token_count=row.token_count,
         source_raw_ngram=row.source_raw_ngram,
         source_yake=row.source_yake,
-        source_spacy_ner=row.source_spacy_ner,
-        source_spacy_noun_chunk=row.source_spacy_noun_chunk,
         source_capitalized=row.source_capitalized,
         source_metadata=row.source_metadata,
-        spacy_label=row.spacy_label,
         raw_count=row.raw_count,
         chapter_count=row.chapter_count,
         yake_score=row.yake_score,
@@ -334,11 +331,8 @@ def candidate_row_values(
         "token_count": candidate.token_count,
         "source_raw_ngram": candidate.source_raw_ngram,
         "source_yake": candidate.source_yake,
-        "source_spacy_ner": candidate.source_spacy_ner,
-        "source_spacy_noun_chunk": candidate.source_spacy_noun_chunk,
         "source_capitalized": candidate.source_capitalized,
         "source_metadata": candidate.source_metadata,
-        "spacy_label": candidate.spacy_label,
         "raw_count": candidate.raw_count,
         "chapter_count": candidate.chapter_count,
         "yake_score": candidate.yake_score,
@@ -460,11 +454,8 @@ def new_candidate_row(book_id: int, series_id: int | None, candidate: PhraseCand
     row.token_count = candidate.token_count
     row.source_raw_ngram = candidate.source_raw_ngram
     row.source_yake = candidate.source_yake
-    row.source_spacy_ner = candidate.source_spacy_ner
-    row.source_spacy_noun_chunk = candidate.source_spacy_noun_chunk
     row.source_capitalized = candidate.source_capitalized
     row.source_metadata = candidate.source_metadata
-    row.spacy_label = candidate.spacy_label
     row.raw_count = candidate.raw_count
     row.chapter_count = candidate.chapter_count
     row.yake_score = candidate.yake_score

python/orm/richie/ebook.py

@@ -167,11 +167,8 @@ class EbookCandidatePhrase(TableBase):
     token_count: Mapped[int]
     source_raw_ngram: Mapped[bool] = mapped_column(default=False)
     source_yake: Mapped[bool] = mapped_column(default=False)
-    source_spacy_ner: Mapped[bool] = mapped_column(default=False)
-    source_spacy_noun_chunk: Mapped[bool] = mapped_column(default=False)
     source_capitalized: Mapped[bool] = mapped_column(default=False)
     source_metadata: Mapped[bool] = mapped_column(default=False)
-    spacy_label: Mapped[str | None]
     raw_count: Mapped[int] = mapped_column(default=0)
     chapter_count: Mapped[int] = mapped_column(default=0)
     yake_score: Mapped[float | None]