diff --git a/python/alembic/richie/versions/2026_07_09-remove_spacy_ner_751260fc3228.py b/python/alembic/richie/versions/2026_07_09-remove_spacy_ner_751260fc3228.py new file mode 100644 index 0000000..e1724b2 --- /dev/null +++ b/python/alembic/richie/versions/2026_07_09-remove_spacy_ner_751260fc3228.py @@ -0,0 +1,55 @@ +"""remove spaCy-ner. + +Revision ID: 751260fc3228 +Revises: dddee09eddcc +Create Date: 2026-07-09 23:03:39.554083 + +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import sqlalchemy as sa +from alembic import op + +from python.orm import RichieBase + +if TYPE_CHECKING: + from collections.abc import Sequence + +# revision identifiers, used by Alembic. +revision: str = "751260fc3228" +down_revision: str | None = "dddee09eddcc" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +schema = RichieBase.schema_name + + +def upgrade() -> None: + """Upgrade.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("candidate_phrases", "source_spacy_noun_chunk", schema=schema) + op.drop_column("candidate_phrases", "source_spacy_ner", schema=schema) + op.drop_column("candidate_phrases", "spacy_label", schema=schema) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade.""" + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "candidate_phrases", sa.Column("spacy_label", sa.VARCHAR(), autoincrement=False, nullable=True), schema=schema + ) + op.add_column( + "candidate_phrases", + sa.Column("source_spacy_ner", sa.BOOLEAN(), autoincrement=False, nullable=False), + schema=schema, + ) + op.add_column( + "candidate_phrases", + sa.Column("source_spacy_noun_chunk", sa.BOOLEAN(), autoincrement=False, nullable=False), + schema=schema, + ) + # ### end Alembic commands ### diff --git a/python/ebook_search/protected_phrases/extraction.py b/python/ebook_search/protected_phrases/extraction.py index 1370e5b..84dd6e9 100644 --- a/python/ebook_search/protected_phrases/extraction.py +++ b/python/ebook_search/protected_phrases/extraction.py @@ -36,32 +36,6 @@ MULTI_SOURCE_MIN_SOURCES = 2 CAPITALIZED_PHRASE_RE = re.compile(r"\b(?:[A-Z][a-zA-Z']+)(?:\s+(?:of|the|and|in|on|for|[A-Z][a-zA-Z']+)){0,6}") -class SpacySpan(Protocol): - """Small protocol for the spaCy span attributes used by this module.""" - - text: str - - -class SpacyEntity(SpacySpan, Protocol): - """Small protocol for the spaCy entity attributes used by this module.""" - - label_: str - - -class SpacyDoc(Protocol): - """Small protocol for the spaCy doc attributes used by this module.""" - - ents: Iterable[SpacyEntity] - noun_chunks: Iterable[SpacySpan] - - -class SpacyLanguage(Protocol): - """Small protocol for a callable spaCy language pipeline.""" - - def __call__(self, text: str) -> SpacyDoc: - """Parse text into a spaCy-like doc.""" - - class YakeExtractor(Protocol): """Small protocol for the YAKE extractor used by this module.""" @@ -242,55 +216,6 @@ def extract_yake_candidates( return out -def extract_spacy_candidates( - book_text: str, - nlp: SpacyLanguage, - config: EbookSearchConfig, -) -> dict[str, PhraseCandidate]: - """Extract spaCy named entities and noun chunks from one text block. - - Args: - book_text (str): Text block to parse with spaCy. - nlp (SpacyLanguage): Callable spaCy language pipeline. - config (EbookSearchConfig): Runtime phrase-tuning settings. - - Returns: - dict[str, PhraseCandidate]: Candidates keyed by normalized phrase from entities and noun chunks. - """ - out: dict[str, PhraseCandidate] = {} - doc = nlp(book_text) - - for ent in doc.ents: - normalized = normalize_candidate_phrase( - ent.text, - config, - max_tokens=config.phrase_max_entity_tokens, - ) - if normalized is None: - continue - phrase_text, phrase_norm, token_count = normalized - out[phrase_norm] = PhraseCandidate( - phrase_text=phrase_text, - phrase_norm=phrase_norm, - token_count=token_count, - source_spacy_ner=True, - spacy_label=ent.label_, - ) - - for chunk in doc.noun_chunks: - normalized = normalize_candidate_phrase(chunk.text, config, strip_leading_article=True) - if normalized is None: - continue - phrase_text, phrase_norm, token_count = normalized - out[phrase_norm] = PhraseCandidate( - phrase_text=phrase_text, - phrase_norm=phrase_norm, - token_count=token_count, - source_spacy_noun_chunk=True, - ) - return out - - def extract_capitalized_phrases(original_text: str, config: EbookSearchConfig) -> dict[str, PhraseCandidate]: """Extract capitalized phrase runs that often carry fictional terms. @@ -392,16 +317,12 @@ def merge_candidate(existing: PhraseCandidate, item: PhraseCandidate) -> None: """ existing.source_raw_ngram = existing.source_raw_ngram or item.source_raw_ngram existing.source_yake = existing.source_yake or item.source_yake - existing.source_spacy_ner = existing.source_spacy_ner or item.source_spacy_ner - existing.source_spacy_noun_chunk = existing.source_spacy_noun_chunk or item.source_spacy_noun_chunk existing.source_capitalized = existing.source_capitalized or item.source_capitalized existing.source_metadata = existing.source_metadata or item.source_metadata existing.raw_count += item.raw_count existing.chapter_count = max(existing.chapter_count, item.chapter_count) if item.yake_score is not None: existing.yake_score = item.yake_score - if item.spacy_label: - existing.spacy_label = item.spacy_label def enrich_with_frequency_and_chapter_counts( @@ -599,8 +520,6 @@ def non_raw_source_count(candidate: PhraseCandidate) -> int: return sum( ( candidate.source_yake, - candidate.source_spacy_ner, - candidate.source_spacy_noun_chunk, candidate.source_capitalized, candidate.source_metadata, ) @@ -646,8 +565,6 @@ def source_score(candidate: PhraseCandidate) -> float: weight for enabled, weight in ( (candidate.source_yake, 2.0), - (candidate.source_spacy_ner, 2.5), - (candidate.source_spacy_noun_chunk, 1.5), (candidate.source_capitalized, 2.0), (candidate.source_metadata, 2.0), (candidate.source_raw_ngram, 0.5), @@ -738,10 +655,6 @@ def candidate_source_names(candidate: PhraseCandidate) -> list[str]: names.append("raw_ngram") if candidate.source_yake: names.append("yake") - if candidate.source_spacy_ner: - names.append("spacy_ner") - if candidate.source_spacy_noun_chunk: - names.append("spacy_noun_chunk") if candidate.source_capitalized: names.append("capitalized") if candidate.source_metadata: @@ -754,16 +667,14 @@ def extract_phrase_candidates_for_book( chapters: Sequence[str], config: EbookSearchConfig, *, - nlp: SpacyLanguage | None = None, metadata: Mapping[str, object] | None = None, ) -> list[PhraseCandidate]: """Extract, score, and limit phrase candidates for one book. Args: book_text (str): Full book text used for most extraction sources. - chapters (Sequence[str]): Chapter-like text blocks used for spaCy and frequency counts. + chapters (Sequence[str]): Chapter-like text blocks used for frequency counts. config (EbookSearchConfig): Runtime phrase-tuning settings. - nlp (SpacyLanguage | None): Optional spaCy pipeline for entity and noun-chunk sources. metadata (Mapping[str, object] | None): Optional book metadata used as a candidate source. Returns: @@ -792,16 +703,6 @@ def extract_phrase_candidates_for_book( len(yake_candidates), (perf_counter() - yake_started_at) * 1000, ) - spacy_candidates: dict[str, PhraseCandidate] = {} - if nlp is not None: - spacy_started_at = perf_counter() - for chapter in chapters: - spacy_candidates = merge_candidate_sources(spacy_candidates, extract_spacy_candidates(chapter, nlp, config)) - logger.info( - "ebook_phrase_candidate_extract_spacy_complete candidates=%s duration_ms=%.1f", - len(spacy_candidates), - (perf_counter() - spacy_started_at) * 1000, - ) capitalized_started_at = perf_counter() capitalized = extract_capitalized_phrases(book_text, config) logger.info( @@ -811,7 +712,7 @@ def extract_phrase_candidates_for_book( ) metadata_candidates = extract_metadata_candidates(metadata, config) - candidates = merge_candidate_sources(raw, yake_candidates, spacy_candidates, capitalized, metadata_candidates) + candidates = merge_candidate_sources(raw, yake_candidates, capitalized, metadata_candidates) enriched_started_at = perf_counter() # Raw n-gram sizes were already counted per chapter above, so only enrich the remaining # (entity-length) sizes here instead of re-sliding every size over the whole book. @@ -831,12 +732,11 @@ def extract_phrase_candidates_for_book( : config.protected_phrase_max_candidates_per_book ] logger.info( - "ebook_phrase_candidate_extract_complete raw=%s yake=%s spacy=%s capitalized=%s metadata=%s " + "ebook_phrase_candidate_extract_complete raw=%s yake=%s capitalized=%s metadata=%s " "merged=%s filtered_too_short=%s filtered_too_rare=%s filtered_too_common=%s filtered_junk=%s " "min_uses=%s storable=%s limited=%s enrich_score_ms=%.1f duration_ms=%.1f", len(raw), len(yake_candidates), - len(spacy_candidates), len(capitalized), len(metadata_candidates), pre_filter_count, diff --git a/python/ebook_search/protected_phrases/models.py b/python/ebook_search/protected_phrases/models.py index 36f810f..ab0fa99 100644 --- a/python/ebook_search/protected_phrases/models.py +++ b/python/ebook_search/protected_phrases/models.py @@ -19,11 +19,8 @@ class PhraseCandidate: token_count (int): Number of normalized tokens in the phrase. source_raw_ngram (bool): Whether the raw n-gram extractor produced the phrase. source_yake (bool): Whether YAKE keyword extraction produced the phrase. - source_spacy_ner (bool): Whether spaCy named-entity recognition produced the phrase. - source_spacy_noun_chunk (bool): Whether spaCy noun chunking produced the phrase. source_capitalized (bool): Whether the capitalized-run extractor produced the phrase. source_metadata (bool): Whether book metadata produced the phrase. - spacy_label (str | None): spaCy entity label when NER produced the phrase. raw_count (int): Occurrences counted across the book text. chapter_count (int): Number of chapters containing the phrase. yake_score (float | None): Raw YAKE score when available; lower is better. @@ -36,11 +33,8 @@ class PhraseCandidate: token_count: int source_raw_ngram: bool = False source_yake: bool = False - source_spacy_ner: bool = False - source_spacy_noun_chunk: bool = False source_capitalized: bool = False source_metadata: bool = False - spacy_label: str | None = None raw_count: int = 0 chapter_count: int = 0 yake_score: float | None = None diff --git a/python/ebook_search/protected_phrases/store.py b/python/ebook_search/protected_phrases/store.py index 86aafd0..345da8d 100644 --- a/python/ebook_search/protected_phrases/store.py +++ b/python/ebook_search/protected_phrases/store.py @@ -295,11 +295,8 @@ def phrase_candidate_from_row(row: EbookCandidatePhrase) -> PhraseCandidate: token_count=row.token_count, source_raw_ngram=row.source_raw_ngram, source_yake=row.source_yake, - source_spacy_ner=row.source_spacy_ner, - source_spacy_noun_chunk=row.source_spacy_noun_chunk, source_capitalized=row.source_capitalized, source_metadata=row.source_metadata, - spacy_label=row.spacy_label, raw_count=row.raw_count, chapter_count=row.chapter_count, yake_score=row.yake_score, @@ -334,11 +331,8 @@ def candidate_row_values( "token_count": candidate.token_count, "source_raw_ngram": candidate.source_raw_ngram, "source_yake": candidate.source_yake, - "source_spacy_ner": candidate.source_spacy_ner, - "source_spacy_noun_chunk": candidate.source_spacy_noun_chunk, "source_capitalized": candidate.source_capitalized, "source_metadata": candidate.source_metadata, - "spacy_label": candidate.spacy_label, "raw_count": candidate.raw_count, "chapter_count": candidate.chapter_count, "yake_score": candidate.yake_score, @@ -460,11 +454,8 @@ def new_candidate_row(book_id: int, series_id: int | None, candidate: PhraseCand row.token_count = candidate.token_count row.source_raw_ngram = candidate.source_raw_ngram row.source_yake = candidate.source_yake - row.source_spacy_ner = candidate.source_spacy_ner - row.source_spacy_noun_chunk = candidate.source_spacy_noun_chunk row.source_capitalized = candidate.source_capitalized row.source_metadata = candidate.source_metadata - row.spacy_label = candidate.spacy_label row.raw_count = candidate.raw_count row.chapter_count = candidate.chapter_count row.yake_score = candidate.yake_score diff --git a/python/orm/richie/ebook.py b/python/orm/richie/ebook.py index e2da0f3..64ecbcb 100644 --- a/python/orm/richie/ebook.py +++ b/python/orm/richie/ebook.py @@ -167,11 +167,8 @@ class EbookCandidatePhrase(TableBase): token_count: Mapped[int] source_raw_ngram: Mapped[bool] = mapped_column(default=False) source_yake: Mapped[bool] = mapped_column(default=False) - source_spacy_ner: Mapped[bool] = mapped_column(default=False) - source_spacy_noun_chunk: Mapped[bool] = mapped_column(default=False) source_capitalized: Mapped[bool] = mapped_column(default=False) source_metadata: Mapped[bool] = mapped_column(default=False) - spacy_label: Mapped[str | None] raw_count: Mapped[int] = mapped_column(default=0) chapter_count: Mapped[int] = mapped_column(default=0) yake_score: Mapped[float | None]