feat(ebook): add phrase metadata tables for protected phrase matching
Introduce four ORM models and their Alembic migration to support phrase-based query matching in the ebook RAG engine: - EbookCandidatePhrase: high-recall phrase candidates extracted per book, with source flags (ngram/yake/spacy/capitalized/metadata), scoring, and LLM judge results. - EbookProtectedPhrase: phrases accepted by the LLM judge, with canonical id, importance, and nesting controls. - EbookPhraseAlias: normalized aliases mapping to protected phrases. - EbookChunkPhraseMention: precomputed phrase occurrences within chunks. Export the new models from python.orm.richie and add a JSON_DOCUMENT helper (JSON with JSONB postgres variant) for storing sample contexts.
This commit is contained in:
+206
@@ -0,0 +1,206 @@
|
||||
"""adding Phrase metadata tables.
|
||||
|
||||
Revision ID: dddee09eddcc
|
||||
Revises: 96d72c748c24
|
||||
Create Date: 2026-06-29 00:49:07.344159
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "dddee09eddcc"
|
||||
down_revision: str | None = "96d72c748c24"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"candidate_phrases",
|
||||
sa.Column("book_id", sa.Integer(), nullable=False),
|
||||
sa.Column("series_id", sa.Integer(), nullable=True),
|
||||
sa.Column("phrase_text", sa.Text(), nullable=False),
|
||||
sa.Column("phrase_norm", sa.Text(), nullable=False),
|
||||
sa.Column("token_count", sa.Integer(), nullable=False),
|
||||
sa.Column("source_raw_ngram", sa.Boolean(), nullable=False),
|
||||
sa.Column("source_yake", sa.Boolean(), nullable=False),
|
||||
sa.Column("source_spacy_ner", sa.Boolean(), nullable=False),
|
||||
sa.Column("source_spacy_noun_chunk", sa.Boolean(), nullable=False),
|
||||
sa.Column("source_capitalized", sa.Boolean(), nullable=False),
|
||||
sa.Column("source_metadata", sa.Boolean(), nullable=False),
|
||||
sa.Column("spacy_label", sa.String(), nullable=True),
|
||||
sa.Column("raw_count", sa.Integer(), nullable=False),
|
||||
sa.Column("chapter_count", sa.Integer(), nullable=False),
|
||||
sa.Column("yake_score", sa.Float(), nullable=True),
|
||||
sa.Column("candidate_score", sa.Float(), nullable=False),
|
||||
sa.Column(
|
||||
"sample_contexts",
|
||||
sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), "postgresql"),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column("llm_judged", sa.Boolean(), nullable=False),
|
||||
sa.Column("llm_keep", sa.Boolean(), nullable=True),
|
||||
sa.Column("llm_confidence", sa.Float(), nullable=True),
|
||||
sa.Column("llm_category", sa.String(), nullable=True),
|
||||
sa.Column("llm_reason", sa.Text(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["book_id"],
|
||||
[f"{schema}.ebook_source.id"],
|
||||
name=op.f("fk_candidate_phrases_book_id_ebook_source"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_candidate_phrases")),
|
||||
sa.UniqueConstraint("book_id", "phrase_norm", name="uq_candidate_phrases_book_id_phrase_norm"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"candidate_phrases_book_norm_idx", "candidate_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
|
||||
)
|
||||
op.create_index(
|
||||
"candidate_phrases_book_score_idx",
|
||||
"candidate_phrases",
|
||||
["book_id", "candidate_score"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"protected_phrases",
|
||||
sa.Column("book_id", sa.Integer(), nullable=True),
|
||||
sa.Column("series_id", sa.Integer(), nullable=True),
|
||||
sa.Column("phrase_text", sa.Text(), nullable=False),
|
||||
sa.Column("phrase_norm", sa.Text(), nullable=False),
|
||||
sa.Column("canonical_id", sa.String(), nullable=False),
|
||||
sa.Column("phrase_type", sa.String(), nullable=True),
|
||||
sa.Column("token_count", sa.Integer(), nullable=False),
|
||||
sa.Column("confidence", sa.Float(), nullable=False),
|
||||
sa.Column("importance", sa.Float(), nullable=False),
|
||||
sa.Column("allow_nested", sa.Boolean(), nullable=False),
|
||||
sa.Column("suppress_children", sa.Boolean(), nullable=False),
|
||||
sa.Column("source_candidate_id", sa.Integer(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["book_id"],
|
||||
[f"{schema}.ebook_source.id"],
|
||||
name=op.f("fk_protected_phrases_book_id_ebook_source"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_candidate_id"],
|
||||
[f"{schema}.candidate_phrases.id"],
|
||||
name=op.f("fk_protected_phrases_source_candidate_id_candidate_phrases"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_protected_phrases")),
|
||||
sa.UniqueConstraint("book_id", "phrase_norm", name="uq_protected_phrases_book_id_phrase_norm"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"protected_phrases_book_norm_idx", "protected_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
|
||||
)
|
||||
op.create_index("protected_phrases_norm_idx", "protected_phrases", ["phrase_norm"], unique=False, schema=schema)
|
||||
op.create_index(
|
||||
"protected_phrases_series_norm_idx",
|
||||
"protected_phrases",
|
||||
["series_id", "phrase_norm"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"chunk_phrase_mentions",
|
||||
sa.Column("chunk_id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("phrase_id", sa.Integer(), nullable=False),
|
||||
sa.Column("book_id", sa.Integer(), nullable=True),
|
||||
sa.Column("series_id", sa.Integer(), nullable=True),
|
||||
sa.Column("start_char", sa.Integer(), nullable=False),
|
||||
sa.Column("end_char", sa.Integer(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["book_id"],
|
||||
[f"{schema}.ebook_source.id"],
|
||||
name=op.f("fk_chunk_phrase_mentions_book_id_ebook_source"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["chunk_id"],
|
||||
[f"{schema}.ebook_chunk.id"],
|
||||
name=op.f("fk_chunk_phrase_mentions_chunk_id_ebook_chunk"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["phrase_id"],
|
||||
[f"{schema}.protected_phrases.id"],
|
||||
name=op.f("fk_chunk_phrase_mentions_phrase_id_protected_phrases"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_chunk_phrase_mentions")),
|
||||
sa.UniqueConstraint("chunk_id", "phrase_id", "start_char", name="uq_chunk_phrase_mentions_chunk_phrase_start"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"chunk_phrase_mentions_chunk_idx", "chunk_phrase_mentions", ["chunk_id"], unique=False, schema=schema
|
||||
)
|
||||
op.create_index(
|
||||
"chunk_phrase_mentions_phrase_idx", "chunk_phrase_mentions", ["phrase_id"], unique=False, schema=schema
|
||||
)
|
||||
op.create_table(
|
||||
"phrase_aliases",
|
||||
sa.Column("phrase_id", sa.Integer(), nullable=False),
|
||||
sa.Column("alias_text", sa.Text(), nullable=False),
|
||||
sa.Column("alias_norm", sa.Text(), nullable=False),
|
||||
sa.Column("confidence", sa.Float(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["phrase_id"],
|
||||
[f"{schema}.protected_phrases.id"],
|
||||
name=op.f("fk_phrase_aliases_phrase_id_protected_phrases"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_phrase_aliases")),
|
||||
sa.UniqueConstraint("phrase_id", "alias_norm", name="uq_phrase_aliases_phrase_id_alias_norm"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index("phrase_aliases_norm_idx", "phrase_aliases", ["alias_norm"], unique=False, schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index("phrase_aliases_norm_idx", table_name="phrase_aliases", schema=schema)
|
||||
op.drop_table("phrase_aliases", schema=schema)
|
||||
op.drop_index("chunk_phrase_mentions_phrase_idx", table_name="chunk_phrase_mentions", schema=schema)
|
||||
op.drop_index("chunk_phrase_mentions_chunk_idx", table_name="chunk_phrase_mentions", schema=schema)
|
||||
op.drop_table("chunk_phrase_mentions", schema=schema)
|
||||
op.drop_index("protected_phrases_series_norm_idx", table_name="protected_phrases", schema=schema)
|
||||
op.drop_index("protected_phrases_norm_idx", table_name="protected_phrases", schema=schema)
|
||||
op.drop_index("protected_phrases_book_norm_idx", table_name="protected_phrases", schema=schema)
|
||||
op.drop_table("protected_phrases", schema=schema)
|
||||
op.drop_index("candidate_phrases_book_score_idx", table_name="candidate_phrases", schema=schema)
|
||||
op.drop_index("candidate_phrases_book_norm_idx", table_name="candidate_phrases", schema=schema)
|
||||
op.drop_table("candidate_phrases", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
Reference in New Issue
Block a user