feat(ebook): add phrase metadata tables for protected phrase matching
Introduce four ORM models and their Alembic migration to support phrase-based query matching in the ebook RAG engine: - EbookCandidatePhrase: high-recall phrase candidates extracted per book, with source flags (ngram/yake/spacy/capitalized/metadata), scoring, and LLM judge results. - EbookProtectedPhrase: phrases accepted by the LLM judge, with canonical id, importance, and nesting controls. - EbookPhraseAlias: normalized aliases mapping to protected phrases. - EbookChunkPhraseMention: precomputed phrase occurrences within chunks. Export the new models from python.orm.richie and add a JSON_DOCUMENT helper (JSON with JSONB postgres variant) for storing sample contexts.
This commit is contained in:
@@ -12,12 +12,16 @@ from python.orm.richie.contact import (
|
||||
RelationshipType,
|
||||
)
|
||||
from python.orm.richie.ebook import (
|
||||
EbookCandidatePhrase,
|
||||
EbookChapter,
|
||||
EbookChunk,
|
||||
EbookChunkEmbedding1024,
|
||||
EbookChunkEmbedding2560,
|
||||
EbookChunkEmbedding4096,
|
||||
EbookChunkPhraseMention,
|
||||
EbookEmbeddingModel,
|
||||
EbookPhraseAlias,
|
||||
EbookProtectedPhrase,
|
||||
EbookSource,
|
||||
)
|
||||
|
||||
@@ -28,12 +32,16 @@ __all__ = [
|
||||
"Contact",
|
||||
"ContactNeed",
|
||||
"ContactRelationship",
|
||||
"EbookCandidatePhrase",
|
||||
"EbookChapter",
|
||||
"EbookChunk",
|
||||
"EbookChunkEmbedding1024",
|
||||
"EbookChunkEmbedding2560",
|
||||
"EbookChunkEmbedding4096",
|
||||
"EbookChunkPhraseMention",
|
||||
"EbookEmbeddingModel",
|
||||
"EbookPhraseAlias",
|
||||
"EbookProtectedPhrase",
|
||||
"EbookSource",
|
||||
"Need",
|
||||
"RelationshipType",
|
||||
|
||||
+108
-2
@@ -5,11 +5,23 @@ from __future__ import annotations
|
||||
from datetime import datetime
|
||||
|
||||
from pgvector.sqlalchemy import Vector
|
||||
from sqlalchemy import BigInteger, Boolean, DateTime, ForeignKey, Index, String, UniqueConstraint
|
||||
from sqlalchemy import (
|
||||
JSON,
|
||||
BigInteger,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from python.orm.richie.base import TableBase, TableBaseBig
|
||||
|
||||
JSON_DOCUMENT = JSON().with_variant(JSONB, "postgresql")
|
||||
|
||||
|
||||
class EbookSource(TableBase):
|
||||
"""One indexed EPUB file."""
|
||||
@@ -94,7 +106,7 @@ class EbookEmbeddingModel(TableBase):
|
||||
|
||||
name: Mapped[str] = mapped_column(String, unique=True)
|
||||
dimension: Mapped[int]
|
||||
is_default: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
is_default: Mapped[bool] = mapped_column(default=False)
|
||||
|
||||
|
||||
class EbookChunkEmbedding1024(TableBaseBig):
|
||||
@@ -136,3 +148,97 @@ class EbookChunkEmbedding4096(TableBaseBig):
|
||||
chunk_id: Mapped[int] = mapped_column(ForeignKey("main.ebook_chunk.id", ondelete="CASCADE"))
|
||||
model_id: Mapped[int] = mapped_column(ForeignKey("main.ebook_embedding_model.id", ondelete="CASCADE"))
|
||||
embedding: Mapped[list[float]] = mapped_column(Vector(4096))
|
||||
|
||||
|
||||
class EbookCandidatePhrase(TableBase):
|
||||
"""A high-recall phrase candidate extracted from one book."""
|
||||
|
||||
__tablename__ = "candidate_phrases"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("book_id", "phrase_norm", name="uq_candidate_phrases_book_id_phrase_norm"),
|
||||
Index("candidate_phrases_book_score_idx", "book_id", "candidate_score"),
|
||||
Index("candidate_phrases_book_norm_idx", "book_id", "phrase_norm"),
|
||||
)
|
||||
|
||||
book_id: Mapped[int] = mapped_column(ForeignKey("main.ebook_source.id", ondelete="CASCADE"))
|
||||
series_id: Mapped[int | None]
|
||||
phrase_text: Mapped[str] = mapped_column(Text)
|
||||
phrase_norm: Mapped[str] = mapped_column(Text)
|
||||
token_count: Mapped[int]
|
||||
source_raw_ngram: Mapped[bool] = mapped_column(default=False)
|
||||
source_yake: Mapped[bool] = mapped_column(default=False)
|
||||
source_spacy_ner: Mapped[bool] = mapped_column(default=False)
|
||||
source_spacy_noun_chunk: Mapped[bool] = mapped_column(default=False)
|
||||
source_capitalized: Mapped[bool] = mapped_column(default=False)
|
||||
source_metadata: Mapped[bool] = mapped_column(default=False)
|
||||
spacy_label: Mapped[str | None]
|
||||
raw_count: Mapped[int] = mapped_column(default=0)
|
||||
chapter_count: Mapped[int] = mapped_column(default=0)
|
||||
yake_score: Mapped[float | None]
|
||||
candidate_score: Mapped[float] = mapped_column(default=0.0)
|
||||
sample_contexts: Mapped[list[str] | None] = mapped_column(JSON_DOCUMENT)
|
||||
llm_judged: Mapped[bool] = mapped_column(default=False)
|
||||
llm_keep: Mapped[bool | None]
|
||||
llm_confidence: Mapped[float | None]
|
||||
llm_category: Mapped[str | None]
|
||||
llm_reason: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
|
||||
class EbookProtectedPhrase(TableBase):
|
||||
"""A phrase accepted by the LLM judge for protected query matching."""
|
||||
|
||||
__tablename__ = "protected_phrases"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("book_id", "phrase_norm", name="uq_protected_phrases_book_id_phrase_norm"),
|
||||
Index("protected_phrases_norm_idx", "phrase_norm"),
|
||||
Index("protected_phrases_book_norm_idx", "book_id", "phrase_norm"),
|
||||
Index("protected_phrases_series_norm_idx", "series_id", "phrase_norm"),
|
||||
)
|
||||
|
||||
book_id: Mapped[int | None] = mapped_column(ForeignKey("main.ebook_source.id", ondelete="CASCADE"))
|
||||
series_id: Mapped[int | None]
|
||||
phrase_text: Mapped[str] = mapped_column(Text)
|
||||
phrase_norm: Mapped[str] = mapped_column(Text)
|
||||
canonical_id: Mapped[str]
|
||||
phrase_type: Mapped[str | None]
|
||||
token_count: Mapped[int]
|
||||
confidence: Mapped[float]
|
||||
importance: Mapped[float] = mapped_column(default=0.5)
|
||||
allow_nested: Mapped[bool] = mapped_column(default=False)
|
||||
suppress_children: Mapped[bool] = mapped_column(default=True)
|
||||
source_candidate_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("main.candidate_phrases.id", ondelete="SET NULL")
|
||||
)
|
||||
|
||||
|
||||
class EbookPhraseAlias(TableBase):
|
||||
"""A normalized alias that maps to a protected phrase."""
|
||||
|
||||
__tablename__ = "phrase_aliases"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("phrase_id", "alias_norm", name="uq_phrase_aliases_phrase_id_alias_norm"),
|
||||
Index("phrase_aliases_norm_idx", "alias_norm"),
|
||||
)
|
||||
|
||||
phrase_id: Mapped[int] = mapped_column(ForeignKey("main.protected_phrases.id", ondelete="CASCADE"))
|
||||
alias_text: Mapped[str] = mapped_column(Text)
|
||||
alias_norm: Mapped[str] = mapped_column(Text)
|
||||
confidence: Mapped[float] = mapped_column(default=1.0)
|
||||
|
||||
|
||||
class EbookChunkPhraseMention(TableBase):
|
||||
"""A precomputed occurrence of a protected phrase inside one chunk."""
|
||||
|
||||
__tablename__ = "chunk_phrase_mentions"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("chunk_id", "phrase_id", "start_char", name="uq_chunk_phrase_mentions_chunk_phrase_start"),
|
||||
Index("chunk_phrase_mentions_phrase_idx", "phrase_id"),
|
||||
Index("chunk_phrase_mentions_chunk_idx", "chunk_id"),
|
||||
)
|
||||
|
||||
chunk_id: Mapped[int] = mapped_column(ForeignKey("main.ebook_chunk.id", ondelete="CASCADE"))
|
||||
phrase_id: Mapped[int] = mapped_column(ForeignKey("main.protected_phrases.id", ondelete="CASCADE"))
|
||||
book_id: Mapped[int | None] = mapped_column(ForeignKey("main.ebook_source.id", ondelete="CASCADE"))
|
||||
series_id: Mapped[int | None]
|
||||
start_char: Mapped[int]
|
||||
end_char: Mapped[int | None]
|
||||
|
||||
Reference in New Issue
Block a user