setting up ORM
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,60 @@
|
||||
"""adding FailedIngestion.
|
||||
|
||||
Revision ID: 2f43120e3ffc
|
||||
Revises: f99be864fe69
|
||||
Create Date: 2026-03-24 23:46:17.277897
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from pipelines.orm import DataScienceDevBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "2f43120e3ffc"
|
||||
down_revision: str | None = "f99be864fe69"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = DataScienceDevBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"failed_ingestion",
|
||||
sa.Column("raw_line", sa.Text(), nullable=False),
|
||||
sa.Column("error", sa.Text(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_failed_ingestion")),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("failed_ingestion", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
File diff suppressed because it is too large
Load Diff
+1391
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,79 @@
|
||||
"""Attach all partition tables to the posts parent table.
|
||||
|
||||
Alembic autogenerate creates partition tables as standalone tables but does not
|
||||
emit the ALTER TABLE ... ATTACH PARTITION statements needed for PostgreSQL to
|
||||
route inserts to the correct partition.
|
||||
|
||||
Revision ID: a1b2c3d4e5f6
|
||||
Revises: 605b1794838f
|
||||
Create Date: 2026-03-25 10:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import text
|
||||
|
||||
from pipelines.orm import DataScienceDevBase
|
||||
from pipelines.orm.data_science_dev.posts.partitions import (
|
||||
PARTITION_END_YEAR,
|
||||
PARTITION_START_YEAR,
|
||||
iso_weeks_in_year,
|
||||
week_bounds,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "a1b2c3d4e5f6"
|
||||
down_revision: str | None = "605b1794838f"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = DataScienceDevBase.schema_name
|
||||
|
||||
ALREADY_ATTACHED_QUERY = text("""
|
||||
SELECT inhrelid::regclass::text
|
||||
FROM pg_inherits
|
||||
WHERE inhparent = :parent::regclass
|
||||
""")
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Attach all weekly partition tables to the posts parent table."""
|
||||
connection = op.get_bind()
|
||||
already_attached = {
|
||||
row[0]
|
||||
for row in connection.execute(
|
||||
ALREADY_ATTACHED_QUERY, {"parent": f"{schema}.posts"}
|
||||
)
|
||||
}
|
||||
|
||||
for year in range(PARTITION_START_YEAR, PARTITION_END_YEAR + 1):
|
||||
for week in range(1, iso_weeks_in_year(year) + 1):
|
||||
table_name = f"posts_{year}_{week:02d}"
|
||||
qualified_name = f"{schema}.{table_name}"
|
||||
if qualified_name in already_attached:
|
||||
continue
|
||||
start, end = week_bounds(year, week)
|
||||
start_str = start.strftime("%Y-%m-%d %H:%M:%S")
|
||||
end_str = end.strftime("%Y-%m-%d %H:%M:%S")
|
||||
op.execute(
|
||||
f"ALTER TABLE {schema}.posts "
|
||||
f"ATTACH PARTITION {qualified_name} "
|
||||
f"FOR VALUES FROM ('{start_str}') TO ('{end_str}')"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Detach all weekly partition tables from the posts parent table."""
|
||||
for year in range(PARTITION_START_YEAR, PARTITION_END_YEAR + 1):
|
||||
for week in range(1, iso_weeks_in_year(year) + 1):
|
||||
table_name = f"posts_{year}_{week:02d}"
|
||||
op.execute(
|
||||
f"ALTER TABLE {schema}.posts DETACH PARTITION {schema}.{table_name}"
|
||||
)
|
||||
@@ -0,0 +1,229 @@
|
||||
"""adding congress data.
|
||||
|
||||
Revision ID: 83bfc8af92d8
|
||||
Revises: a1b2c3d4e5f6
|
||||
Create Date: 2026-03-27 10:43:02.324510
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from pipelines.orm import DataScienceDevBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "83bfc8af92d8"
|
||||
down_revision: str | None = "a1b2c3d4e5f6"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = DataScienceDevBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"bill",
|
||||
sa.Column("congress", sa.Integer(), nullable=False),
|
||||
sa.Column("bill_type", sa.String(), nullable=False),
|
||||
sa.Column("number", sa.Integer(), nullable=False),
|
||||
sa.Column("title", sa.String(), nullable=True),
|
||||
sa.Column("title_short", sa.String(), nullable=True),
|
||||
sa.Column("official_title", sa.String(), nullable=True),
|
||||
sa.Column("status", sa.String(), nullable=True),
|
||||
sa.Column("status_at", sa.Date(), nullable=True),
|
||||
sa.Column("sponsor_bioguide_id", sa.String(), nullable=True),
|
||||
sa.Column("subjects_top_term", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_bill")),
|
||||
sa.UniqueConstraint(
|
||||
"congress", "bill_type", "number", name="uq_bill_congress_type_number"
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_bill_congress", "bill", ["congress"], unique=False, schema=schema
|
||||
)
|
||||
op.create_table(
|
||||
"legislator",
|
||||
sa.Column("bioguide_id", sa.Text(), nullable=False),
|
||||
sa.Column("thomas_id", sa.String(), nullable=True),
|
||||
sa.Column("lis_id", sa.String(), nullable=True),
|
||||
sa.Column("govtrack_id", sa.Integer(), nullable=True),
|
||||
sa.Column("opensecrets_id", sa.String(), nullable=True),
|
||||
sa.Column("fec_ids", sa.String(), nullable=True),
|
||||
sa.Column("first_name", sa.String(), nullable=False),
|
||||
sa.Column("last_name", sa.String(), nullable=False),
|
||||
sa.Column("official_full_name", sa.String(), nullable=True),
|
||||
sa.Column("nickname", sa.String(), nullable=True),
|
||||
sa.Column("birthday", sa.Date(), nullable=True),
|
||||
sa.Column("gender", sa.String(), nullable=True),
|
||||
sa.Column("current_party", sa.String(), nullable=True),
|
||||
sa.Column("current_state", sa.String(), nullable=True),
|
||||
sa.Column("current_district", sa.Integer(), nullable=True),
|
||||
sa.Column("current_chamber", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_legislator")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_legislator_bioguide_id"),
|
||||
"legislator",
|
||||
["bioguide_id"],
|
||||
unique=True,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"bill_text",
|
||||
sa.Column("bill_id", sa.Integer(), nullable=False),
|
||||
sa.Column("version_code", sa.String(), nullable=False),
|
||||
sa.Column("version_name", sa.String(), nullable=True),
|
||||
sa.Column("text_content", sa.String(), nullable=True),
|
||||
sa.Column("date", sa.Date(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["bill_id"],
|
||||
[f"{schema}.bill.id"],
|
||||
name=op.f("fk_bill_text_bill_id_bill"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_bill_text")),
|
||||
sa.UniqueConstraint(
|
||||
"bill_id", "version_code", name="uq_bill_text_bill_id_version_code"
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"vote",
|
||||
sa.Column("congress", sa.Integer(), nullable=False),
|
||||
sa.Column("chamber", sa.String(), nullable=False),
|
||||
sa.Column("session", sa.Integer(), nullable=False),
|
||||
sa.Column("number", sa.Integer(), nullable=False),
|
||||
sa.Column("vote_type", sa.String(), nullable=True),
|
||||
sa.Column("question", sa.String(), nullable=True),
|
||||
sa.Column("result", sa.String(), nullable=True),
|
||||
sa.Column("result_text", sa.String(), nullable=True),
|
||||
sa.Column("vote_date", sa.Date(), nullable=False),
|
||||
sa.Column("yea_count", sa.Integer(), nullable=True),
|
||||
sa.Column("nay_count", sa.Integer(), nullable=True),
|
||||
sa.Column("not_voting_count", sa.Integer(), nullable=True),
|
||||
sa.Column("present_count", sa.Integer(), nullable=True),
|
||||
sa.Column("bill_id", sa.Integer(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["bill_id"], [f"{schema}.bill.id"], name=op.f("fk_vote_bill_id_bill")
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_vote")),
|
||||
sa.UniqueConstraint(
|
||||
"congress",
|
||||
"chamber",
|
||||
"session",
|
||||
"number",
|
||||
name="uq_vote_congress_chamber_session_number",
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_vote_congress_chamber",
|
||||
"vote",
|
||||
["congress", "chamber"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index("ix_vote_date", "vote", ["vote_date"], unique=False, schema=schema)
|
||||
op.create_table(
|
||||
"vote_record",
|
||||
sa.Column("vote_id", sa.Integer(), nullable=False),
|
||||
sa.Column("legislator_id", sa.Integer(), nullable=False),
|
||||
sa.Column("position", sa.String(), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["legislator_id"],
|
||||
[f"{schema}.legislator.id"],
|
||||
name=op.f("fk_vote_record_legislator_id_legislator"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["vote_id"],
|
||||
[f"{schema}.vote.id"],
|
||||
name=op.f("fk_vote_record_vote_id_vote"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint(
|
||||
"vote_id", "legislator_id", name=op.f("pk_vote_record")
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("vote_record", schema=schema)
|
||||
op.drop_index("ix_vote_date", table_name="vote", schema=schema)
|
||||
op.drop_index("ix_vote_congress_chamber", table_name="vote", schema=schema)
|
||||
op.drop_table("vote", schema=schema)
|
||||
op.drop_table("bill_text", schema=schema)
|
||||
op.drop_index(
|
||||
op.f("ix_legislator_bioguide_id"), table_name="legislator", schema=schema
|
||||
)
|
||||
op.drop_table("legislator", schema=schema)
|
||||
op.drop_index("ix_bill_congress", table_name="bill", schema=schema)
|
||||
op.drop_table("bill", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
"""adding LegislatorSocialMedia.
|
||||
|
||||
Revision ID: 5cd7eee3549d
|
||||
Revises: 83bfc8af92d8
|
||||
Create Date: 2026-03-29 11:53:44.224799
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from pipelines.orm import DataScienceDevBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "5cd7eee3549d"
|
||||
down_revision: str | None = "83bfc8af92d8"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = DataScienceDevBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"legislator_social_media",
|
||||
sa.Column("legislator_id", sa.Integer(), nullable=False),
|
||||
sa.Column("platform", sa.String(), nullable=False),
|
||||
sa.Column("account_name", sa.String(), nullable=False),
|
||||
sa.Column("url", sa.String(), nullable=True),
|
||||
sa.Column("source", sa.String(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["legislator_id"],
|
||||
[f"{schema}.legislator.id"],
|
||||
name=op.f("fk_legislator_social_media_legislator_id_legislator"),
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_legislator_social_media")),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("legislator_social_media", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
+245
@@ -0,0 +1,245 @@
|
||||
"""adding LegislatorScore and BillTopic.
|
||||
|
||||
Revision ID: ef4bc5411176
|
||||
Revises: 5cd7eee3549d
|
||||
Create Date: 2026-04-21 11:35:18.977213
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
from pipelines.orm import DataScienceDevBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "ef4bc5411176"
|
||||
down_revision: str | None = "5cd7eee3549d"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = DataScienceDevBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"bill_topic",
|
||||
sa.Column("bill_id", sa.Integer(), nullable=False),
|
||||
sa.Column("topic", sa.String(), nullable=False),
|
||||
sa.Column(
|
||||
"support_position",
|
||||
sa.Enum("for", "against", name="bill_topic_position", native_enum=False),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["bill_id"],
|
||||
[f"{schema}.bill.id"],
|
||||
name=op.f("fk_bill_topic_bill_id_bill"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_bill_topic")),
|
||||
sa.UniqueConstraint(
|
||||
"bill_id",
|
||||
"topic",
|
||||
"support_position",
|
||||
name="uq_bill_topic_bill_id_topic_support_position",
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_bill_topic_topic", "bill_topic", ["topic"], unique=False, schema=schema
|
||||
)
|
||||
op.create_table(
|
||||
"legislator_score",
|
||||
sa.Column("legislator_id", sa.Integer(), nullable=False),
|
||||
sa.Column("year", sa.Integer(), nullable=False),
|
||||
sa.Column("topic", sa.String(), nullable=False),
|
||||
sa.Column("score", sa.Float(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["legislator_id"],
|
||||
[f"{schema}.legislator.id"],
|
||||
name=op.f("fk_legislator_score_legislator_id_legislator"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_legislator_score")),
|
||||
sa.UniqueConstraint(
|
||||
"legislator_id",
|
||||
"year",
|
||||
"topic",
|
||||
name="uq_legislator_score_legislator_id_year_topic",
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_legislator_score_legislator_id"),
|
||||
"legislator_score",
|
||||
["legislator_id"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_legislator_score_year_topic",
|
||||
"legislator_score",
|
||||
["year", "topic"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"legislator_bill_score",
|
||||
sa.Column("bill_id", sa.Integer(), nullable=False),
|
||||
sa.Column("bill_topic_id", sa.Integer(), nullable=False),
|
||||
sa.Column("legislator_id", sa.Integer(), nullable=False),
|
||||
sa.Column("year", sa.Integer(), nullable=False),
|
||||
sa.Column("topic", sa.String(), nullable=False),
|
||||
sa.Column("score", sa.Float(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["bill_id"],
|
||||
[f"{schema}.bill.id"],
|
||||
name=op.f("fk_legislator_bill_score_bill_id_bill"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["bill_topic_id"],
|
||||
[f"{schema}.bill_topic.id"],
|
||||
name=op.f("fk_legislator_bill_score_bill_topic_id_bill_topic"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["legislator_id"],
|
||||
[f"{schema}.legislator.id"],
|
||||
name=op.f("fk_legislator_bill_score_legislator_id_legislator"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_legislator_bill_score")),
|
||||
sa.UniqueConstraint(
|
||||
"bill_topic_id",
|
||||
"legislator_id",
|
||||
"year",
|
||||
name="uq_legislator_bill_score_bill_topic_id_legislator_id_year",
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_legislator_bill_score_bill_id"),
|
||||
"legislator_bill_score",
|
||||
["bill_id"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_legislator_bill_score_bill_topic_id"),
|
||||
"legislator_bill_score",
|
||||
["bill_topic_id"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_legislator_bill_score_legislator_id"),
|
||||
"legislator_bill_score",
|
||||
["legislator_id"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_legislator_bill_score_year_topic",
|
||||
"legislator_bill_score",
|
||||
["year", "topic"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.add_column(
|
||||
"bill",
|
||||
sa.Column("score_processed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
schema=schema,
|
||||
)
|
||||
op.add_column(
|
||||
"bill_text", sa.Column("summary", sa.String(), nullable=True), schema=schema
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_column("bill_text", "summary", schema=schema)
|
||||
op.drop_column("bill", "score_processed_at", schema=schema)
|
||||
op.drop_index(
|
||||
"ix_legislator_bill_score_year_topic",
|
||||
table_name="legislator_bill_score",
|
||||
schema=schema,
|
||||
)
|
||||
op.drop_index(
|
||||
op.f("ix_legislator_bill_score_legislator_id"),
|
||||
table_name="legislator_bill_score",
|
||||
schema=schema,
|
||||
)
|
||||
op.drop_index(
|
||||
op.f("ix_legislator_bill_score_bill_topic_id"),
|
||||
table_name="legislator_bill_score",
|
||||
schema=schema,
|
||||
)
|
||||
op.drop_index(
|
||||
op.f("ix_legislator_bill_score_bill_id"),
|
||||
table_name="legislator_bill_score",
|
||||
schema=schema,
|
||||
)
|
||||
op.drop_table("legislator_bill_score", schema=schema)
|
||||
op.drop_index(
|
||||
"ix_legislator_score_year_topic", table_name="legislator_score", schema=schema
|
||||
)
|
||||
op.drop_index(
|
||||
op.f("ix_legislator_score_legislator_id"),
|
||||
table_name="legislator_score",
|
||||
schema=schema,
|
||||
)
|
||||
op.drop_table("legislator_score", schema=schema)
|
||||
op.drop_index("ix_bill_topic_topic", table_name="bill_topic", schema=schema)
|
||||
op.drop_table("bill_topic", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
"""removed LegislatorBillScore.
|
||||
|
||||
Revision ID: b63ed11d6775
|
||||
Revises: 7d15f9b7c8a2
|
||||
Create Date: 2026-04-21 22:46:48.058542
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
from alembic import op
|
||||
from pipelines.orm import DataScienceDevBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "b63ed11d6775"
|
||||
down_revision: str | None = "7d15f9b7c8a2"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = DataScienceDevBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index(
|
||||
op.f("ix_legislator_bill_score_bill_id"),
|
||||
table_name="legislator_bill_score",
|
||||
schema=schema,
|
||||
)
|
||||
op.drop_index(
|
||||
op.f("ix_legislator_bill_score_bill_topic_id"),
|
||||
table_name="legislator_bill_score",
|
||||
schema=schema,
|
||||
)
|
||||
op.drop_index(
|
||||
op.f("ix_legislator_bill_score_legislator_id"),
|
||||
table_name="legislator_bill_score",
|
||||
schema=schema,
|
||||
)
|
||||
op.drop_index(
|
||||
op.f("ix_legislator_bill_score_year_topic"),
|
||||
table_name="legislator_bill_score",
|
||||
schema=schema,
|
||||
)
|
||||
op.drop_table("legislator_bill_score", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"legislator_bill_score",
|
||||
sa.Column("bill_id", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("bill_topic_id", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("legislator_id", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("year", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("topic", sa.VARCHAR(), autoincrement=False, nullable=False),
|
||||
sa.Column(
|
||||
"score",
|
||||
sa.DOUBLE_PRECISION(precision=53),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["bill_id"],
|
||||
[f"{schema}.bill.id"],
|
||||
name=op.f("fk_legislator_bill_score_bill_id_bill"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["bill_topic_id"],
|
||||
[f"{schema}.bill_topic.id"],
|
||||
name=op.f("fk_legislator_bill_score_bill_topic_id_bill_topic"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["legislator_id"],
|
||||
[f"{schema}.legislator.id"],
|
||||
name=op.f("fk_legislator_bill_score_legislator_id_legislator"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_legislator_bill_score")),
|
||||
sa.UniqueConstraint(
|
||||
"bill_topic_id",
|
||||
"legislator_id",
|
||||
"year",
|
||||
name=op.f("uq_legislator_bill_score_bill_topic_id_legislator_id_year"),
|
||||
postgresql_include=[],
|
||||
postgresql_nulls_not_distinct=False,
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_legislator_bill_score_year_topic"),
|
||||
"legislator_bill_score",
|
||||
["year", "topic"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_legislator_bill_score_legislator_id"),
|
||||
"legislator_bill_score",
|
||||
["legislator_id"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_legislator_bill_score_bill_topic_id"),
|
||||
"legislator_bill_score",
|
||||
["bill_topic_id"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_legislator_bill_score_bill_id"),
|
||||
"legislator_bill_score",
|
||||
["bill_id"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
"""add bill_text summarization metadata.
|
||||
|
||||
Revision ID: 7d15f9b7c8a2
|
||||
Revises: ef4bc5411176
|
||||
Create Date: 2026-04-22 00:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
from pipelines.orm import DataScienceDevBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "7d15f9b7c8a2"
|
||||
down_revision: str | None = "ef4bc5411176"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = DataScienceDevBase.schema_name
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
op.add_column(
|
||||
"bill_text",
|
||||
sa.Column("summarization_model", sa.String(), nullable=True),
|
||||
schema=schema,
|
||||
)
|
||||
op.add_column(
|
||||
"bill_text",
|
||||
sa.Column("summarization_user_prompt_version", sa.String(), nullable=True),
|
||||
schema=schema,
|
||||
)
|
||||
op.add_column(
|
||||
"bill_text",
|
||||
sa.Column("summarization_system_prompt_version", sa.String(), nullable=True),
|
||||
schema=schema,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
op.drop_column(
|
||||
"bill_text", "summarization_system_prompt_version", schema=schema
|
||||
)
|
||||
op.drop_column("bill_text", "summarization_user_prompt_version", schema=schema)
|
||||
op.drop_column("bill_text", "summarization_model", schema=schema)
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
"""adding LegislatorScoreFake.
|
||||
|
||||
Revision ID: 06f833813bd7
|
||||
Revises: b63ed11d6775
|
||||
Create Date: 2026-04-22 18:41:07.484609
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
from pipelines.orm import DataScienceDevBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "06f833813bd7"
|
||||
down_revision: str | None = "b63ed11d6775"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = DataScienceDevBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"legislator_score_fake",
|
||||
sa.Column("legislator_id", sa.Integer(), nullable=False),
|
||||
sa.Column("year", sa.Integer(), nullable=False),
|
||||
sa.Column("topic", sa.String(), nullable=False),
|
||||
sa.Column("score", sa.Float(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["legislator_id"],
|
||||
[f"{schema}.legislator.id"],
|
||||
name=op.f("fk_legislator_score_fake_legislator_id_legislator"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_legislator_score_fake")),
|
||||
sa.UniqueConstraint(
|
||||
"legislator_id",
|
||||
"year",
|
||||
"topic",
|
||||
name="uq_legislator_score_fake_legislator_id_year_topic",
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_legislator_score_fake_legislator_id"),
|
||||
"legislator_score_fake",
|
||||
["legislator_id"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_legislator_score_fake_year_topic",
|
||||
"legislator_score_fake",
|
||||
["year", "topic"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index(
|
||||
"ix_legislator_score_fake_year_topic",
|
||||
table_name="legislator_score_fake",
|
||||
schema=schema,
|
||||
)
|
||||
op.drop_index(
|
||||
op.f("ix_legislator_score_fake_legislator_id"),
|
||||
table_name="legislator_score_fake",
|
||||
schema=schema,
|
||||
)
|
||||
op.drop_table("legislator_score_fake", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
@@ -0,0 +1,64 @@
|
||||
"""add vote.bill_text_id linkage.
|
||||
|
||||
Revision ID: 9c7d4a2e1b10
|
||||
Revises: 06f833813bd7
|
||||
Create Date: 2026-04-23 00:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
from pipelines.orm import DataScienceDevBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "9c7d4a2e1b10"
|
||||
down_revision: str | None = "06f833813bd7"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = DataScienceDevBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
op.add_column(
|
||||
"vote",
|
||||
sa.Column("bill_text_id", sa.Integer(), nullable=True),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_vote_bill_text_id",
|
||||
"vote",
|
||||
["bill_text_id"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_foreign_key(
|
||||
"fk_vote_bill_text_id_bill_text",
|
||||
"vote",
|
||||
"bill_text",
|
||||
["bill_text_id"],
|
||||
["id"],
|
||||
source_schema=schema,
|
||||
referent_schema=schema,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
op.drop_constraint(
|
||||
"fk_vote_bill_text_id_bill_text",
|
||||
"vote",
|
||||
schema=schema,
|
||||
type_="foreignkey",
|
||||
)
|
||||
op.drop_index("ix_vote_bill_text_id", table_name="vote", schema=schema)
|
||||
op.drop_column("vote", "bill_text_id", schema=schema)
|
||||
+844
@@ -0,0 +1,844 @@
|
||||
"""canonical vote context v3.
|
||||
|
||||
Revision ID: 1f8c0e7a9d21
|
||||
Revises: 9c7d4a2e1b10
|
||||
Create Date: 2026-04-25 00:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
from alembic import op
|
||||
from pipelines.orm import DataScienceDevBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
|
||||
revision: str = "1f8c0e7a9d21"
|
||||
down_revision: str | None = "9c7d4a2e1b10"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = DataScienceDevBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
op.create_table(
|
||||
"ingest_run",
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("git_sha", sa.String(), nullable=True),
|
||||
sa.Column("classifier_version", sa.String(), nullable=True),
|
||||
sa.Column("source_snapshot_label", sa.String(), nullable=True),
|
||||
sa.Column("status", sa.String(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_ingest_run")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"source_artifact",
|
||||
sa.Column("source_kind", sa.String(), nullable=False),
|
||||
sa.Column("congress", sa.Integer(), nullable=False),
|
||||
sa.Column("chamber", sa.String(), nullable=True),
|
||||
sa.Column("local_path", sa.String(), nullable=False),
|
||||
sa.Column("source_url", sa.String(), nullable=True),
|
||||
sa.Column("sha256", sa.String(), nullable=False),
|
||||
sa.Column("byte_size", sa.Integer(), nullable=False),
|
||||
sa.Column("modified_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("ingested_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("ingest_run_id", sa.Integer(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["ingest_run_id"],
|
||||
[f"{schema}.ingest_run.id"],
|
||||
name=op.f("fk_source_artifact_ingest_run_id_ingest_run"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_source_artifact")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_source_artifact_source_kind",
|
||||
"source_artifact",
|
||||
["source_kind"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_source_artifact_congress",
|
||||
"source_artifact",
|
||||
["congress"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"score_run",
|
||||
sa.Column("ingest_run_id", sa.Integer(), nullable=True),
|
||||
sa.Column("classifier_version", sa.String(), nullable=True),
|
||||
sa.Column("scoring_version", sa.String(), nullable=True),
|
||||
sa.Column("included_vote_count", sa.Integer(), nullable=False),
|
||||
sa.Column("excluded_vote_count", sa.Integer(), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["ingest_run_id"],
|
||||
[f"{schema}.ingest_run.id"],
|
||||
name=op.f("fk_score_run_ingest_run_id_ingest_run"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_score_run")),
|
||||
schema=schema,
|
||||
)
|
||||
op.add_column(
|
||||
"legislator_score",
|
||||
sa.Column("score_run_id", sa.Integer(), nullable=True),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_legislator_score_score_run_id"),
|
||||
"legislator_score",
|
||||
["score_run_id"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_foreign_key(
|
||||
op.f("fk_legislator_score_score_run_id_score_run"),
|
||||
"legislator_score",
|
||||
"score_run",
|
||||
["score_run_id"],
|
||||
["id"],
|
||||
source_schema=schema,
|
||||
referent_schema=schema,
|
||||
ondelete="CASCADE",
|
||||
)
|
||||
op.add_column(
|
||||
"bill_text",
|
||||
sa.Column("source_datetime_raw", sa.String(), nullable=True),
|
||||
schema=schema,
|
||||
)
|
||||
op.add_column(
|
||||
"bill_text", sa.Column("text_url_xml", sa.String(), nullable=True), schema=schema
|
||||
)
|
||||
op.add_column(
|
||||
"bill_text", sa.Column("text_url_pdf", sa.String(), nullable=True), schema=schema
|
||||
)
|
||||
op.add_column(
|
||||
"bill_text",
|
||||
sa.Column("text_url_html", sa.String(), nullable=True),
|
||||
schema=schema,
|
||||
)
|
||||
op.add_column(
|
||||
"bill_text",
|
||||
sa.Column("source_artifact_id", sa.Integer(), nullable=True),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_foreign_key(
|
||||
op.f("fk_bill_text_source_artifact_id_source_artifact"),
|
||||
"bill_text",
|
||||
"source_artifact",
|
||||
["source_artifact_id"],
|
||||
["id"],
|
||||
source_schema=schema,
|
||||
referent_schema=schema,
|
||||
ondelete="SET NULL",
|
||||
)
|
||||
op.create_table(
|
||||
"bill_action",
|
||||
sa.Column("bill_id", sa.Integer(), nullable=False),
|
||||
sa.Column("sequence", sa.Integer(), nullable=False),
|
||||
sa.Column("action_date", sa.Date(), nullable=False),
|
||||
sa.Column("action_time", sa.String(), nullable=True),
|
||||
sa.Column("action_text", sa.String(), nullable=False),
|
||||
sa.Column("action_type", sa.String(), nullable=True),
|
||||
sa.Column("action_code", sa.String(), nullable=True),
|
||||
sa.Column("source_system_code", sa.String(), nullable=True),
|
||||
sa.Column("source_system_name", sa.String(), nullable=True),
|
||||
sa.Column("source_artifact_id", sa.Integer(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["bill_id"],
|
||||
[f"{schema}.bill.id"],
|
||||
name=op.f("fk_bill_action_bill_id_bill"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_artifact_id"],
|
||||
[f"{schema}.source_artifact.id"],
|
||||
name=op.f("fk_bill_action_source_artifact_id_source_artifact"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_bill_action")),
|
||||
sa.UniqueConstraint("bill_id", "sequence", name="uq_bill_action_bill_id_sequence"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"bill_action_recorded_vote",
|
||||
sa.Column("bill_action_id", sa.Integer(), nullable=False),
|
||||
sa.Column("congress", sa.Integer(), nullable=False),
|
||||
sa.Column("chamber", sa.String(), nullable=False),
|
||||
sa.Column("session_number", sa.Integer(), nullable=False),
|
||||
sa.Column("roll_number", sa.Integer(), nullable=False),
|
||||
sa.Column("vote_datetime", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("vote_url", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["bill_action_id"],
|
||||
[f"{schema}.bill_action.id"],
|
||||
name=op.f("fk_bill_action_recorded_vote_bill_action_id_bill_action"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_bill_action_recorded_vote")),
|
||||
sa.UniqueConstraint(
|
||||
"bill_action_id",
|
||||
"congress",
|
||||
"chamber",
|
||||
"session_number",
|
||||
"roll_number",
|
||||
name="uq_bill_action_recorded_vote_match_key",
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"bill_relation",
|
||||
sa.Column("bill_id", sa.Integer(), nullable=False),
|
||||
sa.Column("related_bill_id", sa.Integer(), nullable=False),
|
||||
sa.Column("relationship_type", sa.String(), nullable=False),
|
||||
sa.Column("identified_by", sa.String(), nullable=True),
|
||||
sa.Column("latest_action_date", sa.Date(), nullable=True),
|
||||
sa.Column("latest_action_text", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["bill_id"],
|
||||
[f"{schema}.bill.id"],
|
||||
name=op.f("fk_bill_relation_bill_id_bill"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["related_bill_id"],
|
||||
[f"{schema}.bill.id"],
|
||||
name=op.f("fk_bill_relation_related_bill_id_bill"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_bill_relation")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_bill_relation_bill_id",
|
||||
"bill_relation",
|
||||
["bill_id"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_bill_relation_related_bill_id",
|
||||
"bill_relation",
|
||||
["related_bill_id"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"amendment",
|
||||
sa.Column("congress", sa.Integer(), nullable=False),
|
||||
sa.Column("amendment_type", sa.String(), nullable=False),
|
||||
sa.Column("number", sa.Integer(), nullable=False),
|
||||
sa.Column("chamber", sa.String(), nullable=False),
|
||||
sa.Column("description", sa.String(), nullable=True),
|
||||
sa.Column("purpose", sa.String(), nullable=True),
|
||||
sa.Column("amended_bill_id", sa.Integer(), nullable=True),
|
||||
sa.Column("amended_amendment_id", sa.Integer(), nullable=True),
|
||||
sa.Column("source_path", sa.String(), nullable=True),
|
||||
sa.Column("source_artifact_id", sa.Integer(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["amended_amendment_id"],
|
||||
[f"{schema}.amendment.id"],
|
||||
name=op.f("fk_amendment_amended_amendment_id_amendment"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["amended_bill_id"],
|
||||
[f"{schema}.bill.id"],
|
||||
name=op.f("fk_amendment_amended_bill_id_bill"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_artifact_id"],
|
||||
[f"{schema}.source_artifact.id"],
|
||||
name=op.f("fk_amendment_source_artifact_id_source_artifact"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_amendment")),
|
||||
sa.UniqueConstraint(
|
||||
"congress",
|
||||
"amendment_type",
|
||||
"number",
|
||||
name="uq_amendment_congress_type_number",
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"amendment_action",
|
||||
sa.Column("amendment_id", sa.Integer(), nullable=False),
|
||||
sa.Column("sequence", sa.Integer(), nullable=False),
|
||||
sa.Column("action_date", sa.Date(), nullable=False),
|
||||
sa.Column("action_time", sa.String(), nullable=True),
|
||||
sa.Column("action_text", sa.String(), nullable=False),
|
||||
sa.Column("action_type", sa.String(), nullable=True),
|
||||
sa.Column("action_code", sa.String(), nullable=True),
|
||||
sa.Column("source_system_code", sa.String(), nullable=True),
|
||||
sa.Column("source_system_name", sa.String(), nullable=True),
|
||||
sa.Column("source_artifact_id", sa.Integer(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["amendment_id"],
|
||||
[f"{schema}.amendment.id"],
|
||||
name=op.f("fk_amendment_action_amendment_id_amendment"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_artifact_id"],
|
||||
[f"{schema}.source_artifact.id"],
|
||||
name=op.f("fk_amendment_action_source_artifact_id_source_artifact"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_amendment_action")),
|
||||
sa.UniqueConstraint(
|
||||
"amendment_id",
|
||||
"sequence",
|
||||
name="uq_amendment_action_amendment_id_sequence",
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"amendment_action_recorded_vote",
|
||||
sa.Column("amendment_action_id", sa.Integer(), nullable=False),
|
||||
sa.Column("congress", sa.Integer(), nullable=False),
|
||||
sa.Column("chamber", sa.String(), nullable=False),
|
||||
sa.Column("session_number", sa.Integer(), nullable=False),
|
||||
sa.Column("roll_number", sa.Integer(), nullable=False),
|
||||
sa.Column("vote_datetime", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("vote_url", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["amendment_action_id"],
|
||||
[f"{schema}.amendment_action.id"],
|
||||
name=op.f(
|
||||
"fk_amendment_action_recorded_vote_amendment_action_id_amendment_action"
|
||||
),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_amendment_action_recorded_vote")),
|
||||
sa.UniqueConstraint(
|
||||
"amendment_action_id",
|
||||
"congress",
|
||||
"chamber",
|
||||
"session_number",
|
||||
"roll_number",
|
||||
name="uq_amendment_action_recorded_vote_match_key",
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
|
||||
op.drop_constraint(
|
||||
"uq_vote_congress_chamber_session_number",
|
||||
"vote",
|
||||
schema=schema,
|
||||
type_="unique",
|
||||
)
|
||||
op.alter_column("vote", "session", new_column_name="session_year", schema=schema)
|
||||
op.alter_column("vote", "number", new_column_name="roll_number", schema=schema)
|
||||
op.add_column("vote", sa.Column("session_number", sa.Integer(), nullable=True), schema=schema)
|
||||
op.add_column(
|
||||
"vote",
|
||||
sa.Column("vote_datetime", sa.DateTime(timezone=True), nullable=True),
|
||||
schema=schema,
|
||||
)
|
||||
op.add_column(
|
||||
"vote", sa.Column("raw_vote_source_url", sa.String(), nullable=True), schema=schema
|
||||
)
|
||||
op.add_column(
|
||||
"vote",
|
||||
sa.Column("raw_bill_ref", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
|
||||
schema=schema,
|
||||
)
|
||||
op.add_column(
|
||||
"vote",
|
||||
sa.Column(
|
||||
"raw_amendment_ref",
|
||||
postgresql.JSONB(astext_type=sa.Text()),
|
||||
nullable=True,
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.add_column(
|
||||
"vote",
|
||||
sa.Column(
|
||||
"raw_nomination_ref",
|
||||
postgresql.JSONB(astext_type=sa.Text()),
|
||||
nullable=True,
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.add_column(
|
||||
"vote",
|
||||
sa.Column(
|
||||
"raw_treaty_ref",
|
||||
postgresql.JSONB(astext_type=sa.Text()),
|
||||
nullable=True,
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.add_column(
|
||||
"vote",
|
||||
sa.Column("raw_vote_source_artifact_id", sa.Integer(), nullable=True),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_foreign_key(
|
||||
op.f("fk_vote_raw_vote_source_artifact_id_source_artifact"),
|
||||
"vote",
|
||||
"source_artifact",
|
||||
["raw_vote_source_artifact_id"],
|
||||
["id"],
|
||||
source_schema=schema,
|
||||
referent_schema=schema,
|
||||
ondelete="SET NULL",
|
||||
)
|
||||
op.execute(
|
||||
sa.text(
|
||||
f"""
|
||||
UPDATE {schema}.vote
|
||||
SET session_number = session_year - (((congress - 1) * 2) + 1789) + 1
|
||||
"""
|
||||
)
|
||||
)
|
||||
op.alter_column("vote", "session_number", nullable=False, schema=schema)
|
||||
op.create_unique_constraint(
|
||||
"uq_vote_congress_chamber_session_number_roll_number",
|
||||
"vote",
|
||||
["congress", "chamber", "session_number", "roll_number"],
|
||||
schema=schema,
|
||||
)
|
||||
op.drop_constraint(
|
||||
op.f("fk_vote_bill_id_bill"),
|
||||
"vote",
|
||||
schema=schema,
|
||||
type_="foreignkey",
|
||||
)
|
||||
op.drop_constraint(
|
||||
"fk_vote_bill_text_id_bill_text",
|
||||
"vote",
|
||||
schema=schema,
|
||||
type_="foreignkey",
|
||||
)
|
||||
op.drop_index("ix_vote_bill_text_id", table_name="vote", schema=schema)
|
||||
op.drop_column("vote", "bill_id", schema=schema)
|
||||
op.drop_column("vote", "bill_text_id", schema=schema)
|
||||
|
||||
op.create_table(
|
||||
"vote_action_match",
|
||||
sa.Column("vote_id", sa.Integer(), nullable=False),
|
||||
sa.Column("action_scope", sa.String(), nullable=False),
|
||||
sa.Column("bill_action_id", sa.Integer(), nullable=True),
|
||||
sa.Column("amendment_action_id", sa.Integer(), nullable=True),
|
||||
sa.Column("is_selected", sa.Boolean(), nullable=False),
|
||||
sa.Column("match_method", sa.String(), nullable=False),
|
||||
sa.Column("match_reason", sa.String(), nullable=True),
|
||||
sa.Column("match_confidence", sa.String(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["amendment_action_id"],
|
||||
[f"{schema}.amendment_action.id"],
|
||||
name=op.f("fk_vote_action_match_amendment_action_id_amendment_action"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["bill_action_id"],
|
||||
[f"{schema}.bill_action.id"],
|
||||
name=op.f("fk_vote_action_match_bill_action_id_bill_action"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["vote_id"],
|
||||
[f"{schema}.vote.id"],
|
||||
name=op.f("fk_vote_action_match_vote_id_vote"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_vote_action_match")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_vote_action_match_vote_id",
|
||||
"vote_action_match",
|
||||
["vote_id"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"uq_vote_action_match_selected_vote_id",
|
||||
"vote_action_match",
|
||||
["vote_id"],
|
||||
unique=True,
|
||||
schema=schema,
|
||||
postgresql_where=sa.text("is_selected"),
|
||||
)
|
||||
op.create_table(
|
||||
"vote_classification",
|
||||
sa.Column("vote_id", sa.Integer(), nullable=False),
|
||||
sa.Column("subject_type", sa.String(), nullable=False),
|
||||
sa.Column("measure_type", sa.String(), nullable=True),
|
||||
sa.Column("measure_subtype", sa.String(), nullable=True),
|
||||
sa.Column("measure_function", sa.String(), nullable=True),
|
||||
sa.Column("vote_relationship", sa.String(), nullable=False),
|
||||
sa.Column("is_legislation_related", sa.Boolean(), nullable=False),
|
||||
sa.Column("is_direct_vote_on_legislative_text", sa.Boolean(), nullable=False),
|
||||
sa.Column("is_substantive_policy_vote", sa.Boolean(), nullable=False),
|
||||
sa.Column("is_lawmaking_vehicle", sa.Boolean(), nullable=False),
|
||||
sa.Column("is_special_rule", sa.Boolean(), nullable=False),
|
||||
sa.Column("classification_method", sa.String(), nullable=False),
|
||||
sa.Column("classification_confidence_reason", sa.String(), nullable=True),
|
||||
sa.Column("confidence", sa.String(), nullable=False),
|
||||
sa.Column("classified_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("classification_version", sa.String(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["vote_id"],
|
||||
[f"{schema}.vote.id"],
|
||||
name=op.f("fk_vote_classification_vote_id_vote"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_vote_classification")),
|
||||
sa.UniqueConstraint("vote_id", name=op.f("uq_vote_classification_vote_id")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_vote_classification_subject_type",
|
||||
"vote_classification",
|
||||
["subject_type"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"vote_measure_link",
|
||||
sa.Column("vote_id", sa.Integer(), nullable=False),
|
||||
sa.Column("measure_id", sa.Integer(), nullable=False),
|
||||
sa.Column("role", sa.String(), nullable=False),
|
||||
sa.Column("source", sa.String(), nullable=False),
|
||||
sa.Column("confidence", sa.String(), nullable=False),
|
||||
sa.Column("notes", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["measure_id"],
|
||||
[f"{schema}.bill.id"],
|
||||
name=op.f("fk_vote_measure_link_measure_id_bill"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_vote_measure_link")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_vote_measure_link_vote_id",
|
||||
"vote_measure_link",
|
||||
["vote_id"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_foreign_key(
|
||||
op.f("fk_vote_measure_link_vote_id_vote"),
|
||||
"vote_measure_link",
|
||||
"vote",
|
||||
["vote_id"],
|
||||
["id"],
|
||||
source_schema=schema,
|
||||
referent_schema=schema,
|
||||
ondelete="CASCADE",
|
||||
)
|
||||
op.create_table(
|
||||
"vote_text_target",
|
||||
sa.Column("vote_id", sa.Integer(), nullable=False),
|
||||
sa.Column("text_target_type", sa.String(), nullable=False),
|
||||
sa.Column("voted_text_version_id", sa.Integer(), nullable=True),
|
||||
sa.Column("resulting_text_version_id", sa.Integer(), nullable=True),
|
||||
sa.Column("related_amendment_id", sa.Integer(), nullable=True),
|
||||
sa.Column("text_target_basis", sa.String(), nullable=False),
|
||||
sa.Column("text_resolution_method", sa.String(), nullable=False),
|
||||
sa.Column("text_resolution_confidence_reason", sa.String(), nullable=True),
|
||||
sa.Column("confidence", sa.String(), nullable=False),
|
||||
sa.Column("notes", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["related_amendment_id"],
|
||||
[f"{schema}.amendment.id"],
|
||||
name=op.f("fk_vote_text_target_related_amendment_id_amendment"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["resulting_text_version_id"],
|
||||
[f"{schema}.bill_text.id"],
|
||||
name=op.f("fk_vote_text_target_resulting_text_version_id_bill_text"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["vote_id"],
|
||||
[f"{schema}.vote.id"],
|
||||
name=op.f("fk_vote_text_target_vote_id_vote"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["voted_text_version_id"],
|
||||
[f"{schema}.bill_text.id"],
|
||||
name=op.f("fk_vote_text_target_voted_text_version_id_bill_text"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_vote_text_target")),
|
||||
sa.UniqueConstraint("vote_id", name=op.f("uq_vote_text_target_vote_id")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"vote_position_meaning",
|
||||
sa.Column("vote_id", sa.Integer(), nullable=False),
|
||||
sa.Column("yea_effect", sa.String(), nullable=False),
|
||||
sa.Column("nay_effect", sa.String(), nullable=False),
|
||||
sa.Column("present_effect", sa.String(), nullable=False),
|
||||
sa.Column("polarity_confidence", sa.String(), nullable=False),
|
||||
sa.Column("polarity_method", sa.String(), nullable=False),
|
||||
sa.Column("notes", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["vote_id"],
|
||||
[f"{schema}.vote.id"],
|
||||
name=op.f("fk_vote_position_meaning_vote_id_vote"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_vote_position_meaning")),
|
||||
sa.UniqueConstraint("vote_id", name=op.f("uq_vote_position_meaning_vote_id")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"vote_context_audit",
|
||||
sa.Column("vote_id", sa.Integer(), nullable=False),
|
||||
sa.Column("step", sa.String(), nullable=False),
|
||||
sa.Column("message", sa.String(), nullable=False),
|
||||
sa.Column("severity", sa.String(), nullable=False),
|
||||
sa.Column("source_path", sa.String(), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["vote_id"],
|
||||
[f"{schema}.vote.id"],
|
||||
name=op.f("fk_vote_context_audit_vote_id_vote"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_vote_context_audit")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_vote_context_audit_vote_id",
|
||||
"vote_context_audit",
|
||||
["vote_id"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
raise NotImplementedError("Downgrade is not supported for canonical vote context v3.")
|
||||
@@ -0,0 +1,203 @@
|
||||
"""add supporting indexes for congress vote context and scoring.
|
||||
|
||||
Revision ID: a7b91c4e2d30
|
||||
Revises: 1f8c0e7a9d21
|
||||
Create Date: 2026-04-26 00:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
from pipelines.orm import DataScienceDevBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
|
||||
revision: str = "a7b91c4e2d30"
|
||||
down_revision: str | None = "1f8c0e7a9d21"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = DataScienceDevBase.schema_name
|
||||
|
||||
|
||||
def _dedupe_source_artifacts() -> None:
|
||||
op.execute(
|
||||
sa.text(
|
||||
f"""
|
||||
CREATE TEMP TABLE tmp_source_artifact_dups AS
|
||||
WITH ranked AS (
|
||||
SELECT
|
||||
id,
|
||||
first_value(id) OVER (
|
||||
PARTITION BY ingest_run_id, local_path, sha256
|
||||
ORDER BY id
|
||||
) AS keep_id,
|
||||
row_number() OVER (
|
||||
PARTITION BY ingest_run_id, local_path, sha256
|
||||
ORDER BY id
|
||||
) AS rn
|
||||
FROM {schema}.source_artifact
|
||||
WHERE ingest_run_id IS NOT NULL
|
||||
)
|
||||
SELECT id, keep_id
|
||||
FROM ranked
|
||||
WHERE rn > 1
|
||||
"""
|
||||
)
|
||||
)
|
||||
for table_name, column_name in (
|
||||
("bill_text", "source_artifact_id"),
|
||||
("bill_action", "source_artifact_id"),
|
||||
("amendment", "source_artifact_id"),
|
||||
("amendment_action", "source_artifact_id"),
|
||||
("vote", "raw_vote_source_artifact_id"),
|
||||
):
|
||||
op.execute(
|
||||
sa.text(
|
||||
f"""
|
||||
UPDATE {schema}.{table_name} AS target
|
||||
SET {column_name} = d.keep_id
|
||||
FROM tmp_source_artifact_dups AS d
|
||||
WHERE target.{column_name} = d.id
|
||||
"""
|
||||
)
|
||||
)
|
||||
op.execute(
|
||||
sa.text(
|
||||
f"""
|
||||
DELETE FROM {schema}.source_artifact AS source_artifact
|
||||
USING tmp_source_artifact_dups AS d
|
||||
WHERE source_artifact.id = d.id
|
||||
"""
|
||||
)
|
||||
)
|
||||
op.execute(sa.text("DROP TABLE tmp_source_artifact_dups"))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
_dedupe_source_artifacts()
|
||||
|
||||
op.create_index(
|
||||
"uq_source_artifact_ingest_identity",
|
||||
"source_artifact",
|
||||
["ingest_run_id", "local_path", "sha256"],
|
||||
unique=True,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_bill_action_recorded_vote_match_tuple",
|
||||
"bill_action_recorded_vote",
|
||||
["congress", "chamber", "session_number", "roll_number"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_amendment_action_recorded_vote_match_tuple",
|
||||
"amendment_action_recorded_vote",
|
||||
["congress", "chamber", "session_number", "roll_number"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_vote_classification_eligible_vote_id",
|
||||
"vote_classification",
|
||||
["vote_id"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
postgresql_where=sa.text(
|
||||
"subject_type = 'measure' "
|
||||
"AND vote_relationship = 'direct_text_vote' "
|
||||
"AND is_direct_vote_on_legislative_text "
|
||||
"AND is_substantive_policy_vote "
|
||||
"AND NOT is_special_rule"
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_vote_measure_link_vote_id_role",
|
||||
"vote_measure_link",
|
||||
["vote_id", "role"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_vote_measure_link_measure_id_role",
|
||||
"vote_measure_link",
|
||||
["measure_id", "role"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_vote_text_target_voted_text_version_id",
|
||||
"vote_text_target",
|
||||
["voted_text_version_id"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
postgresql_where=sa.text("voted_text_version_id IS NOT NULL"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_vote_context_audit_severity_vote_id",
|
||||
"vote_context_audit",
|
||||
["severity", "vote_id"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_legislator_current_chamber",
|
||||
"legislator",
|
||||
["current_chamber"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
op.drop_index("ix_legislator_current_chamber", table_name="legislator", schema=schema)
|
||||
op.drop_index(
|
||||
"ix_vote_context_audit_severity_vote_id",
|
||||
table_name="vote_context_audit",
|
||||
schema=schema,
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_vote_text_target_voted_text_version_id",
|
||||
table_name="vote_text_target",
|
||||
schema=schema,
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_vote_measure_link_measure_id_role",
|
||||
table_name="vote_measure_link",
|
||||
schema=schema,
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_vote_measure_link_vote_id_role",
|
||||
table_name="vote_measure_link",
|
||||
schema=schema,
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_vote_classification_eligible_vote_id",
|
||||
table_name="vote_classification",
|
||||
schema=schema,
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_amendment_action_recorded_vote_match_tuple",
|
||||
table_name="amendment_action_recorded_vote",
|
||||
schema=schema,
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_bill_action_recorded_vote_match_tuple",
|
||||
table_name="bill_action_recorded_vote",
|
||||
schema=schema,
|
||||
)
|
||||
op.drop_index(
|
||||
"uq_source_artifact_ingest_identity",
|
||||
table_name="source_artifact",
|
||||
schema=schema,
|
||||
)
|
||||
@@ -0,0 +1,66 @@
|
||||
"""adding PostTopic.
|
||||
|
||||
Revision ID: 032e26bbfcb5
|
||||
Revises: a7b91c4e2d30
|
||||
Create Date: 2026-04-26 14:34:35.688341
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
from pipelines.orm import DataScienceDevBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "032e26bbfcb5"
|
||||
down_revision: str | None = "a7b91c4e2d30"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = DataScienceDevBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"post_topic",
|
||||
sa.Column("post_id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("topic_id", sa.SmallInteger(), nullable=False),
|
||||
sa.Column("topic_label", sa.String(), nullable=True),
|
||||
sa.Column("model_version", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_post_topic")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_post_topic_post_id", "post_topic", ["post_id"], unique=False, schema=schema
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index("ix_post_topic_post_id", table_name="post_topic", schema=schema)
|
||||
op.drop_table("post_topic", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
@@ -0,0 +1,35 @@
|
||||
"""adding PG Vector.
|
||||
|
||||
Revision ID: b9360b0b0c22
|
||||
Revises: 032e26bbfcb5
|
||||
Create Date: 2026-04-26 14:35:08.770128
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from alembic import op
|
||||
from pipelines.orm import DataScienceDevBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "b9360b0b0c22"
|
||||
down_revision: str | None = "032e26bbfcb5"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = DataScienceDevBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
op.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
op.execute("DROP EXTENSION IF EXISTS vector")
|
||||
Reference in New Issue
Block a user