Compare commits
3 Commits
f77c9657a3
...
5d3a851137
| Author | SHA1 | Date | |
|---|---|---|---|
| 5d3a851137 | |||
| e05e5c77bc | |||
| b0a2ebc052 |
@@ -1,12 +0,0 @@
|
||||
## Dev environment tips
|
||||
|
||||
- use treefmt to format all files
|
||||
- make python code ruff compliant
|
||||
- use pytest to test python code
|
||||
- always use the minimum amount of complexity
|
||||
- if judgment calls are easy to reverse make them. if not ask me first
|
||||
- Match existing code style.
|
||||
- Use builtin helpers getenv() over os.environ.get.
|
||||
- Prefer single-purpose functions over “do everything” helpers.
|
||||
- Avoid compatibility branches like PG_USER and POSTGRESQL_URL unless requested.
|
||||
- Keep helpers only if reused or they simplify the code otherwise inline.
|
||||
File diff suppressed because one or more lines are too long
@@ -84,9 +84,6 @@ lint.ignore = [
|
||||
"python/alembic/**" = [
|
||||
"INP001", # (perm) this creates LSP issues for alembic
|
||||
]
|
||||
"python/signal_bot/**" = [
|
||||
"D107", # (perm) class docstrings cover __init__
|
||||
]
|
||||
|
||||
[tool.ruff.lint.pydocstyle]
|
||||
convention = "google"
|
||||
|
||||
-1417
File diff suppressed because it is too large
Load Diff
-50
@@ -1,50 +0,0 @@
|
||||
"""adding FailedIngestion.
|
||||
|
||||
Revision ID: 2f43120e3ffc
|
||||
Revises: f99be864fe69
|
||||
Create Date: 2026-03-24 23:46:17.277897
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import DataScienceDevBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "2f43120e3ffc"
|
||||
down_revision: str | None = "f99be864fe69"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = DataScienceDevBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"failed_ingestion",
|
||||
sa.Column("raw_line", sa.Text(), nullable=False),
|
||||
sa.Column("error", sa.Text(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_failed_ingestion")),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("failed_ingestion", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
-2770
File diff suppressed because it is too large
Load Diff
-1391
File diff suppressed because it is too large
Load Diff
@@ -1,72 +0,0 @@
|
||||
"""Attach all partition tables to the posts parent table.
|
||||
|
||||
Alembic autogenerate creates partition tables as standalone tables but does not
|
||||
emit the ALTER TABLE ... ATTACH PARTITION statements needed for PostgreSQL to
|
||||
route inserts to the correct partition.
|
||||
|
||||
Revision ID: a1b2c3d4e5f6
|
||||
Revises: 605b1794838f
|
||||
Create Date: 2026-03-25 10:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import text
|
||||
|
||||
from python.orm import DataScienceDevBase
|
||||
from python.orm.data_science_dev.posts.partitions import (
|
||||
PARTITION_END_YEAR,
|
||||
PARTITION_START_YEAR,
|
||||
iso_weeks_in_year,
|
||||
week_bounds,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "a1b2c3d4e5f6"
|
||||
down_revision: str | None = "605b1794838f"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = DataScienceDevBase.schema_name
|
||||
|
||||
ALREADY_ATTACHED_QUERY = text("""
|
||||
SELECT inhrelid::regclass::text
|
||||
FROM pg_inherits
|
||||
WHERE inhparent = :parent::regclass
|
||||
""")
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Attach all weekly partition tables to the posts parent table."""
|
||||
connection = op.get_bind()
|
||||
already_attached = {row[0] for row in connection.execute(ALREADY_ATTACHED_QUERY, {"parent": f"{schema}.posts"})}
|
||||
|
||||
for year in range(PARTITION_START_YEAR, PARTITION_END_YEAR + 1):
|
||||
for week in range(1, iso_weeks_in_year(year) + 1):
|
||||
table_name = f"posts_{year}_{week:02d}"
|
||||
qualified_name = f"{schema}.{table_name}"
|
||||
if qualified_name in already_attached:
|
||||
continue
|
||||
start, end = week_bounds(year, week)
|
||||
start_str = start.strftime("%Y-%m-%d %H:%M:%S")
|
||||
end_str = end.strftime("%Y-%m-%d %H:%M:%S")
|
||||
op.execute(
|
||||
f"ALTER TABLE {schema}.posts "
|
||||
f"ATTACH PARTITION {qualified_name} "
|
||||
f"FOR VALUES FROM ('{start_str}') TO ('{end_str}')"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Detach all weekly partition tables from the posts parent table."""
|
||||
for year in range(PARTITION_START_YEAR, PARTITION_END_YEAR + 1):
|
||||
for week in range(1, iso_weeks_in_year(year) + 1):
|
||||
table_name = f"posts_{year}_{week:02d}"
|
||||
op.execute(f"ALTER TABLE {schema}.posts DETACH PARTITION {schema}.{table_name}")
|
||||
-153
@@ -1,153 +0,0 @@
|
||||
"""adding congress data.
|
||||
|
||||
Revision ID: 83bfc8af92d8
|
||||
Revises: a1b2c3d4e5f6
|
||||
Create Date: 2026-03-27 10:43:02.324510
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import DataScienceDevBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "83bfc8af92d8"
|
||||
down_revision: str | None = "a1b2c3d4e5f6"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = DataScienceDevBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"bill",
|
||||
sa.Column("congress", sa.Integer(), nullable=False),
|
||||
sa.Column("bill_type", sa.String(), nullable=False),
|
||||
sa.Column("number", sa.Integer(), nullable=False),
|
||||
sa.Column("title", sa.String(), nullable=True),
|
||||
sa.Column("title_short", sa.String(), nullable=True),
|
||||
sa.Column("official_title", sa.String(), nullable=True),
|
||||
sa.Column("status", sa.String(), nullable=True),
|
||||
sa.Column("status_at", sa.Date(), nullable=True),
|
||||
sa.Column("sponsor_bioguide_id", sa.String(), nullable=True),
|
||||
sa.Column("subjects_top_term", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_bill")),
|
||||
sa.UniqueConstraint("congress", "bill_type", "number", name="uq_bill_congress_type_number"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index("ix_bill_congress", "bill", ["congress"], unique=False, schema=schema)
|
||||
op.create_table(
|
||||
"legislator",
|
||||
sa.Column("bioguide_id", sa.Text(), nullable=False),
|
||||
sa.Column("thomas_id", sa.String(), nullable=True),
|
||||
sa.Column("lis_id", sa.String(), nullable=True),
|
||||
sa.Column("govtrack_id", sa.Integer(), nullable=True),
|
||||
sa.Column("opensecrets_id", sa.String(), nullable=True),
|
||||
sa.Column("fec_ids", sa.String(), nullable=True),
|
||||
sa.Column("first_name", sa.String(), nullable=False),
|
||||
sa.Column("last_name", sa.String(), nullable=False),
|
||||
sa.Column("official_full_name", sa.String(), nullable=True),
|
||||
sa.Column("nickname", sa.String(), nullable=True),
|
||||
sa.Column("birthday", sa.Date(), nullable=True),
|
||||
sa.Column("gender", sa.String(), nullable=True),
|
||||
sa.Column("current_party", sa.String(), nullable=True),
|
||||
sa.Column("current_state", sa.String(), nullable=True),
|
||||
sa.Column("current_district", sa.Integer(), nullable=True),
|
||||
sa.Column("current_chamber", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_legislator")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(op.f("ix_legislator_bioguide_id"), "legislator", ["bioguide_id"], unique=True, schema=schema)
|
||||
op.create_table(
|
||||
"bill_text",
|
||||
sa.Column("bill_id", sa.Integer(), nullable=False),
|
||||
sa.Column("version_code", sa.String(), nullable=False),
|
||||
sa.Column("version_name", sa.String(), nullable=True),
|
||||
sa.Column("text_content", sa.String(), nullable=True),
|
||||
sa.Column("date", sa.Date(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["bill_id"], [f"{schema}.bill.id"], name=op.f("fk_bill_text_bill_id_bill"), ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_bill_text")),
|
||||
sa.UniqueConstraint("bill_id", "version_code", name="uq_bill_text_bill_id_version_code"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"vote",
|
||||
sa.Column("congress", sa.Integer(), nullable=False),
|
||||
sa.Column("chamber", sa.String(), nullable=False),
|
||||
sa.Column("session", sa.Integer(), nullable=False),
|
||||
sa.Column("number", sa.Integer(), nullable=False),
|
||||
sa.Column("vote_type", sa.String(), nullable=True),
|
||||
sa.Column("question", sa.String(), nullable=True),
|
||||
sa.Column("result", sa.String(), nullable=True),
|
||||
sa.Column("result_text", sa.String(), nullable=True),
|
||||
sa.Column("vote_date", sa.Date(), nullable=False),
|
||||
sa.Column("yea_count", sa.Integer(), nullable=True),
|
||||
sa.Column("nay_count", sa.Integer(), nullable=True),
|
||||
sa.Column("not_voting_count", sa.Integer(), nullable=True),
|
||||
sa.Column("present_count", sa.Integer(), nullable=True),
|
||||
sa.Column("bill_id", sa.Integer(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(["bill_id"], [f"{schema}.bill.id"], name=op.f("fk_vote_bill_id_bill")),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_vote")),
|
||||
sa.UniqueConstraint("congress", "chamber", "session", "number", name="uq_vote_congress_chamber_session_number"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index("ix_vote_congress_chamber", "vote", ["congress", "chamber"], unique=False, schema=schema)
|
||||
op.create_index("ix_vote_date", "vote", ["vote_date"], unique=False, schema=schema)
|
||||
op.create_table(
|
||||
"vote_record",
|
||||
sa.Column("vote_id", sa.Integer(), nullable=False),
|
||||
sa.Column("legislator_id", sa.Integer(), nullable=False),
|
||||
sa.Column("position", sa.String(), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["legislator_id"],
|
||||
[f"{schema}.legislator.id"],
|
||||
name=op.f("fk_vote_record_legislator_id_legislator"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["vote_id"], [f"{schema}.vote.id"], name=op.f("fk_vote_record_vote_id_vote"), ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("vote_id", "legislator_id", name=op.f("pk_vote_record")),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("vote_record", schema=schema)
|
||||
op.drop_index("ix_vote_date", table_name="vote", schema=schema)
|
||||
op.drop_index("ix_vote_congress_chamber", table_name="vote", schema=schema)
|
||||
op.drop_table("vote", schema=schema)
|
||||
op.drop_table("bill_text", schema=schema)
|
||||
op.drop_index(op.f("ix_legislator_bioguide_id"), table_name="legislator", schema=schema)
|
||||
op.drop_table("legislator", schema=schema)
|
||||
op.drop_index("ix_bill_congress", table_name="bill", schema=schema)
|
||||
op.drop_table("bill", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
-58
@@ -1,58 +0,0 @@
|
||||
"""adding LegislatorSocialMedia.
|
||||
|
||||
Revision ID: 5cd7eee3549d
|
||||
Revises: 83bfc8af92d8
|
||||
Create Date: 2026-03-29 11:53:44.224799
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import DataScienceDevBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "5cd7eee3549d"
|
||||
down_revision: str | None = "83bfc8af92d8"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = DataScienceDevBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"legislator_social_media",
|
||||
sa.Column("legislator_id", sa.Integer(), nullable=False),
|
||||
sa.Column("platform", sa.String(), nullable=False),
|
||||
sa.Column("account_name", sa.String(), nullable=False),
|
||||
sa.Column("url", sa.String(), nullable=True),
|
||||
sa.Column("source", sa.String(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["legislator_id"],
|
||||
[f"{schema}.legislator.id"],
|
||||
name=op.f("fk_legislator_social_media_legislator_id_legislator"),
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_legislator_social_media")),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("legislator_social_media", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
-100
@@ -1,100 +0,0 @@
|
||||
"""seprating signal_bot database.
|
||||
|
||||
Revision ID: 6eaf696e07a5
|
||||
Revises:
|
||||
Create Date: 2026-03-17 21:35:37.612672
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
from python.orm import SignalBotBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "6eaf696e07a5"
|
||||
down_revision: str | None = None
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = SignalBotBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"dead_letter_message",
|
||||
sa.Column("source", sa.String(), nullable=False),
|
||||
sa.Column("message", sa.Text(), nullable=False),
|
||||
sa.Column("received_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column(
|
||||
"status", postgresql.ENUM("UNPROCESSED", "PROCESSED", name="message_status", schema=schema), nullable=False
|
||||
),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_dead_letter_message")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"role",
|
||||
sa.Column("name", sa.String(length=50), nullable=False),
|
||||
sa.Column("id", sa.SmallInteger(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_role")),
|
||||
sa.UniqueConstraint("name", name=op.f("uq_role_name")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"signal_device",
|
||||
sa.Column("phone_number", sa.String(length=50), nullable=False),
|
||||
sa.Column("safety_number", sa.String(), nullable=True),
|
||||
sa.Column(
|
||||
"trust_level",
|
||||
postgresql.ENUM("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", schema=schema),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("last_seen", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_signal_device")),
|
||||
sa.UniqueConstraint("phone_number", name=op.f("uq_signal_device_phone_number")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"device_role",
|
||||
sa.Column("device_id", sa.Integer(), nullable=False),
|
||||
sa.Column("role_id", sa.SmallInteger(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["device_id"], [f"{schema}.signal_device.id"], name=op.f("fk_device_role_device_id_signal_device")
|
||||
),
|
||||
sa.ForeignKeyConstraint(["role_id"], [f"{schema}.role.id"], name=op.f("fk_device_role_role_id_role")),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_device_role")),
|
||||
sa.UniqueConstraint("device_id", "role_id", name="uq_device_role_device_role"),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("device_role", schema=schema)
|
||||
op.drop_table("signal_device", schema=schema)
|
||||
op.drop_table("role", schema=schema)
|
||||
op.drop_table("dead_letter_message", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
@@ -1,72 +0,0 @@
|
||||
"""test.
|
||||
|
||||
Revision ID: 66bdd532bcab
|
||||
Revises: 6eaf696e07a5
|
||||
Create Date: 2026-03-18 19:21:14.561568
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
from python.orm import SignalBotBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "66bdd532bcab"
|
||||
down_revision: str | None = "6eaf696e07a5"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = SignalBotBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.alter_column(
|
||||
"dead_letter_message",
|
||||
"status",
|
||||
existing_type=postgresql.ENUM("UNPROCESSED", "PROCESSED", name="message_status", schema=schema),
|
||||
type_=sa.Enum("UNPROCESSED", "PROCESSED", name="message_status", native_enum=False),
|
||||
existing_nullable=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.alter_column(
|
||||
"signal_device",
|
||||
"trust_level",
|
||||
existing_type=postgresql.ENUM("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", schema=schema),
|
||||
type_=sa.Enum("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", native_enum=False),
|
||||
existing_nullable=False,
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.alter_column(
|
||||
"signal_device",
|
||||
"trust_level",
|
||||
existing_type=sa.Enum("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", native_enum=False),
|
||||
type_=postgresql.ENUM("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", schema=schema),
|
||||
existing_nullable=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.alter_column(
|
||||
"dead_letter_message",
|
||||
"status",
|
||||
existing_type=sa.Enum("UNPROCESSED", "PROCESSED", name="message_status", native_enum=False),
|
||||
type_=postgresql.ENUM("UNPROCESSED", "PROCESSED", name="message_status", schema=schema),
|
||||
existing_nullable=False,
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
@@ -1,3 +0,0 @@
|
||||
"""Data science CLI tools."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -1,613 +0,0 @@
|
||||
"""Ingestion pipeline for loading congress data from unitedstates/congress JSON files.
|
||||
|
||||
Loads legislators, bills, votes, vote records, and bill text into the data_science_dev database.
|
||||
Expects the parent directory to contain congress-tracker/ and congress-legislators/ as siblings.
|
||||
|
||||
Usage:
|
||||
ingest-congress /path/to/parent/
|
||||
ingest-congress /path/to/parent/ --congress 118
|
||||
ingest-congress /path/to/parent/ --congress 118 --only bills
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path # noqa: TC003 needed at runtime for typer CLI argument
|
||||
from typing import TYPE_CHECKING, Annotated
|
||||
|
||||
import orjson
|
||||
import typer
|
||||
import yaml
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from python.common import configure_logger
|
||||
from python.orm.common import get_postgres_engine
|
||||
from python.orm.data_science_dev.congress import Bill, BillText, Legislator, LegislatorSocialMedia, Vote, VoteRecord
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BATCH_SIZE = 10_000
|
||||
|
||||
app = typer.Typer(help="Ingest unitedstates/congress data into data_science_dev.")
|
||||
|
||||
|
||||
@app.command()
|
||||
def main(
|
||||
parent_dir: Annotated[
|
||||
Path,
|
||||
typer.Argument(help="Parent directory containing congress-tracker/ and congress-legislators/"),
|
||||
],
|
||||
congress: Annotated[int | None, typer.Option(help="Only ingest a specific congress number")] = None,
|
||||
only: Annotated[
|
||||
str | None,
|
||||
typer.Option(help="Only run a specific step: legislators, social-media, bills, votes, bill-text"),
|
||||
] = None,
|
||||
) -> None:
|
||||
"""Ingest congress data from unitedstates/congress JSON files."""
|
||||
configure_logger(level="INFO")
|
||||
|
||||
data_dir = parent_dir / "congress-tracker/congress/data/"
|
||||
legislators_dir = parent_dir / "congress-legislators"
|
||||
|
||||
if not data_dir.is_dir():
|
||||
typer.echo(f"Expected congress-tracker/ directory: {data_dir}", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
if not legislators_dir.is_dir():
|
||||
typer.echo(f"Expected congress-legislators/ directory: {legislators_dir}", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
engine = get_postgres_engine(name="DATA_SCIENCE_DEV")
|
||||
|
||||
congress_dirs = _resolve_congress_dirs(data_dir, congress)
|
||||
if not congress_dirs:
|
||||
typer.echo("No congress directories found.", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
logger.info("Found %d congress directories to process", len(congress_dirs))
|
||||
|
||||
steps: dict[str, tuple] = {
|
||||
"legislators": (ingest_legislators, (engine, legislators_dir)),
|
||||
"legislators-social-media": (ingest_social_media, (engine, legislators_dir)),
|
||||
"bills": (ingest_bills, (engine, congress_dirs)),
|
||||
"votes": (ingest_votes, (engine, congress_dirs)),
|
||||
"bill-text": (ingest_bill_text, (engine, congress_dirs)),
|
||||
}
|
||||
|
||||
if only:
|
||||
if only not in steps:
|
||||
typer.echo(f"Unknown step: {only}. Choose from: {', '.join(steps)}", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
steps = {only: steps[only]}
|
||||
|
||||
for step_name, (step_func, step_args) in steps.items():
|
||||
logger.info("=== Starting step: %s ===", step_name)
|
||||
step_func(*step_args)
|
||||
logger.info("=== Finished step: %s ===", step_name)
|
||||
|
||||
logger.info("ingest-congress done")
|
||||
|
||||
|
||||
def _resolve_congress_dirs(data_dir: Path, congress: int | None) -> list[Path]:
|
||||
"""Find congress number directories under data_dir."""
|
||||
if congress is not None:
|
||||
target = data_dir / str(congress)
|
||||
return [target] if target.is_dir() else []
|
||||
return sorted(path for path in data_dir.iterdir() if path.is_dir() and path.name.isdigit())
|
||||
|
||||
|
||||
def _flush_batch(session: Session, batch: list[object], label: str) -> int:
|
||||
"""Add a batch of ORM objects to the session and commit. Returns count added."""
|
||||
if not batch:
|
||||
return 0
|
||||
session.add_all(batch)
|
||||
session.commit()
|
||||
count = len(batch)
|
||||
logger.info("Committed %d %s", count, label)
|
||||
batch.clear()
|
||||
return count
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Legislators — loaded from congress-legislators YAML files
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def ingest_legislators(engine: Engine, legislators_dir: Path) -> None:
|
||||
"""Load legislators from congress-legislators YAML files."""
|
||||
legislators_data = _load_legislators_yaml(legislators_dir)
|
||||
logger.info("Loaded %d legislators from YAML files", len(legislators_data))
|
||||
|
||||
with Session(engine) as session:
|
||||
existing_legislators = {
|
||||
legislator.bioguide_id: legislator for legislator in session.scalars(select(Legislator)).all()
|
||||
}
|
||||
logger.info("Found %d existing legislators in DB", len(existing_legislators))
|
||||
|
||||
total_inserted = 0
|
||||
total_updated = 0
|
||||
for entry in legislators_data:
|
||||
bioguide_id = entry.get("id", {}).get("bioguide")
|
||||
if not bioguide_id:
|
||||
continue
|
||||
|
||||
fields = _parse_legislator(entry)
|
||||
if existing := existing_legislators.get(bioguide_id):
|
||||
changed = False
|
||||
for field, value in fields.items():
|
||||
if value is not None and getattr(existing, field) != value:
|
||||
setattr(existing, field, value)
|
||||
changed = True
|
||||
if changed:
|
||||
total_updated += 1
|
||||
else:
|
||||
session.add(Legislator(bioguide_id=bioguide_id, **fields))
|
||||
total_inserted += 1
|
||||
|
||||
session.commit()
|
||||
logger.info("Inserted %d new legislators, updated %d existing", total_inserted, total_updated)
|
||||
|
||||
|
||||
def _load_legislators_yaml(legislators_dir: Path) -> list[dict]:
|
||||
"""Load and combine legislators-current.yaml and legislators-historical.yaml."""
|
||||
legislators: list[dict] = []
|
||||
for filename in ("legislators-current.yaml", "legislators-historical.yaml"):
|
||||
path = legislators_dir / filename
|
||||
if not path.exists():
|
||||
logger.warning("Legislators file not found: %s", path)
|
||||
continue
|
||||
with path.open() as file:
|
||||
data = yaml.safe_load(file)
|
||||
if isinstance(data, list):
|
||||
legislators.extend(data)
|
||||
return legislators
|
||||
|
||||
|
||||
def _parse_legislator(entry: dict) -> dict:
|
||||
"""Extract Legislator fields from a congress-legislators YAML entry."""
|
||||
ids = entry.get("id", {})
|
||||
name = entry.get("name", {})
|
||||
bio = entry.get("bio", {})
|
||||
terms = entry.get("terms", [])
|
||||
latest_term = terms[-1] if terms else {}
|
||||
|
||||
fec_ids = ids.get("fec")
|
||||
fec_ids_joined = ",".join(fec_ids) if isinstance(fec_ids, list) else fec_ids
|
||||
|
||||
chamber = latest_term.get("type")
|
||||
chamber_normalized = {"rep": "House", "sen": "Senate"}.get(chamber, chamber)
|
||||
|
||||
return {
|
||||
"thomas_id": ids.get("thomas"),
|
||||
"lis_id": ids.get("lis"),
|
||||
"govtrack_id": ids.get("govtrack"),
|
||||
"opensecrets_id": ids.get("opensecrets"),
|
||||
"fec_ids": fec_ids_joined,
|
||||
"first_name": name.get("first"),
|
||||
"last_name": name.get("last"),
|
||||
"official_full_name": name.get("official_full"),
|
||||
"nickname": name.get("nickname"),
|
||||
"birthday": bio.get("birthday"),
|
||||
"gender": bio.get("gender"),
|
||||
"current_party": latest_term.get("party"),
|
||||
"current_state": latest_term.get("state"),
|
||||
"current_district": latest_term.get("district"),
|
||||
"current_chamber": chamber_normalized,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Social Media — loaded from legislators-social-media.yaml
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SOCIAL_MEDIA_PLATFORMS = {
|
||||
"twitter": "https://twitter.com/{account}",
|
||||
"facebook": "https://facebook.com/{account}",
|
||||
"youtube": "https://youtube.com/{account}",
|
||||
"instagram": "https://instagram.com/{account}",
|
||||
"mastodon": None,
|
||||
}
|
||||
|
||||
|
||||
def ingest_social_media(engine: Engine, legislators_dir: Path) -> None:
|
||||
"""Load social media accounts from legislators-social-media.yaml."""
|
||||
social_media_path = legislators_dir / "legislators-social-media.yaml"
|
||||
if not social_media_path.exists():
|
||||
logger.warning("Social media file not found: %s", social_media_path)
|
||||
return
|
||||
|
||||
with social_media_path.open() as file:
|
||||
social_media_data = yaml.safe_load(file)
|
||||
|
||||
if not isinstance(social_media_data, list):
|
||||
logger.warning("Unexpected format in %s", social_media_path)
|
||||
return
|
||||
|
||||
logger.info("Loaded %d entries from legislators-social-media.yaml", len(social_media_data))
|
||||
|
||||
with Session(engine) as session:
|
||||
legislator_map = _build_legislator_map(session)
|
||||
existing_accounts = {
|
||||
(account.legislator_id, account.platform)
|
||||
for account in session.scalars(select(LegislatorSocialMedia)).all()
|
||||
}
|
||||
logger.info("Found %d existing social media accounts in DB", len(existing_accounts))
|
||||
|
||||
total_inserted = 0
|
||||
total_updated = 0
|
||||
for entry in social_media_data:
|
||||
bioguide_id = entry.get("id", {}).get("bioguide")
|
||||
if not bioguide_id:
|
||||
continue
|
||||
|
||||
legislator_id = legislator_map.get(bioguide_id)
|
||||
if legislator_id is None:
|
||||
continue
|
||||
|
||||
social = entry.get("social", {})
|
||||
for platform, url_template in SOCIAL_MEDIA_PLATFORMS.items():
|
||||
account_name = social.get(platform)
|
||||
if not account_name:
|
||||
continue
|
||||
|
||||
url = url_template.format(account=account_name) if url_template else None
|
||||
|
||||
if (legislator_id, platform) in existing_accounts:
|
||||
total_updated += 1
|
||||
else:
|
||||
session.add(
|
||||
LegislatorSocialMedia(
|
||||
legislator_id=legislator_id,
|
||||
platform=platform,
|
||||
account_name=str(account_name),
|
||||
url=url,
|
||||
source="https://github.com/unitedstates/congress-legislators",
|
||||
)
|
||||
)
|
||||
existing_accounts.add((legislator_id, platform))
|
||||
total_inserted += 1
|
||||
|
||||
session.commit()
|
||||
logger.info("Inserted %d new social media accounts, updated %d existing", total_inserted, total_updated)
|
||||
|
||||
|
||||
def _iter_voters(position_group: object) -> Iterator[dict]:
|
||||
"""Yield voter dicts from a vote position group (handles list, single dict, or string)."""
|
||||
if isinstance(position_group, dict):
|
||||
yield position_group
|
||||
elif isinstance(position_group, list):
|
||||
for voter in position_group:
|
||||
if isinstance(voter, dict):
|
||||
yield voter
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bills
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def ingest_bills(engine: Engine, congress_dirs: list[Path]) -> None:
|
||||
"""Load bill data.json files."""
|
||||
with Session(engine) as session:
|
||||
existing_bills = {(bill.congress, bill.bill_type, bill.number) for bill in session.scalars(select(Bill)).all()}
|
||||
logger.info("Found %d existing bills in DB", len(existing_bills))
|
||||
|
||||
total_inserted = 0
|
||||
batch: list[Bill] = []
|
||||
for congress_dir in congress_dirs:
|
||||
bills_dir = congress_dir / "bills"
|
||||
if not bills_dir.is_dir():
|
||||
continue
|
||||
logger.info("Scanning bills from %s", congress_dir.name)
|
||||
for bill_file in bills_dir.rglob("data.json"):
|
||||
data = _read_json(bill_file)
|
||||
if data is None:
|
||||
continue
|
||||
bill = _parse_bill(data, existing_bills)
|
||||
if bill is not None:
|
||||
batch.append(bill)
|
||||
if len(batch) >= BATCH_SIZE:
|
||||
total_inserted += _flush_batch(session, batch, "bills")
|
||||
|
||||
total_inserted += _flush_batch(session, batch, "bills")
|
||||
logger.info("Inserted %d new bills total", total_inserted)
|
||||
|
||||
|
||||
def _parse_bill(data: dict, existing_bills: set[tuple[int, str, int]]) -> Bill | None:
|
||||
"""Parse a bill data.json dict into a Bill ORM object, skipping existing."""
|
||||
raw_congress = data.get("congress")
|
||||
bill_type = data.get("bill_type")
|
||||
raw_number = data.get("number")
|
||||
if raw_congress is None or bill_type is None or raw_number is None:
|
||||
return None
|
||||
congress = int(raw_congress)
|
||||
number = int(raw_number)
|
||||
if (congress, bill_type, number) in existing_bills:
|
||||
return None
|
||||
|
||||
sponsor_bioguide = None
|
||||
sponsor = data.get("sponsor")
|
||||
if sponsor:
|
||||
sponsor_bioguide = sponsor.get("bioguide_id")
|
||||
|
||||
return Bill(
|
||||
congress=congress,
|
||||
bill_type=bill_type,
|
||||
number=number,
|
||||
title=data.get("short_title") or data.get("official_title"),
|
||||
title_short=data.get("short_title"),
|
||||
official_title=data.get("official_title"),
|
||||
status=data.get("status"),
|
||||
status_at=data.get("status_at"),
|
||||
sponsor_bioguide_id=sponsor_bioguide,
|
||||
subjects_top_term=data.get("subjects_top_term"),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Votes (and vote records)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def ingest_votes(engine: Engine, congress_dirs: list[Path]) -> None:
|
||||
"""Load vote data.json files with their vote records."""
|
||||
with Session(engine) as session:
|
||||
legislator_map = _build_legislator_map(session)
|
||||
logger.info("Loaded %d legislators into lookup map", len(legislator_map))
|
||||
bill_map = _build_bill_map(session)
|
||||
logger.info("Loaded %d bills into lookup map", len(bill_map))
|
||||
existing_votes = {
|
||||
(vote.congress, vote.chamber, vote.session, vote.number) for vote in session.scalars(select(Vote)).all()
|
||||
}
|
||||
logger.info("Found %d existing votes in DB", len(existing_votes))
|
||||
|
||||
total_inserted = 0
|
||||
batch: list[Vote] = []
|
||||
for congress_dir in congress_dirs:
|
||||
votes_dir = congress_dir / "votes"
|
||||
if not votes_dir.is_dir():
|
||||
continue
|
||||
logger.info("Scanning votes from %s", congress_dir.name)
|
||||
for vote_file in votes_dir.rglob("data.json"):
|
||||
data = _read_json(vote_file)
|
||||
if data is None:
|
||||
continue
|
||||
vote = _parse_vote(data, legislator_map, bill_map, existing_votes)
|
||||
if vote is not None:
|
||||
batch.append(vote)
|
||||
if len(batch) >= BATCH_SIZE:
|
||||
total_inserted += _flush_batch(session, batch, "votes")
|
||||
|
||||
total_inserted += _flush_batch(session, batch, "votes")
|
||||
logger.info("Inserted %d new votes total", total_inserted)
|
||||
|
||||
|
||||
def _build_legislator_map(session: Session) -> dict[str, int]:
|
||||
"""Build a mapping of bioguide_id -> legislator.id."""
|
||||
return {legislator.bioguide_id: legislator.id for legislator in session.scalars(select(Legislator)).all()}
|
||||
|
||||
|
||||
def _build_bill_map(session: Session) -> dict[tuple[int, str, int], int]:
|
||||
"""Build a mapping of (congress, bill_type, number) -> bill.id."""
|
||||
return {(bill.congress, bill.bill_type, bill.number): bill.id for bill in session.scalars(select(Bill)).all()}
|
||||
|
||||
|
||||
def _parse_vote(
|
||||
data: dict,
|
||||
legislator_map: dict[str, int],
|
||||
bill_map: dict[tuple[int, str, int], int],
|
||||
existing_votes: set[tuple[int, str, int, int]],
|
||||
) -> Vote | None:
|
||||
"""Parse a vote data.json dict into a Vote ORM object with records."""
|
||||
raw_congress = data.get("congress")
|
||||
chamber = data.get("chamber")
|
||||
raw_number = data.get("number")
|
||||
vote_date = data.get("date")
|
||||
if raw_congress is None or chamber is None or raw_number is None or vote_date is None:
|
||||
return None
|
||||
|
||||
raw_session = data.get("session")
|
||||
if raw_session is None:
|
||||
return None
|
||||
|
||||
congress = int(raw_congress)
|
||||
number = int(raw_number)
|
||||
session_number = int(raw_session)
|
||||
|
||||
# Normalize chamber from "h"/"s" to "House"/"Senate"
|
||||
chamber_normalized = {"h": "House", "s": "Senate"}.get(chamber, chamber)
|
||||
|
||||
if (congress, chamber_normalized, session_number, number) in existing_votes:
|
||||
return None
|
||||
|
||||
# Resolve linked bill
|
||||
bill_id = None
|
||||
bill_ref = data.get("bill")
|
||||
if bill_ref:
|
||||
bill_key = (
|
||||
int(bill_ref.get("congress", congress)),
|
||||
bill_ref.get("type"),
|
||||
int(bill_ref.get("number", 0)),
|
||||
)
|
||||
bill_id = bill_map.get(bill_key)
|
||||
|
||||
raw_votes = data.get("votes", {})
|
||||
vote_counts = _count_votes(raw_votes)
|
||||
vote_records = _build_vote_records(raw_votes, legislator_map)
|
||||
|
||||
return Vote(
|
||||
congress=congress,
|
||||
chamber=chamber_normalized,
|
||||
session=session_number,
|
||||
number=number,
|
||||
vote_type=data.get("type"),
|
||||
question=data.get("question"),
|
||||
result=data.get("result"),
|
||||
result_text=data.get("result_text"),
|
||||
vote_date=vote_date[:10] if isinstance(vote_date, str) else vote_date,
|
||||
bill_id=bill_id,
|
||||
vote_records=vote_records,
|
||||
**vote_counts,
|
||||
)
|
||||
|
||||
|
||||
def _count_votes(raw_votes: dict) -> dict[str, int]:
|
||||
"""Count voters per position category, correctly handling dict and list formats."""
|
||||
yea_count = 0
|
||||
nay_count = 0
|
||||
not_voting_count = 0
|
||||
present_count = 0
|
||||
|
||||
for position, position_group in raw_votes.items():
|
||||
voter_count = sum(1 for _ in _iter_voters(position_group))
|
||||
if position in ("Yea", "Aye"):
|
||||
yea_count += voter_count
|
||||
elif position in ("Nay", "No"):
|
||||
nay_count += voter_count
|
||||
elif position == "Not Voting":
|
||||
not_voting_count += voter_count
|
||||
elif position == "Present":
|
||||
present_count += voter_count
|
||||
|
||||
return {
|
||||
"yea_count": yea_count,
|
||||
"nay_count": nay_count,
|
||||
"not_voting_count": not_voting_count,
|
||||
"present_count": present_count,
|
||||
}
|
||||
|
||||
|
||||
def _build_vote_records(raw_votes: dict, legislator_map: dict[str, int]) -> list[VoteRecord]:
|
||||
"""Build VoteRecord objects from raw vote data."""
|
||||
records: list[VoteRecord] = []
|
||||
for position, position_group in raw_votes.items():
|
||||
for voter in _iter_voters(position_group):
|
||||
bioguide_id = voter.get("id")
|
||||
if not bioguide_id:
|
||||
continue
|
||||
legislator_id = legislator_map.get(bioguide_id)
|
||||
if legislator_id is None:
|
||||
continue
|
||||
records.append(
|
||||
VoteRecord(
|
||||
legislator_id=legislator_id,
|
||||
position=position,
|
||||
)
|
||||
)
|
||||
return records
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bill Text
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def ingest_bill_text(engine: Engine, congress_dirs: list[Path]) -> None:
|
||||
"""Load bill text from text-versions directories."""
|
||||
with Session(engine) as session:
|
||||
bill_map = _build_bill_map(session)
|
||||
logger.info("Loaded %d bills into lookup map", len(bill_map))
|
||||
existing_bill_texts = {
|
||||
(bill_text.bill_id, bill_text.version_code) for bill_text in session.scalars(select(BillText)).all()
|
||||
}
|
||||
logger.info("Found %d existing bill text versions in DB", len(existing_bill_texts))
|
||||
|
||||
total_inserted = 0
|
||||
batch: list[BillText] = []
|
||||
for congress_dir in congress_dirs:
|
||||
logger.info("Scanning bill texts from %s", congress_dir.name)
|
||||
for bill_text in _iter_bill_texts(congress_dir, bill_map, existing_bill_texts):
|
||||
batch.append(bill_text)
|
||||
if len(batch) >= BATCH_SIZE:
|
||||
total_inserted += _flush_batch(session, batch, "bill texts")
|
||||
|
||||
total_inserted += _flush_batch(session, batch, "bill texts")
|
||||
logger.info("Inserted %d new bill text versions total", total_inserted)
|
||||
|
||||
|
||||
def _iter_bill_texts(
|
||||
congress_dir: Path,
|
||||
bill_map: dict[tuple[int, str, int], int],
|
||||
existing_bill_texts: set[tuple[int, str]],
|
||||
) -> Iterator[BillText]:
|
||||
"""Yield BillText objects for a single congress directory, skipping existing."""
|
||||
bills_dir = congress_dir / "bills"
|
||||
if not bills_dir.is_dir():
|
||||
return
|
||||
|
||||
for bill_dir in bills_dir.rglob("text-versions"):
|
||||
if not bill_dir.is_dir():
|
||||
continue
|
||||
bill_key = _bill_key_from_dir(bill_dir.parent, congress_dir)
|
||||
if bill_key is None:
|
||||
continue
|
||||
bill_id = bill_map.get(bill_key)
|
||||
if bill_id is None:
|
||||
continue
|
||||
|
||||
for version_dir in sorted(bill_dir.iterdir()):
|
||||
if not version_dir.is_dir():
|
||||
continue
|
||||
if (bill_id, version_dir.name) in existing_bill_texts:
|
||||
continue
|
||||
text_content = _read_bill_text(version_dir)
|
||||
version_data = _read_json(version_dir / "data.json")
|
||||
yield BillText(
|
||||
bill_id=bill_id,
|
||||
version_code=version_dir.name,
|
||||
version_name=version_data.get("version_name") if version_data else None,
|
||||
date=version_data.get("issued_on") if version_data else None,
|
||||
text_content=text_content,
|
||||
)
|
||||
|
||||
|
||||
def _bill_key_from_dir(bill_dir: Path, congress_dir: Path) -> tuple[int, str, int] | None:
|
||||
"""Extract (congress, bill_type, number) from directory structure."""
|
||||
congress = int(congress_dir.name)
|
||||
bill_type = bill_dir.parent.name
|
||||
name = bill_dir.name
|
||||
# Directory name is like "hr3590" — strip the type prefix to get the number
|
||||
number_str = name[len(bill_type) :]
|
||||
if not number_str.isdigit():
|
||||
return None
|
||||
return (congress, bill_type, int(number_str))
|
||||
|
||||
|
||||
def _read_bill_text(version_dir: Path) -> str | None:
|
||||
"""Read bill text from a version directory, preferring .txt over .xml."""
|
||||
for extension in ("txt", "htm", "html", "xml"):
|
||||
candidates = list(version_dir.glob(f"document.{extension}"))
|
||||
if not candidates:
|
||||
candidates = list(version_dir.glob(f"*.{extension}"))
|
||||
if candidates:
|
||||
try:
|
||||
return candidates[0].read_text(encoding="utf-8")
|
||||
except Exception:
|
||||
logger.exception("Failed to read %s", candidates[0])
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict | None:
|
||||
"""Read and parse a JSON file, returning None on failure."""
|
||||
try:
|
||||
return orjson.loads(path.read_bytes())
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except Exception:
|
||||
logger.exception("Failed to parse %s", path)
|
||||
return None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
@@ -1,247 +0,0 @@
|
||||
"""Ingestion pipeline for loading JSONL post files into the weekly-partitioned posts table.
|
||||
|
||||
Usage:
|
||||
ingest-posts /path/to/files/
|
||||
ingest-posts /path/to/single_file.jsonl
|
||||
ingest-posts /data/dir/ --workers 4 --batch-size 5000
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path # noqa: TC003 this is needed for typer
|
||||
from typing import TYPE_CHECKING, Annotated
|
||||
|
||||
import orjson
|
||||
import psycopg
|
||||
import typer
|
||||
|
||||
from python.common import configure_logger
|
||||
from python.orm.common import get_connection_info
|
||||
from python.parallelize import parallelize_process
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
app = typer.Typer(help="Ingest JSONL post files into the partitioned posts table.")
|
||||
|
||||
|
||||
@app.command()
|
||||
def main(
|
||||
path: Annotated[Path, typer.Argument(help="Directory containing JSONL files, or a single JSONL file")],
|
||||
batch_size: Annotated[int, typer.Option(help="Rows per INSERT batch")] = 10000,
|
||||
workers: Annotated[int, typer.Option(help="Parallel workers for multi-file ingestion")] = 4,
|
||||
pattern: Annotated[str, typer.Option(help="Glob pattern for JSONL files")] = "*.jsonl",
|
||||
) -> None:
|
||||
"""Ingest JSONL post files into the weekly-partitioned posts table."""
|
||||
configure_logger(level="INFO")
|
||||
|
||||
logger.info("starting ingest-posts")
|
||||
logger.info("path=%s batch_size=%d workers=%d pattern=%s", path, batch_size, workers, pattern)
|
||||
if path.is_file():
|
||||
ingest_file(path, batch_size=batch_size)
|
||||
elif path.is_dir():
|
||||
ingest_directory(path, batch_size=batch_size, max_workers=workers, pattern=pattern)
|
||||
else:
|
||||
typer.echo(f"Path does not exist: {path}", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
logger.info("ingest-posts done")
|
||||
|
||||
|
||||
def ingest_directory(
|
||||
directory: Path,
|
||||
*,
|
||||
batch_size: int,
|
||||
max_workers: int,
|
||||
pattern: str = "*.jsonl",
|
||||
) -> None:
|
||||
"""Ingest all JSONL files in a directory using parallel workers."""
|
||||
files = sorted(directory.glob(pattern))
|
||||
if not files:
|
||||
logger.warning("No JSONL files found in %s", directory)
|
||||
return
|
||||
|
||||
logger.info("Found %d JSONL files to ingest", len(files))
|
||||
|
||||
kwargs_list = [{"path": fp, "batch_size": batch_size} for fp in files]
|
||||
parallelize_process(ingest_file, kwargs_list, max_workers=max_workers)
|
||||
|
||||
|
||||
SCHEMA = "main"
|
||||
|
||||
COLUMNS = (
|
||||
"post_id",
|
||||
"user_id",
|
||||
"instance",
|
||||
"date",
|
||||
"text",
|
||||
"langs",
|
||||
"like_count",
|
||||
"reply_count",
|
||||
"repost_count",
|
||||
"reply_to",
|
||||
"replied_author",
|
||||
"thread_root",
|
||||
"thread_root_author",
|
||||
"repost_from",
|
||||
"reposted_author",
|
||||
"quotes",
|
||||
"quoted_author",
|
||||
"labels",
|
||||
"sent_label",
|
||||
"sent_score",
|
||||
)
|
||||
|
||||
INSERT_FROM_STAGING = f"""
|
||||
INSERT INTO {SCHEMA}.posts ({", ".join(COLUMNS)})
|
||||
SELECT {", ".join(COLUMNS)} FROM pg_temp.staging
|
||||
ON CONFLICT (post_id, date) DO NOTHING
|
||||
""" # noqa: S608
|
||||
|
||||
FAILED_INSERT = f"""
|
||||
INSERT INTO {SCHEMA}.failed_ingestion (raw_line, error)
|
||||
VALUES (%(raw_line)s, %(error)s)
|
||||
""" # noqa: S608
|
||||
|
||||
|
||||
def get_psycopg_connection() -> psycopg.Connection:
|
||||
"""Create a raw psycopg3 connection from environment variables."""
|
||||
database, host, port, username, password = get_connection_info("DATA_SCIENCE_DEV")
|
||||
return psycopg.connect(
|
||||
dbname=database,
|
||||
host=host,
|
||||
port=int(port),
|
||||
user=username,
|
||||
password=password,
|
||||
autocommit=False,
|
||||
)
|
||||
|
||||
|
||||
def ingest_file(path: Path, *, batch_size: int) -> None:
|
||||
"""Ingest a single JSONL file into the posts table."""
|
||||
log_trigger = max(100_000 // batch_size, 1)
|
||||
failed_lines: list[dict] = []
|
||||
try:
|
||||
with get_psycopg_connection() as connection:
|
||||
for index, batch in enumerate(read_jsonl_batches(path, batch_size, failed_lines), 1):
|
||||
ingest_batch(connection, batch)
|
||||
if index % log_trigger == 0:
|
||||
logger.info("Ingested %d batches (%d rows) from %s", index, index * batch_size, path)
|
||||
|
||||
if failed_lines:
|
||||
logger.warning("Recording %d malformed lines from %s", len(failed_lines), path.name)
|
||||
with connection.cursor() as cursor:
|
||||
cursor.executemany(FAILED_INSERT, failed_lines)
|
||||
connection.commit()
|
||||
except Exception:
|
||||
logger.exception("Failed to ingest file: %s", path)
|
||||
raise
|
||||
|
||||
|
||||
def ingest_batch(connection: psycopg.Connection, batch: list[dict]) -> None:
|
||||
"""COPY batch into a temp staging table, then INSERT ... ON CONFLICT into posts."""
|
||||
if not batch:
|
||||
return
|
||||
|
||||
try:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(f"""
|
||||
CREATE TEMP TABLE IF NOT EXISTS staging
|
||||
(LIKE {SCHEMA}.posts INCLUDING DEFAULTS)
|
||||
ON COMMIT DELETE ROWS
|
||||
""")
|
||||
cursor.execute("TRUNCATE pg_temp.staging")
|
||||
|
||||
with cursor.copy(f"COPY pg_temp.staging ({', '.join(COLUMNS)}) FROM STDIN") as copy:
|
||||
for row in batch:
|
||||
copy.write_row(tuple(row.get(column) for column in COLUMNS))
|
||||
|
||||
cursor.execute(INSERT_FROM_STAGING)
|
||||
connection.commit()
|
||||
except Exception as error:
|
||||
connection.rollback()
|
||||
|
||||
if len(batch) == 1:
|
||||
logger.exception("Skipping bad row post_id=%s", batch[0].get("post_id"))
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
FAILED_INSERT,
|
||||
{
|
||||
"raw_line": orjson.dumps(batch[0], default=str).decode(),
|
||||
"error": str(error),
|
||||
},
|
||||
)
|
||||
connection.commit()
|
||||
return
|
||||
|
||||
midpoint = len(batch) // 2
|
||||
ingest_batch(connection, batch[:midpoint])
|
||||
ingest_batch(connection, batch[midpoint:])
|
||||
|
||||
|
||||
def read_jsonl_batches(file_path: Path, batch_size: int, failed_lines: list[dict]) -> Iterator[list[dict]]:
|
||||
"""Stream a JSONL file and yield batches of transformed rows."""
|
||||
batch: list[dict] = []
|
||||
with file_path.open("r", encoding="utf-8") as handle:
|
||||
for raw_line in handle:
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
batch.extend(parse_line(line, file_path, failed_lines))
|
||||
if len(batch) >= batch_size:
|
||||
yield batch
|
||||
batch = []
|
||||
if batch:
|
||||
yield batch
|
||||
|
||||
|
||||
def parse_line(line: str, file_path: Path, failed_lines: list[dict]) -> Iterator[dict]:
|
||||
"""Parse a JSONL line, handling concatenated JSON objects."""
|
||||
try:
|
||||
yield transform_row(orjson.loads(line))
|
||||
except orjson.JSONDecodeError:
|
||||
if "}{" not in line:
|
||||
logger.warning("Skipping malformed line in %s: %s", file_path.name, line[:120])
|
||||
failed_lines.append({"raw_line": line, "error": "malformed JSON"})
|
||||
return
|
||||
fragments = line.replace("}{", "}\n{").split("\n")
|
||||
for fragment in fragments:
|
||||
try:
|
||||
yield transform_row(orjson.loads(fragment))
|
||||
except (orjson.JSONDecodeError, KeyError, ValueError) as error:
|
||||
logger.warning("Skipping malformed fragment in %s: %s", file_path.name, fragment[:120])
|
||||
failed_lines.append({"raw_line": fragment, "error": str(error)})
|
||||
except Exception as error:
|
||||
logger.exception("Skipping bad row in %s: %s", file_path.name, line[:120])
|
||||
failed_lines.append({"raw_line": line, "error": str(error)})
|
||||
|
||||
|
||||
def transform_row(raw: dict) -> dict:
|
||||
"""Transform a raw JSONL row into a dict matching the Posts table columns."""
|
||||
raw["date"] = parse_date(raw["date"])
|
||||
if raw.get("langs") is not None:
|
||||
raw["langs"] = orjson.dumps(raw["langs"])
|
||||
if raw.get("text") is not None:
|
||||
raw["text"] = raw["text"].replace("\x00", "")
|
||||
return raw
|
||||
|
||||
|
||||
def parse_date(raw_date: int) -> datetime:
|
||||
"""Parse compact YYYYMMDDHHmm integer into a naive datetime (input is UTC by spec)."""
|
||||
return datetime(
|
||||
raw_date // 100000000,
|
||||
(raw_date // 1000000) % 100,
|
||||
(raw_date // 10000) % 100,
|
||||
(raw_date // 100) % 100,
|
||||
raw_date % 100,
|
||||
tzinfo=UTC,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
@@ -83,20 +83,6 @@ DATABASES: dict[str, DatabaseConfig] = {
|
||||
base_class_name="VanInventoryBase",
|
||||
models_module="python.orm.van_inventory.models",
|
||||
),
|
||||
"signal_bot": DatabaseConfig(
|
||||
env_prefix="SIGNALBOT",
|
||||
version_location="python/alembic/signal_bot/versions",
|
||||
base_module="python.orm.signal_bot.base",
|
||||
base_class_name="SignalBotBase",
|
||||
models_module="python.orm.signal_bot.models",
|
||||
),
|
||||
"data_science_dev": DatabaseConfig(
|
||||
env_prefix="DATA_SCIENCE_DEV",
|
||||
version_location="python/alembic/data_science_dev/versions",
|
||||
base_module="python.orm.data_science_dev.base",
|
||||
base_class_name="DataScienceDevBase",
|
||||
models_module="python.orm.data_science_dev.models",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,13 +1,9 @@
|
||||
"""ORM package exports."""
|
||||
|
||||
from python.orm.data_science_dev.base import DataScienceDevBase
|
||||
from python.orm.richie.base import RichieBase
|
||||
from python.orm.signal_bot.base import SignalBotBase
|
||||
from python.orm.van_inventory.base import VanInventoryBase
|
||||
|
||||
__all__ = [
|
||||
"DataScienceDevBase",
|
||||
"RichieBase",
|
||||
"SignalBotBase",
|
||||
"VanInventoryBase",
|
||||
]
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
"""Data science dev database ORM exports."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from python.orm.data_science_dev.base import DataScienceDevBase, DataScienceDevTableBase, DataScienceDevTableBaseBig
|
||||
|
||||
__all__ = [
|
||||
"DataScienceDevBase",
|
||||
"DataScienceDevTableBase",
|
||||
"DataScienceDevTableBaseBig",
|
||||
]
|
||||
@@ -1,52 +0,0 @@
|
||||
"""Data science dev database ORM base."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import BigInteger, DateTime, MetaData, func
|
||||
from sqlalchemy.ext.declarative import AbstractConcreteBase
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
|
||||
from python.orm.common import NAMING_CONVENTION
|
||||
|
||||
|
||||
class DataScienceDevBase(DeclarativeBase):
|
||||
"""Base class for data_science_dev database ORM models."""
|
||||
|
||||
schema_name = "main"
|
||||
|
||||
metadata = MetaData(
|
||||
schema=schema_name,
|
||||
naming_convention=NAMING_CONVENTION,
|
||||
)
|
||||
|
||||
|
||||
class _TableMixin:
|
||||
"""Shared timestamp columns for all table bases."""
|
||||
|
||||
created: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
server_default=func.now(),
|
||||
)
|
||||
updated: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
server_default=func.now(),
|
||||
onupdate=func.now(),
|
||||
)
|
||||
|
||||
|
||||
class DataScienceDevTableBase(_TableMixin, AbstractConcreteBase, DataScienceDevBase):
|
||||
"""Table with Integer primary key."""
|
||||
|
||||
__abstract__ = True
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
|
||||
|
||||
class DataScienceDevTableBaseBig(_TableMixin, AbstractConcreteBase, DataScienceDevBase):
|
||||
"""Table with BigInteger primary key."""
|
||||
|
||||
__abstract__ = True
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
@@ -1,14 +0,0 @@
|
||||
"""init."""
|
||||
|
||||
from python.orm.data_science_dev.congress.bill import Bill, BillText
|
||||
from python.orm.data_science_dev.congress.legislator import Legislator, LegislatorSocialMedia
|
||||
from python.orm.data_science_dev.congress.vote import Vote, VoteRecord
|
||||
|
||||
__all__ = [
|
||||
"Bill",
|
||||
"BillText",
|
||||
"Legislator",
|
||||
"LegislatorSocialMedia",
|
||||
"Vote",
|
||||
"VoteRecord",
|
||||
]
|
||||
@@ -1,66 +0,0 @@
|
||||
"""Bill model - legislation introduced in Congress."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import ForeignKey, Index, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from python.orm.data_science_dev.base import DataScienceDevTableBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from python.orm.data_science_dev.congress.vote import Vote
|
||||
|
||||
|
||||
class Bill(DataScienceDevTableBase):
|
||||
"""Legislation with congress number, type, titles, status, and sponsor."""
|
||||
|
||||
__tablename__ = "bill"
|
||||
|
||||
congress: Mapped[int]
|
||||
bill_type: Mapped[str]
|
||||
number: Mapped[int]
|
||||
|
||||
title: Mapped[str | None]
|
||||
title_short: Mapped[str | None]
|
||||
official_title: Mapped[str | None]
|
||||
|
||||
status: Mapped[str | None]
|
||||
status_at: Mapped[date | None]
|
||||
|
||||
sponsor_bioguide_id: Mapped[str | None]
|
||||
|
||||
subjects_top_term: Mapped[str | None]
|
||||
|
||||
votes: Mapped[list[Vote]] = relationship(
|
||||
"Vote",
|
||||
back_populates="bill",
|
||||
)
|
||||
bill_texts: Mapped[list[BillText]] = relationship(
|
||||
"BillText",
|
||||
back_populates="bill",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("congress", "bill_type", "number", name="uq_bill_congress_type_number"),
|
||||
Index("ix_bill_congress", "congress"),
|
||||
)
|
||||
|
||||
|
||||
class BillText(DataScienceDevTableBase):
|
||||
"""Stores different text versions of a bill (introduced, enrolled, etc.)."""
|
||||
|
||||
__tablename__ = "bill_text"
|
||||
|
||||
bill_id: Mapped[int] = mapped_column(ForeignKey("main.bill.id", ondelete="CASCADE"))
|
||||
version_code: Mapped[str]
|
||||
version_name: Mapped[str | None]
|
||||
text_content: Mapped[str | None]
|
||||
date: Mapped[date | None]
|
||||
|
||||
bill: Mapped[Bill] = relationship("Bill", back_populates="bill_texts")
|
||||
|
||||
__table_args__ = (UniqueConstraint("bill_id", "version_code", name="uq_bill_text_bill_id_version_code"),)
|
||||
@@ -1,66 +0,0 @@
|
||||
"""Legislator model - members of Congress."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import ForeignKey, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from python.orm.data_science_dev.base import DataScienceDevTableBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from python.orm.data_science_dev.congress.vote import VoteRecord
|
||||
|
||||
|
||||
class Legislator(DataScienceDevTableBase):
|
||||
"""Members of Congress with identification and current term info."""
|
||||
|
||||
__tablename__ = "legislator"
|
||||
|
||||
bioguide_id: Mapped[str] = mapped_column(Text, unique=True, index=True)
|
||||
|
||||
thomas_id: Mapped[str | None]
|
||||
lis_id: Mapped[str | None]
|
||||
govtrack_id: Mapped[int | None]
|
||||
opensecrets_id: Mapped[str | None]
|
||||
fec_ids: Mapped[str | None]
|
||||
|
||||
first_name: Mapped[str]
|
||||
last_name: Mapped[str]
|
||||
official_full_name: Mapped[str | None]
|
||||
nickname: Mapped[str | None]
|
||||
|
||||
birthday: Mapped[date | None]
|
||||
gender: Mapped[str | None]
|
||||
|
||||
current_party: Mapped[str | None]
|
||||
current_state: Mapped[str | None]
|
||||
current_district: Mapped[int | None]
|
||||
current_chamber: Mapped[str | None]
|
||||
|
||||
social_media_accounts: Mapped[list[LegislatorSocialMedia]] = relationship(
|
||||
"LegislatorSocialMedia",
|
||||
back_populates="legislator",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
vote_records: Mapped[list[VoteRecord]] = relationship(
|
||||
"VoteRecord",
|
||||
back_populates="legislator",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
|
||||
|
||||
class LegislatorSocialMedia(DataScienceDevTableBase):
|
||||
"""Social media account linked to a legislator."""
|
||||
|
||||
__tablename__ = "legislator_social_media"
|
||||
|
||||
legislator_id: Mapped[int] = mapped_column(ForeignKey("main.legislator.id"))
|
||||
platform: Mapped[str]
|
||||
account_name: Mapped[str]
|
||||
url: Mapped[str | None]
|
||||
source: Mapped[str]
|
||||
|
||||
legislator: Mapped[Legislator] = relationship(back_populates="social_media_accounts")
|
||||
@@ -1,79 +0,0 @@
|
||||
"""Vote model - roll call votes in Congress."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import ForeignKey, Index, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from python.orm.data_science_dev.base import DataScienceDevBase, DataScienceDevTableBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from python.orm.data_science_dev.congress.bill import Bill
|
||||
from python.orm.data_science_dev.congress.legislator import Legislator
|
||||
from python.orm.data_science_dev.congress.vote import Vote
|
||||
|
||||
|
||||
class VoteRecord(DataScienceDevBase):
|
||||
"""Links a vote to a legislator with their position (Yea, Nay, etc.)."""
|
||||
|
||||
__tablename__ = "vote_record"
|
||||
|
||||
vote_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("main.vote.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
)
|
||||
legislator_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("main.legislator.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
)
|
||||
position: Mapped[str]
|
||||
|
||||
vote: Mapped[Vote] = relationship("Vote", back_populates="vote_records")
|
||||
legislator: Mapped[Legislator] = relationship("Legislator", back_populates="vote_records")
|
||||
|
||||
|
||||
class Vote(DataScienceDevTableBase):
|
||||
"""Roll call votes with counts and optional bill linkage."""
|
||||
|
||||
__tablename__ = "vote"
|
||||
|
||||
congress: Mapped[int]
|
||||
chamber: Mapped[str]
|
||||
session: Mapped[int]
|
||||
number: Mapped[int]
|
||||
|
||||
vote_type: Mapped[str | None]
|
||||
question: Mapped[str | None]
|
||||
result: Mapped[str | None]
|
||||
result_text: Mapped[str | None]
|
||||
|
||||
vote_date: Mapped[date]
|
||||
|
||||
yea_count: Mapped[int | None]
|
||||
nay_count: Mapped[int | None]
|
||||
not_voting_count: Mapped[int | None]
|
||||
present_count: Mapped[int | None]
|
||||
|
||||
bill_id: Mapped[int | None] = mapped_column(ForeignKey("main.bill.id"))
|
||||
|
||||
bill: Mapped[Bill | None] = relationship("Bill", back_populates="votes")
|
||||
vote_records: Mapped[list[VoteRecord]] = relationship(
|
||||
"VoteRecord",
|
||||
back_populates="vote",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"congress",
|
||||
"chamber",
|
||||
"session",
|
||||
"number",
|
||||
name="uq_vote_congress_chamber_session_number",
|
||||
),
|
||||
Index("ix_vote_date", "vote_date"),
|
||||
Index("ix_vote_congress_chamber", "congress", "chamber"),
|
||||
)
|
||||
@@ -1,16 +0,0 @@
|
||||
"""Data science dev database ORM models."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from python.orm.data_science_dev.congress import Bill, BillText, Legislator, Vote, VoteRecord
|
||||
from python.orm.data_science_dev.posts import partitions # noqa: F401 — registers partition classes in metadata
|
||||
from python.orm.data_science_dev.posts.tables import Posts
|
||||
|
||||
__all__ = [
|
||||
"Bill",
|
||||
"BillText",
|
||||
"Legislator",
|
||||
"Posts",
|
||||
"Vote",
|
||||
"VoteRecord",
|
||||
]
|
||||
@@ -1,11 +0,0 @@
|
||||
"""Posts module — weekly-partitioned posts table and partition ORM models."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from python.orm.data_science_dev.posts.failed_ingestion import FailedIngestion
|
||||
from python.orm.data_science_dev.posts.tables import Posts
|
||||
|
||||
__all__ = [
|
||||
"FailedIngestion",
|
||||
"Posts",
|
||||
]
|
||||
@@ -1,33 +0,0 @@
|
||||
"""Shared column definitions for the posts partitioned table family."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import BigInteger, SmallInteger, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
|
||||
class PostsColumns:
|
||||
"""Mixin providing all posts columns. Used by both the parent table and partitions."""
|
||||
|
||||
post_id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(BigInteger)
|
||||
instance: Mapped[str]
|
||||
date: Mapped[datetime] = mapped_column(primary_key=True)
|
||||
text: Mapped[str] = mapped_column(Text)
|
||||
langs: Mapped[str | None]
|
||||
like_count: Mapped[int]
|
||||
reply_count: Mapped[int]
|
||||
repost_count: Mapped[int]
|
||||
reply_to: Mapped[int | None] = mapped_column(BigInteger)
|
||||
replied_author: Mapped[int | None] = mapped_column(BigInteger)
|
||||
thread_root: Mapped[int | None] = mapped_column(BigInteger)
|
||||
thread_root_author: Mapped[int | None] = mapped_column(BigInteger)
|
||||
repost_from: Mapped[int | None] = mapped_column(BigInteger)
|
||||
reposted_author: Mapped[int | None] = mapped_column(BigInteger)
|
||||
quotes: Mapped[int | None] = mapped_column(BigInteger)
|
||||
quoted_author: Mapped[int | None] = mapped_column(BigInteger)
|
||||
labels: Mapped[str | None]
|
||||
sent_label: Mapped[int | None] = mapped_column(SmallInteger)
|
||||
sent_score: Mapped[float | None]
|
||||
@@ -1,17 +0,0 @@
|
||||
"""Table for storing JSONL lines that failed during post ingestion."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from python.orm.data_science_dev.base import DataScienceDevTableBase
|
||||
|
||||
|
||||
class FailedIngestion(DataScienceDevTableBase):
|
||||
"""Stores raw JSONL lines and their error messages when ingestion fails."""
|
||||
|
||||
__tablename__ = "failed_ingestion"
|
||||
|
||||
raw_line: Mapped[str] = mapped_column(Text)
|
||||
error: Mapped[str] = mapped_column(Text)
|
||||
@@ -1,71 +0,0 @@
|
||||
"""Dynamically generated ORM classes for each weekly partition of the posts table.
|
||||
|
||||
Each class maps to a PostgreSQL partition table (e.g. posts_2024_01).
|
||||
These are real ORM models tracked by Alembic autogenerate.
|
||||
|
||||
Uses ISO week numbering (datetime.isocalendar().week). ISO years can have
|
||||
52 or 53 weeks, and week boundaries are always Monday to Monday.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from python.orm.data_science_dev.base import DataScienceDevBase
|
||||
from python.orm.data_science_dev.posts.columns import PostsColumns
|
||||
|
||||
PARTITION_START_YEAR = 2023
|
||||
PARTITION_END_YEAR = 2026
|
||||
|
||||
_current_module = sys.modules[__name__]
|
||||
|
||||
|
||||
def iso_weeks_in_year(year: int) -> int:
|
||||
"""Return the number of ISO weeks in a given year (52 or 53)."""
|
||||
dec_28 = datetime(year, 12, 28, tzinfo=UTC)
|
||||
return dec_28.isocalendar().week
|
||||
|
||||
|
||||
def week_bounds(year: int, week: int) -> tuple[datetime, datetime]:
|
||||
"""Return (start, end) datetimes for an ISO week.
|
||||
|
||||
Start = Monday 00:00:00 UTC of the given ISO week.
|
||||
End = Monday 00:00:00 UTC of the following ISO week.
|
||||
"""
|
||||
start = datetime.fromisocalendar(year, week, 1).replace(tzinfo=UTC)
|
||||
if week < iso_weeks_in_year(year):
|
||||
end = datetime.fromisocalendar(year, week + 1, 1).replace(tzinfo=UTC)
|
||||
else:
|
||||
end = datetime.fromisocalendar(year + 1, 1, 1).replace(tzinfo=UTC)
|
||||
return start, end
|
||||
|
||||
|
||||
def _build_partition_classes() -> dict[str, type]:
|
||||
"""Generate one ORM class per ISO week partition."""
|
||||
classes: dict[str, type] = {}
|
||||
|
||||
for year in range(PARTITION_START_YEAR, PARTITION_END_YEAR + 1):
|
||||
for week in range(1, iso_weeks_in_year(year) + 1):
|
||||
class_name = f"PostsWeek{year}W{week:02d}"
|
||||
table_name = f"posts_{year}_{week:02d}"
|
||||
|
||||
partition_class = type(
|
||||
class_name,
|
||||
(PostsColumns, DataScienceDevBase),
|
||||
{
|
||||
"__tablename__": table_name,
|
||||
"__table_args__": ({"implicit_returning": False},),
|
||||
},
|
||||
)
|
||||
|
||||
classes[class_name] = partition_class
|
||||
|
||||
return classes
|
||||
|
||||
|
||||
# Generate all partition classes and register them on this module
|
||||
_partition_classes = _build_partition_classes()
|
||||
for _name, _cls in _partition_classes.items():
|
||||
setattr(_current_module, _name, _cls)
|
||||
__all__ = list(_partition_classes.keys())
|
||||
@@ -1,13 +0,0 @@
|
||||
"""Posts parent table with PostgreSQL weekly range partitioning on date column."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from python.orm.data_science_dev.base import DataScienceDevBase
|
||||
from python.orm.data_science_dev.posts.columns import PostsColumns
|
||||
|
||||
|
||||
class Posts(PostsColumns, DataScienceDevBase):
|
||||
"""Parent partitioned table for posts, partitioned by week on `date`."""
|
||||
|
||||
__tablename__ = "posts"
|
||||
__table_args__ = ({"postgresql_partition_by": "RANGE (date)"},)
|
||||
@@ -1,16 +0,0 @@
|
||||
"""Signal bot database ORM exports."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from python.orm.signal_bot.base import SignalBotBase, SignalBotTableBase, SignalBotTableBaseSmall
|
||||
from python.orm.signal_bot.models import DeadLetterMessage, DeviceRole, RoleRecord, SignalDevice
|
||||
|
||||
__all__ = [
|
||||
"DeadLetterMessage",
|
||||
"DeviceRole",
|
||||
"RoleRecord",
|
||||
"SignalBotBase",
|
||||
"SignalBotTableBase",
|
||||
"SignalBotTableBaseSmall",
|
||||
"SignalDevice",
|
||||
]
|
||||
@@ -1,52 +0,0 @@
|
||||
"""Signal bot database ORM base."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, MetaData, SmallInteger, func
|
||||
from sqlalchemy.ext.declarative import AbstractConcreteBase
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
|
||||
from python.orm.common import NAMING_CONVENTION
|
||||
|
||||
|
||||
class SignalBotBase(DeclarativeBase):
|
||||
"""Base class for signal_bot database ORM models."""
|
||||
|
||||
schema_name = "main"
|
||||
|
||||
metadata = MetaData(
|
||||
schema=schema_name,
|
||||
naming_convention=NAMING_CONVENTION,
|
||||
)
|
||||
|
||||
|
||||
class _TableMixin:
|
||||
"""Shared timestamp columns for all table bases."""
|
||||
|
||||
created: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
server_default=func.now(),
|
||||
)
|
||||
updated: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
server_default=func.now(),
|
||||
onupdate=func.now(),
|
||||
)
|
||||
|
||||
|
||||
class SignalBotTableBaseSmall(_TableMixin, AbstractConcreteBase, SignalBotBase):
|
||||
"""Table with SmallInteger primary key."""
|
||||
|
||||
__abstract__ = True
|
||||
|
||||
id: Mapped[int] = mapped_column(SmallInteger, primary_key=True)
|
||||
|
||||
|
||||
class SignalBotTableBase(_TableMixin, AbstractConcreteBase, SignalBotBase):
|
||||
"""Table with Integer primary key."""
|
||||
|
||||
__abstract__ = True
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
@@ -1,62 +0,0 @@
|
||||
"""Signal bot device, role, and dead letter ORM models."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, Enum, ForeignKey, SmallInteger, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from python.orm.signal_bot.base import SignalBotTableBase, SignalBotTableBaseSmall
|
||||
from python.signal_bot.models import MessageStatus, TrustLevel
|
||||
|
||||
|
||||
class RoleRecord(SignalBotTableBaseSmall):
|
||||
"""Lookup table for RBAC roles, keyed by smallint."""
|
||||
|
||||
__tablename__ = "role"
|
||||
|
||||
name: Mapped[str] = mapped_column(String(50), unique=True)
|
||||
|
||||
|
||||
class DeviceRole(SignalBotTableBase):
|
||||
"""Association between a device and a role."""
|
||||
|
||||
__tablename__ = "device_role"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("device_id", "role_id", name="uq_device_role_device_role"),
|
||||
{"schema": "main"},
|
||||
)
|
||||
|
||||
device_id: Mapped[int] = mapped_column(ForeignKey("main.signal_device.id"))
|
||||
role_id: Mapped[int] = mapped_column(SmallInteger, ForeignKey("main.role.id"))
|
||||
|
||||
|
||||
class SignalDevice(SignalBotTableBase):
|
||||
"""A Signal device tracked by phone number and safety number."""
|
||||
|
||||
__tablename__ = "signal_device"
|
||||
|
||||
phone_number: Mapped[str] = mapped_column(String(50), unique=True)
|
||||
safety_number: Mapped[str | None]
|
||||
trust_level: Mapped[TrustLevel] = mapped_column(
|
||||
Enum(TrustLevel, name="trust_level", create_constraint=False, native_enum=False),
|
||||
default=TrustLevel.UNVERIFIED,
|
||||
)
|
||||
last_seen: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
roles: Mapped[list[RoleRecord]] = relationship(secondary=DeviceRole.__table__)
|
||||
|
||||
|
||||
class DeadLetterMessage(SignalBotTableBase):
|
||||
"""A Signal message that failed processing and was sent to the dead letter queue."""
|
||||
|
||||
__tablename__ = "dead_letter_message"
|
||||
|
||||
source: Mapped[str]
|
||||
message: Mapped[str] = mapped_column(Text)
|
||||
received_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
status: Mapped[MessageStatus] = mapped_column(
|
||||
Enum(MessageStatus, name="message_status", create_constraint=False, native_enum=False),
|
||||
default=MessageStatus.UNPROCESSED,
|
||||
)
|
||||
@@ -1 +0,0 @@
|
||||
"""Signal command and control bot."""
|
||||
@@ -1 +0,0 @@
|
||||
"""Signal bot commands."""
|
||||
@@ -1,137 +0,0 @@
|
||||
"""Van inventory command — parse receipts and item lists via LLM, push to API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import httpx
|
||||
|
||||
from python.signal_bot.models import InventoryItem, InventoryUpdate
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from python.signal_bot.llm_client import LLMClient
|
||||
from python.signal_bot.models import SignalMessage
|
||||
from python.signal_bot.signal_client import SignalClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SYSTEM_PROMPT = """\
|
||||
You are an inventory assistant. Extract items from the input and return ONLY
|
||||
a JSON array. Each element must have these fields:
|
||||
- "name": item name (string)
|
||||
- "quantity": numeric count or amount (default 1)
|
||||
- "unit": unit of measure (e.g. "each", "lb", "oz", "gallon", "bag", "box")
|
||||
- "category": category like "food", "tools", "supplies", etc.
|
||||
- "notes": any extra detail (empty string if none)
|
||||
|
||||
Example output:
|
||||
[{"name": "water bottles", "quantity": 6, "unit": "gallon", "category": "supplies", "notes": "1 gallon each"}]
|
||||
|
||||
Return ONLY the JSON array, no other text.\
|
||||
"""
|
||||
|
||||
IMAGE_PROMPT = "Extract all items from this receipt or inventory photo."
|
||||
TEXT_PROMPT = "Extract all items from this inventory list."
|
||||
|
||||
|
||||
def parse_llm_response(raw: str) -> list[InventoryItem]:
|
||||
"""Parse the LLM JSON response into InventoryItem list."""
|
||||
text = raw.strip()
|
||||
# Strip markdown code fences if present
|
||||
if text.startswith("```"):
|
||||
lines = text.split("\n")
|
||||
lines = [line for line in lines if not line.startswith("```")]
|
||||
text = "\n".join(lines)
|
||||
|
||||
items_data: list[dict[str, Any]] = json.loads(text)
|
||||
return [InventoryItem.model_validate(item) for item in items_data]
|
||||
|
||||
|
||||
def _upsert_item(api_url: str, item: InventoryItem) -> None:
|
||||
"""Create or update an item via the van_inventory API.
|
||||
|
||||
Fetches existing items, and if one with the same name exists,
|
||||
patches its quantity (summing). Otherwise creates a new item.
|
||||
"""
|
||||
base = api_url.rstrip("/")
|
||||
response = httpx.get(f"{base}/api/items", timeout=10)
|
||||
response.raise_for_status()
|
||||
existing: list[dict[str, Any]] = response.json()
|
||||
|
||||
match = next((e for e in existing if e["name"].lower() == item.name.lower()), None)
|
||||
|
||||
if match:
|
||||
new_qty = match["quantity"] + item.quantity
|
||||
patch = {"quantity": new_qty}
|
||||
if item.category:
|
||||
patch["category"] = item.category
|
||||
response = httpx.patch(f"{base}/api/items/{match['id']}", json=patch, timeout=10)
|
||||
response.raise_for_status()
|
||||
return
|
||||
payload = {
|
||||
"name": item.name,
|
||||
"quantity": item.quantity,
|
||||
"unit": item.unit,
|
||||
"category": item.category or None,
|
||||
}
|
||||
response = httpx.post(f"{base}/api/items", json=payload, timeout=10)
|
||||
response.raise_for_status()
|
||||
|
||||
|
||||
def handle_inventory_update(
|
||||
message: SignalMessage,
|
||||
signal: SignalClient,
|
||||
llm: LLMClient,
|
||||
api_url: str,
|
||||
) -> InventoryUpdate:
|
||||
"""Process an inventory update from a Signal message.
|
||||
|
||||
Accepts either an image (receipt photo) or text list.
|
||||
Uses the LLM to extract structured items, then pushes to the van_inventory API.
|
||||
"""
|
||||
try:
|
||||
logger.info(f"Processing inventory update from {message.source}")
|
||||
if message.attachments:
|
||||
image_data = signal.get_attachment(message.attachments[0])
|
||||
raw_response = llm.chat(
|
||||
IMAGE_PROMPT,
|
||||
image_data=image_data,
|
||||
system=SYSTEM_PROMPT,
|
||||
)
|
||||
source_type = "receipt_photo"
|
||||
elif message.message.strip():
|
||||
raw_response = llm.chat(
|
||||
f"{TEXT_PROMPT}\n\n{message.message}",
|
||||
system=SYSTEM_PROMPT,
|
||||
)
|
||||
source_type = "text_list"
|
||||
else:
|
||||
signal.reply(message, "Send a photo of a receipt or a text list of items to update inventory.")
|
||||
return InventoryUpdate()
|
||||
|
||||
logger.info(f"{raw_response=}")
|
||||
|
||||
new_items = parse_llm_response(raw_response)
|
||||
|
||||
logger.info(f"{new_items=}")
|
||||
|
||||
for item in new_items:
|
||||
_upsert_item(api_url, item)
|
||||
|
||||
summary = _format_summary(new_items)
|
||||
signal.reply(message, f"Inventory updated with {len(new_items)} item(s):\n{summary}")
|
||||
|
||||
return InventoryUpdate(items=new_items, raw_response=raw_response, source_type=source_type)
|
||||
|
||||
except Exception:
|
||||
logger.exception("Failed to process inventory update")
|
||||
signal.reply(message, "Failed to process inventory update. Check logs for details.")
|
||||
return InventoryUpdate()
|
||||
|
||||
|
||||
def _format_summary(items: list[InventoryItem]) -> str:
|
||||
"""Format items into a readable summary."""
|
||||
lines = [f" - {item.name} x{item.quantity} {item.unit} [{item.category}]" for item in items]
|
||||
return "\n".join(lines)
|
||||
@@ -1,64 +0,0 @@
|
||||
"""Location command for the Signal bot."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import httpx
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from python.signal_bot.models import SignalMessage
|
||||
from python.signal_bot.signal_client import SignalClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_entity_state(ha_url: str, ha_token: str, entity_id: str) -> dict[str, Any]:
|
||||
"""Fetch an entity's state from Home Assistant."""
|
||||
entity_url = f"{ha_url}/api/states/{entity_id}"
|
||||
logger.debug(f"Fetching {entity_url=}")
|
||||
response = httpx.get(
|
||||
entity_url,
|
||||
headers={"Authorization": f"Bearer {ha_token}"},
|
||||
timeout=30,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
|
||||
def _format_location(latitude: str, longitude: str) -> str:
|
||||
"""Render a friendly location response."""
|
||||
return f"Van location: {latitude}, {longitude}\nhttps://maps.google.com/?q={latitude},{longitude}"
|
||||
|
||||
|
||||
def handle_location_request(
|
||||
message: SignalMessage,
|
||||
signal: SignalClient,
|
||||
ha_url: str | None,
|
||||
ha_token: str | None,
|
||||
) -> None:
|
||||
"""Reply with van location from Home Assistant."""
|
||||
if ha_url is None or ha_token is None:
|
||||
signal.reply(message, "Location command is not configured (missing HA_URL or HA_TOKEN).")
|
||||
return
|
||||
|
||||
lat_payload = None
|
||||
lon_payload = None
|
||||
try:
|
||||
lat_payload = _get_entity_state(ha_url, ha_token, "sensor.van_last_known_latitude")
|
||||
lon_payload = _get_entity_state(ha_url, ha_token, "sensor.van_last_known_longitude")
|
||||
except httpx.HTTPError:
|
||||
logger.exception("Couldn't fetch van location from Home Assistant right now.")
|
||||
logger.debug(f"{ha_url=} {lat_payload=} {lon_payload=}")
|
||||
signal.reply(message, "Couldn't fetch van location from Home Assistant right now.")
|
||||
return
|
||||
|
||||
latitude = lat_payload.get("state", "")
|
||||
longitude = lon_payload.get("state", "")
|
||||
|
||||
if not latitude or not longitude or latitude == "unavailable" or longitude == "unavailable":
|
||||
signal.reply(message, "Van location is unavailable in Home Assistant right now.")
|
||||
return
|
||||
|
||||
signal.reply(message, _format_location(latitude, longitude))
|
||||
@@ -1,284 +0,0 @@
|
||||
"""Device registry — tracks verified/unverified devices by safety number."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from typing import TYPE_CHECKING, NamedTuple
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from python.common import utcnow
|
||||
from python.orm.signal_bot.models import RoleRecord, SignalDevice
|
||||
from python.signal_bot.models import Role, TrustLevel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from python.signal_bot.signal_client import SignalClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_BLOCKED_TTL = timedelta(minutes=60)
|
||||
_DEFAULT_TTL = timedelta(minutes=5)
|
||||
|
||||
|
||||
class _CacheEntry(NamedTuple):
|
||||
expires: datetime
|
||||
trust_level: TrustLevel
|
||||
has_safety_number: bool
|
||||
safety_number: str | None
|
||||
roles: list[Role]
|
||||
|
||||
|
||||
class DeviceRegistry:
|
||||
"""Manage device trust based on Signal safety numbers.
|
||||
|
||||
Devices start as UNVERIFIED. An admin verifies them over SSH by calling
|
||||
``verify(phone_number)`` which marks the device VERIFIED and also tells
|
||||
signal-cli to trust the identity.
|
||||
|
||||
Only VERIFIED devices may execute commands.
|
||||
"""
|
||||
|
||||
def __init__(self, signal_client: SignalClient, engine: Engine) -> None:
|
||||
self.signal_client = signal_client
|
||||
self.engine = engine
|
||||
self._contact_cache: dict[str, _CacheEntry] = {}
|
||||
|
||||
def is_verified(self, phone_number: str) -> bool:
|
||||
"""Check if a phone number is verified."""
|
||||
if entry := self._cached(phone_number):
|
||||
return entry.trust_level == TrustLevel.VERIFIED
|
||||
device = self._load_device(phone_number)
|
||||
return device is not None and device.trust_level == TrustLevel.VERIFIED
|
||||
|
||||
def record_contact(self, phone_number: str, safety_number: str | None = None) -> None:
|
||||
"""Record seeing a device. Creates entry if new, updates last_seen."""
|
||||
now = utcnow()
|
||||
|
||||
entry = self._cached(phone_number)
|
||||
if entry and entry.safety_number == safety_number:
|
||||
return
|
||||
|
||||
with Session(self.engine) as session:
|
||||
device = session.scalars(
|
||||
select(SignalDevice).where(SignalDevice.phone_number == phone_number)
|
||||
).one_or_none()
|
||||
|
||||
if device:
|
||||
if device.safety_number != safety_number and device.trust_level != TrustLevel.BLOCKED:
|
||||
logger.warning(f"Safety number changed for {phone_number}, resetting to UNVERIFIED")
|
||||
device.safety_number = safety_number
|
||||
device.trust_level = TrustLevel.UNVERIFIED
|
||||
device.last_seen = now
|
||||
else:
|
||||
device = SignalDevice(
|
||||
phone_number=phone_number,
|
||||
safety_number=safety_number,
|
||||
trust_level=TrustLevel.UNVERIFIED,
|
||||
last_seen=now,
|
||||
)
|
||||
session.add(device)
|
||||
logger.info(f"New device registered: {phone_number}")
|
||||
|
||||
session.commit()
|
||||
self._update_cache(phone_number, device)
|
||||
|
||||
def has_safety_number(self, phone_number: str) -> bool:
|
||||
"""Check if a device has a safety number on file."""
|
||||
if entry := self._cached(phone_number):
|
||||
return entry.has_safety_number
|
||||
device = self._load_device(phone_number)
|
||||
return device is not None and device.safety_number is not None
|
||||
|
||||
def verify(self, phone_number: str) -> bool:
|
||||
"""Mark a device as verified. Called by admin over SSH.
|
||||
|
||||
Returns True if the device was found and verified.
|
||||
"""
|
||||
with Session(self.engine) as session:
|
||||
device = session.scalars(
|
||||
select(SignalDevice).where(SignalDevice.phone_number == phone_number)
|
||||
).one_or_none()
|
||||
|
||||
if not device:
|
||||
logger.warning(f"Cannot verify unknown device: {phone_number}")
|
||||
return False
|
||||
|
||||
device.trust_level = TrustLevel.VERIFIED
|
||||
self.signal_client.trust_identity(phone_number, trust_all_known_keys=True)
|
||||
session.commit()
|
||||
self._update_cache(phone_number, device)
|
||||
logger.info(f"Device verified: {phone_number}")
|
||||
return True
|
||||
|
||||
def block(self, phone_number: str) -> bool:
|
||||
"""Block a device."""
|
||||
return self._set_trust(phone_number, TrustLevel.BLOCKED, "Device blocked")
|
||||
|
||||
def unverify(self, phone_number: str) -> bool:
|
||||
"""Reset a device to unverified."""
|
||||
return self._set_trust(phone_number, TrustLevel.UNVERIFIED)
|
||||
|
||||
# -- role management ------------------------------------------------------
|
||||
|
||||
def get_roles(self, phone_number: str) -> list[Role]:
|
||||
"""Return the roles for a device, defaulting to empty."""
|
||||
if entry := self._cached(phone_number):
|
||||
return entry.roles
|
||||
device = self._load_device(phone_number)
|
||||
return _extract_roles(device) if device else []
|
||||
|
||||
def has_role(self, phone_number: str, role: Role) -> bool:
|
||||
"""Check if a device has a specific role or is admin."""
|
||||
roles = self.get_roles(phone_number)
|
||||
return Role.ADMIN in roles or role in roles
|
||||
|
||||
def grant_role(self, phone_number: str, role: Role) -> bool:
|
||||
"""Add a role to a device. Called by admin over SSH."""
|
||||
with Session(self.engine) as session:
|
||||
device = session.scalars(
|
||||
select(SignalDevice).where(SignalDevice.phone_number == phone_number)
|
||||
).one_or_none()
|
||||
|
||||
if not device:
|
||||
logger.warning(f"Cannot grant role for unknown device: {phone_number}")
|
||||
return False
|
||||
|
||||
if any(record.name == role for record in device.roles):
|
||||
return True
|
||||
|
||||
role_record = session.scalars(select(RoleRecord).where(RoleRecord.name == role)).one_or_none()
|
||||
|
||||
if not role_record:
|
||||
logger.warning(f"Unknown role: {role}")
|
||||
return False
|
||||
|
||||
device.roles.append(role_record)
|
||||
session.commit()
|
||||
self._update_cache(phone_number, device)
|
||||
logger.info(f"Device {phone_number} granted role {role}")
|
||||
return True
|
||||
|
||||
def revoke_role(self, phone_number: str, role: Role) -> bool:
|
||||
"""Remove a role from a device. Called by admin over SSH."""
|
||||
with Session(self.engine) as session:
|
||||
device = session.scalars(
|
||||
select(SignalDevice).where(SignalDevice.phone_number == phone_number)
|
||||
).one_or_none()
|
||||
|
||||
if not device:
|
||||
logger.warning(f"Cannot revoke role for unknown device: {phone_number}")
|
||||
return False
|
||||
|
||||
device.roles = [record for record in device.roles if record.name != role]
|
||||
session.commit()
|
||||
self._update_cache(phone_number, device)
|
||||
logger.info(f"Device {phone_number} revoked role {role}")
|
||||
return True
|
||||
|
||||
def set_roles(self, phone_number: str, roles: list[Role]) -> bool:
|
||||
"""Replace all roles for a device. Called by admin over SSH."""
|
||||
with Session(self.engine) as session:
|
||||
device = session.scalars(
|
||||
select(SignalDevice).where(SignalDevice.phone_number == phone_number)
|
||||
).one_or_none()
|
||||
|
||||
if not device:
|
||||
logger.warning(f"Cannot set roles for unknown device: {phone_number}")
|
||||
return False
|
||||
|
||||
role_names = [str(role) for role in roles]
|
||||
records = session.scalars(select(RoleRecord).where(RoleRecord.name.in_(role_names))).all()
|
||||
device.roles = records
|
||||
session.commit()
|
||||
self._update_cache(phone_number, device)
|
||||
logger.info(f"Device {phone_number} roles set to {role_names}")
|
||||
return True
|
||||
|
||||
# -- queries --------------------------------------------------------------
|
||||
|
||||
def list_devices(self) -> list[SignalDevice]:
|
||||
"""Return all known devices."""
|
||||
with Session(self.engine) as session:
|
||||
return list(session.scalars(select(SignalDevice)).all())
|
||||
|
||||
def sync_identities(self) -> None:
|
||||
"""Pull identity list from signal-cli and record any new ones."""
|
||||
identities = self.signal_client.get_identities()
|
||||
for identity in identities:
|
||||
number = identity.get("number", "")
|
||||
safety = identity.get("safety_number", identity.get("fingerprint", ""))
|
||||
if number:
|
||||
self.record_contact(number, safety)
|
||||
|
||||
# -- internals ------------------------------------------------------------
|
||||
|
||||
def _cached(self, phone_number: str) -> _CacheEntry | None:
|
||||
"""Return the cache entry if it exists and hasn't expired."""
|
||||
entry = self._contact_cache.get(phone_number)
|
||||
if entry and utcnow() < entry.expires:
|
||||
return entry
|
||||
return None
|
||||
|
||||
def _load_device(self, phone_number: str) -> SignalDevice | None:
|
||||
"""Fetch a device by phone number (with joined roles)."""
|
||||
with Session(self.engine) as session:
|
||||
return session.scalars(select(SignalDevice).where(SignalDevice.phone_number == phone_number)).one_or_none()
|
||||
|
||||
def _update_cache(self, phone_number: str, device: SignalDevice) -> None:
|
||||
"""Refresh the cache entry for a device."""
|
||||
ttl = _BLOCKED_TTL if device.trust_level == TrustLevel.BLOCKED else _DEFAULT_TTL
|
||||
self._contact_cache[phone_number] = _CacheEntry(
|
||||
expires=utcnow() + ttl,
|
||||
trust_level=device.trust_level,
|
||||
has_safety_number=device.safety_number is not None,
|
||||
safety_number=device.safety_number,
|
||||
roles=_extract_roles(device),
|
||||
)
|
||||
|
||||
def _set_trust(self, phone_number: str, level: str, log_msg: str | None = None) -> bool:
|
||||
"""Update the trust level for a device."""
|
||||
with Session(self.engine) as session:
|
||||
device = session.scalars(
|
||||
select(SignalDevice).where(SignalDevice.phone_number == phone_number)
|
||||
).one_or_none()
|
||||
|
||||
if not device:
|
||||
return False
|
||||
|
||||
device.trust_level = level
|
||||
session.commit()
|
||||
self._update_cache(phone_number, device)
|
||||
if log_msg:
|
||||
logger.info(f"{log_msg}: {phone_number}")
|
||||
return True
|
||||
|
||||
|
||||
def _extract_roles(device: SignalDevice) -> list[Role]:
|
||||
"""Convert a device's RoleRecord objects to a list of Role enums."""
|
||||
return [Role(record.name) for record in device.roles]
|
||||
|
||||
|
||||
def sync_roles(engine: Engine) -> None:
|
||||
"""Sync the Role enum to the role table, adding new and removing stale entries."""
|
||||
expected = {role.value for role in Role}
|
||||
|
||||
with Session(engine) as session:
|
||||
existing = set(session.scalars(select(RoleRecord.name)).all())
|
||||
|
||||
to_add = expected - existing
|
||||
to_remove = existing - expected
|
||||
|
||||
for name in to_add:
|
||||
session.add(RoleRecord(name=name))
|
||||
logger.info(f"Role added: {name}")
|
||||
|
||||
if to_remove:
|
||||
session.execute(delete(RoleRecord).where(RoleRecord.name.in_(to_remove)))
|
||||
for name in to_remove:
|
||||
logger.info(f"Role removed: {name}")
|
||||
|
||||
session.commit()
|
||||
@@ -1,80 +0,0 @@
|
||||
"""Flexible LLM client for ollama backends."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import logging
|
||||
from typing import Any, Self
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LLMClient:
|
||||
"""Talk to an ollama instance.
|
||||
|
||||
Args:
|
||||
model: Ollama model name.
|
||||
host: Ollama host.
|
||||
port: Ollama port.
|
||||
temperature: Sampling temperature.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
model: str,
|
||||
host: str,
|
||||
port: int = 11434,
|
||||
temperature: float = 0.1,
|
||||
timeout: int = 300,
|
||||
) -> None:
|
||||
self.model = model
|
||||
self.temperature = temperature
|
||||
self._client = httpx.Client(base_url=f"http://{host}:{port}", timeout=timeout)
|
||||
|
||||
def chat(self, prompt: str, image_data: bytes | None = None, system: str | None = None) -> str:
|
||||
"""Send a text prompt and return the response."""
|
||||
messages: list[dict[str, Any]] = []
|
||||
if system:
|
||||
messages.append({"role": "system", "content": system})
|
||||
|
||||
user_msg = {"role": "user", "content": prompt}
|
||||
if image_data:
|
||||
user_msg["images"] = [base64.b64encode(image_data).decode()]
|
||||
|
||||
messages.append(user_msg)
|
||||
return self._generate(messages)
|
||||
|
||||
def _generate(self, messages: list[dict[str, Any]]) -> str:
|
||||
"""Call the ollama chat API."""
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"options": {"temperature": self.temperature},
|
||||
}
|
||||
logger.info(f"LLM request to {self.model}")
|
||||
response = self._client.post("/api/chat", json=payload)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data["message"]["content"]
|
||||
|
||||
def list_models(self) -> list[str]:
|
||||
"""List available models on the ollama instance."""
|
||||
response = self._client.get("/api/tags")
|
||||
response.raise_for_status()
|
||||
return [m["name"] for m in response.json().get("models", [])]
|
||||
|
||||
def __enter__(self) -> Self:
|
||||
"""Enter the context manager."""
|
||||
return self
|
||||
|
||||
def __exit__(self, *args: object) -> None:
|
||||
"""Close the HTTP client on exit."""
|
||||
self.close()
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the HTTP client."""
|
||||
self._client.close()
|
||||
@@ -1,239 +0,0 @@
|
||||
"""Signal command and control bot — main entry point."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from os import getenv
|
||||
from typing import TYPE_CHECKING, Annotated
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
import typer
|
||||
from alembic.command import upgrade
|
||||
from sqlalchemy.orm import Session
|
||||
from tenacity import before_sleep_log, retry, stop_after_attempt, wait_exponential
|
||||
|
||||
from python.common import configure_logger, utcnow
|
||||
from python.database_cli import DATABASES
|
||||
from python.orm.common import get_postgres_engine
|
||||
from python.orm.signal_bot.models import DeadLetterMessage
|
||||
from python.signal_bot.commands.inventory import handle_inventory_update
|
||||
from python.signal_bot.commands.location import handle_location_request
|
||||
from python.signal_bot.device_registry import DeviceRegistry, sync_roles
|
||||
from python.signal_bot.llm_client import LLMClient
|
||||
from python.signal_bot.models import BotConfig, MessageStatus, Role, SignalMessage
|
||||
from python.signal_bot.signal_client import SignalClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Command:
|
||||
"""A registered bot command."""
|
||||
|
||||
action: Callable[[SignalMessage, str], None]
|
||||
help_text: str
|
||||
role: Role | None # None = no role required (always allowed)
|
||||
|
||||
|
||||
class Bot:
|
||||
"""Holds shared resources and dispatches incoming messages to command handlers."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
signal: SignalClient,
|
||||
llm: LLMClient,
|
||||
registry: DeviceRegistry,
|
||||
config: BotConfig,
|
||||
) -> None:
|
||||
self.signal = signal
|
||||
self.llm = llm
|
||||
self.registry = registry
|
||||
self.config = config
|
||||
self.commands: dict[str, Command] = {
|
||||
"help": Command(action=self._help, help_text="show this help message", role=None),
|
||||
"status": Command(action=self._status, help_text="show bot status", role=Role.STATUS),
|
||||
"inventory": Command(
|
||||
action=self._inventory,
|
||||
help_text="update van inventory from a text list or receipt photo",
|
||||
role=Role.INVENTORY,
|
||||
),
|
||||
"location": Command(
|
||||
action=self._location,
|
||||
help_text="get current van location",
|
||||
role=Role.LOCATION,
|
||||
),
|
||||
}
|
||||
|
||||
# -- actions --------------------------------------------------------------
|
||||
|
||||
def _help(self, message: SignalMessage, _cmd: str) -> None:
|
||||
"""Return help text filtered to the sender's roles."""
|
||||
self.signal.reply(message, self._build_help(self.registry.get_roles(message.source)))
|
||||
|
||||
def _status(self, message: SignalMessage, _cmd: str) -> None:
|
||||
"""Return the status of the bot."""
|
||||
models = self.llm.list_models()
|
||||
model_list = ", ".join(models[:10])
|
||||
device_count = len(self.registry.list_devices())
|
||||
self.signal.reply(
|
||||
message,
|
||||
f"Bot online.\nLLM: {self.llm.model}\nAvailable models: {model_list}\nKnown devices: {device_count}",
|
||||
)
|
||||
|
||||
def _inventory(self, message: SignalMessage, _cmd: str) -> None:
|
||||
"""Process an inventory update."""
|
||||
handle_inventory_update(message, self.signal, self.llm, self.config.inventory_api_url)
|
||||
|
||||
def _location(self, message: SignalMessage, _cmd: str) -> None:
|
||||
"""Reply with current van location."""
|
||||
handle_location_request(message, self.signal, self.config.ha_url, self.config.ha_token)
|
||||
|
||||
# -- dispatch -------------------------------------------------------------
|
||||
|
||||
def _build_help(self, roles: list[Role]) -> str:
|
||||
"""Build help text showing only the commands the user can access."""
|
||||
is_admin = Role.ADMIN in roles
|
||||
lines = ["Available commands:"]
|
||||
for name, cmd in self.commands.items():
|
||||
if cmd.role is None or is_admin or cmd.role in roles:
|
||||
lines.append(f" {name:20s} — {cmd.help_text}")
|
||||
return "\n".join(lines)
|
||||
|
||||
def dispatch(self, message: SignalMessage) -> None:
|
||||
"""Route an incoming message to the right command handler."""
|
||||
source = message.source
|
||||
|
||||
if not self.registry.is_verified(source):
|
||||
logger.info(f"Device {source} not verified, ignoring message")
|
||||
return
|
||||
|
||||
if not self.registry.has_safety_number(source) and self.registry.has_role(source, Role.ADMIN):
|
||||
logger.warning(f"Admin device {source} missing safety number, ignoring message")
|
||||
return
|
||||
|
||||
text = message.message.strip()
|
||||
parts = text.split()
|
||||
|
||||
if not parts and not message.attachments:
|
||||
return
|
||||
|
||||
cmd = parts[0].lower() if parts else ""
|
||||
|
||||
logger.info(f"f{source=} running {cmd=} with {message=}")
|
||||
|
||||
command = self.commands.get(cmd)
|
||||
if command is None:
|
||||
if message.attachments:
|
||||
command = self.commands["inventory"]
|
||||
cmd = "inventory"
|
||||
else:
|
||||
return
|
||||
|
||||
if command.role is not None and not self.registry.has_role(source, command.role):
|
||||
logger.warning(f"Device {source} denied access to {cmd!r}")
|
||||
self.signal.reply(message, f"Permission denied: you do not have the '{command.role}' role.")
|
||||
return
|
||||
|
||||
command.action(message, cmd)
|
||||
|
||||
def process_message(self, message: SignalMessage) -> None:
|
||||
"""Process a single message, sending it to the dead letter queue after repeated failures."""
|
||||
max_attempts = self.config.max_message_attempts
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
safety_number = self.signal.get_safety_number(message.source)
|
||||
self.registry.record_contact(message.source, safety_number)
|
||||
self.dispatch(message)
|
||||
except Exception:
|
||||
logger.exception(f"Failed to process message (attempt {attempt}/{max_attempts})")
|
||||
else:
|
||||
return
|
||||
|
||||
logger.error(f"Message from {message.source} failed {max_attempts} times, sending to dead letter queue")
|
||||
with Session(self.config.engine) as session:
|
||||
session.add(
|
||||
DeadLetterMessage(
|
||||
source=message.source,
|
||||
message=message.message,
|
||||
received_at=utcnow(),
|
||||
status=MessageStatus.UNPROCESSED,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
def run(self) -> None:
|
||||
"""Listen for messages via WebSocket, reconnecting on failure."""
|
||||
logger.info("Bot started — listening via WebSocket")
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(self.config.max_retries),
|
||||
wait=wait_exponential(multiplier=self.config.reconnect_delay, max=self.config.max_reconnect_delay),
|
||||
before_sleep=before_sleep_log(logger, logging.WARNING),
|
||||
reraise=True,
|
||||
)
|
||||
def _listen() -> None:
|
||||
for message in self.signal.listen():
|
||||
logger.info(f"Message from {message.source}: {message.message[:80]}")
|
||||
self.process_message(message)
|
||||
|
||||
try:
|
||||
_listen()
|
||||
except Exception:
|
||||
logger.critical("Max retries exceeded, shutting down")
|
||||
raise
|
||||
|
||||
|
||||
def main(
|
||||
log_level: Annotated[str, typer.Option()] = "DEBUG",
|
||||
llm_timeout: Annotated[int, typer.Option()] = 600,
|
||||
) -> None:
|
||||
"""Run the Signal command and control bot."""
|
||||
configure_logger(log_level)
|
||||
signal_api_url = getenv("SIGNAL_API_URL")
|
||||
phone_number = getenv("SIGNAL_PHONE_NUMBER")
|
||||
inventory_api_url = getenv("INVENTORY_API_URL")
|
||||
|
||||
if signal_api_url is None:
|
||||
error = "SIGNAL_API_URL environment variable not set"
|
||||
raise ValueError(error)
|
||||
if phone_number is None:
|
||||
error = "SIGNAL_PHONE_NUMBER environment variable not set"
|
||||
raise ValueError(error)
|
||||
if inventory_api_url is None:
|
||||
error = "INVENTORY_API_URL environment variable not set"
|
||||
raise ValueError(error)
|
||||
|
||||
signal_bot_config = DATABASES["signal_bot"].alembic_config()
|
||||
upgrade(signal_bot_config, "head")
|
||||
engine = get_postgres_engine(name="SIGNALBOT")
|
||||
sync_roles(engine)
|
||||
config = BotConfig(
|
||||
signal_api_url=signal_api_url,
|
||||
phone_number=phone_number,
|
||||
inventory_api_url=inventory_api_url,
|
||||
ha_url=getenv("HA_URL"),
|
||||
ha_token=getenv("HA_TOKEN"),
|
||||
engine=engine,
|
||||
)
|
||||
|
||||
llm_host = getenv("LLM_HOST")
|
||||
llm_model = getenv("LLM_MODEL", "qwen3-vl:32b")
|
||||
llm_port = int(getenv("LLM_PORT", "11434"))
|
||||
if llm_host is None:
|
||||
error = "LLM_HOST environment variable not set"
|
||||
raise ValueError(error)
|
||||
|
||||
with (
|
||||
SignalClient(config.signal_api_url, config.phone_number) as signal,
|
||||
LLMClient(model=llm_model, host=llm_host, port=llm_port, timeout=llm_timeout) as llm,
|
||||
):
|
||||
registry = DeviceRegistry(signal, engine)
|
||||
bot = Bot(signal, llm, registry, config)
|
||||
bot.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
typer.run(main)
|
||||
@@ -1,97 +0,0 @@
|
||||
"""Models for the Signal command and control bot."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime # noqa: TC003 - pydantic needs this at runtime
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from sqlalchemy.engine import Engine # noqa: TC002 - pydantic needs this at runtime
|
||||
|
||||
|
||||
class TrustLevel(StrEnum):
|
||||
"""Device trust level."""
|
||||
|
||||
VERIFIED = "verified"
|
||||
UNVERIFIED = "unverified"
|
||||
BLOCKED = "blocked"
|
||||
|
||||
|
||||
class Role(StrEnum):
|
||||
"""RBAC roles — one per command, plus admin which grants all."""
|
||||
|
||||
ADMIN = "admin"
|
||||
STATUS = "status"
|
||||
INVENTORY = "inventory"
|
||||
LOCATION = "location"
|
||||
|
||||
|
||||
class MessageStatus(StrEnum):
|
||||
"""Dead letter queue message status."""
|
||||
|
||||
UNPROCESSED = "unprocessed"
|
||||
PROCESSED = "processed"
|
||||
|
||||
|
||||
class Device(BaseModel):
|
||||
"""A registered device tracked by safety number."""
|
||||
|
||||
phone_number: str
|
||||
safety_number: str
|
||||
trust_level: TrustLevel = TrustLevel.UNVERIFIED
|
||||
first_seen: datetime
|
||||
last_seen: datetime
|
||||
|
||||
|
||||
class SignalMessage(BaseModel):
|
||||
"""An incoming Signal message."""
|
||||
|
||||
source: str
|
||||
timestamp: int
|
||||
message: str = ""
|
||||
attachments: list[str] = []
|
||||
group_id: str | None = None
|
||||
is_receipt: bool = False
|
||||
|
||||
|
||||
class SignalEnvelope(BaseModel):
|
||||
"""Raw envelope from signal-cli-rest-api."""
|
||||
|
||||
envelope: dict[str, Any]
|
||||
account: str | None = None
|
||||
|
||||
|
||||
class InventoryItem(BaseModel):
|
||||
"""An item in the van inventory."""
|
||||
|
||||
name: str
|
||||
quantity: float = 1
|
||||
unit: str = "each"
|
||||
category: str = ""
|
||||
notes: str = ""
|
||||
|
||||
|
||||
class InventoryUpdate(BaseModel):
|
||||
"""Result of processing an inventory update."""
|
||||
|
||||
items: list[InventoryItem] = []
|
||||
raw_response: str = ""
|
||||
source_type: str = "" # "receipt_photo" or "text_list"
|
||||
|
||||
|
||||
class BotConfig(BaseModel):
|
||||
"""Top-level bot configuration."""
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
signal_api_url: str
|
||||
phone_number: str
|
||||
inventory_api_url: str
|
||||
ha_url: str | None = None
|
||||
ha_token: str | None = None
|
||||
engine: Engine
|
||||
reconnect_delay: int = 5
|
||||
max_reconnect_delay: int = 300
|
||||
max_retries: int = 10
|
||||
max_message_attempts: int = 3
|
||||
@@ -1,141 +0,0 @@
|
||||
"""Client for the signal-cli-rest-api."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, Self
|
||||
|
||||
import httpx
|
||||
import websockets.sync.client
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
|
||||
from python.signal_bot.models import SignalMessage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _parse_envelope(envelope: dict[str, Any]) -> SignalMessage | None:
|
||||
"""Parse a signal-cli envelope into a SignalMessage, or None if not a data message."""
|
||||
data_message = envelope.get("dataMessage")
|
||||
if not data_message:
|
||||
return None
|
||||
|
||||
attachment_ids = [att["id"] for att in data_message.get("attachments", []) if "id" in att]
|
||||
|
||||
group_info = data_message.get("groupInfo")
|
||||
group_id = group_info.get("groupId") if group_info else None
|
||||
|
||||
return SignalMessage(
|
||||
source=envelope.get("source", ""),
|
||||
timestamp=envelope.get("timestamp", 0),
|
||||
message=data_message.get("message", "") or "",
|
||||
attachments=attachment_ids,
|
||||
group_id=group_id,
|
||||
)
|
||||
|
||||
|
||||
class SignalClient:
|
||||
"""Communicate with signal-cli-rest-api.
|
||||
|
||||
Args:
|
||||
base_url: URL of the signal-cli-rest-api (e.g. http://localhost:8989).
|
||||
phone_number: The registered phone number to send/receive as.
|
||||
"""
|
||||
|
||||
def __init__(self, base_url: str, phone_number: str) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.phone_number = phone_number
|
||||
self._client = httpx.Client(base_url=self.base_url, timeout=30)
|
||||
|
||||
def _ws_url(self) -> str:
|
||||
"""Build the WebSocket URL from the base HTTP URL."""
|
||||
url = self.base_url.replace("http://", "ws://").replace("https://", "wss://")
|
||||
return f"{url}/v1/receive/{self.phone_number}"
|
||||
|
||||
def listen(self) -> Generator[SignalMessage]:
|
||||
"""Connect via WebSocket and yield messages as they arrive."""
|
||||
ws_url = self._ws_url()
|
||||
logger.info(f"Connecting to WebSocket: {ws_url}")
|
||||
|
||||
with websockets.sync.client.connect(ws_url) as ws:
|
||||
for raw in ws:
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
envelope = data.get("envelope", {})
|
||||
message = _parse_envelope(envelope)
|
||||
if message:
|
||||
yield message
|
||||
except json.JSONDecodeError:
|
||||
logger.warning(f"Non-JSON WebSocket frame: {raw[:200]}")
|
||||
|
||||
def send(self, recipient: str, message: str) -> None:
|
||||
"""Send a text message."""
|
||||
payload = {
|
||||
"message": message,
|
||||
"number": self.phone_number,
|
||||
"recipients": [recipient],
|
||||
}
|
||||
response = self._client.post("/v2/send", json=payload)
|
||||
response.raise_for_status()
|
||||
|
||||
def send_to_group(self, group_id: str, message: str) -> None:
|
||||
"""Send a message to a group."""
|
||||
payload = {
|
||||
"message": message,
|
||||
"number": self.phone_number,
|
||||
"recipients": [group_id],
|
||||
}
|
||||
response = self._client.post("/v2/send", json=payload)
|
||||
response.raise_for_status()
|
||||
|
||||
def get_attachment(self, attachment_id: str) -> bytes:
|
||||
"""Download an attachment by ID."""
|
||||
response = self._client.get(f"/v1/attachments/{attachment_id}")
|
||||
response.raise_for_status()
|
||||
return response.content
|
||||
|
||||
def get_identities(self) -> list[dict[str, Any]]:
|
||||
"""List known identities and their trust levels."""
|
||||
response = self._client.get(f"/v1/identities/{self.phone_number}")
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
def get_safety_number(self, phone_number: str) -> str | None:
|
||||
"""Look up the safety number for a contact from signal-cli's local store."""
|
||||
for identity in self.get_identities():
|
||||
if identity.get("number") == phone_number:
|
||||
return identity.get("safety_number", identity.get("fingerprint", ""))
|
||||
return None
|
||||
|
||||
def trust_identity(self, number_to_trust: str, *, trust_all_known_keys: bool = False) -> None:
|
||||
"""Trust an identity (verify safety number)."""
|
||||
payload: dict[str, Any] = {}
|
||||
if trust_all_known_keys:
|
||||
payload["trust_all_known_keys"] = True
|
||||
response = self._client.put(
|
||||
f"/v1/identities/{self.phone_number}/trust/{number_to_trust}",
|
||||
json=payload,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
def reply(self, message: SignalMessage, text: str) -> None:
|
||||
"""Reply to a message, routing to group or individual."""
|
||||
if message.group_id:
|
||||
self.send_to_group(message.group_id, text)
|
||||
else:
|
||||
self.send(message.source, text)
|
||||
|
||||
def __enter__(self) -> Self:
|
||||
"""Enter the context manager."""
|
||||
return self
|
||||
|
||||
def __exit__(self, *args: object) -> None:
|
||||
"""Close the HTTP client on exit."""
|
||||
self.close()
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the HTTP client."""
|
||||
self._client.close()
|
||||
@@ -1,40 +0,0 @@
|
||||
"""Shared test fixtures."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine, event
|
||||
|
||||
from python.orm.signal_bot.base import SignalBotBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def sqlite_engine() -> Generator[Engine]:
|
||||
"""Create an in-memory SQLite engine for testing."""
|
||||
engine = create_engine("sqlite:///:memory:")
|
||||
|
||||
@event.listens_for(engine, "connect")
|
||||
def _set_sqlite_pragma(dbapi_connection, _connection_record):
|
||||
cursor = dbapi_connection.cursor()
|
||||
cursor.execute("PRAGMA foreign_keys=ON")
|
||||
cursor.close()
|
||||
|
||||
SignalBotBase.metadata.create_all(engine)
|
||||
yield engine
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def engine(sqlite_engine: Engine) -> Generator[Engine]:
|
||||
"""Yield the shared engine after cleaning all tables between tests."""
|
||||
yield sqlite_engine
|
||||
with sqlite_engine.begin() as connection:
|
||||
for table in reversed(SignalBotBase.metadata.sorted_tables):
|
||||
connection.execute(table.delete())
|
||||
@@ -1,321 +0,0 @@
|
||||
"""Tests for the Signal command and control bot."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import timedelta
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from python.signal_bot.commands.inventory import (
|
||||
_format_summary,
|
||||
parse_llm_response,
|
||||
)
|
||||
from python.signal_bot.commands.location import _format_location, handle_location_request
|
||||
from python.signal_bot.device_registry import _BLOCKED_TTL, _DEFAULT_TTL, DeviceRegistry, _CacheEntry
|
||||
from python.signal_bot.llm_client import LLMClient
|
||||
from python.signal_bot.main import Bot
|
||||
from python.signal_bot.models import (
|
||||
BotConfig,
|
||||
InventoryItem,
|
||||
SignalMessage,
|
||||
TrustLevel,
|
||||
)
|
||||
from python.signal_bot.signal_client import SignalClient
|
||||
|
||||
|
||||
class TestModels:
|
||||
def test_trust_level_values(self):
|
||||
assert TrustLevel.VERIFIED == "verified"
|
||||
assert TrustLevel.UNVERIFIED == "unverified"
|
||||
assert TrustLevel.BLOCKED == "blocked"
|
||||
|
||||
def test_signal_message_defaults(self):
|
||||
msg = SignalMessage(source="+1234", timestamp=0)
|
||||
assert msg.message == ""
|
||||
assert msg.attachments == []
|
||||
assert msg.group_id is None
|
||||
|
||||
def test_inventory_item_defaults(self):
|
||||
item = InventoryItem(name="wrench")
|
||||
assert item.quantity == 1
|
||||
assert item.unit == "each"
|
||||
assert item.category == ""
|
||||
|
||||
|
||||
class TestInventoryParsing:
|
||||
def test_parse_llm_response_basic(self):
|
||||
raw = '[{"name": "water", "quantity": 6, "unit": "gallon", "category": "supplies", "notes": ""}]'
|
||||
items = parse_llm_response(raw)
|
||||
assert len(items) == 1
|
||||
assert items[0].name == "water"
|
||||
assert items[0].quantity == 6
|
||||
assert items[0].unit == "gallon"
|
||||
|
||||
def test_parse_llm_response_with_code_fence(self):
|
||||
raw = '```json\n[{"name": "tape", "quantity": 1, "unit": "each", "category": "tools", "notes": ""}]\n```'
|
||||
items = parse_llm_response(raw)
|
||||
assert len(items) == 1
|
||||
assert items[0].name == "tape"
|
||||
|
||||
def test_parse_llm_response_invalid_json(self):
|
||||
with pytest.raises(json.JSONDecodeError):
|
||||
parse_llm_response("not json at all")
|
||||
|
||||
def test_format_summary(self):
|
||||
items = [InventoryItem(name="water", quantity=6, unit="gallon", category="supplies")]
|
||||
summary = _format_summary(items)
|
||||
assert "water" in summary
|
||||
assert "x6" in summary
|
||||
assert "gallon" in summary
|
||||
|
||||
|
||||
class TestDeviceRegistry:
|
||||
@pytest.fixture
|
||||
def signal_mock(self):
|
||||
return MagicMock(spec=SignalClient)
|
||||
|
||||
@pytest.fixture
|
||||
def registry(self, signal_mock, engine):
|
||||
return DeviceRegistry(signal_mock, engine)
|
||||
|
||||
def test_new_device_is_unverified(self, registry):
|
||||
registry.record_contact("+1234", "abc123")
|
||||
assert not registry.is_verified("+1234")
|
||||
|
||||
def test_verify_device(self, registry):
|
||||
registry.record_contact("+1234", "abc123")
|
||||
assert registry.verify("+1234")
|
||||
assert registry.is_verified("+1234")
|
||||
|
||||
def test_verify_unknown_device(self, registry):
|
||||
assert not registry.verify("+9999")
|
||||
|
||||
def test_block_device(self, registry):
|
||||
registry.record_contact("+1234", "abc123")
|
||||
assert registry.block("+1234")
|
||||
assert not registry.is_verified("+1234")
|
||||
|
||||
def test_safety_number_change_resets_trust(self, registry):
|
||||
registry.record_contact("+1234", "abc123")
|
||||
registry.verify("+1234")
|
||||
assert registry.is_verified("+1234")
|
||||
registry.record_contact("+1234", "different_safety_number")
|
||||
assert not registry.is_verified("+1234")
|
||||
|
||||
def test_persistence(self, signal_mock, engine):
|
||||
reg1 = DeviceRegistry(signal_mock, engine)
|
||||
reg1.record_contact("+1234", "abc123")
|
||||
reg1.verify("+1234")
|
||||
|
||||
reg2 = DeviceRegistry(signal_mock, engine)
|
||||
assert reg2.is_verified("+1234")
|
||||
|
||||
def test_list_devices(self, registry):
|
||||
registry.record_contact("+1234", "abc")
|
||||
registry.record_contact("+5678", "def")
|
||||
assert len(registry.list_devices()) == 2
|
||||
|
||||
|
||||
class TestContactCache:
|
||||
@pytest.fixture
|
||||
def signal_mock(self):
|
||||
return MagicMock(spec=SignalClient)
|
||||
|
||||
@pytest.fixture
|
||||
def registry(self, signal_mock, engine):
|
||||
return DeviceRegistry(signal_mock, engine)
|
||||
|
||||
def test_second_call_uses_cache(self, registry):
|
||||
registry.record_contact("+1234", "abc")
|
||||
assert "+1234" in registry._contact_cache
|
||||
|
||||
with patch.object(registry, "engine") as mock_engine:
|
||||
registry.record_contact("+1234", "abc")
|
||||
mock_engine.assert_not_called()
|
||||
|
||||
def test_unverified_gets_default_ttl(self, registry):
|
||||
registry.record_contact("+1234", "abc")
|
||||
from python.common import utcnow
|
||||
|
||||
entry = registry._contact_cache["+1234"]
|
||||
expected = utcnow() + _DEFAULT_TTL
|
||||
assert abs((entry.expires - expected).total_seconds()) < 2
|
||||
assert entry.trust_level == TrustLevel.UNVERIFIED
|
||||
assert entry.has_safety_number is True
|
||||
|
||||
def test_blocked_gets_blocked_ttl(self, registry):
|
||||
registry.record_contact("+1234", "abc")
|
||||
registry.block("+1234")
|
||||
from python.common import utcnow
|
||||
|
||||
entry = registry._contact_cache["+1234"]
|
||||
expected = utcnow() + _BLOCKED_TTL
|
||||
assert abs((entry.expires - expected).total_seconds()) < 2
|
||||
assert entry.trust_level == TrustLevel.BLOCKED
|
||||
|
||||
def test_verify_updates_cache(self, registry):
|
||||
registry.record_contact("+1234", "abc")
|
||||
registry.verify("+1234")
|
||||
entry = registry._contact_cache["+1234"]
|
||||
assert entry.trust_level == TrustLevel.VERIFIED
|
||||
|
||||
def test_block_updates_cache(self, registry):
|
||||
registry.record_contact("+1234", "abc")
|
||||
registry.block("+1234")
|
||||
entry = registry._contact_cache["+1234"]
|
||||
assert entry.trust_level == TrustLevel.BLOCKED
|
||||
|
||||
def test_unverify_updates_cache(self, registry):
|
||||
registry.record_contact("+1234", "abc")
|
||||
registry.verify("+1234")
|
||||
registry.unverify("+1234")
|
||||
entry = registry._contact_cache["+1234"]
|
||||
assert entry.trust_level == TrustLevel.UNVERIFIED
|
||||
|
||||
def test_is_verified_uses_cache(self, registry):
|
||||
registry.record_contact("+1234", "abc")
|
||||
registry.verify("+1234")
|
||||
with patch.object(registry, "engine") as mock_engine:
|
||||
assert registry.is_verified("+1234") is True
|
||||
mock_engine.assert_not_called()
|
||||
|
||||
def test_has_safety_number_uses_cache(self, registry):
|
||||
registry.record_contact("+1234", "abc")
|
||||
with patch.object(registry, "engine") as mock_engine:
|
||||
assert registry.has_safety_number("+1234") is True
|
||||
mock_engine.assert_not_called()
|
||||
|
||||
def test_no_safety_number_cached(self, registry):
|
||||
registry.record_contact("+1234", None)
|
||||
with patch.object(registry, "engine") as mock_engine:
|
||||
assert registry.has_safety_number("+1234") is False
|
||||
mock_engine.assert_not_called()
|
||||
|
||||
def test_expired_cache_hits_db(self, registry):
|
||||
registry.record_contact("+1234", "abc")
|
||||
old = registry._contact_cache["+1234"]
|
||||
registry._contact_cache["+1234"] = _CacheEntry(
|
||||
expires=old.expires - timedelta(minutes=10),
|
||||
trust_level=old.trust_level,
|
||||
has_safety_number=old.has_safety_number,
|
||||
safety_number=old.safety_number,
|
||||
roles=old.roles,
|
||||
)
|
||||
|
||||
with patch("python.signal_bot.device_registry.Session") as mock_session_cls:
|
||||
mock_session = MagicMock()
|
||||
mock_session_cls.return_value.__enter__ = MagicMock(return_value=mock_session)
|
||||
mock_session_cls.return_value.__exit__ = MagicMock(return_value=False)
|
||||
mock_device = MagicMock()
|
||||
mock_device.trust_level = TrustLevel.UNVERIFIED
|
||||
mock_session.scalars.return_value.one_or_none.return_value = mock_device
|
||||
registry.record_contact("+1234", "abc")
|
||||
mock_session.scalars.assert_called_once()
|
||||
|
||||
|
||||
class TestLocationCommand:
|
||||
def test_format_location(self):
|
||||
response = _format_location("12.34", "56.78")
|
||||
assert "12.34, 56.78" in response
|
||||
assert "maps.google.com" in response
|
||||
|
||||
def test_handle_location_request_without_config(self):
|
||||
signal = MagicMock(spec=SignalClient)
|
||||
message = SignalMessage(source="+1234", timestamp=0, message="location")
|
||||
handle_location_request(message, signal, None, None)
|
||||
signal.reply.assert_called_once()
|
||||
assert "not configured" in signal.reply.call_args[0][1]
|
||||
|
||||
|
||||
class TestDispatch:
|
||||
@pytest.fixture
|
||||
def signal_mock(self):
|
||||
return MagicMock(spec=SignalClient)
|
||||
|
||||
@pytest.fixture
|
||||
def llm_mock(self):
|
||||
return MagicMock(spec=LLMClient)
|
||||
|
||||
@pytest.fixture
|
||||
def registry_mock(self):
|
||||
mock = MagicMock(spec=DeviceRegistry)
|
||||
mock.is_verified.return_value = True
|
||||
mock.has_safety_number.return_value = True
|
||||
mock.has_role.return_value = False
|
||||
mock.get_roles.return_value = []
|
||||
return mock
|
||||
|
||||
@pytest.fixture
|
||||
def config(self, engine):
|
||||
return BotConfig(
|
||||
signal_api_url="http://localhost:8080",
|
||||
phone_number="+1234567890",
|
||||
inventory_api_url="http://localhost:9090",
|
||||
engine=engine,
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def bot(self, signal_mock, llm_mock, registry_mock, config):
|
||||
return Bot(signal_mock, llm_mock, registry_mock, config)
|
||||
|
||||
def test_unverified_device_ignored(self, bot, signal_mock, registry_mock):
|
||||
registry_mock.is_verified.return_value = False
|
||||
msg = SignalMessage(source="+1234", timestamp=0, message="help")
|
||||
bot.dispatch(msg)
|
||||
signal_mock.reply.assert_not_called()
|
||||
|
||||
def test_admin_without_safety_number_ignored(self, bot, signal_mock, registry_mock):
|
||||
registry_mock.has_safety_number.return_value = False
|
||||
registry_mock.has_role.return_value = True
|
||||
msg = SignalMessage(source="+1234", timestamp=0, message="help")
|
||||
bot.dispatch(msg)
|
||||
signal_mock.reply.assert_not_called()
|
||||
|
||||
def test_non_admin_without_safety_number_allowed(self, bot, signal_mock, registry_mock):
|
||||
registry_mock.has_safety_number.return_value = False
|
||||
registry_mock.has_role.return_value = False
|
||||
registry_mock.get_roles.return_value = []
|
||||
msg = SignalMessage(source="+1234", timestamp=0, message="help")
|
||||
bot.dispatch(msg)
|
||||
signal_mock.reply.assert_called_once()
|
||||
|
||||
def test_help_command(self, bot, signal_mock, registry_mock):
|
||||
msg = SignalMessage(source="+1234", timestamp=0, message="help")
|
||||
bot.dispatch(msg)
|
||||
signal_mock.reply.assert_called_once()
|
||||
assert "Available commands" in signal_mock.reply.call_args[0][1]
|
||||
|
||||
def test_unknown_command_ignored(self, bot, signal_mock):
|
||||
msg = SignalMessage(source="+1234", timestamp=0, message="foobar")
|
||||
bot.dispatch(msg)
|
||||
signal_mock.reply.assert_not_called()
|
||||
|
||||
def test_non_command_message_ignored(self, bot, signal_mock):
|
||||
msg = SignalMessage(source="+1234", timestamp=0, message="hello there")
|
||||
bot.dispatch(msg)
|
||||
signal_mock.reply.assert_not_called()
|
||||
|
||||
def test_status_command(self, bot, signal_mock, llm_mock, registry_mock):
|
||||
llm_mock.list_models.return_value = ["model1", "model2"]
|
||||
llm_mock.model = "test:7b"
|
||||
registry_mock.list_devices.return_value = []
|
||||
registry_mock.has_role.return_value = True
|
||||
msg = SignalMessage(source="+1234", timestamp=0, message="status")
|
||||
bot.dispatch(msg)
|
||||
signal_mock.reply.assert_called_once()
|
||||
assert "Bot online" in signal_mock.reply.call_args[0][1]
|
||||
|
||||
def test_location_command(self, bot, signal_mock, registry_mock, config):
|
||||
registry_mock.has_role.return_value = True
|
||||
msg = SignalMessage(source="+1234", timestamp=0, message="location")
|
||||
with patch("python.signal_bot.main.handle_location_request") as mock_location:
|
||||
bot.dispatch(msg)
|
||||
|
||||
mock_location.assert_called_once_with(
|
||||
msg,
|
||||
signal_mock,
|
||||
config.ha_url,
|
||||
config.ha_token,
|
||||
)
|
||||
Reference in New Issue
Block a user