Compare commits

..
Author SHA1 Message Date
Richie 1d3a2d6fb7 added math the jeeves 2025-08-06 22:11:13 -04:00
335 changed files with 1607 additions and 31153 deletions
-15
View File
@@ -1,15 +0,0 @@
.git
.direnv
.mypy_cache
.pytest_cache
.ruff_cache
__pycache__
**/__pycache__
*.pyc
*.pyo
.ebook_search_bm25
result
result-*
*.egg-info
dist
build
+3 -4
View File
@@ -15,13 +15,12 @@ jobs:
matrix:
system:
- "bob"
- "brain"
- "jeeves"
- "rhapsody-in-green"
continue-on-error: true
steps:
- uses: actions/checkout@v4
- name: Build default package
run: "nixos-rebuild build --accept-flake-config --flake ./#${{ matrix.system }}"
run: "nixos-rebuild build --flake ./#${{ matrix.system }}"
- name: copy to nix-cache
run: nix copy --accept-flake-config --to unix:///host-nix/var/nix/daemon-socket/socket .#nixosConfigurations.${{ matrix.system }}.config.system.build.toplevel
run: nix copy --to ssh://jeeves .#nixosConfigurations.${{ matrix.system }}.config.system.build.toplevel
+13 -7
View File
@@ -6,18 +6,24 @@ on:
jobs:
merge:
runs-on: self-hosted
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: merge_flake_lock_update
run: >-
nix develop .#devShells.x86_64-linux.default -c
python -m python.gitea_flake_lock merge
--repo "${{ github.repository }}"
run: |
pr_number=$(gh pr list --state open --author RichieCahill --label flake_lock_update --json number --jq '.[0].number')
echo "pr_number=$pr_number" >> $GITHUB_ENV
if [ -n "$pr_number" ]; then
gh pr merge "$pr_number" --rebase
else
echo "No open PR found with label flake_lock_update"
fi
env:
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
GITEA_URL: https://gitea.tmmworkshop.com
GITHUB_TOKEN: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
-19
View File
@@ -1,19 +0,0 @@
name: pytest
on:
workflow_dispatch:
push:
branches:
- main
pull_request:
branches:
- main
jobs:
pytest:
runs-on: self-hosted
steps:
- uses: actions/checkout@v4
- name: Run tests
run: nix develop .#devShells.x86_64-linux.default -c pytest tests
+11 -14
View File
@@ -6,21 +6,18 @@ on:
jobs:
lockfile:
runs-on: self-hosted
permissions:
actions: write
contents: write
pull-requests: write
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install Nix
uses: DeterminateSystems/nix-installer-action@main
- name: Update flake.lock
run: nix flake update
- name: Create or update flake.lock PR
env:
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
GITEA_URL: https://gitea.tmmworkshop.com
run: >-
nix develop .#devShells.x86_64-linux.default -c
python -m python.gitea_flake_lock update
--repo "${{ github.repository }}"
uses: DeterminateSystems/update-flake-lock@main
with:
token: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
pr-title: "Update flake.lock"
pr-labels: |
dependencies
automated
flake_lock_update
-8
View File
@@ -165,11 +165,3 @@ test.*
# syncthing
.stfolder
# Frontend build output
frontend/dist/
frontend/node_modules/
# data from testing llms
data/*
.ebook_search_bm25
+2 -5
View File
@@ -1,12 +1,9 @@
# Generate AGE keys from SSH keys with:
# ssh-keygen -A
# nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
keys:
- &admin_richie age1u8zj599elqqvcmhxn8zuwrufsz8w8w366d3ayrljjejljt2q45kq8mxw9c # cspell:disable-line
- &system_bob age1q47vup0tjhulkg7d6xwmdsgrw64h4ax3la3evzqpxyy4adsmk9fs56qz3y # cspell:disable-line
- &system_brain age1jhf7vm0005j60mjq63696frrmjhpy8kpc2d66mw044lqap5mjv4snmwvwm # cspell:disable-line
- &system_jeeves age13lmqgc3jvkyah5e3vcwmj4s5wsc2akctcga0lpc0x8v8du3fxprqp4ldkv # cspell:disable-line
- &system_router age1xzxryqq63x65yuza9lmmkud7crjjxpnkdew070yhx6xn7xe4tdws5twxsv # cspell:disable-line
- &system_rhapsody age1ufnewppysaq2wwcl4ugngjz8pfzc5a35yg7luq0qmuqvctajcycs5lf6k4 # cspell:disable-line
creation_rules:
@@ -15,6 +12,6 @@ creation_rules:
- age:
- *admin_richie
- *system_bob
- *system_brain
- *system_jeeves
- *system_router
- *system_rhapsody
+8 -48
View File
@@ -2,7 +2,6 @@
"cSpell.words": [
"aboutwelcome",
"acltype",
"addopts",
"addstr",
"advplyr",
"ahci",
@@ -10,7 +9,6 @@
"aiounifi",
"alsa",
"apiclient",
"apscheduler",
"archlinux",
"ashift",
"asrouter",
@@ -40,6 +38,7 @@
"cgroupdriver",
"charliermarsh",
"Checkpointing",
"cloudflared",
"codellama",
"codezombiech",
"compactmode",
@@ -71,17 +70,16 @@
"ehci",
"emerg",
"endlessh",
"ents",
"errorlens",
"esbenp",
"esphome",
"extest",
"fadvise",
"fastfetch",
"fastforwardteam",
"FASTFOX",
"ffmpegthumbnailer",
"filebot",
"filebrowser",
"fileroller",
"findbar",
"Fira",
@@ -98,7 +96,6 @@
"getch",
"getmaxyx",
"ghdeploy",
"gitea",
"globalprivacycontrol",
"gparted",
"gtts",
@@ -117,9 +114,6 @@
"httpchk",
"hurlenko",
"hwloc",
"ical",
"ignorelist",
"improv",
"INITDB",
"iocharset",
"ioit",
@@ -129,8 +123,6 @@
"jnoortheen",
"jsbc",
"kagi",
"keyformat",
"keylocation",
"kuma",
"lazer",
"levelname",
@@ -156,25 +148,20 @@
"mixtral",
"mklabel",
"mkpart",
"modbus",
"modbuss",
"modesetting",
"mountpoint",
"mountpoints",
"mousewheel",
"mqtt",
"mtxr",
"mypy",
"ncdu",
"nemo",
"neofetch",
"nerdfonts",
"netdev",
"netdevs",
"Networkd",
"networkmanager",
"newtabpage",
"ngram",
"ngrams",
"nixfmt",
"nixos",
"nixpkgs",
@@ -197,7 +184,6 @@
"overalljails",
"overscroll",
"overseerr",
"paho",
"partitionwise",
"pbmode",
"pciutils",
@@ -206,7 +192,6 @@
"peerconnection",
"PESKYFOX",
"PGID",
"pgvector",
"pipewire",
"pkgs",
"plugdev",
@@ -226,16 +211,13 @@
"pulseaudio",
"punycode",
"pychromecast",
"pydocstyle",
"pyfakefs",
"pylance",
"pylint",
"pymetno",
"pymodbus",
"pyopenweathermap",
"pyownet",
"pytest",
"qalculate",
"qbit",
"qbittorrent",
"qbittorrentvpn",
"qbitvpn",
"quicksuggest",
"radarr",
"readahead",
@@ -245,7 +227,6 @@
"referer",
"REFERERS",
"relatime",
"rerank",
"Rhosts",
"ripgrep",
"roboto",
@@ -258,10 +239,8 @@
"schemeless",
"scrollback",
"SECUREFOX",
"sessionmaker",
"sessionstore",
"shellcheck",
"signalbot",
"signon",
"Signons",
"skia",
@@ -271,7 +250,6 @@
"socialtracking",
"sonarr",
"sponsorblock",
"sqlalchemy",
"sqltools",
"ssdp",
"SSHOPTS",
@@ -283,7 +261,6 @@
"tabmanager",
"tamasfe",
"TCPIP",
"testdisk",
"tiktok",
"timonwong",
"titlebar",
@@ -293,17 +270,11 @@
"topstories",
"treefmt",
"twimg",
"typedmonarchmoney",
"typer",
"uaccess",
"ubiquiti",
"ublock",
"uiprotect",
"uitour",
"unifi",
"unjudged",
"unrar",
"unstorable",
"unsubmitted",
"uptimekuma",
"urlbar",
@@ -313,10 +284,7 @@
"useragent",
"usernamehw",
"userprefs",
"vaninventory",
"vdev",
"vfat",
"victron",
"virt",
"virtualisation",
"vpnpromourl",
@@ -328,10 +296,7 @@
"wireshark",
"Workqueues",
"xattr",
"xcursorgen",
"xdist",
"xhci",
"yake",
"yazi",
"yubikey",
"yubioath",
@@ -342,10 +307,5 @@
"zoxide",
"zram",
"zstd"
],
"python-envs.defaultEnvManager": "ms-python.python:system",
"python-envs.pythonProjects": [],
"python.testing.pytestArgs": ["tests"],
"python.testing.unittestEnabled": false,
"python.testing.pytestEnabled": true
]
}
-50
View File
@@ -1,51 +1 @@
# dotfiles
## Installer ISO
Build a bootable NixOS ISO with the installer preinstalled:
```sh
nix build .#iso
```
Write `result/iso/nixos-zfs-installer.iso` to a USB stick (for example with `dd`) or boot it in a VM. The image is the minimal NixOS installation CD with ZFS enabled and `nixos-installer` on `PATH`. SSH is enabled and the `nixos` and `root` accounts use the password `nixos`, so you can also run the installer remotely. Once booted:
```sh
sudo nixos-installer
```
The ISO bundles the `.#installer-nixos` package, a variant of the binary that keeps its Nix store linkage instead of being patched for foreign distributions.
## Installer binary
Build the self-contained installer executable with:
```sh
nix build .#installer
```
The flake package (defined in `python/installer/package.nix`) uses the Python builder in `python/installer/build.py`, which stages only the installer modules before running PyInstaller. You can also call it directly when `pyinstaller` and `patchelf` are on `PATH`:
```sh
python -m python.installer.build --output ./nixos-installer
```
Copy `result/bin/nixos-installer` to the installer USB stick and run it as root from the NixOS live environment:
```sh
sudo ./nixos-installer
```
Validate the live environment first with:
```sh
./nixos-installer --check
```
Paste a value into the TUI encryption password field to enable LUKS during install, or set `ENCRYPT_KEY`:
```sh
sudo env ENCRYPT_KEY='change-me' ./nixos-installer
```
The binary bundles the Python runtime and only the installer modules it imports. It still expects the NixOS installer environment to provide system install tools such as `parted`, `zfs`, `zpool`, `cryptsetup`, `nixos-generate-config`, and `nixos-install`.
+3 -16
View File
@@ -23,10 +23,7 @@
boot = {
tmp.useTmpfs = true;
kernelPackages = lib.mkDefault pkgs.linuxPackages_6_12;
zfs = {
package = lib.mkDefault pkgs.zfs_2_4;
forceImportRoot = lib.mkDefault false;
};
zfs.package = lib.mkDefault pkgs.zfs_2_3;
};
hardware.enableRedistributableFirmware = true;
@@ -40,24 +37,14 @@
nixpkgs = {
overlays = builtins.attrValues outputs.overlays;
config = {
allowUnfree = true;
permittedInsecurePackages = [
"openssl-1.1.1w" # This is for discord-canary
];
};
config.allowUnfree = true;
};
services = {
dbus.implementation = "dbus";
# firmware update
fwupd.enable = true;
snapshot_manager = {
enable = lib.mkDefault true;
PYTHONPATH = "${inputs.self}/";
};
snapshot_manager.enable = lib.mkDefault true;
zfs = {
trim.enable = lib.mkDefault true;
-12
View File
@@ -1,10 +1,4 @@
{ lib, pkgs, ... }:
let
libPath = pkgs.lib.makeLibraryPath [
pkgs.zlib
pkgs.stdenv.cc.cc.lib
];
in
{
programs.nix-ld = {
enable = lib.mkDefault true;
@@ -21,7 +15,6 @@ in
libxml2
openssl
stdenv.cc.cc
stdenv.cc.cc.lib
systemd
util-linux
xz
@@ -30,9 +23,4 @@ in
zstd
];
};
environment = {
sessionVariables.LD_LIBRARY_PATH = lib.mkDefault libPath;
variables.LD_LIBRARY_PATH = lib.mkDefault libPath;
};
}
-3
View File
@@ -33,9 +33,6 @@ in
];
warn-dirty = false;
flake-registry = ""; # disable global flake registries
connect-timeout = 10;
download-buffer-size = 536870912;
fallback = true;
};
# Add each flake input as a registry and nix_path
+1 -1
View File
@@ -2,6 +2,6 @@
{
environment.systemPackages = with pkgs; [
git
my_python
python313
];
}
+11 -25
View File
@@ -1,4 +1,5 @@
{
inputs,
pkgs,
lib,
config,
@@ -10,48 +11,33 @@ in
{
options = {
services.snapshot_manager = {
enable = lib.mkEnableOption "ZFS snapshot manager";
enable = lib.mkOption {
default = true;
example = true;
description = "Whether to enable k3s-net.";
type = lib.types.bool;
};
path = lib.mkOption {
type = lib.types.path;
description = "Path that needs to be updated via git pull";
default = ./snapshot_config.toml;
description = "Path to the snapshot_manager TOML config.";
};
PYTHONPATH = lib.mkOption {
type = lib.types.str;
description = ''
the PYTHONPATH to use for the snapshot_manager service.
'';
};
EnvironmentFile = lib.mkOption {
type = lib.types.nullOr (lib.types.coercedTo lib.types.path toString lib.types.str);
default = null;
description = ''
Single environment file for the service (e.g. /etc/snapshot-manager/env).
Use a leading "-" to ignore if missing (systemd feature).
'';
};
};
};
config = lib.mkIf cfg.enable {
systemd = {
services.snapshot_manager = {
services."snapshot_manager" = {
description = "ZFS Snapshot Manager";
requires = [ "zfs-import.target" ];
after = [ "zfs-import.target" ];
path = [ pkgs.zfs ];
environment = {
PYTHONPATH = cfg.PYTHONPATH;
};
serviceConfig = {
Type = "oneshot";
ExecStart = "${pkgs.my_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
}
// lib.optionalAttrs (cfg.EnvironmentFile != null) {
EnvironmentFile = cfg.EnvironmentFile;
ExecStart = "${inputs.system_tools.packages.x86_64-linux.default}/bin/snapshot_manager --config-file='${cfg.path}'";
};
};
timers.snapshot_manager = {
timers."snapshot_manager" = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnBootSec = "15m";
-2
View File
@@ -37,8 +37,6 @@
TcpKeepAlive = "no";
X11Forwarding = lib.mkDefault false;
KexAlgorithms = [
"sntrup761x25519-sha512@openssh.com"
"mlkem768x25519-sha256"
"curve25519-sha256@libssh.org"
"diffie-hellman-group-exchange-sha256"
];
-6
View File
@@ -1,6 +0,0 @@
{
nix.settings = {
trusted-substituters = [ "http://192.168.95.35:5000" ];
substituters = [ "http://192.168.95.35:5000/?priority=1&want-mass-query=true" ];
};
}
+2 -2
View File
@@ -1,8 +1,8 @@
{ pkgs, ... }:
{
boot = {
kernelPackages = pkgs.linuxPackages_6_18;
zfs.package = pkgs.zfs_2_4;
kernelPackages = pkgs.linuxPackages_6_15;
zfs.package = pkgs.zfs_2_3;
};
hardware.bluetooth = {
-256
View File
@@ -1,256 +0,0 @@
{
config,
lib,
pkgs,
...
}:
let
monitoringInterface = "ztwfunumly";
nodeTextfileDir = "/var/lib/prometheus-node-exporter-textfile";
mkProcessNameTemplate =
perPid: template: if perPid then "${template}:{{.PID}}:{{.StartTime}}" else template;
mkProcessMatchers = perPid: [
{
name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Module}}";
cmdline = [ "^/nix/store[^ ]*/bin/python[^ ]* -m (?P<Module>[^ ]+)" ];
}
{
name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Wrapped}}";
cmdline = [
"^/nix/store[^ ]*/bin/python[^ ]* /nix/store[^ ]*/bin/\\.?(?P<Wrapped>[^ /]+?)(?:-wrapped)?(?:\\s|$)"
];
}
{
name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Wrapped}}";
cmdline = [
"^/nix/store[^ ]*/bin/node /nix/store[^ ]*-(?P<Wrapped>[A-Za-z0-9._+-]+)-[0-9][^ /]*/"
];
}
{
name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Wrapped}}";
cmdline = [ "^/nix/store[^ ]*/(?:bin/|lib/[^ ]*/)?\\.?(?P<Wrapped>[^ /]+?)(?:-wrapped)?(?:\\s|$)" ];
}
{
name = mkProcessNameTemplate perPid "{{.Username}}:{{.ExeBase}}";
cmdline = [ ".+" ];
}
];
perPidConfig = pkgs.writeText "process-exporter-per-pid.yaml" (
builtins.toJSON {
process_names = mkProcessMatchers true;
}
);
zpoolLatencyScript = pkgs.writeShellScript "zpool-latency-exporter" ''
set -euo pipefail
out_dir=${lib.escapeShellArg nodeTextfileDir}
host=${lib.escapeShellArg config.networking.hostName}
tmp_file="$(mktemp "$out_dir/zpool.prom.XXXXXX")"
trap 'rm -f "$tmp_file"' EXIT
pools="$(zpool list -H -o name | paste -sd, -)"
cat >"$tmp_file" <<'EOF'
# HELP zpool_iostat_total_wait_read_ns Average total read wait time reported by zpool iostat.
# TYPE zpool_iostat_total_wait_read_ns gauge
# HELP zpool_iostat_total_wait_write_ns Average total write wait time reported by zpool iostat.
# TYPE zpool_iostat_total_wait_write_ns gauge
# HELP zpool_iostat_disk_wait_read_ns Average disk read wait time reported by zpool iostat.
# TYPE zpool_iostat_disk_wait_read_ns gauge
# HELP zpool_iostat_disk_wait_write_ns Average disk write wait time reported by zpool iostat.
# TYPE zpool_iostat_disk_wait_write_ns gauge
# HELP zpool_iostat_syncq_wait_read_ns Average synchronous queue read wait time reported by zpool iostat.
# TYPE zpool_iostat_syncq_wait_read_ns gauge
# HELP zpool_iostat_syncq_wait_write_ns Average synchronous queue write wait time reported by zpool iostat.
# TYPE zpool_iostat_syncq_wait_write_ns gauge
# HELP zpool_iostat_asyncq_wait_read_ns Average asynchronous queue read wait time reported by zpool iostat.
# TYPE zpool_iostat_asyncq_wait_read_ns gauge
# HELP zpool_iostat_asyncq_wait_write_ns Average asynchronous queue write wait time reported by zpool iostat.
# TYPE zpool_iostat_asyncq_wait_write_ns gauge
EOF
zpool iostat -Hplvy -y 1 1 | awk -F '\t' -v host="$host" -v pools="$pools" '
function esc(str, out) {
out = str
gsub(/\\/, "\\\\", out)
gsub(/"/, "\\\"", out)
return out
}
function emit(metric, pool, vdev, value) {
if (value == "" || value == "-") {
return
}
printf "%s{host=\"%s\",pool=\"%s\",vdev=\"%s\"} %s\n",
metric,
esc(host),
esc(pool),
esc(vdev),
value
}
BEGIN {
split(pools, pool_names, ",")
for (idx in pool_names) {
if (pool_names[idx] != "") {
known_pools[pool_names[idx]] = 1
}
}
}
NF == 0 {
next
}
{
row_name = $1
if (row_name in known_pools) {
current_pool = row_name
current_vdev = "_pool"
} else if (current_pool == "") {
next
} else {
current_vdev = row_name
}
emit("zpool_iostat_total_wait_read_ns", current_pool, current_vdev, $8)
emit("zpool_iostat_total_wait_write_ns", current_pool, current_vdev, $9)
emit("zpool_iostat_disk_wait_read_ns", current_pool, current_vdev, $10)
emit("zpool_iostat_disk_wait_write_ns", current_pool, current_vdev, $11)
emit("zpool_iostat_syncq_wait_read_ns", current_pool, current_vdev, $12)
emit("zpool_iostat_syncq_wait_write_ns", current_pool, current_vdev, $13)
emit("zpool_iostat_asyncq_wait_read_ns", current_pool, current_vdev, $14)
emit("zpool_iostat_asyncq_wait_write_ns", current_pool, current_vdev, $15)
}
' >>"$tmp_file"
mv "$tmp_file" "$out_dir/zpool.prom"
trap - EXIT
'';
in
{
networking.firewall.interfaces.${monitoringInterface}.allowedTCPPorts = [
9100
9134
9256
9257
9633
];
services.prometheus.exporters = {
node = {
enable = true;
enabledCollectors = [
"pressure"
"processes"
"systemd"
];
extraFlags = [ "--collector.textfile.directory=${nodeTextfileDir}" ];
};
process = {
enable = true;
user = "root";
group = "root";
settings.process_names = mkProcessMatchers false;
extraFlags = [
"-gather-smaps=false"
"-remove-empty-groups=true"
"-threads=false"
];
};
smartctl.enable = true;
zfs.enable = true;
};
programs.atop = {
enable = true;
atopService.enable = true;
atopRotateTimer.enable = true;
atopacctService.enable = true;
settings.interval = 30;
};
systemd = {
services = {
prometheus-process-pid-exporter = {
description = "Prometheus process exporter with per-PID naming";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
serviceConfig = {
ExecStart = ''
${pkgs.prometheus-process-exporter}/bin/process-exporter \
--web.listen-address 0.0.0.0:9257 \
--config.path ${perPidConfig} \
-children=false \
-gather-smaps=false \
-remove-empty-groups=true \
-threads=false
'';
User = "root";
Group = "root";
Restart = "always";
WorkingDirectory = "/tmp";
CapabilityBoundingSet = [ "" ];
DeviceAllow = [ "" ];
LockPersonality = true;
MemoryDenyWriteExecute = true;
NoNewPrivileges = true;
PrivateDevices = true;
PrivateTmp = true;
ProtectClock = true;
ProtectControlGroups = true;
ProtectHome = true;
ProtectHostname = true;
ProtectKernelLogs = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectSystem = "strict";
RemoveIPC = true;
RestrictAddressFamilies = [
"AF_INET"
"AF_INET6"
];
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
SystemCallArchitectures = "native";
UMask = "0077";
};
};
zpool-latency-exporter = {
description = "Exports ZFS latency metrics for node_exporter textfile collection";
after = [ "zfs-import.target" ];
requires = [ "zfs-import.target" ];
path = [
config.boot.zfs.package
pkgs.coreutils
pkgs.gawk
];
serviceConfig = {
Type = "oneshot";
ExecStart = zpoolLatencyScript;
};
};
};
timers.zpool-latency-exporter = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnBootSec = "2m";
OnUnitActiveSec = "60s";
Unit = "zpool-latency-exporter.service";
};
};
tmpfiles.rules = [ "d ${nodeTextfileDir} 0755 root root - -" ];
};
}
+1 -4
View File
@@ -10,9 +10,6 @@
authorizedKeys = config.users.users.richie.openssh.authorizedKeys.keys;
};
};
availableKernelModules = [
"igb"
"r8152"
];
availableKernelModules = [ "igb" ];
};
}
+3 -4
View File
@@ -8,11 +8,10 @@
dataDir = "/home/richie/Syncthing";
configDir = "/home/richie/.config/syncthing";
settings.devices = {
bob.id = "CJIAPEJ-VO74RR4-F75VU6M-QNZAMYG-FYUJG7Y-6AT62HJ-355PRPL-PJFETAZ"; # cspell:disable-line
brain.id = "SSCGIPI-IV3VYKB-TRNIJE3-COV4T2H-CDBER7F-I2CGHYA-NWOEUDU-3T5QAAN"; # cspell:disable-line
ipad.id = "KI76T3X-SFUGV2L-VSNYTKR-TSIUV5L-SHWD3HE-GQRGRCN-GY4UFMD-CW6Z6AX"; # cspell:disable-line
phone.id = "TBRULKD-7DZPGGZ-F6LLB7J-MSO54AY-7KLPBIN-QOFK6PX-W2HBEWI-PHM2CQI"; # cspell:disable-line
jeeves.id = "ICRHXZW-ECYJCUZ-I4CZ64R-3XRK7CG-LL2HAAK-FGOHD22-BQA4AI6-5OAL6AG"; # cspell:disable-line
phone.id = "JPVQKQW-CFXOJXT-Q5G5F3H-QIDHDRE-GKHPTQB-GXZUQSP-U7FR7F7-INP3AAH"; # cspell:disable-line
ipad.id = "KI76T3X-SFUGV2L-VSNYTKR-TSIUV5L-SHWD3HE-GQRGRCN-GY4UFMD-CW6Z6AX"; # cspell:disable-line
bob.id = "CJIAPEJ-VO74RR4-F75VU6M-QNZAMYG-FYUJG7Y-6AT62HJ-355PRPL-PJFETAZ"; # cspell:disable-line
rhapsody-in-green.id = "ASL3KC4-3XEN6PA-7BQBRKE-A7JXLI6-DJT43BY-Q4WPOER-7UALUAZ-VTPQ6Q4"; # cspell:disable-line
};
};
+1 -1
View File
@@ -4,7 +4,7 @@
flags = [ "--accept-flake-config" ];
randomizedDelaySec = "1h";
persistent = true;
flake = "git+https://gitea.tmmworkshop.com/richie/dotfiles?ref=main";
flake = "github:RichieCahill/dotfiles";
allowReboot = true;
dates = "Sat *-*-* 06:00:00";
};
-4
View File
@@ -1,4 +0,0 @@
source "https://rubygems.org"
# The github-pages gem pins all compatible versions of Jekyll and its plugins
gem "github-pages", group: :jekyll_plugins
-23
View File
@@ -1,23 +0,0 @@
title: "Richie Cahill"
description: "ALL THE CHAOS THAT I CANT DO AT WORK"
baseurl: "/dotfiles"
url: "https://richiecahill.github.io"
remote_theme: pages-themes/hacker@v0.2.0
plugins:
- jekyll-feed
- jekyll-remote-theme
- jekyll-seo-tag
- jekyll-sitemap
- jekyll-paginate
paginate: 5
paginate_path: "/page:num"
author:
name: "Richie Cahill"
email: "richie@tmmworkshop.com"
social_links:
github: "RichieCahill"
website: "https://tmmworkshop.com"
-13
View File
@@ -1,13 +0,0 @@
# The MONOREPO experiment
Im testing a [MONOREPO](https://en.wikipedia.org/wiki/Monorepo) because Phil said this was a bad idea. To that i say hold my beer.
In all seriousness, I Think that for a small dev team/solo dev. The simplicity is worth higher barer to entry. One of my most annoying processes was updating my system tools. I had to build my update in a feature branch and then merge it into my main branch. then go to my dotfiles create a feature branch update the system tools merge it into main.
It will be starting with my Nix Dotfiles Python tools and now my blog.
I will be reaching ot to phil on 2030-10-31 and 2035-10-31 to give him updates on the progress.
Known Issues:
- the python tests are running on the current derivation not the one the derivation im updating to.
-17
View File
@@ -1,17 +0,0 @@
---
layout: default
title: "Welcome"
---
Welcome to my build logs, notes, and experiments.
You can read my latest posts below
<ul>
{% for post in site.posts %}
<li>
<a href="{{ post.url | relative_url }}">{{ post.title }}</a>
<small>— {{ post.date | date: "%Y-%m-%d" }}</small>
</li>
{% endfor %}
</ul>
-76
View File
@@ -1,76 +0,0 @@
# ZFS failed root import recovery
## Fast path
If the machine fails to boot because ZFS refuses to import `root_pool`:
### GRUB
1. At the bootloader menu, select the normal NixOS entry.
2. Press `e`.
3. Find the line that starts with `linux`.
4. Append this to the end of that line:
```text
zfs_force=1
```
5. Boot once with `Ctrl+x` or `F10`.
### systemd-boot
1. At the bootloader menu, highlight the normal NixOS entry.
2. Press `e`.
3. Append this to the end of the options line:
```text
zfs_force=1
```
4. Press `Enter` to boot once.
## After boot
Run:
```bash
sudo zpool status
sudo zpool import
journalctl -b | rg "ZFS|zfs|import|root_pool"
```
## Expected result
`sudo zpool status` should show `root_pool` as `ONLINE`.
## Reboot test
Run:
```bash
sudo reboot
```
Do not add `zfs_force=1` the second time.
## If it still fails
Boot once more with:
```text
zfs_force=1
```
Then run:
```bash
sudo zpool status -v
sudo zpool history | tail -n 50
journalctl -b | rg "ZFS|zfs|import|root_pool"
```
## Notes
- Root pool name is `root_pool`.
- This is a one-time recovery path after disk moves, controller changes, dirty exports, or interrupted imports.
- Some hosts also need the LUKS unlock USB key inserted before boot.
-3
View File
@@ -1,3 +0,0 @@
# esphome
/.esphome/
/secrets.yaml
-132
View File
@@ -1,132 +0,0 @@
esphome:
name: batteries
friendly_name: batteries
esp32:
board: esp32dev
framework:
type: arduino
logger:
api:
encryption:
key: !secret api_key
external_components:
- source: github://syssi/esphome-jk-bms@main
ota:
- platform: esphome
password: !secret ota_password
wifi:
ssid: !secret wifi_ssid
password: !secret wifi_password
fast_connect: on
captive_portal:
esp32_ble_tracker:
scan_parameters:
interval: 1100ms
window: 1100ms
active: true
ble_client:
- mac_address: "C8:47:80:29:0F:DB"
id: jk_ble0
jk_bms_ble:
- ble_client_id: jk_ble0
protocol_version: JK02_32S
throttle: 1s
id: jk_bms0
button:
- platform: jk_bms_ble
retrieve_settings:
name: "JK0 retrieve settings"
retrieve_device_info:
name: "JK0 retrieve device info"
sensor:
- platform: jk_bms_ble
jk_bms_ble_id: jk_bms0
total_voltage:
name: "JK0 Total Voltage"
state_of_charge:
name: "JK0 SoC"
charging_power:
name: "JK0 charging power"
discharging_power:
name: "JK0 discharging power"
temperature_sensor_1:
name: "JK0 Temp 1"
temperature_sensor_2:
name: "JK0 Temp 2"
balancing:
name: "JK0 balancing"
total_runtime:
name: "JK0 total runtime"
balancing_current:
name: "JK0 balancing current"
delta_cell_voltage:
name: "JK0 cell delta voltage"
average_cell_voltage:
name: "JK0 cell average voltage"
cell_voltage_1:
name: "JK0 cell voltage 1"
cell_voltage_2:
name: "JK0 cell voltage 2"
cell_voltage_3:
name: "JK0 cell voltage 3"
cell_voltage_4:
name: "JK0 cell voltage 4"
cell_voltage_5:
name: "JK0 cell voltage 5"
cell_voltage_6:
name: "JK0 cell voltage 6"
cell_voltage_7:
name: "JK0 cell voltage 7"
cell_voltage_8:
name: "JK0 cell voltage 8"
cell_resistance_1:
name: "JK0 cell resistance 1"
cell_resistance_2:
name: "JK0 cell resistance 2"
cell_resistance_3:
name: "JK0 cell resistance 3"
cell_resistance_4:
name: "JK0 cell resistance 4"
cell_resistance_5:
name: "JK0 cell resistance 5"
cell_resistance_6:
name: "JK0 cell resistance 6"
cell_resistance_7:
name: "JK0 cell resistance 7"
cell_resistance_8:
name: "JK0 cell resistance 8"
total_charging_cycle_capacity:
name: "JK0 total charging cycle capacity"
text_sensor:
- platform: jk_bms_ble
jk_bms_ble_id: jk_bms0
errors:
name: "JK0 Errors"
switch:
- platform: jk_bms_ble
jk_bms_ble_id: jk_bms0
charging:
name: "JK0 Charging"
discharging:
name: "JK0 Discharging"
balancer:
name: "JK0 Balancing"
- platform: ble_client
ble_client_id: jk_ble0
name: "JK0 enable bluetooth connection"
id: ble_client_switch0
-132
View File
@@ -1,132 +0,0 @@
esphome:
name: battery1
friendly_name: battery1
esp32:
board: esp32dev
framework:
type: arduino
logger:
api:
encryption:
key: !secret api_key
external_components:
- source: github://syssi/esphome-jk-bms@main
ota:
- platform: esphome
password: !secret ota_password
wifi:
ssid: !secret wifi_ssid
password: !secret wifi_password
fast_connect: on
captive_portal:
esp32_ble_tracker:
scan_parameters:
interval: 1100ms
window: 1100ms
active: true
ble_client:
- mac_address: "C8:47:80:37:9D:DD"
id: jk_ble1
jk_bms_ble:
- ble_client_id: jk_ble1
protocol_version: JK02_32S
throttle: 1s
id: jk_bms1
button:
- platform: jk_bms_ble
retrieve_settings:
name: "JK1 retrieve settings"
retrieve_device_info:
name: "JK1 retrieve device info"
sensor:
- platform: jk_bms_ble
jk_bms_ble_id: jk_bms1
total_voltage:
name: "JK1 Total Voltage"
state_of_charge:
name: "JK1 SoC"
charging_power:
name: "JK1 charging power"
discharging_power:
name: "JK1 discharging power"
temperature_sensor_1:
name: "JK1 Temp 1"
temperature_sensor_2:
name: "JK1 Temp 2"
balancing:
name: "JK1 balancing"
total_runtime:
name: "JK1 total runtime"
balancing_current:
name: "JK1 balancing current"
delta_cell_voltage:
name: "JK1 cell delta voltage"
average_cell_voltage:
name: "JK1 cell average voltage"
cell_voltage_1:
name: "JK1 cell voltage 1"
cell_voltage_2:
name: "JK1 cell voltage 2"
cell_voltage_3:
name: "JK1 cell voltage 3"
cell_voltage_4:
name: "JK1 cell voltage 4"
cell_voltage_5:
name: "JK1 cell voltage 5"
cell_voltage_6:
name: "JK1 cell voltage 6"
cell_voltage_7:
name: "JK1 cell voltage 7"
cell_voltage_8:
name: "JK1 cell voltage 8"
cell_resistance_1:
name: "JK1 cell resistance 1"
cell_resistance_2:
name: "JK1 cell resistance 2"
cell_resistance_3:
name: "JK1 cell resistance 3"
cell_resistance_4:
name: "JK1 cell resistance 4"
cell_resistance_5:
name: "JK1 cell resistance 5"
cell_resistance_6:
name: "JK1 cell resistance 6"
cell_resistance_7:
name: "JK1 cell resistance 7"
cell_resistance_8:
name: "JK1 cell resistance 8"
total_charging_cycle_capacity:
name: "JK1 total charging cycle capacity"
text_sensor:
- platform: jk_bms_ble
jk_bms_ble_id: jk_bms1
errors:
name: "JK1 Errors"
switch:
- platform: jk_bms_ble
jk_bms_ble_id: jk_bms1
charging:
name: "JK1 Charging"
discharging:
name: "JK1 Discharging"
balancer:
name: "JK1 Balancing"
- platform: ble_client
ble_client_id: jk_ble1
name: "JK1 enable bluetooth connection"
id: ble_client_switch0
-48
View File
@@ -1,48 +0,0 @@
esphome:
name: "environment"
friendly_name: "environment"
esp32:
board: esp32dev
framework:
type: arduino
i2c:
sda: GPIO21
scl: GPIO22
scan: True
id: bus_a
sensor:
- platform: aht10
i2c_id: bus_a
address: 0x38
variant: AHT20
temperature:
name: "environment Temperature"
id: aht10_temperature
humidity:
name: "environment Humidity"
id: aht10_humidity
update_interval: 5s
web_server:
port: 80
logger:
level: DEBUG
api:
encryption:
key: !secret api_key
ota:
- platform: esphome
password: !secret ota_password
wifi:
ssid: !secret wifi_ssid
password: !secret wifi_password
fast_connect: on
captive_portal:
Generated
+118 -35
View File
@@ -8,11 +8,11 @@
},
"locked": {
"dir": "pkgs/firefox-addons",
"lastModified": 1782964936,
"narHash": "sha256-wXEBDr7/dFQYhVpDwCKc9fkrYQQE4x0bdirX1bsLBGA=",
"lastModified": 1753761817,
"narHash": "sha256-FE908x/ihUlr5yn1f+PTMyOjcwotGUodzn7Ej6zZf5U=",
"owner": "rycee",
"repo": "nur-expressions",
"rev": "64feee871e0373dd6121e412c3fb12e372d1bfb5",
"rev": "b657cfddb78408e9b53b4a8aaeaac71fc7ea182e",
"type": "gitlab"
},
"original": {
@@ -29,11 +29,11 @@
]
},
"locked": {
"lastModified": 1783005591,
"narHash": "sha256-NcLHV5uBAeggDUE2wPbKszjfyaSLsoqaYt7izOphkZw=",
"lastModified": 1754085240,
"narHash": "sha256-kVHCrTWEe8B1thAhFag1bk4QPY0ZP45V9vPbrwPHoNo=",
"owner": "nix-community",
"repo": "home-manager",
"rev": "f469c79b955609d6a8fdd9e689be76a93b1621d7",
"rev": "e102920c1becb114645c6f92fe14edc0b05cc229",
"type": "github"
},
"original": {
@@ -43,15 +43,12 @@
}
},
"nixos-hardware": {
"inputs": {
"nixpkgs": "nixpkgs"
},
"locked": {
"lastModified": 1782562157,
"narHash": "sha256-a7+T6QSeowynwZ1ZJJbP8T8ntAytvrui8kFGJmIZt2c=",
"lastModified": 1753122741,
"narHash": "sha256-nFxE8lk9JvGelxClCmwuJYftbHqwnc01dRN4DVLUroM=",
"owner": "nixos",
"repo": "nixos-hardware",
"rev": "a9cf7546a938c737b079e738de73934a13de9784",
"rev": "cc66fddc6cb04ab479a1bb062f4d4da27c936a22",
"type": "github"
},
"original": {
@@ -63,24 +60,27 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1767892417,
"narHash": "sha256-8bW3q88CEg2u4hSP66Vf4lpbLonHz7hqDNBMcCY7E9U=",
"rev": "3497aa5c9457a9d88d71fa93a4a8368816fbeeba",
"type": "tarball",
"url": "https://releases.nixos.org/nixos/unstable/nixos-26.05pre924538.3497aa5c9457/nixexprs.tar.xz"
"lastModified": 1753939845,
"narHash": "sha256-K2ViRJfdVGE8tpJejs8Qpvvejks1+A4GQej/lBk5y7I=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "94def634a20494ee057c76998843c015909d6311",
"type": "github"
},
"original": {
"type": "tarball",
"url": "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz"
"owner": "nixos",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs-master": {
"locked": {
"lastModified": 1783021952,
"narHash": "sha256-8PghAtSGGZ0umfVI8Qbd7ZbFrfZPiH1UwtVbgLeikDA=",
"lastModified": 1754089779,
"narHash": "sha256-bklCdnAaAerEacvmy+HkltNR9QE/M4Ygy+m0L2C6LJI=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "f136374c679c54171a3ace589d15e9e79a8bd086",
"rev": "3683d033f7994c92834f45f166275738dcdd959d",
"type": "github"
},
"original": {
@@ -106,19 +106,53 @@
"type": "github"
}
},
"nixpkgs_2": {
"pyproject-build-systems": {
"inputs": {
"nixpkgs": [
"system_tools",
"nixpkgs"
],
"pyproject-nix": [
"system_tools",
"pyproject-nix"
],
"uv2nix": [
"system_tools",
"uv2nix"
]
},
"locked": {
"lastModified": 1782723713,
"narHash": "sha256-oPXCU/SSUokcGaJREHibG1CBX3+s/W7orDWQOZDsEeQ=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "b5aa0fbd538984f6e3d201be0005b4463d8b09f8",
"lastModified": 1744599653,
"narHash": "sha256-nysSwVVjG4hKoOjhjvE6U5lIKA8sEr1d1QzEfZsannU=",
"owner": "pyproject-nix",
"repo": "build-system-pkgs",
"rev": "7dba6dbc73120e15b558754c26024f6c93015dd7",
"type": "github"
},
"original": {
"owner": "nixos",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"owner": "pyproject-nix",
"repo": "build-system-pkgs",
"type": "github"
}
},
"pyproject-nix": {
"inputs": {
"nixpkgs": [
"system_tools",
"nixpkgs"
]
},
"locked": {
"lastModified": 1746540146,
"narHash": "sha256-QxdHGNpbicIrw5t6U3x+ZxeY/7IEJ6lYbvsjXmcxFIM=",
"owner": "pyproject-nix",
"repo": "pyproject.nix",
"rev": "e09c10c24ebb955125fda449939bfba664c467fd",
"type": "github"
},
"original": {
"owner": "pyproject-nix",
"repo": "pyproject.nix",
"type": "github"
}
},
@@ -127,10 +161,11 @@
"firefox-addons": "firefox-addons",
"home-manager": "home-manager",
"nixos-hardware": "nixos-hardware",
"nixpkgs": "nixpkgs_2",
"nixpkgs": "nixpkgs",
"nixpkgs-master": "nixpkgs-master",
"nixpkgs-stable": "nixpkgs-stable",
"sops-nix": "sops-nix",
"system_tools": "system_tools",
"systems": "systems"
}
},
@@ -141,11 +176,11 @@
]
},
"locked": {
"lastModified": 1782165805,
"narHash": "sha256-478kKQBvK6SYTOdN2h9jhKJv94nbXRbFMfuL1WshErg=",
"lastModified": 1752544651,
"narHash": "sha256-GllP7cmQu7zLZTs9z0J2gIL42IZHa9CBEXwBY9szT0U=",
"owner": "Mic92",
"repo": "sops-nix",
"rev": "56b24064fdcaedca53553b1a6d607fd23b613a24",
"rev": "2c8def626f54708a9c38a5861866660395bb3461",
"type": "github"
},
"original": {
@@ -154,6 +189,29 @@
"type": "github"
}
},
"system_tools": {
"inputs": {
"nixpkgs": [
"nixpkgs"
],
"pyproject-build-systems": "pyproject-build-systems",
"pyproject-nix": "pyproject-nix",
"uv2nix": "uv2nix"
},
"locked": {
"lastModified": 1753716864,
"narHash": "sha256-5G32jskzdJ3u87vnW02kJxUaudvbGN4CeBz751J4+JE=",
"owner": "RichieCahill",
"repo": "system_tools",
"rev": "86e4b9ca02db37612d1e8a381c14045de39c7ca9",
"type": "github"
},
"original": {
"owner": "RichieCahill",
"repo": "system_tools",
"type": "github"
}
},
"systems": {
"locked": {
"lastModified": 1689347949,
@@ -168,6 +226,31 @@
"repo": "default-linux",
"type": "github"
}
},
"uv2nix": {
"inputs": {
"nixpkgs": [
"system_tools",
"nixpkgs"
],
"pyproject-nix": [
"system_tools",
"pyproject-nix"
]
},
"locked": {
"lastModified": 1747441483,
"narHash": "sha256-W8BFXk5R0TuJcjIhcGoMpSOaIufGXpizK0pm+uTqynA=",
"owner": "pyproject-nix",
"repo": "uv2nix",
"rev": "582024dc64663e9f88d467c2f7f7b20d278349de",
"type": "github"
},
"original": {
"owner": "pyproject-nix",
"repo": "uv2nix",
"type": "github"
}
}
},
"root": "root",
+25 -43
View File
@@ -31,6 +31,11 @@
inputs.nixpkgs.follows = "nixpkgs";
};
system_tools = {
url = "github:RichieCahill/system_tools";
inputs.nixpkgs.follows = "nixpkgs";
};
sops-nix = {
url = "github:Mic92/sops-nix";
inputs.nixpkgs.follows = "nixpkgs";
@@ -54,7 +59,6 @@
system:
import nixpkgs {
inherit system;
overlays = builtins.attrValues outputs.overlays;
config.allowUnfree = true;
}
);
@@ -65,48 +69,26 @@
devShells = forEachSystem (pkgs: import ./shell.nix { inherit pkgs; });
formatter = forEachSystem (pkgs: pkgs.treefmt);
packages = forEachSystem (
pkgs:
let
installer = pkgs.callPackage ./python/installer/package.nix { };
installer-nixos = pkgs.callPackage ./python/installer/package.nix { patchElf = false; };
in
{
inherit installer installer-nixos;
default = installer;
}
// lib.optionalAttrs (pkgs.stdenv.hostPlatform.system == "x86_64-linux") {
iso = self.nixosConfigurations.iso.config.system.build.isoImage;
}
);
apps = forEachSystem (
pkgs:
let
system = pkgs.stdenv.hostPlatform.system;
installer = {
type = "app";
program = "${self.packages.${system}.installer}/bin/nixos-installer";
meta.description = "One-file NixOS ZFS installer.";
};
in
{
inherit installer;
default = installer;
}
);
nixosConfigurations =
let
hosts = builtins.attrNames (
lib.filterAttrs (_: type: type == "directory") (builtins.readDir ./systems)
);
mkHost =
name:
lib.nixosSystem {
modules = [ ./systems/${name} ];
specialArgs = { inherit inputs outputs; };
};
in
lib.genAttrs hosts mkHost;
nixosConfigurations = {
bob = lib.nixosSystem {
modules = [
./systems/bob
];
specialArgs = { inherit inputs outputs; };
};
jeeves = lib.nixosSystem {
modules = [
./systems/jeeves
];
specialArgs = { inherit inputs outputs; };
};
rhapsody-in-green = lib.nixosSystem {
modules = [
./systems/rhapsody-in-green
];
specialArgs = { inherit inputs outputs; };
};
};
};
}
+2 -33
View File
@@ -3,46 +3,15 @@
# When applied, the stable nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.stable'
stable = final: _prev: {
stable = import inputs.nixpkgs-stable {
system = final.stdenv.hostPlatform.system;
system = final.system;
config.allowUnfree = true;
};
};
# When applied, the master nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.master'
master = final: _prev: {
master = import inputs.nixpkgs-master {
system = final.stdenv.hostPlatform.system;
system = final.system;
config.allowUnfree = true;
};
};
python-env = final: _prev: {
my_python = final.python314.withPackages (
ps:
with ps;
[
alembic
apprise
apscheduler
fastapi
fastapi-cli
httpx
mypy
pgvector
psycopg
pydantic
pyfakefs
pytest
pytest-cov
pytest-mock
pytest-xdist
python-multipart
ruff
sqlalchemy
tenacity
tinytuya
typer
websockets
]
);
};
}
-125
View File
@@ -1,125 +0,0 @@
[project]
name = "system_tools"
version = "0.1.0"
description = ""
authors = [{ name = "Richie Cahill", email = "richie@tmmworkshop.com" }]
requires-python = "~=3.14.0"
readme = "README.md"
license = "MIT"
# these dependencies are a best effort and aren't guaranteed to work
# for up-to-date dependencies, see overlays/default.nix
dependencies = [
"alembic",
"apprise",
"apscheduler",
"beautifulsoup4",
"bm25s",
"ebooklib",
"fastapi",
"fastapi-cli",
"httpx",
"jinja2",
"pgvector",
"polars",
"psycopg[binary]",
"pydantic",
"pydantic-settings",
"python-multipart",
"sqlalchemy[asyncio]",
"tenacity",
"tiktoken",
"tinytuya",
"typer",
"uvicorn",
"websockets",
"yake",
]
[project.scripts]
database = "python.database_cli:app"
whisper-transcribe = "python.tools.whisper.transcribe:main"
[dependency-groups]
dev = [
"aiosqlite",
"mypy",
"pyfakefs",
"pytest-asyncio",
"pytest-cov",
"pytest-mock",
"pytest-xdist",
"pytest",
"ruff",
]
[tool.ruff]
target-version = "py314"
line-length = 120
lint.select = ["ALL"]
lint.ignore = [
"G004", # (PERM) This is a performers nit
"COM812", # (TEMP) conflicts when used with the formatter
"ISC001", # (TEMP) conflicts when used with the formatter
"S603", # (PERM) This is known to cause a false positive
"S607", # (PERM) This is becoming a consistent annoyance
]
[tool.ruff.lint.per-file-ignores]
"tests/**" = [
"ANN", # (perm) type annotations not needed in tests
"D", # (perm) docstrings not needed in tests
"PLR2004", # (perm) magic values are fine in test assertions
"S101", # (perm) pytest needs asserts
]
"python/stuff/**" = [
"T201", # (perm) I don't care about print statements dir
]
"python/testing/**" = [
"T201", # (perm) I don't care about print statements dir
"ERA001", # (perm) I don't care about print statements dir
]
"python/splendor/**" = [
"S311", # (perm) there is no security issue here
"T201", # (perm) I don't care about print statements dir
"PLR2004", # (temps) need to think about this
]
"python/orm/**" = [
"TC003", # (perm) this creates issues because sqlalchemy uses these at runtime
]
"python/congress_tracker/**" = [
"TC003", # (perm) this creates issues because sqlalchemy uses these at runtime
]
"python/alembic/**" = [
"INP001", # (perm) this creates LSP issues for alembic
]
[tool.ruff.lint.pydocstyle]
convention = "google"
[tool.ruff.lint.flake8-builtins]
builtins-ignorelist = ["id"]
[tool.ruff.lint.pylint]
max-args = 9
[tool.coverage.run]
source = ["system_tools"]
[tool.coverage.report]
exclude_lines = [
"pragma: no cover",
"if TYPE_CHECKING:",
"raise NotImplementedError",
"if __name__ == \"__main__\":",
]
[tool.pytest.ini_options]
addopts = "-n auto -ra"
asyncio_mode = "auto"
testpaths = ["tests"]
# --cov=system_tools --cov-report=term-missing --cov-report=xml --cov-report=html --cov-branch
-1
View File
@@ -1 +0,0 @@
"""Server Tools."""
-122
View File
@@ -1,122 +0,0 @@
"""Alembic."""
from __future__ import annotations
import logging
import sys
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal
from alembic import context
from alembic.script import write_hooks
from sqlalchemy.schema import CreateSchema
from python.common import bash_wrapper
from python.orm.common import get_postgres_engine
if TYPE_CHECKING:
from collections.abc import MutableMapping
from sqlalchemy.orm import DeclarativeBase
config = context.config
base_class: type[DeclarativeBase] = config.attributes.get("base")
if base_class is None:
error = "No base class provided. Use the database CLI to run alembic commands."
raise RuntimeError(error)
target_metadata = base_class.metadata
logging.basicConfig(
level="DEBUG",
datefmt="%Y-%m-%dT%H:%M:%S%z",
format="%(asctime)s %(levelname)s %(filename)s:%(lineno)d - %(message)s",
handlers=[logging.StreamHandler(sys.stdout)],
)
@write_hooks.register("dynamic_schema")
def dynamic_schema(filename: str, _options: dict[Any, Any]) -> None:
"""Dynamic schema."""
original_file = Path(filename).read_text()
schema_name = base_class.schema_name
dynamic_schema_file_part1 = original_file.replace(f"schema='{schema_name}'", "schema=schema")
dynamic_schema_file = dynamic_schema_file_part1.replace(f"'{schema_name}.", "f'{schema}.")
Path(filename).write_text(dynamic_schema_file)
@write_hooks.register("import_postgresql")
def import_postgresql(filename: str, _options: dict[Any, Any]) -> None:
"""Add postgresql dialect import when postgresql types are used."""
content = Path(filename).read_text()
if "postgresql." in content and "from sqlalchemy.dialects import postgresql" not in content:
content = content.replace(
"import sqlalchemy as sa\n",
"import sqlalchemy as sa\nfrom sqlalchemy.dialects import postgresql\n",
)
Path(filename).write_text(content)
@write_hooks.register("ruff")
def ruff_check_and_format(filename: str, _options: dict[Any, Any]) -> None:
"""Docstring for ruff_check_and_format."""
bash_wrapper(f"ruff check --fix {filename}")
bash_wrapper(f"ruff format {filename}")
def include_name(
name: str | None,
type_: Literal["schema", "table", "column", "index", "unique_constraint", "foreign_key_constraint"],
_parent_names: MutableMapping[Literal["schema_name", "table_name", "schema_qualified_table_name"], str | None],
) -> bool:
"""Filter tables to be included in the migration.
Args:
name (str): The name of the table.
type_ (str): The type of the table.
_parent_names (MutableMapping): The names of the parent tables.
Returns:
bool: True if the table should be included, False otherwise.
"""
if type_ == "schema":
# allows a database with multiple schemas to have separate alembic revisions
return name == target_metadata.schema
return True
def run_migrations_online() -> None:
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
env_prefix = config.attributes.get("env_prefix", "POSTGRES")
connectable = get_postgres_engine(name=env_prefix)
with connectable.connect() as connection:
schema = base_class.schema_name
if not connectable.dialect.has_schema(connection, schema):
answer = input(f"Schema {schema!r} does not exist. Create it? [y/N] ")
if answer.lower() != "y":
error = f"Schema {schema!r} does not exist. Exiting."
raise SystemExit(error)
connection.execute(CreateSchema(schema))
connection.commit()
context.configure(
connection=connection,
target_metadata=target_metadata,
include_schemas=True,
version_table_schema=schema,
include_name=include_name,
)
with context.begin_transaction():
context.run_migrations()
connection.commit()
run_migrations_online()
@@ -1,113 +0,0 @@
"""created contact api.
Revision ID: edd7dd61a3d2
Revises:
Create Date: 2026-01-11 15:45:59.909266
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "edd7dd61a3d2"
down_revision: str | None = None
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"contact",
sa.Column("name", sa.String(), nullable=False),
sa.Column("age", sa.Integer(), nullable=True),
sa.Column("bio", sa.String(), nullable=True),
sa.Column("current_job", sa.String(), nullable=True),
sa.Column("gender", sa.String(), nullable=True),
sa.Column("goals", sa.String(), nullable=True),
sa.Column("legal_name", sa.String(), nullable=True),
sa.Column("profile_pic", sa.String(), nullable=True),
sa.Column("safe_conversation_starters", sa.String(), nullable=True),
sa.Column("self_sufficiency_score", sa.Integer(), nullable=True),
sa.Column("social_structure_style", sa.String(), nullable=True),
sa.Column("ssn", sa.String(), nullable=True),
sa.Column("suffix", sa.String(), nullable=True),
sa.Column("timezone", sa.String(), nullable=True),
sa.Column("topics_to_avoid", sa.String(), nullable=True),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_contact")),
schema=schema,
)
op.create_table(
"need",
sa.Column("name", sa.String(), nullable=False),
sa.Column("description", sa.String(), nullable=True),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_need")),
schema=schema,
)
op.create_table(
"contact_need",
sa.Column("contact_id", sa.Integer(), nullable=False),
sa.Column("need_id", sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(
["contact_id"],
[f"{schema}.contact.id"],
name=op.f("fk_contact_need_contact_id_contact"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["need_id"], [f"{schema}.need.id"], name=op.f("fk_contact_need_need_id_need"), ondelete="CASCADE"
),
sa.PrimaryKeyConstraint("contact_id", "need_id", name=op.f("pk_contact_need")),
schema=schema,
)
op.create_table(
"contact_relationship",
sa.Column("contact_id", sa.Integer(), nullable=False),
sa.Column("related_contact_id", sa.Integer(), nullable=False),
sa.Column("relationship_type", sa.String(length=100), nullable=False),
sa.Column("closeness_weight", sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(
["contact_id"],
[f"{schema}.contact.id"],
name=op.f("fk_contact_relationship_contact_id_contact"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["related_contact_id"],
[f"{schema}.contact.id"],
name=op.f("fk_contact_relationship_related_contact_id_contact"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("contact_id", "related_contact_id", name=op.f("pk_contact_relationship")),
schema=schema,
)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("contact_relationship", schema=schema)
op.drop_table("contact_need", schema=schema)
op.drop_table("need", schema=schema)
op.drop_table("contact", schema=schema)
# ### end Alembic commands ###
@@ -1,135 +0,0 @@
"""add congress tracker tables.
Revision ID: 3f71565e38de
Revises: edd7dd61a3d2
Create Date: 2026-02-12 16:36:09.457303
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "3f71565e38de"
down_revision: str | None = "edd7dd61a3d2"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"bill",
sa.Column("congress", sa.Integer(), nullable=False),
sa.Column("bill_type", sa.String(), nullable=False),
sa.Column("number", sa.Integer(), nullable=False),
sa.Column("title", sa.String(), nullable=True),
sa.Column("title_short", sa.String(), nullable=True),
sa.Column("official_title", sa.String(), nullable=True),
sa.Column("status", sa.String(), nullable=True),
sa.Column("status_at", sa.Date(), nullable=True),
sa.Column("sponsor_bioguide_id", sa.String(), nullable=True),
sa.Column("subjects_top_term", sa.String(), nullable=True),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_bill")),
sa.UniqueConstraint("congress", "bill_type", "number", name="uq_bill_congress_type_number"),
schema=schema,
)
op.create_index("ix_bill_congress", "bill", ["congress"], unique=False, schema=schema)
op.create_table(
"legislator",
sa.Column("bioguide_id", sa.Text(), nullable=False),
sa.Column("thomas_id", sa.String(), nullable=True),
sa.Column("lis_id", sa.String(), nullable=True),
sa.Column("govtrack_id", sa.Integer(), nullable=True),
sa.Column("opensecrets_id", sa.String(), nullable=True),
sa.Column("fec_ids", sa.String(), nullable=True),
sa.Column("first_name", sa.String(), nullable=False),
sa.Column("last_name", sa.String(), nullable=False),
sa.Column("official_full_name", sa.String(), nullable=True),
sa.Column("nickname", sa.String(), nullable=True),
sa.Column("birthday", sa.Date(), nullable=True),
sa.Column("gender", sa.String(), nullable=True),
sa.Column("current_party", sa.String(), nullable=True),
sa.Column("current_state", sa.String(), nullable=True),
sa.Column("current_district", sa.Integer(), nullable=True),
sa.Column("current_chamber", sa.String(), nullable=True),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_legislator")),
schema=schema,
)
op.create_index(op.f("ix_legislator_bioguide_id"), "legislator", ["bioguide_id"], unique=True, schema=schema)
op.create_table(
"vote",
sa.Column("congress", sa.Integer(), nullable=False),
sa.Column("chamber", sa.String(), nullable=False),
sa.Column("session", sa.Integer(), nullable=False),
sa.Column("number", sa.Integer(), nullable=False),
sa.Column("vote_type", sa.String(), nullable=True),
sa.Column("question", sa.String(), nullable=True),
sa.Column("result", sa.String(), nullable=True),
sa.Column("result_text", sa.String(), nullable=True),
sa.Column("vote_date", sa.Date(), nullable=False),
sa.Column("yea_count", sa.Integer(), nullable=True),
sa.Column("nay_count", sa.Integer(), nullable=True),
sa.Column("not_voting_count", sa.Integer(), nullable=True),
sa.Column("present_count", sa.Integer(), nullable=True),
sa.Column("bill_id", sa.Integer(), nullable=True),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(["bill_id"], [f"{schema}.bill.id"], name=op.f("fk_vote_bill_id_bill")),
sa.PrimaryKeyConstraint("id", name=op.f("pk_vote")),
sa.UniqueConstraint("congress", "chamber", "session", "number", name="uq_vote_congress_chamber_session_number"),
schema=schema,
)
op.create_index("ix_vote_congress_chamber", "vote", ["congress", "chamber"], unique=False, schema=schema)
op.create_index("ix_vote_date", "vote", ["vote_date"], unique=False, schema=schema)
op.create_table(
"vote_record",
sa.Column("vote_id", sa.Integer(), nullable=False),
sa.Column("legislator_id", sa.Integer(), nullable=False),
sa.Column("position", sa.String(), nullable=False),
sa.ForeignKeyConstraint(
["legislator_id"],
[f"{schema}.legislator.id"],
name=op.f("fk_vote_record_legislator_id_legislator"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["vote_id"], [f"{schema}.vote.id"], name=op.f("fk_vote_record_vote_id_vote"), ondelete="CASCADE"
),
sa.PrimaryKeyConstraint("vote_id", "legislator_id", name=op.f("pk_vote_record")),
schema=schema,
)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("vote_record", schema=schema)
op.drop_index("ix_vote_date", table_name="vote", schema=schema)
op.drop_index("ix_vote_congress_chamber", table_name="vote", schema=schema)
op.drop_table("vote", schema=schema)
op.drop_index(op.f("ix_legislator_bioguide_id"), table_name="legislator", schema=schema)
op.drop_table("legislator", schema=schema)
op.drop_index("ix_bill_congress", table_name="bill", schema=schema)
op.drop_table("bill", schema=schema)
# ### end Alembic commands ###
@@ -1,58 +0,0 @@
"""adding SignalDevice for DeviceRegistry for signal bot.
Revision ID: 4c410c16e39c
Revises: 3f71565e38de
Create Date: 2026-03-09 14:51:24.228976
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "4c410c16e39c"
down_revision: str | None = "3f71565e38de"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"signal_device",
sa.Column("phone_number", sa.String(length=50), nullable=False),
sa.Column("safety_number", sa.String(), nullable=False),
sa.Column(
"trust_level",
postgresql.ENUM("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", schema=schema),
nullable=False,
),
sa.Column("last_seen", sa.DateTime(timezone=True), nullable=False),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_signal_device")),
sa.UniqueConstraint("phone_number", name=op.f("uq_signal_device_phone_number")),
schema=schema,
)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("signal_device", schema=schema)
# ### end Alembic commands ###
@@ -1,41 +0,0 @@
"""fixed safety number logic.
Revision ID: 99fec682516c
Revises: 4c410c16e39c
Create Date: 2026-03-09 16:25:25.085806
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "99fec682516c"
down_revision: str | None = "4c410c16e39c"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column("signal_device", "safety_number", existing_type=sa.VARCHAR(), nullable=True, schema=schema)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column("signal_device", "safety_number", existing_type=sa.VARCHAR(), nullable=False, schema=schema)
# ### end Alembic commands ###
@@ -1,54 +0,0 @@
"""add dead_letter_message table.
Revision ID: a1b2c3d4e5f6
Revises: 99fec682516c
Create Date: 2026-03-10 12:00:00.000000
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "a1b2c3d4e5f6"
down_revision: str | None = "99fec682516c"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
op.create_table(
"dead_letter_message",
sa.Column("source", sa.String(), nullable=False),
sa.Column("message", sa.Text(), nullable=False),
sa.Column("received_at", sa.DateTime(timezone=True), nullable=False),
sa.Column(
"status",
postgresql.ENUM("UNPROCESSED", "PROCESSED", name="message_status", schema=schema),
nullable=False,
),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_dead_letter_message")),
schema=schema,
)
def downgrade() -> None:
"""Downgrade."""
op.drop_table("dead_letter_message", schema=schema)
op.execute(sa.text(f"DROP TYPE IF EXISTS {schema}.message_status"))
@@ -1,66 +0,0 @@
"""adding roles to signal devices.
Revision ID: 2ef7ba690159
Revises: a1b2c3d4e5f6
Create Date: 2026-03-16 19:22:38.020350
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "2ef7ba690159"
down_revision: str | None = "a1b2c3d4e5f6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"role",
sa.Column("name", sa.String(length=50), nullable=False),
sa.Column("id", sa.SmallInteger(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_role")),
sa.UniqueConstraint("name", name=op.f("uq_role_name")),
schema=schema,
)
op.create_table(
"device_role",
sa.Column("device_id", sa.Integer(), nullable=False),
sa.Column("role_id", sa.SmallInteger(), nullable=False),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["device_id"], [f"{schema}.signal_device.id"], name=op.f("fk_device_role_device_id_signal_device")
),
sa.ForeignKeyConstraint(["role_id"], [f"{schema}.role.id"], name=op.f("fk_device_role_role_id_role")),
sa.PrimaryKeyConstraint("id", name=op.f("pk_device_role")),
sa.UniqueConstraint("device_id", "role_id", name="uq_device_role_device_role"),
schema=schema,
)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("device_role", schema=schema)
op.drop_table("role", schema=schema)
# ### end Alembic commands ###
@@ -1,171 +0,0 @@
"""seprating signal_bot database.
Revision ID: 6b275323f435
Revises: 2ef7ba690159
Create Date: 2026-03-18 08:34:28.785885
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "6b275323f435"
down_revision: str | None = "2ef7ba690159"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("device_role", schema=schema)
op.drop_table("signal_device", schema=schema)
op.drop_table("role", schema=schema)
op.drop_table("dead_letter_message", schema=schema)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"dead_letter_message",
sa.Column("source", sa.VARCHAR(), autoincrement=False, nullable=False),
sa.Column("message", sa.TEXT(), autoincrement=False, nullable=False),
sa.Column("received_at", postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False),
sa.Column(
"status",
postgresql.ENUM("UNPROCESSED", "PROCESSED", name="message_status", schema=schema),
autoincrement=False,
nullable=False,
),
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
sa.Column(
"created",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.Column(
"updated",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_dead_letter_message")),
schema=schema,
)
op.create_table(
"role",
sa.Column("name", sa.VARCHAR(length=50), autoincrement=False, nullable=False),
sa.Column(
"id",
sa.SMALLINT(),
server_default=sa.text(f"nextval('{schema}.role_id_seq'::regclass)"),
autoincrement=True,
nullable=False,
),
sa.Column(
"created",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.Column(
"updated",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_role")),
sa.UniqueConstraint(
"name", name=op.f("uq_role_name"), postgresql_include=[], postgresql_nulls_not_distinct=False
),
schema=schema,
)
op.create_table(
"signal_device",
sa.Column("phone_number", sa.VARCHAR(length=50), autoincrement=False, nullable=False),
sa.Column("safety_number", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column(
"trust_level",
postgresql.ENUM("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", schema=schema),
autoincrement=False,
nullable=False,
),
sa.Column("last_seen", postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False),
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
sa.Column(
"created",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.Column(
"updated",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_signal_device")),
sa.UniqueConstraint(
"phone_number",
name=op.f("uq_signal_device_phone_number"),
postgresql_include=[],
postgresql_nulls_not_distinct=False,
),
schema=schema,
)
op.create_table(
"device_role",
sa.Column("device_id", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("role_id", sa.SMALLINT(), autoincrement=False, nullable=False),
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
sa.Column(
"created",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.Column(
"updated",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.ForeignKeyConstraint(
["device_id"], [f"{schema}.signal_device.id"], name=op.f("fk_device_role_device_id_signal_device")
),
sa.ForeignKeyConstraint(["role_id"], [f"{schema}.role.id"], name=op.f("fk_device_role_role_id_role")),
sa.PrimaryKeyConstraint("id", name=op.f("pk_device_role")),
sa.UniqueConstraint(
"device_id",
"role_id",
name=op.f("uq_device_role_device_role"),
postgresql_include=[],
postgresql_nulls_not_distinct=False,
),
schema=schema,
)
# ### end Alembic commands ###
@@ -1,187 +0,0 @@
"""removed ds table from richie DB.
Revision ID: c8a794340928
Revises: 6b275323f435
Create Date: 2026-03-29 15:29:23.643146
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "c8a794340928"
down_revision: str | None = "6b275323f435"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("vote_record", schema=schema)
op.drop_index(op.f("ix_vote_congress_chamber"), table_name="vote", schema=schema)
op.drop_index(op.f("ix_vote_date"), table_name="vote", schema=schema)
op.drop_index(op.f("ix_legislator_bioguide_id"), table_name="legislator", schema=schema)
op.drop_table("legislator", schema=schema)
op.drop_table("vote", schema=schema)
op.drop_index(op.f("ix_bill_congress"), table_name="bill", schema=schema)
op.drop_table("bill", schema=schema)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"vote",
sa.Column("congress", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("chamber", sa.VARCHAR(), autoincrement=False, nullable=False),
sa.Column("session", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("number", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("vote_type", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("question", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("result", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("result_text", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("vote_date", sa.DATE(), autoincrement=False, nullable=False),
sa.Column("yea_count", sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column("nay_count", sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column("not_voting_count", sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column("present_count", sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column("bill_id", sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
sa.Column(
"created",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.Column(
"updated",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.ForeignKeyConstraint(["bill_id"], [f"{schema}.bill.id"], name=op.f("fk_vote_bill_id_bill")),
sa.PrimaryKeyConstraint("id", name=op.f("pk_vote")),
sa.UniqueConstraint(
"congress",
"chamber",
"session",
"number",
name=op.f("uq_vote_congress_chamber_session_number"),
postgresql_include=[],
postgresql_nulls_not_distinct=False,
),
schema=schema,
)
op.create_index(op.f("ix_vote_date"), "vote", ["vote_date"], unique=False, schema=schema)
op.create_index(op.f("ix_vote_congress_chamber"), "vote", ["congress", "chamber"], unique=False, schema=schema)
op.create_table(
"vote_record",
sa.Column("vote_id", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("legislator_id", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("position", sa.VARCHAR(), autoincrement=False, nullable=False),
sa.ForeignKeyConstraint(
["legislator_id"],
[f"{schema}.legislator.id"],
name=op.f("fk_vote_record_legislator_id_legislator"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["vote_id"], [f"{schema}.vote.id"], name=op.f("fk_vote_record_vote_id_vote"), ondelete="CASCADE"
),
sa.PrimaryKeyConstraint("vote_id", "legislator_id", name=op.f("pk_vote_record")),
schema=schema,
)
op.create_table(
"legislator",
sa.Column("bioguide_id", sa.TEXT(), autoincrement=False, nullable=False),
sa.Column("thomas_id", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("lis_id", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("govtrack_id", sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column("opensecrets_id", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("fec_ids", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("first_name", sa.VARCHAR(), autoincrement=False, nullable=False),
sa.Column("last_name", sa.VARCHAR(), autoincrement=False, nullable=False),
sa.Column("official_full_name", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("nickname", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("birthday", sa.DATE(), autoincrement=False, nullable=True),
sa.Column("gender", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("current_party", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("current_state", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("current_district", sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column("current_chamber", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
sa.Column(
"created",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.Column(
"updated",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_legislator")),
schema=schema,
)
op.create_index(op.f("ix_legislator_bioguide_id"), "legislator", ["bioguide_id"], unique=True, schema=schema)
op.create_table(
"bill",
sa.Column("congress", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("bill_type", sa.VARCHAR(), autoincrement=False, nullable=False),
sa.Column("number", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("title", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("title_short", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("official_title", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("status", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("status_at", sa.DATE(), autoincrement=False, nullable=True),
sa.Column("sponsor_bioguide_id", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("subjects_top_term", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
sa.Column(
"created",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.Column(
"updated",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_bill")),
sa.UniqueConstraint(
"congress",
"bill_type",
"number",
name=op.f("uq_bill_congress_type_number"),
postgresql_include=[],
postgresql_nulls_not_distinct=False,
),
schema=schema,
)
op.create_index(op.f("ix_bill_congress"), "bill", ["congress"], unique=False, schema=schema)
# ### end Alembic commands ###
@@ -1,93 +0,0 @@
"""adding audiobook libreary metadata.
Revision ID: d7864d1ffc17
Revises: c8a794340928
Create Date: 2026-06-03 20:24:09.200837
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "d7864d1ffc17"
down_revision: str | None = "c8a794340928"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"audiobook_author",
sa.Column("name", sa.String(), nullable=False),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_audiobook_author")),
sa.UniqueConstraint("name", name=op.f("uq_audiobook_author_name")),
schema=schema,
)
op.create_table(
"audiobook_series",
sa.Column("name", sa.String(), nullable=False),
sa.Column("author_id", sa.Integer(), nullable=False),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["author_id"],
[f"{schema}.audiobook_author.id"],
name=op.f("fk_audiobook_series_author_id_audiobook_author"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_audiobook_series")),
sa.UniqueConstraint("author_id", "name", name=op.f("uq_audiobook_series_author_id")),
schema=schema,
)
op.create_table(
"audiobook",
sa.Column("title", sa.String(), nullable=False),
sa.Column("author_id", sa.Integer(), nullable=False),
sa.Column("series_id", sa.Integer(), nullable=True),
sa.Column("series_index", sa.Integer(), nullable=False),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["author_id"],
[f"{schema}.audiobook_author.id"],
name=op.f("fk_audiobook_author_id_audiobook_author"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["series_id"],
[f"{schema}.audiobook_series.id"],
name=op.f("fk_audiobook_series_id_audiobook_series"),
ondelete="SET NULL",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_audiobook")),
schema=schema,
)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("audiobook", schema=schema)
op.drop_table("audiobook_series", schema=schema)
op.drop_table("audiobook_author", schema=schema)
# ### end Alembic commands ###
@@ -1,200 +0,0 @@
"""add ebook search tables.
Revision ID: 2db132cace1a
Revises: b3c60cc5beb5
Create Date: 2026-06-10 22:10:54.379159
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pgvector
import sqlalchemy as sa
from alembic import op
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "2db132cace1a"
down_revision: str | None = "b3c60cc5beb5"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"ebook_embedding_model",
sa.Column("name", sa.String(), nullable=False),
sa.Column("dimension", sa.Integer(), nullable=False),
sa.Column("is_default", sa.Boolean(), nullable=False),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_embedding_model")),
sa.UniqueConstraint("name", name=op.f("uq_ebook_embedding_model_name")),
schema=schema,
)
op.create_table(
"ebook_source",
sa.Column("title", sa.String(), nullable=False),
sa.Column("author", sa.String(), nullable=True),
sa.Column("language", sa.String(), nullable=True),
sa.Column("publisher", sa.String(), nullable=True),
sa.Column("identifier", sa.String(), nullable=True),
sa.Column("file_path", sa.String(), nullable=False),
sa.Column("file_sha256", sa.String(length=64), nullable=False),
sa.Column("file_mtime", sa.DateTime(timezone=True), nullable=False),
sa.Column("file_size", sa.BigInteger(), nullable=False),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_source")),
sa.UniqueConstraint("file_path", name=op.f("uq_ebook_source_file_path")),
sa.UniqueConstraint("file_sha256", name=op.f("uq_ebook_source_file_sha256")),
schema=schema,
)
op.create_table(
"ebook_chapter",
sa.Column("source_id", sa.Integer(), nullable=False),
sa.Column("spine_index", sa.Integer(), nullable=False),
sa.Column("title", sa.String(), nullable=True),
sa.Column("href", sa.String(), nullable=True),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["source_id"],
[f"{schema}.ebook_source.id"],
name=op.f("fk_ebook_chapter_source_id_ebook_source"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chapter")),
sa.UniqueConstraint("source_id", "spine_index", name=op.f("uq_ebook_chapter_source_id")),
schema=schema,
)
op.create_table(
"ebook_chunk",
sa.Column("source_id", sa.Integer(), nullable=False),
sa.Column("chapter_id", sa.Integer(), nullable=True),
sa.Column("chunk_index", sa.Integer(), nullable=False),
sa.Column("text", sa.String(), nullable=False),
sa.Column("token_start", sa.Integer(), nullable=False),
sa.Column("token_count", sa.Integer(), nullable=False),
sa.Column("page_label", sa.String(), nullable=True),
sa.Column("content_sha256", sa.String(length=64), nullable=False),
sa.Column("search_text", sa.String(), nullable=False),
sa.Column("id", sa.BigInteger(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["chapter_id"],
[f"{schema}.ebook_chapter.id"],
name=op.f("fk_ebook_chunk_chapter_id_ebook_chapter"),
ondelete="SET NULL",
),
sa.ForeignKeyConstraint(
["source_id"],
[f"{schema}.ebook_source.id"],
name=op.f("fk_ebook_chunk_source_id_ebook_source"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk")),
sa.UniqueConstraint("source_id", "chunk_index", name="uq_ebook_chunk_source_id_chunk_index"),
sa.UniqueConstraint("source_id", "content_sha256", name="uq_ebook_chunk_source_id_content_sha256"),
schema=schema,
)
op.create_table(
"ebook_chunk_embedding_1024",
sa.Column("chunk_id", sa.BigInteger(), nullable=False),
sa.Column("model_id", sa.Integer(), nullable=False),
sa.Column("embedding", pgvector.sqlalchemy.vector.VECTOR(dim=1024), nullable=False),
sa.Column("id", sa.BigInteger(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["chunk_id"],
[f"{schema}.ebook_chunk.id"],
name=op.f("fk_ebook_chunk_embedding_1024_chunk_id_ebook_chunk"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["model_id"],
[f"{schema}.ebook_embedding_model.id"],
name=op.f("fk_ebook_chunk_embedding_1024_model_id_ebook_embedding_model"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk_embedding_1024")),
sa.UniqueConstraint("chunk_id", "model_id", name=op.f("uq_ebook_chunk_embedding_1024_chunk_id")),
schema=schema,
)
op.create_table(
"ebook_chunk_embedding_2560",
sa.Column("chunk_id", sa.BigInteger(), nullable=False),
sa.Column("model_id", sa.Integer(), nullable=False),
sa.Column("embedding", pgvector.sqlalchemy.vector.VECTOR(dim=2560), nullable=False),
sa.Column("id", sa.BigInteger(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["chunk_id"],
[f"{schema}.ebook_chunk.id"],
name=op.f("fk_ebook_chunk_embedding_2560_chunk_id_ebook_chunk"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["model_id"],
[f"{schema}.ebook_embedding_model.id"],
name=op.f("fk_ebook_chunk_embedding_2560_model_id_ebook_embedding_model"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk_embedding_2560")),
sa.UniqueConstraint("chunk_id", "model_id", name=op.f("uq_ebook_chunk_embedding_2560_chunk_id")),
schema=schema,
)
op.create_table(
"ebook_chunk_embedding_4096",
sa.Column("chunk_id", sa.BigInteger(), nullable=False),
sa.Column("model_id", sa.Integer(), nullable=False),
sa.Column("embedding", pgvector.sqlalchemy.vector.VECTOR(dim=4096), nullable=False),
sa.Column("id", sa.BigInteger(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["chunk_id"],
[f"{schema}.ebook_chunk.id"],
name=op.f("fk_ebook_chunk_embedding_4096_chunk_id_ebook_chunk"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["model_id"],
[f"{schema}.ebook_embedding_model.id"],
name=op.f("fk_ebook_chunk_embedding_4096_model_id_ebook_embedding_model"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk_embedding_4096")),
sa.UniqueConstraint("chunk_id", "model_id", name=op.f("uq_ebook_chunk_embedding_4096_chunk_id")),
schema=schema,
)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("ebook_chunk_embedding_4096", schema=schema)
op.drop_table("ebook_chunk_embedding_2560", schema=schema)
op.drop_table("ebook_chunk_embedding_1024", schema=schema)
op.drop_table("ebook_chunk", schema=schema)
op.drop_table("ebook_chapter", schema=schema)
op.drop_table("ebook_source", schema=schema)
op.drop_table("ebook_embedding_model", schema=schema)
# ### end Alembic commands ###
@@ -1,63 +0,0 @@
"""updated series_index to float and added UniqueConstraint to audiobook and audiobook_author.
Revision ID: b3c60cc5beb5
Revises: d7864d1ffc17
Create Date: 2026-06-10 20:02:43.073725
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "b3c60cc5beb5"
down_revision: str | None = "d7864d1ffc17"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column(
"audiobook",
"series_index",
existing_type=sa.INTEGER(),
type_=sa.Float(),
existing_nullable=False,
schema=schema,
)
op.create_unique_constraint(
op.f("uq_audiobook_author_id"),
"audiobook",
["author_id", "series_id", "title"],
schema=schema,
postgresql_nulls_not_distinct=True,
)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_constraint(op.f("uq_audiobook_author_id"), "audiobook", schema=schema, type_="unique")
op.alter_column(
"audiobook",
"series_index",
existing_type=sa.Float(),
type_=sa.INTEGER(),
existing_nullable=False,
schema=schema,
)
# ### end Alembic commands ###
@@ -1,54 +0,0 @@
"""add 1024 ebook embedding cosine index.
Revision ID: c460105682d2
Revises: 2db132cace1a
Create Date: 2026-06-13 19:53:45.680289
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from alembic import op
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "c460105682d2"
down_revision: str | None = "2db132cace1a"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_index(
"ix_ebook_chunk_embedding_1024_embedding_cosine",
"ebook_chunk_embedding_1024",
["embedding"],
unique=False,
schema=schema,
postgresql_using="hnsw",
postgresql_ops={"embedding": "vector_cosine_ops"},
)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(
"ix_ebook_chunk_embedding_1024_embedding_cosine",
table_name="ebook_chunk_embedding_1024",
schema=schema,
postgresql_using="hnsw",
postgresql_ops={"embedding": "vector_cosine_ops"},
)
# ### end Alembic commands ###
@@ -1,103 +0,0 @@
"""adding haproxy data.
Revision ID: 96d72c748c24
Revises: c460105682d2
Create Date: 2026-06-23 16:37:17.768851
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "96d72c748c24"
down_revision: str | None = "c460105682d2"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"haproxy_request",
sa.Column("line_hash", sa.String(), nullable=False),
sa.Column("requested_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("client_ip", sa.String(), nullable=False),
sa.Column("client_port", sa.Integer(), nullable=False),
sa.Column("frontend", sa.String(), nullable=False),
sa.Column("ssl", sa.Boolean(), nullable=False),
sa.Column("backend", sa.String(), nullable=False),
sa.Column("server", sa.String(), nullable=False),
sa.Column("time_request", sa.Integer(), nullable=False),
sa.Column("time_queue", sa.Integer(), nullable=False),
sa.Column("time_connect", sa.Integer(), nullable=False),
sa.Column("time_response", sa.Integer(), nullable=False),
sa.Column("time_total", sa.Integer(), nullable=False),
sa.Column("status_code", sa.Integer(), nullable=False),
sa.Column("bytes_read", sa.BigInteger(), nullable=False),
sa.Column("termination_state", sa.String(), nullable=False),
sa.Column("active_connections", sa.Integer(), nullable=False),
sa.Column("frontend_connections", sa.Integer(), nullable=False),
sa.Column("backend_connections", sa.Integer(), nullable=False),
sa.Column("server_connections", sa.Integer(), nullable=False),
sa.Column("retries", sa.Integer(), nullable=False),
sa.Column("server_queue", sa.Integer(), nullable=False),
sa.Column("backend_queue", sa.Integer(), nullable=False),
sa.Column("host", sa.String(), nullable=True),
sa.Column("user_agent", sa.String(), nullable=True),
sa.Column("method", sa.String(), nullable=False),
sa.Column("target", sa.String(), nullable=False),
sa.Column("path", sa.String(), nullable=False),
sa.Column("query", sa.String(), nullable=True),
sa.Column("http_version", sa.String(), nullable=False),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_haproxy_request")),
sa.UniqueConstraint("line_hash", name=op.f("uq_haproxy_request_line_hash")),
schema=schema,
)
op.create_index(op.f("ix_haproxy_request_backend"), "haproxy_request", ["backend"], unique=False, schema=schema)
op.create_index(op.f("ix_haproxy_request_client_ip"), "haproxy_request", ["client_ip"], unique=False, schema=schema)
op.create_index(op.f("ix_haproxy_request_host"), "haproxy_request", ["host"], unique=False, schema=schema)
op.create_index(op.f("ix_haproxy_request_path"), "haproxy_request", ["path"], unique=False, schema=schema)
op.create_index(
op.f("ix_haproxy_request_requested_at"), "haproxy_request", ["requested_at"], unique=False, schema=schema
)
op.create_index(
op.f("ix_haproxy_request_status_code"), "haproxy_request", ["status_code"], unique=False, schema=schema
)
op.create_index(
op.f("ix_haproxy_request_time_response"), "haproxy_request", ["time_response"], unique=False, schema=schema
)
op.create_index(
op.f("ix_haproxy_request_user_agent"), "haproxy_request", ["user_agent"], unique=False, schema=schema
)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f("ix_haproxy_request_user_agent"), table_name="haproxy_request", schema=schema)
op.drop_index(op.f("ix_haproxy_request_time_response"), table_name="haproxy_request", schema=schema)
op.drop_index(op.f("ix_haproxy_request_status_code"), table_name="haproxy_request", schema=schema)
op.drop_index(op.f("ix_haproxy_request_requested_at"), table_name="haproxy_request", schema=schema)
op.drop_index(op.f("ix_haproxy_request_path"), table_name="haproxy_request", schema=schema)
op.drop_index(op.f("ix_haproxy_request_host"), table_name="haproxy_request", schema=schema)
op.drop_index(op.f("ix_haproxy_request_client_ip"), table_name="haproxy_request", schema=schema)
op.drop_index(op.f("ix_haproxy_request_backend"), table_name="haproxy_request", schema=schema)
op.drop_table("haproxy_request", schema=schema)
# ### end Alembic commands ###
@@ -1,206 +0,0 @@
"""adding Phrase metadata tables.
Revision ID: dddee09eddcc
Revises: 96d72c748c24
Create Date: 2026-06-29 00:49:07.344159
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "dddee09eddcc"
down_revision: str | None = "96d72c748c24"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"candidate_phrases",
sa.Column("book_id", sa.Integer(), nullable=False),
sa.Column("series_id", sa.Integer(), nullable=True),
sa.Column("phrase_text", sa.Text(), nullable=False),
sa.Column("phrase_norm", sa.Text(), nullable=False),
sa.Column("token_count", sa.Integer(), nullable=False),
sa.Column("source_raw_ngram", sa.Boolean(), nullable=False),
sa.Column("source_yake", sa.Boolean(), nullable=False),
sa.Column("source_spacy_ner", sa.Boolean(), nullable=False),
sa.Column("source_spacy_noun_chunk", sa.Boolean(), nullable=False),
sa.Column("source_capitalized", sa.Boolean(), nullable=False),
sa.Column("source_metadata", sa.Boolean(), nullable=False),
sa.Column("spacy_label", sa.String(), nullable=True),
sa.Column("raw_count", sa.Integer(), nullable=False),
sa.Column("chapter_count", sa.Integer(), nullable=False),
sa.Column("yake_score", sa.Float(), nullable=True),
sa.Column("candidate_score", sa.Float(), nullable=False),
sa.Column(
"sample_contexts",
sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), "postgresql"),
nullable=True,
),
sa.Column("llm_judged", sa.Boolean(), nullable=False),
sa.Column("llm_keep", sa.Boolean(), nullable=True),
sa.Column("llm_confidence", sa.Float(), nullable=True),
sa.Column("llm_category", sa.String(), nullable=True),
sa.Column("llm_reason", sa.Text(), nullable=True),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["book_id"],
[f"{schema}.ebook_source.id"],
name=op.f("fk_candidate_phrases_book_id_ebook_source"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_candidate_phrases")),
sa.UniqueConstraint("book_id", "phrase_norm", name="uq_candidate_phrases_book_id_phrase_norm"),
schema=schema,
)
op.create_index(
"candidate_phrases_book_norm_idx", "candidate_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
)
op.create_index(
"candidate_phrases_book_score_idx",
"candidate_phrases",
["book_id", "candidate_score"],
unique=False,
schema=schema,
)
op.create_table(
"protected_phrases",
sa.Column("book_id", sa.Integer(), nullable=True),
sa.Column("series_id", sa.Integer(), nullable=True),
sa.Column("phrase_text", sa.Text(), nullable=False),
sa.Column("phrase_norm", sa.Text(), nullable=False),
sa.Column("canonical_id", sa.String(), nullable=False),
sa.Column("phrase_type", sa.String(), nullable=True),
sa.Column("token_count", sa.Integer(), nullable=False),
sa.Column("confidence", sa.Float(), nullable=False),
sa.Column("importance", sa.Float(), nullable=False),
sa.Column("allow_nested", sa.Boolean(), nullable=False),
sa.Column("suppress_children", sa.Boolean(), nullable=False),
sa.Column("source_candidate_id", sa.Integer(), nullable=True),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["book_id"],
[f"{schema}.ebook_source.id"],
name=op.f("fk_protected_phrases_book_id_ebook_source"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["source_candidate_id"],
[f"{schema}.candidate_phrases.id"],
name=op.f("fk_protected_phrases_source_candidate_id_candidate_phrases"),
ondelete="SET NULL",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_protected_phrases")),
sa.UniqueConstraint("book_id", "phrase_norm", name="uq_protected_phrases_book_id_phrase_norm"),
schema=schema,
)
op.create_index(
"protected_phrases_book_norm_idx", "protected_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
)
op.create_index("protected_phrases_norm_idx", "protected_phrases", ["phrase_norm"], unique=False, schema=schema)
op.create_index(
"protected_phrases_series_norm_idx",
"protected_phrases",
["series_id", "phrase_norm"],
unique=False,
schema=schema,
)
op.create_table(
"chunk_phrase_mentions",
sa.Column("chunk_id", sa.BigInteger(), nullable=False),
sa.Column("phrase_id", sa.Integer(), nullable=False),
sa.Column("book_id", sa.Integer(), nullable=True),
sa.Column("series_id", sa.Integer(), nullable=True),
sa.Column("start_char", sa.Integer(), nullable=False),
sa.Column("end_char", sa.Integer(), nullable=True),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["book_id"],
[f"{schema}.ebook_source.id"],
name=op.f("fk_chunk_phrase_mentions_book_id_ebook_source"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["chunk_id"],
[f"{schema}.ebook_chunk.id"],
name=op.f("fk_chunk_phrase_mentions_chunk_id_ebook_chunk"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["phrase_id"],
[f"{schema}.protected_phrases.id"],
name=op.f("fk_chunk_phrase_mentions_phrase_id_protected_phrases"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_chunk_phrase_mentions")),
sa.UniqueConstraint("chunk_id", "phrase_id", "start_char", name="uq_chunk_phrase_mentions_chunk_phrase_start"),
schema=schema,
)
op.create_index(
"chunk_phrase_mentions_chunk_idx", "chunk_phrase_mentions", ["chunk_id"], unique=False, schema=schema
)
op.create_index(
"chunk_phrase_mentions_phrase_idx", "chunk_phrase_mentions", ["phrase_id"], unique=False, schema=schema
)
op.create_table(
"phrase_aliases",
sa.Column("phrase_id", sa.Integer(), nullable=False),
sa.Column("alias_text", sa.Text(), nullable=False),
sa.Column("alias_norm", sa.Text(), nullable=False),
sa.Column("confidence", sa.Float(), nullable=False),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["phrase_id"],
[f"{schema}.protected_phrases.id"],
name=op.f("fk_phrase_aliases_phrase_id_protected_phrases"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_phrase_aliases")),
sa.UniqueConstraint("phrase_id", "alias_norm", name="uq_phrase_aliases_phrase_id_alias_norm"),
schema=schema,
)
op.create_index("phrase_aliases_norm_idx", "phrase_aliases", ["alias_norm"], unique=False, schema=schema)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index("phrase_aliases_norm_idx", table_name="phrase_aliases", schema=schema)
op.drop_table("phrase_aliases", schema=schema)
op.drop_index("chunk_phrase_mentions_phrase_idx", table_name="chunk_phrase_mentions", schema=schema)
op.drop_index("chunk_phrase_mentions_chunk_idx", table_name="chunk_phrase_mentions", schema=schema)
op.drop_table("chunk_phrase_mentions", schema=schema)
op.drop_index("protected_phrases_series_norm_idx", table_name="protected_phrases", schema=schema)
op.drop_index("protected_phrases_norm_idx", table_name="protected_phrases", schema=schema)
op.drop_index("protected_phrases_book_norm_idx", table_name="protected_phrases", schema=schema)
op.drop_table("protected_phrases", schema=schema)
op.drop_index("candidate_phrases_book_score_idx", table_name="candidate_phrases", schema=schema)
op.drop_index("candidate_phrases_book_norm_idx", table_name="candidate_phrases", schema=schema)
op.drop_table("candidate_phrases", schema=schema)
# ### end Alembic commands ###
-36
View File
@@ -1,36 +0,0 @@
"""${message}.
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from python.orm import ${config.attributes["base"].__name__}
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = ${repr(up_revision)}
down_revision: str | None = ${repr(down_revision)}
branch_labels: str | Sequence[str] | None = ${repr(branch_labels)}
depends_on: str | Sequence[str] | None = ${repr(depends_on)}
schema=${config.attributes["base"].__name__}.schema_name
def upgrade() -> None:
"""Upgrade."""
${upgrades if upgrades else "pass"}
def downgrade() -> None:
"""Downgrade."""
${downgrades if downgrades else "pass"}
@@ -1,80 +0,0 @@
"""starting van invintory.
Revision ID: 15e733499804
Revises:
Create Date: 2026-03-08 00:18:20.759720
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from python.orm import VanInventoryBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "15e733499804"
down_revision: str | None = None
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = VanInventoryBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"items",
sa.Column("name", sa.String(), nullable=False),
sa.Column("quantity", sa.Float(), nullable=False),
sa.Column("unit", sa.String(), nullable=False),
sa.Column("category", sa.String(), nullable=True),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_items")),
sa.UniqueConstraint("name", name=op.f("uq_items_name")),
schema=schema,
)
op.create_table(
"meals",
sa.Column("name", sa.String(), nullable=False),
sa.Column("instructions", sa.String(), nullable=True),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_meals")),
sa.UniqueConstraint("name", name=op.f("uq_meals_name")),
schema=schema,
)
op.create_table(
"meal_ingredients",
sa.Column("meal_id", sa.Integer(), nullable=False),
sa.Column("item_id", sa.Integer(), nullable=False),
sa.Column("quantity_needed", sa.Float(), nullable=False),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(["item_id"], [f"{schema}.items.id"], name=op.f("fk_meal_ingredients_item_id_items")),
sa.ForeignKeyConstraint(["meal_id"], [f"{schema}.meals.id"], name=op.f("fk_meal_ingredients_meal_id_meals")),
sa.PrimaryKeyConstraint("id", name=op.f("pk_meal_ingredients")),
sa.UniqueConstraint("meal_id", "item_id", name=op.f("uq_meal_ingredients_meal_id")),
schema=schema,
)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("meal_ingredients", schema=schema)
op.drop_table("meals", schema=schema)
op.drop_table("items", schema=schema)
# ### end Alembic commands ###
-1
View File
@@ -1 +0,0 @@
"""FastAPI applications."""
-56
View File
@@ -1,56 +0,0 @@
"""FastAPI interface for Contact database."""
from __future__ import annotations
import logging
from contextlib import asynccontextmanager
from typing import TYPE_CHECKING, Annotated
import typer
import uvicorn
from fastapi import FastAPI
from python.api.routers import contact_router, views_router
from python.common import configure_logger
from python.fastapi_tools import ZstdMiddleware
from python.orm.common import get_postgres_engine
if TYPE_CHECKING:
from collections.abc import AsyncIterator
logger = logging.getLogger(__name__)
def create_app() -> FastAPI:
"""Create and configure the FastAPI application."""
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
"""Manage application lifespan."""
app.state.engine = get_postgres_engine()
yield
app.state.engine.dispose()
app = FastAPI(title="Contact Database API", lifespan=lifespan)
app.add_middleware(ZstdMiddleware)
app.include_router(contact_router)
app.include_router(views_router)
return app
def serve(
host: Annotated[str, typer.Option("--host", "-h", help="Host to bind to")],
port: Annotated[int, typer.Option("--port", "-p", help="Port to bind to")] = 8000,
log_level: Annotated[str, typer.Option("--log-level", "-l", help="Log level")] = "INFO",
) -> None:
"""Start the Contact API server."""
configure_logger(log_level)
app = create_app()
uvicorn.run(app, host=host, port=port)
if __name__ == "__main__":
typer.run(serve)
-6
View File
@@ -1,6 +0,0 @@
"""API routers."""
from python.api.routers.contact import router as contact_router
from python.api.routers.views import router as views_router
__all__ = ["contact_router", "views_router"]
-481
View File
@@ -1,481 +0,0 @@
"""Contact API router."""
from pathlib import Path
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import HTMLResponse
from fastapi.templating import Jinja2Templates
from pydantic import BaseModel
from sqlalchemy import select
from sqlalchemy.orm import selectinload
from python.fastapi_tools.db import DbSession # noqa: TC001 this is a FastAPI needed at runtime
from python.orm.richie.contact import Contact, ContactRelationship, Need, RelationshipType
TEMPLATES_DIR = Path(__file__).parent.parent / "templates"
templates = Jinja2Templates(directory=TEMPLATES_DIR)
def _is_htmx(request: Request) -> bool:
"""Check if the request is from HTMX."""
return request.headers.get("HX-Request") == "true"
class NeedBase(BaseModel):
"""Base schema for Need."""
name: str
description: str | None = None
class NeedCreate(NeedBase):
"""Schema for creating a Need."""
class NeedResponse(NeedBase):
"""Schema for Need response."""
id: int
model_config = {"from_attributes": True}
class ContactRelationshipCreate(BaseModel):
"""Schema for creating a contact relationship."""
related_contact_id: int
relationship_type: RelationshipType
closeness_weight: int | None = None
class ContactRelationshipUpdate(BaseModel):
"""Schema for updating a contact relationship."""
relationship_type: RelationshipType | None = None
closeness_weight: int | None = None
class ContactRelationshipResponse(BaseModel):
"""Schema for contact relationship response."""
contact_id: int
related_contact_id: int
relationship_type: str
closeness_weight: int
model_config = {"from_attributes": True}
class RelationshipTypeInfo(BaseModel):
"""Information about a relationship type."""
value: str
display_name: str
default_weight: int
class GraphNode(BaseModel):
"""Node in the relationship graph."""
id: int
name: str
current_job: str | None = None
class GraphEdge(BaseModel):
"""Edge in the relationship graph."""
source: int
target: int
relationship_type: str
closeness_weight: int
class GraphData(BaseModel):
"""Complete graph data for visualization."""
nodes: list[GraphNode]
edges: list[GraphEdge]
class ContactBase(BaseModel):
"""Base schema for Contact."""
name: str
age: int | None = None
bio: str | None = None
current_job: str | None = None
gender: str | None = None
goals: str | None = None
legal_name: str | None = None
profile_pic: str | None = None
safe_conversation_starters: str | None = None
self_sufficiency_score: int | None = None
social_structure_style: str | None = None
ssn: str | None = None
suffix: str | None = None
timezone: str | None = None
topics_to_avoid: str | None = None
class ContactCreate(ContactBase):
"""Schema for creating a Contact."""
need_ids: list[int] = []
class ContactUpdate(BaseModel):
"""Schema for updating a Contact."""
name: str | None = None
age: int | None = None
bio: str | None = None
current_job: str | None = None
gender: str | None = None
goals: str | None = None
legal_name: str | None = None
profile_pic: str | None = None
safe_conversation_starters: str | None = None
self_sufficiency_score: int | None = None
social_structure_style: str | None = None
ssn: str | None = None
suffix: str | None = None
timezone: str | None = None
topics_to_avoid: str | None = None
need_ids: list[int] | None = None
class ContactResponse(ContactBase):
"""Schema for Contact response with relationships."""
id: int
needs: list[NeedResponse] = []
related_to: list[ContactRelationshipResponse] = []
related_from: list[ContactRelationshipResponse] = []
model_config = {"from_attributes": True}
class ContactListResponse(ContactBase):
"""Schema for Contact list response."""
id: int
model_config = {"from_attributes": True}
router = APIRouter(prefix="/api", tags=["contacts"])
@router.post("/needs", response_model=NeedResponse)
def create_need(need: NeedCreate, db: DbSession) -> Need:
"""Create a new need."""
db_need = Need(name=need.name, description=need.description)
db.add(db_need)
db.commit()
db.refresh(db_need)
return db_need
@router.get("/needs", response_model=list[NeedResponse])
def list_needs(db: DbSession) -> list[Need]:
"""List all needs."""
return list(db.scalars(select(Need)).all())
@router.get("/needs/{need_id}", response_model=NeedResponse)
def get_need(need_id: int, db: DbSession) -> Need:
"""Get a need by ID."""
need = db.get(Need, need_id)
if not need:
raise HTTPException(status_code=404, detail="Need not found")
return need
@router.delete("/needs/{need_id}", response_model=None)
def delete_need(need_id: int, request: Request, db: DbSession) -> dict[str, bool] | HTMLResponse:
"""Delete a need by ID."""
need = db.get(Need, need_id)
if not need:
raise HTTPException(status_code=404, detail="Need not found")
db.delete(need)
db.commit()
if _is_htmx(request):
return HTMLResponse("")
return {"deleted": True}
@router.post("/contacts", response_model=ContactResponse)
def create_contact(contact: ContactCreate, db: DbSession) -> Contact:
"""Create a new contact."""
need_ids = contact.need_ids
contact_data = contact.model_dump(exclude={"need_ids"})
db_contact = Contact(**contact_data)
if need_ids:
needs = list(db.scalars(select(Need).where(Need.id.in_(need_ids))).all())
db_contact.needs = needs
db.add(db_contact)
db.commit()
db.refresh(db_contact)
return db_contact
@router.get("/contacts", response_model=list[ContactListResponse])
def list_contacts(
db: DbSession,
skip: int = 0,
limit: int = 100,
) -> list[Contact]:
"""List all contacts with pagination."""
return list(db.scalars(select(Contact).offset(skip).limit(limit)).all())
@router.get("/contacts/{contact_id}", response_model=ContactResponse)
def get_contact(contact_id: int, db: DbSession) -> Contact:
"""Get a contact by ID with all relationships."""
contact = db.scalar(
select(Contact)
.where(Contact.id == contact_id)
.options(
selectinload(Contact.needs),
selectinload(Contact.related_to),
selectinload(Contact.related_from),
)
)
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
return contact
@router.patch("/contacts/{contact_id}", response_model=ContactResponse)
def update_contact(
contact_id: int,
contact: ContactUpdate,
db: DbSession,
) -> Contact:
"""Update a contact by ID."""
db_contact = db.get(Contact, contact_id)
if not db_contact:
raise HTTPException(status_code=404, detail="Contact not found")
update_data = contact.model_dump(exclude_unset=True)
need_ids = update_data.pop("need_ids", None)
for key, value in update_data.items():
setattr(db_contact, key, value)
if need_ids is not None:
needs = list(db.scalars(select(Need).where(Need.id.in_(need_ids))).all())
db_contact.needs = needs
db.commit()
db.refresh(db_contact)
return db_contact
@router.delete("/contacts/{contact_id}", response_model=None)
def delete_contact(contact_id: int, request: Request, db: DbSession) -> dict[str, bool] | HTMLResponse:
"""Delete a contact by ID."""
contact = db.get(Contact, contact_id)
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
db.delete(contact)
db.commit()
if _is_htmx(request):
return HTMLResponse("")
return {"deleted": True}
@router.post("/contacts/{contact_id}/needs/{need_id}")
def add_need_to_contact(
contact_id: int,
need_id: int,
db: DbSession,
) -> dict[str, bool]:
"""Add a need to a contact."""
contact = db.get(Contact, contact_id)
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
need = db.get(Need, need_id)
if not need:
raise HTTPException(status_code=404, detail="Need not found")
if need not in contact.needs:
contact.needs.append(need)
db.commit()
return {"added": True}
@router.delete("/contacts/{contact_id}/needs/{need_id}", response_model=None)
def remove_need_from_contact(
contact_id: int,
need_id: int,
request: Request,
db: DbSession,
) -> dict[str, bool] | HTMLResponse:
"""Remove a need from a contact."""
contact = db.get(Contact, contact_id)
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
need = db.get(Need, need_id)
if not need:
raise HTTPException(status_code=404, detail="Need not found")
if need in contact.needs:
contact.needs.remove(need)
db.commit()
if _is_htmx(request):
return HTMLResponse("")
return {"removed": True}
@router.post(
"/contacts/{contact_id}/relationships",
response_model=ContactRelationshipResponse,
)
def add_contact_relationship(
contact_id: int,
relationship: ContactRelationshipCreate,
db: DbSession,
) -> ContactRelationship:
"""Add a relationship between two contacts."""
contact = db.get(Contact, contact_id)
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
related_contact = db.get(Contact, relationship.related_contact_id)
if not related_contact:
raise HTTPException(status_code=404, detail="Related contact not found")
if contact_id == relationship.related_contact_id:
raise HTTPException(status_code=400, detail="Cannot relate contact to itself")
# Use provided weight or default from relationship type
weight = relationship.closeness_weight
if weight is None:
weight = relationship.relationship_type.default_weight
db_relationship = ContactRelationship(
contact_id=contact_id,
related_contact_id=relationship.related_contact_id,
relationship_type=relationship.relationship_type.value,
closeness_weight=weight,
)
db.add(db_relationship)
db.commit()
db.refresh(db_relationship)
return db_relationship
@router.get(
"/contacts/{contact_id}/relationships",
response_model=list[ContactRelationshipResponse],
)
def get_contact_relationships(
contact_id: int,
db: DbSession,
) -> list[ContactRelationship]:
"""Get all relationships for a contact."""
contact = db.get(Contact, contact_id)
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
outgoing = list(db.scalars(select(ContactRelationship).where(ContactRelationship.contact_id == contact_id)).all())
incoming = list(
db.scalars(select(ContactRelationship).where(ContactRelationship.related_contact_id == contact_id)).all()
)
return outgoing + incoming
@router.patch(
"/contacts/{contact_id}/relationships/{related_contact_id}",
response_model=ContactRelationshipResponse,
)
def update_contact_relationship(
contact_id: int,
related_contact_id: int,
update: ContactRelationshipUpdate,
db: DbSession,
) -> ContactRelationship:
"""Update a relationship between two contacts."""
relationship = db.scalar(
select(ContactRelationship).where(
ContactRelationship.contact_id == contact_id,
ContactRelationship.related_contact_id == related_contact_id,
)
)
if not relationship:
raise HTTPException(status_code=404, detail="Relationship not found")
if update.relationship_type is not None:
relationship.relationship_type = update.relationship_type.value
if update.closeness_weight is not None:
relationship.closeness_weight = update.closeness_weight
db.commit()
db.refresh(relationship)
return relationship
@router.delete("/contacts/{contact_id}/relationships/{related_contact_id}", response_model=None)
def remove_contact_relationship(
contact_id: int,
related_contact_id: int,
request: Request,
db: DbSession,
) -> dict[str, bool] | HTMLResponse:
"""Remove a relationship between two contacts."""
relationship = db.scalar(
select(ContactRelationship).where(
ContactRelationship.contact_id == contact_id,
ContactRelationship.related_contact_id == related_contact_id,
)
)
if not relationship:
raise HTTPException(status_code=404, detail="Relationship not found")
db.delete(relationship)
db.commit()
if _is_htmx(request):
return HTMLResponse("")
return {"deleted": True}
@router.get("/relationship-types")
def list_relationship_types() -> list[RelationshipTypeInfo]:
"""List all available relationship types with their default weights."""
return [
RelationshipTypeInfo(
value=rt.value,
display_name=rt.display_name,
default_weight=rt.default_weight,
)
for rt in RelationshipType
]
@router.get("/graph")
def get_relationship_graph(db: DbSession) -> GraphData:
"""Get all contacts and relationships as graph data for visualization."""
contacts = list(db.scalars(select(Contact)).all())
relationships = list(db.scalars(select(ContactRelationship)).all())
nodes = [GraphNode(id=c.id, name=c.name, current_job=c.current_job) for c in contacts]
edges = [
GraphEdge(
source=rel.contact_id,
target=rel.related_contact_id,
relationship_type=rel.relationship_type,
closeness_weight=rel.closeness_weight,
)
for rel in relationships
]
return GraphData(nodes=nodes, edges=edges)
-345
View File
@@ -1,345 +0,0 @@
"""HTMX server-rendered view router."""
from pathlib import Path
from typing import Annotated, Any
from fastapi import APIRouter, Form, HTTPException, Request
from fastapi.responses import HTMLResponse, RedirectResponse
from fastapi.templating import Jinja2Templates
from sqlalchemy import select
from sqlalchemy.orm import Session, selectinload
from python.fastapi_tools.db import DbSession # noqa: TC001 this is a FastAPI needed at runtime
from python.orm.richie.contact import Contact, ContactRelationship, Need, RelationshipType
TEMPLATES_DIR = Path(__file__).parent.parent / "templates"
templates = Jinja2Templates(directory=TEMPLATES_DIR)
router = APIRouter(tags=["views"])
FAMILIAL_TYPES = {
"parent",
"child",
"sibling",
"grandparent",
"grandchild",
"aunt_uncle",
"niece_nephew",
"cousin",
"in_law",
}
FRIEND_TYPES = {"best_friend", "close_friend", "friend", "acquaintance", "neighbor"}
PARTNER_TYPES = {"spouse", "partner"}
PROFESSIONAL_TYPES = {"mentor", "mentee", "business_partner", "colleague", "manager", "direct_report", "client"}
CONTACT_STRING_FIELDS = (
"name",
"legal_name",
"suffix",
"gender",
"current_job",
"timezone",
"profile_pic",
"bio",
"goals",
"social_structure_style",
"safe_conversation_starters",
"topics_to_avoid",
"ssn",
)
CONTACT_INT_FIELDS = ("age", "self_sufficiency_score")
def _group_relationships(relationships: list[ContactRelationship]) -> dict[str, list[ContactRelationship]]:
"""Group relationships by category."""
groups: dict[str, list[ContactRelationship]] = {
"familial": [],
"partners": [],
"friends": [],
"professional": [],
"other": [],
}
for rel in relationships:
if rel.relationship_type in FAMILIAL_TYPES:
groups["familial"].append(rel)
elif rel.relationship_type in PARTNER_TYPES:
groups["partners"].append(rel)
elif rel.relationship_type in FRIEND_TYPES:
groups["friends"].append(rel)
elif rel.relationship_type in PROFESSIONAL_TYPES:
groups["professional"].append(rel)
else:
groups["other"].append(rel)
return groups
def _build_contact_name_map(database: Session, contact: Contact) -> dict[int, str]:
"""Build a mapping of contact IDs to names for relationship display."""
related_ids = {rel.related_contact_id for rel in contact.related_to}
related_ids |= {rel.contact_id for rel in contact.related_from}
related_ids.discard(contact.id)
if not related_ids:
return {}
related_contacts = list(database.scalars(select(Contact).where(Contact.id.in_(related_ids))).all())
return {related.id: related.name for related in related_contacts}
def _get_relationship_type_display() -> dict[str, str]:
"""Build a mapping of relationship type values to display names."""
return {rel_type.value: rel_type.display_name for rel_type in RelationshipType}
async def _parse_contact_form(request: Request) -> dict[str, Any]:
"""Parse contact form data from a multipart/form request."""
form_data = await request.form()
result: dict[str, Any] = {}
for field in CONTACT_STRING_FIELDS:
value = form_data.get(field, "")
result[field] = str(value) if value else None
for field in CONTACT_INT_FIELDS:
value = form_data.get(field, "")
result[field] = int(value) if value else None
result["need_ids"] = [int(value) for value in form_data.getlist("need_ids")]
return result
def _save_contact_from_form(database: Session, contact: Contact, form_result: dict[str, Any]) -> None:
"""Apply parsed form data to a Contact and save associated needs."""
need_ids = form_result.pop("need_ids")
for key, value in form_result.items():
setattr(contact, key, value)
if need_ids:
contact.needs = list(database.scalars(select(Need).where(Need.id.in_(need_ids))).all())
else:
contact.needs = []
@router.get("/", response_class=HTMLResponse)
@router.get("/contacts", response_class=HTMLResponse)
def contact_list_page(request: Request, database: DbSession) -> HTMLResponse:
"""Render the contacts list page."""
contacts = list(database.scalars(select(Contact)).all())
return templates.TemplateResponse(request, "contact_list.html", {"contacts": contacts})
@router.get("/contacts/new", response_class=HTMLResponse)
def new_contact_page(request: Request, database: DbSession) -> HTMLResponse:
"""Render the new contact form page."""
all_needs = list(database.scalars(select(Need)).all())
return templates.TemplateResponse(request, "contact_form.html", {"contact": None, "all_needs": all_needs})
@router.post("/htmx/contacts/new")
async def create_contact_form(request: Request, database: DbSession) -> RedirectResponse:
"""Handle the create contact form submission."""
form_result = await _parse_contact_form(request)
contact = Contact()
_save_contact_from_form(database, contact, form_result)
database.add(contact)
database.commit()
database.refresh(contact)
return RedirectResponse(url=f"/contacts/{contact.id}", status_code=303)
@router.get("/contacts/{contact_id}", response_class=HTMLResponse)
def contact_detail_page(contact_id: int, request: Request, database: DbSession) -> HTMLResponse:
"""Render the contact detail page."""
contact = database.scalar(
select(Contact)
.where(Contact.id == contact_id)
.options(
selectinload(Contact.needs),
selectinload(Contact.related_to),
selectinload(Contact.related_from),
)
)
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
contact_names = _build_contact_name_map(database, contact)
grouped_relationships = _group_relationships(contact.related_to)
all_contacts = list(database.scalars(select(Contact)).all())
all_needs = list(database.scalars(select(Need)).all())
available_needs = [need for need in all_needs if need not in contact.needs]
return templates.TemplateResponse(
request,
"contact_detail.html",
{
"contact": contact,
"contact_names": contact_names,
"grouped_relationships": grouped_relationships,
"all_contacts": all_contacts,
"available_needs": available_needs,
"relationship_types": list(RelationshipType),
},
)
@router.get("/contacts/{contact_id}/edit", response_class=HTMLResponse)
def edit_contact_page(contact_id: int, request: Request, database: DbSession) -> HTMLResponse:
"""Render the edit contact form page."""
contact = database.scalar(select(Contact).where(Contact.id == contact_id).options(selectinload(Contact.needs)))
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
all_needs = list(database.scalars(select(Need)).all())
return templates.TemplateResponse(request, "contact_form.html", {"contact": contact, "all_needs": all_needs})
@router.post("/htmx/contacts/{contact_id}/edit")
async def update_contact_form(contact_id: int, request: Request, database: DbSession) -> RedirectResponse:
"""Handle the edit contact form submission."""
contact = database.get(Contact, contact_id)
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
form_result = await _parse_contact_form(request)
_save_contact_from_form(database, contact, form_result)
database.commit()
return RedirectResponse(url=f"/contacts/{contact_id}", status_code=303)
@router.post("/htmx/contacts/{contact_id}/add-need", response_class=HTMLResponse)
def add_need_to_contact_htmx(
contact_id: int,
request: Request,
database: DbSession,
need_id: Annotated[int, Form()],
) -> HTMLResponse:
"""Add a need to a contact and return updated manage-needs partial."""
contact = database.scalar(select(Contact).where(Contact.id == contact_id).options(selectinload(Contact.needs)))
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
need = database.get(Need, need_id)
if not need:
raise HTTPException(status_code=404, detail="Need not found")
if need not in contact.needs:
contact.needs.append(need)
database.commit()
database.refresh(contact)
return templates.TemplateResponse(request, "partials/manage_needs.html", {"contact": contact})
@router.post("/htmx/contacts/{contact_id}/add-relationship", response_class=HTMLResponse)
def add_relationship_htmx(
contact_id: int,
request: Request,
database: DbSession,
related_contact_id: Annotated[int, Form()],
relationship_type: Annotated[str, Form()],
) -> HTMLResponse:
"""Add a relationship and return updated manage-relationships partial."""
contact = database.scalar(select(Contact).where(Contact.id == contact_id).options(selectinload(Contact.related_to)))
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
related_contact = database.get(Contact, related_contact_id)
if not related_contact:
raise HTTPException(status_code=404, detail="Related contact not found")
rel_type = RelationshipType(relationship_type)
weight = rel_type.default_weight
relationship = ContactRelationship(
contact_id=contact_id,
related_contact_id=related_contact_id,
relationship_type=relationship_type,
closeness_weight=weight,
)
database.add(relationship)
database.commit()
database.refresh(contact)
contact_names = _build_contact_name_map(database, contact)
return templates.TemplateResponse(
request,
"partials/manage_relationships.html",
{"contact": contact, "contact_names": contact_names},
)
@router.post("/htmx/contacts/{contact_id}/relationships/{related_contact_id}/weight")
def update_relationship_weight_htmx(
contact_id: int,
related_contact_id: int,
database: DbSession,
closeness_weight: Annotated[int, Form()],
) -> HTMLResponse:
"""Update a relationship's closeness weight from HTMX range input."""
relationship = database.scalar(
select(ContactRelationship).where(
ContactRelationship.contact_id == contact_id,
ContactRelationship.related_contact_id == related_contact_id,
)
)
if not relationship:
raise HTTPException(status_code=404, detail="Relationship not found")
relationship.closeness_weight = closeness_weight
database.commit()
return HTMLResponse("")
@router.post("/htmx/needs", response_class=HTMLResponse)
def create_need_htmx(
request: Request,
database: DbSession,
name: Annotated[str, Form()],
description: Annotated[str, Form()] = "",
) -> HTMLResponse:
"""Create a need via form data and return updated needs list."""
need = Need(name=name, description=description or None)
database.add(need)
database.commit()
needs = list(database.scalars(select(Need)).all())
return templates.TemplateResponse(request, "partials/need_items.html", {"needs": needs})
@router.get("/needs", response_class=HTMLResponse)
def needs_page(request: Request, database: DbSession) -> HTMLResponse:
"""Render the needs list page."""
needs = list(database.scalars(select(Need)).all())
return templates.TemplateResponse(request, "need_list.html", {"needs": needs})
@router.get("/graph", response_class=HTMLResponse)
def graph_page(request: Request, database: DbSession) -> HTMLResponse:
"""Render the relationship graph page."""
contacts = list(database.scalars(select(Contact)).all())
relationships = list(database.scalars(select(ContactRelationship)).all())
graph_data = {
"nodes": [{"id": contact.id, "name": contact.name, "current_job": contact.current_job} for contact in contacts],
"edges": [
{
"source": rel.contact_id,
"target": rel.related_contact_id,
"relationship_type": rel.relationship_type,
"closeness_weight": rel.closeness_weight,
}
for rel in relationships
],
}
return templates.TemplateResponse(
request,
"graph.html",
{
"graph_data": graph_data,
"relationship_type_display": _get_relationship_type_display(),
},
)
-198
View File
@@ -1,198 +0,0 @@
<!DOCTYPE html>
<html lang="en" data-theme="light">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>{% block title %}Contact Database{% endblock %}</title>
<script src="https://unpkg.com/htmx.org@2.0.4"></script>
<style>
:root {
--color-bg: #f5f5f5;
--color-bg-card: #ffffff;
--color-bg-hover: #f0f0f0;
--color-bg-muted: #f9f9f9;
--color-bg-error: #ffe0e0;
--color-text: #333333;
--color-text-muted: #666666;
--color-text-error: #cc0000;
--color-border: #dddddd;
--color-border-light: #eeeeee;
--color-border-lighter: #f0f0f0;
--color-primary: #0066cc;
--color-primary-hover: #0055aa;
--color-danger: #cc3333;
--color-danger-hover: #aa2222;
--color-tag-bg: #e0e0e0;
--shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
line-height: 1.5;
color: var(--color-text);
background-color: var(--color-bg);
}
[data-theme="dark"] {
--color-bg: #1a1a1a;
--color-bg-card: #2d2d2d;
--color-bg-hover: #3d3d3d;
--color-bg-muted: #252525;
--color-bg-error: #4a2020;
--color-text: #e0e0e0;
--color-text-muted: #a0a0a0;
--color-text-error: #ff6b6b;
--color-border: #404040;
--color-border-light: #353535;
--color-border-lighter: #303030;
--color-primary: #4da6ff;
--color-primary-hover: #7dbfff;
--color-danger: #ff6b6b;
--color-danger-hover: #ff8a8a;
--color-tag-bg: #404040;
--shadow: 0 1px 3px rgba(0, 0, 0, 0.3);
}
* { box-sizing: border-box; }
body { margin: 0; background: var(--color-bg); color: var(--color-text); }
.app { max-width: 1000px; margin: 0 auto; padding: 20px; }
nav { display: flex; align-items: center; gap: 20px; padding: 15px 0; border-bottom: 1px solid var(--color-border); margin-bottom: 20px; }
nav a { color: var(--color-primary); text-decoration: none; font-weight: 500; }
nav a:hover { text-decoration: underline; }
.theme-toggle { margin-left: auto; }
main { background: var(--color-bg-card); padding: 20px; border-radius: 8px; box-shadow: var(--shadow); }
.header { display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px; }
.header h1 { margin: 0; }
a { color: var(--color-primary); }
a:hover { text-decoration: underline; }
.btn { display: inline-block; padding: 8px 16px; border: 1px solid var(--color-border); border-radius: 4px; background: var(--color-bg-card); color: var(--color-text); text-decoration: none; cursor: pointer; font-size: 14px; margin-left: 8px; }
.btn:hover { background: var(--color-bg-hover); }
.btn-primary { background: var(--color-primary); border-color: var(--color-primary); color: white; }
.btn-primary:hover { background: var(--color-primary-hover); }
.btn-danger { background: var(--color-danger); border-color: var(--color-danger); color: white; }
.btn-danger:hover { background: var(--color-danger-hover); }
.btn-small { padding: 4px 8px; font-size: 12px; }
.btn:disabled { opacity: 0.6; cursor: not-allowed; }
table { width: 100%; border-collapse: collapse; }
th, td { padding: 12px; text-align: left; border-bottom: 1px solid var(--color-border-light); }
th { font-weight: 600; background: var(--color-bg-muted); }
tr:hover { background: var(--color-bg-muted); }
.error { background: var(--color-bg-error); color: var(--color-text-error); padding: 10px; border-radius: 4px; margin-bottom: 20px; }
.tag { display: inline-block; background: var(--color-tag-bg); padding: 2px 8px; border-radius: 12px; font-size: 12px; color: var(--color-text-muted); }
.add-form { display: flex; gap: 10px; margin-top: 15px; flex-wrap: wrap; }
.add-form select, .add-form input { padding: 8px; border: 1px solid var(--color-border); border-radius: 4px; min-width: 200px; background: var(--color-bg-card); color: var(--color-text); }
.form-group { margin-bottom: 20px; }
.form-group label { display: block; font-weight: 500; margin-bottom: 5px; }
.form-group input, .form-group textarea, .form-group select { width: 100%; padding: 10px; border: 1px solid var(--color-border); border-radius: 4px; font-size: 14px; background: var(--color-bg-card); color: var(--color-text); }
.form-group textarea { resize: vertical; }
.form-row { display: grid; grid-template-columns: 1fr 1fr; gap: 20px; }
.checkbox-group { display: flex; flex-wrap: wrap; gap: 15px; }
.checkbox-label { display: flex; align-items: center; gap: 5px; cursor: pointer; }
.form-actions { display: flex; gap: 10px; margin-top: 30px; padding-top: 20px; border-top: 1px solid var(--color-border-light); }
.need-form { background: var(--color-bg-muted); padding: 20px; border-radius: 4px; margin-bottom: 20px; }
.need-items { list-style: none; padding: 0; }
.need-items li { display: flex; justify-content: space-between; align-items: flex-start; padding: 15px; border: 1px solid var(--color-border-light); border-radius: 4px; margin-bottom: 10px; }
.need-info p { margin: 5px 0 0; color: var(--color-text-muted); font-size: 14px; }
.graph-container { width: 100%; }
.graph-hint { color: var(--color-text-muted); font-size: 14px; margin-bottom: 15px; }
.selected-info { margin-top: 15px; padding: 15px; background: var(--color-bg-muted); border-radius: 8px; }
.selected-info h3 { margin: 0 0 10px; }
.selected-info p { margin: 5px 0; color: var(--color-text-muted); }
.legend { margin-top: 20px; padding: 15px; background: var(--color-bg-muted); border-radius: 8px; }
.legend h4 { margin: 0 0 10px; font-size: 14px; }
.legend-items { display: flex; flex-wrap: wrap; gap: 15px; }
.legend-item { display: flex; align-items: center; gap: 8px; font-size: 12px; color: var(--color-text-muted); }
.legend-line { width: 30px; border-radius: 2px; }
.id-card { width: 100%; }
.id-card-inner { background: linear-gradient(135deg, #0a0a0f 0%, #1a1a2e 50%, #0a0a0f 100%); background-image: radial-gradient(white 1px, transparent 1px), linear-gradient(135deg, #0a0a0f 0%, #1a1a2e 50%, #0a0a0f 100%); background-size: 50px 50px, 100% 100%; color: #fff; border-radius: 12px; padding: 25px; min-height: 500px; position: relative; overflow: hidden; }
.id-card-header { display: flex; justify-content: space-between; align-items: flex-start; margin-bottom: 15px; }
.id-card-header-left { flex: 1; }
.id-card-header-right { display: flex; flex-direction: column; align-items: flex-end; gap: 10px; }
.id-card-title { font-size: 2.5rem; font-weight: 700; margin: 0; color: #fff; text-shadow: 2px 2px 4px rgba(0,0,0,0.5); }
.id-profile-pic { width: 80px; height: 80px; border-radius: 8px; object-fit: cover; border: 2px solid rgba(255,255,255,0.3); }
.id-profile-placeholder { width: 80px; height: 80px; border-radius: 8px; background: linear-gradient(135deg, #4ecdc4 0%, #44a8a0 100%); display: flex; align-items: center; justify-content: center; border: 2px solid rgba(255,255,255,0.3); }
.id-profile-placeholder span { font-size: 2rem; font-weight: 700; color: #fff; text-shadow: 1px 1px 2px rgba(0,0,0,0.3); }
.id-card-actions { display: flex; gap: 8px; }
.id-card-actions .btn { background: rgba(255,255,255,0.1); border-color: rgba(255,255,255,0.3); color: #fff; }
.id-card-actions .btn:hover { background: rgba(255,255,255,0.2); }
.id-card-body { display: grid; grid-template-columns: 1fr 1.5fr; gap: 30px; }
.id-card-left { display: flex; flex-direction: column; gap: 8px; }
.id-field { font-size: 1rem; line-height: 1.4; }
.id-field-block { margin-top: 15px; font-size: 0.95rem; line-height: 1.5; }
.id-label { color: #4ecdc4; font-weight: 500; }
.id-card-right { display: flex; flex-direction: column; gap: 20px; }
.id-bio { font-size: 0.9rem; line-height: 1.6; color: #e0e0e0; }
.id-relationships { margin-top: 10px; }
.id-section-title { font-size: 1.5rem; margin: 0 0 15px; color: #fff; border-bottom: 1px solid rgba(255,255,255,0.2); padding-bottom: 8px; }
.id-rel-group { margin-bottom: 12px; font-size: 0.9rem; line-height: 1.6; }
.id-rel-label { color: #a0a0a0; }
.id-rel-group a { color: #4ecdc4; text-decoration: none; }
.id-rel-group a:hover { text-decoration: underline; }
.id-rel-type { color: #888; font-size: 0.85em; }
.id-card-warnings { margin-top: 30px; padding-top: 20px; border-top: 1px solid rgba(255,255,255,0.2); display: flex; flex-wrap: wrap; gap: 20px; }
.id-warning { display: flex; align-items: center; gap: 8px; font-size: 0.9rem; color: #ff6b6b; }
.warning-dot { width: 8px; height: 8px; background: #ff6b6b; border-radius: 50%; flex-shrink: 0; }
.warning-desc { color: #ccc; }
.id-card-manage { margin-top: 20px; background: var(--color-bg-muted); border-radius: 8px; padding: 15px; }
.id-card-manage summary { cursor: pointer; font-weight: 600; font-size: 1.1rem; padding: 5px 0; }
.id-card-manage[open] summary { margin-bottom: 15px; border-bottom: 1px solid var(--color-border-light); padding-bottom: 10px; }
.manage-section { margin-bottom: 25px; }
.manage-section h3 { margin: 0 0 15px; font-size: 1rem; }
.manage-relationships { display: flex; flex-direction: column; gap: 10px; margin-bottom: 15px; }
.manage-rel-item { display: flex; align-items: center; gap: 12px; padding: 10px; background: var(--color-bg-card); border-radius: 6px; flex-wrap: wrap; }
.manage-rel-item a { font-weight: 500; min-width: 120px; }
.weight-control { display: flex; align-items: center; gap: 8px; font-size: 12px; color: var(--color-text-muted); }
.weight-control input[type="range"] { width: 80px; cursor: pointer; }
.weight-value { min-width: 20px; text-align: center; font-weight: 600; }
.manage-needs-list { list-style: none; padding: 0; margin: 0 0 15px; }
.manage-needs-list li { display: flex; align-items: center; gap: 12px; padding: 10px; background: var(--color-bg-card); border-radius: 6px; margin-bottom: 8px; }
.manage-needs-list li .btn { margin-left: auto; }
.htmx-indicator { display: none; }
.htmx-request .htmx-indicator { display: inline; }
.htmx-request.htmx-indicator { display: inline; }
@media (max-width: 768px) {
.id-card-body { grid-template-columns: 1fr; }
.id-card-title { font-size: 1.8rem; }
.id-card-header { flex-direction: column; gap: 15px; }
}
</style>
</head>
<body>
<div class="app">
<nav>
<a href="/contacts">Contacts</a>
<a href="/graph">Graph</a>
<a href="/needs">Needs</a>
<button class="btn btn-small theme-toggle" onclick="toggleTheme()">
<span id="theme-label">Dark</span>
</button>
</nav>
<main id="main-content">
{% block content %}{% endblock %}
</main>
</div>
<script>
function toggleTheme() {
const html = document.documentElement;
const current = html.getAttribute('data-theme');
const next = current === 'light' ? 'dark' : 'light';
html.setAttribute('data-theme', next);
localStorage.setItem('theme', next);
document.getElementById('theme-label').textContent = next === 'light' ? 'Dark' : 'Light';
}
(function() {
const saved = localStorage.getItem('theme') || 'light';
document.documentElement.setAttribute('data-theme', saved);
document.getElementById('theme-label').textContent = saved === 'light' ? 'Dark' : 'Light';
})();
</script>
</body>
</html>
-204
View File
@@ -1,204 +0,0 @@
{% extends "base.html" %}
{% block title %}{{ contact.name }}{% endblock %}
{% block content %}
<div class="id-card">
<div class="id-card-inner">
<div class="id-card-header">
<div class="id-card-header-left">
<h1 class="id-card-title">I.D.: {{ contact.name }}</h1>
</div>
<div class="id-card-header-right">
{% if contact.profile_pic %}
<img src="{{ contact.profile_pic }}" alt="{{ contact.name }}'s profile" class="id-profile-pic">
{% else %}
<div class="id-profile-placeholder">
<span>{{ contact.name[0]|upper }}</span>
</div>
{% endif %}
<div class="id-card-actions">
<a href="/contacts/{{ contact.id }}/edit" class="btn btn-small">Edit</a>
<a href="/contacts" class="btn btn-small">Back</a>
</div>
</div>
</div>
<div class="id-card-body">
<div class="id-card-left">
{% if contact.legal_name %}
<div class="id-field">Legal name: {{ contact.legal_name }}</div>
{% endif %}
{% if contact.suffix %}
<div class="id-field">Suffix: {{ contact.suffix }}</div>
{% endif %}
{% if contact.gender %}
<div class="id-field">Gender: {{ contact.gender }}</div>
{% endif %}
{% if contact.age %}
<div class="id-field">Age: {{ contact.age }}</div>
{% endif %}
{% if contact.current_job %}
<div class="id-field">Job: {{ contact.current_job }}</div>
{% endif %}
{% if contact.social_structure_style %}
<div class="id-field">Social style: {{ contact.social_structure_style }}</div>
{% endif %}
{% if contact.self_sufficiency_score is not none %}
<div class="id-field">Self-Sufficiency: {{ contact.self_sufficiency_score }}</div>
{% endif %}
{% if contact.timezone %}
<div class="id-field">Timezone: {{ contact.timezone }}</div>
{% endif %}
{% if contact.safe_conversation_starters %}
<div class="id-field-block">
<span class="id-label">Safe con starters:</span> {{ contact.safe_conversation_starters }}
</div>
{% endif %}
{% if contact.topics_to_avoid %}
<div class="id-field-block">
<span class="id-label">Topics to avoid:</span> {{ contact.topics_to_avoid }}
</div>
{% endif %}
{% if contact.goals %}
<div class="id-field-block">
<span class="id-label">Goals:</span> {{ contact.goals }}
</div>
{% endif %}
</div>
<div class="id-card-right">
{% if contact.bio %}
<div class="id-bio">
<span class="id-label">Bio:</span> {{ contact.bio }}
</div>
{% endif %}
<div class="id-relationships">
<h2 class="id-section-title">Relationships</h2>
{% if grouped_relationships.familial %}
<div class="id-rel-group">
<span class="id-rel-label">Familial:</span>
{% for rel in grouped_relationships.familial %}
<a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a><span class="id-rel-type">({{ rel.relationship_type|replace("_", " ")|title }})</span>{% if not loop.last %}, {% endif %}
{% endfor %}
</div>
{% endif %}
{% if grouped_relationships.partners %}
<div class="id-rel-group">
<span class="id-rel-label">Partners:</span>
{% for rel in grouped_relationships.partners %}
<a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a>{% if not loop.last %}, {% endif %}
{% endfor %}
</div>
{% endif %}
{% if grouped_relationships.friends %}
<div class="id-rel-group">
<span class="id-rel-label">Friends:</span>
{% for rel in grouped_relationships.friends %}
<a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a>{% if not loop.last %}, {% endif %}
{% endfor %}
</div>
{% endif %}
{% if grouped_relationships.professional %}
<div class="id-rel-group">
<span class="id-rel-label">Professional:</span>
{% for rel in grouped_relationships.professional %}
<a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a><span class="id-rel-type">({{ rel.relationship_type|replace("_", " ")|title }})</span>{% if not loop.last %}, {% endif %}
{% endfor %}
</div>
{% endif %}
{% if grouped_relationships.other %}
<div class="id-rel-group">
<span class="id-rel-label">Other:</span>
{% for rel in grouped_relationships.other %}
<a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a><span class="id-rel-type">({{ rel.relationship_type|replace("_", " ")|title }})</span>{% if not loop.last %}, {% endif %}
{% endfor %}
</div>
{% endif %}
{% if contact.related_from %}
<div class="id-rel-group">
<span class="id-rel-label">Known by:</span>
{% for rel in contact.related_from %}
<a href="/contacts/{{ rel.contact_id }}">{{ contact_names[rel.contact_id] }}</a>{% if not loop.last %}, {% endif %}
{% endfor %}
</div>
{% endif %}
</div>
</div>
</div>
{% if contact.needs %}
<div class="id-card-warnings">
{% for need in contact.needs %}
<div class="id-warning">
<span class="warning-dot"></span>
Warning: {{ need.name }}
{% if need.description %}<span class="warning-desc"> - {{ need.description }}</span>{% endif %}
</div>
{% endfor %}
</div>
{% endif %}
</div>
<details class="id-card-manage">
<summary>Manage Contact</summary>
<div class="manage-section">
<h3>Manage Relationships</h3>
<div id="manage-relationships" class="manage-relationships">
{% include "partials/manage_relationships.html" %}
</div>
{% if all_contacts %}
<form hx-post="/htmx/contacts/{{ contact.id }}/add-relationship"
hx-target="#manage-relationships"
hx-swap="innerHTML"
class="add-form">
<select name="related_contact_id" required>
<option value="">Select contact...</option>
{% for other in all_contacts %}
{% if other.id != contact.id %}
<option value="{{ other.id }}">{{ other.name }}</option>
{% endif %}
{% endfor %}
</select>
<select name="relationship_type" required>
<option value="">Select relationship type...</option>
{% for rel_type in relationship_types %}
<option value="{{ rel_type.value }}">{{ rel_type.display_name }}</option>
{% endfor %}
</select>
<button type="submit" class="btn btn-primary">Add Relationship</button>
</form>
{% endif %}
</div>
<div class="manage-section">
<h3>Manage Needs/Warnings</h3>
<div id="manage-needs">
{% include "partials/manage_needs.html" %}
</div>
{% if available_needs %}
<form hx-post="/htmx/contacts/{{ contact.id }}/add-need"
hx-target="#manage-needs"
hx-swap="innerHTML"
class="add-form">
<select name="need_id" required>
<option value="">Select a need...</option>
{% for need in available_needs %}
<option value="{{ need.id }}">{{ need.name }}</option>
{% endfor %}
</select>
<button type="submit" class="btn btn-primary">Add Need</button>
</form>
{% endif %}
</div>
</details>
</div>
{% endblock %}
-115
View File
@@ -1,115 +0,0 @@
{% extends "base.html" %}
{% block title %}{{ "Edit " + contact.name if contact else "New Contact" }}{% endblock %}
{% block content %}
<div class="contact-form">
<h1>{{ "Edit Contact" if contact else "New Contact" }}</h1>
{% if contact %}
<form method="post" action="/htmx/contacts/{{ contact.id }}/edit">
{% else %}
<form method="post" action="/htmx/contacts/new">
{% endif %}
<div class="form-group">
<label for="name">Name *</label>
<input id="name" name="name" type="text" value="{{ contact.name if contact else '' }}" required>
</div>
<div class="form-row">
<div class="form-group">
<label for="legal_name">Legal Name</label>
<input id="legal_name" name="legal_name" type="text" value="{{ contact.legal_name or '' }}">
</div>
<div class="form-group">
<label for="suffix">Suffix</label>
<input id="suffix" name="suffix" type="text" value="{{ contact.suffix or '' }}">
</div>
</div>
<div class="form-row">
<div class="form-group">
<label for="age">Age</label>
<input id="age" name="age" type="number" value="{{ contact.age if contact and contact.age is not none else '' }}">
</div>
<div class="form-group">
<label for="gender">Gender</label>
<input id="gender" name="gender" type="text" value="{{ contact.gender or '' }}">
</div>
</div>
<div class="form-group">
<label for="current_job">Current Job</label>
<input id="current_job" name="current_job" type="text" value="{{ contact.current_job or '' }}">
</div>
<div class="form-group">
<label for="timezone">Timezone</label>
<input id="timezone" name="timezone" type="text" value="{{ contact.timezone or '' }}">
</div>
<div class="form-group">
<label for="profile_pic">Profile Picture URL</label>
<input id="profile_pic" name="profile_pic" type="url" placeholder="https://example.com/photo.jpg" value="{{ contact.profile_pic or '' }}">
</div>
<div class="form-group">
<label for="bio">Bio</label>
<textarea id="bio" name="bio" rows="3">{{ contact.bio or '' }}</textarea>
</div>
<div class="form-group">
<label for="goals">Goals</label>
<textarea id="goals" name="goals" rows="3">{{ contact.goals or '' }}</textarea>
</div>
<div class="form-group">
<label for="social_structure_style">Social Structure Style</label>
<input id="social_structure_style" name="social_structure_style" type="text" value="{{ contact.social_structure_style or '' }}">
</div>
<div class="form-group">
<label for="self_sufficiency_score">Self-Sufficiency Score (1-10)</label>
<input id="self_sufficiency_score" name="self_sufficiency_score" type="number" min="1" max="10" value="{{ contact.self_sufficiency_score if contact and contact.self_sufficiency_score is not none else '' }}">
</div>
<div class="form-group">
<label for="safe_conversation_starters">Safe Conversation Starters</label>
<textarea id="safe_conversation_starters" name="safe_conversation_starters" rows="2">{{ contact.safe_conversation_starters or '' }}</textarea>
</div>
<div class="form-group">
<label for="topics_to_avoid">Topics to Avoid</label>
<textarea id="topics_to_avoid" name="topics_to_avoid" rows="2">{{ contact.topics_to_avoid or '' }}</textarea>
</div>
<div class="form-group">
<label for="ssn">SSN</label>
<input id="ssn" name="ssn" type="text" value="{{ contact.ssn or '' }}">
</div>
{% if all_needs %}
<div class="form-group">
<label>Needs/Accommodations</label>
<div class="checkbox-group">
{% for need in all_needs %}
<label class="checkbox-label">
<input type="checkbox" name="need_ids" value="{{ need.id }}"
{% if contact and need in contact.needs %}checked{% endif %}>
{{ need.name }}
</label>
{% endfor %}
</div>
</div>
{% endif %}
<div class="form-actions">
<button type="submit" class="btn btn-primary">Save</button>
{% if contact %}
<a href="/contacts/{{ contact.id }}" class="btn">Cancel</a>
{% else %}
<a href="/contacts" class="btn">Cancel</a>
{% endif %}
</div>
</form>
</div>
{% endblock %}
-14
View File
@@ -1,14 +0,0 @@
{% extends "base.html" %}
{% block title %}Contacts{% endblock %}
{% block content %}
<div class="contact-list">
<div class="header">
<h1>Contacts</h1>
<a href="/contacts/new" class="btn btn-primary">Add Contact</a>
</div>
<div id="contact-table">
{% include "partials/contact_table.html" %}
</div>
</div>
{% endblock %}
-198
View File
@@ -1,198 +0,0 @@
{% extends "base.html" %}
{% block title %}Relationship Graph{% endblock %}
{% block content %}
<div class="graph-container">
<div class="header">
<h1>Relationship Graph</h1>
</div>
<p class="graph-hint">Drag nodes to reposition. Closer relationships have shorter, darker edges.</p>
<canvas id="graph-canvas" width="900" height="600"
style="border: 1px solid var(--color-border); border-radius: 8px; background: var(--color-bg); cursor: grab;">
</canvas>
<div id="selected-info"></div>
<div class="legend">
<h4>Relationship Closeness (1-10)</h4>
<div class="legend-items">
<div class="legend-item">
<span class="legend-line" style="background: hsl(220, 70%, 40%); height: 4px; display: inline-block;"></span>
<span>10 - Very Close (Spouse, Partner)</span>
</div>
<div class="legend-item">
<span class="legend-line" style="background: hsl(220, 70%, 52%); height: 3px; display: inline-block;"></span>
<span>7 - Close (Family, Best Friend)</span>
</div>
<div class="legend-item">
<span class="legend-line" style="background: hsl(220, 70%, 64%); height: 2px; display: inline-block;"></span>
<span>4 - Moderate (Friend, Colleague)</span>
</div>
<div class="legend-item">
<span class="legend-line" style="background: hsl(220, 70%, 72%); height: 1px; display: inline-block;"></span>
<span>2 - Distant (Acquaintance)</span>
</div>
</div>
</div>
</div>
<script>
(function() {
const RELATIONSHIP_DISPLAY = {{ relationship_type_display|tojson }};
const graphData = {{ graph_data|tojson }};
const canvas = document.getElementById('graph-canvas');
const ctx = canvas.getContext('2d');
const width = canvas.width;
const height = canvas.height;
const centerX = width / 2;
const centerY = height / 2;
const nodes = graphData.nodes.map(function(node) {
return Object.assign({}, node, {
x: centerX + (Math.random() - 0.5) * 300,
y: centerY + (Math.random() - 0.5) * 300,
vx: 0,
vy: 0
});
});
const nodeMap = new Map(nodes.map(function(node) { return [node.id, node]; }));
const edges = graphData.edges.map(function(edge) {
const sourceNode = nodeMap.get(edge.source);
const targetNode = nodeMap.get(edge.target);
if (!sourceNode || !targetNode) return null;
return Object.assign({}, edge, { sourceNode: sourceNode, targetNode: targetNode });
}).filter(function(edge) { return edge !== null; });
let dragNode = null;
let selectedNode = null;
const repulsion = 5000;
const springStrength = 0.05;
const baseSpringLength = 150;
const damping = 0.9;
const centerPull = 0.01;
function simulate() {
for (const node of nodes) { node.vx = 0; node.vy = 0; }
for (let i = 0; i < nodes.length; i++) {
for (let j = i + 1; j < nodes.length; j++) {
const dx = nodes[j].x - nodes[i].x;
const dy = nodes[j].y - nodes[i].y;
const dist = Math.sqrt(dx * dx + dy * dy) || 1;
const force = repulsion / (dist * dist);
const fx = (dx / dist) * force;
const fy = (dy / dist) * force;
nodes[i].vx -= fx; nodes[i].vy -= fy;
nodes[j].vx += fx; nodes[j].vy += fy;
}
}
for (const edge of edges) {
const dx = edge.targetNode.x - edge.sourceNode.x;
const dy = edge.targetNode.y - edge.sourceNode.y;
const dist = Math.sqrt(dx * dx + dy * dy) || 1;
const normalizedWeight = edge.closeness_weight / 10;
const idealLength = baseSpringLength * (1.5 - normalizedWeight);
const displacement = dist - idealLength;
const force = springStrength * displacement;
const fx = (dx / dist) * force;
const fy = (dy / dist) * force;
edge.sourceNode.vx += fx; edge.sourceNode.vy += fy;
edge.targetNode.vx -= fx; edge.targetNode.vy -= fy;
}
for (const node of nodes) {
node.vx += (centerX - node.x) * centerPull;
node.vy += (centerY - node.y) * centerPull;
}
for (const node of nodes) {
if (node === dragNode) continue;
node.x += node.vx * damping;
node.y += node.vy * damping;
node.x = Math.max(30, Math.min(width - 30, node.x));
node.y = Math.max(30, Math.min(height - 30, node.y));
}
}
function getEdgeColor(weight) {
const normalized = weight / 10;
return 'hsl(220, 70%, ' + (80 - normalized * 40) + '%)';
}
function draw() {
ctx.clearRect(0, 0, width, height);
for (const edge of edges) {
const lineWidth = 1 + (edge.closeness_weight / 10) * 3;
ctx.strokeStyle = getEdgeColor(edge.closeness_weight);
ctx.lineWidth = lineWidth;
ctx.beginPath();
ctx.moveTo(edge.sourceNode.x, edge.sourceNode.y);
ctx.lineTo(edge.targetNode.x, edge.targetNode.y);
ctx.stroke();
const midX = (edge.sourceNode.x + edge.targetNode.x) / 2;
const midY = (edge.sourceNode.y + edge.targetNode.y) / 2;
ctx.fillStyle = '#666';
ctx.font = '10px sans-serif';
ctx.textAlign = 'center';
const label = RELATIONSHIP_DISPLAY[edge.relationship_type] || edge.relationship_type;
ctx.fillText(label, midX, midY - 5);
}
for (const node of nodes) {
const isSelected = node === selectedNode;
const radius = isSelected ? 25 : 20;
ctx.beginPath();
ctx.arc(node.x, node.y, radius, 0, Math.PI * 2);
ctx.fillStyle = isSelected ? '#0066cc' : '#fff';
ctx.fill();
ctx.strokeStyle = '#0066cc';
ctx.lineWidth = 2;
ctx.stroke();
ctx.fillStyle = isSelected ? '#fff' : '#333';
ctx.font = '12px sans-serif';
ctx.textAlign = 'center';
ctx.textBaseline = 'middle';
const name = node.name.length > 10 ? node.name.slice(0, 9) + '\u2026' : node.name;
ctx.fillText(name, node.x, node.y);
}
}
function animate() {
simulate();
draw();
requestAnimationFrame(animate);
}
animate();
function getNodeAt(x, y) {
for (const node of nodes) {
const dx = x - node.x;
const dy = y - node.y;
if (dx * dx + dy * dy < 400) return node;
}
return null;
}
canvas.addEventListener('mousedown', function(event) {
const rect = canvas.getBoundingClientRect();
const node = getNodeAt(event.clientX - rect.left, event.clientY - rect.top);
if (node) {
dragNode = node;
selectedNode = node;
const infoDiv = document.getElementById('selected-info');
let html = '<div class="selected-info"><h3>' + node.name + '</h3>';
if (node.current_job) html += '<p>Job: ' + node.current_job + '</p>';
html += '<a href="/contacts/' + node.id + '">View details</a></div>';
infoDiv.innerHTML = html;
}
});
canvas.addEventListener('mousemove', function(event) {
if (!dragNode) return;
const rect = canvas.getBoundingClientRect();
dragNode.x = event.clientX - rect.left;
dragNode.y = event.clientY - rect.top;
});
canvas.addEventListener('mouseup', function() { dragNode = null; });
canvas.addEventListener('mouseleave', function() { dragNode = null; });
})();
</script>
{% endblock %}
-31
View File
@@ -1,31 +0,0 @@
{% extends "base.html" %}
{% block title %}Needs{% endblock %}
{% block content %}
<div class="need-list">
<div class="header">
<h1>Needs / Accommodations</h1>
<button class="btn btn-primary" onclick="document.getElementById('need-form').toggleAttribute('hidden')">Add Need</button>
</div>
<form id="need-form" hidden
hx-post="/htmx/needs"
hx-target="#need-items"
hx-swap="innerHTML"
hx-on::after-request="if(event.detail.successful) this.reset()"
class="need-form">
<div class="form-group">
<label for="name">Name *</label>
<input id="name" name="name" type="text" placeholder="e.g., Light Sensitive, ADHD" required>
</div>
<div class="form-group">
<label for="description">Description</label>
<textarea id="description" name="description" placeholder="Optional description..." rows="2"></textarea>
</div>
<button type="submit" class="btn btn-primary">Create</button>
</form>
<div id="need-items">
{% include "partials/need_items.html" %}
</div>
</div>
{% endblock %}
@@ -1,33 +0,0 @@
{% if contacts %}
<table>
<thead>
<tr>
<th>Name</th>
<th>Job</th>
<th>Timezone</th>
<th>Actions</th>
</tr>
</thead>
<tbody>
{% for contact in contacts %}
<tr id="contact-row-{{ contact.id }}">
<td><a href="/contacts/{{ contact.id }}">{{ contact.name }}</a></td>
<td>{{ contact.current_job or "-" }}</td>
<td>{{ contact.timezone or "-" }}</td>
<td>
<a href="/contacts/{{ contact.id }}/edit" class="btn">Edit</a>
<button class="btn btn-danger"
hx-delete="/api/contacts/{{ contact.id }}"
hx-target="#contact-row-{{ contact.id }}"
hx-swap="outerHTML"
hx-confirm="Delete this contact?">
Delete
</button>
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p>No contacts yet.</p>
{% endif %}
@@ -1,14 +0,0 @@
<ul class="manage-needs-list">
{% for need in contact.needs %}
<li id="contact-need-{{ need.id }}">
<strong>{{ need.name }}</strong>
{% if need.description %}<span> - {{ need.description }}</span>{% endif %}
<button class="btn btn-small btn-danger"
hx-delete="/api/contacts/{{ contact.id }}/needs/{{ need.id }}"
hx-target="#contact-need-{{ need.id }}"
hx-swap="outerHTML">
Remove
</button>
</li>
{% endfor %}
</ul>
@@ -1,23 +0,0 @@
{% for rel in contact.related_to %}
<div class="manage-rel-item" id="rel-{{ contact.id }}-{{ rel.related_contact_id }}">
<a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a>
<span class="tag">{{ rel.relationship_type|replace("_", " ")|title }}</span>
<label class="weight-control">
<span>Closeness:</span>
<input type="range" min="1" max="10" value="{{ rel.closeness_weight }}"
hx-post="/htmx/contacts/{{ contact.id }}/relationships/{{ rel.related_contact_id }}/weight"
hx-trigger="change"
hx-include="this"
name="closeness_weight"
hx-swap="none"
oninput="this.nextElementSibling.textContent = this.value">
<span class="weight-value">{{ rel.closeness_weight }}</span>
</label>
<button class="btn btn-small btn-danger"
hx-delete="/api/contacts/{{ contact.id }}/relationships/{{ rel.related_contact_id }}"
hx-target="#rel-{{ contact.id }}-{{ rel.related_contact_id }}"
hx-swap="outerHTML">
Remove
</button>
</div>
{% endfor %}
@@ -1,21 +0,0 @@
{% if needs %}
<ul class="need-items">
{% for need in needs %}
<li id="need-item-{{ need.id }}">
<div class="need-info">
<strong>{{ need.name }}</strong>
{% if need.description %}<p>{{ need.description }}</p>{% endif %}
</div>
<button class="btn btn-danger"
hx-delete="/api/needs/{{ need.id }}"
hx-target="#need-item-{{ need.id }}"
hx-swap="outerHTML"
hx-confirm="Delete this need?">
Delete
</button>
</li>
{% endfor %}
</ul>
{% else %}
<p>No needs defined yet.</p>
{% endif %}
-47
View File
@@ -1,47 +0,0 @@
"""common."""
from __future__ import annotations
import logging
from datetime import UTC, datetime
from pathlib import Path
from subprocess import PIPE, Popen
from python.logging_config import configure_logger as _configure_logger
logger = logging.getLogger(__name__)
def get_repo_dir() -> Path:
"""Return the repository root directory."""
return Path(__file__).resolve().parents[1]
def configure_logger(level: str = "INFO") -> None:
"""Configure the logger."""
_configure_logger(level)
def bash_wrapper(command: str) -> tuple[str, int]:
"""Execute a bash command and capture the output.
Args:
command (str): The bash command to be executed.
Returns:
Tuple[str, int]: A tuple containing the output of the command (stdout) as a string,
the error output (stderr) as a string (optional), and the return code as an integer.
"""
# This is a acceptable risk
process = Popen(command.split(), stdout=PIPE, stderr=PIPE)
output, error = process.communicate()
if error:
logger.error(f"{error=}")
return error.decode(), process.returncode
return output.decode(), process.returncode
def utcnow() -> datetime:
"""Get the current UTC time."""
return datetime.now(tz=UTC)
-59
View File
@@ -1,59 +0,0 @@
"""database."""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
from sqlalchemy import inspect
from sqlalchemy.exc import NoInspectionAvailable
if TYPE_CHECKING:
from collections.abc import Sequence
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
def safe_insert(orm_objects: Sequence[object], session: Session) -> list[tuple[Exception, object]]:
"""Safer insert at allows for partial rollbacks.
Args:
orm_objects (Sequence[object]): Tables to insert.
session (Session): Database session.
"""
if unmapped := [orm_object for orm_object in orm_objects if not _is_mapped_instance(orm_object)]:
error = f"safe_insert expects ORM-mapped instances {unmapped}"
raise TypeError(error)
return _safe_insert(orm_objects, session)
def _safe_insert(objects: Sequence[object], session: Session) -> list[tuple[Exception, object]]:
exceptions: list[tuple[Exception, object]] = []
try:
session.add_all(objects)
session.commit()
except Exception as error:
session.rollback()
objects_len = len(objects)
if objects_len == 1:
logger.exception(objects)
return [(error, objects[0])]
middle = objects_len // 2
exceptions.extend(_safe_insert(objects=objects[:middle], session=session))
exceptions.extend(_safe_insert(objects=objects[middle:], session=session))
return exceptions
def _is_mapped_instance(obj: object) -> bool:
"""Return True if `obj` is a SQLAlchemy ORM-mapped instance."""
try:
inspect(obj) # raises NoInspectionAvailable if not mapped
except NoInspectionAvailable:
return False
else:
return True
-103
View File
@@ -1,103 +0,0 @@
"""CLI wrapper around alembic for multi-database support.
Usage:
database <db_name> <command> [args...]
Examples:
database richie check
database richie upgrade head
database richie downgrade head-1
database richie revision --autogenerate -m "add meals table"
"""
from __future__ import annotations
from dataclasses import dataclass
from importlib import import_module
from typing import TYPE_CHECKING, Annotated
import typer
from alembic.config import CommandLine, Config
if TYPE_CHECKING:
from sqlalchemy.orm import DeclarativeBase
@dataclass(frozen=True)
class DatabaseConfig:
"""Configuration for a database."""
env_prefix: str
version_location: str
base_module: str
base_class_name: str
models_module: str
script_location: str = "python/alembic"
file_template: str = "%%(year)d_%%(month).2d_%%(day).2d-%%(slug)s_%%(rev)s"
def get_base(self) -> type[DeclarativeBase]:
"""Import and return the Base class."""
module = import_module(self.base_module)
return getattr(module, self.base_class_name)
def import_models(self) -> None:
"""Import ORM models so alembic autogenerate can detect them."""
import_module(self.models_module)
def alembic_config(self) -> Config:
"""Build an alembic Config for this database."""
cfg = Config()
cfg.set_main_option("script_location", self.script_location)
cfg.set_main_option("file_template", self.file_template)
cfg.set_main_option("prepend_sys_path", ".")
cfg.set_main_option("version_path_separator", "os")
cfg.set_main_option("version_locations", self.version_location)
cfg.set_main_option("revision_environment", "true")
cfg.set_section_option("post_write_hooks", "hooks", "dynamic_schema,import_postgresql,ruff")
cfg.set_section_option("post_write_hooks", "dynamic_schema.type", "dynamic_schema")
cfg.set_section_option("post_write_hooks", "import_postgresql.type", "import_postgresql")
cfg.set_section_option("post_write_hooks", "ruff.type", "ruff")
cfg.attributes["base"] = self.get_base()
cfg.attributes["env_prefix"] = self.env_prefix
self.import_models()
return cfg
DATABASES: dict[str, DatabaseConfig] = {
"richie": DatabaseConfig(
env_prefix="RICHIE",
version_location="python/alembic/richie/versions",
base_module="python.orm.richie.base",
base_class_name="RichieBase",
models_module="python.orm.richie",
),
}
app = typer.Typer(help="Multi-database alembic wrapper.")
@app.command(
context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
)
def main(
ctx: typer.Context,
db_name: Annotated[str, typer.Argument(help=f"Database name. Options: {', '.join(DATABASES)}")],
command: Annotated[str, typer.Argument(help="Alembic command (upgrade, downgrade, revision, check, etc.)")],
) -> None:
"""Run an alembic command against the specified database."""
db_config = DATABASES.get(db_name)
if not db_config:
typer.echo(f"Unknown database: {db_name!r}. Available: {', '.join(DATABASES)}", err=True)
raise typer.Exit(code=1)
alembic_cfg = db_config.alembic_config()
cmd_line = CommandLine()
options = cmd_line.parser.parse_args([command, *ctx.args])
cmd_line.run_cmd(alembic_cfg, options)
if __name__ == "__main__":
app()
-1
View File
@@ -1 +0,0 @@
"""EPUB search package."""
-65
View File
@@ -1,65 +0,0 @@
"""Grounded answer generation."""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
from python.ebook_search.llm_interface import request_chat_completion
if TYPE_CHECKING:
import httpx
from python.ebook_search.config import EbookSearchConfig
from python.ebook_search.search import SearchResult
logger = logging.getLogger(__name__)
async def answer_query(
client: httpx.AsyncClient,
query: str,
results: list[SearchResult],
config: EbookSearchConfig,
) -> str:
"""Answer a question using only retrieved chunks."""
if not config.answer_enabled:
logger.info("ebook_answer_skipped_disabled")
return "Answer generation is disabled. Source chunks are shown below."
if not results:
logger.info("ebook_answer_skipped_no_results")
return "No relevant sources were found."
logger.info(
"ebook_answer_request_start base_url=%s model=%s sources=%s query_length=%s",
config.vllm_base_url,
config.chat_model,
len(results),
len(query),
)
context = "\n\n".join(
f"[{index}] {result.source_title}{' - ' + result.chapter_title if result.chapter_title else ''}\n{result.text}"
for index, result in enumerate(results, start=1)
)
content = await request_chat_completion(
client,
config,
[
{
"role": "system",
"content": (
"Answer only from the provided context. Cite sources with bracketed numbers like [1]. "
"If the context is insufficient, say so."
),
},
{"role": "user", "content": f"Question:\n{query}\n\nContext:\n{context}"},
],
)
logger.info(
"ebook_answer_request_complete model=%s answer_length=%s",
config.chat_model,
len(content),
)
return content or "The model returned an empty answer."
-1
View File
@@ -1 +0,0 @@
"""Web and external API adapters for EPUB search."""
-73
View File
@@ -1,73 +0,0 @@
"""Background BM25 refresh tasks for the web app.
The refresh is scheduled on the event loop instead of a thread because the async psycopg
driver only works from the loop; a bare thread cannot open a session on the async engine.
"""
from __future__ import annotations
import asyncio
import logging
from typing import TYPE_CHECKING
from sqlalchemy.ext.asyncio import AsyncSession
from python.ebook_search.bm25_corpus import load_bm25_corpus, refresh_bm25_corpus
if TYPE_CHECKING:
from fastapi import FastAPI
from sqlalchemy.ext.asyncio import AsyncEngine
from python.ebook_search.config import EbookSearchConfig
logger = logging.getLogger(__name__)
def schedule_bm25_refresh(app: FastAPI) -> None:
"""Schedule a delayed BM25 corpus refresh, replacing any pending refresh.
Only called from route handlers, so a running event loop is guaranteed.
"""
cancel_bm25_refresh(app)
loop = asyncio.get_running_loop()
def start_refresh() -> None:
app.state.bm25_refresh_task = loop.create_task(refresh_bm25_for_app(app))
app.state.bm25_refresh_timer = loop.call_later(app.state.config.bm25_refresh_delay_seconds, start_refresh)
logger.info(
"ebook_bm25_refresh_scheduled delay_seconds=%s",
app.state.config.bm25_refresh_delay_seconds,
)
def cancel_bm25_refresh(app: FastAPI) -> None:
"""Cancel any pending BM25 corpus refresh timer and in-flight refresh task."""
existing_timer = getattr(app.state, "bm25_refresh_timer", None)
if existing_timer is not None:
existing_timer.cancel()
app.state.bm25_refresh_timer = None
logger.info("ebook_bm25_refresh_cancelled")
existing_task = getattr(app.state, "bm25_refresh_task", None)
if existing_task is not None:
if not existing_task.done():
existing_task.cancel()
app.state.bm25_refresh_task = None
async def refresh_bm25_for_app(app: FastAPI) -> None:
"""Refresh the BM25 corpus using the app engine and config."""
try:
await refresh_bm25_for_engine(app.state.engine, app.state.config)
except Exception:
logger.exception("ebook_bm25_refresh_failed")
async def refresh_bm25_for_engine(engine: AsyncEngine, config: EbookSearchConfig) -> None:
"""Refresh the BM25 corpus using an async SQLAlchemy engine."""
async with AsyncSession(engine) as session:
await refresh_bm25_corpus(session, config)
load_bm25_corpus.cache_clear()
logger.info("ebook_bm25_corpus_cache_cleared_after_refresh")
-31
View File
@@ -1,31 +0,0 @@
"""FastAPI dependencies for the EPUB search app."""
from __future__ import annotations
from typing import Annotated
import httpx
from fastapi import Depends, Request
from sqlalchemy.ext.asyncio import AsyncEngine
from python.ebook_search.config import EbookSearchConfig
def get_config(request: Request) -> EbookSearchConfig:
"""Get the loaded search config from app state."""
return request.app.state.config
def get_engine(request: Request) -> AsyncEngine:
"""Get the database engine from app state."""
return request.app.state.engine
def get_http_client(request: Request) -> httpx.AsyncClient:
"""Get the shared LLM HTTP client from app state."""
return request.app.state.http_client
AppConfig = Annotated[EbookSearchConfig, Depends(get_config)]
AppEngine = Annotated[AsyncEngine, Depends(get_engine)]
AppHttpClient = Annotated[httpx.AsyncClient, Depends(get_http_client)]
-131
View File
@@ -1,131 +0,0 @@
"""Background phrase-judging tasks for the web app.
Judging a book sends one LLM request per candidate phrase, which can take minutes, so it must
not run inside the request where it would block the UI. Judgments run as async FastAPI
background tasks, awaited on the event loop after the response is sent, and are tracked per
book in app state so a second judge request for a book that is already being judged is
rejected instead of doubling the work.
State is loop-confined: every read and mutation happens on the event loop (async route
handlers and async background tasks) and no critical section contains an ``await``, so each
mutation is atomic per loop iteration and no locking is needed.
"""
from __future__ import annotations
import logging
from dataclasses import dataclass, field
from typing import TYPE_CHECKING
from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
if TYPE_CHECKING:
from fastapi import BackgroundTasks, FastAPI
logger = logging.getLogger(__name__)
@dataclass
class JudgeTaskState:
"""Running book judgments and last outcome messages, keyed by book id."""
running_book_ids: set[int] = field(default_factory=set)
outcome_messages: dict[int, str] = field(default_factory=dict)
def get_judge_task_state(app: FastAPI) -> JudgeTaskState:
"""Return the app's judge task state, creating it on first use.
Args:
app (FastAPI): App whose state holds the judge task registry.
Returns:
JudgeTaskState: The shared judge task state for this app.
"""
state = getattr(app.state, "judge_tasks", None)
if state is None:
state = JudgeTaskState()
app.state.judge_tasks = state
return state
def start_book_phrase_judgment(app: FastAPI, background_tasks: BackgroundTasks, source_id: int) -> bool:
"""Queue judging of one book's candidate phrases as a FastAPI background task.
The book is claimed before the response returns, so a repeated judge request cannot queue
a second run while one is pending or running.
Args:
app (FastAPI): App supplying the engine, config, and judge task state.
background_tasks (BackgroundTasks): Request's background tasks to queue the judgment on.
source_id (int): Book to judge candidates for.
Returns:
bool: True when a judgment was queued, False when one is already running for this book.
"""
state = get_judge_task_state(app)
if source_id in state.running_book_ids:
logger.info("ebook_book_phrase_judgment_already_running source_id=%s", source_id)
return False
state.running_book_ids.add(source_id)
state.outcome_messages.pop(source_id, None)
background_tasks.add_task(judge_book_phrases_for_app, app, source_id)
logger.info("ebook_book_phrase_judgment_queued source_id=%s", source_id)
return True
async def judge_book_phrases_for_app(app: FastAPI, source_id: int) -> None:
"""Judge one book using the app engine and config, recording the outcome message.
Args:
app (FastAPI): App supplying the engine, config, and judge task state.
source_id (int): Book to judge candidates for.
"""
state = get_judge_task_state(app)
try:
result = await judge_candidate_phrases_for_books(app.state.engine, app.state.config, source_ids=[source_id])
logger.info(
"ebook_book_phrase_judgment_complete source_id=%s judged=%s protected=%s mentions=%s failed=%s",
source_id,
result.candidates_judged,
result.protected_phrases,
result.phrase_mentions,
result.books_failed,
)
if result.books_failed:
message = "Judging failed; see server logs for details"
else:
message = (
f"Judged {result.candidates_judged} candidates; {result.protected_phrases} protected phrases promoted"
)
except Exception:
logger.exception("ebook_book_phrase_judgment_task_failed source_id=%s", source_id)
message = "Judging failed; see server logs for details"
state.running_book_ids.discard(source_id)
state.outcome_messages[source_id] = message
def is_judging_book(app: FastAPI, source_id: int) -> bool:
"""Report whether a judgment is currently queued or running for one book.
Args:
app (FastAPI): App supplying the judge task state.
source_id (int): Book to check.
Returns:
bool: True while the book's judgment is pending or running.
"""
return source_id in get_judge_task_state(app).running_book_ids
def pop_book_judgment_outcome(app: FastAPI, source_id: int) -> str | None:
"""Return and clear the outcome message from one book's last finished judgment.
Args:
app (FastAPI): App supplying the judge task state.
source_id (int): Book to fetch the outcome for.
Returns:
str | None: The outcome message, or None when there is nothing new to report.
"""
return get_judge_task_state(app).outcome_messages.pop(source_id, None)
-98
View File
@@ -1,98 +0,0 @@
"""FastAPI HTMX app for EPUB search."""
from __future__ import annotations
import logging
from contextlib import asynccontextmanager
from typing import TYPE_CHECKING, Annotated
import httpx
import typer
import uvicorn
from fastapi import FastAPI
from fastapi.staticfiles import StaticFiles
from sqlalchemy.ext.asyncio import AsyncSession
from python.common import configure_logger
from python.ebook_search.api.bm25_tasks import cancel_bm25_refresh
from python.ebook_search.api.routes import admin_router, health_router, page_router, search_router
from python.ebook_search.api.web import STATIC_DIR
from python.ebook_search.bm25_corpus import ensure_bm25_corpus
from python.ebook_search.config import load_config
from python.ebook_search.protected_phrases.pool import shutdown_extraction_pool
from python.fastapi_tools import ZstdMiddleware
from python.orm.common import get_async_postgres_engine
if TYPE_CHECKING:
from collections.abc import AsyncIterator
logger = logging.getLogger(__name__)
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
"""Manage application startup and shutdown resources."""
logger.info("ebook_search_startup")
config = load_config()
app.state.config = config
logger.info(
"ebook_search_config_loaded top_k=%s embedding_model=%s embedding_base_url=%s vllm_base_url=%s "
"rerank_enabled=%s phrase_matching_enabled=%s answer_enabled=%s library_paths=%s",
config.top_k,
config.embedding_model,
config.embedding_base_url,
config.vllm_base_url,
config.rerank.enabled,
config.phrase_matching_enabled,
config.answer_enabled,
len(config.library_paths),
)
if not config.library_paths:
logger.warning("ebook_search_no_library_paths_configured")
# Concurrent phrase judging opens one session per book worker on this engine, so size the pool
# to cover those plus headroom for ordinary web requests.
app.state.engine = get_async_postgres_engine(
name="RICHIE",
vector_engine=True,
pool_size=config.phrase_judge_book_workers + 10,
)
app.state.http_client = httpx.AsyncClient()
async with AsyncSession(app.state.engine, expire_on_commit=False) as session:
await ensure_bm25_corpus(session, config)
try:
yield
finally:
logger.info("ebook_search_shutdown")
cancel_bm25_refresh(app)
shutdown_extraction_pool()
await app.state.http_client.aclose()
await app.state.engine.dispose()
def create_app() -> FastAPI:
"""Create the EPUB search web app."""
app = FastAPI(title="EPUB Search", lifespan=lifespan)
app.add_middleware(ZstdMiddleware)
app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
app.include_router(admin_router)
app.include_router(health_router)
app.include_router(page_router)
app.include_router(search_router)
return app
def serve(
host: Annotated[str, typer.Option("--host", "-h", help="Host to bind to")] = "127.0.0.1",
port: Annotated[int, typer.Option("--port", "-p", help="Port to bind to")] = 8070,
log_level: Annotated[str, typer.Option("--log-level", "-l", help="Log level")] = "INFO",
) -> None:
"""Start the EPUB search server."""
configure_logger(log_level)
uvicorn.run(create_app(), host=host, port=port)
if __name__ == "__main__":
typer.run(serve)
@@ -1,13 +0,0 @@
"""EPUB search web route modules."""
from python.ebook_search.api.routes.admin import router as admin_router
from python.ebook_search.api.routes.health import router as health_router
from python.ebook_search.api.routes.page import router as page_router
from python.ebook_search.api.routes.search import router as search_router
__all__ = [
"admin_router",
"health_router",
"page_router",
"search_router",
]
-263
View File
@@ -1,263 +0,0 @@
"""Admin routes for the EPUB search web UI."""
from __future__ import annotations
import logging
from fastapi import APIRouter, Request
from fastapi.responses import HTMLResponse
from python.ebook_search.api.bm25_tasks import schedule_bm25_refresh
from python.ebook_search.api.dependencies import ( # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
AppConfig,
AppEngine,
AppHttpClient,
)
from python.ebook_search.api.web import templates
from python.ebook_search.embeddings import embed_missing_chunks, embedding_model_stats
from python.ebook_search.ingest import ingest_configured_paths
from python.ebook_search.protected_phrases.generate_ngrams import generate_candidate_phrases_for_books
from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
from python.ebook_search.protected_phrases.store import book_ids_pending_first_judgment, corpus_phrase_stats
from python.fastapi_tools import AsyncDbSession # noqa: TC001 FastAPI resolves this annotated dependency at runtime
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/admin")
@router.get("", response_class=HTMLResponse)
async def admin(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
"""Render the admin page."""
stats = await embedding_model_stats(session)
phrase_stats = await corpus_phrase_stats(session)
logger.info(
"ebook_admin_page_loaded models=%s candidate_phrases=%s protected_phrases=%s",
len(stats),
phrase_stats.candidate_phrases,
phrase_stats.protected_phrases,
)
return templates.TemplateResponse(
request,
"admin.html",
{"config": config, "stats": stats, "phrase_stats": phrase_stats},
)
@router.post("/scan", response_class=HTMLResponse)
async def scan_library(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
"""Scan configured library paths for EPUB changes."""
try:
count = await ingest_configured_paths(session, config)
await session.commit()
except Exception as error:
logger.exception("ebook_admin_scan_failed")
return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
logger.info("ebook_admin_scan_complete changed_files=%s", count)
if count > 0:
schedule_bm25_refresh(request.app)
return templates.TemplateResponse(request, "partials/admin_status.html", {"message": f"Indexed {count} EPUBs"})
@router.post("/phrases/generate-all", response_class=HTMLResponse)
async def generate_all_phrases(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
"""Regenerate candidate phrases for every indexed book without LLM judging."""
return await run_phrase_generation(request, config, session, only_missing=False)
@router.post("/phrases/generate-missing", response_class=HTMLResponse)
async def generate_missing_phrases(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
"""Generate candidate phrases only for books that have none yet."""
return await run_phrase_generation(request, config, session, only_missing=True)
async def run_phrase_generation(
request: Request,
config: AppConfig,
session: AsyncDbSession,
*,
only_missing: bool,
) -> HTMLResponse:
"""Run candidate phrase generation and render the outcome as an admin status partial.
Args:
request (Request): Current request, for template rendering.
config (AppConfig): Runtime phrase-tuning settings.
session (AsyncDbSession): Active database session.
only_missing (bool): Only generate for books without candidates instead of every book.
Returns:
HTMLResponse: Status partial describing the generation outcome.
"""
try:
result = await generate_candidate_phrases_for_books(session, config, only_missing=only_missing)
await session.commit()
except Exception as error:
await session.rollback()
logger.exception("ebook_admin_generate_phrases_failed only_missing=%s", only_missing)
return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
logger.info(
"ebook_admin_generate_phrases_complete only_missing=%s books_seen=%s books_built=%s candidates=%s",
only_missing,
result.books_seen,
result.books_built,
result.candidate_phrases,
)
if only_missing and result.books_seen == 0:
return templates.TemplateResponse(
request,
"partials/admin_status.html",
{"message": "All books already have candidate phrases"},
)
return templates.TemplateResponse(
request,
"partials/admin_status.html",
{
"message": (
f"Generated phrases for {result.books_built} of {result.books_seen} books; "
f"{result.candidate_phrases} candidates stored"
)
},
)
@router.post("/phrases/judge-all", response_class=HTMLResponse)
async def judge_all_phrases(request: Request, engine: AppEngine, config: AppConfig) -> HTMLResponse:
"""Judge unjudged candidate phrases across every indexed book."""
return await run_phrase_judgment(request, engine, config, source_ids=None)
@router.post("/phrases/judge-missing", response_class=HTMLResponse)
async def judge_missing_phrases(
request: Request,
engine: AppEngine,
config: AppConfig,
session: AsyncDbSession,
) -> HTMLResponse:
"""Judge candidate phrases only for books where judging has never run."""
source_ids = await book_ids_pending_first_judgment(session)
if not source_ids:
return templates.TemplateResponse(
request,
"partials/admin_status.html",
{"message": "All books with candidate phrases have been judged"},
)
return await run_phrase_judgment(request, engine, config, source_ids=source_ids)
async def run_phrase_judgment(
request: Request,
engine: AppEngine,
config: AppConfig,
*,
source_ids: list[int] | None,
) -> HTMLResponse:
"""Run LLM judging for candidate phrases and render the outcome as an admin status partial.
Args:
request (Request): Current request, for template rendering.
engine (AppEngine): Engine used to open per-book judging sessions.
config (AppConfig): Runtime phrase-tuning settings.
source_ids (list[int] | None): Books to judge; ``None`` judges every indexed book.
Returns:
HTMLResponse: Status partial describing the judging outcome.
"""
try:
result = await judge_candidate_phrases_for_books(engine, config, source_ids=source_ids)
except Exception as error:
logger.exception("ebook_admin_judge_phrases_failed")
return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
logger.info(
"ebook_admin_judge_phrases_complete books_seen=%s books_judged=%s books_failed=%s candidates_judged=%s "
"protected=%s mentions=%s",
result.books_seen,
result.books_judged,
result.books_failed,
result.candidates_judged,
result.protected_phrases,
result.phrase_mentions,
)
return templates.TemplateResponse(
request,
"partials/admin_status.html",
{
"message": (
f"Judged {result.candidates_judged} candidates across {result.books_judged} of "
f"{result.books_seen} books; {result.protected_phrases} protected phrases, "
f"{result.phrase_mentions} mentions"
+ (f"; {result.books_failed} books failed" if result.books_failed else "")
)
},
)
@router.post("/embed-missing", response_class=HTMLResponse)
async def embed_missing(
request: Request,
config: AppConfig,
session: AsyncDbSession,
client: AppHttpClient,
) -> HTMLResponse:
"""Embed chunks missing vectors for the configured model."""
try:
count = await embed_missing_chunks(session, client, config)
await session.commit()
except Exception as error:
logger.exception("ebook_admin_embed_missing_failed")
return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
logger.info("ebook_admin_embed_missing_complete chunks=%s", count)
return templates.TemplateResponse(
request,
"partials/admin_status.html",
{"message": f"Embedded {count} chunks"},
)
@router.post("/embed-all", response_class=HTMLResponse)
async def embed_all(
request: Request,
config: AppConfig,
session: AsyncDbSession,
client: AppHttpClient,
) -> HTMLResponse:
"""Embed all chunks missing vectors in fixed-size batches."""
total = 0
batches = 0
try:
while True:
count = await embed_missing_chunks(session, client, config)
if count == 0:
break
await session.commit()
total += count
batches += 1
logger.info(
"ebook_admin_embed_all_batch_complete batch=%s chunks=%s total_chunks=%s",
batches,
count,
total,
)
except Exception as error:
logger.exception(
"ebook_admin_embed_all_failed batches=%s chunks=%s",
batches,
total,
)
return templates.TemplateResponse(
request,
"partials/error.html",
{"message": f"Embed all failed after {total} chunks in {batches} batches: {error}"},
status_code=500,
)
logger.info("ebook_admin_embed_all_complete batches=%s chunks=%s", batches, total)
return templates.TemplateResponse(
request,
"partials/admin_status.html",
{"message": f"Embedded {total} chunks in {batches} batches of {config.embedding_batch_size}"},
)
-99
View File
@@ -1,99 +0,0 @@
"""Liveness and readiness routes for the EPUB search service."""
from __future__ import annotations
import logging
from http import HTTPStatus
from typing import TYPE_CHECKING
from fastapi import APIRouter
from fastapi.responses import JSONResponse
from sqlalchemy import literal, select
from sqlalchemy.exc import SQLAlchemyError
from python.ebook_search.api.dependencies import ( # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
AppConfig,
AppHttpClient,
)
from python.ebook_search.bm25_corpus import bm25_index_exists, bm25_index_path, read_bm25_manifest
from python.ebook_search.llm_interface import check_chat_endpoint, check_embedding_endpoint
from python.fastapi_tools import AsyncDbSession # noqa: TC001 FastAPI resolves this annotated dependency at runtime
if TYPE_CHECKING:
import httpx
from sqlalchemy.ext.asyncio import AsyncSession
from python.ebook_search.config import EbookSearchConfig
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/health")
async def health() -> dict[str, str]:
"""Liveness probe that returns ok without touching dependencies."""
return {"status": "ok"}
@router.get("/ready")
async def ready(config: AppConfig, session: AsyncDbSession, client: AppHttpClient) -> JSONResponse:
"""Readiness probe reporting database, embedding endpoint, and BM25 index status."""
database_ok = await check_database(session)
embedding_ok = await check_embedding_endpoint(client, config)
chat_status = await chat_endpoint_status(client, config)
bm25_status = check_bm25_status(config)
checks = {
"database": "ok" if database_ok else "fail",
"embedding": "ok" if embedding_ok else "fail",
"chat": chat_status,
"bm25": bm25_status,
}
if not database_ok:
status = "unavailable"
status_code = HTTPStatus.SERVICE_UNAVAILABLE
elif not embedding_ok or chat_status == "fail" or bm25_status == "missing":
status = "degraded"
status_code = HTTPStatus.OK
else:
status = "ready"
status_code = HTTPStatus.OK
logger.info(
"ebook_ready_check status=%s database=%s embedding=%s chat=%s bm25=%s",
status,
database_ok,
embedding_ok,
chat_status,
bm25_status,
)
return JSONResponse(content={"status": status, "checks": checks}, status_code=status_code)
async def chat_endpoint_status(client: httpx.AsyncClient, config: EbookSearchConfig) -> str:
"""Return the answering chat endpoint status, or disabled when answers are off."""
if not config.answer_enabled:
return "disabled"
return "ok" if await check_chat_endpoint(client, config) else "fail"
async def check_database(session: AsyncSession) -> bool:
"""Return whether the database answers a trivial query."""
try:
await session.execute(select(literal(1)))
except SQLAlchemyError as error:
logger.warning("ebook_ready_database_unavailable error=%s", error)
return False
return True
def check_bm25_status(config: EbookSearchConfig) -> str:
"""Return the persisted BM25 index status without loading it into memory."""
index_path = bm25_index_path(config)
manifest = read_bm25_manifest(index_path)
if manifest is None or not bm25_index_exists(index_path, manifest):
return "missing"
if manifest.chunk_count == 0:
return "empty"
return "ok"
-202
View File
@@ -1,202 +0,0 @@
"""Page routes for the EPUB search web UI."""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
from fastapi.responses import HTMLResponse, RedirectResponse
from sqlalchemy import func, select
from python.ebook_search.api.dependencies import (
AppConfig, # noqa: TC001 FastAPI resolves this annotated dependency at runtime
)
from python.ebook_search.api.judge_tasks import is_judging_book, pop_book_judgment_outcome, start_book_phrase_judgment
from python.ebook_search.api.web import templates
from python.ebook_search.protected_phrases.generate_ngrams import recalculate_candidate_phrases_for_book
from python.fastapi_tools import AsyncDbSession # noqa: TC001 FastAPI resolves this annotated dependency at runtime
from python.orm.richie import EbookCandidatePhrase, EbookChapter, EbookChunk, EbookProtectedPhrase, EbookSource
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/", response_class=HTMLResponse)
async def index(request: Request, config: AppConfig) -> HTMLResponse:
"""Render the search page."""
return templates.TemplateResponse(request, "search.html", {"config": config})
@router.get("/books", response_class=HTMLResponse)
async def books(request: Request, session: AsyncDbSession) -> HTMLResponse:
"""Render the indexed books page."""
sources = list((await session.scalars(select(EbookSource).order_by(EbookSource.title))).all())
logger.info("ebook_books_page_loaded count=%s", len(sources))
return templates.TemplateResponse(request, "books.html", {"sources": sources})
async def get_chapter_count(session: AsyncSession, book_id: int) -> int:
"""Return the number of indexed chapters for one book."""
return await session.scalar(select(func.count(EbookChapter.id)).where(EbookChapter.source_id == book_id)) or 0
async def get_chunk_count(session: AsyncSession, book_id: int) -> int:
"""Return the number of indexed chunks for one book."""
return await session.scalar(select(func.count(EbookChunk.id)).where(EbookChunk.source_id == book_id)) or 0
async def get_candidate_count(session: AsyncSession, book_id: int) -> int:
"""Return the number of indexed candidates for one book."""
return (
await session.scalar(select(func.count(EbookCandidatePhrase.id)).where(EbookCandidatePhrase.book_id == book_id))
or 0
)
async def get_judged_candidate_count(session: AsyncSession, book_id: int) -> int:
"""Return the number of judged candidates for one book."""
return (
await session.scalar(
select(func.count(EbookCandidatePhrase.id)).where(
EbookCandidatePhrase.book_id == book_id,
EbookCandidatePhrase.llm_judged.is_(True),
)
)
or 0
)
async def get_protected_count(session: AsyncSession, book_id: int) -> int:
"""Return the number of protected phrases for one book."""
return (
await session.scalar(select(func.count(EbookProtectedPhrase.id)).where(EbookProtectedPhrase.book_id == book_id))
or 0
)
async def get_candidates(session: AsyncSession, book_id: int) -> list[EbookCandidatePhrase]:
"""Return the indexed candidates for one book."""
return list(
await session.scalars(
select(EbookCandidatePhrase)
.where(EbookCandidatePhrase.book_id == book_id)
.order_by(EbookCandidatePhrase.candidate_score.desc())
.limit(100)
)
)
async def get_protected_phrases(session: AsyncSession, book_id: int) -> list[EbookProtectedPhrase]:
"""Return the protected phrases for one book."""
return list(
await session.scalars(
select(EbookProtectedPhrase)
.where(EbookProtectedPhrase.book_id == book_id)
.order_by(EbookProtectedPhrase.importance.desc())
.limit(100)
)
)
@router.get("/books/{source_id}", response_class=HTMLResponse)
async def book_detail(source_id: int, request: Request, session: AsyncDbSession) -> HTMLResponse:
"""Render details for one indexed book."""
source = await session.get(EbookSource, source_id)
phrase_status_message = None
recalculated = request.query_params.get("phrases_recalculated")
if recalculated is not None:
phrase_status_message = f"Recalculated phrases; {recalculated} candidates generated"
judgment_outcome = pop_book_judgment_outcome(request.app, source_id)
if judgment_outcome is not None:
phrase_status_message = judgment_outcome
judging_in_progress = is_judging_book(request.app, source_id)
if judging_in_progress:
phrase_status_message = "Judging candidate phrases in the background; refresh to see progress"
if source is not None:
chapter_count = await get_chapter_count(session, source.id)
chunk_count = await get_chunk_count(session, source.id)
candidate_count = await get_candidate_count(session, source.id)
judged_candidate_count = await get_judged_candidate_count(session, source.id)
protected_count = await get_protected_count(session, source.id)
candidates = await get_candidates(session, source.id)
protected_phrases = await get_protected_phrases(session, source.id)
else:
chapter_count = 0
chunk_count = 0
candidate_count = 0
judged_candidate_count = 0
protected_count = 0
candidates = []
protected_phrases = []
logger.info(
"ebook_book_detail_loaded source_id=%s found=%s chapters=%s chunks=%s candidates=%s judged=%s protected=%s",
source_id,
source is not None,
chapter_count,
chunk_count,
candidate_count,
judged_candidate_count,
protected_count,
)
return templates.TemplateResponse(
request,
"book_detail.html",
{
"candidate_count": candidate_count,
"candidates": candidates,
"chapter_count": chapter_count,
"chunk_count": chunk_count,
"judged_candidate_count": judged_candidate_count,
"judging_in_progress": judging_in_progress,
"protected_count": protected_count,
"protected_phrases": protected_phrases,
"phrase_status_message": phrase_status_message,
"source": source,
},
)
@router.post("/books/{source_id}/recalculate-phrases")
async def recalculate_book_phrases(source_id: int, config: AppConfig, session: AsyncDbSession) -> RedirectResponse:
"""Clear and regenerate candidate phrases for one indexed book."""
source = await session.get(EbookSource, source_id)
if source is None:
raise HTTPException(status_code=404, detail="Book not found")
result = await recalculate_candidate_phrases_for_book(session, source, config, use_process_pool=True)
logger.info(
"ebook_book_phrase_recalculation_complete source_id=%s candidates=%s deleted_candidates=%s "
"deleted_protected=%s deleted_aliases=%s deleted_mentions=%s",
source_id,
result.candidate_phrases,
result.deleted_candidates,
result.deleted_protected_phrases,
result.deleted_aliases,
result.deleted_mentions,
)
return RedirectResponse(
url=f"/books/{source_id}?phrases_recalculated={result.candidate_phrases}",
status_code=303,
)
@router.post("/books/{source_id}/judge-phrases")
async def judge_book_phrases(
source_id: int,
request: Request,
background_tasks: BackgroundTasks,
session: AsyncDbSession,
) -> RedirectResponse:
"""Queue background judging of one book's candidate phrases and return immediately."""
source = await session.get(EbookSource, source_id)
if source is None:
raise HTTPException(status_code=404, detail="Book not found")
started = start_book_phrase_judgment(request.app, background_tasks, source.id)
logger.info("ebook_book_phrase_judgment_requested source_id=%s started=%s", source_id, started)
return RedirectResponse(url=f"/books/{source_id}", status_code=303)
-129
View File
@@ -1,129 +0,0 @@
"""Search routes for the EPUB search web UI."""
from __future__ import annotations
import logging
from dataclasses import replace
from time import perf_counter
from typing import TYPE_CHECKING, Annotated
from fastapi import APIRouter, Form, Request
from fastapi.responses import HTMLResponse
from python.ebook_search.answer import answer_query
from python.ebook_search.api.dependencies import ( # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
AppConfig,
AppEngine,
AppHttpClient,
)
from python.ebook_search.api.web import templates
from python.ebook_search.guardrails import (
CitationReport,
is_confident,
retrieval_confidence,
validate_citations,
)
from python.ebook_search.search import SearchResponse, search_ebooks
from python.ebook_search.timing import runtime_step_from_start
if TYPE_CHECKING:
import httpx
from python.ebook_search.config import EbookSearchConfig
logger = logging.getLogger(__name__)
router = APIRouter()
async def build_answer(
client: httpx.AsyncClient,
query: str,
response: SearchResponse,
config: EbookSearchConfig,
) -> tuple[str, bool, CitationReport | None]:
"""Generate the answer for a search, returning ``(answer, low_confidence, citation_report)``."""
if not config.answer_enabled:
logger.info("ebook_answer_skipped_disabled")
return "Answer generation is disabled. Source chunks are shown below.", False, None
if not is_confident(response.results, config):
logger.info(
"ebook_answer_low_confidence confidence=%.4f threshold=%.4f",
retrieval_confidence(response.results),
config.min_retrieval_confidence,
)
answer = (
"Retrieval confidence is low for this query, so answer generation was skipped. "
"Source chunks are shown below."
)
return answer, True, None
try:
answer = await answer_query(client, query, response.results, config)
except RuntimeError as error:
logger.warning("ebook_answer_request_failed_falling_back error=%s", error)
return "Answer generation failed. Source chunks are still shown below.", False, None
citation_report = None
if config.validate_citations_enabled and response.results:
citation_report = validate_citations(answer, len(response.results))
if citation_report.invalid or not citation_report.grounded:
logger.warning(
"ebook_answer_citation_issue invalid=%s grounded=%s",
citation_report.invalid,
citation_report.grounded,
)
return answer, False, citation_report
@router.post("/search", response_class=HTMLResponse)
async def search(
request: Request,
config: AppConfig,
engine: AppEngine,
client: AppHttpClient,
query: Annotated[str, Form()],
rerank: Annotated[str | None, Form()] = None,
phrase_matching: Annotated[str | None, Form()] = None,
) -> HTMLResponse:
"""Run a search and render HTMX results."""
try:
response = await search_ebooks(
engine,
client,
query,
config,
rerank=rerank == "true",
phrase_matching=phrase_matching == "true",
)
except Exception as error:
logger.exception("ebook_search_request_failed")
return templates.TemplateResponse(request, "partials/error.html", {"message": str(error)}, status_code=500)
answer_start = perf_counter()
answer, low_confidence, citation_report = await build_answer(client, query, response, config)
answer_step_name = "Answer generation" if config.answer_enabled else "Answer skipped"
response = replace(
response,
timings=(*response.timings, runtime_step_from_start(answer_step_name, answer_start)),
)
for step in response.timings:
logger.info("ebook_search_timing step=%r runtime_ms=%.1f", step.name, step.duration_ms)
logger.info(
"ebook_search_request_complete results=%s rank_label=%s runtime_ms=%.1f",
len(response.results),
response.rank_label,
response.total_runtime_ms,
)
return templates.TemplateResponse(
request,
"partials/results.html",
{
"answer": answer,
"response": response,
"low_confidence": low_confidence,
"citation_report": citation_report,
},
)
-447
View File
@@ -1,447 +0,0 @@
:root {
--bg: #f4f5f7;
--surface: #ffffff;
--border: #e3e5ea;
--text: #1c1f24;
--muted: #6b7280;
--accent: #4f46e5;
--accent-soft: #eef0fe;
--danger: #b42318;
--warn-bg: #fff8eb;
--warn-border: #e0a92e;
--warn-text: #7a5008;
--radius: 12px;
--shadow: 0 1px 2px rgba(16, 24, 40, 0.04), 0 1px 3px rgba(16, 24, 40, 0.08);
}
html.theme-dark {
--bg: #0f1117;
--surface: #1a1d25;
--border: #2b303b;
--text: #e6e8ec;
--muted: #9aa1ad;
--accent: #818cf8;
--accent-soft: #262b45;
--danger: #f97066;
--warn-bg: #2a2410;
--warn-border: #b9881f;
--warn-text: #e8c97a;
--shadow: 0 1px 2px rgba(0, 0, 0, 0.3), 0 1px 3px rgba(0, 0, 0, 0.4);
color-scheme: dark;
}
* {
box-sizing: border-box;
}
body {
margin: 0;
background: var(--bg);
color: var(--text);
font-family: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
line-height: 1.55;
}
main {
max-width: 820px;
margin: 0 auto;
padding: 32px 20px 64px;
}
/* Header / nav */
.site-header {
background: var(--surface);
border-bottom: 1px solid var(--border);
position: sticky;
top: 0;
z-index: 10;
}
.site-nav {
max-width: 820px;
margin: 0 auto;
padding: 12px 20px;
display: flex;
align-items: center;
gap: 20px;
}
.brand {
font-weight: 700;
font-size: 1.05rem;
color: var(--text);
text-decoration: none;
}
.nav-links {
display: flex;
gap: 6px;
margin-right: auto;
}
.nav-links a {
padding: 6px 12px;
border-radius: 8px;
color: var(--muted);
text-decoration: none;
font-size: 0.94rem;
transition: background 0.15s, color 0.15s;
}
.nav-links a:hover {
background: var(--accent-soft);
color: var(--accent);
}
.dev-toggle {
display: inline-flex;
align-items: center;
gap: 6px;
font-size: 0.85rem;
color: var(--muted);
cursor: pointer;
user-select: none;
}
.theme-toggle {
display: inline-flex;
align-items: center;
justify-content: center;
width: 34px;
height: 34px;
padding: 0;
font-size: 1rem;
line-height: 1;
color: var(--text);
background: var(--bg);
border: 1px solid var(--border);
border-radius: 8px;
cursor: pointer;
}
.theme-toggle:hover {
border-color: var(--accent);
filter: none;
}
h1 {
font-size: 1.6rem;
margin: 0 0 20px;
}
h2 {
font-size: 1.15rem;
margin: 0 0 8px;
}
/* Cards */
.card {
background: var(--surface);
border: 1px solid var(--border);
border-radius: var(--radius);
box-shadow: var(--shadow);
padding: 20px;
}
/* Search form */
form {
margin: 0;
}
label {
font-weight: 600;
font-size: 0.92rem;
}
textarea {
display: block;
width: 100%;
margin: 8px 0 16px;
padding: 12px 14px;
font: inherit;
color: var(--text);
background: var(--surface);
border: 1px solid var(--border);
border-radius: 10px;
resize: vertical;
transition: border-color 0.15s, box-shadow 0.15s;
}
textarea:focus {
outline: none;
border-color: var(--accent);
box-shadow: 0 0 0 3px var(--accent-soft);
}
.form-row {
display: flex;
align-items: center;
justify-content: space-between;
gap: 12px;
flex-wrap: wrap;
}
.search-toggles {
display: flex;
flex-wrap: wrap;
gap: 14px;
}
button {
padding: 10px 20px;
font: inherit;
font-weight: 600;
color: #fff;
background: var(--accent);
border: none;
border-radius: 10px;
cursor: pointer;
transition: filter 0.15s;
}
button:hover {
filter: brightness(1.08);
}
.check {
display: inline-flex;
gap: 8px;
align-items: center;
font-weight: 500;
color: var(--muted);
}
.actions {
display: flex;
flex-wrap: wrap;
gap: 12px;
margin-bottom: 24px;
}
.actions-grid {
display: grid;
grid-template-columns: repeat(2, max-content);
}
/* Answer + results */
#results {
display: block;
margin-top: 28px;
}
.rank-label {
font-size: 0.82rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.04em;
color: var(--muted);
margin-bottom: 16px;
}
.answer {
background: var(--surface);
border: 1px solid var(--border);
border-radius: var(--radius);
box-shadow: var(--shadow);
padding: 20px;
margin-bottom: 24px;
}
.answer p:last-child {
margin-bottom: 0;
}
.results {
list-style: none;
padding: 0;
margin: 0;
display: grid;
gap: 16px;
}
.results > li {
background: var(--surface);
border: 1px solid var(--border);
border-radius: var(--radius);
box-shadow: var(--shadow);
padding: 18px 20px;
}
.results h2 {
font-size: 1.05rem;
}
.results h2 a {
color: var(--text);
text-decoration: none;
}
.results h2 a:hover {
color: var(--accent);
}
.meta {
color: var(--muted);
font-size: 0.88rem;
margin: 0 0 10px;
}
.scores {
display: flex;
flex-wrap: wrap;
gap: 8px;
margin: 14px 0 0;
}
.scores div {
display: inline-flex;
gap: 6px;
align-items: baseline;
padding: 3px 10px;
background: var(--bg);
border: 1px solid var(--border);
border-radius: 999px;
font-size: 0.78rem;
}
.scores dt {
font-weight: 600;
color: var(--muted);
}
.scores dd {
margin: 0;
font-variant-numeric: tabular-nums;
}
.phrase-matches {
display: flex;
flex-wrap: wrap;
gap: 8px;
align-items: baseline;
margin: 10px 0 0;
font-size: 0.78rem;
}
.phrase-matches-label {
color: var(--muted);
font-weight: 600;
}
.phrase-match {
padding: 3px 10px;
background: var(--bg);
border: 1px solid var(--border);
border-radius: 999px;
color: var(--accent);
}
/* Runtime — developer diagnostics, hidden unless dev mode is on */
.runtime {
display: none;
background: var(--surface);
border: 1px solid var(--border);
border-radius: var(--radius);
box-shadow: var(--shadow);
padding: 18px 20px;
margin-bottom: 24px;
}
html.dev .runtime {
display: block;
}
.timing-chart {
display: grid;
gap: 8px;
padding: 0;
margin: 12px 0 0;
list-style: none;
}
.timing-chart li {
display: grid;
grid-template-columns: minmax(150px, 1fr) minmax(160px, 2fr) auto auto;
gap: 10px;
align-items: center;
font-size: 0.85rem;
}
.timing-bar {
height: 8px;
overflow: hidden;
background: var(--bg);
border-radius: 999px;
}
.timing-bar span {
display: block;
height: 100%;
background: var(--accent);
border-radius: 999px;
}
.timing-value,
.timing-remaining {
color: var(--muted);
font-variant-numeric: tabular-nums;
text-align: right;
}
/* Tables */
table {
width: 100%;
border-collapse: collapse;
background: var(--surface);
border: 1px solid var(--border);
border-radius: var(--radius);
overflow: hidden;
}
th,
td {
padding: 10px 14px;
border-bottom: 1px solid var(--border);
text-align: left;
font-size: 0.9rem;
}
th {
font-weight: 600;
color: var(--muted);
background: var(--bg);
}
tbody tr:last-child td {
border-bottom: none;
}
dl dt {
font-weight: 600;
color: var(--muted);
font-size: 0.85rem;
}
dl dd {
margin: 0 0 12px;
}
/* States */
.error {
color: var(--danger);
font-weight: 600;
}
.notice {
margin: 12px 0;
padding: 10px 14px;
border-left: 3px solid var(--warn-border);
border-radius: 6px;
background: var(--warn-bg);
color: var(--warn-text);
font-weight: 500;
}
.status {
color: var(--muted);
}
@@ -1,110 +0,0 @@
{% extends "base.html" %} {% block title %}EPUB Admin{% endblock %} {% block
head %}
<script src="https://unpkg.com/htmx.org@2.0.4"></script>
{% endblock %} {% block content %}
<h1>Admin</h1>
<section id="admin-status"></section>
<section class="actions">
<form hx-post="/admin/scan" hx-target="#admin-status" hx-swap="innerHTML">
<button type="submit">Scan</button>
</form>
</section>
<section>
<h2>Embeddings</h2>
<section class="actions">
<form
hx-post="/admin/embed-missing"
hx-target="#admin-status"
hx-swap="innerHTML"
>
<button type="submit">Embed</button>
</form>
<form
hx-post="/admin/embed-all"
hx-target="#admin-status"
hx-swap="innerHTML"
>
<button type="submit">Embed all</button>
</form>
</section>
<table>
<thead>
<tr>
<th>Model</th>
<th>Dimensions</th>
<th>Embedded</th>
<th>Missing</th>
<th>Total chunks</th>
</tr>
</thead>
<tbody>
{% for item in stats %}
<tr>
<td>{{ item.model_name }}</td>
<td>{{ item.dimension }}</td>
<td>{{ item.embedded_chunks }}</td>
<td>{{ item.missing_chunks }}</td>
<td>{{ item.total_chunks }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</section>
<section>
<h2>Protected phrases</h2>
<section class="actions actions-grid">
<form
hx-post="/admin/phrases/generate-all"
hx-target="#admin-status"
hx-swap="innerHTML"
>
<button type="submit">Regenerate all phrases</button>
</form>
<form
hx-post="/admin/phrases/generate-missing"
hx-target="#admin-status"
hx-swap="innerHTML"
>
<button type="submit">Add missing phrases</button>
</form>
<form
hx-post="/admin/phrases/judge-all"
hx-target="#admin-status"
hx-swap="innerHTML"
>
<button type="submit">Judge all phrases</button>
</form>
<form
hx-post="/admin/phrases/judge-missing"
hx-target="#admin-status"
hx-swap="innerHTML"
>
<button type="submit">Judge missing phrases</button>
</form>
</section>
<table>
<thead>
<tr>
<th>Candidates</th>
<th>Judged</th>
<th>Unjudged</th>
<th>Protected</th>
<th>Books indexed</th>
<th>Books generated</th>
<th>Books fully judged</th>
</tr>
</thead>
<tbody>
<tr>
<td>{{ phrase_stats.candidate_phrases }}</td>
<td>{{ phrase_stats.judged_candidates }}</td>
<td>{{ phrase_stats.unjudged_candidates }}</td>
<td>{{ phrase_stats.protected_phrases }}</td>
<td>{{ phrase_stats.total_books }}</td>
<td>{{ phrase_stats.books_with_candidates }}</td>
<td>{{ phrase_stats.books_fully_judged }}</td>
</tr>
</tbody>
</table>
</section>
{% endblock %}
@@ -1,71 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>{% block title %}EPUB Search{% endblock %}</title>
{% block head %}{% endblock %}
<link rel="stylesheet" href="/static/style.css?v={{ static_version('style.css') }}">
<script>
// Apply theme and dev mode before paint to avoid a flash of unstyled/wrong content.
(function () {
var stored = localStorage.getItem("ebook-theme");
var prefersDark = window.matchMedia("(prefers-color-scheme: dark)").matches;
var theme = stored || (prefersDark ? "dark" : "light");
document.documentElement.classList.add("theme-" + theme);
if (localStorage.getItem("ebook-dev-mode") === "on") {
document.documentElement.classList.add("dev");
}
})();
</script>
</head>
<body>
<header class="site-header">
<nav class="site-nav">
<a class="brand" href="/">EPUB Search</a>
<div class="nav-links">
<a href="/">Search</a>
<a href="/books">Books</a>
<a href="/admin">Admin</a>
</div>
<button type="button" id="theme-toggle" class="theme-toggle" title="Toggle light / dark theme" aria-label="Toggle theme"></button>
<label class="dev-toggle" title="Show developer diagnostics">
<input type="checkbox" id="dev-mode-toggle">
<span>Dev</span>
</label>
</nav>
</header>
<main>
{% block content %}{% endblock %}
</main>
<script>
(function () {
var toggle = document.getElementById("dev-mode-toggle");
if (toggle) {
toggle.checked = document.documentElement.classList.contains("dev");
toggle.addEventListener("change", function () {
document.documentElement.classList.toggle("dev", toggle.checked);
localStorage.setItem("ebook-dev-mode", toggle.checked ? "on" : "off");
});
}
var themeButton = document.getElementById("theme-toggle");
if (themeButton) {
var root = document.documentElement;
var sync = function () {
var isDark = root.classList.contains("theme-dark");
themeButton.textContent = isDark ? "☀️" : "🌙";
};
sync();
themeButton.addEventListener("click", function () {
var next = root.classList.contains("theme-dark") ? "light" : "dark";
root.classList.remove("theme-dark", "theme-light");
root.classList.add("theme-" + next);
localStorage.setItem("ebook-theme", next);
sync();
});
}
})();
</script>
</body>
</html>
@@ -1,109 +0,0 @@
{% extends "base.html" %}
{% block title %}{% if source %}{{ source.title }}{% else %}Book not found{% endif %}{% endblock %}
{% block content %}
{% if source %}
<h1>{{ source.title }}</h1>
<p class="meta">{{ source.author or "Unknown author" }}</p>
{% if phrase_status_message %}
<p class="status">{{ phrase_status_message }}</p>
{% endif %}
<dl class="card">
<dt>File</dt>
<dd>{{ source.file_path }}</dd>
<dt>Chapters</dt>
<dd>{{ chapter_count }}</dd>
<dt>Chunks</dt>
<dd>{{ chunk_count }}</dd>
<dt>Candidates</dt>
<dd>{{ candidate_count }}</dd>
<dt>Judged</dt>
<dd>{{ judged_candidate_count }}</dd>
<dt>Protected</dt>
<dd>{{ protected_count }}</dd>
</dl>
<form
method="post"
action="/books/{{ source.id }}/recalculate-phrases"
onsubmit="return confirm('Remove old phrases for this book and generate new candidates?');"
>
<button type="submit">Recalculate phrases</button>
</form>
<form
method="post"
action="/books/{{ source.id }}/judge-phrases"
onsubmit="return confirm('Judge candidate phrases for this book with the LLM?');"
>
<button type="submit"{% if judging_in_progress %} disabled{% endif %}>
{% if judging_in_progress %}Judging&hellip;{% else %}Judge phrases{% endif %}
</button>
</form>
<section>
<h2>Candidate n-grams</h2>
{% if candidates %}
<table>
<thead>
<tr>
<th>Phrase</th>
<th>Status</th>
<th>Score</th>
<th>Count</th>
<th>Chapters</th>
</tr>
</thead>
<tbody>
{% for candidate in candidates %}
<tr>
<td>{{ candidate.phrase_text }}</td>
<td>
{% if candidate.llm_judged %}
{% if candidate.llm_keep %}Kept{% else %}Rejected{% endif %}
{% else %}
Candidate
{% endif %}
</td>
<td>{{ "%.2f"|format(candidate.candidate_score) }}</td>
<td>{{ candidate.raw_count }}</td>
<td>{{ candidate.chapter_count }}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p>No candidate n-grams.</p>
{% endif %}
</section>
<section>
<h2>Protected phrases</h2>
{% if protected_phrases %}
<table>
<thead>
<tr>
<th>Phrase</th>
<th>Type</th>
<th>Confidence</th>
<th>Importance</th>
</tr>
</thead>
<tbody>
{% for phrase in protected_phrases %}
<tr>
<td>{{ phrase.phrase_text }}</td>
<td>{{ phrase.phrase_type or "phrase" }}</td>
<td>{{ "%.2f"|format(phrase.confidence) }}</td>
<td>{{ "%.2f"|format(phrase.importance) }}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p>No protected phrases.</p>
{% endif %}
</section>
{% else %}
<h1>Book not found</h1>
{% endif %}
{% endblock %}
@@ -1,19 +0,0 @@
{% extends "base.html" %}
{% block title %}EPUB Books{% endblock %}
{% block content %}
<h1>Books</h1>
{% if sources %}
<ol class="results">
{% for source in sources %}
<li>
<h2><a href="/books/{{ source.id }}">{{ source.title }}</a></h2>
<p class="meta">{{ source.author or "Unknown author" }}</p>
</li>
{% endfor %}
</ol>
{% else %}
<p>No EPUBs indexed.</p>
{% endif %}
{% endblock %}
@@ -1 +0,0 @@
<p class="status">{{ message }}</p>
@@ -1 +0,0 @@
<p class="error">{{ message }}</p>
@@ -1,98 +0,0 @@
<div class="rank-label">{{ response.rank_label }}</div>
{% if response.timings %}
<section class="runtime">
<h2>Runtime</h2>
<p class="meta">Total {{ "%.1f"|format(response.total_runtime_ms) }} ms</p>
<ol class="timing-chart">
{% set total = response.total_runtime_ms %}
{% set ns = namespace(remaining=total) %}
{% for step in response.timings %}
{% set width = (step.duration_ms / total * 100) if total else 0 %}
{% if step.counts_toward_total %}
{% set ns.remaining = ns.remaining - step.duration_ms %}
{% endif %}
<li>
<span class="timing-label">{{ step.name }}</span>
<span class="timing-bar"><span style="width: {{ "%.2f"|format(width) }}%"></span></span>
<span class="timing-value">{{ "%.1f"|format(step.duration_ms) }} ms</span>
<span class="timing-remaining">{{ "%.1f"|format([ns.remaining, 0]|max) }} ms left</span>
</li>
{% endfor %}
</ol>
</section>
{% endif %}
<section class="answer">
<h2>Answer</h2>
{% if low_confidence|default(false) %}
<p class="notice">Low retrieval confidence — answer generation was skipped.</p>
{% endif %}
{% set report = citation_report|default(none) %}
{% if report is not none and not report.grounded %}
<p class="notice">Unverified — no source citations were found in this answer.</p>
{% endif %}
{% if report is not none and report.invalid %}
<p class="notice">Invalid citations: {{ report.invalid|join(", ") }} (no matching source).</p>
{% endif %}
<p>{{ answer }}</p>
</section>
{% if response.results %}
<ol class="results">
{% for result in response.results %}
<li>
<h2>
{% if result.source_id %}
<a href="/books/{{ result.source_id }}">{{ result.source_title }}</a>
{% else %}
{{ result.source_title }}
{% endif %}
</h2>
<p class="meta">
{% if result.source_author %}{{ result.source_author }}{% endif %}
{% if result.chapter_title %} · {{ result.chapter_title }}{% endif %}
{% if result.page_label %} · page {{ result.page_label }}{% endif %}
</p>
<p>{{ result.text }}</p>
<dl class="scores">
<div>
<dt>final</dt>
<dd>{{ "%.3f"|format(result.score) }}</dd>
</div>
{% if result.rerank_score is not none %}
<div>
<dt>rerank</dt>
<dd>{{ "%.3f"|format(result.rerank_score) }}</dd>
</div>
{% endif %}
{% if result.vector_score is not none %}
<div>
<dt>vector cosine</dt>
<dd>{{ "%.3f"|format(result.vector_score) }}</dd>
</div>
{% endif %}
{% if result.bm25_score is not none %}
<div>
<dt>BM25</dt>
<dd>{{ "%.6f"|format(result.bm25_score) }}</dd>
</div>
{% endif %}
{% if result.fused_score is not none %}
<div>
<dt>RRF</dt>
<dd>{{ "%.3f"|format(result.fused_score) }}</dd>
</div>
{% endif %}
</dl>
{% if result.matched_phrases %}
<p class="phrase-matches">
<span class="phrase-matches-label">boosted by</span>
{% for phrase in result.matched_phrases %}
<span class="phrase-match">{{ phrase }}</span>
{% endfor %}
</p>
{% endif %}
</li>
{% endfor %}
</ol>
{% else %}
<p>No results.</p>
{% endif %}
@@ -1,32 +0,0 @@
{% extends "base.html" %}
{% block title %}EPUB Search{% endblock %}
{% block head %}<script src="https://unpkg.com/htmx.org@2.0.4"></script>{% endblock %}
{% block content %}
<h1>Search</h1>
<form class="card" hx-post="/search" hx-target="#results" hx-swap="innerHTML">
<label for="query">What are you looking for?</label>
<textarea id="query" name="query" rows="4" placeholder="Ask a question or paste a passage…" required
onkeydown="if (event.key === 'Enter' && !event.shiftKey) { event.preventDefault(); this.form.requestSubmit(); }"></textarea>
<div class="form-row">
<div class="search-toggles">
<label class="check">
<input type="checkbox" name="rerank" value="true" {% if config.rerank.enabled %}checked{% endif %}>
Rerank
</label>
<label class="check">
<input
type="checkbox"
name="phrase_matching"
value="true"
{% if config.phrase_matching_enabled %}checked{% endif %}
>
Phrase matching
</label>
</div>
<button type="submit">Search</button>
</div>
</form>
<section id="results"></section>
{% endblock %}
-23
View File
@@ -1,23 +0,0 @@
"""Shared web UI resources for EPUB search."""
from __future__ import annotations
from pathlib import Path
from fastapi.templating import Jinja2Templates
PACKAGE_DIR = Path(__file__).resolve().parent
TEMPLATE_DIR = PACKAGE_DIR / "templates"
STATIC_DIR = PACKAGE_DIR / "static"
def static_version(filename: str) -> int:
"""Return a cache-busting token for a static file based on its modification time."""
try:
return int((STATIC_DIR / filename).stat().st_mtime)
except OSError:
return 0
templates = Jinja2Templates(directory=TEMPLATE_DIR)
templates.env.globals["static_version"] = static_version
-286
View File
@@ -1,286 +0,0 @@
"""Persisted BM25 corpus management."""
from __future__ import annotations
import asyncio
import json
import logging
import shutil
from dataclasses import dataclass
from datetime import UTC, datetime
from functools import cache
from pathlib import Path
from typing import TYPE_CHECKING
import bm25s
from sqlalchemy import func, select, union_all
from python.orm.richie import EbookChapter, EbookChunk, EbookSource
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession
from python.ebook_search.config import EbookSearchConfig
logger = logging.getLogger(__name__)
MANIFEST_NAME = "manifest.json"
REQUIRED_INDEX_FILES = frozenset(
{
"data.csc.index.npy",
"indices.csc.index.npy",
"indptr.csc.index.npy",
"params.index.json",
"vocab.index.json",
"corpus.jsonl",
}
)
@dataclass(frozen=True)
class BM25Manifest:
"""Metadata describing a persisted BM25 corpus."""
created_at: datetime
db_updated_at: datetime | None
chunk_count: int
@dataclass(frozen=True)
class BM25Corpus:
"""Loaded persisted BM25 corpus and retriever."""
retriever: object | None
records: tuple[dict[str, object], ...]
manifest: BM25Manifest
class BM25CorpusUnavailableError(RuntimeError):
"""Raised when the persisted BM25 corpus cannot be loaded."""
def bm25_index_path(config: EbookSearchConfig) -> Path:
"""Return the configured BM25 index root path relative to the current working directory."""
path = Path(config.bm25_index_dir).expanduser()
if path.is_absolute():
return path
return Path.cwd() / path
def get_current_bm25_index(index_path: Path) -> Path:
"""Return the live BM25 index directory."""
current_path = index_path / "current"
if current_path.exists() or current_path.is_symlink():
return current_path
return index_path
async def ensure_bm25_corpus(session: AsyncSession, config: EbookSearchConfig) -> None:
"""Create or refresh the persisted BM25 corpus when it is missing or stale."""
index_path = bm25_index_path(config)
manifest = read_bm25_manifest(index_path)
db_updated_at = await corpus_last_updated_at(session)
if not bm25_index_exists(index_path, manifest):
logger.info("ebook_bm25_index_missing path=%s", index_path)
await refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
return
if db_updated_at is not None and manifest is not None and manifest.created_at < db_updated_at:
logger.info(
"ebook_bm25_index_stale path=%s created_at=%s db_updated_at=%s",
index_path,
manifest.created_at.isoformat(),
db_updated_at.isoformat(),
)
await refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
return
logger.info(
"ebook_bm25_index_current path=%s chunks=%s created_at=%s",
index_path,
manifest.chunk_count if manifest else 0,
manifest.created_at.isoformat() if manifest else None,
)
async def refresh_bm25_corpus(
session: AsyncSession,
config: EbookSearchConfig,
*,
db_updated_at: datetime | None = None,
) -> BM25Manifest:
"""Rebuild and persist the BM25 corpus from the current database chunks.
The index build is CPU and disk work, so it runs in a worker thread.
"""
index_path = bm25_index_path(config)
records, texts = await fetch_bm25_corpus_records(session)
manifest = BM25Manifest(
created_at=datetime.now(tz=UTC),
db_updated_at=db_updated_at if db_updated_at is not None else await corpus_last_updated_at(session),
chunk_count=len(records),
)
await asyncio.to_thread(write_bm25_corpus, index_path, records, texts, manifest)
logger.info(
"ebook_bm25_index_refreshed path=%s chunks=%s created_at=%s",
index_path,
manifest.chunk_count,
manifest.created_at.isoformat(),
)
return manifest
@cache
def load_bm25_corpus(config: EbookSearchConfig) -> BM25Corpus:
"""Load the BM25 corpus into memory once per process.
Background refresh tasks clear this cache after rebuilding the on-disk corpus.
"""
index_path = bm25_index_path(config)
active_index_path = get_current_bm25_index(index_path)
logger.info("ebook_bm25_corpus_cache_load path=%s active_path=%s", index_path, active_index_path)
manifest = read_bm25_manifest(index_path)
if manifest is None or not bm25_index_exists(index_path, manifest):
msg = f"BM25 corpus is not available: {index_path}"
raise BM25CorpusUnavailableError(msg)
if manifest.chunk_count == 0:
return BM25Corpus(retriever=None, records=(), manifest=manifest)
retriever = bm25s.BM25.load(active_index_path, load_corpus=True, mmap=True)
records = tuple(dict(record) for record in retriever.corpus)
return BM25Corpus(retriever=retriever, records=records, manifest=manifest)
def score_bm25_corpus(query: str, corpus: BM25Corpus, *, limit: int) -> list[tuple[dict[str, object], float]]:
"""Score a query against a loaded BM25 corpus."""
if corpus.retriever is None or not corpus.records:
return []
k = min(limit, len(corpus.records))
documents, scores = corpus.retriever.retrieve(
bm25s.tokenize(query, show_progress=False),
corpus=list(corpus.records),
k=k,
show_progress=False,
)
results: list[tuple[dict[str, object], float]] = []
for document, score in zip(documents[0], scores[0], strict=True):
score_value = float(score)
if score_value <= 0:
continue
results.append((dict(document), score_value))
return results
async def fetch_bm25_corpus_records(session: AsyncSession) -> tuple[list[dict[str, object]], list[str]]:
"""Fetch persistable BM25 corpus records and their matching index texts from the database.
search_text is only needed to build the index, so it is returned separately instead of
being persisted into the corpus records, which would double the corpus size.
"""
statement = (
select(
EbookChunk.id.label("chunk_id"),
EbookChunk.text.label("text"),
EbookSource.id.label("source_id"),
EbookSource.title.label("source_title"),
EbookSource.author.label("source_author"),
EbookChapter.title.label("chapter_title"),
EbookChunk.page_label.label("page_label"),
EbookChunk.search_text.label("bm25_text"),
)
.select_from(EbookChunk)
.join(EbookSource, EbookSource.id == EbookChunk.source_id)
.outerjoin(EbookChapter, EbookChapter.id == EbookChunk.chapter_id)
.order_by(EbookChunk.id)
)
records: list[dict[str, object]] = []
texts: list[str] = []
for row in (await session.execute(statement)).mappings():
record = dict(row)
texts.append(str(record.pop("bm25_text")))
records.append(record)
return records, texts
async def corpus_last_updated_at(session: AsyncSession) -> datetime | None:
"""Return the latest source/chapter/chunk update timestamp relevant to BM25 text."""
update_times = union_all(
select(func.max(EbookSource.updated).label("updated")),
select(func.max(EbookChapter.updated).label("updated")),
select(func.max(EbookChunk.updated).label("updated")),
).subquery()
return await session.scalar(select(func.max(update_times.c.updated)))
def write_bm25_corpus(
index_path: Path,
records: list[dict[str, object]],
texts: list[str],
manifest: BM25Manifest,
) -> None:
"""Write a BM25 corpus generation and publish it through the current symlink."""
index_path.mkdir(parents=True, exist_ok=True)
generations_path = index_path / "generations"
generations_path.mkdir(exist_ok=True)
generation_path = next_bm25_generation_path(generations_path, manifest.created_at)
current_path = index_path / "current"
next_current_path = index_path / f".current.{generation_path.name}.tmp"
try:
generation_path.mkdir()
# Empty corpora publish a manifest-only generation so startup succeeds before any chunks exist.
if records:
retriever = bm25s.BM25()
retriever.index(bm25s.tokenize(texts, show_progress=False), show_progress=False)
retriever.save(generation_path, corpus=records, show_progress=False)
write_bm25_manifest(generation_path, manifest)
next_current_path.unlink(missing_ok=True)
next_current_path.symlink_to(generation_path, target_is_directory=True)
next_current_path.replace(current_path)
except Exception:
next_current_path.unlink(missing_ok=True)
shutil.rmtree(generation_path, ignore_errors=True)
raise
def read_bm25_manifest(index_path: Path) -> BM25Manifest | None:
"""Read the BM25 manifest if it exists and is valid."""
manifest_path = get_current_bm25_index(index_path) / MANIFEST_NAME
if not manifest_path.exists():
return None
body = json.loads(manifest_path.read_text(encoding="utf-8"))
return BM25Manifest(
created_at=datetime.fromisoformat(str(body["created_at"])),
db_updated_at=datetime.fromisoformat(str(body["db_updated_at"])) if body.get("db_updated_at") else None,
chunk_count=int(body["chunk_count"]),
)
def write_bm25_manifest(index_path: Path, manifest: BM25Manifest) -> None:
"""Write the BM25 manifest to an index directory."""
body = {
"created_at": manifest.created_at.isoformat(),
"db_updated_at": manifest.db_updated_at.isoformat() if manifest.db_updated_at else None,
"chunk_count": manifest.chunk_count,
}
(index_path / MANIFEST_NAME).write_text(json.dumps(body, indent=2, sort_keys=True), encoding="utf-8")
def bm25_index_exists(index_path: Path, manifest: BM25Manifest | None) -> bool:
"""Return whether a usable persisted BM25 index exists."""
active_index_path = get_current_bm25_index(index_path)
if manifest is None or not active_index_path.is_dir():
return False
if manifest.chunk_count == 0:
return True
return all((active_index_path / file_name).exists() for file_name in REQUIRED_INDEX_FILES)
def next_bm25_generation_path(generations_path: Path, created_at: datetime) -> Path:
"""Return an unused dated BM25 generation path."""
base_name = created_at.astimezone(UTC).strftime("%Y%m%dT%H%M%S.%fZ")
generation_path = generations_path / base_name
suffix = 1
while generation_path.exists():
generation_path = generations_path / f"{base_name}.{suffix}"
suffix += 1
return generation_path
-145
View File
@@ -1,145 +0,0 @@
"""Configuration for the EPUB search app."""
from __future__ import annotations
from os import getenv
from typing import Annotated, Self
from pydantic import AliasChoices, Field, field_validator, model_validator
from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
def normalize_embedding_alias(model: str) -> str:
"""Normalize a supported embedding alias to its provider model name."""
aliases = {
"Qwen3-Embedding-0.6B": "qwen3-embedding-0.6b",
"Qwen3-Embedding-4B": "qwen3-embedding-4b",
"Qwen3-Embedding-8B": "qwen3-embedding-8b",
"Qwen/Qwen3-Embedding-0.6B": "qwen3-embedding-0.6b",
"Qwen/Qwen3-Embedding-4B": "qwen3-embedding-4b",
"Qwen/Qwen3-Embedding-8B": "qwen3-embedding-8b",
"qwen3-embedding:0.6b": "qwen3-embedding-0.6b",
"qwen3-embedding:4b": "qwen3-embedding-4b",
"qwen3-embedding:8b": "qwen3-embedding-8b",
"qwen3-embedding-0.6b": "qwen3-embedding-0.6b",
"qwen3-embedding-4b": "qwen3-embedding-4b",
"qwen3-embedding-8b": "qwen3-embedding-8b",
}
standard_model = aliases.get(model)
if standard_model is None:
error = f"Embedding model {model} is not supported. Supported models are {aliases.keys()}"
raise ValueError(error)
return standard_model
def normalize_embedding_model(default: str = "qwen3-embedding-0.6b") -> str:
"""Normalize the configured embedding alias to its provider model name."""
return normalize_embedding_alias(getenv("EBOOK_SEARCH_EMBEDDING_MODEL", default))
class RerankConfig(BaseSettings):
"""vLLM reranker settings."""
model_config = SettingsConfigDict(env_prefix="EBOOK_SEARCH_RERANK_", frozen=True, protected_namespaces=())
enabled: bool = True
base_url: str = "http://192.168.90.25:8001"
model: str = "qwen3-reranker-06b"
candidates: int = 24
timeout_seconds: float = 30.0
score_weight: float = 0.7
hybrid_weight: float = 0.3
class EbookSearchConfig(BaseSettings):
"""Runtime settings for EPUB search."""
model_config = SettingsConfigDict(
env_prefix="EBOOK_SEARCH_",
frozen=True,
populate_by_name=True,
protected_namespaces=(),
)
rerank: RerankConfig = Field(default_factory=RerankConfig)
top_k: int = 12
library_paths: Annotated[tuple[str, ...], NoDecode] = ()
chunk_tokens: int = 700
chunk_overlap: int = 100
vllm_base_url: str = "https://ollama.com/v1"
vllm_api_key: str = Field(
default="not-needed",
validation_alias=AliasChoices("EBOOK_SEARCH_VLLM_API_KEY", "OLLAMA_API_KEY"),
)
chat_model: str = "deepseek-v4-flash"
answer_enabled: bool = True
embedding_base_url: str = "http://192.168.90.25:8000/v1"
embedding_api_key: str = "not-needed"
embedding_model: str = "qwen3-embedding-0.6b"
embedding_batch_size: int = 32
embedding_timeout_seconds: float = 60.0
chat_timeout_seconds: float = 60.0
vector_candidate_multiplier: int = 4
bm25_candidate_limit: int = 120
rrf_rank_constant: int = 60
min_retrieval_confidence: float = 0.0
validate_citations_enabled: bool = True
bm25_index_dir: str = ".ebook_search_bm25"
bm25_refresh_delay_seconds: int = 60
protected_phrase_max_candidates_per_book: int = 5000
protected_phrase_llm_candidates_per_book: int = 500
protected_phrase_extraction_workers: int = 16
phrase_judge_book_workers: int = 20
phrase_judge_phrase_workers: int = 100
protected_phrase_confidence_threshold: float = 0.80
phrase_matching_enabled: bool = True
phrase_hit_boost: float = 0.25
phrase_min_tokens: int = 2
phrase_max_tokens: int = 5
phrase_max_entity_tokens: int = 8
phrase_raw_ngram_min_count: int = 2
phrase_raw_count_score_threshold: int = 3
phrase_raw_count_high_score_threshold: int = 10
phrase_chapter_count_score_threshold: int = 2
phrase_chapter_count_high_score_threshold: int = 5
phrase_target_protected_per_book: int = 100
phrase_default_allow_nested: bool = False
phrase_default_suppress_children: bool = True
@field_validator("library_paths", mode="before")
@classmethod
def split_library_paths(cls, value: object) -> object:
"""Split a colon-separated library path string into a tuple of paths."""
if isinstance(value, str):
return tuple(path for path in value.split(":") if path)
return value
@field_validator("embedding_model")
@classmethod
def normalize_embedding(cls, value: str) -> str:
"""Normalize the configured embedding alias to its provider model name."""
return normalize_embedding_alias(value)
@model_validator(mode="after")
def validate_runtime_consistency(self) -> Self:
"""Reject configurations that cannot serve the features they enable."""
if not self.embedding_base_url.strip():
msg = "embedding_base_url must be set"
raise ValueError(msg)
if self.answer_enabled and (not self.vllm_base_url.strip() or not self.chat_model.strip()):
msg = "answer_enabled requires vllm_base_url and chat_model to be set"
raise ValueError(msg)
if self.rerank.enabled and not self.rerank.base_url.strip():
msg = "rerank.enabled requires rerank.base_url to be set"
raise ValueError(msg)
return self
def load_rerank_config() -> RerankConfig:
"""Load reranker config from environment variables."""
return RerankConfig()
def load_config() -> EbookSearchConfig:
"""Load EPUB search config from environment variables."""
return EbookSearchConfig()
-49
View File
@@ -1,49 +0,0 @@
FROM python:3.14-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1 \
APP_DIR=/home/richie/dotfiles \
EBOOK_SEARCH_HOST=0.0.0.0 \
EBOOK_SEARCH_PORT=8070 \
EBOOK_SEARCH_BM25_INDEX_DIR=/data/bm25
WORKDIR ${APP_DIR}
RUN apt-get update \
&& apt-get install -y --no-install-recommends build-essential curl \
&& rm -rf /var/lib/apt/lists/*
COPY pyproject.toml README.md LICENSE ./
COPY python ./python
RUN python -m pip install --upgrade pip \
&& python -m pip install \
"alembic" \
"beautifulsoup4" \
"bm25s" \
"ebooklib" \
"fastapi" \
"httpx" \
"jinja2" \
"pgvector" \
"psycopg[binary]" \
"pydantic" \
"pydantic-settings" \
"python-multipart" \
"sqlalchemy[asyncio]" \
"tiktoken" \
"typer" \
"uvicorn[standard]" \
"yake" \
&& python -m pip install --no-deps --editable "${APP_DIR}"
RUN useradd --create-home --uid 10001 app \
&& mkdir -p /data \
&& chown -R app:app /home/richie /data
USER app
EXPOSE 8070
CMD ["sh", "-c", "exec python -m python.ebook_search.api.main --host \"${EBOOK_SEARCH_HOST}\" --port \"${EBOOK_SEARCH_PORT}\" --log-level \"${EBOOK_SEARCH_LOG_LEVEL:-INFO}\""]
-40
View File
@@ -1,40 +0,0 @@
# Ebook Search Docker
Run the EPUB search app against the existing Postgres database on `jeeves`:
```sh
ebook-search-containers start --library-path /path/to/epubs --build
```
All ebook-search Docker files live in this directory:
- `Dockerfile`
- `docker-compose.yml`
- `containers.py`
- `container.py`
The app listens on `http://localhost:8070`.
Useful lifecycle commands:
```sh
ebook-search-containers build
ebook-search-containers start --library-path /path/to/epubs
ebook-search-containers logs
ebook-search-containers ps
ebook-search-containers stop
```
Direct compose usage from the repo root:
```sh
docker compose -f python/ebook_search/docker/docker-compose.yml ps
```
The compose service also loads the repo root `.env` into the container via `env_file`.
Mount your EPUB directory by setting `EBOOK_LIBRARY_HOST_PATH` in an env file or on the command line. The container sees it as `/library`, and `EBOOK_SEARCH_LIBRARY_PATHS` is set to `/library` inside the container.
Database connection settings are controlled by `RICHIE_DB`, `RICHIE_HOST`, `RICHIE_PORT`, `RICHIE_USER`, and `RICHIE_PASSWORD`. The default host is `jeeves`.
Startup runs the Richie Alembic migrations automatically after creating the `main` schema and `vector` extension.
-1
View File
@@ -1 +0,0 @@
"""Docker packaging and lifecycle tooling for ebook search."""
-229
View File
@@ -1,229 +0,0 @@
"""Docker container lifecycle management for ebook search."""
from __future__ import annotations
import logging
import os
import subprocess
from pathlib import Path
from typing import Annotated
import typer
from python.common import configure_logger, get_repo_dir
logger = logging.getLogger(__name__)
def get_compose_file() -> Path:
"""Return the path to the docker-compose.yml file."""
return Path(__file__).resolve().with_name("docker-compose.yml")
def compose_base_args() -> list[str]:
"""Return the common docker compose arguments for the ebook search stack."""
return ["compose", "-f", str(get_compose_file())]
def docker_run(
arguments: list[str],
*,
env: dict[str, str] | None = None,
capture_output: bool = False,
) -> subprocess.CompletedProcess[str]:
"""Run docker with repo-root cwd and consistent error handling."""
logger.info("docker %s", " ".join(arguments))
return subprocess.run(
["docker", *arguments],
cwd=get_repo_dir(),
env=env,
text=True,
check=False,
capture_output=capture_output,
)
def compose_env(*, library_path: Path | None = None, port: int | None = None) -> dict[str, str]:
"""Return environment variables passed to docker compose."""
env = os.environ.copy()
if library_path is not None:
resolved_library = library_path.expanduser().resolve()
if not resolved_library.exists():
msg = f"EPUB library path does not exist: {resolved_library}"
raise FileNotFoundError(msg)
env["EBOOK_LIBRARY_HOST_PATH"] = str(resolved_library)
if port is not None:
env["EBOOK_SEARCH_PORT"] = str(port)
return env
def ensure_compose_file() -> None:
"""Raise if the ebook search compose file is missing."""
if not get_compose_file().is_file():
msg = f"Compose file not found: {get_compose_file()}"
raise FileNotFoundError(msg)
def build_image() -> None:
"""Build the ebook search app image."""
ensure_compose_file()
result = docker_run([*compose_base_args(), "build"])
if result.returncode != 0:
msg = "Failed to build ebook search image"
raise RuntimeError(msg)
def start_stack(
*,
library_path: Path | None = None,
port: int | None = None,
build: bool = False,
) -> None:
"""Start the ebook search Docker compose stack."""
ensure_compose_file()
env = compose_env(library_path=library_path, port=port)
if build:
build_image()
result = docker_run(
[*compose_base_args(), "up", "-d"],
env=env,
)
if result.returncode != 0:
msg = f"Ebook search stack failed to start with code {result.returncode}"
raise RuntimeError(msg)
logger.info("Ebook search started.")
def stop_stack(
*,
volumes: bool = False,
) -> None:
"""Stop and remove ebook search containers."""
ensure_compose_file()
command = [*compose_base_args(), "down"]
if volumes:
command.append("-v")
result = docker_run(command)
if result.returncode != 0:
msg = f"Ebook search stack failed to stop with code {result.returncode}"
raise RuntimeError(msg)
def logs_stack(
*,
service: str | None = None,
tail: int = 100,
follow: bool = False,
) -> str | None:
"""Return recent logs from the ebook search stack."""
ensure_compose_file()
command = [*compose_base_args(), "logs", "--tail", str(tail)]
if follow:
command.append("--follow")
if service:
command.append(service)
result = docker_run(command, capture_output=not follow)
if result.returncode != 0:
return None
if follow:
return ""
return result.stdout + result.stderr
def ps_stack() -> str | None:
"""Return docker compose ps output for the ebook search stack."""
ensure_compose_file()
result = docker_run([*compose_base_args(), "ps"], capture_output=True)
if result.returncode != 0:
return None
return result.stdout + result.stderr
app = typer.Typer(help="Ebook search Docker container management.", no_args_is_help=True)
@app.command()
def build() -> None:
"""Build the ebook search Docker image."""
build_image()
@app.command()
def start(
library_path: Annotated[Path | None, typer.Option(help="Override host path containing EPUB files.")] = None,
port: Annotated[int | None, typer.Option(help="Override host port for the web UI.")] = None,
*,
build: Annotated[bool, typer.Option("--build", help="Build the image before starting.")] = False,
log_level: Annotated[str, typer.Option(help="Log level.")] = "INFO",
) -> None:
"""Start the ebook search container."""
configure_logger(log_level)
start_stack(
library_path=library_path,
port=port,
build=build,
)
@app.command()
def stop(
*,
volumes: Annotated[bool, typer.Option("--volumes", help="Also remove ebook search data volumes.")] = False,
log_level: Annotated[str, typer.Option(help="Log level.")] = "INFO",
) -> None:
"""Stop and remove ebook search containers."""
configure_logger(log_level)
stop_stack(volumes=volumes)
@app.command()
def restart(
library_path: Annotated[Path | None, typer.Option(help="Override host path containing EPUB files.")] = None,
port: Annotated[int | None, typer.Option(help="Override host port for the web UI.")] = None,
*,
build: Annotated[bool, typer.Option("--build", help="Build the image before starting.")] = False,
log_level: Annotated[str, typer.Option(help="Log level.")] = "INFO",
) -> None:
"""Restart the ebook search stack."""
configure_logger(log_level)
stop_stack()
start_stack(
library_path=library_path,
port=port,
build=build,
)
@app.command()
def logs(
service: Annotated[str | None, typer.Option(help="Service name, or omit for all services.")] = None,
tail: Annotated[int, typer.Option(help="Number of recent log lines.")] = 100,
*,
follow: Annotated[bool, typer.Option("--follow", "-f", help="Follow logs.")] = False,
) -> None:
"""Show recent ebook search container logs."""
output = logs_stack(service=service, tail=tail, follow=follow)
if output is None:
typer.echo("No ebook search containers found.")
raise typer.Exit(code=1)
if output:
typer.echo(output)
@app.command("ps")
def ps() -> None:
"""Show ebook search container status."""
output = ps_stack()
if output is None:
typer.echo("No ebook search containers found.")
raise typer.Exit(code=1)
typer.echo(output)
def cli() -> None:
"""Typer entry point."""
app()
if __name__ == "__main__":
cli()
@@ -1,36 +0,0 @@
name: ebook-search
services:
ebook-search:
build:
context: ../../..
dockerfile: python/ebook_search/docker/Dockerfile
image: ebook-search:latest
restart: unless-stopped
ports:
- "${EBOOK_SEARCH_PORT:-8070}:8070"
extra_hosts:
- "jeeves:192.168.90.40"
env_file:
- ../../../.env
environment:
EBOOK_SEARCH_HOST: "0.0.0.0"
EBOOK_SEARCH_PORT: "8070"
EBOOK_SEARCH_LIBRARY_PATHS: "/library"
EBOOK_SEARCH_BM25_INDEX_DIR: "/data/bm25"
volumes:
- "${EBOOK_LIBRARY_HOST_PATH:-/home/richie/ebooks}:/library:ro"
- ebook-search-data:/data
healthcheck:
test:
[
"CMD-SHELL",
"curl -fsS http://127.0.0.1:8070/health >/dev/null || exit 1",
]
interval: 30s
timeout: 5s
retries: 5
start_period: 30s
volumes:
ebook-search-data:
-175
View File
@@ -1,175 +0,0 @@
"""Embedding model helpers."""
from __future__ import annotations
import logging
from dataclasses import dataclass
from typing import TYPE_CHECKING
from sqlalchemy import func, select
from sqlalchemy.dialects.postgresql import insert
from python.ebook_search.llm_interface import request_embeddings
from python.orm.richie import (
EbookChunk,
EbookChunkEmbedding1024,
EbookChunkEmbedding2560,
EbookChunkEmbedding4096,
EbookEmbeddingModel,
)
logger = logging.getLogger(__name__)
if TYPE_CHECKING:
from collections.abc import Sequence
import httpx
from sqlalchemy.ext.asyncio import AsyncSession
from python.ebook_search.config import EbookSearchConfig
MODEL_DIMENSIONS = {
"qwen3-embedding-0.6b": 1024,
"qwen3-embedding-4b": 2560,
"qwen3-embedding-8b": 4096,
}
def get_embedding_table(
dimension: int,
) -> type[EbookChunkEmbedding1024 | EbookChunkEmbedding2560 | EbookChunkEmbedding4096]:
"""Return the embedding table mapped to an embedding dimension."""
embedding_tables = {
1024: EbookChunkEmbedding1024,
2560: EbookChunkEmbedding2560,
4096: EbookChunkEmbedding4096,
}
table = embedding_tables.get(dimension)
if not table:
msg = f"Embedding dimension {dimension} is not supported"
raise ValueError(msg)
return table
@dataclass(frozen=True)
class EmbeddingModelStats:
"""Embedding coverage for one model."""
model_name: str
dimension: int
embedded_chunks: int
total_chunks: int
@property
def missing_chunks(self) -> int:
"""Return chunks missing this embedding model."""
return max(self.total_chunks - self.embedded_chunks, 0)
async def embed_texts(
client: httpx.AsyncClient,
texts: Sequence[str],
config: EbookSearchConfig,
) -> list[list[float]]:
"""Embed text with the configured vLLM embedding model."""
logger.info(
"ebook_embed_request_start base_url=%s model=%s count=%s",
config.embedding_base_url,
config.embedding_model,
len(texts),
)
vectors = await request_embeddings(client, texts, config)
expected_dimension = MODEL_DIMENSIONS[config.embedding_model]
for vector in vectors:
if len(vector) != expected_dimension:
msg = f"Expected {expected_dimension} dimensions, got {len(vector)}"
raise ValueError(msg)
logger.info(
"ebook_embed_request_complete model=%s count=%s dimension=%s",
config.embedding_model,
len(vectors),
expected_dimension,
)
return vectors
async def embed_query(client: httpx.AsyncClient, query: str, config: EbookSearchConfig) -> list[float]:
"""Embed a search query with the Qwen retrieval instruction."""
instructed_query = f"Instruct: Retrieve relevant passages for the query.\nQuery: {query}"
return (await embed_texts(client, [instructed_query], config))[0]
async def ensure_embedding_models(session: AsyncSession) -> None:
"""Ensure supported embedding model rows exist."""
for name, dimension in MODEL_DIMENSIONS.items():
existing = await session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == name))
if existing is None:
session.add(EbookEmbeddingModel(name=name, dimension=dimension, is_default=name == "qwen3-embedding-0.6b"))
logger.info("ebook_embedding_model_created model=%s dimension=%s", name, dimension)
await session.flush()
async def embedding_model_stats(session: AsyncSession) -> list[EmbeddingModelStats]:
"""Return embedding coverage counts for every supported model."""
total_chunks = await session.scalar(select(func.count(EbookChunk.id))) or 0
models = {
model.name: model
for model in await session.scalars(
select(EbookEmbeddingModel)
.where(EbookEmbeddingModel.name.in_(MODEL_DIMENSIONS))
.order_by(EbookEmbeddingModel.name)
)
}
stats: list[EmbeddingModelStats] = []
for model_name, dimension in MODEL_DIMENSIONS.items():
model = models.get(model_name)
embedded_chunks = 0
if model is not None:
table = get_embedding_table(dimension)
embedded_chunks = await session.scalar(select(func.count(table.id)).where(table.model_id == model.id)) or 0
stats.append(
EmbeddingModelStats(
model_name=model_name,
dimension=dimension,
embedded_chunks=embedded_chunks,
total_chunks=total_chunks,
)
)
return stats
async def embed_missing_chunks(session: AsyncSession, client: httpx.AsyncClient, config: EbookSearchConfig) -> int:
"""Embed chunks missing embeddings for the configured model."""
await ensure_embedding_models(session)
model = await session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == config.embedding_model))
if model is None:
supported_models = ", ".join(MODEL_DIMENSIONS)
msg = f"Unknown embedding model: {config.embedding_model}. Supported models: {supported_models}"
raise ValueError(msg)
table = get_embedding_table(model.dimension)
chunks = list(
await session.scalars(
select(EbookChunk)
.outerjoin(table, (table.chunk_id == EbookChunk.id) & (table.model_id == model.id))
.where(table.id.is_(None))
.order_by(EbookChunk.id)
.limit(config.embedding_batch_size)
)
)
if not chunks:
logger.info("ebook_embed_missing_none model=%s", config.embedding_model)
return 0
logger.info("ebook_embed_missing_batch_start model=%s count=%s", config.embedding_model, len(chunks))
vectors = await embed_texts(client, [chunk.text for chunk in chunks], config)
rows = [
{"chunk_id": chunk.id, "model_id": model.id, "embedding": vector}
for chunk, vector in zip(chunks, vectors, strict=True)
]
statement = insert(table).values(rows).on_conflict_do_nothing(index_elements=["chunk_id", "model_id"])
await session.execute(statement)
await session.flush()
logger.info("ebook_embed_missing_batch_complete model=%s count=%s", config.embedding_model, len(rows))
return len(rows)

Some files were not shown because too many files have changed in this diff Show More