feat(settings): add configuration to disable YAML schema detection for specific workflow files

Move protected phrase detection into the retrieval gather so it runs
concurrently with vector and BM25 candidates instead of sequentially
before them. Make the search API accept real bool form fields for
rerank/phrase_matching, gate phrase matching on both the request and
config kill switch, and reflow log f-strings for readability.

.dockerignore

@@ -0,0 +1,28 @@
+.git
+.direnv
+.mypy_cache
+.pytest_cache
+.ruff_cache
+.venv
+**/.venv
+.env
+.cache
+.claude
+.coverage
+.vscode
+.stfolder
+.literotica_data
+esphome
+htmlcov
+data
+ebooks
+__pycache__
+**/__pycache__
+*.pyc
+*.pyo
+.ebook_search_bm25
+result
+result-*
+*.egg-info
+dist
+build

.github/workflows/build_systems.yml

@@ -17,12 +17,11 @@ jobs:
           - "bob"
           - "brain"
           - "jeeves"
-          - "leviathan"
           - "rhapsody-in-green"
     continue-on-error: true
     steps:
       - uses: actions/checkout@v4
       - name: Build default package
-        run: "nixos-rebuild build --flake ./#${{ matrix.system }}"
+        run: "nixos-rebuild build --accept-flake-config --flake ./#${{ matrix.system }}"
       - name: copy to nix-cache
         run: nix copy --accept-flake-config --to unix:///host-nix/var/nix/daemon-socket/socket .#nixosConfigurations.${{ matrix.system }}.config.system.build.toplevel

.github/workflows/fix_eval_warnings.yml

@@ -1,30 +0,0 @@
-name: fix_eval_warnings
-on:
-  workflow_run:
-    workflows: ["build_systems"]
-    types: [completed]
-
-jobs:
-  check-warnings:
-    if: >-
-      github.event.workflow_run.conclusion != 'cancelled' &&
-      github.event.workflow_run.head_branch == 'main' &&
-      (github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'schedule')
-    runs-on: self-hosted
-    permissions:
-      contents: write
-      pull-requests: write
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Fix eval warnings
-        env:
-          GH_TOKEN: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
-        run: >-
-          nix develop .#devShells.x86_64-linux.default -c
-          python -m python.eval_warnings.main
-          --run-id "${{ github.event.workflow_run.id }}"
-          --repo "${{ github.repository }}"
-          --ollama-url "${{ secrets.OLLAMA_URL }}"
-          --run-url "${{ github.event.workflow_run.html_url }}"

.github/workflows/merge_flake_lock_update.yml

@@ -6,24 +6,18 @@ on:
 
 jobs:
   merge:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
 
     permissions:
       contents: write
       pull-requests: write
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
       - name: merge_flake_lock_update
-        run: |
-          pr_number=$(gh pr list --state open --author RichieCahill --label flake_lock_update --json number --jq '.[0].number')
-          echo "pr_number=$pr_number" >> $GITHUB_ENV
-          if [ -n "$pr_number" ]; then
-            gh pr merge "$pr_number" --rebase
-          else
-            echo "No open PR found with label flake_lock_update"
-          fi
+        run: >-
+          nix develop .#devShells.x86_64-linux.default -c
+          python -m python.gitea_flake_lock merge
+          --repo "${{ github.repository }}"
         env:
-          GITHUB_TOKEN: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+          GITEA_URL: https://gitea.tmmworkshop.com

.github/workflows/pytest.yml

@@ -1,13 +1,13 @@
 name: pytest
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - main
   pull_request:
     branches:
       - main
-  merge_group:
 
 jobs:
   pytest:

.github/workflows/test_ebook_search.yml

@@ -0,0 +1,26 @@
+name: test ebook search
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+env:
+  UV_PYTHON_DOWNLOADS: never
+  UV_CACHE_DIR: /var/cache/uv
+  UV_LINK_MODE: copy
+
+jobs:
+  test-ebook-search:
+    runs-on: self-hosted
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install dependencies
+        run: nix develop .#devShells.x86_64-linux.ebook-search -c uv sync --locked --project python/ebook_search/docker
+      - name: Run ebook search tests
+        run: nix develop .#devShells.x86_64-linux.ebook-search -c uv run --project python/ebook_search/docker --no-sync pytest tests/ebook_search --override-ini addopts="-n auto -ra"

.github/workflows/update-flake-lock.yml

@@ -6,18 +6,21 @@ on:
 
 jobs:
   lockfile:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
+    permissions:
+      actions: write
+      contents: write
+      pull-requests: write
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - name: Install Nix
-        uses: DeterminateSystems/nix-installer-action@main
       - name: Update flake.lock
-        uses: DeterminateSystems/update-flake-lock@main
-        with:
-          token: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
-          pr-title: "Update flake.lock"
-          pr-labels: |
-            dependencies
-            automated
-            flake_lock_update
+        run: nix flake update
+      - name: Create or update flake.lock PR
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+          GITEA_URL: https://gitea.tmmworkshop.com
+        run: >-
+          nix develop .#devShells.x86_64-linux.default -c
+          python -m python.gitea_flake_lock update
+          --repo "${{ github.repository }}"

.gitignore

@@ -169,3 +169,10 @@ test.*
 # Frontend build output
 frontend/dist/
 frontend/node_modules/
+
+# data from testing llms
+data/*
+.ebook_search_bm25
+
+# gems data
+.gems

.sops.yaml

@@ -7,7 +7,6 @@ keys:
   - &system_bob age1q47vup0tjhulkg7d6xwmdsgrw64h4ax3la3evzqpxyy4adsmk9fs56qz3y # cspell:disable-line
   - &system_brain age1jhf7vm0005j60mjq63696frrmjhpy8kpc2d66mw044lqap5mjv4snmwvwm # cspell:disable-line
   - &system_jeeves age13lmqgc3jvkyah5e3vcwmj4s5wsc2akctcga0lpc0x8v8du3fxprqp4ldkv # cspell:disable-line
-  - &system_leviathan age1l272y8udvg60z7edgje42fu49uwt4x2gxn5zvywssnv9h2krms8s094m4k # cspell:disable-line
   - &system_rhapsody age1ufnewppysaq2wwcl4ugngjz8pfzc5a35yg7luq0qmuqvctajcycs5lf6k4 # cspell:disable-line
 
 creation_rules:
@@ -18,5 +17,4 @@ creation_rules:
           - *system_bob
           - *system_brain
           - *system_jeeves
-          - *system_leviathan
           - *system_rhapsody

.vscode/settings.json

@@ -40,7 +40,6 @@
     "cgroupdriver",
     "charliermarsh",
     "Checkpointing",
-    "cloudflared",
     "codellama",
     "codezombiech",
     "compactmode",
@@ -72,6 +71,7 @@
     "ehci",
     "emerg",
     "endlessh",
+    "ents",
     "errorlens",
     "esbenp",
     "esphome",
@@ -173,6 +173,8 @@
     "Networkd",
     "networkmanager",
     "newtabpage",
+    "ngram",
+    "ngrams",
     "nixfmt",
     "nixos",
     "nixpkgs",
@@ -204,6 +206,7 @@
     "peerconnection",
     "PESKYFOX",
     "PGID",
+    "pgvector",
     "pipewire",
     "pkgs",
     "plugdev",
@@ -242,6 +245,7 @@
     "referer",
     "REFERERS",
     "relatime",
+    "rerank",
     "Rhosts",
     "ripgrep",
     "roboto",
@@ -297,7 +301,9 @@
     "uiprotect",
     "uitour",
     "unifi",
+    "unjudged",
     "unrar",
+    "unstorable",
     "unsubmitted",
     "uptimekuma",
     "urlbar",
@@ -308,6 +314,7 @@
     "usernamehw",
     "userprefs",
     "vaninventory",
+    "vdev",
     "vfat",
     "victron",
     "virt",
@@ -324,6 +331,7 @@
     "xcursorgen",
     "xdist",
     "xhci",
+    "yake",
     "yazi",
     "yubikey",
     "yubioath",

AGENTS.md

@@ -1,12 +0,0 @@
-## Dev environment tips
-
-- use treefmt to format all files
-- make python code ruff compliant
-- use pytest to test python code
-- always use the minimum amount of complexity
-- if judgment calls are easy to reverse make them. if not ask me first
-- Match existing code style.
-- Use builtin helpers getenv() over os.environ.get.
-- Prefer single-purpose functions over “do everything” helpers.
-- Avoid compatibility branches like PG_USER and POSTGRESQL_URL unless requested.
-- Keep helpers only if reused or they simplify the code otherwise inline.

README.md

@@ -1 +1,51 @@
 # dotfiles
+
+## Installer ISO
+
+Build a bootable NixOS ISO with the installer preinstalled:
+
+```sh
+nix build .#iso
+```
+
+Write `result/iso/nixos-zfs-installer.iso` to a USB stick (for example with `dd`) or boot it in a VM. The image is the minimal NixOS installation CD with ZFS enabled and `nixos-installer` on `PATH`. SSH is enabled and the `nixos` and `root` accounts use the password `nixos`, so you can also run the installer remotely. Once booted:
+
+```sh
+sudo nixos-installer
+```
+
+The ISO bundles the `.#installer-nixos` package, a variant of the binary that keeps its Nix store linkage instead of being patched for foreign distributions.
+
+## Installer binary
+
+Build the self-contained installer executable with:
+
+```sh
+nix build .#installer
+```
+
+The flake package (defined in `python/installer/package.nix`) uses the Python builder in `python/installer/build.py`, which stages only the installer modules before running PyInstaller. You can also call it directly when `pyinstaller` and `patchelf` are on `PATH`:
+
+```sh
+python -m python.installer.build --output ./nixos-installer
+```
+
+Copy `result/bin/nixos-installer` to the installer USB stick and run it as root from the NixOS live environment:
+
+```sh
+sudo ./nixos-installer
+```
+
+Validate the live environment first with:
+
+```sh
+./nixos-installer --check
+```
+
+Paste a value into the TUI encryption password field to enable LUKS during install, or set `ENCRYPT_KEY`:
+
+```sh
+sudo env ENCRYPT_KEY='change-me' ./nixos-installer
+```
+
+The binary bundles the Python runtime and only the installer modules it imports. It still expects the NixOS installer environment to provide system install tools such as `parted`, `zfs`, `zpool`, `cryptsetup`, `nixos-generate-config`, and `nixos-install`.

common/global/default.nix

@@ -23,7 +23,10 @@
   boot = {
     tmp.useTmpfs = true;
     kernelPackages = lib.mkDefault pkgs.linuxPackages_6_12;
-    zfs.package = lib.mkDefault pkgs.zfs_2_4;
+    zfs = {
+      package = lib.mkDefault pkgs.zfs_2_4;
+      forceImportRoot = lib.mkDefault false;
+    };
   };
 
   hardware.enableRedistributableFirmware = true;
@@ -37,10 +40,17 @@
 
   nixpkgs = {
     overlays = builtins.attrValues outputs.overlays;
-    config.allowUnfree = true;
+    config = {
+      allowUnfree = true;
+      permittedInsecurePackages = [
+        "openssl-1.1.1w" # This is for discord-canary
+      ];
+    };
   };
 
   services = {
+    dbus.implementation = "dbus";
+
     # firmware update
     fwupd.enable = true;
 

common/global/nix.nix

@@ -34,6 +34,7 @@ in
       warn-dirty = false;
       flake-registry = ""; # disable global flake registries
       connect-timeout = 10;
+      download-buffer-size = 536870912;
       fallback = true;
     };
 

common/optional/monitoring-agent.nix

@@ -0,0 +1,256 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  monitoringInterface = "ztwfunumly";
+  nodeTextfileDir = "/var/lib/prometheus-node-exporter-textfile";
+
+  mkProcessNameTemplate =
+    perPid: template: if perPid then "${template}:{{.PID}}:{{.StartTime}}" else template;
+
+  mkProcessMatchers = perPid: [
+    {
+      name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Module}}";
+      cmdline = [ "^/nix/store[^ ]*/bin/python[^ ]* -m (?P<Module>[^ ]+)" ];
+    }
+    {
+      name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Wrapped}}";
+      cmdline = [
+        "^/nix/store[^ ]*/bin/python[^ ]* /nix/store[^ ]*/bin/\\.?(?P<Wrapped>[^ /]+?)(?:-wrapped)?(?:\\s|$)"
+      ];
+    }
+    {
+      name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Wrapped}}";
+      cmdline = [
+        "^/nix/store[^ ]*/bin/node /nix/store[^ ]*-(?P<Wrapped>[A-Za-z0-9._+-]+)-[0-9][^ /]*/"
+      ];
+    }
+    {
+      name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Wrapped}}";
+      cmdline = [ "^/nix/store[^ ]*/(?:bin/|lib/[^ ]*/)?\\.?(?P<Wrapped>[^ /]+?)(?:-wrapped)?(?:\\s|$)" ];
+    }
+    {
+      name = mkProcessNameTemplate perPid "{{.Username}}:{{.ExeBase}}";
+      cmdline = [ ".+" ];
+    }
+  ];
+
+  perPidConfig = pkgs.writeText "process-exporter-per-pid.yaml" (
+    builtins.toJSON {
+      process_names = mkProcessMatchers true;
+    }
+  );
+
+  zpoolLatencyScript = pkgs.writeShellScript "zpool-latency-exporter" ''
+        set -euo pipefail
+
+        out_dir=${lib.escapeShellArg nodeTextfileDir}
+        host=${lib.escapeShellArg config.networking.hostName}
+        tmp_file="$(mktemp "$out_dir/zpool.prom.XXXXXX")"
+        trap 'rm -f "$tmp_file"' EXIT
+
+        pools="$(zpool list -H -o name | paste -sd, -)"
+
+        cat >"$tmp_file" <<'EOF'
+    # HELP zpool_iostat_total_wait_read_ns Average total read wait time reported by zpool iostat.
+    # TYPE zpool_iostat_total_wait_read_ns gauge
+    # HELP zpool_iostat_total_wait_write_ns Average total write wait time reported by zpool iostat.
+    # TYPE zpool_iostat_total_wait_write_ns gauge
+    # HELP zpool_iostat_disk_wait_read_ns Average disk read wait time reported by zpool iostat.
+    # TYPE zpool_iostat_disk_wait_read_ns gauge
+    # HELP zpool_iostat_disk_wait_write_ns Average disk write wait time reported by zpool iostat.
+    # TYPE zpool_iostat_disk_wait_write_ns gauge
+    # HELP zpool_iostat_syncq_wait_read_ns Average synchronous queue read wait time reported by zpool iostat.
+    # TYPE zpool_iostat_syncq_wait_read_ns gauge
+    # HELP zpool_iostat_syncq_wait_write_ns Average synchronous queue write wait time reported by zpool iostat.
+    # TYPE zpool_iostat_syncq_wait_write_ns gauge
+    # HELP zpool_iostat_asyncq_wait_read_ns Average asynchronous queue read wait time reported by zpool iostat.
+    # TYPE zpool_iostat_asyncq_wait_read_ns gauge
+    # HELP zpool_iostat_asyncq_wait_write_ns Average asynchronous queue write wait time reported by zpool iostat.
+    # TYPE zpool_iostat_asyncq_wait_write_ns gauge
+    EOF
+
+        zpool iostat -Hplvy -y 1 1 | awk -F '\t' -v host="$host" -v pools="$pools" '
+          function esc(str, out) {
+            out = str
+            gsub(/\\/, "\\\\", out)
+            gsub(/"/, "\\\"", out)
+            return out
+          }
+
+          function emit(metric, pool, vdev, value) {
+            if (value == "" || value == "-") {
+              return
+            }
+
+            printf "%s{host=\"%s\",pool=\"%s\",vdev=\"%s\"} %s\n",
+              metric,
+              esc(host),
+              esc(pool),
+              esc(vdev),
+              value
+          }
+
+          BEGIN {
+            split(pools, pool_names, ",")
+            for (idx in pool_names) {
+              if (pool_names[idx] != "") {
+                known_pools[pool_names[idx]] = 1
+              }
+            }
+          }
+
+          NF == 0 {
+            next
+          }
+
+          {
+            row_name = $1
+
+            if (row_name in known_pools) {
+              current_pool = row_name
+              current_vdev = "_pool"
+            } else if (current_pool == "") {
+              next
+            } else {
+              current_vdev = row_name
+            }
+
+            emit("zpool_iostat_total_wait_read_ns", current_pool, current_vdev, $8)
+            emit("zpool_iostat_total_wait_write_ns", current_pool, current_vdev, $9)
+            emit("zpool_iostat_disk_wait_read_ns", current_pool, current_vdev, $10)
+            emit("zpool_iostat_disk_wait_write_ns", current_pool, current_vdev, $11)
+            emit("zpool_iostat_syncq_wait_read_ns", current_pool, current_vdev, $12)
+            emit("zpool_iostat_syncq_wait_write_ns", current_pool, current_vdev, $13)
+            emit("zpool_iostat_asyncq_wait_read_ns", current_pool, current_vdev, $14)
+            emit("zpool_iostat_asyncq_wait_write_ns", current_pool, current_vdev, $15)
+          }
+        ' >>"$tmp_file"
+
+        mv "$tmp_file" "$out_dir/zpool.prom"
+        trap - EXIT
+  '';
+in
+{
+  networking.firewall.interfaces.${monitoringInterface}.allowedTCPPorts = [
+    9100
+    9134
+    9256
+    9257
+    9633
+  ];
+
+  services.prometheus.exporters = {
+    node = {
+      enable = true;
+      enabledCollectors = [
+        "pressure"
+        "processes"
+        "systemd"
+      ];
+      extraFlags = [ "--collector.textfile.directory=${nodeTextfileDir}" ];
+    };
+
+    process = {
+      enable = true;
+      user = "root";
+      group = "root";
+      settings.process_names = mkProcessMatchers false;
+      extraFlags = [
+        "-gather-smaps=false"
+        "-remove-empty-groups=true"
+        "-threads=false"
+      ];
+    };
+
+    smartctl.enable = true;
+    zfs.enable = true;
+  };
+
+  programs.atop = {
+    enable = true;
+    atopService.enable = true;
+    atopRotateTimer.enable = true;
+    atopacctService.enable = true;
+    settings.interval = 30;
+  };
+
+  systemd = {
+    services = {
+      prometheus-process-pid-exporter = {
+        description = "Prometheus process exporter with per-PID naming";
+        wantedBy = [ "multi-user.target" ];
+        after = [ "network.target" ];
+        serviceConfig = {
+          ExecStart = ''
+            ${pkgs.prometheus-process-exporter}/bin/process-exporter \
+              --web.listen-address 0.0.0.0:9257 \
+              --config.path ${perPidConfig} \
+              -children=false \
+              -gather-smaps=false \
+              -remove-empty-groups=true \
+              -threads=false
+          '';
+          User = "root";
+          Group = "root";
+          Restart = "always";
+          WorkingDirectory = "/tmp";
+          CapabilityBoundingSet = [ "" ];
+          DeviceAllow = [ "" ];
+          LockPersonality = true;
+          MemoryDenyWriteExecute = true;
+          NoNewPrivileges = true;
+          PrivateDevices = true;
+          PrivateTmp = true;
+          ProtectClock = true;
+          ProtectControlGroups = true;
+          ProtectHome = true;
+          ProtectHostname = true;
+          ProtectKernelLogs = true;
+          ProtectKernelModules = true;
+          ProtectKernelTunables = true;
+          ProtectSystem = "strict";
+          RemoveIPC = true;
+          RestrictAddressFamilies = [
+            "AF_INET"
+            "AF_INET6"
+          ];
+          RestrictNamespaces = true;
+          RestrictRealtime = true;
+          RestrictSUIDSGID = true;
+          SystemCallArchitectures = "native";
+          UMask = "0077";
+        };
+      };
+
+      zpool-latency-exporter = {
+        description = "Exports ZFS latency metrics for node_exporter textfile collection";
+        after = [ "zfs-import.target" ];
+        requires = [ "zfs-import.target" ];
+        path = [
+          config.boot.zfs.package
+          pkgs.coreutils
+          pkgs.gawk
+        ];
+        serviceConfig = {
+          Type = "oneshot";
+          ExecStart = zpoolLatencyScript;
+        };
+      };
+    };
+
+    timers.zpool-latency-exporter = {
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnBootSec = "2m";
+        OnUnitActiveSec = "60s";
+        Unit = "zpool-latency-exporter.service";
+      };
+    };
+
+    tmpfiles.rules = [ "d ${nodeTextfileDir} 0755 root root - -" ];
+  };
+}

common/optional/syncthing_base.nix

@@ -12,7 +12,7 @@
       brain.id = "SSCGIPI-IV3VYKB-TRNIJE3-COV4T2H-CDBER7F-I2CGHYA-NWOEUDU-3T5QAAN"; # cspell:disable-line
       ipad.id = "KI76T3X-SFUGV2L-VSNYTKR-TSIUV5L-SHWD3HE-GQRGRCN-GY4UFMD-CW6Z6AX"; # cspell:disable-line
       jeeves.id = "ICRHXZW-ECYJCUZ-I4CZ64R-3XRK7CG-LL2HAAK-FGOHD22-BQA4AI6-5OAL6AG"; # cspell:disable-line
-      phone.id = "TBRULKD-7DZPGGZ-F6LLB7J-MSO54AY-7KLPBIN-QOFK6PX-W2HBEWI-PHM2CQI"; # cspell:disable-line
+      phone.id = "JPVQKQW-CFXOJXT-Q5G5F3H-QIDHDRE-GKHPTQB-GXZUQSP-U7FR7F7-INP3AAH"; # cspell:disable-line
       rhapsody-in-green.id = "ASL3KC4-3XEN6PA-7BQBRKE-A7JXLI6-DJT43BY-Q4WPOER-7UALUAZ-VTPQ6Q4"; # cspell:disable-line
     };
   };

common/optional/update.nix

@@ -4,7 +4,7 @@
     flags = [ "--accept-flake-config" ];
     randomizedDelaySec = "1h";
     persistent = true;
-    flake = "github:RichieCahill/dotfiles";
+    flake = "git+https://gitea.tmmworkshop.com/richie/dotfiles?ref=main";
     allowReboot = true;
     dates = "Sat *-*-* 06:00:00";
   };

docs/zfs_failed_root_import_recovry.md

@@ -0,0 +1,76 @@
+# ZFS failed root import recovery
+
+## Fast path
+
+If the machine fails to boot because ZFS refuses to import `root_pool`:
+
+### GRUB
+
+1. At the bootloader menu, select the normal NixOS entry.
+2. Press `e`.
+3. Find the line that starts with `linux`.
+4. Append this to the end of that line:
+
+```text
+zfs_force=1
+```
+
+5. Boot once with `Ctrl+x` or `F10`.
+
+### systemd-boot
+
+1. At the bootloader menu, highlight the normal NixOS entry.
+2. Press `e`.
+3. Append this to the end of the options line:
+
+```text
+zfs_force=1
+```
+
+4. Press `Enter` to boot once.
+
+## After boot
+
+Run:
+
+```bash
+sudo zpool status
+sudo zpool import
+journalctl -b | rg "ZFS|zfs|import|root_pool"
+```
+
+## Expected result
+
+`sudo zpool status` should show `root_pool` as `ONLINE`.
+
+## Reboot test
+
+Run:
+
+```bash
+sudo reboot
+```
+
+Do not add `zfs_force=1` the second time.
+
+## If it still fails
+
+Boot once more with:
+
+```text
+zfs_force=1
+```
+
+Then run:
+
+```bash
+sudo zpool status -v
+sudo zpool history | tail -n 50
+journalctl -b | rg "ZFS|zfs|import|root_pool"
+```
+
+## Notes
+
+- Root pool name is `root_pool`.
+- This is a one-time recovery path after disk moves, controller changes, dirty exports, or interrupted imports.
+- Some hosts also need the LUKS unlock USB key inserted before boot.

flake.lock

@@ -8,11 +8,11 @@
       },
       "locked": {
         "dir": "pkgs/firefox-addons",
-        "lastModified": 1773979456,
-        "narHash": "sha256-9kBMJ5IvxqNlkkj/swmE8uK1Sc7TL/LIRUI958m7uBM=",
+        "lastModified": 1784963334,
+        "narHash": "sha256-Xbj8fFapX+ufz8kx0sH33RAvF0Pauzfg5oihn38SWtE=",
         "owner": "rycee",
         "repo": "nur-expressions",
-        "rev": "81e28f47ac18d9e89513929c77e711e657b64851",
+        "rev": "557935d63e75f4bfbd61ca3cede3a632edbed7e5",
         "type": "gitlab"
       },
       "original": {
@@ -29,11 +29,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1774007980,
-        "narHash": "sha256-FOnZjElEI8pqqCvB6K/1JRHTE8o4rer8driivTpq2uo=",
+        "lastModified": 1784913159,
+        "narHash": "sha256-JWq0BfjO4ktpH5USfQNQzdvHpIDT8fSKD5K7LvdMRFs=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "9670de2921812bc4e0452f6e3efd8c859696c183",
+        "rev": "079a3b5d1aa6a719920a51316253b7d6dd22738d",
         "type": "github"
       },
       "original": {
@@ -43,12 +43,15 @@
       }
     },
     "nixos-hardware": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      },
       "locked": {
-        "lastModified": 1774018263,
-        "narHash": "sha256-HHYEwK1A22aSaxv2ibhMMkKvrDGKGlA/qObG4smrSqc=",
+        "lastModified": 1784723954,
+        "narHash": "sha256-1CfD8ZUjCkTgjsneLZ/lxCHhgDfqxxE7/GX0MmsgiqA=",
         "owner": "nixos",
         "repo": "nixos-hardware",
-        "rev": "2d4b4717b2534fad5c715968c1cece04a172b365",
+        "rev": "a017f5b72210026af5b3ac5949f08d94380a6fbd",
         "type": "github"
       },
       "original": {
@@ -60,27 +63,24 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1773821835,
-        "narHash": "sha256-TJ3lSQtW0E2JrznGVm8hOQGVpXjJyXY2guAxku2O9A4=",
-        "owner": "nixos",
-        "repo": "nixpkgs",
-        "rev": "b40629efe5d6ec48dd1efba650c797ddbd39ace0",
-        "type": "github"
+        "lastModified": 1767892417,
+        "narHash": "sha256-8bW3q88CEg2u4hSP66Vf4lpbLonHz7hqDNBMcCY7E9U=",
+        "rev": "3497aa5c9457a9d88d71fa93a4a8368816fbeeba",
+        "type": "tarball",
+        "url": "https://releases.nixos.org/nixos/unstable/nixos-26.05pre924538.3497aa5c9457/nixexprs.tar.xz"
       },
       "original": {
-        "owner": "nixos",
-        "ref": "nixos-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
+        "type": "tarball",
+        "url": "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz"
       }
     },
     "nixpkgs-master": {
       "locked": {
-        "lastModified": 1774051532,
-        "narHash": "sha256-d3CGMweyYIcPuTj5BKq+1Lx4zwlgL31nVtN647tOZKo=",
+        "lastModified": 1785001027,
+        "narHash": "sha256-Cc+7WLCh3sss1d+4DDLuEma4a2v3YdXGez/N+MJoomY=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "8620c0b5cc8fbe76502442181be1d0514bc3a1b7",
+        "rev": "b2a8a65ad069640cf1a0656b355338a65c41ac12",
         "type": "github"
       },
       "original": {
@@ -106,12 +106,28 @@
         "type": "github"
       }
     },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1784796856,
+        "narHash": "sha256-wWFrV5/Qbm+lyt5x20E/bSbfJiGKMo4RCxZV8cl/WZI=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "e2587caef70cea85dd97d7daab492899902dbf5d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "firefox-addons": "firefox-addons",
         "home-manager": "home-manager",
         "nixos-hardware": "nixos-hardware",
-        "nixpkgs": "nixpkgs",
+        "nixpkgs": "nixpkgs_2",
         "nixpkgs-master": "nixpkgs-master",
         "nixpkgs-stable": "nixpkgs-stable",
         "sops-nix": "sops-nix",
@@ -125,11 +141,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1773889674,
-        "narHash": "sha256-+ycaiVAk3MEshJTg35cBTUa0MizGiS+bgpYw/f8ohkg=",
+        "lastModified": 1783174389,
+        "narHash": "sha256-aCWC8ngycU7OdJrU2+Je3qf+1a2ykuBvpPhZT/9tXMc=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "29b6519f3e0780452bca0ac0be4584f04ac16cc5",
+        "rev": "f1406619a3884cd5c47992a70b8b35c9c0fcb4c9",
         "type": "github"
       },
       "original": {

flake.nix

@@ -65,38 +65,48 @@
 
       devShells = forEachSystem (pkgs: import ./shell.nix { inherit pkgs; });
       formatter = forEachSystem (pkgs: pkgs.treefmt);
+      packages = forEachSystem (
+        pkgs:
+        let
+          installer = pkgs.callPackage ./python/installer/package.nix { };
+          installer-nixos = pkgs.callPackage ./python/installer/package.nix { patchElf = false; };
+        in
+        {
+          inherit installer installer-nixos;
+          default = installer;
+        }
+        // lib.optionalAttrs (pkgs.stdenv.hostPlatform.system == "x86_64-linux") {
+          iso = self.nixosConfigurations.iso.config.system.build.isoImage;
+        }
+      );
+      apps = forEachSystem (
+        pkgs:
+        let
+          system = pkgs.stdenv.hostPlatform.system;
+          installer = {
+            type = "app";
+            program = "${self.packages.${system}.installer}/bin/nixos-installer";
+            meta.description = "One-file NixOS ZFS installer.";
+          };
+        in
+        {
+          inherit installer;
+          default = installer;
+        }
+      );
 
-      nixosConfigurations = {
-        bob = lib.nixosSystem {
-          modules = [
-            ./systems/bob
-          ];
-          specialArgs = { inherit inputs outputs; };
-        };
-        brain = lib.nixosSystem {
-          modules = [
-            ./systems/brain
-          ];
-          specialArgs = { inherit inputs outputs; };
-        };
-        jeeves = lib.nixosSystem {
-          modules = [
-            ./systems/jeeves
-          ];
-          specialArgs = { inherit inputs outputs; };
-        };
-        rhapsody-in-green = lib.nixosSystem {
-          modules = [
-            ./systems/rhapsody-in-green
-          ];
-          specialArgs = { inherit inputs outputs; };
-        };
-        leviathan = lib.nixosSystem {
-          modules = [
-            ./systems/leviathan
-          ];
-          specialArgs = { inherit inputs outputs; };
-        };
-      };
+      nixosConfigurations =
+        let
+          hosts = builtins.attrNames (
+            lib.filterAttrs (_: type: type == "directory") (builtins.readDir ./systems)
+          );
+          mkHost =
+            name:
+            lib.nixosSystem {
+              modules = [ ./systems/${name} ];
+              specialArgs = { inherit inputs outputs; };
+            };
+        in
+        lib.genAttrs hosts mkHost;
     };
 }

frontend/.gitignore

@@ -1,24 +0,0 @@
-# Logs
-logs
-*.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-pnpm-debug.log*
-lerna-debug.log*
-
-node_modules
-dist
-dist-ssr
-*.local
-
-# Editor directories and files
-.vscode/*
-!.vscode/extensions.json
-.idea
-.DS_Store
-*.suo
-*.ntvs*
-*.njsproj
-*.sln
-*.sw?

overlays/default.nix

@@ -21,13 +21,12 @@
         alembic
         apprise
         apscheduler
-        confluent-kafka
         fastapi
         fastapi-cli
         httpx
+        jinja2
         mypy
-        orjson
-        polars
+        pgvector
         psycopg
         pydantic
         pyfakefs
@@ -36,14 +35,13 @@
         pytest-mock
         pytest-xdist
         python-multipart
+        pydantic-settings
         ruff
-        scalene
-        sqlalchemy
         sqlalchemy
         tenacity
-        textual
         tinytuya
         typer
+        uvicorn
         websockets
       ]
     );

pyproject.toml

@@ -3,7 +3,7 @@ name = "system_tools"
 version = "0.1.0"
 description = ""
 authors = [{ name = "Richie Cahill", email = "richie@tmmworkshop.com" }]
-requires-python = "~=3.13.0"
+requires-python = "~=3.14.0"
 readme = "README.md"
 license = "MIT"
 # these dependencies are a best effort and aren't guaranteed to work
@@ -12,25 +12,39 @@ dependencies = [
     "alembic",
     "apprise",
     "apscheduler",
+    "beautifulsoup4",
+    "bm25s",
+    "ebooklib",
+    "fastapi",
+    "fastapi-cli",
     "httpx",
-    "python-multipart",
+    "jinja2",
+    "pgvector",
     "polars",
     "psycopg[binary]",
     "pydantic",
-    "pyyaml",
-    "sqlalchemy",
+    "pydantic-settings",
+    "python-multipart",
+    "sqlalchemy[asyncio]",
+    "tenacity",
+    "tiktoken",
+    "tinytuya",
     "typer",
+    "uvicorn",
     "websockets",
+    "yake",
 ]
 
 [project.scripts]
 database = "python.database_cli:app"
-van-inventory = "python.van_inventory.main:serve"
+whisper-transcribe = "python.tools.whisper.transcribe:main"
 
 [dependency-groups]
 dev = [
+    "aiosqlite",
     "mypy",
     "pyfakefs",
+    "pytest-asyncio",
     "pytest-cov",
     "pytest-mock",
     "pytest-xdist",
@@ -40,7 +54,7 @@ dev = [
 
 [tool.ruff]
 
-target-version = "py313"
+target-version = "py314"
 
 line-length = 120
 
@@ -50,6 +64,7 @@ lint.ignore = [
     "COM812", # (TEMP) conflicts when used with the formatter
     "ISC001", # (TEMP) conflicts when used with the formatter
     "S603",   # (PERM) This is known to cause a false positive
+    "S607",   # (PERM) This is becoming a consistent annoyance
 ]
 
 [tool.ruff.lint.per-file-ignores]
@@ -78,15 +93,10 @@ lint.ignore = [
 "python/congress_tracker/**" = [
     "TC003", # (perm) this creates issues because sqlalchemy uses these at runtime
 ]
-"python/eval_warnings/**" = [
-    "S607", # (perm) gh and git are expected on PATH in the runner environment
-]
+
 "python/alembic/**" = [
     "INP001", # (perm) this creates LSP issues for alembic
 ]
-"python/signal_bot/**" = [
-    "D107", # (perm) class docstrings cover __init__
-]
 
 [tool.ruff.lint.pydocstyle]
 convention = "google"
@@ -109,6 +119,7 @@ exclude_lines = [
 ]
 
 [tool.pytest.ini_options]
-addopts = "-n auto -ra"
+addopts = "-n auto -ra --ignore=tests/ebook_search"
+asyncio_mode = "auto"
 testpaths = ["tests"]
 # --cov=system_tools --cov-report=term-missing --cov-report=xml --cov-report=html --cov-branch

python/alembic/data_science_dev/versions/2026_03_24-adding_failedingestion_2f43120e3ffc.py

@@ -1,50 +0,0 @@
-"""adding FailedIngestion.
-
-Revision ID: 2f43120e3ffc
-Revises: f99be864fe69
-Create Date: 2026-03-24 23:46:17.277897
-
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import sqlalchemy as sa
-from alembic import op
-
-from python.orm import DataScienceDevBase
-
-if TYPE_CHECKING:
-    from collections.abc import Sequence
-
-# revision identifiers, used by Alembic.
-revision: str = "2f43120e3ffc"
-down_revision: str | None = "f99be864fe69"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-schema = DataScienceDevBase.schema_name
-
-
-def upgrade() -> None:
-    """Upgrade."""
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table(
-        "failed_ingestion",
-        sa.Column("raw_line", sa.Text(), nullable=False),
-        sa.Column("error", sa.Text(), nullable=False),
-        sa.Column("id", sa.Integer(), nullable=False),
-        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
-        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
-        sa.PrimaryKeyConstraint("id", name=op.f("pk_failed_ingestion")),
-        schema=schema,
-    )
-    # ### end Alembic commands ###
-
-
-def downgrade() -> None:
-    """Downgrade."""
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table("failed_ingestion", schema=schema)
-    # ### end Alembic commands ###

python/alembic/data_science_dev/versions/2026_03_25-attach_partitions_to_posts.py

@@ -1,80 +0,0 @@
-"""Attach all partition tables to the posts parent table.
-
-Alembic autogenerate creates partition tables as standalone tables but does not
-emit the ALTER TABLE ... ATTACH PARTITION statements needed for PostgreSQL to
-route inserts to the correct partition.
-
-Revision ID: a1b2c3d4e5f6
-Revises: 605b1794838f
-Create Date: 2026-03-25 10:00:00.000000
-
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-from alembic import op
-from sqlalchemy import text
-
-from python.orm import DataScienceDevBase
-from python.orm.data_science_dev.posts.partitions import (
-    PARTITION_END_YEAR,
-    PARTITION_START_YEAR,
-    iso_weeks_in_year,
-    week_bounds,
-)
-
-if TYPE_CHECKING:
-    from collections.abc import Sequence
-
-# revision identifiers, used by Alembic.
-revision: str = "a1b2c3d4e5f6"
-down_revision: str | None = "605b1794838f"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-schema = DataScienceDevBase.schema_name
-
-ALREADY_ATTACHED_QUERY = text("""
-    SELECT inhrelid::regclass::text
-    FROM pg_inherits
-    WHERE inhparent = :parent::regclass
-""")
-
-
-def upgrade() -> None:
-    """Attach all weekly partition tables to the posts parent table."""
-    connection = op.get_bind()
-    already_attached = {
-        row[0]
-        for row in connection.execute(
-            ALREADY_ATTACHED_QUERY, {"parent": f"{schema}.posts"}
-        )
-    }
-
-    for year in range(PARTITION_START_YEAR, PARTITION_END_YEAR + 1):
-        for week in range(1, iso_weeks_in_year(year) + 1):
-            table_name = f"posts_{year}_{week:02d}"
-            qualified_name = f"{schema}.{table_name}"
-            if qualified_name in already_attached:
-                continue
-            start, end = week_bounds(year, week)
-            start_str = start.strftime("%Y-%m-%d %H:%M:%S")
-            end_str = end.strftime("%Y-%m-%d %H:%M:%S")
-            op.execute(
-                f"ALTER TABLE {schema}.posts "
-                f"ATTACH PARTITION {qualified_name} "
-                f"FOR VALUES FROM ('{start_str}') TO ('{end_str}')"
-            )
-
-
-def downgrade() -> None:
-    """Detach all weekly partition tables from the posts parent table."""
-    for year in range(PARTITION_START_YEAR, PARTITION_END_YEAR + 1):
-        for week in range(1, iso_weeks_in_year(year) + 1):
-            table_name = f"posts_{year}_{week:02d}"
-            op.execute(
-                f"ALTER TABLE {schema}.posts "
-                f"DETACH PARTITION {schema}.{table_name}"
-            )

python/alembic/env.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import logging
-import re
 import sys
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Literal

python/alembic/richie/versions/2026_03_29-removed_ds_table_from_richie_db_c8a794340928.py

@@ -0,0 +1,187 @@
+"""removed ds table from richie DB.
+
+Revision ID: c8a794340928
+Revises: 6b275323f435
+Create Date: 2026-03-29 15:29:23.643146
+
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+from python.orm import RichieBase
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+# revision identifiers, used by Alembic.
+revision: str = "c8a794340928"
+down_revision: str | None = "6b275323f435"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+schema = RichieBase.schema_name
+
+
+def upgrade() -> None:
+    """Upgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("vote_record", schema=schema)
+    op.drop_index(op.f("ix_vote_congress_chamber"), table_name="vote", schema=schema)
+    op.drop_index(op.f("ix_vote_date"), table_name="vote", schema=schema)
+    op.drop_index(op.f("ix_legislator_bioguide_id"), table_name="legislator", schema=schema)
+    op.drop_table("legislator", schema=schema)
+    op.drop_table("vote", schema=schema)
+    op.drop_index(op.f("ix_bill_congress"), table_name="bill", schema=schema)
+    op.drop_table("bill", schema=schema)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "vote",
+        sa.Column("congress", sa.INTEGER(), autoincrement=False, nullable=False),
+        sa.Column("chamber", sa.VARCHAR(), autoincrement=False, nullable=False),
+        sa.Column("session", sa.INTEGER(), autoincrement=False, nullable=False),
+        sa.Column("number", sa.INTEGER(), autoincrement=False, nullable=False),
+        sa.Column("vote_type", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("question", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("result", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("result_text", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("vote_date", sa.DATE(), autoincrement=False, nullable=False),
+        sa.Column("yea_count", sa.INTEGER(), autoincrement=False, nullable=True),
+        sa.Column("nay_count", sa.INTEGER(), autoincrement=False, nullable=True),
+        sa.Column("not_voting_count", sa.INTEGER(), autoincrement=False, nullable=True),
+        sa.Column("present_count", sa.INTEGER(), autoincrement=False, nullable=True),
+        sa.Column("bill_id", sa.INTEGER(), autoincrement=False, nullable=True),
+        sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
+        sa.Column(
+            "created",
+            postgresql.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            autoincrement=False,
+            nullable=False,
+        ),
+        sa.Column(
+            "updated",
+            postgresql.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            autoincrement=False,
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(["bill_id"], [f"{schema}.bill.id"], name=op.f("fk_vote_bill_id_bill")),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_vote")),
+        sa.UniqueConstraint(
+            "congress",
+            "chamber",
+            "session",
+            "number",
+            name=op.f("uq_vote_congress_chamber_session_number"),
+            postgresql_include=[],
+            postgresql_nulls_not_distinct=False,
+        ),
+        schema=schema,
+    )
+    op.create_index(op.f("ix_vote_date"), "vote", ["vote_date"], unique=False, schema=schema)
+    op.create_index(op.f("ix_vote_congress_chamber"), "vote", ["congress", "chamber"], unique=False, schema=schema)
+    op.create_table(
+        "vote_record",
+        sa.Column("vote_id", sa.INTEGER(), autoincrement=False, nullable=False),
+        sa.Column("legislator_id", sa.INTEGER(), autoincrement=False, nullable=False),
+        sa.Column("position", sa.VARCHAR(), autoincrement=False, nullable=False),
+        sa.ForeignKeyConstraint(
+            ["legislator_id"],
+            [f"{schema}.legislator.id"],
+            name=op.f("fk_vote_record_legislator_id_legislator"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["vote_id"], [f"{schema}.vote.id"], name=op.f("fk_vote_record_vote_id_vote"), ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("vote_id", "legislator_id", name=op.f("pk_vote_record")),
+        schema=schema,
+    )
+    op.create_table(
+        "legislator",
+        sa.Column("bioguide_id", sa.TEXT(), autoincrement=False, nullable=False),
+        sa.Column("thomas_id", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("lis_id", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("govtrack_id", sa.INTEGER(), autoincrement=False, nullable=True),
+        sa.Column("opensecrets_id", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("fec_ids", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("first_name", sa.VARCHAR(), autoincrement=False, nullable=False),
+        sa.Column("last_name", sa.VARCHAR(), autoincrement=False, nullable=False),
+        sa.Column("official_full_name", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("nickname", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("birthday", sa.DATE(), autoincrement=False, nullable=True),
+        sa.Column("gender", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("current_party", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("current_state", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("current_district", sa.INTEGER(), autoincrement=False, nullable=True),
+        sa.Column("current_chamber", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
+        sa.Column(
+            "created",
+            postgresql.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            autoincrement=False,
+            nullable=False,
+        ),
+        sa.Column(
+            "updated",
+            postgresql.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            autoincrement=False,
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_legislator")),
+        schema=schema,
+    )
+    op.create_index(op.f("ix_legislator_bioguide_id"), "legislator", ["bioguide_id"], unique=True, schema=schema)
+    op.create_table(
+        "bill",
+        sa.Column("congress", sa.INTEGER(), autoincrement=False, nullable=False),
+        sa.Column("bill_type", sa.VARCHAR(), autoincrement=False, nullable=False),
+        sa.Column("number", sa.INTEGER(), autoincrement=False, nullable=False),
+        sa.Column("title", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("title_short", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("official_title", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("status", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("status_at", sa.DATE(), autoincrement=False, nullable=True),
+        sa.Column("sponsor_bioguide_id", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("subjects_top_term", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
+        sa.Column(
+            "created",
+            postgresql.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            autoincrement=False,
+            nullable=False,
+        ),
+        sa.Column(
+            "updated",
+            postgresql.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            autoincrement=False,
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_bill")),
+        sa.UniqueConstraint(
+            "congress",
+            "bill_type",
+            "number",
+            name=op.f("uq_bill_congress_type_number"),
+            postgresql_include=[],
+            postgresql_nulls_not_distinct=False,
+        ),
+        schema=schema,
+    )
+    op.create_index(op.f("ix_bill_congress"), "bill", ["congress"], unique=False, schema=schema)
+    # ### end Alembic commands ###

python/alembic/richie/versions/2026_06_03-adding_audiobook_libreary_metadata_d7864d1ffc17.py

@@ -0,0 +1,93 @@
+"""adding audiobook libreary metadata.
+
+Revision ID: d7864d1ffc17
+Revises: c8a794340928
+Create Date: 2026-06-03 20:24:09.200837
+
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import sqlalchemy as sa
+from alembic import op
+
+from python.orm import RichieBase
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+# revision identifiers, used by Alembic.
+revision: str = "d7864d1ffc17"
+down_revision: str | None = "c8a794340928"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+schema = RichieBase.schema_name
+
+
+def upgrade() -> None:
+    """Upgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "audiobook_author",
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_audiobook_author")),
+        sa.UniqueConstraint("name", name=op.f("uq_audiobook_author_name")),
+        schema=schema,
+    )
+    op.create_table(
+        "audiobook_series",
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("author_id", sa.Integer(), nullable=False),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["author_id"],
+            [f"{schema}.audiobook_author.id"],
+            name=op.f("fk_audiobook_series_author_id_audiobook_author"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_audiobook_series")),
+        sa.UniqueConstraint("author_id", "name", name=op.f("uq_audiobook_series_author_id")),
+        schema=schema,
+    )
+    op.create_table(
+        "audiobook",
+        sa.Column("title", sa.String(), nullable=False),
+        sa.Column("author_id", sa.Integer(), nullable=False),
+        sa.Column("series_id", sa.Integer(), nullable=True),
+        sa.Column("series_index", sa.Integer(), nullable=False),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["author_id"],
+            [f"{schema}.audiobook_author.id"],
+            name=op.f("fk_audiobook_author_id_audiobook_author"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["series_id"],
+            [f"{schema}.audiobook_series.id"],
+            name=op.f("fk_audiobook_series_id_audiobook_series"),
+            ondelete="SET NULL",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_audiobook")),
+        schema=schema,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("audiobook", schema=schema)
+    op.drop_table("audiobook_series", schema=schema)
+    op.drop_table("audiobook_author", schema=schema)
+    # ### end Alembic commands ###

python/alembic/richie/versions/2026_06_10-add_ebook_search_tables_2db132cace1a.py

@@ -0,0 +1,200 @@
+"""add ebook search tables.
+
+Revision ID: 2db132cace1a
+Revises: b3c60cc5beb5
+Create Date: 2026-06-10 22:10:54.379159
+
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pgvector
+import sqlalchemy as sa
+from alembic import op
+
+from python.orm import RichieBase
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+# revision identifiers, used by Alembic.
+revision: str = "2db132cace1a"
+down_revision: str | None = "b3c60cc5beb5"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+schema = RichieBase.schema_name
+
+
+def upgrade() -> None:
+    """Upgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "ebook_embedding_model",
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("dimension", sa.Integer(), nullable=False),
+        sa.Column("is_default", sa.Boolean(), nullable=False),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_embedding_model")),
+        sa.UniqueConstraint("name", name=op.f("uq_ebook_embedding_model_name")),
+        schema=schema,
+    )
+    op.create_table(
+        "ebook_source",
+        sa.Column("title", sa.String(), nullable=False),
+        sa.Column("author", sa.String(), nullable=True),
+        sa.Column("language", sa.String(), nullable=True),
+        sa.Column("publisher", sa.String(), nullable=True),
+        sa.Column("identifier", sa.String(), nullable=True),
+        sa.Column("file_path", sa.String(), nullable=False),
+        sa.Column("file_sha256", sa.String(length=64), nullable=False),
+        sa.Column("file_mtime", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("file_size", sa.BigInteger(), nullable=False),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_source")),
+        sa.UniqueConstraint("file_path", name=op.f("uq_ebook_source_file_path")),
+        sa.UniqueConstraint("file_sha256", name=op.f("uq_ebook_source_file_sha256")),
+        schema=schema,
+    )
+    op.create_table(
+        "ebook_chapter",
+        sa.Column("source_id", sa.Integer(), nullable=False),
+        sa.Column("spine_index", sa.Integer(), nullable=False),
+        sa.Column("title", sa.String(), nullable=True),
+        sa.Column("href", sa.String(), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["source_id"],
+            [f"{schema}.ebook_source.id"],
+            name=op.f("fk_ebook_chapter_source_id_ebook_source"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chapter")),
+        sa.UniqueConstraint("source_id", "spine_index", name=op.f("uq_ebook_chapter_source_id")),
+        schema=schema,
+    )
+    op.create_table(
+        "ebook_chunk",
+        sa.Column("source_id", sa.Integer(), nullable=False),
+        sa.Column("chapter_id", sa.Integer(), nullable=True),
+        sa.Column("chunk_index", sa.Integer(), nullable=False),
+        sa.Column("text", sa.String(), nullable=False),
+        sa.Column("token_start", sa.Integer(), nullable=False),
+        sa.Column("token_count", sa.Integer(), nullable=False),
+        sa.Column("page_label", sa.String(), nullable=True),
+        sa.Column("content_sha256", sa.String(length=64), nullable=False),
+        sa.Column("search_text", sa.String(), nullable=False),
+        sa.Column("id", sa.BigInteger(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["chapter_id"],
+            [f"{schema}.ebook_chapter.id"],
+            name=op.f("fk_ebook_chunk_chapter_id_ebook_chapter"),
+            ondelete="SET NULL",
+        ),
+        sa.ForeignKeyConstraint(
+            ["source_id"],
+            [f"{schema}.ebook_source.id"],
+            name=op.f("fk_ebook_chunk_source_id_ebook_source"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk")),
+        sa.UniqueConstraint("source_id", "chunk_index", name="uq_ebook_chunk_source_id_chunk_index"),
+        sa.UniqueConstraint("source_id", "content_sha256", name="uq_ebook_chunk_source_id_content_sha256"),
+        schema=schema,
+    )
+    op.create_table(
+        "ebook_chunk_embedding_1024",
+        sa.Column("chunk_id", sa.BigInteger(), nullable=False),
+        sa.Column("model_id", sa.Integer(), nullable=False),
+        sa.Column("embedding", pgvector.sqlalchemy.vector.VECTOR(dim=1024), nullable=False),
+        sa.Column("id", sa.BigInteger(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["chunk_id"],
+            [f"{schema}.ebook_chunk.id"],
+            name=op.f("fk_ebook_chunk_embedding_1024_chunk_id_ebook_chunk"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["model_id"],
+            [f"{schema}.ebook_embedding_model.id"],
+            name=op.f("fk_ebook_chunk_embedding_1024_model_id_ebook_embedding_model"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk_embedding_1024")),
+        sa.UniqueConstraint("chunk_id", "model_id", name=op.f("uq_ebook_chunk_embedding_1024_chunk_id")),
+        schema=schema,
+    )
+    op.create_table(
+        "ebook_chunk_embedding_2560",
+        sa.Column("chunk_id", sa.BigInteger(), nullable=False),
+        sa.Column("model_id", sa.Integer(), nullable=False),
+        sa.Column("embedding", pgvector.sqlalchemy.vector.VECTOR(dim=2560), nullable=False),
+        sa.Column("id", sa.BigInteger(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["chunk_id"],
+            [f"{schema}.ebook_chunk.id"],
+            name=op.f("fk_ebook_chunk_embedding_2560_chunk_id_ebook_chunk"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["model_id"],
+            [f"{schema}.ebook_embedding_model.id"],
+            name=op.f("fk_ebook_chunk_embedding_2560_model_id_ebook_embedding_model"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk_embedding_2560")),
+        sa.UniqueConstraint("chunk_id", "model_id", name=op.f("uq_ebook_chunk_embedding_2560_chunk_id")),
+        schema=schema,
+    )
+    op.create_table(
+        "ebook_chunk_embedding_4096",
+        sa.Column("chunk_id", sa.BigInteger(), nullable=False),
+        sa.Column("model_id", sa.Integer(), nullable=False),
+        sa.Column("embedding", pgvector.sqlalchemy.vector.VECTOR(dim=4096), nullable=False),
+        sa.Column("id", sa.BigInteger(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["chunk_id"],
+            [f"{schema}.ebook_chunk.id"],
+            name=op.f("fk_ebook_chunk_embedding_4096_chunk_id_ebook_chunk"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["model_id"],
+            [f"{schema}.ebook_embedding_model.id"],
+            name=op.f("fk_ebook_chunk_embedding_4096_model_id_ebook_embedding_model"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk_embedding_4096")),
+        sa.UniqueConstraint("chunk_id", "model_id", name=op.f("uq_ebook_chunk_embedding_4096_chunk_id")),
+        schema=schema,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("ebook_chunk_embedding_4096", schema=schema)
+    op.drop_table("ebook_chunk_embedding_2560", schema=schema)
+    op.drop_table("ebook_chunk_embedding_1024", schema=schema)
+    op.drop_table("ebook_chunk", schema=schema)
+    op.drop_table("ebook_chapter", schema=schema)
+    op.drop_table("ebook_source", schema=schema)
+    op.drop_table("ebook_embedding_model", schema=schema)
+    # ### end Alembic commands ###

python/alembic/richie/versions/2026_06_10-updated_series_index_to_float_and_added__b3c60cc5beb5.py

@@ -0,0 +1,63 @@
+"""updated series_index to float and added UniqueConstraint to audiobook and audiobook_author.
+
+Revision ID: b3c60cc5beb5
+Revises: d7864d1ffc17
+Create Date: 2026-06-10 20:02:43.073725
+
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import sqlalchemy as sa
+from alembic import op
+
+from python.orm import RichieBase
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+# revision identifiers, used by Alembic.
+revision: str = "b3c60cc5beb5"
+down_revision: str | None = "d7864d1ffc17"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+schema = RichieBase.schema_name
+
+
+def upgrade() -> None:
+    """Upgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "audiobook",
+        "series_index",
+        existing_type=sa.INTEGER(),
+        type_=sa.Float(),
+        existing_nullable=False,
+        schema=schema,
+    )
+    op.create_unique_constraint(
+        op.f("uq_audiobook_author_id"),
+        "audiobook",
+        ["author_id", "series_id", "title"],
+        schema=schema,
+        postgresql_nulls_not_distinct=True,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(op.f("uq_audiobook_author_id"), "audiobook", schema=schema, type_="unique")
+    op.alter_column(
+        "audiobook",
+        "series_index",
+        existing_type=sa.Float(),
+        type_=sa.INTEGER(),
+        existing_nullable=False,
+        schema=schema,
+    )
+    # ### end Alembic commands ###

python/alembic/richie/versions/2026_06_13-add_1024_ebook_embedding_cosine_index_c460105682d2.py

@@ -0,0 +1,54 @@
+"""add 1024 ebook embedding cosine index.
+
+Revision ID: c460105682d2
+Revises: 2db132cace1a
+Create Date: 2026-06-13 19:53:45.680289
+
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from alembic import op
+
+from python.orm import RichieBase
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+# revision identifiers, used by Alembic.
+revision: str = "c460105682d2"
+down_revision: str | None = "2db132cace1a"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+schema = RichieBase.schema_name
+
+
+def upgrade() -> None:
+    """Upgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_index(
+        "ix_ebook_chunk_embedding_1024_embedding_cosine",
+        "ebook_chunk_embedding_1024",
+        ["embedding"],
+        unique=False,
+        schema=schema,
+        postgresql_using="hnsw",
+        postgresql_ops={"embedding": "vector_cosine_ops"},
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        "ix_ebook_chunk_embedding_1024_embedding_cosine",
+        table_name="ebook_chunk_embedding_1024",
+        schema=schema,
+        postgresql_using="hnsw",
+        postgresql_ops={"embedding": "vector_cosine_ops"},
+    )
+    # ### end Alembic commands ###

python/alembic/richie/versions/2026_06_23-adding_haproxy_data_96d72c748c24.py

@@ -0,0 +1,103 @@
+"""adding haproxy data.
+
+Revision ID: 96d72c748c24
+Revises: c460105682d2
+Create Date: 2026-06-23 16:37:17.768851
+
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import sqlalchemy as sa
+from alembic import op
+
+from python.orm import RichieBase
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+# revision identifiers, used by Alembic.
+revision: str = "96d72c748c24"
+down_revision: str | None = "c460105682d2"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+schema = RichieBase.schema_name
+
+
+def upgrade() -> None:
+    """Upgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "haproxy_request",
+        sa.Column("line_hash", sa.String(), nullable=False),
+        sa.Column("requested_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("client_ip", sa.String(), nullable=False),
+        sa.Column("client_port", sa.Integer(), nullable=False),
+        sa.Column("frontend", sa.String(), nullable=False),
+        sa.Column("ssl", sa.Boolean(), nullable=False),
+        sa.Column("backend", sa.String(), nullable=False),
+        sa.Column("server", sa.String(), nullable=False),
+        sa.Column("time_request", sa.Integer(), nullable=False),
+        sa.Column("time_queue", sa.Integer(), nullable=False),
+        sa.Column("time_connect", sa.Integer(), nullable=False),
+        sa.Column("time_response", sa.Integer(), nullable=False),
+        sa.Column("time_total", sa.Integer(), nullable=False),
+        sa.Column("status_code", sa.Integer(), nullable=False),
+        sa.Column("bytes_read", sa.BigInteger(), nullable=False),
+        sa.Column("termination_state", sa.String(), nullable=False),
+        sa.Column("active_connections", sa.Integer(), nullable=False),
+        sa.Column("frontend_connections", sa.Integer(), nullable=False),
+        sa.Column("backend_connections", sa.Integer(), nullable=False),
+        sa.Column("server_connections", sa.Integer(), nullable=False),
+        sa.Column("retries", sa.Integer(), nullable=False),
+        sa.Column("server_queue", sa.Integer(), nullable=False),
+        sa.Column("backend_queue", sa.Integer(), nullable=False),
+        sa.Column("host", sa.String(), nullable=True),
+        sa.Column("user_agent", sa.String(), nullable=True),
+        sa.Column("method", sa.String(), nullable=False),
+        sa.Column("target", sa.String(), nullable=False),
+        sa.Column("path", sa.String(), nullable=False),
+        sa.Column("query", sa.String(), nullable=True),
+        sa.Column("http_version", sa.String(), nullable=False),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_haproxy_request")),
+        sa.UniqueConstraint("line_hash", name=op.f("uq_haproxy_request_line_hash")),
+        schema=schema,
+    )
+    op.create_index(op.f("ix_haproxy_request_backend"), "haproxy_request", ["backend"], unique=False, schema=schema)
+    op.create_index(op.f("ix_haproxy_request_client_ip"), "haproxy_request", ["client_ip"], unique=False, schema=schema)
+    op.create_index(op.f("ix_haproxy_request_host"), "haproxy_request", ["host"], unique=False, schema=schema)
+    op.create_index(op.f("ix_haproxy_request_path"), "haproxy_request", ["path"], unique=False, schema=schema)
+    op.create_index(
+        op.f("ix_haproxy_request_requested_at"), "haproxy_request", ["requested_at"], unique=False, schema=schema
+    )
+    op.create_index(
+        op.f("ix_haproxy_request_status_code"), "haproxy_request", ["status_code"], unique=False, schema=schema
+    )
+    op.create_index(
+        op.f("ix_haproxy_request_time_response"), "haproxy_request", ["time_response"], unique=False, schema=schema
+    )
+    op.create_index(
+        op.f("ix_haproxy_request_user_agent"), "haproxy_request", ["user_agent"], unique=False, schema=schema
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f("ix_haproxy_request_user_agent"), table_name="haproxy_request", schema=schema)
+    op.drop_index(op.f("ix_haproxy_request_time_response"), table_name="haproxy_request", schema=schema)
+    op.drop_index(op.f("ix_haproxy_request_status_code"), table_name="haproxy_request", schema=schema)
+    op.drop_index(op.f("ix_haproxy_request_requested_at"), table_name="haproxy_request", schema=schema)
+    op.drop_index(op.f("ix_haproxy_request_path"), table_name="haproxy_request", schema=schema)
+    op.drop_index(op.f("ix_haproxy_request_host"), table_name="haproxy_request", schema=schema)
+    op.drop_index(op.f("ix_haproxy_request_client_ip"), table_name="haproxy_request", schema=schema)
+    op.drop_index(op.f("ix_haproxy_request_backend"), table_name="haproxy_request", schema=schema)
+    op.drop_table("haproxy_request", schema=schema)
+    # ### end Alembic commands ###

python/alembic/richie/versions/2026_06_29-adding_phrase_metadata_tables_dddee09eddcc.py

@@ -0,0 +1,206 @@
+"""adding Phrase metadata tables.
+
+Revision ID: dddee09eddcc
+Revises: 96d72c748c24
+Create Date: 2026-06-29 00:49:07.344159
+
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+from python.orm import RichieBase
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+# revision identifiers, used by Alembic.
+revision: str = "dddee09eddcc"
+down_revision: str | None = "96d72c748c24"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+schema = RichieBase.schema_name
+
+
+def upgrade() -> None:
+    """Upgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "candidate_phrases",
+        sa.Column("book_id", sa.Integer(), nullable=False),
+        sa.Column("series_id", sa.Integer(), nullable=True),
+        sa.Column("phrase_text", sa.Text(), nullable=False),
+        sa.Column("phrase_norm", sa.Text(), nullable=False),
+        sa.Column("token_count", sa.Integer(), nullable=False),
+        sa.Column("source_raw_ngram", sa.Boolean(), nullable=False),
+        sa.Column("source_yake", sa.Boolean(), nullable=False),
+        sa.Column("source_spacy_ner", sa.Boolean(), nullable=False),
+        sa.Column("source_spacy_noun_chunk", sa.Boolean(), nullable=False),
+        sa.Column("source_capitalized", sa.Boolean(), nullable=False),
+        sa.Column("source_metadata", sa.Boolean(), nullable=False),
+        sa.Column("spacy_label", sa.String(), nullable=True),
+        sa.Column("raw_count", sa.Integer(), nullable=False),
+        sa.Column("chapter_count", sa.Integer(), nullable=False),
+        sa.Column("yake_score", sa.Float(), nullable=True),
+        sa.Column("candidate_score", sa.Float(), nullable=False),
+        sa.Column(
+            "sample_contexts",
+            sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), "postgresql"),
+            nullable=True,
+        ),
+        sa.Column("llm_judged", sa.Boolean(), nullable=False),
+        sa.Column("llm_keep", sa.Boolean(), nullable=True),
+        sa.Column("llm_confidence", sa.Float(), nullable=True),
+        sa.Column("llm_category", sa.String(), nullable=True),
+        sa.Column("llm_reason", sa.Text(), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["book_id"],
+            [f"{schema}.ebook_source.id"],
+            name=op.f("fk_candidate_phrases_book_id_ebook_source"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_candidate_phrases")),
+        sa.UniqueConstraint("book_id", "phrase_norm", name="uq_candidate_phrases_book_id_phrase_norm"),
+        schema=schema,
+    )
+    op.create_index(
+        "candidate_phrases_book_norm_idx", "candidate_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
+    )
+    op.create_index(
+        "candidate_phrases_book_score_idx",
+        "candidate_phrases",
+        ["book_id", "candidate_score"],
+        unique=False,
+        schema=schema,
+    )
+    op.create_table(
+        "protected_phrases",
+        sa.Column("book_id", sa.Integer(), nullable=True),
+        sa.Column("series_id", sa.Integer(), nullable=True),
+        sa.Column("phrase_text", sa.Text(), nullable=False),
+        sa.Column("phrase_norm", sa.Text(), nullable=False),
+        sa.Column("canonical_id", sa.String(), nullable=False),
+        sa.Column("phrase_type", sa.String(), nullable=True),
+        sa.Column("token_count", sa.Integer(), nullable=False),
+        sa.Column("confidence", sa.Float(), nullable=False),
+        sa.Column("importance", sa.Float(), nullable=False),
+        sa.Column("allow_nested", sa.Boolean(), nullable=False),
+        sa.Column("suppress_children", sa.Boolean(), nullable=False),
+        sa.Column("source_candidate_id", sa.Integer(), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["book_id"],
+            [f"{schema}.ebook_source.id"],
+            name=op.f("fk_protected_phrases_book_id_ebook_source"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["source_candidate_id"],
+            [f"{schema}.candidate_phrases.id"],
+            name=op.f("fk_protected_phrases_source_candidate_id_candidate_phrases"),
+            ondelete="SET NULL",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_protected_phrases")),
+        sa.UniqueConstraint("book_id", "phrase_norm", name="uq_protected_phrases_book_id_phrase_norm"),
+        schema=schema,
+    )
+    op.create_index(
+        "protected_phrases_book_norm_idx", "protected_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
+    )
+    op.create_index("protected_phrases_norm_idx", "protected_phrases", ["phrase_norm"], unique=False, schema=schema)
+    op.create_index(
+        "protected_phrases_series_norm_idx",
+        "protected_phrases",
+        ["series_id", "phrase_norm"],
+        unique=False,
+        schema=schema,
+    )
+    op.create_table(
+        "chunk_phrase_mentions",
+        sa.Column("chunk_id", sa.BigInteger(), nullable=False),
+        sa.Column("phrase_id", sa.Integer(), nullable=False),
+        sa.Column("book_id", sa.Integer(), nullable=True),
+        sa.Column("series_id", sa.Integer(), nullable=True),
+        sa.Column("start_char", sa.Integer(), nullable=False),
+        sa.Column("end_char", sa.Integer(), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["book_id"],
+            [f"{schema}.ebook_source.id"],
+            name=op.f("fk_chunk_phrase_mentions_book_id_ebook_source"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["chunk_id"],
+            [f"{schema}.ebook_chunk.id"],
+            name=op.f("fk_chunk_phrase_mentions_chunk_id_ebook_chunk"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["phrase_id"],
+            [f"{schema}.protected_phrases.id"],
+            name=op.f("fk_chunk_phrase_mentions_phrase_id_protected_phrases"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_chunk_phrase_mentions")),
+        sa.UniqueConstraint("chunk_id", "phrase_id", "start_char", name="uq_chunk_phrase_mentions_chunk_phrase_start"),
+        schema=schema,
+    )
+    op.create_index(
+        "chunk_phrase_mentions_chunk_idx", "chunk_phrase_mentions", ["chunk_id"], unique=False, schema=schema
+    )
+    op.create_index(
+        "chunk_phrase_mentions_phrase_idx", "chunk_phrase_mentions", ["phrase_id"], unique=False, schema=schema
+    )
+    op.create_table(
+        "phrase_aliases",
+        sa.Column("phrase_id", sa.Integer(), nullable=False),
+        sa.Column("alias_text", sa.Text(), nullable=False),
+        sa.Column("alias_norm", sa.Text(), nullable=False),
+        sa.Column("confidence", sa.Float(), nullable=False),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["phrase_id"],
+            [f"{schema}.protected_phrases.id"],
+            name=op.f("fk_phrase_aliases_phrase_id_protected_phrases"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_phrase_aliases")),
+        sa.UniqueConstraint("phrase_id", "alias_norm", name="uq_phrase_aliases_phrase_id_alias_norm"),
+        schema=schema,
+    )
+    op.create_index("phrase_aliases_norm_idx", "phrase_aliases", ["alias_norm"], unique=False, schema=schema)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index("phrase_aliases_norm_idx", table_name="phrase_aliases", schema=schema)
+    op.drop_table("phrase_aliases", schema=schema)
+    op.drop_index("chunk_phrase_mentions_phrase_idx", table_name="chunk_phrase_mentions", schema=schema)
+    op.drop_index("chunk_phrase_mentions_chunk_idx", table_name="chunk_phrase_mentions", schema=schema)
+    op.drop_table("chunk_phrase_mentions", schema=schema)
+    op.drop_index("protected_phrases_series_norm_idx", table_name="protected_phrases", schema=schema)
+    op.drop_index("protected_phrases_norm_idx", table_name="protected_phrases", schema=schema)
+    op.drop_index("protected_phrases_book_norm_idx", table_name="protected_phrases", schema=schema)
+    op.drop_table("protected_phrases", schema=schema)
+    op.drop_index("candidate_phrases_book_score_idx", table_name="candidate_phrases", schema=schema)
+    op.drop_index("candidate_phrases_book_norm_idx", table_name="candidate_phrases", schema=schema)
+    op.drop_table("candidate_phrases", schema=schema)
+    # ### end Alembic commands ###

python/alembic/richie/versions/2026_07_09-remove_spacy_ner_751260fc3228.py

@@ -0,0 +1,55 @@
+"""remove spaCy-ner.
+
+Revision ID: 751260fc3228
+Revises: dddee09eddcc
+Create Date: 2026-07-09 23:03:39.554083
+
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import sqlalchemy as sa
+from alembic import op
+
+from python.orm import RichieBase
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+# revision identifiers, used by Alembic.
+revision: str = "751260fc3228"
+down_revision: str | None = "dddee09eddcc"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+schema = RichieBase.schema_name
+
+
+def upgrade() -> None:
+    """Upgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("candidate_phrases", "source_spacy_noun_chunk", schema=schema)
+    op.drop_column("candidate_phrases", "source_spacy_ner", schema=schema)
+    op.drop_column("candidate_phrases", "spacy_label", schema=schema)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "candidate_phrases", sa.Column("spacy_label", sa.VARCHAR(), autoincrement=False, nullable=True), schema=schema
+    )
+    op.add_column(
+        "candidate_phrases",
+        sa.Column("source_spacy_ner", sa.BOOLEAN(), autoincrement=False, nullable=False),
+        schema=schema,
+    )
+    op.add_column(
+        "candidate_phrases",
+        sa.Column("source_spacy_noun_chunk", sa.BOOLEAN(), autoincrement=False, nullable=False),
+        schema=schema,
+    )
+    # ### end Alembic commands ###

python/alembic/signal_bot/versions/2026_03_17-seprating_signal_bot_database_6eaf696e07a5.py

@@ -1,100 +0,0 @@
-"""seprating signal_bot database.
-
-Revision ID: 6eaf696e07a5
-Revises:
-Create Date: 2026-03-17 21:35:37.612672
-
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import sqlalchemy as sa
-from alembic import op
-from sqlalchemy.dialects import postgresql
-
-from python.orm import SignalBotBase
-
-if TYPE_CHECKING:
-    from collections.abc import Sequence
-
-# revision identifiers, used by Alembic.
-revision: str = "6eaf696e07a5"
-down_revision: str | None = None
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-schema = SignalBotBase.schema_name
-
-
-def upgrade() -> None:
-    """Upgrade."""
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table(
-        "dead_letter_message",
-        sa.Column("source", sa.String(), nullable=False),
-        sa.Column("message", sa.Text(), nullable=False),
-        sa.Column("received_at", sa.DateTime(timezone=True), nullable=False),
-        sa.Column(
-            "status", postgresql.ENUM("UNPROCESSED", "PROCESSED", name="message_status", schema=schema), nullable=False
-        ),
-        sa.Column("id", sa.Integer(), nullable=False),
-        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
-        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
-        sa.PrimaryKeyConstraint("id", name=op.f("pk_dead_letter_message")),
-        schema=schema,
-    )
-    op.create_table(
-        "role",
-        sa.Column("name", sa.String(length=50), nullable=False),
-        sa.Column("id", sa.SmallInteger(), nullable=False),
-        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
-        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
-        sa.PrimaryKeyConstraint("id", name=op.f("pk_role")),
-        sa.UniqueConstraint("name", name=op.f("uq_role_name")),
-        schema=schema,
-    )
-    op.create_table(
-        "signal_device",
-        sa.Column("phone_number", sa.String(length=50), nullable=False),
-        sa.Column("safety_number", sa.String(), nullable=True),
-        sa.Column(
-            "trust_level",
-            postgresql.ENUM("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", schema=schema),
-            nullable=False,
-        ),
-        sa.Column("last_seen", sa.DateTime(timezone=True), nullable=False),
-        sa.Column("id", sa.Integer(), nullable=False),
-        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
-        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
-        sa.PrimaryKeyConstraint("id", name=op.f("pk_signal_device")),
-        sa.UniqueConstraint("phone_number", name=op.f("uq_signal_device_phone_number")),
-        schema=schema,
-    )
-    op.create_table(
-        "device_role",
-        sa.Column("device_id", sa.Integer(), nullable=False),
-        sa.Column("role_id", sa.SmallInteger(), nullable=False),
-        sa.Column("id", sa.Integer(), nullable=False),
-        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
-        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
-        sa.ForeignKeyConstraint(
-            ["device_id"], [f"{schema}.signal_device.id"], name=op.f("fk_device_role_device_id_signal_device")
-        ),
-        sa.ForeignKeyConstraint(["role_id"], [f"{schema}.role.id"], name=op.f("fk_device_role_role_id_role")),
-        sa.PrimaryKeyConstraint("id", name=op.f("pk_device_role")),
-        sa.UniqueConstraint("device_id", "role_id", name="uq_device_role_device_role"),
-        schema=schema,
-    )
-    # ### end Alembic commands ###
-
-
-def downgrade() -> None:
-    """Downgrade."""
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table("device_role", schema=schema)
-    op.drop_table("signal_device", schema=schema)
-    op.drop_table("role", schema=schema)
-    op.drop_table("dead_letter_message", schema=schema)
-    # ### end Alembic commands ###

python/alembic/signal_bot/versions/2026_03_18-test_66bdd532bcab.py

@@ -1,72 +0,0 @@
-"""test.
-
-Revision ID: 66bdd532bcab
-Revises: 6eaf696e07a5
-Create Date: 2026-03-18 19:21:14.561568
-
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import sqlalchemy as sa
-from alembic import op
-from sqlalchemy.dialects import postgresql
-
-from python.orm import SignalBotBase
-
-if TYPE_CHECKING:
-    from collections.abc import Sequence
-
-# revision identifiers, used by Alembic.
-revision: str = "66bdd532bcab"
-down_revision: str | None = "6eaf696e07a5"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-schema = SignalBotBase.schema_name
-
-
-def upgrade() -> None:
-    """Upgrade."""
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.alter_column(
-        "dead_letter_message",
-        "status",
-        existing_type=postgresql.ENUM("UNPROCESSED", "PROCESSED", name="message_status", schema=schema),
-        type_=sa.Enum("UNPROCESSED", "PROCESSED", name="message_status", native_enum=False),
-        existing_nullable=False,
-        schema=schema,
-    )
-    op.alter_column(
-        "signal_device",
-        "trust_level",
-        existing_type=postgresql.ENUM("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", schema=schema),
-        type_=sa.Enum("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", native_enum=False),
-        existing_nullable=False,
-        schema=schema,
-    )
-    # ### end Alembic commands ###
-
-
-def downgrade() -> None:
-    """Downgrade."""
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.alter_column(
-        "signal_device",
-        "trust_level",
-        existing_type=sa.Enum("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", native_enum=False),
-        type_=postgresql.ENUM("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", schema=schema),
-        existing_nullable=False,
-        schema=schema,
-    )
-    op.alter_column(
-        "dead_letter_message",
-        "status",
-        existing_type=sa.Enum("UNPROCESSED", "PROCESSED", name="message_status", native_enum=False),
-        type_=postgresql.ENUM("UNPROCESSED", "PROCESSED", name="message_status", schema=schema),
-        existing_nullable=False,
-        schema=schema,
-    )
-    # ### end Alembic commands ###

python/api/__init__.py

@@ -1 +0,0 @@
-"""FastAPI applications."""

python/api/dependencies.py

@@ -1,16 +0,0 @@
-"""FastAPI dependencies."""
-
-from collections.abc import Iterator
-from typing import Annotated
-
-from fastapi import Depends, Request
-from sqlalchemy.orm import Session
-
-
-def get_db(request: Request) -> Iterator[Session]:
-    """Get database session from app state."""
-    with Session(request.app.state.engine) as session:
-        yield session
-
-
-DbSession = Annotated[Session, Depends(get_db)]

python/api/main.py

@@ -1,52 +0,0 @@
-"""FastAPI interface for Contact database."""
-
-import logging
-from collections.abc import AsyncIterator
-from contextlib import asynccontextmanager
-from typing import Annotated
-
-import typer
-import uvicorn
-from fastapi import FastAPI
-
-from python.api.middleware import ZstdMiddleware
-from python.api.routers import contact_router, views_router
-from python.common import configure_logger
-from python.orm.common import get_postgres_engine
-
-logger = logging.getLogger(__name__)
-
-
-def create_app() -> FastAPI:
-    """Create and configure the FastAPI application."""
-
-    @asynccontextmanager
-    async def lifespan(app: FastAPI) -> AsyncIterator[None]:
-        """Manage application lifespan."""
-        app.state.engine = get_postgres_engine()
-        yield
-        app.state.engine.dispose()
-
-    app = FastAPI(title="Contact Database API", lifespan=lifespan)
-    app.add_middleware(ZstdMiddleware)
-
-    app.include_router(contact_router)
-    app.include_router(views_router)
-
-    return app
-
-
-def serve(
-    host: Annotated[str, typer.Option("--host", "-h", help="Host to bind to")],
-    port: Annotated[int, typer.Option("--port", "-p", help="Port to bind to")] = 8000,
-    log_level: Annotated[str, typer.Option("--log-level", "-l", help="Log level")] = "INFO",
-) -> None:
-    """Start the Contact API server."""
-    configure_logger(log_level)
-
-    app = create_app()
-    uvicorn.run(app, host=host, port=port)
-
-
-if __name__ == "__main__":
-    typer.run(serve)

python/api/routers/__init__.py

@@ -1,6 +0,0 @@
-"""API routers."""
-
-from python.api.routers.contact import router as contact_router
-from python.api.routers.views import router as views_router
-
-__all__ = ["contact_router", "views_router"]

python/api/routers/contact.py

@@ -1,481 +0,0 @@
-"""Contact API router."""
-
-from pathlib import Path
-
-from fastapi import APIRouter, HTTPException, Request
-from fastapi.responses import HTMLResponse
-from fastapi.templating import Jinja2Templates
-from pydantic import BaseModel
-from sqlalchemy import select
-from sqlalchemy.orm import selectinload
-
-from python.api.dependencies import DbSession
-from python.orm.richie.contact import Contact, ContactRelationship, Need, RelationshipType
-
-TEMPLATES_DIR = Path(__file__).parent.parent / "templates"
-templates = Jinja2Templates(directory=TEMPLATES_DIR)
-
-
-def _is_htmx(request: Request) -> bool:
-    """Check if the request is from HTMX."""
-    return request.headers.get("HX-Request") == "true"
-
-
-class NeedBase(BaseModel):
-    """Base schema for Need."""
-
-    name: str
-    description: str | None = None
-
-
-class NeedCreate(NeedBase):
-    """Schema for creating a Need."""
-
-
-class NeedResponse(NeedBase):
-    """Schema for Need response."""
-
-    id: int
-
-    model_config = {"from_attributes": True}
-
-
-class ContactRelationshipCreate(BaseModel):
-    """Schema for creating a contact relationship."""
-
-    related_contact_id: int
-    relationship_type: RelationshipType
-    closeness_weight: int | None = None
-
-
-class ContactRelationshipUpdate(BaseModel):
-    """Schema for updating a contact relationship."""
-
-    relationship_type: RelationshipType | None = None
-    closeness_weight: int | None = None
-
-
-class ContactRelationshipResponse(BaseModel):
-    """Schema for contact relationship response."""
-
-    contact_id: int
-    related_contact_id: int
-    relationship_type: str
-    closeness_weight: int
-
-    model_config = {"from_attributes": True}
-
-
-class RelationshipTypeInfo(BaseModel):
-    """Information about a relationship type."""
-
-    value: str
-    display_name: str
-    default_weight: int
-
-
-class GraphNode(BaseModel):
-    """Node in the relationship graph."""
-
-    id: int
-    name: str
-    current_job: str | None = None
-
-
-class GraphEdge(BaseModel):
-    """Edge in the relationship graph."""
-
-    source: int
-    target: int
-    relationship_type: str
-    closeness_weight: int
-
-
-class GraphData(BaseModel):
-    """Complete graph data for visualization."""
-
-    nodes: list[GraphNode]
-    edges: list[GraphEdge]
-
-
-class ContactBase(BaseModel):
-    """Base schema for Contact."""
-
-    name: str
-    age: int | None = None
-    bio: str | None = None
-    current_job: str | None = None
-    gender: str | None = None
-    goals: str | None = None
-    legal_name: str | None = None
-    profile_pic: str | None = None
-    safe_conversation_starters: str | None = None
-    self_sufficiency_score: int | None = None
-    social_structure_style: str | None = None
-    ssn: str | None = None
-    suffix: str | None = None
-    timezone: str | None = None
-    topics_to_avoid: str | None = None
-
-
-class ContactCreate(ContactBase):
-    """Schema for creating a Contact."""
-
-    need_ids: list[int] = []
-
-
-class ContactUpdate(BaseModel):
-    """Schema for updating a Contact."""
-
-    name: str | None = None
-    age: int | None = None
-    bio: str | None = None
-    current_job: str | None = None
-    gender: str | None = None
-    goals: str | None = None
-    legal_name: str | None = None
-    profile_pic: str | None = None
-    safe_conversation_starters: str | None = None
-    self_sufficiency_score: int | None = None
-    social_structure_style: str | None = None
-    ssn: str | None = None
-    suffix: str | None = None
-    timezone: str | None = None
-    topics_to_avoid: str | None = None
-    need_ids: list[int] | None = None
-
-
-class ContactResponse(ContactBase):
-    """Schema for Contact response with relationships."""
-
-    id: int
-    needs: list[NeedResponse] = []
-    related_to: list[ContactRelationshipResponse] = []
-    related_from: list[ContactRelationshipResponse] = []
-
-    model_config = {"from_attributes": True}
-
-
-class ContactListResponse(ContactBase):
-    """Schema for Contact list response."""
-
-    id: int
-
-    model_config = {"from_attributes": True}
-
-
-router = APIRouter(prefix="/api", tags=["contacts"])
-
-
-@router.post("/needs", response_model=NeedResponse)
-def create_need(need: NeedCreate, db: DbSession) -> Need:
-    """Create a new need."""
-    db_need = Need(name=need.name, description=need.description)
-    db.add(db_need)
-    db.commit()
-    db.refresh(db_need)
-    return db_need
-
-
-@router.get("/needs", response_model=list[NeedResponse])
-def list_needs(db: DbSession) -> list[Need]:
-    """List all needs."""
-    return list(db.scalars(select(Need)).all())
-
-
-@router.get("/needs/{need_id}", response_model=NeedResponse)
-def get_need(need_id: int, db: DbSession) -> Need:
-    """Get a need by ID."""
-    need = db.get(Need, need_id)
-    if not need:
-        raise HTTPException(status_code=404, detail="Need not found")
-    return need
-
-
-@router.delete("/needs/{need_id}", response_model=None)
-def delete_need(need_id: int, request: Request, db: DbSession) -> dict[str, bool] | HTMLResponse:
-    """Delete a need by ID."""
-    need = db.get(Need, need_id)
-    if not need:
-        raise HTTPException(status_code=404, detail="Need not found")
-    db.delete(need)
-    db.commit()
-    if _is_htmx(request):
-        return HTMLResponse("")
-    return {"deleted": True}
-
-
-@router.post("/contacts", response_model=ContactResponse)
-def create_contact(contact: ContactCreate, db: DbSession) -> Contact:
-    """Create a new contact."""
-    need_ids = contact.need_ids
-    contact_data = contact.model_dump(exclude={"need_ids"})
-    db_contact = Contact(**contact_data)
-
-    if need_ids:
-        needs = list(db.scalars(select(Need).where(Need.id.in_(need_ids))).all())
-        db_contact.needs = needs
-
-    db.add(db_contact)
-    db.commit()
-    db.refresh(db_contact)
-    return db_contact
-
-
-@router.get("/contacts", response_model=list[ContactListResponse])
-def list_contacts(
-    db: DbSession,
-    skip: int = 0,
-    limit: int = 100,
-) -> list[Contact]:
-    """List all contacts with pagination."""
-    return list(db.scalars(select(Contact).offset(skip).limit(limit)).all())
-
-
-@router.get("/contacts/{contact_id}", response_model=ContactResponse)
-def get_contact(contact_id: int, db: DbSession) -> Contact:
-    """Get a contact by ID with all relationships."""
-    contact = db.scalar(
-        select(Contact)
-        .where(Contact.id == contact_id)
-        .options(
-            selectinload(Contact.needs),
-            selectinload(Contact.related_to),
-            selectinload(Contact.related_from),
-        )
-    )
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-    return contact
-
-
-@router.patch("/contacts/{contact_id}", response_model=ContactResponse)
-def update_contact(
-    contact_id: int,
-    contact: ContactUpdate,
-    db: DbSession,
-) -> Contact:
-    """Update a contact by ID."""
-    db_contact = db.get(Contact, contact_id)
-    if not db_contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    update_data = contact.model_dump(exclude_unset=True)
-    need_ids = update_data.pop("need_ids", None)
-
-    for key, value in update_data.items():
-        setattr(db_contact, key, value)
-
-    if need_ids is not None:
-        needs = list(db.scalars(select(Need).where(Need.id.in_(need_ids))).all())
-        db_contact.needs = needs
-
-    db.commit()
-    db.refresh(db_contact)
-    return db_contact
-
-
-@router.delete("/contacts/{contact_id}", response_model=None)
-def delete_contact(contact_id: int, request: Request, db: DbSession) -> dict[str, bool] | HTMLResponse:
-    """Delete a contact by ID."""
-    contact = db.get(Contact, contact_id)
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-    db.delete(contact)
-    db.commit()
-    if _is_htmx(request):
-        return HTMLResponse("")
-    return {"deleted": True}
-
-
-@router.post("/contacts/{contact_id}/needs/{need_id}")
-def add_need_to_contact(
-    contact_id: int,
-    need_id: int,
-    db: DbSession,
-) -> dict[str, bool]:
-    """Add a need to a contact."""
-    contact = db.get(Contact, contact_id)
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    need = db.get(Need, need_id)
-    if not need:
-        raise HTTPException(status_code=404, detail="Need not found")
-
-    if need not in contact.needs:
-        contact.needs.append(need)
-        db.commit()
-
-    return {"added": True}
-
-
-@router.delete("/contacts/{contact_id}/needs/{need_id}", response_model=None)
-def remove_need_from_contact(
-    contact_id: int,
-    need_id: int,
-    request: Request,
-    db: DbSession,
-) -> dict[str, bool] | HTMLResponse:
-    """Remove a need from a contact."""
-    contact = db.get(Contact, contact_id)
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    need = db.get(Need, need_id)
-    if not need:
-        raise HTTPException(status_code=404, detail="Need not found")
-
-    if need in contact.needs:
-        contact.needs.remove(need)
-        db.commit()
-
-    if _is_htmx(request):
-        return HTMLResponse("")
-    return {"removed": True}
-
-
-@router.post(
-    "/contacts/{contact_id}/relationships",
-    response_model=ContactRelationshipResponse,
-)
-def add_contact_relationship(
-    contact_id: int,
-    relationship: ContactRelationshipCreate,
-    db: DbSession,
-) -> ContactRelationship:
-    """Add a relationship between two contacts."""
-    contact = db.get(Contact, contact_id)
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    related_contact = db.get(Contact, relationship.related_contact_id)
-    if not related_contact:
-        raise HTTPException(status_code=404, detail="Related contact not found")
-
-    if contact_id == relationship.related_contact_id:
-        raise HTTPException(status_code=400, detail="Cannot relate contact to itself")
-
-    # Use provided weight or default from relationship type
-    weight = relationship.closeness_weight
-    if weight is None:
-        weight = relationship.relationship_type.default_weight
-
-    db_relationship = ContactRelationship(
-        contact_id=contact_id,
-        related_contact_id=relationship.related_contact_id,
-        relationship_type=relationship.relationship_type.value,
-        closeness_weight=weight,
-    )
-    db.add(db_relationship)
-    db.commit()
-    db.refresh(db_relationship)
-    return db_relationship
-
-
-@router.get(
-    "/contacts/{contact_id}/relationships",
-    response_model=list[ContactRelationshipResponse],
-)
-def get_contact_relationships(
-    contact_id: int,
-    db: DbSession,
-) -> list[ContactRelationship]:
-    """Get all relationships for a contact."""
-    contact = db.get(Contact, contact_id)
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    outgoing = list(db.scalars(select(ContactRelationship).where(ContactRelationship.contact_id == contact_id)).all())
-    incoming = list(
-        db.scalars(select(ContactRelationship).where(ContactRelationship.related_contact_id == contact_id)).all()
-    )
-    return outgoing + incoming
-
-
-@router.patch(
-    "/contacts/{contact_id}/relationships/{related_contact_id}",
-    response_model=ContactRelationshipResponse,
-)
-def update_contact_relationship(
-    contact_id: int,
-    related_contact_id: int,
-    update: ContactRelationshipUpdate,
-    db: DbSession,
-) -> ContactRelationship:
-    """Update a relationship between two contacts."""
-    relationship = db.scalar(
-        select(ContactRelationship).where(
-            ContactRelationship.contact_id == contact_id,
-            ContactRelationship.related_contact_id == related_contact_id,
-        )
-    )
-    if not relationship:
-        raise HTTPException(status_code=404, detail="Relationship not found")
-
-    if update.relationship_type is not None:
-        relationship.relationship_type = update.relationship_type.value
-    if update.closeness_weight is not None:
-        relationship.closeness_weight = update.closeness_weight
-
-    db.commit()
-    db.refresh(relationship)
-    return relationship
-
-
-@router.delete("/contacts/{contact_id}/relationships/{related_contact_id}", response_model=None)
-def remove_contact_relationship(
-    contact_id: int,
-    related_contact_id: int,
-    request: Request,
-    db: DbSession,
-) -> dict[str, bool] | HTMLResponse:
-    """Remove a relationship between two contacts."""
-    relationship = db.scalar(
-        select(ContactRelationship).where(
-            ContactRelationship.contact_id == contact_id,
-            ContactRelationship.related_contact_id == related_contact_id,
-        )
-    )
-    if not relationship:
-        raise HTTPException(status_code=404, detail="Relationship not found")
-
-    db.delete(relationship)
-    db.commit()
-    if _is_htmx(request):
-        return HTMLResponse("")
-    return {"deleted": True}
-
-
-@router.get("/relationship-types")
-def list_relationship_types() -> list[RelationshipTypeInfo]:
-    """List all available relationship types with their default weights."""
-    return [
-        RelationshipTypeInfo(
-            value=rt.value,
-            display_name=rt.display_name,
-            default_weight=rt.default_weight,
-        )
-        for rt in RelationshipType
-    ]
-
-
-@router.get("/graph")
-def get_relationship_graph(db: DbSession) -> GraphData:
-    """Get all contacts and relationships as graph data for visualization."""
-    contacts = list(db.scalars(select(Contact)).all())
-    relationships = list(db.scalars(select(ContactRelationship)).all())
-
-    nodes = [GraphNode(id=c.id, name=c.name, current_job=c.current_job) for c in contacts]
-
-    edges = [
-        GraphEdge(
-            source=rel.contact_id,
-            target=rel.related_contact_id,
-            relationship_type=rel.relationship_type,
-            closeness_weight=rel.closeness_weight,
-        )
-        for rel in relationships
-    ]
-
-    return GraphData(nodes=nodes, edges=edges)

python/api/routers/views.py

@@ -1,345 +0,0 @@
-"""HTMX server-rendered view router."""
-
-from pathlib import Path
-from typing import Annotated, Any
-
-from fastapi import APIRouter, Form, HTTPException, Request
-from fastapi.responses import HTMLResponse, RedirectResponse
-from fastapi.templating import Jinja2Templates
-from sqlalchemy import select
-from sqlalchemy.orm import Session, selectinload
-
-from python.api.dependencies import DbSession
-from python.orm.richie.contact import Contact, ContactRelationship, Need, RelationshipType
-
-TEMPLATES_DIR = Path(__file__).parent.parent / "templates"
-templates = Jinja2Templates(directory=TEMPLATES_DIR)
-
-router = APIRouter(tags=["views"])
-
-FAMILIAL_TYPES = {
-    "parent",
-    "child",
-    "sibling",
-    "grandparent",
-    "grandchild",
-    "aunt_uncle",
-    "niece_nephew",
-    "cousin",
-    "in_law",
-}
-FRIEND_TYPES = {"best_friend", "close_friend", "friend", "acquaintance", "neighbor"}
-PARTNER_TYPES = {"spouse", "partner"}
-PROFESSIONAL_TYPES = {"mentor", "mentee", "business_partner", "colleague", "manager", "direct_report", "client"}
-
-CONTACT_STRING_FIELDS = (
-    "name",
-    "legal_name",
-    "suffix",
-    "gender",
-    "current_job",
-    "timezone",
-    "profile_pic",
-    "bio",
-    "goals",
-    "social_structure_style",
-    "safe_conversation_starters",
-    "topics_to_avoid",
-    "ssn",
-)
-
-CONTACT_INT_FIELDS = ("age", "self_sufficiency_score")
-
-
-def _group_relationships(relationships: list[ContactRelationship]) -> dict[str, list[ContactRelationship]]:
-    """Group relationships by category."""
-    groups: dict[str, list[ContactRelationship]] = {
-        "familial": [],
-        "partners": [],
-        "friends": [],
-        "professional": [],
-        "other": [],
-    }
-    for rel in relationships:
-        if rel.relationship_type in FAMILIAL_TYPES:
-            groups["familial"].append(rel)
-        elif rel.relationship_type in PARTNER_TYPES:
-            groups["partners"].append(rel)
-        elif rel.relationship_type in FRIEND_TYPES:
-            groups["friends"].append(rel)
-        elif rel.relationship_type in PROFESSIONAL_TYPES:
-            groups["professional"].append(rel)
-        else:
-            groups["other"].append(rel)
-    return groups
-
-
-def _build_contact_name_map(database: Session, contact: Contact) -> dict[int, str]:
-    """Build a mapping of contact IDs to names for relationship display."""
-    related_ids = {rel.related_contact_id for rel in contact.related_to}
-    related_ids |= {rel.contact_id for rel in contact.related_from}
-    related_ids.discard(contact.id)
-
-    if not related_ids:
-        return {}
-
-    related_contacts = list(database.scalars(select(Contact).where(Contact.id.in_(related_ids))).all())
-    return {related.id: related.name for related in related_contacts}
-
-
-def _get_relationship_type_display() -> dict[str, str]:
-    """Build a mapping of relationship type values to display names."""
-    return {rel_type.value: rel_type.display_name for rel_type in RelationshipType}
-
-
-async def _parse_contact_form(request: Request) -> dict[str, Any]:
-    """Parse contact form data from a multipart/form request."""
-    form_data = await request.form()
-    result: dict[str, Any] = {}
-
-    for field in CONTACT_STRING_FIELDS:
-        value = form_data.get(field, "")
-        result[field] = str(value) if value else None
-
-    for field in CONTACT_INT_FIELDS:
-        value = form_data.get(field, "")
-        result[field] = int(value) if value else None
-
-    result["need_ids"] = [int(value) for value in form_data.getlist("need_ids")]
-    return result
-
-
-def _save_contact_from_form(database: Session, contact: Contact, form_result: dict[str, Any]) -> None:
-    """Apply parsed form data to a Contact and save associated needs."""
-    need_ids = form_result.pop("need_ids")
-
-    for key, value in form_result.items():
-        setattr(contact, key, value)
-
-    if need_ids:
-        contact.needs = list(database.scalars(select(Need).where(Need.id.in_(need_ids))).all())
-    else:
-        contact.needs = []
-
-
-@router.get("/", response_class=HTMLResponse)
-@router.get("/contacts", response_class=HTMLResponse)
-def contact_list_page(request: Request, database: DbSession) -> HTMLResponse:
-    """Render the contacts list page."""
-    contacts = list(database.scalars(select(Contact)).all())
-    return templates.TemplateResponse(request, "contact_list.html", {"contacts": contacts})
-
-
-@router.get("/contacts/new", response_class=HTMLResponse)
-def new_contact_page(request: Request, database: DbSession) -> HTMLResponse:
-    """Render the new contact form page."""
-    all_needs = list(database.scalars(select(Need)).all())
-    return templates.TemplateResponse(request, "contact_form.html", {"contact": None, "all_needs": all_needs})
-
-
-@router.post("/htmx/contacts/new")
-async def create_contact_form(request: Request, database: DbSession) -> RedirectResponse:
-    """Handle the create contact form submission."""
-    form_result = await _parse_contact_form(request)
-    contact = Contact()
-    _save_contact_from_form(database, contact, form_result)
-
-    database.add(contact)
-    database.commit()
-    database.refresh(contact)
-    return RedirectResponse(url=f"/contacts/{contact.id}", status_code=303)
-
-
-@router.get("/contacts/{contact_id}", response_class=HTMLResponse)
-def contact_detail_page(contact_id: int, request: Request, database: DbSession) -> HTMLResponse:
-    """Render the contact detail page."""
-    contact = database.scalar(
-        select(Contact)
-        .where(Contact.id == contact_id)
-        .options(
-            selectinload(Contact.needs),
-            selectinload(Contact.related_to),
-            selectinload(Contact.related_from),
-        )
-    )
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    contact_names = _build_contact_name_map(database, contact)
-    grouped_relationships = _group_relationships(contact.related_to)
-    all_contacts = list(database.scalars(select(Contact)).all())
-    all_needs = list(database.scalars(select(Need)).all())
-    available_needs = [need for need in all_needs if need not in contact.needs]
-
-    return templates.TemplateResponse(
-        request,
-        "contact_detail.html",
-        {
-            "contact": contact,
-            "contact_names": contact_names,
-            "grouped_relationships": grouped_relationships,
-            "all_contacts": all_contacts,
-            "available_needs": available_needs,
-            "relationship_types": list(RelationshipType),
-        },
-    )
-
-
-@router.get("/contacts/{contact_id}/edit", response_class=HTMLResponse)
-def edit_contact_page(contact_id: int, request: Request, database: DbSession) -> HTMLResponse:
-    """Render the edit contact form page."""
-    contact = database.scalar(select(Contact).where(Contact.id == contact_id).options(selectinload(Contact.needs)))
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    all_needs = list(database.scalars(select(Need)).all())
-    return templates.TemplateResponse(request, "contact_form.html", {"contact": contact, "all_needs": all_needs})
-
-
-@router.post("/htmx/contacts/{contact_id}/edit")
-async def update_contact_form(contact_id: int, request: Request, database: DbSession) -> RedirectResponse:
-    """Handle the edit contact form submission."""
-    contact = database.get(Contact, contact_id)
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    form_result = await _parse_contact_form(request)
-    _save_contact_from_form(database, contact, form_result)
-
-    database.commit()
-    return RedirectResponse(url=f"/contacts/{contact_id}", status_code=303)
-
-
-@router.post("/htmx/contacts/{contact_id}/add-need", response_class=HTMLResponse)
-def add_need_to_contact_htmx(
-    contact_id: int,
-    request: Request,
-    database: DbSession,
-    need_id: Annotated[int, Form()],
-) -> HTMLResponse:
-    """Add a need to a contact and return updated manage-needs partial."""
-    contact = database.scalar(select(Contact).where(Contact.id == contact_id).options(selectinload(Contact.needs)))
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    need = database.get(Need, need_id)
-    if not need:
-        raise HTTPException(status_code=404, detail="Need not found")
-
-    if need not in contact.needs:
-        contact.needs.append(need)
-        database.commit()
-        database.refresh(contact)
-
-    return templates.TemplateResponse(request, "partials/manage_needs.html", {"contact": contact})
-
-
-@router.post("/htmx/contacts/{contact_id}/add-relationship", response_class=HTMLResponse)
-def add_relationship_htmx(
-    contact_id: int,
-    request: Request,
-    database: DbSession,
-    related_contact_id: Annotated[int, Form()],
-    relationship_type: Annotated[str, Form()],
-) -> HTMLResponse:
-    """Add a relationship and return updated manage-relationships partial."""
-    contact = database.scalar(select(Contact).where(Contact.id == contact_id).options(selectinload(Contact.related_to)))
-    if not contact:
-        raise HTTPException(status_code=404, detail="Contact not found")
-
-    related_contact = database.get(Contact, related_contact_id)
-    if not related_contact:
-        raise HTTPException(status_code=404, detail="Related contact not found")
-
-    rel_type = RelationshipType(relationship_type)
-    weight = rel_type.default_weight
-
-    relationship = ContactRelationship(
-        contact_id=contact_id,
-        related_contact_id=related_contact_id,
-        relationship_type=relationship_type,
-        closeness_weight=weight,
-    )
-    database.add(relationship)
-    database.commit()
-    database.refresh(contact)
-
-    contact_names = _build_contact_name_map(database, contact)
-    return templates.TemplateResponse(
-        request,
-        "partials/manage_relationships.html",
-        {"contact": contact, "contact_names": contact_names},
-    )
-
-
-@router.post("/htmx/contacts/{contact_id}/relationships/{related_contact_id}/weight")
-def update_relationship_weight_htmx(
-    contact_id: int,
-    related_contact_id: int,
-    database: DbSession,
-    closeness_weight: Annotated[int, Form()],
-) -> HTMLResponse:
-    """Update a relationship's closeness weight from HTMX range input."""
-    relationship = database.scalar(
-        select(ContactRelationship).where(
-            ContactRelationship.contact_id == contact_id,
-            ContactRelationship.related_contact_id == related_contact_id,
-        )
-    )
-    if not relationship:
-        raise HTTPException(status_code=404, detail="Relationship not found")
-
-    relationship.closeness_weight = closeness_weight
-    database.commit()
-    return HTMLResponse("")
-
-
-@router.post("/htmx/needs", response_class=HTMLResponse)
-def create_need_htmx(
-    request: Request,
-    database: DbSession,
-    name: Annotated[str, Form()],
-    description: Annotated[str, Form()] = "",
-) -> HTMLResponse:
-    """Create a need via form data and return updated needs list."""
-    need = Need(name=name, description=description or None)
-    database.add(need)
-    database.commit()
-    needs = list(database.scalars(select(Need)).all())
-    return templates.TemplateResponse(request, "partials/need_items.html", {"needs": needs})
-
-
-@router.get("/needs", response_class=HTMLResponse)
-def needs_page(request: Request, database: DbSession) -> HTMLResponse:
-    """Render the needs list page."""
-    needs = list(database.scalars(select(Need)).all())
-    return templates.TemplateResponse(request, "need_list.html", {"needs": needs})
-
-
-@router.get("/graph", response_class=HTMLResponse)
-def graph_page(request: Request, database: DbSession) -> HTMLResponse:
-    """Render the relationship graph page."""
-    contacts = list(database.scalars(select(Contact)).all())
-    relationships = list(database.scalars(select(ContactRelationship)).all())
-
-    graph_data = {
-        "nodes": [{"id": contact.id, "name": contact.name, "current_job": contact.current_job} for contact in contacts],
-        "edges": [
-            {
-                "source": rel.contact_id,
-                "target": rel.related_contact_id,
-                "relationship_type": rel.relationship_type,
-                "closeness_weight": rel.closeness_weight,
-            }
-            for rel in relationships
-        ],
-    }
-
-    return templates.TemplateResponse(
-        request,
-        "graph.html",
-        {
-            "graph_data": graph_data,
-            "relationship_type_display": _get_relationship_type_display(),
-        },
-    )

python/api/templates/base.html

@@ -1,198 +0,0 @@
-<!DOCTYPE html>
-<html lang="en" data-theme="light">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>{% block title %}Contact Database{% endblock %}</title>
-    <script src="https://unpkg.com/htmx.org@2.0.4"></script>
-    <style>
-        :root {
-            --color-bg: #f5f5f5;
-            --color-bg-card: #ffffff;
-            --color-bg-hover: #f0f0f0;
-            --color-bg-muted: #f9f9f9;
-            --color-bg-error: #ffe0e0;
-            --color-text: #333333;
-            --color-text-muted: #666666;
-            --color-text-error: #cc0000;
-            --color-border: #dddddd;
-            --color-border-light: #eeeeee;
-            --color-border-lighter: #f0f0f0;
-            --color-primary: #0066cc;
-            --color-primary-hover: #0055aa;
-            --color-danger: #cc3333;
-            --color-danger-hover: #aa2222;
-            --color-tag-bg: #e0e0e0;
-            --shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
-            font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
-            line-height: 1.5;
-            color: var(--color-text);
-            background-color: var(--color-bg);
-        }
-        [data-theme="dark"] {
-            --color-bg: #1a1a1a;
-            --color-bg-card: #2d2d2d;
-            --color-bg-hover: #3d3d3d;
-            --color-bg-muted: #252525;
-            --color-bg-error: #4a2020;
-            --color-text: #e0e0e0;
-            --color-text-muted: #a0a0a0;
-            --color-text-error: #ff6b6b;
-            --color-border: #404040;
-            --color-border-light: #353535;
-            --color-border-lighter: #303030;
-            --color-primary: #4da6ff;
-            --color-primary-hover: #7dbfff;
-            --color-danger: #ff6b6b;
-            --color-danger-hover: #ff8a8a;
-            --color-tag-bg: #404040;
-            --shadow: 0 1px 3px rgba(0, 0, 0, 0.3);
-        }
-        * { box-sizing: border-box; }
-        body { margin: 0; background: var(--color-bg); color: var(--color-text); }
-        .app { max-width: 1000px; margin: 0 auto; padding: 20px; }
-        nav { display: flex; align-items: center; gap: 20px; padding: 15px 0; border-bottom: 1px solid var(--color-border); margin-bottom: 20px; }
-        nav a { color: var(--color-primary); text-decoration: none; font-weight: 500; }
-        nav a:hover { text-decoration: underline; }
-        .theme-toggle { margin-left: auto; }
-        main { background: var(--color-bg-card); padding: 20px; border-radius: 8px; box-shadow: var(--shadow); }
-        .header { display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px; }
-        .header h1 { margin: 0; }
-        a { color: var(--color-primary); }
-        a:hover { text-decoration: underline; }
-
-        .btn { display: inline-block; padding: 8px 16px; border: 1px solid var(--color-border); border-radius: 4px; background: var(--color-bg-card); color: var(--color-text); text-decoration: none; cursor: pointer; font-size: 14px; margin-left: 8px; }
-        .btn:hover { background: var(--color-bg-hover); }
-        .btn-primary { background: var(--color-primary); border-color: var(--color-primary); color: white; }
-        .btn-primary:hover { background: var(--color-primary-hover); }
-        .btn-danger { background: var(--color-danger); border-color: var(--color-danger); color: white; }
-        .btn-danger:hover { background: var(--color-danger-hover); }
-        .btn-small { padding: 4px 8px; font-size: 12px; }
-        .btn:disabled { opacity: 0.6; cursor: not-allowed; }
-
-        table { width: 100%; border-collapse: collapse; }
-        th, td { padding: 12px; text-align: left; border-bottom: 1px solid var(--color-border-light); }
-        th { font-weight: 600; background: var(--color-bg-muted); }
-        tr:hover { background: var(--color-bg-muted); }
-
-        .error { background: var(--color-bg-error); color: var(--color-text-error); padding: 10px; border-radius: 4px; margin-bottom: 20px; }
-        .tag { display: inline-block; background: var(--color-tag-bg); padding: 2px 8px; border-radius: 12px; font-size: 12px; color: var(--color-text-muted); }
-
-        .add-form { display: flex; gap: 10px; margin-top: 15px; flex-wrap: wrap; }
-        .add-form select, .add-form input { padding: 8px; border: 1px solid var(--color-border); border-radius: 4px; min-width: 200px; background: var(--color-bg-card); color: var(--color-text); }
-
-        .form-group { margin-bottom: 20px; }
-        .form-group label { display: block; font-weight: 500; margin-bottom: 5px; }
-        .form-group input, .form-group textarea, .form-group select { width: 100%; padding: 10px; border: 1px solid var(--color-border); border-radius: 4px; font-size: 14px; background: var(--color-bg-card); color: var(--color-text); }
-        .form-group textarea { resize: vertical; }
-        .form-row { display: grid; grid-template-columns: 1fr 1fr; gap: 20px; }
-        .checkbox-group { display: flex; flex-wrap: wrap; gap: 15px; }
-        .checkbox-label { display: flex; align-items: center; gap: 5px; cursor: pointer; }
-        .form-actions { display: flex; gap: 10px; margin-top: 30px; padding-top: 20px; border-top: 1px solid var(--color-border-light); }
-
-        .need-form { background: var(--color-bg-muted); padding: 20px; border-radius: 4px; margin-bottom: 20px; }
-        .need-items { list-style: none; padding: 0; }
-        .need-items li { display: flex; justify-content: space-between; align-items: flex-start; padding: 15px; border: 1px solid var(--color-border-light); border-radius: 4px; margin-bottom: 10px; }
-        .need-info p { margin: 5px 0 0; color: var(--color-text-muted); font-size: 14px; }
-
-        .graph-container { width: 100%; }
-        .graph-hint { color: var(--color-text-muted); font-size: 14px; margin-bottom: 15px; }
-        .selected-info { margin-top: 15px; padding: 15px; background: var(--color-bg-muted); border-radius: 8px; }
-        .selected-info h3 { margin: 0 0 10px; }
-        .selected-info p { margin: 5px 0; color: var(--color-text-muted); }
-        .legend { margin-top: 20px; padding: 15px; background: var(--color-bg-muted); border-radius: 8px; }
-        .legend h4 { margin: 0 0 10px; font-size: 14px; }
-        .legend-items { display: flex; flex-wrap: wrap; gap: 15px; }
-        .legend-item { display: flex; align-items: center; gap: 8px; font-size: 12px; color: var(--color-text-muted); }
-        .legend-line { width: 30px; border-radius: 2px; }
-
-        .id-card { width: 100%; }
-        .id-card-inner { background: linear-gradient(135deg, #0a0a0f 0%, #1a1a2e 50%, #0a0a0f 100%); background-image: radial-gradient(white 1px, transparent 1px), linear-gradient(135deg, #0a0a0f 0%, #1a1a2e 50%, #0a0a0f 100%); background-size: 50px 50px, 100% 100%; color: #fff; border-radius: 12px; padding: 25px; min-height: 500px; position: relative; overflow: hidden; }
-        .id-card-header { display: flex; justify-content: space-between; align-items: flex-start; margin-bottom: 15px; }
-        .id-card-header-left { flex: 1; }
-        .id-card-header-right { display: flex; flex-direction: column; align-items: flex-end; gap: 10px; }
-        .id-card-title { font-size: 2.5rem; font-weight: 700; margin: 0; color: #fff; text-shadow: 2px 2px 4px rgba(0,0,0,0.5); }
-        .id-profile-pic { width: 80px; height: 80px; border-radius: 8px; object-fit: cover; border: 2px solid rgba(255,255,255,0.3); }
-        .id-profile-placeholder { width: 80px; height: 80px; border-radius: 8px; background: linear-gradient(135deg, #4ecdc4 0%, #44a8a0 100%); display: flex; align-items: center; justify-content: center; border: 2px solid rgba(255,255,255,0.3); }
-        .id-profile-placeholder span { font-size: 2rem; font-weight: 700; color: #fff; text-shadow: 1px 1px 2px rgba(0,0,0,0.3); }
-        .id-card-actions { display: flex; gap: 8px; }
-        .id-card-actions .btn { background: rgba(255,255,255,0.1); border-color: rgba(255,255,255,0.3); color: #fff; }
-        .id-card-actions .btn:hover { background: rgba(255,255,255,0.2); }
-        .id-card-body { display: grid; grid-template-columns: 1fr 1.5fr; gap: 30px; }
-        .id-card-left { display: flex; flex-direction: column; gap: 8px; }
-        .id-field { font-size: 1rem; line-height: 1.4; }
-        .id-field-block { margin-top: 15px; font-size: 0.95rem; line-height: 1.5; }
-        .id-label { color: #4ecdc4; font-weight: 500; }
-        .id-card-right { display: flex; flex-direction: column; gap: 20px; }
-        .id-bio { font-size: 0.9rem; line-height: 1.6; color: #e0e0e0; }
-        .id-relationships { margin-top: 10px; }
-        .id-section-title { font-size: 1.5rem; margin: 0 0 15px; color: #fff; border-bottom: 1px solid rgba(255,255,255,0.2); padding-bottom: 8px; }
-        .id-rel-group { margin-bottom: 12px; font-size: 0.9rem; line-height: 1.6; }
-        .id-rel-label { color: #a0a0a0; }
-        .id-rel-group a { color: #4ecdc4; text-decoration: none; }
-        .id-rel-group a:hover { text-decoration: underline; }
-        .id-rel-type { color: #888; font-size: 0.85em; }
-        .id-card-warnings { margin-top: 30px; padding-top: 20px; border-top: 1px solid rgba(255,255,255,0.2); display: flex; flex-wrap: wrap; gap: 20px; }
-        .id-warning { display: flex; align-items: center; gap: 8px; font-size: 0.9rem; color: #ff6b6b; }
-        .warning-dot { width: 8px; height: 8px; background: #ff6b6b; border-radius: 50%; flex-shrink: 0; }
-        .warning-desc { color: #ccc; }
-
-        .id-card-manage { margin-top: 20px; background: var(--color-bg-muted); border-radius: 8px; padding: 15px; }
-        .id-card-manage summary { cursor: pointer; font-weight: 600; font-size: 1.1rem; padding: 5px 0; }
-        .id-card-manage[open] summary { margin-bottom: 15px; border-bottom: 1px solid var(--color-border-light); padding-bottom: 10px; }
-        .manage-section { margin-bottom: 25px; }
-        .manage-section h3 { margin: 0 0 15px; font-size: 1rem; }
-        .manage-relationships { display: flex; flex-direction: column; gap: 10px; margin-bottom: 15px; }
-        .manage-rel-item { display: flex; align-items: center; gap: 12px; padding: 10px; background: var(--color-bg-card); border-radius: 6px; flex-wrap: wrap; }
-        .manage-rel-item a { font-weight: 500; min-width: 120px; }
-        .weight-control { display: flex; align-items: center; gap: 8px; font-size: 12px; color: var(--color-text-muted); }
-        .weight-control input[type="range"] { width: 80px; cursor: pointer; }
-        .weight-value { min-width: 20px; text-align: center; font-weight: 600; }
-        .manage-needs-list { list-style: none; padding: 0; margin: 0 0 15px; }
-        .manage-needs-list li { display: flex; align-items: center; gap: 12px; padding: 10px; background: var(--color-bg-card); border-radius: 6px; margin-bottom: 8px; }
-        .manage-needs-list li .btn { margin-left: auto; }
-
-        .htmx-indicator { display: none; }
-        .htmx-request .htmx-indicator { display: inline; }
-        .htmx-request.htmx-indicator { display: inline; }
-
-        @media (max-width: 768px) {
-            .id-card-body { grid-template-columns: 1fr; }
-            .id-card-title { font-size: 1.8rem; }
-            .id-card-header { flex-direction: column; gap: 15px; }
-        }
-    </style>
-</head>
-<body>
-    <div class="app">
-        <nav>
-            <a href="/contacts">Contacts</a>
-            <a href="/graph">Graph</a>
-            <a href="/needs">Needs</a>
-            <button class="btn btn-small theme-toggle" onclick="toggleTheme()">
-                <span id="theme-label">Dark</span>
-            </button>
-        </nav>
-
-        <main id="main-content">
-            {% block content %}{% endblock %}
-        </main>
-    </div>
-
-    <script>
-        function toggleTheme() {
-            const html = document.documentElement;
-            const current = html.getAttribute('data-theme');
-            const next = current === 'light' ? 'dark' : 'light';
-            html.setAttribute('data-theme', next);
-            localStorage.setItem('theme', next);
-            document.getElementById('theme-label').textContent = next === 'light' ? 'Dark' : 'Light';
-        }
-        (function() {
-            const saved = localStorage.getItem('theme') || 'light';
-            document.documentElement.setAttribute('data-theme', saved);
-            document.getElementById('theme-label').textContent = saved === 'light' ? 'Dark' : 'Light';
-        })();
-    </script>
-</body>
-</html>

python/api/templates/contact_detail.html

@@ -1,204 +0,0 @@
-{% extends "base.html" %}
-{% block title %}{{ contact.name }}{% endblock %}
-{% block content %}
-<div class="id-card">
-    <div class="id-card-inner">
-        <div class="id-card-header">
-            <div class="id-card-header-left">
-                <h1 class="id-card-title">I.D.: {{ contact.name }}</h1>
-            </div>
-            <div class="id-card-header-right">
-                {% if contact.profile_pic %}
-                <img src="{{ contact.profile_pic }}" alt="{{ contact.name }}'s profile" class="id-profile-pic">
-                {% else %}
-                <div class="id-profile-placeholder">
-                    <span>{{ contact.name[0]|upper }}</span>
-                </div>
-                {% endif %}
-                <div class="id-card-actions">
-                    <a href="/contacts/{{ contact.id }}/edit" class="btn btn-small">Edit</a>
-                    <a href="/contacts" class="btn btn-small">Back</a>
-                </div>
-            </div>
-        </div>
-
-        <div class="id-card-body">
-            <div class="id-card-left">
-                {% if contact.legal_name %}
-                <div class="id-field">Legal name: {{ contact.legal_name }}</div>
-                {% endif %}
-                {% if contact.suffix %}
-                <div class="id-field">Suffix: {{ contact.suffix }}</div>
-                {% endif %}
-                {% if contact.gender %}
-                <div class="id-field">Gender: {{ contact.gender }}</div>
-                {% endif %}
-                {% if contact.age %}
-                <div class="id-field">Age: {{ contact.age }}</div>
-                {% endif %}
-                {% if contact.current_job %}
-                <div class="id-field">Job: {{ contact.current_job }}</div>
-                {% endif %}
-                {% if contact.social_structure_style %}
-                <div class="id-field">Social style: {{ contact.social_structure_style }}</div>
-                {% endif %}
-                {% if contact.self_sufficiency_score is not none %}
-                <div class="id-field">Self-Sufficiency: {{ contact.self_sufficiency_score }}</div>
-                {% endif %}
-                {% if contact.timezone %}
-                <div class="id-field">Timezone: {{ contact.timezone }}</div>
-                {% endif %}
-                {% if contact.safe_conversation_starters %}
-                <div class="id-field-block">
-                    <span class="id-label">Safe con starters:</span> {{ contact.safe_conversation_starters }}
-                </div>
-                {% endif %}
-                {% if contact.topics_to_avoid %}
-                <div class="id-field-block">
-                    <span class="id-label">Topics to avoid:</span> {{ contact.topics_to_avoid }}
-                </div>
-                {% endif %}
-                {% if contact.goals %}
-                <div class="id-field-block">
-                    <span class="id-label">Goals:</span> {{ contact.goals }}
-                </div>
-                {% endif %}
-            </div>
-
-            <div class="id-card-right">
-                {% if contact.bio %}
-                <div class="id-bio">
-                    <span class="id-label">Bio:</span> {{ contact.bio }}
-                </div>
-                {% endif %}
-
-                <div class="id-relationships">
-                    <h2 class="id-section-title">Relationships</h2>
-
-                    {% if grouped_relationships.familial %}
-                    <div class="id-rel-group">
-                        <span class="id-rel-label">Familial:</span>
-                        {% for rel in grouped_relationships.familial %}
-                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a><span class="id-rel-type">({{ rel.relationship_type|replace("_", " ")|title }})</span>{% if not loop.last %}, {% endif %}
-                        {% endfor %}
-                    </div>
-                    {% endif %}
-
-                    {% if grouped_relationships.partners %}
-                    <div class="id-rel-group">
-                        <span class="id-rel-label">Partners:</span>
-                        {% for rel in grouped_relationships.partners %}
-                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a>{% if not loop.last %}, {% endif %}
-                        {% endfor %}
-                    </div>
-                    {% endif %}
-
-                    {% if grouped_relationships.friends %}
-                    <div class="id-rel-group">
-                        <span class="id-rel-label">Friends:</span>
-                        {% for rel in grouped_relationships.friends %}
-                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a>{% if not loop.last %}, {% endif %}
-                        {% endfor %}
-                    </div>
-                    {% endif %}
-
-                    {% if grouped_relationships.professional %}
-                    <div class="id-rel-group">
-                        <span class="id-rel-label">Professional:</span>
-                        {% for rel in grouped_relationships.professional %}
-                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a><span class="id-rel-type">({{ rel.relationship_type|replace("_", " ")|title }})</span>{% if not loop.last %}, {% endif %}
-                        {% endfor %}
-                    </div>
-                    {% endif %}
-
-                    {% if grouped_relationships.other %}
-                    <div class="id-rel-group">
-                        <span class="id-rel-label">Other:</span>
-                        {% for rel in grouped_relationships.other %}
-                        <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a><span class="id-rel-type">({{ rel.relationship_type|replace("_", " ")|title }})</span>{% if not loop.last %}, {% endif %}
-                        {% endfor %}
-                    </div>
-                    {% endif %}
-
-                    {% if contact.related_from %}
-                    <div class="id-rel-group">
-                        <span class="id-rel-label">Known by:</span>
-                        {% for rel in contact.related_from %}
-                        <a href="/contacts/{{ rel.contact_id }}">{{ contact_names[rel.contact_id] }}</a>{% if not loop.last %}, {% endif %}
-                        {% endfor %}
-                    </div>
-                    {% endif %}
-                </div>
-            </div>
-        </div>
-
-        {% if contact.needs %}
-        <div class="id-card-warnings">
-            {% for need in contact.needs %}
-            <div class="id-warning">
-                <span class="warning-dot"></span>
-                Warning: {{ need.name }}
-                {% if need.description %}<span class="warning-desc"> - {{ need.description }}</span>{% endif %}
-            </div>
-            {% endfor %}
-        </div>
-        {% endif %}
-    </div>
-
-    <details class="id-card-manage">
-        <summary>Manage Contact</summary>
-
-        <div class="manage-section">
-            <h3>Manage Relationships</h3>
-            <div id="manage-relationships" class="manage-relationships">
-                {% include "partials/manage_relationships.html" %}
-            </div>
-
-            {% if all_contacts %}
-            <form hx-post="/htmx/contacts/{{ contact.id }}/add-relationship"
-                  hx-target="#manage-relationships"
-                  hx-swap="innerHTML"
-                  class="add-form">
-                <select name="related_contact_id" required>
-                    <option value="">Select contact...</option>
-                    {% for other in all_contacts %}
-                    {% if other.id != contact.id %}
-                    <option value="{{ other.id }}">{{ other.name }}</option>
-                    {% endif %}
-                    {% endfor %}
-                </select>
-                <select name="relationship_type" required>
-                    <option value="">Select relationship type...</option>
-                    {% for rel_type in relationship_types %}
-                    <option value="{{ rel_type.value }}">{{ rel_type.display_name }}</option>
-                    {% endfor %}
-                </select>
-                <button type="submit" class="btn btn-primary">Add Relationship</button>
-            </form>
-            {% endif %}
-        </div>
-
-        <div class="manage-section">
-            <h3>Manage Needs/Warnings</h3>
-            <div id="manage-needs">
-                {% include "partials/manage_needs.html" %}
-            </div>
-
-            {% if available_needs %}
-            <form hx-post="/htmx/contacts/{{ contact.id }}/add-need"
-                  hx-target="#manage-needs"
-                  hx-swap="innerHTML"
-                  class="add-form">
-                <select name="need_id" required>
-                    <option value="">Select a need...</option>
-                    {% for need in available_needs %}
-                    <option value="{{ need.id }}">{{ need.name }}</option>
-                    {% endfor %}
-                </select>
-                <button type="submit" class="btn btn-primary">Add Need</button>
-            </form>
-            {% endif %}
-        </div>
-    </details>
-</div>
-{% endblock %}

python/api/templates/contact_form.html

@@ -1,115 +0,0 @@
-{% extends "base.html" %}
-{% block title %}{{ "Edit " + contact.name if contact else "New Contact" }}{% endblock %}
-{% block content %}
-<div class="contact-form">
-    <h1>{{ "Edit Contact" if contact else "New Contact" }}</h1>
-
-    {% if contact %}
-    <form method="post" action="/htmx/contacts/{{ contact.id }}/edit">
-    {% else %}
-    <form method="post" action="/htmx/contacts/new">
-    {% endif %}
-
-        <div class="form-group">
-            <label for="name">Name *</label>
-            <input id="name" name="name" type="text" value="{{ contact.name if contact else '' }}" required>
-        </div>
-
-        <div class="form-row">
-            <div class="form-group">
-                <label for="legal_name">Legal Name</label>
-                <input id="legal_name" name="legal_name" type="text" value="{{ contact.legal_name or '' }}">
-            </div>
-            <div class="form-group">
-                <label for="suffix">Suffix</label>
-                <input id="suffix" name="suffix" type="text" value="{{ contact.suffix or '' }}">
-            </div>
-        </div>
-
-        <div class="form-row">
-            <div class="form-group">
-                <label for="age">Age</label>
-                <input id="age" name="age" type="number" value="{{ contact.age if contact and contact.age is not none else '' }}">
-            </div>
-            <div class="form-group">
-                <label for="gender">Gender</label>
-                <input id="gender" name="gender" type="text" value="{{ contact.gender or '' }}">
-            </div>
-        </div>
-
-        <div class="form-group">
-            <label for="current_job">Current Job</label>
-            <input id="current_job" name="current_job" type="text" value="{{ contact.current_job or '' }}">
-        </div>
-
-        <div class="form-group">
-            <label for="timezone">Timezone</label>
-            <input id="timezone" name="timezone" type="text" value="{{ contact.timezone or '' }}">
-        </div>
-
-        <div class="form-group">
-            <label for="profile_pic">Profile Picture URL</label>
-            <input id="profile_pic" name="profile_pic" type="url" placeholder="https://example.com/photo.jpg" value="{{ contact.profile_pic or '' }}">
-        </div>
-
-        <div class="form-group">
-            <label for="bio">Bio</label>
-            <textarea id="bio" name="bio" rows="3">{{ contact.bio or '' }}</textarea>
-        </div>
-
-        <div class="form-group">
-            <label for="goals">Goals</label>
-            <textarea id="goals" name="goals" rows="3">{{ contact.goals or '' }}</textarea>
-        </div>
-
-        <div class="form-group">
-            <label for="social_structure_style">Social Structure Style</label>
-            <input id="social_structure_style" name="social_structure_style" type="text" value="{{ contact.social_structure_style or '' }}">
-        </div>
-
-        <div class="form-group">
-            <label for="self_sufficiency_score">Self-Sufficiency Score (1-10)</label>
-            <input id="self_sufficiency_score" name="self_sufficiency_score" type="number" min="1" max="10" value="{{ contact.self_sufficiency_score if contact and contact.self_sufficiency_score is not none else '' }}">
-        </div>
-
-        <div class="form-group">
-            <label for="safe_conversation_starters">Safe Conversation Starters</label>
-            <textarea id="safe_conversation_starters" name="safe_conversation_starters" rows="2">{{ contact.safe_conversation_starters or '' }}</textarea>
-        </div>
-
-        <div class="form-group">
-            <label for="topics_to_avoid">Topics to Avoid</label>
-            <textarea id="topics_to_avoid" name="topics_to_avoid" rows="2">{{ contact.topics_to_avoid or '' }}</textarea>
-        </div>
-
-        <div class="form-group">
-            <label for="ssn">SSN</label>
-            <input id="ssn" name="ssn" type="text" value="{{ contact.ssn or '' }}">
-        </div>
-
-        {% if all_needs %}
-        <div class="form-group">
-            <label>Needs/Accommodations</label>
-            <div class="checkbox-group">
-                {% for need in all_needs %}
-                <label class="checkbox-label">
-                    <input type="checkbox" name="need_ids" value="{{ need.id }}"
-                           {% if contact and need in contact.needs %}checked{% endif %}>
-                    {{ need.name }}
-                </label>
-                {% endfor %}
-            </div>
-        </div>
-        {% endif %}
-
-        <div class="form-actions">
-            <button type="submit" class="btn btn-primary">Save</button>
-            {% if contact %}
-            <a href="/contacts/{{ contact.id }}" class="btn">Cancel</a>
-            {% else %}
-            <a href="/contacts" class="btn">Cancel</a>
-            {% endif %}
-        </div>
-    </form>
-</div>
-{% endblock %}

python/api/templates/contact_list.html

@@ -1,14 +0,0 @@
-{% extends "base.html" %}
-{% block title %}Contacts{% endblock %}
-{% block content %}
-<div class="contact-list">
-    <div class="header">
-        <h1>Contacts</h1>
-        <a href="/contacts/new" class="btn btn-primary">Add Contact</a>
-    </div>
-
-    <div id="contact-table">
-        {% include "partials/contact_table.html" %}
-    </div>
-</div>
-{% endblock %}

python/api/templates/graph.html

@@ -1,198 +0,0 @@
-{% extends "base.html" %}
-{% block title %}Relationship Graph{% endblock %}
-{% block content %}
-<div class="graph-container">
-    <div class="header">
-        <h1>Relationship Graph</h1>
-    </div>
-    <p class="graph-hint">Drag nodes to reposition. Closer relationships have shorter, darker edges.</p>
-    <canvas id="graph-canvas" width="900" height="600"
-            style="border: 1px solid var(--color-border); border-radius: 8px; background: var(--color-bg); cursor: grab;">
-    </canvas>
-    <div id="selected-info"></div>
-    <div class="legend">
-        <h4>Relationship Closeness (1-10)</h4>
-        <div class="legend-items">
-            <div class="legend-item">
-                <span class="legend-line" style="background: hsl(220, 70%, 40%); height: 4px; display: inline-block;"></span>
-                <span>10 - Very Close (Spouse, Partner)</span>
-            </div>
-            <div class="legend-item">
-                <span class="legend-line" style="background: hsl(220, 70%, 52%); height: 3px; display: inline-block;"></span>
-                <span>7 - Close (Family, Best Friend)</span>
-            </div>
-            <div class="legend-item">
-                <span class="legend-line" style="background: hsl(220, 70%, 64%); height: 2px; display: inline-block;"></span>
-                <span>4 - Moderate (Friend, Colleague)</span>
-            </div>
-            <div class="legend-item">
-                <span class="legend-line" style="background: hsl(220, 70%, 72%); height: 1px; display: inline-block;"></span>
-                <span>2 - Distant (Acquaintance)</span>
-            </div>
-        </div>
-    </div>
-</div>
-
-<script>
-(function() {
-    const RELATIONSHIP_DISPLAY = {{ relationship_type_display|tojson }};
-    const graphData = {{ graph_data|tojson }};
-
-    const canvas = document.getElementById('graph-canvas');
-    const ctx = canvas.getContext('2d');
-    const width = canvas.width;
-    const height = canvas.height;
-    const centerX = width / 2;
-    const centerY = height / 2;
-
-    const nodes = graphData.nodes.map(function(node) {
-        return Object.assign({}, node, {
-            x: centerX + (Math.random() - 0.5) * 300,
-            y: centerY + (Math.random() - 0.5) * 300,
-            vx: 0,
-            vy: 0
-        });
-    });
-
-    const nodeMap = new Map(nodes.map(function(node) { return [node.id, node]; }));
-
-    const edges = graphData.edges.map(function(edge) {
-        const sourceNode = nodeMap.get(edge.source);
-        const targetNode = nodeMap.get(edge.target);
-        if (!sourceNode || !targetNode) return null;
-        return Object.assign({}, edge, { sourceNode: sourceNode, targetNode: targetNode });
-    }).filter(function(edge) { return edge !== null; });
-
-    let dragNode = null;
-    let selectedNode = null;
-
-    const repulsion = 5000;
-    const springStrength = 0.05;
-    const baseSpringLength = 150;
-    const damping = 0.9;
-    const centerPull = 0.01;
-
-    function simulate() {
-        for (const node of nodes) { node.vx = 0; node.vy = 0; }
-        for (let i = 0; i < nodes.length; i++) {
-            for (let j = i + 1; j < nodes.length; j++) {
-                const dx = nodes[j].x - nodes[i].x;
-                const dy = nodes[j].y - nodes[i].y;
-                const dist = Math.sqrt(dx * dx + dy * dy) || 1;
-                const force = repulsion / (dist * dist);
-                const fx = (dx / dist) * force;
-                const fy = (dy / dist) * force;
-                nodes[i].vx -= fx; nodes[i].vy -= fy;
-                nodes[j].vx += fx; nodes[j].vy += fy;
-            }
-        }
-        for (const edge of edges) {
-            const dx = edge.targetNode.x - edge.sourceNode.x;
-            const dy = edge.targetNode.y - edge.sourceNode.y;
-            const dist = Math.sqrt(dx * dx + dy * dy) || 1;
-            const normalizedWeight = edge.closeness_weight / 10;
-            const idealLength = baseSpringLength * (1.5 - normalizedWeight);
-            const displacement = dist - idealLength;
-            const force = springStrength * displacement;
-            const fx = (dx / dist) * force;
-            const fy = (dy / dist) * force;
-            edge.sourceNode.vx += fx; edge.sourceNode.vy += fy;
-            edge.targetNode.vx -= fx; edge.targetNode.vy -= fy;
-        }
-        for (const node of nodes) {
-            node.vx += (centerX - node.x) * centerPull;
-            node.vy += (centerY - node.y) * centerPull;
-        }
-        for (const node of nodes) {
-            if (node === dragNode) continue;
-            node.x += node.vx * damping;
-            node.y += node.vy * damping;
-            node.x = Math.max(30, Math.min(width - 30, node.x));
-            node.y = Math.max(30, Math.min(height - 30, node.y));
-        }
-    }
-
-    function getEdgeColor(weight) {
-        const normalized = weight / 10;
-        return 'hsl(220, 70%, ' + (80 - normalized * 40) + '%)';
-    }
-
-    function draw() {
-        ctx.clearRect(0, 0, width, height);
-        for (const edge of edges) {
-            const lineWidth = 1 + (edge.closeness_weight / 10) * 3;
-            ctx.strokeStyle = getEdgeColor(edge.closeness_weight);
-            ctx.lineWidth = lineWidth;
-            ctx.beginPath();
-            ctx.moveTo(edge.sourceNode.x, edge.sourceNode.y);
-            ctx.lineTo(edge.targetNode.x, edge.targetNode.y);
-            ctx.stroke();
-            const midX = (edge.sourceNode.x + edge.targetNode.x) / 2;
-            const midY = (edge.sourceNode.y + edge.targetNode.y) / 2;
-            ctx.fillStyle = '#666';
-            ctx.font = '10px sans-serif';
-            ctx.textAlign = 'center';
-            const label = RELATIONSHIP_DISPLAY[edge.relationship_type] || edge.relationship_type;
-            ctx.fillText(label, midX, midY - 5);
-        }
-        for (const node of nodes) {
-            const isSelected = node === selectedNode;
-            const radius = isSelected ? 25 : 20;
-            ctx.beginPath();
-            ctx.arc(node.x, node.y, radius, 0, Math.PI * 2);
-            ctx.fillStyle = isSelected ? '#0066cc' : '#fff';
-            ctx.fill();
-            ctx.strokeStyle = '#0066cc';
-            ctx.lineWidth = 2;
-            ctx.stroke();
-            ctx.fillStyle = isSelected ? '#fff' : '#333';
-            ctx.font = '12px sans-serif';
-            ctx.textAlign = 'center';
-            ctx.textBaseline = 'middle';
-            const name = node.name.length > 10 ? node.name.slice(0, 9) + '\u2026' : node.name;
-            ctx.fillText(name, node.x, node.y);
-        }
-    }
-
-    function animate() {
-        simulate();
-        draw();
-        requestAnimationFrame(animate);
-    }
-    animate();
-
-    function getNodeAt(x, y) {
-        for (const node of nodes) {
-            const dx = x - node.x;
-            const dy = y - node.y;
-            if (dx * dx + dy * dy < 400) return node;
-        }
-        return null;
-    }
-
-    canvas.addEventListener('mousedown', function(event) {
-        const rect = canvas.getBoundingClientRect();
-        const node = getNodeAt(event.clientX - rect.left, event.clientY - rect.top);
-        if (node) {
-            dragNode = node;
-            selectedNode = node;
-            const infoDiv = document.getElementById('selected-info');
-            let html = '<div class="selected-info"><h3>' + node.name + '</h3>';
-            if (node.current_job) html += '<p>Job: ' + node.current_job + '</p>';
-            html += '<a href="/contacts/' + node.id + '">View details</a></div>';
-            infoDiv.innerHTML = html;
-        }
-    });
-
-    canvas.addEventListener('mousemove', function(event) {
-        if (!dragNode) return;
-        const rect = canvas.getBoundingClientRect();
-        dragNode.x = event.clientX - rect.left;
-        dragNode.y = event.clientY - rect.top;
-    });
-
-    canvas.addEventListener('mouseup', function() { dragNode = null; });
-    canvas.addEventListener('mouseleave', function() { dragNode = null; });
-})();
-</script>
-{% endblock %}

python/api/templates/need_list.html

@@ -1,31 +0,0 @@
-{% extends "base.html" %}
-{% block title %}Needs{% endblock %}
-{% block content %}
-<div class="need-list">
-    <div class="header">
-        <h1>Needs / Accommodations</h1>
-        <button class="btn btn-primary" onclick="document.getElementById('need-form').toggleAttribute('hidden')">Add Need</button>
-    </div>
-
-    <form id="need-form" hidden
-          hx-post="/htmx/needs"
-          hx-target="#need-items"
-          hx-swap="innerHTML"
-          hx-on::after-request="if(event.detail.successful) this.reset()"
-          class="need-form">
-        <div class="form-group">
-            <label for="name">Name *</label>
-            <input id="name" name="name" type="text" placeholder="e.g., Light Sensitive, ADHD" required>
-        </div>
-        <div class="form-group">
-            <label for="description">Description</label>
-            <textarea id="description" name="description" placeholder="Optional description..." rows="2"></textarea>
-        </div>
-        <button type="submit" class="btn btn-primary">Create</button>
-    </form>
-
-    <div id="need-items">
-        {% include "partials/need_items.html" %}
-    </div>
-</div>
-{% endblock %}

python/api/templates/partials/contact_table.html

@@ -1,33 +0,0 @@
-{% if contacts %}
-<table>
-    <thead>
-        <tr>
-            <th>Name</th>
-            <th>Job</th>
-            <th>Timezone</th>
-            <th>Actions</th>
-        </tr>
-    </thead>
-    <tbody>
-        {% for contact in contacts %}
-        <tr id="contact-row-{{ contact.id }}">
-            <td><a href="/contacts/{{ contact.id }}">{{ contact.name }}</a></td>
-            <td>{{ contact.current_job or "-" }}</td>
-            <td>{{ contact.timezone or "-" }}</td>
-            <td>
-                <a href="/contacts/{{ contact.id }}/edit" class="btn">Edit</a>
-                <button class="btn btn-danger"
-                        hx-delete="/api/contacts/{{ contact.id }}"
-                        hx-target="#contact-row-{{ contact.id }}"
-                        hx-swap="outerHTML"
-                        hx-confirm="Delete this contact?">
-                    Delete
-                </button>
-            </td>
-        </tr>
-        {% endfor %}
-    </tbody>
-</table>
-{% else %}
-<p>No contacts yet.</p>
-{% endif %}

python/api/templates/partials/manage_needs.html

@@ -1,14 +0,0 @@
-<ul class="manage-needs-list">
-    {% for need in contact.needs %}
-    <li id="contact-need-{{ need.id }}">
-        <strong>{{ need.name }}</strong>
-        {% if need.description %}<span> - {{ need.description }}</span>{% endif %}
-        <button class="btn btn-small btn-danger"
-                hx-delete="/api/contacts/{{ contact.id }}/needs/{{ need.id }}"
-                hx-target="#contact-need-{{ need.id }}"
-                hx-swap="outerHTML">
-            Remove
-        </button>
-    </li>
-    {% endfor %}
-</ul>

python/api/templates/partials/manage_relationships.html

@@ -1,23 +0,0 @@
-{% for rel in contact.related_to %}
-<div class="manage-rel-item" id="rel-{{ contact.id }}-{{ rel.related_contact_id }}">
-    <a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a>
-    <span class="tag">{{ rel.relationship_type|replace("_", " ")|title }}</span>
-    <label class="weight-control">
-        <span>Closeness:</span>
-        <input type="range" min="1" max="10" value="{{ rel.closeness_weight }}"
-               hx-post="/htmx/contacts/{{ contact.id }}/relationships/{{ rel.related_contact_id }}/weight"
-               hx-trigger="change"
-               hx-include="this"
-               name="closeness_weight"
-               hx-swap="none"
-               oninput="this.nextElementSibling.textContent = this.value">
-        <span class="weight-value">{{ rel.closeness_weight }}</span>
-    </label>
-    <button class="btn btn-small btn-danger"
-            hx-delete="/api/contacts/{{ contact.id }}/relationships/{{ rel.related_contact_id }}"
-            hx-target="#rel-{{ contact.id }}-{{ rel.related_contact_id }}"
-            hx-swap="outerHTML">
-        Remove
-    </button>
-</div>
-{% endfor %}

python/api/templates/partials/need_items.html

@@ -1,21 +0,0 @@
-{% if needs %}
-<ul class="need-items">
-    {% for need in needs %}
-    <li id="need-item-{{ need.id }}">
-        <div class="need-info">
-            <strong>{{ need.name }}</strong>
-            {% if need.description %}<p>{{ need.description }}</p>{% endif %}
-        </div>
-        <button class="btn btn-danger"
-                hx-delete="/api/needs/{{ need.id }}"
-                hx-target="#need-item-{{ need.id }}"
-                hx-swap="outerHTML"
-                hx-confirm="Delete this need?">
-            Delete
-        </button>
-    </li>
-    {% endfor %}
-</ul>
-{% else %}
-<p>No needs defined yet.</p>
-{% endif %}

python/common.py

@@ -3,28 +3,23 @@
 from __future__ import annotations
 
 import logging
-import sys
 from datetime import UTC, datetime
-from os import getenv
+from pathlib import Path
 from subprocess import PIPE, Popen
 
-from apprise import Apprise
+from python.logging_config import configure_logger as _configure_logger
 
 logger = logging.getLogger(__name__)
 
 
-def configure_logger(level: str = "INFO") -> None:
-    """Configure the logger.
+def get_repo_dir() -> Path:
+    """Return the repository root directory."""
+    return Path(__file__).resolve().parents[1]
 
-    Args:
-        level (str, optional): The logging level. Defaults to "INFO".
-    """
-    logging.basicConfig(
-        level=level,
-        datefmt="%Y-%m-%dT%H:%M:%S%z",
-        format="%(asctime)s %(levelname)s %(filename)s:%(lineno)d - %(message)s",
-        handlers=[logging.StreamHandler(sys.stdout)],
-    )
+
+def configure_logger(level: str = "INFO") -> None:
+    """Configure the logger."""
+    _configure_logger(level)
 
 
 def bash_wrapper(command: str) -> tuple[str, int]:
@@ -47,26 +42,6 @@ def bash_wrapper(command: str) -> tuple[str, int]:
     return output.decode(), process.returncode
 
 
-def signal_alert(body: str, title: str = "") -> None:
-    """Send a signal alert.
-
-    Args:
-        body (str): The body of the alert.
-        title (str, optional): The title of the alert. Defaults to "".
-    """
-    apprise_client = Apprise()
-
-    from_phone = getenv("SIGNAL_ALERT_FROM_PHONE")
-    to_phone = getenv("SIGNAL_ALERT_TO_PHONE")
-    if not from_phone or not to_phone:
-        logger.info("SIGNAL_ALERT_FROM_PHONE or SIGNAL_ALERT_TO_PHONE not set")
-        return
-
-    apprise_client.add(f"signal://localhost:8989/{from_phone}/{to_phone}")
-
-    apprise_client.notify(title=title, body=body)
-
-
 def utcnow() -> datetime:
     """Get the current UTC time."""
     return datetime.now(tz=UTC)

python/data_science/__init__.py

@@ -1,3 +0,0 @@
-"""Data science CLI tools."""
-
-from __future__ import annotations

python/data_science/bluesky_ids.py

@@ -1,104 +0,0 @@
-"""Utilities for converting Bluesky identifiers to numeric database IDs.
-
-Handles DID-to-user_id hashing, TID-to-post_id decoding, and AT-URI parsing.
-"""
-
-from __future__ import annotations
-
-import hashlib
-
-TID_CHARSET = "234567abcdefghijklmnopqrstuvwxyz"
-_TID_LENGTH = 13
-_BIGINT_MASK = 0x7FFFFFFFFFFFFFFF
-_AT_URI_SEGMENT_COUNT = 3
-
-
-def did_to_user_id(did: str) -> int:
-    """Convert a DID string to a deterministic 63-bit integer for user_id.
-
-    Uses SHA-256, truncated to 63 bits (positive signed BigInteger range).
-    Collision probability is negligible at Bluesky's scale (~tens of millions of users).
-
-    Args:
-        did: A Bluesky DID string, e.g. "did:plc:abc123".
-
-    Returns:
-        A positive 63-bit integer suitable for BigInteger storage.
-    """
-    digest = hashlib.sha256(did.encode()).digest()
-    return int.from_bytes(digest[:8], "big") & _BIGINT_MASK
-
-
-def tid_to_integer(tid: str) -> int:
-    """Decode a Bluesky TID (base32-sortbase) into a 64-bit integer for post_id.
-
-    TIDs are 13-character, base32-sortbase encoded identifiers that encode a
-    microsecond timestamp plus a clock ID. They are globally unique by construction.
-
-    Args:
-        tid: A 13-character TID string, e.g. "3abc2defghijk".
-
-    Returns:
-        A positive integer suitable for BigInteger storage.
-
-    Raises:
-        ValueError: If the TID is malformed (wrong length or invalid characters).
-    """
-    if len(tid) != _TID_LENGTH:
-        message = f"TID must be {_TID_LENGTH} characters, got {len(tid)}: {tid!r}"
-        raise ValueError(message)
-
-    result = 0
-    for char in tid:
-        index = TID_CHARSET.find(char)
-        if index == -1:
-            message = f"Invalid character {char!r} in TID {tid!r}"
-            raise ValueError(message)
-        result = result * 32 + index
-    return result
-
-
-def parse_at_uri(uri: str) -> tuple[str, str, str]:
-    """Parse an AT-URI into its components.
-
-    Args:
-        uri: An AT-URI string, e.g. "at://did:plc:abc123/app.bsky.feed.post/3abc2defghijk".
-
-    Returns:
-        A tuple of (did, collection, rkey).
-
-    Raises:
-        ValueError: If the URI doesn't have the expected format.
-    """
-    stripped = uri.removeprefix("at://")
-    parts = stripped.split("/", maxsplit=2)
-    if len(parts) != _AT_URI_SEGMENT_COUNT:
-        message = f"Expected {_AT_URI_SEGMENT_COUNT} path segments in AT-URI, got {len(parts)}: {uri!r}"
-        raise ValueError(message)
-    return parts[0], parts[1], parts[2]
-
-
-def post_id_from_uri(uri: str) -> int:
-    """Extract and decode the post_id (TID) from an AT-URI.
-
-    Args:
-        uri: An AT-URI pointing to a post.
-
-    Returns:
-        The post_id as an integer.
-    """
-    _did, _collection, rkey = parse_at_uri(uri)
-    return tid_to_integer(rkey)
-
-
-def user_id_from_uri(uri: str) -> int:
-    """Extract and hash the user_id (DID) from an AT-URI.
-
-    Args:
-        uri: An AT-URI pointing to a post.
-
-    Returns:
-        The user_id as an integer.
-    """
-    did, _collection, _rkey = parse_at_uri(uri)
-    return did_to_user_id(did)

python/data_science/bluesky_transform.py

@@ -1,143 +0,0 @@
-"""Transform Bluesky Jetstream messages into rows matching the Posts table schema."""
-
-from __future__ import annotations
-
-import json
-import logging
-from datetime import datetime
-
-from python.data_science.bluesky_ids import (
-    did_to_user_id,
-    post_id_from_uri,
-    tid_to_integer,
-    user_id_from_uri,
-)
-
-logger = logging.getLogger(__name__)
-
-INSTANCE = "bsky"
-POST_COLLECTION = "app.bsky.feed.post"
-EMBED_RECORD_TYPE = "app.bsky.embed.record"
-EMBED_RECORD_WITH_MEDIA_TYPE = "app.bsky.embed.recordWithMedia"
-
-
-def transform_jetstream_post(message: dict) -> dict:
-    """Transform a Jetstream commit message into a dict matching Posts table columns.
-
-    Expects a Jetstream message with kind=commit, operation=create,
-    collection=app.bsky.feed.post.
-
-    Args:
-        message: The full Jetstream JSON message.
-
-    Returns:
-        A dict with keys matching the Posts table columns.
-    """
-    did = message["did"]
-    commit = message["commit"]
-    record = commit["record"]
-
-    row: dict = {
-        "post_id": tid_to_integer(commit["rkey"]),
-        "user_id": did_to_user_id(did),
-        "instance": INSTANCE,
-        "date": datetime.fromisoformat(record["createdAt"]),
-        "text": record.get("text", ""),
-        "langs": _extract_langs(record),
-        "like_count": 0,
-        "reply_count": 0,
-        "repost_count": 0,
-        "reply_to": None,
-        "replied_author": None,
-        "thread_root": None,
-        "thread_root_author": None,
-        "repost_from": None,
-        "reposted_author": None,
-        "quotes": None,
-        "quoted_author": None,
-        "labels": _extract_labels(record),
-        "sent_label": None,
-        "sent_score": None,
-    }
-
-    _extract_reply_refs(record, row)
-    _extract_quote_refs(record, row)
-
-    return row
-
-
-def is_post_create(message: dict) -> bool:
-    """Check if a Jetstream message is a post creation event.
-
-    Args:
-        message: The full Jetstream JSON message.
-
-    Returns:
-        True if this is a create commit for app.bsky.feed.post.
-    """
-    if message.get("kind") != "commit":
-        return False
-    commit = message.get("commit", {})
-    return commit.get("operation") == "create" and commit.get("collection") == POST_COLLECTION
-
-
-def _extract_langs(record: dict) -> str | None:
-    """Extract langs array as a JSON string, or None if absent."""
-    langs = record.get("langs")
-    if langs is None:
-        return None
-    return json.dumps(langs)
-
-
-def _extract_labels(record: dict) -> str | None:
-    """Extract self-labels as a JSON string, or None if absent."""
-    labels_obj = record.get("labels")
-    if labels_obj is None:
-        return None
-    values = labels_obj.get("values", [])
-    if not values:
-        return None
-    label_strings = [label.get("val", "") for label in values]
-    return json.dumps(label_strings)
-
-
-def _extract_reply_refs(record: dict, row: dict) -> None:
-    """Populate reply_to, replied_author, thread_root, thread_root_author from record.reply."""
-    reply = record.get("reply")
-    if reply is None:
-        return
-
-    parent = reply.get("parent", {})
-    parent_uri = parent.get("uri")
-    if parent_uri:
-        row["reply_to"] = post_id_from_uri(parent_uri)
-        row["replied_author"] = user_id_from_uri(parent_uri)
-
-    root = reply.get("root", {})
-    root_uri = root.get("uri")
-    if root_uri:
-        row["thread_root"] = post_id_from_uri(root_uri)
-        row["thread_root_author"] = user_id_from_uri(root_uri)
-
-
-def _extract_quote_refs(record: dict, row: dict) -> None:
-    """Populate quotes and quoted_author from embed record references."""
-    embed = record.get("embed")
-    if embed is None:
-        return
-
-    embed_type = embed.get("$type", "")
-
-    if embed_type == EMBED_RECORD_TYPE:
-        _set_quote_from_record(embed.get("record", {}), row)
-    elif embed_type == EMBED_RECORD_WITH_MEDIA_TYPE:
-        inner_record = embed.get("record", {}).get("record", {})
-        _set_quote_from_record(inner_record, row)
-
-
-def _set_quote_from_record(record_ref: dict, row: dict) -> None:
-    """Set quotes and quoted_author from a record reference object."""
-    uri = record_ref.get("uri")
-    if uri and POST_COLLECTION in uri:
-        row["quotes"] = post_id_from_uri(uri)
-        row["quoted_author"] = user_id_from_uri(uri)

python/data_science/firehose_consumer.py

@@ -1,203 +0,0 @@
-"""Kafka consumer that ingests Bluesky posts into the partitioned Posts table.
-
-Consumes Jetstream messages from Kafka, transforms them into Posts rows,
-and batch-inserts them into PostgreSQL with manual offset commits.
-
-Usage:
-    firehose-consumer
-    firehose-consumer --kafka-servers kafka:9092 --batch-size 500
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import signal
-from os import getenv
-from threading import Event
-from typing import Annotated
-
-import typer
-from confluent_kafka import Consumer, KafkaError, KafkaException
-from sqlalchemy.orm import Session
-
-from python.data_science.bluesky_transform import is_post_create, transform_jetstream_post
-from python.data_science.ingest_posts import ingest_batch
-from python.orm.common import get_postgres_engine
-from python.orm.data_science_dev.posts.failed_ingestion import FailedIngestion
-
-logger = logging.getLogger(__name__)
-
-DEFAULT_TOPIC = "bluesky.firehose.posts"
-DEFAULT_KAFKA_SERVERS = "localhost:9092"
-DEFAULT_GROUP_ID = "bluesky-posts-ingestor"
-DEFAULT_BATCH_SIZE = 500
-POLL_TIMEOUT_SECONDS = 5.0
-
-shutdown_event = Event()
-
-app = typer.Typer(help="Consume Bluesky posts from Kafka and ingest into PostgreSQL.")
-
-
-@app.command()
-def main(
-    kafka_servers: Annotated[str, typer.Option(help="Kafka bootstrap servers")] = "",
-    topic: Annotated[str, typer.Option(help="Kafka topic to consume from")] = "",
-    group_id: Annotated[str, typer.Option(help="Kafka consumer group ID")] = "",
-    batch_size: Annotated[int, typer.Option(help="Messages per DB insert batch")] = DEFAULT_BATCH_SIZE,
-) -> None:
-    """Consume Bluesky posts from Kafka and ingest into the partitioned posts table."""
-    logging.basicConfig(
-        level=logging.INFO,
-        format="%(asctime)s %(levelname)s %(message)s",
-        datefmt="%H:%M:%S",
-    )
-
-    servers = kafka_servers or getenv("KAFKA_BOOTSTRAP_SERVERS", DEFAULT_KAFKA_SERVERS)
-    topic_name = topic or getenv("BLUESKY_FIREHOSE_TOPIC", DEFAULT_TOPIC)
-    group = group_id or getenv("KAFKA_GROUP_ID", DEFAULT_GROUP_ID)
-
-    signal.signal(signal.SIGTERM, _handle_shutdown)
-    signal.signal(signal.SIGINT, _handle_shutdown)
-
-    consumer = _create_consumer(servers, group)
-    consumer.subscribe([topic_name])
-
-    engine = get_postgres_engine(name="DATA_SCIENCE_DEV")
-    total_inserted = 0
-
-    logger.info("Starting firehose consumer: topic=%s group=%s batch_size=%d", topic_name, group, batch_size)
-
-    try:
-        with Session(engine) as session:
-            while not shutdown_event.is_set():
-                inserted = _consume_batch(consumer, session, batch_size)
-                total_inserted += inserted
-                if inserted > 0:
-                    logger.info("Batch inserted %d rows (total: %d)", inserted, total_inserted)
-    except KafkaException:
-        logger.exception("Fatal Kafka error")
-    finally:
-        logger.info("Closing consumer (total inserted: %d)", total_inserted)
-        consumer.close()
-
-
-def _consume_batch(consumer: Consumer, session: Session, batch_size: int) -> int:
-    """Poll a batch of messages, transform, and insert into the database.
-
-    Args:
-        consumer: The Kafka consumer instance.
-        session: SQLAlchemy database session.
-        batch_size: Maximum number of messages to consume per batch.
-
-    Returns:
-        Number of rows successfully inserted.
-    """
-    messages = consumer.consume(num_messages=batch_size, timeout=POLL_TIMEOUT_SECONDS)
-    if not messages:
-        return 0
-
-    rows: list[dict] = []
-    for message in messages:
-        error = message.error()
-        if error is not None:
-            if error.code() == KafkaError._PARTITION_EOF:  # noqa: SLF001 — confluent-kafka exposes this as a pseudo-private constant; no public alternative exists
-                continue
-            logger.error("Consumer error: %s", error)
-            continue
-
-        row = _safe_transform(message.value(), session)
-        if row is not None:
-            rows.append(row)
-
-    if not rows:
-        consumer.commit(asynchronous=False)
-        return 0
-
-    inserted = ingest_batch(session, rows)
-    consumer.commit(asynchronous=False)
-    return inserted
-
-
-def _safe_transform(raw_value: bytes | None, session: Session) -> dict | None:
-    """Transform a Kafka message value into a Posts row, logging failures.
-
-    Args:
-        raw_value: Raw message bytes from Kafka.
-        session: SQLAlchemy session for logging failures.
-
-    Returns:
-        A transformed row dict, or None if transformation failed.
-    """
-    if raw_value is None:
-        return None
-
-    try:
-        message = json.loads(raw_value)
-    except (json.JSONDecodeError, UnicodeDecodeError):
-        logger.exception("Failed to decode Kafka message")
-        _log_failed_ingestion(session, raw_value, "JSON decode error")
-        return None
-
-    if not is_post_create(message):
-        return None
-
-    try:
-        return transform_jetstream_post(message)
-    except (KeyError, ValueError, TypeError):
-        logger.exception("Failed to transform Jetstream message")
-        _log_failed_ingestion(session, raw_value, "Transform error")
-        return None
-
-
-def _log_failed_ingestion(session: Session, raw_value: bytes, error: str) -> None:
-    """Log a failed ingestion to the FailedIngestion table.
-
-    Args:
-        session: SQLAlchemy session.
-        raw_value: The raw message bytes.
-        error: Description of the error.
-    """
-    try:
-        session.add(
-            FailedIngestion(
-                raw_line=raw_value.decode(errors="replace")[:10000],
-                error=error,
-            )
-        )
-        session.commit()
-    except Exception:
-        session.rollback()
-        logger.exception("Failed to log ingestion failure")
-
-
-def _create_consumer(servers: str, group: str) -> Consumer:
-    """Create a configured Kafka consumer.
-
-    Args:
-        servers: Kafka bootstrap servers string.
-        group: Consumer group ID.
-
-    Returns:
-        A configured confluent_kafka.Consumer.
-    """
-    config = {
-        "bootstrap.servers": servers,
-        "group.id": group,
-        "auto.offset.reset": "earliest",
-        "enable.auto.commit": False,
-        "max.poll.interval.ms": 300000,
-        "fetch.min.bytes": 1024,
-        "session.timeout.ms": 30000,
-    }
-    return Consumer(config)
-
-
-def _handle_shutdown(_signum: int, _frame: object) -> None:
-    """Signal handler to trigger graceful shutdown."""
-    logger.info("Shutdown signal received")
-    shutdown_event.set()
-
-
-if __name__ == "__main__":
-    app()

python/data_science/firehose_producer.py

@@ -1,230 +0,0 @@
-"""Bluesky Jetstream firehose to Kafka producer.
-
-Connects to the Bluesky Jetstream WebSocket API with zstd compression,
-filters for post creation events, and produces them to a Kafka topic.
-
-Usage:
-    firehose-producer
-    firehose-producer --kafka-servers kafka:9092 --topic bluesky.firehose.posts
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import signal
-from os import getenv
-from threading import Event
-from typing import Annotated
-
-import typer
-from compression import zstd
-from confluent_kafka import KafkaError, KafkaException, Producer
-from websockets.exceptions import ConnectionClosed
-from websockets.sync.client import connect
-
-logger = logging.getLogger(__name__)
-
-JETSTREAM_URL = "wss://jetstream2.us-east.bsky.network/subscribe"
-DEFAULT_TOPIC = "bluesky.firehose.posts"
-DEFAULT_KAFKA_SERVERS = "localhost:9092"
-POLL_INTERVAL = 100
-POST_COLLECTION = "app.bsky.feed.post"
-
-shutdown_event = Event()
-
-app = typer.Typer(help="Stream Bluesky firehose posts into Kafka.")
-
-
-@app.command()
-def main(
-    kafka_servers: Annotated[str, typer.Option(help="Kafka bootstrap servers")] = "",
-    topic: Annotated[str, typer.Option(help="Kafka topic to produce to")] = "",
-    collections: Annotated[str, typer.Option(help="Comma-separated collections to subscribe to")] = POST_COLLECTION,
-) -> None:
-    """Connect to Bluesky Jetstream and produce post events to Kafka."""
-    logging.basicConfig(
-        level=logging.INFO,
-        format="%(asctime)s %(levelname)s %(message)s",
-        datefmt="%H:%M:%S",
-    )
-
-    servers = kafka_servers or getenv("KAFKA_BOOTSTRAP_SERVERS", DEFAULT_KAFKA_SERVERS)
-    topic_name = topic or getenv("BLUESKY_FIREHOSE_TOPIC", DEFAULT_TOPIC)
-
-    signal.signal(signal.SIGTERM, _handle_shutdown)
-    signal.signal(signal.SIGINT, _handle_shutdown)
-
-    producer = _create_producer(servers)
-    cursor: int | None = None
-
-    logger.info("Starting firehose producer → %s on %s", topic_name, servers)
-
-    while not shutdown_event.is_set():
-        try:
-            cursor = _stream_loop(producer, topic_name, collections, cursor)
-        except (ConnectionClosed, OSError):
-            logger.exception("WebSocket disconnected, reconnecting")
-        except KafkaException:
-            logger.exception("Kafka error, reconnecting")
-
-        if not shutdown_event.is_set():
-            logger.info("Reconnecting in 5 seconds (cursor=%s)", cursor)
-            shutdown_event.wait(timeout=5)
-
-    logger.info("Shutting down, flushing producer")
-    producer.flush(timeout=30)
-    logger.info("Producer shutdown complete")
-
-
-def _stream_loop(
-    producer: Producer,
-    topic: str,
-    collections: str,
-    cursor: int | None,
-) -> int | None:
-    """Connect to Jetstream and stream messages to Kafka until disconnected.
-
-    Args:
-        producer: The Kafka producer instance.
-        topic: Kafka topic name.
-        collections: Comma-separated AT Protocol collections to subscribe to.
-        cursor: Optional microsecond timestamp to resume from.
-
-    Returns:
-        The last processed time_us cursor value.
-    """
-    url = _build_jetstream_url(collections, cursor)
-    logger.info("Connecting to %s", url)
-
-    message_count = 0
-    last_cursor = cursor
-
-    with connect(url, additional_headers={"Accept-Encoding": "zstd"}) as websocket:
-        logger.info("Connected to Jetstream")
-
-        while not shutdown_event.is_set():
-            try:
-                raw_frame = websocket.recv(timeout=10)
-            except TimeoutError:
-                producer.poll(0)
-                continue
-
-            text = _decode_frame(raw_frame)
-            message = json.loads(text)
-
-            time_us = message.get("time_us")
-            if time_us is not None:
-                last_cursor = time_us
-
-            if not _is_post_create(message):
-                continue
-
-            did = message.get("did", "")
-
-            try:
-                producer.produce(
-                    topic,
-                    key=did.encode(),
-                    value=text.encode() if isinstance(text, str) else text,
-                    callback=_delivery_callback,
-                )
-            except BufferError:
-                logger.warning("Producer buffer full, flushing")
-                producer.flush(timeout=10)
-                producer.produce(
-                    topic,
-                    key=did.encode(),
-                    value=text.encode() if isinstance(text, str) else text,
-                    callback=_delivery_callback,
-                )
-
-            message_count += 1
-            if message_count % POLL_INTERVAL == 0:
-                producer.poll(0)
-
-            if message_count % 10000 == 0:
-                logger.info("Produced %d messages (cursor=%s)", message_count, last_cursor)
-
-    return last_cursor
-
-
-def _build_jetstream_url(collections: str, cursor: int | None) -> str:
-    """Build the Jetstream WebSocket URL with query parameters.
-
-    Args:
-        collections: Comma-separated collection names.
-        cursor: Optional microsecond timestamp for resumption.
-
-    Returns:
-        The full WebSocket URL.
-    """
-    params = ["compress=true"]
-    for raw_collection in collections.split(","):
-        cleaned = raw_collection.strip()
-        if cleaned:
-            params.append(f"wantedCollections={cleaned}")
-    if cursor is not None:
-        params.append(f"cursor={cursor}")
-    return f"{JETSTREAM_URL}?{'&'.join(params)}"
-
-
-def _decode_frame(frame: str | bytes) -> str:
-    """Decode a WebSocket frame, decompressing zstd if binary.
-
-    Jetstream with compress=true sends zstd-compressed binary frames.
-
-    Args:
-        frame: Raw WebSocket frame data.
-
-    Returns:
-        The decoded JSON string.
-    """
-    if isinstance(frame, bytes):
-        return zstd.decompress(frame).decode()
-    return frame
-
-
-def _is_post_create(message: dict) -> bool:
-    """Check if a Jetstream message is a post creation commit."""
-    if message.get("kind") != "commit":
-        return False
-    commit = message.get("commit", {})
-    return commit.get("operation") == "create" and commit.get("collection") == POST_COLLECTION
-
-
-def _create_producer(servers: str) -> Producer:
-    """Create a configured Kafka producer.
-
-    Args:
-        servers: Kafka bootstrap servers string.
-
-    Returns:
-        A configured confluent_kafka.Producer.
-    """
-    config = {
-        "bootstrap.servers": servers,
-        "linger.ms": 50,
-        "batch.size": 65536,
-        "compression.type": "zstd",
-        "acks": "all",
-        "retries": 5,
-        "retry.backoff.ms": 500,
-    }
-    return Producer(config)
-
-
-def _delivery_callback(error: KafkaError | None, _message: object) -> None:
-    """Log delivery failures from the Kafka producer."""
-    if error is not None:
-        logger.error("Kafka delivery failed: %s", error)
-
-
-def _handle_shutdown(_signum: int, _frame: object) -> None:
-    """Signal handler to trigger graceful shutdown."""
-    logger.info("Shutdown signal received")
-    shutdown_event.set()
-
-
-if __name__ == "__main__":
-    app()

python/data_science/ingest_posts.py

@@ -1,247 +0,0 @@
-"""Ingestion pipeline for loading JSONL post files into the weekly-partitioned posts table.
-
-Usage:
-    ingest-posts /path/to/files/
-    ingest-posts /path/to/single_file.jsonl
-    ingest-posts /data/dir/ --workers 4 --batch-size 5000
-"""
-
-from __future__ import annotations
-
-import logging
-from datetime import UTC, datetime
-from pathlib import Path  # noqa: TC003 this is needed for typer
-from typing import TYPE_CHECKING, Annotated
-
-import orjson
-import psycopg
-import typer
-
-from python.common import configure_logger
-from python.orm.common import get_connection_info
-from python.parallelize import parallelize_process
-
-if TYPE_CHECKING:
-    from collections.abc import Iterator
-
-logger = logging.getLogger(__name__)
-
-
-app = typer.Typer(help="Ingest JSONL post files into the partitioned posts table.")
-
-
-@app.command()
-def main(
-    path: Annotated[Path, typer.Argument(help="Directory containing JSONL files, or a single JSONL file")],
-    batch_size: Annotated[int, typer.Option(help="Rows per INSERT batch")] = 10000,
-    workers: Annotated[int, typer.Option(help="Parallel workers for multi-file ingestion")] = 4,
-    pattern: Annotated[str, typer.Option(help="Glob pattern for JSONL files")] = "*.jsonl",
-) -> None:
-    """Ingest JSONL post files into the weekly-partitioned posts table."""
-    configure_logger(level="INFO")
-
-    logger.info("starting ingest-posts")
-    logger.info("path=%s batch_size=%d workers=%d pattern=%s", path, batch_size, workers, pattern)
-    if path.is_file():
-        ingest_file(path, batch_size=batch_size)
-    elif path.is_dir():
-        ingest_directory(path, batch_size=batch_size, max_workers=workers, pattern=pattern)
-    else:
-        typer.echo(f"Path does not exist: {path}", err=True)
-        raise typer.Exit(code=1)
-
-    logger.info("ingest-posts done")
-
-
-def ingest_directory(
-    directory: Path,
-    *,
-    batch_size: int,
-    max_workers: int,
-    pattern: str = "*.jsonl",
-) -> None:
-    """Ingest all JSONL files in a directory using parallel workers."""
-    files = sorted(directory.glob(pattern))
-    if not files:
-        logger.warning("No JSONL files found in %s", directory)
-        return
-
-    logger.info("Found %d JSONL files to ingest", len(files))
-
-    kwargs_list = [{"path": fp, "batch_size": batch_size} for fp in files]
-    parallelize_process(ingest_file, kwargs_list, max_workers=max_workers)
-
-
-SCHEMA = "main"
-
-COLUMNS = (
-    "post_id",
-    "user_id",
-    "instance",
-    "date",
-    "text",
-    "langs",
-    "like_count",
-    "reply_count",
-    "repost_count",
-    "reply_to",
-    "replied_author",
-    "thread_root",
-    "thread_root_author",
-    "repost_from",
-    "reposted_author",
-    "quotes",
-    "quoted_author",
-    "labels",
-    "sent_label",
-    "sent_score",
-)
-
-INSERT_FROM_STAGING = f"""
-    INSERT INTO {SCHEMA}.posts ({", ".join(COLUMNS)})
-    SELECT {", ".join(COLUMNS)} FROM pg_temp.staging
-    ON CONFLICT (post_id, date) DO NOTHING
-"""  # noqa: S608
-
-FAILED_INSERT = f"""
-    INSERT INTO {SCHEMA}.failed_ingestion (raw_line, error)
-    VALUES (%(raw_line)s, %(error)s)
-"""  # noqa: S608
-
-
-def get_psycopg_connection() -> psycopg.Connection:
-    """Create a raw psycopg3 connection from environment variables."""
-    database, host, port, username, password = get_connection_info("DATA_SCIENCE_DEV")
-    return psycopg.connect(
-        dbname=database,
-        host=host,
-        port=int(port),
-        user=username,
-        password=password,
-        autocommit=False,
-    )
-
-
-def ingest_file(path: Path, *, batch_size: int) -> None:
-    """Ingest a single JSONL file into the posts table."""
-    log_trigger = max(100_000 // batch_size, 1)
-    failed_lines: list[dict] = []
-    try:
-        with get_psycopg_connection() as connection:
-            for index, batch in enumerate(read_jsonl_batches(path, batch_size, failed_lines), 1):
-                ingest_batch(connection, batch)
-                if index % log_trigger == 0:
-                    logger.info("Ingested %d batches (%d rows) from %s", index, index * batch_size, path)
-
-            if failed_lines:
-                logger.warning("Recording %d malformed lines from %s", len(failed_lines), path.name)
-                with connection.cursor() as cursor:
-                    cursor.executemany(FAILED_INSERT, failed_lines)
-                connection.commit()
-    except Exception:
-        logger.exception("Failed to ingest file: %s", path)
-        raise
-
-
-def ingest_batch(connection: psycopg.Connection, batch: list[dict]) -> None:
-    """COPY batch into a temp staging table, then INSERT ... ON CONFLICT into posts."""
-    if not batch:
-        return
-
-    try:
-        with connection.cursor() as cursor:
-            cursor.execute(f"""
-                CREATE TEMP TABLE IF NOT EXISTS staging
-                (LIKE {SCHEMA}.posts INCLUDING DEFAULTS)
-                ON COMMIT DELETE ROWS
-            """)
-            cursor.execute("TRUNCATE pg_temp.staging")
-
-            with cursor.copy(f"COPY pg_temp.staging ({', '.join(COLUMNS)}) FROM STDIN") as copy:
-                for row in batch:
-                    copy.write_row(tuple(row.get(column) for column in COLUMNS))
-
-            cursor.execute(INSERT_FROM_STAGING)
-        connection.commit()
-    except Exception as error:
-        connection.rollback()
-
-        if len(batch) == 1:
-            logger.exception("Skipping bad row post_id=%s", batch[0].get("post_id"))
-            with connection.cursor() as cursor:
-                cursor.execute(
-                    FAILED_INSERT,
-                    {
-                        "raw_line": orjson.dumps(batch[0], default=str).decode(),
-                        "error": str(error),
-                    },
-                )
-            connection.commit()
-            return
-
-        midpoint = len(batch) // 2
-        ingest_batch(connection, batch[:midpoint])
-        ingest_batch(connection, batch[midpoint:])
-
-
-def read_jsonl_batches(file_path: Path, batch_size: int, failed_lines: list[dict]) -> Iterator[list[dict]]:
-    """Stream a JSONL file and yield batches of transformed rows."""
-    batch: list[dict] = []
-    with file_path.open("r", encoding="utf-8") as handle:
-        for raw_line in handle:
-            line = raw_line.strip()
-            if not line:
-                continue
-            batch.extend(parse_line(line, file_path, failed_lines))
-            if len(batch) >= batch_size:
-                yield batch
-                batch = []
-    if batch:
-        yield batch
-
-
-def parse_line(line: str, file_path: Path, failed_lines: list[dict]) -> Iterator[dict]:
-    """Parse a JSONL line, handling concatenated JSON objects."""
-    try:
-        yield transform_row(orjson.loads(line))
-    except orjson.JSONDecodeError:
-        if "}{" not in line:
-            logger.warning("Skipping malformed line in %s: %s", file_path.name, line[:120])
-            failed_lines.append({"raw_line": line, "error": "malformed JSON"})
-            return
-        fragments = line.replace("}{", "}\n{").split("\n")
-        for fragment in fragments:
-            try:
-                yield transform_row(orjson.loads(fragment))
-            except (orjson.JSONDecodeError, KeyError, ValueError) as error:
-                logger.warning("Skipping malformed fragment in %s: %s", file_path.name, fragment[:120])
-                failed_lines.append({"raw_line": fragment, "error": str(error)})
-    except Exception as error:
-        logger.exception("Skipping bad row in %s: %s", file_path.name, line[:120])
-        failed_lines.append({"raw_line": line, "error": str(error)})
-
-
-def transform_row(raw: dict) -> dict:
-    """Transform a raw JSONL row into a dict matching the Posts table columns."""
-    raw["date"] = parse_date(raw["date"])
-    if raw.get("langs") is not None:
-        raw["langs"] = orjson.dumps(raw["langs"])
-    if raw.get("text") is not None:
-        raw["text"] = raw["text"].replace("\x00", "")
-    return raw
-
-
-def parse_date(raw_date: int) -> datetime:
-    """Parse compact YYYYMMDDHHmm integer into a naive datetime (input is UTC by spec)."""
-    return datetime(
-        raw_date // 100000000,
-        (raw_date // 1000000) % 100,
-        (raw_date // 10000) % 100,
-        (raw_date // 100) % 100,
-        raw_date % 100,
-        tzinfo=UTC,
-    )
-
-
-if __name__ == "__main__":
-    app()

python/database_cli.py

@@ -4,12 +4,10 @@ Usage:
     database <db_name> <command> [args...]
 
 Examples:
-    database van_inventory upgrade head
-    database van_inventory downgrade head-1
-    database van_inventory revision --autogenerate -m "add meals table"
-    database van_inventory check
     database richie check
     database richie upgrade head
+    database richie downgrade head-1
+    database richie revision --autogenerate -m "add meals table"
 """
 
 from __future__ import annotations
@@ -48,10 +46,7 @@ class DatabaseConfig:
 
     def alembic_config(self) -> Config:
         """Build an alembic Config for this database."""
-        # Runtime import needed — Config is in TYPE_CHECKING for the return type annotation
-        from alembic.config import Config as AlembicConfig  # noqa: PLC0415
-
-        cfg = AlembicConfig()
+        cfg = Config()
         cfg.set_main_option("script_location", self.script_location)
         cfg.set_main_option("file_template", self.file_template)
         cfg.set_main_option("prepend_sys_path", ".")
@@ -76,27 +71,6 @@ DATABASES: dict[str, DatabaseConfig] = {
         base_class_name="RichieBase",
         models_module="python.orm.richie",
     ),
-    "van_inventory": DatabaseConfig(
-        env_prefix="VAN_INVENTORY",
-        version_location="python/alembic/van_inventory/versions",
-        base_module="python.orm.van_inventory.base",
-        base_class_name="VanInventoryBase",
-        models_module="python.orm.van_inventory.models",
-    ),
-    "signal_bot": DatabaseConfig(
-        env_prefix="SIGNALBOT",
-        version_location="python/alembic/signal_bot/versions",
-        base_module="python.orm.signal_bot.base",
-        base_class_name="SignalBotBase",
-        models_module="python.orm.signal_bot.models",
-    ),
-    "data_science_dev": DatabaseConfig(
-        env_prefix="DATA_SCIENCE_DEV",
-        version_location="python/alembic/data_science_dev/versions",
-        base_module="python.orm.data_science_dev.base",
-        base_class_name="DataScienceDevBase",
-        models_module="python.orm.data_science_dev.models",
-    ),
 }
 
 

python/ebook_search/__init__.py

@@ -0,0 +1 @@
+"""EPUB search package."""

python/ebook_search/answer.py

@@ -0,0 +1,50 @@
+"""Grounded answer generation."""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+from python.ebook_search.llm_interface import request_chat_completion
+from python.ebook_search.prompts import load_prompt
+
+if TYPE_CHECKING:
+    import httpx
+
+    from python.ebook_search.config import EbookSearchConfig
+    from python.ebook_search.search import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+async def answer_query(
+    client: httpx.AsyncClient,
+    query: str,
+    results: list[SearchResult],
+    config: EbookSearchConfig,
+) -> str:
+    """Answer a question using only retrieved chunks."""
+    if not config.answer_enabled:
+        logger.info("ebook_answer_skipped_disabled")
+        return "Answer generation is disabled. Source chunks are shown below."
+
+    if not results:
+        logger.info("ebook_answer_skipped_no_results")
+        return "No relevant sources were found."
+
+    logger.info(
+        f"ebook_answer_request_start {config.vllm_base_url=} {config.chat_model=} sources={len(results)} "
+        f"query_length={len(query)}"
+    )
+    context = "\n\n".join(
+        f"[{index}] {result.source_title}{' - ' + result.chapter_title if result.chapter_title else ''}\n{result.text}"
+        for index, result in enumerate(results, start=1)
+    )
+    content = await request_chat_completion(
+        client,
+        config,
+        load_prompt("answer").messages(query=query, context=context),
+    )
+
+    logger.info(f"ebook_answer_request_complete {config.chat_model=} answer_length={len(content)}")
+    return content or "The model returned an empty answer."

python/ebook_search/api/__init__.py

@@ -0,0 +1 @@
+"""Web and external API adapters for EPUB search."""

python/ebook_search/api/bm25_tasks.py

@@ -0,0 +1,70 @@
+"""Background BM25 refresh tasks for the web app.
+
+The refresh is scheduled on the event loop instead of a thread because the async psycopg
+driver only works from the loop; a bare thread cannot open a session on the async engine.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import TYPE_CHECKING
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from python.ebook_search.bm25_corpus import load_bm25_corpus, refresh_bm25_corpus
+
+if TYPE_CHECKING:
+    from fastapi import FastAPI
+    from sqlalchemy.ext.asyncio import AsyncEngine
+
+    from python.ebook_search.config import EbookSearchConfig
+
+logger = logging.getLogger(__name__)
+
+
+def schedule_bm25_refresh(app: FastAPI) -> None:
+    """Schedule a delayed BM25 corpus refresh, replacing any pending refresh.
+
+    Only called from route handlers, so a running event loop is guaranteed.
+    """
+    cancel_bm25_refresh(app)
+
+    loop = asyncio.get_running_loop()
+
+    def start_refresh() -> None:
+        app.state.bm25_refresh_task = loop.create_task(refresh_bm25_for_app(app))
+
+    app.state.bm25_refresh_timer = loop.call_later(app.state.config.bm25_refresh_delay_seconds, start_refresh)
+    logger.info(f"ebook_bm25_refresh_scheduled {app.state.config.bm25_refresh_delay_seconds=}")
+
+
+def cancel_bm25_refresh(app: FastAPI) -> None:
+    """Cancel any pending BM25 corpus refresh timer and in-flight refresh task."""
+    existing_timer = getattr(app.state, "bm25_refresh_timer", None)
+    if existing_timer is not None:
+        existing_timer.cancel()
+        app.state.bm25_refresh_timer = None
+        logger.info("ebook_bm25_refresh_cancelled")
+
+    existing_task = getattr(app.state, "bm25_refresh_task", None)
+    if existing_task is not None:
+        if not existing_task.done():
+            existing_task.cancel()
+        app.state.bm25_refresh_task = None
+
+
+async def refresh_bm25_for_app(app: FastAPI) -> None:
+    """Refresh the BM25 corpus using the app engine and config."""
+    try:
+        await refresh_bm25_for_engine(app.state.engine, app.state.config)
+    except Exception:
+        logger.exception("ebook_bm25_refresh_failed")
+
+
+async def refresh_bm25_for_engine(engine: AsyncEngine, config: EbookSearchConfig) -> None:
+    """Refresh the BM25 corpus using an async SQLAlchemy engine."""
+    async with AsyncSession(engine) as session:
+        await refresh_bm25_corpus(session, config)
+    load_bm25_corpus.cache_clear()
+    logger.info("ebook_bm25_corpus_cache_cleared_after_refresh")

python/ebook_search/api/dependencies.py

@@ -0,0 +1,24 @@
+"""FastAPI dependencies for the EPUB search app."""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+import httpx
+from fastapi import Depends, Request
+
+from python.ebook_search.config import EbookSearchConfig
+
+
+def get_config(request: Request) -> EbookSearchConfig:
+    """Get the loaded search config from app state."""
+    return request.app.state.config
+
+
+def get_http_client(request: Request) -> httpx.AsyncClient:
+    """Get the shared LLM HTTP client from app state."""
+    return request.app.state.http_client
+
+
+AppConfig = Annotated[EbookSearchConfig, Depends(get_config)]
+AppHttpClient = Annotated[httpx.AsyncClient, Depends(get_http_client)]

python/ebook_search/api/judge_tasks.py

@@ -0,0 +1,127 @@
+"""Background phrase-judging tasks for the web app.
+
+Judging a book sends one LLM request per candidate phrase, which can take minutes, so it must
+not run inside the request where it would block the UI. Judgments run as async FastAPI
+background tasks, awaited on the event loop after the response is sent, and are tracked per
+book in app state so a second judge request for a book that is already being judged is
+rejected instead of doubling the work.
+
+State is loop-confined: every read and mutation happens on the event loop (async route
+handlers and async background tasks) and no critical section contains an ``await``, so each
+mutation is atomic per loop iteration and no locking is needed.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
+
+if TYPE_CHECKING:
+    from fastapi import BackgroundTasks, FastAPI
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class JudgeTaskState:
+    """Running book judgments and last outcome messages, keyed by book id."""
+
+    running_book_ids: set[int] = field(default_factory=set)
+    outcome_messages: dict[int, str] = field(default_factory=dict)
+
+
+def get_judge_task_state(app: FastAPI) -> JudgeTaskState:
+    """Return the app's judge task state, creating it on first use.
+
+    Args:
+        app (FastAPI): App whose state holds the judge task registry.
+
+    Returns:
+        JudgeTaskState: The shared judge task state for this app.
+    """
+    state = getattr(app.state, "judge_tasks", None)
+    if state is None:
+        state = JudgeTaskState()
+        app.state.judge_tasks = state
+    return state
+
+
+def start_book_phrase_judgment(app: FastAPI, background_tasks: BackgroundTasks, source_id: int) -> bool:
+    """Queue judging of one book's candidate phrases as a FastAPI background task.
+
+    The book is claimed before the response returns, so a repeated judge request cannot queue
+    a second run while one is pending or running.
+
+    Args:
+        app (FastAPI): App supplying the engine, config, and judge task state.
+        background_tasks (BackgroundTasks): Request's background tasks to queue the judgment on.
+        source_id (int): Book to judge candidates for.
+
+    Returns:
+        bool: True when a judgment was queued, False when one is already running for this book.
+    """
+    state = get_judge_task_state(app)
+    if source_id in state.running_book_ids:
+        logger.info(f"ebook_book_phrase_judgment_already_running {source_id=}")
+        return False
+    state.running_book_ids.add(source_id)
+    state.outcome_messages.pop(source_id, None)
+    background_tasks.add_task(judge_book_phrases_for_app, app, source_id)
+    logger.info(f"ebook_book_phrase_judgment_queued {source_id=}")
+    return True
+
+
+async def judge_book_phrases_for_app(app: FastAPI, source_id: int) -> None:
+    """Judge one book using the app engine and config, recording the outcome message.
+
+    Args:
+        app (FastAPI): App supplying the engine, config, and judge task state.
+        source_id (int): Book to judge candidates for.
+    """
+    state = get_judge_task_state(app)
+    try:
+        result = await judge_candidate_phrases_for_books(app.state.engine, app.state.config, source_ids=[source_id])
+        logger.info(
+            f"ebook_book_phrase_judgment_complete {source_id=} {result.candidates_judged=} {result.protected_phrases=} "
+            f"{result.phrase_mentions=} {result.books_failed=}"
+        )
+        if result.books_failed:
+            message = "Judging failed; see server logs for details"
+        else:
+            message = (
+                f"Judged {result.candidates_judged} candidates; {result.protected_phrases} protected phrases promoted"
+            )
+    except Exception:
+        logger.exception(f"ebook_book_phrase_judgment_task_failed {source_id=}")
+        message = "Judging failed; see server logs for details"
+    state.running_book_ids.discard(source_id)
+    state.outcome_messages[source_id] = message
+
+
+def is_judging_book(app: FastAPI, source_id: int) -> bool:
+    """Report whether a judgment is currently queued or running for one book.
+
+    Args:
+        app (FastAPI): App supplying the judge task state.
+        source_id (int): Book to check.
+
+    Returns:
+        bool: True while the book's judgment is pending or running.
+    """
+    return source_id in get_judge_task_state(app).running_book_ids
+
+
+def pop_book_judgment_outcome(app: FastAPI, source_id: int) -> str | None:
+    """Return and clear the outcome message from one book's last finished judgment.
+
+    Args:
+        app (FastAPI): App supplying the judge task state.
+        source_id (int): Book to fetch the outcome for.
+
+    Returns:
+        str | None: The outcome message, or None when there is nothing new to report.
+    """
+    return get_judge_task_state(app).outcome_messages.pop(source_id, None)

python/ebook_search/api/main.py

@@ -0,0 +1,91 @@
+"""FastAPI HTMX app for EPUB search."""
+
+from __future__ import annotations
+
+import logging
+from contextlib import asynccontextmanager
+from typing import TYPE_CHECKING, Annotated
+
+import httpx
+import typer
+import uvicorn
+from fastapi import FastAPI
+from fastapi.staticfiles import StaticFiles
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from python.common import configure_logger
+from python.ebook_search.api.bm25_tasks import cancel_bm25_refresh
+from python.ebook_search.api.routes import admin_router, health_router, page_router, search_router
+from python.ebook_search.api.web import STATIC_DIR
+from python.ebook_search.bm25_corpus import ensure_bm25_corpus
+from python.ebook_search.config import load_config
+from python.ebook_search.protected_phrases.pool import shutdown_extraction_pool
+from python.fastapi_tools import ZstdMiddleware
+from python.orm.common import get_async_postgres_engine
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
+
+logger = logging.getLogger(__name__)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI) -> AsyncIterator[None]:
+    """Manage application startup and shutdown resources."""
+    logger.info("ebook_search_startup")
+    config = load_config()
+    app.state.config = config
+    logger.info(
+        f"ebook_search_config_loaded {config.top_k=} {config.embedding_model=} {config.embedding_base_url=} "
+        f"{config.vllm_base_url=} {config.rerank.enabled=} {config.phrase_matching_enabled=} {config.answer_enabled=} "
+        f"library_paths={len(config.library_paths)}"
+    )
+    if not config.library_paths:
+        logger.warning("ebook_search_no_library_paths_configured")
+    # Concurrent phrase judging opens one session per book worker on this engine, so size the pool
+    # to cover those plus headroom for ordinary web requests.
+    app.state.engine = get_async_postgres_engine(
+        name="RICHIE",
+        vector_engine=True,
+        pool_size=config.phrase_judge_book_workers + 10,
+    )
+    app.state.http_client = httpx.AsyncClient()
+    async with AsyncSession(app.state.engine, expire_on_commit=False) as session:
+        await ensure_bm25_corpus(session, config)
+    try:
+        yield
+    finally:
+        logger.info("ebook_search_shutdown")
+        cancel_bm25_refresh(app)
+        shutdown_extraction_pool()
+        await app.state.http_client.aclose()
+        await app.state.engine.dispose()
+
+
+def create_app() -> FastAPI:
+    """Create the EPUB search web app."""
+    app = FastAPI(title="EPUB Search", lifespan=lifespan)
+    app.add_middleware(ZstdMiddleware)
+    app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
+
+    app.include_router(admin_router)
+    app.include_router(health_router)
+    app.include_router(page_router)
+    app.include_router(search_router)
+
+    return app
+
+
+def serve(
+    host: Annotated[str, typer.Option("--host", "-h", help="Host to bind to")] = "127.0.0.1",
+    port: Annotated[int, typer.Option("--port", "-p", help="Port to bind to")] = 8070,
+    log_level: Annotated[str, typer.Option("--log-level", "-l", help="Log level")] = "INFO",
+) -> None:
+    """Start the EPUB search server."""
+    configure_logger(log_level)
+    uvicorn.run(create_app(), host=host, port=port)
+
+
+if __name__ == "__main__":
+    typer.run(serve)

python/ebook_search/api/routes/__init__.py

@@ -0,0 +1,13 @@
+"""EPUB search web route modules."""
+
+from python.ebook_search.api.routes.admin import router as admin_router
+from python.ebook_search.api.routes.health import router as health_router
+from python.ebook_search.api.routes.page import router as page_router
+from python.ebook_search.api.routes.search import router as search_router
+
+__all__ = [
+    "admin_router",
+    "health_router",
+    "page_router",
+    "search_router",
+]

python/ebook_search/api/routes/admin.py

@@ -0,0 +1,204 @@
+"""Admin routes for the EPUB search web UI."""
+
+from __future__ import annotations
+
+import logging
+
+from fastapi import APIRouter, Request
+from fastapi.responses import HTMLResponse
+
+from python.ebook_search.api.bm25_tasks import schedule_bm25_refresh
+from python.ebook_search.api.dependencies import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
+    AppConfig,
+    AppHttpClient,
+)
+from python.ebook_search.api.web import error_response, templates
+from python.ebook_search.embeddings import embed_missing_chunks, embedding_model_stats
+from python.ebook_search.ingest import ingest_configured_paths
+from python.ebook_search.protected_phrases.generate_ngrams import generate_candidate_phrases_for_books
+from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
+from python.ebook_search.protected_phrases.store import book_ids_pending_first_judgment, corpus_phrase_stats
+from python.fastapi_tools import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
+    AppAsyncEngine,
+    AsyncDbSession,
+)
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/admin")
+
+
+@router.get("", response_class=HTMLResponse)
+async def admin(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
+    """Render the admin page."""
+    stats = await embedding_model_stats(session)
+    phrase_stats = await corpus_phrase_stats(session)
+    logger.info(
+        f"ebook_admin_page_loaded models={len(stats)} {phrase_stats.candidate_phrases=} "
+        f"{phrase_stats.protected_phrases=}"
+    )
+    return templates.TemplateResponse(
+        request,
+        "admin.html",
+        {"config": config, "stats": stats, "phrase_stats": phrase_stats},
+    )
+
+
+@router.post("/scan", response_class=HTMLResponse)
+async def scan_library(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
+    """Scan configured library paths for EPUB changes."""
+    try:
+        count = await ingest_configured_paths(session, config)
+        await session.commit()
+    except Exception as error:
+        logger.exception("ebook_admin_scan_failed")
+        return error_response(request, error)
+
+    logger.info(f"ebook_admin_scan_complete {count=}")
+    if count > 0:
+        schedule_bm25_refresh(request.app)
+    return templates.TemplateResponse(request, "partials/admin_status.html", {"message": f"Indexed {count} EPUBs"})
+
+
+@router.post("/phrases/generate-all", response_class=HTMLResponse)
+async def generate_all_phrases(request: Request, config: AppConfig, engine: AppAsyncEngine) -> HTMLResponse:
+    """Regenerate candidate phrases for every indexed book without LLM judging."""
+    try:
+        result = await generate_candidate_phrases_for_books(engine, config)
+    except Exception as error:
+        logger.exception("ebook_admin_generate_phrases_failed")
+        return error_response(request, error)
+
+    logger.info(
+        f"ebook_admin_generate_phrases_complete {result.books_seen=} {result.books_built=} {result.candidate_phrases=}"
+    )
+    return templates.TemplateResponse(
+        request,
+        "partials/admin_status.html",
+        {
+            "message": (
+                f"Generated phrases for {result.books_built} of {result.books_seen} books; "
+                f"{result.candidate_phrases} candidates stored"
+            )
+        },
+    )
+
+
+@router.post("/phrases/judge-all", response_class=HTMLResponse)
+async def judge_all_phrases(request: Request, engine: AppAsyncEngine, config: AppConfig) -> HTMLResponse:
+    """Judge unjudged candidate phrases across every indexed book."""
+    return await run_phrase_judgment(request, engine, config, source_ids=None)
+
+
+@router.post("/phrases/judge-missing", response_class=HTMLResponse)
+async def judge_missing_phrases(
+    request: Request,
+    engine: AppAsyncEngine,
+    config: AppConfig,
+    session: AsyncDbSession,
+) -> HTMLResponse:
+    """Judge candidate phrases only for books where judging has never run."""
+    source_ids = await book_ids_pending_first_judgment(session)
+    if not source_ids:
+        return templates.TemplateResponse(
+            request,
+            "partials/admin_status.html",
+            {"message": "All books with candidate phrases have been judged"},
+        )
+    return await run_phrase_judgment(request, engine, config, source_ids=source_ids)
+
+
+async def run_phrase_judgment(
+    request: Request,
+    engine: AppAsyncEngine,
+    config: AppConfig,
+    *,
+    source_ids: list[int] | None,
+) -> HTMLResponse:
+    """Run LLM judging for candidate phrases and render the outcome as an admin status partial.
+
+    Args:
+        request (Request): Current request, for template rendering.
+        engine (AppAsyncEngine): Engine used to open per-book judging sessions.
+        config (AppConfig): Runtime phrase-tuning settings.
+        source_ids (list[int] | None): Books to judge; ``None`` judges every indexed book.
+
+    Returns:
+        HTMLResponse: Status partial describing the judging outcome.
+    """
+    try:
+        result = await judge_candidate_phrases_for_books(engine, config, source_ids=source_ids)
+    except Exception as error:
+        logger.exception("ebook_admin_judge_phrases_failed")
+        return error_response(request, error)
+
+    logger.info(
+        f"ebook_admin_judge_phrases_complete {result.books_seen=} {result.books_judged=} {result.books_failed=} "
+        f"{result.candidates_judged=} {result.protected_phrases=} {result.phrase_mentions=}"
+    )
+    return templates.TemplateResponse(
+        request,
+        "partials/admin_status.html",
+        {
+            "message": (
+                f"Judged {result.candidates_judged} candidates across {result.books_judged} of "
+                f"{result.books_seen} books; {result.protected_phrases} protected phrases, "
+                f"{result.phrase_mentions} mentions"
+                + (f"; {result.books_failed} books failed" if result.books_failed else "")
+            )
+        },
+    )
+
+
+@router.post("/embed-missing", response_class=HTMLResponse)
+async def embed_missing(
+    request: Request,
+    config: AppConfig,
+    session: AsyncDbSession,
+    client: AppHttpClient,
+) -> HTMLResponse:
+    """Embed chunks missing vectors for the configured model."""
+    try:
+        count = await embed_missing_chunks(session, client, config)
+        await session.commit()
+    except Exception as error:
+        logger.exception("ebook_admin_embed_missing_failed")
+        return error_response(request, error)
+
+    logger.info(f"ebook_admin_embed_missing_complete {count=}")
+    return templates.TemplateResponse(
+        request,
+        "partials/admin_status.html",
+        {"message": f"Embedded {count} chunks"},
+    )
+
+
+@router.post("/embed-all", response_class=HTMLResponse)
+async def embed_all(
+    request: Request,
+    config: AppConfig,
+    session: AsyncDbSession,
+    client: AppHttpClient,
+) -> HTMLResponse:
+    """Embed all chunks missing vectors in fixed-size batches."""
+    total = 0
+    batches = 0
+    try:
+        while True:
+            count = await embed_missing_chunks(session, client, config)
+            if count == 0:
+                break
+            await session.commit()
+            total += count
+            batches += 1
+            logger.info(f"ebook_admin_embed_all_batch_complete {batches=} {count=} {total=}")
+    except Exception as error:
+        logger.exception(f"ebook_admin_embed_all_failed {batches=} {total=}")
+        return error_response(request, f"Embed all failed after {total} chunks in {batches} batches: {error}")
+
+    logger.info(f"ebook_admin_embed_all_complete {batches=} {total=}")
+    return templates.TemplateResponse(
+        request,
+        "partials/admin_status.html",
+        {"message": f"Embedded {total} chunks in {batches} batches of {config.embedding_batch_size}"},
+    )

python/ebook_search/api/routes/health.py

@@ -0,0 +1,92 @@
+"""Liveness and readiness routes for the EPUB search service."""
+
+from __future__ import annotations
+
+import logging
+from http import HTTPStatus
+from typing import TYPE_CHECKING
+
+from fastapi import APIRouter
+from fastapi.responses import JSONResponse
+from sqlalchemy import literal, select
+from sqlalchemy.exc import SQLAlchemyError
+
+from python.ebook_search.api.dependencies import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
+    AppConfig,
+    AppHttpClient,
+)
+from python.ebook_search.bm25_corpus import bm25_index_exists, bm25_index_path, read_bm25_manifest
+from python.ebook_search.llm_interface import check_chat_endpoint, check_embedding_endpoint
+from python.fastapi_tools import AsyncDbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+
+if TYPE_CHECKING:
+    import httpx
+    from sqlalchemy.ext.asyncio import AsyncSession
+
+    from python.ebook_search.config import EbookSearchConfig
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("/health")
+async def health() -> dict[str, str]:
+    """Liveness probe that returns ok without touching dependencies."""
+    return {"status": "ok"}
+
+
+@router.get("/ready")
+async def ready(config: AppConfig, session: AsyncDbSession, client: AppHttpClient) -> JSONResponse:
+    """Readiness probe reporting database, embedding endpoint, and BM25 index status."""
+    database_ok = await check_database(session)
+    embedding_ok = await check_embedding_endpoint(client, config)
+    chat_status = await chat_endpoint_status(client, config)
+    bm25_status = check_bm25_status(config)
+
+    checks = {
+        "database": "ok" if database_ok else "fail",
+        "embedding": "ok" if embedding_ok else "fail",
+        "chat": chat_status,
+        "bm25": bm25_status,
+    }
+    if not database_ok:
+        status = "unavailable"
+        status_code = HTTPStatus.SERVICE_UNAVAILABLE
+    elif not embedding_ok or chat_status == "fail" or bm25_status == "missing":
+        status = "degraded"
+        status_code = HTTPStatus.OK
+    else:
+        status = "ready"
+        status_code = HTTPStatus.OK
+
+    logger.info(f"ebook_ready_check {status=} {database_ok=} {embedding_ok=} {chat_status=} {bm25_status=}")
+    return JSONResponse(content={"status": status, "checks": checks}, status_code=status_code)
+
+
+async def chat_endpoint_status(client: httpx.AsyncClient, config: EbookSearchConfig) -> str:
+    """Return the answering chat endpoint status, or disabled when answers are off."""
+    if not config.answer_enabled:
+        return "disabled"
+    return "ok" if await check_chat_endpoint(client, config) else "fail"
+
+
+async def check_database(session: AsyncSession) -> bool:
+    """Return whether the database answers a trivial query."""
+    try:
+        await session.execute(select(literal(1)))
+    except SQLAlchemyError as error:
+        logger.warning(f"ebook_ready_database_unavailable {error=}")
+        return False
+    return True
+
+
+def check_bm25_status(config: EbookSearchConfig) -> str:
+    """Return the persisted BM25 index status without loading it into memory."""
+    index_path = bm25_index_path(config)
+    manifest = read_bm25_manifest(index_path)
+    if manifest is None or not bm25_index_exists(index_path, manifest):
+        return "missing"
+    if manifest.chunk_count == 0:
+        return "empty"
+    return "ok"

python/ebook_search/api/routes/page.py

@@ -0,0 +1,187 @@
+"""Page routes for the EPUB search web UI."""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
+from fastapi.responses import HTMLResponse, RedirectResponse
+from sqlalchemy import func, select
+
+from python.ebook_search.api.dependencies import (
+    AppConfig,  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+)
+from python.ebook_search.api.judge_tasks import is_judging_book, pop_book_judgment_outcome, start_book_phrase_judgment
+from python.ebook_search.api.web import templates
+from python.ebook_search.protected_phrases.generate_ngrams import recalculate_candidate_phrases_for_book
+from python.ebook_search.protected_phrases.store import count_protected_phrases
+from python.fastapi_tools import AsyncDbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+from python.orm.richie import EbookCandidatePhrase, EbookChapter, EbookChunk, EbookProtectedPhrase, EbookSource
+
+if TYPE_CHECKING:
+    from sqlalchemy.ext.asyncio import AsyncSession
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("/", response_class=HTMLResponse)
+async def index(request: Request, config: AppConfig) -> HTMLResponse:
+    """Render the search page."""
+    return templates.TemplateResponse(request, "search.html", {"config": config})
+
+
+@router.get("/books", response_class=HTMLResponse)
+async def books(request: Request, session: AsyncDbSession) -> HTMLResponse:
+    """Render the indexed books page."""
+    sources = list((await session.scalars(select(EbookSource).order_by(EbookSource.title))).all())
+    logger.info(f"ebook_books_page_loaded count={len(sources)}")
+    return templates.TemplateResponse(request, "books.html", {"sources": sources})
+
+
+async def get_chapter_count(session: AsyncSession, book_id: int) -> int:
+    """Return the number of indexed chapters for one book."""
+    return await session.scalar(select(func.count(EbookChapter.id)).where(EbookChapter.source_id == book_id)) or 0
+
+
+async def get_chunk_count(session: AsyncSession, book_id: int) -> int:
+    """Return the number of indexed chunks for one book."""
+    return await session.scalar(select(func.count(EbookChunk.id)).where(EbookChunk.source_id == book_id)) or 0
+
+
+async def get_candidate_count(session: AsyncSession, book_id: int) -> int:
+    """Return the number of indexed candidates for one book."""
+    return (
+        await session.scalar(select(func.count(EbookCandidatePhrase.id)).where(EbookCandidatePhrase.book_id == book_id))
+        or 0
+    )
+
+
+async def get_judged_candidate_count(session: AsyncSession, book_id: int) -> int:
+    """Return the number of judged candidates for one book."""
+    return (
+        await session.scalar(
+            select(func.count(EbookCandidatePhrase.id)).where(
+                EbookCandidatePhrase.book_id == book_id,
+                EbookCandidatePhrase.llm_judged.is_(True),
+            )
+        )
+        or 0
+    )
+
+
+async def get_candidates(session: AsyncSession, book_id: int) -> list[EbookCandidatePhrase]:
+    """Return the indexed candidates for one book."""
+    return list(
+        await session.scalars(
+            select(EbookCandidatePhrase)
+            .where(EbookCandidatePhrase.book_id == book_id)
+            .order_by(EbookCandidatePhrase.candidate_score.desc())
+            .limit(100)
+        )
+    )
+
+
+async def get_protected_phrases(session: AsyncSession, book_id: int) -> list[EbookProtectedPhrase]:
+    """Return the protected phrases for one book."""
+    return list(
+        await session.scalars(
+            select(EbookProtectedPhrase)
+            .where(EbookProtectedPhrase.book_id == book_id)
+            .order_by(EbookProtectedPhrase.importance.desc())
+            .limit(100)
+        )
+    )
+
+
+@router.get("/books/{source_id}", response_class=HTMLResponse)
+async def book_detail(source_id: int, request: Request, session: AsyncDbSession) -> HTMLResponse:
+    """Render details for one indexed book."""
+    source = await session.get(EbookSource, source_id)
+    phrase_status_message = None
+    recalculated = request.query_params.get("phrases_recalculated")
+    if recalculated is not None:
+        phrase_status_message = f"Recalculated phrases; {recalculated} candidates generated"
+    judgment_outcome = pop_book_judgment_outcome(request.app, source_id)
+    if judgment_outcome is not None:
+        phrase_status_message = judgment_outcome
+    judging_in_progress = is_judging_book(request.app, source_id)
+    if judging_in_progress:
+        phrase_status_message = "Judging candidate phrases in the background; refresh to see progress"
+    if source is not None:
+        chapter_count = await get_chapter_count(session, source.id)
+        chunk_count = await get_chunk_count(session, source.id)
+        candidate_count = await get_candidate_count(session, source.id)
+        judged_candidate_count = await get_judged_candidate_count(session, source.id)
+        protected_count = await count_protected_phrases(session, source.id)
+        candidates = await get_candidates(session, source.id)
+        protected_phrases = await get_protected_phrases(session, source.id)
+    else:
+        chapter_count = 0
+        chunk_count = 0
+        candidate_count = 0
+        judged_candidate_count = 0
+        protected_count = 0
+        candidates = []
+        protected_phrases = []
+    logger.info(
+        f"ebook_book_detail_loaded {source_id=} found={source is not None} {chapter_count=} {chunk_count=} "
+        f"{candidate_count=} {judged_candidate_count=} {protected_count=}"
+    )
+    return templates.TemplateResponse(
+        request,
+        "book_detail.html",
+        {
+            "candidate_count": candidate_count,
+            "candidates": candidates,
+            "chapter_count": chapter_count,
+            "chunk_count": chunk_count,
+            "judged_candidate_count": judged_candidate_count,
+            "judging_in_progress": judging_in_progress,
+            "protected_count": protected_count,
+            "protected_phrases": protected_phrases,
+            "phrase_status_message": phrase_status_message,
+            "source": source,
+        },
+    )
+
+
+@router.post("/books/{source_id}/recalculate-phrases")
+async def recalculate_book_phrases(source_id: int, config: AppConfig, session: AsyncDbSession) -> RedirectResponse:
+    """Clear and regenerate candidate phrases for one indexed book."""
+    source = await session.get(EbookSource, source_id)
+    if source is None:
+        raise HTTPException(status_code=404, detail="Book not found")
+
+    try:
+        result = await recalculate_candidate_phrases_for_book(session, source, config)
+    except ValueError as error:
+        raise HTTPException(status_code=409, detail=str(error)) from error
+    logger.info(
+        f"ebook_book_phrase_recalculation_complete {source_id=} {result.candidate_phrases=} "
+        f"{result.deleted_candidates=} {result.deleted_protected_phrases=} {result.deleted_aliases=} "
+        f"{result.deleted_mentions=}"
+    )
+    return RedirectResponse(
+        url=f"/books/{source_id}?phrases_recalculated={result.candidate_phrases}",
+        status_code=303,
+    )
+
+
+@router.post("/books/{source_id}/judge-phrases")
+async def judge_book_phrases(
+    source_id: int,
+    request: Request,
+    background_tasks: BackgroundTasks,
+    session: AsyncDbSession,
+) -> RedirectResponse:
+    """Queue background judging of one book's candidate phrases and return immediately."""
+    source = await session.get(EbookSource, source_id)
+    if source is None:
+        raise HTTPException(status_code=404, detail="Book not found")
+
+    started = start_book_phrase_judgment(request.app, background_tasks, source.id)
+    logger.info(f"ebook_book_phrase_judgment_requested {source_id=} {started=}")
+    return RedirectResponse(url=f"/books/{source_id}", status_code=303)

python/ebook_search/api/routes/search.py

@@ -0,0 +1,123 @@
+"""Search routes for the EPUB search web UI."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import replace
+from time import perf_counter
+from typing import TYPE_CHECKING, Annotated
+
+from fastapi import APIRouter, Form, Request
+from fastapi.responses import HTMLResponse
+
+from python.ebook_search.answer import answer_query
+from python.ebook_search.api.dependencies import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
+    AppConfig,
+    AppHttpClient,
+)
+from python.ebook_search.api.web import error_response, templates
+from python.ebook_search.guardrails import (
+    CitationReport,
+    is_confident,
+    retrieval_confidence,
+    validate_citations,
+)
+from python.ebook_search.search import SearchResponse, search_ebooks
+from python.ebook_search.timing import runtime_step_from_start
+from python.fastapi_tools import AppAsyncEngine  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
+
+if TYPE_CHECKING:
+    import httpx
+
+    from python.ebook_search.config import EbookSearchConfig
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+async def build_answer(
+    client: httpx.AsyncClient,
+    query: str,
+    response: SearchResponse,
+    config: EbookSearchConfig,
+) -> tuple[str, bool, CitationReport | None]:
+    """Generate the answer for a search, returning ``(answer, low_confidence, citation_report)``."""
+    if not config.answer_enabled:
+        logger.info("ebook_answer_skipped_disabled")
+        return "Answer generation is disabled. Source chunks are shown below.", False, None
+
+    if not is_confident(response.results, config):
+        logger.info(
+            f"ebook_answer_low_confidence confidence={retrieval_confidence(response.results):.4f} "
+            f"{config.min_retrieval_confidence=:.4f}"
+        )
+        answer = (
+            "Retrieval confidence is low for this query, so answer generation was skipped. "
+            "Source chunks are shown below."
+        )
+        return answer, True, None
+
+    try:
+        answer = await answer_query(client, query, response.results, config)
+    except RuntimeError as error:
+        logger.warning(f"ebook_answer_request_failed_falling_back {error=}")
+        return "Answer generation failed. Source chunks are still shown below.", False, None
+
+    citation_report = None
+    if config.validate_citations_enabled and response.results:
+        citation_report = validate_citations(answer, len(response.results))
+        if citation_report.invalid or not citation_report.grounded:
+            logger.warning(f"ebook_answer_citation_issue {citation_report.invalid=} {citation_report.grounded=}")
+    return answer, False, citation_report
+
+
+@router.post("/search", response_class=HTMLResponse)
+async def search(
+    request: Request,
+    config: AppConfig,
+    engine: AppAsyncEngine,
+    client: AppHttpClient,
+    query: Annotated[str, Form()],
+    *,
+    rerank: Annotated[bool, Form()] = False,
+    phrase_matching: Annotated[bool, Form()] = False,
+) -> HTMLResponse:
+    """Run a search and render HTMX results."""
+    try:
+        response = await search_ebooks(
+            engine,
+            client,
+            query,
+            config,
+            rerank=rerank,
+            phrase_matching=phrase_matching,
+        )
+    except Exception as error:
+        logger.exception("ebook_search_request_failed")
+        return error_response(request, error)
+
+    answer_start = perf_counter()
+    answer, low_confidence, citation_report = await build_answer(client, query, response, config)
+    answer_step_name = "Answer generation" if config.answer_enabled else "Answer skipped"
+    response = replace(
+        response,
+        timings=(*response.timings, runtime_step_from_start(answer_step_name, answer_start)),
+    )
+
+    for step in response.timings:
+        logger.info(f"ebook_search_timing {step.name=} {step.duration_ms=:.1f}")
+    logger.info(
+        f"ebook_search_request_complete results={len(response.results)} {response.rank_label=} "
+        f"{response.total_runtime_ms=:.1f}"
+    )
+    return templates.TemplateResponse(
+        request,
+        "partials/results.html",
+        {
+            "answer": answer,
+            "response": response,
+            "low_confidence": low_confidence,
+            "citation_report": citation_report,
+        },
+    )

python/ebook_search/api/static/style.css

@@ -0,0 +1,447 @@
+:root {
+    --bg: #f4f5f7;
+    --surface: #ffffff;
+    --border: #e3e5ea;
+    --text: #1c1f24;
+    --muted: #6b7280;
+    --accent: #4f46e5;
+    --accent-soft: #eef0fe;
+    --danger: #b42318;
+    --warn-bg: #fff8eb;
+    --warn-border: #e0a92e;
+    --warn-text: #7a5008;
+    --radius: 12px;
+    --shadow: 0 1px 2px rgba(16, 24, 40, 0.04), 0 1px 3px rgba(16, 24, 40, 0.08);
+}
+
+html.theme-dark {
+    --bg: #0f1117;
+    --surface: #1a1d25;
+    --border: #2b303b;
+    --text: #e6e8ec;
+    --muted: #9aa1ad;
+    --accent: #818cf8;
+    --accent-soft: #262b45;
+    --danger: #f97066;
+    --warn-bg: #2a2410;
+    --warn-border: #b9881f;
+    --warn-text: #e8c97a;
+    --shadow: 0 1px 2px rgba(0, 0, 0, 0.3), 0 1px 3px rgba(0, 0, 0, 0.4);
+    color-scheme: dark;
+}
+
+* {
+    box-sizing: border-box;
+}
+
+body {
+    margin: 0;
+    background: var(--bg);
+    color: var(--text);
+    font-family: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+    line-height: 1.55;
+}
+
+main {
+    max-width: 820px;
+    margin: 0 auto;
+    padding: 32px 20px 64px;
+}
+
+/* Header / nav */
+.site-header {
+    background: var(--surface);
+    border-bottom: 1px solid var(--border);
+    position: sticky;
+    top: 0;
+    z-index: 10;
+}
+
+.site-nav {
+    max-width: 820px;
+    margin: 0 auto;
+    padding: 12px 20px;
+    display: flex;
+    align-items: center;
+    gap: 20px;
+}
+
+.brand {
+    font-weight: 700;
+    font-size: 1.05rem;
+    color: var(--text);
+    text-decoration: none;
+}
+
+.nav-links {
+    display: flex;
+    gap: 6px;
+    margin-right: auto;
+}
+
+.nav-links a {
+    padding: 6px 12px;
+    border-radius: 8px;
+    color: var(--muted);
+    text-decoration: none;
+    font-size: 0.94rem;
+    transition: background 0.15s, color 0.15s;
+}
+
+.nav-links a:hover {
+    background: var(--accent-soft);
+    color: var(--accent);
+}
+
+.dev-toggle {
+    display: inline-flex;
+    align-items: center;
+    gap: 6px;
+    font-size: 0.85rem;
+    color: var(--muted);
+    cursor: pointer;
+    user-select: none;
+}
+
+.theme-toggle {
+    display: inline-flex;
+    align-items: center;
+    justify-content: center;
+    width: 34px;
+    height: 34px;
+    padding: 0;
+    font-size: 1rem;
+    line-height: 1;
+    color: var(--text);
+    background: var(--bg);
+    border: 1px solid var(--border);
+    border-radius: 8px;
+    cursor: pointer;
+}
+
+.theme-toggle:hover {
+    border-color: var(--accent);
+    filter: none;
+}
+
+h1 {
+    font-size: 1.6rem;
+    margin: 0 0 20px;
+}
+
+h2 {
+    font-size: 1.15rem;
+    margin: 0 0 8px;
+}
+
+/* Cards */
+.card {
+    background: var(--surface);
+    border: 1px solid var(--border);
+    border-radius: var(--radius);
+    box-shadow: var(--shadow);
+    padding: 20px;
+}
+
+/* Search form */
+form {
+    margin: 0;
+}
+
+label {
+    font-weight: 600;
+    font-size: 0.92rem;
+}
+
+textarea {
+    display: block;
+    width: 100%;
+    margin: 8px 0 16px;
+    padding: 12px 14px;
+    font: inherit;
+    color: var(--text);
+    background: var(--surface);
+    border: 1px solid var(--border);
+    border-radius: 10px;
+    resize: vertical;
+    transition: border-color 0.15s, box-shadow 0.15s;
+}
+
+textarea:focus {
+    outline: none;
+    border-color: var(--accent);
+    box-shadow: 0 0 0 3px var(--accent-soft);
+}
+
+.form-row {
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    gap: 12px;
+    flex-wrap: wrap;
+}
+
+.search-toggles {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 14px;
+}
+
+button {
+    padding: 10px 20px;
+    font: inherit;
+    font-weight: 600;
+    color: #fff;
+    background: var(--accent);
+    border: none;
+    border-radius: 10px;
+    cursor: pointer;
+    transition: filter 0.15s;
+}
+
+button:hover {
+    filter: brightness(1.08);
+}
+
+.check {
+    display: inline-flex;
+    gap: 8px;
+    align-items: center;
+    font-weight: 500;
+    color: var(--muted);
+}
+
+.actions {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 12px;
+    margin-bottom: 24px;
+}
+
+.actions-grid {
+    display: grid;
+    grid-template-columns: repeat(2, max-content);
+}
+
+/* Answer + results */
+#results {
+    display: block;
+    margin-top: 28px;
+}
+
+.rank-label {
+    font-size: 0.82rem;
+    font-weight: 600;
+    text-transform: uppercase;
+    letter-spacing: 0.04em;
+    color: var(--muted);
+    margin-bottom: 16px;
+}
+
+.answer {
+    background: var(--surface);
+    border: 1px solid var(--border);
+    border-radius: var(--radius);
+    box-shadow: var(--shadow);
+    padding: 20px;
+    margin-bottom: 24px;
+}
+
+.answer p:last-child {
+    margin-bottom: 0;
+}
+
+.results {
+    list-style: none;
+    padding: 0;
+    margin: 0;
+    display: grid;
+    gap: 16px;
+}
+
+.results > li {
+    background: var(--surface);
+    border: 1px solid var(--border);
+    border-radius: var(--radius);
+    box-shadow: var(--shadow);
+    padding: 18px 20px;
+}
+
+.results h2 {
+    font-size: 1.05rem;
+}
+
+.results h2 a {
+    color: var(--text);
+    text-decoration: none;
+}
+
+.results h2 a:hover {
+    color: var(--accent);
+}
+
+.meta {
+    color: var(--muted);
+    font-size: 0.88rem;
+    margin: 0 0 10px;
+}
+
+.scores {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 8px;
+    margin: 14px 0 0;
+}
+
+.scores div {
+    display: inline-flex;
+    gap: 6px;
+    align-items: baseline;
+    padding: 3px 10px;
+    background: var(--bg);
+    border: 1px solid var(--border);
+    border-radius: 999px;
+    font-size: 0.78rem;
+}
+
+.scores dt {
+    font-weight: 600;
+    color: var(--muted);
+}
+
+.scores dd {
+    margin: 0;
+    font-variant-numeric: tabular-nums;
+}
+
+.phrase-matches {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 8px;
+    align-items: baseline;
+    margin: 10px 0 0;
+    font-size: 0.78rem;
+}
+
+.phrase-matches-label {
+    color: var(--muted);
+    font-weight: 600;
+}
+
+.phrase-match {
+    padding: 3px 10px;
+    background: var(--bg);
+    border: 1px solid var(--border);
+    border-radius: 999px;
+    color: var(--accent);
+}
+
+/* Runtime — developer diagnostics, hidden unless dev mode is on */
+.runtime {
+    display: none;
+    background: var(--surface);
+    border: 1px solid var(--border);
+    border-radius: var(--radius);
+    box-shadow: var(--shadow);
+    padding: 18px 20px;
+    margin-bottom: 24px;
+}
+
+html.dev .runtime {
+    display: block;
+}
+
+.timing-chart {
+    display: grid;
+    gap: 8px;
+    padding: 0;
+    margin: 12px 0 0;
+    list-style: none;
+}
+
+.timing-chart li {
+    display: grid;
+    grid-template-columns: minmax(150px, 1fr) minmax(160px, 2fr) auto auto;
+    gap: 10px;
+    align-items: center;
+    font-size: 0.85rem;
+}
+
+.timing-bar {
+    height: 8px;
+    overflow: hidden;
+    background: var(--bg);
+    border-radius: 999px;
+}
+
+.timing-bar span {
+    display: block;
+    height: 100%;
+    background: var(--accent);
+    border-radius: 999px;
+}
+
+.timing-value,
+.timing-remaining {
+    color: var(--muted);
+    font-variant-numeric: tabular-nums;
+    text-align: right;
+}
+
+/* Tables */
+table {
+    width: 100%;
+    border-collapse: collapse;
+    background: var(--surface);
+    border: 1px solid var(--border);
+    border-radius: var(--radius);
+    overflow: hidden;
+}
+
+th,
+td {
+    padding: 10px 14px;
+    border-bottom: 1px solid var(--border);
+    text-align: left;
+    font-size: 0.9rem;
+}
+
+th {
+    font-weight: 600;
+    color: var(--muted);
+    background: var(--bg);
+}
+
+tbody tr:last-child td {
+    border-bottom: none;
+}
+
+dl dt {
+    font-weight: 600;
+    color: var(--muted);
+    font-size: 0.85rem;
+}
+
+dl dd {
+    margin: 0 0 12px;
+}
+
+/* States */
+.error {
+    color: var(--danger);
+    font-weight: 600;
+}
+
+.notice {
+    margin: 12px 0;
+    padding: 10px 14px;
+    border-left: 3px solid var(--warn-border);
+    border-radius: 6px;
+    background: var(--warn-bg);
+    color: var(--warn-text);
+    font-weight: 500;
+}
+
+.status {
+    color: var(--muted);
+}

python/ebook_search/api/templates/admin.html

@@ -0,0 +1,103 @@
+{% extends "base.html" %} {% block title %}EPUB Admin{% endblock %} {% block
+head %}
+<script src="https://unpkg.com/htmx.org@2.0.4"></script>
+{% endblock %} {% block content %}
+<h1>Admin</h1>
+<section id="admin-status"></section>
+<section class="actions">
+  <form hx-post="/admin/scan" hx-target="#admin-status" hx-swap="innerHTML">
+    <button type="submit">Scan</button>
+  </form>
+</section>
+<section>
+  <h2>Embeddings</h2>
+  <section class="actions">
+    <form
+      hx-post="/admin/embed-missing"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Embed</button>
+    </form>
+    <form
+      hx-post="/admin/embed-all"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Embed all</button>
+    </form>
+  </section>
+  <table>
+    <thead>
+      <tr>
+        <th>Model</th>
+        <th>Dimensions</th>
+        <th>Embedded</th>
+        <th>Missing</th>
+        <th>Total chunks</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for item in stats %}
+      <tr>
+        <td>{{ item.model_name }}</td>
+        <td>{{ item.dimension }}</td>
+        <td>{{ item.embedded_chunks }}</td>
+        <td>{{ item.missing_chunks }}</td>
+        <td>{{ item.total_chunks }}</td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+</section>
+<section>
+  <h2>Protected phrases</h2>
+  <section class="actions actions-grid">
+    <form
+      hx-post="/admin/phrases/generate-all"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Regenerate all phrases</button>
+    </form>
+    <form
+      hx-post="/admin/phrases/judge-all"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Judge all phrases</button>
+    </form>
+    <form
+      hx-post="/admin/phrases/judge-missing"
+      hx-target="#admin-status"
+      hx-swap="innerHTML"
+    >
+      <button type="submit">Judge missing phrases</button>
+    </form>
+  </section>
+  <table>
+    <thead>
+      <tr>
+        <th>Candidates</th>
+        <th>Judged</th>
+        <th>Unjudged</th>
+        <th>Protected</th>
+        <th>Books indexed</th>
+        <th>Books generated</th>
+        <th>Books fully judged</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+        <td>{{ phrase_stats.candidate_phrases }}</td>
+        <td>{{ phrase_stats.judged_candidates }}</td>
+        <td>{{ phrase_stats.unjudged_candidates }}</td>
+        <td>{{ phrase_stats.protected_phrases }}</td>
+        <td>{{ phrase_stats.total_books }}</td>
+        <td>{{ phrase_stats.books_with_candidates }}</td>
+        <td>{{ phrase_stats.books_fully_judged }}</td>
+      </tr>
+    </tbody>
+  </table>
+</section>
+{% endblock %}

python/ebook_search/api/templates/base.html

@@ -0,0 +1,71 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>{% block title %}EPUB Search{% endblock %}</title>
+    {% block head %}{% endblock %}
+    <link rel="stylesheet" href="/static/style.css?v={{ static_version('style.css') }}">
+    <script>
+        // Apply theme and dev mode before paint to avoid a flash of unstyled/wrong content.
+        (function () {
+            var stored = localStorage.getItem("ebook-theme");
+            var prefersDark = window.matchMedia("(prefers-color-scheme: dark)").matches;
+            var theme = stored || (prefersDark ? "dark" : "light");
+            document.documentElement.classList.add("theme-" + theme);
+            if (localStorage.getItem("ebook-dev-mode") === "on") {
+                document.documentElement.classList.add("dev");
+            }
+        })();
+    </script>
+</head>
+<body>
+    <header class="site-header">
+        <nav class="site-nav">
+            <a class="brand" href="/">EPUB Search</a>
+            <div class="nav-links">
+                <a href="/">Search</a>
+                <a href="/books">Books</a>
+                <a href="/admin">Admin</a>
+            </div>
+            <button type="button" id="theme-toggle" class="theme-toggle" title="Toggle light / dark theme" aria-label="Toggle theme"></button>
+            <label class="dev-toggle" title="Show developer diagnostics">
+                <input type="checkbox" id="dev-mode-toggle">
+                <span>Dev</span>
+            </label>
+        </nav>
+    </header>
+    <main>
+        {% block content %}{% endblock %}
+    </main>
+    <script>
+        (function () {
+            var toggle = document.getElementById("dev-mode-toggle");
+            if (toggle) {
+                toggle.checked = document.documentElement.classList.contains("dev");
+                toggle.addEventListener("change", function () {
+                    document.documentElement.classList.toggle("dev", toggle.checked);
+                    localStorage.setItem("ebook-dev-mode", toggle.checked ? "on" : "off");
+                });
+            }
+
+            var themeButton = document.getElementById("theme-toggle");
+            if (themeButton) {
+                var root = document.documentElement;
+                var sync = function () {
+                    var isDark = root.classList.contains("theme-dark");
+                    themeButton.textContent = isDark ? "☀️" : "🌙";
+                };
+                sync();
+                themeButton.addEventListener("click", function () {
+                    var next = root.classList.contains("theme-dark") ? "light" : "dark";
+                    root.classList.remove("theme-dark", "theme-light");
+                    root.classList.add("theme-" + next);
+                    localStorage.setItem("ebook-theme", next);
+                    sync();
+                });
+            }
+        })();
+    </script>
+</body>
+</html>

python/ebook_search/api/templates/book_detail.html

@@ -0,0 +1,109 @@
+{% extends "base.html" %}
+
+{% block title %}{% if source %}{{ source.title }}{% else %}Book not found{% endif %}{% endblock %}
+
+{% block content %}
+    {% if source %}
+        <h1>{{ source.title }}</h1>
+        <p class="meta">{{ source.author or "Unknown author" }}</p>
+        {% if phrase_status_message %}
+            <p class="status">{{ phrase_status_message }}</p>
+        {% endif %}
+        <dl class="card">
+            <dt>File</dt>
+            <dd>{{ source.file_path }}</dd>
+            <dt>Chapters</dt>
+            <dd>{{ chapter_count }}</dd>
+            <dt>Chunks</dt>
+            <dd>{{ chunk_count }}</dd>
+            <dt>Candidates</dt>
+            <dd>{{ candidate_count }}</dd>
+            <dt>Judged</dt>
+            <dd>{{ judged_candidate_count }}</dd>
+            <dt>Protected</dt>
+            <dd>{{ protected_count }}</dd>
+        </dl>
+        <form
+            method="post"
+            action="/books/{{ source.id }}/recalculate-phrases"
+            onsubmit="return confirm('Remove old phrases for this book and generate new candidates?');"
+        >
+            <button type="submit">Recalculate phrases</button>
+        </form>
+        <form
+            method="post"
+            action="/books/{{ source.id }}/judge-phrases"
+            onsubmit="return confirm('Judge candidate phrases for this book with the LLM?');"
+        >
+            <button type="submit"{% if judging_in_progress %} disabled{% endif %}>
+                {% if judging_in_progress %}Judging&hellip;{% else %}Judge phrases{% endif %}
+            </button>
+        </form>
+
+        <section>
+            <h2>Candidate n-grams</h2>
+            {% if candidates %}
+                <table>
+                    <thead>
+                        <tr>
+                            <th>Phrase</th>
+                            <th>Status</th>
+                            <th>Score</th>
+                            <th>Count</th>
+                            <th>Chapters</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for candidate in candidates %}
+                            <tr>
+                                <td>{{ candidate.phrase_text }}</td>
+                                <td>
+                                    {% if candidate.llm_judged %}
+                                        {% if candidate.llm_keep %}Kept{% else %}Rejected{% endif %}
+                                    {% else %}
+                                        Candidate
+                                    {% endif %}
+                                </td>
+                                <td>{{ "%.2f"|format(candidate.candidate_score) }}</td>
+                                <td>{{ candidate.raw_count }}</td>
+                                <td>{{ candidate.chapter_count }}</td>
+                            </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+            {% else %}
+                <p>No candidate n-grams.</p>
+            {% endif %}
+        </section>
+
+        <section>
+            <h2>Protected phrases</h2>
+            {% if protected_phrases %}
+                <table>
+                    <thead>
+                        <tr>
+                            <th>Phrase</th>
+                            <th>Type</th>
+                            <th>Confidence</th>
+                            <th>Importance</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for phrase in protected_phrases %}
+                            <tr>
+                                <td>{{ phrase.phrase_text }}</td>
+                                <td>{{ phrase.phrase_type or "phrase" }}</td>
+                                <td>{{ "%.2f"|format(phrase.confidence) }}</td>
+                                <td>{{ "%.2f"|format(phrase.importance) }}</td>
+                            </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+            {% else %}
+                <p>No protected phrases.</p>
+            {% endif %}
+        </section>
+    {% else %}
+        <h1>Book not found</h1>
+    {% endif %}
+{% endblock %}

python/ebook_search/api/templates/books.html

@@ -0,0 +1,19 @@
+{% extends "base.html" %}
+
+{% block title %}EPUB Books{% endblock %}
+
+{% block content %}
+    <h1>Books</h1>
+    {% if sources %}
+        <ol class="results">
+            {% for source in sources %}
+                <li>
+                    <h2><a href="/books/{{ source.id }}">{{ source.title }}</a></h2>
+                    <p class="meta">{{ source.author or "Unknown author" }}</p>
+                </li>
+            {% endfor %}
+        </ol>
+    {% else %}
+        <p>No EPUBs indexed.</p>
+    {% endif %}
+{% endblock %}

python/ebook_search/api/templates/partials/admin_status.html

@@ -0,0 +1 @@
+<p class="status">{{ message }}</p>

python/ebook_search/api/templates/partials/error.html

@@ -0,0 +1 @@
+<p class="error">{{ message }}</p>

python/ebook_search/api/templates/partials/results.html

@@ -0,0 +1,98 @@
+<div class="rank-label">{{ response.rank_label }}</div>
+{% if response.timings %}
+    <section class="runtime">
+        <h2>Runtime</h2>
+        <p class="meta">Total {{ "%.1f"|format(response.total_runtime_ms) }} ms</p>
+        <ol class="timing-chart">
+            {% set total = response.total_runtime_ms %}
+            {% set ns = namespace(remaining=total) %}
+            {% for step in response.timings %}
+                {% set width = (step.duration_ms / total * 100) if total else 0 %}
+                {% if step.counts_toward_total %}
+                    {% set ns.remaining = ns.remaining - step.duration_ms %}
+                {% endif %}
+                <li>
+                    <span class="timing-label">{{ step.name }}</span>
+                    <span class="timing-bar"><span style="width: {{ "%.2f"|format(width) }}%"></span></span>
+                    <span class="timing-value">{{ "%.1f"|format(step.duration_ms) }} ms</span>
+                    <span class="timing-remaining">{{ "%.1f"|format([ns.remaining, 0]|max) }} ms left</span>
+                </li>
+            {% endfor %}
+        </ol>
+    </section>
+{% endif %}
+<section class="answer">
+    <h2>Answer</h2>
+    {% if low_confidence|default(false) %}
+        <p class="notice">Low retrieval confidence — answer generation was skipped.</p>
+    {% endif %}
+    {% set report = citation_report|default(none) %}
+    {% if report is not none and not report.grounded %}
+        <p class="notice">Unverified — no source citations were found in this answer.</p>
+    {% endif %}
+    {% if report is not none and report.invalid %}
+        <p class="notice">Invalid citations: {{ report.invalid|join(", ") }} (no matching source).</p>
+    {% endif %}
+    <p>{{ answer }}</p>
+</section>
+{% if response.results %}
+    <ol class="results">
+        {% for result in response.results %}
+            <li>
+                <h2>
+                    {% if result.source_id %}
+                        <a href="/books/{{ result.source_id }}">{{ result.source_title }}</a>
+                    {% else %}
+                        {{ result.source_title }}
+                    {% endif %}
+                </h2>
+                <p class="meta">
+                    {% if result.source_author %}{{ result.source_author }}{% endif %}
+                    {% if result.chapter_title %} · {{ result.chapter_title }}{% endif %}
+                    {% if result.page_label %} · page {{ result.page_label }}{% endif %}
+                </p>
+                <p>{{ result.text }}</p>
+                <dl class="scores">
+                    <div>
+                        <dt>final</dt>
+                        <dd>{{ "%.3f"|format(result.score) }}</dd>
+                    </div>
+                    {% if result.rerank_score is not none %}
+                        <div>
+                            <dt>rerank</dt>
+                            <dd>{{ "%.3f"|format(result.rerank_score) }}</dd>
+                        </div>
+                    {% endif %}
+                    {% if result.vector_score is not none %}
+                        <div>
+                            <dt>vector cosine</dt>
+                            <dd>{{ "%.3f"|format(result.vector_score) }}</dd>
+                        </div>
+                    {% endif %}
+                    {% if result.bm25_score is not none %}
+                        <div>
+                            <dt>BM25</dt>
+                            <dd>{{ "%.6f"|format(result.bm25_score) }}</dd>
+                        </div>
+                    {% endif %}
+                    {% if result.fused_score is not none %}
+                        <div>
+                            <dt>RRF</dt>
+                            <dd>{{ "%.3f"|format(result.fused_score) }}</dd>
+                        </div>
+                    {% endif %}
+                </dl>
+                {% if result.matched_phrases %}
+                    <p class="phrase-matches">
+                        <span class="phrase-matches-label">boosted by</span>
+                        {% for phrase in result.matched_phrases %}
+                            <span class="phrase-match">{{ phrase }}</span>
+                        {% endfor %}
+                    </p>
+                {% endif %}
+            </li>
+        {% endfor %}
+    </ol>
+{% else %}
+    <p>No results.</p>
+{% endif %}

python/ebook_search/api/templates/search.html

@@ -0,0 +1,32 @@
+{% extends "base.html" %}
+
+{% block title %}EPUB Search{% endblock %}
+{% block head %}<script src="https://unpkg.com/htmx.org@2.0.4"></script>{% endblock %}
+
+{% block content %}
+    <h1>Search</h1>
+    <form class="card" hx-post="/search" hx-target="#results" hx-swap="innerHTML">
+        <label for="query">What are you looking for?</label>
+        <textarea id="query" name="query" rows="4" placeholder="Ask a question or paste a passage…" required
+            onkeydown="if (event.key === 'Enter' && !event.shiftKey) { event.preventDefault(); this.form.requestSubmit(); }"></textarea>
+        <div class="form-row">
+            <div class="search-toggles">
+                <label class="check">
+                    <input type="checkbox" name="rerank" value="true" {% if config.rerank.enabled %}checked{% endif %}>
+                    Rerank
+                </label>
+                <label class="check">
+                    <input
+                        type="checkbox"
+                        name="phrase_matching"
+                        value="true"
+                        {% if config.phrase_matching_enabled %}checked{% endif %}
+                    >
+                    Phrase matching
+                </label>
+            </div>
+            <button type="submit">Search</button>
+        </div>
+    </form>
+    <section id="results"></section>
+{% endblock %}

python/ebook_search/api/web.py

@@ -0,0 +1,33 @@
+"""Shared web UI resources for EPUB search."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from fastapi.templating import Jinja2Templates
+
+if TYPE_CHECKING:
+    from fastapi import Request
+    from fastapi.responses import HTMLResponse
+
+PACKAGE_DIR = Path(__file__).resolve().parent
+TEMPLATE_DIR = PACKAGE_DIR / "templates"
+STATIC_DIR = PACKAGE_DIR / "static"
+
+
+def static_version(filename: str) -> int:
+    """Return a cache-busting token for a static file based on its modification time."""
+    try:
+        return int((STATIC_DIR / filename).stat().st_mtime)
+    except OSError:
+        return 0
+
+
+templates = Jinja2Templates(directory=TEMPLATE_DIR)
+templates.env.globals["static_version"] = static_version
+
+
+def error_response(request: Request, message: object) -> HTMLResponse:
+    """Render the shared error partial for a failed UI request."""
+    return templates.TemplateResponse(request, "partials/error.html", {"message": str(message)}, status_code=500)

python/ebook_search/bm25_corpus.py

@@ -0,0 +1,274 @@
+"""Persisted BM25 corpus management."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import shutil
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from functools import cache
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import bm25s
+from sqlalchemy import func, select, union_all
+
+from python.ebook_search.chunk_records import CHUNK_RECORD_COLUMNS
+from python.orm.richie import EbookChapter, EbookChunk, EbookSource
+
+if TYPE_CHECKING:
+    from sqlalchemy.ext.asyncio import AsyncSession
+
+    from python.ebook_search.config import EbookSearchConfig
+
+logger = logging.getLogger(__name__)
+MANIFEST_NAME = "manifest.json"
+REQUIRED_INDEX_FILES = frozenset(
+    {
+        "data.csc.index.npy",
+        "indices.csc.index.npy",
+        "indptr.csc.index.npy",
+        "params.index.json",
+        "vocab.index.json",
+        "corpus.jsonl",
+    }
+)
+
+
+@dataclass(frozen=True)
+class BM25Manifest:
+    """Metadata describing a persisted BM25 corpus."""
+
+    created_at: datetime
+    db_updated_at: datetime | None
+    chunk_count: int
+
+
+@dataclass(frozen=True)
+class BM25Corpus:
+    """Loaded persisted BM25 corpus and retriever."""
+
+    retriever: object | None
+    records: tuple[dict[str, object], ...]
+    manifest: BM25Manifest
+
+
+class BM25CorpusUnavailableError(RuntimeError):
+    """Raised when the persisted BM25 corpus cannot be loaded."""
+
+
+def bm25_index_path(config: EbookSearchConfig) -> Path:
+    """Return the configured BM25 index root path relative to the current working directory."""
+    path = Path(config.bm25_index_dir).expanduser()
+    if path.is_absolute():
+        return path
+    return Path.cwd() / path
+
+
+def get_current_bm25_index(index_path: Path) -> Path:
+    """Return the live BM25 index directory."""
+    current_path = index_path / "current"
+    if current_path.exists() or current_path.is_symlink():
+        return current_path
+    return index_path
+
+
+async def ensure_bm25_corpus(session: AsyncSession, config: EbookSearchConfig) -> None:
+    """Create or refresh the persisted BM25 corpus when it is missing or stale."""
+    index_path = bm25_index_path(config)
+    manifest = read_bm25_manifest(index_path)
+    db_updated_at = await corpus_last_updated_at(session)
+    if not bm25_index_exists(index_path, manifest):
+        logger.info(f"ebook_bm25_index_missing {index_path=}")
+        await refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
+        return
+    if db_updated_at is not None and manifest is not None and manifest.created_at < db_updated_at:
+        logger.info(
+            f"ebook_bm25_index_stale {index_path=} created_at={manifest.created_at.isoformat()} "
+            f"db_updated_at={db_updated_at.isoformat()}"
+        )
+        await refresh_bm25_corpus(session, config, db_updated_at=db_updated_at)
+        return
+    logger.info(
+        f"ebook_bm25_index_current {index_path=} chunks={manifest.chunk_count if manifest else 0} "
+        f"created_at={manifest.created_at.isoformat() if manifest else None}"
+    )
+
+
+async def refresh_bm25_corpus(
+    session: AsyncSession,
+    config: EbookSearchConfig,
+    *,
+    db_updated_at: datetime | None = None,
+) -> BM25Manifest:
+    """Rebuild and persist the BM25 corpus from the current database chunks.
+
+    The index build is CPU and disk work, so it runs in a worker thread.
+    """
+    index_path = bm25_index_path(config)
+    records, texts = await fetch_bm25_corpus_records(session)
+    manifest = BM25Manifest(
+        created_at=datetime.now(tz=UTC),
+        db_updated_at=db_updated_at if db_updated_at is not None else await corpus_last_updated_at(session),
+        chunk_count=len(records),
+    )
+    await asyncio.to_thread(write_bm25_corpus, index_path, records, texts, manifest)
+    logger.info(
+        f"ebook_bm25_index_refreshed {index_path=} {manifest.chunk_count=} created_at={manifest.created_at.isoformat()}"
+    )
+    return manifest
+
+
+@cache
+def load_bm25_corpus(config: EbookSearchConfig) -> BM25Corpus:
+    """Load the BM25 corpus into memory once per process.
+
+    Background refresh tasks clear this cache after rebuilding the on-disk corpus.
+    """
+    index_path = bm25_index_path(config)
+    active_index_path = get_current_bm25_index(index_path)
+    logger.info(f"ebook_bm25_corpus_cache_load {index_path=} {active_index_path=}")
+    manifest = read_bm25_manifest(index_path)
+    if manifest is None or not bm25_index_exists(index_path, manifest):
+        msg = f"BM25 corpus is not available: {index_path}"
+        raise BM25CorpusUnavailableError(msg)
+    if manifest.chunk_count == 0:
+        return BM25Corpus(retriever=None, records=(), manifest=manifest)
+
+    retriever = bm25s.BM25.load(active_index_path, load_corpus=True, mmap=True)
+    records = tuple(dict(record) for record in retriever.corpus)
+    return BM25Corpus(retriever=retriever, records=records, manifest=manifest)
+
+
+def score_bm25_corpus(query: str, corpus: BM25Corpus, *, limit: int) -> list[tuple[dict[str, object], float]]:
+    """Score a query against a loaded BM25 corpus."""
+    if corpus.retriever is None or not corpus.records:
+        return []
+    k = min(limit, len(corpus.records))
+    documents, scores = corpus.retriever.retrieve(
+        bm25s.tokenize(query, show_progress=False),
+        corpus=list(corpus.records),
+        k=k,
+        show_progress=False,
+    )
+    results: list[tuple[dict[str, object], float]] = []
+    for document, score in zip(documents[0], scores[0], strict=True):
+        score_value = float(score)
+        if score_value <= 0:
+            continue
+        results.append((dict(document), score_value))
+    return results
+
+
+async def fetch_bm25_corpus_records(session: AsyncSession) -> tuple[list[dict[str, object]], list[str]]:
+    """Fetch persistable BM25 corpus records and their matching index texts from the database.
+
+    search_text is only needed to build the index, so it is returned separately instead of
+    being persisted into the corpus records, which would double the corpus size.
+    """
+    statement = (
+        select(
+            *CHUNK_RECORD_COLUMNS,
+            EbookChunk.search_text.label("bm25_text"),
+        )
+        .select_from(EbookChunk)
+        .join(EbookSource, EbookSource.id == EbookChunk.source_id)
+        .outerjoin(EbookChapter, EbookChapter.id == EbookChunk.chapter_id)
+        .order_by(EbookChunk.id)
+    )
+    records: list[dict[str, object]] = []
+    texts: list[str] = []
+    for row in (await session.execute(statement)).mappings():
+        record = dict(row)
+        texts.append(str(record.pop("bm25_text")))
+        records.append(record)
+    return records, texts
+
+
+async def corpus_last_updated_at(session: AsyncSession) -> datetime | None:
+    """Return the latest source/chapter/chunk update timestamp relevant to BM25 text."""
+    update_times = union_all(
+        select(func.max(EbookSource.updated).label("updated")),
+        select(func.max(EbookChapter.updated).label("updated")),
+        select(func.max(EbookChunk.updated).label("updated")),
+    ).subquery()
+    return await session.scalar(select(func.max(update_times.c.updated)))
+
+
+def write_bm25_corpus(
+    index_path: Path,
+    records: list[dict[str, object]],
+    texts: list[str],
+    manifest: BM25Manifest,
+) -> None:
+    """Write a BM25 corpus generation and publish it through the current symlink."""
+    index_path.mkdir(parents=True, exist_ok=True)
+
+    generations_path = index_path / "generations"
+    generations_path.mkdir(exist_ok=True)
+
+    generation_path = next_bm25_generation_path(generations_path, manifest.created_at)
+    current_path = index_path / "current"
+    next_current_path = index_path / f".current.{generation_path.name}.tmp"
+    try:
+        generation_path.mkdir()
+
+        # Empty corpora publish a manifest-only generation so startup succeeds before any chunks exist.
+        if records:
+            retriever = bm25s.BM25()
+            retriever.index(bm25s.tokenize(texts, show_progress=False), show_progress=False)
+            retriever.save(generation_path, corpus=records, show_progress=False)
+        write_bm25_manifest(generation_path, manifest)
+        next_current_path.unlink(missing_ok=True)
+        next_current_path.symlink_to(generation_path, target_is_directory=True)
+        next_current_path.replace(current_path)
+    except Exception:
+        next_current_path.unlink(missing_ok=True)
+        shutil.rmtree(generation_path, ignore_errors=True)
+        raise
+
+
+def read_bm25_manifest(index_path: Path) -> BM25Manifest | None:
+    """Read the BM25 manifest if it exists and is valid."""
+    manifest_path = get_current_bm25_index(index_path) / MANIFEST_NAME
+    if not manifest_path.exists():
+        return None
+    body = json.loads(manifest_path.read_text(encoding="utf-8"))
+    return BM25Manifest(
+        created_at=datetime.fromisoformat(str(body["created_at"])),
+        db_updated_at=datetime.fromisoformat(str(body["db_updated_at"])) if body.get("db_updated_at") else None,
+        chunk_count=int(body["chunk_count"]),
+    )
+
+
+def write_bm25_manifest(index_path: Path, manifest: BM25Manifest) -> None:
+    """Write the BM25 manifest to an index directory."""
+    body = {
+        "created_at": manifest.created_at.isoformat(),
+        "db_updated_at": manifest.db_updated_at.isoformat() if manifest.db_updated_at else None,
+        "chunk_count": manifest.chunk_count,
+    }
+    (index_path / MANIFEST_NAME).write_text(json.dumps(body, indent=2, sort_keys=True), encoding="utf-8")
+
+
+def bm25_index_exists(index_path: Path, manifest: BM25Manifest | None) -> bool:
+    """Return whether a usable persisted BM25 index exists."""
+    active_index_path = get_current_bm25_index(index_path)
+    if manifest is None or not active_index_path.is_dir():
+        return False
+    if manifest.chunk_count == 0:
+        return True
+    return all((active_index_path / file_name).exists() for file_name in REQUIRED_INDEX_FILES)
+
+
+def next_bm25_generation_path(generations_path: Path, created_at: datetime) -> Path:
+    """Return an unused dated BM25 generation path."""
+    base_name = created_at.astimezone(UTC).strftime("%Y%m%dT%H%M%S.%fZ")
+    generation_path = generations_path / base_name
+    suffix = 1
+    while generation_path.exists():
+        generation_path = generations_path / f"{base_name}.{suffix}"
+        suffix += 1
+    return generation_path

python/ebook_search/chunk_records.py

@@ -0,0 +1,13 @@
+"""Shared database columns used to build search-result records."""
+
+from python.orm.richie import EbookChapter, EbookChunk, EbookSource
+
+CHUNK_RECORD_COLUMNS = (
+    EbookChunk.id.label("chunk_id"),
+    EbookChunk.text.label("text"),
+    EbookSource.id.label("source_id"),
+    EbookSource.title.label("source_title"),
+    EbookSource.author.label("source_author"),
+    EbookChapter.title.label("chapter_title"),
+    EbookChunk.page_label.label("page_label"),
+)

python/ebook_search/config.py

@@ -0,0 +1,145 @@
+"""Configuration for the EPUB search app."""
+
+from __future__ import annotations
+
+from typing import Annotated, Self
+
+from pydantic import AliasChoices, Field, field_validator, model_validator
+from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
+
+
+def normalize_embedding_alias(model: str) -> str:
+    """Normalize a supported embedding alias to its provider model name."""
+    aliases = {
+        "Qwen3-Embedding-0.6B": "qwen3-embedding-0.6b",
+        "Qwen3-Embedding-4B": "qwen3-embedding-4b",
+        "Qwen3-Embedding-8B": "qwen3-embedding-8b",
+        "Qwen/Qwen3-Embedding-0.6B": "qwen3-embedding-0.6b",
+        "Qwen/Qwen3-Embedding-4B": "qwen3-embedding-4b",
+        "Qwen/Qwen3-Embedding-8B": "qwen3-embedding-8b",
+        "qwen3-embedding:0.6b": "qwen3-embedding-0.6b",
+        "qwen3-embedding:4b": "qwen3-embedding-4b",
+        "qwen3-embedding:8b": "qwen3-embedding-8b",
+        "qwen3-embedding-0.6b": "qwen3-embedding-0.6b",
+        "qwen3-embedding-4b": "qwen3-embedding-4b",
+        "qwen3-embedding-8b": "qwen3-embedding-8b",
+    }
+    standard_model = aliases.get(model)
+    if standard_model is None:
+        error = f"Embedding model {model} is not supported. Supported models are {aliases.keys()}"
+        raise ValueError(error)
+    return standard_model
+
+
+class RerankConfig(BaseSettings):
+    """vLLM reranker settings."""
+
+    model_config = SettingsConfigDict(env_prefix="EBOOK_SEARCH_RERANK_", frozen=True, protected_namespaces=())
+
+    enabled: bool = True
+    base_url: str = "http://192.168.90.25:8001"
+    model: str = "qwen3-reranker-06b"
+    candidates: int = 24
+    timeout_seconds: float = 30.0
+    score_weight: float = 0.7
+    hybrid_weight: float = 0.3
+
+
+class EbookSearchConfig(BaseSettings):
+    """Runtime settings for EPUB search."""
+
+    model_config = SettingsConfigDict(
+        env_prefix="EBOOK_SEARCH_",
+        frozen=True,
+        populate_by_name=True,
+        protected_namespaces=(),
+    )
+
+    rerank: RerankConfig = Field(default_factory=RerankConfig)
+    top_k: int = 12
+    library_paths: Annotated[tuple[str, ...], NoDecode] = ()
+    chunk_tokens: int = 700
+    chunk_overlap: int = 100
+    vllm_base_url: str = "https://ollama.com/v1"
+    vllm_api_key: str = Field(
+        default="not-needed",
+        validation_alias=AliasChoices("EBOOK_SEARCH_VLLM_API_KEY", "OLLAMA_API_KEY"),
+    )
+    chat_model: str = "deepseek-v4-flash"
+    answer_enabled: bool = True
+    embedding_base_url: str = "http://192.168.90.25:8000/v1"
+    embedding_api_key: str = "not-needed"
+    embedding_model: str = "qwen3-embedding-0.6b"
+    embedding_batch_size: int = 32
+    embedding_timeout_seconds: float = 60.0
+    chat_timeout_seconds: float = 60.0
+    vector_candidate_multiplier: int = 4
+    bm25_candidate_limit: int = 120
+    rrf_rank_constant: int = 60
+    min_retrieval_confidence: float = 0.0
+    validate_citations_enabled: bool = True
+    bm25_index_dir: str = ".ebook_search_bm25"
+    bm25_refresh_delay_seconds: int = 60
+    protected_phrase_max_candidates_per_book: int = 5000
+    protected_phrase_llm_candidates_per_book: int = 500
+    protected_phrase_extraction_workers: int = 16
+    phrase_judge_book_workers: int = 20
+    phrase_judge_phrase_workers: int = 100
+    protected_phrase_confidence_threshold: float = 0.80
+    phrase_matching_enabled: bool = True
+    phrase_hit_boost: float = 0.25
+    phrase_min_tokens: int = 2
+    phrase_max_tokens: int = 5
+    phrase_max_entity_tokens: int = 8
+    phrase_yake_top_k: int = 1000
+    phrase_yake_dedup_limit: float = 0.85
+    phrase_raw_ngram_min_count: int = 2
+    phrase_raw_count_score_threshold: int = 3
+    phrase_raw_count_high_score_threshold: int = 10
+    phrase_chapter_count_score_threshold: int = 2
+    phrase_chapter_count_high_score_threshold: int = 5
+    phrase_target_protected_per_book: int = 100
+    phrase_default_allow_nested: bool = False
+    phrase_default_suppress_children: bool = True
+    phrase_bad_start_score_penalty: float = 10.0
+    phrase_bad_end_score_penalty: float = 10.0
+    phrase_multi_source_score_bonus: float = 2.0
+    phrase_multi_source_min_sources: int = 2
+
+    @field_validator("library_paths", mode="before")
+    @classmethod
+    def split_library_paths(cls, value: object) -> object:
+        """Split a colon-separated library path string into a tuple of paths."""
+        if isinstance(value, str):
+            return tuple(path for path in value.split(":") if path)
+        return value
+
+    @field_validator("embedding_model")
+    @classmethod
+    def normalize_embedding(cls, value: str) -> str:
+        """Normalize the configured embedding alias to its provider model name."""
+        return normalize_embedding_alias(value)
+
+    @model_validator(mode="after")
+    def validate_runtime_consistency(self) -> Self:
+        """Reject configurations that cannot serve the features they enable."""
+        if not self.embedding_base_url.strip():
+            msg = "embedding_base_url must be set"
+            raise ValueError(msg)
+        if self.answer_enabled and (not self.vllm_base_url.strip() or not self.chat_model.strip()):
+            msg = "answer_enabled requires vllm_base_url and chat_model to be set"
+            raise ValueError(msg)
+        if self.rerank.enabled and not self.rerank.base_url.strip():
+            msg = "rerank.enabled requires rerank.base_url to be set"
+            raise ValueError(msg)
+        return self
+
+
+def load_rerank_config() -> RerankConfig:
+    """Load reranker config from environment variables."""
+    return RerankConfig()
+
+
+def load_config() -> EbookSearchConfig:
+    """Load EPUB search config from environment variables."""
+    return EbookSearchConfig()

python/ebook_search/docker/Dockerfile

@@ -0,0 +1,54 @@
+FROM python:3.14-slim AS base
+
+COPY --from=ghcr.io/astral-sh/uv:0.11.26 /uv /uvx /bin/
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    APP_DIR=/home/richie/dotfiles \
+    UV_PROJECT_ENVIRONMENT=/opt/venv \
+    UV_PYTHON_DOWNLOADS=never \
+    UV_NO_CACHE=1
+
+# Separate ENV instruction so ${APP_DIR} and ${PATH} from above resolve.
+ENV PYTHONPATH=${APP_DIR} \
+    PATH=/opt/venv/bin:${PATH}
+
+WORKDIR ${APP_DIR}
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends build-essential curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY python/ebook_search/docker/pyproject.toml python/ebook_search/docker/uv.lock ./
+
+RUN uv sync --locked --no-dev
+
+
+FROM base AS test
+
+RUN uv sync --locked
+
+COPY python ./python
+COPY tests/__init__.py ./tests/__init__.py
+COPY tests/ebook_search ./tests/ebook_search
+
+CMD ["pytest"]
+
+
+FROM base AS runtime
+
+ENV EBOOK_SEARCH_HOST=0.0.0.0 \
+    EBOOK_SEARCH_PORT=8070 \
+    EBOOK_SEARCH_BM25_INDEX_DIR=/data/bm25
+
+COPY python ./python
+
+RUN useradd --create-home --uid 10001 app \
+    && mkdir -p /data \
+    && chown -R app:app /home/richie /data
+
+USER app
+
+EXPOSE 8070
+
+CMD ["sh", "-c", "exec python -m python.ebook_search.api.main --host \"${EBOOK_SEARCH_HOST}\" --port \"${EBOOK_SEARCH_PORT}\" --log-level \"${EBOOK_SEARCH_LOG_LEVEL:-INFO}\""]

python/ebook_search/docker/README.md

@@ -0,0 +1,77 @@
+# Ebook Search Docker
+
+Run the EPUB search app against the existing Postgres database on `jeeves`:
+
+```sh
+python -m python.ebook_search.docker.containers start --library-path /path/to/epubs --build
+```
+
+All ebook-search Docker files live in this directory:
+
+- `Dockerfile` — multi-stage: `test` (runs pytest) and `runtime` (default target, the app image)
+- `docker-compose.yml`
+- `containers.py` — Typer lifecycle CLI
+- `pyproject.toml` / `uv.lock` — the container's uv-locked dependencies
+
+The app listens on `http://localhost:8070`.
+
+Useful lifecycle commands:
+
+```sh
+python -m python.ebook_search.docker.containers build
+python -m python.ebook_search.docker.containers start --library-path /path/to/epubs
+python -m python.ebook_search.docker.containers test
+python -m python.ebook_search.docker.containers logs
+python -m python.ebook_search.docker.containers ps
+python -m python.ebook_search.docker.containers stop
+```
+
+Direct compose usage from the repo root:
+
+```sh
+docker compose -f python/ebook_search/docker/docker-compose.yml ps
+```
+
+## Dependencies
+
+The image builds its environment with uv from `pyproject.toml` + `uv.lock` in this
+directory — this is the source of truth for the container's dependencies. To add or
+update a dependency, edit `pyproject.toml` here and regenerate the lock (uv is
+available in the `ebook-search` dev shell):
+
+```sh
+nix develop .#ebook-search -c uv lock --project python/ebook_search/docker
+```
+
+## Tests
+
+The main pytest suite excludes `tests/ebook_search` (its dependencies are no longer
+in the nix dev shell). The `test ebook search` CI workflow runs them in a uv env
+built from the lockfile in this directory — same commands work locally from the
+repo root (the `--override-ini` drops the main suite's ignore):
+
+```sh
+uv sync --locked --project python/ebook_search/docker
+uv run --project python/ebook_search/docker --no-sync pytest tests/ebook_search --override-ini addopts="-n auto -ra"
+```
+
+They can also run inside the Docker `test` image, which validates the image itself:
+
+```sh
+python -m python.ebook_search.docker.containers test
+```
+
+or the raw docker equivalent:
+
+```sh
+docker build --file python/ebook_search/docker/Dockerfile --target test --tag ebook-search:test .
+docker run --rm ebook-search:test
+```
+
+## Configuration
+
+The compose service loads the repo root `.env` into the container via `env_file`.
+
+Mount your EPUB directory by setting `EBOOK_LIBRARY_HOST_PATH` in an env file or on the command line. The container sees it as `/library`, and `EBOOK_SEARCH_LIBRARY_PATHS` is set to `/library` inside the container.
+
+Database connection settings are controlled by `RICHIE_DB`, `RICHIE_HOST`, `RICHIE_PORT`, `RICHIE_USER`, and `RICHIE_PASSWORD`. The default host is `jeeves`.

python/ebook_search/docker/__init__.py

@@ -0,0 +1 @@
+"""Docker packaging and lifecycle tooling for ebook search."""

python/ebook_search/docker/containers.py

@@ -0,0 +1,259 @@
+"""Docker container lifecycle management for ebook search."""
+
+from __future__ import annotations
+
+import logging
+import os
+import subprocess
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from python.common import configure_logger, get_repo_dir
+
+logger = logging.getLogger(__name__)
+
+
+def get_compose_file() -> Path:
+    """Return the path to the docker-compose.yml file."""
+    return Path(__file__).resolve().with_name("docker-compose.yml")
+
+
+def compose_base_args() -> list[str]:
+    """Return the common docker compose arguments for the ebook search stack."""
+    return ["compose", "-f", str(get_compose_file())]
+
+
+def docker_run(
+    arguments: list[str],
+    *,
+    env: dict[str, str] | None = None,
+    capture_output: bool = False,
+) -> subprocess.CompletedProcess[str]:
+    """Run docker with repo-root cwd and consistent error handling."""
+    logger.info(f"docker {' '.join(arguments)}")
+    return subprocess.run(
+        ["docker", *arguments],
+        cwd=get_repo_dir(),
+        env=env,
+        text=True,
+        check=False,
+        capture_output=capture_output,
+    )
+
+
+def compose_env(*, library_path: Path | None = None, port: int | None = None) -> dict[str, str]:
+    """Return environment variables passed to docker compose."""
+    env = os.environ.copy()
+    if library_path is not None:
+        resolved_library = library_path.expanduser().resolve()
+        if not resolved_library.exists():
+            msg = f"EPUB library path does not exist: {resolved_library}"
+            raise FileNotFoundError(msg)
+        env["EBOOK_LIBRARY_HOST_PATH"] = str(resolved_library)
+    if port is not None:
+        env["EBOOK_SEARCH_PORT"] = str(port)
+    return env
+
+
+def ensure_compose_file() -> None:
+    """Raise if the ebook search compose file is missing."""
+    if not get_compose_file().is_file():
+        msg = f"Compose file not found: {get_compose_file()}"
+        raise FileNotFoundError(msg)
+
+
+def build_image() -> None:
+    """Build the ebook search app image."""
+    ensure_compose_file()
+    result = docker_run([*compose_base_args(), "build"])
+    if result.returncode != 0:
+        msg = "Failed to build ebook search image"
+        raise RuntimeError(msg)
+
+
+def build_test_image() -> None:
+    """Build the ebook search test Docker image."""
+    dockerfile = Path(__file__).resolve().with_name("Dockerfile")
+    result = docker_run(["build", "--file", str(dockerfile), "--target", "test", "--tag", "ebook-search:test", "."])
+    if result.returncode != 0:
+        msg = "Failed to build ebook search test image"
+        raise RuntimeError(msg)
+
+
+def run_test_image() -> None:
+    """Run the ebook search test suite inside Docker."""
+    result = docker_run(["run", "--rm", "ebook-search:test"])
+    if result.returncode != 0:
+        msg = f"Ebook search tests failed with code {result.returncode}"
+        raise RuntimeError(msg)
+
+
+def start_stack(
+    *,
+    library_path: Path | None = None,
+    port: int | None = None,
+    build: bool = False,
+) -> None:
+    """Start the ebook search Docker compose stack."""
+    ensure_compose_file()
+    env = compose_env(library_path=library_path, port=port)
+    if build:
+        build_image()
+    result = docker_run(
+        [*compose_base_args(), "up", "-d"],
+        env=env,
+    )
+    if result.returncode != 0:
+        msg = f"Ebook search stack failed to start with code {result.returncode}"
+        raise RuntimeError(msg)
+    logger.info("Ebook search started.")
+
+
+def stop_stack(
+    *,
+    volumes: bool = False,
+) -> None:
+    """Stop and remove ebook search containers."""
+    ensure_compose_file()
+    command = [*compose_base_args(), "down"]
+    if volumes:
+        command.append("-v")
+    result = docker_run(command)
+    if result.returncode != 0:
+        msg = f"Ebook search stack failed to stop with code {result.returncode}"
+        raise RuntimeError(msg)
+
+
+def logs_stack(
+    *,
+    service: str | None = None,
+    tail: int = 100,
+    follow: bool = False,
+) -> str | None:
+    """Return recent logs from the ebook search stack."""
+    ensure_compose_file()
+    command = [*compose_base_args(), "logs", "--tail", str(tail)]
+    if follow:
+        command.append("--follow")
+    if service:
+        command.append(service)
+    result = docker_run(command, capture_output=not follow)
+    if result.returncode != 0:
+        return None
+    if follow:
+        return ""
+    return result.stdout + result.stderr
+
+
+def ps_stack() -> str | None:
+    """Return docker compose ps output for the ebook search stack."""
+    ensure_compose_file()
+    result = docker_run([*compose_base_args(), "ps"], capture_output=True)
+    if result.returncode != 0:
+        return None
+    return result.stdout + result.stderr
+
+
+app = typer.Typer(help="Ebook search Docker container management.", no_args_is_help=True)
+
+
+@app.command()
+def build() -> None:
+    """Build the ebook search Docker image."""
+    build_image()
+
+
+@app.command()
+def start(
+    library_path: Annotated[Path | None, typer.Option(help="Override host path containing EPUB files.")] = None,
+    port: Annotated[int | None, typer.Option(help="Override host port for the web UI.")] = None,
+    *,
+    build: Annotated[bool, typer.Option("--build", help="Build the image before starting.")] = False,
+    log_level: Annotated[str, typer.Option(help="Log level.")] = "INFO",
+) -> None:
+    """Start the ebook search container."""
+    configure_logger(log_level)
+    start_stack(
+        library_path=library_path,
+        port=port,
+        build=build,
+    )
+
+
+@app.command()
+def stop(
+    *,
+    volumes: Annotated[bool, typer.Option("--volumes", help="Also remove ebook search data volumes.")] = False,
+    log_level: Annotated[str, typer.Option(help="Log level.")] = "INFO",
+) -> None:
+    """Stop and remove ebook search containers."""
+    configure_logger(log_level)
+    stop_stack(volumes=volumes)
+
+
+@app.command()
+def restart(
+    library_path: Annotated[Path | None, typer.Option(help="Override host path containing EPUB files.")] = None,
+    port: Annotated[int | None, typer.Option(help="Override host port for the web UI.")] = None,
+    *,
+    build: Annotated[bool, typer.Option("--build", help="Build the image before starting.")] = False,
+    log_level: Annotated[str, typer.Option(help="Log level.")] = "INFO",
+) -> None:
+    """Restart the ebook search stack."""
+    configure_logger(log_level)
+    stop_stack()
+    start_stack(
+        library_path=library_path,
+        port=port,
+        build=build,
+    )
+
+
+@app.command()
+def logs(
+    service: Annotated[str | None, typer.Option(help="Service name, or omit for all services.")] = None,
+    tail: Annotated[int, typer.Option(help="Number of recent log lines.")] = 100,
+    *,
+    follow: Annotated[bool, typer.Option("--follow", "-f", help="Follow logs.")] = False,
+) -> None:
+    """Show recent ebook search container logs."""
+    output = logs_stack(service=service, tail=tail, follow=follow)
+    if output is None:
+        typer.echo("No ebook search containers found.")
+        raise typer.Exit(code=1)
+    if output:
+        typer.echo(output)
+
+
+@app.command("test")
+def run_tests(
+    *,
+    build: Annotated[bool, typer.Option("--build/--no-build", help="Build the test image before running.")] = True,
+    log_level: Annotated[str, typer.Option(help="Log level.")] = "INFO",
+) -> None:
+    """Run ebook search tests inside the Docker test image."""
+    configure_logger(log_level)
+    if build:
+        build_test_image()
+    run_test_image()
+
+
+@app.command("ps")
+def ps() -> None:
+    """Show ebook search container status."""
+    output = ps_stack()
+    if output is None:
+        typer.echo("No ebook search containers found.")
+        raise typer.Exit(code=1)
+    typer.echo(output)
+
+
+def cli() -> None:
+    """Typer entry point."""
+    app()
+
+
+if __name__ == "__main__":
+    cli()

python/ebook_search/docker/docker-compose.yml

@@ -0,0 +1,36 @@
+name: ebook-search
+
+services:
+  ebook-search:
+    build:
+      context: ../../..
+      dockerfile: python/ebook_search/docker/Dockerfile
+    image: ebook-search:latest
+    restart: unless-stopped
+    ports:
+      - "${EBOOK_SEARCH_PORT:-8070}:8070"
+    extra_hosts:
+      - "jeeves:192.168.90.40"
+    env_file:
+      - ../../../.env
+    environment:
+      EBOOK_SEARCH_HOST: "0.0.0.0"
+      EBOOK_SEARCH_PORT: "8070"
+      EBOOK_SEARCH_LIBRARY_PATHS: "/library"
+      EBOOK_SEARCH_BM25_INDEX_DIR: "/data/bm25"
+    volumes:
+      - "${EBOOK_LIBRARY_HOST_PATH:-/home/richie/ebooks}:/library:ro"
+      - ebook-search-data:/data
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -fsS http://127.0.0.1:8070/health >/dev/null || exit 1",
+        ]
+      interval: 30s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
+
+volumes:
+  ebook-search-data:

python/ebook_search/docker/pyproject.toml

@@ -0,0 +1,41 @@
+[project]
+name = "ebook-search"
+version = "0.1.0"
+description = "Locked runtime environment for the ebook search container."
+requires-python = "~=3.14.0"
+dependencies = [
+    "alembic",
+    "beautifulsoup4",
+    "bm25s",
+    "ebooklib",
+    "fastapi",
+    "httpx",
+    "jinja2",
+    "pgvector",
+    "psycopg[binary]",
+    "pydantic",
+    "pydantic-settings",
+    "python-multipart",
+    "sqlalchemy[asyncio]",
+    "tiktoken",
+    "typer",
+    "uvicorn[standard]",
+    "yake",
+]
+
+[dependency-groups]
+dev = [
+    "aiosqlite",
+    "pytest",
+    "pytest-asyncio",
+    "pytest-mock",
+    "pytest-xdist",
+]
+
+[tool.uv]
+package = false
+
+[tool.pytest.ini_options]
+addopts = "-n auto -ra"
+asyncio_mode = "auto"
+testpaths = ["tests/ebook_search"]

python/ebook_search/embeddings.py

@@ -0,0 +1,165 @@
+"""Embedding model helpers."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from sqlalchemy import func, select
+from sqlalchemy.dialects.postgresql import insert
+
+from python.ebook_search.llm_interface import request_embeddings
+from python.orm.richie import (
+    EbookChunk,
+    EbookChunkEmbedding1024,
+    EbookChunkEmbedding2560,
+    EbookChunkEmbedding4096,
+    EbookEmbeddingModel,
+)
+
+logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    import httpx
+    from sqlalchemy.ext.asyncio import AsyncSession
+
+    from python.ebook_search.config import EbookSearchConfig
+
+MODEL_DIMENSIONS = {
+    "qwen3-embedding-0.6b": 1024,
+    "qwen3-embedding-4b": 2560,
+    "qwen3-embedding-8b": 4096,
+}
+
+
+def get_embedding_table(
+    dimension: int,
+) -> type[EbookChunkEmbedding1024 | EbookChunkEmbedding2560 | EbookChunkEmbedding4096]:
+    """Return the embedding table mapped to an embedding dimension."""
+    embedding_tables = {
+        1024: EbookChunkEmbedding1024,
+        2560: EbookChunkEmbedding2560,
+        4096: EbookChunkEmbedding4096,
+    }
+    table = embedding_tables.get(dimension)
+    if not table:
+        msg = f"Embedding dimension {dimension} is not supported"
+        raise ValueError(msg)
+    return table
+
+
+@dataclass(frozen=True)
+class EmbeddingModelStats:
+    """Embedding coverage for one model."""
+
+    model_name: str
+    dimension: int
+    embedded_chunks: int
+    total_chunks: int
+
+    @property
+    def missing_chunks(self) -> int:
+        """Return chunks missing this embedding model."""
+        return max(self.total_chunks - self.embedded_chunks, 0)
+
+
+async def embed_texts(
+    client: httpx.AsyncClient,
+    texts: Sequence[str],
+    config: EbookSearchConfig,
+) -> list[list[float]]:
+    """Embed text with the configured vLLM embedding model."""
+    logger.info(f"ebook_embed_request_start {config.embedding_base_url=} {config.embedding_model=} count={len(texts)}")
+    vectors = await request_embeddings(client, texts, config)
+    expected_dimension = MODEL_DIMENSIONS[config.embedding_model]
+    for vector in vectors:
+        if len(vector) != expected_dimension:
+            msg = f"Expected {expected_dimension} dimensions, got {len(vector)}"
+            raise ValueError(msg)
+    logger.info(f"ebook_embed_request_complete {config.embedding_model=} count={len(vectors)} {expected_dimension=}")
+    return vectors
+
+
+async def embed_query(client: httpx.AsyncClient, query: str, config: EbookSearchConfig) -> list[float]:
+    """Embed a search query with the Qwen retrieval instruction."""
+    instructed_query = f"Instruct: Retrieve relevant passages for the query.\nQuery: {query}"
+    return (await embed_texts(client, [instructed_query], config))[0]
+
+
+async def ensure_embedding_models(session: AsyncSession) -> None:
+    """Ensure supported embedding model rows exist."""
+    for name, dimension in MODEL_DIMENSIONS.items():
+        existing = await session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == name))
+        if existing is None:
+            session.add(EbookEmbeddingModel(name=name, dimension=dimension, is_default=name == "qwen3-embedding-0.6b"))
+            logger.info(f"ebook_embedding_model_created {name=} {dimension=}")
+    await session.flush()
+
+
+async def embedding_model_stats(session: AsyncSession) -> list[EmbeddingModelStats]:
+    """Return embedding coverage counts for every supported model."""
+    total_chunks = await session.scalar(select(func.count(EbookChunk.id))) or 0
+    models = {
+        model.name: model
+        for model in await session.scalars(
+            select(EbookEmbeddingModel)
+            .where(EbookEmbeddingModel.name.in_(MODEL_DIMENSIONS))
+            .order_by(EbookEmbeddingModel.name)
+        )
+    }
+
+    stats: list[EmbeddingModelStats] = []
+    for model_name, dimension in MODEL_DIMENSIONS.items():
+        model = models.get(model_name)
+        embedded_chunks = 0
+        if model is not None:
+            table = get_embedding_table(dimension)
+            embedded_chunks = await session.scalar(select(func.count(table.id)).where(table.model_id == model.id)) or 0
+        stats.append(
+            EmbeddingModelStats(
+                model_name=model_name,
+                dimension=dimension,
+                embedded_chunks=embedded_chunks,
+                total_chunks=total_chunks,
+            )
+        )
+    return stats
+
+
+async def embed_missing_chunks(session: AsyncSession, client: httpx.AsyncClient, config: EbookSearchConfig) -> int:
+    """Embed chunks missing embeddings for the configured model."""
+    await ensure_embedding_models(session)
+    model = await session.scalar(select(EbookEmbeddingModel).where(EbookEmbeddingModel.name == config.embedding_model))
+    if model is None:
+        supported_models = ", ".join(MODEL_DIMENSIONS)
+        msg = f"Unknown embedding model: {config.embedding_model}. Supported models: {supported_models}"
+        raise ValueError(msg)
+
+    table = get_embedding_table(model.dimension)
+    chunks = list(
+        await session.scalars(
+            select(EbookChunk)
+            .outerjoin(table, (table.chunk_id == EbookChunk.id) & (table.model_id == model.id))
+            .where(table.id.is_(None))
+            .order_by(EbookChunk.id)
+            .limit(config.embedding_batch_size)
+        )
+    )
+    if not chunks:
+        logger.info(f"ebook_embed_missing_none {config.embedding_model=}")
+        return 0
+
+    logger.info(f"ebook_embed_missing_batch_start {config.embedding_model=} count={len(chunks)}")
+    vectors = await embed_texts(client, [chunk.text for chunk in chunks], config)
+    rows = [
+        {"chunk_id": chunk.id, "model_id": model.id, "embedding": vector}
+        for chunk, vector in zip(chunks, vectors, strict=True)
+    ]
+    statement = insert(table).values(rows).on_conflict_do_nothing(index_elements=["chunk_id", "model_id"])
+    await session.execute(statement)
+    await session.flush()
+    logger.info(f"ebook_embed_missing_batch_complete {config.embedding_model=} count={len(rows)}")
+    return len(rows)

python/ebook_search/epub_parse.py

@@ -0,0 +1,95 @@
+"""EPUB parsing helpers."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from bs4 import BeautifulSoup
+from ebooklib import ITEM_DOCUMENT, epub
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+WHITESPACE_RE = re.compile(r"\s+")
+
+
+@dataclass(frozen=True)
+class ParsedChapter:
+    """Text extracted from one EPUB spine document."""
+
+    title: str | None
+    href: str | None
+    text: str
+    page_labels: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class ParsedEpub:
+    """Parsed EPUB metadata and text."""
+
+    title: str
+    author: str | None
+    language: str | None
+    publisher: str | None
+    identifier: str | None
+    chapters: tuple[ParsedChapter, ...]
+
+
+def parse_epub(path: Path) -> ParsedEpub:
+    """Parse EPUB metadata and spine text."""
+    book = epub.read_epub(path)
+    chapters = []
+    for item in book.get_items_of_type(ITEM_DOCUMENT):
+        soup = BeautifulSoup(item.get_content(), "html.parser")
+        title = chapter_title(soup)
+        page_labels = tuple(extract_page_labels(soup))
+        text = clean_text(soup.get_text(" "))
+        if text:
+            chapters.append(ParsedChapter(title=title, href=item.get_name(), text=text, page_labels=page_labels))
+
+    return ParsedEpub(
+        title=metadata_value(book, "title") or path.stem,
+        author=metadata_value(book, "creator"),
+        language=metadata_value(book, "language"),
+        publisher=metadata_value(book, "publisher"),
+        identifier=metadata_value(book, "identifier"),
+        chapters=tuple(chapters),
+    )
+
+
+def metadata_value(book: epub.EpubBook, name: str) -> str | None:
+    """Return the first non-empty Dublin Core metadata value for a name."""
+    values = book.get_metadata("DC", name)
+    if not values:
+        return None
+    value = values[0][0]
+    return str(value).strip() or None
+
+
+def chapter_title(soup: BeautifulSoup) -> str | None:
+    """Extract the best available title from an EPUB document soup."""
+    heading = soup.find(["h1", "h2", "h3"])
+    if heading is None:
+        title = soup.find("title")
+        if title is None:
+            return None
+        return clean_text(title.get_text(" ")) or None
+    return clean_text(heading.get_text(" ")) or None
+
+
+def extract_page_labels(soup: BeautifulSoup) -> list[str]:
+    """Extract EPUB page-break labels from a document soup."""
+    labels: list[str] = []
+    for tag in soup.find_all(attrs={"epub:type": "pagebreak"}):
+        label = tag.get("title") or tag.get("aria-label") or tag.get_text(" ")
+        clean = clean_text(str(label))
+        if clean:
+            labels.append(clean)
+    return labels
+
+
+def clean_text(text: str) -> str:
+    """Normalize whitespace in extracted EPUB text."""
+    return WHITESPACE_RE.sub(" ", text).strip()

python/ebook_search/eval/__init__.py

@@ -0,0 +1 @@
+"""Offline evaluation tooling for the ebook search pipeline."""

python/ebook_search/eval/data/queries.jsonl

@@ -0,0 +1,71 @@
+{"query": "Who is Damien Montgomery and how does he become a Jump Mage?", "answer": null, "answerable": true, "relevant_sources": ["Starship's Mage"]}
+{"query": "What is a Rune Wright and why is Damien so rare?", "answer": null, "answerable": true, "relevant_sources": ["Starship's Mage"]}
+{"query": "How does jump magic let starships travel faster than light?", "answer": null, "answerable": true, "relevant_sources": ["Starship's Mage"]}
+{"query": "What is the role of the Mage-King of Mars in the Protectorate?", "answer": null, "answerable": true, "relevant_sources": ["Starship's Mage"]}
+{"query": "What happened aboard the Blue Jay in the first Starship's Mage book?", "answer": null, "answerable": true, "relevant_sources": ["Starship's Mage"]}
+{"query": "Who is Captain David Rice?", "answer": null, "answerable": true, "relevant_sources": ["Starship's Mage"]}
+{"query": "How are amplifiers and simulacrums used to power a ship's jump?", "answer": null, "answerable": true, "relevant_sources": ["Starship's Mage"]}
+{"query": "What duties does a Hand of the Mage-King carry out?", "answer": null, "answerable": true, "relevant_sources": ["Starship's Mage"]}
+{"query": "Explain the structure of the Royal Martian Navy.", "answer": null, "answerable": true, "relevant_sources": ["Starship's Mage"]}
+{"query": "How do mages carve runes to enchant a starship?", "answer": null, "answerable": true, "relevant_sources": ["Starship's Mage"]}
+{"query": "What threat do the Legatan rebels pose to the Protectorate?", "answer": null, "answerable": true, "relevant_sources": ["Starship's Mage"]}
+{"query": "How does Damien handle his first command?", "answer": null, "answerable": true, "relevant_sources": ["Starship's Mage"]}
+{"query": "What is the significance of the simulacrum on a jump ship?", "answer": null, "answerable": true, "relevant_sources": ["Starship's Mage"]}
+{"query": "Describe a mage duel in the Starship's Mage series.", "answer": null, "answerable": true, "relevant_sources": ["Starship's Mage"]}
+{"query": "What moral conflicts does Damien face as a Hand of the Mage-King?", "answer": null, "answerable": true, "relevant_sources": ["Starship's Mage"]}
+{"query": "How does the Protectorate keep peace among its member worlds?", "answer": null, "answerable": true, "relevant_sources": ["Starship's Mage"]}
+{"query": "Who is the Keeper of Oaths and how does Damien work with them?", "answer": null, "answerable": true, "relevant_sources": ["Starship's Mage"]}
+{"query": "What event is known as the Onset and how does it change the world?", "answer": null, "answerable": true, "relevant_sources": ["The Onset"]}
+{"query": "Who is the main character at the start of the Onset?", "answer": null, "answerable": true, "relevant_sources": ["The Onset"]}
+{"query": "How do survivors adapt after the Onset begins?", "answer": null, "answerable": true, "relevant_sources": ["The Onset"]}
+{"query": "What new abilities emerge during the Onset?", "answer": null, "answerable": true, "relevant_sources": ["The Onset"]}
+{"query": "Describe the primary antagonist in the Onset series.", "answer": null, "answerable": true, "relevant_sources": ["The Onset"]}
+{"query": "How does society collapse and reorganize after the Onset?", "answer": null, "answerable": true, "relevant_sources": ["The Onset"]}
+{"query": "What factions form in the aftermath of the Onset?", "answer": null, "answerable": true, "relevant_sources": ["The Onset"]}
+{"query": "How does the protagonist gain power throughout the Onset?", "answer": null, "answerable": true, "relevant_sources": ["The Onset"]}
+{"query": "What is the cause or origin of the Onset?", "answer": null, "answerable": true, "relevant_sources": ["The Onset"]}
+{"query": "Describe an early survival challenge faced after the Onset.", "answer": null, "answerable": true, "relevant_sources": ["The Onset"]}
+{"query": "How do the characters defend their stronghold during the Onset?", "answer": null, "answerable": true, "relevant_sources": ["The Onset"]}
+{"query": "What relationships drive the protagonist's choices in the Onset?", "answer": null, "answerable": true, "relevant_sources": ["The Onset"]}
+{"query": "How does the Onset escalate by the end of the first book?", "answer": null, "answerable": true, "relevant_sources": ["The Onset"]}
+{"query": "What mysteries about the Onset remain unresolved?", "answer": null, "answerable": true, "relevant_sources": ["The Onset"]}
+{"query": "How do the rules of the world change once the Onset takes hold?", "answer": null, "answerable": true, "relevant_sources": ["The Onset"]}
+{"query": "What weapons or tactics work best against the threats of the Onset?", "answer": null, "answerable": true, "relevant_sources": ["The Onset"]}
+{"query": "How does Bob Johansson become a von Neumann probe?", "answer": null, "answerable": true, "relevant_sources": ["We Are Legion (We Are Bob)"]}
+{"query": "What is a replicant and why do Bob's copies have different personalities?", "answer": null, "answerable": true, "relevant_sources": ["We Are Legion (We Are Bob)"]}
+{"query": "Who are Riker, Homer, and Bill among the Bob clones?", "answer": null, "answerable": true, "relevant_sources": ["We Are Legion (We Are Bob)"]}
+{"query": "What is GUPPI and how does Bob use it?", "answer": null, "answerable": true, "relevant_sources": ["We Are Legion (We Are Bob)"]}
+{"query": "Describe the threat posed by the Others.", "answer": null, "answerable": true, "relevant_sources": ["We Are Legion (We Are Bob)"]}
+{"query": "How does Bob protect and uplift the Deltans?", "answer": null, "answerable": true, "relevant_sources": ["We Are Legion (We Are Bob)"]}
+{"query": "Why do the replicants drift apart in personality over time?", "answer": null, "answerable": true, "relevant_sources": ["We Are Legion (We Are Bob)"]}
+{"query": "What is the role of FAITH and the Brazilian Empire on Earth?", "answer": null, "answerable": true, "relevant_sources": ["We Are Legion (We Are Bob)"]}
+{"query": "How does subspace communication work for the Bobs?", "answer": null, "answerable": true, "relevant_sources": ["We Are Legion (We Are Bob)"]}
+{"query": "What happens to Bender after he goes missing?", "answer": null, "answerable": true, "relevant_sources": ["We Are Legion (We Are Bob)"]}
+{"query": "How do the Bobs build self-replicating probes across the galaxy?", "answer": null, "answerable": true, "relevant_sources": ["We Are Legion (We Are Bob)"]}
+{"query": "How does Bob evacuate humanity after Earth becomes uninhabitable?", "answer": null, "answerable": true, "relevant_sources": ["We Are Legion (We Are Bob)"]}
+{"query": "Describe the conflict between different factions of Bobs.", "answer": null, "answerable": true, "relevant_sources": ["We Are Legion (We Are Bob)"]}
+{"query": "What ethical dilemmas does Bob face when interfering with primitive species?", "answer": null, "answerable": true, "relevant_sources": ["We Are Legion (We Are Bob)"]}
+{"query": "How does the original Bob differ from later generations of clones?", "answer": null, "answerable": true, "relevant_sources": ["We Are Legion (We Are Bob)"]}
+{"query": "How do the Bobs defeat the Others' system-harvesting fleets?", "answer": null, "answerable": true, "relevant_sources": ["We Are Legion (We Are Bob)"]}
+{"query": "What role does Howard play in the human colonies?", "answer": null, "answerable": true, "relevant_sources": ["We Are Legion (We Are Bob)"]}
+// querys not it the dataset
+{"query": "How does Frodo destroy the One Ring in The Lord of the Rings?", "answer": null, "answerable": false, "relevant_sources": []}
+{"query": "Who killed Dumbledore in Harry Potter and the Half-Blood Prince?", "answer": null, "answerable": false, "relevant_sources": []}
+{"query": "What house does Tyrion Lannister belong to in A Game of Thrones?", "answer": null, "answerable": false, "relevant_sources": []}
+{"query": "How does Paul Atreides control the spice on Arrakis in Dune?", "answer": null, "answerable": false, "relevant_sources": []}
+{"query": "What does the green light at the end of the dock mean in The Great Gatsby?", "answer": null, "answerable": false, "relevant_sources": []}
+{"query": "Why does Hester Prynne wear a scarlet letter?", "answer": null, "answerable": false, "relevant_sources": []}
+{"query": "What does the white whale represent in Moby-Dick?", "answer": null, "answerable": false, "relevant_sources": []}
+{"query": "How does Elizabeth Bennet's view of Mr. Darcy change in Pride and Prejudice?", "answer": null, "answerable": false, "relevant_sources": []}
+{"query": "What crime does Raskolnikov commit in Crime and Punishment?", "answer": null, "answerable": false, "relevant_sources": []}
+{"query": "How does Katniss volunteer for the Hunger Games?", "answer": null, "answerable": false, "relevant_sources": []}
+{"query": "What is Winston Smith's job in Nineteen Eighty-Four?", "answer": null, "answerable": false, "relevant_sources": []}
+{"query": "Who is Atticus Finch defending in To Kill a Mockingbird?", "answer": null, "answerable": false, "relevant_sources": []}
+{"query": "What is the capital of Australia?", "answer": null, "answerable": false, "relevant_sources": []}
+{"query": "How do I bake a sourdough loaf from scratch?", "answer": null, "answerable": false, "relevant_sources": []}
+{"query": "Explain how photosynthesis converts sunlight into energy.", "answer": null, "answerable": false, "relevant_sources": []}
+{"query": "What were the main causes of World War I?", "answer": null, "answerable": false, "relevant_sources": []}
+{"query": "How does compound interest work?", "answer": null, "answerable": false, "relevant_sources": []}
+{"query": "How do I change a flat tire on a car?", "answer": null, "answerable": false, "relevant_sources": []}
+{"query": "What is the boiling point of water at sea level?", "answer": null, "answerable": false, "relevant_sources": []}
+{"query": "What is the recommended daily intake of vitamin D?", "answer": null, "answerable": false, "relevant_sources": []}