removing postgresql was no longer able to reproduse the error

fix(scipy): relax STFT test tolerances
refactor(overlays): consolidate test patches
2026-09-19 22:42:44 -04:00 · 2026-09-19 22:42:44 -04:00 · 2026-09-19 22:42:44 -04:00 · 2026-09-19 22:42:44 -04:00 · 2026-09-19 22:42:44 -04:00 · 2026-09-19 22:42:44 -04:00
402 changed files with 42370 additions and 2290 deletions
@@ -0,0 +1,28 @@
 .git
 .direnv
 .mypy_cache
 .pytest_cache
 .ruff_cache
 .venv
 **/.venv
 .env
 .cache
 .claude
 .coverage
 .vscode
 .stfolder
 .literotica_data
 esphome
 htmlcov
 data
 ebooks
 __pycache__
 **/__pycache__
 *.pyc
 *.pyo
 .ebook_search_bm25
 result
 result-*
 *.egg-info
 dist
 build
@@ -8,19 +8,36 @@ on:
    - cron: "0 22 * * *"
 jobs:
  prebuild-common:
    name: prebuild-common-x86-64-v3
    runs-on: nix-cache-builder
    steps:
      - uses: actions/checkout@v4
      # portal-1 is the smallest system closure: 95% of its derivations are
      # shared by all five systems, so it is a maintainable common cache seed.
      # Keep going so one failing package does not stop unrelated cache entries
      # from being built.
      - name: Build common packages
        run: nixos-rebuild build --keep-going --accept-flake-config --flake ./#portal-1
      - name: Copy common packages to nix-cache
        run: nix copy --accept-flake-config --to unix:///host-nix/var/nix/daemon-socket/socket .#nixosConfigurations.portal-1.config.system.build.toplevel
  build:
    name: build-${{ matrix.system }}
    needs: prebuild-common
    runs-on: self-hosted
    strategy:
      matrix:
        system:
          - "bob"
          - "brain"
          - "jeeves"
          - "rhapsody-in-green"
          - "portal-1"
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4
      - name: Build default package
-        run: "nixos-rebuild build --flake ./#${{ matrix.system }}"
+        run: "nixos-rebuild build --accept-flake-config --flake ./#${{ matrix.system }}"
      - name: copy to nix-cache
-        run: nix copy --to ssh://jeeves .#nixosConfigurations.${{ matrix.system }}.config.system.build.toplevel
+        run: nix copy --accept-flake-config --to unix:///host-nix/var/nix/daemon-socket/socket .#nixosConfigurations.${{ matrix.system }}.config.system.build.toplevel
@@ -0,0 +1,22 @@
 name: merge_flake_lock_update
 on:
  workflow_dispatch:
  schedule:
    - cron: "0 2 * * 6"
 jobs:
  merge:
    runs-on: self-hosted
    permissions:
      contents: write
      pull-requests: write
    steps:
      - name: merge_flake_lock_update
        run: >-
          python -m python.gitea_flake_lock merge
          --repo "${{ github.repository }}"
        env:
          JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
          GITEA_URL: https://gitea.tmmworkshop.com
@@ -0,0 +1,26 @@
 name: test ebook search
 on:
  workflow_dispatch:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
 env:
  UV_PYTHON_DOWNLOADS: never
  UV_CACHE_DIR: /var/cache/uv
  UV_LINK_MODE: copy
 jobs:
  test-ebook-search:
    runs-on: self-hosted
    steps:
      - uses: actions/checkout@v4
      - name: Install dependencies
        run: nix develop .#devShells.x86_64-linux.ebook-search -c uv sync --locked --project python/ebook_search/docker
      - name: Run ebook search tests
        run: nix develop .#devShells.x86_64-linux.ebook-search -c uv run --project python/ebook_search/docker --no-sync pytest tests/ebook_search --override-ini addopts="-n auto -ra"
@@ -2,21 +2,24 @@ name: update-flake-lock
 on:
  workflow_dispatch:
  schedule:
-    - cron: "0 0 * * *"
+    - cron: "0 0 * * 6"
 jobs:
  lockfile:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
    permissions:
      actions: write
      contents: write
      pull-requests: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Install Nix
        uses: DeterminateSystems/nix-installer-action@main
      - name: Update flake.lock
-        uses: DeterminateSystems/update-flake-lock@main
+        run: nix flake update
-        with:
+      - name: Create or update flake.lock PR
-          token: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
+        env:
-          pr-title: "Update flake.lock"
+          JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
-          pr-labels: |
+          GITEA_URL: https://gitea.tmmworkshop.com
-            dependencies
+        run: >-
-            automated
+          python -m python.gitea_flake_lock update
          --repo "${{ github.repository }}"
@@ -165,3 +165,14 @@ test.*
 # syncthing
 .stfolder
 # Frontend build output
 frontend/dist/
 frontend/node_modules/
 # data from testing llms
 data/*
 .ebook_search_bm25
 # gems data
 .gems
@@ -1,10 +1,17 @@
 # Generate AGE keys from SSH keys with:
 #   ssh-keygen -A
 #   nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
 # update keys after addin/removing a key
 #   nix-shell -p sops --run "sops updatekeys users/secrets.yaml" users/secrets.yaml
 keys:
  - &admin_richie age1u8zj599elqqvcmhxn8zuwrufsz8w8w366d3ayrljjejljt2q45kq8mxw9c # cspell:disable-line
  - &system_bob age1q47vup0tjhulkg7d6xwmdsgrw64h4ax3la3evzqpxyy4adsmk9fs56qz3y # cspell:disable-line
  - &system_brain age1jhf7vm0005j60mjq63696frrmjhpy8kpc2d66mw044lqap5mjv4snmwvwm # cspell:disable-line
  - &system_jeeves age13lmqgc3jvkyah5e3vcwmj4s5wsc2akctcga0lpc0x8v8du3fxprqp4ldkv # cspell:disable-line
  - &system_router age1xzxryqq63x65yuza9lmmkud7crjjxpnkdew070yhx6xn7xe4tdws5twxsv # cspell:disable-line
  - &system_rhapsody age1ufnewppysaq2wwcl4ugngjz8pfzc5a35yg7luq0qmuqvctajcycs5lf6k4 # cspell:disable-line
  - &system_portal_1 age1vyav6kxtvt3z4vtnkkjj38eu8hlts5m7ygyckhskvalg2gpjk52su53d0a # cspell:disable-line
 creation_rules:
  - path_regex: users/secrets\.yaml$
@@ -12,6 +19,7 @@ creation_rules:
      - age:
          - *admin_richie
          - *system_bob
          - *system_brain
          - *system_jeeves
          - *system_router
          - *system_rhapsody
          - *system_portal_1
@@ -2,6 +2,7 @@
  "cSpell.words": [
    "aboutwelcome",
    "acltype",
    "addopts",
    "addstr",
    "advplyr",
    "ahci",
@@ -38,7 +39,6 @@
    "cgroupdriver",
    "charliermarsh",
    "Checkpointing",
    "cloudflared",
    "codellama",
    "codezombiech",
    "compactmode",
@@ -70,16 +70,17 @@
    "ehci",
    "emerg",
    "endlessh",
    "ents",
    "errorlens",
    "esbenp",
    "esphome",
    "extest",
    "fadvise",
    "fastfetch",
    "fastforwardteam",
    "FASTFOX",
    "ffmpegthumbnailer",
    "filebot",
    "filebrowser",
    "fileroller",
    "findbar",
    "Fira",
@@ -96,6 +97,7 @@
    "getch",
    "getmaxyx",
    "ghdeploy",
    "gitea",
    "globalprivacycontrol",
    "gparted",
    "gtts",
@@ -114,6 +116,9 @@
    "httpchk",
    "hurlenko",
    "hwloc",
    "ical",
    "ignorelist",
    "improv",
    "INITDB",
    "iocharset",
    "ioit",
@@ -123,6 +128,8 @@
    "jnoortheen",
    "jsbc",
    "kagi",
    "keyformat",
    "keylocation",
    "kuma",
    "lazer",
    "levelname",
@@ -148,20 +155,25 @@
    "mixtral",
    "mklabel",
    "mkpart",
    "modbus",
    "modbuss",
    "modesetting",
    "mountpoint",
    "mountpoints",
    "mousewheel",
    "mqtt",
    "mtxr",
    "mypy",
    "ncdu",
    "nemo",
    "neofetch",
    "nerdfonts",
    "netdev",
    "netdevs",
    "Networkd",
    "networkmanager",
    "newtabpage",
    "ngram",
    "ngrams",
    "nixfmt",
    "nixos",
    "nixpkgs",
@@ -184,6 +196,7 @@
    "overalljails",
    "overscroll",
    "overseerr",
    "paho",
    "partitionwise",
    "pbmode",
    "pciutils",
@@ -192,6 +205,7 @@
    "peerconnection",
    "PESKYFOX",
    "PGID",
    "pgvector",
    "pipewire",
    "pkgs",
    "plugdev",
@@ -211,13 +225,16 @@
    "pulseaudio",
    "punycode",
    "pychromecast",
    "pydocstyle",
    "pyfakefs",
    "pylance",
    "pylint",
    "pymetno",
    "pymodbus",
    "pyopenweathermap",
    "pyownet",
-    "qbit",
+    "pytest",
-    "qbittorrent",
+    "qalculate",
    "qbittorrentvpn",
    "qbitvpn",
    "quicksuggest",
    "radarr",
    "readahead",
@@ -227,6 +244,7 @@
    "referer",
    "REFERERS",
    "relatime",
    "rerank",
    "Rhosts",
    "ripgrep",
    "roboto",
@@ -239,8 +257,10 @@
    "schemeless",
    "scrollback",
    "SECUREFOX",
    "sessionmaker",
    "sessionstore",
    "shellcheck",
    "signalbot",
    "signon",
    "Signons",
    "skia",
@@ -250,6 +270,7 @@
    "socialtracking",
    "sonarr",
    "sponsorblock",
    "sqlalchemy",
    "sqltools",
    "ssdp",
    "SSHOPTS",
@@ -261,6 +282,7 @@
    "tabmanager",
    "tamasfe",
    "TCPIP",
    "testdisk",
    "tiktok",
    "timonwong",
    "titlebar",
@@ -270,11 +292,17 @@
    "topstories",
    "treefmt",
    "twimg",
    "typedmonarchmoney",
    "typer",
    "uaccess",
    "ubiquiti",
    "ublock",
    "uiprotect",
    "uitour",
    "unifi",
    "unjudged",
    "unrar",
    "unstorable",
    "unsubmitted",
    "uptimekuma",
    "urlbar",
@@ -284,7 +312,10 @@
    "useragent",
    "usernamehw",
    "userprefs",
    "vaninventory",
    "vdev",
    "vfat",
    "victron",
    "virt",
    "virtualisation",
    "vpnpromourl",
@@ -296,16 +327,22 @@
    "wireshark",
    "Workqueues",
    "xattr",
    "xcursorgen",
    "xdist",
    "xhci",
    "yake",
    "yazi",
    "yubikey",
    "yubioath",
    "yzhang",
    "zeroconf",
    "zerotier",
    "zerotierone",
    "zoxide",
    "zram",
    "zstd"
-  ]
+  ],
  "python-envs.defaultEnvManager": "ms-python.python:system",
  "python-envs.pythonProjects": [],
  "python.testing.pytestArgs": ["tests"],
  "python.testing.unittestEnabled": false,
  "python.testing.pytestEnabled": true
 }
@@ -0,0 +1,4 @@
 [workspace]
 resolver = "2"
 members = ["rust/*"]
@@ -1 +1,51 @@
 # dotfiles
 ## Installer ISO
 Build a bootable NixOS ISO with the installer preinstalled:
 ```sh
 nix build .#iso
 ```
 Write `result/iso/nixos-zfs-installer.iso` to a USB stick (for example with `dd`) or boot it in a VM. The image is the minimal NixOS installation CD with ZFS enabled and `nixos-installer` on `PATH`. SSH is enabled and the `nixos` and `root` accounts use the password `nixos`, so you can also run the installer remotely. Once booted:
 ```sh
 sudo nixos-installer
 ```
 The ISO bundles the `.#installer-nixos` package, a variant of the binary that keeps its Nix store linkage instead of being patched for foreign distributions.
 ## Installer binary
 Build the self-contained installer executable with:
 ```sh
 nix build .#installer
 ```
 The flake package (defined in `python/installer/package.nix`) uses the Python builder in `python/installer/build.py`, which stages only the installer modules before running PyInstaller. You can also call it directly when `pyinstaller` and `patchelf` are on `PATH`:
 ```sh
 python -m python.installer.build --output ./nixos-installer
 ```
 Copy `result/bin/nixos-installer` to the installer USB stick and run it as root from the NixOS live environment:
 ```sh
 sudo ./nixos-installer
 ```
 Validate the live environment first with:
 ```sh
 ./nixos-installer --check
 ```
 Paste a value into the TUI encryption password field to enable LUKS during install, or set `ENCRYPT_KEY`:
 ```sh
 sudo env ENCRYPT_KEY='change-me' ./nixos-installer
 ```
 The binary bundles the Python runtime and only the installer modules it imports. It still expects the NixOS installer environment to provide system install tools such as `parted`, `zfs`, `zpool`, `cryptsetup`, `nixos-generate-config`, and `nixos-install`.
@@ -17,13 +17,11 @@
    ./nix.nix
    ./programs.nix
    ./ssh.nix
    ./snapshot_manager.nix
  ];
  boot = {
-    tmp.useTmpfs = true;
+    tmp.useTmpfs = lib.mkDefault true;
    kernelPackages = lib.mkDefault pkgs.linuxPackages_6_12;
    zfs.package = lib.mkDefault pkgs.zfs_2_3;
  };
  hardware.enableRedistributableFirmware = true;
@@ -37,19 +35,16 @@
  nixpkgs = {
    overlays = builtins.attrValues outputs.overlays;
-    config.allowUnfree = true;
+    config = {
      allowUnfree = true;
    };
  };
  services = {
    dbus.implementation = "dbus";
    # firmware update
    fwupd.enable = true;
    snapshot_manager.enable = lib.mkDefault true;
    zfs = {
      trim.enable = lib.mkDefault true;
      autoScrub.enable = lib.mkDefault true;
    };
  };
  powerManagement.powertop.enable = lib.mkDefault true;
@@ -1,4 +1,10 @@
 { lib, pkgs, ... }:
 let
  libPath = pkgs.lib.makeLibraryPath [
    pkgs.zlib
    pkgs.stdenv.cc.cc.lib
  ];
 in
 {
  programs.nix-ld = {
    enable = lib.mkDefault true;
@@ -15,6 +21,7 @@
      libxml2
      openssl
      stdenv.cc.cc
      stdenv.cc.cc.lib
      systemd
      util-linux
      xz
@@ -23,4 +30,9 @@
      zstd
    ];
  };
  environment = {
    sessionVariables.LD_LIBRARY_PATH = lib.mkDefault libPath;
    variables.LD_LIBRARY_PATH = lib.mkDefault libPath;
  };
 }
@@ -31,8 +31,15 @@ in
        "flakes"
        "ca-derivations"
      ];
      system-features = lib.mkAfter [
        "gccarch-x86-64-v2"
        "gccarch-x86-64-v3"
      ];
      warn-dirty = false;
      flake-registry = ""; # disable global flake registries
      connect-timeout = 10;
      download-buffer-size = 536870912;
      fallback = true;
    };
    # Add each flake input as a registry and nix_path
@@ -2,6 +2,5 @@
 {
  environment.systemPackages = with pkgs; [
    git
    python313
  ];
 }
@@ -1,50 +0,0 @@
 {
  inputs,
  pkgs,
  lib,
  config,
  ...
 }:
 let
  cfg = config.services.snapshot_manager;
 in
 {
  options = {
    services.snapshot_manager = {
      enable = lib.mkOption {
        default = true;
        example = true;
        description = "Whether to enable k3s-net.";
        type = lib.types.bool;
      };
      path = lib.mkOption {
        type = lib.types.path;
        description = "Path that needs to be updated via git pull";
        default = ./snapshot_config.toml;
      };
    };
  };
  config = lib.mkIf cfg.enable {
    systemd = {
      services."snapshot_manager" = {
        description = "ZFS Snapshot Manager";
        requires = [ "zfs-import.target" ];
        after = [ "zfs-import.target" ];
        path = [ pkgs.zfs ];
        serviceConfig = {
          Type = "oneshot";
          ExecStart = "${inputs.system_tools.packages.x86_64-linux.default}/bin/snapshot_manager --config-file='${cfg.path}'";
        };
      };
      timers."snapshot_manager" = {
        wantedBy = [ "timers.target" ];
        timerConfig = {
          OnBootSec = "15m";
          OnUnitActiveSec = "15m";
          Unit = "snapshot_manager.service";
        };
      };
    };
  };
 }
@@ -37,6 +37,8 @@
        TcpKeepAlive = "no";
        X11Forwarding = lib.mkDefault false;
        KexAlgorithms = [
          "sntrup761x25519-sha512@openssh.com"
          "mlkem768x25519-sha256"
          "curve25519-sha256@libssh.org"
          "diffie-hellman-group-exchange-sha256"
        ];
@@ -0,0 +1,6 @@
 {
  nix.settings = {
    trusted-substituters = [ "http://192.168.95.35:5000" ];
    substituters = [ "http://192.168.95.35:5000/?priority=1&want-mass-query=true" ];
  };
 }
@@ -1,8 +1,8 @@
 { pkgs, ... }:
 {
  boot = {
-    kernelPackages = pkgs.linuxPackages_6_13;
+    kernelPackages = pkgs.linuxPackages_6_18;
-    zfs.package = pkgs.zfs_2_3;
+    zfs.package = pkgs.zfs_2_4;
  };
  hardware.bluetooth = {
@@ -17,6 +17,7 @@
    logDriver = "local";
    storageDriver = "overlay2";
    daemon.settings = {
      live-restore = false;
      experimental = true;
      exec-opts = [ "native.cgroupdriver=systemd" ];
      log-opts = {
@@ -0,0 +1,256 @@
 {
  config,
  lib,
  pkgs,
  ...
 }:
 let
  monitoringInterface = "tailscale0";
  nodeTextfileDir = "/var/lib/prometheus-node-exporter-textfile";
  mkProcessNameTemplate =
    perPid: template: if perPid then "${template}:{{.PID}}:{{.StartTime}}" else template;
  mkProcessMatchers = perPid: [
    {
      name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Module}}";
      cmdline = [ "^/nix/store[^ ]*/bin/python[^ ]* -m (?P<Module>[^ ]+)" ];
    }
    {
      name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Wrapped}}";
      cmdline = [
        "^/nix/store[^ ]*/bin/python[^ ]* /nix/store[^ ]*/bin/\\.?(?P<Wrapped>[^ /]+?)(?:-wrapped)?(?:\\s|$)"
      ];
    }
    {
      name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Wrapped}}";
      cmdline = [
        "^/nix/store[^ ]*/bin/node /nix/store[^ ]*-(?P<Wrapped>[A-Za-z0-9._+-]+)-[0-9][^ /]*/"
      ];
    }
    {
      name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Wrapped}}";
      cmdline = [ "^/nix/store[^ ]*/(?:bin/|lib/[^ ]*/)?\\.?(?P<Wrapped>[^ /]+?)(?:-wrapped)?(?:\\s|$)" ];
    }
    {
      name = mkProcessNameTemplate perPid "{{.Username}}:{{.ExeBase}}";
      cmdline = [ ".+" ];
    }
  ];
  perPidConfig = pkgs.writeText "process-exporter-per-pid.yaml" (
    builtins.toJSON {
      process_names = mkProcessMatchers true;
    }
  );
  zpoolLatencyScript = pkgs.writeShellScript "zpool-latency-exporter" ''
        set -euo pipefail
        out_dir=${lib.escapeShellArg nodeTextfileDir}
        host=${lib.escapeShellArg config.networking.hostName}
        tmp_file="$(mktemp "$out_dir/zpool.prom.XXXXXX")"
        trap 'rm -f "$tmp_file"' EXIT
        pools="$(zpool list -H -o name | paste -sd, -)"
        cat >"$tmp_file" <<'EOF'
    # HELP zpool_iostat_total_wait_read_ns Average total read wait time reported by zpool iostat.
    # TYPE zpool_iostat_total_wait_read_ns gauge
    # HELP zpool_iostat_total_wait_write_ns Average total write wait time reported by zpool iostat.
    # TYPE zpool_iostat_total_wait_write_ns gauge
    # HELP zpool_iostat_disk_wait_read_ns Average disk read wait time reported by zpool iostat.
    # TYPE zpool_iostat_disk_wait_read_ns gauge
    # HELP zpool_iostat_disk_wait_write_ns Average disk write wait time reported by zpool iostat.
    # TYPE zpool_iostat_disk_wait_write_ns gauge
    # HELP zpool_iostat_syncq_wait_read_ns Average synchronous queue read wait time reported by zpool iostat.
    # TYPE zpool_iostat_syncq_wait_read_ns gauge
    # HELP zpool_iostat_syncq_wait_write_ns Average synchronous queue write wait time reported by zpool iostat.
    # TYPE zpool_iostat_syncq_wait_write_ns gauge
    # HELP zpool_iostat_asyncq_wait_read_ns Average asynchronous queue read wait time reported by zpool iostat.
    # TYPE zpool_iostat_asyncq_wait_read_ns gauge
    # HELP zpool_iostat_asyncq_wait_write_ns Average asynchronous queue write wait time reported by zpool iostat.
    # TYPE zpool_iostat_asyncq_wait_write_ns gauge
    EOF
        zpool iostat -Hplvy -y 1 1 | awk -F '\t' -v host="$host" -v pools="$pools" '
          function esc(str, out) {
            out = str
            gsub(/\\/, "\\\\", out)
            gsub(/"/, "\\\"", out)
            return out
          }
          function emit(metric, pool, vdev, value) {
            if (value == "" || value == "-") {
              return
            }
            printf "%s{host=\"%s\",pool=\"%s\",vdev=\"%s\"} %s\n",
              metric,
              esc(host),
              esc(pool),
              esc(vdev),
              value
          }
          BEGIN {
            split(pools, pool_names, ",")
            for (idx in pool_names) {
              if (pool_names[idx] != "") {
                known_pools[pool_names[idx]] = 1
              }
            }
          }
          NF == 0 {
            next
          }
          {
            row_name = $1
            if (row_name in known_pools) {
              current_pool = row_name
              current_vdev = "_pool"
            } else if (current_pool == "") {
              next
            } else {
              current_vdev = row_name
            }
            emit("zpool_iostat_total_wait_read_ns", current_pool, current_vdev, $8)
            emit("zpool_iostat_total_wait_write_ns", current_pool, current_vdev, $9)
            emit("zpool_iostat_disk_wait_read_ns", current_pool, current_vdev, $10)
            emit("zpool_iostat_disk_wait_write_ns", current_pool, current_vdev, $11)
            emit("zpool_iostat_syncq_wait_read_ns", current_pool, current_vdev, $12)
            emit("zpool_iostat_syncq_wait_write_ns", current_pool, current_vdev, $13)
            emit("zpool_iostat_asyncq_wait_read_ns", current_pool, current_vdev, $14)
            emit("zpool_iostat_asyncq_wait_write_ns", current_pool, current_vdev, $15)
          }
        ' >>"$tmp_file"
        mv "$tmp_file" "$out_dir/zpool.prom"
        trap - EXIT
  '';
 in
 {
  networking.firewall.interfaces.${monitoringInterface}.allowedTCPPorts = [
    9100
    9134
    9256
    9257
    9633
  ];
  services.prometheus.exporters = {
    node = {
      enable = true;
      enabledCollectors = [
        "pressure"
        "processes"
        "systemd"
      ];
      extraFlags = [ "--collector.textfile.directory=${nodeTextfileDir}" ];
    };
    process = {
      enable = true;
      user = "root";
      group = "root";
      settings.process_names = mkProcessMatchers false;
      extraFlags = [
        "-gather-smaps=false"
        "-remove-empty-groups=true"
        "-threads=false"
      ];
    };
    smartctl.enable = true;
    zfs.enable = true;
  };
  programs.atop = {
    enable = true;
    atopService.enable = true;
    atopRotateTimer.enable = true;
    atopacctService.enable = true;
    settings.interval = 30;
  };
  systemd = {
    services = {
      prometheus-process-pid-exporter = {
        description = "Prometheus process exporter with per-PID naming";
        wantedBy = [ "multi-user.target" ];
        after = [ "network.target" ];
        serviceConfig = {
          ExecStart = ''
            ${pkgs.prometheus-process-exporter}/bin/process-exporter \
              --web.listen-address 0.0.0.0:9257 \
              --config.path ${perPidConfig} \
              -children=false \
              -gather-smaps=false \
              -remove-empty-groups=true \
              -threads=false
          '';
          User = "root";
          Group = "root";
          Restart = "always";
          WorkingDirectory = "/tmp";
          CapabilityBoundingSet = [ "" ];
          DeviceAllow = [ "" ];
          LockPersonality = true;
          MemoryDenyWriteExecute = true;
          NoNewPrivileges = true;
          PrivateDevices = true;
          PrivateTmp = true;
          ProtectClock = true;
          ProtectControlGroups = true;
          ProtectHome = true;
          ProtectHostname = true;
          ProtectKernelLogs = true;
          ProtectKernelModules = true;
          ProtectKernelTunables = true;
          ProtectSystem = "strict";
          RemoveIPC = true;
          RestrictAddressFamilies = [
            "AF_INET"
            "AF_INET6"
          ];
          RestrictNamespaces = true;
          RestrictRealtime = true;
          RestrictSUIDSGID = true;
          SystemCallArchitectures = "native";
          UMask = "0077";
        };
      };
      zpool-latency-exporter = {
        description = "Exports ZFS latency metrics for node_exporter textfile collection";
        after = [ "zfs-import.target" ];
        requires = [ "zfs-import.target" ];
        path = [
          config.boot.zfs.package
          pkgs.coreutils
          pkgs.gawk
        ];
        serviceConfig = {
          Type = "oneshot";
          ExecStart = zpoolLatencyScript;
        };
      };
    };
    timers.zpool-latency-exporter = {
      wantedBy = [ "timers.target" ];
      timerConfig = {
        OnBootSec = "2m";
        OnUnitActiveSec = "60s";
        Unit = "zpool-latency-exporter.service";
      };
    };
    tmpfiles.rules = [ "d ${nodeTextfileDir} 0755 root root - -" ];
  };
 }
@@ -10,6 +10,9 @@
        authorizedKeys = config.users.users.richie.openssh.authorizedKeys.keys;
      };
    };
-    availableKernelModules = [ "igb" ];
+    availableKernelModules = [
      "igb"
      "r8152"
    ];
  };
 }
@@ -8,10 +8,11 @@
    dataDir = "/home/richie/Syncthing";
    configDir = "/home/richie/.config/syncthing";
    settings.devices = {
      phone.id = "TBRULKD-7DZPGGZ-F6LLB7J-MSO54AY-7KLPBIN-QOFK6PX-W2HBEWI-PHM2CQI"; # cspell:disable-line
      jeeves.id = "ICRHXZW-ECYJCUZ-I4CZ64R-3XRK7CG-LL2HAAK-FGOHD22-BQA4AI6-5OAL6AG"; # cspell:disable-line
      ipad.id = "KI76T3X-SFUGV2L-VSNYTKR-TSIUV5L-SHWD3HE-GQRGRCN-GY4UFMD-CW6Z6AX"; # cspell:disable-line
      bob.id = "CJIAPEJ-VO74RR4-F75VU6M-QNZAMYG-FYUJG7Y-6AT62HJ-355PRPL-PJFETAZ"; # cspell:disable-line
      brain.id = "SSCGIPI-IV3VYKB-TRNIJE3-COV4T2H-CDBER7F-I2CGHYA-NWOEUDU-3T5QAAN"; # cspell:disable-line
      ipad.id = "KI76T3X-SFUGV2L-VSNYTKR-TSIUV5L-SHWD3HE-GQRGRCN-GY4UFMD-CW6Z6AX"; # cspell:disable-line
      jeeves.id = "ICRHXZW-ECYJCUZ-I4CZ64R-3XRK7CG-LL2HAAK-FGOHD22-BQA4AI6-5OAL6AG"; # cspell:disable-line
      phone.id = "JPVQKQW-CFXOJXT-Q5G5F3H-QIDHDRE-GKHPTQB-GXZUQSP-U7FR7F7-INP3AAH"; # cspell:disable-line
      rhapsody-in-green.id = "ASL3KC4-3XEN6PA-7BQBRKE-A7JXLI6-DJT43BY-Q4WPOER-7UALUAZ-VTPQ6Q4"; # cspell:disable-line
    };
  };
@@ -0,0 +1,37 @@
 {
  config,
  inputs,
  ...
 }:
 {
  nix.settings = {
    trusted-substituters = [ "http://jeeves:5000" ];
    substituters = [ "http://jeeves:5000/?priority=1&want-mass-query=true" ];
    trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
  };
  services.tailscale = {
    enable = true;
    openFirewall = true;
    authKeyFile = config.sops.secrets.tailscale_auth_key.path;
    # OAuth client secrets create ephemeral nodes by default. NixOS machines
    # are persistent and should enroll without interactive device approval.
    authKeyParameters = {
      ephemeral = false;
      preauthorized = true;
    };
    extraUpFlags = [ "--advertise-tags=tag:nixos" ];
  };
  sops = {
    age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
    secrets.tailscale_auth_key = {
      sopsFile = "${inputs.self}/users/secrets.yaml";
      owner = "root";
      mode = "0400";
    };
  };
 }
@@ -4,6 +4,8 @@
    flags = [ "--accept-flake-config" ];
    randomizedDelaySec = "1h";
    persistent = true;
-    flake = "github:RichieCahill/dotfiles";
+    flake = "git+https://gitea.tmmworkshop.com/richie/dotfiles?ref=main";
    allowReboot = true;
    dates = "Sat *-*-* 06:00:00";
  };
 }
@@ -0,0 +1,9 @@
 {
  nixpkgs.hostPlatform = {
    system = "x86_64-linux";
    gcc = {
      arch = "x86-64-v3";
      tune = "generic";
    };
  };
 }
@@ -1,11 +0,0 @@
 {
  services.zerotierone = {
    enable = true;
    joinNetworks = [ "e4da7455b2ae64ca" ];
  };
  nix.settings = {
    trusted-substituters = [ "http://192.168.90.40:5000" ];
    substituters = [ "http://192.168.90.40:5000/?priority=1&want-mass-query=true" ];
    trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
  };
 }
@@ -0,0 +1,26 @@
 {
  inputs,
  lib,
  pkgs,
  ...
 }:
 {
  imports = [ ./snapshot.nix ];
  boot.zfs = {
    package = lib.mkDefault pkgs.zfs_2_4;
    forceImportRoot = lib.mkDefault false;
  };
  services = {
    snapshot_manager = {
      enable = lib.mkDefault true;
      PYTHONPATH = "${inputs.self}/";
    };
    zfs = {
      trim.enable = lib.mkDefault true;
      autoScrub.enable = lib.mkDefault true;
    };
  };
 }
@@ -0,0 +1,105 @@
 {
  inputs,
  pkgs,
  lib,
  config,
  ...
 }:
 let
  cfg = config.services.snapshot_manager;
  snapshotManagerPackages =
    ps: with ps; [
      httpx
      typer
    ];
 in
 {
  options = {
    services.snapshot_manager = {
      enable = lib.mkEnableOption "ZFS snapshot manager";
      path = lib.mkOption {
        type = lib.types.path;
        default = ./snapshot_config.toml;
        description = "Path to the snapshot_manager TOML config.";
      };
      PYTHONPATH = lib.mkOption {
        type = lib.types.str;
        description = ''
          the PYTHONPATH to use for the snapshot_manager service.
        '';
      };
      EnvironmentFile = lib.mkOption {
        type = lib.types.nullOr (lib.types.coercedTo lib.types.path toString lib.types.str);
        default = null;
        description = ''
          Single environment file for the service (e.g. /etc/snapshot-manager/env).
          Use a leading "-" to ignore if missing (systemd feature).
        '';
      };
    };
  };
  config = lib.mkIf cfg.enable {
    nixpkgs.overlays = [
      (final: _prev: {
        snapshot_manager_python = final.python314.withPackages snapshotManagerPackages;
        snapshot_manager_test_python = final.python314.withPackages (
          ps:
          snapshotManagerPackages ps
          ++ (with ps; [
            pyfakefs
            pytest
            pytest-asyncio
            pytest-mock
            pytest-xdist
          ])
        );
        snapshot_manager_tests =
          final.runCommand "snapshot-manager-tests"
            {
              nativeBuildInputs = [ final.snapshot_manager_test_python ];
            }
            ''
              export HOME="$TMPDIR"
              cd ${inputs.self}
              pytest \
                -o cache_dir="$TMPDIR/pytest-cache" \
                tests/test_common.py \
                tests/test_signal_alert.py \
                tests/test_snapshot_manager.py \
                tests/test_zfs.py
              touch "$out"
            '';
      })
    ];
    system.checks = [ pkgs.snapshot_manager_tests ];
    systemd = {
      services.snapshot_manager = {
        description = "ZFS Snapshot Manager";
        requires = [ "zfs-import.target" ];
        after = [ "zfs-import.target" ];
        path = [ pkgs.zfs ];
        environment = {
          PYTHONPATH = cfg.PYTHONPATH;
        };
        serviceConfig = {
          Type = "oneshot";
          ExecStart = "${pkgs.snapshot_manager_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
        }
        // lib.optionalAttrs (cfg.EnvironmentFile != null) {
          EnvironmentFile = cfg.EnvironmentFile;
        };
      };
      timers.snapshot_manager = {
        wantedBy = [ "timers.target" ];
        timerConfig = {
          OnBootSec = "15m";
          OnUnitActiveSec = "15m";
          Unit = "snapshot_manager.service";
        };
      };
    };
  };
 }
@@ -22,6 +22,12 @@ hourly = 0
 daily = 0
 monthly = 0
 ["root_pool/nix_build"]
 15_min = 1
 hourly = 0
 daily = 0
 monthly = 0
 ["root_pool/var"]
 15_min = 8
 hourly = 24
@@ -0,0 +1,4 @@
 source "https://rubygems.org"
 # The github-pages gem pins all compatible versions of Jekyll and its plugins
 gem "github-pages", group: :jekyll_plugins
@@ -0,0 +1,23 @@
 title: "Richie Cahill"
 description: "ALL THE CHAOS THAT I CANT DO AT WORK"
 baseurl: "/dotfiles"
 url: "https://richiecahill.github.io"
 remote_theme: pages-themes/hacker@v0.2.0
 plugins:
  - jekyll-feed
  - jekyll-remote-theme
  - jekyll-seo-tag
  - jekyll-sitemap
  - jekyll-paginate
 paginate: 5
 paginate_path: "/page:num"
 author:
  name: "Richie Cahill"
  email: "richie@tmmworkshop.com"
 social_links:
  github: "RichieCahill"
  website: "https://tmmworkshop.com"
@@ -0,0 +1,13 @@
 # The MONOREPO experiment
 Im testing a [MONOREPO](https://en.wikipedia.org/wiki/Monorepo) because Phil said this was a bad idea. To that i say hold my beer.
 In all seriousness, I Think that for a small dev team/solo dev. The simplicity is worth higher barer to entry. One of my most annoying processes was updating my system tools. I had to build my update in a feature branch and then merge it into my main branch. then go to my dotfiles create a feature branch update the system tools merge it into main.
 It will be starting with my Nix Dotfiles Python tools and now my blog.
 I will be reaching ot to phil on 2030-10-31 and 2035-10-31 to give him updates on the progress.
 Known Issues:
 - the python tests are running on the current derivation not the one the derivation im updating to.
@@ -0,0 +1,17 @@
 ---
 layout: default
 title: "Welcome"
 ---
 Welcome to my build logs, notes, and experiments.
 You can read my latest posts below
 <ul>
  {% for post in site.posts %}
  <li>
    <a href="{{ post.url | relative_url }}">{{ post.title }}</a>
    <small>— {{ post.date | date: "%Y-%m-%d" }}</small>
  </li>
  {% endfor %}
 </ul>
@@ -0,0 +1,76 @@
 # ZFS failed root import recovery
 ## Fast path
 If the machine fails to boot because ZFS refuses to import `root_pool`:
 ### GRUB
 1. At the bootloader menu, select the normal NixOS entry.
 2. Press `e`.
 3. Find the line that starts with `linux`.
 4. Append this to the end of that line:
 ```text
 zfs_force=1
 ```
 5. Boot once with `Ctrl+x` or `F10`.
 ### systemd-boot
 1. At the bootloader menu, highlight the normal NixOS entry.
 2. Press `e`.
 3. Append this to the end of the options line:
 ```text
 zfs_force=1
 ```
 4. Press `Enter` to boot once.
 ## After boot
 Run:
 ```bash
 sudo zpool status
 sudo zpool import
 journalctl -b | rg "ZFS|zfs|import|root_pool"
 ```
 ## Expected result
 `sudo zpool status` should show `root_pool` as `ONLINE`.
 ## Reboot test
 Run:
 ```bash
 sudo reboot
 ```
 Do not add `zfs_force=1` the second time.
 ## If it still fails
 Boot once more with:
 ```text
 zfs_force=1
 ```
 Then run:
 ```bash
 sudo zpool status -v
 sudo zpool history | tail -n 50
 journalctl -b | rg "ZFS|zfs|import|root_pool"
 ```
 ## Notes
 - Root pool name is `root_pool`.
 - This is a one-time recovery path after disk moves, controller changes, dirty exports, or interrupted imports.
 - Some hosts also need the LUKS unlock USB key inserted before boot.
@@ -0,0 +1,3 @@
 # esphome
 /.esphome/
 /secrets.yaml
@@ -0,0 +1,132 @@
 esphome:
  name: batteries
  friendly_name: batteries
 esp32:
  board: esp32dev
  framework:
    type: arduino
 logger:
 api:
  encryption:
    key: !secret api_key
 external_components:
  - source: github://syssi/esphome-jk-bms@main
 ota:
  - platform: esphome
    password: !secret ota_password
 wifi:
  ssid: !secret wifi_ssid
  password: !secret wifi_password
  fast_connect: on
 captive_portal:
 esp32_ble_tracker:
  scan_parameters:
    interval: 1100ms
    window: 1100ms
    active: true
 ble_client:
  - mac_address: "C8:47:80:29:0F:DB"
    id: jk_ble0
 jk_bms_ble:
  - ble_client_id: jk_ble0
    protocol_version: JK02_32S
    throttle: 1s
    id: jk_bms0
 button:
  - platform: jk_bms_ble
    retrieve_settings:
      name: "JK0 retrieve settings"
    retrieve_device_info:
      name: "JK0 retrieve device info"
 sensor:
  - platform: jk_bms_ble
    jk_bms_ble_id: jk_bms0
    total_voltage:
      name: "JK0 Total Voltage"
    state_of_charge:
      name: "JK0 SoC"
    charging_power:
      name: "JK0 charging power"
    discharging_power:
      name: "JK0 discharging power"
    temperature_sensor_1:
      name: "JK0 Temp 1"
    temperature_sensor_2:
      name: "JK0 Temp 2"
    balancing:
      name: "JK0 balancing"
    total_runtime:
      name: "JK0 total runtime"
    balancing_current:
      name: "JK0 balancing current"
    delta_cell_voltage:
      name: "JK0 cell delta voltage"
    average_cell_voltage:
      name: "JK0 cell average voltage"
    cell_voltage_1:
      name: "JK0 cell voltage 1"
    cell_voltage_2:
      name: "JK0 cell voltage 2"
    cell_voltage_3:
      name: "JK0 cell voltage 3"
    cell_voltage_4:
      name: "JK0 cell voltage 4"
    cell_voltage_5:
      name: "JK0 cell voltage 5"
    cell_voltage_6:
      name: "JK0 cell voltage 6"
    cell_voltage_7:
      name: "JK0 cell voltage 7"
    cell_voltage_8:
      name: "JK0 cell voltage 8"
    cell_resistance_1:
      name: "JK0 cell resistance 1"
    cell_resistance_2:
      name: "JK0 cell resistance 2"
    cell_resistance_3:
      name: "JK0 cell resistance 3"
    cell_resistance_4:
      name: "JK0 cell resistance 4"
    cell_resistance_5:
      name: "JK0 cell resistance 5"
    cell_resistance_6:
      name: "JK0 cell resistance 6"
    cell_resistance_7:
      name: "JK0 cell resistance 7"
    cell_resistance_8:
      name: "JK0 cell resistance 8"
    total_charging_cycle_capacity:
      name: "JK0 total charging cycle capacity"
 text_sensor:
  - platform: jk_bms_ble
    jk_bms_ble_id: jk_bms0
    errors:
      name: "JK0 Errors"
 switch:
  - platform: jk_bms_ble
    jk_bms_ble_id: jk_bms0
    charging:
      name: "JK0 Charging"
    discharging:
      name: "JK0 Discharging"
    balancer:
      name: "JK0 Balancing"
  - platform: ble_client
    ble_client_id: jk_ble0
    name: "JK0 enable bluetooth connection"
    id: ble_client_switch0
@@ -0,0 +1,132 @@
 esphome:
  name: battery1
  friendly_name: battery1
 esp32:
  board: esp32dev
  framework:
    type: arduino
 logger:
 api:
  encryption:
    key: !secret api_key
 external_components:
  - source: github://syssi/esphome-jk-bms@main
 ota:
  - platform: esphome
    password: !secret ota_password
 wifi:
  ssid: !secret wifi_ssid
  password: !secret wifi_password
  fast_connect: on
 captive_portal:
 esp32_ble_tracker:
  scan_parameters:
    interval: 1100ms
    window: 1100ms
    active: true
 ble_client:
  - mac_address: "C8:47:80:37:9D:DD"
    id: jk_ble1
 jk_bms_ble:
  - ble_client_id: jk_ble1
    protocol_version: JK02_32S
    throttle: 1s
    id: jk_bms1
 button:
  - platform: jk_bms_ble
    retrieve_settings:
      name: "JK1 retrieve settings"
    retrieve_device_info:
      name: "JK1 retrieve device info"
 sensor:
  - platform: jk_bms_ble
    jk_bms_ble_id: jk_bms1
    total_voltage:
      name: "JK1 Total Voltage"
    state_of_charge:
      name: "JK1 SoC"
    charging_power:
      name: "JK1 charging power"
    discharging_power:
      name: "JK1 discharging power"
    temperature_sensor_1:
      name: "JK1 Temp 1"
    temperature_sensor_2:
      name: "JK1 Temp 2"
    balancing:
      name: "JK1 balancing"
    total_runtime:
      name: "JK1 total runtime"
    balancing_current:
      name: "JK1 balancing current"
    delta_cell_voltage:
      name: "JK1 cell delta voltage"
    average_cell_voltage:
      name: "JK1 cell average voltage"
    cell_voltage_1:
      name: "JK1 cell voltage 1"
    cell_voltage_2:
      name: "JK1 cell voltage 2"
    cell_voltage_3:
      name: "JK1 cell voltage 3"
    cell_voltage_4:
      name: "JK1 cell voltage 4"
    cell_voltage_5:
      name: "JK1 cell voltage 5"
    cell_voltage_6:
      name: "JK1 cell voltage 6"
    cell_voltage_7:
      name: "JK1 cell voltage 7"
    cell_voltage_8:
      name: "JK1 cell voltage 8"
    cell_resistance_1:
      name: "JK1 cell resistance 1"
    cell_resistance_2:
      name: "JK1 cell resistance 2"
    cell_resistance_3:
      name: "JK1 cell resistance 3"
    cell_resistance_4:
      name: "JK1 cell resistance 4"
    cell_resistance_5:
      name: "JK1 cell resistance 5"
    cell_resistance_6:
      name: "JK1 cell resistance 6"
    cell_resistance_7:
      name: "JK1 cell resistance 7"
    cell_resistance_8:
      name: "JK1 cell resistance 8"
    total_charging_cycle_capacity:
      name: "JK1 total charging cycle capacity"
 text_sensor:
  - platform: jk_bms_ble
    jk_bms_ble_id: jk_bms1
    errors:
      name: "JK1 Errors"
 switch:
  - platform: jk_bms_ble
    jk_bms_ble_id: jk_bms1
    charging:
      name: "JK1 Charging"
    discharging:
      name: "JK1 Discharging"
    balancer:
      name: "JK1 Balancing"
  - platform: ble_client
    ble_client_id: jk_ble1
    name: "JK1 enable bluetooth connection"
    id: ble_client_switch0
@@ -0,0 +1,48 @@
 esphome:
  name: "environment"
  friendly_name: "environment"
 esp32:
  board: esp32dev
  framework:
    type: arduino
 i2c:
  sda: GPIO21
  scl: GPIO22
  scan: True
  id: bus_a
 sensor:
  - platform: aht10
    i2c_id: bus_a
    address: 0x38
    variant: AHT20
    temperature:
      name: "environment Temperature"
      id: aht10_temperature
    humidity:
      name: "environment Humidity"
      id: aht10_humidity
    update_interval: 5s
 web_server:
  port: 80
 logger:
  level: DEBUG
 api:
  encryption:
    key: !secret api_key
 ota:
  - platform: esphome
    password: !secret ota_password
 wifi:
  ssid: !secret wifi_ssid
  password: !secret wifi_password
  fast_connect: on
 captive_portal:
@@ -1,25 +1,23 @@
 {
  "nodes": {
-    "firefox-addons": {
+    "disko": {
      "inputs": {
        "nixpkgs": [
          "nixpkgs"
        ]
      },
      "locked": {
-        "dir": "pkgs/firefox-addons",
+        "lastModified": 1781152676,
-        "lastModified": 1750219402,
+        "narHash": "sha256-RxWs5ND31KzTG7wvMM+PMfUjyNpmIEr999lqNARaM5o=",
-        "narHash": "sha256-b3y7V7db0VwLGtpcLRmT1Aa9dpAKoHQdem55UhgB/fw=",
+        "owner": "nix-community",
-        "owner": "rycee",
+        "repo": "disko",
-        "repo": "nur-expressions",
+        "rev": "ff8702b4de27f72b4c78573dfb89ec74e36abdf1",
-        "rev": "a00ce73b626ed274fbfe9f51627861e140b08f6d",
+        "type": "github"
        "type": "gitlab"
      },
      "original": {
-        "dir": "pkgs/firefox-addons",
+        "owner": "nix-community",
-        "owner": "rycee",
+        "repo": "disko",
-        "repo": "nur-expressions",
+        "type": "github"
        "type": "gitlab"
      }
    },
    "home-manager": {
@@ -29,11 +27,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1750275572,
+        "lastModified": 1788651960,
-        "narHash": "sha256-upC/GIlsIgtdtWRGd1obzdXWYQptNkfzZeyAFWgsgf0=",
+        "narHash": "sha256-v9wJd32eZ2bvhBzVOd7TIjLQd011P7nwOhjKtWlci5I=",
        "owner": "nix-community",
        "repo": "home-manager",
-        "rev": "0f355844e54e4c70906b1ef5cc35a0047d666c04",
+        "rev": "2c0350c759688177331b8f5242311fae8877bdb3",
        "type": "github"
      },
      "original": {
@@ -43,12 +41,15 @@
      }
    },
    "nixos-hardware": {
      "inputs": {
        "nixpkgs": "nixpkgs"
      },
      "locked": {
-        "lastModified": 1750083401,
+        "lastModified": 1788860136,
-        "narHash": "sha256-ynqbgIYrg7P1fAKYqe8I/PMiLABBcNDYG9YaAP/d/C4=",
+        "narHash": "sha256-MhPMOFV4pVkygWEbQ8t1De/uQ9cWF1u++tRe2L5tG48=",
        "owner": "nixos",
        "repo": "nixos-hardware",
-        "rev": "61837d2a33ccc1582c5fabb7bf9130d39fee59ad",
+        "rev": "62173785b9a18c78b4a15aca2623d02bceb9d077",
        "type": "github"
      },
      "original": {
@@ -60,27 +61,24 @@
    },
    "nixpkgs": {
      "locked": {
-        "lastModified": 1750134718,
+        "lastModified": 1767892417,
-        "narHash": "sha256-v263g4GbxXv87hMXMCpjkIxd/viIF7p3JpJrwgKdNiI=",
+        "narHash": "sha256-8bW3q88CEg2u4hSP66Vf4lpbLonHz7hqDNBMcCY7E9U=",
-        "owner": "nixos",
+        "rev": "3497aa5c9457a9d88d71fa93a4a8368816fbeeba",
-        "repo": "nixpkgs",
+        "type": "tarball",
-        "rev": "9e83b64f727c88a7711a2c463a7b16eedb69a84c",
+        "url": "https://releases.nixos.org/nixos/unstable/nixos-26.05pre924538.3497aa5c9457/nixexprs.tar.xz"
        "type": "github"
      },
      "original": {
-        "owner": "nixos",
+        "type": "tarball",
-        "ref": "nixos-unstable",
+        "url": "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz"
        "repo": "nixpkgs",
        "type": "github"
      }
    },
    "nixpkgs-master": {
      "locked": {
-        "lastModified": 1750291913,
+        "lastModified": 1788892992,
-        "narHash": "sha256-JW40+zIiDS+rZavb9IYdIN40/GmErO2+0+A66rM6/b8=",
+        "narHash": "sha256-cIMFh9gyU4/aLeB3JCcsWM3tTAvD9pAq9Smr1Wa8aIU=",
        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "ba92ab5dc0759a8740003ca34b5c1b888f4766d4",
+        "rev": "dff6994123e257ec9901c271bc2b52e64d7c8f05",
        "type": "github"
      },
      "original": {
@@ -106,66 +104,31 @@
        "type": "github"
      }
    },
-    "pyproject-build-systems": {
+    "nixpkgs_2": {
      "inputs": {
        "nixpkgs": [
          "system_tools",
          "nixpkgs"
        ],
        "pyproject-nix": [
          "system_tools",
          "pyproject-nix"
        ],
        "uv2nix": [
          "system_tools",
          "uv2nix"
        ]
      },
      "locked": {
-        "lastModified": 1744599653,
+        "lastModified": 1788752844,
-        "narHash": "sha256-nysSwVVjG4hKoOjhjvE6U5lIKA8sEr1d1QzEfZsannU=",
+        "narHash": "sha256-VaWGJ6+cIYN2erfSecbRV+4ljI185Ty2wUrXyvQbgOw=",
-        "owner": "pyproject-nix",
+        "owner": "nixos",
-        "repo": "build-system-pkgs",
+        "repo": "nixpkgs",
-        "rev": "7dba6dbc73120e15b558754c26024f6c93015dd7",
+        "rev": "dc5d91f840324650bac8c379428c7037a416959a",
        "type": "github"
      },
      "original": {
-        "owner": "pyproject-nix",
+        "owner": "nixos",
-        "repo": "build-system-pkgs",
+        "ref": "nixos-unstable",
-        "type": "github"
+        "repo": "nixpkgs",
      }
    },
    "pyproject-nix": {
      "inputs": {
        "nixpkgs": [
          "system_tools",
          "nixpkgs"
        ]
      },
      "locked": {
        "lastModified": 1746540146,
        "narHash": "sha256-QxdHGNpbicIrw5t6U3x+ZxeY/7IEJ6lYbvsjXmcxFIM=",
        "owner": "pyproject-nix",
        "repo": "pyproject.nix",
        "rev": "e09c10c24ebb955125fda449939bfba664c467fd",
        "type": "github"
      },
      "original": {
        "owner": "pyproject-nix",
        "repo": "pyproject.nix",
        "type": "github"
      }
    },
    "root": {
      "inputs": {
-        "firefox-addons": "firefox-addons",
+        "disko": "disko",
        "home-manager": "home-manager",
        "nixos-hardware": "nixos-hardware",
-        "nixpkgs": "nixpkgs",
+        "nixpkgs": "nixpkgs_2",
        "nixpkgs-master": "nixpkgs-master",
        "nixpkgs-stable": "nixpkgs-stable",
        "sops-nix": "sops-nix",
        "system_tools": "system_tools",
        "systems": "systems"
      }
    },
@@ -176,11 +139,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1750119275,
+        "lastModified": 1788337237,
-        "narHash": "sha256-Rr7Pooz9zQbhdVxux16h7URa6mA80Pb/G07T4lHvh0M=",
+        "narHash": "sha256-gkSH8VUtCo6hnysNmb9DbTuDepH2t5pv+QWjP75xKAk=",
        "owner": "Mic92",
        "repo": "sops-nix",
-        "rev": "77c423a03b9b2b79709ea2cb63336312e78b72e2",
+        "rev": "fbf759290e0cb0a98dfc813a4eb7d53ad1dacb57",
        "type": "github"
      },
      "original": {
@@ -189,29 +152,6 @@
        "type": "github"
      }
    },
    "system_tools": {
      "inputs": {
        "nixpkgs": [
          "nixpkgs"
        ],
        "pyproject-build-systems": "pyproject-build-systems",
        "pyproject-nix": "pyproject-nix",
        "uv2nix": "uv2nix"
      },
      "locked": {
        "lastModified": 1747501237,
        "narHash": "sha256-woyaUwmZurfNTXBEFM6M7ueSd/Udixs+4DUInhL835c=",
        "owner": "RichieCahill",
        "repo": "system_tools",
        "rev": "68ab5d1c17ac3fe2487f73dbbb4848bd2291139e",
        "type": "github"
      },
      "original": {
        "owner": "RichieCahill",
        "repo": "system_tools",
        "type": "github"
      }
    },
    "systems": {
      "locked": {
        "lastModified": 1689347949,
@@ -226,31 +166,6 @@
        "repo": "default-linux",
        "type": "github"
      }
    },
    "uv2nix": {
      "inputs": {
        "nixpkgs": [
          "system_tools",
          "nixpkgs"
        ],
        "pyproject-nix": [
          "system_tools",
          "pyproject-nix"
        ]
      },
      "locked": {
        "lastModified": 1747441483,
        "narHash": "sha256-W8BFXk5R0TuJcjIhcGoMpSOaIufGXpizK0pm+uTqynA=",
        "owner": "pyproject-nix",
        "repo": "uv2nix",
        "rev": "582024dc64663e9f88d467c2f7f7b20d278349de",
        "type": "github"
      },
      "original": {
        "owner": "pyproject-nix",
        "repo": "uv2nix",
        "type": "github"
      }
    }
  },
  "root": "root",
@@ -26,20 +26,15 @@
      inputs.nixpkgs.follows = "nixpkgs";
    };
    firefox-addons = {
      url = "gitlab:rycee/nur-expressions?dir=pkgs/firefox-addons";
      inputs.nixpkgs.follows = "nixpkgs";
    };
    system_tools = {
      url = "github:RichieCahill/system_tools";
      inputs.nixpkgs.follows = "nixpkgs";
    };
    sops-nix = {
      url = "github:Mic92/sops-nix";
      inputs.nixpkgs.follows = "nixpkgs";
    };
    disko = {
      url = "github:nix-community/disko";
      inputs.nixpkgs.follows = "nixpkgs";
    };
  };
  outputs =
@@ -59,6 +54,7 @@
        system:
        import nixpkgs {
          inherit system;
          overlays = builtins.attrValues outputs.overlays;
          config.allowUnfree = true;
        }
      );
@@ -69,26 +65,48 @@
      devShells = forEachSystem (pkgs: import ./shell.nix { inherit pkgs; });
      formatter = forEachSystem (pkgs: pkgs.treefmt);
      packages = forEachSystem (
        pkgs:
        let
          installer = pkgs.callPackage ./python/installer/package.nix { };
          installer-nixos = pkgs.callPackage ./python/installer/package.nix { patchElf = false; };
        in
        {
          inherit installer installer-nixos;
          default = installer;
        }
        // lib.optionalAttrs (pkgs.stdenv.hostPlatform.system == "x86_64-linux") {
          iso = self.nixosConfigurations.iso.config.system.build.isoImage;
        }
      );
      apps = forEachSystem (
        pkgs:
        let
          system = pkgs.stdenv.hostPlatform.system;
          installer = {
            type = "app";
            program = "${self.packages.${system}.installer}/bin/nixos-installer";
            meta.description = "One-file NixOS ZFS installer.";
          };
        in
        {
          inherit installer;
          default = installer;
        }
      );
-      nixosConfigurations = {
+      nixosConfigurations =
-        bob = lib.nixosSystem {
+        let
-          modules = [
+          hosts = builtins.attrNames (
-            ./systems/bob
+            lib.filterAttrs (_: type: type == "directory") (builtins.readDir ./systems)
-          ];
+          );
-          specialArgs = { inherit inputs outputs; };
+          mkHost =
-        };
+            name:
-        jeeves = lib.nixosSystem {
+            lib.nixosSystem {
-          modules = [
+              modules = [ ./systems/${name} ];
-            ./systems/jeeves
+              specialArgs = { inherit inputs outputs; };
-          ];
+            };
-          specialArgs = { inherit inputs outputs; };
+        in
-        };
+        lib.genAttrs hosts mkHost;
        rhapsody-in-green = lib.nixosSystem {
          modules = [
            ./systems/rhapsody-in-green
          ];
          specialArgs = { inherit inputs outputs; };
        };
      };
    };
 }
@@ -3,15 +3,17 @@
  # When applied, the stable nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.stable'
  stable = final: _prev: {
    stable = import inputs.nixpkgs-stable {
-      system = final.system;
+      system = final.stdenv.hostPlatform.system;
      config.allowUnfree = true;
    };
  };
  # When applied, the master nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.master'
  master = final: _prev: {
    master = import inputs.nixpkgs-master {
-      system = final.system;
+      system = final.stdenv.hostPlatform.system;
      config.allowUnfree = true;
    };
  };
  patches = import ./patches;
 }
@@ -0,0 +1,46 @@
 # Package patches
 Each package follows the [GnuTLS layout](gnutls/README.md):
 - `default.nix` applies the patch through the package overlay.
 - A descriptive `.patch` file contains the standalone upstream source change.
 - `README.md` explains the problem, scope, reproduction, upstream status,
  Nix integration, and recorded validation limits.
 - Companion `verify-*` tools live beside the patch when needed; otherwise
  the README gives commands for the package's existing tests.
 Keep package-specific evidence in its directory. Patch headers explain the
 change independently of Nix, and `default.nix` preserves existing patches.
 | Package | Repair |
 | --- | --- |
 | [Abseil](abseil/README.md) | Public BMI2 header in Electron, Deno, and Signal's vendored copies |
 | [Backrefs](backrefs/README.md) | Match the regex timeout's CPU clock |
 | [GnuTLS](gnutls/README.md) | Wait for the UDP server socket before connecting |
 | [Jupyter Server](jupyter-server/README.md) | Exercise the correct shared future during reconnect |
 | [Prometheus](prometheus/README.md) | Complete parsing before inspecting the test editor state |
 | [pytest-xdist](pytest-xdist/README.md) | Check worker replacements and allow startup on loaded builders |
 | [SciPy](scipy/README.md) | Account for floating-point rounding in STFT tests |
 | [Sentry SDK](sentry-sdk/README.md) | Isolate SDK thread mocks from Python's threading module |
 | [Torchaudio](torchaudio/README.md) | Compare pitch-shift batches at appropriate precision |
 | [TorchCodec](torchcodec/README.md) | Match the reference MP3 encoder's sample format |
 ## Local NixOS integration
 [`../default.nix`](../default.nix) imports this directory's
 [`default.nix`](default.nix), which wires each package's override into the
 package set. Abseil repairs several vendored copies and is gated on
 `x86-64-v3`; Prometheus patches its separate assets derivation; Python
 packages use `pythonPackagesExtensions`.
 The [pytest-xdist directory](pytest-xdist/README.md) also owns its outer-worker
 limit and remote-worker event timeout. These package overrides add no skipped
 tests. Existing nixpkgs exclusions remain separate from these repairs.
 The test-exclusion review used Python 3.14.7 and the pinned x86-64-v3 package
 set. Host-flake evaluation verified patch wiring, Python install checks,
 removal of the local skips, and Prometheus's reference to the patched assets.
 Jupyter and Sentry package tests used the preceding dependency set with the
 new package patch to avoid unrelated rebuilds after pytest-xdist changed.
 No complete NixOS rebuild was performed. Individual READMEs distinguish
 package builds, focused tests, and checks that have not been run.
@@ -0,0 +1,59 @@
 # Abseil BMI2 public header
 Vendored Abseil includes `bmi2intrin.h` directly when `__BMI2__` is enabled.
 Compilers reject that internal header without the umbrella-header setup.
 `bmi2-public-header.patch` includes `immintrin.h` instead, allowing builds
 that enable BMI2 through `-march=x86-64-v3`.
 ## Scope and behavior
 The patch changes one include in
 `third_party/abseil-cpp/absl/container/internal/raw_hash_set.h`.
 `default.nix` applies it to Electron 43's unwrapped package, Deno's
 `librusty_v8`, and Signal's WebRTC dependency. It also supplies the patched
 Electron package to Signal. These overrides apply only to `x86-64-v3`.
 The shared file path is relative to each vendoring project's source root,
 not the root of a standalone Abseil checkout. No hash-table algorithm or
 test exclusion changes.
 ## Reproduction and focused checks
 From this directory, check and apply the patch to each vendored source tree:
 ```sh
 patch --dry-run --fuzz=0 -d /path/to/vendor-source -p1 < bmi2-public-header.patch
 patch --fuzz=0 -d /path/to/vendor-source -p1 < bmi2-public-header.patch
 ```
 A small compiler check isolates the header requirement. With GCC or Clang
 on x86-64, compile `#include <bmi2intrin.h>` using `-march=x86-64-v3`; the
 compiler rejects the direct include. Changing it to `#include <immintrin.h>`
 should compile. The full consumer builds below check integration with their
 actual toolchains.
 ## Upstream status
 Abseil addressed this issue through
 [PR #2071](https://github.com/abseil/abseil-cpp/pull/2071), imported by its
 upstream workflow. That change uses `x86gprintrin.h`; this local variant uses
 the public `immintrin.h` umbrella header for the vendored toolchains.
 Keep the workaround until all three bundled copies include a compatible fix.
 This file is a local adaptation, not a verbatim copy of the upstream diff.
 ## Local NixOS integration and build results
 [`../default.nix`](../default.nix) merges this directory's overlay fragment
 because it repairs multiple packages. From the repository root, the consumer
 build commands are:
 ```sh
 nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.deno
 nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.electron_43
 nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.signal-desktop
 ```
 The earlier extraction checked the vendored header snapshots and evaluated
 all three patch attachments. Those records do not establish successful full
 consumer rebuilds. No new compiler or consumer build was run for the layout
 change; the patch and override are unchanged.
@@ -0,0 +1,20 @@
 Subject: [PATCH] abseil: include BMI2 intrinsics through the public header
 GCC and Clang reject direct inclusion of bmi2intrin.h. Include immintrin.h
 instead so that the compiler supplies the required intrinsic setup when
 BMI2 is enabled, including builds targeting x86-64-v3.
 This patch is shared by the vendored Abseil copies in Electron, rusty_v8
 (Deno), and Signal's WebRTC build.
 --- a/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
 +++ b/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
@@ -226,7 +226,7 @@
 #endif
 #ifdef __BMI2__
 -#include <bmi2intrin.h>
 +#include <immintrin.h>
 #endif  // __BMI2__
 namespace absl {
@@ -0,0 +1,38 @@
 # Abseil accepted the upstream fix: https://github.com/abseil/abseil-cpp/pull/2071
 # Keep this workaround until Electron, Deno's rusty_v8, and Signal's WebRTC
 # update their bundled Abseil copies to include it.
 { prev }:
 let
  patchAbseilBmi2Include =
    package:
    package.overrideAttrs (old: {
      # GCC and Clang require the public umbrella header for BMI2 intrinsics.
      patches = (old.patches or [ ]) ++ [ ./bmi2-public-header.patch ];
    });
  electron43Unwrapped = patchAbseilBmi2Include prev.electron_43.unwrapped;
  electron43 = prev.electron_43.override {
    electron-unwrapped = electron43Unwrapped;
  };
  signalCallPackage =
    path: args:
    let
      package = prev.callPackage path args;
    in
    if builtins.baseNameOf path == "webrtc.nix" then patchAbseilBmi2Include package else package;
 in
 prev.lib.optionalAttrs ((prev.stdenv.hostPlatform.gcc.arch or null) == "x86-64-v3") {
  deno =
    let
      librusty_v8 = patchAbseilBmi2Include prev.deno.passthru.librusty_v8;
    in
    prev.deno.override { inherit librusty_v8; };
  electron_43 = electron43;
  signal-desktop = prev.signal-desktop.override {
    electron_43 = electron43;
    callPackage = signalCallPackage;
  };
 }
@@ -0,0 +1,14 @@
 _final: prev:
 (import ./abseil { inherit prev; })
 // {
  gnutls = import ./gnutls { inherit (prev) gnutls; };
  prometheus = import ./prometheus { inherit (prev) prometheus; };
  pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
    (_pythonFinal: pythonPrev: {
      pytest-xdist = import ./pytest-xdist { inherit (pythonPrev) pytest-xdist; };
      scipy = import ./scipy { inherit (pythonPrev) scipy; };
      sentry-sdk = import ./sentry-sdk { inherit (pythonPrev) sentry-sdk; };
    })
  ];
 }
@@ -0,0 +1,160 @@
 # GnuTLS UDP server readiness
 Under load, the test client can start before `gnutls-serv` binds its UDP
 socket, and the first handshake fails with `Connection refused`.
 `serv-udp.sh` currently waits a fixed four seconds; elapsed time does not
 establish server readiness. `udp-server-readiness.patch` replaces that wait
 with polling for the local IPv4 UDP endpoint.
 ## Scope and waiting behavior
 The patch changes the existing `wait_udp_server()` and adds a new
 `check_if_udp_port_bound()` beside it in `tests/scripts/common.sh`.
 `serv-udp.sh` is its only caller in 3.8.13. The TCP helpers `wait_server()`
 and `wait_for_port()`, including their existing sleeps, are unchanged.
 Both original DTLS handshake checks remain unchanged.
 Each iteration checks process liveness and the socket **before sleeping**.
 A ready socket returns immediately. An unsuccessful check sleeps two
 seconds only if another attempt remains: at most 90 attempts, consistent
 with the existing `wait_server()` budget implemented by `wait_for_port()`,
 with no sleep after the final check. Server exit fails early; exhausting the
 budget fails and terminates the server. No handshake is retried, and no
 protocol timeout is changed. Once bound, the kernel can queue datagrams
 while the server is scheduled; the probe itself sends no packets.
 The existing `have_port_finder()` prefers `ss`, then `netstat`. If neither
 exists, it prints `neither ss nor netstat found` and exits **77 (skip)**.
 In the normal test flow, port selection calls it before launching a server.
 The probe runs in a subshell so that, even if this skip occurs after launch,
 the waiting helper can terminate and reap the server before exiting 77.
 ## Why an IPv4 socket is expected
 This is specific to the server used by this test, not a general rule that
 IPv6 sockets cannot serve IPv4 clients. The client explicitly uses
 `127.0.0.1`. The server's `--udp` path calls `udp_server()`, which calls
 `listen_socket(..., SOCK_DGRAM)`. That function iterates the wildcard
 addresses returned by `getaddrinfo(NULL, ..., AI_PASSIVE)`:
 | Server build / Linux setting | Binding behavior |
 | --- | --- |
 | IPv6 enabled, `net.ipv6.bindv6only=0` | Requests `IPV6_V6ONLY=1` on the IPv6 socket, binds `[::]:PORT`, and separately binds `0.0.0.0:PORT`. It overrides the system's dual-stack default. |
 | IPv6 enabled, `net.ipv6.bindv6only=1` | The same explicit socket option and separate IPv4/IPv6 binds. |
 | `HAVE_IPV6` undefined | Skips every address family except `AF_INET`; only the IPv4 wildcard is attempted. |
 `udp_server()` uses `wait_for_connection()`, which puts **every listener**
 from that list into `select()` and returns a readable socket for `recvfrom()`;
 it does not permanently choose one socket based on `getaddrinfo()` order.
 The first two cases were traced with the actual GnuTLS 3.8.13 binary in
 separate Linux network namespaces: `setsockopt(IPV6_V6ONLY, [1])` and both
 UDP binds returned success under each setting. The no-IPv6 case was checked
 in source, not by building a second binary. The same bind implementation
 was checked directly on GitLab master.
 Thus, successful normal startup for this invocation provides an explicit
 IPv4 socket; a lone IPv6 wildcard is not the expected success path.
 There is one portability caveat: upstream discards the return value of
 `setsockopt(IPV6_V6ONLY)`. On a platform where that call fails and the server
 ends up with only a dual-stack socket, this helper would time out despite
 IPv4 reachability. Such a platform needs additional handling before this
 patch can claim support. Blindly accepting every IPv6 wildcard would also
 accept IPv6-only sockets before the separate IPv4 bind finishes.
 Source: [`src/serv.c`, `listen_socket()`](https://gitlab.com/gnutls/gnutls/-/blob/master/src/serv.c#L937),
 [`src/udp-serv.c`](https://gitlab.com/gnutls/gnutls/-/blob/master/src/udp-serv.c),
 and [`tests/serv-udp.sh`](https://gitlab.com/gnutls/gnutls/-/blob/master/tests/serv-udp.sh).
 ## Port matching and ownership limit
 Only `-an` is passed to the socket-listing tool: BSD `netstat -u` selects
 Unix-domain sockets, whereas Linux `netstat -u` selects UDP. The parser
 handles the extra state column in `ss`, Linux colon-separated endpoints,
 and BSD dot-separated endpoints, including `*.PORT`. It matches the full
 local port and rejects TCP, IPv6 entries, peer ports, and longer numbers.
 A live PID plus a bound port does **not** prove that PID owns the socket.
 Existing `GETPORT` selection checks for an unused port and uses a test
 port-lock directory; `launch_bare_server()` also calls
 `wait_for_free_port()` before starting the process. These are advisory:
 the launcher does not enforce the latter's result, and another process
 can bind between the check and launch. The patch does not close that race
 or add nonportable PID parsing. An unrelated process can satisfy the
 socket check; the real handshakes remain the functional check and may
 fail (or reach the wrong server). This is a startup-order fix, not a
 socket-ownership guarantee.
 ## Reproduction and focused checks
 Apply the patch to an unpacked source tree, then run the companion checks
 with Python's standard library and a shell:
 ```sh
 patch --fuzz=0 -d /path/to/gnutls -p1 < udp-server-readiness.patch
 SHELL=/bin/sh python3 verify-readiness.py /path/to/gnutls/tests/scripts/common.sh -v
 ```
 Set `NETSTAT=/path/to/netstat` to exercise one outside `PATH`. The checks
 cover Linux/BSD output samples, false matches, immediate readiness,
 missing tools, process exit, timeout cleanup, and real IPv4 UDP sockets
 whose bind is delayed six seconds. The missing-tools fixture is skipped
 if an absolute fallback `ss` path cannot be hidden with `PATH`. Native
 BSD execution remains untested.
 To reproduce with GnuTLS itself, run `tests/serv-udp.sh` with `SERV` pointing
 to a wrapper that sleeps six seconds, then `exec`s `gnutls-serv` with all
 arguments. Set `CLI` to the matching `gnutls-cli`, `srcdir` to the source
 `tests` directory, and `abs_top_builddir` to a writable build directory.
 With GnuTLS 3.8.13, the original helper failed the first handshake with
 `Connection refused`; the patched helper passed both with the same binaries.
 ## GnuTLS submission
 Development and merge requests are on [GitLab](https://gitlab.com/gnutls/gnutls).
 [`CONTRIBUTING.md` on master](https://gitlab.com/gnutls/gnutls/-/blob/master/CONTRIBUTING.md)
 was read directly for this review. It requires the contributor's DCO
 `Signed-off-by`, successful and failure test coverage, consistent coding
 style, and adequate documentation; GitLab CI runs for merge requests.
 Its commenting guidance asks for comments explaining non-obvious behavior
 or protocol expectations. It does not prescribe an additional special
 test-suite comment. The patch now explains its IPv4 binding assumption
 next to the probe.
 The submission will contain the shell patch, without the Python verifier
 or a new Python test dependency. The existing `serv-udp.sh` supplies the
 functional success check. Running it through the six-second startup
 wrapper supplies a reproducible regression case: it fails before the fix
 and passes after it. The local verifier was used to validate socket-output
 parsing and the helper's success, process-exit, skip-cleanup, and timeout
 branches. Those branch checks are local evidence, not new automated
 coverage in the upstream suite; the MR must state that distinction.
 No dedicated unit-test harness for these shell helpers was found in the
 3.8.13 tests inspected. That does not establish that Python cannot be used
 upstream; keeping this submission dependency-free is a scope choice. Use
 the existing test and before/after reproduction as the submission's
 coverage argument, retaining the platform limitations above. Apply the
 patch in an upstream checkout and include those results with the
 contributor's own sign-off. No MR or sign-off has been created.
 ## Local NixOS integration and build results
 `overlays/default.nix` imports the `overlays/patches` overlay, which loads
 `gnutls/default.nix` to apply the patch and keep `serv-udp.sh` enabled.
 The patch itself has no Nix dependencies and applies to 3.8.13 and GitLab
 master without fuzz.
 The final patch was rebuilt with:
 ```sh
 nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.gnutls
 ```
 That x86-64-v3 build passed: 927 tests, 796 passes, 131 existing skips,
 zero failures/errors, and `PASS: serv-udp.sh`. The patch bytes in the built
 derivation were compared with the repository artifact; both have SHA-256
 `59013d47fd446f2dd065012a2259ccc1898fedc8a053a630e13efa0076368760`.
 All seven local checks passed, including skip cleanup and exactly 90
 probes with 89 sleeps on timeout. The six-second before/after reproduction
 was also repeated successfully with the final helper.
@@ -0,0 +1,6 @@
 { gnutls }:
 gnutls.overrideAttrs (old: {
  # Keep the UDP handshake test enabled on loaded builders by waiting for
  # the server to bind its socket. Kept as a standalone patch for upstream.
  patches = (old.patches or [ ]) ++ [ ./udp-server-readiness.patch ];
 })
@@ -0,0 +1,70 @@
 Subject: [PATCH] tests: wait for the UDP server socket before connecting
 A fixed four-second sleep does not guarantee that gnutls-serv has bound
 its UDP socket on a busy builder. Poll the local IPv4 UDP endpoint using
 the existing ss/netstat discovery, with the same retry budget as the TCP
 helper. Fail early if the server exits, and retain the original handshake
 checks in serv-udp.sh.
 Use flags common to ss and BSD/Linux netstat. Match the local endpoint
 and complete port number, excluding TCP, IPv6-only and peer endpoints.
 --- a/tests/scripts/common.sh
 +++ b/tests/scripts/common.sh
@@ -185,10 +185,55 @@
 	fi
 }
 +check_if_udp_port_bound() {
 +	local PORT=$1
 +	have_port_finder
 +	# Use only -an, which is shared by ss and BSD/Linux netstat. UDP has
 +	# no LISTEN state. Match the local IPv4 endpoint, not a peer port or
 +	# a longer port number. serv-udp.sh connects to 127.0.0.1;
 +	# listen_socket() in serv.c binds IPv4 separately and requests
 +	# IPV6_V6ONLY=1 for its IPv6 socket.
 +	$PFCMD -an | awk -v port="$PORT" '
 +		$1 == "udp" || $1 == "udp4" {
 +			# ss includes a state column; netstat does not.
 +			address = ($2 == "UNCONN" || $2 == "ESTAB") ? $5 : $4
 +			if (address ~ ("^[0-9.]+[.:]" port "$") ||
 +			    address == "*." port)
 +				found = 1
 +		}
 +		END { exit !found }
 +	'
 +}
 +
 wait_udp_server() {
 	local PID=$1
 +	local ret
 	trap "test -n \"${PID}\" && kill ${PID};exit 1" 1 15 2
 -	sleep 4
 +	local i=0
 +	# Use the same retry budget as wait_for_port(), but also stop if the
 +	# server exits before binding its socket.
 +	while test $i -lt 90; do
 +		if ! kill -0 "$PID" 2>/dev/null; then
 +			fail "" "UDP server $PID exited before binding port $PORT"
 +		fi
 +		# Contain have_port_finder's exit so a skip also stops the server.
 +		if (check_if_udp_port_bound "$PORT"); then
 +			return 0
 +		else
 +			ret=$?
 +			if test "$ret" = 77; then
 +				kill "$PID" 2>/dev/null || :
 +				wait "$PID" 2>/dev/null || :
 +				exit 77
 +			fi
 +		fi
 +		i=$((i + 1))
 +		if test $i -lt 90; then
 +			echo "try $i: waiting for UDP port $PORT"
 +			sleep 2
 +		fi
 +	done
 +	fail "$PID" "UDP server $PORT did not come up"
 }
 create_testdir() {
@@ -0,0 +1,180 @@
 #!/usr/bin/env python3
 """Exercise patched common.sh without building GnuTLS (Python standard library only).
 Usage: python3 verify-readiness.py /path/to/patched/tests/scripts/common.sh
 Set SHELL to test another shell, and NETSTAT to test a netstat outside PATH.
 """
 # Use unittest so this upstream companion tool needs no pytest installation.
 # ruff: noqa: PT009
 import os
 import shutil
 import socket
 import subprocess
 import sys
 import tempfile
 import time
 import unittest
 from pathlib import Path
 COMMON = str(Path(sys.argv.pop(1)).resolve())
 SHELL = os.environ.get("SHELL", "/bin/sh")
 class ReadinessTests(unittest.TestCase):
    """Check endpoint parsing and the server startup lifecycle."""
    def setUp(self) -> None:
        """Create a socket-listing fixture for each check."""
        self.tmp = tempfile.TemporaryDirectory()
        self.addCleanup(self.tmp.cleanup)
        self.root = Path(self.tmp.name)
        self.fixture = self.root / "sockets"
        self.fixture.write_text("")
        self.finder = self.root / "port-finder"
        self.finder.write_text('#!/bin/sh\ncat "$SOCKET_FIXTURE"\n')
        self.finder.chmod(0o755)
    def run_shell(self, body: str, **env: str) -> subprocess.CompletedProcess[str]:
        """Source the actual helper and run a shell scenario."""
        return subprocess.run(
            [SHELL, "-c", '. "$COMMON"\n' + body],
            env={
                **os.environ,
                "COMMON": COMMON,
                "SOCKET_FIXTURE": str(self.fixture),
                "PFCMD": str(self.finder),
                "PORT": "12345",
                **env,
            },
            capture_output=True,
            text=True,
            timeout=20,
            check=False,
        )
    def test_socket_formats_and_false_matches(self) -> None:
        """Accept IPv4 UDP local endpoints and reject unrelated sockets."""
        cases = [
            ("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*", True),
            ("udp UNCONN 0 0 127.0.0.1:12345 0.0.0.0:*", True),
            ("udp 0 0 0.0.0.0:12345 0.0.0.0:*", True),
            ("udp4 0 0 *.12345 *.*", True),
            ("udp 0 0 127.0.0.1.12345 *.*", True),
            ("udp 0 0 *.12345 *.*", True),
            ("udp UNCONN 0 0 0.0.0.0:123456 0.0.0.0:*", False),
            ("udp 0 0 0.0.0.0:123456 0.0.0.0:*", False),
            ("udp ESTAB 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
            ("udp 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
            ("tcp LISTEN 0 128 0.0.0.0:12345 0.0.0.0:*", False),
            ("tcp 0 0 0.0.0.0:12345 0.0.0.0:* LISTEN", False),
            ("udp UNCONN 0 0 [::]:12345 [::]:*", False),
            ("udp UNCONN 0 0 *:12345 *:*", False),
            ("udp6 0 0 :::12345 :::*", False),
            ("udp6 0 0 *.12345 *.*", False),
            ("", False),
        ]
        for row, ready in cases:
            with self.subTest(row=row):
                self.fixture.write_text(row + "\n")
                result = self.run_shell('check_if_udp_port_bound "$PORT"')
                self.assertEqual(result.returncode, 0 if ready else 1, result.stderr)
    def test_exited_server_fails_immediately(self) -> None:
        """Fail without sleeping when the server has already exited."""
        result = self.run_shell(
            'true &\npid=$!\nwait "$pid"\nsleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$pid"'
        )
        self.assertEqual(result.returncode, 1)
        self.assertIn("exited before binding", result.stderr)
        self.assertNotIn("unexpected sleep", result.stderr)
    def test_ready_socket_does_not_sleep(self) -> None:
        """Check readiness before the first sleep."""
        self.fixture.write_text("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*\n")
        result = self.run_shell('sleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$$"')
        self.assertEqual(result.returncode, 0, result.stderr)
        self.assertNotIn("unexpected sleep", result.stderr)
    def test_missing_port_finders_skip(self) -> None:
        """Skip and stop the live server when no finder is available."""
        # have_port_finder also tries these paths independently of PATH.
        if any(os.access(f"{directory}/ss", os.X_OK) for directory in ("/sbin", "/usr/sbin", "/usr/local/sbin")):
            self.skipTest("an absolute ss path cannot be hidden by this PATH-only fixture")
        with subprocess.Popen(["sleep", "60"]) as server:
            try:
                result = self.run_shell(
                    'unset PFCMD\nPATH=/nonexistent\nwait_udp_server "$SERVER_PID"',
                    SERVER_PID=str(server.pid),
                )
                self.assertEqual(result.returncode, 77)
                self.assertIn("neither ss nor netstat found", result.stderr)
                server.wait(timeout=3)
                self.assertLess(server.returncode, 0)
            finally:
                if server.poll() is None:
                    server.kill()
    def test_timeout_is_bounded_and_cleans_up(self) -> None:
        """Stop polling after the retry budget and terminate the server."""
        # Only accelerate the polling delay; keep a real live server process.
        self.finder.write_text('#!/bin/sh\necho probe >&2\ncat "$SOCKET_FIXTURE"\n')
        with subprocess.Popen(["sleep", "60"]) as server:
            try:
                result = self.run_shell(
                    'sleep() { echo polling-sleep; }\nwait_udp_server "$SERVER_PID"',
                    SERVER_PID=str(server.pid),
                )
                self.assertEqual(result.returncode, 1)
                self.assertIn("did not come up", result.stderr)
                self.assertEqual(result.stderr.count("probe\n"), 90)
                self.assertEqual(result.stdout.count("polling-sleep"), 89)
                server.wait(timeout=3)
                self.assertLess(server.returncode, 0)
            finally:
                if server.poll() is None:
                    server.kill()
    def test_server_exits_while_waiting(self) -> None:
        """Detect a startup failure that happens after polling begins."""
        result = self.run_shell('sleep 1 &\npid=$!\nwait_udp_server "$pid"')
        self.assertEqual(result.returncode, 1)
        self.assertIn("exited before binding", result.stderr)
        self.assertIn("waiting for UDP port", result.stdout)
    def test_real_socket_delayed_beyond_four_seconds(self) -> None:
        """Wait for a real delayed bind with each installed port finder."""
        finders = [shutil.which("ss"), os.environ.get("NETSTAT") or shutil.which("netstat")]
        finders = [finder for finder in finders if finder]
        if not finders:
            self.skipTest("neither ss nor netstat available")
        for finder in finders:
            with self.subTest(finder=finder):
                with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
                    sock.bind(("127.0.0.1", 0))
                    port = sock.getsockname()[1]
                code = (
                    "import socket,time,sys; time.sleep(6); "
                    "s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); "
                    "s.bind(('127.0.0.1',int(sys.argv[1]))); time.sleep(30)"
                )
                with subprocess.Popen([sys.executable, "-c", code, str(port)]) as server:
                    try:
                        started = time.monotonic()
                        result = self.run_shell(
                            'wait_udp_server "$SERVER_PID"',
                            SERVER_PID=str(server.pid),
                            PORT=str(port),
                            PFCMD=finder,
                        )
                        self.assertEqual(result.returncode, 0, result.stderr)
                        self.assertGreaterEqual(time.monotonic() - started, 6)
                        self.assertIsNone(server.poll())
                    finally:
                        server.terminate()
                        server.wait(timeout=3)
 if __name__ == "__main__":
    unittest.main()
@@ -0,0 +1,66 @@
 # Prometheus complete test parsing
 CodeMirror gives editor-state creation a 20 ms synchronous parsing budget.
 The shared `createEditorState()` test helper can therefore return an
 incomplete syntax tree when the process is descheduled. The completion and
 vector-matching tests immediately inspect that tree.
 ## Scope and behavior
 `complete-test-parsing.patch` changes only
 `module/codemirror-promql/src/test/utils-test.ts` inside `web/ui`. It completes
 the small test expression with `ensureSyntaxTree(..., Infinity)` and publishes
 the completed parse through an empty transaction so `syntaxTree(state)` sees
 it. Failure to obtain a tree raises an error.
 The original assertions remain enabled, including `autocomplete topk params 2`
 and `foo * on(test,blub) bar`. The unlimited budget applies to the test helper;
 production editor parsing budgets are unchanged.
 ## Reproduction and focused checks
 Use a disposable Prometheus 3.14.0 checkout. The patch root is `web/ui`, matching
 the Nix assets derivation. From this directory:
 ```sh
 patch --fuzz=0 -d /path/to/prometheus/web/ui -p1 < complete-test-parsing.patch
 cd /path/to/prometheus/web/ui
 pnpm install --frozen-lockfile
 pnpm --filter @prometheus-io/lezer-promql build
 pnpm --filter @prometheus-io/codemirror-promql test
 ```
 To force the scheduling condition, temporarily append this clock to
 `module/codemirror-promql/setupJest.cjs` in the disposable checkout:
 ```js
 let parseClock = 0;
 Date.now = () => (parseClock += 25);
 ```
 Each clock read crosses the editor's initial parsing budget. Against the
 original helper, the hybrid and vector suites have 186 failures, including
 both locally excluded cases. With the patch, all 386 CodeMirror tests pass
 under that same clock. Remove the injected clock before normal builds.
 ## Upstream status
 This is a standalone test-helper patch for Prometheus 3.14.0. No upstream
 submission was made during this work. Recheck the helper when updating
 Prometheus or CodeMirror, including how an ensured parse becomes visible
 through the editor state.
 ## Local NixOS integration and build results
 [`../default.nix`](../default.nix) loads `default.nix`, which patches the
 separate assets derivation. It updates both `passthru.assets` and the main
 Prometheus build's reference to those assets. From the repository root:
 ```sh
 nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.prometheus.assets
 ```
 The full x86-64-v3 assets build passed with the normal clock, including the
 CodeMirror and UI suites. Host-flake evaluation confirmed that the main
 Prometheus derivation refers to these patched assets. The Go server package
 was not rebuilt for this test-helper change.
@@ -0,0 +1,36 @@
 Subject: [PATCH] tests: finish parsing before inspecting editor state
 EditorState creation has a 20 ms parsing budget. A descheduled test can
 therefore observe an incomplete tree. Finish these small test documents
 without an interactive deadline and publish the result with a transaction.
 Keep the original completion and vector-matching assertions enabled.
 --- a/module/codemirror-promql/src/test/utils-test.ts
 +++ b/module/codemirror-promql/src/test/utils-test.ts
@@ -13,7 +13,7 @@
 import { parser } from '@prometheus-io/lezer-promql';
 import { EditorState } from '@codemirror/state';
 -import { LRLanguage } from '@codemirror/language';
 +import { ensureSyntaxTree, LRLanguage } from '@codemirror/language';
 import nock from 'nock';
 import path from 'path';
 import { fileURLToPath } from 'url';
@@ -23,10 +23,16 @@
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 export function createEditorState(expr: string): EditorState {
 -  return EditorState.create({
 +  const state = EditorState.create({
     doc: expr,
     extensions: lightPromQLSyntax,
   });
 +  // These tests need a complete tree, independent of the editor's time budget.
 +  if (!ensureSyntaxTree(state, state.doc.length, Infinity)) {
 +    throw new Error('Unable to parse the test expression');
 +  }
 +  // Publish the completed parse so syntaxTree(state) sees it too.
 +  return state.update({}).state;
 }
 export function mockPrometheusServer(): void {
@@ -0,0 +1,17 @@
 { prometheus }:
 prometheus.overrideAttrs (
  old:
  let
    assets = old.passthru.assets.overrideAttrs (assetsOld: {
      patches = (assetsOld.patches or [ ]) ++ [ ./complete-test-parsing.patch ];
    });
  in
  {
    postPatch = builtins.replaceStrings [ "${old.passthru.assets}" ] [ "${assets}" ] (
      builtins.unsafeDiscardStringContext old.postPatch
    );
    passthru = old.passthru // {
      inherit assets;
    };
  }
 )
@@ -0,0 +1,79 @@
 # pytest-xdist test fixes
 With two workers and a restart limit of three, the fourth worker crash
 requests shutdown while another test can still be running. That test may
 also crash. The original queued-work test requires exactly four failures,
 even though five failures can occur without exceeding the replacement limit.
 ## Scope and behavior
 `concurrent-worker-crashes.patch` changes the assertions in
 `TestNodeFailure.test_max_worker_restart_tests_queued` in
 `testing/acceptance_test.py`. It requires exactly three replacements, four or
 five failed tests, the failed-tests exit status, the limit message, and no
 internal error. It retains the two-worker workload and ten queued tests.
 `worker-startup-timeout.patch` changes the remote-test helper's event timeout
 from 10 to 60 seconds so loaded builders have time to start workers. The
 helper returns immediately when an event arrives and still has a bounded wait.
 The existing nixpkgs pytest-9 compatibility patches remain in place.
 Production scheduling and worker-restart behavior are unchanged.
 ## Reproduction and focused checks
 Use a disposable pytest-xdist 3.8.0 checkout with its test dependencies and
 the nixpkgs pytest-9 compatibility patches where required. From this directory:
 ```sh
 patch --fuzz=0 -d /path/to/pytest-xdist -p1 < concurrent-worker-crashes.patch
 patch --fuzz=0 -d /path/to/pytest-xdist -p1 < worker-startup-timeout.patch
 cd /path/to/pytest-xdist
 python -m pytest testing/acceptance_test.py \
  -k test_max_worker_restart_tests_queued -q
 python -m pytest testing/test_remote.py -q
 ```
 Twenty unmodified runs passed during the review. To force the failing
 schedule, modify the generated crashing test in a disposable checkout to
 accept `worker_id`: make `gw3` wait for a marker created by `gw4`, and make
 `gw4` pause 0.1 seconds after creating the marker. Then both have in-flight
 tests when shutdown starts. Bound the marker wait so a reproduction failure
 cannot hang the suite. The original assertion fails on five reported
 failures; the patched test passes.
 ## Worker startup and outer concurrency
 [`default.nix`](default.nix) runs the outer suite with one worker to limit
 nested process pools. This is a Nix test-runner setting; the source timeout
 change lives in [`worker-startup-timeout.patch`](worker-startup-timeout.patch).
 A separate reproduction inserts an 11-second `pytest_sessionstart` delay
 into the child created by `test_basic_collect_and_runtests` in
 `testing/test_remote.py`. The original 10-second channel wait fails; the
 60-second wait passes. This bounds waits for test worker events, including
 startup, rather than changing a product deadline.
 ## Upstream status
 These are standalone test patches for pytest-xdist 3.8.0. No upstream submission
 was made during this work. Recheck the allowed in-flight failures, replacement
 count, and remote-test wait when updating the scheduler or worker behavior.
 ## Local NixOS integration and build results
 [`../default.nix`](../default.nix) loads `default.nix` through
 `pythonPackagesExtensions`. From the repository root:
 ```sh
 nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.python314Packages.pytest-xdist
 ```
 After consolidating the settings in this directory, the full x86-64-v3 package
 build passed 185 tests, with 6 existing skips and 10 expected failures. Nix
 evaluation confirmed the same outer-worker limit and preserved existing
 patches, with the timeout now applied as a source patch.
 The earlier forced concurrent-crash and delayed-startup reproductions passed
 after their fixes; the focused crash test also passed after formatting its
 assertion.
@@ -0,0 +1,29 @@
 Subject: [PATCH] tests: count replacements when checking the worker restart limit
 With two workers, another in-flight test may crash after the fourth
 crash requests shutdown. Either four or five failed tests is valid.
 Require exactly three replacements and the failed-tests exit status,
 while preserving the queued-work and no-internal-error assertions.
 --- a/testing/acceptance_test.py
 +++ b/testing/acceptance_test.py
@@ -1011,9 +1011,18 @@
                 "worker*crashed while running*",
                 "worker*crashed while running*",
                 "* xdist: maximum crashed workers reached: 3 *",
 -                "* 4 failed in *",
             ]
         )
 +        # A second in-flight test may crash after shutdown is requested.
 +        # The restart limit constrains replacements, not concurrent failures.
 +        replacements = sum(
 +            line.startswith("replacing crashed worker ") for line in res.stdout.lines
 +        )
 +        assert replacements == 3
 +        failed = res.parseoutcomes()["failed"]
 +        assert failed in (4, 5)
 +        res.assert_outcomes(failed=failed)
 +        assert res.ret == pytest.ExitCode.TESTS_FAILED
         assert "INTERNALERROR" not in res.stdout.str()
     def test_max_worker_restart_die(self, pytester: pytest.Pytester) -> None:
@@ -0,0 +1,12 @@
 { pytest-xdist }:
 pytest-xdist.overridePythonAttrs (old: {
  patches = (old.patches or [ ]) ++ [
    ./concurrent-worker-crashes.patch
    ./worker-startup-timeout.patch
  ];
  # The suite exercises its own worker pools. Limit the outer suite to one worker.
  preCheck = builtins.replaceStrings [ "--numprocesses=$NIX_BUILD_CORES" ] [ "--numprocesses=1" ] (
    old.preCheck or ""
  );
 })
@@ -0,0 +1,19 @@
 Subject: [PATCH] tests: allow more time for remote worker events
 Worker startup can exceed ten seconds on heavily loaded builders. Allow
 the remote-test helper to wait up to sixty seconds for worker events.
 The wait still returns as soon as an event arrives and remains bounded.
 Production worker timeouts and test assertions are unchanged.
 --- a/testing/test_remote.py
 +++ b/testing/test_remote.py
@@ -17,7 +17,8 @@
 from xdist.workermanage import WorkerController
 -WAIT_TIMEOUT = 10.0
 +# Allow worker events extra time on heavily loaded builders.
 +WAIT_TIMEOUT = 60.0
 def check_marshallable(d: object) -> None:
@@ -0,0 +1,70 @@
 # SciPy STFT test tolerances
 The x86-64-v3 build can produce small floating-point residuals in inverse-STFT
 comparisons and scaling round trips. The original bounds reject these results,
 including residuals around `4e-17` where a round trip expects zero for a signal
 with amplitude 2.
 ## Scope and behavior
 `stft-test-tolerances.patch` changes only the signal tests:
 - The inverse-STFT comparison in `_scipy_spectral_test_shim.py` uses
  `max(1e-7, 2 * np.finfo(x.dtype).eps)` as its relative tolerance. Float64
  keeps the original bound, and the existing i686 override remains.
 - Three scaling round trips in `test_spectral.py` gain an absolute tolerance
  of one epsilon for the input dtype, allowing small residuals near zero.
 The tests remain enabled, and the production STFT implementation is unchanged.
 ## Reproduction and focused checks
 From this directory, apply the patch to a disposable SciPy 1.18.0 checkout:
 ```sh
 patch --fuzz=0 -d /path/to/scipy -p1 < stft-test-tolerances.patch
 ```
 Build and install that tree with SciPy's test dependencies. From outside the
 source directory, run the installed tests:
 ```sh
 python -m pytest --pyargs scipy.signal.tests.test_spectral \
  -k 'roundtrip_float32 or roundtrip_scaling' -q
 ```
 Use the same compiler flags and numerical libraries for before/after runs.
 The earlier reproduction called `TestSTFT.test_roundtrip_float32` and
 `TestSTFT.test_roundtrip_scaling` against the x86-64-v3 libraries, then loaded
 patched copies of the test modules. Both failed with the original bounds
 and passed with the adjusted bounds.
 ## Upstream status
 [SciPy issue #25488](https://github.com/scipy/scipy/issues/25488) records
 related test failures with architecture-specific compiler flags. It is
 context for the local tolerance repair; this exact patch has not been
 submitted upstream during this work.
 ## Local NixOS integration and build results
 [`../default.nix`](../default.nix) loads [`default.nix`](default.nix) through
 `pythonPackagesExtensions`, preserving the package's existing patches.
 The override also covers SciPy used to test other Python dependencies,
 including pgvector in portal's shared Python environment.
 From the repository root:
 ```sh
 nix build --no-link -L .#nixosConfigurations.portal-1.pkgs.python314Packages.scipy
 ```
 The original remote build of patched SciPy 1.18.0 passed 87,723 tests, with
 8,342 skips, 300 expected failures, and 22 unexpected passes. The patch and
 override have been restored byte-for-byte from commit `24cbf74f`; those counts
 describe the earlier full build.
 Restoration checks confirmed that the patch applies to the pinned source
 without fuzz, portal's evaluated SciPy retains its existing patch and install
 checks, and pgvector uses the patched SciPy. A full package or system rebuild
 was not repeated for this restoration.
@@ -0,0 +1,5 @@
 { scipy }:
 scipy.overridePythonAttrs (old: {
  # Keep the STFT tests enabled with tolerances for x86-64-v3 rounding.
  patches = (old.patches or [ ]) ++ [ ./stft-test-tolerances.patch ];
 })
@@ -0,0 +1,51 @@
 Subject: [PATCH] signal: allow floating-point rounding in STFT tests
 Keep the STFT tests enabled for x86-64-v3 builds. Allow two float32
 epsilons of relative error when comparing inverse-STFT implementations;
 float64 and the existing i686 override remain unchanged. Allow one
 float64 epsilon of absolute error in all three scaling round trips,
 which otherwise require exact zeros (observed residual: 4e-17 for a
 signal with amplitude 2).
 Upstream issue: https://github.com/scipy/scipy/issues/25488
 --- a/scipy/signal/tests/_scipy_spectral_test_shim.py
 +++ b/scipy/signal/tests/_scipy_spectral_test_shim.py
@@ -294,7 +294,7 @@
     # Adapted tolerances to account for resolution loss:
     atol = np.finfo(x.dtype).resolution*2  # instead of default atol = 0
 -    rtol = 1e-7  # default for np.allclose()
 +    rtol = max(1e-7, 2 * np.finfo(x.dtype).eps)
     # Relax atol on 32-Bit platforms a bit to pass CI tests.
     #  - Not clear why there are discrepancies (in the FFT maybe?)
 --- a/scipy/signal/tests/test_spectral.py
 +++ b/scipy/signal/tests/test_spectral.py
@@ -2044,7 +2044,7 @@
         # Test round trip:
         x1 = istft(Zs, boundary=True, scaling='spectrum')[1]
 -        assert_allclose(x1, x)
 +        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
         # For a Hann-windowed 256 sample length FFT, we expect a peak at
         # frequency 64 (since it is 1/4 the length of X) with a height of 1
@@ -2074,7 +2074,7 @@
         # Test round trip:
         x1 = istft(Zp, input_onesided=False, boundary=True, scaling='psd')[1]
 -        assert_allclose(x1, x)
 +        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
         # The power of the one-sided psd-scaled STFT can be determined
         # analogously (note that the two sides are not of equal shape):
@@ -2094,7 +2094,7 @@
         # Test round trip:
         x1 = istft(Zp0, input_onesided=True, boundary=True, scaling='psd')[1]
 -        assert_allclose(x1, x)
 +        assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
 class TestSampledSpectralRepresentations:
@@ -0,0 +1,58 @@
 # Sentry SDK thread-metadata test isolation
 The fallback tests globally mock `threading.current_thread` while a worker
 is running. Python 3.14's `Thread.join()` also calls that function. A one-use
 mock can therefore be consumed by the wrong caller or raise `StopIteration`
 when the main thread joins the worker.
 ## Scope and behavior
 `isolate-threading-mocks.patch` changes three neighboring thread-metadata
 tests in `tests/test_utils.py`, including the formerly excluded
 `test_get_current_thread_meta_main_thread`.
 Each test replaces only `sentry_sdk.utils.threading`, wraps the real module
 for unmocked operations, and sets the SDK lookup's return value. The real
 `Thread.join()` continues using Python's unmodified `threading` module.
 The fallback-result assertions remain; SDK production code is unchanged.
 ## Reproduction and focused checks
 Use a disposable Sentry SDK 2.66.0 checkout and its Python test dependencies.
 From this directory:
 ```sh
 patch --fuzz=0 -d /path/to/sentry-python -p1 < isolate-threading-mocks.patch
 cd /path/to/sentry-python
 python -m pytest tests/test_utils.py -k get_current_thread_meta -q
 ```
 To reproduce the race, hold the worker inside its mock just after
 `get_current_thread_meta()` returns, signal that point to the main thread,
 and call `Thread.join()` before releasing the worker. Use an independent
 bounded release so the patched join can finish. The original test raises
 `StopIteration` in `join`; the patched test passes under the same schedule.
 Perform this scheduling instrumentation only in a disposable checkout.
 ## Upstream status
 This is a standalone test patch for Sentry SDK 2.66.0. No upstream submission
 was made during this work. Recheck mock isolation and Python threading
 behavior when upgrading the SDK or interpreter.
 ## Local NixOS integration and build results
 [`../default.nix`](../default.nix) loads `default.nix` through
 `pythonPackagesExtensions`. From the repository root:
 ```sh
 nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.python314Packages.sentry-sdk
 ```
 The patched package passed 2,356 tests with 116 existing skips on Python
 3.14.7. The controlled join reproduction failed before the fix and passed
 after it.
 That package build used the preceding dependency set with this patch to avoid
 unrelated rebuilds after pytest-xdist changed. The integrated host derivation
 was evaluated; a complete NixOS rebuild was not performed.
@@ -0,0 +1,4 @@
 { sentry-sdk }:
 sentry-sdk.overridePythonAttrs (old: {
  patches = (old.patches or [ ]) ++ [ ./isolate-threading-mocks.patch ];
 })
@@ -0,0 +1,41 @@
 Subject: [PATCH] tests: isolate SDK thread lookup mocks from Python threading
 Thread.join also calls threading.current_thread on Python 3.14. A global
 single-use side effect can be consumed by join instead of the SDK, or
 raise StopIteration in join after the SDK consumes it. Patch the SDK's
 module binding and delegate unmocked operations to the real module.
 Apply the same isolation to the adjacent invalid-thread fallback tests.
 --- a/tests/test_utils.py
 +++ b/tests/test_utils.py
@@ -914,7 +914,8 @@
     results = Queue(maxsize=1)
     def target():
 -        with mock.patch("threading.current_thread", side_effect=["fake thread"]):
 +        with mock.patch("sentry_sdk.utils.threading", wraps=threading) as sdk_threading:
 +            sdk_threading.current_thread.return_value = "fake thread"
             results.put(get_current_thread_meta())
     thread = threading.Thread(target=target)
@@ -930,7 +931,9 @@
     def target():
         # mock that somehow the current thread doesn't exist
 -        with mock.patch("threading.current_thread", side_effect=[None]):
 +        # Keep the real threading module intact for concurrent Thread.join calls.
 +        with mock.patch("sentry_sdk.utils.threading", wraps=threading) as sdk_threading:
 +            sdk_threading.current_thread.return_value = None
             results.put(get_current_thread_meta())
     main_thread = threading.main_thread()
@@ -945,7 +948,8 @@
     results = Queue(maxsize=1)
     def target():
 -        with mock.patch("threading.current_thread", return_value="fake thread"):
 +        with mock.patch("sentry_sdk.utils.threading", wraps=threading) as sdk_threading:
 +            sdk_threading.current_thread.return_value = "fake thread"
             results.put(get_current_thread_meta())
     main_thread = threading.main_thread()
@@ -0,0 +1,81 @@
 [project]
 name = "system_tools"
 version = "0.1.0"
 description = ""
 authors = [{ name = "Richie Cahill", email = "richie@tmmworkshop.com" }]
 requires-python = "~=3.14.0"
 readme = "README.md"
 license = "MIT"
 [tool.ruff]
 target-version = "py314"
 line-length = 120
 lint.select = ["ALL"]
 lint.ignore = [
    "G004",   # (PERM) This is a performers nit
    "COM812", # (TEMP) conflicts when used with the formatter
    "ISC001", # (TEMP) conflicts when used with the formatter
    "S603",   # (PERM) This is known to cause a false positive
    "S607",   # (PERM) This is becoming a consistent annoyance
    "CPY001",  # (PERM) I don't include the license in every file
 ]
 [tool.ruff.lint.per-file-ignores]
 "tests/**" = [
    "ANN",     # (perm) type annotations not needed in tests
    "D",       # (perm) docstrings not needed in tests
    "PLR2004", # (perm) magic values are fine in test assertions
    "S101",    # (perm) pytest needs asserts
 ]
 "python/stuff/**" = [
    "T201", # (perm) I don't care about print statements dir
 ]
 "python/testing/**" = [
    "T201",   # (perm) I don't care about print statements dir
    "ERA001", # (perm) I don't care about print statements dir
 ]
 "python/splendor/**" = [
    "S311",    # (perm) there is no security issue here
    "T201",    # (perm) I don't care about print statements dir
    "PLR2004", # (temps) need to think about this
 ]
 "python/orm/**" = [
    "TC003", # (perm) this creates issues because sqlalchemy uses these at runtime
 ]
 "python/congress_tracker/**" = [
    "TC003", # (perm) this creates issues because sqlalchemy uses these at runtime
 ]
 "python/alembic/**" = [
    "INP001", # (perm) this creates LSP issues for alembic
 ]
 [tool.ruff.lint.pydocstyle]
 convention = "google"
 [tool.ruff.lint.flake8-builtins]
 builtins-ignorelist = ["id"]
 [tool.ruff.lint.pylint]
 max-args = 9
 [tool.coverage.run]
 source = ["system_tools"]
 [tool.coverage.report]
 exclude_lines = [
    "pragma: no cover",
    "if TYPE_CHECKING:",
    "raise NotImplementedError",
    "if __name__ == \"__main__\":",
 ]
 [tool.pytest.ini_options]
 addopts = "-n auto -ra --ignore=tests/ebook_search"
 asyncio_mode = "auto"
 testpaths = ["tests"]
 # --cov=system_tools --cov-report=term-missing --cov-report=xml --cov-report=html --cov-branch
@@ -0,0 +1 @@
 """Server Tools."""
@@ -0,0 +1,122 @@
 """Alembic."""
 from __future__ import annotations
 import logging
 import sys
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Literal
 from alembic import context
 from alembic.script import write_hooks
 from sqlalchemy.schema import CreateSchema
 from python.common import bash_wrapper
 from python.orm.common import get_postgres_engine
 if TYPE_CHECKING:
    from collections.abc import MutableMapping
    from sqlalchemy.orm import DeclarativeBase
 config = context.config
 base_class: type[DeclarativeBase] = config.attributes.get("base")
 if base_class is None:
    error = "No base class provided. Use the database CLI to run alembic commands."
    raise RuntimeError(error)
 target_metadata = base_class.metadata
 logging.basicConfig(
    level="DEBUG",
    datefmt="%Y-%m-%dT%H:%M:%S%z",
    format="%(asctime)s %(levelname)s %(filename)s:%(lineno)d - %(message)s",
    handlers=[logging.StreamHandler(sys.stdout)],
 )
@write_hooks.register("dynamic_schema")
 def dynamic_schema(filename: str, _options: dict[Any, Any]) -> None:
    """Dynamic schema."""
    original_file = Path(filename).read_text()
    schema_name = base_class.schema_name
    dynamic_schema_file_part1 = original_file.replace(f"schema='{schema_name}'", "schema=schema")
    dynamic_schema_file = dynamic_schema_file_part1.replace(f"'{schema_name}.", "f'{schema}.")
    Path(filename).write_text(dynamic_schema_file)
@write_hooks.register("import_postgresql")
 def import_postgresql(filename: str, _options: dict[Any, Any]) -> None:
    """Add postgresql dialect import when postgresql types are used."""
    content = Path(filename).read_text()
    if "postgresql." in content and "from sqlalchemy.dialects import postgresql" not in content:
        content = content.replace(
            "import sqlalchemy as sa\n",
            "import sqlalchemy as sa\nfrom sqlalchemy.dialects import postgresql\n",
        )
        Path(filename).write_text(content)
@write_hooks.register("ruff")
 def ruff_check_and_format(filename: str, _options: dict[Any, Any]) -> None:
    """Docstring for ruff_check_and_format."""
    bash_wrapper(f"ruff check --fix {filename}")
    bash_wrapper(f"ruff format {filename}")
 def include_name(
    name: str | None,
    type_: Literal["schema", "table", "column", "index", "unique_constraint", "foreign_key_constraint"],
    _parent_names: MutableMapping[Literal["schema_name", "table_name", "schema_qualified_table_name"], str | None],
 ) -> bool:
    """Filter tables to be included in the migration.
    Args:
        name (str): The name of the table.
        type_ (str): The type of the table.
        _parent_names (MutableMapping): The names of the parent tables.
    Returns:
        bool: True if the table should be included, False otherwise.
    """
    if type_ == "schema":
        # allows a database with multiple schemas to have separate alembic revisions
        return name == target_metadata.schema
    return True
 def run_migrations_online() -> None:
    """Run migrations in 'online' mode.
    In this scenario we need to create an Engine
    and associate a connection with the context.
    """
    env_prefix = config.attributes.get("env_prefix", "POSTGRES")
    connectable = get_postgres_engine(name=env_prefix)
    with connectable.connect() as connection:
        schema = base_class.schema_name
        if not connectable.dialect.has_schema(connection, schema):
            answer = input(f"Schema {schema!r} does not exist. Create it? [y/N] ")
            if answer.lower() != "y":
                error = f"Schema {schema!r} does not exist. Exiting."
                raise SystemExit(error)
            connection.execute(CreateSchema(schema))
            connection.commit()
        context.configure(
            connection=connection,
            target_metadata=target_metadata,
            include_schemas=True,
            version_table_schema=schema,
            include_name=include_name,
        )
        with context.begin_transaction():
            context.run_migrations()
        connection.commit()
 run_migrations_online()
@@ -0,0 +1,113 @@
 """created contact api.
 Revision ID: edd7dd61a3d2
 Revises:
 Create Date: 2026-01-11 15:45:59.909266
 """
 from __future__ import annotations
 from typing import TYPE_CHECKING
 import sqlalchemy as sa
 from alembic import op
 from python.orm import RichieBase
 if TYPE_CHECKING:
    from collections.abc import Sequence
 # revision identifiers, used by Alembic.
 revision: str = "edd7dd61a3d2"
 down_revision: str | None = None
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 schema = RichieBase.schema_name
 def upgrade() -> None:
    """Upgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "contact",
        sa.Column("name", sa.String(), nullable=False),
        sa.Column("age", sa.Integer(), nullable=True),
        sa.Column("bio", sa.String(), nullable=True),
        sa.Column("current_job", sa.String(), nullable=True),
        sa.Column("gender", sa.String(), nullable=True),
        sa.Column("goals", sa.String(), nullable=True),
        sa.Column("legal_name", sa.String(), nullable=True),
        sa.Column("profile_pic", sa.String(), nullable=True),
        sa.Column("safe_conversation_starters", sa.String(), nullable=True),
        sa.Column("self_sufficiency_score", sa.Integer(), nullable=True),
        sa.Column("social_structure_style", sa.String(), nullable=True),
        sa.Column("ssn", sa.String(), nullable=True),
        sa.Column("suffix", sa.String(), nullable=True),
        sa.Column("timezone", sa.String(), nullable=True),
        sa.Column("topics_to_avoid", sa.String(), nullable=True),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_contact")),
        schema=schema,
    )
    op.create_table(
        "need",
        sa.Column("name", sa.String(), nullable=False),
        sa.Column("description", sa.String(), nullable=True),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_need")),
        schema=schema,
    )
    op.create_table(
        "contact_need",
        sa.Column("contact_id", sa.Integer(), nullable=False),
        sa.Column("need_id", sa.Integer(), nullable=False),
        sa.ForeignKeyConstraint(
            ["contact_id"],
            [f"{schema}.contact.id"],
            name=op.f("fk_contact_need_contact_id_contact"),
            ondelete="CASCADE",
        ),
        sa.ForeignKeyConstraint(
            ["need_id"], [f"{schema}.need.id"], name=op.f("fk_contact_need_need_id_need"), ondelete="CASCADE"
        ),
        sa.PrimaryKeyConstraint("contact_id", "need_id", name=op.f("pk_contact_need")),
        schema=schema,
    )
    op.create_table(
        "contact_relationship",
        sa.Column("contact_id", sa.Integer(), nullable=False),
        sa.Column("related_contact_id", sa.Integer(), nullable=False),
        sa.Column("relationship_type", sa.String(length=100), nullable=False),
        sa.Column("closeness_weight", sa.Integer(), nullable=False),
        sa.ForeignKeyConstraint(
            ["contact_id"],
            [f"{schema}.contact.id"],
            name=op.f("fk_contact_relationship_contact_id_contact"),
            ondelete="CASCADE",
        ),
        sa.ForeignKeyConstraint(
            ["related_contact_id"],
            [f"{schema}.contact.id"],
            name=op.f("fk_contact_relationship_related_contact_id_contact"),
            ondelete="CASCADE",
        ),
        sa.PrimaryKeyConstraint("contact_id", "related_contact_id", name=op.f("pk_contact_relationship")),
        schema=schema,
    )
    # ### end Alembic commands ###
 def downgrade() -> None:
    """Downgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table("contact_relationship", schema=schema)
    op.drop_table("contact_need", schema=schema)
    op.drop_table("need", schema=schema)
    op.drop_table("contact", schema=schema)
    # ### end Alembic commands ###
@@ -0,0 +1,135 @@
 """add congress tracker tables.
 Revision ID: 3f71565e38de
 Revises: edd7dd61a3d2
 Create Date: 2026-02-12 16:36:09.457303
 """
 from __future__ import annotations
 from typing import TYPE_CHECKING
 import sqlalchemy as sa
 from alembic import op
 from python.orm import RichieBase
 if TYPE_CHECKING:
    from collections.abc import Sequence
 # revision identifiers, used by Alembic.
 revision: str = "3f71565e38de"
 down_revision: str | None = "edd7dd61a3d2"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 schema = RichieBase.schema_name
 def upgrade() -> None:
    """Upgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "bill",
        sa.Column("congress", sa.Integer(), nullable=False),
        sa.Column("bill_type", sa.String(), nullable=False),
        sa.Column("number", sa.Integer(), nullable=False),
        sa.Column("title", sa.String(), nullable=True),
        sa.Column("title_short", sa.String(), nullable=True),
        sa.Column("official_title", sa.String(), nullable=True),
        sa.Column("status", sa.String(), nullable=True),
        sa.Column("status_at", sa.Date(), nullable=True),
        sa.Column("sponsor_bioguide_id", sa.String(), nullable=True),
        sa.Column("subjects_top_term", sa.String(), nullable=True),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_bill")),
        sa.UniqueConstraint("congress", "bill_type", "number", name="uq_bill_congress_type_number"),
        schema=schema,
    )
    op.create_index("ix_bill_congress", "bill", ["congress"], unique=False, schema=schema)
    op.create_table(
        "legislator",
        sa.Column("bioguide_id", sa.Text(), nullable=False),
        sa.Column("thomas_id", sa.String(), nullable=True),
        sa.Column("lis_id", sa.String(), nullable=True),
        sa.Column("govtrack_id", sa.Integer(), nullable=True),
        sa.Column("opensecrets_id", sa.String(), nullable=True),
        sa.Column("fec_ids", sa.String(), nullable=True),
        sa.Column("first_name", sa.String(), nullable=False),
        sa.Column("last_name", sa.String(), nullable=False),
        sa.Column("official_full_name", sa.String(), nullable=True),
        sa.Column("nickname", sa.String(), nullable=True),
        sa.Column("birthday", sa.Date(), nullable=True),
        sa.Column("gender", sa.String(), nullable=True),
        sa.Column("current_party", sa.String(), nullable=True),
        sa.Column("current_state", sa.String(), nullable=True),
        sa.Column("current_district", sa.Integer(), nullable=True),
        sa.Column("current_chamber", sa.String(), nullable=True),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_legislator")),
        schema=schema,
    )
    op.create_index(op.f("ix_legislator_bioguide_id"), "legislator", ["bioguide_id"], unique=True, schema=schema)
    op.create_table(
        "vote",
        sa.Column("congress", sa.Integer(), nullable=False),
        sa.Column("chamber", sa.String(), nullable=False),
        sa.Column("session", sa.Integer(), nullable=False),
        sa.Column("number", sa.Integer(), nullable=False),
        sa.Column("vote_type", sa.String(), nullable=True),
        sa.Column("question", sa.String(), nullable=True),
        sa.Column("result", sa.String(), nullable=True),
        sa.Column("result_text", sa.String(), nullable=True),
        sa.Column("vote_date", sa.Date(), nullable=False),
        sa.Column("yea_count", sa.Integer(), nullable=True),
        sa.Column("nay_count", sa.Integer(), nullable=True),
        sa.Column("not_voting_count", sa.Integer(), nullable=True),
        sa.Column("present_count", sa.Integer(), nullable=True),
        sa.Column("bill_id", sa.Integer(), nullable=True),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.ForeignKeyConstraint(["bill_id"], [f"{schema}.bill.id"], name=op.f("fk_vote_bill_id_bill")),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_vote")),
        sa.UniqueConstraint("congress", "chamber", "session", "number", name="uq_vote_congress_chamber_session_number"),
        schema=schema,
    )
    op.create_index("ix_vote_congress_chamber", "vote", ["congress", "chamber"], unique=False, schema=schema)
    op.create_index("ix_vote_date", "vote", ["vote_date"], unique=False, schema=schema)
    op.create_table(
        "vote_record",
        sa.Column("vote_id", sa.Integer(), nullable=False),
        sa.Column("legislator_id", sa.Integer(), nullable=False),
        sa.Column("position", sa.String(), nullable=False),
        sa.ForeignKeyConstraint(
            ["legislator_id"],
            [f"{schema}.legislator.id"],
            name=op.f("fk_vote_record_legislator_id_legislator"),
            ondelete="CASCADE",
        ),
        sa.ForeignKeyConstraint(
            ["vote_id"], [f"{schema}.vote.id"], name=op.f("fk_vote_record_vote_id_vote"), ondelete="CASCADE"
        ),
        sa.PrimaryKeyConstraint("vote_id", "legislator_id", name=op.f("pk_vote_record")),
        schema=schema,
    )
    # ### end Alembic commands ###
 def downgrade() -> None:
    """Downgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table("vote_record", schema=schema)
    op.drop_index("ix_vote_date", table_name="vote", schema=schema)
    op.drop_index("ix_vote_congress_chamber", table_name="vote", schema=schema)
    op.drop_table("vote", schema=schema)
    op.drop_index(op.f("ix_legislator_bioguide_id"), table_name="legislator", schema=schema)
    op.drop_table("legislator", schema=schema)
    op.drop_index("ix_bill_congress", table_name="bill", schema=schema)
    op.drop_table("bill", schema=schema)
    # ### end Alembic commands ###
@@ -0,0 +1,58 @@
 """adding SignalDevice for DeviceRegistry for signal bot.
 Revision ID: 4c410c16e39c
 Revises: 3f71565e38de
 Create Date: 2026-03-09 14:51:24.228976
 """
 from __future__ import annotations
 from typing import TYPE_CHECKING
 import sqlalchemy as sa
 from alembic import op
 from sqlalchemy.dialects import postgresql
 from python.orm import RichieBase
 if TYPE_CHECKING:
    from collections.abc import Sequence
 # revision identifiers, used by Alembic.
 revision: str = "4c410c16e39c"
 down_revision: str | None = "3f71565e38de"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 schema = RichieBase.schema_name
 def upgrade() -> None:
    """Upgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "signal_device",
        sa.Column("phone_number", sa.String(length=50), nullable=False),
        sa.Column("safety_number", sa.String(), nullable=False),
        sa.Column(
            "trust_level",
            postgresql.ENUM("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", schema=schema),
            nullable=False,
        ),
        sa.Column("last_seen", sa.DateTime(timezone=True), nullable=False),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_signal_device")),
        sa.UniqueConstraint("phone_number", name=op.f("uq_signal_device_phone_number")),
        schema=schema,
    )
    # ### end Alembic commands ###
 def downgrade() -> None:
    """Downgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table("signal_device", schema=schema)
    # ### end Alembic commands ###
@@ -0,0 +1,41 @@
 """fixed safety number logic.
 Revision ID: 99fec682516c
 Revises: 4c410c16e39c
 Create Date: 2026-03-09 16:25:25.085806
 """
 from __future__ import annotations
 from typing import TYPE_CHECKING
 import sqlalchemy as sa
 from alembic import op
 from python.orm import RichieBase
 if TYPE_CHECKING:
    from collections.abc import Sequence
 # revision identifiers, used by Alembic.
 revision: str = "99fec682516c"
 down_revision: str | None = "4c410c16e39c"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 schema = RichieBase.schema_name
 def upgrade() -> None:
    """Upgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.alter_column("signal_device", "safety_number", existing_type=sa.VARCHAR(), nullable=True, schema=schema)
    # ### end Alembic commands ###
 def downgrade() -> None:
    """Downgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.alter_column("signal_device", "safety_number", existing_type=sa.VARCHAR(), nullable=False, schema=schema)
    # ### end Alembic commands ###
@@ -0,0 +1,54 @@
 """add dead_letter_message table.
 Revision ID: a1b2c3d4e5f6
 Revises: 99fec682516c
 Create Date: 2026-03-10 12:00:00.000000
 """
 from __future__ import annotations
 from typing import TYPE_CHECKING
 import sqlalchemy as sa
 from alembic import op
 from sqlalchemy.dialects import postgresql
 from python.orm import RichieBase
 if TYPE_CHECKING:
    from collections.abc import Sequence
 # revision identifiers, used by Alembic.
 revision: str = "a1b2c3d4e5f6"
 down_revision: str | None = "99fec682516c"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 schema = RichieBase.schema_name
 def upgrade() -> None:
    """Upgrade."""
    op.create_table(
        "dead_letter_message",
        sa.Column("source", sa.String(), nullable=False),
        sa.Column("message", sa.Text(), nullable=False),
        sa.Column("received_at", sa.DateTime(timezone=True), nullable=False),
        sa.Column(
            "status",
            postgresql.ENUM("UNPROCESSED", "PROCESSED", name="message_status", schema=schema),
            nullable=False,
        ),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_dead_letter_message")),
        schema=schema,
    )
 def downgrade() -> None:
    """Downgrade."""
    op.drop_table("dead_letter_message", schema=schema)
    op.execute(sa.text(f"DROP TYPE IF EXISTS {schema}.message_status"))
@@ -0,0 +1,66 @@
 """adding roles to signal devices.
 Revision ID: 2ef7ba690159
 Revises: a1b2c3d4e5f6
 Create Date: 2026-03-16 19:22:38.020350
 """
 from __future__ import annotations
 from typing import TYPE_CHECKING
 import sqlalchemy as sa
 from alembic import op
 from python.orm import RichieBase
 if TYPE_CHECKING:
    from collections.abc import Sequence
 # revision identifiers, used by Alembic.
 revision: str = "2ef7ba690159"
 down_revision: str | None = "a1b2c3d4e5f6"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 schema = RichieBase.schema_name
 def upgrade() -> None:
    """Upgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "role",
        sa.Column("name", sa.String(length=50), nullable=False),
        sa.Column("id", sa.SmallInteger(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_role")),
        sa.UniqueConstraint("name", name=op.f("uq_role_name")),
        schema=schema,
    )
    op.create_table(
        "device_role",
        sa.Column("device_id", sa.Integer(), nullable=False),
        sa.Column("role_id", sa.SmallInteger(), nullable=False),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.ForeignKeyConstraint(
            ["device_id"], [f"{schema}.signal_device.id"], name=op.f("fk_device_role_device_id_signal_device")
        ),
        sa.ForeignKeyConstraint(["role_id"], [f"{schema}.role.id"], name=op.f("fk_device_role_role_id_role")),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_device_role")),
        sa.UniqueConstraint("device_id", "role_id", name="uq_device_role_device_role"),
        schema=schema,
    )
    # ### end Alembic commands ###
 def downgrade() -> None:
    """Downgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table("device_role", schema=schema)
    op.drop_table("role", schema=schema)
    # ### end Alembic commands ###
@@ -0,0 +1,171 @@
 """seprating signal_bot database.
 Revision ID: 6b275323f435
 Revises: 2ef7ba690159
 Create Date: 2026-03-18 08:34:28.785885
 """
 from __future__ import annotations
 from typing import TYPE_CHECKING
 import sqlalchemy as sa
 from alembic import op
 from sqlalchemy.dialects import postgresql
 from python.orm import RichieBase
 if TYPE_CHECKING:
    from collections.abc import Sequence
 # revision identifiers, used by Alembic.
 revision: str = "6b275323f435"
 down_revision: str | None = "2ef7ba690159"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 schema = RichieBase.schema_name
 def upgrade() -> None:
    """Upgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table("device_role", schema=schema)
    op.drop_table("signal_device", schema=schema)
    op.drop_table("role", schema=schema)
    op.drop_table("dead_letter_message", schema=schema)
    # ### end Alembic commands ###
 def downgrade() -> None:
    """Downgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "dead_letter_message",
        sa.Column("source", sa.VARCHAR(), autoincrement=False, nullable=False),
        sa.Column("message", sa.TEXT(), autoincrement=False, nullable=False),
        sa.Column("received_at", postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False),
        sa.Column(
            "status",
            postgresql.ENUM("UNPROCESSED", "PROCESSED", name="message_status", schema=schema),
            autoincrement=False,
            nullable=False,
        ),
        sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
        sa.Column(
            "created",
            postgresql.TIMESTAMP(timezone=True),
            server_default=sa.text("now()"),
            autoincrement=False,
            nullable=False,
        ),
        sa.Column(
            "updated",
            postgresql.TIMESTAMP(timezone=True),
            server_default=sa.text("now()"),
            autoincrement=False,
            nullable=False,
        ),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_dead_letter_message")),
        schema=schema,
    )
    op.create_table(
        "role",
        sa.Column("name", sa.VARCHAR(length=50), autoincrement=False, nullable=False),
        sa.Column(
            "id",
            sa.SMALLINT(),
            server_default=sa.text(f"nextval('{schema}.role_id_seq'::regclass)"),
            autoincrement=True,
            nullable=False,
        ),
        sa.Column(
            "created",
            postgresql.TIMESTAMP(timezone=True),
            server_default=sa.text("now()"),
            autoincrement=False,
            nullable=False,
        ),
        sa.Column(
            "updated",
            postgresql.TIMESTAMP(timezone=True),
            server_default=sa.text("now()"),
            autoincrement=False,
            nullable=False,
        ),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_role")),
        sa.UniqueConstraint(
            "name", name=op.f("uq_role_name"), postgresql_include=[], postgresql_nulls_not_distinct=False
        ),
        schema=schema,
    )
    op.create_table(
        "signal_device",
        sa.Column("phone_number", sa.VARCHAR(length=50), autoincrement=False, nullable=False),
        sa.Column("safety_number", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column(
            "trust_level",
            postgresql.ENUM("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", schema=schema),
            autoincrement=False,
            nullable=False,
        ),
        sa.Column("last_seen", postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False),
        sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
        sa.Column(
            "created",
            postgresql.TIMESTAMP(timezone=True),
            server_default=sa.text("now()"),
            autoincrement=False,
            nullable=False,
        ),
        sa.Column(
            "updated",
            postgresql.TIMESTAMP(timezone=True),
            server_default=sa.text("now()"),
            autoincrement=False,
            nullable=False,
        ),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_signal_device")),
        sa.UniqueConstraint(
            "phone_number",
            name=op.f("uq_signal_device_phone_number"),
            postgresql_include=[],
            postgresql_nulls_not_distinct=False,
        ),
        schema=schema,
    )
    op.create_table(
        "device_role",
        sa.Column("device_id", sa.INTEGER(), autoincrement=False, nullable=False),
        sa.Column("role_id", sa.SMALLINT(), autoincrement=False, nullable=False),
        sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
        sa.Column(
            "created",
            postgresql.TIMESTAMP(timezone=True),
            server_default=sa.text("now()"),
            autoincrement=False,
            nullable=False,
        ),
        sa.Column(
            "updated",
            postgresql.TIMESTAMP(timezone=True),
            server_default=sa.text("now()"),
            autoincrement=False,
            nullable=False,
        ),
        sa.ForeignKeyConstraint(
            ["device_id"], [f"{schema}.signal_device.id"], name=op.f("fk_device_role_device_id_signal_device")
        ),
        sa.ForeignKeyConstraint(["role_id"], [f"{schema}.role.id"], name=op.f("fk_device_role_role_id_role")),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_device_role")),
        sa.UniqueConstraint(
            "device_id",
            "role_id",
            name=op.f("uq_device_role_device_role"),
            postgresql_include=[],
            postgresql_nulls_not_distinct=False,
        ),
        schema=schema,
    )
    # ### end Alembic commands ###
@@ -0,0 +1,187 @@
 """removed ds table from richie DB.
 Revision ID: c8a794340928
 Revises: 6b275323f435
 Create Date: 2026-03-29 15:29:23.643146
 """
 from __future__ import annotations
 from typing import TYPE_CHECKING
 import sqlalchemy as sa
 from alembic import op
 from sqlalchemy.dialects import postgresql
 from python.orm import RichieBase
 if TYPE_CHECKING:
    from collections.abc import Sequence
 # revision identifiers, used by Alembic.
 revision: str = "c8a794340928"
 down_revision: str | None = "6b275323f435"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 schema = RichieBase.schema_name
 def upgrade() -> None:
    """Upgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table("vote_record", schema=schema)
    op.drop_index(op.f("ix_vote_congress_chamber"), table_name="vote", schema=schema)
    op.drop_index(op.f("ix_vote_date"), table_name="vote", schema=schema)
    op.drop_index(op.f("ix_legislator_bioguide_id"), table_name="legislator", schema=schema)
    op.drop_table("legislator", schema=schema)
    op.drop_table("vote", schema=schema)
    op.drop_index(op.f("ix_bill_congress"), table_name="bill", schema=schema)
    op.drop_table("bill", schema=schema)
    # ### end Alembic commands ###
 def downgrade() -> None:
    """Downgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "vote",
        sa.Column("congress", sa.INTEGER(), autoincrement=False, nullable=False),
        sa.Column("chamber", sa.VARCHAR(), autoincrement=False, nullable=False),
        sa.Column("session", sa.INTEGER(), autoincrement=False, nullable=False),
        sa.Column("number", sa.INTEGER(), autoincrement=False, nullable=False),
        sa.Column("vote_type", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("question", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("result", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("result_text", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("vote_date", sa.DATE(), autoincrement=False, nullable=False),
        sa.Column("yea_count", sa.INTEGER(), autoincrement=False, nullable=True),
        sa.Column("nay_count", sa.INTEGER(), autoincrement=False, nullable=True),
        sa.Column("not_voting_count", sa.INTEGER(), autoincrement=False, nullable=True),
        sa.Column("present_count", sa.INTEGER(), autoincrement=False, nullable=True),
        sa.Column("bill_id", sa.INTEGER(), autoincrement=False, nullable=True),
        sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
        sa.Column(
            "created",
            postgresql.TIMESTAMP(timezone=True),
            server_default=sa.text("now()"),
            autoincrement=False,
            nullable=False,
        ),
        sa.Column(
            "updated",
            postgresql.TIMESTAMP(timezone=True),
            server_default=sa.text("now()"),
            autoincrement=False,
            nullable=False,
        ),
        sa.ForeignKeyConstraint(["bill_id"], [f"{schema}.bill.id"], name=op.f("fk_vote_bill_id_bill")),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_vote")),
        sa.UniqueConstraint(
            "congress",
            "chamber",
            "session",
            "number",
            name=op.f("uq_vote_congress_chamber_session_number"),
            postgresql_include=[],
            postgresql_nulls_not_distinct=False,
        ),
        schema=schema,
    )
    op.create_index(op.f("ix_vote_date"), "vote", ["vote_date"], unique=False, schema=schema)
    op.create_index(op.f("ix_vote_congress_chamber"), "vote", ["congress", "chamber"], unique=False, schema=schema)
    op.create_table(
        "vote_record",
        sa.Column("vote_id", sa.INTEGER(), autoincrement=False, nullable=False),
        sa.Column("legislator_id", sa.INTEGER(), autoincrement=False, nullable=False),
        sa.Column("position", sa.VARCHAR(), autoincrement=False, nullable=False),
        sa.ForeignKeyConstraint(
            ["legislator_id"],
            [f"{schema}.legislator.id"],
            name=op.f("fk_vote_record_legislator_id_legislator"),
            ondelete="CASCADE",
        ),
        sa.ForeignKeyConstraint(
            ["vote_id"], [f"{schema}.vote.id"], name=op.f("fk_vote_record_vote_id_vote"), ondelete="CASCADE"
        ),
        sa.PrimaryKeyConstraint("vote_id", "legislator_id", name=op.f("pk_vote_record")),
        schema=schema,
    )
    op.create_table(
        "legislator",
        sa.Column("bioguide_id", sa.TEXT(), autoincrement=False, nullable=False),
        sa.Column("thomas_id", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("lis_id", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("govtrack_id", sa.INTEGER(), autoincrement=False, nullable=True),
        sa.Column("opensecrets_id", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("fec_ids", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("first_name", sa.VARCHAR(), autoincrement=False, nullable=False),
        sa.Column("last_name", sa.VARCHAR(), autoincrement=False, nullable=False),
        sa.Column("official_full_name", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("nickname", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("birthday", sa.DATE(), autoincrement=False, nullable=True),
        sa.Column("gender", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("current_party", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("current_state", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("current_district", sa.INTEGER(), autoincrement=False, nullable=True),
        sa.Column("current_chamber", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
        sa.Column(
            "created",
            postgresql.TIMESTAMP(timezone=True),
            server_default=sa.text("now()"),
            autoincrement=False,
            nullable=False,
        ),
        sa.Column(
            "updated",
            postgresql.TIMESTAMP(timezone=True),
            server_default=sa.text("now()"),
            autoincrement=False,
            nullable=False,
        ),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_legislator")),
        schema=schema,
    )
    op.create_index(op.f("ix_legislator_bioguide_id"), "legislator", ["bioguide_id"], unique=True, schema=schema)
    op.create_table(
        "bill",
        sa.Column("congress", sa.INTEGER(), autoincrement=False, nullable=False),
        sa.Column("bill_type", sa.VARCHAR(), autoincrement=False, nullable=False),
        sa.Column("number", sa.INTEGER(), autoincrement=False, nullable=False),
        sa.Column("title", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("title_short", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("official_title", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("status", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("status_at", sa.DATE(), autoincrement=False, nullable=True),
        sa.Column("sponsor_bioguide_id", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("subjects_top_term", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
        sa.Column(
            "created",
            postgresql.TIMESTAMP(timezone=True),
            server_default=sa.text("now()"),
            autoincrement=False,
            nullable=False,
        ),
        sa.Column(
            "updated",
            postgresql.TIMESTAMP(timezone=True),
            server_default=sa.text("now()"),
            autoincrement=False,
            nullable=False,
        ),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_bill")),
        sa.UniqueConstraint(
            "congress",
            "bill_type",
            "number",
            name=op.f("uq_bill_congress_type_number"),
            postgresql_include=[],
            postgresql_nulls_not_distinct=False,
        ),
        schema=schema,
    )
    op.create_index(op.f("ix_bill_congress"), "bill", ["congress"], unique=False, schema=schema)
    # ### end Alembic commands ###
@@ -0,0 +1,93 @@
 """adding audiobook libreary metadata.
 Revision ID: d7864d1ffc17
 Revises: c8a794340928
 Create Date: 2026-06-03 20:24:09.200837
 """
 from __future__ import annotations
 from typing import TYPE_CHECKING
 import sqlalchemy as sa
 from alembic import op
 from python.orm import RichieBase
 if TYPE_CHECKING:
    from collections.abc import Sequence
 # revision identifiers, used by Alembic.
 revision: str = "d7864d1ffc17"
 down_revision: str | None = "c8a794340928"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 schema = RichieBase.schema_name
 def upgrade() -> None:
    """Upgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "audiobook_author",
        sa.Column("name", sa.String(), nullable=False),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_audiobook_author")),
        sa.UniqueConstraint("name", name=op.f("uq_audiobook_author_name")),
        schema=schema,
    )
    op.create_table(
        "audiobook_series",
        sa.Column("name", sa.String(), nullable=False),
        sa.Column("author_id", sa.Integer(), nullable=False),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.ForeignKeyConstraint(
            ["author_id"],
            [f"{schema}.audiobook_author.id"],
            name=op.f("fk_audiobook_series_author_id_audiobook_author"),
            ondelete="CASCADE",
        ),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_audiobook_series")),
        sa.UniqueConstraint("author_id", "name", name=op.f("uq_audiobook_series_author_id")),
        schema=schema,
    )
    op.create_table(
        "audiobook",
        sa.Column("title", sa.String(), nullable=False),
        sa.Column("author_id", sa.Integer(), nullable=False),
        sa.Column("series_id", sa.Integer(), nullable=True),
        sa.Column("series_index", sa.Integer(), nullable=False),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.ForeignKeyConstraint(
            ["author_id"],
            [f"{schema}.audiobook_author.id"],
            name=op.f("fk_audiobook_author_id_audiobook_author"),
            ondelete="CASCADE",
        ),
        sa.ForeignKeyConstraint(
            ["series_id"],
            [f"{schema}.audiobook_series.id"],
            name=op.f("fk_audiobook_series_id_audiobook_series"),
            ondelete="SET NULL",
        ),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_audiobook")),
        schema=schema,
    )
    # ### end Alembic commands ###
 def downgrade() -> None:
    """Downgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table("audiobook", schema=schema)
    op.drop_table("audiobook_series", schema=schema)
    op.drop_table("audiobook_author", schema=schema)
    # ### end Alembic commands ###
@@ -0,0 +1,200 @@
 """add ebook search tables.
 Revision ID: 2db132cace1a
 Revises: b3c60cc5beb5
 Create Date: 2026-06-10 22:10:54.379159
 """
 from __future__ import annotations
 from typing import TYPE_CHECKING
 import pgvector
 import sqlalchemy as sa
 from alembic import op
 from python.orm import RichieBase
 if TYPE_CHECKING:
    from collections.abc import Sequence
 # revision identifiers, used by Alembic.
 revision: str = "2db132cace1a"
 down_revision: str | None = "b3c60cc5beb5"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 schema = RichieBase.schema_name
 def upgrade() -> None:
    """Upgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "ebook_embedding_model",
        sa.Column("name", sa.String(), nullable=False),
        sa.Column("dimension", sa.Integer(), nullable=False),
        sa.Column("is_default", sa.Boolean(), nullable=False),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_embedding_model")),
        sa.UniqueConstraint("name", name=op.f("uq_ebook_embedding_model_name")),
        schema=schema,
    )
    op.create_table(
        "ebook_source",
        sa.Column("title", sa.String(), nullable=False),
        sa.Column("author", sa.String(), nullable=True),
        sa.Column("language", sa.String(), nullable=True),
        sa.Column("publisher", sa.String(), nullable=True),
        sa.Column("identifier", sa.String(), nullable=True),
        sa.Column("file_path", sa.String(), nullable=False),
        sa.Column("file_sha256", sa.String(length=64), nullable=False),
        sa.Column("file_mtime", sa.DateTime(timezone=True), nullable=False),
        sa.Column("file_size", sa.BigInteger(), nullable=False),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_source")),
        sa.UniqueConstraint("file_path", name=op.f("uq_ebook_source_file_path")),
        sa.UniqueConstraint("file_sha256", name=op.f("uq_ebook_source_file_sha256")),
        schema=schema,
    )
    op.create_table(
        "ebook_chapter",
        sa.Column("source_id", sa.Integer(), nullable=False),
        sa.Column("spine_index", sa.Integer(), nullable=False),
        sa.Column("title", sa.String(), nullable=True),
        sa.Column("href", sa.String(), nullable=True),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.ForeignKeyConstraint(
            ["source_id"],
            [f"{schema}.ebook_source.id"],
            name=op.f("fk_ebook_chapter_source_id_ebook_source"),
            ondelete="CASCADE",
        ),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chapter")),
        sa.UniqueConstraint("source_id", "spine_index", name=op.f("uq_ebook_chapter_source_id")),
        schema=schema,
    )
    op.create_table(
        "ebook_chunk",
        sa.Column("source_id", sa.Integer(), nullable=False),
        sa.Column("chapter_id", sa.Integer(), nullable=True),
        sa.Column("chunk_index", sa.Integer(), nullable=False),
        sa.Column("text", sa.String(), nullable=False),
        sa.Column("token_start", sa.Integer(), nullable=False),
        sa.Column("token_count", sa.Integer(), nullable=False),
        sa.Column("page_label", sa.String(), nullable=True),
        sa.Column("content_sha256", sa.String(length=64), nullable=False),
        sa.Column("search_text", sa.String(), nullable=False),
        sa.Column("id", sa.BigInteger(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.ForeignKeyConstraint(
            ["chapter_id"],
            [f"{schema}.ebook_chapter.id"],
            name=op.f("fk_ebook_chunk_chapter_id_ebook_chapter"),
            ondelete="SET NULL",
        ),
        sa.ForeignKeyConstraint(
            ["source_id"],
            [f"{schema}.ebook_source.id"],
            name=op.f("fk_ebook_chunk_source_id_ebook_source"),
            ondelete="CASCADE",
        ),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk")),
        sa.UniqueConstraint("source_id", "chunk_index", name="uq_ebook_chunk_source_id_chunk_index"),
        sa.UniqueConstraint("source_id", "content_sha256", name="uq_ebook_chunk_source_id_content_sha256"),
        schema=schema,
    )
    op.create_table(
        "ebook_chunk_embedding_1024",
        sa.Column("chunk_id", sa.BigInteger(), nullable=False),
        sa.Column("model_id", sa.Integer(), nullable=False),
        sa.Column("embedding", pgvector.sqlalchemy.vector.VECTOR(dim=1024), nullable=False),
        sa.Column("id", sa.BigInteger(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.ForeignKeyConstraint(
            ["chunk_id"],
            [f"{schema}.ebook_chunk.id"],
            name=op.f("fk_ebook_chunk_embedding_1024_chunk_id_ebook_chunk"),
            ondelete="CASCADE",
        ),
        sa.ForeignKeyConstraint(
            ["model_id"],
            [f"{schema}.ebook_embedding_model.id"],
            name=op.f("fk_ebook_chunk_embedding_1024_model_id_ebook_embedding_model"),
            ondelete="CASCADE",
        ),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk_embedding_1024")),
        sa.UniqueConstraint("chunk_id", "model_id", name=op.f("uq_ebook_chunk_embedding_1024_chunk_id")),
        schema=schema,
    )
    op.create_table(
        "ebook_chunk_embedding_2560",
        sa.Column("chunk_id", sa.BigInteger(), nullable=False),
        sa.Column("model_id", sa.Integer(), nullable=False),
        sa.Column("embedding", pgvector.sqlalchemy.vector.VECTOR(dim=2560), nullable=False),
        sa.Column("id", sa.BigInteger(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.ForeignKeyConstraint(
            ["chunk_id"],
            [f"{schema}.ebook_chunk.id"],
            name=op.f("fk_ebook_chunk_embedding_2560_chunk_id_ebook_chunk"),
            ondelete="CASCADE",
        ),
        sa.ForeignKeyConstraint(
            ["model_id"],
            [f"{schema}.ebook_embedding_model.id"],
            name=op.f("fk_ebook_chunk_embedding_2560_model_id_ebook_embedding_model"),
            ondelete="CASCADE",
        ),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk_embedding_2560")),
        sa.UniqueConstraint("chunk_id", "model_id", name=op.f("uq_ebook_chunk_embedding_2560_chunk_id")),
        schema=schema,
    )
    op.create_table(
        "ebook_chunk_embedding_4096",
        sa.Column("chunk_id", sa.BigInteger(), nullable=False),
        sa.Column("model_id", sa.Integer(), nullable=False),
        sa.Column("embedding", pgvector.sqlalchemy.vector.VECTOR(dim=4096), nullable=False),
        sa.Column("id", sa.BigInteger(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.ForeignKeyConstraint(
            ["chunk_id"],
            [f"{schema}.ebook_chunk.id"],
            name=op.f("fk_ebook_chunk_embedding_4096_chunk_id_ebook_chunk"),
            ondelete="CASCADE",
        ),
        sa.ForeignKeyConstraint(
            ["model_id"],
            [f"{schema}.ebook_embedding_model.id"],
            name=op.f("fk_ebook_chunk_embedding_4096_model_id_ebook_embedding_model"),
            ondelete="CASCADE",
        ),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk_embedding_4096")),
        sa.UniqueConstraint("chunk_id", "model_id", name=op.f("uq_ebook_chunk_embedding_4096_chunk_id")),
        schema=schema,
    )
    # ### end Alembic commands ###
 def downgrade() -> None:
    """Downgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table("ebook_chunk_embedding_4096", schema=schema)
    op.drop_table("ebook_chunk_embedding_2560", schema=schema)
    op.drop_table("ebook_chunk_embedding_1024", schema=schema)
    op.drop_table("ebook_chunk", schema=schema)
    op.drop_table("ebook_chapter", schema=schema)
    op.drop_table("ebook_source", schema=schema)
    op.drop_table("ebook_embedding_model", schema=schema)
    # ### end Alembic commands ###
@@ -0,0 +1,63 @@
 """updated series_index to float and added UniqueConstraint to audiobook and audiobook_author.
 Revision ID: b3c60cc5beb5
 Revises: d7864d1ffc17
 Create Date: 2026-06-10 20:02:43.073725
 """
 from __future__ import annotations
 from typing import TYPE_CHECKING
 import sqlalchemy as sa
 from alembic import op
 from python.orm import RichieBase
 if TYPE_CHECKING:
    from collections.abc import Sequence
 # revision identifiers, used by Alembic.
 revision: str = "b3c60cc5beb5"
 down_revision: str | None = "d7864d1ffc17"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 schema = RichieBase.schema_name
 def upgrade() -> None:
    """Upgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.alter_column(
        "audiobook",
        "series_index",
        existing_type=sa.INTEGER(),
        type_=sa.Float(),
        existing_nullable=False,
        schema=schema,
    )
    op.create_unique_constraint(
        op.f("uq_audiobook_author_id"),
        "audiobook",
        ["author_id", "series_id", "title"],
        schema=schema,
        postgresql_nulls_not_distinct=True,
    )
    # ### end Alembic commands ###
 def downgrade() -> None:
    """Downgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_constraint(op.f("uq_audiobook_author_id"), "audiobook", schema=schema, type_="unique")
    op.alter_column(
        "audiobook",
        "series_index",
        existing_type=sa.Float(),
        type_=sa.INTEGER(),
        existing_nullable=False,
        schema=schema,
    )
    # ### end Alembic commands ###
@@ -0,0 +1,54 @@
 """add 1024 ebook embedding cosine index.
 Revision ID: c460105682d2
 Revises: 2db132cace1a
 Create Date: 2026-06-13 19:53:45.680289
 """
 from __future__ import annotations
 from typing import TYPE_CHECKING
 from alembic import op
 from python.orm import RichieBase
 if TYPE_CHECKING:
    from collections.abc import Sequence
 # revision identifiers, used by Alembic.
 revision: str = "c460105682d2"
 down_revision: str | None = "2db132cace1a"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 schema = RichieBase.schema_name
 def upgrade() -> None:
    """Upgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_index(
        "ix_ebook_chunk_embedding_1024_embedding_cosine",
        "ebook_chunk_embedding_1024",
        ["embedding"],
        unique=False,
        schema=schema,
        postgresql_using="hnsw",
        postgresql_ops={"embedding": "vector_cosine_ops"},
    )
    # ### end Alembic commands ###
 def downgrade() -> None:
    """Downgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index(
        "ix_ebook_chunk_embedding_1024_embedding_cosine",
        table_name="ebook_chunk_embedding_1024",
        schema=schema,
        postgresql_using="hnsw",
        postgresql_ops={"embedding": "vector_cosine_ops"},
    )
    # ### end Alembic commands ###
@@ -0,0 +1,103 @@
 """adding haproxy data.
 Revision ID: 96d72c748c24
 Revises: c460105682d2
 Create Date: 2026-06-23 16:37:17.768851
 """
 from __future__ import annotations
 from typing import TYPE_CHECKING
 import sqlalchemy as sa
 from alembic import op
 from python.orm import RichieBase
 if TYPE_CHECKING:
    from collections.abc import Sequence
 # revision identifiers, used by Alembic.
 revision: str = "96d72c748c24"
 down_revision: str | None = "c460105682d2"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 schema = RichieBase.schema_name
 def upgrade() -> None:
    """Upgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "haproxy_request",
        sa.Column("line_hash", sa.String(), nullable=False),
        sa.Column("requested_at", sa.DateTime(timezone=True), nullable=False),
        sa.Column("client_ip", sa.String(), nullable=False),
        sa.Column("client_port", sa.Integer(), nullable=False),
        sa.Column("frontend", sa.String(), nullable=False),
        sa.Column("ssl", sa.Boolean(), nullable=False),
        sa.Column("backend", sa.String(), nullable=False),
        sa.Column("server", sa.String(), nullable=False),
        sa.Column("time_request", sa.Integer(), nullable=False),
        sa.Column("time_queue", sa.Integer(), nullable=False),
        sa.Column("time_connect", sa.Integer(), nullable=False),
        sa.Column("time_response", sa.Integer(), nullable=False),
        sa.Column("time_total", sa.Integer(), nullable=False),
        sa.Column("status_code", sa.Integer(), nullable=False),
        sa.Column("bytes_read", sa.BigInteger(), nullable=False),
        sa.Column("termination_state", sa.String(), nullable=False),
        sa.Column("active_connections", sa.Integer(), nullable=False),
        sa.Column("frontend_connections", sa.Integer(), nullable=False),
        sa.Column("backend_connections", sa.Integer(), nullable=False),
        sa.Column("server_connections", sa.Integer(), nullable=False),
        sa.Column("retries", sa.Integer(), nullable=False),
        sa.Column("server_queue", sa.Integer(), nullable=False),
        sa.Column("backend_queue", sa.Integer(), nullable=False),
        sa.Column("host", sa.String(), nullable=True),
        sa.Column("user_agent", sa.String(), nullable=True),
        sa.Column("method", sa.String(), nullable=False),
        sa.Column("target", sa.String(), nullable=False),
        sa.Column("path", sa.String(), nullable=False),
        sa.Column("query", sa.String(), nullable=True),
        sa.Column("http_version", sa.String(), nullable=False),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_haproxy_request")),
        sa.UniqueConstraint("line_hash", name=op.f("uq_haproxy_request_line_hash")),
        schema=schema,
    )
    op.create_index(op.f("ix_haproxy_request_backend"), "haproxy_request", ["backend"], unique=False, schema=schema)
    op.create_index(op.f("ix_haproxy_request_client_ip"), "haproxy_request", ["client_ip"], unique=False, schema=schema)
    op.create_index(op.f("ix_haproxy_request_host"), "haproxy_request", ["host"], unique=False, schema=schema)
    op.create_index(op.f("ix_haproxy_request_path"), "haproxy_request", ["path"], unique=False, schema=schema)
    op.create_index(
        op.f("ix_haproxy_request_requested_at"), "haproxy_request", ["requested_at"], unique=False, schema=schema
    )
    op.create_index(
        op.f("ix_haproxy_request_status_code"), "haproxy_request", ["status_code"], unique=False, schema=schema
    )
    op.create_index(
        op.f("ix_haproxy_request_time_response"), "haproxy_request", ["time_response"], unique=False, schema=schema
    )
    op.create_index(
        op.f("ix_haproxy_request_user_agent"), "haproxy_request", ["user_agent"], unique=False, schema=schema
    )
    # ### end Alembic commands ###
 def downgrade() -> None:
    """Downgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index(op.f("ix_haproxy_request_user_agent"), table_name="haproxy_request", schema=schema)
    op.drop_index(op.f("ix_haproxy_request_time_response"), table_name="haproxy_request", schema=schema)
    op.drop_index(op.f("ix_haproxy_request_status_code"), table_name="haproxy_request", schema=schema)
    op.drop_index(op.f("ix_haproxy_request_requested_at"), table_name="haproxy_request", schema=schema)
    op.drop_index(op.f("ix_haproxy_request_path"), table_name="haproxy_request", schema=schema)
    op.drop_index(op.f("ix_haproxy_request_host"), table_name="haproxy_request", schema=schema)
    op.drop_index(op.f("ix_haproxy_request_client_ip"), table_name="haproxy_request", schema=schema)
    op.drop_index(op.f("ix_haproxy_request_backend"), table_name="haproxy_request", schema=schema)
    op.drop_table("haproxy_request", schema=schema)
    # ### end Alembic commands ###
@@ -0,0 +1,206 @@
 """adding Phrase metadata tables.
 Revision ID: dddee09eddcc
 Revises: 96d72c748c24
 Create Date: 2026-06-29 00:49:07.344159
 """
 from __future__ import annotations
 from typing import TYPE_CHECKING
 import sqlalchemy as sa
 from alembic import op
 from sqlalchemy.dialects import postgresql
 from python.orm import RichieBase
 if TYPE_CHECKING:
    from collections.abc import Sequence
 # revision identifiers, used by Alembic.
 revision: str = "dddee09eddcc"
 down_revision: str | None = "96d72c748c24"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 schema = RichieBase.schema_name
 def upgrade() -> None:
    """Upgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "candidate_phrases",
        sa.Column("book_id", sa.Integer(), nullable=False),
        sa.Column("series_id", sa.Integer(), nullable=True),
        sa.Column("phrase_text", sa.Text(), nullable=False),
        sa.Column("phrase_norm", sa.Text(), nullable=False),
        sa.Column("token_count", sa.Integer(), nullable=False),
        sa.Column("source_raw_ngram", sa.Boolean(), nullable=False),
        sa.Column("source_yake", sa.Boolean(), nullable=False),
        sa.Column("source_spacy_ner", sa.Boolean(), nullable=False),
        sa.Column("source_spacy_noun_chunk", sa.Boolean(), nullable=False),
        sa.Column("source_capitalized", sa.Boolean(), nullable=False),
        sa.Column("source_metadata", sa.Boolean(), nullable=False),
        sa.Column("spacy_label", sa.String(), nullable=True),
        sa.Column("raw_count", sa.Integer(), nullable=False),
        sa.Column("chapter_count", sa.Integer(), nullable=False),
        sa.Column("yake_score", sa.Float(), nullable=True),
        sa.Column("candidate_score", sa.Float(), nullable=False),
        sa.Column(
            "sample_contexts",
            sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), "postgresql"),
            nullable=True,
        ),
        sa.Column("llm_judged", sa.Boolean(), nullable=False),
        sa.Column("llm_keep", sa.Boolean(), nullable=True),
        sa.Column("llm_confidence", sa.Float(), nullable=True),
        sa.Column("llm_category", sa.String(), nullable=True),
        sa.Column("llm_reason", sa.Text(), nullable=True),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.ForeignKeyConstraint(
            ["book_id"],
            [f"{schema}.ebook_source.id"],
            name=op.f("fk_candidate_phrases_book_id_ebook_source"),
            ondelete="CASCADE",
        ),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_candidate_phrases")),
        sa.UniqueConstraint("book_id", "phrase_norm", name="uq_candidate_phrases_book_id_phrase_norm"),
        schema=schema,
    )
    op.create_index(
        "candidate_phrases_book_norm_idx", "candidate_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
    )
    op.create_index(
        "candidate_phrases_book_score_idx",
        "candidate_phrases",
        ["book_id", "candidate_score"],
        unique=False,
        schema=schema,
    )
    op.create_table(
        "protected_phrases",
        sa.Column("book_id", sa.Integer(), nullable=True),
        sa.Column("series_id", sa.Integer(), nullable=True),
        sa.Column("phrase_text", sa.Text(), nullable=False),
        sa.Column("phrase_norm", sa.Text(), nullable=False),
        sa.Column("canonical_id", sa.String(), nullable=False),
        sa.Column("phrase_type", sa.String(), nullable=True),
        sa.Column("token_count", sa.Integer(), nullable=False),
        sa.Column("confidence", sa.Float(), nullable=False),
        sa.Column("importance", sa.Float(), nullable=False),
        sa.Column("allow_nested", sa.Boolean(), nullable=False),
        sa.Column("suppress_children", sa.Boolean(), nullable=False),
        sa.Column("source_candidate_id", sa.Integer(), nullable=True),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.ForeignKeyConstraint(
            ["book_id"],
            [f"{schema}.ebook_source.id"],
            name=op.f("fk_protected_phrases_book_id_ebook_source"),
            ondelete="CASCADE",
        ),
        sa.ForeignKeyConstraint(
            ["source_candidate_id"],
            [f"{schema}.candidate_phrases.id"],
            name=op.f("fk_protected_phrases_source_candidate_id_candidate_phrases"),
            ondelete="SET NULL",
        ),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_protected_phrases")),
        sa.UniqueConstraint("book_id", "phrase_norm", name="uq_protected_phrases_book_id_phrase_norm"),
        schema=schema,
    )
    op.create_index(
        "protected_phrases_book_norm_idx", "protected_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
    )
    op.create_index("protected_phrases_norm_idx", "protected_phrases", ["phrase_norm"], unique=False, schema=schema)
    op.create_index(
        "protected_phrases_series_norm_idx",
        "protected_phrases",
        ["series_id", "phrase_norm"],
        unique=False,
        schema=schema,
    )
    op.create_table(
        "chunk_phrase_mentions",
        sa.Column("chunk_id", sa.BigInteger(), nullable=False),
        sa.Column("phrase_id", sa.Integer(), nullable=False),
        sa.Column("book_id", sa.Integer(), nullable=True),
        sa.Column("series_id", sa.Integer(), nullable=True),
        sa.Column("start_char", sa.Integer(), nullable=False),
        sa.Column("end_char", sa.Integer(), nullable=True),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.ForeignKeyConstraint(
            ["book_id"],
            [f"{schema}.ebook_source.id"],
            name=op.f("fk_chunk_phrase_mentions_book_id_ebook_source"),
            ondelete="CASCADE",
        ),
        sa.ForeignKeyConstraint(
            ["chunk_id"],
            [f"{schema}.ebook_chunk.id"],
            name=op.f("fk_chunk_phrase_mentions_chunk_id_ebook_chunk"),
            ondelete="CASCADE",
        ),
        sa.ForeignKeyConstraint(
            ["phrase_id"],
            [f"{schema}.protected_phrases.id"],
            name=op.f("fk_chunk_phrase_mentions_phrase_id_protected_phrases"),
            ondelete="CASCADE",
        ),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_chunk_phrase_mentions")),
        sa.UniqueConstraint("chunk_id", "phrase_id", "start_char", name="uq_chunk_phrase_mentions_chunk_phrase_start"),
        schema=schema,
    )
    op.create_index(
        "chunk_phrase_mentions_chunk_idx", "chunk_phrase_mentions", ["chunk_id"], unique=False, schema=schema
    )
    op.create_index(
        "chunk_phrase_mentions_phrase_idx", "chunk_phrase_mentions", ["phrase_id"], unique=False, schema=schema
    )
    op.create_table(
        "phrase_aliases",
        sa.Column("phrase_id", sa.Integer(), nullable=False),
        sa.Column("alias_text", sa.Text(), nullable=False),
        sa.Column("alias_norm", sa.Text(), nullable=False),
        sa.Column("confidence", sa.Float(), nullable=False),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.ForeignKeyConstraint(
            ["phrase_id"],
            [f"{schema}.protected_phrases.id"],
            name=op.f("fk_phrase_aliases_phrase_id_protected_phrases"),
            ondelete="CASCADE",
        ),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_phrase_aliases")),
        sa.UniqueConstraint("phrase_id", "alias_norm", name="uq_phrase_aliases_phrase_id_alias_norm"),
        schema=schema,
    )
    op.create_index("phrase_aliases_norm_idx", "phrase_aliases", ["alias_norm"], unique=False, schema=schema)
    # ### end Alembic commands ###
 def downgrade() -> None:
    """Downgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index("phrase_aliases_norm_idx", table_name="phrase_aliases", schema=schema)
    op.drop_table("phrase_aliases", schema=schema)
    op.drop_index("chunk_phrase_mentions_phrase_idx", table_name="chunk_phrase_mentions", schema=schema)
    op.drop_index("chunk_phrase_mentions_chunk_idx", table_name="chunk_phrase_mentions", schema=schema)
    op.drop_table("chunk_phrase_mentions", schema=schema)
    op.drop_index("protected_phrases_series_norm_idx", table_name="protected_phrases", schema=schema)
    op.drop_index("protected_phrases_norm_idx", table_name="protected_phrases", schema=schema)
    op.drop_index("protected_phrases_book_norm_idx", table_name="protected_phrases", schema=schema)
    op.drop_table("protected_phrases", schema=schema)
    op.drop_index("candidate_phrases_book_score_idx", table_name="candidate_phrases", schema=schema)
    op.drop_index("candidate_phrases_book_norm_idx", table_name="candidate_phrases", schema=schema)
    op.drop_table("candidate_phrases", schema=schema)
    # ### end Alembic commands ###
@@ -0,0 +1,55 @@
 """remove spaCy-ner.
 Revision ID: 751260fc3228
 Revises: dddee09eddcc
 Create Date: 2026-07-09 23:03:39.554083
 """
 from __future__ import annotations
 from typing import TYPE_CHECKING
 import sqlalchemy as sa
 from alembic import op
 from python.orm import RichieBase
 if TYPE_CHECKING:
    from collections.abc import Sequence
 # revision identifiers, used by Alembic.
 revision: str = "751260fc3228"
 down_revision: str | None = "dddee09eddcc"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 schema = RichieBase.schema_name
 def upgrade() -> None:
    """Upgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column("candidate_phrases", "source_spacy_noun_chunk", schema=schema)
    op.drop_column("candidate_phrases", "source_spacy_ner", schema=schema)
    op.drop_column("candidate_phrases", "spacy_label", schema=schema)
    # ### end Alembic commands ###
 def downgrade() -> None:
    """Downgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column(
        "candidate_phrases", sa.Column("spacy_label", sa.VARCHAR(), autoincrement=False, nullable=True), schema=schema
    )
    op.add_column(
        "candidate_phrases",
        sa.Column("source_spacy_ner", sa.BOOLEAN(), autoincrement=False, nullable=False),
        schema=schema,
    )
    op.add_column(
        "candidate_phrases",
        sa.Column("source_spacy_noun_chunk", sa.BOOLEAN(), autoincrement=False, nullable=False),
        schema=schema,
    )
    # ### end Alembic commands ###
@@ -0,0 +1,36 @@
 """${message}.
 Revision ID: ${up_revision}
 Revises: ${down_revision | comma,n}
 Create Date: ${create_date}
 """
 from __future__ import annotations
 from typing import TYPE_CHECKING
 import sqlalchemy as sa
 from alembic import op
 from python.orm import ${config.attributes["base"].__name__}
 if TYPE_CHECKING:
    from collections.abc import Sequence
 # revision identifiers, used by Alembic.
 revision: str = ${repr(up_revision)}
 down_revision: str | None = ${repr(down_revision)}
 branch_labels: str | Sequence[str] | None = ${repr(branch_labels)}
 depends_on: str | Sequence[str] | None = ${repr(depends_on)}
 schema=${config.attributes["base"].__name__}.schema_name
 def upgrade() -> None:
    """Upgrade."""
    ${upgrades if upgrades else "pass"}
 def downgrade() -> None:
    """Downgrade."""
    ${downgrades if downgrades else "pass"}
@@ -0,0 +1,80 @@
 """starting van invintory.
 Revision ID: 15e733499804
 Revises:
 Create Date: 2026-03-08 00:18:20.759720
 """
 from __future__ import annotations
 from typing import TYPE_CHECKING
 import sqlalchemy as sa
 from alembic import op
 from python.orm import VanInventoryBase
 if TYPE_CHECKING:
    from collections.abc import Sequence
 # revision identifiers, used by Alembic.
 revision: str = "15e733499804"
 down_revision: str | None = None
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 schema = VanInventoryBase.schema_name
 def upgrade() -> None:
    """Upgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "items",
        sa.Column("name", sa.String(), nullable=False),
        sa.Column("quantity", sa.Float(), nullable=False),
        sa.Column("unit", sa.String(), nullable=False),
        sa.Column("category", sa.String(), nullable=True),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_items")),
        sa.UniqueConstraint("name", name=op.f("uq_items_name")),
        schema=schema,
    )
    op.create_table(
        "meals",
        sa.Column("name", sa.String(), nullable=False),
        sa.Column("instructions", sa.String(), nullable=True),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_meals")),
        sa.UniqueConstraint("name", name=op.f("uq_meals_name")),
        schema=schema,
    )
    op.create_table(
        "meal_ingredients",
        sa.Column("meal_id", sa.Integer(), nullable=False),
        sa.Column("item_id", sa.Integer(), nullable=False),
        sa.Column("quantity_needed", sa.Float(), nullable=False),
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
        sa.ForeignKeyConstraint(["item_id"], [f"{schema}.items.id"], name=op.f("fk_meal_ingredients_item_id_items")),
        sa.ForeignKeyConstraint(["meal_id"], [f"{schema}.meals.id"], name=op.f("fk_meal_ingredients_meal_id_meals")),
        sa.PrimaryKeyConstraint("id", name=op.f("pk_meal_ingredients")),
        sa.UniqueConstraint("meal_id", "item_id", name=op.f("uq_meal_ingredients_meal_id")),
        schema=schema,
    )
    # ### end Alembic commands ###
 def downgrade() -> None:
    """Downgrade."""
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table("meal_ingredients", schema=schema)
    op.drop_table("meals", schema=schema)
    op.drop_table("items", schema=schema)
    # ### end Alembic commands ###
@@ -0,0 +1,47 @@
 """common."""
 from __future__ import annotations
 import logging
 from datetime import UTC, datetime
 from pathlib import Path
 from subprocess import PIPE, Popen
 from python.logging_config import configure_logger as _configure_logger
 logger = logging.getLogger(__name__)
 def get_repo_dir() -> Path:
    """Return the repository root directory."""
    return Path(__file__).resolve().parents[1]
 def configure_logger(level: str = "INFO") -> None:
    """Configure the logger."""
    _configure_logger(level)
 def bash_wrapper(command: str) -> tuple[str, int]:
    """Execute a bash command and capture the output.
    Args:
        command (str): The bash command to be executed.
    Returns:
        Tuple[str, int]: A tuple containing the output of the command (stdout) as a string,
        the error output (stderr) as a string (optional), and the return code as an integer.
    """
    # This is a acceptable risk
    process = Popen(command.split(), stdout=PIPE, stderr=PIPE)
    output, error = process.communicate()
    if error:
        logger.error(f"{error=}")
        return error.decode(), process.returncode
    return output.decode(), process.returncode
 def utcnow() -> datetime:
    """Get the current UTC time."""
    return datetime.now(tz=UTC)
@@ -0,0 +1,59 @@
 """database."""
 from __future__ import annotations
 import logging
 from typing import TYPE_CHECKING
 from sqlalchemy import inspect
 from sqlalchemy.exc import NoInspectionAvailable
 if TYPE_CHECKING:
    from collections.abc import Sequence
    from sqlalchemy.orm import Session
 logger = logging.getLogger(__name__)
 def safe_insert(orm_objects: Sequence[object], session: Session) -> list[tuple[Exception, object]]:
    """Safer insert at allows for partial rollbacks.
    Args:
        orm_objects (Sequence[object]): Tables to insert.
        session (Session): Database session.
    """
    if unmapped := [orm_object for orm_object in orm_objects if not _is_mapped_instance(orm_object)]:
        error = f"safe_insert expects ORM-mapped instances {unmapped}"
        raise TypeError(error)
    return _safe_insert(orm_objects, session)
 def _safe_insert(objects: Sequence[object], session: Session) -> list[tuple[Exception, object]]:
    exceptions: list[tuple[Exception, object]] = []
    try:
        session.add_all(objects)
        session.commit()
    except Exception as error:
        session.rollback()
        objects_len = len(objects)
        if objects_len == 1:
            logger.exception(objects)
            return [(error, objects[0])]
        middle = objects_len // 2
        exceptions.extend(_safe_insert(objects=objects[:middle], session=session))
        exceptions.extend(_safe_insert(objects=objects[middle:], session=session))
    return exceptions
 def _is_mapped_instance(obj: object) -> bool:
    """Return True if `obj` is a SQLAlchemy ORM-mapped instance."""
    try:
        inspect(obj)  # raises NoInspectionAvailable if not mapped
    except NoInspectionAvailable:
        return False
    else:
        return True
@@ -0,0 +1,103 @@
 """CLI wrapper around alembic for multi-database support.
 Usage:
    database <db_name> <command> [args...]
 Examples:
    database richie check
    database richie upgrade head
    database richie downgrade head-1
    database richie revision --autogenerate -m "add meals table"
 """
 from __future__ import annotations
 from dataclasses import dataclass
 from importlib import import_module
 from typing import TYPE_CHECKING, Annotated
 import typer
 from alembic.config import CommandLine, Config
 if TYPE_CHECKING:
    from sqlalchemy.orm import DeclarativeBase
@dataclass(frozen=True)
 class DatabaseConfig:
    """Configuration for a database."""
    env_prefix: str
    version_location: str
    base_module: str
    base_class_name: str
    models_module: str
    script_location: str = "python/alembic"
    file_template: str = "%%(year)d_%%(month).2d_%%(day).2d-%%(slug)s_%%(rev)s"
    def get_base(self) -> type[DeclarativeBase]:
        """Import and return the Base class."""
        module = import_module(self.base_module)
        return getattr(module, self.base_class_name)
    def import_models(self) -> None:
        """Import ORM models so alembic autogenerate can detect them."""
        import_module(self.models_module)
    def alembic_config(self) -> Config:
        """Build an alembic Config for this database."""
        cfg = Config()
        cfg.set_main_option("script_location", self.script_location)
        cfg.set_main_option("file_template", self.file_template)
        cfg.set_main_option("prepend_sys_path", ".")
        cfg.set_main_option("version_path_separator", "os")
        cfg.set_main_option("version_locations", self.version_location)
        cfg.set_main_option("revision_environment", "true")
        cfg.set_section_option("post_write_hooks", "hooks", "dynamic_schema,import_postgresql,ruff")
        cfg.set_section_option("post_write_hooks", "dynamic_schema.type", "dynamic_schema")
        cfg.set_section_option("post_write_hooks", "import_postgresql.type", "import_postgresql")
        cfg.set_section_option("post_write_hooks", "ruff.type", "ruff")
        cfg.attributes["base"] = self.get_base()
        cfg.attributes["env_prefix"] = self.env_prefix
        self.import_models()
        return cfg
 DATABASES: dict[str, DatabaseConfig] = {
    "richie": DatabaseConfig(
        env_prefix="RICHIE",
        version_location="python/alembic/richie/versions",
        base_module="python.orm.richie.base",
        base_class_name="RichieBase",
        models_module="python.orm.richie",
    ),
 }
 app = typer.Typer(help="Multi-database alembic wrapper.")
@app.command(
    context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
 )
 def main(
    ctx: typer.Context,
    db_name: Annotated[str, typer.Argument(help=f"Database name. Options: {', '.join(DATABASES)}")],
    command: Annotated[str, typer.Argument(help="Alembic command (upgrade, downgrade, revision, check, etc.)")],
 ) -> None:
    """Run an alembic command against the specified database."""
    db_config = DATABASES.get(db_name)
    if not db_config:
        typer.echo(f"Unknown database: {db_name!r}. Available: {', '.join(DATABASES)}", err=True)
        raise typer.Exit(code=1)
    alembic_cfg = db_config.alembic_config()
    cmd_line = CommandLine()
    options = cmd_line.parser.parse_args([command, *ctx.args])
    cmd_line.run_cmd(alembic_cfg, options)
 if __name__ == "__main__":
    app()
@@ -0,0 +1 @@
 """EPUB search package."""
@@ -0,0 +1,50 @@
 """Grounded answer generation."""
 from __future__ import annotations
 import logging
 from typing import TYPE_CHECKING
 from python.ebook_search.llm_interface import request_chat_completion
 from python.ebook_search.prompts import load_prompt
 if TYPE_CHECKING:
    import httpx
    from python.ebook_search.config import EbookSearchConfig
    from python.ebook_search.search import SearchResult
 logger = logging.getLogger(__name__)
 async def answer_query(
    client: httpx.AsyncClient,
    query: str,
    results: list[SearchResult],
    config: EbookSearchConfig,
 ) -> str:
    """Answer a question using only retrieved chunks."""
    if not config.answer_enabled:
        logger.info("ebook_answer_skipped_disabled")
        return "Answer generation is disabled. Source chunks are shown below."
    if not results:
        logger.info("ebook_answer_skipped_no_results")
        return "No relevant sources were found."
    logger.info(
        f"ebook_answer_request_start {config.vllm_base_url=} {config.chat_model=} sources={len(results)} "
        f"query_length={len(query)}"
    )
    context = "\n\n".join(
        f"[{index}] {result.source_title}{' - ' + result.chapter_title if result.chapter_title else ''}\n{result.text}"
        for index, result in enumerate(results, start=1)
    )
    content = await request_chat_completion(
        client,
        config,
        load_prompt("answer").messages(query=query, context=context),
    )
    logger.info(f"ebook_answer_request_complete {config.chat_model=} answer_length={len(content)}")
    return content or "The model returned an empty answer."
@@ -0,0 +1 @@
 """Web and external API adapters for EPUB search."""
@@ -0,0 +1,70 @@
 """Background BM25 refresh tasks for the web app.
 The refresh is scheduled on the event loop instead of a thread because the async psycopg
 driver only works from the loop; a bare thread cannot open a session on the async engine.
 """
 from __future__ import annotations
 import asyncio
 import logging
 from typing import TYPE_CHECKING
 from sqlalchemy.ext.asyncio import AsyncSession
 from python.ebook_search.bm25_corpus import load_bm25_corpus, refresh_bm25_corpus
 if TYPE_CHECKING:
    from fastapi import FastAPI
    from sqlalchemy.ext.asyncio import AsyncEngine
    from python.ebook_search.config import EbookSearchConfig
 logger = logging.getLogger(__name__)
 def schedule_bm25_refresh(app: FastAPI) -> None:
    """Schedule a delayed BM25 corpus refresh, replacing any pending refresh.
    Only called from route handlers, so a running event loop is guaranteed.
    """
    cancel_bm25_refresh(app)
    loop = asyncio.get_running_loop()
    def start_refresh() -> None:
        app.state.bm25_refresh_task = loop.create_task(refresh_bm25_for_app(app))
    app.state.bm25_refresh_timer = loop.call_later(app.state.config.bm25_refresh_delay_seconds, start_refresh)
    logger.info(f"ebook_bm25_refresh_scheduled {app.state.config.bm25_refresh_delay_seconds=}")
 def cancel_bm25_refresh(app: FastAPI) -> None:
    """Cancel any pending BM25 corpus refresh timer and in-flight refresh task."""
    existing_timer = getattr(app.state, "bm25_refresh_timer", None)
    if existing_timer is not None:
        existing_timer.cancel()
        app.state.bm25_refresh_timer = None
        logger.info("ebook_bm25_refresh_cancelled")
    existing_task = getattr(app.state, "bm25_refresh_task", None)
    if existing_task is not None:
        if not existing_task.done():
            existing_task.cancel()
        app.state.bm25_refresh_task = None
 async def refresh_bm25_for_app(app: FastAPI) -> None:
    """Refresh the BM25 corpus using the app engine and config."""
    try:
        await refresh_bm25_for_engine(app.state.engine, app.state.config)
    except Exception:
        logger.exception("ebook_bm25_refresh_failed")
 async def refresh_bm25_for_engine(engine: AsyncEngine, config: EbookSearchConfig) -> None:
    """Refresh the BM25 corpus using an async SQLAlchemy engine."""
    async with AsyncSession(engine) as session:
        await refresh_bm25_corpus(session, config)
    load_bm25_corpus.cache_clear()
    logger.info("ebook_bm25_corpus_cache_cleared_after_refresh")
@@ -0,0 +1,24 @@
 """FastAPI dependencies for the EPUB search app."""
 from __future__ import annotations
 from typing import Annotated
 import httpx
 from fastapi import Depends, Request
 from python.ebook_search.config import EbookSearchConfig
 def get_config(request: Request) -> EbookSearchConfig:
    """Get the loaded search config from app state."""
    return request.app.state.config
 def get_http_client(request: Request) -> httpx.AsyncClient:
    """Get the shared LLM HTTP client from app state."""
    return request.app.state.http_client
 AppConfig = Annotated[EbookSearchConfig, Depends(get_config)]
 AppHttpClient = Annotated[httpx.AsyncClient, Depends(get_http_client)]
@@ -0,0 +1,127 @@
 """Background phrase-judging tasks for the web app.
 Judging a book sends one LLM request per candidate phrase, which can take minutes, so it must
 not run inside the request where it would block the UI. Judgments run as async FastAPI
 background tasks, awaited on the event loop after the response is sent, and are tracked per
 book in app state so a second judge request for a book that is already being judged is
 rejected instead of doubling the work.
 State is loop-confined: every read and mutation happens on the event loop (async route
 handlers and async background tasks) and no critical section contains an ``await``, so each
 mutation is atomic per loop iteration and no locking is needed.
 """
 from __future__ import annotations
 import logging
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
 from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
 if TYPE_CHECKING:
    from fastapi import BackgroundTasks, FastAPI
 logger = logging.getLogger(__name__)
@dataclass
 class JudgeTaskState:
    """Running book judgments and last outcome messages, keyed by book id."""
    running_book_ids: set[int] = field(default_factory=set)
    outcome_messages: dict[int, str] = field(default_factory=dict)
 def get_judge_task_state(app: FastAPI) -> JudgeTaskState:
    """Return the app's judge task state, creating it on first use.
    Args:
        app (FastAPI): App whose state holds the judge task registry.
    Returns:
        JudgeTaskState: The shared judge task state for this app.
    """
    state = getattr(app.state, "judge_tasks", None)
    if state is None:
        state = JudgeTaskState()
        app.state.judge_tasks = state
    return state
 def start_book_phrase_judgment(app: FastAPI, background_tasks: BackgroundTasks, source_id: int) -> bool:
    """Queue judging of one book's candidate phrases as a FastAPI background task.
    The book is claimed before the response returns, so a repeated judge request cannot queue
    a second run while one is pending or running.
    Args:
        app (FastAPI): App supplying the engine, config, and judge task state.
        background_tasks (BackgroundTasks): Request's background tasks to queue the judgment on.
        source_id (int): Book to judge candidates for.
    Returns:
        bool: True when a judgment was queued, False when one is already running for this book.
    """
    state = get_judge_task_state(app)
    if source_id in state.running_book_ids:
        logger.info(f"ebook_book_phrase_judgment_already_running {source_id=}")
        return False
    state.running_book_ids.add(source_id)
    state.outcome_messages.pop(source_id, None)
    background_tasks.add_task(judge_book_phrases_for_app, app, source_id)
    logger.info(f"ebook_book_phrase_judgment_queued {source_id=}")
    return True
 async def judge_book_phrases_for_app(app: FastAPI, source_id: int) -> None:
    """Judge one book using the app engine and config, recording the outcome message.
    Args:
        app (FastAPI): App supplying the engine, config, and judge task state.
        source_id (int): Book to judge candidates for.
    """
    state = get_judge_task_state(app)
    try:
        result = await judge_candidate_phrases_for_books(app.state.engine, app.state.config, source_ids=[source_id])
        logger.info(
            f"ebook_book_phrase_judgment_complete {source_id=} {result.candidates_judged=} {result.protected_phrases=} "
            f"{result.phrase_mentions=} {result.books_failed=}"
        )
        if result.books_failed:
            message = "Judging failed; see server logs for details"
        else:
            message = (
                f"Judged {result.candidates_judged} candidates; {result.protected_phrases} protected phrases promoted"
            )
    except Exception:
        logger.exception(f"ebook_book_phrase_judgment_task_failed {source_id=}")
        message = "Judging failed; see server logs for details"
    state.running_book_ids.discard(source_id)
    state.outcome_messages[source_id] = message
 def is_judging_book(app: FastAPI, source_id: int) -> bool:
    """Report whether a judgment is currently queued or running for one book.
    Args:
        app (FastAPI): App supplying the judge task state.
        source_id (int): Book to check.
    Returns:
        bool: True while the book's judgment is pending or running.
    """
    return source_id in get_judge_task_state(app).running_book_ids
 def pop_book_judgment_outcome(app: FastAPI, source_id: int) -> str | None:
    """Return and clear the outcome message from one book's last finished judgment.
    Args:
        app (FastAPI): App supplying the judge task state.
        source_id (int): Book to fetch the outcome for.
    Returns:
        str | None: The outcome message, or None when there is nothing new to report.
    """
    return get_judge_task_state(app).outcome_messages.pop(source_id, None)
@@ -0,0 +1,91 @@
 """FastAPI HTMX app for EPUB search."""
 from __future__ import annotations
 import logging
 from contextlib import asynccontextmanager
 from typing import TYPE_CHECKING, Annotated
 import httpx
 import typer
 import uvicorn
 from fastapi import FastAPI
 from fastapi.staticfiles import StaticFiles
 from sqlalchemy.ext.asyncio import AsyncSession
 from python.common import configure_logger
 from python.ebook_search.api.bm25_tasks import cancel_bm25_refresh
 from python.ebook_search.api.routes import admin_router, health_router, page_router, search_router
 from python.ebook_search.api.web import STATIC_DIR
 from python.ebook_search.bm25_corpus import ensure_bm25_corpus
 from python.ebook_search.config import load_config
 from python.ebook_search.protected_phrases.pool import shutdown_extraction_pool
 from python.fastapi_tools import ZstdMiddleware
 from python.orm.common import get_async_postgres_engine
 if TYPE_CHECKING:
    from collections.abc import AsyncIterator
 logger = logging.getLogger(__name__)
@asynccontextmanager
 async def lifespan(app: FastAPI) -> AsyncIterator[None]:
    """Manage application startup and shutdown resources."""
    logger.info("ebook_search_startup")
    config = load_config()
    app.state.config = config
    logger.info(
        f"ebook_search_config_loaded {config.top_k=} {config.embedding_model=} {config.embedding_base_url=} "
        f"{config.vllm_base_url=} {config.rerank.enabled=} {config.phrase_matching_enabled=} {config.answer_enabled=} "
        f"library_paths={len(config.library_paths)}"
    )
    if not config.library_paths:
        logger.warning("ebook_search_no_library_paths_configured")
    # Concurrent phrase judging opens one session per book worker on this engine, so size the pool
    # to cover those plus headroom for ordinary web requests.
    app.state.engine = get_async_postgres_engine(
        name="RICHIE",
        vector_engine=True,
        pool_size=config.phrase_judge_book_workers + 10,
    )
    app.state.http_client = httpx.AsyncClient()
    async with AsyncSession(app.state.engine, expire_on_commit=False) as session:
        await ensure_bm25_corpus(session, config)
    try:
        yield
    finally:
        logger.info("ebook_search_shutdown")
        cancel_bm25_refresh(app)
        shutdown_extraction_pool()
        await app.state.http_client.aclose()
        await app.state.engine.dispose()
 def create_app() -> FastAPI:
    """Create the EPUB search web app."""
    app = FastAPI(title="EPUB Search", lifespan=lifespan)
    app.add_middleware(ZstdMiddleware)
    app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
    app.include_router(admin_router)
    app.include_router(health_router)
    app.include_router(page_router)
    app.include_router(search_router)
    return app
 def serve(
    host: Annotated[str, typer.Option("--host", "-h", help="Host to bind to")] = "127.0.0.1",
    port: Annotated[int, typer.Option("--port", "-p", help="Port to bind to")] = 8070,
    log_level: Annotated[str, typer.Option("--log-level", "-l", help="Log level")] = "INFO",
 ) -> None:
    """Start the EPUB search server."""
    configure_logger(log_level)
    uvicorn.run(create_app(), host=host, port=port)
 if __name__ == "__main__":
    typer.run(serve)
@@ -0,0 +1,13 @@
 """EPUB search web route modules."""
 from python.ebook_search.api.routes.admin import router as admin_router
 from python.ebook_search.api.routes.health import router as health_router
 from python.ebook_search.api.routes.page import router as page_router
 from python.ebook_search.api.routes.search import router as search_router
 __all__ = [
    "admin_router",
    "health_router",
    "page_router",
    "search_router",
 ]
@@ -0,0 +1,204 @@
 """Admin routes for the EPUB search web UI."""
 from __future__ import annotations
 import logging
 from fastapi import APIRouter, Request
 from fastapi.responses import HTMLResponse
 from python.ebook_search.api.bm25_tasks import schedule_bm25_refresh
 from python.ebook_search.api.dependencies import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
    AppConfig,
    AppHttpClient,
 )
 from python.ebook_search.api.web import error_response, templates
 from python.ebook_search.embeddings import embed_missing_chunks, embedding_model_stats
 from python.ebook_search.ingest import ingest_configured_paths
 from python.ebook_search.protected_phrases.generate_ngrams import generate_candidate_phrases_for_books
 from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
 from python.ebook_search.protected_phrases.store import book_ids_pending_first_judgment, corpus_phrase_stats
 from python.fastapi_tools import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
    AppAsyncEngine,
    AsyncDbSession,
 )
 logger = logging.getLogger(__name__)
 router = APIRouter(prefix="/admin")
@router.get("", response_class=HTMLResponse)
 async def admin(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
    """Render the admin page."""
    stats = await embedding_model_stats(session)
    phrase_stats = await corpus_phrase_stats(session)
    logger.info(
        f"ebook_admin_page_loaded models={len(stats)} {phrase_stats.candidate_phrases=} "
        f"{phrase_stats.protected_phrases=}"
    )
    return templates.TemplateResponse(
        request,
        "admin.html",
        {"config": config, "stats": stats, "phrase_stats": phrase_stats},
    )
@router.post("/scan", response_class=HTMLResponse)
 async def scan_library(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
    """Scan configured library paths for EPUB changes."""
    try:
        count = await ingest_configured_paths(session, config)
        await session.commit()
    except Exception as error:
        logger.exception("ebook_admin_scan_failed")
        return error_response(request, error)
    logger.info(f"ebook_admin_scan_complete {count=}")
    if count > 0:
        schedule_bm25_refresh(request.app)
    return templates.TemplateResponse(request, "partials/admin_status.html", {"message": f"Indexed {count} EPUBs"})
@router.post("/phrases/generate-all", response_class=HTMLResponse)
 async def generate_all_phrases(request: Request, config: AppConfig, engine: AppAsyncEngine) -> HTMLResponse:
    """Regenerate candidate phrases for every indexed book without LLM judging."""
    try:
        result = await generate_candidate_phrases_for_books(engine, config)
    except Exception as error:
        logger.exception("ebook_admin_generate_phrases_failed")
        return error_response(request, error)
    logger.info(
        f"ebook_admin_generate_phrases_complete {result.books_seen=} {result.books_built=} {result.candidate_phrases=}"
    )
    return templates.TemplateResponse(
        request,
        "partials/admin_status.html",
        {
            "message": (
                f"Generated phrases for {result.books_built} of {result.books_seen} books; "
                f"{result.candidate_phrases} candidates stored"
            )
        },
    )
@router.post("/phrases/judge-all", response_class=HTMLResponse)
 async def judge_all_phrases(request: Request, engine: AppAsyncEngine, config: AppConfig) -> HTMLResponse:
    """Judge unjudged candidate phrases across every indexed book."""
    return await run_phrase_judgment(request, engine, config, source_ids=None)
@router.post("/phrases/judge-missing", response_class=HTMLResponse)
 async def judge_missing_phrases(
    request: Request,
    engine: AppAsyncEngine,
    config: AppConfig,
    session: AsyncDbSession,
 ) -> HTMLResponse:
    """Judge candidate phrases only for books where judging has never run."""
    source_ids = await book_ids_pending_first_judgment(session)
    if not source_ids:
        return templates.TemplateResponse(
            request,
            "partials/admin_status.html",
            {"message": "All books with candidate phrases have been judged"},
        )
    return await run_phrase_judgment(request, engine, config, source_ids=source_ids)
 async def run_phrase_judgment(
    request: Request,
    engine: AppAsyncEngine,
    config: AppConfig,
    *,
    source_ids: list[int] | None,
 ) -> HTMLResponse:
    """Run LLM judging for candidate phrases and render the outcome as an admin status partial.
    Args:
        request (Request): Current request, for template rendering.
        engine (AppAsyncEngine): Engine used to open per-book judging sessions.
        config (AppConfig): Runtime phrase-tuning settings.
        source_ids (list[int] | None): Books to judge; ``None`` judges every indexed book.
    Returns:
        HTMLResponse: Status partial describing the judging outcome.
    """
    try:
        result = await judge_candidate_phrases_for_books(engine, config, source_ids=source_ids)
    except Exception as error:
        logger.exception("ebook_admin_judge_phrases_failed")
        return error_response(request, error)
    logger.info(
        f"ebook_admin_judge_phrases_complete {result.books_seen=} {result.books_judged=} {result.books_failed=} "
        f"{result.candidates_judged=} {result.protected_phrases=} {result.phrase_mentions=}"
    )
    return templates.TemplateResponse(
        request,
        "partials/admin_status.html",
        {
            "message": (
                f"Judged {result.candidates_judged} candidates across {result.books_judged} of "
                f"{result.books_seen} books; {result.protected_phrases} protected phrases, "
                f"{result.phrase_mentions} mentions"
                + (f"; {result.books_failed} books failed" if result.books_failed else "")
            )
        },
    )
@router.post("/embed-missing", response_class=HTMLResponse)
 async def embed_missing(
    request: Request,
    config: AppConfig,
    session: AsyncDbSession,
    client: AppHttpClient,
 ) -> HTMLResponse:
    """Embed chunks missing vectors for the configured model."""
    try:
        count = await embed_missing_chunks(session, client, config)
        await session.commit()
    except Exception as error:
        logger.exception("ebook_admin_embed_missing_failed")
        return error_response(request, error)
    logger.info(f"ebook_admin_embed_missing_complete {count=}")
    return templates.TemplateResponse(
        request,
        "partials/admin_status.html",
        {"message": f"Embedded {count} chunks"},
    )
@router.post("/embed-all", response_class=HTMLResponse)
 async def embed_all(
    request: Request,
    config: AppConfig,
    session: AsyncDbSession,
    client: AppHttpClient,
 ) -> HTMLResponse:
    """Embed all chunks missing vectors in fixed-size batches."""
    total = 0
    batches = 0
    try:
        while True:
            count = await embed_missing_chunks(session, client, config)
            if count == 0:
                break
            await session.commit()
            total += count
            batches += 1
            logger.info(f"ebook_admin_embed_all_batch_complete {batches=} {count=} {total=}")
    except Exception as error:
        logger.exception(f"ebook_admin_embed_all_failed {batches=} {total=}")
        return error_response(request, f"Embed all failed after {total} chunks in {batches} batches: {error}")
    logger.info(f"ebook_admin_embed_all_complete {batches=} {total=}")
    return templates.TemplateResponse(
        request,
        "partials/admin_status.html",
        {"message": f"Embedded {total} chunks in {batches} batches of {config.embedding_batch_size}"},
    )
@@ -0,0 +1,92 @@
 """Liveness and readiness routes for the EPUB search service."""
 from __future__ import annotations
 import logging
 from http import HTTPStatus
 from typing import TYPE_CHECKING
 from fastapi import APIRouter
 from fastapi.responses import JSONResponse
 from sqlalchemy import literal, select
 from sqlalchemy.exc import SQLAlchemyError
 from python.ebook_search.api.dependencies import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
    AppConfig,
    AppHttpClient,
 )
 from python.ebook_search.bm25_corpus import bm25_index_exists, bm25_index_path, read_bm25_manifest
 from python.ebook_search.llm_interface import check_chat_endpoint, check_embedding_endpoint
 from python.fastapi_tools import AsyncDbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
 if TYPE_CHECKING:
    import httpx
    from sqlalchemy.ext.asyncio import AsyncSession
    from python.ebook_search.config import EbookSearchConfig
 logger = logging.getLogger(__name__)
 router = APIRouter()
@router.get("/health")
 async def health() -> dict[str, str]:
    """Liveness probe that returns ok without touching dependencies."""
    return {"status": "ok"}
@router.get("/ready")
 async def ready(config: AppConfig, session: AsyncDbSession, client: AppHttpClient) -> JSONResponse:
    """Readiness probe reporting database, embedding endpoint, and BM25 index status."""
    database_ok = await check_database(session)
    embedding_ok = await check_embedding_endpoint(client, config)
    chat_status = await chat_endpoint_status(client, config)
    bm25_status = check_bm25_status(config)
    checks = {
        "database": "ok" if database_ok else "fail",
        "embedding": "ok" if embedding_ok else "fail",
        "chat": chat_status,
        "bm25": bm25_status,
    }
    if not database_ok:
        status = "unavailable"
        status_code = HTTPStatus.SERVICE_UNAVAILABLE
    elif not embedding_ok or chat_status == "fail" or bm25_status == "missing":
        status = "degraded"
        status_code = HTTPStatus.OK
    else:
        status = "ready"
        status_code = HTTPStatus.OK
    logger.info(f"ebook_ready_check {status=} {database_ok=} {embedding_ok=} {chat_status=} {bm25_status=}")
    return JSONResponse(content={"status": status, "checks": checks}, status_code=status_code)
 async def chat_endpoint_status(client: httpx.AsyncClient, config: EbookSearchConfig) -> str:
    """Return the answering chat endpoint status, or disabled when answers are off."""
    if not config.answer_enabled:
        return "disabled"
    return "ok" if await check_chat_endpoint(client, config) else "fail"
 async def check_database(session: AsyncSession) -> bool:
    """Return whether the database answers a trivial query."""
    try:
        await session.execute(select(literal(1)))
    except SQLAlchemyError as error:
        logger.warning(f"ebook_ready_database_unavailable {error=}")
        return False
    return True
 def check_bm25_status(config: EbookSearchConfig) -> str:
    """Return the persisted BM25 index status without loading it into memory."""
    index_path = bm25_index_path(config)
    manifest = read_bm25_manifest(index_path)
    if manifest is None or not bm25_index_exists(index_path, manifest):
        return "missing"
    if manifest.chunk_count == 0:
        return "empty"
    return "ok"
@@ -0,0 +1,187 @@
 """Page routes for the EPUB search web UI."""
 from __future__ import annotations
 import logging
 from typing import TYPE_CHECKING
 from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
 from fastapi.responses import HTMLResponse, RedirectResponse
 from sqlalchemy import func, select
 from python.ebook_search.api.dependencies import (
    AppConfig,  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
 )
 from python.ebook_search.api.judge_tasks import is_judging_book, pop_book_judgment_outcome, start_book_phrase_judgment
 from python.ebook_search.api.web import templates
 from python.ebook_search.protected_phrases.generate_ngrams import recalculate_candidate_phrases_for_book
 from python.ebook_search.protected_phrases.store import count_protected_phrases
 from python.fastapi_tools import AsyncDbSession  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
 from python.orm.richie import EbookCandidatePhrase, EbookChapter, EbookChunk, EbookProtectedPhrase, EbookSource
 if TYPE_CHECKING:
    from sqlalchemy.ext.asyncio import AsyncSession
 logger = logging.getLogger(__name__)
 router = APIRouter()
@router.get("/", response_class=HTMLResponse)
 async def index(request: Request, config: AppConfig) -> HTMLResponse:
    """Render the search page."""
    return templates.TemplateResponse(request, "search.html", {"config": config})
@router.get("/books", response_class=HTMLResponse)
 async def books(request: Request, session: AsyncDbSession) -> HTMLResponse:
    """Render the indexed books page."""
    sources = list((await session.scalars(select(EbookSource).order_by(EbookSource.title))).all())
    logger.info(f"ebook_books_page_loaded count={len(sources)}")
    return templates.TemplateResponse(request, "books.html", {"sources": sources})
 async def get_chapter_count(session: AsyncSession, book_id: int) -> int:
    """Return the number of indexed chapters for one book."""
    return await session.scalar(select(func.count(EbookChapter.id)).where(EbookChapter.source_id == book_id)) or 0
 async def get_chunk_count(session: AsyncSession, book_id: int) -> int:
    """Return the number of indexed chunks for one book."""
    return await session.scalar(select(func.count(EbookChunk.id)).where(EbookChunk.source_id == book_id)) or 0
 async def get_candidate_count(session: AsyncSession, book_id: int) -> int:
    """Return the number of indexed candidates for one book."""
    return (
        await session.scalar(select(func.count(EbookCandidatePhrase.id)).where(EbookCandidatePhrase.book_id == book_id))
        or 0
    )
 async def get_judged_candidate_count(session: AsyncSession, book_id: int) -> int:
    """Return the number of judged candidates for one book."""
    return (
        await session.scalar(
            select(func.count(EbookCandidatePhrase.id)).where(
                EbookCandidatePhrase.book_id == book_id,
                EbookCandidatePhrase.llm_judged.is_(True),
            )
        )
        or 0
    )
 async def get_candidates(session: AsyncSession, book_id: int) -> list[EbookCandidatePhrase]:
    """Return the indexed candidates for one book."""
    return list(
        await session.scalars(
            select(EbookCandidatePhrase)
            .where(EbookCandidatePhrase.book_id == book_id)
            .order_by(EbookCandidatePhrase.candidate_score.desc())
            .limit(100)
        )
    )
 async def get_protected_phrases(session: AsyncSession, book_id: int) -> list[EbookProtectedPhrase]:
    """Return the protected phrases for one book."""
    return list(
        await session.scalars(
            select(EbookProtectedPhrase)
            .where(EbookProtectedPhrase.book_id == book_id)
            .order_by(EbookProtectedPhrase.importance.desc())
            .limit(100)
        )
    )
@router.get("/books/{source_id}", response_class=HTMLResponse)
 async def book_detail(source_id: int, request: Request, session: AsyncDbSession) -> HTMLResponse:
    """Render details for one indexed book."""
    source = await session.get(EbookSource, source_id)
    phrase_status_message = None
    recalculated = request.query_params.get("phrases_recalculated")
    if recalculated is not None:
        phrase_status_message = f"Recalculated phrases; {recalculated} candidates generated"
    judgment_outcome = pop_book_judgment_outcome(request.app, source_id)
    if judgment_outcome is not None:
        phrase_status_message = judgment_outcome
    judging_in_progress = is_judging_book(request.app, source_id)
    if judging_in_progress:
        phrase_status_message = "Judging candidate phrases in the background; refresh to see progress"
    if source is not None:
        chapter_count = await get_chapter_count(session, source.id)
        chunk_count = await get_chunk_count(session, source.id)
        candidate_count = await get_candidate_count(session, source.id)
        judged_candidate_count = await get_judged_candidate_count(session, source.id)
        protected_count = await count_protected_phrases(session, source.id)
        candidates = await get_candidates(session, source.id)
        protected_phrases = await get_protected_phrases(session, source.id)
    else:
        chapter_count = 0
        chunk_count = 0
        candidate_count = 0
        judged_candidate_count = 0
        protected_count = 0
        candidates = []
        protected_phrases = []
    logger.info(
        f"ebook_book_detail_loaded {source_id=} found={source is not None} {chapter_count=} {chunk_count=} "
        f"{candidate_count=} {judged_candidate_count=} {protected_count=}"
    )
    return templates.TemplateResponse(
        request,
        "book_detail.html",
        {
            "candidate_count": candidate_count,
            "candidates": candidates,
            "chapter_count": chapter_count,
            "chunk_count": chunk_count,
            "judged_candidate_count": judged_candidate_count,
            "judging_in_progress": judging_in_progress,
            "protected_count": protected_count,
            "protected_phrases": protected_phrases,
            "phrase_status_message": phrase_status_message,
            "source": source,
        },
    )
@router.post("/books/{source_id}/recalculate-phrases")
 async def recalculate_book_phrases(source_id: int, config: AppConfig, session: AsyncDbSession) -> RedirectResponse:
    """Clear and regenerate candidate phrases for one indexed book."""
    source = await session.get(EbookSource, source_id)
    if source is None:
        raise HTTPException(status_code=404, detail="Book not found")
    try:
        result = await recalculate_candidate_phrases_for_book(session, source, config)
    except ValueError as error:
        raise HTTPException(status_code=409, detail=str(error)) from error
    logger.info(
        f"ebook_book_phrase_recalculation_complete {source_id=} {result.candidate_phrases=} "
        f"{result.deleted_candidates=} {result.deleted_protected_phrases=} {result.deleted_aliases=} "
        f"{result.deleted_mentions=}"
    )
    return RedirectResponse(
        url=f"/books/{source_id}?phrases_recalculated={result.candidate_phrases}",
        status_code=303,
    )
@router.post("/books/{source_id}/judge-phrases")
 async def judge_book_phrases(
    source_id: int,
    request: Request,
    background_tasks: BackgroundTasks,
    session: AsyncDbSession,
 ) -> RedirectResponse:
    """Queue background judging of one book's candidate phrases and return immediately."""
    source = await session.get(EbookSource, source_id)
    if source is None:
        raise HTTPException(status_code=404, detail="Book not found")
    started = start_book_phrase_judgment(request.app, background_tasks, source.id)
    logger.info(f"ebook_book_phrase_judgment_requested {source_id=} {started=}")
    return RedirectResponse(url=f"/books/{source_id}", status_code=303)
@@ -0,0 +1,123 @@
 """Search routes for the EPUB search web UI."""
 from __future__ import annotations
 import logging
 from dataclasses import replace
 from time import perf_counter
 from typing import TYPE_CHECKING, Annotated
 from fastapi import APIRouter, Form, Request
 from fastapi.responses import HTMLResponse
 from python.ebook_search.answer import answer_query
 from python.ebook_search.api.dependencies import (  # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
    AppConfig,
    AppHttpClient,
 )
 from python.ebook_search.api.web import error_response, templates
 from python.ebook_search.guardrails import (
    CitationReport,
    is_confident,
    retrieval_confidence,
    validate_citations,
 )
 from python.ebook_search.search import SearchResponse, search_ebooks
 from python.ebook_search.timing import runtime_step_from_start
 from python.fastapi_tools import AppAsyncEngine  # noqa: TC001 FastAPI resolves this annotated dependency at runtime
 if TYPE_CHECKING:
    import httpx
    from python.ebook_search.config import EbookSearchConfig
 logger = logging.getLogger(__name__)
 router = APIRouter()
 async def build_answer(
    client: httpx.AsyncClient,
    query: str,
    response: SearchResponse,
    config: EbookSearchConfig,
 ) -> tuple[str, bool, CitationReport | None]:
    """Generate the answer for a search, returning ``(answer, low_confidence, citation_report)``."""
    if not config.answer_enabled:
        logger.info("ebook_answer_skipped_disabled")
        return "Answer generation is disabled. Source chunks are shown below.", False, None
    if not is_confident(response.results, config):
        logger.info(
            f"ebook_answer_low_confidence confidence={retrieval_confidence(response.results):.4f} "
            f"{config.min_retrieval_confidence=:.4f}"
        )
        answer = (
            "Retrieval confidence is low for this query, so answer generation was skipped. "
            "Source chunks are shown below."
        )
        return answer, True, None
    try:
        answer = await answer_query(client, query, response.results, config)
    except RuntimeError as error:
        logger.warning(f"ebook_answer_request_failed_falling_back {error=}")
        return "Answer generation failed. Source chunks are still shown below.", False, None
    citation_report = None
    if config.validate_citations_enabled and response.results:
        citation_report = validate_citations(answer, len(response.results))
        if citation_report.invalid or not citation_report.grounded:
            logger.warning(f"ebook_answer_citation_issue {citation_report.invalid=} {citation_report.grounded=}")
    return answer, False, citation_report
@router.post("/search", response_class=HTMLResponse)
 async def search(
    request: Request,
    config: AppConfig,
    engine: AppAsyncEngine,
    client: AppHttpClient,
    query: Annotated[str, Form()],
    *,
    rerank: Annotated[bool, Form()] = False,
    phrase_matching: Annotated[bool, Form()] = False,
 ) -> HTMLResponse:
    """Run a search and render HTMX results."""
    try:
        response = await search_ebooks(
            engine,
            client,
            query,
            config,
            rerank=rerank,
            phrase_matching=phrase_matching,
        )
    except Exception as error:
        logger.exception("ebook_search_request_failed")
        return error_response(request, error)
    answer_start = perf_counter()
    answer, low_confidence, citation_report = await build_answer(client, query, response, config)
    answer_step_name = "Answer generation" if config.answer_enabled else "Answer skipped"
    response = replace(
        response,
        timings=(*response.timings, runtime_step_from_start(answer_step_name, answer_start)),
    )
    for step in response.timings:
        logger.info(f"ebook_search_timing {step.name=} {step.duration_ms=:.1f}")
    logger.info(
        f"ebook_search_request_complete results={len(response.results)} {response.rank_label=} "
        f"{response.total_runtime_ms=:.1f}"
    )
    return templates.TemplateResponse(
        request,
        "partials/results.html",
        {
            "answer": answer,
            "response": response,
            "low_confidence": low_confidence,
            "citation_report": citation_report,
        },
    )
@@ -0,0 +1,447 @@
 :root {
    --bg: #f4f5f7;
    --surface: #ffffff;
    --border: #e3e5ea;
    --text: #1c1f24;
    --muted: #6b7280;
    --accent: #4f46e5;
    --accent-soft: #eef0fe;
    --danger: #b42318;
    --warn-bg: #fff8eb;
    --warn-border: #e0a92e;
    --warn-text: #7a5008;
    --radius: 12px;
    --shadow: 0 1px 2px rgba(16, 24, 40, 0.04), 0 1px 3px rgba(16, 24, 40, 0.08);
 }
 html.theme-dark {
    --bg: #0f1117;
    --surface: #1a1d25;
    --border: #2b303b;
    --text: #e6e8ec;
    --muted: #9aa1ad;
    --accent: #818cf8;
    --accent-soft: #262b45;
    --danger: #f97066;
    --warn-bg: #2a2410;
    --warn-border: #b9881f;
    --warn-text: #e8c97a;
    --shadow: 0 1px 2px rgba(0, 0, 0, 0.3), 0 1px 3px rgba(0, 0, 0, 0.4);
    color-scheme: dark;
 }
 * {
    box-sizing: border-box;
 }
 body {
    margin: 0;
    background: var(--bg);
    color: var(--text);
    font-family: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
    line-height: 1.55;
 }
 main {
    max-width: 820px;
    margin: 0 auto;
    padding: 32px 20px 64px;
 }
 /* Header / nav */
 .site-header {
    background: var(--surface);
    border-bottom: 1px solid var(--border);
    position: sticky;
    top: 0;
    z-index: 10;
 }
 .site-nav {
    max-width: 820px;
    margin: 0 auto;
    padding: 12px 20px;
    display: flex;
    align-items: center;
    gap: 20px;
 }
 .brand {
    font-weight: 700;
    font-size: 1.05rem;
    color: var(--text);
    text-decoration: none;
 }
 .nav-links {
    display: flex;
    gap: 6px;
    margin-right: auto;
 }
 .nav-links a {
    padding: 6px 12px;
    border-radius: 8px;
    color: var(--muted);
    text-decoration: none;
    font-size: 0.94rem;
    transition: background 0.15s, color 0.15s;
 }
 .nav-links a:hover {
    background: var(--accent-soft);
    color: var(--accent);
 }
 .dev-toggle {
    display: inline-flex;
    align-items: center;
    gap: 6px;
    font-size: 0.85rem;
    color: var(--muted);
    cursor: pointer;
    user-select: none;
 }
 .theme-toggle {
    display: inline-flex;
    align-items: center;
    justify-content: center;
    width: 34px;
    height: 34px;
    padding: 0;
    font-size: 1rem;
    line-height: 1;
    color: var(--text);
    background: var(--bg);
    border: 1px solid var(--border);
    border-radius: 8px;
    cursor: pointer;
 }
 .theme-toggle:hover {
    border-color: var(--accent);
    filter: none;
 }
 h1 {
    font-size: 1.6rem;
    margin: 0 0 20px;
 }
 h2 {
    font-size: 1.15rem;
    margin: 0 0 8px;
 }
 /* Cards */
 .card {
    background: var(--surface);
    border: 1px solid var(--border);
    border-radius: var(--radius);
    box-shadow: var(--shadow);
    padding: 20px;
 }
 /* Search form */
 form {
    margin: 0;
 }
 label {
    font-weight: 600;
    font-size: 0.92rem;
 }
 textarea {
    display: block;
    width: 100%;
    margin: 8px 0 16px;
    padding: 12px 14px;
    font: inherit;
    color: var(--text);
    background: var(--surface);
    border: 1px solid var(--border);
    border-radius: 10px;
    resize: vertical;
    transition: border-color 0.15s, box-shadow 0.15s;
 }
 textarea:focus {
    outline: none;
    border-color: var(--accent);
    box-shadow: 0 0 0 3px var(--accent-soft);
 }
 .form-row {
    display: flex;
    align-items: center;
    justify-content: space-between;
    gap: 12px;
    flex-wrap: wrap;
 }
 .search-toggles {
    display: flex;
    flex-wrap: wrap;
    gap: 14px;
 }
 button {
    padding: 10px 20px;
    font: inherit;
    font-weight: 600;
    color: #fff;
    background: var(--accent);
    border: none;
    border-radius: 10px;
    cursor: pointer;
    transition: filter 0.15s;
 }
 button:hover {
    filter: brightness(1.08);
 }
 .check {
    display: inline-flex;
    gap: 8px;
    align-items: center;
    font-weight: 500;
    color: var(--muted);
 }
 .actions {
    display: flex;
    flex-wrap: wrap;
    gap: 12px;
    margin-bottom: 24px;
 }
 .actions-grid {
    display: grid;
    grid-template-columns: repeat(2, max-content);
 }
 /* Answer + results */
 #results {
    display: block;
    margin-top: 28px;
 }
 .rank-label {
    font-size: 0.82rem;
    font-weight: 600;
    text-transform: uppercase;
    letter-spacing: 0.04em;
    color: var(--muted);
    margin-bottom: 16px;
 }
 .answer {
    background: var(--surface);
    border: 1px solid var(--border);
    border-radius: var(--radius);
    box-shadow: var(--shadow);
    padding: 20px;
    margin-bottom: 24px;
 }
 .answer p:last-child {
    margin-bottom: 0;
 }
 .results {
    list-style: none;
    padding: 0;
    margin: 0;
    display: grid;
    gap: 16px;
 }
 .results > li {
    background: var(--surface);
    border: 1px solid var(--border);
    border-radius: var(--radius);
    box-shadow: var(--shadow);
    padding: 18px 20px;
 }
 .results h2 {
    font-size: 1.05rem;
 }
 .results h2 a {
    color: var(--text);
    text-decoration: none;
 }
 .results h2 a:hover {
    color: var(--accent);
 }
 .meta {
    color: var(--muted);
    font-size: 0.88rem;
    margin: 0 0 10px;
 }
 .scores {
    display: flex;
    flex-wrap: wrap;
    gap: 8px;
    margin: 14px 0 0;
 }
 .scores div {
    display: inline-flex;
    gap: 6px;
    align-items: baseline;
    padding: 3px 10px;
    background: var(--bg);
    border: 1px solid var(--border);
    border-radius: 999px;
    font-size: 0.78rem;
 }
 .scores dt {
    font-weight: 600;
    color: var(--muted);
 }
 .scores dd {
    margin: 0;
    font-variant-numeric: tabular-nums;
 }
 .phrase-matches {
    display: flex;
    flex-wrap: wrap;
    gap: 8px;
    align-items: baseline;
    margin: 10px 0 0;
    font-size: 0.78rem;
 }
 .phrase-matches-label {
    color: var(--muted);
    font-weight: 600;
 }
 .phrase-match {
    padding: 3px 10px;
    background: var(--bg);
    border: 1px solid var(--border);
    border-radius: 999px;
    color: var(--accent);
 }
 /* Runtime — developer diagnostics, hidden unless dev mode is on */
 .runtime {
    display: none;
    background: var(--surface);
    border: 1px solid var(--border);
    border-radius: var(--radius);
    box-shadow: var(--shadow);
    padding: 18px 20px;
    margin-bottom: 24px;
 }
 html.dev .runtime {
    display: block;
 }
 .timing-chart {
    display: grid;
    gap: 8px;
    padding: 0;
    margin: 12px 0 0;
    list-style: none;
 }
 .timing-chart li {
    display: grid;
    grid-template-columns: minmax(150px, 1fr) minmax(160px, 2fr) auto auto;
    gap: 10px;
    align-items: center;
    font-size: 0.85rem;
 }
 .timing-bar {
    height: 8px;
    overflow: hidden;
    background: var(--bg);
    border-radius: 999px;
 }
 .timing-bar span {
    display: block;
    height: 100%;
    background: var(--accent);
    border-radius: 999px;
 }
 .timing-value,
 .timing-remaining {
    color: var(--muted);
    font-variant-numeric: tabular-nums;
    text-align: right;
 }
 /* Tables */
 table {
    width: 100%;
    border-collapse: collapse;
    background: var(--surface);
    border: 1px solid var(--border);
    border-radius: var(--radius);
    overflow: hidden;
 }
 th,
 td {
    padding: 10px 14px;
    border-bottom: 1px solid var(--border);
    text-align: left;
    font-size: 0.9rem;
 }
 th {
    font-weight: 600;
    color: var(--muted);
    background: var(--bg);
 }
 tbody tr:last-child td {
    border-bottom: none;
 }
 dl dt {
    font-weight: 600;
    color: var(--muted);
    font-size: 0.85rem;
 }
 dl dd {
    margin: 0 0 12px;
 }
 /* States */
 .error {
    color: var(--danger);
    font-weight: 600;
 }
 .notice {
    margin: 12px 0;
    padding: 10px 14px;
    border-left: 3px solid var(--warn-border);
    border-radius: 6px;
    background: var(--warn-bg);
    color: var(--warn-text);
    font-weight: 500;
 }
 .status {
    color: var(--muted);
 }
--- a/Show More
+++ b/Show More
		`@@ -0,0 +1 @@`
							`"""Web and external API adapters for EPUB search."""`