Compare commits

..
434 changed files with 10711 additions and 35078 deletions

No files matched your search

-28
View File
@@ -1,28 +0,0 @@
.git
.direnv
.mypy_cache
.pytest_cache
.ruff_cache
.venv
**/.venv
.env
.cache
.claude
.coverage
.vscode
.stfolder
.literotica_data
esphome
htmlcov
data
ebooks
__pycache__
**/__pycache__
*.pyc
*.pyo
.ebook_search_bm25
result
result-*
*.egg-info
dist
build
+2 -17
View File
@@ -8,23 +8,8 @@ on:
- cron: "0 22 * * *"
jobs:
prebuild-common:
name: prebuild-common-x86-64-v3
runs-on: nix-cache-builder
steps:
- uses: actions/checkout@v4
# portal-1 is the smallest system closure: 95% of its derivations are
# shared by all five systems, so it is a maintainable common cache seed.
# Keep going so one failing package does not stop unrelated cache entries
# from being built.
- name: Build common packages
run: nixos-rebuild build --keep-going --accept-flake-config --flake ./#portal-1
- name: Copy common packages to nix-cache
run: nix copy --accept-flake-config --to unix:///host-nix/var/nix/daemon-socket/socket .#nixosConfigurations.portal-1.config.system.build.toplevel
build:
name: build-${{ matrix.system }}
needs: prebuild-common
runs-on: self-hosted
strategy:
matrix:
@@ -32,12 +17,12 @@ jobs:
- "bob"
- "brain"
- "jeeves"
- "leviathan"
- "rhapsody-in-green"
- "portal-1"
continue-on-error: true
steps:
- uses: actions/checkout@v4
- name: Build default package
run: "nixos-rebuild build --accept-flake-config --flake ./#${{ matrix.system }}"
run: "nixos-rebuild build --flake ./#${{ matrix.system }}"
- name: copy to nix-cache
run: nix copy --accept-flake-config --to unix:///host-nix/var/nix/daemon-socket/socket .#nixosConfigurations.${{ matrix.system }}.config.system.build.toplevel
+30
View File
@@ -0,0 +1,30 @@
name: fix_eval_warnings
on:
workflow_run:
workflows: ["build_systems"]
types: [completed]
jobs:
check-warnings:
if: >-
github.event.workflow_run.conclusion != 'cancelled' &&
github.event.workflow_run.head_branch == 'main' &&
(github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'schedule')
runs-on: self-hosted
permissions:
contents: write
pull-requests: write
steps:
- uses: actions/checkout@v4
- name: Fix eval warnings
env:
GH_TOKEN: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
run: >-
nix develop .#devShells.x86_64-linux.default -c
python -m python.eval_warnings.main
--run-id "${{ github.event.workflow_run.id }}"
--repo "${{ github.repository }}"
--ollama-url "${{ secrets.OLLAMA_URL }}"
--run-url "${{ github.event.workflow_run.html_url }}"
+13 -6
View File
@@ -6,17 +6,24 @@ on:
jobs:
merge:
runs-on: self-hosted
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: merge_flake_lock_update
run: >-
python -m python.gitea_flake_lock merge
--repo "${{ github.repository }}"
run: |
pr_number=$(gh pr list --state open --author RichieCahill --label flake_lock_update --json number --jq '.[0].number')
echo "pr_number=$pr_number" >> $GITHUB_ENV
if [ -n "$pr_number" ]; then
gh pr merge "$pr_number" --rebase
else
echo "No open PR found with label flake_lock_update"
fi
env:
JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
GITEA_URL: https://gitea.tmmworkshop.com
GITHUB_TOKEN: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
+19
View File
@@ -0,0 +1,19 @@
name: pytest
on:
push:
branches:
- main
pull_request:
branches:
- main
merge_group:
jobs:
pytest:
runs-on: self-hosted
steps:
- uses: actions/checkout@v4
- name: Run tests
run: nix develop .#devShells.x86_64-linux.default -c pytest tests
-26
View File
@@ -1,26 +0,0 @@
name: test ebook search
on:
workflow_dispatch:
push:
branches:
- main
pull_request:
branches:
- main
env:
UV_PYTHON_DOWNLOADS: never
UV_CACHE_DIR: /var/cache/uv
UV_LINK_MODE: copy
jobs:
test-ebook-search:
runs-on: self-hosted
steps:
- uses: actions/checkout@v4
- name: Install dependencies
run: nix develop .#devShells.x86_64-linux.ebook-search -c uv sync --locked --project python/ebook_search/docker
- name: Run ebook search tests
run: nix develop .#devShells.x86_64-linux.ebook-search -c uv run --project python/ebook_search/docker --no-sync pytest tests/ebook_search --override-ini addopts="-n auto -ra"
+11 -13
View File
@@ -6,20 +6,18 @@ on:
jobs:
lockfile:
runs-on: self-hosted
permissions:
actions: write
contents: write
pull-requests: write
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install Nix
uses: DeterminateSystems/nix-installer-action@main
- name: Update flake.lock
run: nix flake update
- name: Create or update flake.lock PR
env:
JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
GITEA_URL: https://gitea.tmmworkshop.com
run: >-
python -m python.gitea_flake_lock update
--repo "${{ github.repository }}"
uses: DeterminateSystems/update-flake-lock@main
with:
token: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
pr-title: "Update flake.lock"
pr-labels: |
dependencies
automated
flake_lock_update
-7
View File
@@ -169,10 +169,3 @@ test.*
# Frontend build output
frontend/dist/
frontend/node_modules/
# data from testing llms
data/*
.ebook_search_bm25
# gems data
.gems
+2 -5
View File
@@ -1,17 +1,14 @@
# Generate AGE keys from SSH keys with:
# ssh-keygen -A
# nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
# update keys after addin/removing a key
# nix-shell -p sops --run "sops updatekeys users/secrets.yaml" users/secrets.yaml
keys:
- &admin_richie age1u8zj599elqqvcmhxn8zuwrufsz8w8w366d3ayrljjejljt2q45kq8mxw9c # cspell:disable-line
- &system_bob age1q47vup0tjhulkg7d6xwmdsgrw64h4ax3la3evzqpxyy4adsmk9fs56qz3y # cspell:disable-line
- &system_brain age1jhf7vm0005j60mjq63696frrmjhpy8kpc2d66mw044lqap5mjv4snmwvwm # cspell:disable-line
- &system_jeeves age13lmqgc3jvkyah5e3vcwmj4s5wsc2akctcga0lpc0x8v8du3fxprqp4ldkv # cspell:disable-line
- &system_leviathan age1l272y8udvg60z7edgje42fu49uwt4x2gxn5zvywssnv9h2krms8s094m4k # cspell:disable-line
- &system_rhapsody age1ufnewppysaq2wwcl4ugngjz8pfzc5a35yg7luq0qmuqvctajcycs5lf6k4 # cspell:disable-line
- &system_portal_1 age1vyav6kxtvt3z4vtnkkjj38eu8hlts5m7ygyckhskvalg2gpjk52su53d0a # cspell:disable-line
creation_rules:
- path_regex: users/secrets\.yaml$
@@ -21,5 +18,5 @@ creation_rules:
- *system_bob
- *system_brain
- *system_jeeves
- *system_leviathan
- *system_rhapsody
- *system_portal_1
+4 -9
View File
@@ -10,6 +10,7 @@
"aiounifi",
"alsa",
"apiclient",
"apscheduler",
"archlinux",
"ashift",
"asrouter",
@@ -39,6 +40,7 @@
"cgroupdriver",
"charliermarsh",
"Checkpointing",
"cloudflared",
"codellama",
"codezombiech",
"compactmode",
@@ -70,7 +72,6 @@
"ehci",
"emerg",
"endlessh",
"ents",
"errorlens",
"esbenp",
"esphome",
@@ -172,8 +173,6 @@
"Networkd",
"networkmanager",
"newtabpage",
"ngram",
"ngrams",
"nixfmt",
"nixos",
"nixpkgs",
@@ -205,7 +204,6 @@
"peerconnection",
"PESKYFOX",
"PGID",
"pgvector",
"pipewire",
"pkgs",
"plugdev",
@@ -244,7 +242,6 @@
"referer",
"REFERERS",
"relatime",
"rerank",
"Rhosts",
"ripgrep",
"roboto",
@@ -300,9 +297,7 @@
"uiprotect",
"uitour",
"unifi",
"unjudged",
"unrar",
"unstorable",
"unsubmitted",
"uptimekuma",
"urlbar",
@@ -313,7 +308,6 @@
"usernamehw",
"userprefs",
"vaninventory",
"vdev",
"vfat",
"victron",
"virt",
@@ -330,12 +324,13 @@
"xcursorgen",
"xdist",
"xhci",
"yake",
"yazi",
"yubikey",
"yubioath",
"yzhang",
"zeroconf",
"zerotier",
"zerotierone",
"zoxide",
"zram",
"zstd"
+12
View File
@@ -0,0 +1,12 @@
## Dev environment tips
- use treefmt to format all files
- make python code ruff compliant
- use pytest to test python code
- always use the minimum amount of complexity
- if judgment calls are easy to reverse make them. if not ask me first
- Match existing code style.
- Use builtin helpers getenv() over os.environ.get.
- Prefer single-purpose functions over “do everything” helpers.
- Avoid compatibility branches like PG_USER and POSTGRESQL_URL unless requested.
- Keep helpers only if reused or they simplify the code otherwise inline.
Generated
-1686
View File
File diff suppressed because it is too large. Load diff
-4
View File
@@ -1,4 +0,0 @@
[workspace]
resolver = "2"
members = ["rust/*"]
+23 -48
View File
@@ -1,51 +1,26 @@
# dotfiles
## Installer ISO
<!-- LINE-COUNT-START -->
This repo has **20,055** lines of technical debt.
Build a bootable NixOS ISO with the installer preinstalled:
```sh
nix build .#iso
```
Write `result/iso/nixos-zfs-installer.iso` to a USB stick (for example with `dd`) or boot it in a VM. The image is the minimal NixOS installation CD with ZFS enabled and `nixos-installer` on `PATH`. SSH is enabled and the `nixos` and `root` accounts use the password `nixos`, so you can also run the installer remotely. Once booted:
```sh
sudo nixos-installer
```
The ISO bundles the `.#installer-nixos` package, a variant of the binary that keeps its Nix store linkage instead of being patched for foreign distributions.
## Installer binary
Build the self-contained installer executable with:
```sh
nix build .#installer
```
The flake package (defined in `python/installer/package.nix`) uses the Python builder in `python/installer/build.py`, which stages only the installer modules before running PyInstaller. You can also call it directly when `pyinstaller` and `patchelf` are on `PATH`:
```sh
python -m python.installer.build --output ./nixos-installer
```
Copy `result/bin/nixos-installer` to the installer USB stick and run it as root from the NixOS live environment:
```sh
sudo ./nixos-installer
```
Validate the live environment first with:
```sh
./nixos-installer --check
```
Paste a value into the TUI encryption password field to enable LUKS during install, or set `ENCRYPT_KEY`:
```sh
sudo env ENCRYPT_KEY='change-me' ./nixos-installer
```
The binary bundles the Python runtime and only the installer modules it imports. It still expects the NixOS installer environment to provide system install tools such as `parted`, `zfs`, `zpool`, `cryptsetup`, `nixos-generate-config`, and `nixos-install`.
| File Type | Lines | Percentage |
|-----------|------:|-----------:|
| .py | 11,441 | 57.0% |
| .nix | 4,471 | 22.3% |
| .yaml | 1,121 | 5.6% |
| .html | 1,009 | 5.0% |
| .json | 555 | 2.8% |
| .yml | 479 | 2.4% |
| .toml | 290 | 1.4% |
| .css | 212 | 1.1% |
| .gitignore | 199 | 1.0% |
| .md | 75 | 0.4% |
| .cfg | 73 | 0.4% |
| .sh | 48 | 0.2% |
| .mako | 36 | 0.2% |
| .LICENSE | 21 | 0.1% |
| .conf | 17 | 0.1% |
| .Gemfile | 4 | 0.0% |
| .svg | 3 | 0.0% |
| .new | 1 | 0.0% |
<!-- LINE-COUNT-END -->
+15 -7
View File
@@ -17,11 +17,13 @@
./nix.nix
./programs.nix
./ssh.nix
./snapshot_manager.nix
];
boot = {
tmp.useTmpfs = lib.mkDefault true;
kernelPackages = lib.mkDefault pkgs.linuxPackages_6_18;
tmp.useTmpfs = true;
kernelPackages = lib.mkDefault pkgs.linuxPackages_6_12;
zfs.package = lib.mkDefault pkgs.zfs_2_4;
};
hardware.enableRedistributableFirmware = true;
@@ -35,16 +37,22 @@
nixpkgs = {
overlays = builtins.attrValues outputs.overlays;
config = {
allowUnfree = true;
};
config.allowUnfree = true;
};
services = {
dbus.implementation = "dbus";
# firmware update
fwupd.enable = true;
snapshot_manager = {
enable = lib.mkDefault true;
PYTHONPATH = "${inputs.self}/";
};
zfs = {
trim.enable = lib.mkDefault true;
autoScrub.enable = lib.mkDefault true;
};
};
powerManagement.powertop.enable = lib.mkDefault true;
-5
View File
@@ -31,14 +31,9 @@ in
"flakes"
"ca-derivations"
];
system-features = lib.mkAfter [
"gccarch-x86-64-v2"
"gccarch-x86-64-v3"
];
warn-dirty = false;
flake-registry = ""; # disable global flake registries
connect-timeout = 10;
download-buffer-size = 536870912;
fallback = true;
};
+1
View File
@@ -2,5 +2,6 @@
{
environment.systemPackages = with pkgs; [
git
my_python
];
}
@@ -22,12 +22,6 @@ hourly = 0
daily = 0
monthly = 0
["root_pool/nix_build"]
15_min = 1
hourly = 0
daily = 0
monthly = 0
["root_pool/var"]
15_min = 8
hourly = 24
@@ -1,5 +1,4 @@
{
inputs,
pkgs,
lib,
config,
@@ -7,7 +6,6 @@
}:
let
cfg = config.services.snapshot_manager;
snapshotManagerPackages = ps: with ps; [ typer ];
in
{
options = {
@@ -36,41 +34,6 @@ in
};
config = lib.mkIf cfg.enable {
nixpkgs.overlays = [
(final: _prev: {
snapshot_manager_python = final.python314.withPackages snapshotManagerPackages;
snapshot_manager_test_python = final.python314.withPackages (
ps:
snapshotManagerPackages ps
++ (with ps; [
pyfakefs
pytest
pytest-asyncio
pytest-mock
pytest-xdist
])
);
snapshot_manager_tests =
final.runCommand "snapshot-manager-tests"
{
nativeBuildInputs = [ final.snapshot_manager_test_python ];
}
''
export HOME="$TMPDIR"
cd ${inputs.self}
pytest \
-o cache_dir="$TMPDIR/pytest-cache" \
tests/test_common.py \
tests/test_signal_alert.py \
tests/test_snapshot_manager.py \
tests/test_zfs.py
touch "$out"
'';
})
];
system.checks = [ pkgs.snapshot_manager_tests ];
systemd = {
services.snapshot_manager = {
description = "ZFS Snapshot Manager";
@@ -82,7 +45,7 @@ in
};
serviceConfig = {
Type = "oneshot";
ExecStart = "${pkgs.snapshot_manager_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
ExecStart = "${pkgs.my_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
}
// lib.optionalAttrs (cfg.EnvironmentFile != null) {
EnvironmentFile = cfg.EnvironmentFile;
+42
View File
@@ -0,0 +1,42 @@
{ pkgs, ... }:
{
boot = {
kernelPackages = pkgs.linuxPackages_6_18;
zfs.package = pkgs.zfs_2_4;
};
hardware.bluetooth = {
enable = true;
powerOnBoot = true;
};
# rtkit is optional but recommended for pipewire
security.rtkit.enable = true;
services = {
displayManager.sddm = {
enable = true;
wayland.enable = true;
};
desktopManager.plasma6.enable = true;
xserver = {
enable = true;
xkb = {
layout = "us";
variant = "";
};
};
pulseaudio.enable = false;
pipewire = {
enable = true;
alsa.enable = true;
alsa.support32Bit = true;
pulse.enable = true;
wireplumber.enable = true;
};
};
}
-17
View File
@@ -1,17 +0,0 @@
{ pkgs, ... }:
{
imports = [
./kde.nix
./pipewire.nix
];
boot = {
kernelPackages = pkgs.linuxPackages_6_18;
zfs.package = pkgs.zfs_2_4;
};
hardware.bluetooth = {
enable = true;
powerOnBoot = true;
};
}
-45
View File
@@ -1,45 +0,0 @@
{ pkgs, ... }:
{
environment.plasma6.excludePackages = with pkgs.kdePackages; [
aurorae # theme
elisa # music player
kate # text editor
kconfig # config editor
khelpcenter # help center
kinfocenter # system info
konsole # terminal
krdp # remote desktop
ktexteditor # thing for kate
okular # pdf reader
plasma-keyboard # used by plasma-keyboard KCM
plasma-sdk # plasma development tools
plasma-workspace-wallpapers # wallpapers
qrca # qr code scanner
qtvirtualkeyboard # virtual keyboard
union # theme
];
# disable KDE PIM (Personal Information Management) applications
programs.kde-pim.enable = false;
services = {
# removes screen reader and speech dispatcher from the system
orca.enable = false;
speechd.enable = false;
displayManager.sddm = {
enable = true;
wayland.enable = true;
};
desktopManager.plasma6.enable = true;
xserver = {
enable = true;
xkb = {
layout = "us";
variant = "";
};
};
};
}
-16
View File
@@ -1,16 +0,0 @@
{
# rtkit is optional but recommended for pipewire
security.rtkit.enable = true;
services = {
pulseaudio.enable = false;
pipewire = {
enable = true;
alsa.enable = true;
alsa.support32Bit = true;
pulse.enable = true;
wireplumber.enable = true;
};
};
}
-1
View File
@@ -17,7 +17,6 @@
logDriver = "local";
storageDriver = "overlay2";
daemon.settings = {
live-restore = false;
experimental = true;
exec-opts = [ "native.cgroupdriver=systemd" ];
log-opts = {
-256
View File
@@ -1,256 +0,0 @@
{
config,
lib,
pkgs,
...
}:
let
monitoringInterface = "tailscale0";
nodeTextfileDir = "/var/lib/prometheus-node-exporter-textfile";
mkProcessNameTemplate =
perPid: template: if perPid then "${template}:{{.PID}}:{{.StartTime}}" else template;
mkProcessMatchers = perPid: [
{
name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Module}}";
cmdline = [ "^/nix/store[^ ]*/bin/python[^ ]* -m (?P<Module>[^ ]+)" ];
}
{
name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Wrapped}}";
cmdline = [
"^/nix/store[^ ]*/bin/python[^ ]* /nix/store[^ ]*/bin/\\.?(?P<Wrapped>[^ /]+?)(?:-wrapped)?(?:\\s|$)"
];
}
{
name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Wrapped}}";
cmdline = [
"^/nix/store[^ ]*/bin/node /nix/store[^ ]*-(?P<Wrapped>[A-Za-z0-9._+-]+)-[0-9][^ /]*/"
];
}
{
name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Wrapped}}";
cmdline = [ "^/nix/store[^ ]*/(?:bin/|lib/[^ ]*/)?\\.?(?P<Wrapped>[^ /]+?)(?:-wrapped)?(?:\\s|$)" ];
}
{
name = mkProcessNameTemplate perPid "{{.Username}}:{{.ExeBase}}";
cmdline = [ ".+" ];
}
];
perPidConfig = pkgs.writeText "process-exporter-per-pid.yaml" (
builtins.toJSON {
process_names = mkProcessMatchers true;
}
);
zpoolLatencyScript = pkgs.writeShellScript "zpool-latency-exporter" ''
set -euo pipefail
out_dir=${lib.escapeShellArg nodeTextfileDir}
host=${lib.escapeShellArg config.networking.hostName}
tmp_file="$(mktemp "$out_dir/zpool.prom.XXXXXX")"
trap 'rm -f "$tmp_file"' EXIT
pools="$(zpool list -H -o name | paste -sd, -)"
cat >"$tmp_file" <<'EOF'
# HELP zpool_iostat_total_wait_read_ns Average total read wait time reported by zpool iostat.
# TYPE zpool_iostat_total_wait_read_ns gauge
# HELP zpool_iostat_total_wait_write_ns Average total write wait time reported by zpool iostat.
# TYPE zpool_iostat_total_wait_write_ns gauge
# HELP zpool_iostat_disk_wait_read_ns Average disk read wait time reported by zpool iostat.
# TYPE zpool_iostat_disk_wait_read_ns gauge
# HELP zpool_iostat_disk_wait_write_ns Average disk write wait time reported by zpool iostat.
# TYPE zpool_iostat_disk_wait_write_ns gauge
# HELP zpool_iostat_syncq_wait_read_ns Average synchronous queue read wait time reported by zpool iostat.
# TYPE zpool_iostat_syncq_wait_read_ns gauge
# HELP zpool_iostat_syncq_wait_write_ns Average synchronous queue write wait time reported by zpool iostat.
# TYPE zpool_iostat_syncq_wait_write_ns gauge
# HELP zpool_iostat_asyncq_wait_read_ns Average asynchronous queue read wait time reported by zpool iostat.
# TYPE zpool_iostat_asyncq_wait_read_ns gauge
# HELP zpool_iostat_asyncq_wait_write_ns Average asynchronous queue write wait time reported by zpool iostat.
# TYPE zpool_iostat_asyncq_wait_write_ns gauge
EOF
zpool iostat -Hplvy -y 1 1 | awk -F '\t' -v host="$host" -v pools="$pools" '
function esc(str, out) {
out = str
gsub(/\\/, "\\\\", out)
gsub(/"/, "\\\"", out)
return out
}
function emit(metric, pool, vdev, value) {
if (value == "" || value == "-") {
return
}
printf "%s{host=\"%s\",pool=\"%s\",vdev=\"%s\"} %s\n",
metric,
esc(host),
esc(pool),
esc(vdev),
value
}
BEGIN {
split(pools, pool_names, ",")
for (idx in pool_names) {
if (pool_names[idx] != "") {
known_pools[pool_names[idx]] = 1
}
}
}
NF == 0 {
next
}
{
row_name = $1
if (row_name in known_pools) {
current_pool = row_name
current_vdev = "_pool"
} else if (current_pool == "") {
next
} else {
current_vdev = row_name
}
emit("zpool_iostat_total_wait_read_ns", current_pool, current_vdev, $8)
emit("zpool_iostat_total_wait_write_ns", current_pool, current_vdev, $9)
emit("zpool_iostat_disk_wait_read_ns", current_pool, current_vdev, $10)
emit("zpool_iostat_disk_wait_write_ns", current_pool, current_vdev, $11)
emit("zpool_iostat_syncq_wait_read_ns", current_pool, current_vdev, $12)
emit("zpool_iostat_syncq_wait_write_ns", current_pool, current_vdev, $13)
emit("zpool_iostat_asyncq_wait_read_ns", current_pool, current_vdev, $14)
emit("zpool_iostat_asyncq_wait_write_ns", current_pool, current_vdev, $15)
}
' >>"$tmp_file"
mv "$tmp_file" "$out_dir/zpool.prom"
trap - EXIT
'';
in
{
networking.firewall.interfaces.${monitoringInterface}.allowedTCPPorts = [
9100
9134
9256
9257
9633
];
services.prometheus.exporters = {
node = {
enable = true;
enabledCollectors = [
"pressure"
"processes"
"systemd"
];
extraFlags = [ "--collector.textfile.directory=${nodeTextfileDir}" ];
};
process = {
enable = true;
user = "root";
group = "root";
settings.process_names = mkProcessMatchers false;
extraFlags = [
"-gather-smaps=false"
"-remove-empty-groups=true"
"-threads=false"
];
};
smartctl.enable = true;
zfs.enable = true;
};
programs.atop = {
enable = true;
atopService.enable = true;
atopRotateTimer.enable = true;
atopacctService.enable = true;
settings.interval = 30;
};
systemd = {
services = {
prometheus-process-pid-exporter = {
description = "Prometheus process exporter with per-PID naming";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
serviceConfig = {
ExecStart = ''
${pkgs.prometheus-process-exporter}/bin/process-exporter \
--web.listen-address 0.0.0.0:9257 \
--config.path ${perPidConfig} \
-children=false \
-gather-smaps=false \
-remove-empty-groups=true \
-threads=false
'';
User = "root";
Group = "root";
Restart = "always";
WorkingDirectory = "/tmp";
CapabilityBoundingSet = [ "" ];
DeviceAllow = [ "" ];
LockPersonality = true;
MemoryDenyWriteExecute = true;
NoNewPrivileges = true;
PrivateDevices = true;
PrivateTmp = true;
ProtectClock = true;
ProtectControlGroups = true;
ProtectHome = true;
ProtectHostname = true;
ProtectKernelLogs = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectSystem = "strict";
RemoveIPC = true;
RestrictAddressFamilies = [
"AF_INET"
"AF_INET6"
];
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
SystemCallArchitectures = "native";
UMask = "0077";
};
};
zpool-latency-exporter = {
description = "Exports ZFS latency metrics for node_exporter textfile collection";
after = [ "zfs-import.target" ];
requires = [ "zfs-import.target" ];
path = [
config.boot.zfs.package
pkgs.coreutils
pkgs.gawk
];
serviceConfig = {
Type = "oneshot";
ExecStart = zpoolLatencyScript;
};
};
};
timers.zpool-latency-exporter = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnBootSec = "2m";
OnUnitActiveSec = "60s";
Unit = "zpool-latency-exporter.service";
};
};
tmpfiles.rules = [ "d ${nodeTextfileDir} 0755 root root - -" ];
};
}
-46
View File
@@ -1,46 +0,0 @@
{ pkgs, ... }:
{
environment.systemPackages = [
pkgs.signal-cli
];
users = {
groups.signal-cli = { };
users.signal-cli = {
isSystemUser = true;
group = "signal-cli";
home = "/var/lib/signal-cli";
};
};
systemd.tmpfiles.rules = [
"d /var/lib/signal-cli 0700 signal-cli signal-cli - -"
"Z /var/lib/signal-cli - signal-cli signal-cli - -"
];
systemd.services.signal-cli = {
description = "Signal CLI JSON-RPC service";
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
unitConfig.RequiresMountsFor = [ "/var/lib/signal-cli" ];
serviceConfig = {
Type = "simple";
User = "signal-cli";
Group = "signal-cli";
ExecStart = "${pkgs.signal-cli}/bin/signal-cli --data-dir /var/lib/signal-cli daemon --socket /run/signal-cli/socket";
Restart = "on-failure";
RestartSec = "5s";
SuccessExitStatus = 143;
RuntimeDirectory = "signal-cli";
RuntimeDirectoryMode = "0750";
UMask = "0007";
NoNewPrivileges = true;
PrivateTmp = true;
ProtectHome = true;
ProtectSystem = "strict";
ReadWritePaths = [ "/var/lib/signal-cli" ];
};
};
}
+1
View File
@@ -1,6 +1,7 @@
{ pkgs, ... }:
{
environment.systemPackages = with pkgs; [
mangohud
steam-run
];
hardware.steam-hardware.enable = true;
+1 -1
View File
@@ -12,7 +12,7 @@
brain.id = "SSCGIPI-IV3VYKB-TRNIJE3-COV4T2H-CDBER7F-I2CGHYA-NWOEUDU-3T5QAAN"; # cspell:disable-line
ipad.id = "KI76T3X-SFUGV2L-VSNYTKR-TSIUV5L-SHWD3HE-GQRGRCN-GY4UFMD-CW6Z6AX"; # cspell:disable-line
jeeves.id = "ICRHXZW-ECYJCUZ-I4CZ64R-3XRK7CG-LL2HAAK-FGOHD22-BQA4AI6-5OAL6AG"; # cspell:disable-line
phone.id = "JPVQKQW-CFXOJXT-Q5G5F3H-QIDHDRE-GKHPTQB-GXZUQSP-U7FR7F7-INP3AAH"; # cspell:disable-line
phone.id = "TBRULKD-7DZPGGZ-F6LLB7J-MSO54AY-7KLPBIN-QOFK6PX-W2HBEWI-PHM2CQI"; # cspell:disable-line
rhapsody-in-green.id = "ASL3KC4-3XEN6PA-7BQBRKE-A7JXLI6-DJT43BY-Q4WPOER-7UALUAZ-VTPQ6Q4"; # cspell:disable-line
};
};
-37
View File
@@ -1,37 +0,0 @@
{
config,
inputs,
...
}:
{
nix.settings = {
trusted-substituters = [ "http://jeeves:5000" ];
substituters = [ "http://jeeves:5000/?priority=1&want-mass-query=true" ];
trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
};
services.tailscale = {
enable = true;
openFirewall = true;
authKeyFile = config.sops.secrets.tailscale_auth_key.path;
# OAuth client secrets create ephemeral nodes by default. NixOS machines
# are persistent and should enroll without interactive device approval.
authKeyParameters = {
ephemeral = false;
preauthorized = true;
};
extraUpFlags = [ "--advertise-tags=tag:nixos" ];
};
sops = {
age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
secrets.tailscale_auth_key = {
sopsFile = "${inputs.self}/users/secrets.yaml";
owner = "root";
mode = "0400";
};
};
}
+1 -1
View File
@@ -4,7 +4,7 @@
flags = [ "--accept-flake-config" ];
randomizedDelaySec = "1h";
persistent = true;
flake = "git+https://gitea.tmmworkshop.com/richie/dotfiles?ref=main";
flake = "github:RichieCahill/dotfiles";
allowReboot = true;
dates = "Sat *-*-* 06:00:00";
};
-15
View File
@@ -1,15 +0,0 @@
{
nixpkgs = {
hostPlatform = {
system = "x86_64-linux";
gcc = {
arch = "x86-64-v3";
tune = "generic";
};
};
# These patches repair tests and bundled dependencies that are sensitive
# to the compiler flags used by the x86-64-v3 package set.
overlays = [ (import ./patches) ];
};
}
@@ -1,20 +0,0 @@
Subject: [PATCH] abseil: include BMI2 intrinsics through the public header
GCC and Clang reject direct inclusion of bmi2intrin.h. Include immintrin.h
instead so that the compiler supplies the required intrinsic setup when
BMI2 is enabled, including builds targeting x86-64-v3.
This patch is shared by the vendored Abseil copies in Electron, rusty_v8
(Deno), and Signal's WebRTC build.
--- a/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
+++ b/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
@@ -226,7 +226,7 @@
#endif
#ifdef __BMI2__
-#include <bmi2intrin.h>
+#include <immintrin.h>
#endif // __BMI2__
namespace absl {
@@ -1,38 +0,0 @@
# Abseil accepted the upstream fix: https://github.com/abseil/abseil-cpp/pull/2071
# Keep this workaround until Electron, Deno's rusty_v8, and Signal's WebRTC
# update their bundled Abseil copies to include it.
{ prev }:
let
patchAbseilBmi2Include =
package:
package.overrideAttrs (old: {
# GCC and Clang require the public umbrella header for BMI2 intrinsics.
patches = (old.patches or [ ]) ++ [ ./bmi2-public-header.patch ];
});
electron43Unwrapped = patchAbseilBmi2Include prev.electron_43.unwrapped;
electron43 = prev.electron_43.override {
electron-unwrapped = electron43Unwrapped;
};
signalCallPackage =
path: args:
let
package = prev.callPackage path args;
in
if builtins.baseNameOf path == "webrtc.nix" then patchAbseilBmi2Include package else package;
in
prev.lib.optionalAttrs ((prev.stdenv.hostPlatform.gcc.arch or null) == "x86-64-v3") {
deno =
let
librusty_v8 = patchAbseilBmi2Include prev.deno.passthru.librusty_v8;
in
prev.deno.override { inherit librusty_v8; };
electron_43 = electron43;
signal-desktop = prev.signal-desktop.override {
electron_43 = electron43;
callPackage = signalCallPackage;
};
}
@@ -1,19 +0,0 @@
Subject: [PATCH] Compare font content in the overlap-removal regression
PFA_SKIP ends with a separator, producing an empty prefix that matches every
line. Remove the empty prefix for this test so glyph differences are checked,
and assert that tx exits successfully before inspecting the output.
--- a/tests/tx_test.py
+++ b/tests/tx_test.py
@@ -1064,6 +1064,8 @@
output_path = get_temp_file_path()
args = [TOOL, '-t1', '+V', '-o', output_path, input_path]
- subprocess.call(args)
- assert differ([expected_path, output_path, '-s', PFA_SKIP[0]])
+ subprocess.check_call(args)
+ # An empty skip prefix matches every line and hides font differences.
+ skip_headers = SPLIT_MARKER.join(filter(None, PFA_SKIP[0].split(SPLIT_MARKER)))
+ assert differ([expected_path, output_path, '-s', skip_headers])
@@ -1,10 +0,0 @@
{ afdko }:
afdko.overridePythonAttrs (old: {
# FMA changes overlap-removal coordinates by 0.01 units on x86-64-v3.
# Separate multiply/add rounding reproduces the reference Type 1 font exactly.
env = (old.env or { }) // {
NIX_CFLAGS_COMPILE = (old.env.NIX_CFLAGS_COMPILE or "") + " -ffp-contract=off";
};
patches = (old.patches or [ ]) ++ [ ./check-overlap-font-content.patch ];
})
@@ -1,16 +0,0 @@
_final: prev:
(import ./abseil { inherit prev; })
// {
gnutls = import ./gnutls { inherit (prev) gnutls; };
prometheus = import ./prometheus { inherit (prev) prometheus; };
zopfli = import ./zopfli { inherit (prev) zopfli; };
pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
(_pythonFinal: pythonPrev: {
afdko = import ./afdko { inherit (pythonPrev) afdko; };
psutil = import ./psutil { inherit (pythonPrev) psutil; };
pytest-xdist = import ./pytest-xdist { inherit (pythonPrev) pytest-xdist; };
scipy = import ./scipy { inherit (pythonPrev) scipy; };
})
];
}
@@ -1,6 +0,0 @@
{ gnutls }:
gnutls.overrideAttrs (old: {
# Keep the UDP handshake test enabled on loaded builders by waiting for
# the server to bind its socket. Kept as a standalone patch for upstream.
patches = (old.patches or [ ]) ++ [ ./udp-server-readiness.patch ];
})
@@ -1,70 +0,0 @@
Subject: [PATCH] tests: wait for the UDP server socket before connecting
A fixed four-second sleep does not guarantee that gnutls-serv has bound
its UDP socket on a busy builder. Poll the local IPv4 UDP endpoint using
the existing ss/netstat discovery, with the same retry budget as the TCP
helper. Fail early if the server exits, and retain the original handshake
checks in serv-udp.sh.
Use flags common to ss and BSD/Linux netstat. Match the local endpoint
and complete port number, excluding TCP, IPv6-only and peer endpoints.
--- a/tests/scripts/common.sh
+++ b/tests/scripts/common.sh
@@ -185,10 +185,55 @@
fi
}
+check_if_udp_port_bound() {
+ local PORT=$1
+ have_port_finder
+ # Use only -an, which is shared by ss and BSD/Linux netstat. UDP has
+ # no LISTEN state. Match the local IPv4 endpoint, not a peer port or
+ # a longer port number. serv-udp.sh connects to 127.0.0.1;
+ # listen_socket() in serv.c binds IPv4 separately and requests
+ # IPV6_V6ONLY=1 for its IPv6 socket.
+ $PFCMD -an | awk -v port="$PORT" '
+ $1 == "udp" || $1 == "udp4" {
+ # ss includes a state column; netstat does not.
+ address = ($2 == "UNCONN" || $2 == "ESTAB") ? $5 : $4
+ if (address ~ ("^[0-9.]+[.:]" port "$") ||
+ address == "*." port)
+ found = 1
+ }
+ END { exit !found }
+ '
+}
+
wait_udp_server() {
local PID=$1
+ local ret
trap "test -n \"${PID}\" && kill ${PID};exit 1" 1 15 2
- sleep 4
+ local i=0
+ # Use the same retry budget as wait_for_port(), but also stop if the
+ # server exits before binding its socket.
+ while test $i -lt 90; do
+ if ! kill -0 "$PID" 2>/dev/null; then
+ fail "" "UDP server $PID exited before binding port $PORT"
+ fi
+ # Contain have_port_finder's exit so a skip also stops the server.
+ if (check_if_udp_port_bound "$PORT"); then
+ return 0
+ else
+ ret=$?
+ if test "$ret" = 77; then
+ kill "$PID" 2>/dev/null || :
+ wait "$PID" 2>/dev/null || :
+ exit 77
+ fi
+ fi
+ i=$((i + 1))
+ if test $i -lt 90; then
+ echo "try $i: waiting for UDP port $PORT"
+ sleep 2
+ fi
+ done
+ fail "$PID" "UDP server $PORT did not come up"
}
create_testdir() {
@@ -1,180 +0,0 @@
#!/usr/bin/env python3
"""Exercise patched common.sh without building GnuTLS (Python standard library only).
Usage: python3 verify-readiness.py /path/to/patched/tests/scripts/common.sh
Set SHELL to test another shell, and NETSTAT to test a netstat outside PATH.
"""
# Use unittest so this upstream companion tool needs no pytest installation.
# ruff: noqa: PT009
import os
import shutil
import socket
import subprocess
import sys
import tempfile
import time
import unittest
from pathlib import Path
COMMON = str(Path(sys.argv.pop(1)).resolve())
SHELL = os.environ.get("SHELL", "/bin/sh")
class ReadinessTests(unittest.TestCase):
"""Check endpoint parsing and the server startup lifecycle."""
def setUp(self) -> None:
"""Create a socket-listing fixture for each check."""
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name)
self.fixture = self.root / "sockets"
self.fixture.write_text("")
self.finder = self.root / "port-finder"
self.finder.write_text('#!/bin/sh\ncat "$SOCKET_FIXTURE"\n')
self.finder.chmod(0o755)
def run_shell(self, body: str, **env: str) -> subprocess.CompletedProcess[str]:
"""Source the actual helper and run a shell scenario."""
return subprocess.run(
[SHELL, "-c", '. "$COMMON"\n' + body],
env={
**os.environ,
"COMMON": COMMON,
"SOCKET_FIXTURE": str(self.fixture),
"PFCMD": str(self.finder),
"PORT": "12345",
**env,
},
capture_output=True,
text=True,
timeout=20,
check=False,
)
def test_socket_formats_and_false_matches(self) -> None:
"""Accept IPv4 UDP local endpoints and reject unrelated sockets."""
cases = [
("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*", True),
("udp UNCONN 0 0 127.0.0.1:12345 0.0.0.0:*", True),
("udp 0 0 0.0.0.0:12345 0.0.0.0:*", True),
("udp4 0 0 *.12345 *.*", True),
("udp 0 0 127.0.0.1.12345 *.*", True),
("udp 0 0 *.12345 *.*", True),
("udp UNCONN 0 0 0.0.0.0:123456 0.0.0.0:*", False),
("udp 0 0 0.0.0.0:123456 0.0.0.0:*", False),
("udp ESTAB 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
("udp 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
("tcp LISTEN 0 128 0.0.0.0:12345 0.0.0.0:*", False),
("tcp 0 0 0.0.0.0:12345 0.0.0.0:* LISTEN", False),
("udp UNCONN 0 0 [::]:12345 [::]:*", False),
("udp UNCONN 0 0 *:12345 *:*", False),
("udp6 0 0 :::12345 :::*", False),
("udp6 0 0 *.12345 *.*", False),
("", False),
]
for row, ready in cases:
with self.subTest(row=row):
self.fixture.write_text(row + "\n")
result = self.run_shell('check_if_udp_port_bound "$PORT"')
self.assertEqual(result.returncode, 0 if ready else 1, result.stderr)
def test_exited_server_fails_immediately(self) -> None:
"""Fail without sleeping when the server has already exited."""
result = self.run_shell(
'true &\npid=$!\nwait "$pid"\nsleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$pid"'
)
self.assertEqual(result.returncode, 1)
self.assertIn("exited before binding", result.stderr)
self.assertNotIn("unexpected sleep", result.stderr)
def test_ready_socket_does_not_sleep(self) -> None:
"""Check readiness before the first sleep."""
self.fixture.write_text("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*\n")
result = self.run_shell('sleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$$"')
self.assertEqual(result.returncode, 0, result.stderr)
self.assertNotIn("unexpected sleep", result.stderr)
def test_missing_port_finders_skip(self) -> None:
"""Skip and stop the live server when no finder is available."""
# have_port_finder also tries these paths independently of PATH.
if any(os.access(f"{directory}/ss", os.X_OK) for directory in ("/sbin", "/usr/sbin", "/usr/local/sbin")):
self.skipTest("an absolute ss path cannot be hidden by this PATH-only fixture")
with subprocess.Popen(["sleep", "60"]) as server:
try:
result = self.run_shell(
'unset PFCMD\nPATH=/nonexistent\nwait_udp_server "$SERVER_PID"',
SERVER_PID=str(server.pid),
)
self.assertEqual(result.returncode, 77)
self.assertIn("neither ss nor netstat found", result.stderr)
server.wait(timeout=3)
self.assertLess(server.returncode, 0)
finally:
if server.poll() is None:
server.kill()
def test_timeout_is_bounded_and_cleans_up(self) -> None:
"""Stop polling after the retry budget and terminate the server."""
# Only accelerate the polling delay; keep a real live server process.
self.finder.write_text('#!/bin/sh\necho probe >&2\ncat "$SOCKET_FIXTURE"\n')
with subprocess.Popen(["sleep", "60"]) as server:
try:
result = self.run_shell(
'sleep() { echo polling-sleep; }\nwait_udp_server "$SERVER_PID"',
SERVER_PID=str(server.pid),
)
self.assertEqual(result.returncode, 1)
self.assertIn("did not come up", result.stderr)
self.assertEqual(result.stderr.count("probe\n"), 90)
self.assertEqual(result.stdout.count("polling-sleep"), 89)
server.wait(timeout=3)
self.assertLess(server.returncode, 0)
finally:
if server.poll() is None:
server.kill()
def test_server_exits_while_waiting(self) -> None:
"""Detect a startup failure that happens after polling begins."""
result = self.run_shell('sleep 1 &\npid=$!\nwait_udp_server "$pid"')
self.assertEqual(result.returncode, 1)
self.assertIn("exited before binding", result.stderr)
self.assertIn("waiting for UDP port", result.stdout)
def test_real_socket_delayed_beyond_four_seconds(self) -> None:
"""Wait for a real delayed bind with each installed port finder."""
finders = [shutil.which("ss"), os.environ.get("NETSTAT") or shutil.which("netstat")]
finders = [finder for finder in finders if finder]
if not finders:
self.skipTest("neither ss nor netstat available")
for finder in finders:
with self.subTest(finder=finder):
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
sock.bind(("127.0.0.1", 0))
port = sock.getsockname()[1]
code = (
"import socket,time,sys; time.sleep(6); "
"s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); "
"s.bind(('127.0.0.1',int(sys.argv[1]))); time.sleep(30)"
)
with subprocess.Popen([sys.executable, "-c", code, str(port)]) as server:
try:
started = time.monotonic()
result = self.run_shell(
'wait_udp_server "$SERVER_PID"',
SERVER_PID=str(server.pid),
PORT=str(port),
PFCMD=finder,
)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertGreaterEqual(time.monotonic() - started, 6)
self.assertIsNone(server.poll())
finally:
server.terminate()
server.wait(timeout=3)
if __name__ == "__main__":
unittest.main()
@@ -1,36 +0,0 @@
Subject: [PATCH] tests: finish parsing before inspecting editor state
EditorState creation has a 20 ms parsing budget. A descheduled test can
therefore observe an incomplete tree. Finish these small test documents
without an interactive deadline and publish the result with a transaction.
Keep the original completion and vector-matching assertions enabled.
--- a/module/codemirror-promql/src/test/utils-test.ts
+++ b/module/codemirror-promql/src/test/utils-test.ts
@@ -13,7 +13,7 @@
import { parser } from '@prometheus-io/lezer-promql';
import { EditorState } from '@codemirror/state';
-import { LRLanguage } from '@codemirror/language';
+import { ensureSyntaxTree, LRLanguage } from '@codemirror/language';
import nock from 'nock';
import path from 'path';
import { fileURLToPath } from 'url';
@@ -23,10 +23,16 @@
const __dirname = path.dirname(fileURLToPath(import.meta.url));
export function createEditorState(expr: string): EditorState {
- return EditorState.create({
+ const state = EditorState.create({
doc: expr,
extensions: lightPromQLSyntax,
});
+ // These tests need a complete tree, independent of the editor's time budget.
+ if (!ensureSyntaxTree(state, state.doc.length, Infinity)) {
+ throw new Error('Unable to parse the test expression');
+ }
+ // Publish the completed parse so syntaxTree(state) sees it too.
+ return state.update({}).state;
}
export function mockPrometheusServer(): void {
@@ -1,17 +0,0 @@
{ prometheus }:
prometheus.overrideAttrs (
old:
let
assets = old.passthru.assets.overrideAttrs (assetsOld: {
patches = (assetsOld.patches or [ ]) ++ [ ./complete-test-parsing.patch ];
});
in
{
postPatch = builtins.replaceStrings [ "${old.passthru.assets}" ] [ "${assets}" ] (
builtins.unsafeDiscardStringContext old.postPatch
);
passthru = old.passthru // {
inherit assets;
};
}
)
@@ -1,4 +0,0 @@
{ psutil }:
psutil.overridePythonAttrs (old: {
patches = (old.patches or [ ]) ++ [ ./heap-info-zero-mmap.patch ];
})
@@ -1,17 +0,0 @@
Subject: [PATCH] Allow an empty mmap allocation total in heap_info
The allocator may satisfy all live allocations from the heap. A zero
mmap_used value is valid and depends on the worker allocation history.
Keep the heap and platform checks without requiring an mmap allocation.
--- a/tests/test_system.py
+++ b/tests/test_system.py
@@ -267,6 +267,7 @@
if MACOS:
assert m.mmap_used == 0 # not supported
else:
- assert m.mmap_used > 0
+ # A process can have no live mmap-backed malloc allocations.
+ assert m.mmap_used >= 0
if WINDOWS:
assert m.heap_count >= 0
@@ -1,29 +0,0 @@
Subject: [PATCH] tests: count replacements when checking the worker restart limit
With two workers, another in-flight test may crash after the fourth
crash requests shutdown. Either four or five failed tests is valid.
Require exactly three replacements and the failed-tests exit status,
while preserving the queued-work and no-internal-error assertions.
--- a/testing/acceptance_test.py
+++ b/testing/acceptance_test.py
@@ -1011,9 +1011,18 @@
"worker*crashed while running*",
"worker*crashed while running*",
"* xdist: maximum crashed workers reached: 3 *",
- "* 4 failed in *",
]
)
+ # A second in-flight test may crash after shutdown is requested.
+ # The restart limit constrains replacements, not concurrent failures.
+ replacements = sum(
+ line.startswith("replacing crashed worker ") for line in res.stdout.lines
+ )
+ assert replacements == 3
+ failed = res.parseoutcomes()["failed"]
+ assert failed in (4, 5)
+ res.assert_outcomes(failed=failed)
+ assert res.ret == pytest.ExitCode.TESTS_FAILED
assert "INTERNALERROR" not in res.stdout.str()
def test_max_worker_restart_die(self, pytester: pytest.Pytester) -> None:
@@ -1,12 +0,0 @@
{ pytest-xdist }:
pytest-xdist.overridePythonAttrs (old: {
patches = (old.patches or [ ]) ++ [
./concurrent-worker-crashes.patch
./worker-startup-timeout.patch
];
# The suite exercises its own worker pools. Limit the outer suite to one worker.
preCheck = builtins.replaceStrings [ "--numprocesses=$NIX_BUILD_CORES" ] [ "--numprocesses=1" ] (
old.preCheck or ""
);
})
@@ -1,19 +0,0 @@
Subject: [PATCH] tests: allow more time for remote worker events
Worker startup can exceed ten seconds on heavily loaded builders. Allow
the remote-test helper to wait up to sixty seconds for worker events.
The wait still returns as soon as an event arrives and remains bounded.
Production worker timeouts and test assertions are unchanged.
--- a/testing/test_remote.py
+++ b/testing/test_remote.py
@@ -17,7 +17,8 @@
from xdist.workermanage import WorkerController
-WAIT_TIMEOUT = 10.0
+# Allow worker events extra time on heavily loaded builders.
+WAIT_TIMEOUT = 60.0
def check_marshallable(d: object) -> None:
@@ -1,5 +0,0 @@
{ scipy }:
scipy.overridePythonAttrs (old: {
# Keep the STFT tests enabled with tolerances for x86-64-v3 rounding.
patches = (old.patches or [ ]) ++ [ ./stft-test-tolerances.patch ];
})
@@ -1,51 +0,0 @@
Subject: [PATCH] signal: allow floating-point rounding in STFT tests
Keep the STFT tests enabled for x86-64-v3 builds. Allow two float32
epsilons of relative error when comparing inverse-STFT implementations;
float64 and the existing i686 override remain unchanged. Allow one
float64 epsilon of absolute error in all three scaling round trips,
which otherwise require exact zeros (observed residual: 4e-17 for a
signal with amplitude 2).
Upstream issue: https://github.com/scipy/scipy/issues/25488
--- a/scipy/signal/tests/_scipy_spectral_test_shim.py
+++ b/scipy/signal/tests/_scipy_spectral_test_shim.py
@@ -294,7 +294,7 @@
# Adapted tolerances to account for resolution loss:
atol = np.finfo(x.dtype).resolution*2 # instead of default atol = 0
- rtol = 1e-7 # default for np.allclose()
+ rtol = max(1e-7, 2 * np.finfo(x.dtype).eps)
# Relax atol on 32-Bit platforms a bit to pass CI tests.
# - Not clear why there are discrepancies (in the FFT maybe?)
--- a/scipy/signal/tests/test_spectral.py
+++ b/scipy/signal/tests/test_spectral.py
@@ -2044,7 +2044,7 @@
# Test round trip:
x1 = istft(Zs, boundary=True, scaling='spectrum')[1]
- assert_allclose(x1, x)
+ assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
# For a Hann-windowed 256 sample length FFT, we expect a peak at
# frequency 64 (since it is 1/4 the length of X) with a height of 1
@@ -2074,7 +2074,7 @@
# Test round trip:
x1 = istft(Zp, input_onesided=False, boundary=True, scaling='psd')[1]
- assert_allclose(x1, x)
+ assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
# The power of the one-sided psd-scaled STFT can be determined
# analogously (note that the two sides are not of equal shape):
@@ -2094,7 +2094,7 @@
# Test round trip:
x1 = istft(Zp0, input_onesided=True, boundary=True, scaling='psd')[1]
- assert_allclose(x1, x)
+ assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
class TestSampledSpectralRepresentations:
@@ -1,4 +0,0 @@
{ zopfli }:
zopfli.overrideAttrs (old: {
patches = (old.patches or [ ]) ++ [ ./unaligned-match-loads.patch ];
})
@@ -1,32 +0,0 @@
Subject: [PATCH] Read unaligned match buffers with memcpy
Byte buffers need not satisfy integer alignment or aliasing requirements.
GCC 16 with x86-64-v3 vectorizes the cast loads using aligned AVX reads,
causing a segmentation fault. Copy into local integers instead.
--- a/src/zopfli/lz77.c
+++ b/src/zopfli/lz77.c
@@ -302,13 +302,20 @@
if (sizeof(size_t) == 8) {
/* 8 checks at once per array bounds check (size_t is 64-bit). */
- while (scan < safe_end && *((size_t*)scan) == *((size_t*)match)) {
+ while (scan < safe_end) {
+ size_t scan_word, match_word;
+ memcpy(&scan_word, scan, sizeof(scan_word));
+ memcpy(&match_word, match, sizeof(match_word));
+ if (scan_word != match_word) break;
scan += 8;
match += 8;
}
} else if (sizeof(unsigned int) == 4) {
/* 4 checks at once per array bounds check (unsigned int is 32-bit). */
- while (scan < safe_end
- && *((unsigned int*)scan) == *((unsigned int*)match)) {
+ while (scan < safe_end) {
+ unsigned int scan_word, match_word;
+ memcpy(&scan_word, scan, sizeof(scan_word));
+ memcpy(&match_word, match, sizeof(match_word));
+ if (scan_word != match_word) break;
scan += 4;
match += 4;
}
+11
View File
@@ -0,0 +1,11 @@
{
services.zerotierone = {
enable = true;
joinNetworks = [ "e4da7455b2ae64ca" ];
};
nix.settings = {
trusted-substituters = [ "http://192.168.90.40:5000" ];
substituters = [ "http://192.168.90.40:5000/?priority=1&want-mass-query=true" ];
trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
};
}
-26
View File
@@ -1,26 +0,0 @@
{
inputs,
lib,
pkgs,
...
}:
{
imports = [ ./snapshot.nix ];
boot.zfs = {
package = lib.mkDefault pkgs.zfs_2_4;
forceImportRoot = lib.mkDefault false;
};
services = {
snapshot_manager = {
enable = lib.mkDefault true;
PYTHONPATH = "${inputs.self}/";
};
zfs = {
trim.enable = lib.mkDefault true;
autoScrub.enable = lib.mkDefault true;
};
};
}
-76
View File
@@ -1,76 +0,0 @@
# ZFS failed root import recovery
## Fast path
If the machine fails to boot because ZFS refuses to import `root_pool`:
### GRUB
1. At the bootloader menu, select the normal NixOS entry.
2. Press `e`.
3. Find the line that starts with `linux`.
4. Append this to the end of that line:
```text
zfs_force=1
```
5. Boot once with `Ctrl+x` or `F10`.
### systemd-boot
1. At the bootloader menu, highlight the normal NixOS entry.
2. Press `e`.
3. Append this to the end of the options line:
```text
zfs_force=1
```
4. Press `Enter` to boot once.
## After boot
Run:
```bash
sudo zpool status
sudo zpool import
journalctl -b | rg "ZFS|zfs|import|root_pool"
```
## Expected result
`sudo zpool status` should show `root_pool` as `ONLINE`.
## Reboot test
Run:
```bash
sudo reboot
```
Do not add `zfs_force=1` the second time.
## If it still fails
Boot once more with:
```text
zfs_force=1
```
Then run:
```bash
sudo zpool status -v
sudo zpool history | tail -n 50
journalctl -b | rg "ZFS|zfs|import|root_pool"
```
## Notes
- Root pool name is `root_pool`.
- This is a one-time recovery path after disk moves, controller changes, dirty exports, or interrupted imports.
- Some hosts also need the LUKS unlock USB key inserted before boot.
+1
View File
@@ -0,0 +1 @@
defaultdict(set[str], {1: {'3279788.jsonl', '3540261.jsonl', '3650306.jsonl', '2901931.jsonl', '1614768.jsonl', '3140371.jsonl', '3548235.jsonl', '1993792.jsonl', '3691009.jsonl', '1414659.jsonl', '2724180.jsonl', '1974895.jsonl', '2734331.jsonl', '1156760.jsonl', '3129472.jsonl', '1103333.jsonl', '920422.jsonl', '1069680.jsonl', '3389701.jsonl', '1407502.jsonl', '2835562.jsonl', '3820601.jsonl', '551282.jsonl', '2406007.jsonl', '1783305.jsonl', '1848515.jsonl', '1265277.jsonl', '3016581.jsonl', '1834941.jsonl', '3657968.jsonl', '3809328.jsonl', '2127784.jsonl', '458059.jsonl', '2901855.jsonl', '385686.jsonl', '2314072.jsonl', '3968115.jsonl', '1080750.jsonl', '2504964.jsonl', '2368770.jsonl', '1989017.jsonl', '2302821.jsonl', '2084438.jsonl', '1465590.jsonl', '3749509.jsonl', '2028622.jsonl', '1752462.jsonl', '3880689.jsonl', '1870430.jsonl', '883298.jsonl', '2810266.jsonl', '3956172.jsonl', '466350.jsonl', '3833342.jsonl', '1941808.jsonl', '1151892.jsonl', '3029902.jsonl', '3423250.jsonl', '1655019.jsonl', '3415180.jsonl', '2690355.jsonl', '806775.jsonl', '1556284.jsonl', '3936160.jsonl', '1438699.jsonl', '1692724.jsonl', '3044082.jsonl', '3871581.jsonl', '2477674.jsonl', '2910735.jsonl', '2083221.jsonl', '3197201.jsonl', '3396628.jsonl', '1944123.jsonl', '2655340.jsonl', '1853046.jsonl', '1539860.jsonl', '2327092.jsonl', '2295460.jsonl', '2760874.jsonl', '3662012.jsonl', '2258353.jsonl', '1655035.jsonl', '584766.jsonl', '3589506.jsonl', '3333321.jsonl', '2149240.jsonl', '3089270.jsonl', '981146.jsonl', '1521142.jsonl', '3196145.jsonl', '822873.jsonl', '2583801.jsonl', '3023631.jsonl', '2082683.jsonl', '432937.jsonl', '2332140.jsonl', '222815.jsonl', '1561588.jsonl', '3287525.jsonl', '1106811.jsonl', '2546051.jsonl', '1227794.jsonl', '1825994.jsonl', '3021693.jsonl', '1135173.jsonl', '1465527.jsonl', '1562103.jsonl', '483466.jsonl', '3038483.jsonl', '1582285.jsonl', '1661483.jsonl', '3358493.jsonl', '2569231.jsonl', '2967343.jsonl', '63461.jsonl', '3004916.jsonl', '2714868.jsonl', '2542966.jsonl', '3090109.jsonl', '1662435.jsonl', '2366706.jsonl', '2604410.jsonl', '2226642.jsonl', '380838.jsonl', '3676321.jsonl', '1557349.jsonl', '2622227.jsonl', '2124076.jsonl', '2754508.jsonl', '3101954.jsonl', '2858448.jsonl', '2260602.jsonl', '2111885.jsonl', '791888.jsonl', '2467072.jsonl', '3679102.jsonl', '3142337.jsonl', '3352858.jsonl', '1544588.jsonl', '4033334.jsonl', '2072128.jsonl', '1962409.jsonl', '3539593.jsonl', '1534296.jsonl', '3638461.jsonl', '2406298.jsonl', '1455038.jsonl', '2345264.jsonl', '2572252.jsonl', '3657110.jsonl', '78066.jsonl', '3522058.jsonl', '2370417.jsonl', '2780284.jsonl', '2217500.jsonl', '3780324.jsonl', '1236038.jsonl', '995746.jsonl', '3310141.jsonl', '2863401.jsonl', '402039.jsonl', '621782.jsonl', '1921613.jsonl', '1890717.jsonl', '2967679.jsonl', '2858477.jsonl', '296484.jsonl', '2577007.jsonl', '1512793.jsonl', '2657813.jsonl', '2902918.jsonl', '2911023.jsonl', '1420300.jsonl', '930290.jsonl', '784023.jsonl', '1155580.jsonl', '1637704.jsonl', '2173155.jsonl', '2936853.jsonl', '3466262.jsonl', '2876969.jsonl', '1765591.jsonl', '1173493.jsonl', '2594589.jsonl', '3880915.jsonl', '2786298.jsonl', '3465909.jsonl', '568116.jsonl', '3452901.jsonl', '2950523.jsonl', '3309573.jsonl', '2258210.jsonl', '3076883.jsonl', '3166670.jsonl', '1030980.jsonl', '3299065.jsonl', '1676590.jsonl', '3771288.jsonl', '2602228.jsonl', '1614223.jsonl', '3441235.jsonl', '3218360.jsonl', '4026882.jsonl', '2112293.jsonl', '3329722.jsonl', '1435113.jsonl', '3306825.jsonl', '2040903.jsonl', '3578066.jsonl', '354273.jsonl', '2286891.jsonl', '3862366.jsonl', '2679507.jsonl', '2800291.jsonl', '3467967.jsonl', '1931456.jsonl', '601413.jsonl', '2229048.jsonl', '1356601.jsonl', '1774535.jsonl', '2506409.jsonl', '755110.jsonl', '3297092.jsonl', '3075825.jsonl', '1940150.jsonl', '3416788.jsonl', '3399818.jsonl', '2182741.jsonl', '783351.jsonl', '2431492.jsonl', '2945353.jsonl', '1501481.jsonl', '2651542.jsonl', '1536019.jsonl', '1091050.jsonl', '3668244.jsonl', '2598954.jsonl', '3415438.jsonl', '1738848.jsonl', '2151834.jsonl', '3388733.jsonl', '1909217.jsonl', '3213867.jsonl', '1552325.jsonl', '3990557.jsonl', '3423402.jsonl', '746106.jsonl', '786455.jsonl', '2148630.jsonl', '3872796.jsonl', '2551235.jsonl', '846557.jsonl', '1628133.jsonl', '1766820.jsonl', '1740039.jsonl', '3898812.jsonl', '3966944.jsonl', '1443136.jsonl', '1508291.jsonl', '2761338.jsonl', '2947131.jsonl', '3060604.jsonl', '1543756.jsonl', '2287089.jsonl', '2362745.jsonl', '1480233.jsonl', '1564217.jsonl', '3794626.jsonl', '1906777.jsonl', '1854136.jsonl', '1624677.jsonl', '3008141.jsonl', '2520018.jsonl', '3509776.jsonl', '3122035.jsonl', '2927611.jsonl', '462911.jsonl', '2214248.jsonl', '2171198.jsonl', '1042578.jsonl', '1248976.jsonl', '1605098.jsonl', '3619217.jsonl', '3673731.jsonl', '2092449.jsonl', '2226760.jsonl', '2875325.jsonl', '1429869.jsonl', '773647.jsonl', '3427060.jsonl', '1939851.jsonl', '2494900.jsonl', '1281179.jsonl', '2104073.Line truncated
Generated
+37 -128
View File
@@ -1,41 +1,25 @@
{
"nodes": {
"disko": {
"firefox-addons": {
"inputs": {
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1789770686,
"narHash": "sha256-uZkBR7yHdIKUFB5SZdfgh1qkGfI3XmYmI/lTiquxbck=",
"owner": "nix-community",
"repo": "disko",
"rev": "725ea35e410ad83be4931d1bff7e090eacaf3563",
"type": "github"
"dir": "pkgs/firefox-addons",
"lastModified": 1773979456,
"narHash": "sha256-9kBMJ5IvxqNlkkj/swmE8uK1Sc7TL/LIRUI958m7uBM=",
"owner": "rycee",
"repo": "nur-expressions",
"rev": "81e28f47ac18d9e89513929c77e711e657b64851",
"type": "gitlab"
},
"original": {
"owner": "nix-community",
"repo": "disko",
"type": "github"
}
},
"flake-utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1731533236,
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
"dir": "pkgs/firefox-addons",
"owner": "rycee",
"repo": "nur-expressions",
"type": "gitlab"
}
},
"home-manager": {
@@ -45,11 +29,11 @@
]
},
"locked": {
"lastModified": 1791293500,
"narHash": "sha256-3hwJd2/vG/VYV1P/g7oDgdbNsWODhN5F//Od+hPMm2k=",
"lastModified": 1774007980,
"narHash": "sha256-FOnZjElEI8pqqCvB6K/1JRHTE8o4rer8driivTpq2uo=",
"owner": "nix-community",
"repo": "home-manager",
"rev": "0e11b7bcba1d74ef8ea10bfdfb1e1a5811a21087",
"rev": "9670de2921812bc4e0452f6e3efd8c859696c183",
"type": "github"
},
"original": {
@@ -58,38 +42,13 @@
"type": "github"
}
},
"jellyswarrm": {
"inputs": {
"flake-utils": "flake-utils",
"nixpkgs": [
"nixpkgs"
],
"rust-overlay": "rust-overlay"
},
"locked": {
"lastModified": 1791127101,
"narHash": "sha256-SmrBBhcv9iNHA2vlafi7Zpe8bkcsMjVNSmCCVf5/jrQ=",
"owner": "LLukas22",
"repo": "Jellyswarrm",
"rev": "0b4301c6c74efff9c3634296913032579c98faf6",
"type": "github"
},
"original": {
"owner": "LLukas22",
"repo": "Jellyswarrm",
"type": "github"
}
},
"nixos-hardware": {
"inputs": {
"nixpkgs": "nixpkgs"
},
"locked": {
"lastModified": 1791128201,
"narHash": "sha256-rlsYY/Dg/LhbIj51Aa6TLvRpZeb3KZgUTd7llQ5XcJE=",
"lastModified": 1774018263,
"narHash": "sha256-HHYEwK1A22aSaxv2ibhMMkKvrDGKGlA/qObG4smrSqc=",
"owner": "nixos",
"repo": "nixos-hardware",
"rev": "31cc5f4d9b9ba601071e8b8504601b9b176e2756",
"rev": "2d4b4717b2534fad5c715968c1cece04a172b365",
"type": "github"
},
"original": {
@@ -101,24 +60,27 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1789546076,
"narHash": "sha256-vWkSk5bbfTqdtMoSgD9FshACO8JCvXTFi+3cqEp0mH0=",
"rev": "b1b875982b17dabde9b4a37f3e229e74913e6db3",
"type": "tarball",
"url": "https://releases.nixos.org/nixos/unstable/nixos-26.11pre1074753.b1b875982b17/nixexprs.tar.xz"
"lastModified": 1773821835,
"narHash": "sha256-TJ3lSQtW0E2JrznGVm8hOQGVpXjJyXY2guAxku2O9A4=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "b40629efe5d6ec48dd1efba650c797ddbd39ace0",
"type": "github"
},
"original": {
"type": "tarball",
"url": "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz"
"owner": "nixos",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs-master": {
"locked": {
"lastModified": 1791296764,
"narHash": "sha256-N2b6Uxdr1MFhscEPqDyntTTtGZASgq3XYE0ujRzfeNI=",
"lastModified": 1774051532,
"narHash": "sha256-d3CGMweyYIcPuTj5BKq+1Lx4zwlgL31nVtN647tOZKo=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "a15afac8691bd24f746255ffbd76306693e4cf7a",
"rev": "8620c0b5cc8fbe76502442181be1d0514bc3a1b7",
"type": "github"
},
"original": {
@@ -144,54 +106,16 @@
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1791260994,
"narHash": "sha256-Miqqk/ammqnTUxaoCyvtwoPeLbI1ksFyLxi/CazZpWY=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "151fa4e8ddfdd8dd25d945ad94ed54a13de9f6e4",
"type": "github"
},
"original": {
"owner": "nixos",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"disko": "disko",
"firefox-addons": "firefox-addons",
"home-manager": "home-manager",
"jellyswarrm": "jellyswarrm",
"nixos-hardware": "nixos-hardware",
"nixpkgs": "nixpkgs_2",
"nixpkgs": "nixpkgs",
"nixpkgs-master": "nixpkgs-master",
"nixpkgs-stable": "nixpkgs-stable",
"sops-nix": "sops-nix",
"systems": "systems_2"
}
},
"rust-overlay": {
"inputs": {
"nixpkgs": [
"jellyswarrm",
"nixpkgs"
]
},
"locked": {
"lastModified": 1779419951,
"narHash": "sha256-dMX0PUslUHPajP6o8FEoRdFv9afq/dec4POR0vVfjK4=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "5b5c521d6cae9ef4aa32f888eb2c0ce595c9be52",
"type": "github"
},
"original": {
"owner": "oxalica",
"repo": "rust-overlay",
"type": "github"
"systems": "systems"
}
},
"sops-nix": {
@@ -201,11 +125,11 @@
]
},
"locked": {
"lastModified": 1791103873,
"narHash": "sha256-nFxM+pKoZ8LJAEnUXARyCaOAloWgaW9kZQOSjWzKcTE=",
"lastModified": 1773889674,
"narHash": "sha256-+ycaiVAk3MEshJTg35cBTUa0MizGiS+bgpYw/f8ohkg=",
"owner": "Mic92",
"repo": "sops-nix",
"rev": "dcd241ba97088c22569d1573286e1b9daad340c0",
"rev": "29b6519f3e0780452bca0ac0be4584f04ac16cc5",
"type": "github"
},
"original": {
@@ -215,21 +139,6 @@
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"systems_2": {
"locked": {
"lastModified": 1689347949,
"narHash": "sha256-12tWmuL2zgBgZkdoB6qXZsgJEH9LR3oUgpaQq2RbI80=",
+36 -51
View File
@@ -21,23 +21,18 @@
nixos-hardware.url = "github:nixos/nixos-hardware/master";
jellyswarrm = {
url = "github:LLukas22/Jellyswarrm";
inputs.nixpkgs.follows = "nixpkgs";
};
home-manager = {
url = "github:nix-community/home-manager";
inputs.nixpkgs.follows = "nixpkgs";
};
sops-nix = {
url = "github:Mic92/sops-nix";
firefox-addons = {
url = "gitlab:rycee/nur-expressions?dir=pkgs/firefox-addons";
inputs.nixpkgs.follows = "nixpkgs";
};
disko = {
url = "github:nix-community/disko";
sops-nix = {
url = "github:Mic92/sops-nix";
inputs.nixpkgs.follows = "nixpkgs";
};
};
@@ -70,48 +65,38 @@
devShells = forEachSystem (pkgs: import ./shell.nix { inherit pkgs; });
formatter = forEachSystem (pkgs: pkgs.treefmt);
packages = forEachSystem (
pkgs:
let
installer = pkgs.callPackage ./python/installer/package.nix { };
installer-nixos = pkgs.callPackage ./python/installer/package.nix { patchElf = false; };
in
{
inherit installer installer-nixos;
default = installer;
}
// lib.optionalAttrs (pkgs.stdenv.hostPlatform.system == "x86_64-linux") {
iso = self.nixosConfigurations.iso.config.system.build.isoImage;
}
);
apps = forEachSystem (
pkgs:
let
system = pkgs.stdenv.hostPlatform.system;
installer = {
type = "app";
program = "${self.packages.${system}.installer}/bin/nixos-installer";
meta.description = "One-file NixOS ZFS installer.";
};
in
{
inherit installer;
default = installer;
}
);
nixosConfigurations =
let
hosts = builtins.attrNames (
lib.filterAttrs (_: type: type == "directory") (builtins.readDir ./systems)
);
mkHost =
name:
lib.nixosSystem {
modules = [ ./systems/${name} ];
specialArgs = { inherit inputs outputs; };
};
in
lib.genAttrs hosts mkHost;
nixosConfigurations = {
bob = lib.nixosSystem {
modules = [
./systems/bob
];
specialArgs = { inherit inputs outputs; };
};
brain = lib.nixosSystem {
modules = [
./systems/brain
];
specialArgs = { inherit inputs outputs; };
};
jeeves = lib.nixosSystem {
modules = [
./systems/jeeves
];
specialArgs = { inherit inputs outputs; };
};
rhapsody-in-green = lib.nixosSystem {
modules = [
./systems/rhapsody-in-green
];
specialArgs = { inherit inputs outputs; };
};
leviathan = lib.nixosSystem {
modules = [
./systems/leviathan
];
specialArgs = { inherit inputs outputs; };
};
};
};
}
+24
View File
@@ -0,0 +1,24 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
lerna-debug.log*
node_modules
dist
dist-ssr
*.local
# Editor directories and files
.vscode/*
!.vscode/extensions.json
.idea
.DS_Store
*.suo
*.ntvs*
*.njsproj
*.sln
*.sw?
+32 -9
View File
@@ -1,13 +1,13 @@
{ inputs, ... }:
{
# Additional package sets are accessible through `pkgs.<name>`.
# When applied, the stable nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.stable'
stable = final: _prev: {
stable = import inputs.nixpkgs-stable {
system = final.stdenv.hostPlatform.system;
config.allowUnfree = true;
};
};
# When applied, the master nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.master'
master = final: _prev: {
master = import inputs.nixpkgs-master {
system = final.stdenv.hostPlatform.system;
@@ -15,12 +15,35 @@
};
};
# Baseline x86-64 (v1) packages for prebuilt applications that should not
# inherit an x86-64-v3 host platform.
x86-v1 = final: _prev: {
x86-v1 = import inputs.nixpkgs {
system = final.stdenv.hostPlatform.system;
config.allowUnfree = true;
};
python-env = final: _prev: {
my_python = final.python314.withPackages (
ps: with ps; [
alembic
apprise
apscheduler
fastapi
fastapi-cli
httpx
mypy
polars
psycopg
pydantic
pyfakefs
pytest
pytest-cov
pytest-mock
pytest-xdist
python-multipart
ruff
scalene
sqlalchemy
sqlalchemy
tenacity
textual
tinytuya
typer
websockets
]
);
};
}
+40 -7
View File
@@ -3,13 +3,44 @@ name = "system_tools"
version = "0.1.0"
description = ""
authors = [{ name = "Richie Cahill", email = "richie@tmmworkshop.com" }]
requires-python = "~=3.14.0"
requires-python = "~=3.13.0"
readme = "README.md"
license = "MIT"
# these dependencies are a best effort and aren't guaranteed to work
# for up-to-date dependencies, see overlays/default.nix
dependencies = [
"alembic",
"apprise",
"apscheduler",
"httpx",
"python-multipart",
"polars",
"psycopg[binary]",
"pydantic",
"pyyaml",
"sqlalchemy",
"typer",
"websockets",
]
[project.scripts]
database = "python.database_cli:app"
van-inventory = "python.van_inventory.main:serve"
[dependency-groups]
dev = [
"mypy",
"pyfakefs",
"pytest-cov",
"pytest-mock",
"pytest-xdist",
"pytest",
"ruff",
]
[tool.ruff]
target-version = "py314"
target-version = "py313"
line-length = 120
@@ -19,8 +50,6 @@ lint.ignore = [
"COM812", # (TEMP) conflicts when used with the formatter
"ISC001", # (TEMP) conflicts when used with the formatter
"S603", # (PERM) This is known to cause a false positive
"S607", # (PERM) This is becoming a consistent annoyance
"CPY001", # (PERM) I don't include the license in every file
]
[tool.ruff.lint.per-file-ignores]
@@ -49,10 +78,15 @@ lint.ignore = [
"python/congress_tracker/**" = [
"TC003", # (perm) this creates issues because sqlalchemy uses these at runtime
]
"python/eval_warnings/**" = [
"S607", # (perm) gh and git are expected on PATH in the runner environment
]
"python/alembic/**" = [
"INP001", # (perm) this creates LSP issues for alembic
]
"python/signal_bot/**" = [
"D107", # (perm) class docstrings cover __init__
]
[tool.ruff.lint.pydocstyle]
convention = "google"
@@ -75,7 +109,6 @@ exclude_lines = [
]
[tool.pytest.ini_options]
addopts = "-n auto -ra --ignore=tests/ebook_search"
asyncio_mode = "auto"
addopts = "-n auto -ra"
testpaths = ["tests"]
# --cov=system_tools --cov-report=term-missing --cov-report=xml --cov-report=html --cov-branch
-1
View File
@@ -81,7 +81,6 @@ def include_name(
"""
if type_ == "schema":
# allows a database with multiple schemas to have separate alembic revisions
return name == target_metadata.schema
return True
@@ -1,187 +0,0 @@
"""removed ds table from richie DB.
Revision ID: c8a794340928
Revises: 6b275323f435
Create Date: 2026-03-29 15:29:23.643146
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "c8a794340928"
down_revision: str | None = "6b275323f435"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("vote_record", schema=schema)
op.drop_index(op.f("ix_vote_congress_chamber"), table_name="vote", schema=schema)
op.drop_index(op.f("ix_vote_date"), table_name="vote", schema=schema)
op.drop_index(op.f("ix_legislator_bioguide_id"), table_name="legislator", schema=schema)
op.drop_table("legislator", schema=schema)
op.drop_table("vote", schema=schema)
op.drop_index(op.f("ix_bill_congress"), table_name="bill", schema=schema)
op.drop_table("bill", schema=schema)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"vote",
sa.Column("congress", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("chamber", sa.VARCHAR(), autoincrement=False, nullable=False),
sa.Column("session", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("number", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("vote_type", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("question", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("result", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("result_text", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("vote_date", sa.DATE(), autoincrement=False, nullable=False),
sa.Column("yea_count", sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column("nay_count", sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column("not_voting_count", sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column("present_count", sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column("bill_id", sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
sa.Column(
"created",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.Column(
"updated",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.ForeignKeyConstraint(["bill_id"], [f"{schema}.bill.id"], name=op.f("fk_vote_bill_id_bill")),
sa.PrimaryKeyConstraint("id", name=op.f("pk_vote")),
sa.UniqueConstraint(
"congress",
"chamber",
"session",
"number",
name=op.f("uq_vote_congress_chamber_session_number"),
postgresql_include=[],
postgresql_nulls_not_distinct=False,
),
schema=schema,
)
op.create_index(op.f("ix_vote_date"), "vote", ["vote_date"], unique=False, schema=schema)
op.create_index(op.f("ix_vote_congress_chamber"), "vote", ["congress", "chamber"], unique=False, schema=schema)
op.create_table(
"vote_record",
sa.Column("vote_id", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("legislator_id", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("position", sa.VARCHAR(), autoincrement=False, nullable=False),
sa.ForeignKeyConstraint(
["legislator_id"],
[f"{schema}.legislator.id"],
name=op.f("fk_vote_record_legislator_id_legislator"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["vote_id"], [f"{schema}.vote.id"], name=op.f("fk_vote_record_vote_id_vote"), ondelete="CASCADE"
),
sa.PrimaryKeyConstraint("vote_id", "legislator_id", name=op.f("pk_vote_record")),
schema=schema,
)
op.create_table(
"legislator",
sa.Column("bioguide_id", sa.TEXT(), autoincrement=False, nullable=False),
sa.Column("thomas_id", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("lis_id", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("govtrack_id", sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column("opensecrets_id", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("fec_ids", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("first_name", sa.VARCHAR(), autoincrement=False, nullable=False),
sa.Column("last_name", sa.VARCHAR(), autoincrement=False, nullable=False),
sa.Column("official_full_name", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("nickname", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("birthday", sa.DATE(), autoincrement=False, nullable=True),
sa.Column("gender", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("current_party", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("current_state", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("current_district", sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column("current_chamber", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
sa.Column(
"created",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.Column(
"updated",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_legislator")),
schema=schema,
)
op.create_index(op.f("ix_legislator_bioguide_id"), "legislator", ["bioguide_id"], unique=True, schema=schema)
op.create_table(
"bill",
sa.Column("congress", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("bill_type", sa.VARCHAR(), autoincrement=False, nullable=False),
sa.Column("number", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("title", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("title_short", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("official_title", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("status", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("status_at", sa.DATE(), autoincrement=False, nullable=True),
sa.Column("sponsor_bioguide_id", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("subjects_top_term", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
sa.Column(
"created",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.Column(
"updated",
postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
autoincrement=False,
nullable=False,
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_bill")),
sa.UniqueConstraint(
"congress",
"bill_type",
"number",
name=op.f("uq_bill_congress_type_number"),
postgresql_include=[],
postgresql_nulls_not_distinct=False,
),
schema=schema,
)
op.create_index(op.f("ix_bill_congress"), "bill", ["congress"], unique=False, schema=schema)
# ### end Alembic commands ###
@@ -1,93 +0,0 @@
"""adding audiobook libreary metadata.
Revision ID: d7864d1ffc17
Revises: c8a794340928
Create Date: 2026-06-03 20:24:09.200837
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "d7864d1ffc17"
down_revision: str | None = "c8a794340928"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"audiobook_author",
sa.Column("name", sa.String(), nullable=False),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_audiobook_author")),
sa.UniqueConstraint("name", name=op.f("uq_audiobook_author_name")),
schema=schema,
)
op.create_table(
"audiobook_series",
sa.Column("name", sa.String(), nullable=False),
sa.Column("author_id", sa.Integer(), nullable=False),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["author_id"],
[f"{schema}.audiobook_author.id"],
name=op.f("fk_audiobook_series_author_id_audiobook_author"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_audiobook_series")),
sa.UniqueConstraint("author_id", "name", name=op.f("uq_audiobook_series_author_id")),
schema=schema,
)
op.create_table(
"audiobook",
sa.Column("title", sa.String(), nullable=False),
sa.Column("author_id", sa.Integer(), nullable=False),
sa.Column("series_id", sa.Integer(), nullable=True),
sa.Column("series_index", sa.Integer(), nullable=False),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["author_id"],
[f"{schema}.audiobook_author.id"],
name=op.f("fk_audiobook_author_id_audiobook_author"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["series_id"],
[f"{schema}.audiobook_series.id"],
name=op.f("fk_audiobook_series_id_audiobook_series"),
ondelete="SET NULL",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_audiobook")),
schema=schema,
)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("audiobook", schema=schema)
op.drop_table("audiobook_series", schema=schema)
op.drop_table("audiobook_author", schema=schema)
# ### end Alembic commands ###
@@ -1,200 +0,0 @@
"""add ebook search tables.
Revision ID: 2db132cace1a
Revises: b3c60cc5beb5
Create Date: 2026-06-10 22:10:54.379159
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pgvector
import sqlalchemy as sa
from alembic import op
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "2db132cace1a"
down_revision: str | None = "b3c60cc5beb5"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"ebook_embedding_model",
sa.Column("name", sa.String(), nullable=False),
sa.Column("dimension", sa.Integer(), nullable=False),
sa.Column("is_default", sa.Boolean(), nullable=False),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_embedding_model")),
sa.UniqueConstraint("name", name=op.f("uq_ebook_embedding_model_name")),
schema=schema,
)
op.create_table(
"ebook_source",
sa.Column("title", sa.String(), nullable=False),
sa.Column("author", sa.String(), nullable=True),
sa.Column("language", sa.String(), nullable=True),
sa.Column("publisher", sa.String(), nullable=True),
sa.Column("identifier", sa.String(), nullable=True),
sa.Column("file_path", sa.String(), nullable=False),
sa.Column("file_sha256", sa.String(length=64), nullable=False),
sa.Column("file_mtime", sa.DateTime(timezone=True), nullable=False),
sa.Column("file_size", sa.BigInteger(), nullable=False),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_source")),
sa.UniqueConstraint("file_path", name=op.f("uq_ebook_source_file_path")),
sa.UniqueConstraint("file_sha256", name=op.f("uq_ebook_source_file_sha256")),
schema=schema,
)
op.create_table(
"ebook_chapter",
sa.Column("source_id", sa.Integer(), nullable=False),
sa.Column("spine_index", sa.Integer(), nullable=False),
sa.Column("title", sa.String(), nullable=True),
sa.Column("href", sa.String(), nullable=True),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["source_id"],
[f"{schema}.ebook_source.id"],
name=op.f("fk_ebook_chapter_source_id_ebook_source"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chapter")),
sa.UniqueConstraint("source_id", "spine_index", name=op.f("uq_ebook_chapter_source_id")),
schema=schema,
)
op.create_table(
"ebook_chunk",
sa.Column("source_id", sa.Integer(), nullable=False),
sa.Column("chapter_id", sa.Integer(), nullable=True),
sa.Column("chunk_index", sa.Integer(), nullable=False),
sa.Column("text", sa.String(), nullable=False),
sa.Column("token_start", sa.Integer(), nullable=False),
sa.Column("token_count", sa.Integer(), nullable=False),
sa.Column("page_label", sa.String(), nullable=True),
sa.Column("content_sha256", sa.String(length=64), nullable=False),
sa.Column("search_text", sa.String(), nullable=False),
sa.Column("id", sa.BigInteger(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["chapter_id"],
[f"{schema}.ebook_chapter.id"],
name=op.f("fk_ebook_chunk_chapter_id_ebook_chapter"),
ondelete="SET NULL",
),
sa.ForeignKeyConstraint(
["source_id"],
[f"{schema}.ebook_source.id"],
name=op.f("fk_ebook_chunk_source_id_ebook_source"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk")),
sa.UniqueConstraint("source_id", "chunk_index", name="uq_ebook_chunk_source_id_chunk_index"),
sa.UniqueConstraint("source_id", "content_sha256", name="uq_ebook_chunk_source_id_content_sha256"),
schema=schema,
)
op.create_table(
"ebook_chunk_embedding_1024",
sa.Column("chunk_id", sa.BigInteger(), nullable=False),
sa.Column("model_id", sa.Integer(), nullable=False),
sa.Column("embedding", pgvector.sqlalchemy.vector.VECTOR(dim=1024), nullable=False),
sa.Column("id", sa.BigInteger(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["chunk_id"],
[f"{schema}.ebook_chunk.id"],
name=op.f("fk_ebook_chunk_embedding_1024_chunk_id_ebook_chunk"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["model_id"],
[f"{schema}.ebook_embedding_model.id"],
name=op.f("fk_ebook_chunk_embedding_1024_model_id_ebook_embedding_model"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk_embedding_1024")),
sa.UniqueConstraint("chunk_id", "model_id", name=op.f("uq_ebook_chunk_embedding_1024_chunk_id")),
schema=schema,
)
op.create_table(
"ebook_chunk_embedding_2560",
sa.Column("chunk_id", sa.BigInteger(), nullable=False),
sa.Column("model_id", sa.Integer(), nullable=False),
sa.Column("embedding", pgvector.sqlalchemy.vector.VECTOR(dim=2560), nullable=False),
sa.Column("id", sa.BigInteger(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["chunk_id"],
[f"{schema}.ebook_chunk.id"],
name=op.f("fk_ebook_chunk_embedding_2560_chunk_id_ebook_chunk"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["model_id"],
[f"{schema}.ebook_embedding_model.id"],
name=op.f("fk_ebook_chunk_embedding_2560_model_id_ebook_embedding_model"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk_embedding_2560")),
sa.UniqueConstraint("chunk_id", "model_id", name=op.f("uq_ebook_chunk_embedding_2560_chunk_id")),
schema=schema,
)
op.create_table(
"ebook_chunk_embedding_4096",
sa.Column("chunk_id", sa.BigInteger(), nullable=False),
sa.Column("model_id", sa.Integer(), nullable=False),
sa.Column("embedding", pgvector.sqlalchemy.vector.VECTOR(dim=4096), nullable=False),
sa.Column("id", sa.BigInteger(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["chunk_id"],
[f"{schema}.ebook_chunk.id"],
name=op.f("fk_ebook_chunk_embedding_4096_chunk_id_ebook_chunk"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["model_id"],
[f"{schema}.ebook_embedding_model.id"],
name=op.f("fk_ebook_chunk_embedding_4096_model_id_ebook_embedding_model"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk_embedding_4096")),
sa.UniqueConstraint("chunk_id", "model_id", name=op.f("uq_ebook_chunk_embedding_4096_chunk_id")),
schema=schema,
)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("ebook_chunk_embedding_4096", schema=schema)
op.drop_table("ebook_chunk_embedding_2560", schema=schema)
op.drop_table("ebook_chunk_embedding_1024", schema=schema)
op.drop_table("ebook_chunk", schema=schema)
op.drop_table("ebook_chapter", schema=schema)
op.drop_table("ebook_source", schema=schema)
op.drop_table("ebook_embedding_model", schema=schema)
# ### end Alembic commands ###
@@ -1,63 +0,0 @@
"""updated series_index to float and added UniqueConstraint to audiobook and audiobook_author.
Revision ID: b3c60cc5beb5
Revises: d7864d1ffc17
Create Date: 2026-06-10 20:02:43.073725
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "b3c60cc5beb5"
down_revision: str | None = "d7864d1ffc17"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column(
"audiobook",
"series_index",
existing_type=sa.INTEGER(),
type_=sa.Float(),
existing_nullable=False,
schema=schema,
)
op.create_unique_constraint(
op.f("uq_audiobook_author_id"),
"audiobook",
["author_id", "series_id", "title"],
schema=schema,
postgresql_nulls_not_distinct=True,
)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_constraint(op.f("uq_audiobook_author_id"), "audiobook", schema=schema, type_="unique")
op.alter_column(
"audiobook",
"series_index",
existing_type=sa.Float(),
type_=sa.INTEGER(),
existing_nullable=False,
schema=schema,
)
# ### end Alembic commands ###
@@ -1,54 +0,0 @@
"""add 1024 ebook embedding cosine index.
Revision ID: c460105682d2
Revises: 2db132cace1a
Create Date: 2026-06-13 19:53:45.680289
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from alembic import op
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "c460105682d2"
down_revision: str | None = "2db132cace1a"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_index(
"ix_ebook_chunk_embedding_1024_embedding_cosine",
"ebook_chunk_embedding_1024",
["embedding"],
unique=False,
schema=schema,
postgresql_using="hnsw",
postgresql_ops={"embedding": "vector_cosine_ops"},
)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(
"ix_ebook_chunk_embedding_1024_embedding_cosine",
table_name="ebook_chunk_embedding_1024",
schema=schema,
postgresql_using="hnsw",
postgresql_ops={"embedding": "vector_cosine_ops"},
)
# ### end Alembic commands ###
@@ -1,103 +0,0 @@
"""adding haproxy data.
Revision ID: 96d72c748c24
Revises: c460105682d2
Create Date: 2026-06-23 16:37:17.768851
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "96d72c748c24"
down_revision: str | None = "c460105682d2"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"haproxy_request",
sa.Column("line_hash", sa.String(), nullable=False),
sa.Column("requested_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("client_ip", sa.String(), nullable=False),
sa.Column("client_port", sa.Integer(), nullable=False),
sa.Column("frontend", sa.String(), nullable=False),
sa.Column("ssl", sa.Boolean(), nullable=False),
sa.Column("backend", sa.String(), nullable=False),
sa.Column("server", sa.String(), nullable=False),
sa.Column("time_request", sa.Integer(), nullable=False),
sa.Column("time_queue", sa.Integer(), nullable=False),
sa.Column("time_connect", sa.Integer(), nullable=False),
sa.Column("time_response", sa.Integer(), nullable=False),
sa.Column("time_total", sa.Integer(), nullable=False),
sa.Column("status_code", sa.Integer(), nullable=False),
sa.Column("bytes_read", sa.BigInteger(), nullable=False),
sa.Column("termination_state", sa.String(), nullable=False),
sa.Column("active_connections", sa.Integer(), nullable=False),
sa.Column("frontend_connections", sa.Integer(), nullable=False),
sa.Column("backend_connections", sa.Integer(), nullable=False),
sa.Column("server_connections", sa.Integer(), nullable=False),
sa.Column("retries", sa.Integer(), nullable=False),
sa.Column("server_queue", sa.Integer(), nullable=False),
sa.Column("backend_queue", sa.Integer(), nullable=False),
sa.Column("host", sa.String(), nullable=True),
sa.Column("user_agent", sa.String(), nullable=True),
sa.Column("method", sa.String(), nullable=False),
sa.Column("target", sa.String(), nullable=False),
sa.Column("path", sa.String(), nullable=False),
sa.Column("query", sa.String(), nullable=True),
sa.Column("http_version", sa.String(), nullable=False),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_haproxy_request")),
sa.UniqueConstraint("line_hash", name=op.f("uq_haproxy_request_line_hash")),
schema=schema,
)
op.create_index(op.f("ix_haproxy_request_backend"), "haproxy_request", ["backend"], unique=False, schema=schema)
op.create_index(op.f("ix_haproxy_request_client_ip"), "haproxy_request", ["client_ip"], unique=False, schema=schema)
op.create_index(op.f("ix_haproxy_request_host"), "haproxy_request", ["host"], unique=False, schema=schema)
op.create_index(op.f("ix_haproxy_request_path"), "haproxy_request", ["path"], unique=False, schema=schema)
op.create_index(
op.f("ix_haproxy_request_requested_at"), "haproxy_request", ["requested_at"], unique=False, schema=schema
)
op.create_index(
op.f("ix_haproxy_request_status_code"), "haproxy_request", ["status_code"], unique=False, schema=schema
)
op.create_index(
op.f("ix_haproxy_request_time_response"), "haproxy_request", ["time_response"], unique=False, schema=schema
)
op.create_index(
op.f("ix_haproxy_request_user_agent"), "haproxy_request", ["user_agent"], unique=False, schema=schema
)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f("ix_haproxy_request_user_agent"), table_name="haproxy_request", schema=schema)
op.drop_index(op.f("ix_haproxy_request_time_response"), table_name="haproxy_request", schema=schema)
op.drop_index(op.f("ix_haproxy_request_status_code"), table_name="haproxy_request", schema=schema)
op.drop_index(op.f("ix_haproxy_request_requested_at"), table_name="haproxy_request", schema=schema)
op.drop_index(op.f("ix_haproxy_request_path"), table_name="haproxy_request", schema=schema)
op.drop_index(op.f("ix_haproxy_request_host"), table_name="haproxy_request", schema=schema)
op.drop_index(op.f("ix_haproxy_request_client_ip"), table_name="haproxy_request", schema=schema)
op.drop_index(op.f("ix_haproxy_request_backend"), table_name="haproxy_request", schema=schema)
op.drop_table("haproxy_request", schema=schema)
# ### end Alembic commands ###
@@ -1,206 +0,0 @@
"""adding Phrase metadata tables.
Revision ID: dddee09eddcc
Revises: 96d72c748c24
Create Date: 2026-06-29 00:49:07.344159
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "dddee09eddcc"
down_revision: str | None = "96d72c748c24"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"candidate_phrases",
sa.Column("book_id", sa.Integer(), nullable=False),
sa.Column("series_id", sa.Integer(), nullable=True),
sa.Column("phrase_text", sa.Text(), nullable=False),
sa.Column("phrase_norm", sa.Text(), nullable=False),
sa.Column("token_count", sa.Integer(), nullable=False),
sa.Column("source_raw_ngram", sa.Boolean(), nullable=False),
sa.Column("source_yake", sa.Boolean(), nullable=False),
sa.Column("source_spacy_ner", sa.Boolean(), nullable=False),
sa.Column("source_spacy_noun_chunk", sa.Boolean(), nullable=False),
sa.Column("source_capitalized", sa.Boolean(), nullable=False),
sa.Column("source_metadata", sa.Boolean(), nullable=False),
sa.Column("spacy_label", sa.String(), nullable=True),
sa.Column("raw_count", sa.Integer(), nullable=False),
sa.Column("chapter_count", sa.Integer(), nullable=False),
sa.Column("yake_score", sa.Float(), nullable=True),
sa.Column("candidate_score", sa.Float(), nullable=False),
sa.Column(
"sample_contexts",
sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), "postgresql"),
nullable=True,
),
sa.Column("llm_judged", sa.Boolean(), nullable=False),
sa.Column("llm_keep", sa.Boolean(), nullable=True),
sa.Column("llm_confidence", sa.Float(), nullable=True),
sa.Column("llm_category", sa.String(), nullable=True),
sa.Column("llm_reason", sa.Text(), nullable=True),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["book_id"],
[f"{schema}.ebook_source.id"],
name=op.f("fk_candidate_phrases_book_id_ebook_source"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_candidate_phrases")),
sa.UniqueConstraint("book_id", "phrase_norm", name="uq_candidate_phrases_book_id_phrase_norm"),
schema=schema,
)
op.create_index(
"candidate_phrases_book_norm_idx", "candidate_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
)
op.create_index(
"candidate_phrases_book_score_idx",
"candidate_phrases",
["book_id", "candidate_score"],
unique=False,
schema=schema,
)
op.create_table(
"protected_phrases",
sa.Column("book_id", sa.Integer(), nullable=True),
sa.Column("series_id", sa.Integer(), nullable=True),
sa.Column("phrase_text", sa.Text(), nullable=False),
sa.Column("phrase_norm", sa.Text(), nullable=False),
sa.Column("canonical_id", sa.String(), nullable=False),
sa.Column("phrase_type", sa.String(), nullable=True),
sa.Column("token_count", sa.Integer(), nullable=False),
sa.Column("confidence", sa.Float(), nullable=False),
sa.Column("importance", sa.Float(), nullable=False),
sa.Column("allow_nested", sa.Boolean(), nullable=False),
sa.Column("suppress_children", sa.Boolean(), nullable=False),
sa.Column("source_candidate_id", sa.Integer(), nullable=True),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["book_id"],
[f"{schema}.ebook_source.id"],
name=op.f("fk_protected_phrases_book_id_ebook_source"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["source_candidate_id"],
[f"{schema}.candidate_phrases.id"],
name=op.f("fk_protected_phrases_source_candidate_id_candidate_phrases"),
ondelete="SET NULL",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_protected_phrases")),
sa.UniqueConstraint("book_id", "phrase_norm", name="uq_protected_phrases_book_id_phrase_norm"),
schema=schema,
)
op.create_index(
"protected_phrases_book_norm_idx", "protected_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
)
op.create_index("protected_phrases_norm_idx", "protected_phrases", ["phrase_norm"], unique=False, schema=schema)
op.create_index(
"protected_phrases_series_norm_idx",
"protected_phrases",
["series_id", "phrase_norm"],
unique=False,
schema=schema,
)
op.create_table(
"chunk_phrase_mentions",
sa.Column("chunk_id", sa.BigInteger(), nullable=False),
sa.Column("phrase_id", sa.Integer(), nullable=False),
sa.Column("book_id", sa.Integer(), nullable=True),
sa.Column("series_id", sa.Integer(), nullable=True),
sa.Column("start_char", sa.Integer(), nullable=False),
sa.Column("end_char", sa.Integer(), nullable=True),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["book_id"],
[f"{schema}.ebook_source.id"],
name=op.f("fk_chunk_phrase_mentions_book_id_ebook_source"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["chunk_id"],
[f"{schema}.ebook_chunk.id"],
name=op.f("fk_chunk_phrase_mentions_chunk_id_ebook_chunk"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["phrase_id"],
[f"{schema}.protected_phrases.id"],
name=op.f("fk_chunk_phrase_mentions_phrase_id_protected_phrases"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_chunk_phrase_mentions")),
sa.UniqueConstraint("chunk_id", "phrase_id", "start_char", name="uq_chunk_phrase_mentions_chunk_phrase_start"),
schema=schema,
)
op.create_index(
"chunk_phrase_mentions_chunk_idx", "chunk_phrase_mentions", ["chunk_id"], unique=False, schema=schema
)
op.create_index(
"chunk_phrase_mentions_phrase_idx", "chunk_phrase_mentions", ["phrase_id"], unique=False, schema=schema
)
op.create_table(
"phrase_aliases",
sa.Column("phrase_id", sa.Integer(), nullable=False),
sa.Column("alias_text", sa.Text(), nullable=False),
sa.Column("alias_norm", sa.Text(), nullable=False),
sa.Column("confidence", sa.Float(), nullable=False),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["phrase_id"],
[f"{schema}.protected_phrases.id"],
name=op.f("fk_phrase_aliases_phrase_id_protected_phrases"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_phrase_aliases")),
sa.UniqueConstraint("phrase_id", "alias_norm", name="uq_phrase_aliases_phrase_id_alias_norm"),
schema=schema,
)
op.create_index("phrase_aliases_norm_idx", "phrase_aliases", ["alias_norm"], unique=False, schema=schema)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index("phrase_aliases_norm_idx", table_name="phrase_aliases", schema=schema)
op.drop_table("phrase_aliases", schema=schema)
op.drop_index("chunk_phrase_mentions_phrase_idx", table_name="chunk_phrase_mentions", schema=schema)
op.drop_index("chunk_phrase_mentions_chunk_idx", table_name="chunk_phrase_mentions", schema=schema)
op.drop_table("chunk_phrase_mentions", schema=schema)
op.drop_index("protected_phrases_series_norm_idx", table_name="protected_phrases", schema=schema)
op.drop_index("protected_phrases_norm_idx", table_name="protected_phrases", schema=schema)
op.drop_index("protected_phrases_book_norm_idx", table_name="protected_phrases", schema=schema)
op.drop_table("protected_phrases", schema=schema)
op.drop_index("candidate_phrases_book_score_idx", table_name="candidate_phrases", schema=schema)
op.drop_index("candidate_phrases_book_norm_idx", table_name="candidate_phrases", schema=schema)
op.drop_table("candidate_phrases", schema=schema)
# ### end Alembic commands ###
@@ -1,55 +0,0 @@
"""remove spaCy-ner.
Revision ID: 751260fc3228
Revises: dddee09eddcc
Create Date: 2026-07-09 23:03:39.554083
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from python.orm import RichieBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "751260fc3228"
down_revision: str | None = "dddee09eddcc"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = RichieBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("candidate_phrases", "source_spacy_noun_chunk", schema=schema)
op.drop_column("candidate_phrases", "source_spacy_ner", schema=schema)
op.drop_column("candidate_phrases", "spacy_label", schema=schema)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.add_column(
"candidate_phrases", sa.Column("spacy_label", sa.VARCHAR(), autoincrement=False, nullable=True), schema=schema
)
op.add_column(
"candidate_phrases",
sa.Column("source_spacy_ner", sa.BOOLEAN(), autoincrement=False, nullable=False),
schema=schema,
)
op.add_column(
"candidate_phrases",
sa.Column("source_spacy_noun_chunk", sa.BOOLEAN(), autoincrement=False, nullable=False),
schema=schema,
)
# ### end Alembic commands ###
@@ -0,0 +1,100 @@
"""seprating signal_bot database.
Revision ID: 6eaf696e07a5
Revises:
Create Date: 2026-03-17 21:35:37.612672
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
from python.orm import SignalBotBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "6eaf696e07a5"
down_revision: str | None = None
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = SignalBotBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"dead_letter_message",
sa.Column("source", sa.String(), nullable=False),
sa.Column("message", sa.Text(), nullable=False),
sa.Column("received_at", sa.DateTime(timezone=True), nullable=False),
sa.Column(
"status", postgresql.ENUM("UNPROCESSED", "PROCESSED", name="message_status", schema=schema), nullable=False
),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_dead_letter_message")),
schema=schema,
)
op.create_table(
"role",
sa.Column("name", sa.String(length=50), nullable=False),
sa.Column("id", sa.SmallInteger(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_role")),
sa.UniqueConstraint("name", name=op.f("uq_role_name")),
schema=schema,
)
op.create_table(
"signal_device",
sa.Column("phone_number", sa.String(length=50), nullable=False),
sa.Column("safety_number", sa.String(), nullable=True),
sa.Column(
"trust_level",
postgresql.ENUM("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", schema=schema),
nullable=False,
),
sa.Column("last_seen", sa.DateTime(timezone=True), nullable=False),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_signal_device")),
sa.UniqueConstraint("phone_number", name=op.f("uq_signal_device_phone_number")),
schema=schema,
)
op.create_table(
"device_role",
sa.Column("device_id", sa.Integer(), nullable=False),
sa.Column("role_id", sa.SmallInteger(), nullable=False),
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(
["device_id"], [f"{schema}.signal_device.id"], name=op.f("fk_device_role_device_id_signal_device")
),
sa.ForeignKeyConstraint(["role_id"], [f"{schema}.role.id"], name=op.f("fk_device_role_role_id_role")),
sa.PrimaryKeyConstraint("id", name=op.f("pk_device_role")),
sa.UniqueConstraint("device_id", "role_id", name="uq_device_role_device_role"),
schema=schema,
)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("device_role", schema=schema)
op.drop_table("signal_device", schema=schema)
op.drop_table("role", schema=schema)
op.drop_table("dead_letter_message", schema=schema)
# ### end Alembic commands ###
@@ -0,0 +1,72 @@
"""test.
Revision ID: 66bdd532bcab
Revises: 6eaf696e07a5
Create Date: 2026-03-18 19:21:14.561568
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
from python.orm import SignalBotBase
if TYPE_CHECKING:
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "66bdd532bcab"
down_revision: str | None = "6eaf696e07a5"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = SignalBotBase.schema_name
def upgrade() -> None:
"""Upgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column(
"dead_letter_message",
"status",
existing_type=postgresql.ENUM("UNPROCESSED", "PROCESSED", name="message_status", schema=schema),
type_=sa.Enum("UNPROCESSED", "PROCESSED", name="message_status", native_enum=False),
existing_nullable=False,
schema=schema,
)
op.alter_column(
"signal_device",
"trust_level",
existing_type=postgresql.ENUM("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", schema=schema),
type_=sa.Enum("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", native_enum=False),
existing_nullable=False,
schema=schema,
)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade."""
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column(
"signal_device",
"trust_level",
existing_type=sa.Enum("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", native_enum=False),
type_=postgresql.ENUM("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", schema=schema),
existing_nullable=False,
schema=schema,
)
op.alter_column(
"dead_letter_message",
"status",
existing_type=sa.Enum("UNPROCESSED", "PROCESSED", name="message_status", native_enum=False),
type_=postgresql.ENUM("UNPROCESSED", "PROCESSED", name="message_status", schema=schema),
existing_nullable=False,
schema=schema,
)
# ### end Alembic commands ###
+1
View File
@@ -0,0 +1 @@
"""FastAPI applications."""
+16
View File
@@ -0,0 +1,16 @@
"""FastAPI dependencies."""
from collections.abc import Iterator
from typing import Annotated
from fastapi import Depends, Request
from sqlalchemy.orm import Session
def get_db(request: Request) -> Iterator[Session]:
"""Get database session from app state."""
with Session(request.app.state.engine) as session:
yield session
DbSession = Annotated[Session, Depends(get_db)]
+52
View File
@@ -0,0 +1,52 @@
"""FastAPI interface for Contact database."""
import logging
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from typing import Annotated
import typer
import uvicorn
from fastapi import FastAPI
from python.api.middleware import ZstdMiddleware
from python.api.routers import contact_router, views_router
from python.common import configure_logger
from python.orm.common import get_postgres_engine
logger = logging.getLogger(__name__)
def create_app() -> FastAPI:
"""Create and configure the FastAPI application."""
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
"""Manage application lifespan."""
app.state.engine = get_postgres_engine()
yield
app.state.engine.dispose()
app = FastAPI(title="Contact Database API", lifespan=lifespan)
app.add_middleware(ZstdMiddleware)
app.include_router(contact_router)
app.include_router(views_router)
return app
def serve(
host: Annotated[str, typer.Option("--host", "-h", help="Host to bind to")],
port: Annotated[int, typer.Option("--port", "-p", help="Port to bind to")] = 8000,
log_level: Annotated[str, typer.Option("--log-level", "-l", help="Log level")] = "INFO",
) -> None:
"""Start the Contact API server."""
configure_logger(log_level)
app = create_app()
uvicorn.run(app, host=host, port=port)
if __name__ == "__main__":
typer.run(serve)
@@ -1,14 +1,10 @@
"""Zstd response compression middleware."""
"""Middleware for the FastAPI application."""
from compression import zstd
from typing import TYPE_CHECKING
from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
from starlette.requests import Request
from starlette.responses import Response
if TYPE_CHECKING:
from starlette.requests import Request
MINIMUM_RESPONSE_SIZE = 500
+6
View File
@@ -0,0 +1,6 @@
"""API routers."""
from python.api.routers.contact import router as contact_router
from python.api.routers.views import router as views_router
__all__ = ["contact_router", "views_router"]
+481
View File
@@ -0,0 +1,481 @@
"""Contact API router."""
from pathlib import Path
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import HTMLResponse
from fastapi.templating import Jinja2Templates
from pydantic import BaseModel
from sqlalchemy import select
from sqlalchemy.orm import selectinload
from python.api.dependencies import DbSession
from python.orm.richie.contact import Contact, ContactRelationship, Need, RelationshipType
TEMPLATES_DIR = Path(__file__).parent.parent / "templates"
templates = Jinja2Templates(directory=TEMPLATES_DIR)
def _is_htmx(request: Request) -> bool:
"""Check if the request is from HTMX."""
return request.headers.get("HX-Request") == "true"
class NeedBase(BaseModel):
"""Base schema for Need."""
name: str
description: str | None = None
class NeedCreate(NeedBase):
"""Schema for creating a Need."""
class NeedResponse(NeedBase):
"""Schema for Need response."""
id: int
model_config = {"from_attributes": True}
class ContactRelationshipCreate(BaseModel):
"""Schema for creating a contact relationship."""
related_contact_id: int
relationship_type: RelationshipType
closeness_weight: int | None = None
class ContactRelationshipUpdate(BaseModel):
"""Schema for updating a contact relationship."""
relationship_type: RelationshipType | None = None
closeness_weight: int | None = None
class ContactRelationshipResponse(BaseModel):
"""Schema for contact relationship response."""
contact_id: int
related_contact_id: int
relationship_type: str
closeness_weight: int
model_config = {"from_attributes": True}
class RelationshipTypeInfo(BaseModel):
"""Information about a relationship type."""
value: str
display_name: str
default_weight: int
class GraphNode(BaseModel):
"""Node in the relationship graph."""
id: int
name: str
current_job: str | None = None
class GraphEdge(BaseModel):
"""Edge in the relationship graph."""
source: int
target: int
relationship_type: str
closeness_weight: int
class GraphData(BaseModel):
"""Complete graph data for visualization."""
nodes: list[GraphNode]
edges: list[GraphEdge]
class ContactBase(BaseModel):
"""Base schema for Contact."""
name: str
age: int | None = None
bio: str | None = None
current_job: str | None = None
gender: str | None = None
goals: str | None = None
legal_name: str | None = None
profile_pic: str | None = None
safe_conversation_starters: str | None = None
self_sufficiency_score: int | None = None
social_structure_style: str | None = None
ssn: str | None = None
suffix: str | None = None
timezone: str | None = None
topics_to_avoid: str | None = None
class ContactCreate(ContactBase):
"""Schema for creating a Contact."""
need_ids: list[int] = []
class ContactUpdate(BaseModel):
"""Schema for updating a Contact."""
name: str | None = None
age: int | None = None
bio: str | None = None
current_job: str | None = None
gender: str | None = None
goals: str | None = None
legal_name: str | None = None
profile_pic: str | None = None
safe_conversation_starters: str | None = None
self_sufficiency_score: int | None = None
social_structure_style: str | None = None
ssn: str | None = None
suffix: str | None = None
timezone: str | None = None
topics_to_avoid: str | None = None
need_ids: list[int] | None = None
class ContactResponse(ContactBase):
"""Schema for Contact response with relationships."""
id: int
needs: list[NeedResponse] = []
related_to: list[ContactRelationshipResponse] = []
related_from: list[ContactRelationshipResponse] = []
model_config = {"from_attributes": True}
class ContactListResponse(ContactBase):
"""Schema for Contact list response."""
id: int
model_config = {"from_attributes": True}
router = APIRouter(prefix="/api", tags=["contacts"])
@router.post("/needs", response_model=NeedResponse)
def create_need(need: NeedCreate, db: DbSession) -> Need:
"""Create a new need."""
db_need = Need(name=need.name, description=need.description)
db.add(db_need)
db.commit()
db.refresh(db_need)
return db_need
@router.get("/needs", response_model=list[NeedResponse])
def list_needs(db: DbSession) -> list[Need]:
"""List all needs."""
return list(db.scalars(select(Need)).all())
@router.get("/needs/{need_id}", response_model=NeedResponse)
def get_need(need_id: int, db: DbSession) -> Need:
"""Get a need by ID."""
need = db.get(Need, need_id)
if not need:
raise HTTPException(status_code=404, detail="Need not found")
return need
@router.delete("/needs/{need_id}", response_model=None)
def delete_need(need_id: int, request: Request, db: DbSession) -> dict[str, bool] | HTMLResponse:
"""Delete a need by ID."""
need = db.get(Need, need_id)
if not need:
raise HTTPException(status_code=404, detail="Need not found")
db.delete(need)
db.commit()
if _is_htmx(request):
return HTMLResponse("")
return {"deleted": True}
@router.post("/contacts", response_model=ContactResponse)
def create_contact(contact: ContactCreate, db: DbSession) -> Contact:
"""Create a new contact."""
need_ids = contact.need_ids
contact_data = contact.model_dump(exclude={"need_ids"})
db_contact = Contact(**contact_data)
if need_ids:
needs = list(db.scalars(select(Need).where(Need.id.in_(need_ids))).all())
db_contact.needs = needs
db.add(db_contact)
db.commit()
db.refresh(db_contact)
return db_contact
@router.get("/contacts", response_model=list[ContactListResponse])
def list_contacts(
db: DbSession,
skip: int = 0,
limit: int = 100,
) -> list[Contact]:
"""List all contacts with pagination."""
return list(db.scalars(select(Contact).offset(skip).limit(limit)).all())
@router.get("/contacts/{contact_id}", response_model=ContactResponse)
def get_contact(contact_id: int, db: DbSession) -> Contact:
"""Get a contact by ID with all relationships."""
contact = db.scalar(
select(Contact)
.where(Contact.id == contact_id)
.options(
selectinload(Contact.needs),
selectinload(Contact.related_to),
selectinload(Contact.related_from),
)
)
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
return contact
@router.patch("/contacts/{contact_id}", response_model=ContactResponse)
def update_contact(
contact_id: int,
contact: ContactUpdate,
db: DbSession,
) -> Contact:
"""Update a contact by ID."""
db_contact = db.get(Contact, contact_id)
if not db_contact:
raise HTTPException(status_code=404, detail="Contact not found")
update_data = contact.model_dump(exclude_unset=True)
need_ids = update_data.pop("need_ids", None)
for key, value in update_data.items():
setattr(db_contact, key, value)
if need_ids is not None:
needs = list(db.scalars(select(Need).where(Need.id.in_(need_ids))).all())
db_contact.needs = needs
db.commit()
db.refresh(db_contact)
return db_contact
@router.delete("/contacts/{contact_id}", response_model=None)
def delete_contact(contact_id: int, request: Request, db: DbSession) -> dict[str, bool] | HTMLResponse:
"""Delete a contact by ID."""
contact = db.get(Contact, contact_id)
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
db.delete(contact)
db.commit()
if _is_htmx(request):
return HTMLResponse("")
return {"deleted": True}
@router.post("/contacts/{contact_id}/needs/{need_id}")
def add_need_to_contact(
contact_id: int,
need_id: int,
db: DbSession,
) -> dict[str, bool]:
"""Add a need to a contact."""
contact = db.get(Contact, contact_id)
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
need = db.get(Need, need_id)
if not need:
raise HTTPException(status_code=404, detail="Need not found")
if need not in contact.needs:
contact.needs.append(need)
db.commit()
return {"added": True}
@router.delete("/contacts/{contact_id}/needs/{need_id}", response_model=None)
def remove_need_from_contact(
contact_id: int,
need_id: int,
request: Request,
db: DbSession,
) -> dict[str, bool] | HTMLResponse:
"""Remove a need from a contact."""
contact = db.get(Contact, contact_id)
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
need = db.get(Need, need_id)
if not need:
raise HTTPException(status_code=404, detail="Need not found")
if need in contact.needs:
contact.needs.remove(need)
db.commit()
if _is_htmx(request):
return HTMLResponse("")
return {"removed": True}
@router.post(
"/contacts/{contact_id}/relationships",
response_model=ContactRelationshipResponse,
)
def add_contact_relationship(
contact_id: int,
relationship: ContactRelationshipCreate,
db: DbSession,
) -> ContactRelationship:
"""Add a relationship between two contacts."""
contact = db.get(Contact, contact_id)
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
related_contact = db.get(Contact, relationship.related_contact_id)
if not related_contact:
raise HTTPException(status_code=404, detail="Related contact not found")
if contact_id == relationship.related_contact_id:
raise HTTPException(status_code=400, detail="Cannot relate contact to itself")
# Use provided weight or default from relationship type
weight = relationship.closeness_weight
if weight is None:
weight = relationship.relationship_type.default_weight
db_relationship = ContactRelationship(
contact_id=contact_id,
related_contact_id=relationship.related_contact_id,
relationship_type=relationship.relationship_type.value,
closeness_weight=weight,
)
db.add(db_relationship)
db.commit()
db.refresh(db_relationship)
return db_relationship
@router.get(
"/contacts/{contact_id}/relationships",
response_model=list[ContactRelationshipResponse],
)
def get_contact_relationships(
contact_id: int,
db: DbSession,
) -> list[ContactRelationship]:
"""Get all relationships for a contact."""
contact = db.get(Contact, contact_id)
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
outgoing = list(db.scalars(select(ContactRelationship).where(ContactRelationship.contact_id == contact_id)).all())
incoming = list(
db.scalars(select(ContactRelationship).where(ContactRelationship.related_contact_id == contact_id)).all()
)
return outgoing + incoming
@router.patch(
"/contacts/{contact_id}/relationships/{related_contact_id}",
response_model=ContactRelationshipResponse,
)
def update_contact_relationship(
contact_id: int,
related_contact_id: int,
update: ContactRelationshipUpdate,
db: DbSession,
) -> ContactRelationship:
"""Update a relationship between two contacts."""
relationship = db.scalar(
select(ContactRelationship).where(
ContactRelationship.contact_id == contact_id,
ContactRelationship.related_contact_id == related_contact_id,
)
)
if not relationship:
raise HTTPException(status_code=404, detail="Relationship not found")
if update.relationship_type is not None:
relationship.relationship_type = update.relationship_type.value
if update.closeness_weight is not None:
relationship.closeness_weight = update.closeness_weight
db.commit()
db.refresh(relationship)
return relationship
@router.delete("/contacts/{contact_id}/relationships/{related_contact_id}", response_model=None)
def remove_contact_relationship(
contact_id: int,
related_contact_id: int,
request: Request,
db: DbSession,
) -> dict[str, bool] | HTMLResponse:
"""Remove a relationship between two contacts."""
relationship = db.scalar(
select(ContactRelationship).where(
ContactRelationship.contact_id == contact_id,
ContactRelationship.related_contact_id == related_contact_id,
)
)
if not relationship:
raise HTTPException(status_code=404, detail="Relationship not found")
db.delete(relationship)
db.commit()
if _is_htmx(request):
return HTMLResponse("")
return {"deleted": True}
@router.get("/relationship-types")
def list_relationship_types() -> list[RelationshipTypeInfo]:
"""List all available relationship types with their default weights."""
return [
RelationshipTypeInfo(
value=rt.value,
display_name=rt.display_name,
default_weight=rt.default_weight,
)
for rt in RelationshipType
]
@router.get("/graph")
def get_relationship_graph(db: DbSession) -> GraphData:
"""Get all contacts and relationships as graph data for visualization."""
contacts = list(db.scalars(select(Contact)).all())
relationships = list(db.scalars(select(ContactRelationship)).all())
nodes = [GraphNode(id=c.id, name=c.name, current_job=c.current_job) for c in contacts]
edges = [
GraphEdge(
source=rel.contact_id,
target=rel.related_contact_id,
relationship_type=rel.relationship_type,
closeness_weight=rel.closeness_weight,
)
for rel in relationships
]
return GraphData(nodes=nodes, edges=edges)
+345
View File
@@ -0,0 +1,345 @@
"""HTMX server-rendered view router."""
from pathlib import Path
from typing import Annotated, Any
from fastapi import APIRouter, Form, HTTPException, Request
from fastapi.responses import HTMLResponse, RedirectResponse
from fastapi.templating import Jinja2Templates
from sqlalchemy import select
from sqlalchemy.orm import Session, selectinload
from python.api.dependencies import DbSession
from python.orm.richie.contact import Contact, ContactRelationship, Need, RelationshipType
TEMPLATES_DIR = Path(__file__).parent.parent / "templates"
templates = Jinja2Templates(directory=TEMPLATES_DIR)
router = APIRouter(tags=["views"])
FAMILIAL_TYPES = {
"parent",
"child",
"sibling",
"grandparent",
"grandchild",
"aunt_uncle",
"niece_nephew",
"cousin",
"in_law",
}
FRIEND_TYPES = {"best_friend", "close_friend", "friend", "acquaintance", "neighbor"}
PARTNER_TYPES = {"spouse", "partner"}
PROFESSIONAL_TYPES = {"mentor", "mentee", "business_partner", "colleague", "manager", "direct_report", "client"}
CONTACT_STRING_FIELDS = (
"name",
"legal_name",
"suffix",
"gender",
"current_job",
"timezone",
"profile_pic",
"bio",
"goals",
"social_structure_style",
"safe_conversation_starters",
"topics_to_avoid",
"ssn",
)
CONTACT_INT_FIELDS = ("age", "self_sufficiency_score")
def _group_relationships(relationships: list[ContactRelationship]) -> dict[str, list[ContactRelationship]]:
"""Group relationships by category."""
groups: dict[str, list[ContactRelationship]] = {
"familial": [],
"partners": [],
"friends": [],
"professional": [],
"other": [],
}
for rel in relationships:
if rel.relationship_type in FAMILIAL_TYPES:
groups["familial"].append(rel)
elif rel.relationship_type in PARTNER_TYPES:
groups["partners"].append(rel)
elif rel.relationship_type in FRIEND_TYPES:
groups["friends"].append(rel)
elif rel.relationship_type in PROFESSIONAL_TYPES:
groups["professional"].append(rel)
else:
groups["other"].append(rel)
return groups
def _build_contact_name_map(database: Session, contact: Contact) -> dict[int, str]:
"""Build a mapping of contact IDs to names for relationship display."""
related_ids = {rel.related_contact_id for rel in contact.related_to}
related_ids |= {rel.contact_id for rel in contact.related_from}
related_ids.discard(contact.id)
if not related_ids:
return {}
related_contacts = list(database.scalars(select(Contact).where(Contact.id.in_(related_ids))).all())
return {related.id: related.name for related in related_contacts}
def _get_relationship_type_display() -> dict[str, str]:
"""Build a mapping of relationship type values to display names."""
return {rel_type.value: rel_type.display_name for rel_type in RelationshipType}
async def _parse_contact_form(request: Request) -> dict[str, Any]:
"""Parse contact form data from a multipart/form request."""
form_data = await request.form()
result: dict[str, Any] = {}
for field in CONTACT_STRING_FIELDS:
value = form_data.get(field, "")
result[field] = str(value) if value else None
for field in CONTACT_INT_FIELDS:
value = form_data.get(field, "")
result[field] = int(value) if value else None
result["need_ids"] = [int(value) for value in form_data.getlist("need_ids")]
return result
def _save_contact_from_form(database: Session, contact: Contact, form_result: dict[str, Any]) -> None:
"""Apply parsed form data to a Contact and save associated needs."""
need_ids = form_result.pop("need_ids")
for key, value in form_result.items():
setattr(contact, key, value)
if need_ids:
contact.needs = list(database.scalars(select(Need).where(Need.id.in_(need_ids))).all())
else:
contact.needs = []
@router.get("/", response_class=HTMLResponse)
@router.get("/contacts", response_class=HTMLResponse)
def contact_list_page(request: Request, database: DbSession) -> HTMLResponse:
"""Render the contacts list page."""
contacts = list(database.scalars(select(Contact)).all())
return templates.TemplateResponse(request, "contact_list.html", {"contacts": contacts})
@router.get("/contacts/new", response_class=HTMLResponse)
def new_contact_page(request: Request, database: DbSession) -> HTMLResponse:
"""Render the new contact form page."""
all_needs = list(database.scalars(select(Need)).all())
return templates.TemplateResponse(request, "contact_form.html", {"contact": None, "all_needs": all_needs})
@router.post("/htmx/contacts/new")
async def create_contact_form(request: Request, database: DbSession) -> RedirectResponse:
"""Handle the create contact form submission."""
form_result = await _parse_contact_form(request)
contact = Contact()
_save_contact_from_form(database, contact, form_result)
database.add(contact)
database.commit()
database.refresh(contact)
return RedirectResponse(url=f"/contacts/{contact.id}", status_code=303)
@router.get("/contacts/{contact_id}", response_class=HTMLResponse)
def contact_detail_page(contact_id: int, request: Request, database: DbSession) -> HTMLResponse:
"""Render the contact detail page."""
contact = database.scalar(
select(Contact)
.where(Contact.id == contact_id)
.options(
selectinload(Contact.needs),
selectinload(Contact.related_to),
selectinload(Contact.related_from),
)
)
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
contact_names = _build_contact_name_map(database, contact)
grouped_relationships = _group_relationships(contact.related_to)
all_contacts = list(database.scalars(select(Contact)).all())
all_needs = list(database.scalars(select(Need)).all())
available_needs = [need for need in all_needs if need not in contact.needs]
return templates.TemplateResponse(
request,
"contact_detail.html",
{
"contact": contact,
"contact_names": contact_names,
"grouped_relationships": grouped_relationships,
"all_contacts": all_contacts,
"available_needs": available_needs,
"relationship_types": list(RelationshipType),
},
)
@router.get("/contacts/{contact_id}/edit", response_class=HTMLResponse)
def edit_contact_page(contact_id: int, request: Request, database: DbSession) -> HTMLResponse:
"""Render the edit contact form page."""
contact = database.scalar(select(Contact).where(Contact.id == contact_id).options(selectinload(Contact.needs)))
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
all_needs = list(database.scalars(select(Need)).all())
return templates.TemplateResponse(request, "contact_form.html", {"contact": contact, "all_needs": all_needs})
@router.post("/htmx/contacts/{contact_id}/edit")
async def update_contact_form(contact_id: int, request: Request, database: DbSession) -> RedirectResponse:
"""Handle the edit contact form submission."""
contact = database.get(Contact, contact_id)
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
form_result = await _parse_contact_form(request)
_save_contact_from_form(database, contact, form_result)
database.commit()
return RedirectResponse(url=f"/contacts/{contact_id}", status_code=303)
@router.post("/htmx/contacts/{contact_id}/add-need", response_class=HTMLResponse)
def add_need_to_contact_htmx(
contact_id: int,
request: Request,
database: DbSession,
need_id: Annotated[int, Form()],
) -> HTMLResponse:
"""Add a need to a contact and return updated manage-needs partial."""
contact = database.scalar(select(Contact).where(Contact.id == contact_id).options(selectinload(Contact.needs)))
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
need = database.get(Need, need_id)
if not need:
raise HTTPException(status_code=404, detail="Need not found")
if need not in contact.needs:
contact.needs.append(need)
database.commit()
database.refresh(contact)
return templates.TemplateResponse(request, "partials/manage_needs.html", {"contact": contact})
@router.post("/htmx/contacts/{contact_id}/add-relationship", response_class=HTMLResponse)
def add_relationship_htmx(
contact_id: int,
request: Request,
database: DbSession,
related_contact_id: Annotated[int, Form()],
relationship_type: Annotated[str, Form()],
) -> HTMLResponse:
"""Add a relationship and return updated manage-relationships partial."""
contact = database.scalar(select(Contact).where(Contact.id == contact_id).options(selectinload(Contact.related_to)))
if not contact:
raise HTTPException(status_code=404, detail="Contact not found")
related_contact = database.get(Contact, related_contact_id)
if not related_contact:
raise HTTPException(status_code=404, detail="Related contact not found")
rel_type = RelationshipType(relationship_type)
weight = rel_type.default_weight
relationship = ContactRelationship(
contact_id=contact_id,
related_contact_id=related_contact_id,
relationship_type=relationship_type,
closeness_weight=weight,
)
database.add(relationship)
database.commit()
database.refresh(contact)
contact_names = _build_contact_name_map(database, contact)
return templates.TemplateResponse(
request,
"partials/manage_relationships.html",
{"contact": contact, "contact_names": contact_names},
)
@router.post("/htmx/contacts/{contact_id}/relationships/{related_contact_id}/weight")
def update_relationship_weight_htmx(
contact_id: int,
related_contact_id: int,
database: DbSession,
closeness_weight: Annotated[int, Form()],
) -> HTMLResponse:
"""Update a relationship's closeness weight from HTMX range input."""
relationship = database.scalar(
select(ContactRelationship).where(
ContactRelationship.contact_id == contact_id,
ContactRelationship.related_contact_id == related_contact_id,
)
)
if not relationship:
raise HTTPException(status_code=404, detail="Relationship not found")
relationship.closeness_weight = closeness_weight
database.commit()
return HTMLResponse("")
@router.post("/htmx/needs", response_class=HTMLResponse)
def create_need_htmx(
request: Request,
database: DbSession,
name: Annotated[str, Form()],
description: Annotated[str, Form()] = "",
) -> HTMLResponse:
"""Create a need via form data and return updated needs list."""
need = Need(name=name, description=description or None)
database.add(need)
database.commit()
needs = list(database.scalars(select(Need)).all())
return templates.TemplateResponse(request, "partials/need_items.html", {"needs": needs})
@router.get("/needs", response_class=HTMLResponse)
def needs_page(request: Request, database: DbSession) -> HTMLResponse:
"""Render the needs list page."""
needs = list(database.scalars(select(Need)).all())
return templates.TemplateResponse(request, "need_list.html", {"needs": needs})
@router.get("/graph", response_class=HTMLResponse)
def graph_page(request: Request, database: DbSession) -> HTMLResponse:
"""Render the relationship graph page."""
contacts = list(database.scalars(select(Contact)).all())
relationships = list(database.scalars(select(ContactRelationship)).all())
graph_data = {
"nodes": [{"id": contact.id, "name": contact.name, "current_job": contact.current_job} for contact in contacts],
"edges": [
{
"source": rel.contact_id,
"target": rel.related_contact_id,
"relationship_type": rel.relationship_type,
"closeness_weight": rel.closeness_weight,
}
for rel in relationships
],
}
return templates.TemplateResponse(
request,
"graph.html",
{
"graph_data": graph_data,
"relationship_type_display": _get_relationship_type_display(),
},
)
+198
View File
@@ -0,0 +1,198 @@
<!DOCTYPE html>
<html lang="en" data-theme="light">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>{% block title %}Contact Database{% endblock %}</title>
<script src="https://unpkg.com/htmx.org@2.0.4"></script>
<style>
:root {
--color-bg: #f5f5f5;
--color-bg-card: #ffffff;
--color-bg-hover: #f0f0f0;
--color-bg-muted: #f9f9f9;
--color-bg-error: #ffe0e0;
--color-text: #333333;
--color-text-muted: #666666;
--color-text-error: #cc0000;
--color-border: #dddddd;
--color-border-light: #eeeeee;
--color-border-lighter: #f0f0f0;
--color-primary: #0066cc;
--color-primary-hover: #0055aa;
--color-danger: #cc3333;
--color-danger-hover: #aa2222;
--color-tag-bg: #e0e0e0;
--shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
line-height: 1.5;
color: var(--color-text);
background-color: var(--color-bg);
}
[data-theme="dark"] {
--color-bg: #1a1a1a;
--color-bg-card: #2d2d2d;
--color-bg-hover: #3d3d3d;
--color-bg-muted: #252525;
--color-bg-error: #4a2020;
--color-text: #e0e0e0;
--color-text-muted: #a0a0a0;
--color-text-error: #ff6b6b;
--color-border: #404040;
--color-border-light: #353535;
--color-border-lighter: #303030;
--color-primary: #4da6ff;
--color-primary-hover: #7dbfff;
--color-danger: #ff6b6b;
--color-danger-hover: #ff8a8a;
--color-tag-bg: #404040;
--shadow: 0 1px 3px rgba(0, 0, 0, 0.3);
}
* { box-sizing: border-box; }
body { margin: 0; background: var(--color-bg); color: var(--color-text); }
.app { max-width: 1000px; margin: 0 auto; padding: 20px; }
nav { display: flex; align-items: center; gap: 20px; padding: 15px 0; border-bottom: 1px solid var(--color-border); margin-bottom: 20px; }
nav a { color: var(--color-primary); text-decoration: none; font-weight: 500; }
nav a:hover { text-decoration: underline; }
.theme-toggle { margin-left: auto; }
main { background: var(--color-bg-card); padding: 20px; border-radius: 8px; box-shadow: var(--shadow); }
.header { display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px; }
.header h1 { margin: 0; }
a { color: var(--color-primary); }
a:hover { text-decoration: underline; }
.btn { display: inline-block; padding: 8px 16px; border: 1px solid var(--color-border); border-radius: 4px; background: var(--color-bg-card); color: var(--color-text); text-decoration: none; cursor: pointer; font-size: 14px; margin-left: 8px; }
.btn:hover { background: var(--color-bg-hover); }
.btn-primary { background: var(--color-primary); border-color: var(--color-primary); color: white; }
.btn-primary:hover { background: var(--color-primary-hover); }
.btn-danger { background: var(--color-danger); border-color: var(--color-danger); color: white; }
.btn-danger:hover { background: var(--color-danger-hover); }
.btn-small { padding: 4px 8px; font-size: 12px; }
.btn:disabled { opacity: 0.6; cursor: not-allowed; }
table { width: 100%; border-collapse: collapse; }
th, td { padding: 12px; text-align: left; border-bottom: 1px solid var(--color-border-light); }
th { font-weight: 600; background: var(--color-bg-muted); }
tr:hover { background: var(--color-bg-muted); }
.error { background: var(--color-bg-error); color: var(--color-text-error); padding: 10px; border-radius: 4px; margin-bottom: 20px; }
.tag { display: inline-block; background: var(--color-tag-bg); padding: 2px 8px; border-radius: 12px; font-size: 12px; color: var(--color-text-muted); }
.add-form { display: flex; gap: 10px; margin-top: 15px; flex-wrap: wrap; }
.add-form select, .add-form input { padding: 8px; border: 1px solid var(--color-border); border-radius: 4px; min-width: 200px; background: var(--color-bg-card); color: var(--color-text); }
.form-group { margin-bottom: 20px; }
.form-group label { display: block; font-weight: 500; margin-bottom: 5px; }
.form-group input, .form-group textarea, .form-group select { width: 100%; padding: 10px; border: 1px solid var(--color-border); border-radius: 4px; font-size: 14px; background: var(--color-bg-card); color: var(--color-text); }
.form-group textarea { resize: vertical; }
.form-row { display: grid; grid-template-columns: 1fr 1fr; gap: 20px; }
.checkbox-group { display: flex; flex-wrap: wrap; gap: 15px; }
.checkbox-label { display: flex; align-items: center; gap: 5px; cursor: pointer; }
.form-actions { display: flex; gap: 10px; margin-top: 30px; padding-top: 20px; border-top: 1px solid var(--color-border-light); }
.need-form { background: var(--color-bg-muted); padding: 20px; border-radius: 4px; margin-bottom: 20px; }
.need-items { list-style: none; padding: 0; }
.need-items li { display: flex; justify-content: space-between; align-items: flex-start; padding: 15px; border: 1px solid var(--color-border-light); border-radius: 4px; margin-bottom: 10px; }
.need-info p { margin: 5px 0 0; color: var(--color-text-muted); font-size: 14px; }
.graph-container { width: 100%; }
.graph-hint { color: var(--color-text-muted); font-size: 14px; margin-bottom: 15px; }
.selected-info { margin-top: 15px; padding: 15px; background: var(--color-bg-muted); border-radius: 8px; }
.selected-info h3 { margin: 0 0 10px; }
.selected-info p { margin: 5px 0; color: var(--color-text-muted); }
.legend { margin-top: 20px; padding: 15px; background: var(--color-bg-muted); border-radius: 8px; }
.legend h4 { margin: 0 0 10px; font-size: 14px; }
.legend-items { display: flex; flex-wrap: wrap; gap: 15px; }
.legend-item { display: flex; align-items: center; gap: 8px; font-size: 12px; color: var(--color-text-muted); }
.legend-line { width: 30px; border-radius: 2px; }
.id-card { width: 100%; }
.id-card-inner { background: linear-gradient(135deg, #0a0a0f 0%, #1a1a2e 50%, #0a0a0f 100%); background-image: radial-gradient(white 1px, transparent 1px), linear-gradient(135deg, #0a0a0f 0%, #1a1a2e 50%, #0a0a0f 100%); background-size: 50px 50px, 100% 100%; color: #fff; border-radius: 12px; padding: 25px; min-height: 500px; position: relative; overflow: hidden; }
.id-card-header { display: flex; justify-content: space-between; align-items: flex-start; margin-bottom: 15px; }
.id-card-header-left { flex: 1; }
.id-card-header-right { display: flex; flex-direction: column; align-items: flex-end; gap: 10px; }
.id-card-title { font-size: 2.5rem; font-weight: 700; margin: 0; color: #fff; text-shadow: 2px 2px 4px rgba(0,0,0,0.5); }
.id-profile-pic { width: 80px; height: 80px; border-radius: 8px; object-fit: cover; border: 2px solid rgba(255,255,255,0.3); }
.id-profile-placeholder { width: 80px; height: 80px; border-radius: 8px; background: linear-gradient(135deg, #4ecdc4 0%, #44a8a0 100%); display: flex; align-items: center; justify-content: center; border: 2px solid rgba(255,255,255,0.3); }
.id-profile-placeholder span { font-size: 2rem; font-weight: 700; color: #fff; text-shadow: 1px 1px 2px rgba(0,0,0,0.3); }
.id-card-actions { display: flex; gap: 8px; }
.id-card-actions .btn { background: rgba(255,255,255,0.1); border-color: rgba(255,255,255,0.3); color: #fff; }
.id-card-actions .btn:hover { background: rgba(255,255,255,0.2); }
.id-card-body { display: grid; grid-template-columns: 1fr 1.5fr; gap: 30px; }
.id-card-left { display: flex; flex-direction: column; gap: 8px; }
.id-field { font-size: 1rem; line-height: 1.4; }
.id-field-block { margin-top: 15px; font-size: 0.95rem; line-height: 1.5; }
.id-label { color: #4ecdc4; font-weight: 500; }
.id-card-right { display: flex; flex-direction: column; gap: 20px; }
.id-bio { font-size: 0.9rem; line-height: 1.6; color: #e0e0e0; }
.id-relationships { margin-top: 10px; }
.id-section-title { font-size: 1.5rem; margin: 0 0 15px; color: #fff; border-bottom: 1px solid rgba(255,255,255,0.2); padding-bottom: 8px; }
.id-rel-group { margin-bottom: 12px; font-size: 0.9rem; line-height: 1.6; }
.id-rel-label { color: #a0a0a0; }
.id-rel-group a { color: #4ecdc4; text-decoration: none; }
.id-rel-group a:hover { text-decoration: underline; }
.id-rel-type { color: #888; font-size: 0.85em; }
.id-card-warnings { margin-top: 30px; padding-top: 20px; border-top: 1px solid rgba(255,255,255,0.2); display: flex; flex-wrap: wrap; gap: 20px; }
.id-warning { display: flex; align-items: center; gap: 8px; font-size: 0.9rem; color: #ff6b6b; }
.warning-dot { width: 8px; height: 8px; background: #ff6b6b; border-radius: 50%; flex-shrink: 0; }
.warning-desc { color: #ccc; }
.id-card-manage { margin-top: 20px; background: var(--color-bg-muted); border-radius: 8px; padding: 15px; }
.id-card-manage summary { cursor: pointer; font-weight: 600; font-size: 1.1rem; padding: 5px 0; }
.id-card-manage[open] summary { margin-bottom: 15px; border-bottom: 1px solid var(--color-border-light); padding-bottom: 10px; }
.manage-section { margin-bottom: 25px; }
.manage-section h3 { margin: 0 0 15px; font-size: 1rem; }
.manage-relationships { display: flex; flex-direction: column; gap: 10px; margin-bottom: 15px; }
.manage-rel-item { display: flex; align-items: center; gap: 12px; padding: 10px; background: var(--color-bg-card); border-radius: 6px; flex-wrap: wrap; }
.manage-rel-item a { font-weight: 500; min-width: 120px; }
.weight-control { display: flex; align-items: center; gap: 8px; font-size: 12px; color: var(--color-text-muted); }
.weight-control input[type="range"] { width: 80px; cursor: pointer; }
.weight-value { min-width: 20px; text-align: center; font-weight: 600; }
.manage-needs-list { list-style: none; padding: 0; margin: 0 0 15px; }
.manage-needs-list li { display: flex; align-items: center; gap: 12px; padding: 10px; background: var(--color-bg-card); border-radius: 6px; margin-bottom: 8px; }
.manage-needs-list li .btn { margin-left: auto; }
.htmx-indicator { display: none; }
.htmx-request .htmx-indicator { display: inline; }
.htmx-request.htmx-indicator { display: inline; }
@media (max-width: 768px) {
.id-card-body { grid-template-columns: 1fr; }
.id-card-title { font-size: 1.8rem; }
.id-card-header { flex-direction: column; gap: 15px; }
}
</style>
</head>
<body>
<div class="app">
<nav>
<a href="/contacts">Contacts</a>
<a href="/graph">Graph</a>
<a href="/needs">Needs</a>
<button class="btn btn-small theme-toggle" onclick="toggleTheme()">
<span id="theme-label">Dark</span>
</button>
</nav>
<main id="main-content">
{% block content %}{% endblock %}
</main>
</div>
<script>
function toggleTheme() {
const html = document.documentElement;
const current = html.getAttribute('data-theme');
const next = current === 'light' ? 'dark' : 'light';
html.setAttribute('data-theme', next);
localStorage.setItem('theme', next);
document.getElementById('theme-label').textContent = next === 'light' ? 'Dark' : 'Light';
}
(function() {
const saved = localStorage.getItem('theme') || 'light';
document.documentElement.setAttribute('data-theme', saved);
document.getElementById('theme-label').textContent = saved === 'light' ? 'Dark' : 'Light';
})();
</script>
</body>
</html>
+204
View File
@@ -0,0 +1,204 @@
{% extends "base.html" %}
{% block title %}{{ contact.name }}{% endblock %}
{% block content %}
<div class="id-card">
<div class="id-card-inner">
<div class="id-card-header">
<div class="id-card-header-left">
<h1 class="id-card-title">I.D.: {{ contact.name }}</h1>
</div>
<div class="id-card-header-right">
{% if contact.profile_pic %}
<img src="{{ contact.profile_pic }}" alt="{{ contact.name }}'s profile" class="id-profile-pic">
{% else %}
<div class="id-profile-placeholder">
<span>{{ contact.name[0]|upper }}</span>
</div>
{% endif %}
<div class="id-card-actions">
<a href="/contacts/{{ contact.id }}/edit" class="btn btn-small">Edit</a>
<a href="/contacts" class="btn btn-small">Back</a>
</div>
</div>
</div>
<div class="id-card-body">
<div class="id-card-left">
{% if contact.legal_name %}
<div class="id-field">Legal name: {{ contact.legal_name }}</div>
{% endif %}
{% if contact.suffix %}
<div class="id-field">Suffix: {{ contact.suffix }}</div>
{% endif %}
{% if contact.gender %}
<div class="id-field">Gender: {{ contact.gender }}</div>
{% endif %}
{% if contact.age %}
<div class="id-field">Age: {{ contact.age }}</div>
{% endif %}
{% if contact.current_job %}
<div class="id-field">Job: {{ contact.current_job }}</div>
{% endif %}
{% if contact.social_structure_style %}
<div class="id-field">Social style: {{ contact.social_structure_style }}</div>
{% endif %}
{% if contact.self_sufficiency_score is not none %}
<div class="id-field">Self-Sufficiency: {{ contact.self_sufficiency_score }}</div>
{% endif %}
{% if contact.timezone %}
<div class="id-field">Timezone: {{ contact.timezone }}</div>
{% endif %}
{% if contact.safe_conversation_starters %}
<div class="id-field-block">
<span class="id-label">Safe con starters:</span> {{ contact.safe_conversation_starters }}
</div>
{% endif %}
{% if contact.topics_to_avoid %}
<div class="id-field-block">
<span class="id-label">Topics to avoid:</span> {{ contact.topics_to_avoid }}
</div>
{% endif %}
{% if contact.goals %}
<div class="id-field-block">
<span class="id-label">Goals:</span> {{ contact.goals }}
</div>
{% endif %}
</div>
<div class="id-card-right">
{% if contact.bio %}
<div class="id-bio">
<span class="id-label">Bio:</span> {{ contact.bio }}
</div>
{% endif %}
<div class="id-relationships">
<h2 class="id-section-title">Relationships</h2>
{% if grouped_relationships.familial %}
<div class="id-rel-group">
<span class="id-rel-label">Familial:</span>
{% for rel in grouped_relationships.familial %}
<a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a><span class="id-rel-type">({{ rel.relationship_type|replace("_", " ")|title }})</span>{% if not loop.last %}, {% endif %}
{% endfor %}
</div>
{% endif %}
{% if grouped_relationships.partners %}
<div class="id-rel-group">
<span class="id-rel-label">Partners:</span>
{% for rel in grouped_relationships.partners %}
<a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a>{% if not loop.last %}, {% endif %}
{% endfor %}
</div>
{% endif %}
{% if grouped_relationships.friends %}
<div class="id-rel-group">
<span class="id-rel-label">Friends:</span>
{% for rel in grouped_relationships.friends %}
<a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a>{% if not loop.last %}, {% endif %}
{% endfor %}
</div>
{% endif %}
{% if grouped_relationships.professional %}
<div class="id-rel-group">
<span class="id-rel-label">Professional:</span>
{% for rel in grouped_relationships.professional %}
<a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a><span class="id-rel-type">({{ rel.relationship_type|replace("_", " ")|title }})</span>{% if not loop.last %}, {% endif %}
{% endfor %}
</div>
{% endif %}
{% if grouped_relationships.other %}
<div class="id-rel-group">
<span class="id-rel-label">Other:</span>
{% for rel in grouped_relationships.other %}
<a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a><span class="id-rel-type">({{ rel.relationship_type|replace("_", " ")|title }})</span>{% if not loop.last %}, {% endif %}
{% endfor %}
</div>
{% endif %}
{% if contact.related_from %}
<div class="id-rel-group">
<span class="id-rel-label">Known by:</span>
{% for rel in contact.related_from %}
<a href="/contacts/{{ rel.contact_id }}">{{ contact_names[rel.contact_id] }}</a>{% if not loop.last %}, {% endif %}
{% endfor %}
</div>
{% endif %}
</div>
</div>
</div>
{% if contact.needs %}
<div class="id-card-warnings">
{% for need in contact.needs %}
<div class="id-warning">
<span class="warning-dot"></span>
Warning: {{ need.name }}
{% if need.description %}<span class="warning-desc"> - {{ need.description }}</span>{% endif %}
</div>
{% endfor %}
</div>
{% endif %}
</div>
<details class="id-card-manage">
<summary>Manage Contact</summary>
<div class="manage-section">
<h3>Manage Relationships</h3>
<div id="manage-relationships" class="manage-relationships">
{% include "partials/manage_relationships.html" %}
</div>
{% if all_contacts %}
<form hx-post="/htmx/contacts/{{ contact.id }}/add-relationship"
hx-target="#manage-relationships"
hx-swap="innerHTML"
class="add-form">
<select name="related_contact_id" required>
<option value="">Select contact...</option>
{% for other in all_contacts %}
{% if other.id != contact.id %}
<option value="{{ other.id }}">{{ other.name }}</option>
{% endif %}
{% endfor %}
</select>
<select name="relationship_type" required>
<option value="">Select relationship type...</option>
{% for rel_type in relationship_types %}
<option value="{{ rel_type.value }}">{{ rel_type.display_name }}</option>
{% endfor %}
</select>
<button type="submit" class="btn btn-primary">Add Relationship</button>
</form>
{% endif %}
</div>
<div class="manage-section">
<h3>Manage Needs/Warnings</h3>
<div id="manage-needs">
{% include "partials/manage_needs.html" %}
</div>
{% if available_needs %}
<form hx-post="/htmx/contacts/{{ contact.id }}/add-need"
hx-target="#manage-needs"
hx-swap="innerHTML"
class="add-form">
<select name="need_id" required>
<option value="">Select a need...</option>
{% for need in available_needs %}
<option value="{{ need.id }}">{{ need.name }}</option>
{% endfor %}
</select>
<button type="submit" class="btn btn-primary">Add Need</button>
</form>
{% endif %}
</div>
</details>
</div>
{% endblock %}
+115
View File
@@ -0,0 +1,115 @@
{% extends "base.html" %}
{% block title %}{{ "Edit " + contact.name if contact else "New Contact" }}{% endblock %}
{% block content %}
<div class="contact-form">
<h1>{{ "Edit Contact" if contact else "New Contact" }}</h1>
{% if contact %}
<form method="post" action="/htmx/contacts/{{ contact.id }}/edit">
{% else %}
<form method="post" action="/htmx/contacts/new">
{% endif %}
<div class="form-group">
<label for="name">Name *</label>
<input id="name" name="name" type="text" value="{{ contact.name if contact else '' }}" required>
</div>
<div class="form-row">
<div class="form-group">
<label for="legal_name">Legal Name</label>
<input id="legal_name" name="legal_name" type="text" value="{{ contact.legal_name or '' }}">
</div>
<div class="form-group">
<label for="suffix">Suffix</label>
<input id="suffix" name="suffix" type="text" value="{{ contact.suffix or '' }}">
</div>
</div>
<div class="form-row">
<div class="form-group">
<label for="age">Age</label>
<input id="age" name="age" type="number" value="{{ contact.age if contact and contact.age is not none else '' }}">
</div>
<div class="form-group">
<label for="gender">Gender</label>
<input id="gender" name="gender" type="text" value="{{ contact.gender or '' }}">
</div>
</div>
<div class="form-group">
<label for="current_job">Current Job</label>
<input id="current_job" name="current_job" type="text" value="{{ contact.current_job or '' }}">
</div>
<div class="form-group">
<label for="timezone">Timezone</label>
<input id="timezone" name="timezone" type="text" value="{{ contact.timezone or '' }}">
</div>
<div class="form-group">
<label for="profile_pic">Profile Picture URL</label>
<input id="profile_pic" name="profile_pic" type="url" placeholder="https://example.com/photo.jpg" value="{{ contact.profile_pic or '' }}">
</div>
<div class="form-group">
<label for="bio">Bio</label>
<textarea id="bio" name="bio" rows="3">{{ contact.bio or '' }}</textarea>
</div>
<div class="form-group">
<label for="goals">Goals</label>
<textarea id="goals" name="goals" rows="3">{{ contact.goals or '' }}</textarea>
</div>
<div class="form-group">
<label for="social_structure_style">Social Structure Style</label>
<input id="social_structure_style" name="social_structure_style" type="text" value="{{ contact.social_structure_style or '' }}">
</div>
<div class="form-group">
<label for="self_sufficiency_score">Self-Sufficiency Score (1-10)</label>
<input id="self_sufficiency_score" name="self_sufficiency_score" type="number" min="1" max="10" value="{{ contact.self_sufficiency_score if contact and contact.self_sufficiency_score is not none else '' }}">
</div>
<div class="form-group">
<label for="safe_conversation_starters">Safe Conversation Starters</label>
<textarea id="safe_conversation_starters" name="safe_conversation_starters" rows="2">{{ contact.safe_conversation_starters or '' }}</textarea>
</div>
<div class="form-group">
<label for="topics_to_avoid">Topics to Avoid</label>
<textarea id="topics_to_avoid" name="topics_to_avoid" rows="2">{{ contact.topics_to_avoid or '' }}</textarea>
</div>
<div class="form-group">
<label for="ssn">SSN</label>
<input id="ssn" name="ssn" type="text" value="{{ contact.ssn or '' }}">
</div>
{% if all_needs %}
<div class="form-group">
<label>Needs/Accommodations</label>
<div class="checkbox-group">
{% for need in all_needs %}
<label class="checkbox-label">
<input type="checkbox" name="need_ids" value="{{ need.id }}"
{% if contact and need in contact.needs %}checked{% endif %}>
{{ need.name }}
</label>
{% endfor %}
</div>
</div>
{% endif %}
<div class="form-actions">
<button type="submit" class="btn btn-primary">Save</button>
{% if contact %}
<a href="/contacts/{{ contact.id }}" class="btn">Cancel</a>
{% else %}
<a href="/contacts" class="btn">Cancel</a>
{% endif %}
</div>
</form>
</div>
{% endblock %}
+14
View File
@@ -0,0 +1,14 @@
{% extends "base.html" %}
{% block title %}Contacts{% endblock %}
{% block content %}
<div class="contact-list">
<div class="header">
<h1>Contacts</h1>
<a href="/contacts/new" class="btn btn-primary">Add Contact</a>
</div>
<div id="contact-table">
{% include "partials/contact_table.html" %}
</div>
</div>
{% endblock %}
+198
View File
@@ -0,0 +1,198 @@
{% extends "base.html" %}
{% block title %}Relationship Graph{% endblock %}
{% block content %}
<div class="graph-container">
<div class="header">
<h1>Relationship Graph</h1>
</div>
<p class="graph-hint">Drag nodes to reposition. Closer relationships have shorter, darker edges.</p>
<canvas id="graph-canvas" width="900" height="600"
style="border: 1px solid var(--color-border); border-radius: 8px; background: var(--color-bg); cursor: grab;">
</canvas>
<div id="selected-info"></div>
<div class="legend">
<h4>Relationship Closeness (1-10)</h4>
<div class="legend-items">
<div class="legend-item">
<span class="legend-line" style="background: hsl(220, 70%, 40%); height: 4px; display: inline-block;"></span>
<span>10 - Very Close (Spouse, Partner)</span>
</div>
<div class="legend-item">
<span class="legend-line" style="background: hsl(220, 70%, 52%); height: 3px; display: inline-block;"></span>
<span>7 - Close (Family, Best Friend)</span>
</div>
<div class="legend-item">
<span class="legend-line" style="background: hsl(220, 70%, 64%); height: 2px; display: inline-block;"></span>
<span>4 - Moderate (Friend, Colleague)</span>
</div>
<div class="legend-item">
<span class="legend-line" style="background: hsl(220, 70%, 72%); height: 1px; display: inline-block;"></span>
<span>2 - Distant (Acquaintance)</span>
</div>
</div>
</div>
</div>
<script>
(function() {
const RELATIONSHIP_DISPLAY = {{ relationship_type_display|tojson }};
const graphData = {{ graph_data|tojson }};
const canvas = document.getElementById('graph-canvas');
const ctx = canvas.getContext('2d');
const width = canvas.width;
const height = canvas.height;
const centerX = width / 2;
const centerY = height / 2;
const nodes = graphData.nodes.map(function(node) {
return Object.assign({}, node, {
x: centerX + (Math.random() - 0.5) * 300,
y: centerY + (Math.random() - 0.5) * 300,
vx: 0,
vy: 0
});
});
const nodeMap = new Map(nodes.map(function(node) { return [node.id, node]; }));
const edges = graphData.edges.map(function(edge) {
const sourceNode = nodeMap.get(edge.source);
const targetNode = nodeMap.get(edge.target);
if (!sourceNode || !targetNode) return null;
return Object.assign({}, edge, { sourceNode: sourceNode, targetNode: targetNode });
}).filter(function(edge) { return edge !== null; });
let dragNode = null;
let selectedNode = null;
const repulsion = 5000;
const springStrength = 0.05;
const baseSpringLength = 150;
const damping = 0.9;
const centerPull = 0.01;
function simulate() {
for (const node of nodes) { node.vx = 0; node.vy = 0; }
for (let i = 0; i < nodes.length; i++) {
for (let j = i + 1; j < nodes.length; j++) {
const dx = nodes[j].x - nodes[i].x;
const dy = nodes[j].y - nodes[i].y;
const dist = Math.sqrt(dx * dx + dy * dy) || 1;
const force = repulsion / (dist * dist);
const fx = (dx / dist) * force;
const fy = (dy / dist) * force;
nodes[i].vx -= fx; nodes[i].vy -= fy;
nodes[j].vx += fx; nodes[j].vy += fy;
}
}
for (const edge of edges) {
const dx = edge.targetNode.x - edge.sourceNode.x;
const dy = edge.targetNode.y - edge.sourceNode.y;
const dist = Math.sqrt(dx * dx + dy * dy) || 1;
const normalizedWeight = edge.closeness_weight / 10;
const idealLength = baseSpringLength * (1.5 - normalizedWeight);
const displacement = dist - idealLength;
const force = springStrength * displacement;
const fx = (dx / dist) * force;
const fy = (dy / dist) * force;
edge.sourceNode.vx += fx; edge.sourceNode.vy += fy;
edge.targetNode.vx -= fx; edge.targetNode.vy -= fy;
}
for (const node of nodes) {
node.vx += (centerX - node.x) * centerPull;
node.vy += (centerY - node.y) * centerPull;
}
for (const node of nodes) {
if (node === dragNode) continue;
node.x += node.vx * damping;
node.y += node.vy * damping;
node.x = Math.max(30, Math.min(width - 30, node.x));
node.y = Math.max(30, Math.min(height - 30, node.y));
}
}
function getEdgeColor(weight) {
const normalized = weight / 10;
return 'hsl(220, 70%, ' + (80 - normalized * 40) + '%)';
}
function draw() {
ctx.clearRect(0, 0, width, height);
for (const edge of edges) {
const lineWidth = 1 + (edge.closeness_weight / 10) * 3;
ctx.strokeStyle = getEdgeColor(edge.closeness_weight);
ctx.lineWidth = lineWidth;
ctx.beginPath();
ctx.moveTo(edge.sourceNode.x, edge.sourceNode.y);
ctx.lineTo(edge.targetNode.x, edge.targetNode.y);
ctx.stroke();
const midX = (edge.sourceNode.x + edge.targetNode.x) / 2;
const midY = (edge.sourceNode.y + edge.targetNode.y) / 2;
ctx.fillStyle = '#666';
ctx.font = '10px sans-serif';
ctx.textAlign = 'center';
const label = RELATIONSHIP_DISPLAY[edge.relationship_type] || edge.relationship_type;
ctx.fillText(label, midX, midY - 5);
}
for (const node of nodes) {
const isSelected = node === selectedNode;
const radius = isSelected ? 25 : 20;
ctx.beginPath();
ctx.arc(node.x, node.y, radius, 0, Math.PI * 2);
ctx.fillStyle = isSelected ? '#0066cc' : '#fff';
ctx.fill();
ctx.strokeStyle = '#0066cc';
ctx.lineWidth = 2;
ctx.stroke();
ctx.fillStyle = isSelected ? '#fff' : '#333';
ctx.font = '12px sans-serif';
ctx.textAlign = 'center';
ctx.textBaseline = 'middle';
const name = node.name.length > 10 ? node.name.slice(0, 9) + '\u2026' : node.name;
ctx.fillText(name, node.x, node.y);
}
}
function animate() {
simulate();
draw();
requestAnimationFrame(animate);
}
animate();
function getNodeAt(x, y) {
for (const node of nodes) {
const dx = x - node.x;
const dy = y - node.y;
if (dx * dx + dy * dy < 400) return node;
}
return null;
}
canvas.addEventListener('mousedown', function(event) {
const rect = canvas.getBoundingClientRect();
const node = getNodeAt(event.clientX - rect.left, event.clientY - rect.top);
if (node) {
dragNode = node;
selectedNode = node;
const infoDiv = document.getElementById('selected-info');
let html = '<div class="selected-info"><h3>' + node.name + '</h3>';
if (node.current_job) html += '<p>Job: ' + node.current_job + '</p>';
html += '<a href="/contacts/' + node.id + '">View details</a></div>';
infoDiv.innerHTML = html;
}
});
canvas.addEventListener('mousemove', function(event) {
if (!dragNode) return;
const rect = canvas.getBoundingClientRect();
dragNode.x = event.clientX - rect.left;
dragNode.y = event.clientY - rect.top;
});
canvas.addEventListener('mouseup', function() { dragNode = null; });
canvas.addEventListener('mouseleave', function() { dragNode = null; });
})();
</script>
{% endblock %}
+31
View File
@@ -0,0 +1,31 @@
{% extends "base.html" %}
{% block title %}Needs{% endblock %}
{% block content %}
<div class="need-list">
<div class="header">
<h1>Needs / Accommodations</h1>
<button class="btn btn-primary" onclick="document.getElementById('need-form').toggleAttribute('hidden')">Add Need</button>
</div>
<form id="need-form" hidden
hx-post="/htmx/needs"
hx-target="#need-items"
hx-swap="innerHTML"
hx-on::after-request="if(event.detail.successful) this.reset()"
class="need-form">
<div class="form-group">
<label for="name">Name *</label>
<input id="name" name="name" type="text" placeholder="e.g., Light Sensitive, ADHD" required>
</div>
<div class="form-group">
<label for="description">Description</label>
<textarea id="description" name="description" placeholder="Optional description..." rows="2"></textarea>
</div>
<button type="submit" class="btn btn-primary">Create</button>
</form>
<div id="need-items">
{% include "partials/need_items.html" %}
</div>
</div>
{% endblock %}
@@ -0,0 +1,33 @@
{% if contacts %}
<table>
<thead>
<tr>
<th>Name</th>
<th>Job</th>
<th>Timezone</th>
<th>Actions</th>
</tr>
</thead>
<tbody>
{% for contact in contacts %}
<tr id="contact-row-{{ contact.id }}">
<td><a href="/contacts/{{ contact.id }}">{{ contact.name }}</a></td>
<td>{{ contact.current_job or "-" }}</td>
<td>{{ contact.timezone or "-" }}</td>
<td>
<a href="/contacts/{{ contact.id }}/edit" class="btn">Edit</a>
<button class="btn btn-danger"
hx-delete="/api/contacts/{{ contact.id }}"
hx-target="#contact-row-{{ contact.id }}"
hx-swap="outerHTML"
hx-confirm="Delete this contact?">
Delete
</button>
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p>No contacts yet.</p>
{% endif %}
@@ -0,0 +1,14 @@
<ul class="manage-needs-list">
{% for need in contact.needs %}
<li id="contact-need-{{ need.id }}">
<strong>{{ need.name }}</strong>
{% if need.description %}<span> - {{ need.description }}</span>{% endif %}
<button class="btn btn-small btn-danger"
hx-delete="/api/contacts/{{ contact.id }}/needs/{{ need.id }}"
hx-target="#contact-need-{{ need.id }}"
hx-swap="outerHTML">
Remove
</button>
</li>
{% endfor %}
</ul>
@@ -0,0 +1,23 @@
{% for rel in contact.related_to %}
<div class="manage-rel-item" id="rel-{{ contact.id }}-{{ rel.related_contact_id }}">
<a href="/contacts/{{ rel.related_contact_id }}">{{ contact_names[rel.related_contact_id] }}</a>
<span class="tag">{{ rel.relationship_type|replace("_", " ")|title }}</span>
<label class="weight-control">
<span>Closeness:</span>
<input type="range" min="1" max="10" value="{{ rel.closeness_weight }}"
hx-post="/htmx/contacts/{{ contact.id }}/relationships/{{ rel.related_contact_id }}/weight"
hx-trigger="change"
hx-include="this"
name="closeness_weight"
hx-swap="none"
oninput="this.nextElementSibling.textContent = this.value">
<span class="weight-value">{{ rel.closeness_weight }}</span>
</label>
<button class="btn btn-small btn-danger"
hx-delete="/api/contacts/{{ contact.id }}/relationships/{{ rel.related_contact_id }}"
hx-target="#rel-{{ contact.id }}-{{ rel.related_contact_id }}"
hx-swap="outerHTML">
Remove
</button>
</div>
{% endfor %}
@@ -0,0 +1,21 @@
{% if needs %}
<ul class="need-items">
{% for need in needs %}
<li id="need-item-{{ need.id }}">
<div class="need-info">
<strong>{{ need.name }}</strong>
{% if need.description %}<p>{{ need.description }}</p>{% endif %}
</div>
<button class="btn btn-danger"
hx-delete="/api/needs/{{ need.id }}"
hx-target="#need-item-{{ need.id }}"
hx-swap="outerHTML"
hx-confirm="Delete this need?">
Delete
</button>
</li>
{% endfor %}
</ul>
{% else %}
<p>No needs defined yet.</p>
{% endif %}
+34 -9
View File
@@ -3,23 +3,28 @@
from __future__ import annotations
import logging
import sys
from datetime import UTC, datetime
from pathlib import Path
from os import getenv
from subprocess import PIPE, Popen
from python.logging_config import configure_logger as _configure_logger
from apprise import Apprise
logger = logging.getLogger(__name__)
def get_repo_dir() -> Path:
"""Return the repository root directory."""
return Path(__file__).resolve().parents[1]
def configure_logger(level: str = "INFO") -> None:
"""Configure the logger."""
_configure_logger(level)
"""Configure the logger.
Args:
level (str, optional): The logging level. Defaults to "INFO".
"""
logging.basicConfig(
level=level,
datefmt="%Y-%m-%dT%H:%M:%S%z",
format="%(asctime)s %(levelname)s %(filename)s:%(lineno)d - %(message)s",
handlers=[logging.StreamHandler(sys.stdout)],
)
def bash_wrapper(command: str) -> tuple[str, int]:
@@ -42,6 +47,26 @@ def bash_wrapper(command: str) -> tuple[str, int]:
return output.decode(), process.returncode
def signal_alert(body: str, title: str = "") -> None:
"""Send a signal alert.
Args:
body (str): The body of the alert.
title (str, optional): The title of the alert. Defaults to "".
"""
apprise_client = Apprise()
from_phone = getenv("SIGNAL_ALERT_FROM_PHONE")
to_phone = getenv("SIGNAL_ALERT_TO_PHONE")
if not from_phone or not to_phone:
logger.info("SIGNAL_ALERT_FROM_PHONE or SIGNAL_ALERT_TO_PHONE not set")
return
apprise_client.add(f"signal://localhost:8989/{from_phone}/{to_phone}")
apprise_client.notify(title=title, body=body)
def utcnow() -> datetime:
"""Get the current UTC time."""
return datetime.now(tz=UTC)
+22 -3
View File
@@ -4,10 +4,12 @@ Usage:
database <db_name> <command> [args...]
Examples:
database van_inventory upgrade head
database van_inventory downgrade head-1
database van_inventory revision --autogenerate -m "add meals table"
database van_inventory check
database richie check
database richie upgrade head
database richie downgrade head-1
database richie revision --autogenerate -m "add meals table"
"""
from __future__ import annotations
@@ -46,7 +48,10 @@ class DatabaseConfig:
def alembic_config(self) -> Config:
"""Build an alembic Config for this database."""
cfg = Config()
# Runtime import needed — Config is in TYPE_CHECKING for the return type annotation
from alembic.config import Config as AlembicConfig # noqa: PLC0415
cfg = AlembicConfig()
cfg.set_main_option("script_location", self.script_location)
cfg.set_main_option("file_template", self.file_template)
cfg.set_main_option("prepend_sys_path", ".")
@@ -71,6 +76,20 @@ DATABASES: dict[str, DatabaseConfig] = {
base_class_name="RichieBase",
models_module="python.orm.richie",
),
"van_inventory": DatabaseConfig(
env_prefix="VAN_INVENTORY",
version_location="python/alembic/van_inventory/versions",
base_module="python.orm.van_inventory.base",
base_class_name="VanInventoryBase",
models_module="python.orm.van_inventory.models",
),
"signal_bot": DatabaseConfig(
env_prefix="SIGNALBOT",
version_location="python/alembic/signal_bot/versions",
base_module="python.orm.signal_bot.base",
base_class_name="SignalBotBase",
models_module="python.orm.signal_bot.models",
),
}
-1
View File
@@ -1 +0,0 @@
"""EPUB search package."""
-50
View File
@@ -1,50 +0,0 @@
"""Grounded answer generation."""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
from python.ebook_search.llm_interface import request_chat_completion
from python.ebook_search.prompts import load_prompt
if TYPE_CHECKING:
import httpx
from python.ebook_search.config import EbookSearchConfig
from python.ebook_search.search import SearchResult
logger = logging.getLogger(__name__)
async def answer_query(
client: httpx.AsyncClient,
query: str,
results: list[SearchResult],
config: EbookSearchConfig,
) -> str:
"""Answer a question using only retrieved chunks."""
if not config.answer_enabled:
logger.info("ebook_answer_skipped_disabled")
return "Answer generation is disabled. Source chunks are shown below."
if not results:
logger.info("ebook_answer_skipped_no_results")
return "No relevant sources were found."
logger.info(
f"ebook_answer_request_start {config.vllm_base_url=} {config.chat_model=} sources={len(results)} "
f"query_length={len(query)}"
)
context = "\n\n".join(
f"[{index}] {result.source_title}{' - ' + result.chapter_title if result.chapter_title else ''}\n{result.text}"
for index, result in enumerate(results, start=1)
)
content = await request_chat_completion(
client,
config,
load_prompt("answer").messages(query=query, context=context),
)
logger.info(f"ebook_answer_request_complete {config.chat_model=} answer_length={len(content)}")
return content or "The model returned an empty answer."
-1
View File
@@ -1 +0,0 @@
"""Web and external API adapters for EPUB search."""
-70
View File
@@ -1,70 +0,0 @@
"""Background BM25 refresh tasks for the web app.
The refresh is scheduled on the event loop instead of a thread because the async psycopg
driver only works from the loop; a bare thread cannot open a session on the async engine.
"""
from __future__ import annotations
import asyncio
import logging
from typing import TYPE_CHECKING
from sqlalchemy.ext.asyncio import AsyncSession
from python.ebook_search.bm25_corpus import load_bm25_corpus, refresh_bm25_corpus
if TYPE_CHECKING:
from fastapi import FastAPI
from sqlalchemy.ext.asyncio import AsyncEngine
from python.ebook_search.config import EbookSearchConfig
logger = logging.getLogger(__name__)
def schedule_bm25_refresh(app: FastAPI) -> None:
"""Schedule a delayed BM25 corpus refresh, replacing any pending refresh.
Only called from route handlers, so a running event loop is guaranteed.
"""
cancel_bm25_refresh(app)
loop = asyncio.get_running_loop()
def start_refresh() -> None:
app.state.bm25_refresh_task = loop.create_task(refresh_bm25_for_app(app))
app.state.bm25_refresh_timer = loop.call_later(app.state.config.bm25_refresh_delay_seconds, start_refresh)
logger.info(f"ebook_bm25_refresh_scheduled {app.state.config.bm25_refresh_delay_seconds=}")
def cancel_bm25_refresh(app: FastAPI) -> None:
"""Cancel any pending BM25 corpus refresh timer and in-flight refresh task."""
existing_timer = getattr(app.state, "bm25_refresh_timer", None)
if existing_timer is not None:
existing_timer.cancel()
app.state.bm25_refresh_timer = None
logger.info("ebook_bm25_refresh_cancelled")
existing_task = getattr(app.state, "bm25_refresh_task", None)
if existing_task is not None:
if not existing_task.done():
existing_task.cancel()
app.state.bm25_refresh_task = None
async def refresh_bm25_for_app(app: FastAPI) -> None:
"""Refresh the BM25 corpus using the app engine and config."""
try:
await refresh_bm25_for_engine(app.state.engine, app.state.config)
except Exception:
logger.exception("ebook_bm25_refresh_failed")
async def refresh_bm25_for_engine(engine: AsyncEngine, config: EbookSearchConfig) -> None:
"""Refresh the BM25 corpus using an async SQLAlchemy engine."""
async with AsyncSession(engine) as session:
await refresh_bm25_corpus(session, config)
load_bm25_corpus.cache_clear()
logger.info("ebook_bm25_corpus_cache_cleared_after_refresh")
-24
View File
@@ -1,24 +0,0 @@
"""FastAPI dependencies for the EPUB search app."""
from __future__ import annotations
from typing import Annotated
import httpx
from fastapi import Depends, Request
from python.ebook_search.config import EbookSearchConfig
def get_config(request: Request) -> EbookSearchConfig:
"""Get the loaded search config from app state."""
return request.app.state.config
def get_http_client(request: Request) -> httpx.AsyncClient:
"""Get the shared LLM HTTP client from app state."""
return request.app.state.http_client
AppConfig = Annotated[EbookSearchConfig, Depends(get_config)]
AppHttpClient = Annotated[httpx.AsyncClient, Depends(get_http_client)]
-127
View File
@@ -1,127 +0,0 @@
"""Background phrase-judging tasks for the web app.
Judging a book sends one LLM request per candidate phrase, which can take minutes, so it must
not run inside the request where it would block the UI. Judgments run as async FastAPI
background tasks, awaited on the event loop after the response is sent, and are tracked per
book in app state so a second judge request for a book that is already being judged is
rejected instead of doubling the work.
State is loop-confined: every read and mutation happens on the event loop (async route
handlers and async background tasks) and no critical section contains an ``await``, so each
mutation is atomic per loop iteration and no locking is needed.
"""
from __future__ import annotations
import logging
from dataclasses import dataclass, field
from typing import TYPE_CHECKING
from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
if TYPE_CHECKING:
from fastapi import BackgroundTasks, FastAPI
logger = logging.getLogger(__name__)
@dataclass
class JudgeTaskState:
"""Running book judgments and last outcome messages, keyed by book id."""
running_book_ids: set[int] = field(default_factory=set)
outcome_messages: dict[int, str] = field(default_factory=dict)
def get_judge_task_state(app: FastAPI) -> JudgeTaskState:
"""Return the app's judge task state, creating it on first use.
Args:
app (FastAPI): App whose state holds the judge task registry.
Returns:
JudgeTaskState: The shared judge task state for this app.
"""
state = getattr(app.state, "judge_tasks", None)
if state is None:
state = JudgeTaskState()
app.state.judge_tasks = state
return state
def start_book_phrase_judgment(app: FastAPI, background_tasks: BackgroundTasks, source_id: int) -> bool:
"""Queue judging of one book's candidate phrases as a FastAPI background task.
The book is claimed before the response returns, so a repeated judge request cannot queue
a second run while one is pending or running.
Args:
app (FastAPI): App supplying the engine, config, and judge task state.
background_tasks (BackgroundTasks): Request's background tasks to queue the judgment on.
source_id (int): Book to judge candidates for.
Returns:
bool: True when a judgment was queued, False when one is already running for this book.
"""
state = get_judge_task_state(app)
if source_id in state.running_book_ids:
logger.info(f"ebook_book_phrase_judgment_already_running {source_id=}")
return False
state.running_book_ids.add(source_id)
state.outcome_messages.pop(source_id, None)
background_tasks.add_task(judge_book_phrases_for_app, app, source_id)
logger.info(f"ebook_book_phrase_judgment_queued {source_id=}")
return True
async def judge_book_phrases_for_app(app: FastAPI, source_id: int) -> None:
"""Judge one book using the app engine and config, recording the outcome message.
Args:
app (FastAPI): App supplying the engine, config, and judge task state.
source_id (int): Book to judge candidates for.
"""
state = get_judge_task_state(app)
try:
result = await judge_candidate_phrases_for_books(app.state.engine, app.state.config, source_ids=[source_id])
logger.info(
f"ebook_book_phrase_judgment_complete {source_id=} {result.candidates_judged=} {result.protected_phrases=} "
f"{result.phrase_mentions=} {result.books_failed=}"
)
if result.books_failed:
message = "Judging failed; see server logs for details"
else:
message = (
f"Judged {result.candidates_judged} candidates; {result.protected_phrases} protected phrases promoted"
)
except Exception:
logger.exception(f"ebook_book_phrase_judgment_task_failed {source_id=}")
message = "Judging failed; see server logs for details"
state.running_book_ids.discard(source_id)
state.outcome_messages[source_id] = message
def is_judging_book(app: FastAPI, source_id: int) -> bool:
"""Report whether a judgment is currently queued or running for one book.
Args:
app (FastAPI): App supplying the judge task state.
source_id (int): Book to check.
Returns:
bool: True while the book's judgment is pending or running.
"""
return source_id in get_judge_task_state(app).running_book_ids
def pop_book_judgment_outcome(app: FastAPI, source_id: int) -> str | None:
"""Return and clear the outcome message from one book's last finished judgment.
Args:
app (FastAPI): App supplying the judge task state.
source_id (int): Book to fetch the outcome for.
Returns:
str | None: The outcome message, or None when there is nothing new to report.
"""
return get_judge_task_state(app).outcome_messages.pop(source_id, None)
-91
View File
@@ -1,91 +0,0 @@
"""FastAPI HTMX app for EPUB search."""
from __future__ import annotations
import logging
from contextlib import asynccontextmanager
from typing import TYPE_CHECKING, Annotated
import httpx
import typer
import uvicorn
from fastapi import FastAPI
from fastapi.staticfiles import StaticFiles
from sqlalchemy.ext.asyncio import AsyncSession
from python.common import configure_logger
from python.ebook_search.api.bm25_tasks import cancel_bm25_refresh
from python.ebook_search.api.routes import admin_router, health_router, page_router, search_router
from python.ebook_search.api.web import STATIC_DIR
from python.ebook_search.bm25_corpus import ensure_bm25_corpus
from python.ebook_search.config import load_config
from python.ebook_search.protected_phrases.pool import shutdown_extraction_pool
from python.fastapi_tools import ZstdMiddleware
from python.orm.common import get_async_postgres_engine
if TYPE_CHECKING:
from collections.abc import AsyncIterator
logger = logging.getLogger(__name__)
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
"""Manage application startup and shutdown resources."""
logger.info("ebook_search_startup")
config = load_config()
app.state.config = config
logger.info(
f"ebook_search_config_loaded {config.top_k=} {config.embedding_model=} {config.embedding_base_url=} "
f"{config.vllm_base_url=} {config.rerank.enabled=} {config.phrase_matching_enabled=} {config.answer_enabled=} "
f"library_paths={len(config.library_paths)}"
)
if not config.library_paths:
logger.warning("ebook_search_no_library_paths_configured")
# Concurrent phrase judging opens one session per book worker on this engine, so size the pool
# to cover those plus headroom for ordinary web requests.
app.state.engine = get_async_postgres_engine(
name="RICHIE",
vector_engine=True,
pool_size=config.phrase_judge_book_workers + 10,
)
app.state.http_client = httpx.AsyncClient()
async with AsyncSession(app.state.engine, expire_on_commit=False) as session:
await ensure_bm25_corpus(session, config)
try:
yield
finally:
logger.info("ebook_search_shutdown")
cancel_bm25_refresh(app)
shutdown_extraction_pool()
await app.state.http_client.aclose()
await app.state.engine.dispose()
def create_app() -> FastAPI:
"""Create the EPUB search web app."""
app = FastAPI(title="EPUB Search", lifespan=lifespan)
app.add_middleware(ZstdMiddleware)
app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
app.include_router(admin_router)
app.include_router(health_router)
app.include_router(page_router)
app.include_router(search_router)
return app
def serve(
host: Annotated[str, typer.Option("--host", "-h", help="Host to bind to")] = "127.0.0.1",
port: Annotated[int, typer.Option("--port", "-p", help="Port to bind to")] = 8070,
log_level: Annotated[str, typer.Option("--log-level", "-l", help="Log level")] = "INFO",
) -> None:
"""Start the EPUB search server."""
configure_logger(log_level)
uvicorn.run(create_app(), host=host, port=port)
if __name__ == "__main__":
typer.run(serve)
@@ -1,13 +0,0 @@
"""EPUB search web route modules."""
from python.ebook_search.api.routes.admin import router as admin_router
from python.ebook_search.api.routes.health import router as health_router
from python.ebook_search.api.routes.page import router as page_router
from python.ebook_search.api.routes.search import router as search_router
__all__ = [
"admin_router",
"health_router",
"page_router",
"search_router",
]
-204
View File
@@ -1,204 +0,0 @@
"""Admin routes for the EPUB search web UI."""
from __future__ import annotations
import logging
from fastapi import APIRouter, Request
from fastapi.responses import HTMLResponse
from python.ebook_search.api.bm25_tasks import schedule_bm25_refresh
from python.ebook_search.api.dependencies import ( # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
AppConfig,
AppHttpClient,
)
from python.ebook_search.api.web import error_response, templates
from python.ebook_search.embeddings import embed_missing_chunks, embedding_model_stats
from python.ebook_search.ingest import ingest_configured_paths
from python.ebook_search.protected_phrases.generate_ngrams import generate_candidate_phrases_for_books
from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
from python.ebook_search.protected_phrases.store import book_ids_pending_first_judgment, corpus_phrase_stats
from python.fastapi_tools import ( # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
AppAsyncEngine,
AsyncDbSession,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/admin")
@router.get("", response_class=HTMLResponse)
async def admin(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
"""Render the admin page."""
stats = await embedding_model_stats(session)
phrase_stats = await corpus_phrase_stats(session)
logger.info(
f"ebook_admin_page_loaded models={len(stats)} {phrase_stats.candidate_phrases=} "
f"{phrase_stats.protected_phrases=}"
)
return templates.TemplateResponse(
request,
"admin.html",
{"config": config, "stats": stats, "phrase_stats": phrase_stats},
)
@router.post("/scan", response_class=HTMLResponse)
async def scan_library(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
"""Scan configured library paths for EPUB changes."""
try:
count = await ingest_configured_paths(session, config)
await session.commit()
except Exception as error:
logger.exception("ebook_admin_scan_failed")
return error_response(request, error)
logger.info(f"ebook_admin_scan_complete {count=}")
if count > 0:
schedule_bm25_refresh(request.app)
return templates.TemplateResponse(request, "partials/admin_status.html", {"message": f"Indexed {count} EPUBs"})
@router.post("/phrases/generate-all", response_class=HTMLResponse)
async def generate_all_phrases(request: Request, config: AppConfig, engine: AppAsyncEngine) -> HTMLResponse:
"""Regenerate candidate phrases for every indexed book without LLM judging."""
try:
result = await generate_candidate_phrases_for_books(engine, config)
except Exception as error:
logger.exception("ebook_admin_generate_phrases_failed")
return error_response(request, error)
logger.info(
f"ebook_admin_generate_phrases_complete {result.books_seen=} {result.books_built=} {result.candidate_phrases=}"
)
return templates.TemplateResponse(
request,
"partials/admin_status.html",
{
"message": (
f"Generated phrases for {result.books_built} of {result.books_seen} books; "
f"{result.candidate_phrases} candidates stored"
)
},
)
@router.post("/phrases/judge-all", response_class=HTMLResponse)
async def judge_all_phrases(request: Request, engine: AppAsyncEngine, config: AppConfig) -> HTMLResponse:
"""Judge unjudged candidate phrases across every indexed book."""
return await run_phrase_judgment(request, engine, config, source_ids=None)
@router.post("/phrases/judge-missing", response_class=HTMLResponse)
async def judge_missing_phrases(
request: Request,
engine: AppAsyncEngine,
config: AppConfig,
session: AsyncDbSession,
) -> HTMLResponse:
"""Judge candidate phrases only for books where judging has never run."""
source_ids = await book_ids_pending_first_judgment(session)
if not source_ids:
return templates.TemplateResponse(
request,
"partials/admin_status.html",
{"message": "All books with candidate phrases have been judged"},
)
return await run_phrase_judgment(request, engine, config, source_ids=source_ids)
async def run_phrase_judgment(
request: Request,
engine: AppAsyncEngine,
config: AppConfig,
*,
source_ids: list[int] | None,
) -> HTMLResponse:
"""Run LLM judging for candidate phrases and render the outcome as an admin status partial.
Args:
request (Request): Current request, for template rendering.
engine (AppAsyncEngine): Engine used to open per-book judging sessions.
config (AppConfig): Runtime phrase-tuning settings.
source_ids (list[int] | None): Books to judge; ``None`` judges every indexed book.
Returns:
HTMLResponse: Status partial describing the judging outcome.
"""
try:
result = await judge_candidate_phrases_for_books(engine, config, source_ids=source_ids)
except Exception as error:
logger.exception("ebook_admin_judge_phrases_failed")
return error_response(request, error)
logger.info(
f"ebook_admin_judge_phrases_complete {result.books_seen=} {result.books_judged=} {result.books_failed=} "
f"{result.candidates_judged=} {result.protected_phrases=} {result.phrase_mentions=}"
)
return templates.TemplateResponse(
request,
"partials/admin_status.html",
{
"message": (
f"Judged {result.candidates_judged} candidates across {result.books_judged} of "
f"{result.books_seen} books; {result.protected_phrases} protected phrases, "
f"{result.phrase_mentions} mentions"
+ (f"; {result.books_failed} books failed" if result.books_failed else "")
)
},
)
@router.post("/embed-missing", response_class=HTMLResponse)
async def embed_missing(
request: Request,
config: AppConfig,
session: AsyncDbSession,
client: AppHttpClient,
) -> HTMLResponse:
"""Embed chunks missing vectors for the configured model."""
try:
count = await embed_missing_chunks(session, client, config)
await session.commit()
except Exception as error:
logger.exception("ebook_admin_embed_missing_failed")
return error_response(request, error)
logger.info(f"ebook_admin_embed_missing_complete {count=}")
return templates.TemplateResponse(
request,
"partials/admin_status.html",
{"message": f"Embedded {count} chunks"},
)
@router.post("/embed-all", response_class=HTMLResponse)
async def embed_all(
request: Request,
config: AppConfig,
session: AsyncDbSession,
client: AppHttpClient,
) -> HTMLResponse:
"""Embed all chunks missing vectors in fixed-size batches."""
total = 0
batches = 0
try:
while True:
count = await embed_missing_chunks(session, client, config)
if count == 0:
break
await session.commit()
total += count
batches += 1
logger.info(f"ebook_admin_embed_all_batch_complete {batches=} {count=} {total=}")
except Exception as error:
logger.exception(f"ebook_admin_embed_all_failed {batches=} {total=}")
return error_response(request, f"Embed all failed after {total} chunks in {batches} batches: {error}")
logger.info(f"ebook_admin_embed_all_complete {batches=} {total=}")
return templates.TemplateResponse(
request,
"partials/admin_status.html",
{"message": f"Embedded {total} chunks in {batches} batches of {config.embedding_batch_size}"},
)
-92
View File
@@ -1,92 +0,0 @@
"""Liveness and readiness routes for the EPUB search service."""
from __future__ import annotations
import logging
from http import HTTPStatus
from typing import TYPE_CHECKING
from fastapi import APIRouter
from fastapi.responses import JSONResponse
from sqlalchemy import literal, select
from sqlalchemy.exc import SQLAlchemyError
from python.ebook_search.api.dependencies import ( # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
AppConfig,
AppHttpClient,
)
from python.ebook_search.bm25_corpus import bm25_index_exists, bm25_index_path, read_bm25_manifest
from python.ebook_search.llm_interface import check_chat_endpoint, check_embedding_endpoint
from python.fastapi_tools import AsyncDbSession # noqa: TC001 FastAPI resolves this annotated dependency at runtime
if TYPE_CHECKING:
import httpx
from sqlalchemy.ext.asyncio import AsyncSession
from python.ebook_search.config import EbookSearchConfig
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/health")
async def health() -> dict[str, str]:
"""Liveness probe that returns ok without touching dependencies."""
return {"status": "ok"}
@router.get("/ready")
async def ready(config: AppConfig, session: AsyncDbSession, client: AppHttpClient) -> JSONResponse:
"""Readiness probe reporting database, embedding endpoint, and BM25 index status."""
database_ok = await check_database(session)
embedding_ok = await check_embedding_endpoint(client, config)
chat_status = await chat_endpoint_status(client, config)
bm25_status = check_bm25_status(config)
checks = {
"database": "ok" if database_ok else "fail",
"embedding": "ok" if embedding_ok else "fail",
"chat": chat_status,
"bm25": bm25_status,
}
if not database_ok:
status = "unavailable"
status_code = HTTPStatus.SERVICE_UNAVAILABLE
elif not embedding_ok or chat_status == "fail" or bm25_status == "missing":
status = "degraded"
status_code = HTTPStatus.OK
else:
status = "ready"
status_code = HTTPStatus.OK
logger.info(f"ebook_ready_check {status=} {database_ok=} {embedding_ok=} {chat_status=} {bm25_status=}")
return JSONResponse(content={"status": status, "checks": checks}, status_code=status_code)
async def chat_endpoint_status(client: httpx.AsyncClient, config: EbookSearchConfig) -> str:
"""Return the answering chat endpoint status, or disabled when answers are off."""
if not config.answer_enabled:
return "disabled"
return "ok" if await check_chat_endpoint(client, config) else "fail"
async def check_database(session: AsyncSession) -> bool:
"""Return whether the database answers a trivial query."""
try:
await session.execute(select(literal(1)))
except SQLAlchemyError as error:
logger.warning(f"ebook_ready_database_unavailable {error=}")
return False
return True
def check_bm25_status(config: EbookSearchConfig) -> str:
"""Return the persisted BM25 index status without loading it into memory."""
index_path = bm25_index_path(config)
manifest = read_bm25_manifest(index_path)
if manifest is None or not bm25_index_exists(index_path, manifest):
return "missing"
if manifest.chunk_count == 0:
return "empty"
return "ok"
-187
View File
@@ -1,187 +0,0 @@
"""Page routes for the EPUB search web UI."""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
from fastapi.responses import HTMLResponse, RedirectResponse
from sqlalchemy import func, select
from python.ebook_search.api.dependencies import (
AppConfig, # noqa: TC001 FastAPI resolves this annotated dependency at runtime
)
from python.ebook_search.api.judge_tasks import is_judging_book, pop_book_judgment_outcome, start_book_phrase_judgment
from python.ebook_search.api.web import templates
from python.ebook_search.protected_phrases.generate_ngrams import recalculate_candidate_phrases_for_book
from python.ebook_search.protected_phrases.store import count_protected_phrases
from python.fastapi_tools import AsyncDbSession # noqa: TC001 FastAPI resolves this annotated dependency at runtime
from python.orm.richie import EbookCandidatePhrase, EbookChapter, EbookChunk, EbookProtectedPhrase, EbookSource
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/", response_class=HTMLResponse)
async def index(request: Request, config: AppConfig) -> HTMLResponse:
"""Render the search page."""
return templates.TemplateResponse(request, "search.html", {"config": config})
@router.get("/books", response_class=HTMLResponse)
async def books(request: Request, session: AsyncDbSession) -> HTMLResponse:
"""Render the indexed books page."""
sources = list((await session.scalars(select(EbookSource).order_by(EbookSource.title))).all())
logger.info(f"ebook_books_page_loaded count={len(sources)}")
return templates.TemplateResponse(request, "books.html", {"sources": sources})
async def get_chapter_count(session: AsyncSession, book_id: int) -> int:
"""Return the number of indexed chapters for one book."""
return await session.scalar(select(func.count(EbookChapter.id)).where(EbookChapter.source_id == book_id)) or 0
async def get_chunk_count(session: AsyncSession, book_id: int) -> int:
"""Return the number of indexed chunks for one book."""
return await session.scalar(select(func.count(EbookChunk.id)).where(EbookChunk.source_id == book_id)) or 0
async def get_candidate_count(session: AsyncSession, book_id: int) -> int:
"""Return the number of indexed candidates for one book."""
return (
await session.scalar(select(func.count(EbookCandidatePhrase.id)).where(EbookCandidatePhrase.book_id == book_id))
or 0
)
async def get_judged_candidate_count(session: AsyncSession, book_id: int) -> int:
"""Return the number of judged candidates for one book."""
return (
await session.scalar(
select(func.count(EbookCandidatePhrase.id)).where(
EbookCandidatePhrase.book_id == book_id,
EbookCandidatePhrase.llm_judged.is_(True),
)
)
or 0
)
async def get_candidates(session: AsyncSession, book_id: int) -> list[EbookCandidatePhrase]:
"""Return the indexed candidates for one book."""
return list(
await session.scalars(
select(EbookCandidatePhrase)
.where(EbookCandidatePhrase.book_id == book_id)
.order_by(EbookCandidatePhrase.candidate_score.desc())
.limit(100)
)
)
async def get_protected_phrases(session: AsyncSession, book_id: int) -> list[EbookProtectedPhrase]:
"""Return the protected phrases for one book."""
return list(
await session.scalars(
select(EbookProtectedPhrase)
.where(EbookProtectedPhrase.book_id == book_id)
.order_by(EbookProtectedPhrase.importance.desc())
.limit(100)
)
)
@router.get("/books/{source_id}", response_class=HTMLResponse)
async def book_detail(source_id: int, request: Request, session: AsyncDbSession) -> HTMLResponse:
"""Render details for one indexed book."""
source = await session.get(EbookSource, source_id)
phrase_status_message = None
recalculated = request.query_params.get("phrases_recalculated")
if recalculated is not None:
phrase_status_message = f"Recalculated phrases; {recalculated} candidates generated"
judgment_outcome = pop_book_judgment_outcome(request.app, source_id)
if judgment_outcome is not None:
phrase_status_message = judgment_outcome
judging_in_progress = is_judging_book(request.app, source_id)
if judging_in_progress:
phrase_status_message = "Judging candidate phrases in the background; refresh to see progress"
if source is not None:
chapter_count = await get_chapter_count(session, source.id)
chunk_count = await get_chunk_count(session, source.id)
candidate_count = await get_candidate_count(session, source.id)
judged_candidate_count = await get_judged_candidate_count(session, source.id)
protected_count = await count_protected_phrases(session, source.id)
candidates = await get_candidates(session, source.id)
protected_phrases = await get_protected_phrases(session, source.id)
else:
chapter_count = 0
chunk_count = 0
candidate_count = 0
judged_candidate_count = 0
protected_count = 0
candidates = []
protected_phrases = []
logger.info(
f"ebook_book_detail_loaded {source_id=} found={source is not None} {chapter_count=} {chunk_count=} "
f"{candidate_count=} {judged_candidate_count=} {protected_count=}"
)
return templates.TemplateResponse(
request,
"book_detail.html",
{
"candidate_count": candidate_count,
"candidates": candidates,
"chapter_count": chapter_count,
"chunk_count": chunk_count,
"judged_candidate_count": judged_candidate_count,
"judging_in_progress": judging_in_progress,
"protected_count": protected_count,
"protected_phrases": protected_phrases,
"phrase_status_message": phrase_status_message,
"source": source,
},
)
@router.post("/books/{source_id}/recalculate-phrases")
async def recalculate_book_phrases(source_id: int, config: AppConfig, session: AsyncDbSession) -> RedirectResponse:
"""Clear and regenerate candidate phrases for one indexed book."""
source = await session.get(EbookSource, source_id)
if source is None:
raise HTTPException(status_code=404, detail="Book not found")
try:
result = await recalculate_candidate_phrases_for_book(session, source, config)
except ValueError as error:
raise HTTPException(status_code=409, detail=str(error)) from error
logger.info(
f"ebook_book_phrase_recalculation_complete {source_id=} {result.candidate_phrases=} "
f"{result.deleted_candidates=} {result.deleted_protected_phrases=} {result.deleted_aliases=} "
f"{result.deleted_mentions=}"
)
return RedirectResponse(
url=f"/books/{source_id}?phrases_recalculated={result.candidate_phrases}",
status_code=303,
)
@router.post("/books/{source_id}/judge-phrases")
async def judge_book_phrases(
source_id: int,
request: Request,
background_tasks: BackgroundTasks,
session: AsyncDbSession,
) -> RedirectResponse:
"""Queue background judging of one book's candidate phrases and return immediately."""
source = await session.get(EbookSource, source_id)
if source is None:
raise HTTPException(status_code=404, detail="Book not found")
started = start_book_phrase_judgment(request.app, background_tasks, source.id)
logger.info(f"ebook_book_phrase_judgment_requested {source_id=} {started=}")
return RedirectResponse(url=f"/books/{source_id}", status_code=303)
Loaded 100 of 434 files, more files were not shown because too many files have changed in this diff. Show more