Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Richie
|
0248bc320c |
@@ -1,28 +0,0 @@
|
||||
.git
|
||||
.direnv
|
||||
.mypy_cache
|
||||
.pytest_cache
|
||||
.ruff_cache
|
||||
.venv
|
||||
**/.venv
|
||||
.env
|
||||
.cache
|
||||
.claude
|
||||
.coverage
|
||||
.vscode
|
||||
.stfolder
|
||||
.literotica_data
|
||||
esphome
|
||||
htmlcov
|
||||
data
|
||||
ebooks
|
||||
__pycache__
|
||||
**/__pycache__
|
||||
*.pyc
|
||||
*.pyo
|
||||
.ebook_search_bm25
|
||||
result
|
||||
result-*
|
||||
*.egg-info
|
||||
dist
|
||||
build
|
||||
@@ -8,36 +8,19 @@ on:
|
||||
- cron: "0 22 * * *"
|
||||
|
||||
jobs:
|
||||
prebuild-common:
|
||||
name: prebuild-common-x86-64-v3
|
||||
runs-on: nix-cache-builder
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# portal-1 is the smallest system closure: 95% of its derivations are
|
||||
# shared by all five systems, so it is a maintainable common cache seed.
|
||||
# Keep going so one failing package does not stop unrelated cache entries
|
||||
# from being built.
|
||||
- name: Build common packages
|
||||
run: nixos-rebuild build --keep-going --accept-flake-config --flake ./#portal-1
|
||||
- name: Copy common packages to nix-cache
|
||||
run: nix copy --accept-flake-config --to unix:///host-nix/var/nix/daemon-socket/socket .#nixosConfigurations.portal-1.config.system.build.toplevel
|
||||
|
||||
build:
|
||||
name: build-${{ matrix.system }}
|
||||
needs: prebuild-common
|
||||
runs-on: self-hosted
|
||||
strategy:
|
||||
matrix:
|
||||
system:
|
||||
- "bob"
|
||||
- "brain"
|
||||
- "jeeves"
|
||||
- "rhapsody-in-green"
|
||||
- "portal-1"
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Build default package
|
||||
run: "nixos-rebuild build --accept-flake-config --flake ./#${{ matrix.system }}"
|
||||
run: "nixos-rebuild build --flake ./#${{ matrix.system }}"
|
||||
- name: copy to nix-cache
|
||||
run: nix copy --accept-flake-config --to unix:///host-nix/var/nix/daemon-socket/socket .#nixosConfigurations.${{ matrix.system }}.config.system.build.toplevel
|
||||
run: nix copy --to ssh://jeeves .#nixosConfigurations.${{ matrix.system }}.config.system.build.toplevel
|
||||
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
name: merge_flake_lock_update
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 2 * * 6"
|
||||
|
||||
jobs:
|
||||
merge:
|
||||
runs-on: self-hosted
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- name: merge_flake_lock_update
|
||||
run: >-
|
||||
python -m python.gitea_flake_lock merge
|
||||
--repo "${{ github.repository }}"
|
||||
env:
|
||||
JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
|
||||
GITEA_URL: https://gitea.tmmworkshop.com
|
||||
@@ -1,26 +0,0 @@
|
||||
name: test ebook search
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
env:
|
||||
UV_PYTHON_DOWNLOADS: never
|
||||
UV_CACHE_DIR: /var/cache/uv
|
||||
UV_LINK_MODE: copy
|
||||
|
||||
jobs:
|
||||
test-ebook-search:
|
||||
runs-on: self-hosted
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install dependencies
|
||||
run: nix develop .#devShells.x86_64-linux.ebook-search -c uv sync --locked --project python/ebook_search/docker
|
||||
- name: Run ebook search tests
|
||||
run: nix develop .#devShells.x86_64-linux.ebook-search -c uv run --project python/ebook_search/docker --no-sync pytest tests/ebook_search --override-ini addopts="-n auto -ra"
|
||||
@@ -2,24 +2,21 @@ name: update-flake-lock
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 0 * * 6"
|
||||
- cron: "0 0 * * *"
|
||||
|
||||
jobs:
|
||||
lockfile:
|
||||
runs-on: self-hosted
|
||||
permissions:
|
||||
actions: write
|
||||
contents: write
|
||||
pull-requests: write
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Install Nix
|
||||
uses: DeterminateSystems/nix-installer-action@main
|
||||
- name: Update flake.lock
|
||||
run: nix flake update
|
||||
- name: Create or update flake.lock PR
|
||||
env:
|
||||
JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
|
||||
GITEA_URL: https://gitea.tmmworkshop.com
|
||||
run: >-
|
||||
python -m python.gitea_flake_lock update
|
||||
--repo "${{ github.repository }}"
|
||||
uses: DeterminateSystems/update-flake-lock@main
|
||||
with:
|
||||
token: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
|
||||
pr-title: "Update flake.lock"
|
||||
pr-labels: |
|
||||
dependencies
|
||||
automated
|
||||
|
||||
-11
@@ -165,14 +165,3 @@ test.*
|
||||
|
||||
# syncthing
|
||||
.stfolder
|
||||
|
||||
# Frontend build output
|
||||
frontend/dist/
|
||||
frontend/node_modules/
|
||||
|
||||
# data from testing llms
|
||||
data/*
|
||||
.ebook_search_bm25
|
||||
|
||||
# gems data
|
||||
.gems
|
||||
|
||||
+2
-10
@@ -1,17 +1,10 @@
|
||||
# Generate AGE keys from SSH keys with:
|
||||
# ssh-keygen -A
|
||||
# nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
|
||||
# update keys after addin/removing a key
|
||||
# nix-shell -p sops --run "sops updatekeys users/secrets.yaml" users/secrets.yaml
|
||||
|
||||
keys:
|
||||
- &admin_richie age1u8zj599elqqvcmhxn8zuwrufsz8w8w366d3ayrljjejljt2q45kq8mxw9c # cspell:disable-line
|
||||
|
||||
- &system_bob age1q47vup0tjhulkg7d6xwmdsgrw64h4ax3la3evzqpxyy4adsmk9fs56qz3y # cspell:disable-line
|
||||
- &system_brain age1jhf7vm0005j60mjq63696frrmjhpy8kpc2d66mw044lqap5mjv4snmwvwm # cspell:disable-line
|
||||
- &system_jeeves age13lmqgc3jvkyah5e3vcwmj4s5wsc2akctcga0lpc0x8v8du3fxprqp4ldkv # cspell:disable-line
|
||||
- &system_router age1xzxryqq63x65yuza9lmmkud7crjjxpnkdew070yhx6xn7xe4tdws5twxsv # cspell:disable-line
|
||||
- &system_rhapsody age1ufnewppysaq2wwcl4ugngjz8pfzc5a35yg7luq0qmuqvctajcycs5lf6k4 # cspell:disable-line
|
||||
- &system_portal_1 age1vyav6kxtvt3z4vtnkkjj38eu8hlts5m7ygyckhskvalg2gpjk52su53d0a # cspell:disable-line
|
||||
|
||||
creation_rules:
|
||||
- path_regex: users/secrets\.yaml$
|
||||
@@ -19,7 +12,6 @@ creation_rules:
|
||||
- age:
|
||||
- *admin_richie
|
||||
- *system_bob
|
||||
- *system_brain
|
||||
- *system_jeeves
|
||||
- *system_router
|
||||
- *system_rhapsody
|
||||
- *system_portal_1
|
||||
|
||||
Vendored
+10
-47
@@ -2,7 +2,6 @@
|
||||
"cSpell.words": [
|
||||
"aboutwelcome",
|
||||
"acltype",
|
||||
"addopts",
|
||||
"addstr",
|
||||
"advplyr",
|
||||
"ahci",
|
||||
@@ -39,6 +38,7 @@
|
||||
"cgroupdriver",
|
||||
"charliermarsh",
|
||||
"Checkpointing",
|
||||
"cloudflared",
|
||||
"codellama",
|
||||
"codezombiech",
|
||||
"compactmode",
|
||||
@@ -70,17 +70,16 @@
|
||||
"ehci",
|
||||
"emerg",
|
||||
"endlessh",
|
||||
"ents",
|
||||
"errorlens",
|
||||
"esbenp",
|
||||
"esphome",
|
||||
"extest",
|
||||
"fadvise",
|
||||
"fastfetch",
|
||||
"fastforwardteam",
|
||||
"FASTFOX",
|
||||
"ffmpegthumbnailer",
|
||||
"filebot",
|
||||
"filebrowser",
|
||||
"fileroller",
|
||||
"findbar",
|
||||
"Fira",
|
||||
@@ -97,7 +96,6 @@
|
||||
"getch",
|
||||
"getmaxyx",
|
||||
"ghdeploy",
|
||||
"gitea",
|
||||
"globalprivacycontrol",
|
||||
"gparted",
|
||||
"gtts",
|
||||
@@ -116,9 +114,6 @@
|
||||
"httpchk",
|
||||
"hurlenko",
|
||||
"hwloc",
|
||||
"ical",
|
||||
"ignorelist",
|
||||
"improv",
|
||||
"INITDB",
|
||||
"iocharset",
|
||||
"ioit",
|
||||
@@ -128,8 +123,6 @@
|
||||
"jnoortheen",
|
||||
"jsbc",
|
||||
"kagi",
|
||||
"keyformat",
|
||||
"keylocation",
|
||||
"kuma",
|
||||
"lazer",
|
||||
"levelname",
|
||||
@@ -155,25 +148,20 @@
|
||||
"mixtral",
|
||||
"mklabel",
|
||||
"mkpart",
|
||||
"modbus",
|
||||
"modbuss",
|
||||
"modesetting",
|
||||
"mountpoint",
|
||||
"mountpoints",
|
||||
"mousewheel",
|
||||
"mqtt",
|
||||
"mtxr",
|
||||
"mypy",
|
||||
"ncdu",
|
||||
"nemo",
|
||||
"neofetch",
|
||||
"nerdfonts",
|
||||
"netdev",
|
||||
"netdevs",
|
||||
"Networkd",
|
||||
"networkmanager",
|
||||
"newtabpage",
|
||||
"ngram",
|
||||
"ngrams",
|
||||
"nixfmt",
|
||||
"nixos",
|
||||
"nixpkgs",
|
||||
@@ -196,7 +184,6 @@
|
||||
"overalljails",
|
||||
"overscroll",
|
||||
"overseerr",
|
||||
"paho",
|
||||
"partitionwise",
|
||||
"pbmode",
|
||||
"pciutils",
|
||||
@@ -205,7 +192,6 @@
|
||||
"peerconnection",
|
||||
"PESKYFOX",
|
||||
"PGID",
|
||||
"pgvector",
|
||||
"pipewire",
|
||||
"pkgs",
|
||||
"plugdev",
|
||||
@@ -225,16 +211,13 @@
|
||||
"pulseaudio",
|
||||
"punycode",
|
||||
"pychromecast",
|
||||
"pydocstyle",
|
||||
"pyfakefs",
|
||||
"pylance",
|
||||
"pylint",
|
||||
"pymetno",
|
||||
"pymodbus",
|
||||
"pyopenweathermap",
|
||||
"pyownet",
|
||||
"pytest",
|
||||
"qalculate",
|
||||
"qbit",
|
||||
"qbittorrent",
|
||||
"qbittorrentvpn",
|
||||
"qbitvpn",
|
||||
"quicksuggest",
|
||||
"radarr",
|
||||
"readahead",
|
||||
@@ -244,7 +227,6 @@
|
||||
"referer",
|
||||
"REFERERS",
|
||||
"relatime",
|
||||
"rerank",
|
||||
"Rhosts",
|
||||
"ripgrep",
|
||||
"roboto",
|
||||
@@ -257,10 +239,8 @@
|
||||
"schemeless",
|
||||
"scrollback",
|
||||
"SECUREFOX",
|
||||
"sessionmaker",
|
||||
"sessionstore",
|
||||
"shellcheck",
|
||||
"signalbot",
|
||||
"signon",
|
||||
"Signons",
|
||||
"skia",
|
||||
@@ -270,7 +250,6 @@
|
||||
"socialtracking",
|
||||
"sonarr",
|
||||
"sponsorblock",
|
||||
"sqlalchemy",
|
||||
"sqltools",
|
||||
"ssdp",
|
||||
"SSHOPTS",
|
||||
@@ -282,7 +261,6 @@
|
||||
"tabmanager",
|
||||
"tamasfe",
|
||||
"TCPIP",
|
||||
"testdisk",
|
||||
"tiktok",
|
||||
"timonwong",
|
||||
"titlebar",
|
||||
@@ -292,17 +270,11 @@
|
||||
"topstories",
|
||||
"treefmt",
|
||||
"twimg",
|
||||
"typedmonarchmoney",
|
||||
"typer",
|
||||
"uaccess",
|
||||
"ubiquiti",
|
||||
"ublock",
|
||||
"uiprotect",
|
||||
"uitour",
|
||||
"unifi",
|
||||
"unjudged",
|
||||
"unrar",
|
||||
"unstorable",
|
||||
"unsubmitted",
|
||||
"uptimekuma",
|
||||
"urlbar",
|
||||
@@ -312,10 +284,7 @@
|
||||
"useragent",
|
||||
"usernamehw",
|
||||
"userprefs",
|
||||
"vaninventory",
|
||||
"vdev",
|
||||
"vfat",
|
||||
"victron",
|
||||
"virt",
|
||||
"virtualisation",
|
||||
"vpnpromourl",
|
||||
@@ -327,22 +296,16 @@
|
||||
"wireshark",
|
||||
"Workqueues",
|
||||
"xattr",
|
||||
"xcursorgen",
|
||||
"xdist",
|
||||
"xhci",
|
||||
"yake",
|
||||
"yazi",
|
||||
"yubikey",
|
||||
"yubioath",
|
||||
"yzhang",
|
||||
"zeroconf",
|
||||
"zerotier",
|
||||
"zerotierone",
|
||||
"zoxide",
|
||||
"zram",
|
||||
"zstd"
|
||||
],
|
||||
"python-envs.defaultEnvManager": "ms-python.python:system",
|
||||
"python-envs.pythonProjects": [],
|
||||
"python.testing.pytestArgs": ["tests"],
|
||||
"python.testing.unittestEnabled": false,
|
||||
"python.testing.pytestEnabled": true
|
||||
]
|
||||
}
|
||||
|
||||
Generated
-1686
File diff suppressed because it is too large
Load Diff
@@ -1,4 +0,0 @@
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
|
||||
members = ["rust/*"]
|
||||
@@ -1,51 +1 @@
|
||||
# dotfiles
|
||||
|
||||
## Installer ISO
|
||||
|
||||
Build a bootable NixOS ISO with the installer preinstalled:
|
||||
|
||||
```sh
|
||||
nix build .#iso
|
||||
```
|
||||
|
||||
Write `result/iso/nixos-zfs-installer.iso` to a USB stick (for example with `dd`) or boot it in a VM. The image is the minimal NixOS installation CD with ZFS enabled and `nixos-installer` on `PATH`. SSH is enabled and the `nixos` and `root` accounts use the password `nixos`, so you can also run the installer remotely. Once booted:
|
||||
|
||||
```sh
|
||||
sudo nixos-installer
|
||||
```
|
||||
|
||||
The ISO bundles the `.#installer-nixos` package, a variant of the binary that keeps its Nix store linkage instead of being patched for foreign distributions.
|
||||
|
||||
## Installer binary
|
||||
|
||||
Build the self-contained installer executable with:
|
||||
|
||||
```sh
|
||||
nix build .#installer
|
||||
```
|
||||
|
||||
The flake package (defined in `python/installer/package.nix`) uses the Python builder in `python/installer/build.py`, which stages only the installer modules before running PyInstaller. You can also call it directly when `pyinstaller` and `patchelf` are on `PATH`:
|
||||
|
||||
```sh
|
||||
python -m python.installer.build --output ./nixos-installer
|
||||
```
|
||||
|
||||
Copy `result/bin/nixos-installer` to the installer USB stick and run it as root from the NixOS live environment:
|
||||
|
||||
```sh
|
||||
sudo ./nixos-installer
|
||||
```
|
||||
|
||||
Validate the live environment first with:
|
||||
|
||||
```sh
|
||||
./nixos-installer --check
|
||||
```
|
||||
|
||||
Paste a value into the TUI encryption password field to enable LUKS during install, or set `ENCRYPT_KEY`:
|
||||
|
||||
```sh
|
||||
sudo env ENCRYPT_KEY='change-me' ./nixos-installer
|
||||
```
|
||||
|
||||
The binary bundles the Python runtime and only the installer modules it imports. It still expects the NixOS installer environment to provide system install tools such as `parted`, `zfs`, `zpool`, `cryptsetup`, `nixos-generate-config`, and `nixos-install`.
|
||||
|
||||
@@ -17,11 +17,13 @@
|
||||
./nix.nix
|
||||
./programs.nix
|
||||
./ssh.nix
|
||||
./snapshot_manager.nix
|
||||
];
|
||||
|
||||
boot = {
|
||||
tmp.useTmpfs = lib.mkDefault true;
|
||||
tmp.useTmpfs = true;
|
||||
kernelPackages = lib.mkDefault pkgs.linuxPackages_6_12;
|
||||
zfs.package = lib.mkDefault pkgs.zfs_2_3;
|
||||
};
|
||||
|
||||
hardware.enableRedistributableFirmware = true;
|
||||
@@ -35,16 +37,19 @@
|
||||
|
||||
nixpkgs = {
|
||||
overlays = builtins.attrValues outputs.overlays;
|
||||
config = {
|
||||
allowUnfree = true;
|
||||
};
|
||||
config.allowUnfree = true;
|
||||
};
|
||||
|
||||
services = {
|
||||
dbus.implementation = "dbus";
|
||||
|
||||
# firmware update
|
||||
fwupd.enable = true;
|
||||
|
||||
snapshot_manager.enable = lib.mkDefault true;
|
||||
|
||||
zfs = {
|
||||
trim.enable = lib.mkDefault true;
|
||||
autoScrub.enable = lib.mkDefault true;
|
||||
};
|
||||
};
|
||||
|
||||
powerManagement.powertop.enable = lib.mkDefault true;
|
||||
|
||||
@@ -1,10 +1,4 @@
|
||||
{ lib, pkgs, ... }:
|
||||
let
|
||||
libPath = pkgs.lib.makeLibraryPath [
|
||||
pkgs.zlib
|
||||
pkgs.stdenv.cc.cc.lib
|
||||
];
|
||||
in
|
||||
{
|
||||
programs.nix-ld = {
|
||||
enable = lib.mkDefault true;
|
||||
@@ -21,7 +15,6 @@ in
|
||||
libxml2
|
||||
openssl
|
||||
stdenv.cc.cc
|
||||
stdenv.cc.cc.lib
|
||||
systemd
|
||||
util-linux
|
||||
xz
|
||||
@@ -30,9 +23,4 @@ in
|
||||
zstd
|
||||
];
|
||||
};
|
||||
|
||||
environment = {
|
||||
sessionVariables.LD_LIBRARY_PATH = lib.mkDefault libPath;
|
||||
variables.LD_LIBRARY_PATH = lib.mkDefault libPath;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -31,15 +31,8 @@ in
|
||||
"flakes"
|
||||
"ca-derivations"
|
||||
];
|
||||
system-features = lib.mkAfter [
|
||||
"gccarch-x86-64-v2"
|
||||
"gccarch-x86-64-v3"
|
||||
];
|
||||
warn-dirty = false;
|
||||
flake-registry = ""; # disable global flake registries
|
||||
connect-timeout = 10;
|
||||
download-buffer-size = 536870912;
|
||||
fallback = true;
|
||||
};
|
||||
|
||||
# Add each flake input as a registry and nix_path
|
||||
|
||||
@@ -2,5 +2,6 @@
|
||||
{
|
||||
environment.systemPackages = with pkgs; [
|
||||
git
|
||||
python313
|
||||
];
|
||||
}
|
||||
|
||||
@@ -22,12 +22,6 @@ hourly = 0
|
||||
daily = 0
|
||||
monthly = 0
|
||||
|
||||
["root_pool/nix_build"]
|
||||
15_min = 1
|
||||
hourly = 0
|
||||
daily = 0
|
||||
monthly = 0
|
||||
|
||||
["root_pool/var"]
|
||||
15_min = 8
|
||||
hourly = 24
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
inputs,
|
||||
pkgs,
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.services.snapshot_manager;
|
||||
in
|
||||
{
|
||||
options = {
|
||||
services.snapshot_manager = {
|
||||
enable = lib.mkOption {
|
||||
default = true;
|
||||
example = true;
|
||||
description = "Whether to enable k3s-net.";
|
||||
type = lib.types.bool;
|
||||
};
|
||||
path = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
description = "Path that needs to be updated via git pull";
|
||||
default = ./snapshot_config.toml;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
systemd = {
|
||||
services."snapshot_manager" = {
|
||||
description = "ZFS Snapshot Manager";
|
||||
requires = [ "zfs-import.target" ];
|
||||
after = [ "zfs-import.target" ];
|
||||
path = [ pkgs.zfs ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = "${inputs.system_tools.packages.x86_64-linux.default}/bin/snapshot_manager --config-file='${cfg.path}'";
|
||||
};
|
||||
};
|
||||
timers."snapshot_manager" = {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnBootSec = "15m";
|
||||
OnUnitActiveSec = "15m";
|
||||
Unit = "snapshot_manager.service";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -37,8 +37,6 @@
|
||||
TcpKeepAlive = "no";
|
||||
X11Forwarding = lib.mkDefault false;
|
||||
KexAlgorithms = [
|
||||
"sntrup761x25519-sha512@openssh.com"
|
||||
"mlkem768x25519-sha256"
|
||||
"curve25519-sha256@libssh.org"
|
||||
"diffie-hellman-group-exchange-sha256"
|
||||
];
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
nix.settings = {
|
||||
trusted-substituters = [ "http://192.168.95.35:5000" ];
|
||||
substituters = [ "http://192.168.95.35:5000/?priority=1&want-mass-query=true" ];
|
||||
};
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
boot = {
|
||||
kernelPackages = pkgs.linuxPackages_6_18;
|
||||
zfs.package = pkgs.zfs_2_4;
|
||||
kernelPackages = pkgs.linuxPackages_6_15;
|
||||
zfs.package = pkgs.zfs_2_3;
|
||||
};
|
||||
|
||||
hardware.bluetooth = {
|
||||
|
||||
@@ -17,7 +17,6 @@
|
||||
logDriver = "local";
|
||||
storageDriver = "overlay2";
|
||||
daemon.settings = {
|
||||
live-restore = false;
|
||||
experimental = true;
|
||||
exec-opts = [ "native.cgroupdriver=systemd" ];
|
||||
log-opts = {
|
||||
|
||||
@@ -1,256 +0,0 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
monitoringInterface = "tailscale0";
|
||||
nodeTextfileDir = "/var/lib/prometheus-node-exporter-textfile";
|
||||
|
||||
mkProcessNameTemplate =
|
||||
perPid: template: if perPid then "${template}:{{.PID}}:{{.StartTime}}" else template;
|
||||
|
||||
mkProcessMatchers = perPid: [
|
||||
{
|
||||
name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Module}}";
|
||||
cmdline = [ "^/nix/store[^ ]*/bin/python[^ ]* -m (?P<Module>[^ ]+)" ];
|
||||
}
|
||||
{
|
||||
name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Wrapped}}";
|
||||
cmdline = [
|
||||
"^/nix/store[^ ]*/bin/python[^ ]* /nix/store[^ ]*/bin/\\.?(?P<Wrapped>[^ /]+?)(?:-wrapped)?(?:\\s|$)"
|
||||
];
|
||||
}
|
||||
{
|
||||
name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Wrapped}}";
|
||||
cmdline = [
|
||||
"^/nix/store[^ ]*/bin/node /nix/store[^ ]*-(?P<Wrapped>[A-Za-z0-9._+-]+)-[0-9][^ /]*/"
|
||||
];
|
||||
}
|
||||
{
|
||||
name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Wrapped}}";
|
||||
cmdline = [ "^/nix/store[^ ]*/(?:bin/|lib/[^ ]*/)?\\.?(?P<Wrapped>[^ /]+?)(?:-wrapped)?(?:\\s|$)" ];
|
||||
}
|
||||
{
|
||||
name = mkProcessNameTemplate perPid "{{.Username}}:{{.ExeBase}}";
|
||||
cmdline = [ ".+" ];
|
||||
}
|
||||
];
|
||||
|
||||
perPidConfig = pkgs.writeText "process-exporter-per-pid.yaml" (
|
||||
builtins.toJSON {
|
||||
process_names = mkProcessMatchers true;
|
||||
}
|
||||
);
|
||||
|
||||
zpoolLatencyScript = pkgs.writeShellScript "zpool-latency-exporter" ''
|
||||
set -euo pipefail
|
||||
|
||||
out_dir=${lib.escapeShellArg nodeTextfileDir}
|
||||
host=${lib.escapeShellArg config.networking.hostName}
|
||||
tmp_file="$(mktemp "$out_dir/zpool.prom.XXXXXX")"
|
||||
trap 'rm -f "$tmp_file"' EXIT
|
||||
|
||||
pools="$(zpool list -H -o name | paste -sd, -)"
|
||||
|
||||
cat >"$tmp_file" <<'EOF'
|
||||
# HELP zpool_iostat_total_wait_read_ns Average total read wait time reported by zpool iostat.
|
||||
# TYPE zpool_iostat_total_wait_read_ns gauge
|
||||
# HELP zpool_iostat_total_wait_write_ns Average total write wait time reported by zpool iostat.
|
||||
# TYPE zpool_iostat_total_wait_write_ns gauge
|
||||
# HELP zpool_iostat_disk_wait_read_ns Average disk read wait time reported by zpool iostat.
|
||||
# TYPE zpool_iostat_disk_wait_read_ns gauge
|
||||
# HELP zpool_iostat_disk_wait_write_ns Average disk write wait time reported by zpool iostat.
|
||||
# TYPE zpool_iostat_disk_wait_write_ns gauge
|
||||
# HELP zpool_iostat_syncq_wait_read_ns Average synchronous queue read wait time reported by zpool iostat.
|
||||
# TYPE zpool_iostat_syncq_wait_read_ns gauge
|
||||
# HELP zpool_iostat_syncq_wait_write_ns Average synchronous queue write wait time reported by zpool iostat.
|
||||
# TYPE zpool_iostat_syncq_wait_write_ns gauge
|
||||
# HELP zpool_iostat_asyncq_wait_read_ns Average asynchronous queue read wait time reported by zpool iostat.
|
||||
# TYPE zpool_iostat_asyncq_wait_read_ns gauge
|
||||
# HELP zpool_iostat_asyncq_wait_write_ns Average asynchronous queue write wait time reported by zpool iostat.
|
||||
# TYPE zpool_iostat_asyncq_wait_write_ns gauge
|
||||
EOF
|
||||
|
||||
zpool iostat -Hplvy -y 1 1 | awk -F '\t' -v host="$host" -v pools="$pools" '
|
||||
function esc(str, out) {
|
||||
out = str
|
||||
gsub(/\\/, "\\\\", out)
|
||||
gsub(/"/, "\\\"", out)
|
||||
return out
|
||||
}
|
||||
|
||||
function emit(metric, pool, vdev, value) {
|
||||
if (value == "" || value == "-") {
|
||||
return
|
||||
}
|
||||
|
||||
printf "%s{host=\"%s\",pool=\"%s\",vdev=\"%s\"} %s\n",
|
||||
metric,
|
||||
esc(host),
|
||||
esc(pool),
|
||||
esc(vdev),
|
||||
value
|
||||
}
|
||||
|
||||
BEGIN {
|
||||
split(pools, pool_names, ",")
|
||||
for (idx in pool_names) {
|
||||
if (pool_names[idx] != "") {
|
||||
known_pools[pool_names[idx]] = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
NF == 0 {
|
||||
next
|
||||
}
|
||||
|
||||
{
|
||||
row_name = $1
|
||||
|
||||
if (row_name in known_pools) {
|
||||
current_pool = row_name
|
||||
current_vdev = "_pool"
|
||||
} else if (current_pool == "") {
|
||||
next
|
||||
} else {
|
||||
current_vdev = row_name
|
||||
}
|
||||
|
||||
emit("zpool_iostat_total_wait_read_ns", current_pool, current_vdev, $8)
|
||||
emit("zpool_iostat_total_wait_write_ns", current_pool, current_vdev, $9)
|
||||
emit("zpool_iostat_disk_wait_read_ns", current_pool, current_vdev, $10)
|
||||
emit("zpool_iostat_disk_wait_write_ns", current_pool, current_vdev, $11)
|
||||
emit("zpool_iostat_syncq_wait_read_ns", current_pool, current_vdev, $12)
|
||||
emit("zpool_iostat_syncq_wait_write_ns", current_pool, current_vdev, $13)
|
||||
emit("zpool_iostat_asyncq_wait_read_ns", current_pool, current_vdev, $14)
|
||||
emit("zpool_iostat_asyncq_wait_write_ns", current_pool, current_vdev, $15)
|
||||
}
|
||||
' >>"$tmp_file"
|
||||
|
||||
mv "$tmp_file" "$out_dir/zpool.prom"
|
||||
trap - EXIT
|
||||
'';
|
||||
in
|
||||
{
|
||||
networking.firewall.interfaces.${monitoringInterface}.allowedTCPPorts = [
|
||||
9100
|
||||
9134
|
||||
9256
|
||||
9257
|
||||
9633
|
||||
];
|
||||
|
||||
services.prometheus.exporters = {
|
||||
node = {
|
||||
enable = true;
|
||||
enabledCollectors = [
|
||||
"pressure"
|
||||
"processes"
|
||||
"systemd"
|
||||
];
|
||||
extraFlags = [ "--collector.textfile.directory=${nodeTextfileDir}" ];
|
||||
};
|
||||
|
||||
process = {
|
||||
enable = true;
|
||||
user = "root";
|
||||
group = "root";
|
||||
settings.process_names = mkProcessMatchers false;
|
||||
extraFlags = [
|
||||
"-gather-smaps=false"
|
||||
"-remove-empty-groups=true"
|
||||
"-threads=false"
|
||||
];
|
||||
};
|
||||
|
||||
smartctl.enable = true;
|
||||
zfs.enable = true;
|
||||
};
|
||||
|
||||
programs.atop = {
|
||||
enable = true;
|
||||
atopService.enable = true;
|
||||
atopRotateTimer.enable = true;
|
||||
atopacctService.enable = true;
|
||||
settings.interval = 30;
|
||||
};
|
||||
|
||||
systemd = {
|
||||
services = {
|
||||
prometheus-process-pid-exporter = {
|
||||
description = "Prometheus process exporter with per-PID naming";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network.target" ];
|
||||
serviceConfig = {
|
||||
ExecStart = ''
|
||||
${pkgs.prometheus-process-exporter}/bin/process-exporter \
|
||||
--web.listen-address 0.0.0.0:9257 \
|
||||
--config.path ${perPidConfig} \
|
||||
-children=false \
|
||||
-gather-smaps=false \
|
||||
-remove-empty-groups=true \
|
||||
-threads=false
|
||||
'';
|
||||
User = "root";
|
||||
Group = "root";
|
||||
Restart = "always";
|
||||
WorkingDirectory = "/tmp";
|
||||
CapabilityBoundingSet = [ "" ];
|
||||
DeviceAllow = [ "" ];
|
||||
LockPersonality = true;
|
||||
MemoryDenyWriteExecute = true;
|
||||
NoNewPrivileges = true;
|
||||
PrivateDevices = true;
|
||||
PrivateTmp = true;
|
||||
ProtectClock = true;
|
||||
ProtectControlGroups = true;
|
||||
ProtectHome = true;
|
||||
ProtectHostname = true;
|
||||
ProtectKernelLogs = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectSystem = "strict";
|
||||
RemoveIPC = true;
|
||||
RestrictAddressFamilies = [
|
||||
"AF_INET"
|
||||
"AF_INET6"
|
||||
];
|
||||
RestrictNamespaces = true;
|
||||
RestrictRealtime = true;
|
||||
RestrictSUIDSGID = true;
|
||||
SystemCallArchitectures = "native";
|
||||
UMask = "0077";
|
||||
};
|
||||
};
|
||||
|
||||
zpool-latency-exporter = {
|
||||
description = "Exports ZFS latency metrics for node_exporter textfile collection";
|
||||
after = [ "zfs-import.target" ];
|
||||
requires = [ "zfs-import.target" ];
|
||||
path = [
|
||||
config.boot.zfs.package
|
||||
pkgs.coreutils
|
||||
pkgs.gawk
|
||||
];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = zpoolLatencyScript;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
timers.zpool-latency-exporter = {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnBootSec = "2m";
|
||||
OnUnitActiveSec = "60s";
|
||||
Unit = "zpool-latency-exporter.service";
|
||||
};
|
||||
};
|
||||
|
||||
tmpfiles.rules = [ "d ${nodeTextfileDir} 0755 root root - -" ];
|
||||
};
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
environment.systemPackages = [
|
||||
pkgs.signal-cli
|
||||
];
|
||||
|
||||
users = {
|
||||
groups.signal-cli = { };
|
||||
users.signal-cli = {
|
||||
isSystemUser = true;
|
||||
group = "signal-cli";
|
||||
home = "/var/lib/signal-cli";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /var/lib/signal-cli 0700 signal-cli signal-cli - -"
|
||||
"Z /var/lib/signal-cli - signal-cli signal-cli - -"
|
||||
];
|
||||
|
||||
systemd.services.signal-cli = {
|
||||
description = "Signal CLI JSON-RPC service";
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
unitConfig.RequiresMountsFor = [ "/var/lib/signal-cli" ];
|
||||
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
User = "signal-cli";
|
||||
Group = "signal-cli";
|
||||
ExecStart = "${pkgs.signal-cli}/bin/signal-cli --data-dir /var/lib/signal-cli daemon --socket /run/signal-cli/socket";
|
||||
Restart = "on-failure";
|
||||
RestartSec = "5s";
|
||||
SuccessExitStatus = 143;
|
||||
RuntimeDirectory = "signal-cli";
|
||||
RuntimeDirectoryMode = "0750";
|
||||
UMask = "0007";
|
||||
NoNewPrivileges = true;
|
||||
PrivateTmp = true;
|
||||
ProtectHome = true;
|
||||
ProtectSystem = "strict";
|
||||
ReadWritePaths = [ "/var/lib/signal-cli" ];
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -10,9 +10,6 @@
|
||||
authorizedKeys = config.users.users.richie.openssh.authorizedKeys.keys;
|
||||
};
|
||||
};
|
||||
availableKernelModules = [
|
||||
"igb"
|
||||
"r8152"
|
||||
];
|
||||
availableKernelModules = [ "igb" ];
|
||||
};
|
||||
}
|
||||
|
||||
@@ -8,11 +8,10 @@
|
||||
dataDir = "/home/richie/Syncthing";
|
||||
configDir = "/home/richie/.config/syncthing";
|
||||
settings.devices = {
|
||||
bob.id = "CJIAPEJ-VO74RR4-F75VU6M-QNZAMYG-FYUJG7Y-6AT62HJ-355PRPL-PJFETAZ"; # cspell:disable-line
|
||||
brain.id = "SSCGIPI-IV3VYKB-TRNIJE3-COV4T2H-CDBER7F-I2CGHYA-NWOEUDU-3T5QAAN"; # cspell:disable-line
|
||||
ipad.id = "KI76T3X-SFUGV2L-VSNYTKR-TSIUV5L-SHWD3HE-GQRGRCN-GY4UFMD-CW6Z6AX"; # cspell:disable-line
|
||||
phone.id = "TBRULKD-7DZPGGZ-F6LLB7J-MSO54AY-7KLPBIN-QOFK6PX-W2HBEWI-PHM2CQI"; # cspell:disable-line
|
||||
jeeves.id = "ICRHXZW-ECYJCUZ-I4CZ64R-3XRK7CG-LL2HAAK-FGOHD22-BQA4AI6-5OAL6AG"; # cspell:disable-line
|
||||
phone.id = "JPVQKQW-CFXOJXT-Q5G5F3H-QIDHDRE-GKHPTQB-GXZUQSP-U7FR7F7-INP3AAH"; # cspell:disable-line
|
||||
ipad.id = "KI76T3X-SFUGV2L-VSNYTKR-TSIUV5L-SHWD3HE-GQRGRCN-GY4UFMD-CW6Z6AX"; # cspell:disable-line
|
||||
bob.id = "CJIAPEJ-VO74RR4-F75VU6M-QNZAMYG-FYUJG7Y-6AT62HJ-355PRPL-PJFETAZ"; # cspell:disable-line
|
||||
rhapsody-in-green.id = "ASL3KC4-3XEN6PA-7BQBRKE-A7JXLI6-DJT43BY-Q4WPOER-7UALUAZ-VTPQ6Q4"; # cspell:disable-line
|
||||
};
|
||||
};
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
{
|
||||
config,
|
||||
inputs,
|
||||
...
|
||||
}:
|
||||
{
|
||||
nix.settings = {
|
||||
trusted-substituters = [ "http://jeeves:5000" ];
|
||||
substituters = [ "http://jeeves:5000/?priority=1&want-mass-query=true" ];
|
||||
trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
|
||||
};
|
||||
|
||||
services.tailscale = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
authKeyFile = config.sops.secrets.tailscale_auth_key.path;
|
||||
|
||||
# OAuth client secrets create ephemeral nodes by default. NixOS machines
|
||||
# are persistent and should enroll without interactive device approval.
|
||||
authKeyParameters = {
|
||||
ephemeral = false;
|
||||
preauthorized = true;
|
||||
};
|
||||
|
||||
extraUpFlags = [ "--advertise-tags=tag:nixos" ];
|
||||
};
|
||||
|
||||
sops = {
|
||||
age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
|
||||
|
||||
secrets.tailscale_auth_key = {
|
||||
sopsFile = "${inputs.self}/users/secrets.yaml";
|
||||
owner = "root";
|
||||
mode = "0400";
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -4,8 +4,6 @@
|
||||
flags = [ "--accept-flake-config" ];
|
||||
randomizedDelaySec = "1h";
|
||||
persistent = true;
|
||||
flake = "git+https://gitea.tmmworkshop.com/richie/dotfiles?ref=main";
|
||||
allowReboot = true;
|
||||
dates = "Sat *-*-* 06:00:00";
|
||||
flake = "github:RichieCahill/dotfiles";
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
nixpkgs = {
|
||||
hostPlatform = {
|
||||
system = "x86_64-linux";
|
||||
gcc = {
|
||||
arch = "x86-64-v3";
|
||||
tune = "generic";
|
||||
};
|
||||
};
|
||||
|
||||
# These patches repair tests and bundled dependencies that are sensitive
|
||||
# to the compiler flags used by the x86-64-v3 package set.
|
||||
overlays = [ (import ./patches) ];
|
||||
};
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
Subject: [PATCH] abseil: include BMI2 intrinsics through the public header
|
||||
|
||||
GCC and Clang reject direct inclusion of bmi2intrin.h. Include immintrin.h
|
||||
instead so that the compiler supplies the required intrinsic setup when
|
||||
BMI2 is enabled, including builds targeting x86-64-v3.
|
||||
|
||||
This patch is shared by the vendored Abseil copies in Electron, rusty_v8
|
||||
(Deno), and Signal's WebRTC build.
|
||||
|
||||
--- a/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
|
||||
+++ b/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
|
||||
@@ -226,7 +226,7 @@
|
||||
#endif
|
||||
|
||||
#ifdef __BMI2__
|
||||
-#include <bmi2intrin.h>
|
||||
+#include <immintrin.h>
|
||||
#endif // __BMI2__
|
||||
|
||||
namespace absl {
|
||||
@@ -1,38 +0,0 @@
|
||||
# Abseil accepted the upstream fix: https://github.com/abseil/abseil-cpp/pull/2071
|
||||
# Keep this workaround until Electron, Deno's rusty_v8, and Signal's WebRTC
|
||||
# update their bundled Abseil copies to include it.
|
||||
{ prev }:
|
||||
let
|
||||
patchAbseilBmi2Include =
|
||||
package:
|
||||
package.overrideAttrs (old: {
|
||||
# GCC and Clang require the public umbrella header for BMI2 intrinsics.
|
||||
patches = (old.patches or [ ]) ++ [ ./bmi2-public-header.patch ];
|
||||
});
|
||||
|
||||
electron43Unwrapped = patchAbseilBmi2Include prev.electron_43.unwrapped;
|
||||
electron43 = prev.electron_43.override {
|
||||
electron-unwrapped = electron43Unwrapped;
|
||||
};
|
||||
|
||||
signalCallPackage =
|
||||
path: args:
|
||||
let
|
||||
package = prev.callPackage path args;
|
||||
in
|
||||
if builtins.baseNameOf path == "webrtc.nix" then patchAbseilBmi2Include package else package;
|
||||
in
|
||||
prev.lib.optionalAttrs ((prev.stdenv.hostPlatform.gcc.arch or null) == "x86-64-v3") {
|
||||
deno =
|
||||
let
|
||||
librusty_v8 = patchAbseilBmi2Include prev.deno.passthru.librusty_v8;
|
||||
in
|
||||
prev.deno.override { inherit librusty_v8; };
|
||||
|
||||
electron_43 = electron43;
|
||||
|
||||
signal-desktop = prev.signal-desktop.override {
|
||||
electron_43 = electron43;
|
||||
callPackage = signalCallPackage;
|
||||
};
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
Subject: [PATCH] Compare font content in the overlap-removal regression
|
||||
|
||||
PFA_SKIP ends with a separator, producing an empty prefix that matches every
|
||||
line. Remove the empty prefix for this test so glyph differences are checked,
|
||||
and assert that tx exits successfully before inspecting the output.
|
||||
|
||||
--- a/tests/tx_test.py
|
||||
+++ b/tests/tx_test.py
|
||||
@@ -1064,6 +1064,8 @@
|
||||
output_path = get_temp_file_path()
|
||||
args = [TOOL, '-t1', '+V', '-o', output_path, input_path]
|
||||
- subprocess.call(args)
|
||||
- assert differ([expected_path, output_path, '-s', PFA_SKIP[0]])
|
||||
+ subprocess.check_call(args)
|
||||
+ # An empty skip prefix matches every line and hides font differences.
|
||||
+ skip_headers = SPLIT_MARKER.join(filter(None, PFA_SKIP[0].split(SPLIT_MARKER)))
|
||||
+ assert differ([expected_path, output_path, '-s', skip_headers])
|
||||
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
{ afdko }:
|
||||
afdko.overridePythonAttrs (old: {
|
||||
# FMA changes overlap-removal coordinates by 0.01 units on x86-64-v3.
|
||||
# Separate multiply/add rounding reproduces the reference Type 1 font exactly.
|
||||
env = (old.env or { }) // {
|
||||
NIX_CFLAGS_COMPILE = (old.env.NIX_CFLAGS_COMPILE or "") + " -ffp-contract=off";
|
||||
};
|
||||
|
||||
patches = (old.patches or [ ]) ++ [ ./check-overlap-font-content.patch ];
|
||||
})
|
||||
@@ -1,19 +0,0 @@
|
||||
{ prev }:
|
||||
let
|
||||
buildRedistHook = prev.runCommand "buildRedistHook.bash" { } ''
|
||||
cp ${prev.path}/pkgs/development/cuda-modules/buildRedist/buildRedistHook.bash "$out"
|
||||
chmod u+w "$out"
|
||||
patch "$out" < ${./propagated-build-output-arrays.patch}
|
||||
'';
|
||||
in
|
||||
prev.cudaPackages_12_9.overrideScope (
|
||||
_cudaFinal: cudaPrev: {
|
||||
buildRedist =
|
||||
args:
|
||||
(cudaPrev.buildRedist args).overrideAttrs (old: {
|
||||
nativeBuildInputs = map (
|
||||
input: if builtins.baseNameOf input == "buildRedistHook.bash" then buildRedistHook else input
|
||||
) old.nativeBuildInputs;
|
||||
});
|
||||
}
|
||||
)
|
||||
@@ -1,44 +0,0 @@
|
||||
Subject: [PATCH] Preserve structured CUDA output arrays
|
||||
|
||||
The multiple-outputs hook now accepts arrays. The old compatibility export
|
||||
collapses include and lib into one array element, breaking indirect expansion.
|
||||
Keep the array and iterate over each output when propagating to out.
|
||||
|
||||
--- a/buildRedistHook.bash
|
||||
+++ b/buildRedistHook.bash
|
||||
@@ -23,9 +23,6 @@
|
||||
|
||||
postInstallCheckHooks+=(checkCudaNonEmptyOutputs)
|
||||
nixLog "added checkCudaNonEmptyOutputs to postInstallCheckHooks"
|
||||
-
|
||||
- preFixupHooks+=(fixupPropagatedBuildOutputsForMultipleOutputs)
|
||||
- nixLog "added fixupPropagatedBuildOutputsForMultipleOutputs to preFixupHooks"
|
||||
|
||||
postFixupHooks+=(fixupCudaPropagatedBuildOutputsToOut)
|
||||
nixLog "added fixupCudaPropagatedBuildOutputsToOut to postFixupHooks"
|
||||
@@ -193,16 +190,6 @@
|
||||
return 0
|
||||
}
|
||||
|
||||
-# TODO(@connorbaker): https://github.com/NixOS/nixpkgs/issues/323126.
|
||||
-# _multioutPropagateDev() currently expects a space-separated string rather than an array.
|
||||
-# NOTE: Because _multioutPropagateDev is a postFixup hook, we correct it in preFixup.
|
||||
-fixupPropagatedBuildOutputsForMultipleOutputs() {
|
||||
- nixLog "converting propagatedBuildOutputs to a space-separated string"
|
||||
- # shellcheck disable=SC2124
|
||||
- export propagatedBuildOutputs="${propagatedBuildOutputs[@]}"
|
||||
- return 0
|
||||
-}
|
||||
-
|
||||
# The multiple outputs setup hook only propagates build outputs to dev.
|
||||
# We want to propagate them to out as well, in case the user interpolates
|
||||
# the package into a string -- in such a case, the dev output is not selected
|
||||
@@ -217,7 +204,7 @@
|
||||
mkdir -p "${out:?}/nix-support"
|
||||
|
||||
# NOTE: We must use printWords to ensure the output is a single line.
|
||||
- for output in $propagatedBuildOutputs; do
|
||||
+ for output in "${propagatedBuildOutputs[@]}"; do
|
||||
# Propagate the other components to the out output
|
||||
nixLog "adding ${!output:?} to propagatedBuildInputs of ${out:?}"
|
||||
printWords "${!output:?}" >>"${out:?}/nix-support/propagated-build-inputs"
|
||||
@@ -1,18 +0,0 @@
|
||||
_final: prev:
|
||||
(import ./abseil { inherit prev; })
|
||||
// {
|
||||
cudaPackages_12_9 = import ./cuda { inherit prev; };
|
||||
trunk = import ./trunk { inherit (prev) trunk jq; };
|
||||
gnutls = import ./gnutls { inherit (prev) gnutls; };
|
||||
prometheus = import ./prometheus { inherit (prev) prometheus; };
|
||||
zopfli = import ./zopfli { inherit (prev) zopfli; };
|
||||
|
||||
pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
|
||||
(_pythonFinal: pythonPrev: {
|
||||
afdko = import ./afdko { inherit (pythonPrev) afdko; };
|
||||
psutil = import ./psutil { inherit (pythonPrev) psutil; };
|
||||
pytest-xdist = import ./pytest-xdist { inherit (pythonPrev) pytest-xdist; };
|
||||
scipy = import ./scipy { inherit (pythonPrev) scipy; };
|
||||
})
|
||||
];
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
{ gnutls }:
|
||||
gnutls.overrideAttrs (old: {
|
||||
# Keep the UDP handshake test enabled on loaded builders by waiting for
|
||||
# the server to bind its socket. Kept as a standalone patch for upstream.
|
||||
patches = (old.patches or [ ]) ++ [ ./udp-server-readiness.patch ];
|
||||
})
|
||||
@@ -1,70 +0,0 @@
|
||||
Subject: [PATCH] tests: wait for the UDP server socket before connecting
|
||||
|
||||
A fixed four-second sleep does not guarantee that gnutls-serv has bound
|
||||
its UDP socket on a busy builder. Poll the local IPv4 UDP endpoint using
|
||||
the existing ss/netstat discovery, with the same retry budget as the TCP
|
||||
helper. Fail early if the server exits, and retain the original handshake
|
||||
checks in serv-udp.sh.
|
||||
|
||||
Use flags common to ss and BSD/Linux netstat. Match the local endpoint
|
||||
and complete port number, excluding TCP, IPv6-only and peer endpoints.
|
||||
|
||||
--- a/tests/scripts/common.sh
|
||||
+++ b/tests/scripts/common.sh
|
||||
@@ -185,10 +185,55 @@
|
||||
fi
|
||||
}
|
||||
|
||||
+check_if_udp_port_bound() {
|
||||
+ local PORT=$1
|
||||
+ have_port_finder
|
||||
+ # Use only -an, which is shared by ss and BSD/Linux netstat. UDP has
|
||||
+ # no LISTEN state. Match the local IPv4 endpoint, not a peer port or
|
||||
+ # a longer port number. serv-udp.sh connects to 127.0.0.1;
|
||||
+ # listen_socket() in serv.c binds IPv4 separately and requests
|
||||
+ # IPV6_V6ONLY=1 for its IPv6 socket.
|
||||
+ $PFCMD -an | awk -v port="$PORT" '
|
||||
+ $1 == "udp" || $1 == "udp4" {
|
||||
+ # ss includes a state column; netstat does not.
|
||||
+ address = ($2 == "UNCONN" || $2 == "ESTAB") ? $5 : $4
|
||||
+ if (address ~ ("^[0-9.]+[.:]" port "$") ||
|
||||
+ address == "*." port)
|
||||
+ found = 1
|
||||
+ }
|
||||
+ END { exit !found }
|
||||
+ '
|
||||
+}
|
||||
+
|
||||
wait_udp_server() {
|
||||
local PID=$1
|
||||
+ local ret
|
||||
trap "test -n \"${PID}\" && kill ${PID};exit 1" 1 15 2
|
||||
- sleep 4
|
||||
+ local i=0
|
||||
+ # Use the same retry budget as wait_for_port(), but also stop if the
|
||||
+ # server exits before binding its socket.
|
||||
+ while test $i -lt 90; do
|
||||
+ if ! kill -0 "$PID" 2>/dev/null; then
|
||||
+ fail "" "UDP server $PID exited before binding port $PORT"
|
||||
+ fi
|
||||
+ # Contain have_port_finder's exit so a skip also stops the server.
|
||||
+ if (check_if_udp_port_bound "$PORT"); then
|
||||
+ return 0
|
||||
+ else
|
||||
+ ret=$?
|
||||
+ if test "$ret" = 77; then
|
||||
+ kill "$PID" 2>/dev/null || :
|
||||
+ wait "$PID" 2>/dev/null || :
|
||||
+ exit 77
|
||||
+ fi
|
||||
+ fi
|
||||
+ i=$((i + 1))
|
||||
+ if test $i -lt 90; then
|
||||
+ echo "try $i: waiting for UDP port $PORT"
|
||||
+ sleep 2
|
||||
+ fi
|
||||
+ done
|
||||
+ fail "$PID" "UDP server $PORT did not come up"
|
||||
}
|
||||
|
||||
create_testdir() {
|
||||
@@ -1,180 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise patched common.sh without building GnuTLS (Python standard library only).
|
||||
|
||||
Usage: python3 verify-readiness.py /path/to/patched/tests/scripts/common.sh
|
||||
Set SHELL to test another shell, and NETSTAT to test a netstat outside PATH.
|
||||
"""
|
||||
|
||||
# Use unittest so this upstream companion tool needs no pytest installation.
|
||||
# ruff: noqa: PT009
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
COMMON = str(Path(sys.argv.pop(1)).resolve())
|
||||
SHELL = os.environ.get("SHELL", "/bin/sh")
|
||||
|
||||
|
||||
class ReadinessTests(unittest.TestCase):
|
||||
"""Check endpoint parsing and the server startup lifecycle."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
"""Create a socket-listing fixture for each check."""
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.root = Path(self.tmp.name)
|
||||
self.fixture = self.root / "sockets"
|
||||
self.fixture.write_text("")
|
||||
self.finder = self.root / "port-finder"
|
||||
self.finder.write_text('#!/bin/sh\ncat "$SOCKET_FIXTURE"\n')
|
||||
self.finder.chmod(0o755)
|
||||
|
||||
def run_shell(self, body: str, **env: str) -> subprocess.CompletedProcess[str]:
|
||||
"""Source the actual helper and run a shell scenario."""
|
||||
return subprocess.run(
|
||||
[SHELL, "-c", '. "$COMMON"\n' + body],
|
||||
env={
|
||||
**os.environ,
|
||||
"COMMON": COMMON,
|
||||
"SOCKET_FIXTURE": str(self.fixture),
|
||||
"PFCMD": str(self.finder),
|
||||
"PORT": "12345",
|
||||
**env,
|
||||
},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=20,
|
||||
check=False,
|
||||
)
|
||||
|
||||
def test_socket_formats_and_false_matches(self) -> None:
|
||||
"""Accept IPv4 UDP local endpoints and reject unrelated sockets."""
|
||||
cases = [
|
||||
("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*", True),
|
||||
("udp UNCONN 0 0 127.0.0.1:12345 0.0.0.0:*", True),
|
||||
("udp 0 0 0.0.0.0:12345 0.0.0.0:*", True),
|
||||
("udp4 0 0 *.12345 *.*", True),
|
||||
("udp 0 0 127.0.0.1.12345 *.*", True),
|
||||
("udp 0 0 *.12345 *.*", True),
|
||||
("udp UNCONN 0 0 0.0.0.0:123456 0.0.0.0:*", False),
|
||||
("udp 0 0 0.0.0.0:123456 0.0.0.0:*", False),
|
||||
("udp ESTAB 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
|
||||
("udp 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
|
||||
("tcp LISTEN 0 128 0.0.0.0:12345 0.0.0.0:*", False),
|
||||
("tcp 0 0 0.0.0.0:12345 0.0.0.0:* LISTEN", False),
|
||||
("udp UNCONN 0 0 [::]:12345 [::]:*", False),
|
||||
("udp UNCONN 0 0 *:12345 *:*", False),
|
||||
("udp6 0 0 :::12345 :::*", False),
|
||||
("udp6 0 0 *.12345 *.*", False),
|
||||
("", False),
|
||||
]
|
||||
for row, ready in cases:
|
||||
with self.subTest(row=row):
|
||||
self.fixture.write_text(row + "\n")
|
||||
result = self.run_shell('check_if_udp_port_bound "$PORT"')
|
||||
self.assertEqual(result.returncode, 0 if ready else 1, result.stderr)
|
||||
|
||||
def test_exited_server_fails_immediately(self) -> None:
|
||||
"""Fail without sleeping when the server has already exited."""
|
||||
result = self.run_shell(
|
||||
'true &\npid=$!\nwait "$pid"\nsleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$pid"'
|
||||
)
|
||||
self.assertEqual(result.returncode, 1)
|
||||
self.assertIn("exited before binding", result.stderr)
|
||||
self.assertNotIn("unexpected sleep", result.stderr)
|
||||
|
||||
def test_ready_socket_does_not_sleep(self) -> None:
|
||||
"""Check readiness before the first sleep."""
|
||||
self.fixture.write_text("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*\n")
|
||||
result = self.run_shell('sleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$$"')
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertNotIn("unexpected sleep", result.stderr)
|
||||
|
||||
def test_missing_port_finders_skip(self) -> None:
|
||||
"""Skip and stop the live server when no finder is available."""
|
||||
# have_port_finder also tries these paths independently of PATH.
|
||||
if any(os.access(f"{directory}/ss", os.X_OK) for directory in ("/sbin", "/usr/sbin", "/usr/local/sbin")):
|
||||
self.skipTest("an absolute ss path cannot be hidden by this PATH-only fixture")
|
||||
with subprocess.Popen(["sleep", "60"]) as server:
|
||||
try:
|
||||
result = self.run_shell(
|
||||
'unset PFCMD\nPATH=/nonexistent\nwait_udp_server "$SERVER_PID"',
|
||||
SERVER_PID=str(server.pid),
|
||||
)
|
||||
self.assertEqual(result.returncode, 77)
|
||||
self.assertIn("neither ss nor netstat found", result.stderr)
|
||||
server.wait(timeout=3)
|
||||
self.assertLess(server.returncode, 0)
|
||||
finally:
|
||||
if server.poll() is None:
|
||||
server.kill()
|
||||
|
||||
def test_timeout_is_bounded_and_cleans_up(self) -> None:
|
||||
"""Stop polling after the retry budget and terminate the server."""
|
||||
# Only accelerate the polling delay; keep a real live server process.
|
||||
self.finder.write_text('#!/bin/sh\necho probe >&2\ncat "$SOCKET_FIXTURE"\n')
|
||||
with subprocess.Popen(["sleep", "60"]) as server:
|
||||
try:
|
||||
result = self.run_shell(
|
||||
'sleep() { echo polling-sleep; }\nwait_udp_server "$SERVER_PID"',
|
||||
SERVER_PID=str(server.pid),
|
||||
)
|
||||
self.assertEqual(result.returncode, 1)
|
||||
self.assertIn("did not come up", result.stderr)
|
||||
self.assertEqual(result.stderr.count("probe\n"), 90)
|
||||
self.assertEqual(result.stdout.count("polling-sleep"), 89)
|
||||
server.wait(timeout=3)
|
||||
self.assertLess(server.returncode, 0)
|
||||
finally:
|
||||
if server.poll() is None:
|
||||
server.kill()
|
||||
|
||||
def test_server_exits_while_waiting(self) -> None:
|
||||
"""Detect a startup failure that happens after polling begins."""
|
||||
result = self.run_shell('sleep 1 &\npid=$!\nwait_udp_server "$pid"')
|
||||
self.assertEqual(result.returncode, 1)
|
||||
self.assertIn("exited before binding", result.stderr)
|
||||
self.assertIn("waiting for UDP port", result.stdout)
|
||||
|
||||
def test_real_socket_delayed_beyond_four_seconds(self) -> None:
|
||||
"""Wait for a real delayed bind with each installed port finder."""
|
||||
finders = [shutil.which("ss"), os.environ.get("NETSTAT") or shutil.which("netstat")]
|
||||
finders = [finder for finder in finders if finder]
|
||||
if not finders:
|
||||
self.skipTest("neither ss nor netstat available")
|
||||
for finder in finders:
|
||||
with self.subTest(finder=finder):
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
code = (
|
||||
"import socket,time,sys; time.sleep(6); "
|
||||
"s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); "
|
||||
"s.bind(('127.0.0.1',int(sys.argv[1]))); time.sleep(30)"
|
||||
)
|
||||
with subprocess.Popen([sys.executable, "-c", code, str(port)]) as server:
|
||||
try:
|
||||
started = time.monotonic()
|
||||
result = self.run_shell(
|
||||
'wait_udp_server "$SERVER_PID"',
|
||||
SERVER_PID=str(server.pid),
|
||||
PORT=str(port),
|
||||
PFCMD=finder,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertGreaterEqual(time.monotonic() - started, 6)
|
||||
self.assertIsNone(server.poll())
|
||||
finally:
|
||||
server.terminate()
|
||||
server.wait(timeout=3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,36 +0,0 @@
|
||||
Subject: [PATCH] tests: finish parsing before inspecting editor state
|
||||
|
||||
EditorState creation has a 20 ms parsing budget. A descheduled test can
|
||||
therefore observe an incomplete tree. Finish these small test documents
|
||||
without an interactive deadline and publish the result with a transaction.
|
||||
Keep the original completion and vector-matching assertions enabled.
|
||||
|
||||
--- a/module/codemirror-promql/src/test/utils-test.ts
|
||||
+++ b/module/codemirror-promql/src/test/utils-test.ts
|
||||
@@ -13,7 +13,7 @@
|
||||
|
||||
import { parser } from '@prometheus-io/lezer-promql';
|
||||
import { EditorState } from '@codemirror/state';
|
||||
-import { LRLanguage } from '@codemirror/language';
|
||||
+import { ensureSyntaxTree, LRLanguage } from '@codemirror/language';
|
||||
import nock from 'nock';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
@@ -23,10 +23,16 @@
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
export function createEditorState(expr: string): EditorState {
|
||||
- return EditorState.create({
|
||||
+ const state = EditorState.create({
|
||||
doc: expr,
|
||||
extensions: lightPromQLSyntax,
|
||||
});
|
||||
+ // These tests need a complete tree, independent of the editor's time budget.
|
||||
+ if (!ensureSyntaxTree(state, state.doc.length, Infinity)) {
|
||||
+ throw new Error('Unable to parse the test expression');
|
||||
+ }
|
||||
+ // Publish the completed parse so syntaxTree(state) sees it too.
|
||||
+ return state.update({}).state;
|
||||
}
|
||||
|
||||
export function mockPrometheusServer(): void {
|
||||
@@ -1,17 +0,0 @@
|
||||
{ prometheus }:
|
||||
prometheus.overrideAttrs (
|
||||
old:
|
||||
let
|
||||
assets = old.passthru.assets.overrideAttrs (assetsOld: {
|
||||
patches = (assetsOld.patches or [ ]) ++ [ ./complete-test-parsing.patch ];
|
||||
});
|
||||
in
|
||||
{
|
||||
postPatch = builtins.replaceStrings [ "${old.passthru.assets}" ] [ "${assets}" ] (
|
||||
builtins.unsafeDiscardStringContext old.postPatch
|
||||
);
|
||||
passthru = old.passthru // {
|
||||
inherit assets;
|
||||
};
|
||||
}
|
||||
)
|
||||
@@ -1,4 +0,0 @@
|
||||
{ psutil }:
|
||||
psutil.overridePythonAttrs (old: {
|
||||
patches = (old.patches or [ ]) ++ [ ./heap-info-zero-mmap.patch ];
|
||||
})
|
||||
@@ -1,17 +0,0 @@
|
||||
Subject: [PATCH] Allow an empty mmap allocation total in heap_info
|
||||
|
||||
The allocator may satisfy all live allocations from the heap. A zero
|
||||
mmap_used value is valid and depends on the worker allocation history.
|
||||
Keep the heap and platform checks without requiring an mmap allocation.
|
||||
|
||||
--- a/tests/test_system.py
|
||||
+++ b/tests/test_system.py
|
||||
@@ -267,6 +267,7 @@
|
||||
if MACOS:
|
||||
assert m.mmap_used == 0 # not supported
|
||||
else:
|
||||
- assert m.mmap_used > 0
|
||||
+ # A process can have no live mmap-backed malloc allocations.
|
||||
+ assert m.mmap_used >= 0
|
||||
if WINDOWS:
|
||||
assert m.heap_count >= 0
|
||||
@@ -1,29 +0,0 @@
|
||||
Subject: [PATCH] tests: count replacements when checking the worker restart limit
|
||||
|
||||
With two workers, another in-flight test may crash after the fourth
|
||||
crash requests shutdown. Either four or five failed tests is valid.
|
||||
Require exactly three replacements and the failed-tests exit status,
|
||||
while preserving the queued-work and no-internal-error assertions.
|
||||
|
||||
--- a/testing/acceptance_test.py
|
||||
+++ b/testing/acceptance_test.py
|
||||
@@ -1011,9 +1011,18 @@
|
||||
"worker*crashed while running*",
|
||||
"worker*crashed while running*",
|
||||
"* xdist: maximum crashed workers reached: 3 *",
|
||||
- "* 4 failed in *",
|
||||
]
|
||||
)
|
||||
+ # A second in-flight test may crash after shutdown is requested.
|
||||
+ # The restart limit constrains replacements, not concurrent failures.
|
||||
+ replacements = sum(
|
||||
+ line.startswith("replacing crashed worker ") for line in res.stdout.lines
|
||||
+ )
|
||||
+ assert replacements == 3
|
||||
+ failed = res.parseoutcomes()["failed"]
|
||||
+ assert failed in (4, 5)
|
||||
+ res.assert_outcomes(failed=failed)
|
||||
+ assert res.ret == pytest.ExitCode.TESTS_FAILED
|
||||
assert "INTERNALERROR" not in res.stdout.str()
|
||||
|
||||
def test_max_worker_restart_die(self, pytester: pytest.Pytester) -> None:
|
||||
@@ -1,12 +0,0 @@
|
||||
{ pytest-xdist }:
|
||||
pytest-xdist.overridePythonAttrs (old: {
|
||||
patches = (old.patches or [ ]) ++ [
|
||||
./concurrent-worker-crashes.patch
|
||||
./worker-startup-timeout.patch
|
||||
];
|
||||
|
||||
# The suite exercises its own worker pools. Limit the outer suite to one worker.
|
||||
preCheck = builtins.replaceStrings [ "--numprocesses=$NIX_BUILD_CORES" ] [ "--numprocesses=1" ] (
|
||||
old.preCheck or ""
|
||||
);
|
||||
})
|
||||
@@ -1,19 +0,0 @@
|
||||
Subject: [PATCH] tests: allow more time for remote worker events
|
||||
|
||||
Worker startup can exceed ten seconds on heavily loaded builders. Allow
|
||||
the remote-test helper to wait up to sixty seconds for worker events.
|
||||
The wait still returns as soon as an event arrives and remains bounded.
|
||||
Production worker timeouts and test assertions are unchanged.
|
||||
|
||||
--- a/testing/test_remote.py
|
||||
+++ b/testing/test_remote.py
|
||||
@@ -17,7 +17,8 @@
|
||||
from xdist.workermanage import WorkerController
|
||||
|
||||
|
||||
-WAIT_TIMEOUT = 10.0
|
||||
+# Allow worker events extra time on heavily loaded builders.
|
||||
+WAIT_TIMEOUT = 60.0
|
||||
|
||||
|
||||
def check_marshallable(d: object) -> None:
|
||||
@@ -1,5 +0,0 @@
|
||||
{ scipy }:
|
||||
scipy.overridePythonAttrs (old: {
|
||||
# Keep the STFT tests enabled with tolerances for x86-64-v3 rounding.
|
||||
patches = (old.patches or [ ]) ++ [ ./stft-test-tolerances.patch ];
|
||||
})
|
||||
@@ -1,51 +0,0 @@
|
||||
Subject: [PATCH] signal: allow floating-point rounding in STFT tests
|
||||
|
||||
Keep the STFT tests enabled for x86-64-v3 builds. Allow two float32
|
||||
epsilons of relative error when comparing inverse-STFT implementations;
|
||||
float64 and the existing i686 override remain unchanged. Allow one
|
||||
float64 epsilon of absolute error in all three scaling round trips,
|
||||
which otherwise require exact zeros (observed residual: 4e-17 for a
|
||||
signal with amplitude 2).
|
||||
|
||||
Upstream issue: https://github.com/scipy/scipy/issues/25488
|
||||
|
||||
--- a/scipy/signal/tests/_scipy_spectral_test_shim.py
|
||||
+++ b/scipy/signal/tests/_scipy_spectral_test_shim.py
|
||||
@@ -294,7 +294,7 @@
|
||||
|
||||
# Adapted tolerances to account for resolution loss:
|
||||
atol = np.finfo(x.dtype).resolution*2 # instead of default atol = 0
|
||||
- rtol = 1e-7 # default for np.allclose()
|
||||
+ rtol = max(1e-7, 2 * np.finfo(x.dtype).eps)
|
||||
|
||||
# Relax atol on 32-Bit platforms a bit to pass CI tests.
|
||||
# - Not clear why there are discrepancies (in the FFT maybe?)
|
||||
--- a/scipy/signal/tests/test_spectral.py
|
||||
+++ b/scipy/signal/tests/test_spectral.py
|
||||
@@ -2044,7 +2044,7 @@
|
||||
|
||||
# Test round trip:
|
||||
x1 = istft(Zs, boundary=True, scaling='spectrum')[1]
|
||||
- assert_allclose(x1, x)
|
||||
+ assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
|
||||
|
||||
# For a Hann-windowed 256 sample length FFT, we expect a peak at
|
||||
# frequency 64 (since it is 1/4 the length of X) with a height of 1
|
||||
@@ -2074,7 +2074,7 @@
|
||||
|
||||
# Test round trip:
|
||||
x1 = istft(Zp, input_onesided=False, boundary=True, scaling='psd')[1]
|
||||
- assert_allclose(x1, x)
|
||||
+ assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
|
||||
|
||||
# The power of the one-sided psd-scaled STFT can be determined
|
||||
# analogously (note that the two sides are not of equal shape):
|
||||
@@ -2094,7 +2094,7 @@
|
||||
|
||||
# Test round trip:
|
||||
x1 = istft(Zp0, input_onesided=True, boundary=True, scaling='psd')[1]
|
||||
- assert_allclose(x1, x)
|
||||
+ assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
|
||||
|
||||
|
||||
class TestSampledSpectralRepresentations:
|
||||
@@ -1,21 +0,0 @@
|
||||
{ trunk, jq }:
|
||||
trunk.overrideAttrs (old: {
|
||||
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ jq ];
|
||||
|
||||
# Patch the bundled libdeflate before Cargo validates its vendor checksums.
|
||||
postPatch = (old.postPatch or "") + ''
|
||||
libdeflateVendors=( "$cargoDepsCopy"/source-*/libdeflate-sys-1.23.1 )
|
||||
if [ "''${#libdeflateVendors[@]}" -ne 1 ] || [ ! -d "''${libdeflateVendors[0]}" ]; then
|
||||
echo "Expected exactly one vendored libdeflate-sys 1.23.1" >&2
|
||||
exit 1
|
||||
fi
|
||||
libdeflateVendor="''${libdeflateVendors[0]}"
|
||||
patch -d "$libdeflateVendor/libdeflate" -p1 < ${./libdeflate-gcc16.patch}
|
||||
libdeflateHeader=libdeflate/lib/x86/cpu_features.h
|
||||
libdeflateChecksum=$(sha256sum "$libdeflateVendor/$libdeflateHeader" | cut -d ' ' -f 1)
|
||||
jq --arg path "$libdeflateHeader" --arg hash "$libdeflateChecksum" \
|
||||
'.files[$path] = $hash' "$libdeflateVendor/.cargo-checksum.json" \
|
||||
> "$libdeflateVendor/.cargo-checksum.json.new"
|
||||
mv "$libdeflateVendor/.cargo-checksum.json.new" "$libdeflateVendor/.cargo-checksum.json"
|
||||
'';
|
||||
})
|
||||
@@ -1,33 +0,0 @@
|
||||
From 2a3762cf736aff9aa5dda329fdec3c08a816eb9b Mon Sep 17 00:00:00 2001
|
||||
From: Sergei Trofimovich <slyich@gmail.com>
|
||||
Date: Tue, 20 May 2025 21:18:28 +0100
|
||||
Subject: [PATCH] lib/x86/cpu_features.h: drop evex512 on gcc-16
|
||||
|
||||
gcc-16 `master` branch dropped support for `evex512` as:
|
||||
https://gcc.gnu.org/git/?p=gcc.git;a=commitdiff;h=c052a6f4a1c803cb92147ff98fb91cf3511e0856
|
||||
|
||||
As a result libdeflate build started failing as:
|
||||
|
||||
In file included from /build/source/lib/adler32.c:28:
|
||||
/build/source/lib/x86/adler32_template.h:135:12: error: attribute 'target' argument 'no-evex512' is unknown
|
||||
135 | ADD_SUFFIX(reduce_to_32bits)(vec_t v_s1, vec_t v_s2, u32 *s1_p, u32 *s2_p)
|
||||
| ^~~~~~~~~~~~~~~~
|
||||
|
||||
The change adds upper build to `evex512` for `gcc`.
|
||||
---
|
||||
lib/x86/cpu_features.h | 2 +-
|
||||
1 file changed, 1 insertion(+), 1 deletion(-)
|
||||
|
||||
diff --git a/lib/x86/cpu_features.h b/lib/x86/cpu_features.h
|
||||
index fbfdb0eb..f9d69d72 100644
|
||||
--- a/lib/x86/cpu_features.h
|
||||
+++ b/lib/x86/cpu_features.h
|
||||
@@ -165,7 +165,7 @@ static inline u32 get_x86_cpu_features(void) { return 0; }
|
||||
# define HAVE_AVXVNNI(features) ((features) & X86_CPU_FEATURE_AVXVNNI)
|
||||
#endif
|
||||
|
||||
-#if (GCC_PREREQ(14, 0) || CLANG_PREREQ(18, 0, 18000000)) \
|
||||
+#if ((GCC_PREREQ(14, 0) && !(GCC_PREREQ(16, 0))) || CLANG_PREREQ(18, 0, 18000000)) \
|
||||
&& !defined(__EVEX512__) /* avoid subtracting the evex512 feature */
|
||||
# define EVEX512 ",evex512" /* needed to override potential -mno-evex512 */
|
||||
# define NO_EVEX512 ",no-evex512"
|
||||
@@ -1,4 +0,0 @@
|
||||
{ zopfli }:
|
||||
zopfli.overrideAttrs (old: {
|
||||
patches = (old.patches or [ ]) ++ [ ./unaligned-match-loads.patch ];
|
||||
})
|
||||
@@ -1,32 +0,0 @@
|
||||
Subject: [PATCH] Read unaligned match buffers with memcpy
|
||||
|
||||
Byte buffers need not satisfy integer alignment or aliasing requirements.
|
||||
GCC 16 with x86-64-v3 vectorizes the cast loads using aligned AVX reads,
|
||||
causing a segmentation fault. Copy into local integers instead.
|
||||
|
||||
--- a/src/zopfli/lz77.c
|
||||
+++ b/src/zopfli/lz77.c
|
||||
@@ -302,13 +302,20 @@
|
||||
if (sizeof(size_t) == 8) {
|
||||
/* 8 checks at once per array bounds check (size_t is 64-bit). */
|
||||
- while (scan < safe_end && *((size_t*)scan) == *((size_t*)match)) {
|
||||
+ while (scan < safe_end) {
|
||||
+ size_t scan_word, match_word;
|
||||
+ memcpy(&scan_word, scan, sizeof(scan_word));
|
||||
+ memcpy(&match_word, match, sizeof(match_word));
|
||||
+ if (scan_word != match_word) break;
|
||||
scan += 8;
|
||||
match += 8;
|
||||
}
|
||||
} else if (sizeof(unsigned int) == 4) {
|
||||
/* 4 checks at once per array bounds check (unsigned int is 32-bit). */
|
||||
- while (scan < safe_end
|
||||
- && *((unsigned int*)scan) == *((unsigned int*)match)) {
|
||||
+ while (scan < safe_end) {
|
||||
+ unsigned int scan_word, match_word;
|
||||
+ memcpy(&scan_word, scan, sizeof(scan_word));
|
||||
+ memcpy(&match_word, match, sizeof(match_word));
|
||||
+ if (scan_word != match_word) break;
|
||||
scan += 4;
|
||||
match += 4;
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
services.zerotierone = {
|
||||
enable = true;
|
||||
joinNetworks = [ "e4da7455b2ae64ca" ];
|
||||
};
|
||||
nix.settings = {
|
||||
trusted-substituters = [ "http://192.168.90.40:5000" ];
|
||||
substituters = [ "http://192.168.90.40:5000/?priority=1&want-mass-query=true" ];
|
||||
trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
|
||||
};
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
{
|
||||
inputs,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
{
|
||||
imports = [ ./snapshot.nix ];
|
||||
|
||||
boot.zfs = {
|
||||
package = lib.mkDefault pkgs.zfs_2_4;
|
||||
forceImportRoot = lib.mkDefault false;
|
||||
};
|
||||
|
||||
services = {
|
||||
snapshot_manager = {
|
||||
enable = lib.mkDefault true;
|
||||
PYTHONPATH = "${inputs.self}/";
|
||||
};
|
||||
|
||||
zfs = {
|
||||
trim.enable = lib.mkDefault true;
|
||||
autoScrub.enable = lib.mkDefault true;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
{
|
||||
inputs,
|
||||
pkgs,
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.services.snapshot_manager;
|
||||
snapshotManagerPackages = ps: with ps; [ typer ];
|
||||
in
|
||||
{
|
||||
options = {
|
||||
services.snapshot_manager = {
|
||||
enable = lib.mkEnableOption "ZFS snapshot manager";
|
||||
path = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
default = ./snapshot_config.toml;
|
||||
description = "Path to the snapshot_manager TOML config.";
|
||||
};
|
||||
PYTHONPATH = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = ''
|
||||
the PYTHONPATH to use for the snapshot_manager service.
|
||||
'';
|
||||
};
|
||||
EnvironmentFile = lib.mkOption {
|
||||
type = lib.types.nullOr (lib.types.coercedTo lib.types.path toString lib.types.str);
|
||||
default = null;
|
||||
description = ''
|
||||
Single environment file for the service (e.g. /etc/snapshot-manager/env).
|
||||
Use a leading "-" to ignore if missing (systemd feature).
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
nixpkgs.overlays = [
|
||||
(final: _prev: {
|
||||
snapshot_manager_python = final.python314.withPackages snapshotManagerPackages;
|
||||
snapshot_manager_test_python = final.python314.withPackages (
|
||||
ps:
|
||||
snapshotManagerPackages ps
|
||||
++ (with ps; [
|
||||
pyfakefs
|
||||
pytest
|
||||
pytest-asyncio
|
||||
pytest-mock
|
||||
pytest-xdist
|
||||
])
|
||||
);
|
||||
snapshot_manager_tests =
|
||||
final.runCommand "snapshot-manager-tests"
|
||||
{
|
||||
nativeBuildInputs = [ final.snapshot_manager_test_python ];
|
||||
}
|
||||
''
|
||||
export HOME="$TMPDIR"
|
||||
cd ${inputs.self}
|
||||
pytest \
|
||||
-o cache_dir="$TMPDIR/pytest-cache" \
|
||||
tests/test_common.py \
|
||||
tests/test_signal_alert.py \
|
||||
tests/test_snapshot_manager.py \
|
||||
tests/test_zfs.py
|
||||
touch "$out"
|
||||
'';
|
||||
})
|
||||
];
|
||||
|
||||
system.checks = [ pkgs.snapshot_manager_tests ];
|
||||
|
||||
systemd = {
|
||||
services.snapshot_manager = {
|
||||
description = "ZFS Snapshot Manager";
|
||||
requires = [ "zfs-import.target" ];
|
||||
after = [ "zfs-import.target" ];
|
||||
path = [ pkgs.zfs ];
|
||||
environment = {
|
||||
PYTHONPATH = cfg.PYTHONPATH;
|
||||
};
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = "${pkgs.snapshot_manager_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
|
||||
}
|
||||
// lib.optionalAttrs (cfg.EnvironmentFile != null) {
|
||||
EnvironmentFile = cfg.EnvironmentFile;
|
||||
};
|
||||
};
|
||||
timers.snapshot_manager = {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnBootSec = "15m";
|
||||
OnUnitActiveSec = "15m";
|
||||
Unit = "snapshot_manager.service";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
source "https://rubygems.org"
|
||||
|
||||
# The github-pages gem pins all compatible versions of Jekyll and its plugins
|
||||
gem "github-pages", group: :jekyll_plugins
|
||||
@@ -1,23 +0,0 @@
|
||||
title: "Richie Cahill"
|
||||
description: "ALL THE CHAOS THAT I CANT DO AT WORK"
|
||||
baseurl: "/dotfiles"
|
||||
url: "https://richiecahill.github.io"
|
||||
|
||||
remote_theme: pages-themes/hacker@v0.2.0
|
||||
plugins:
|
||||
- jekyll-feed
|
||||
- jekyll-remote-theme
|
||||
- jekyll-seo-tag
|
||||
- jekyll-sitemap
|
||||
- jekyll-paginate
|
||||
|
||||
paginate: 5
|
||||
paginate_path: "/page:num"
|
||||
|
||||
author:
|
||||
name: "Richie Cahill"
|
||||
email: "richie@tmmworkshop.com"
|
||||
|
||||
social_links:
|
||||
github: "RichieCahill"
|
||||
website: "https://tmmworkshop.com"
|
||||
@@ -1,13 +0,0 @@
|
||||
# The MONOREPO experiment
|
||||
|
||||
Im testing a [MONOREPO](https://en.wikipedia.org/wiki/Monorepo) because Phil said this was a bad idea. To that i say hold my beer.
|
||||
|
||||
In all seriousness, I Think that for a small dev team/solo dev. The simplicity is worth higher barer to entry. One of my most annoying processes was updating my system tools. I had to build my update in a feature branch and then merge it into my main branch. then go to my dotfiles create a feature branch update the system tools merge it into main.
|
||||
|
||||
It will be starting with my Nix Dotfiles Python tools and now my blog.
|
||||
|
||||
I will be reaching ot to phil on 2030-10-31 and 2035-10-31 to give him updates on the progress.
|
||||
|
||||
Known Issues:
|
||||
|
||||
- the python tests are running on the current derivation not the one the derivation im updating to.
|
||||
@@ -1,17 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: "Welcome"
|
||||
---
|
||||
|
||||
Welcome to my build logs, notes, and experiments.
|
||||
|
||||
You can read my latest posts below
|
||||
|
||||
<ul>
|
||||
{% for post in site.posts %}
|
||||
<li>
|
||||
<a href="{{ post.url | relative_url }}">{{ post.title }}</a>
|
||||
<small>— {{ post.date | date: "%Y-%m-%d" }}</small>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
@@ -1,76 +0,0 @@
|
||||
# ZFS failed root import recovery
|
||||
|
||||
## Fast path
|
||||
|
||||
If the machine fails to boot because ZFS refuses to import `root_pool`:
|
||||
|
||||
### GRUB
|
||||
|
||||
1. At the bootloader menu, select the normal NixOS entry.
|
||||
2. Press `e`.
|
||||
3. Find the line that starts with `linux`.
|
||||
4. Append this to the end of that line:
|
||||
|
||||
```text
|
||||
zfs_force=1
|
||||
```
|
||||
|
||||
5. Boot once with `Ctrl+x` or `F10`.
|
||||
|
||||
### systemd-boot
|
||||
|
||||
1. At the bootloader menu, highlight the normal NixOS entry.
|
||||
2. Press `e`.
|
||||
3. Append this to the end of the options line:
|
||||
|
||||
```text
|
||||
zfs_force=1
|
||||
```
|
||||
|
||||
4. Press `Enter` to boot once.
|
||||
|
||||
## After boot
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
sudo zpool status
|
||||
sudo zpool import
|
||||
journalctl -b | rg "ZFS|zfs|import|root_pool"
|
||||
```
|
||||
|
||||
## Expected result
|
||||
|
||||
`sudo zpool status` should show `root_pool` as `ONLINE`.
|
||||
|
||||
## Reboot test
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
sudo reboot
|
||||
```
|
||||
|
||||
Do not add `zfs_force=1` the second time.
|
||||
|
||||
## If it still fails
|
||||
|
||||
Boot once more with:
|
||||
|
||||
```text
|
||||
zfs_force=1
|
||||
```
|
||||
|
||||
Then run:
|
||||
|
||||
```bash
|
||||
sudo zpool status -v
|
||||
sudo zpool history | tail -n 50
|
||||
journalctl -b | rg "ZFS|zfs|import|root_pool"
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
- Root pool name is `root_pool`.
|
||||
- This is a one-time recovery path after disk moves, controller changes, dirty exports, or interrupted imports.
|
||||
- Some hosts also need the LUKS unlock USB key inserted before boot.
|
||||
@@ -1,3 +0,0 @@
|
||||
# esphome
|
||||
/.esphome/
|
||||
/secrets.yaml
|
||||
@@ -1,132 +0,0 @@
|
||||
esphome:
|
||||
name: batteries
|
||||
friendly_name: batteries
|
||||
|
||||
esp32:
|
||||
board: esp32dev
|
||||
framework:
|
||||
type: arduino
|
||||
|
||||
logger:
|
||||
|
||||
api:
|
||||
encryption:
|
||||
key: !secret api_key
|
||||
|
||||
external_components:
|
||||
- source: github://syssi/esphome-jk-bms@main
|
||||
|
||||
ota:
|
||||
- platform: esphome
|
||||
password: !secret ota_password
|
||||
|
||||
wifi:
|
||||
ssid: !secret wifi_ssid
|
||||
password: !secret wifi_password
|
||||
fast_connect: on
|
||||
|
||||
captive_portal:
|
||||
|
||||
esp32_ble_tracker:
|
||||
scan_parameters:
|
||||
interval: 1100ms
|
||||
window: 1100ms
|
||||
active: true
|
||||
|
||||
ble_client:
|
||||
- mac_address: "C8:47:80:29:0F:DB"
|
||||
id: jk_ble0
|
||||
|
||||
jk_bms_ble:
|
||||
- ble_client_id: jk_ble0
|
||||
protocol_version: JK02_32S
|
||||
throttle: 1s
|
||||
id: jk_bms0
|
||||
|
||||
button:
|
||||
- platform: jk_bms_ble
|
||||
retrieve_settings:
|
||||
name: "JK0 retrieve settings"
|
||||
retrieve_device_info:
|
||||
name: "JK0 retrieve device info"
|
||||
|
||||
sensor:
|
||||
- platform: jk_bms_ble
|
||||
jk_bms_ble_id: jk_bms0
|
||||
total_voltage:
|
||||
name: "JK0 Total Voltage"
|
||||
state_of_charge:
|
||||
name: "JK0 SoC"
|
||||
charging_power:
|
||||
name: "JK0 charging power"
|
||||
discharging_power:
|
||||
name: "JK0 discharging power"
|
||||
temperature_sensor_1:
|
||||
name: "JK0 Temp 1"
|
||||
temperature_sensor_2:
|
||||
name: "JK0 Temp 2"
|
||||
balancing:
|
||||
name: "JK0 balancing"
|
||||
total_runtime:
|
||||
name: "JK0 total runtime"
|
||||
balancing_current:
|
||||
name: "JK0 balancing current"
|
||||
delta_cell_voltage:
|
||||
name: "JK0 cell delta voltage"
|
||||
average_cell_voltage:
|
||||
name: "JK0 cell average voltage"
|
||||
cell_voltage_1:
|
||||
name: "JK0 cell voltage 1"
|
||||
cell_voltage_2:
|
||||
name: "JK0 cell voltage 2"
|
||||
cell_voltage_3:
|
||||
name: "JK0 cell voltage 3"
|
||||
cell_voltage_4:
|
||||
name: "JK0 cell voltage 4"
|
||||
cell_voltage_5:
|
||||
name: "JK0 cell voltage 5"
|
||||
cell_voltage_6:
|
||||
name: "JK0 cell voltage 6"
|
||||
cell_voltage_7:
|
||||
name: "JK0 cell voltage 7"
|
||||
cell_voltage_8:
|
||||
name: "JK0 cell voltage 8"
|
||||
cell_resistance_1:
|
||||
name: "JK0 cell resistance 1"
|
||||
cell_resistance_2:
|
||||
name: "JK0 cell resistance 2"
|
||||
cell_resistance_3:
|
||||
name: "JK0 cell resistance 3"
|
||||
cell_resistance_4:
|
||||
name: "JK0 cell resistance 4"
|
||||
cell_resistance_5:
|
||||
name: "JK0 cell resistance 5"
|
||||
cell_resistance_6:
|
||||
name: "JK0 cell resistance 6"
|
||||
cell_resistance_7:
|
||||
name: "JK0 cell resistance 7"
|
||||
cell_resistance_8:
|
||||
name: "JK0 cell resistance 8"
|
||||
total_charging_cycle_capacity:
|
||||
name: "JK0 total charging cycle capacity"
|
||||
|
||||
text_sensor:
|
||||
- platform: jk_bms_ble
|
||||
jk_bms_ble_id: jk_bms0
|
||||
errors:
|
||||
name: "JK0 Errors"
|
||||
|
||||
switch:
|
||||
- platform: jk_bms_ble
|
||||
jk_bms_ble_id: jk_bms0
|
||||
charging:
|
||||
name: "JK0 Charging"
|
||||
discharging:
|
||||
name: "JK0 Discharging"
|
||||
balancer:
|
||||
name: "JK0 Balancing"
|
||||
|
||||
- platform: ble_client
|
||||
ble_client_id: jk_ble0
|
||||
name: "JK0 enable bluetooth connection"
|
||||
id: ble_client_switch0
|
||||
@@ -1,132 +0,0 @@
|
||||
esphome:
|
||||
name: battery1
|
||||
friendly_name: battery1
|
||||
|
||||
esp32:
|
||||
board: esp32dev
|
||||
framework:
|
||||
type: arduino
|
||||
|
||||
logger:
|
||||
|
||||
api:
|
||||
encryption:
|
||||
key: !secret api_key
|
||||
|
||||
external_components:
|
||||
- source: github://syssi/esphome-jk-bms@main
|
||||
|
||||
ota:
|
||||
- platform: esphome
|
||||
password: !secret ota_password
|
||||
|
||||
wifi:
|
||||
ssid: !secret wifi_ssid
|
||||
password: !secret wifi_password
|
||||
fast_connect: on
|
||||
|
||||
captive_portal:
|
||||
|
||||
esp32_ble_tracker:
|
||||
scan_parameters:
|
||||
interval: 1100ms
|
||||
window: 1100ms
|
||||
active: true
|
||||
|
||||
ble_client:
|
||||
- mac_address: "C8:47:80:37:9D:DD"
|
||||
id: jk_ble1
|
||||
|
||||
jk_bms_ble:
|
||||
- ble_client_id: jk_ble1
|
||||
protocol_version: JK02_32S
|
||||
throttle: 1s
|
||||
id: jk_bms1
|
||||
|
||||
button:
|
||||
- platform: jk_bms_ble
|
||||
retrieve_settings:
|
||||
name: "JK1 retrieve settings"
|
||||
retrieve_device_info:
|
||||
name: "JK1 retrieve device info"
|
||||
|
||||
sensor:
|
||||
- platform: jk_bms_ble
|
||||
jk_bms_ble_id: jk_bms1
|
||||
total_voltage:
|
||||
name: "JK1 Total Voltage"
|
||||
state_of_charge:
|
||||
name: "JK1 SoC"
|
||||
charging_power:
|
||||
name: "JK1 charging power"
|
||||
discharging_power:
|
||||
name: "JK1 discharging power"
|
||||
temperature_sensor_1:
|
||||
name: "JK1 Temp 1"
|
||||
temperature_sensor_2:
|
||||
name: "JK1 Temp 2"
|
||||
balancing:
|
||||
name: "JK1 balancing"
|
||||
total_runtime:
|
||||
name: "JK1 total runtime"
|
||||
balancing_current:
|
||||
name: "JK1 balancing current"
|
||||
delta_cell_voltage:
|
||||
name: "JK1 cell delta voltage"
|
||||
average_cell_voltage:
|
||||
name: "JK1 cell average voltage"
|
||||
cell_voltage_1:
|
||||
name: "JK1 cell voltage 1"
|
||||
cell_voltage_2:
|
||||
name: "JK1 cell voltage 2"
|
||||
cell_voltage_3:
|
||||
name: "JK1 cell voltage 3"
|
||||
cell_voltage_4:
|
||||
name: "JK1 cell voltage 4"
|
||||
cell_voltage_5:
|
||||
name: "JK1 cell voltage 5"
|
||||
cell_voltage_6:
|
||||
name: "JK1 cell voltage 6"
|
||||
cell_voltage_7:
|
||||
name: "JK1 cell voltage 7"
|
||||
cell_voltage_8:
|
||||
name: "JK1 cell voltage 8"
|
||||
cell_resistance_1:
|
||||
name: "JK1 cell resistance 1"
|
||||
cell_resistance_2:
|
||||
name: "JK1 cell resistance 2"
|
||||
cell_resistance_3:
|
||||
name: "JK1 cell resistance 3"
|
||||
cell_resistance_4:
|
||||
name: "JK1 cell resistance 4"
|
||||
cell_resistance_5:
|
||||
name: "JK1 cell resistance 5"
|
||||
cell_resistance_6:
|
||||
name: "JK1 cell resistance 6"
|
||||
cell_resistance_7:
|
||||
name: "JK1 cell resistance 7"
|
||||
cell_resistance_8:
|
||||
name: "JK1 cell resistance 8"
|
||||
total_charging_cycle_capacity:
|
||||
name: "JK1 total charging cycle capacity"
|
||||
|
||||
text_sensor:
|
||||
- platform: jk_bms_ble
|
||||
jk_bms_ble_id: jk_bms1
|
||||
errors:
|
||||
name: "JK1 Errors"
|
||||
|
||||
switch:
|
||||
- platform: jk_bms_ble
|
||||
jk_bms_ble_id: jk_bms1
|
||||
charging:
|
||||
name: "JK1 Charging"
|
||||
discharging:
|
||||
name: "JK1 Discharging"
|
||||
balancer:
|
||||
name: "JK1 Balancing"
|
||||
|
||||
- platform: ble_client
|
||||
ble_client_id: jk_ble1
|
||||
name: "JK1 enable bluetooth connection"
|
||||
id: ble_client_switch0
|
||||
@@ -1,48 +0,0 @@
|
||||
esphome:
|
||||
name: "environment"
|
||||
friendly_name: "environment"
|
||||
|
||||
esp32:
|
||||
board: esp32dev
|
||||
framework:
|
||||
type: arduino
|
||||
|
||||
i2c:
|
||||
sda: GPIO21
|
||||
scl: GPIO22
|
||||
scan: True
|
||||
id: bus_a
|
||||
|
||||
sensor:
|
||||
- platform: aht10
|
||||
i2c_id: bus_a
|
||||
address: 0x38
|
||||
variant: AHT20
|
||||
temperature:
|
||||
name: "environment Temperature"
|
||||
id: aht10_temperature
|
||||
humidity:
|
||||
name: "environment Humidity"
|
||||
id: aht10_humidity
|
||||
update_interval: 5s
|
||||
|
||||
web_server:
|
||||
port: 80
|
||||
|
||||
logger:
|
||||
level: DEBUG
|
||||
|
||||
api:
|
||||
encryption:
|
||||
key: !secret api_key
|
||||
|
||||
ota:
|
||||
- platform: esphome
|
||||
password: !secret ota_password
|
||||
|
||||
wifi:
|
||||
ssid: !secret wifi_ssid
|
||||
password: !secret wifi_password
|
||||
fast_connect: on
|
||||
|
||||
captive_portal:
|
||||
Generated
+129
-121
@@ -1,41 +1,25 @@
|
||||
{
|
||||
"nodes": {
|
||||
"disko": {
|
||||
"firefox-addons": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1789770686,
|
||||
"narHash": "sha256-uZkBR7yHdIKUFB5SZdfgh1qkGfI3XmYmI/lTiquxbck=",
|
||||
"owner": "nix-community",
|
||||
"repo": "disko",
|
||||
"rev": "725ea35e410ad83be4931d1bff7e090eacaf3563",
|
||||
"type": "github"
|
||||
"dir": "pkgs/firefox-addons",
|
||||
"lastModified": 1750219402,
|
||||
"narHash": "sha256-b3y7V7db0VwLGtpcLRmT1Aa9dpAKoHQdem55UhgB/fw=",
|
||||
"owner": "rycee",
|
||||
"repo": "nur-expressions",
|
||||
"rev": "a00ce73b626ed274fbfe9f51627861e140b08f6d",
|
||||
"type": "gitlab"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-community",
|
||||
"repo": "disko",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-utils": {
|
||||
"inputs": {
|
||||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1731533236,
|
||||
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
"dir": "pkgs/firefox-addons",
|
||||
"owner": "rycee",
|
||||
"repo": "nur-expressions",
|
||||
"type": "gitlab"
|
||||
}
|
||||
},
|
||||
"home-manager": {
|
||||
@@ -45,11 +29,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1790989254,
|
||||
"narHash": "sha256-ZN+riaeFuE+DZ5zfqJiXlM12jLwZzeIXiNkU1YeHMXI=",
|
||||
"lastModified": 1750275572,
|
||||
"narHash": "sha256-upC/GIlsIgtdtWRGd1obzdXWYQptNkfzZeyAFWgsgf0=",
|
||||
"owner": "nix-community",
|
||||
"repo": "home-manager",
|
||||
"rev": "acd21c5a3420a9d5fd0ed06299b10828267ef9ba",
|
||||
"rev": "0f355844e54e4c70906b1ef5cc35a0047d666c04",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -58,38 +42,13 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"jellyswarrm": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils",
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
],
|
||||
"rust-overlay": "rust-overlay"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1790427786,
|
||||
"narHash": "sha256-8+Ip0HFW2oZiRr0zCZmPPIHfTdxxHy4CmSF1uOkM3Bw=",
|
||||
"owner": "LLukas22",
|
||||
"repo": "Jellyswarrm",
|
||||
"rev": "9e67c5ad12116add867ba6c1bf69f767879e9147",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "LLukas22",
|
||||
"repo": "Jellyswarrm",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixos-hardware": {
|
||||
"inputs": {
|
||||
"nixpkgs": "nixpkgs"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1790948122,
|
||||
"narHash": "sha256-fDMYyzY1lI5b8RuCMrvoaIgTeRwNnTG4dNk1ucROlQE=",
|
||||
"lastModified": 1750083401,
|
||||
"narHash": "sha256-ynqbgIYrg7P1fAKYqe8I/PMiLABBcNDYG9YaAP/d/C4=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixos-hardware",
|
||||
"rev": "0953bb1a609df63013176e3463393f8df993d3ef",
|
||||
"rev": "61837d2a33ccc1582c5fabb7bf9130d39fee59ad",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -101,24 +60,27 @@
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1789546076,
|
||||
"narHash": "sha256-vWkSk5bbfTqdtMoSgD9FshACO8JCvXTFi+3cqEp0mH0=",
|
||||
"rev": "b1b875982b17dabde9b4a37f3e229e74913e6db3",
|
||||
"type": "tarball",
|
||||
"url": "https://releases.nixos.org/nixos/unstable/nixos-26.11pre1074753.b1b875982b17/nixexprs.tar.xz"
|
||||
"lastModified": 1750134718,
|
||||
"narHash": "sha256-v263g4GbxXv87hMXMCpjkIxd/viIF7p3JpJrwgKdNiI=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "9e83b64f727c88a7711a2c463a7b16eedb69a84c",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"type": "tarball",
|
||||
"url": "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz"
|
||||
"owner": "nixos",
|
||||
"ref": "nixos-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs-master": {
|
||||
"locked": {
|
||||
"lastModified": 1791033472,
|
||||
"narHash": "sha256-IRpPnIfZ0drkkyMtuuvXaada2RjH+G2WYv14Ap1W8q4=",
|
||||
"lastModified": 1750291913,
|
||||
"narHash": "sha256-JW40+zIiDS+rZavb9IYdIN40/GmErO2+0+A66rM6/b8=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "ad70d223e208533b7ee8cc8e336a34aad134ab19",
|
||||
"rev": "ba92ab5dc0759a8740003ca34b5c1b888f4766d4",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -144,54 +106,67 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs_2": {
|
||||
"pyproject-build-systems": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"system_tools",
|
||||
"nixpkgs"
|
||||
],
|
||||
"pyproject-nix": [
|
||||
"system_tools",
|
||||
"pyproject-nix"
|
||||
],
|
||||
"uv2nix": [
|
||||
"system_tools",
|
||||
"uv2nix"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1790822859,
|
||||
"narHash": "sha256-69xHQhAeMAD2wDXO7T2pcOZIF9Sga2W+JkmY2a11Ops=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "c59305bab2065cfecc4944690d9eedbb56f3a9fa",
|
||||
"lastModified": 1744599653,
|
||||
"narHash": "sha256-nysSwVVjG4hKoOjhjvE6U5lIKA8sEr1d1QzEfZsannU=",
|
||||
"owner": "pyproject-nix",
|
||||
"repo": "build-system-pkgs",
|
||||
"rev": "7dba6dbc73120e15b558754c26024f6c93015dd7",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nixos",
|
||||
"ref": "nixos-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"owner": "pyproject-nix",
|
||||
"repo": "build-system-pkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"pyproject-nix": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"system_tools",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1746540146,
|
||||
"narHash": "sha256-QxdHGNpbicIrw5t6U3x+ZxeY/7IEJ6lYbvsjXmcxFIM=",
|
||||
"owner": "pyproject-nix",
|
||||
"repo": "pyproject.nix",
|
||||
"rev": "e09c10c24ebb955125fda449939bfba664c467fd",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "pyproject-nix",
|
||||
"repo": "pyproject.nix",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"disko": "disko",
|
||||
"firefox-addons": "firefox-addons",
|
||||
"home-manager": "home-manager",
|
||||
"jellyswarrm": "jellyswarrm",
|
||||
"nixos-hardware": "nixos-hardware",
|
||||
"nixpkgs": "nixpkgs_2",
|
||||
"nixpkgs": "nixpkgs",
|
||||
"nixpkgs-master": "nixpkgs-master",
|
||||
"nixpkgs-stable": "nixpkgs-stable",
|
||||
"sops-nix": "sops-nix",
|
||||
"systems": "systems_2"
|
||||
}
|
||||
},
|
||||
"rust-overlay": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"jellyswarrm",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1779419951,
|
||||
"narHash": "sha256-dMX0PUslUHPajP6o8FEoRdFv9afq/dec4POR0vVfjK4=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "5b5c521d6cae9ef4aa32f888eb2c0ce595c9be52",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"type": "github"
|
||||
"system_tools": "system_tools",
|
||||
"systems": "systems"
|
||||
}
|
||||
},
|
||||
"sops-nix": {
|
||||
@@ -201,11 +176,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1790498116,
|
||||
"narHash": "sha256-rs9meAYxW3zzrh43yaW7htrqCD+X9+pupDPHN86fumI=",
|
||||
"lastModified": 1750119275,
|
||||
"narHash": "sha256-Rr7Pooz9zQbhdVxux16h7URa6mA80Pb/G07T4lHvh0M=",
|
||||
"owner": "Mic92",
|
||||
"repo": "sops-nix",
|
||||
"rev": "5efb5a6f4f5ab192817d28557dd4d650fa14d866",
|
||||
"rev": "77c423a03b9b2b79709ea2cb63336312e78b72e2",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -214,22 +189,30 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"system_tools": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
],
|
||||
"pyproject-build-systems": "pyproject-build-systems",
|
||||
"pyproject-nix": "pyproject-nix",
|
||||
"uv2nix": "uv2nix"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1747501237,
|
||||
"narHash": "sha256-woyaUwmZurfNTXBEFM6M7ueSd/Udixs+4DUInhL835c=",
|
||||
"owner": "RichieCahill",
|
||||
"repo": "system_tools",
|
||||
"rev": "68ab5d1c17ac3fe2487f73dbbb4848bd2291139e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "RichieCahill",
|
||||
"repo": "system_tools",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems_2": {
|
||||
"locked": {
|
||||
"lastModified": 1689347949,
|
||||
"narHash": "sha256-12tWmuL2zgBgZkdoB6qXZsgJEH9LR3oUgpaQq2RbI80=",
|
||||
@@ -243,6 +226,31 @@
|
||||
"repo": "default-linux",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"uv2nix": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"system_tools",
|
||||
"nixpkgs"
|
||||
],
|
||||
"pyproject-nix": [
|
||||
"system_tools",
|
||||
"pyproject-nix"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1747441483,
|
||||
"narHash": "sha256-W8BFXk5R0TuJcjIhcGoMpSOaIufGXpizK0pm+uTqynA=",
|
||||
"owner": "pyproject-nix",
|
||||
"repo": "uv2nix",
|
||||
"rev": "582024dc64663e9f88d467c2f7f7b20d278349de",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "pyproject-nix",
|
||||
"repo": "uv2nix",
|
||||
"type": "github"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
|
||||
@@ -21,23 +21,23 @@
|
||||
|
||||
nixos-hardware.url = "github:nixos/nixos-hardware/master";
|
||||
|
||||
jellyswarrm = {
|
||||
url = "github:LLukas22/Jellyswarrm";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
|
||||
home-manager = {
|
||||
url = "github:nix-community/home-manager";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
|
||||
sops-nix = {
|
||||
url = "github:Mic92/sops-nix";
|
||||
firefox-addons = {
|
||||
url = "gitlab:rycee/nur-expressions?dir=pkgs/firefox-addons";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
|
||||
disko = {
|
||||
url = "github:nix-community/disko";
|
||||
system_tools = {
|
||||
url = "github:RichieCahill/system_tools";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
|
||||
sops-nix = {
|
||||
url = "github:Mic92/sops-nix";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
};
|
||||
@@ -59,7 +59,6 @@
|
||||
system:
|
||||
import nixpkgs {
|
||||
inherit system;
|
||||
overlays = builtins.attrValues outputs.overlays;
|
||||
config.allowUnfree = true;
|
||||
}
|
||||
);
|
||||
@@ -70,48 +69,26 @@
|
||||
|
||||
devShells = forEachSystem (pkgs: import ./shell.nix { inherit pkgs; });
|
||||
formatter = forEachSystem (pkgs: pkgs.treefmt);
|
||||
packages = forEachSystem (
|
||||
pkgs:
|
||||
let
|
||||
installer = pkgs.callPackage ./python/installer/package.nix { };
|
||||
installer-nixos = pkgs.callPackage ./python/installer/package.nix { patchElf = false; };
|
||||
in
|
||||
{
|
||||
inherit installer installer-nixos;
|
||||
default = installer;
|
||||
}
|
||||
// lib.optionalAttrs (pkgs.stdenv.hostPlatform.system == "x86_64-linux") {
|
||||
iso = self.nixosConfigurations.iso.config.system.build.isoImage;
|
||||
}
|
||||
);
|
||||
apps = forEachSystem (
|
||||
pkgs:
|
||||
let
|
||||
system = pkgs.stdenv.hostPlatform.system;
|
||||
installer = {
|
||||
type = "app";
|
||||
program = "${self.packages.${system}.installer}/bin/nixos-installer";
|
||||
meta.description = "One-file NixOS ZFS installer.";
|
||||
};
|
||||
in
|
||||
{
|
||||
inherit installer;
|
||||
default = installer;
|
||||
}
|
||||
);
|
||||
|
||||
nixosConfigurations =
|
||||
let
|
||||
hosts = builtins.attrNames (
|
||||
lib.filterAttrs (_: type: type == "directory") (builtins.readDir ./systems)
|
||||
);
|
||||
mkHost =
|
||||
name:
|
||||
lib.nixosSystem {
|
||||
modules = [ ./systems/${name} ];
|
||||
specialArgs = { inherit inputs outputs; };
|
||||
};
|
||||
in
|
||||
lib.genAttrs hosts mkHost;
|
||||
nixosConfigurations = {
|
||||
bob = lib.nixosSystem {
|
||||
modules = [
|
||||
./systems/bob
|
||||
];
|
||||
specialArgs = { inherit inputs outputs; };
|
||||
};
|
||||
jeeves = lib.nixosSystem {
|
||||
modules = [
|
||||
./systems/jeeves
|
||||
];
|
||||
specialArgs = { inherit inputs outputs; };
|
||||
};
|
||||
rhapsody-in-green = lib.nixosSystem {
|
||||
modules = [
|
||||
./systems/rhapsody-in-green
|
||||
];
|
||||
specialArgs = { inherit inputs outputs; };
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
+4
-13
@@ -1,25 +1,16 @@
|
||||
{ inputs, ... }:
|
||||
{
|
||||
# Additional package sets are accessible through `pkgs.<name>`.
|
||||
# When applied, the stable nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.stable'
|
||||
stable = final: _prev: {
|
||||
stable = import inputs.nixpkgs-stable {
|
||||
system = final.stdenv.hostPlatform.system;
|
||||
system = final.system;
|
||||
config.allowUnfree = true;
|
||||
};
|
||||
};
|
||||
|
||||
# When applied, the master nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.master'
|
||||
master = final: _prev: {
|
||||
master = import inputs.nixpkgs-master {
|
||||
system = final.stdenv.hostPlatform.system;
|
||||
config.allowUnfree = true;
|
||||
};
|
||||
};
|
||||
|
||||
# Baseline x86-64 (v1) packages for prebuilt applications that should not
|
||||
# inherit an x86-64-v3 host platform.
|
||||
x86-v1 = final: _prev: {
|
||||
x86-v1 = import inputs.nixpkgs {
|
||||
system = final.stdenv.hostPlatform.system;
|
||||
system = final.system;
|
||||
config.allowUnfree = true;
|
||||
};
|
||||
};
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
[project]
|
||||
name = "system_tools"
|
||||
version = "0.1.0"
|
||||
description = ""
|
||||
authors = [{ name = "Richie Cahill", email = "richie@tmmworkshop.com" }]
|
||||
requires-python = "~=3.14.0"
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
|
||||
[tool.ruff]
|
||||
|
||||
target-version = "py314"
|
||||
|
||||
line-length = 120
|
||||
|
||||
lint.select = ["ALL"]
|
||||
lint.ignore = [
|
||||
"G004", # (PERM) This is a performers nit
|
||||
"COM812", # (TEMP) conflicts when used with the formatter
|
||||
"ISC001", # (TEMP) conflicts when used with the formatter
|
||||
"S603", # (PERM) This is known to cause a false positive
|
||||
"S607", # (PERM) This is becoming a consistent annoyance
|
||||
"CPY001", # (PERM) I don't include the license in every file
|
||||
]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
|
||||
"tests/**" = [
|
||||
"ANN", # (perm) type annotations not needed in tests
|
||||
"D", # (perm) docstrings not needed in tests
|
||||
"PLR2004", # (perm) magic values are fine in test assertions
|
||||
"S101", # (perm) pytest needs asserts
|
||||
]
|
||||
"python/stuff/**" = [
|
||||
"T201", # (perm) I don't care about print statements dir
|
||||
]
|
||||
"python/testing/**" = [
|
||||
"T201", # (perm) I don't care about print statements dir
|
||||
"ERA001", # (perm) I don't care about print statements dir
|
||||
]
|
||||
"python/splendor/**" = [
|
||||
"S311", # (perm) there is no security issue here
|
||||
"T201", # (perm) I don't care about print statements dir
|
||||
"PLR2004", # (temps) need to think about this
|
||||
]
|
||||
"python/orm/**" = [
|
||||
"TC003", # (perm) this creates issues because sqlalchemy uses these at runtime
|
||||
]
|
||||
"python/congress_tracker/**" = [
|
||||
"TC003", # (perm) this creates issues because sqlalchemy uses these at runtime
|
||||
]
|
||||
|
||||
"python/alembic/**" = [
|
||||
"INP001", # (perm) this creates LSP issues for alembic
|
||||
]
|
||||
|
||||
[tool.ruff.lint.pydocstyle]
|
||||
convention = "google"
|
||||
|
||||
[tool.ruff.lint.flake8-builtins]
|
||||
builtins-ignorelist = ["id"]
|
||||
|
||||
[tool.ruff.lint.pylint]
|
||||
max-args = 9
|
||||
|
||||
[tool.coverage.run]
|
||||
source = ["system_tools"]
|
||||
|
||||
[tool.coverage.report]
|
||||
exclude_lines = [
|
||||
"pragma: no cover",
|
||||
"if TYPE_CHECKING:",
|
||||
"raise NotImplementedError",
|
||||
"if __name__ == \"__main__\":",
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
addopts = "-n auto -ra --ignore=tests/ebook_search"
|
||||
asyncio_mode = "auto"
|
||||
testpaths = ["tests"]
|
||||
# --cov=system_tools --cov-report=term-missing --cov-report=xml --cov-report=html --cov-branch
|
||||
@@ -1 +0,0 @@
|
||||
"""Server Tools."""
|
||||
@@ -1,122 +0,0 @@
|
||||
"""Alembic."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Literal
|
||||
|
||||
from alembic import context
|
||||
from alembic.script import write_hooks
|
||||
from sqlalchemy.schema import CreateSchema
|
||||
|
||||
from python.common import bash_wrapper
|
||||
from python.orm.common import get_postgres_engine
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import MutableMapping
|
||||
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
config = context.config
|
||||
|
||||
base_class: type[DeclarativeBase] = config.attributes.get("base")
|
||||
if base_class is None:
|
||||
error = "No base class provided. Use the database CLI to run alembic commands."
|
||||
raise RuntimeError(error)
|
||||
|
||||
target_metadata = base_class.metadata
|
||||
logging.basicConfig(
|
||||
level="DEBUG",
|
||||
datefmt="%Y-%m-%dT%H:%M:%S%z",
|
||||
format="%(asctime)s %(levelname)s %(filename)s:%(lineno)d - %(message)s",
|
||||
handlers=[logging.StreamHandler(sys.stdout)],
|
||||
)
|
||||
|
||||
|
||||
@write_hooks.register("dynamic_schema")
|
||||
def dynamic_schema(filename: str, _options: dict[Any, Any]) -> None:
|
||||
"""Dynamic schema."""
|
||||
original_file = Path(filename).read_text()
|
||||
schema_name = base_class.schema_name
|
||||
dynamic_schema_file_part1 = original_file.replace(f"schema='{schema_name}'", "schema=schema")
|
||||
dynamic_schema_file = dynamic_schema_file_part1.replace(f"'{schema_name}.", "f'{schema}.")
|
||||
Path(filename).write_text(dynamic_schema_file)
|
||||
|
||||
|
||||
@write_hooks.register("import_postgresql")
|
||||
def import_postgresql(filename: str, _options: dict[Any, Any]) -> None:
|
||||
"""Add postgresql dialect import when postgresql types are used."""
|
||||
content = Path(filename).read_text()
|
||||
if "postgresql." in content and "from sqlalchemy.dialects import postgresql" not in content:
|
||||
content = content.replace(
|
||||
"import sqlalchemy as sa\n",
|
||||
"import sqlalchemy as sa\nfrom sqlalchemy.dialects import postgresql\n",
|
||||
)
|
||||
Path(filename).write_text(content)
|
||||
|
||||
|
||||
@write_hooks.register("ruff")
|
||||
def ruff_check_and_format(filename: str, _options: dict[Any, Any]) -> None:
|
||||
"""Docstring for ruff_check_and_format."""
|
||||
bash_wrapper(f"ruff check --fix {filename}")
|
||||
bash_wrapper(f"ruff format {filename}")
|
||||
|
||||
|
||||
def include_name(
|
||||
name: str | None,
|
||||
type_: Literal["schema", "table", "column", "index", "unique_constraint", "foreign_key_constraint"],
|
||||
_parent_names: MutableMapping[Literal["schema_name", "table_name", "schema_qualified_table_name"], str | None],
|
||||
) -> bool:
|
||||
"""Filter tables to be included in the migration.
|
||||
|
||||
Args:
|
||||
name (str): The name of the table.
|
||||
type_ (str): The type of the table.
|
||||
_parent_names (MutableMapping): The names of the parent tables.
|
||||
|
||||
Returns:
|
||||
bool: True if the table should be included, False otherwise.
|
||||
|
||||
"""
|
||||
if type_ == "schema":
|
||||
# allows a database with multiple schemas to have separate alembic revisions
|
||||
return name == target_metadata.schema
|
||||
return True
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations in 'online' mode.
|
||||
|
||||
In this scenario we need to create an Engine
|
||||
and associate a connection with the context.
|
||||
|
||||
"""
|
||||
env_prefix = config.attributes.get("env_prefix", "POSTGRES")
|
||||
connectable = get_postgres_engine(name=env_prefix)
|
||||
|
||||
with connectable.connect() as connection:
|
||||
schema = base_class.schema_name
|
||||
if not connectable.dialect.has_schema(connection, schema):
|
||||
answer = input(f"Schema {schema!r} does not exist. Create it? [y/N] ")
|
||||
if answer.lower() != "y":
|
||||
error = f"Schema {schema!r} does not exist. Exiting."
|
||||
raise SystemExit(error)
|
||||
connection.execute(CreateSchema(schema))
|
||||
connection.commit()
|
||||
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
include_schemas=True,
|
||||
version_table_schema=schema,
|
||||
include_name=include_name,
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
connection.commit()
|
||||
|
||||
|
||||
run_migrations_online()
|
||||
@@ -1,113 +0,0 @@
|
||||
"""created contact api.
|
||||
|
||||
Revision ID: edd7dd61a3d2
|
||||
Revises:
|
||||
Create Date: 2026-01-11 15:45:59.909266
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "edd7dd61a3d2"
|
||||
down_revision: str | None = None
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"contact",
|
||||
sa.Column("name", sa.String(), nullable=False),
|
||||
sa.Column("age", sa.Integer(), nullable=True),
|
||||
sa.Column("bio", sa.String(), nullable=True),
|
||||
sa.Column("current_job", sa.String(), nullable=True),
|
||||
sa.Column("gender", sa.String(), nullable=True),
|
||||
sa.Column("goals", sa.String(), nullable=True),
|
||||
sa.Column("legal_name", sa.String(), nullable=True),
|
||||
sa.Column("profile_pic", sa.String(), nullable=True),
|
||||
sa.Column("safe_conversation_starters", sa.String(), nullable=True),
|
||||
sa.Column("self_sufficiency_score", sa.Integer(), nullable=True),
|
||||
sa.Column("social_structure_style", sa.String(), nullable=True),
|
||||
sa.Column("ssn", sa.String(), nullable=True),
|
||||
sa.Column("suffix", sa.String(), nullable=True),
|
||||
sa.Column("timezone", sa.String(), nullable=True),
|
||||
sa.Column("topics_to_avoid", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_contact")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"need",
|
||||
sa.Column("name", sa.String(), nullable=False),
|
||||
sa.Column("description", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_need")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"contact_need",
|
||||
sa.Column("contact_id", sa.Integer(), nullable=False),
|
||||
sa.Column("need_id", sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["contact_id"],
|
||||
[f"{schema}.contact.id"],
|
||||
name=op.f("fk_contact_need_contact_id_contact"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["need_id"], [f"{schema}.need.id"], name=op.f("fk_contact_need_need_id_need"), ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("contact_id", "need_id", name=op.f("pk_contact_need")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"contact_relationship",
|
||||
sa.Column("contact_id", sa.Integer(), nullable=False),
|
||||
sa.Column("related_contact_id", sa.Integer(), nullable=False),
|
||||
sa.Column("relationship_type", sa.String(length=100), nullable=False),
|
||||
sa.Column("closeness_weight", sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["contact_id"],
|
||||
[f"{schema}.contact.id"],
|
||||
name=op.f("fk_contact_relationship_contact_id_contact"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["related_contact_id"],
|
||||
[f"{schema}.contact.id"],
|
||||
name=op.f("fk_contact_relationship_related_contact_id_contact"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("contact_id", "related_contact_id", name=op.f("pk_contact_relationship")),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("contact_relationship", schema=schema)
|
||||
op.drop_table("contact_need", schema=schema)
|
||||
op.drop_table("need", schema=schema)
|
||||
op.drop_table("contact", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
-135
@@ -1,135 +0,0 @@
|
||||
"""add congress tracker tables.
|
||||
|
||||
Revision ID: 3f71565e38de
|
||||
Revises: edd7dd61a3d2
|
||||
Create Date: 2026-02-12 16:36:09.457303
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "3f71565e38de"
|
||||
down_revision: str | None = "edd7dd61a3d2"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"bill",
|
||||
sa.Column("congress", sa.Integer(), nullable=False),
|
||||
sa.Column("bill_type", sa.String(), nullable=False),
|
||||
sa.Column("number", sa.Integer(), nullable=False),
|
||||
sa.Column("title", sa.String(), nullable=True),
|
||||
sa.Column("title_short", sa.String(), nullable=True),
|
||||
sa.Column("official_title", sa.String(), nullable=True),
|
||||
sa.Column("status", sa.String(), nullable=True),
|
||||
sa.Column("status_at", sa.Date(), nullable=True),
|
||||
sa.Column("sponsor_bioguide_id", sa.String(), nullable=True),
|
||||
sa.Column("subjects_top_term", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_bill")),
|
||||
sa.UniqueConstraint("congress", "bill_type", "number", name="uq_bill_congress_type_number"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index("ix_bill_congress", "bill", ["congress"], unique=False, schema=schema)
|
||||
op.create_table(
|
||||
"legislator",
|
||||
sa.Column("bioguide_id", sa.Text(), nullable=False),
|
||||
sa.Column("thomas_id", sa.String(), nullable=True),
|
||||
sa.Column("lis_id", sa.String(), nullable=True),
|
||||
sa.Column("govtrack_id", sa.Integer(), nullable=True),
|
||||
sa.Column("opensecrets_id", sa.String(), nullable=True),
|
||||
sa.Column("fec_ids", sa.String(), nullable=True),
|
||||
sa.Column("first_name", sa.String(), nullable=False),
|
||||
sa.Column("last_name", sa.String(), nullable=False),
|
||||
sa.Column("official_full_name", sa.String(), nullable=True),
|
||||
sa.Column("nickname", sa.String(), nullable=True),
|
||||
sa.Column("birthday", sa.Date(), nullable=True),
|
||||
sa.Column("gender", sa.String(), nullable=True),
|
||||
sa.Column("current_party", sa.String(), nullable=True),
|
||||
sa.Column("current_state", sa.String(), nullable=True),
|
||||
sa.Column("current_district", sa.Integer(), nullable=True),
|
||||
sa.Column("current_chamber", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_legislator")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(op.f("ix_legislator_bioguide_id"), "legislator", ["bioguide_id"], unique=True, schema=schema)
|
||||
op.create_table(
|
||||
"vote",
|
||||
sa.Column("congress", sa.Integer(), nullable=False),
|
||||
sa.Column("chamber", sa.String(), nullable=False),
|
||||
sa.Column("session", sa.Integer(), nullable=False),
|
||||
sa.Column("number", sa.Integer(), nullable=False),
|
||||
sa.Column("vote_type", sa.String(), nullable=True),
|
||||
sa.Column("question", sa.String(), nullable=True),
|
||||
sa.Column("result", sa.String(), nullable=True),
|
||||
sa.Column("result_text", sa.String(), nullable=True),
|
||||
sa.Column("vote_date", sa.Date(), nullable=False),
|
||||
sa.Column("yea_count", sa.Integer(), nullable=True),
|
||||
sa.Column("nay_count", sa.Integer(), nullable=True),
|
||||
sa.Column("not_voting_count", sa.Integer(), nullable=True),
|
||||
sa.Column("present_count", sa.Integer(), nullable=True),
|
||||
sa.Column("bill_id", sa.Integer(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(["bill_id"], [f"{schema}.bill.id"], name=op.f("fk_vote_bill_id_bill")),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_vote")),
|
||||
sa.UniqueConstraint("congress", "chamber", "session", "number", name="uq_vote_congress_chamber_session_number"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index("ix_vote_congress_chamber", "vote", ["congress", "chamber"], unique=False, schema=schema)
|
||||
op.create_index("ix_vote_date", "vote", ["vote_date"], unique=False, schema=schema)
|
||||
op.create_table(
|
||||
"vote_record",
|
||||
sa.Column("vote_id", sa.Integer(), nullable=False),
|
||||
sa.Column("legislator_id", sa.Integer(), nullable=False),
|
||||
sa.Column("position", sa.String(), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["legislator_id"],
|
||||
[f"{schema}.legislator.id"],
|
||||
name=op.f("fk_vote_record_legislator_id_legislator"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["vote_id"], [f"{schema}.vote.id"], name=op.f("fk_vote_record_vote_id_vote"), ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("vote_id", "legislator_id", name=op.f("pk_vote_record")),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("vote_record", schema=schema)
|
||||
op.drop_index("ix_vote_date", table_name="vote", schema=schema)
|
||||
op.drop_index("ix_vote_congress_chamber", table_name="vote", schema=schema)
|
||||
op.drop_table("vote", schema=schema)
|
||||
op.drop_index(op.f("ix_legislator_bioguide_id"), table_name="legislator", schema=schema)
|
||||
op.drop_table("legislator", schema=schema)
|
||||
op.drop_index("ix_bill_congress", table_name="bill", schema=schema)
|
||||
op.drop_table("bill", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
-58
@@ -1,58 +0,0 @@
|
||||
"""adding SignalDevice for DeviceRegistry for signal bot.
|
||||
|
||||
Revision ID: 4c410c16e39c
|
||||
Revises: 3f71565e38de
|
||||
Create Date: 2026-03-09 14:51:24.228976
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "4c410c16e39c"
|
||||
down_revision: str | None = "3f71565e38de"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"signal_device",
|
||||
sa.Column("phone_number", sa.String(length=50), nullable=False),
|
||||
sa.Column("safety_number", sa.String(), nullable=False),
|
||||
sa.Column(
|
||||
"trust_level",
|
||||
postgresql.ENUM("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", schema=schema),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("last_seen", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_signal_device")),
|
||||
sa.UniqueConstraint("phone_number", name=op.f("uq_signal_device_phone_number")),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("signal_device", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
@@ -1,41 +0,0 @@
|
||||
"""fixed safety number logic.
|
||||
|
||||
Revision ID: 99fec682516c
|
||||
Revises: 4c410c16e39c
|
||||
Create Date: 2026-03-09 16:25:25.085806
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "99fec682516c"
|
||||
down_revision: str | None = "4c410c16e39c"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.alter_column("signal_device", "safety_number", existing_type=sa.VARCHAR(), nullable=True, schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.alter_column("signal_device", "safety_number", existing_type=sa.VARCHAR(), nullable=False, schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
-54
@@ -1,54 +0,0 @@
|
||||
"""add dead_letter_message table.
|
||||
|
||||
Revision ID: a1b2c3d4e5f6
|
||||
Revises: 99fec682516c
|
||||
Create Date: 2026-03-10 12:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "a1b2c3d4e5f6"
|
||||
down_revision: str | None = "99fec682516c"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
op.create_table(
|
||||
"dead_letter_message",
|
||||
sa.Column("source", sa.String(), nullable=False),
|
||||
sa.Column("message", sa.Text(), nullable=False),
|
||||
sa.Column("received_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column(
|
||||
"status",
|
||||
postgresql.ENUM("UNPROCESSED", "PROCESSED", name="message_status", schema=schema),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_dead_letter_message")),
|
||||
schema=schema,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
op.drop_table("dead_letter_message", schema=schema)
|
||||
op.execute(sa.text(f"DROP TYPE IF EXISTS {schema}.message_status"))
|
||||
-66
@@ -1,66 +0,0 @@
|
||||
"""adding roles to signal devices.
|
||||
|
||||
Revision ID: 2ef7ba690159
|
||||
Revises: a1b2c3d4e5f6
|
||||
Create Date: 2026-03-16 19:22:38.020350
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "2ef7ba690159"
|
||||
down_revision: str | None = "a1b2c3d4e5f6"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"role",
|
||||
sa.Column("name", sa.String(length=50), nullable=False),
|
||||
sa.Column("id", sa.SmallInteger(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_role")),
|
||||
sa.UniqueConstraint("name", name=op.f("uq_role_name")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"device_role",
|
||||
sa.Column("device_id", sa.Integer(), nullable=False),
|
||||
sa.Column("role_id", sa.SmallInteger(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["device_id"], [f"{schema}.signal_device.id"], name=op.f("fk_device_role_device_id_signal_device")
|
||||
),
|
||||
sa.ForeignKeyConstraint(["role_id"], [f"{schema}.role.id"], name=op.f("fk_device_role_role_id_role")),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_device_role")),
|
||||
sa.UniqueConstraint("device_id", "role_id", name="uq_device_role_device_role"),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("device_role", schema=schema)
|
||||
op.drop_table("role", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
-171
@@ -1,171 +0,0 @@
|
||||
"""seprating signal_bot database.
|
||||
|
||||
Revision ID: 6b275323f435
|
||||
Revises: 2ef7ba690159
|
||||
Create Date: 2026-03-18 08:34:28.785885
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "6b275323f435"
|
||||
down_revision: str | None = "2ef7ba690159"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("device_role", schema=schema)
|
||||
op.drop_table("signal_device", schema=schema)
|
||||
op.drop_table("role", schema=schema)
|
||||
op.drop_table("dead_letter_message", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"dead_letter_message",
|
||||
sa.Column("source", sa.VARCHAR(), autoincrement=False, nullable=False),
|
||||
sa.Column("message", sa.TEXT(), autoincrement=False, nullable=False),
|
||||
sa.Column("received_at", postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False),
|
||||
sa.Column(
|
||||
"status",
|
||||
postgresql.ENUM("UNPROCESSED", "PROCESSED", name="message_status", schema=schema),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_dead_letter_message")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"role",
|
||||
sa.Column("name", sa.VARCHAR(length=50), autoincrement=False, nullable=False),
|
||||
sa.Column(
|
||||
"id",
|
||||
sa.SMALLINT(),
|
||||
server_default=sa.text(f"nextval('{schema}.role_id_seq'::regclass)"),
|
||||
autoincrement=True,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"created",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_role")),
|
||||
sa.UniqueConstraint(
|
||||
"name", name=op.f("uq_role_name"), postgresql_include=[], postgresql_nulls_not_distinct=False
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"signal_device",
|
||||
sa.Column("phone_number", sa.VARCHAR(length=50), autoincrement=False, nullable=False),
|
||||
sa.Column("safety_number", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column(
|
||||
"trust_level",
|
||||
postgresql.ENUM("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", schema=schema),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("last_seen", postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False),
|
||||
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_signal_device")),
|
||||
sa.UniqueConstraint(
|
||||
"phone_number",
|
||||
name=op.f("uq_signal_device_phone_number"),
|
||||
postgresql_include=[],
|
||||
postgresql_nulls_not_distinct=False,
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"device_role",
|
||||
sa.Column("device_id", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("role_id", sa.SMALLINT(), autoincrement=False, nullable=False),
|
||||
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["device_id"], [f"{schema}.signal_device.id"], name=op.f("fk_device_role_device_id_signal_device")
|
||||
),
|
||||
sa.ForeignKeyConstraint(["role_id"], [f"{schema}.role.id"], name=op.f("fk_device_role_role_id_role")),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_device_role")),
|
||||
sa.UniqueConstraint(
|
||||
"device_id",
|
||||
"role_id",
|
||||
name=op.f("uq_device_role_device_role"),
|
||||
postgresql_include=[],
|
||||
postgresql_nulls_not_distinct=False,
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
-187
@@ -1,187 +0,0 @@
|
||||
"""removed ds table from richie DB.
|
||||
|
||||
Revision ID: c8a794340928
|
||||
Revises: 6b275323f435
|
||||
Create Date: 2026-03-29 15:29:23.643146
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "c8a794340928"
|
||||
down_revision: str | None = "6b275323f435"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("vote_record", schema=schema)
|
||||
op.drop_index(op.f("ix_vote_congress_chamber"), table_name="vote", schema=schema)
|
||||
op.drop_index(op.f("ix_vote_date"), table_name="vote", schema=schema)
|
||||
op.drop_index(op.f("ix_legislator_bioguide_id"), table_name="legislator", schema=schema)
|
||||
op.drop_table("legislator", schema=schema)
|
||||
op.drop_table("vote", schema=schema)
|
||||
op.drop_index(op.f("ix_bill_congress"), table_name="bill", schema=schema)
|
||||
op.drop_table("bill", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"vote",
|
||||
sa.Column("congress", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("chamber", sa.VARCHAR(), autoincrement=False, nullable=False),
|
||||
sa.Column("session", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("number", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("vote_type", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("question", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("result", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("result_text", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("vote_date", sa.DATE(), autoincrement=False, nullable=False),
|
||||
sa.Column("yea_count", sa.INTEGER(), autoincrement=False, nullable=True),
|
||||
sa.Column("nay_count", sa.INTEGER(), autoincrement=False, nullable=True),
|
||||
sa.Column("not_voting_count", sa.INTEGER(), autoincrement=False, nullable=True),
|
||||
sa.Column("present_count", sa.INTEGER(), autoincrement=False, nullable=True),
|
||||
sa.Column("bill_id", sa.INTEGER(), autoincrement=False, nullable=True),
|
||||
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(["bill_id"], [f"{schema}.bill.id"], name=op.f("fk_vote_bill_id_bill")),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_vote")),
|
||||
sa.UniqueConstraint(
|
||||
"congress",
|
||||
"chamber",
|
||||
"session",
|
||||
"number",
|
||||
name=op.f("uq_vote_congress_chamber_session_number"),
|
||||
postgresql_include=[],
|
||||
postgresql_nulls_not_distinct=False,
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(op.f("ix_vote_date"), "vote", ["vote_date"], unique=False, schema=schema)
|
||||
op.create_index(op.f("ix_vote_congress_chamber"), "vote", ["congress", "chamber"], unique=False, schema=schema)
|
||||
op.create_table(
|
||||
"vote_record",
|
||||
sa.Column("vote_id", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("legislator_id", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("position", sa.VARCHAR(), autoincrement=False, nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["legislator_id"],
|
||||
[f"{schema}.legislator.id"],
|
||||
name=op.f("fk_vote_record_legislator_id_legislator"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["vote_id"], [f"{schema}.vote.id"], name=op.f("fk_vote_record_vote_id_vote"), ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("vote_id", "legislator_id", name=op.f("pk_vote_record")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"legislator",
|
||||
sa.Column("bioguide_id", sa.TEXT(), autoincrement=False, nullable=False),
|
||||
sa.Column("thomas_id", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("lis_id", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("govtrack_id", sa.INTEGER(), autoincrement=False, nullable=True),
|
||||
sa.Column("opensecrets_id", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("fec_ids", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("first_name", sa.VARCHAR(), autoincrement=False, nullable=False),
|
||||
sa.Column("last_name", sa.VARCHAR(), autoincrement=False, nullable=False),
|
||||
sa.Column("official_full_name", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("nickname", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("birthday", sa.DATE(), autoincrement=False, nullable=True),
|
||||
sa.Column("gender", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("current_party", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("current_state", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("current_district", sa.INTEGER(), autoincrement=False, nullable=True),
|
||||
sa.Column("current_chamber", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_legislator")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(op.f("ix_legislator_bioguide_id"), "legislator", ["bioguide_id"], unique=True, schema=schema)
|
||||
op.create_table(
|
||||
"bill",
|
||||
sa.Column("congress", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("bill_type", sa.VARCHAR(), autoincrement=False, nullable=False),
|
||||
sa.Column("number", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("title", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("title_short", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("official_title", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("status", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("status_at", sa.DATE(), autoincrement=False, nullable=True),
|
||||
sa.Column("sponsor_bioguide_id", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("subjects_top_term", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_bill")),
|
||||
sa.UniqueConstraint(
|
||||
"congress",
|
||||
"bill_type",
|
||||
"number",
|
||||
name=op.f("uq_bill_congress_type_number"),
|
||||
postgresql_include=[],
|
||||
postgresql_nulls_not_distinct=False,
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(op.f("ix_bill_congress"), "bill", ["congress"], unique=False, schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
-93
@@ -1,93 +0,0 @@
|
||||
"""adding audiobook libreary metadata.
|
||||
|
||||
Revision ID: d7864d1ffc17
|
||||
Revises: c8a794340928
|
||||
Create Date: 2026-06-03 20:24:09.200837
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "d7864d1ffc17"
|
||||
down_revision: str | None = "c8a794340928"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"audiobook_author",
|
||||
sa.Column("name", sa.String(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_audiobook_author")),
|
||||
sa.UniqueConstraint("name", name=op.f("uq_audiobook_author_name")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"audiobook_series",
|
||||
sa.Column("name", sa.String(), nullable=False),
|
||||
sa.Column("author_id", sa.Integer(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["author_id"],
|
||||
[f"{schema}.audiobook_author.id"],
|
||||
name=op.f("fk_audiobook_series_author_id_audiobook_author"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_audiobook_series")),
|
||||
sa.UniqueConstraint("author_id", "name", name=op.f("uq_audiobook_series_author_id")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"audiobook",
|
||||
sa.Column("title", sa.String(), nullable=False),
|
||||
sa.Column("author_id", sa.Integer(), nullable=False),
|
||||
sa.Column("series_id", sa.Integer(), nullable=True),
|
||||
sa.Column("series_index", sa.Integer(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["author_id"],
|
||||
[f"{schema}.audiobook_author.id"],
|
||||
name=op.f("fk_audiobook_author_id_audiobook_author"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["series_id"],
|
||||
[f"{schema}.audiobook_series.id"],
|
||||
name=op.f("fk_audiobook_series_id_audiobook_series"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_audiobook")),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("audiobook", schema=schema)
|
||||
op.drop_table("audiobook_series", schema=schema)
|
||||
op.drop_table("audiobook_author", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
@@ -1,200 +0,0 @@
|
||||
"""add ebook search tables.
|
||||
|
||||
Revision ID: 2db132cace1a
|
||||
Revises: b3c60cc5beb5
|
||||
Create Date: 2026-06-10 22:10:54.379159
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pgvector
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "2db132cace1a"
|
||||
down_revision: str | None = "b3c60cc5beb5"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"ebook_embedding_model",
|
||||
sa.Column("name", sa.String(), nullable=False),
|
||||
sa.Column("dimension", sa.Integer(), nullable=False),
|
||||
sa.Column("is_default", sa.Boolean(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_embedding_model")),
|
||||
sa.UniqueConstraint("name", name=op.f("uq_ebook_embedding_model_name")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"ebook_source",
|
||||
sa.Column("title", sa.String(), nullable=False),
|
||||
sa.Column("author", sa.String(), nullable=True),
|
||||
sa.Column("language", sa.String(), nullable=True),
|
||||
sa.Column("publisher", sa.String(), nullable=True),
|
||||
sa.Column("identifier", sa.String(), nullable=True),
|
||||
sa.Column("file_path", sa.String(), nullable=False),
|
||||
sa.Column("file_sha256", sa.String(length=64), nullable=False),
|
||||
sa.Column("file_mtime", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("file_size", sa.BigInteger(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_source")),
|
||||
sa.UniqueConstraint("file_path", name=op.f("uq_ebook_source_file_path")),
|
||||
sa.UniqueConstraint("file_sha256", name=op.f("uq_ebook_source_file_sha256")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"ebook_chapter",
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("spine_index", sa.Integer(), nullable=False),
|
||||
sa.Column("title", sa.String(), nullable=True),
|
||||
sa.Column("href", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"],
|
||||
[f"{schema}.ebook_source.id"],
|
||||
name=op.f("fk_ebook_chapter_source_id_ebook_source"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chapter")),
|
||||
sa.UniqueConstraint("source_id", "spine_index", name=op.f("uq_ebook_chapter_source_id")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"ebook_chunk",
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("chapter_id", sa.Integer(), nullable=True),
|
||||
sa.Column("chunk_index", sa.Integer(), nullable=False),
|
||||
sa.Column("text", sa.String(), nullable=False),
|
||||
sa.Column("token_start", sa.Integer(), nullable=False),
|
||||
sa.Column("token_count", sa.Integer(), nullable=False),
|
||||
sa.Column("page_label", sa.String(), nullable=True),
|
||||
sa.Column("content_sha256", sa.String(length=64), nullable=False),
|
||||
sa.Column("search_text", sa.String(), nullable=False),
|
||||
sa.Column("id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["chapter_id"],
|
||||
[f"{schema}.ebook_chapter.id"],
|
||||
name=op.f("fk_ebook_chunk_chapter_id_ebook_chapter"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"],
|
||||
[f"{schema}.ebook_source.id"],
|
||||
name=op.f("fk_ebook_chunk_source_id_ebook_source"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk")),
|
||||
sa.UniqueConstraint("source_id", "chunk_index", name="uq_ebook_chunk_source_id_chunk_index"),
|
||||
sa.UniqueConstraint("source_id", "content_sha256", name="uq_ebook_chunk_source_id_content_sha256"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"ebook_chunk_embedding_1024",
|
||||
sa.Column("chunk_id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("model_id", sa.Integer(), nullable=False),
|
||||
sa.Column("embedding", pgvector.sqlalchemy.vector.VECTOR(dim=1024), nullable=False),
|
||||
sa.Column("id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["chunk_id"],
|
||||
[f"{schema}.ebook_chunk.id"],
|
||||
name=op.f("fk_ebook_chunk_embedding_1024_chunk_id_ebook_chunk"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["model_id"],
|
||||
[f"{schema}.ebook_embedding_model.id"],
|
||||
name=op.f("fk_ebook_chunk_embedding_1024_model_id_ebook_embedding_model"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk_embedding_1024")),
|
||||
sa.UniqueConstraint("chunk_id", "model_id", name=op.f("uq_ebook_chunk_embedding_1024_chunk_id")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"ebook_chunk_embedding_2560",
|
||||
sa.Column("chunk_id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("model_id", sa.Integer(), nullable=False),
|
||||
sa.Column("embedding", pgvector.sqlalchemy.vector.VECTOR(dim=2560), nullable=False),
|
||||
sa.Column("id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["chunk_id"],
|
||||
[f"{schema}.ebook_chunk.id"],
|
||||
name=op.f("fk_ebook_chunk_embedding_2560_chunk_id_ebook_chunk"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["model_id"],
|
||||
[f"{schema}.ebook_embedding_model.id"],
|
||||
name=op.f("fk_ebook_chunk_embedding_2560_model_id_ebook_embedding_model"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk_embedding_2560")),
|
||||
sa.UniqueConstraint("chunk_id", "model_id", name=op.f("uq_ebook_chunk_embedding_2560_chunk_id")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"ebook_chunk_embedding_4096",
|
||||
sa.Column("chunk_id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("model_id", sa.Integer(), nullable=False),
|
||||
sa.Column("embedding", pgvector.sqlalchemy.vector.VECTOR(dim=4096), nullable=False),
|
||||
sa.Column("id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["chunk_id"],
|
||||
[f"{schema}.ebook_chunk.id"],
|
||||
name=op.f("fk_ebook_chunk_embedding_4096_chunk_id_ebook_chunk"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["model_id"],
|
||||
[f"{schema}.ebook_embedding_model.id"],
|
||||
name=op.f("fk_ebook_chunk_embedding_4096_model_id_ebook_embedding_model"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk_embedding_4096")),
|
||||
sa.UniqueConstraint("chunk_id", "model_id", name=op.f("uq_ebook_chunk_embedding_4096_chunk_id")),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("ebook_chunk_embedding_4096", schema=schema)
|
||||
op.drop_table("ebook_chunk_embedding_2560", schema=schema)
|
||||
op.drop_table("ebook_chunk_embedding_1024", schema=schema)
|
||||
op.drop_table("ebook_chunk", schema=schema)
|
||||
op.drop_table("ebook_chapter", schema=schema)
|
||||
op.drop_table("ebook_source", schema=schema)
|
||||
op.drop_table("ebook_embedding_model", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
-63
@@ -1,63 +0,0 @@
|
||||
"""updated series_index to float and added UniqueConstraint to audiobook and audiobook_author.
|
||||
|
||||
Revision ID: b3c60cc5beb5
|
||||
Revises: d7864d1ffc17
|
||||
Create Date: 2026-06-10 20:02:43.073725
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "b3c60cc5beb5"
|
||||
down_revision: str | None = "d7864d1ffc17"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.alter_column(
|
||||
"audiobook",
|
||||
"series_index",
|
||||
existing_type=sa.INTEGER(),
|
||||
type_=sa.Float(),
|
||||
existing_nullable=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_unique_constraint(
|
||||
op.f("uq_audiobook_author_id"),
|
||||
"audiobook",
|
||||
["author_id", "series_id", "title"],
|
||||
schema=schema,
|
||||
postgresql_nulls_not_distinct=True,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_constraint(op.f("uq_audiobook_author_id"), "audiobook", schema=schema, type_="unique")
|
||||
op.alter_column(
|
||||
"audiobook",
|
||||
"series_index",
|
||||
existing_type=sa.Float(),
|
||||
type_=sa.INTEGER(),
|
||||
existing_nullable=False,
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
-54
@@ -1,54 +0,0 @@
|
||||
"""add 1024 ebook embedding cosine index.
|
||||
|
||||
Revision ID: c460105682d2
|
||||
Revises: 2db132cace1a
|
||||
Create Date: 2026-06-13 19:53:45.680289
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "c460105682d2"
|
||||
down_revision: str | None = "2db132cace1a"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_index(
|
||||
"ix_ebook_chunk_embedding_1024_embedding_cosine",
|
||||
"ebook_chunk_embedding_1024",
|
||||
["embedding"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
postgresql_using="hnsw",
|
||||
postgresql_ops={"embedding": "vector_cosine_ops"},
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index(
|
||||
"ix_ebook_chunk_embedding_1024_embedding_cosine",
|
||||
table_name="ebook_chunk_embedding_1024",
|
||||
schema=schema,
|
||||
postgresql_using="hnsw",
|
||||
postgresql_ops={"embedding": "vector_cosine_ops"},
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
@@ -1,103 +0,0 @@
|
||||
"""adding haproxy data.
|
||||
|
||||
Revision ID: 96d72c748c24
|
||||
Revises: c460105682d2
|
||||
Create Date: 2026-06-23 16:37:17.768851
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "96d72c748c24"
|
||||
down_revision: str | None = "c460105682d2"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"haproxy_request",
|
||||
sa.Column("line_hash", sa.String(), nullable=False),
|
||||
sa.Column("requested_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("client_ip", sa.String(), nullable=False),
|
||||
sa.Column("client_port", sa.Integer(), nullable=False),
|
||||
sa.Column("frontend", sa.String(), nullable=False),
|
||||
sa.Column("ssl", sa.Boolean(), nullable=False),
|
||||
sa.Column("backend", sa.String(), nullable=False),
|
||||
sa.Column("server", sa.String(), nullable=False),
|
||||
sa.Column("time_request", sa.Integer(), nullable=False),
|
||||
sa.Column("time_queue", sa.Integer(), nullable=False),
|
||||
sa.Column("time_connect", sa.Integer(), nullable=False),
|
||||
sa.Column("time_response", sa.Integer(), nullable=False),
|
||||
sa.Column("time_total", sa.Integer(), nullable=False),
|
||||
sa.Column("status_code", sa.Integer(), nullable=False),
|
||||
sa.Column("bytes_read", sa.BigInteger(), nullable=False),
|
||||
sa.Column("termination_state", sa.String(), nullable=False),
|
||||
sa.Column("active_connections", sa.Integer(), nullable=False),
|
||||
sa.Column("frontend_connections", sa.Integer(), nullable=False),
|
||||
sa.Column("backend_connections", sa.Integer(), nullable=False),
|
||||
sa.Column("server_connections", sa.Integer(), nullable=False),
|
||||
sa.Column("retries", sa.Integer(), nullable=False),
|
||||
sa.Column("server_queue", sa.Integer(), nullable=False),
|
||||
sa.Column("backend_queue", sa.Integer(), nullable=False),
|
||||
sa.Column("host", sa.String(), nullable=True),
|
||||
sa.Column("user_agent", sa.String(), nullable=True),
|
||||
sa.Column("method", sa.String(), nullable=False),
|
||||
sa.Column("target", sa.String(), nullable=False),
|
||||
sa.Column("path", sa.String(), nullable=False),
|
||||
sa.Column("query", sa.String(), nullable=True),
|
||||
sa.Column("http_version", sa.String(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_haproxy_request")),
|
||||
sa.UniqueConstraint("line_hash", name=op.f("uq_haproxy_request_line_hash")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(op.f("ix_haproxy_request_backend"), "haproxy_request", ["backend"], unique=False, schema=schema)
|
||||
op.create_index(op.f("ix_haproxy_request_client_ip"), "haproxy_request", ["client_ip"], unique=False, schema=schema)
|
||||
op.create_index(op.f("ix_haproxy_request_host"), "haproxy_request", ["host"], unique=False, schema=schema)
|
||||
op.create_index(op.f("ix_haproxy_request_path"), "haproxy_request", ["path"], unique=False, schema=schema)
|
||||
op.create_index(
|
||||
op.f("ix_haproxy_request_requested_at"), "haproxy_request", ["requested_at"], unique=False, schema=schema
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_haproxy_request_status_code"), "haproxy_request", ["status_code"], unique=False, schema=schema
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_haproxy_request_time_response"), "haproxy_request", ["time_response"], unique=False, schema=schema
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_haproxy_request_user_agent"), "haproxy_request", ["user_agent"], unique=False, schema=schema
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index(op.f("ix_haproxy_request_user_agent"), table_name="haproxy_request", schema=schema)
|
||||
op.drop_index(op.f("ix_haproxy_request_time_response"), table_name="haproxy_request", schema=schema)
|
||||
op.drop_index(op.f("ix_haproxy_request_status_code"), table_name="haproxy_request", schema=schema)
|
||||
op.drop_index(op.f("ix_haproxy_request_requested_at"), table_name="haproxy_request", schema=schema)
|
||||
op.drop_index(op.f("ix_haproxy_request_path"), table_name="haproxy_request", schema=schema)
|
||||
op.drop_index(op.f("ix_haproxy_request_host"), table_name="haproxy_request", schema=schema)
|
||||
op.drop_index(op.f("ix_haproxy_request_client_ip"), table_name="haproxy_request", schema=schema)
|
||||
op.drop_index(op.f("ix_haproxy_request_backend"), table_name="haproxy_request", schema=schema)
|
||||
op.drop_table("haproxy_request", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
-206
@@ -1,206 +0,0 @@
|
||||
"""adding Phrase metadata tables.
|
||||
|
||||
Revision ID: dddee09eddcc
|
||||
Revises: 96d72c748c24
|
||||
Create Date: 2026-06-29 00:49:07.344159
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "dddee09eddcc"
|
||||
down_revision: str | None = "96d72c748c24"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"candidate_phrases",
|
||||
sa.Column("book_id", sa.Integer(), nullable=False),
|
||||
sa.Column("series_id", sa.Integer(), nullable=True),
|
||||
sa.Column("phrase_text", sa.Text(), nullable=False),
|
||||
sa.Column("phrase_norm", sa.Text(), nullable=False),
|
||||
sa.Column("token_count", sa.Integer(), nullable=False),
|
||||
sa.Column("source_raw_ngram", sa.Boolean(), nullable=False),
|
||||
sa.Column("source_yake", sa.Boolean(), nullable=False),
|
||||
sa.Column("source_spacy_ner", sa.Boolean(), nullable=False),
|
||||
sa.Column("source_spacy_noun_chunk", sa.Boolean(), nullable=False),
|
||||
sa.Column("source_capitalized", sa.Boolean(), nullable=False),
|
||||
sa.Column("source_metadata", sa.Boolean(), nullable=False),
|
||||
sa.Column("spacy_label", sa.String(), nullable=True),
|
||||
sa.Column("raw_count", sa.Integer(), nullable=False),
|
||||
sa.Column("chapter_count", sa.Integer(), nullable=False),
|
||||
sa.Column("yake_score", sa.Float(), nullable=True),
|
||||
sa.Column("candidate_score", sa.Float(), nullable=False),
|
||||
sa.Column(
|
||||
"sample_contexts",
|
||||
sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), "postgresql"),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column("llm_judged", sa.Boolean(), nullable=False),
|
||||
sa.Column("llm_keep", sa.Boolean(), nullable=True),
|
||||
sa.Column("llm_confidence", sa.Float(), nullable=True),
|
||||
sa.Column("llm_category", sa.String(), nullable=True),
|
||||
sa.Column("llm_reason", sa.Text(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["book_id"],
|
||||
[f"{schema}.ebook_source.id"],
|
||||
name=op.f("fk_candidate_phrases_book_id_ebook_source"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_candidate_phrases")),
|
||||
sa.UniqueConstraint("book_id", "phrase_norm", name="uq_candidate_phrases_book_id_phrase_norm"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"candidate_phrases_book_norm_idx", "candidate_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
|
||||
)
|
||||
op.create_index(
|
||||
"candidate_phrases_book_score_idx",
|
||||
"candidate_phrases",
|
||||
["book_id", "candidate_score"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"protected_phrases",
|
||||
sa.Column("book_id", sa.Integer(), nullable=True),
|
||||
sa.Column("series_id", sa.Integer(), nullable=True),
|
||||
sa.Column("phrase_text", sa.Text(), nullable=False),
|
||||
sa.Column("phrase_norm", sa.Text(), nullable=False),
|
||||
sa.Column("canonical_id", sa.String(), nullable=False),
|
||||
sa.Column("phrase_type", sa.String(), nullable=True),
|
||||
sa.Column("token_count", sa.Integer(), nullable=False),
|
||||
sa.Column("confidence", sa.Float(), nullable=False),
|
||||
sa.Column("importance", sa.Float(), nullable=False),
|
||||
sa.Column("allow_nested", sa.Boolean(), nullable=False),
|
||||
sa.Column("suppress_children", sa.Boolean(), nullable=False),
|
||||
sa.Column("source_candidate_id", sa.Integer(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["book_id"],
|
||||
[f"{schema}.ebook_source.id"],
|
||||
name=op.f("fk_protected_phrases_book_id_ebook_source"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_candidate_id"],
|
||||
[f"{schema}.candidate_phrases.id"],
|
||||
name=op.f("fk_protected_phrases_source_candidate_id_candidate_phrases"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_protected_phrases")),
|
||||
sa.UniqueConstraint("book_id", "phrase_norm", name="uq_protected_phrases_book_id_phrase_norm"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"protected_phrases_book_norm_idx", "protected_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
|
||||
)
|
||||
op.create_index("protected_phrases_norm_idx", "protected_phrases", ["phrase_norm"], unique=False, schema=schema)
|
||||
op.create_index(
|
||||
"protected_phrases_series_norm_idx",
|
||||
"protected_phrases",
|
||||
["series_id", "phrase_norm"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"chunk_phrase_mentions",
|
||||
sa.Column("chunk_id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("phrase_id", sa.Integer(), nullable=False),
|
||||
sa.Column("book_id", sa.Integer(), nullable=True),
|
||||
sa.Column("series_id", sa.Integer(), nullable=True),
|
||||
sa.Column("start_char", sa.Integer(), nullable=False),
|
||||
sa.Column("end_char", sa.Integer(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["book_id"],
|
||||
[f"{schema}.ebook_source.id"],
|
||||
name=op.f("fk_chunk_phrase_mentions_book_id_ebook_source"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["chunk_id"],
|
||||
[f"{schema}.ebook_chunk.id"],
|
||||
name=op.f("fk_chunk_phrase_mentions_chunk_id_ebook_chunk"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["phrase_id"],
|
||||
[f"{schema}.protected_phrases.id"],
|
||||
name=op.f("fk_chunk_phrase_mentions_phrase_id_protected_phrases"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_chunk_phrase_mentions")),
|
||||
sa.UniqueConstraint("chunk_id", "phrase_id", "start_char", name="uq_chunk_phrase_mentions_chunk_phrase_start"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"chunk_phrase_mentions_chunk_idx", "chunk_phrase_mentions", ["chunk_id"], unique=False, schema=schema
|
||||
)
|
||||
op.create_index(
|
||||
"chunk_phrase_mentions_phrase_idx", "chunk_phrase_mentions", ["phrase_id"], unique=False, schema=schema
|
||||
)
|
||||
op.create_table(
|
||||
"phrase_aliases",
|
||||
sa.Column("phrase_id", sa.Integer(), nullable=False),
|
||||
sa.Column("alias_text", sa.Text(), nullable=False),
|
||||
sa.Column("alias_norm", sa.Text(), nullable=False),
|
||||
sa.Column("confidence", sa.Float(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["phrase_id"],
|
||||
[f"{schema}.protected_phrases.id"],
|
||||
name=op.f("fk_phrase_aliases_phrase_id_protected_phrases"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_phrase_aliases")),
|
||||
sa.UniqueConstraint("phrase_id", "alias_norm", name="uq_phrase_aliases_phrase_id_alias_norm"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index("phrase_aliases_norm_idx", "phrase_aliases", ["alias_norm"], unique=False, schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index("phrase_aliases_norm_idx", table_name="phrase_aliases", schema=schema)
|
||||
op.drop_table("phrase_aliases", schema=schema)
|
||||
op.drop_index("chunk_phrase_mentions_phrase_idx", table_name="chunk_phrase_mentions", schema=schema)
|
||||
op.drop_index("chunk_phrase_mentions_chunk_idx", table_name="chunk_phrase_mentions", schema=schema)
|
||||
op.drop_table("chunk_phrase_mentions", schema=schema)
|
||||
op.drop_index("protected_phrases_series_norm_idx", table_name="protected_phrases", schema=schema)
|
||||
op.drop_index("protected_phrases_norm_idx", table_name="protected_phrases", schema=schema)
|
||||
op.drop_index("protected_phrases_book_norm_idx", table_name="protected_phrases", schema=schema)
|
||||
op.drop_table("protected_phrases", schema=schema)
|
||||
op.drop_index("candidate_phrases_book_score_idx", table_name="candidate_phrases", schema=schema)
|
||||
op.drop_index("candidate_phrases_book_norm_idx", table_name="candidate_phrases", schema=schema)
|
||||
op.drop_table("candidate_phrases", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
@@ -1,55 +0,0 @@
|
||||
"""remove spaCy-ner.
|
||||
|
||||
Revision ID: 751260fc3228
|
||||
Revises: dddee09eddcc
|
||||
Create Date: 2026-07-09 23:03:39.554083
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "751260fc3228"
|
||||
down_revision: str | None = "dddee09eddcc"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_column("candidate_phrases", "source_spacy_noun_chunk", schema=schema)
|
||||
op.drop_column("candidate_phrases", "source_spacy_ner", schema=schema)
|
||||
op.drop_column("candidate_phrases", "spacy_label", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.add_column(
|
||||
"candidate_phrases", sa.Column("spacy_label", sa.VARCHAR(), autoincrement=False, nullable=True), schema=schema
|
||||
)
|
||||
op.add_column(
|
||||
"candidate_phrases",
|
||||
sa.Column("source_spacy_ner", sa.BOOLEAN(), autoincrement=False, nullable=False),
|
||||
schema=schema,
|
||||
)
|
||||
op.add_column(
|
||||
"candidate_phrases",
|
||||
sa.Column("source_spacy_noun_chunk", sa.BOOLEAN(), autoincrement=False, nullable=False),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
@@ -1,36 +0,0 @@
|
||||
"""${message}.
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
from python.orm import ${config.attributes["base"].__name__}
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: str | None = ${repr(down_revision)}
|
||||
branch_labels: str | Sequence[str] | None = ${repr(branch_labels)}
|
||||
depends_on: str | Sequence[str] | None = ${repr(depends_on)}
|
||||
|
||||
schema=${config.attributes["base"].__name__}.schema_name
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
${downgrades if downgrades else "pass"}
|
||||
-80
@@ -1,80 +0,0 @@
|
||||
"""starting van invintory.
|
||||
|
||||
Revision ID: 15e733499804
|
||||
Revises:
|
||||
Create Date: 2026-03-08 00:18:20.759720
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import VanInventoryBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "15e733499804"
|
||||
down_revision: str | None = None
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = VanInventoryBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"items",
|
||||
sa.Column("name", sa.String(), nullable=False),
|
||||
sa.Column("quantity", sa.Float(), nullable=False),
|
||||
sa.Column("unit", sa.String(), nullable=False),
|
||||
sa.Column("category", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_items")),
|
||||
sa.UniqueConstraint("name", name=op.f("uq_items_name")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"meals",
|
||||
sa.Column("name", sa.String(), nullable=False),
|
||||
sa.Column("instructions", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_meals")),
|
||||
sa.UniqueConstraint("name", name=op.f("uq_meals_name")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"meal_ingredients",
|
||||
sa.Column("meal_id", sa.Integer(), nullable=False),
|
||||
sa.Column("item_id", sa.Integer(), nullable=False),
|
||||
sa.Column("quantity_needed", sa.Float(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(["item_id"], [f"{schema}.items.id"], name=op.f("fk_meal_ingredients_item_id_items")),
|
||||
sa.ForeignKeyConstraint(["meal_id"], [f"{schema}.meals.id"], name=op.f("fk_meal_ingredients_meal_id_meals")),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_meal_ingredients")),
|
||||
sa.UniqueConstraint("meal_id", "item_id", name=op.f("uq_meal_ingredients_meal_id")),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("meal_ingredients", schema=schema)
|
||||
op.drop_table("meals", schema=schema)
|
||||
op.drop_table("items", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
@@ -1,47 +0,0 @@
|
||||
"""common."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from subprocess import PIPE, Popen
|
||||
|
||||
from python.logging_config import configure_logger as _configure_logger
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_repo_dir() -> Path:
|
||||
"""Return the repository root directory."""
|
||||
return Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def configure_logger(level: str = "INFO") -> None:
|
||||
"""Configure the logger."""
|
||||
_configure_logger(level)
|
||||
|
||||
|
||||
def bash_wrapper(command: str) -> tuple[str, int]:
|
||||
"""Execute a bash command and capture the output.
|
||||
|
||||
Args:
|
||||
command (str): The bash command to be executed.
|
||||
|
||||
Returns:
|
||||
Tuple[str, int]: A tuple containing the output of the command (stdout) as a string,
|
||||
the error output (stderr) as a string (optional), and the return code as an integer.
|
||||
"""
|
||||
# This is a acceptable risk
|
||||
process = Popen(command.split(), stdout=PIPE, stderr=PIPE)
|
||||
output, error = process.communicate()
|
||||
if error:
|
||||
logger.error(f"{error=}")
|
||||
return error.decode(), process.returncode
|
||||
|
||||
return output.decode(), process.returncode
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
"""Get the current UTC time."""
|
||||
return datetime.now(tz=UTC)
|
||||
@@ -1,59 +0,0 @@
|
||||
"""database."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy.exc import NoInspectionAvailable
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def safe_insert(orm_objects: Sequence[object], session: Session) -> list[tuple[Exception, object]]:
|
||||
"""Safer insert at allows for partial rollbacks.
|
||||
|
||||
Args:
|
||||
orm_objects (Sequence[object]): Tables to insert.
|
||||
session (Session): Database session.
|
||||
"""
|
||||
if unmapped := [orm_object for orm_object in orm_objects if not _is_mapped_instance(orm_object)]:
|
||||
error = f"safe_insert expects ORM-mapped instances {unmapped}"
|
||||
raise TypeError(error)
|
||||
return _safe_insert(orm_objects, session)
|
||||
|
||||
|
||||
def _safe_insert(objects: Sequence[object], session: Session) -> list[tuple[Exception, object]]:
|
||||
exceptions: list[tuple[Exception, object]] = []
|
||||
try:
|
||||
session.add_all(objects)
|
||||
session.commit()
|
||||
|
||||
except Exception as error:
|
||||
session.rollback()
|
||||
|
||||
objects_len = len(objects)
|
||||
if objects_len == 1:
|
||||
logger.exception(objects)
|
||||
return [(error, objects[0])]
|
||||
|
||||
middle = objects_len // 2
|
||||
exceptions.extend(_safe_insert(objects=objects[:middle], session=session))
|
||||
exceptions.extend(_safe_insert(objects=objects[middle:], session=session))
|
||||
return exceptions
|
||||
|
||||
|
||||
def _is_mapped_instance(obj: object) -> bool:
|
||||
"""Return True if `obj` is a SQLAlchemy ORM-mapped instance."""
|
||||
try:
|
||||
inspect(obj) # raises NoInspectionAvailable if not mapped
|
||||
except NoInspectionAvailable:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
@@ -1,103 +0,0 @@
|
||||
"""CLI wrapper around alembic for multi-database support.
|
||||
|
||||
Usage:
|
||||
database <db_name> <command> [args...]
|
||||
|
||||
Examples:
|
||||
database richie check
|
||||
database richie upgrade head
|
||||
database richie downgrade head-1
|
||||
database richie revision --autogenerate -m "add meals table"
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from importlib import import_module
|
||||
from typing import TYPE_CHECKING, Annotated
|
||||
|
||||
import typer
|
||||
from alembic.config import CommandLine, Config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatabaseConfig:
|
||||
"""Configuration for a database."""
|
||||
|
||||
env_prefix: str
|
||||
version_location: str
|
||||
base_module: str
|
||||
base_class_name: str
|
||||
models_module: str
|
||||
script_location: str = "python/alembic"
|
||||
file_template: str = "%%(year)d_%%(month).2d_%%(day).2d-%%(slug)s_%%(rev)s"
|
||||
|
||||
def get_base(self) -> type[DeclarativeBase]:
|
||||
"""Import and return the Base class."""
|
||||
module = import_module(self.base_module)
|
||||
return getattr(module, self.base_class_name)
|
||||
|
||||
def import_models(self) -> None:
|
||||
"""Import ORM models so alembic autogenerate can detect them."""
|
||||
import_module(self.models_module)
|
||||
|
||||
def alembic_config(self) -> Config:
|
||||
"""Build an alembic Config for this database."""
|
||||
cfg = Config()
|
||||
cfg.set_main_option("script_location", self.script_location)
|
||||
cfg.set_main_option("file_template", self.file_template)
|
||||
cfg.set_main_option("prepend_sys_path", ".")
|
||||
cfg.set_main_option("version_path_separator", "os")
|
||||
cfg.set_main_option("version_locations", self.version_location)
|
||||
cfg.set_main_option("revision_environment", "true")
|
||||
cfg.set_section_option("post_write_hooks", "hooks", "dynamic_schema,import_postgresql,ruff")
|
||||
cfg.set_section_option("post_write_hooks", "dynamic_schema.type", "dynamic_schema")
|
||||
cfg.set_section_option("post_write_hooks", "import_postgresql.type", "import_postgresql")
|
||||
cfg.set_section_option("post_write_hooks", "ruff.type", "ruff")
|
||||
cfg.attributes["base"] = self.get_base()
|
||||
cfg.attributes["env_prefix"] = self.env_prefix
|
||||
self.import_models()
|
||||
return cfg
|
||||
|
||||
|
||||
DATABASES: dict[str, DatabaseConfig] = {
|
||||
"richie": DatabaseConfig(
|
||||
env_prefix="RICHIE",
|
||||
version_location="python/alembic/richie/versions",
|
||||
base_module="python.orm.richie.base",
|
||||
base_class_name="RichieBase",
|
||||
models_module="python.orm.richie",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
app = typer.Typer(help="Multi-database alembic wrapper.")
|
||||
|
||||
|
||||
@app.command(
|
||||
context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
|
||||
)
|
||||
def main(
|
||||
ctx: typer.Context,
|
||||
db_name: Annotated[str, typer.Argument(help=f"Database name. Options: {', '.join(DATABASES)}")],
|
||||
command: Annotated[str, typer.Argument(help="Alembic command (upgrade, downgrade, revision, check, etc.)")],
|
||||
) -> None:
|
||||
"""Run an alembic command against the specified database."""
|
||||
db_config = DATABASES.get(db_name)
|
||||
if not db_config:
|
||||
typer.echo(f"Unknown database: {db_name!r}. Available: {', '.join(DATABASES)}", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
alembic_cfg = db_config.alembic_config()
|
||||
|
||||
cmd_line = CommandLine()
|
||||
options = cmd_line.parser.parse_args([command, *ctx.args])
|
||||
cmd_line.run_cmd(alembic_cfg, options)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
"""EPUB search package."""
|
||||
@@ -1,50 +0,0 @@
|
||||
"""Grounded answer generation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from python.ebook_search.llm_interface import request_chat_completion
|
||||
from python.ebook_search.prompts import load_prompt
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import httpx
|
||||
|
||||
from python.ebook_search.config import EbookSearchConfig
|
||||
from python.ebook_search.search import SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def answer_query(
|
||||
client: httpx.AsyncClient,
|
||||
query: str,
|
||||
results: list[SearchResult],
|
||||
config: EbookSearchConfig,
|
||||
) -> str:
|
||||
"""Answer a question using only retrieved chunks."""
|
||||
if not config.answer_enabled:
|
||||
logger.info("ebook_answer_skipped_disabled")
|
||||
return "Answer generation is disabled. Source chunks are shown below."
|
||||
|
||||
if not results:
|
||||
logger.info("ebook_answer_skipped_no_results")
|
||||
return "No relevant sources were found."
|
||||
|
||||
logger.info(
|
||||
f"ebook_answer_request_start {config.vllm_base_url=} {config.chat_model=} sources={len(results)} "
|
||||
f"query_length={len(query)}"
|
||||
)
|
||||
context = "\n\n".join(
|
||||
f"[{index}] {result.source_title}{' - ' + result.chapter_title if result.chapter_title else ''}\n{result.text}"
|
||||
for index, result in enumerate(results, start=1)
|
||||
)
|
||||
content = await request_chat_completion(
|
||||
client,
|
||||
config,
|
||||
load_prompt("answer").messages(query=query, context=context),
|
||||
)
|
||||
|
||||
logger.info(f"ebook_answer_request_complete {config.chat_model=} answer_length={len(content)}")
|
||||
return content or "The model returned an empty answer."
|
||||
@@ -1 +0,0 @@
|
||||
"""Web and external API adapters for EPUB search."""
|
||||
@@ -1,70 +0,0 @@
|
||||
"""Background BM25 refresh tasks for the web app.
|
||||
|
||||
The refresh is scheduled on the event loop instead of a thread because the async psycopg
|
||||
driver only works from the loop; a bare thread cannot open a session on the async engine.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from python.ebook_search.bm25_corpus import load_bm25_corpus, refresh_bm25_corpus
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import FastAPI
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
|
||||
from python.ebook_search.config import EbookSearchConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def schedule_bm25_refresh(app: FastAPI) -> None:
|
||||
"""Schedule a delayed BM25 corpus refresh, replacing any pending refresh.
|
||||
|
||||
Only called from route handlers, so a running event loop is guaranteed.
|
||||
"""
|
||||
cancel_bm25_refresh(app)
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def start_refresh() -> None:
|
||||
app.state.bm25_refresh_task = loop.create_task(refresh_bm25_for_app(app))
|
||||
|
||||
app.state.bm25_refresh_timer = loop.call_later(app.state.config.bm25_refresh_delay_seconds, start_refresh)
|
||||
logger.info(f"ebook_bm25_refresh_scheduled {app.state.config.bm25_refresh_delay_seconds=}")
|
||||
|
||||
|
||||
def cancel_bm25_refresh(app: FastAPI) -> None:
|
||||
"""Cancel any pending BM25 corpus refresh timer and in-flight refresh task."""
|
||||
existing_timer = getattr(app.state, "bm25_refresh_timer", None)
|
||||
if existing_timer is not None:
|
||||
existing_timer.cancel()
|
||||
app.state.bm25_refresh_timer = None
|
||||
logger.info("ebook_bm25_refresh_cancelled")
|
||||
|
||||
existing_task = getattr(app.state, "bm25_refresh_task", None)
|
||||
if existing_task is not None:
|
||||
if not existing_task.done():
|
||||
existing_task.cancel()
|
||||
app.state.bm25_refresh_task = None
|
||||
|
||||
|
||||
async def refresh_bm25_for_app(app: FastAPI) -> None:
|
||||
"""Refresh the BM25 corpus using the app engine and config."""
|
||||
try:
|
||||
await refresh_bm25_for_engine(app.state.engine, app.state.config)
|
||||
except Exception:
|
||||
logger.exception("ebook_bm25_refresh_failed")
|
||||
|
||||
|
||||
async def refresh_bm25_for_engine(engine: AsyncEngine, config: EbookSearchConfig) -> None:
|
||||
"""Refresh the BM25 corpus using an async SQLAlchemy engine."""
|
||||
async with AsyncSession(engine) as session:
|
||||
await refresh_bm25_corpus(session, config)
|
||||
load_bm25_corpus.cache_clear()
|
||||
logger.info("ebook_bm25_corpus_cache_cleared_after_refresh")
|
||||
@@ -1,24 +0,0 @@
|
||||
"""FastAPI dependencies for the EPUB search app."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
import httpx
|
||||
from fastapi import Depends, Request
|
||||
|
||||
from python.ebook_search.config import EbookSearchConfig
|
||||
|
||||
|
||||
def get_config(request: Request) -> EbookSearchConfig:
|
||||
"""Get the loaded search config from app state."""
|
||||
return request.app.state.config
|
||||
|
||||
|
||||
def get_http_client(request: Request) -> httpx.AsyncClient:
|
||||
"""Get the shared LLM HTTP client from app state."""
|
||||
return request.app.state.http_client
|
||||
|
||||
|
||||
AppConfig = Annotated[EbookSearchConfig, Depends(get_config)]
|
||||
AppHttpClient = Annotated[httpx.AsyncClient, Depends(get_http_client)]
|
||||
@@ -1,127 +0,0 @@
|
||||
"""Background phrase-judging tasks for the web app.
|
||||
|
||||
Judging a book sends one LLM request per candidate phrase, which can take minutes, so it must
|
||||
not run inside the request where it would block the UI. Judgments run as async FastAPI
|
||||
background tasks, awaited on the event loop after the response is sent, and are tracked per
|
||||
book in app state so a second judge request for a book that is already being judged is
|
||||
rejected instead of doubling the work.
|
||||
|
||||
State is loop-confined: every read and mutation happens on the event loop (async route
|
||||
handlers and async background tasks) and no critical section contains an ``await``, so each
|
||||
mutation is atomic per loop iteration and no locking is needed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import BackgroundTasks, FastAPI
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class JudgeTaskState:
|
||||
"""Running book judgments and last outcome messages, keyed by book id."""
|
||||
|
||||
running_book_ids: set[int] = field(default_factory=set)
|
||||
outcome_messages: dict[int, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
def get_judge_task_state(app: FastAPI) -> JudgeTaskState:
|
||||
"""Return the app's judge task state, creating it on first use.
|
||||
|
||||
Args:
|
||||
app (FastAPI): App whose state holds the judge task registry.
|
||||
|
||||
Returns:
|
||||
JudgeTaskState: The shared judge task state for this app.
|
||||
"""
|
||||
state = getattr(app.state, "judge_tasks", None)
|
||||
if state is None:
|
||||
state = JudgeTaskState()
|
||||
app.state.judge_tasks = state
|
||||
return state
|
||||
|
||||
|
||||
def start_book_phrase_judgment(app: FastAPI, background_tasks: BackgroundTasks, source_id: int) -> bool:
|
||||
"""Queue judging of one book's candidate phrases as a FastAPI background task.
|
||||
|
||||
The book is claimed before the response returns, so a repeated judge request cannot queue
|
||||
a second run while one is pending or running.
|
||||
|
||||
Args:
|
||||
app (FastAPI): App supplying the engine, config, and judge task state.
|
||||
background_tasks (BackgroundTasks): Request's background tasks to queue the judgment on.
|
||||
source_id (int): Book to judge candidates for.
|
||||
|
||||
Returns:
|
||||
bool: True when a judgment was queued, False when one is already running for this book.
|
||||
"""
|
||||
state = get_judge_task_state(app)
|
||||
if source_id in state.running_book_ids:
|
||||
logger.info(f"ebook_book_phrase_judgment_already_running {source_id=}")
|
||||
return False
|
||||
state.running_book_ids.add(source_id)
|
||||
state.outcome_messages.pop(source_id, None)
|
||||
background_tasks.add_task(judge_book_phrases_for_app, app, source_id)
|
||||
logger.info(f"ebook_book_phrase_judgment_queued {source_id=}")
|
||||
return True
|
||||
|
||||
|
||||
async def judge_book_phrases_for_app(app: FastAPI, source_id: int) -> None:
|
||||
"""Judge one book using the app engine and config, recording the outcome message.
|
||||
|
||||
Args:
|
||||
app (FastAPI): App supplying the engine, config, and judge task state.
|
||||
source_id (int): Book to judge candidates for.
|
||||
"""
|
||||
state = get_judge_task_state(app)
|
||||
try:
|
||||
result = await judge_candidate_phrases_for_books(app.state.engine, app.state.config, source_ids=[source_id])
|
||||
logger.info(
|
||||
f"ebook_book_phrase_judgment_complete {source_id=} {result.candidates_judged=} {result.protected_phrases=} "
|
||||
f"{result.phrase_mentions=} {result.books_failed=}"
|
||||
)
|
||||
if result.books_failed:
|
||||
message = "Judging failed; see server logs for details"
|
||||
else:
|
||||
message = (
|
||||
f"Judged {result.candidates_judged} candidates; {result.protected_phrases} protected phrases promoted"
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(f"ebook_book_phrase_judgment_task_failed {source_id=}")
|
||||
message = "Judging failed; see server logs for details"
|
||||
state.running_book_ids.discard(source_id)
|
||||
state.outcome_messages[source_id] = message
|
||||
|
||||
|
||||
def is_judging_book(app: FastAPI, source_id: int) -> bool:
|
||||
"""Report whether a judgment is currently queued or running for one book.
|
||||
|
||||
Args:
|
||||
app (FastAPI): App supplying the judge task state.
|
||||
source_id (int): Book to check.
|
||||
|
||||
Returns:
|
||||
bool: True while the book's judgment is pending or running.
|
||||
"""
|
||||
return source_id in get_judge_task_state(app).running_book_ids
|
||||
|
||||
|
||||
def pop_book_judgment_outcome(app: FastAPI, source_id: int) -> str | None:
|
||||
"""Return and clear the outcome message from one book's last finished judgment.
|
||||
|
||||
Args:
|
||||
app (FastAPI): App supplying the judge task state.
|
||||
source_id (int): Book to fetch the outcome for.
|
||||
|
||||
Returns:
|
||||
str | None: The outcome message, or None when there is nothing new to report.
|
||||
"""
|
||||
return get_judge_task_state(app).outcome_messages.pop(source_id, None)
|
||||
@@ -1,91 +0,0 @@
|
||||
"""FastAPI HTMX app for EPUB search."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import TYPE_CHECKING, Annotated
|
||||
|
||||
import httpx
|
||||
import typer
|
||||
import uvicorn
|
||||
from fastapi import FastAPI
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from python.common import configure_logger
|
||||
from python.ebook_search.api.bm25_tasks import cancel_bm25_refresh
|
||||
from python.ebook_search.api.routes import admin_router, health_router, page_router, search_router
|
||||
from python.ebook_search.api.web import STATIC_DIR
|
||||
from python.ebook_search.bm25_corpus import ensure_bm25_corpus
|
||||
from python.ebook_search.config import load_config
|
||||
from python.ebook_search.protected_phrases.pool import shutdown_extraction_pool
|
||||
from python.fastapi_tools import ZstdMiddleware
|
||||
from python.orm.common import get_async_postgres_engine
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
"""Manage application startup and shutdown resources."""
|
||||
logger.info("ebook_search_startup")
|
||||
config = load_config()
|
||||
app.state.config = config
|
||||
logger.info(
|
||||
f"ebook_search_config_loaded {config.top_k=} {config.embedding_model=} {config.embedding_base_url=} "
|
||||
f"{config.vllm_base_url=} {config.rerank.enabled=} {config.phrase_matching_enabled=} {config.answer_enabled=} "
|
||||
f"library_paths={len(config.library_paths)}"
|
||||
)
|
||||
if not config.library_paths:
|
||||
logger.warning("ebook_search_no_library_paths_configured")
|
||||
# Concurrent phrase judging opens one session per book worker on this engine, so size the pool
|
||||
# to cover those plus headroom for ordinary web requests.
|
||||
app.state.engine = get_async_postgres_engine(
|
||||
name="RICHIE",
|
||||
vector_engine=True,
|
||||
pool_size=config.phrase_judge_book_workers + 10,
|
||||
)
|
||||
app.state.http_client = httpx.AsyncClient()
|
||||
async with AsyncSession(app.state.engine, expire_on_commit=False) as session:
|
||||
await ensure_bm25_corpus(session, config)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
logger.info("ebook_search_shutdown")
|
||||
cancel_bm25_refresh(app)
|
||||
shutdown_extraction_pool()
|
||||
await app.state.http_client.aclose()
|
||||
await app.state.engine.dispose()
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
"""Create the EPUB search web app."""
|
||||
app = FastAPI(title="EPUB Search", lifespan=lifespan)
|
||||
app.add_middleware(ZstdMiddleware)
|
||||
app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
|
||||
|
||||
app.include_router(admin_router)
|
||||
app.include_router(health_router)
|
||||
app.include_router(page_router)
|
||||
app.include_router(search_router)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def serve(
|
||||
host: Annotated[str, typer.Option("--host", "-h", help="Host to bind to")] = "127.0.0.1",
|
||||
port: Annotated[int, typer.Option("--port", "-p", help="Port to bind to")] = 8070,
|
||||
log_level: Annotated[str, typer.Option("--log-level", "-l", help="Log level")] = "INFO",
|
||||
) -> None:
|
||||
"""Start the EPUB search server."""
|
||||
configure_logger(log_level)
|
||||
uvicorn.run(create_app(), host=host, port=port)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
typer.run(serve)
|
||||
@@ -1,13 +0,0 @@
|
||||
"""EPUB search web route modules."""
|
||||
|
||||
from python.ebook_search.api.routes.admin import router as admin_router
|
||||
from python.ebook_search.api.routes.health import router as health_router
|
||||
from python.ebook_search.api.routes.page import router as page_router
|
||||
from python.ebook_search.api.routes.search import router as search_router
|
||||
|
||||
__all__ = [
|
||||
"admin_router",
|
||||
"health_router",
|
||||
"page_router",
|
||||
"search_router",
|
||||
]
|
||||
@@ -1,204 +0,0 @@
|
||||
"""Admin routes for the EPUB search web UI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from fastapi.responses import HTMLResponse
|
||||
|
||||
from python.ebook_search.api.bm25_tasks import schedule_bm25_refresh
|
||||
from python.ebook_search.api.dependencies import ( # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
|
||||
AppConfig,
|
||||
AppHttpClient,
|
||||
)
|
||||
from python.ebook_search.api.web import error_response, templates
|
||||
from python.ebook_search.embeddings import embed_missing_chunks, embedding_model_stats
|
||||
from python.ebook_search.ingest import ingest_configured_paths
|
||||
from python.ebook_search.protected_phrases.generate_ngrams import generate_candidate_phrases_for_books
|
||||
from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
|
||||
from python.ebook_search.protected_phrases.store import book_ids_pending_first_judgment, corpus_phrase_stats
|
||||
from python.fastapi_tools import ( # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
|
||||
AppAsyncEngine,
|
||||
AsyncDbSession,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/admin")
|
||||
|
||||
|
||||
@router.get("", response_class=HTMLResponse)
|
||||
async def admin(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
|
||||
"""Render the admin page."""
|
||||
stats = await embedding_model_stats(session)
|
||||
phrase_stats = await corpus_phrase_stats(session)
|
||||
logger.info(
|
||||
f"ebook_admin_page_loaded models={len(stats)} {phrase_stats.candidate_phrases=} "
|
||||
f"{phrase_stats.protected_phrases=}"
|
||||
)
|
||||
return templates.TemplateResponse(
|
||||
request,
|
||||
"admin.html",
|
||||
{"config": config, "stats": stats, "phrase_stats": phrase_stats},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/scan", response_class=HTMLResponse)
|
||||
async def scan_library(request: Request, config: AppConfig, session: AsyncDbSession) -> HTMLResponse:
|
||||
"""Scan configured library paths for EPUB changes."""
|
||||
try:
|
||||
count = await ingest_configured_paths(session, config)
|
||||
await session.commit()
|
||||
except Exception as error:
|
||||
logger.exception("ebook_admin_scan_failed")
|
||||
return error_response(request, error)
|
||||
|
||||
logger.info(f"ebook_admin_scan_complete {count=}")
|
||||
if count > 0:
|
||||
schedule_bm25_refresh(request.app)
|
||||
return templates.TemplateResponse(request, "partials/admin_status.html", {"message": f"Indexed {count} EPUBs"})
|
||||
|
||||
|
||||
@router.post("/phrases/generate-all", response_class=HTMLResponse)
|
||||
async def generate_all_phrases(request: Request, config: AppConfig, engine: AppAsyncEngine) -> HTMLResponse:
|
||||
"""Regenerate candidate phrases for every indexed book without LLM judging."""
|
||||
try:
|
||||
result = await generate_candidate_phrases_for_books(engine, config)
|
||||
except Exception as error:
|
||||
logger.exception("ebook_admin_generate_phrases_failed")
|
||||
return error_response(request, error)
|
||||
|
||||
logger.info(
|
||||
f"ebook_admin_generate_phrases_complete {result.books_seen=} {result.books_built=} {result.candidate_phrases=}"
|
||||
)
|
||||
return templates.TemplateResponse(
|
||||
request,
|
||||
"partials/admin_status.html",
|
||||
{
|
||||
"message": (
|
||||
f"Generated phrases for {result.books_built} of {result.books_seen} books; "
|
||||
f"{result.candidate_phrases} candidates stored"
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/phrases/judge-all", response_class=HTMLResponse)
|
||||
async def judge_all_phrases(request: Request, engine: AppAsyncEngine, config: AppConfig) -> HTMLResponse:
|
||||
"""Judge unjudged candidate phrases across every indexed book."""
|
||||
return await run_phrase_judgment(request, engine, config, source_ids=None)
|
||||
|
||||
|
||||
@router.post("/phrases/judge-missing", response_class=HTMLResponse)
|
||||
async def judge_missing_phrases(
|
||||
request: Request,
|
||||
engine: AppAsyncEngine,
|
||||
config: AppConfig,
|
||||
session: AsyncDbSession,
|
||||
) -> HTMLResponse:
|
||||
"""Judge candidate phrases only for books where judging has never run."""
|
||||
source_ids = await book_ids_pending_first_judgment(session)
|
||||
if not source_ids:
|
||||
return templates.TemplateResponse(
|
||||
request,
|
||||
"partials/admin_status.html",
|
||||
{"message": "All books with candidate phrases have been judged"},
|
||||
)
|
||||
return await run_phrase_judgment(request, engine, config, source_ids=source_ids)
|
||||
|
||||
|
||||
async def run_phrase_judgment(
|
||||
request: Request,
|
||||
engine: AppAsyncEngine,
|
||||
config: AppConfig,
|
||||
*,
|
||||
source_ids: list[int] | None,
|
||||
) -> HTMLResponse:
|
||||
"""Run LLM judging for candidate phrases and render the outcome as an admin status partial.
|
||||
|
||||
Args:
|
||||
request (Request): Current request, for template rendering.
|
||||
engine (AppAsyncEngine): Engine used to open per-book judging sessions.
|
||||
config (AppConfig): Runtime phrase-tuning settings.
|
||||
source_ids (list[int] | None): Books to judge; ``None`` judges every indexed book.
|
||||
|
||||
Returns:
|
||||
HTMLResponse: Status partial describing the judging outcome.
|
||||
"""
|
||||
try:
|
||||
result = await judge_candidate_phrases_for_books(engine, config, source_ids=source_ids)
|
||||
except Exception as error:
|
||||
logger.exception("ebook_admin_judge_phrases_failed")
|
||||
return error_response(request, error)
|
||||
|
||||
logger.info(
|
||||
f"ebook_admin_judge_phrases_complete {result.books_seen=} {result.books_judged=} {result.books_failed=} "
|
||||
f"{result.candidates_judged=} {result.protected_phrases=} {result.phrase_mentions=}"
|
||||
)
|
||||
return templates.TemplateResponse(
|
||||
request,
|
||||
"partials/admin_status.html",
|
||||
{
|
||||
"message": (
|
||||
f"Judged {result.candidates_judged} candidates across {result.books_judged} of "
|
||||
f"{result.books_seen} books; {result.protected_phrases} protected phrases, "
|
||||
f"{result.phrase_mentions} mentions"
|
||||
+ (f"; {result.books_failed} books failed" if result.books_failed else "")
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/embed-missing", response_class=HTMLResponse)
|
||||
async def embed_missing(
|
||||
request: Request,
|
||||
config: AppConfig,
|
||||
session: AsyncDbSession,
|
||||
client: AppHttpClient,
|
||||
) -> HTMLResponse:
|
||||
"""Embed chunks missing vectors for the configured model."""
|
||||
try:
|
||||
count = await embed_missing_chunks(session, client, config)
|
||||
await session.commit()
|
||||
except Exception as error:
|
||||
logger.exception("ebook_admin_embed_missing_failed")
|
||||
return error_response(request, error)
|
||||
|
||||
logger.info(f"ebook_admin_embed_missing_complete {count=}")
|
||||
return templates.TemplateResponse(
|
||||
request,
|
||||
"partials/admin_status.html",
|
||||
{"message": f"Embedded {count} chunks"},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/embed-all", response_class=HTMLResponse)
|
||||
async def embed_all(
|
||||
request: Request,
|
||||
config: AppConfig,
|
||||
session: AsyncDbSession,
|
||||
client: AppHttpClient,
|
||||
) -> HTMLResponse:
|
||||
"""Embed all chunks missing vectors in fixed-size batches."""
|
||||
total = 0
|
||||
batches = 0
|
||||
try:
|
||||
while True:
|
||||
count = await embed_missing_chunks(session, client, config)
|
||||
if count == 0:
|
||||
break
|
||||
await session.commit()
|
||||
total += count
|
||||
batches += 1
|
||||
logger.info(f"ebook_admin_embed_all_batch_complete {batches=} {count=} {total=}")
|
||||
except Exception as error:
|
||||
logger.exception(f"ebook_admin_embed_all_failed {batches=} {total=}")
|
||||
return error_response(request, f"Embed all failed after {total} chunks in {batches} batches: {error}")
|
||||
|
||||
logger.info(f"ebook_admin_embed_all_complete {batches=} {total=}")
|
||||
return templates.TemplateResponse(
|
||||
request,
|
||||
"partials/admin_status.html",
|
||||
{"message": f"Embedded {total} chunks in {batches} batches of {config.embedding_batch_size}"},
|
||||
)
|
||||
@@ -1,92 +0,0 @@
|
||||
"""Liveness and readiness routes for the EPUB search service."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from http import HTTPStatus
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import APIRouter
|
||||
from fastapi.responses import JSONResponse
|
||||
from sqlalchemy import literal, select
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
|
||||
from python.ebook_search.api.dependencies import ( # noqa: TC001 FastAPI resolves these annotated dependencies at runtime
|
||||
AppConfig,
|
||||
AppHttpClient,
|
||||
)
|
||||
from python.ebook_search.bm25_corpus import bm25_index_exists, bm25_index_path, read_bm25_manifest
|
||||
from python.ebook_search.llm_interface import check_chat_endpoint, check_embedding_endpoint
|
||||
from python.fastapi_tools import AsyncDbSession # noqa: TC001 FastAPI resolves this annotated dependency at runtime
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import httpx
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from python.ebook_search.config import EbookSearchConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
async def health() -> dict[str, str]:
|
||||
"""Liveness probe that returns ok without touching dependencies."""
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.get("/ready")
|
||||
async def ready(config: AppConfig, session: AsyncDbSession, client: AppHttpClient) -> JSONResponse:
|
||||
"""Readiness probe reporting database, embedding endpoint, and BM25 index status."""
|
||||
database_ok = await check_database(session)
|
||||
embedding_ok = await check_embedding_endpoint(client, config)
|
||||
chat_status = await chat_endpoint_status(client, config)
|
||||
bm25_status = check_bm25_status(config)
|
||||
|
||||
checks = {
|
||||
"database": "ok" if database_ok else "fail",
|
||||
"embedding": "ok" if embedding_ok else "fail",
|
||||
"chat": chat_status,
|
||||
"bm25": bm25_status,
|
||||
}
|
||||
if not database_ok:
|
||||
status = "unavailable"
|
||||
status_code = HTTPStatus.SERVICE_UNAVAILABLE
|
||||
elif not embedding_ok or chat_status == "fail" or bm25_status == "missing":
|
||||
status = "degraded"
|
||||
status_code = HTTPStatus.OK
|
||||
else:
|
||||
status = "ready"
|
||||
status_code = HTTPStatus.OK
|
||||
|
||||
logger.info(f"ebook_ready_check {status=} {database_ok=} {embedding_ok=} {chat_status=} {bm25_status=}")
|
||||
return JSONResponse(content={"status": status, "checks": checks}, status_code=status_code)
|
||||
|
||||
|
||||
async def chat_endpoint_status(client: httpx.AsyncClient, config: EbookSearchConfig) -> str:
|
||||
"""Return the answering chat endpoint status, or disabled when answers are off."""
|
||||
if not config.answer_enabled:
|
||||
return "disabled"
|
||||
return "ok" if await check_chat_endpoint(client, config) else "fail"
|
||||
|
||||
|
||||
async def check_database(session: AsyncSession) -> bool:
|
||||
"""Return whether the database answers a trivial query."""
|
||||
try:
|
||||
await session.execute(select(literal(1)))
|
||||
except SQLAlchemyError as error:
|
||||
logger.warning(f"ebook_ready_database_unavailable {error=}")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def check_bm25_status(config: EbookSearchConfig) -> str:
|
||||
"""Return the persisted BM25 index status without loading it into memory."""
|
||||
index_path = bm25_index_path(config)
|
||||
manifest = read_bm25_manifest(index_path)
|
||||
if manifest is None or not bm25_index_exists(index_path, manifest):
|
||||
return "missing"
|
||||
if manifest.chunk_count == 0:
|
||||
return "empty"
|
||||
return "ok"
|
||||
@@ -1,187 +0,0 @@
|
||||
"""Page routes for the EPUB search web UI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
|
||||
from fastapi.responses import HTMLResponse, RedirectResponse
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from python.ebook_search.api.dependencies import (
|
||||
AppConfig, # noqa: TC001 FastAPI resolves this annotated dependency at runtime
|
||||
)
|
||||
from python.ebook_search.api.judge_tasks import is_judging_book, pop_book_judgment_outcome, start_book_phrase_judgment
|
||||
from python.ebook_search.api.web import templates
|
||||
from python.ebook_search.protected_phrases.generate_ngrams import recalculate_candidate_phrases_for_book
|
||||
from python.ebook_search.protected_phrases.store import count_protected_phrases
|
||||
from python.fastapi_tools import AsyncDbSession # noqa: TC001 FastAPI resolves this annotated dependency at runtime
|
||||
from python.orm.richie import EbookCandidatePhrase, EbookChapter, EbookChunk, EbookProtectedPhrase, EbookSource
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/", response_class=HTMLResponse)
|
||||
async def index(request: Request, config: AppConfig) -> HTMLResponse:
|
||||
"""Render the search page."""
|
||||
return templates.TemplateResponse(request, "search.html", {"config": config})
|
||||
|
||||
|
||||
@router.get("/books", response_class=HTMLResponse)
|
||||
async def books(request: Request, session: AsyncDbSession) -> HTMLResponse:
|
||||
"""Render the indexed books page."""
|
||||
sources = list((await session.scalars(select(EbookSource).order_by(EbookSource.title))).all())
|
||||
logger.info(f"ebook_books_page_loaded count={len(sources)}")
|
||||
return templates.TemplateResponse(request, "books.html", {"sources": sources})
|
||||
|
||||
|
||||
async def get_chapter_count(session: AsyncSession, book_id: int) -> int:
|
||||
"""Return the number of indexed chapters for one book."""
|
||||
return await session.scalar(select(func.count(EbookChapter.id)).where(EbookChapter.source_id == book_id)) or 0
|
||||
|
||||
|
||||
async def get_chunk_count(session: AsyncSession, book_id: int) -> int:
|
||||
"""Return the number of indexed chunks for one book."""
|
||||
return await session.scalar(select(func.count(EbookChunk.id)).where(EbookChunk.source_id == book_id)) or 0
|
||||
|
||||
|
||||
async def get_candidate_count(session: AsyncSession, book_id: int) -> int:
|
||||
"""Return the number of indexed candidates for one book."""
|
||||
return (
|
||||
await session.scalar(select(func.count(EbookCandidatePhrase.id)).where(EbookCandidatePhrase.book_id == book_id))
|
||||
or 0
|
||||
)
|
||||
|
||||
|
||||
async def get_judged_candidate_count(session: AsyncSession, book_id: int) -> int:
|
||||
"""Return the number of judged candidates for one book."""
|
||||
return (
|
||||
await session.scalar(
|
||||
select(func.count(EbookCandidatePhrase.id)).where(
|
||||
EbookCandidatePhrase.book_id == book_id,
|
||||
EbookCandidatePhrase.llm_judged.is_(True),
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
|
||||
|
||||
async def get_candidates(session: AsyncSession, book_id: int) -> list[EbookCandidatePhrase]:
|
||||
"""Return the indexed candidates for one book."""
|
||||
return list(
|
||||
await session.scalars(
|
||||
select(EbookCandidatePhrase)
|
||||
.where(EbookCandidatePhrase.book_id == book_id)
|
||||
.order_by(EbookCandidatePhrase.candidate_score.desc())
|
||||
.limit(100)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def get_protected_phrases(session: AsyncSession, book_id: int) -> list[EbookProtectedPhrase]:
|
||||
"""Return the protected phrases for one book."""
|
||||
return list(
|
||||
await session.scalars(
|
||||
select(EbookProtectedPhrase)
|
||||
.where(EbookProtectedPhrase.book_id == book_id)
|
||||
.order_by(EbookProtectedPhrase.importance.desc())
|
||||
.limit(100)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@router.get("/books/{source_id}", response_class=HTMLResponse)
|
||||
async def book_detail(source_id: int, request: Request, session: AsyncDbSession) -> HTMLResponse:
|
||||
"""Render details for one indexed book."""
|
||||
source = await session.get(EbookSource, source_id)
|
||||
phrase_status_message = None
|
||||
recalculated = request.query_params.get("phrases_recalculated")
|
||||
if recalculated is not None:
|
||||
phrase_status_message = f"Recalculated phrases; {recalculated} candidates generated"
|
||||
judgment_outcome = pop_book_judgment_outcome(request.app, source_id)
|
||||
if judgment_outcome is not None:
|
||||
phrase_status_message = judgment_outcome
|
||||
judging_in_progress = is_judging_book(request.app, source_id)
|
||||
if judging_in_progress:
|
||||
phrase_status_message = "Judging candidate phrases in the background; refresh to see progress"
|
||||
if source is not None:
|
||||
chapter_count = await get_chapter_count(session, source.id)
|
||||
chunk_count = await get_chunk_count(session, source.id)
|
||||
candidate_count = await get_candidate_count(session, source.id)
|
||||
judged_candidate_count = await get_judged_candidate_count(session, source.id)
|
||||
protected_count = await count_protected_phrases(session, source.id)
|
||||
candidates = await get_candidates(session, source.id)
|
||||
protected_phrases = await get_protected_phrases(session, source.id)
|
||||
else:
|
||||
chapter_count = 0
|
||||
chunk_count = 0
|
||||
candidate_count = 0
|
||||
judged_candidate_count = 0
|
||||
protected_count = 0
|
||||
candidates = []
|
||||
protected_phrases = []
|
||||
logger.info(
|
||||
f"ebook_book_detail_loaded {source_id=} found={source is not None} {chapter_count=} {chunk_count=} "
|
||||
f"{candidate_count=} {judged_candidate_count=} {protected_count=}"
|
||||
)
|
||||
return templates.TemplateResponse(
|
||||
request,
|
||||
"book_detail.html",
|
||||
{
|
||||
"candidate_count": candidate_count,
|
||||
"candidates": candidates,
|
||||
"chapter_count": chapter_count,
|
||||
"chunk_count": chunk_count,
|
||||
"judged_candidate_count": judged_candidate_count,
|
||||
"judging_in_progress": judging_in_progress,
|
||||
"protected_count": protected_count,
|
||||
"protected_phrases": protected_phrases,
|
||||
"phrase_status_message": phrase_status_message,
|
||||
"source": source,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/books/{source_id}/recalculate-phrases")
|
||||
async def recalculate_book_phrases(source_id: int, config: AppConfig, session: AsyncDbSession) -> RedirectResponse:
|
||||
"""Clear and regenerate candidate phrases for one indexed book."""
|
||||
source = await session.get(EbookSource, source_id)
|
||||
if source is None:
|
||||
raise HTTPException(status_code=404, detail="Book not found")
|
||||
|
||||
try:
|
||||
result = await recalculate_candidate_phrases_for_book(session, source, config)
|
||||
except ValueError as error:
|
||||
raise HTTPException(status_code=409, detail=str(error)) from error
|
||||
logger.info(
|
||||
f"ebook_book_phrase_recalculation_complete {source_id=} {result.candidate_phrases=} "
|
||||
f"{result.deleted_candidates=} {result.deleted_protected_phrases=} {result.deleted_aliases=} "
|
||||
f"{result.deleted_mentions=}"
|
||||
)
|
||||
return RedirectResponse(
|
||||
url=f"/books/{source_id}?phrases_recalculated={result.candidate_phrases}",
|
||||
status_code=303,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/books/{source_id}/judge-phrases")
|
||||
async def judge_book_phrases(
|
||||
source_id: int,
|
||||
request: Request,
|
||||
background_tasks: BackgroundTasks,
|
||||
session: AsyncDbSession,
|
||||
) -> RedirectResponse:
|
||||
"""Queue background judging of one book's candidate phrases and return immediately."""
|
||||
source = await session.get(EbookSource, source_id)
|
||||
if source is None:
|
||||
raise HTTPException(status_code=404, detail="Book not found")
|
||||
|
||||
started = start_book_phrase_judgment(request.app, background_tasks, source.id)
|
||||
logger.info(f"ebook_book_phrase_judgment_requested {source_id=} {started=}")
|
||||
return RedirectResponse(url=f"/books/{source_id}", status_code=303)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user