Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Richie
|
bf4e23d503 |
@@ -1,28 +0,0 @@
|
||||
.git
|
||||
.direnv
|
||||
.mypy_cache
|
||||
.pytest_cache
|
||||
.ruff_cache
|
||||
.venv
|
||||
**/.venv
|
||||
.env
|
||||
.cache
|
||||
.claude
|
||||
.coverage
|
||||
.vscode
|
||||
.stfolder
|
||||
.literotica_data
|
||||
esphome
|
||||
htmlcov
|
||||
data
|
||||
ebooks
|
||||
__pycache__
|
||||
**/__pycache__
|
||||
*.pyc
|
||||
*.pyo
|
||||
.ebook_search_bm25
|
||||
result
|
||||
result-*
|
||||
*.egg-info
|
||||
dist
|
||||
build
|
||||
@@ -1,43 +1,23 @@
|
||||
name: build_systems
|
||||
name: build_jeeves
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
schedule:
|
||||
- cron: "0 22 * * *"
|
||||
|
||||
jobs:
|
||||
prebuild-common:
|
||||
name: prebuild-common-x86-64-v3
|
||||
runs-on: nix-cache-builder
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# portal-1 is the smallest system closure: 95% of its derivations are
|
||||
# shared by all five systems, so it is a maintainable common cache seed.
|
||||
# Keep going so one failing package does not stop unrelated cache entries
|
||||
# from being built.
|
||||
- name: Build common packages
|
||||
run: nixos-rebuild build --keep-going --accept-flake-config --flake ./#portal-1
|
||||
- name: Copy common packages to nix-cache
|
||||
run: nix copy --accept-flake-config --to unix:///host-nix/var/nix/daemon-socket/socket .#nixosConfigurations.portal-1.config.system.build.toplevel
|
||||
|
||||
build:
|
||||
name: build-${{ matrix.system }}
|
||||
needs: prebuild-common
|
||||
runs-on: self-hosted
|
||||
strategy:
|
||||
matrix:
|
||||
system:
|
||||
- "bob"
|
||||
- "brain"
|
||||
- "jeeves"
|
||||
- "muninn"
|
||||
- "rhapsody-in-green"
|
||||
- "portal-1"
|
||||
continue-on-error: true
|
||||
- "router"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Build default package
|
||||
run: "nixos-rebuild build --accept-flake-config --flake ./#${{ matrix.system }}"
|
||||
- name: copy to nix-cache
|
||||
run: nix copy --accept-flake-config --to unix:///host-nix/var/nix/daemon-socket/socket .#nixosConfigurations.${{ matrix.system }}.config.system.build.toplevel
|
||||
run: "nixos-rebuild build --flake ./#${{ matrix.system }}"
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
name: merge_flake_lock_update
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 2 * * 6"
|
||||
|
||||
jobs:
|
||||
merge:
|
||||
runs-on: self-hosted
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- name: merge_flake_lock_update
|
||||
run: >-
|
||||
python -m python.gitea_flake_lock merge
|
||||
--repo "${{ github.repository }}"
|
||||
env:
|
||||
JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
|
||||
GITEA_URL: https://gitea.tmmworkshop.com
|
||||
@@ -1,26 +0,0 @@
|
||||
name: test ebook search
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
env:
|
||||
UV_PYTHON_DOWNLOADS: never
|
||||
UV_CACHE_DIR: /var/cache/uv
|
||||
UV_LINK_MODE: copy
|
||||
|
||||
jobs:
|
||||
test-ebook-search:
|
||||
runs-on: self-hosted
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install dependencies
|
||||
run: nix develop .#devShells.x86_64-linux.ebook-search -c uv sync --locked --project python/ebook_search/docker
|
||||
- name: Run ebook search tests
|
||||
run: nix develop .#devShells.x86_64-linux.ebook-search -c uv run --project python/ebook_search/docker --no-sync pytest tests/ebook_search --override-ini addopts="-n auto -ra"
|
||||
@@ -1,15 +0,0 @@
|
||||
name: treefmt
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
treefmt:
|
||||
name: nix fmt
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: runs treefmt
|
||||
run: "treefmt --ci"
|
||||
@@ -2,24 +2,20 @@ name: update-flake-lock
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 0 * * 6"
|
||||
- cron: "0 0 * * *"
|
||||
|
||||
jobs:
|
||||
lockfile:
|
||||
runs-on: self-hosted
|
||||
permissions:
|
||||
actions: write
|
||||
contents: write
|
||||
pull-requests: write
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Install Nix
|
||||
uses: DeterminateSystems/nix-installer-action@main
|
||||
- name: Update flake.lock
|
||||
run: nix flake update
|
||||
- name: Create or update flake.lock PR
|
||||
env:
|
||||
JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
|
||||
GITEA_URL: https://gitea.tmmworkshop.com
|
||||
run: >-
|
||||
python -m python.gitea_flake_lock update
|
||||
--repo "${{ github.repository }}"
|
||||
uses: DeterminateSystems/update-flake-lock@main
|
||||
with:
|
||||
pr-title: "Update flake.lock"
|
||||
pr-labels: |
|
||||
dependencies
|
||||
automated
|
||||
|
||||
-14
@@ -162,17 +162,3 @@ cython_debug/
|
||||
#.idea/
|
||||
|
||||
test.*
|
||||
|
||||
# syncthing
|
||||
.stfolder
|
||||
|
||||
# Frontend build output
|
||||
frontend/dist/
|
||||
frontend/node_modules/
|
||||
|
||||
# data from testing llms
|
||||
data/*
|
||||
.ebook_search_bm25
|
||||
|
||||
# gems data
|
||||
.gems
|
||||
|
||||
+6
-14
@@ -1,17 +1,10 @@
|
||||
# Generate AGE keys from SSH keys with:
|
||||
# ssh-keygen -A
|
||||
# nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
|
||||
# update keys after addin/removing a key
|
||||
# nix-shell -p sops --run "sops updatekeys users/secrets.yaml" users/secrets.yaml
|
||||
|
||||
keys:
|
||||
- &admin_richie age1u8zj599elqqvcmhxn8zuwrufsz8w8w366d3ayrljjejljt2q45kq8mxw9c # cspell:disable-line
|
||||
- &admin_richie age1u8zj599elqqvcmhxn8zuwrufsz8w8w366d3ayrljjejljt2q45kq8mxw9c
|
||||
|
||||
- &system_bob age1q47vup0tjhulkg7d6xwmdsgrw64h4ax3la3evzqpxyy4adsmk9fs56qz3y # cspell:disable-line
|
||||
- &system_brain age1jhf7vm0005j60mjq63696frrmjhpy8kpc2d66mw044lqap5mjv4snmwvwm # cspell:disable-line
|
||||
- &system_jeeves age13lmqgc3jvkyah5e3vcwmj4s5wsc2akctcga0lpc0x8v8du3fxprqp4ldkv # cspell:disable-line
|
||||
- &system_rhapsody age1ufnewppysaq2wwcl4ugngjz8pfzc5a35yg7luq0qmuqvctajcycs5lf6k4 # cspell:disable-line
|
||||
- &system_portal_1 age1vyav6kxtvt3z4vtnkkjj38eu8hlts5m7ygyckhskvalg2gpjk52su53d0a # cspell:disable-line
|
||||
- &system_bob age1q47vup0tjhulkg7d6xwmdsgrw64h4ax3la3evzqpxyy4adsmk9fs56qz3y
|
||||
- &system_jeeves age13lmqgc3jvkyah5e3vcwmj4s5wsc2akctcga0lpc0x8v8du3fxprqp4ldkv
|
||||
- &system_router age1xzxryqq63x65yuza9lmmkud7crjjxpnkdew070yhx6xn7xe4tdws5twxsv
|
||||
- &system_rhapsody age1ufnewppysaq2wwcl4ugngjz8pfzc5a35yg7luq0qmuqvctajcycs5lf6k4
|
||||
|
||||
creation_rules:
|
||||
- path_regex: users/secrets\.yaml$
|
||||
@@ -19,7 +12,6 @@ creation_rules:
|
||||
- age:
|
||||
- *admin_richie
|
||||
- *system_bob
|
||||
- *system_brain
|
||||
- *system_jeeves
|
||||
- *system_router
|
||||
- *system_rhapsody
|
||||
- *system_portal_1
|
||||
|
||||
Vendored
+13
-94
@@ -2,14 +2,11 @@
|
||||
"cSpell.words": [
|
||||
"aboutwelcome",
|
||||
"acltype",
|
||||
"addopts",
|
||||
"addstr",
|
||||
"advplyr",
|
||||
"ahci",
|
||||
"aioesphomeapi",
|
||||
"aiounifi",
|
||||
"alsa",
|
||||
"apiclient",
|
||||
"archlinux",
|
||||
"ashift",
|
||||
"asrouter",
|
||||
@@ -20,14 +17,11 @@
|
||||
"auditd",
|
||||
"autofetch",
|
||||
"autologin",
|
||||
"automations",
|
||||
"autopull",
|
||||
"autotrim",
|
||||
"autoupdate",
|
||||
"azuretools",
|
||||
"bantime",
|
||||
"bazarr",
|
||||
"bgwriter",
|
||||
"binhex",
|
||||
"bitwarden",
|
||||
"blkdiscard",
|
||||
@@ -38,26 +32,20 @@
|
||||
"captivedetect",
|
||||
"cgroupdriver",
|
||||
"charliermarsh",
|
||||
"Checkpointing",
|
||||
"codellama",
|
||||
"cloudflared",
|
||||
"codezombiech",
|
||||
"compactmode",
|
||||
"Compat",
|
||||
"contentblocking",
|
||||
"cookiebanners",
|
||||
"createdb",
|
||||
"createrole",
|
||||
"crlite",
|
||||
"cryptsetup",
|
||||
"cuda",
|
||||
"darkreader",
|
||||
"datagrip",
|
||||
"datareporting",
|
||||
"davidanson",
|
||||
"dconf",
|
||||
"dearrow",
|
||||
"debugpy",
|
||||
"deepseek",
|
||||
"dialout",
|
||||
"diffie",
|
||||
"direnv",
|
||||
@@ -65,29 +53,23 @@
|
||||
"dnodesize",
|
||||
"dotfiles",
|
||||
"drawio",
|
||||
"duckdns",
|
||||
"eamodio",
|
||||
"ehci",
|
||||
"emerg",
|
||||
"endlessh",
|
||||
"ents",
|
||||
"errorlens",
|
||||
"esbenp",
|
||||
"esphome",
|
||||
"extest",
|
||||
"fadvise",
|
||||
"fastfetch",
|
||||
"fastforwardteam",
|
||||
"FASTFOX",
|
||||
"ffmpegthumbnailer",
|
||||
"filebot",
|
||||
"filebrowser",
|
||||
"fileroller",
|
||||
"findbar",
|
||||
"Fira",
|
||||
"fmask",
|
||||
"fontconfig",
|
||||
"formfill",
|
||||
"forwardfor",
|
||||
"foxundermoon",
|
||||
"FULLSCREEN",
|
||||
"fwupd",
|
||||
@@ -97,7 +79,6 @@
|
||||
"getch",
|
||||
"getmaxyx",
|
||||
"ghdeploy",
|
||||
"gitea",
|
||||
"globalprivacycontrol",
|
||||
"gparted",
|
||||
"gtts",
|
||||
@@ -113,32 +94,23 @@
|
||||
"HPKP",
|
||||
"hplip",
|
||||
"htmlaboutaddons",
|
||||
"httpchk",
|
||||
"hurlenko",
|
||||
"hwloc",
|
||||
"ical",
|
||||
"ignorelist",
|
||||
"improv",
|
||||
"INITDB",
|
||||
"iocharset",
|
||||
"ioit",
|
||||
"iperf",
|
||||
"isal",
|
||||
"jellyfin",
|
||||
"jnoortheen",
|
||||
"jsbc",
|
||||
"kagi",
|
||||
"keyformat",
|
||||
"keylocation",
|
||||
"kuma",
|
||||
"lazer",
|
||||
"levelname",
|
||||
"libglvnd",
|
||||
"libmysqlclient",
|
||||
"libsodium",
|
||||
"libssh",
|
||||
"libvirtd",
|
||||
"llms",
|
||||
"localtime",
|
||||
"louislam",
|
||||
"lsnew",
|
||||
@@ -146,58 +118,43 @@
|
||||
"lynis",
|
||||
"mangohud",
|
||||
"markdownlint",
|
||||
"maxconn",
|
||||
"maxpages",
|
||||
"maxretry",
|
||||
"maxtime",
|
||||
"mechatroner",
|
||||
"mediainfo",
|
||||
"mixtral",
|
||||
"mklabel",
|
||||
"mkpart",
|
||||
"modbus",
|
||||
"modbuss",
|
||||
"modesetting",
|
||||
"mountpoint",
|
||||
"mountpoints",
|
||||
"mousewheel",
|
||||
"mqtt",
|
||||
"mtxr",
|
||||
"mypy",
|
||||
"muninn",
|
||||
"ncdu",
|
||||
"nemo",
|
||||
"neofetch",
|
||||
"nerdfonts",
|
||||
"netdev",
|
||||
"netdevs",
|
||||
"Networkd",
|
||||
"networkmanager",
|
||||
"newtabpage",
|
||||
"ngram",
|
||||
"ngrams",
|
||||
"nixfmt",
|
||||
"nixos",
|
||||
"nixpkgs",
|
||||
"nmap",
|
||||
"noauto",
|
||||
"nodev",
|
||||
"noecho",
|
||||
"nonsponsored",
|
||||
"Noto",
|
||||
"nprt",
|
||||
"nvme",
|
||||
"OCSP",
|
||||
"oderwat",
|
||||
"ollama",
|
||||
"ondemand",
|
||||
"oneshot",
|
||||
"openwakeword",
|
||||
"optimise",
|
||||
"optoutstudies",
|
||||
"overalljails",
|
||||
"overscroll",
|
||||
"overseerr",
|
||||
"paho",
|
||||
"partitionwise",
|
||||
"pbmode",
|
||||
"pciutils",
|
||||
"pcscd",
|
||||
@@ -205,18 +162,16 @@
|
||||
"peerconnection",
|
||||
"PESKYFOX",
|
||||
"PGID",
|
||||
"pgvector",
|
||||
"photoprism",
|
||||
"pipewire",
|
||||
"pkgs",
|
||||
"plugdev",
|
||||
"poppler",
|
||||
"posixacl",
|
||||
"powertop",
|
||||
"primarycache",
|
||||
"prismlauncher",
|
||||
"privatebrowsing",
|
||||
"PRIVOXY",
|
||||
"protontricks",
|
||||
"prowlarr",
|
||||
"proxychains",
|
||||
"prusa",
|
||||
@@ -224,31 +179,21 @@
|
||||
"PUID",
|
||||
"pulseaudio",
|
||||
"punycode",
|
||||
"pychromecast",
|
||||
"pydocstyle",
|
||||
"pyfakefs",
|
||||
"pylance",
|
||||
"pylint",
|
||||
"pymetno",
|
||||
"pymodbus",
|
||||
"pyopenweathermap",
|
||||
"pyownet",
|
||||
"pytest",
|
||||
"qalculate",
|
||||
"qbit",
|
||||
"qbittorrent",
|
||||
"qbittorrentvpn",
|
||||
"qbitvpn",
|
||||
"quicksuggest",
|
||||
"radarr",
|
||||
"readahead",
|
||||
"receiveencrypted",
|
||||
"recordsize",
|
||||
"Redistributable",
|
||||
"referer",
|
||||
"REFERERS",
|
||||
"relatime",
|
||||
"rerank",
|
||||
"Rhosts",
|
||||
"ripgrep",
|
||||
"roboto",
|
||||
"rokuecp",
|
||||
"routable",
|
||||
"rspace",
|
||||
"rtkit",
|
||||
@@ -257,10 +202,8 @@
|
||||
"schemeless",
|
||||
"scrollback",
|
||||
"SECUREFOX",
|
||||
"sessionmaker",
|
||||
"sessionstore",
|
||||
"shellcheck",
|
||||
"signalbot",
|
||||
"signon",
|
||||
"Signons",
|
||||
"skia",
|
||||
@@ -270,10 +213,8 @@
|
||||
"socialtracking",
|
||||
"sonarr",
|
||||
"sponsorblock",
|
||||
"sqlalchemy",
|
||||
"sqltools",
|
||||
"ssdp",
|
||||
"SSHOPTS",
|
||||
"stdenv",
|
||||
"subresource",
|
||||
"substituters",
|
||||
@@ -281,30 +222,19 @@
|
||||
"sysstat",
|
||||
"tabmanager",
|
||||
"tamasfe",
|
||||
"TCPIP",
|
||||
"testdisk",
|
||||
"tiktok",
|
||||
"timonwong",
|
||||
"titlebar",
|
||||
"tmmworkshop",
|
||||
"Tmpfs",
|
||||
"topsites",
|
||||
"topstories",
|
||||
"treefmt",
|
||||
"torrenting",
|
||||
"twimg",
|
||||
"typedmonarchmoney",
|
||||
"typer",
|
||||
"uaccess",
|
||||
"ubiquiti",
|
||||
"ublock",
|
||||
"uiprotect",
|
||||
"uitour",
|
||||
"unifi",
|
||||
"unjudged",
|
||||
"unrar",
|
||||
"unstorable",
|
||||
"unsubmitted",
|
||||
"uptimekuma",
|
||||
"urlbar",
|
||||
"urlclassifier",
|
||||
"usbhid",
|
||||
@@ -312,37 +242,26 @@
|
||||
"useragent",
|
||||
"usernamehw",
|
||||
"userprefs",
|
||||
"vaninventory",
|
||||
"vdev",
|
||||
"vfat",
|
||||
"victron",
|
||||
"virt",
|
||||
"virtualisation",
|
||||
"vpnpromourl",
|
||||
"wakeonlan",
|
||||
"webchannel",
|
||||
"WEBRTC",
|
||||
"WEBUI",
|
||||
"wireplumber",
|
||||
"wireshark",
|
||||
"Workqueues",
|
||||
"xattr",
|
||||
"xcursorgen",
|
||||
"xdist",
|
||||
"xhci",
|
||||
"yake",
|
||||
"yazi",
|
||||
"yubikey",
|
||||
"yubioath",
|
||||
"yzhang",
|
||||
"zeroconf",
|
||||
"zerotier",
|
||||
"zerotierone",
|
||||
"zoxide",
|
||||
"zram",
|
||||
"zstd"
|
||||
],
|
||||
"python-envs.defaultEnvManager": "ms-python.python:system",
|
||||
"python-envs.pythonProjects": [],
|
||||
"python.testing.pytestArgs": ["tests"],
|
||||
"python.testing.unittestEnabled": false,
|
||||
"python.testing.pytestEnabled": true
|
||||
]
|
||||
}
|
||||
|
||||
Generated
-1686
File diff suppressed because it is too large
Load Diff
@@ -1,4 +0,0 @@
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
|
||||
members = ["rust/*"]
|
||||
@@ -1,51 +1 @@
|
||||
# dotfiles
|
||||
|
||||
## Installer ISO
|
||||
|
||||
Build a bootable NixOS ISO with the installer preinstalled:
|
||||
|
||||
```sh
|
||||
nix build .#iso
|
||||
```
|
||||
|
||||
Write `result/iso/nixos-zfs-installer.iso` to a USB stick (for example with `dd`) or boot it in a VM. The image is the minimal NixOS installation CD with ZFS enabled and `nixos-installer` on `PATH`. SSH is enabled and the `nixos` and `root` accounts use the password `nixos`, so you can also run the installer remotely. Once booted:
|
||||
|
||||
```sh
|
||||
sudo nixos-installer
|
||||
```
|
||||
|
||||
The ISO bundles the `.#installer-nixos` package, a variant of the binary that keeps its Nix store linkage instead of being patched for foreign distributions.
|
||||
|
||||
## Installer binary
|
||||
|
||||
Build the self-contained installer executable with:
|
||||
|
||||
```sh
|
||||
nix build .#installer
|
||||
```
|
||||
|
||||
The flake package (defined in `python/installer/package.nix`) uses the Python builder in `python/installer/build.py`, which stages only the installer modules before running PyInstaller. You can also call it directly when `pyinstaller` and `patchelf` are on `PATH`:
|
||||
|
||||
```sh
|
||||
python -m python.installer.build --output ./nixos-installer
|
||||
```
|
||||
|
||||
Copy `result/bin/nixos-installer` to the installer USB stick and run it as root from the NixOS live environment:
|
||||
|
||||
```sh
|
||||
sudo ./nixos-installer
|
||||
```
|
||||
|
||||
Validate the live environment first with:
|
||||
|
||||
```sh
|
||||
./nixos-installer --check
|
||||
```
|
||||
|
||||
Paste a value into the TUI encryption password field to enable LUKS during install, or set `ENCRYPT_KEY`:
|
||||
|
||||
```sh
|
||||
sudo env ENCRYPT_KEY='change-me' ./nixos-installer
|
||||
```
|
||||
|
||||
The binary bundles the Python runtime and only the installer modules it imports. It still expects the NixOS installer environment to provide system install tools such as `parted`, `zfs`, `zpool`, `cryptsetup`, `nixos-generate-config`, and `nixos-install`.
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
|
||||
nixos-rebuild build --flake /home/richie/projects/dotfiles#bob
|
||||
nixos-rebuild build --flake /home/richie/projects/dotfiles#jeeves
|
||||
nixos-rebuild build --flake /home/richie/projects/dotfiles#muninn
|
||||
nixos-rebuild build --flake /home/richie/projects/dotfiles#rhapsody-in-green
|
||||
@@ -17,11 +17,13 @@
|
||||
./nix.nix
|
||||
./programs.nix
|
||||
./ssh.nix
|
||||
./snapshot_manager.nix
|
||||
];
|
||||
|
||||
boot = {
|
||||
tmp.useTmpfs = lib.mkDefault true;
|
||||
tmp.useTmpfs = true;
|
||||
kernelPackages = lib.mkDefault pkgs.linuxPackages_6_12;
|
||||
zfs.package = lib.mkDefault pkgs.zfs;
|
||||
};
|
||||
|
||||
hardware.enableRedistributableFirmware = true;
|
||||
@@ -29,8 +31,7 @@
|
||||
home-manager = {
|
||||
useGlobalPkgs = true;
|
||||
useUserPackages = true;
|
||||
extraSpecialArgs = { inherit inputs outputs; };
|
||||
backupFileExtension = "backup";
|
||||
extraSpecialArgs = {inherit inputs outputs;};
|
||||
};
|
||||
|
||||
nixpkgs = {
|
||||
@@ -40,25 +41,11 @@
|
||||
};
|
||||
};
|
||||
|
||||
services = {
|
||||
dbus.implementation = "dbus";
|
||||
|
||||
# firmware update
|
||||
fwupd.enable = true;
|
||||
};
|
||||
|
||||
powerManagement.powertop.enable = lib.mkDefault true;
|
||||
services.fwupd.enable = true;
|
||||
|
||||
programs.zsh.enable = true;
|
||||
|
||||
security = {
|
||||
auditd.enable = lib.mkDefault true;
|
||||
sudo-rs = {
|
||||
enable = true;
|
||||
execWheelOnly = true;
|
||||
};
|
||||
sudo.enable = false;
|
||||
};
|
||||
security.auditd.enable = lib.mkDefault true;
|
||||
|
||||
users.mutableUsers = lib.mkDefault false;
|
||||
|
||||
|
||||
@@ -1,10 +1,4 @@
|
||||
{ lib, pkgs, ... }:
|
||||
let
|
||||
libPath = pkgs.lib.makeLibraryPath [
|
||||
pkgs.zlib
|
||||
pkgs.stdenv.cc.cc.lib
|
||||
];
|
||||
in
|
||||
{
|
||||
programs.nix-ld = {
|
||||
enable = lib.mkDefault true;
|
||||
@@ -21,7 +15,6 @@ in
|
||||
libxml2
|
||||
openssl
|
||||
stdenv.cc.cc
|
||||
stdenv.cc.cc.lib
|
||||
systemd
|
||||
util-linux
|
||||
xz
|
||||
@@ -30,9 +23,4 @@ in
|
||||
zstd
|
||||
];
|
||||
};
|
||||
|
||||
environment = {
|
||||
sessionVariables.LD_LIBRARY_PATH = lib.mkDefault libPath;
|
||||
variables.LD_LIBRARY_PATH = lib.mkDefault libPath;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
console.keyMap = lib.mkDefault "us";
|
||||
|
||||
i18n = {
|
||||
defaultLocale = lib.mkDefault "en_US.UTF-8";
|
||||
defaultLocale = lib.mkDefault "en_US.utf8";
|
||||
supportedLocales = lib.mkDefault [ "en_US.UTF-8/UTF-8" ];
|
||||
extraLocaleSettings = lib.mkDefault {
|
||||
LC_ADDRESS = "en_US.UTF-8";
|
||||
LC_IDENTIFICATION = "en_US.UTF-8";
|
||||
|
||||
+6
-12
@@ -2,11 +2,9 @@
|
||||
inputs,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
}: let
|
||||
flakeInputs = lib.filterAttrs (_: lib.isType "flake") inputs;
|
||||
in
|
||||
{
|
||||
in {
|
||||
nix = {
|
||||
settings = {
|
||||
trusted-users = [
|
||||
@@ -15,14 +13,17 @@ in
|
||||
];
|
||||
trusted-substituters = [
|
||||
"https://cache.nixos.org"
|
||||
"https://cache.tmmworkshop.com"
|
||||
"https://nix-community.cachix.org"
|
||||
];
|
||||
substituters = [
|
||||
"https://cache.nixos.org/?priority=2&want-mass-query=true"
|
||||
"https://cache.tmmworkshop.com/?priority=2&want-mass-query=true"
|
||||
"https://nix-community.cachix.org/?priority=10&want-mass-query=true"
|
||||
];
|
||||
trusted-public-keys = [
|
||||
"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
|
||||
"cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA="
|
||||
"nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
|
||||
];
|
||||
auto-optimise-store = lib.mkDefault true;
|
||||
@@ -31,19 +32,12 @@ in
|
||||
"flakes"
|
||||
"ca-derivations"
|
||||
];
|
||||
system-features = lib.mkAfter [
|
||||
"gccarch-x86-64-v2"
|
||||
"gccarch-x86-64-v3"
|
||||
];
|
||||
warn-dirty = false;
|
||||
flake-registry = ""; # disable global flake registries
|
||||
connect-timeout = 10;
|
||||
download-buffer-size = 536870912;
|
||||
fallback = true;
|
||||
};
|
||||
|
||||
# Add each flake input as a registry and nix_path
|
||||
registry = lib.mapAttrs (_: flake: { inherit flake; }) flakeInputs;
|
||||
registry = lib.mapAttrs (_: flake: {inherit flake;}) flakeInputs;
|
||||
nixPath = lib.mapAttrsToList (n: _: "${n}=flake:${n}") flakeInputs;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2,5 +2,6 @@
|
||||
{
|
||||
environment.systemPackages = with pkgs; [
|
||||
git
|
||||
python313
|
||||
];
|
||||
}
|
||||
|
||||
@@ -22,12 +22,6 @@ hourly = 0
|
||||
daily = 0
|
||||
monthly = 0
|
||||
|
||||
["root_pool/nix_build"]
|
||||
15_min = 1
|
||||
hourly = 0
|
||||
daily = 0
|
||||
monthly = 0
|
||||
|
||||
["root_pool/var"]
|
||||
15_min = 8
|
||||
hourly = 24
|
||||
@@ -0,0 +1,44 @@
|
||||
{ inputs, pkgs, lib, config, ... }:
|
||||
let
|
||||
cfg = config.services.snapshot_manager;
|
||||
in
|
||||
{
|
||||
options = {
|
||||
services.snapshot_manager = {
|
||||
enable = lib.mkOption {
|
||||
default = true;
|
||||
example = true;
|
||||
description = "Whether to enable k3s-net.";
|
||||
type = lib.types.bool;
|
||||
};
|
||||
path = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
description = "Path that needs to be updated via git pull";
|
||||
default = ./snapshot_config.toml;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
systemd = {
|
||||
services."snapshot_manager" = {
|
||||
description = "ZFS Snapshot Manager";
|
||||
requires = [ "zfs-import.target" ];
|
||||
after = [ "zfs-import.target" ];
|
||||
path = [ pkgs.zfs ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = "${inputs.system_tools.packages.x86_64-linux.default}/bin/snapshot_manager --config-file='${cfg.path}'";
|
||||
};
|
||||
};
|
||||
timers."snapshot_manager" = {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnBootSec = "15m";
|
||||
OnUnitActiveSec = "15m";
|
||||
Unit = "snapshot_manager.service";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -37,8 +37,6 @@
|
||||
TcpKeepAlive = "no";
|
||||
X11Forwarding = lib.mkDefault false;
|
||||
KexAlgorithms = [
|
||||
"sntrup761x25519-sha512@openssh.com"
|
||||
"mlkem768x25519-sha256"
|
||||
"curve25519-sha256@libssh.org"
|
||||
"diffie-hellman-group-exchange-sha256"
|
||||
];
|
||||
@@ -69,5 +67,5 @@
|
||||
};
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 22 ];
|
||||
networking.firewall.allowedTCPPorts = [ 22 ];
|
||||
}
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
nix.settings = {
|
||||
trusted-substituters = [ "http://192.168.95.35:5000" ];
|
||||
substituters = [ "http://192.168.95.35:5000/?priority=1&want-mass-query=true" ];
|
||||
};
|
||||
}
|
||||
@@ -1,18 +1,10 @@
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
boot = {
|
||||
kernelPackages = pkgs.linuxPackages_6_18;
|
||||
zfs.package = pkgs.zfs_2_4;
|
||||
kernelPackages = pkgs.linuxPackages_6_12;
|
||||
zfs.package = pkgs.zfs;
|
||||
};
|
||||
|
||||
hardware.bluetooth = {
|
||||
enable = true;
|
||||
powerOnBoot = true;
|
||||
};
|
||||
|
||||
# rtkit is optional but recommended for pipewire
|
||||
security.rtkit.enable = true;
|
||||
|
||||
services = {
|
||||
displayManager.sddm = {
|
||||
enable = true;
|
||||
@@ -29,8 +21,6 @@
|
||||
};
|
||||
};
|
||||
|
||||
pulseaudio.enable = false;
|
||||
|
||||
pipewire = {
|
||||
enable = true;
|
||||
alsa.enable = true;
|
||||
|
||||
@@ -17,7 +17,6 @@
|
||||
logDriver = "local";
|
||||
storageDriver = "overlay2";
|
||||
daemon.settings = {
|
||||
live-restore = false;
|
||||
experimental = true;
|
||||
exec-opts = [ "native.cgroupdriver=systemd" ];
|
||||
log-opts = {
|
||||
|
||||
@@ -1,256 +0,0 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
monitoringInterface = "tailscale0";
|
||||
nodeTextfileDir = "/var/lib/prometheus-node-exporter-textfile";
|
||||
|
||||
mkProcessNameTemplate =
|
||||
perPid: template: if perPid then "${template}:{{.PID}}:{{.StartTime}}" else template;
|
||||
|
||||
mkProcessMatchers = perPid: [
|
||||
{
|
||||
name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Module}}";
|
||||
cmdline = [ "^/nix/store[^ ]*/bin/python[^ ]* -m (?P<Module>[^ ]+)" ];
|
||||
}
|
||||
{
|
||||
name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Wrapped}}";
|
||||
cmdline = [
|
||||
"^/nix/store[^ ]*/bin/python[^ ]* /nix/store[^ ]*/bin/\\.?(?P<Wrapped>[^ /]+?)(?:-wrapped)?(?:\\s|$)"
|
||||
];
|
||||
}
|
||||
{
|
||||
name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Wrapped}}";
|
||||
cmdline = [
|
||||
"^/nix/store[^ ]*/bin/node /nix/store[^ ]*-(?P<Wrapped>[A-Za-z0-9._+-]+)-[0-9][^ /]*/"
|
||||
];
|
||||
}
|
||||
{
|
||||
name = mkProcessNameTemplate perPid "{{.Username}}:{{.Matches.Wrapped}}";
|
||||
cmdline = [ "^/nix/store[^ ]*/(?:bin/|lib/[^ ]*/)?\\.?(?P<Wrapped>[^ /]+?)(?:-wrapped)?(?:\\s|$)" ];
|
||||
}
|
||||
{
|
||||
name = mkProcessNameTemplate perPid "{{.Username}}:{{.ExeBase}}";
|
||||
cmdline = [ ".+" ];
|
||||
}
|
||||
];
|
||||
|
||||
perPidConfig = pkgs.writeText "process-exporter-per-pid.yaml" (
|
||||
builtins.toJSON {
|
||||
process_names = mkProcessMatchers true;
|
||||
}
|
||||
);
|
||||
|
||||
zpoolLatencyScript = pkgs.writeShellScript "zpool-latency-exporter" ''
|
||||
set -euo pipefail
|
||||
|
||||
out_dir=${lib.escapeShellArg nodeTextfileDir}
|
||||
host=${lib.escapeShellArg config.networking.hostName}
|
||||
tmp_file="$(mktemp "$out_dir/zpool.prom.XXXXXX")"
|
||||
trap 'rm -f "$tmp_file"' EXIT
|
||||
|
||||
pools="$(zpool list -H -o name | paste -sd, -)"
|
||||
|
||||
cat >"$tmp_file" <<'EOF'
|
||||
# HELP zpool_iostat_total_wait_read_ns Average total read wait time reported by zpool iostat.
|
||||
# TYPE zpool_iostat_total_wait_read_ns gauge
|
||||
# HELP zpool_iostat_total_wait_write_ns Average total write wait time reported by zpool iostat.
|
||||
# TYPE zpool_iostat_total_wait_write_ns gauge
|
||||
# HELP zpool_iostat_disk_wait_read_ns Average disk read wait time reported by zpool iostat.
|
||||
# TYPE zpool_iostat_disk_wait_read_ns gauge
|
||||
# HELP zpool_iostat_disk_wait_write_ns Average disk write wait time reported by zpool iostat.
|
||||
# TYPE zpool_iostat_disk_wait_write_ns gauge
|
||||
# HELP zpool_iostat_syncq_wait_read_ns Average synchronous queue read wait time reported by zpool iostat.
|
||||
# TYPE zpool_iostat_syncq_wait_read_ns gauge
|
||||
# HELP zpool_iostat_syncq_wait_write_ns Average synchronous queue write wait time reported by zpool iostat.
|
||||
# TYPE zpool_iostat_syncq_wait_write_ns gauge
|
||||
# HELP zpool_iostat_asyncq_wait_read_ns Average asynchronous queue read wait time reported by zpool iostat.
|
||||
# TYPE zpool_iostat_asyncq_wait_read_ns gauge
|
||||
# HELP zpool_iostat_asyncq_wait_write_ns Average asynchronous queue write wait time reported by zpool iostat.
|
||||
# TYPE zpool_iostat_asyncq_wait_write_ns gauge
|
||||
EOF
|
||||
|
||||
zpool iostat -Hplvy -y 1 1 | awk -F '\t' -v host="$host" -v pools="$pools" '
|
||||
function esc(str, out) {
|
||||
out = str
|
||||
gsub(/\\/, "\\\\", out)
|
||||
gsub(/"/, "\\\"", out)
|
||||
return out
|
||||
}
|
||||
|
||||
function emit(metric, pool, vdev, value) {
|
||||
if (value == "" || value == "-") {
|
||||
return
|
||||
}
|
||||
|
||||
printf "%s{host=\"%s\",pool=\"%s\",vdev=\"%s\"} %s\n",
|
||||
metric,
|
||||
esc(host),
|
||||
esc(pool),
|
||||
esc(vdev),
|
||||
value
|
||||
}
|
||||
|
||||
BEGIN {
|
||||
split(pools, pool_names, ",")
|
||||
for (idx in pool_names) {
|
||||
if (pool_names[idx] != "") {
|
||||
known_pools[pool_names[idx]] = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
NF == 0 {
|
||||
next
|
||||
}
|
||||
|
||||
{
|
||||
row_name = $1
|
||||
|
||||
if (row_name in known_pools) {
|
||||
current_pool = row_name
|
||||
current_vdev = "_pool"
|
||||
} else if (current_pool == "") {
|
||||
next
|
||||
} else {
|
||||
current_vdev = row_name
|
||||
}
|
||||
|
||||
emit("zpool_iostat_total_wait_read_ns", current_pool, current_vdev, $8)
|
||||
emit("zpool_iostat_total_wait_write_ns", current_pool, current_vdev, $9)
|
||||
emit("zpool_iostat_disk_wait_read_ns", current_pool, current_vdev, $10)
|
||||
emit("zpool_iostat_disk_wait_write_ns", current_pool, current_vdev, $11)
|
||||
emit("zpool_iostat_syncq_wait_read_ns", current_pool, current_vdev, $12)
|
||||
emit("zpool_iostat_syncq_wait_write_ns", current_pool, current_vdev, $13)
|
||||
emit("zpool_iostat_asyncq_wait_read_ns", current_pool, current_vdev, $14)
|
||||
emit("zpool_iostat_asyncq_wait_write_ns", current_pool, current_vdev, $15)
|
||||
}
|
||||
' >>"$tmp_file"
|
||||
|
||||
mv "$tmp_file" "$out_dir/zpool.prom"
|
||||
trap - EXIT
|
||||
'';
|
||||
in
|
||||
{
|
||||
networking.firewall.interfaces.${monitoringInterface}.allowedTCPPorts = [
|
||||
9100
|
||||
9134
|
||||
9256
|
||||
9257
|
||||
9633
|
||||
];
|
||||
|
||||
services.prometheus.exporters = {
|
||||
node = {
|
||||
enable = true;
|
||||
enabledCollectors = [
|
||||
"pressure"
|
||||
"processes"
|
||||
"systemd"
|
||||
];
|
||||
extraFlags = [ "--collector.textfile.directory=${nodeTextfileDir}" ];
|
||||
};
|
||||
|
||||
process = {
|
||||
enable = true;
|
||||
user = "root";
|
||||
group = "root";
|
||||
settings.process_names = mkProcessMatchers false;
|
||||
extraFlags = [
|
||||
"-gather-smaps=false"
|
||||
"-remove-empty-groups=true"
|
||||
"-threads=false"
|
||||
];
|
||||
};
|
||||
|
||||
smartctl.enable = true;
|
||||
zfs.enable = true;
|
||||
};
|
||||
|
||||
programs.atop = {
|
||||
enable = true;
|
||||
atopService.enable = true;
|
||||
atopRotateTimer.enable = true;
|
||||
atopacctService.enable = true;
|
||||
settings.interval = 30;
|
||||
};
|
||||
|
||||
systemd = {
|
||||
services = {
|
||||
prometheus-process-pid-exporter = {
|
||||
description = "Prometheus process exporter with per-PID naming";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network.target" ];
|
||||
serviceConfig = {
|
||||
ExecStart = ''
|
||||
${pkgs.prometheus-process-exporter}/bin/process-exporter \
|
||||
--web.listen-address 0.0.0.0:9257 \
|
||||
--config.path ${perPidConfig} \
|
||||
-children=false \
|
||||
-gather-smaps=false \
|
||||
-remove-empty-groups=true \
|
||||
-threads=false
|
||||
'';
|
||||
User = "root";
|
||||
Group = "root";
|
||||
Restart = "always";
|
||||
WorkingDirectory = "/tmp";
|
||||
CapabilityBoundingSet = [ "" ];
|
||||
DeviceAllow = [ "" ];
|
||||
LockPersonality = true;
|
||||
MemoryDenyWriteExecute = true;
|
||||
NoNewPrivileges = true;
|
||||
PrivateDevices = true;
|
||||
PrivateTmp = true;
|
||||
ProtectClock = true;
|
||||
ProtectControlGroups = true;
|
||||
ProtectHome = true;
|
||||
ProtectHostname = true;
|
||||
ProtectKernelLogs = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectSystem = "strict";
|
||||
RemoveIPC = true;
|
||||
RestrictAddressFamilies = [
|
||||
"AF_INET"
|
||||
"AF_INET6"
|
||||
];
|
||||
RestrictNamespaces = true;
|
||||
RestrictRealtime = true;
|
||||
RestrictSUIDSGID = true;
|
||||
SystemCallArchitectures = "native";
|
||||
UMask = "0077";
|
||||
};
|
||||
};
|
||||
|
||||
zpool-latency-exporter = {
|
||||
description = "Exports ZFS latency metrics for node_exporter textfile collection";
|
||||
after = [ "zfs-import.target" ];
|
||||
requires = [ "zfs-import.target" ];
|
||||
path = [
|
||||
config.boot.zfs.package
|
||||
pkgs.coreutils
|
||||
pkgs.gawk
|
||||
];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = zpoolLatencyScript;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
timers.zpool-latency-exporter = {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnBootSec = "2m";
|
||||
OnUnitActiveSec = "60s";
|
||||
Unit = "zpool-latency-exporter.service";
|
||||
};
|
||||
};
|
||||
|
||||
tmpfiles.rules = [ "d ${nodeTextfileDir} 0755 root root - -" ];
|
||||
};
|
||||
}
|
||||
@@ -2,9 +2,6 @@
|
||||
{
|
||||
services.printing = {
|
||||
enable = true;
|
||||
drivers = with pkgs; [
|
||||
gutenprint
|
||||
hplip
|
||||
];
|
||||
drivers = with pkgs; [ gutenprint hplip ];
|
||||
};
|
||||
}
|
||||
|
||||
@@ -10,9 +10,6 @@
|
||||
authorizedKeys = config.users.users.richie.openssh.authorizedKeys.keys;
|
||||
};
|
||||
};
|
||||
availableKernelModules = [
|
||||
"igb"
|
||||
"r8152"
|
||||
];
|
||||
availableKernelModules = [ "igb" ];
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
environment.systemPackages = with pkgs; [
|
||||
mangohud
|
||||
steam-run
|
||||
];
|
||||
environment.systemPackages = with pkgs; [mangohud steam-run];
|
||||
hardware.steam-hardware.enable = true;
|
||||
|
||||
programs = {
|
||||
@@ -14,7 +11,7 @@
|
||||
remotePlay.openFirewall = true;
|
||||
localNetworkGameTransfers.openFirewall = true;
|
||||
protontricks.enable = true;
|
||||
extraCompatPackages = with pkgs; [ proton-ge-bin ];
|
||||
extraCompatPackages = with pkgs; [proton-ge-bin];
|
||||
extest.enable = true;
|
||||
};
|
||||
gamescope = {
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
{ lib, ... }:
|
||||
{
|
||||
services.syncthing = {
|
||||
enable = true;
|
||||
user = "richie";
|
||||
overrideDevices = true;
|
||||
overrideFolders = lib.mkDefault true;
|
||||
overrideFolders = true;
|
||||
dataDir = "/home/richie/Syncthing";
|
||||
configDir = "/home/richie/.config/syncthing";
|
||||
settings.devices = {
|
||||
bob.id = "CJIAPEJ-VO74RR4-F75VU6M-QNZAMYG-FYUJG7Y-6AT62HJ-355PRPL-PJFETAZ"; # cspell:disable-line
|
||||
brain.id = "SSCGIPI-IV3VYKB-TRNIJE3-COV4T2H-CDBER7F-I2CGHYA-NWOEUDU-3T5QAAN"; # cspell:disable-line
|
||||
ipad.id = "KI76T3X-SFUGV2L-VSNYTKR-TSIUV5L-SHWD3HE-GQRGRCN-GY4UFMD-CW6Z6AX"; # cspell:disable-line
|
||||
jeeves.id = "ICRHXZW-ECYJCUZ-I4CZ64R-3XRK7CG-LL2HAAK-FGOHD22-BQA4AI6-5OAL6AG"; # cspell:disable-line
|
||||
phone.id = "JPVQKQW-CFXOJXT-Q5G5F3H-QIDHDRE-GKHPTQB-GXZUQSP-U7FR7F7-INP3AAH"; # cspell:disable-line
|
||||
rhapsody-in-green.id = "ASL3KC4-3XEN6PA-7BQBRKE-A7JXLI6-DJT43BY-Q4WPOER-7UALUAZ-VTPQ6Q4"; # cspell:disable-line
|
||||
settings = {
|
||||
devices = {
|
||||
phone.id = "LTGPLAE-M4ZDJTM-TZ3DJGY-SLLAVWF-CQDVEVS-RGCS75T-GAPZYK3-KUM6LA5"; # cspell:disable-line
|
||||
jeeves.id = "ICRHXZW-ECYJCUZ-I4CZ64R-3XRK7CG-LL2HAAK-FGOHD22-BQA4AI6-5OAL6AG"; # cspell:disable-line
|
||||
ipad.id = "KI76T3X-SFUGV2L-VSNYTKR-TSIUV5L-SHWD3HE-GQRGRCN-GY4UFMD-CW6Z6AX"; # cspell:disable-line
|
||||
bob.id = "CJIAPEJ-VO74RR4-F75VU6M-QNZAMYG-FYUJG7Y-6AT62HJ-355PRPL-PJFETAZ"; # cspell:disable-line
|
||||
rhapsody-in-green.id = "ASL3KC4-3XEN6PA-7BQBRKE-A7JXLI6-DJT43BY-Q4WPOER-7UALUAZ-VTPQ6Q4"; # cspell:disable-line
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
{
|
||||
config,
|
||||
inputs,
|
||||
...
|
||||
}:
|
||||
{
|
||||
nix.settings = {
|
||||
trusted-substituters = [ "http://jeeves:5000" ];
|
||||
substituters = [ "http://jeeves:5000/?priority=1&want-mass-query=true" ];
|
||||
trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
|
||||
};
|
||||
|
||||
services.tailscale = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
authKeyFile = config.sops.secrets.tailscale_auth_key.path;
|
||||
|
||||
# OAuth client secrets create ephemeral nodes by default. NixOS machines
|
||||
# are persistent and should enroll without interactive device approval.
|
||||
authKeyParameters = {
|
||||
ephemeral = false;
|
||||
preauthorized = true;
|
||||
};
|
||||
|
||||
extraUpFlags = [ "--advertise-tags=tag:nixos" ];
|
||||
};
|
||||
|
||||
sops = {
|
||||
age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
|
||||
|
||||
secrets.tailscale_auth_key = {
|
||||
sopsFile = "${inputs.self}/users/secrets.yaml";
|
||||
owner = "root";
|
||||
mode = "0400";
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
{
|
||||
nix.settings = {
|
||||
trusted-substituters = [ "http://cache.tmmworkshop.com" ];
|
||||
substituters = [ "http://cache.tmmworkshop.com/?priority=1&want-mass-query=true" ];
|
||||
trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
|
||||
};
|
||||
}
|
||||
@@ -1,11 +1,32 @@
|
||||
{ lib, pkgs, ... }:
|
||||
{
|
||||
systemd = {
|
||||
services."autopull@dotfiles" = {
|
||||
requires = [ "multi-user.target" ];
|
||||
after = [ "multi-user.target" ];
|
||||
description = "Pull the latest data for dotfiles";
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
User = "root";
|
||||
WorkingDirectory = /root/dotfiles;
|
||||
ExecStart = "${pkgs.git}/bin/git pull --all --prune";
|
||||
};
|
||||
};
|
||||
timers."autopull@dotfiles" = {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnBootSec = "1h";
|
||||
OnUnitActiveSec = "1h";
|
||||
Unit = "autopull@dotfiles.service";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
system.autoUpgrade = {
|
||||
enable = true;
|
||||
enable = lib.mkDefault true;
|
||||
flags = [ "--accept-flake-config" ];
|
||||
randomizedDelaySec = "1h";
|
||||
persistent = true;
|
||||
flake = "git+https://gitea.tmmworkshop.com/richie/dotfiles?ref=main";
|
||||
allowReboot = true;
|
||||
dates = "Sat *-*-* 06:00:00";
|
||||
flake = "github:RAD-Development/nix-dotfiles";
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
nixpkgs.hostPlatform = {
|
||||
system = "x86_64-linux";
|
||||
gcc = {
|
||||
arch = "x86-64-v3";
|
||||
tune = "generic";
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
services.zerotierone = {
|
||||
enable = true;
|
||||
joinNetworks = [ "e4da7455b2ae64ca" ];
|
||||
};
|
||||
nix.settings = {
|
||||
trusted-substituters = [ "http://192.168.90.40:5000" ];
|
||||
substituters = [ "http://192.168.90.40:5000/?priority=1&want-mass-query=true" ];
|
||||
trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
|
||||
};
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
{
|
||||
inputs,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
{
|
||||
imports = [ ./snapshot.nix ];
|
||||
|
||||
boot.zfs = {
|
||||
package = lib.mkDefault pkgs.zfs_2_4;
|
||||
forceImportRoot = lib.mkDefault false;
|
||||
};
|
||||
|
||||
services = {
|
||||
snapshot_manager = {
|
||||
enable = lib.mkDefault true;
|
||||
PYTHONPATH = "${inputs.self}/";
|
||||
};
|
||||
|
||||
zfs = {
|
||||
trim.enable = lib.mkDefault true;
|
||||
autoScrub.enable = lib.mkDefault true;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
{
|
||||
inputs,
|
||||
pkgs,
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.services.snapshot_manager;
|
||||
snapshotManagerPackages =
|
||||
ps: with ps; [
|
||||
apprise
|
||||
typer
|
||||
];
|
||||
in
|
||||
{
|
||||
options = {
|
||||
services.snapshot_manager = {
|
||||
enable = lib.mkEnableOption "ZFS snapshot manager";
|
||||
path = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
default = ./snapshot_config.toml;
|
||||
description = "Path to the snapshot_manager TOML config.";
|
||||
};
|
||||
PYTHONPATH = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = ''
|
||||
the PYTHONPATH to use for the snapshot_manager service.
|
||||
'';
|
||||
};
|
||||
EnvironmentFile = lib.mkOption {
|
||||
type = lib.types.nullOr (lib.types.coercedTo lib.types.path toString lib.types.str);
|
||||
default = null;
|
||||
description = ''
|
||||
Single environment file for the service (e.g. /etc/snapshot-manager/env).
|
||||
Use a leading "-" to ignore if missing (systemd feature).
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
nixpkgs.overlays = [
|
||||
(final: _prev: {
|
||||
snapshot_manager_python = final.python314.withPackages snapshotManagerPackages;
|
||||
snapshot_manager_test_python = final.python314.withPackages (
|
||||
ps:
|
||||
snapshotManagerPackages ps
|
||||
++ (with ps; [
|
||||
pyfakefs
|
||||
pytest
|
||||
pytest-asyncio
|
||||
pytest-mock
|
||||
pytest-xdist
|
||||
])
|
||||
);
|
||||
snapshot_manager_tests =
|
||||
final.runCommand "snapshot-manager-tests"
|
||||
{
|
||||
nativeBuildInputs = [ final.snapshot_manager_test_python ];
|
||||
}
|
||||
''
|
||||
export HOME="$TMPDIR"
|
||||
cd ${inputs.self}
|
||||
pytest \
|
||||
-o cache_dir="$TMPDIR/pytest-cache" \
|
||||
tests/test_common.py \
|
||||
tests/test_signal_alert.py \
|
||||
tests/test_snapshot_manager.py \
|
||||
tests/test_zfs.py
|
||||
touch "$out"
|
||||
'';
|
||||
})
|
||||
];
|
||||
|
||||
system.checks = [ pkgs.snapshot_manager_tests ];
|
||||
|
||||
systemd = {
|
||||
services.snapshot_manager = {
|
||||
description = "ZFS Snapshot Manager";
|
||||
requires = [ "zfs-import.target" ];
|
||||
after = [ "zfs-import.target" ];
|
||||
path = [ pkgs.zfs ];
|
||||
environment = {
|
||||
PYTHONPATH = cfg.PYTHONPATH;
|
||||
};
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = "${pkgs.snapshot_manager_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
|
||||
}
|
||||
// lib.optionalAttrs (cfg.EnvironmentFile != null) {
|
||||
EnvironmentFile = cfg.EnvironmentFile;
|
||||
};
|
||||
};
|
||||
timers.snapshot_manager = {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnBootSec = "15m";
|
||||
OnUnitActiveSec = "15m";
|
||||
Unit = "snapshot_manager.service";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
source "https://rubygems.org"
|
||||
|
||||
# The github-pages gem pins all compatible versions of Jekyll and its plugins
|
||||
gem "github-pages", group: :jekyll_plugins
|
||||
@@ -1,23 +0,0 @@
|
||||
title: "Richie Cahill"
|
||||
description: "ALL THE CHAOS THAT I CANT DO AT WORK"
|
||||
baseurl: "/dotfiles"
|
||||
url: "https://richiecahill.github.io"
|
||||
|
||||
remote_theme: pages-themes/hacker@v0.2.0
|
||||
plugins:
|
||||
- jekyll-feed
|
||||
- jekyll-remote-theme
|
||||
- jekyll-seo-tag
|
||||
- jekyll-sitemap
|
||||
- jekyll-paginate
|
||||
|
||||
paginate: 5
|
||||
paginate_path: "/page:num"
|
||||
|
||||
author:
|
||||
name: "Richie Cahill"
|
||||
email: "richie@tmmworkshop.com"
|
||||
|
||||
social_links:
|
||||
github: "RichieCahill"
|
||||
website: "https://tmmworkshop.com"
|
||||
@@ -1,13 +0,0 @@
|
||||
# The MONOREPO experiment
|
||||
|
||||
Im testing a [MONOREPO](https://en.wikipedia.org/wiki/Monorepo) because Phil said this was a bad idea. To that i say hold my beer.
|
||||
|
||||
In all seriousness, I Think that for a small dev team/solo dev. The simplicity is worth higher barer to entry. One of my most annoying processes was updating my system tools. I had to build my update in a feature branch and then merge it into my main branch. then go to my dotfiles create a feature branch update the system tools merge it into main.
|
||||
|
||||
It will be starting with my Nix Dotfiles Python tools and now my blog.
|
||||
|
||||
I will be reaching ot to phil on 2030-10-31 and 2035-10-31 to give him updates on the progress.
|
||||
|
||||
Known Issues:
|
||||
|
||||
- the python tests are running on the current derivation not the one the derivation im updating to.
|
||||
@@ -1,17 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: "Welcome"
|
||||
---
|
||||
|
||||
Welcome to my build logs, notes, and experiments.
|
||||
|
||||
You can read my latest posts below
|
||||
|
||||
<ul>
|
||||
{% for post in site.posts %}
|
||||
<li>
|
||||
<a href="{{ post.url | relative_url }}">{{ post.title }}</a>
|
||||
<small>— {{ post.date | date: "%Y-%m-%d" }}</small>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
@@ -1,76 +0,0 @@
|
||||
# ZFS failed root import recovery
|
||||
|
||||
## Fast path
|
||||
|
||||
If the machine fails to boot because ZFS refuses to import `root_pool`:
|
||||
|
||||
### GRUB
|
||||
|
||||
1. At the bootloader menu, select the normal NixOS entry.
|
||||
2. Press `e`.
|
||||
3. Find the line that starts with `linux`.
|
||||
4. Append this to the end of that line:
|
||||
|
||||
```text
|
||||
zfs_force=1
|
||||
```
|
||||
|
||||
5. Boot once with `Ctrl+x` or `F10`.
|
||||
|
||||
### systemd-boot
|
||||
|
||||
1. At the bootloader menu, highlight the normal NixOS entry.
|
||||
2. Press `e`.
|
||||
3. Append this to the end of the options line:
|
||||
|
||||
```text
|
||||
zfs_force=1
|
||||
```
|
||||
|
||||
4. Press `Enter` to boot once.
|
||||
|
||||
## After boot
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
sudo zpool status
|
||||
sudo zpool import
|
||||
journalctl -b | rg "ZFS|zfs|import|root_pool"
|
||||
```
|
||||
|
||||
## Expected result
|
||||
|
||||
`sudo zpool status` should show `root_pool` as `ONLINE`.
|
||||
|
||||
## Reboot test
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
sudo reboot
|
||||
```
|
||||
|
||||
Do not add `zfs_force=1` the second time.
|
||||
|
||||
## If it still fails
|
||||
|
||||
Boot once more with:
|
||||
|
||||
```text
|
||||
zfs_force=1
|
||||
```
|
||||
|
||||
Then run:
|
||||
|
||||
```bash
|
||||
sudo zpool status -v
|
||||
sudo zpool history | tail -n 50
|
||||
journalctl -b | rg "ZFS|zfs|import|root_pool"
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
- Root pool name is `root_pool`.
|
||||
- This is a one-time recovery path after disk moves, controller changes, dirty exports, or interrupted imports.
|
||||
- Some hosts also need the LUKS unlock USB key inserted before boot.
|
||||
@@ -1,3 +0,0 @@
|
||||
# esphome
|
||||
/.esphome/
|
||||
/secrets.yaml
|
||||
@@ -1,132 +0,0 @@
|
||||
esphome:
|
||||
name: batteries
|
||||
friendly_name: batteries
|
||||
|
||||
esp32:
|
||||
board: esp32dev
|
||||
framework:
|
||||
type: arduino
|
||||
|
||||
logger:
|
||||
|
||||
api:
|
||||
encryption:
|
||||
key: !secret api_key
|
||||
|
||||
external_components:
|
||||
- source: github://syssi/esphome-jk-bms@main
|
||||
|
||||
ota:
|
||||
- platform: esphome
|
||||
password: !secret ota_password
|
||||
|
||||
wifi:
|
||||
ssid: !secret wifi_ssid
|
||||
password: !secret wifi_password
|
||||
fast_connect: on
|
||||
|
||||
captive_portal:
|
||||
|
||||
esp32_ble_tracker:
|
||||
scan_parameters:
|
||||
interval: 1100ms
|
||||
window: 1100ms
|
||||
active: true
|
||||
|
||||
ble_client:
|
||||
- mac_address: "C8:47:80:29:0F:DB"
|
||||
id: jk_ble0
|
||||
|
||||
jk_bms_ble:
|
||||
- ble_client_id: jk_ble0
|
||||
protocol_version: JK02_32S
|
||||
throttle: 1s
|
||||
id: jk_bms0
|
||||
|
||||
button:
|
||||
- platform: jk_bms_ble
|
||||
retrieve_settings:
|
||||
name: "JK0 retrieve settings"
|
||||
retrieve_device_info:
|
||||
name: "JK0 retrieve device info"
|
||||
|
||||
sensor:
|
||||
- platform: jk_bms_ble
|
||||
jk_bms_ble_id: jk_bms0
|
||||
total_voltage:
|
||||
name: "JK0 Total Voltage"
|
||||
state_of_charge:
|
||||
name: "JK0 SoC"
|
||||
charging_power:
|
||||
name: "JK0 charging power"
|
||||
discharging_power:
|
||||
name: "JK0 discharging power"
|
||||
temperature_sensor_1:
|
||||
name: "JK0 Temp 1"
|
||||
temperature_sensor_2:
|
||||
name: "JK0 Temp 2"
|
||||
balancing:
|
||||
name: "JK0 balancing"
|
||||
total_runtime:
|
||||
name: "JK0 total runtime"
|
||||
balancing_current:
|
||||
name: "JK0 balancing current"
|
||||
delta_cell_voltage:
|
||||
name: "JK0 cell delta voltage"
|
||||
average_cell_voltage:
|
||||
name: "JK0 cell average voltage"
|
||||
cell_voltage_1:
|
||||
name: "JK0 cell voltage 1"
|
||||
cell_voltage_2:
|
||||
name: "JK0 cell voltage 2"
|
||||
cell_voltage_3:
|
||||
name: "JK0 cell voltage 3"
|
||||
cell_voltage_4:
|
||||
name: "JK0 cell voltage 4"
|
||||
cell_voltage_5:
|
||||
name: "JK0 cell voltage 5"
|
||||
cell_voltage_6:
|
||||
name: "JK0 cell voltage 6"
|
||||
cell_voltage_7:
|
||||
name: "JK0 cell voltage 7"
|
||||
cell_voltage_8:
|
||||
name: "JK0 cell voltage 8"
|
||||
cell_resistance_1:
|
||||
name: "JK0 cell resistance 1"
|
||||
cell_resistance_2:
|
||||
name: "JK0 cell resistance 2"
|
||||
cell_resistance_3:
|
||||
name: "JK0 cell resistance 3"
|
||||
cell_resistance_4:
|
||||
name: "JK0 cell resistance 4"
|
||||
cell_resistance_5:
|
||||
name: "JK0 cell resistance 5"
|
||||
cell_resistance_6:
|
||||
name: "JK0 cell resistance 6"
|
||||
cell_resistance_7:
|
||||
name: "JK0 cell resistance 7"
|
||||
cell_resistance_8:
|
||||
name: "JK0 cell resistance 8"
|
||||
total_charging_cycle_capacity:
|
||||
name: "JK0 total charging cycle capacity"
|
||||
|
||||
text_sensor:
|
||||
- platform: jk_bms_ble
|
||||
jk_bms_ble_id: jk_bms0
|
||||
errors:
|
||||
name: "JK0 Errors"
|
||||
|
||||
switch:
|
||||
- platform: jk_bms_ble
|
||||
jk_bms_ble_id: jk_bms0
|
||||
charging:
|
||||
name: "JK0 Charging"
|
||||
discharging:
|
||||
name: "JK0 Discharging"
|
||||
balancer:
|
||||
name: "JK0 Balancing"
|
||||
|
||||
- platform: ble_client
|
||||
ble_client_id: jk_ble0
|
||||
name: "JK0 enable bluetooth connection"
|
||||
id: ble_client_switch0
|
||||
@@ -1,132 +0,0 @@
|
||||
esphome:
|
||||
name: battery1
|
||||
friendly_name: battery1
|
||||
|
||||
esp32:
|
||||
board: esp32dev
|
||||
framework:
|
||||
type: arduino
|
||||
|
||||
logger:
|
||||
|
||||
api:
|
||||
encryption:
|
||||
key: !secret api_key
|
||||
|
||||
external_components:
|
||||
- source: github://syssi/esphome-jk-bms@main
|
||||
|
||||
ota:
|
||||
- platform: esphome
|
||||
password: !secret ota_password
|
||||
|
||||
wifi:
|
||||
ssid: !secret wifi_ssid
|
||||
password: !secret wifi_password
|
||||
fast_connect: on
|
||||
|
||||
captive_portal:
|
||||
|
||||
esp32_ble_tracker:
|
||||
scan_parameters:
|
||||
interval: 1100ms
|
||||
window: 1100ms
|
||||
active: true
|
||||
|
||||
ble_client:
|
||||
- mac_address: "C8:47:80:37:9D:DD"
|
||||
id: jk_ble1
|
||||
|
||||
jk_bms_ble:
|
||||
- ble_client_id: jk_ble1
|
||||
protocol_version: JK02_32S
|
||||
throttle: 1s
|
||||
id: jk_bms1
|
||||
|
||||
button:
|
||||
- platform: jk_bms_ble
|
||||
retrieve_settings:
|
||||
name: "JK1 retrieve settings"
|
||||
retrieve_device_info:
|
||||
name: "JK1 retrieve device info"
|
||||
|
||||
sensor:
|
||||
- platform: jk_bms_ble
|
||||
jk_bms_ble_id: jk_bms1
|
||||
total_voltage:
|
||||
name: "JK1 Total Voltage"
|
||||
state_of_charge:
|
||||
name: "JK1 SoC"
|
||||
charging_power:
|
||||
name: "JK1 charging power"
|
||||
discharging_power:
|
||||
name: "JK1 discharging power"
|
||||
temperature_sensor_1:
|
||||
name: "JK1 Temp 1"
|
||||
temperature_sensor_2:
|
||||
name: "JK1 Temp 2"
|
||||
balancing:
|
||||
name: "JK1 balancing"
|
||||
total_runtime:
|
||||
name: "JK1 total runtime"
|
||||
balancing_current:
|
||||
name: "JK1 balancing current"
|
||||
delta_cell_voltage:
|
||||
name: "JK1 cell delta voltage"
|
||||
average_cell_voltage:
|
||||
name: "JK1 cell average voltage"
|
||||
cell_voltage_1:
|
||||
name: "JK1 cell voltage 1"
|
||||
cell_voltage_2:
|
||||
name: "JK1 cell voltage 2"
|
||||
cell_voltage_3:
|
||||
name: "JK1 cell voltage 3"
|
||||
cell_voltage_4:
|
||||
name: "JK1 cell voltage 4"
|
||||
cell_voltage_5:
|
||||
name: "JK1 cell voltage 5"
|
||||
cell_voltage_6:
|
||||
name: "JK1 cell voltage 6"
|
||||
cell_voltage_7:
|
||||
name: "JK1 cell voltage 7"
|
||||
cell_voltage_8:
|
||||
name: "JK1 cell voltage 8"
|
||||
cell_resistance_1:
|
||||
name: "JK1 cell resistance 1"
|
||||
cell_resistance_2:
|
||||
name: "JK1 cell resistance 2"
|
||||
cell_resistance_3:
|
||||
name: "JK1 cell resistance 3"
|
||||
cell_resistance_4:
|
||||
name: "JK1 cell resistance 4"
|
||||
cell_resistance_5:
|
||||
name: "JK1 cell resistance 5"
|
||||
cell_resistance_6:
|
||||
name: "JK1 cell resistance 6"
|
||||
cell_resistance_7:
|
||||
name: "JK1 cell resistance 7"
|
||||
cell_resistance_8:
|
||||
name: "JK1 cell resistance 8"
|
||||
total_charging_cycle_capacity:
|
||||
name: "JK1 total charging cycle capacity"
|
||||
|
||||
text_sensor:
|
||||
- platform: jk_bms_ble
|
||||
jk_bms_ble_id: jk_bms1
|
||||
errors:
|
||||
name: "JK1 Errors"
|
||||
|
||||
switch:
|
||||
- platform: jk_bms_ble
|
||||
jk_bms_ble_id: jk_bms1
|
||||
charging:
|
||||
name: "JK1 Charging"
|
||||
discharging:
|
||||
name: "JK1 Discharging"
|
||||
balancer:
|
||||
name: "JK1 Balancing"
|
||||
|
||||
- platform: ble_client
|
||||
ble_client_id: jk_ble1
|
||||
name: "JK1 enable bluetooth connection"
|
||||
id: ble_client_switch0
|
||||
@@ -1,48 +0,0 @@
|
||||
esphome:
|
||||
name: "environment"
|
||||
friendly_name: "environment"
|
||||
|
||||
esp32:
|
||||
board: esp32dev
|
||||
framework:
|
||||
type: arduino
|
||||
|
||||
i2c:
|
||||
sda: GPIO21
|
||||
scl: GPIO22
|
||||
scan: True
|
||||
id: bus_a
|
||||
|
||||
sensor:
|
||||
- platform: aht10
|
||||
i2c_id: bus_a
|
||||
address: 0x38
|
||||
variant: AHT20
|
||||
temperature:
|
||||
name: "environment Temperature"
|
||||
id: aht10_temperature
|
||||
humidity:
|
||||
name: "environment Humidity"
|
||||
id: aht10_humidity
|
||||
update_interval: 5s
|
||||
|
||||
web_server:
|
||||
port: 80
|
||||
|
||||
logger:
|
||||
level: DEBUG
|
||||
|
||||
api:
|
||||
encryption:
|
||||
key: !secret api_key
|
||||
|
||||
ota:
|
||||
- platform: esphome
|
||||
password: !secret ota_password
|
||||
|
||||
wifi:
|
||||
ssid: !secret wifi_ssid
|
||||
password: !secret wifi_password
|
||||
fast_connect: on
|
||||
|
||||
captive_portal:
|
||||
Generated
+261
-40
@@ -1,22 +1,74 @@
|
||||
{
|
||||
"nodes": {
|
||||
"disko": {
|
||||
"firefox-addons": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils",
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1781152676,
|
||||
"narHash": "sha256-RxWs5ND31KzTG7wvMM+PMfUjyNpmIEr999lqNARaM5o=",
|
||||
"dir": "pkgs/firefox-addons",
|
||||
"lastModified": 1736279099,
|
||||
"narHash": "sha256-WXbwHw+JeIOo3KvGDjrfsuwj8JWBUkFFxmlm07w9i7Q=",
|
||||
"owner": "rycee",
|
||||
"repo": "nur-expressions",
|
||||
"rev": "0ad9cf75bae6a9290e9c83cef19f60595f189979",
|
||||
"type": "gitlab"
|
||||
},
|
||||
"original": {
|
||||
"dir": "pkgs/firefox-addons",
|
||||
"owner": "rycee",
|
||||
"repo": "nur-expressions",
|
||||
"type": "gitlab"
|
||||
}
|
||||
},
|
||||
"flake-compat": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1717312683,
|
||||
"narHash": "sha256-FrlieJH50AuvagamEvWMIE6D2OAnERuDboFDYAED/dE=",
|
||||
"owner": "nix-community",
|
||||
"repo": "disko",
|
||||
"rev": "ff8702b4de27f72b4c78573dfb89ec74e36abdf1",
|
||||
"repo": "flake-compat",
|
||||
"rev": "38fd3954cf65ce6faf3d0d45cd26059e059f07ea",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-community",
|
||||
"repo": "disko",
|
||||
"repo": "flake-compat",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-utils": {
|
||||
"locked": {
|
||||
"lastModified": 1629284811,
|
||||
"narHash": "sha256-JHgasjPR0/J1J3DRm4KxM4zTyAj4IOJY8vIl75v/kPI=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "c5d161cc0af116a2e17f54316f0bf43f0819785c",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-utils_2": {
|
||||
"inputs": {
|
||||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1731533236,
|
||||
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
@@ -27,11 +79,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1788651960,
|
||||
"narHash": "sha256-v9wJd32eZ2bvhBzVOd7TIjLQd011P7nwOhjKtWlci5I=",
|
||||
"lastModified": 1736277415,
|
||||
"narHash": "sha256-kPDXF6cIPsVqSK08XF5EC6KM7BdMnM9vtJDzsnf+lLU=",
|
||||
"owner": "nix-community",
|
||||
"repo": "home-manager",
|
||||
"rev": "2c0350c759688177331b8f5242311fae8877bdb3",
|
||||
"rev": "5c4302313d9207f7ec0886d68f8ff4a3c71209a1",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -40,16 +92,58 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixos-hardware": {
|
||||
"nix-github-actions": {
|
||||
"inputs": {
|
||||
"nixpkgs": "nixpkgs"
|
||||
"nixpkgs": [
|
||||
"system_tools",
|
||||
"poetry2nix",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1788860136,
|
||||
"narHash": "sha256-MhPMOFV4pVkygWEbQ8t1De/uQ9cWF1u++tRe2L5tG48=",
|
||||
"lastModified": 1729742964,
|
||||
"narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
|
||||
"owner": "nix-community",
|
||||
"repo": "nix-github-actions",
|
||||
"rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-community",
|
||||
"repo": "nix-github-actions",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixos-cosmic": {
|
||||
"inputs": {
|
||||
"flake-compat": "flake-compat",
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
],
|
||||
"nixpkgs-stable": "nixpkgs-stable",
|
||||
"rust-overlay": "rust-overlay"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1736214624,
|
||||
"narHash": "sha256-Pi70vbASZ1O9cR8RO5d2hBiNjIJBKKLoABl4sxWyOgg=",
|
||||
"owner": "lilyinstarlight",
|
||||
"repo": "nixos-cosmic",
|
||||
"rev": "0830abeebf3b2d1bae44652ffb2c89cf0d56ddaa",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "lilyinstarlight",
|
||||
"repo": "nixos-cosmic",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixos-hardware": {
|
||||
"locked": {
|
||||
"lastModified": 1736283893,
|
||||
"narHash": "sha256-BG1FfTexFwNty5VhYjaQLMR6CMPfI3QRcaZrFQYu2EM=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixos-hardware",
|
||||
"rev": "62173785b9a18c78b4a15aca2623d02bceb9d077",
|
||||
"rev": "4f339f6be2b61662f957c2ee9eda0fa597d8a6d6",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -61,24 +155,27 @@
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1767892417,
|
||||
"narHash": "sha256-8bW3q88CEg2u4hSP66Vf4lpbLonHz7hqDNBMcCY7E9U=",
|
||||
"rev": "3497aa5c9457a9d88d71fa93a4a8368816fbeeba",
|
||||
"type": "tarball",
|
||||
"url": "https://releases.nixos.org/nixos/unstable/nixos-26.05pre924538.3497aa5c9457/nixexprs.tar.xz"
|
||||
"lastModified": 1736012469,
|
||||
"narHash": "sha256-/qlNWm/IEVVH7GfgAIyP6EsVZI6zjAx1cV5zNyrs+rI=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "8f3e1f807051e32d8c95cd12b9b421623850a34d",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"type": "tarball",
|
||||
"url": "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz"
|
||||
"owner": "nixos",
|
||||
"ref": "nixos-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs-master": {
|
||||
"locked": {
|
||||
"lastModified": 1788892992,
|
||||
"narHash": "sha256-cIMFh9gyU4/aLeB3JCcsWM3tTAvD9pAq9Smr1Wa8aIU=",
|
||||
"lastModified": 1736294919,
|
||||
"narHash": "sha256-6Epm0TQkXt0DcS3akYSeT0PzYMpgkZf0V14+5UGe6oE=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "dff6994123e257ec9901c271bc2b52e64d7c8f05",
|
||||
"rev": "ed6183b173fe27e1d7dceef1ddca2aa64d07aad4",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -89,6 +186,22 @@
|
||||
}
|
||||
},
|
||||
"nixpkgs-stable": {
|
||||
"locked": {
|
||||
"lastModified": 1736061677,
|
||||
"narHash": "sha256-DjkQPnkAfd7eB522PwnkGhOMuT9QVCZspDpJJYyOj60=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "cbd8ec4de4469333c82ff40d057350c30e9f7d36",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-24.11",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs-stable_2": {
|
||||
"locked": {
|
||||
"lastModified": 1735563628,
|
||||
"narHash": "sha256-OnSAY7XDSx7CtDoqNh8jwVwh4xNL/2HaJxGjryLWzX8=",
|
||||
@@ -104,32 +217,67 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs_2": {
|
||||
"poetry2nix": {
|
||||
"inputs": {
|
||||
"flake-utils": [
|
||||
"system_tools",
|
||||
"flake-utils"
|
||||
],
|
||||
"nix-github-actions": "nix-github-actions",
|
||||
"nixpkgs": [
|
||||
"system_tools",
|
||||
"nixpkgs"
|
||||
],
|
||||
"systems": "systems_2",
|
||||
"treefmt-nix": "treefmt-nix"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1788752844,
|
||||
"narHash": "sha256-VaWGJ6+cIYN2erfSecbRV+4ljI185Ty2wUrXyvQbgOw=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "dc5d91f840324650bac8c379428c7037a416959a",
|
||||
"lastModified": 1731205797,
|
||||
"narHash": "sha256-F7N1mxH1VrkVNHR3JGNMRvp9+98KYO4b832KS8Gl2xI=",
|
||||
"owner": "nix-community",
|
||||
"repo": "poetry2nix",
|
||||
"rev": "f554d27c1544d9c56e5f1f8e2b8aff399803674e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nixos",
|
||||
"ref": "nixos-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"owner": "nix-community",
|
||||
"repo": "poetry2nix",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"disko": "disko",
|
||||
"firefox-addons": "firefox-addons",
|
||||
"home-manager": "home-manager",
|
||||
"nixos-cosmic": "nixos-cosmic",
|
||||
"nixos-hardware": "nixos-hardware",
|
||||
"nixpkgs": "nixpkgs_2",
|
||||
"nixpkgs": "nixpkgs",
|
||||
"nixpkgs-master": "nixpkgs-master",
|
||||
"nixpkgs-stable": "nixpkgs-stable",
|
||||
"nixpkgs-stable": "nixpkgs-stable_2",
|
||||
"sops-nix": "sops-nix",
|
||||
"systems": "systems"
|
||||
"system_tools": "system_tools",
|
||||
"systems": "systems_3"
|
||||
}
|
||||
},
|
||||
"rust-overlay": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixos-cosmic",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1736130662,
|
||||
"narHash": "sha256-z+WGez9oTR2OsiUWE5ZhIpETqM1ogrv6Xcd24WFi6KQ=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "2f5d4d9cd31cc02c36e51cb2e21c4b25c4f78c52",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"sops-nix": {
|
||||
@@ -139,11 +287,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1788337237,
|
||||
"narHash": "sha256-gkSH8VUtCo6hnysNmb9DbTuDepH2t5pv+QWjP75xKAk=",
|
||||
"lastModified": 1736203741,
|
||||
"narHash": "sha256-eSjkBwBdQk+TZWFlLbclF2rAh4JxbGg8az4w/Lfe7f4=",
|
||||
"owner": "Mic92",
|
||||
"repo": "sops-nix",
|
||||
"rev": "fbf759290e0cb0a98dfc813a4eb7d53ad1dacb57",
|
||||
"rev": "c9c88f08e3ee495e888b8d7c8624a0b2519cb773",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -152,7 +300,58 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"system_tools": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils_2",
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
],
|
||||
"poetry2nix": "poetry2nix"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1733083480,
|
||||
"narHash": "sha256-B13faNyBbA3MeI7Jp6pFVbp58rI2Rx5Uvd83csW2p48=",
|
||||
"owner": "RichieCahill",
|
||||
"repo": "system_tools",
|
||||
"rev": "bc357d8fabd83c1423611829091e5b1d86dd913c",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "RichieCahill",
|
||||
"repo": "system_tools",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems_2": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"id": "systems",
|
||||
"type": "indirect"
|
||||
}
|
||||
},
|
||||
"systems_3": {
|
||||
"locked": {
|
||||
"lastModified": 1689347949,
|
||||
"narHash": "sha256-12tWmuL2zgBgZkdoB6qXZsgJEH9LR3oUgpaQq2RbI80=",
|
||||
@@ -166,6 +365,28 @@
|
||||
"repo": "default-linux",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"treefmt-nix": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"system_tools",
|
||||
"poetry2nix",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1730120726,
|
||||
"narHash": "sha256-LqHYIxMrl/1p3/kvm2ir925tZ8DkI0KA10djk8wecSk=",
|
||||
"owner": "numtide",
|
||||
"repo": "treefmt-nix",
|
||||
"rev": "9ef337e492a5555d8e17a51c911ff1f02635be15",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "treefmt-nix",
|
||||
"type": "github"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
|
||||
@@ -4,12 +4,14 @@
|
||||
nixConfig = {
|
||||
extra-substituters = [
|
||||
"https://cache.nixos.org/?priority=2&want-mass-query=true"
|
||||
"https://cache.tmmworkshop.com/?priority=2&want-mass-query=true"
|
||||
"https://nix-community.cachix.org/?priority=10&want-mass-query=true"
|
||||
];
|
||||
extra-trusted-public-keys = [
|
||||
"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" # cspell:disable-line
|
||||
"nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" # cspell:disable-line
|
||||
"cache-nix-dot:Od9KN34LXc6Lu7y1ozzV1kIXZa8coClozgth/SYE7dU=" # cspell:disable-line
|
||||
"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
|
||||
"cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA="
|
||||
"nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
|
||||
"cache-nix-dot:Od9KN34LXc6Lu7y1ozzV1kIXZa8coClozgth/SYE7dU="
|
||||
];
|
||||
};
|
||||
|
||||
@@ -26,87 +28,82 @@
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
|
||||
firefox-addons = {
|
||||
url = "gitlab:rycee/nur-expressions?dir=pkgs/firefox-addons";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
|
||||
system_tools = {
|
||||
url = "github:RichieCahill/system_tools";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
|
||||
nixos-cosmic = {
|
||||
url = "github:lilyinstarlight/nixos-cosmic";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
|
||||
sops-nix = {
|
||||
url = "github:Mic92/sops-nix";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
|
||||
disko = {
|
||||
url = "github:nix-community/disko";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
};
|
||||
|
||||
outputs =
|
||||
{
|
||||
self,
|
||||
nixpkgs,
|
||||
home-manager,
|
||||
systems,
|
||||
sops-nix,
|
||||
...
|
||||
}@inputs:
|
||||
let
|
||||
inherit (self) outputs;
|
||||
lib = nixpkgs.lib // home-manager.lib;
|
||||
forEachSystem = f: lib.genAttrs (import systems) (system: f pkgsFor.${system});
|
||||
pkgsFor = lib.genAttrs (import systems) (
|
||||
system:
|
||||
outputs = {
|
||||
self,
|
||||
nixpkgs,
|
||||
home-manager,
|
||||
systems,
|
||||
nixos-cosmic,
|
||||
sops-nix,
|
||||
...
|
||||
} @ inputs: let
|
||||
inherit (self) outputs;
|
||||
lib = nixpkgs.lib // home-manager.lib;
|
||||
forEachSystem = f: lib.genAttrs (import systems) (system: f pkgsFor.${system});
|
||||
pkgsFor = lib.genAttrs (import systems) (
|
||||
system:
|
||||
import nixpkgs {
|
||||
inherit system;
|
||||
overlays = builtins.attrValues outputs.overlays;
|
||||
config.allowUnfree = true;
|
||||
}
|
||||
);
|
||||
in
|
||||
{
|
||||
inherit lib;
|
||||
overlays = import ./overlays { inherit inputs outputs; };
|
||||
);
|
||||
in {
|
||||
inherit lib;
|
||||
overlays = import ./overlays {inherit inputs outputs;};
|
||||
|
||||
devShells = forEachSystem (pkgs: import ./shell.nix { inherit pkgs; });
|
||||
formatter = forEachSystem (pkgs: pkgs.treefmt);
|
||||
packages = forEachSystem (
|
||||
pkgs:
|
||||
let
|
||||
installer = pkgs.callPackage ./python/installer/package.nix { };
|
||||
installer-nixos = pkgs.callPackage ./python/installer/package.nix { patchElf = false; };
|
||||
in
|
||||
{
|
||||
inherit installer installer-nixos;
|
||||
default = installer;
|
||||
}
|
||||
// lib.optionalAttrs (pkgs.stdenv.hostPlatform.system == "x86_64-linux") {
|
||||
iso = self.nixosConfigurations.iso.config.system.build.isoImage;
|
||||
}
|
||||
);
|
||||
apps = forEachSystem (
|
||||
pkgs:
|
||||
let
|
||||
system = pkgs.stdenv.hostPlatform.system;
|
||||
installer = {
|
||||
type = "app";
|
||||
program = "${self.packages.${system}.installer}/bin/nixos-installer";
|
||||
meta.description = "One-file NixOS ZFS installer.";
|
||||
};
|
||||
in
|
||||
{
|
||||
inherit installer;
|
||||
default = installer;
|
||||
}
|
||||
);
|
||||
devShells = forEachSystem (pkgs: import ./shell.nix {inherit pkgs;});
|
||||
formatter = forEachSystem (pkgs: pkgs.alejandra);
|
||||
|
||||
nixosConfigurations =
|
||||
let
|
||||
hosts = builtins.attrNames (
|
||||
lib.filterAttrs (_: type: type == "directory") (builtins.readDir ./systems)
|
||||
);
|
||||
mkHost =
|
||||
name:
|
||||
lib.nixosSystem {
|
||||
modules = [ ./systems/${name} ];
|
||||
specialArgs = { inherit inputs outputs; };
|
||||
};
|
||||
in
|
||||
lib.genAttrs hosts mkHost;
|
||||
nixosConfigurations = {
|
||||
bob = lib.nixosSystem {
|
||||
modules = [
|
||||
./systems/bob
|
||||
];
|
||||
specialArgs = {inherit inputs outputs;};
|
||||
};
|
||||
jeeves = lib.nixosSystem {
|
||||
modules = [
|
||||
./systems/jeeves
|
||||
];
|
||||
specialArgs = {inherit inputs outputs;};
|
||||
};
|
||||
rhapsody-in-green = lib.nixosSystem {
|
||||
modules = [
|
||||
./systems/rhapsody-in-green
|
||||
];
|
||||
specialArgs = {inherit inputs outputs;};
|
||||
};
|
||||
muninn = lib.nixosSystem {
|
||||
modules = [
|
||||
./systems/muninn
|
||||
];
|
||||
specialArgs = {inherit inputs outputs;};
|
||||
};
|
||||
router = lib.nixosSystem {
|
||||
modules = [./systems/router];
|
||||
specialArgs = {inherit inputs outputs;};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,19 +1,16 @@
|
||||
{ inputs, ... }:
|
||||
{
|
||||
{inputs, ...}: {
|
||||
# When applied, the stable nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.stable'
|
||||
stable = final: _prev: {
|
||||
stable = import inputs.nixpkgs-stable {
|
||||
system = final.stdenv.hostPlatform.system;
|
||||
system = final.system;
|
||||
config.allowUnfree = true;
|
||||
};
|
||||
};
|
||||
# When applied, the master nixpkgs set (declared in the flake inputs) will be accessible through 'pkgs.master'
|
||||
master = final: _prev: {
|
||||
master = import inputs.nixpkgs-master {
|
||||
system = final.stdenv.hostPlatform.system;
|
||||
system = final.system;
|
||||
config.allowUnfree = true;
|
||||
};
|
||||
};
|
||||
|
||||
patches = import ./patches;
|
||||
}
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
# Package patches
|
||||
|
||||
Each package follows the [GnuTLS layout](gnutls/README.md):
|
||||
|
||||
- `default.nix` applies the patch through the package overlay.
|
||||
- A descriptive `.patch` file contains the standalone upstream source change.
|
||||
- `README.md` explains the problem, scope, reproduction, upstream status,
|
||||
Nix integration, and recorded validation limits.
|
||||
- Companion `verify-*` tools live beside the patch when needed; otherwise
|
||||
the README gives commands for the package's existing tests.
|
||||
|
||||
Keep package-specific evidence in its directory. Patch headers explain the
|
||||
change independently of Nix, and `default.nix` preserves existing patches.
|
||||
|
||||
| Package | Repair |
|
||||
| --- | --- |
|
||||
| [Abseil](abseil/README.md) | Public BMI2 header in Electron, Deno, and Signal's vendored copies |
|
||||
| [Backrefs](backrefs/README.md) | Match the regex timeout's CPU clock |
|
||||
| [GnuTLS](gnutls/README.md) | Wait for the UDP server socket before connecting |
|
||||
| [Jupyter Server](jupyter-server/README.md) | Exercise the correct shared future during reconnect |
|
||||
| [Prometheus](prometheus/README.md) | Complete parsing before inspecting the test editor state |
|
||||
| [pytest-xdist](pytest-xdist/README.md) | Check worker replacements and allow startup on loaded builders |
|
||||
| [SciPy](scipy/README.md) | Account for floating-point rounding in STFT tests |
|
||||
| [Sentry SDK](sentry-sdk/README.md) | Isolate SDK thread mocks from Python's threading module |
|
||||
| [Torchaudio](torchaudio/README.md) | Compare pitch-shift batches at appropriate precision |
|
||||
| [TorchCodec](torchcodec/README.md) | Match the reference MP3 encoder's sample format |
|
||||
|
||||
## Local NixOS integration
|
||||
|
||||
[`../default.nix`](../default.nix) imports this directory's
|
||||
[`default.nix`](default.nix), which wires each package's override into the
|
||||
package set. Abseil repairs several vendored copies and is gated on
|
||||
`x86-64-v3`; Prometheus patches its separate assets derivation; Python
|
||||
packages use `pythonPackagesExtensions`.
|
||||
|
||||
The [pytest-xdist directory](pytest-xdist/README.md) also owns its outer-worker
|
||||
limit and remote-worker event timeout. These package overrides add no skipped
|
||||
tests. Existing nixpkgs exclusions remain separate from these repairs.
|
||||
|
||||
The test-exclusion review used Python 3.14.7 and the pinned x86-64-v3 package
|
||||
set. Host-flake evaluation verified patch wiring, Python install checks,
|
||||
removal of the local skips, and Prometheus's reference to the patched assets.
|
||||
Jupyter and Sentry package tests used the preceding dependency set with the
|
||||
new package patch to avoid unrelated rebuilds after pytest-xdist changed.
|
||||
No complete NixOS rebuild was performed. Individual READMEs distinguish
|
||||
package builds, focused tests, and checks that have not been run.
|
||||
@@ -1,59 +0,0 @@
|
||||
# Abseil BMI2 public header
|
||||
|
||||
Vendored Abseil includes `bmi2intrin.h` directly when `__BMI2__` is enabled.
|
||||
Compilers reject that internal header without the umbrella-header setup.
|
||||
`bmi2-public-header.patch` includes `immintrin.h` instead, allowing builds
|
||||
that enable BMI2 through `-march=x86-64-v3`.
|
||||
|
||||
## Scope and behavior
|
||||
|
||||
The patch changes one include in
|
||||
`third_party/abseil-cpp/absl/container/internal/raw_hash_set.h`.
|
||||
`default.nix` applies it to Electron 43's unwrapped package, Deno's
|
||||
`librusty_v8`, and Signal's WebRTC dependency. It also supplies the patched
|
||||
Electron package to Signal. These overrides apply only to `x86-64-v3`.
|
||||
|
||||
The shared file path is relative to each vendoring project's source root,
|
||||
not the root of a standalone Abseil checkout. No hash-table algorithm or
|
||||
test exclusion changes.
|
||||
|
||||
## Reproduction and focused checks
|
||||
|
||||
From this directory, check and apply the patch to each vendored source tree:
|
||||
|
||||
```sh
|
||||
patch --dry-run --fuzz=0 -d /path/to/vendor-source -p1 < bmi2-public-header.patch
|
||||
patch --fuzz=0 -d /path/to/vendor-source -p1 < bmi2-public-header.patch
|
||||
```
|
||||
|
||||
A small compiler check isolates the header requirement. With GCC or Clang
|
||||
on x86-64, compile `#include <bmi2intrin.h>` using `-march=x86-64-v3`; the
|
||||
compiler rejects the direct include. Changing it to `#include <immintrin.h>`
|
||||
should compile. The full consumer builds below check integration with their
|
||||
actual toolchains.
|
||||
|
||||
## Upstream status
|
||||
|
||||
Abseil addressed this issue through
|
||||
[PR #2071](https://github.com/abseil/abseil-cpp/pull/2071), imported by its
|
||||
upstream workflow. That change uses `x86gprintrin.h`; this local variant uses
|
||||
the public `immintrin.h` umbrella header for the vendored toolchains.
|
||||
Keep the workaround until all three bundled copies include a compatible fix.
|
||||
This file is a local adaptation, not a verbatim copy of the upstream diff.
|
||||
|
||||
## Local NixOS integration and build results
|
||||
|
||||
[`../default.nix`](../default.nix) merges this directory's overlay fragment
|
||||
because it repairs multiple packages. From the repository root, the consumer
|
||||
build commands are:
|
||||
|
||||
```sh
|
||||
nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.deno
|
||||
nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.electron_43
|
||||
nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.signal-desktop
|
||||
```
|
||||
|
||||
The earlier extraction checked the vendored header snapshots and evaluated
|
||||
all three patch attachments. Those records do not establish successful full
|
||||
consumer rebuilds. No new compiler or consumer build was run for the layout
|
||||
change; the patch and override are unchanged.
|
||||
@@ -1,20 +0,0 @@
|
||||
Subject: [PATCH] abseil: include BMI2 intrinsics through the public header
|
||||
|
||||
GCC and Clang reject direct inclusion of bmi2intrin.h. Include immintrin.h
|
||||
instead so that the compiler supplies the required intrinsic setup when
|
||||
BMI2 is enabled, including builds targeting x86-64-v3.
|
||||
|
||||
This patch is shared by the vendored Abseil copies in Electron, rusty_v8
|
||||
(Deno), and Signal's WebRTC build.
|
||||
|
||||
--- a/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
|
||||
+++ b/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h
|
||||
@@ -226,7 +226,7 @@
|
||||
#endif
|
||||
|
||||
#ifdef __BMI2__
|
||||
-#include <bmi2intrin.h>
|
||||
+#include <immintrin.h>
|
||||
#endif // __BMI2__
|
||||
|
||||
namespace absl {
|
||||
@@ -1,38 +0,0 @@
|
||||
# Abseil accepted the upstream fix: https://github.com/abseil/abseil-cpp/pull/2071
|
||||
# Keep this workaround until Electron, Deno's rusty_v8, and Signal's WebRTC
|
||||
# update their bundled Abseil copies to include it.
|
||||
{ prev }:
|
||||
let
|
||||
patchAbseilBmi2Include =
|
||||
package:
|
||||
package.overrideAttrs (old: {
|
||||
# GCC and Clang require the public umbrella header for BMI2 intrinsics.
|
||||
patches = (old.patches or [ ]) ++ [ ./bmi2-public-header.patch ];
|
||||
});
|
||||
|
||||
electron43Unwrapped = patchAbseilBmi2Include prev.electron_43.unwrapped;
|
||||
electron43 = prev.electron_43.override {
|
||||
electron-unwrapped = electron43Unwrapped;
|
||||
};
|
||||
|
||||
signalCallPackage =
|
||||
path: args:
|
||||
let
|
||||
package = prev.callPackage path args;
|
||||
in
|
||||
if builtins.baseNameOf path == "webrtc.nix" then patchAbseilBmi2Include package else package;
|
||||
in
|
||||
prev.lib.optionalAttrs ((prev.stdenv.hostPlatform.gcc.arch or null) == "x86-64-v3") {
|
||||
deno =
|
||||
let
|
||||
librusty_v8 = patchAbseilBmi2Include prev.deno.passthru.librusty_v8;
|
||||
in
|
||||
prev.deno.override { inherit librusty_v8; };
|
||||
|
||||
electron_43 = electron43;
|
||||
|
||||
signal-desktop = prev.signal-desktop.override {
|
||||
electron_43 = electron43;
|
||||
callPackage = signalCallPackage;
|
||||
};
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
_final: prev:
|
||||
(import ./abseil { inherit prev; })
|
||||
// {
|
||||
gnutls = import ./gnutls { inherit (prev) gnutls; };
|
||||
prometheus = import ./prometheus { inherit (prev) prometheus; };
|
||||
|
||||
pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
|
||||
(_pythonFinal: pythonPrev: {
|
||||
pytest-xdist = import ./pytest-xdist { inherit (pythonPrev) pytest-xdist; };
|
||||
scipy = import ./scipy { inherit (pythonPrev) scipy; };
|
||||
sentry-sdk = import ./sentry-sdk { inherit (pythonPrev) sentry-sdk; };
|
||||
})
|
||||
];
|
||||
}
|
||||
@@ -1,160 +0,0 @@
|
||||
# GnuTLS UDP server readiness
|
||||
|
||||
Under load, the test client can start before `gnutls-serv` binds its UDP
|
||||
socket, and the first handshake fails with `Connection refused`.
|
||||
`serv-udp.sh` currently waits a fixed four seconds; elapsed time does not
|
||||
establish server readiness. `udp-server-readiness.patch` replaces that wait
|
||||
with polling for the local IPv4 UDP endpoint.
|
||||
|
||||
## Scope and waiting behavior
|
||||
|
||||
The patch changes the existing `wait_udp_server()` and adds a new
|
||||
`check_if_udp_port_bound()` beside it in `tests/scripts/common.sh`.
|
||||
`serv-udp.sh` is its only caller in 3.8.13. The TCP helpers `wait_server()`
|
||||
and `wait_for_port()`, including their existing sleeps, are unchanged.
|
||||
Both original DTLS handshake checks remain unchanged.
|
||||
|
||||
Each iteration checks process liveness and the socket **before sleeping**.
|
||||
A ready socket returns immediately. An unsuccessful check sleeps two
|
||||
seconds only if another attempt remains: at most 90 attempts, consistent
|
||||
with the existing `wait_server()` budget implemented by `wait_for_port()`,
|
||||
with no sleep after the final check. Server exit fails early; exhausting the
|
||||
budget fails and terminates the server. No handshake is retried, and no
|
||||
protocol timeout is changed. Once bound, the kernel can queue datagrams
|
||||
while the server is scheduled; the probe itself sends no packets.
|
||||
|
||||
The existing `have_port_finder()` prefers `ss`, then `netstat`. If neither
|
||||
exists, it prints `neither ss nor netstat found` and exits **77 (skip)**.
|
||||
In the normal test flow, port selection calls it before launching a server.
|
||||
The probe runs in a subshell so that, even if this skip occurs after launch,
|
||||
the waiting helper can terminate and reap the server before exiting 77.
|
||||
|
||||
## Why an IPv4 socket is expected
|
||||
|
||||
This is specific to the server used by this test, not a general rule that
|
||||
IPv6 sockets cannot serve IPv4 clients. The client explicitly uses
|
||||
`127.0.0.1`. The server's `--udp` path calls `udp_server()`, which calls
|
||||
`listen_socket(..., SOCK_DGRAM)`. That function iterates the wildcard
|
||||
addresses returned by `getaddrinfo(NULL, ..., AI_PASSIVE)`:
|
||||
|
||||
| Server build / Linux setting | Binding behavior |
|
||||
| --- | --- |
|
||||
| IPv6 enabled, `net.ipv6.bindv6only=0` | Requests `IPV6_V6ONLY=1` on the IPv6 socket, binds `[::]:PORT`, and separately binds `0.0.0.0:PORT`. It overrides the system's dual-stack default. |
|
||||
| IPv6 enabled, `net.ipv6.bindv6only=1` | The same explicit socket option and separate IPv4/IPv6 binds. |
|
||||
| `HAVE_IPV6` undefined | Skips every address family except `AF_INET`; only the IPv4 wildcard is attempted. |
|
||||
|
||||
`udp_server()` uses `wait_for_connection()`, which puts **every listener**
|
||||
from that list into `select()` and returns a readable socket for `recvfrom()`;
|
||||
it does not permanently choose one socket based on `getaddrinfo()` order.
|
||||
|
||||
The first two cases were traced with the actual GnuTLS 3.8.13 binary in
|
||||
separate Linux network namespaces: `setsockopt(IPV6_V6ONLY, [1])` and both
|
||||
UDP binds returned success under each setting. The no-IPv6 case was checked
|
||||
in source, not by building a second binary. The same bind implementation
|
||||
was checked directly on GitLab master.
|
||||
|
||||
Thus, successful normal startup for this invocation provides an explicit
|
||||
IPv4 socket; a lone IPv6 wildcard is not the expected success path.
|
||||
There is one portability caveat: upstream discards the return value of
|
||||
`setsockopt(IPV6_V6ONLY)`. On a platform where that call fails and the server
|
||||
ends up with only a dual-stack socket, this helper would time out despite
|
||||
IPv4 reachability. Such a platform needs additional handling before this
|
||||
patch can claim support. Blindly accepting every IPv6 wildcard would also
|
||||
accept IPv6-only sockets before the separate IPv4 bind finishes.
|
||||
|
||||
Source: [`src/serv.c`, `listen_socket()`](https://gitlab.com/gnutls/gnutls/-/blob/master/src/serv.c#L937),
|
||||
[`src/udp-serv.c`](https://gitlab.com/gnutls/gnutls/-/blob/master/src/udp-serv.c),
|
||||
and [`tests/serv-udp.sh`](https://gitlab.com/gnutls/gnutls/-/blob/master/tests/serv-udp.sh).
|
||||
|
||||
## Port matching and ownership limit
|
||||
|
||||
Only `-an` is passed to the socket-listing tool: BSD `netstat -u` selects
|
||||
Unix-domain sockets, whereas Linux `netstat -u` selects UDP. The parser
|
||||
handles the extra state column in `ss`, Linux colon-separated endpoints,
|
||||
and BSD dot-separated endpoints, including `*.PORT`. It matches the full
|
||||
local port and rejects TCP, IPv6 entries, peer ports, and longer numbers.
|
||||
|
||||
A live PID plus a bound port does **not** prove that PID owns the socket.
|
||||
Existing `GETPORT` selection checks for an unused port and uses a test
|
||||
port-lock directory; `launch_bare_server()` also calls
|
||||
`wait_for_free_port()` before starting the process. These are advisory:
|
||||
the launcher does not enforce the latter's result, and another process
|
||||
can bind between the check and launch. The patch does not close that race
|
||||
or add nonportable PID parsing. An unrelated process can satisfy the
|
||||
socket check; the real handshakes remain the functional check and may
|
||||
fail (or reach the wrong server). This is a startup-order fix, not a
|
||||
socket-ownership guarantee.
|
||||
|
||||
## Reproduction and focused checks
|
||||
|
||||
Apply the patch to an unpacked source tree, then run the companion checks
|
||||
with Python's standard library and a shell:
|
||||
|
||||
```sh
|
||||
patch --fuzz=0 -d /path/to/gnutls -p1 < udp-server-readiness.patch
|
||||
SHELL=/bin/sh python3 verify-readiness.py /path/to/gnutls/tests/scripts/common.sh -v
|
||||
```
|
||||
|
||||
Set `NETSTAT=/path/to/netstat` to exercise one outside `PATH`. The checks
|
||||
cover Linux/BSD output samples, false matches, immediate readiness,
|
||||
missing tools, process exit, timeout cleanup, and real IPv4 UDP sockets
|
||||
whose bind is delayed six seconds. The missing-tools fixture is skipped
|
||||
if an absolute fallback `ss` path cannot be hidden with `PATH`. Native
|
||||
BSD execution remains untested.
|
||||
|
||||
To reproduce with GnuTLS itself, run `tests/serv-udp.sh` with `SERV` pointing
|
||||
to a wrapper that sleeps six seconds, then `exec`s `gnutls-serv` with all
|
||||
arguments. Set `CLI` to the matching `gnutls-cli`, `srcdir` to the source
|
||||
`tests` directory, and `abs_top_builddir` to a writable build directory.
|
||||
With GnuTLS 3.8.13, the original helper failed the first handshake with
|
||||
`Connection refused`; the patched helper passed both with the same binaries.
|
||||
|
||||
## GnuTLS submission
|
||||
|
||||
Development and merge requests are on [GitLab](https://gitlab.com/gnutls/gnutls).
|
||||
[`CONTRIBUTING.md` on master](https://gitlab.com/gnutls/gnutls/-/blob/master/CONTRIBUTING.md)
|
||||
was read directly for this review. It requires the contributor's DCO
|
||||
`Signed-off-by`, successful and failure test coverage, consistent coding
|
||||
style, and adequate documentation; GitLab CI runs for merge requests.
|
||||
Its commenting guidance asks for comments explaining non-obvious behavior
|
||||
or protocol expectations. It does not prescribe an additional special
|
||||
test-suite comment. The patch now explains its IPv4 binding assumption
|
||||
next to the probe.
|
||||
|
||||
The submission will contain the shell patch, without the Python verifier
|
||||
or a new Python test dependency. The existing `serv-udp.sh` supplies the
|
||||
functional success check. Running it through the six-second startup
|
||||
wrapper supplies a reproducible regression case: it fails before the fix
|
||||
and passes after it. The local verifier was used to validate socket-output
|
||||
parsing and the helper's success, process-exit, skip-cleanup, and timeout
|
||||
branches. Those branch checks are local evidence, not new automated
|
||||
coverage in the upstream suite; the MR must state that distinction.
|
||||
|
||||
No dedicated unit-test harness for these shell helpers was found in the
|
||||
3.8.13 tests inspected. That does not establish that Python cannot be used
|
||||
upstream; keeping this submission dependency-free is a scope choice. Use
|
||||
the existing test and before/after reproduction as the submission's
|
||||
coverage argument, retaining the platform limitations above. Apply the
|
||||
patch in an upstream checkout and include those results with the
|
||||
contributor's own sign-off. No MR or sign-off has been created.
|
||||
|
||||
## Local NixOS integration and build results
|
||||
|
||||
`overlays/default.nix` imports the `overlays/patches` overlay, which loads
|
||||
`gnutls/default.nix` to apply the patch and keep `serv-udp.sh` enabled.
|
||||
The patch itself has no Nix dependencies and applies to 3.8.13 and GitLab
|
||||
master without fuzz.
|
||||
|
||||
The final patch was rebuilt with:
|
||||
|
||||
```sh
|
||||
nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.gnutls
|
||||
```
|
||||
|
||||
That x86-64-v3 build passed: 927 tests, 796 passes, 131 existing skips,
|
||||
zero failures/errors, and `PASS: serv-udp.sh`. The patch bytes in the built
|
||||
derivation were compared with the repository artifact; both have SHA-256
|
||||
`59013d47fd446f2dd065012a2259ccc1898fedc8a053a630e13efa0076368760`.
|
||||
All seven local checks passed, including skip cleanup and exactly 90
|
||||
probes with 89 sleeps on timeout. The six-second before/after reproduction
|
||||
was also repeated successfully with the final helper.
|
||||
@@ -1,6 +0,0 @@
|
||||
{ gnutls }:
|
||||
gnutls.overrideAttrs (old: {
|
||||
# Keep the UDP handshake test enabled on loaded builders by waiting for
|
||||
# the server to bind its socket. Kept as a standalone patch for upstream.
|
||||
patches = (old.patches or [ ]) ++ [ ./udp-server-readiness.patch ];
|
||||
})
|
||||
@@ -1,70 +0,0 @@
|
||||
Subject: [PATCH] tests: wait for the UDP server socket before connecting
|
||||
|
||||
A fixed four-second sleep does not guarantee that gnutls-serv has bound
|
||||
its UDP socket on a busy builder. Poll the local IPv4 UDP endpoint using
|
||||
the existing ss/netstat discovery, with the same retry budget as the TCP
|
||||
helper. Fail early if the server exits, and retain the original handshake
|
||||
checks in serv-udp.sh.
|
||||
|
||||
Use flags common to ss and BSD/Linux netstat. Match the local endpoint
|
||||
and complete port number, excluding TCP, IPv6-only and peer endpoints.
|
||||
|
||||
--- a/tests/scripts/common.sh
|
||||
+++ b/tests/scripts/common.sh
|
||||
@@ -185,10 +185,55 @@
|
||||
fi
|
||||
}
|
||||
|
||||
+check_if_udp_port_bound() {
|
||||
+ local PORT=$1
|
||||
+ have_port_finder
|
||||
+ # Use only -an, which is shared by ss and BSD/Linux netstat. UDP has
|
||||
+ # no LISTEN state. Match the local IPv4 endpoint, not a peer port or
|
||||
+ # a longer port number. serv-udp.sh connects to 127.0.0.1;
|
||||
+ # listen_socket() in serv.c binds IPv4 separately and requests
|
||||
+ # IPV6_V6ONLY=1 for its IPv6 socket.
|
||||
+ $PFCMD -an | awk -v port="$PORT" '
|
||||
+ $1 == "udp" || $1 == "udp4" {
|
||||
+ # ss includes a state column; netstat does not.
|
||||
+ address = ($2 == "UNCONN" || $2 == "ESTAB") ? $5 : $4
|
||||
+ if (address ~ ("^[0-9.]+[.:]" port "$") ||
|
||||
+ address == "*." port)
|
||||
+ found = 1
|
||||
+ }
|
||||
+ END { exit !found }
|
||||
+ '
|
||||
+}
|
||||
+
|
||||
wait_udp_server() {
|
||||
local PID=$1
|
||||
+ local ret
|
||||
trap "test -n \"${PID}\" && kill ${PID};exit 1" 1 15 2
|
||||
- sleep 4
|
||||
+ local i=0
|
||||
+ # Use the same retry budget as wait_for_port(), but also stop if the
|
||||
+ # server exits before binding its socket.
|
||||
+ while test $i -lt 90; do
|
||||
+ if ! kill -0 "$PID" 2>/dev/null; then
|
||||
+ fail "" "UDP server $PID exited before binding port $PORT"
|
||||
+ fi
|
||||
+ # Contain have_port_finder's exit so a skip also stops the server.
|
||||
+ if (check_if_udp_port_bound "$PORT"); then
|
||||
+ return 0
|
||||
+ else
|
||||
+ ret=$?
|
||||
+ if test "$ret" = 77; then
|
||||
+ kill "$PID" 2>/dev/null || :
|
||||
+ wait "$PID" 2>/dev/null || :
|
||||
+ exit 77
|
||||
+ fi
|
||||
+ fi
|
||||
+ i=$((i + 1))
|
||||
+ if test $i -lt 90; then
|
||||
+ echo "try $i: waiting for UDP port $PORT"
|
||||
+ sleep 2
|
||||
+ fi
|
||||
+ done
|
||||
+ fail "$PID" "UDP server $PORT did not come up"
|
||||
}
|
||||
|
||||
create_testdir() {
|
||||
@@ -1,180 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise patched common.sh without building GnuTLS (Python standard library only).
|
||||
|
||||
Usage: python3 verify-readiness.py /path/to/patched/tests/scripts/common.sh
|
||||
Set SHELL to test another shell, and NETSTAT to test a netstat outside PATH.
|
||||
"""
|
||||
|
||||
# Use unittest so this upstream companion tool needs no pytest installation.
|
||||
# ruff: noqa: PT009
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
COMMON = str(Path(sys.argv.pop(1)).resolve())
|
||||
SHELL = os.environ.get("SHELL", "/bin/sh")
|
||||
|
||||
|
||||
class ReadinessTests(unittest.TestCase):
|
||||
"""Check endpoint parsing and the server startup lifecycle."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
"""Create a socket-listing fixture for each check."""
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.root = Path(self.tmp.name)
|
||||
self.fixture = self.root / "sockets"
|
||||
self.fixture.write_text("")
|
||||
self.finder = self.root / "port-finder"
|
||||
self.finder.write_text('#!/bin/sh\ncat "$SOCKET_FIXTURE"\n')
|
||||
self.finder.chmod(0o755)
|
||||
|
||||
def run_shell(self, body: str, **env: str) -> subprocess.CompletedProcess[str]:
|
||||
"""Source the actual helper and run a shell scenario."""
|
||||
return subprocess.run(
|
||||
[SHELL, "-c", '. "$COMMON"\n' + body],
|
||||
env={
|
||||
**os.environ,
|
||||
"COMMON": COMMON,
|
||||
"SOCKET_FIXTURE": str(self.fixture),
|
||||
"PFCMD": str(self.finder),
|
||||
"PORT": "12345",
|
||||
**env,
|
||||
},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=20,
|
||||
check=False,
|
||||
)
|
||||
|
||||
def test_socket_formats_and_false_matches(self) -> None:
|
||||
"""Accept IPv4 UDP local endpoints and reject unrelated sockets."""
|
||||
cases = [
|
||||
("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*", True),
|
||||
("udp UNCONN 0 0 127.0.0.1:12345 0.0.0.0:*", True),
|
||||
("udp 0 0 0.0.0.0:12345 0.0.0.0:*", True),
|
||||
("udp4 0 0 *.12345 *.*", True),
|
||||
("udp 0 0 127.0.0.1.12345 *.*", True),
|
||||
("udp 0 0 *.12345 *.*", True),
|
||||
("udp UNCONN 0 0 0.0.0.0:123456 0.0.0.0:*", False),
|
||||
("udp 0 0 0.0.0.0:123456 0.0.0.0:*", False),
|
||||
("udp ESTAB 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
|
||||
("udp 0 0 127.0.0.1:54321 127.0.0.1:12345", False),
|
||||
("tcp LISTEN 0 128 0.0.0.0:12345 0.0.0.0:*", False),
|
||||
("tcp 0 0 0.0.0.0:12345 0.0.0.0:* LISTEN", False),
|
||||
("udp UNCONN 0 0 [::]:12345 [::]:*", False),
|
||||
("udp UNCONN 0 0 *:12345 *:*", False),
|
||||
("udp6 0 0 :::12345 :::*", False),
|
||||
("udp6 0 0 *.12345 *.*", False),
|
||||
("", False),
|
||||
]
|
||||
for row, ready in cases:
|
||||
with self.subTest(row=row):
|
||||
self.fixture.write_text(row + "\n")
|
||||
result = self.run_shell('check_if_udp_port_bound "$PORT"')
|
||||
self.assertEqual(result.returncode, 0 if ready else 1, result.stderr)
|
||||
|
||||
def test_exited_server_fails_immediately(self) -> None:
|
||||
"""Fail without sleeping when the server has already exited."""
|
||||
result = self.run_shell(
|
||||
'true &\npid=$!\nwait "$pid"\nsleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$pid"'
|
||||
)
|
||||
self.assertEqual(result.returncode, 1)
|
||||
self.assertIn("exited before binding", result.stderr)
|
||||
self.assertNotIn("unexpected sleep", result.stderr)
|
||||
|
||||
def test_ready_socket_does_not_sleep(self) -> None:
|
||||
"""Check readiness before the first sleep."""
|
||||
self.fixture.write_text("udp UNCONN 0 0 0.0.0.0:12345 0.0.0.0:*\n")
|
||||
result = self.run_shell('sleep() { echo "unexpected sleep" >&2; }\nwait_udp_server "$$"')
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertNotIn("unexpected sleep", result.stderr)
|
||||
|
||||
def test_missing_port_finders_skip(self) -> None:
|
||||
"""Skip and stop the live server when no finder is available."""
|
||||
# have_port_finder also tries these paths independently of PATH.
|
||||
if any(os.access(f"{directory}/ss", os.X_OK) for directory in ("/sbin", "/usr/sbin", "/usr/local/sbin")):
|
||||
self.skipTest("an absolute ss path cannot be hidden by this PATH-only fixture")
|
||||
with subprocess.Popen(["sleep", "60"]) as server:
|
||||
try:
|
||||
result = self.run_shell(
|
||||
'unset PFCMD\nPATH=/nonexistent\nwait_udp_server "$SERVER_PID"',
|
||||
SERVER_PID=str(server.pid),
|
||||
)
|
||||
self.assertEqual(result.returncode, 77)
|
||||
self.assertIn("neither ss nor netstat found", result.stderr)
|
||||
server.wait(timeout=3)
|
||||
self.assertLess(server.returncode, 0)
|
||||
finally:
|
||||
if server.poll() is None:
|
||||
server.kill()
|
||||
|
||||
def test_timeout_is_bounded_and_cleans_up(self) -> None:
|
||||
"""Stop polling after the retry budget and terminate the server."""
|
||||
# Only accelerate the polling delay; keep a real live server process.
|
||||
self.finder.write_text('#!/bin/sh\necho probe >&2\ncat "$SOCKET_FIXTURE"\n')
|
||||
with subprocess.Popen(["sleep", "60"]) as server:
|
||||
try:
|
||||
result = self.run_shell(
|
||||
'sleep() { echo polling-sleep; }\nwait_udp_server "$SERVER_PID"',
|
||||
SERVER_PID=str(server.pid),
|
||||
)
|
||||
self.assertEqual(result.returncode, 1)
|
||||
self.assertIn("did not come up", result.stderr)
|
||||
self.assertEqual(result.stderr.count("probe\n"), 90)
|
||||
self.assertEqual(result.stdout.count("polling-sleep"), 89)
|
||||
server.wait(timeout=3)
|
||||
self.assertLess(server.returncode, 0)
|
||||
finally:
|
||||
if server.poll() is None:
|
||||
server.kill()
|
||||
|
||||
def test_server_exits_while_waiting(self) -> None:
|
||||
"""Detect a startup failure that happens after polling begins."""
|
||||
result = self.run_shell('sleep 1 &\npid=$!\nwait_udp_server "$pid"')
|
||||
self.assertEqual(result.returncode, 1)
|
||||
self.assertIn("exited before binding", result.stderr)
|
||||
self.assertIn("waiting for UDP port", result.stdout)
|
||||
|
||||
def test_real_socket_delayed_beyond_four_seconds(self) -> None:
|
||||
"""Wait for a real delayed bind with each installed port finder."""
|
||||
finders = [shutil.which("ss"), os.environ.get("NETSTAT") or shutil.which("netstat")]
|
||||
finders = [finder for finder in finders if finder]
|
||||
if not finders:
|
||||
self.skipTest("neither ss nor netstat available")
|
||||
for finder in finders:
|
||||
with self.subTest(finder=finder):
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
code = (
|
||||
"import socket,time,sys; time.sleep(6); "
|
||||
"s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); "
|
||||
"s.bind(('127.0.0.1',int(sys.argv[1]))); time.sleep(30)"
|
||||
)
|
||||
with subprocess.Popen([sys.executable, "-c", code, str(port)]) as server:
|
||||
try:
|
||||
started = time.monotonic()
|
||||
result = self.run_shell(
|
||||
'wait_udp_server "$SERVER_PID"',
|
||||
SERVER_PID=str(server.pid),
|
||||
PORT=str(port),
|
||||
PFCMD=finder,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertGreaterEqual(time.monotonic() - started, 6)
|
||||
self.assertIsNone(server.poll())
|
||||
finally:
|
||||
server.terminate()
|
||||
server.wait(timeout=3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,66 +0,0 @@
|
||||
# Prometheus complete test parsing
|
||||
|
||||
CodeMirror gives editor-state creation a 20 ms synchronous parsing budget.
|
||||
The shared `createEditorState()` test helper can therefore return an
|
||||
incomplete syntax tree when the process is descheduled. The completion and
|
||||
vector-matching tests immediately inspect that tree.
|
||||
|
||||
## Scope and behavior
|
||||
|
||||
`complete-test-parsing.patch` changes only
|
||||
`module/codemirror-promql/src/test/utils-test.ts` inside `web/ui`. It completes
|
||||
the small test expression with `ensureSyntaxTree(..., Infinity)` and publishes
|
||||
the completed parse through an empty transaction so `syntaxTree(state)` sees
|
||||
it. Failure to obtain a tree raises an error.
|
||||
|
||||
The original assertions remain enabled, including `autocomplete topk params 2`
|
||||
and `foo * on(test,blub) bar`. The unlimited budget applies to the test helper;
|
||||
production editor parsing budgets are unchanged.
|
||||
|
||||
## Reproduction and focused checks
|
||||
|
||||
Use a disposable Prometheus 3.14.0 checkout. The patch root is `web/ui`, matching
|
||||
the Nix assets derivation. From this directory:
|
||||
|
||||
```sh
|
||||
patch --fuzz=0 -d /path/to/prometheus/web/ui -p1 < complete-test-parsing.patch
|
||||
cd /path/to/prometheus/web/ui
|
||||
pnpm install --frozen-lockfile
|
||||
pnpm --filter @prometheus-io/lezer-promql build
|
||||
pnpm --filter @prometheus-io/codemirror-promql test
|
||||
```
|
||||
|
||||
To force the scheduling condition, temporarily append this clock to
|
||||
`module/codemirror-promql/setupJest.cjs` in the disposable checkout:
|
||||
|
||||
```js
|
||||
let parseClock = 0;
|
||||
Date.now = () => (parseClock += 25);
|
||||
```
|
||||
|
||||
Each clock read crosses the editor's initial parsing budget. Against the
|
||||
original helper, the hybrid and vector suites have 186 failures, including
|
||||
both locally excluded cases. With the patch, all 386 CodeMirror tests pass
|
||||
under that same clock. Remove the injected clock before normal builds.
|
||||
|
||||
## Upstream status
|
||||
|
||||
This is a standalone test-helper patch for Prometheus 3.14.0. No upstream
|
||||
submission was made during this work. Recheck the helper when updating
|
||||
Prometheus or CodeMirror, including how an ensured parse becomes visible
|
||||
through the editor state.
|
||||
|
||||
## Local NixOS integration and build results
|
||||
|
||||
[`../default.nix`](../default.nix) loads `default.nix`, which patches the
|
||||
separate assets derivation. It updates both `passthru.assets` and the main
|
||||
Prometheus build's reference to those assets. From the repository root:
|
||||
|
||||
```sh
|
||||
nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.prometheus.assets
|
||||
```
|
||||
|
||||
The full x86-64-v3 assets build passed with the normal clock, including the
|
||||
CodeMirror and UI suites. Host-flake evaluation confirmed that the main
|
||||
Prometheus derivation refers to these patched assets. The Go server package
|
||||
was not rebuilt for this test-helper change.
|
||||
@@ -1,36 +0,0 @@
|
||||
Subject: [PATCH] tests: finish parsing before inspecting editor state
|
||||
|
||||
EditorState creation has a 20 ms parsing budget. A descheduled test can
|
||||
therefore observe an incomplete tree. Finish these small test documents
|
||||
without an interactive deadline and publish the result with a transaction.
|
||||
Keep the original completion and vector-matching assertions enabled.
|
||||
|
||||
--- a/module/codemirror-promql/src/test/utils-test.ts
|
||||
+++ b/module/codemirror-promql/src/test/utils-test.ts
|
||||
@@ -13,7 +13,7 @@
|
||||
|
||||
import { parser } from '@prometheus-io/lezer-promql';
|
||||
import { EditorState } from '@codemirror/state';
|
||||
-import { LRLanguage } from '@codemirror/language';
|
||||
+import { ensureSyntaxTree, LRLanguage } from '@codemirror/language';
|
||||
import nock from 'nock';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
@@ -23,10 +23,16 @@
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
export function createEditorState(expr: string): EditorState {
|
||||
- return EditorState.create({
|
||||
+ const state = EditorState.create({
|
||||
doc: expr,
|
||||
extensions: lightPromQLSyntax,
|
||||
});
|
||||
+ // These tests need a complete tree, independent of the editor's time budget.
|
||||
+ if (!ensureSyntaxTree(state, state.doc.length, Infinity)) {
|
||||
+ throw new Error('Unable to parse the test expression');
|
||||
+ }
|
||||
+ // Publish the completed parse so syntaxTree(state) sees it too.
|
||||
+ return state.update({}).state;
|
||||
}
|
||||
|
||||
export function mockPrometheusServer(): void {
|
||||
@@ -1,17 +0,0 @@
|
||||
{ prometheus }:
|
||||
prometheus.overrideAttrs (
|
||||
old:
|
||||
let
|
||||
assets = old.passthru.assets.overrideAttrs (assetsOld: {
|
||||
patches = (assetsOld.patches or [ ]) ++ [ ./complete-test-parsing.patch ];
|
||||
});
|
||||
in
|
||||
{
|
||||
postPatch = builtins.replaceStrings [ "${old.passthru.assets}" ] [ "${assets}" ] (
|
||||
builtins.unsafeDiscardStringContext old.postPatch
|
||||
);
|
||||
passthru = old.passthru // {
|
||||
inherit assets;
|
||||
};
|
||||
}
|
||||
)
|
||||
@@ -1,79 +0,0 @@
|
||||
# pytest-xdist test fixes
|
||||
|
||||
With two workers and a restart limit of three, the fourth worker crash
|
||||
requests shutdown while another test can still be running. That test may
|
||||
also crash. The original queued-work test requires exactly four failures,
|
||||
even though five failures can occur without exceeding the replacement limit.
|
||||
|
||||
## Scope and behavior
|
||||
|
||||
`concurrent-worker-crashes.patch` changes the assertions in
|
||||
`TestNodeFailure.test_max_worker_restart_tests_queued` in
|
||||
`testing/acceptance_test.py`. It requires exactly three replacements, four or
|
||||
five failed tests, the failed-tests exit status, the limit message, and no
|
||||
internal error. It retains the two-worker workload and ten queued tests.
|
||||
|
||||
`worker-startup-timeout.patch` changes the remote-test helper's event timeout
|
||||
from 10 to 60 seconds so loaded builders have time to start workers. The
|
||||
helper returns immediately when an event arrives and still has a bounded wait.
|
||||
|
||||
The existing nixpkgs pytest-9 compatibility patches remain in place.
|
||||
Production scheduling and worker-restart behavior are unchanged.
|
||||
|
||||
## Reproduction and focused checks
|
||||
|
||||
Use a disposable pytest-xdist 3.8.0 checkout with its test dependencies and
|
||||
the nixpkgs pytest-9 compatibility patches where required. From this directory:
|
||||
|
||||
```sh
|
||||
patch --fuzz=0 -d /path/to/pytest-xdist -p1 < concurrent-worker-crashes.patch
|
||||
patch --fuzz=0 -d /path/to/pytest-xdist -p1 < worker-startup-timeout.patch
|
||||
cd /path/to/pytest-xdist
|
||||
python -m pytest testing/acceptance_test.py \
|
||||
-k test_max_worker_restart_tests_queued -q
|
||||
python -m pytest testing/test_remote.py -q
|
||||
```
|
||||
|
||||
Twenty unmodified runs passed during the review. To force the failing
|
||||
schedule, modify the generated crashing test in a disposable checkout to
|
||||
accept `worker_id`: make `gw3` wait for a marker created by `gw4`, and make
|
||||
`gw4` pause 0.1 seconds after creating the marker. Then both have in-flight
|
||||
tests when shutdown starts. Bound the marker wait so a reproduction failure
|
||||
cannot hang the suite. The original assertion fails on five reported
|
||||
failures; the patched test passes.
|
||||
|
||||
## Worker startup and outer concurrency
|
||||
|
||||
[`default.nix`](default.nix) runs the outer suite with one worker to limit
|
||||
nested process pools. This is a Nix test-runner setting; the source timeout
|
||||
change lives in [`worker-startup-timeout.patch`](worker-startup-timeout.patch).
|
||||
|
||||
A separate reproduction inserts an 11-second `pytest_sessionstart` delay
|
||||
into the child created by `test_basic_collect_and_runtests` in
|
||||
`testing/test_remote.py`. The original 10-second channel wait fails; the
|
||||
60-second wait passes. This bounds waits for test worker events, including
|
||||
startup, rather than changing a product deadline.
|
||||
|
||||
## Upstream status
|
||||
|
||||
These are standalone test patches for pytest-xdist 3.8.0. No upstream submission
|
||||
was made during this work. Recheck the allowed in-flight failures, replacement
|
||||
count, and remote-test wait when updating the scheduler or worker behavior.
|
||||
|
||||
## Local NixOS integration and build results
|
||||
|
||||
[`../default.nix`](../default.nix) loads `default.nix` through
|
||||
`pythonPackagesExtensions`. From the repository root:
|
||||
|
||||
```sh
|
||||
nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.python314Packages.pytest-xdist
|
||||
```
|
||||
|
||||
After consolidating the settings in this directory, the full x86-64-v3 package
|
||||
build passed 185 tests, with 6 existing skips and 10 expected failures. Nix
|
||||
evaluation confirmed the same outer-worker limit and preserved existing
|
||||
patches, with the timeout now applied as a source patch.
|
||||
|
||||
The earlier forced concurrent-crash and delayed-startup reproductions passed
|
||||
after their fixes; the focused crash test also passed after formatting its
|
||||
assertion.
|
||||
@@ -1,29 +0,0 @@
|
||||
Subject: [PATCH] tests: count replacements when checking the worker restart limit
|
||||
|
||||
With two workers, another in-flight test may crash after the fourth
|
||||
crash requests shutdown. Either four or five failed tests is valid.
|
||||
Require exactly three replacements and the failed-tests exit status,
|
||||
while preserving the queued-work and no-internal-error assertions.
|
||||
|
||||
--- a/testing/acceptance_test.py
|
||||
+++ b/testing/acceptance_test.py
|
||||
@@ -1011,9 +1011,18 @@
|
||||
"worker*crashed while running*",
|
||||
"worker*crashed while running*",
|
||||
"* xdist: maximum crashed workers reached: 3 *",
|
||||
- "* 4 failed in *",
|
||||
]
|
||||
)
|
||||
+ # A second in-flight test may crash after shutdown is requested.
|
||||
+ # The restart limit constrains replacements, not concurrent failures.
|
||||
+ replacements = sum(
|
||||
+ line.startswith("replacing crashed worker ") for line in res.stdout.lines
|
||||
+ )
|
||||
+ assert replacements == 3
|
||||
+ failed = res.parseoutcomes()["failed"]
|
||||
+ assert failed in (4, 5)
|
||||
+ res.assert_outcomes(failed=failed)
|
||||
+ assert res.ret == pytest.ExitCode.TESTS_FAILED
|
||||
assert "INTERNALERROR" not in res.stdout.str()
|
||||
|
||||
def test_max_worker_restart_die(self, pytester: pytest.Pytester) -> None:
|
||||
@@ -1,12 +0,0 @@
|
||||
{ pytest-xdist }:
|
||||
pytest-xdist.overridePythonAttrs (old: {
|
||||
patches = (old.patches or [ ]) ++ [
|
||||
./concurrent-worker-crashes.patch
|
||||
./worker-startup-timeout.patch
|
||||
];
|
||||
|
||||
# The suite exercises its own worker pools. Limit the outer suite to one worker.
|
||||
preCheck = builtins.replaceStrings [ "--numprocesses=$NIX_BUILD_CORES" ] [ "--numprocesses=1" ] (
|
||||
old.preCheck or ""
|
||||
);
|
||||
})
|
||||
@@ -1,19 +0,0 @@
|
||||
Subject: [PATCH] tests: allow more time for remote worker events
|
||||
|
||||
Worker startup can exceed ten seconds on heavily loaded builders. Allow
|
||||
the remote-test helper to wait up to sixty seconds for worker events.
|
||||
The wait still returns as soon as an event arrives and remains bounded.
|
||||
Production worker timeouts and test assertions are unchanged.
|
||||
|
||||
--- a/testing/test_remote.py
|
||||
+++ b/testing/test_remote.py
|
||||
@@ -17,7 +17,8 @@
|
||||
from xdist.workermanage import WorkerController
|
||||
|
||||
|
||||
-WAIT_TIMEOUT = 10.0
|
||||
+# Allow worker events extra time on heavily loaded builders.
|
||||
+WAIT_TIMEOUT = 60.0
|
||||
|
||||
|
||||
def check_marshallable(d: object) -> None:
|
||||
@@ -1,70 +0,0 @@
|
||||
# SciPy STFT test tolerances
|
||||
|
||||
The x86-64-v3 build can produce small floating-point residuals in inverse-STFT
|
||||
comparisons and scaling round trips. The original bounds reject these results,
|
||||
including residuals around `4e-17` where a round trip expects zero for a signal
|
||||
with amplitude 2.
|
||||
|
||||
## Scope and behavior
|
||||
|
||||
`stft-test-tolerances.patch` changes only the signal tests:
|
||||
|
||||
- The inverse-STFT comparison in `_scipy_spectral_test_shim.py` uses
|
||||
`max(1e-7, 2 * np.finfo(x.dtype).eps)` as its relative tolerance. Float64
|
||||
keeps the original bound, and the existing i686 override remains.
|
||||
- Three scaling round trips in `test_spectral.py` gain an absolute tolerance
|
||||
of one epsilon for the input dtype, allowing small residuals near zero.
|
||||
|
||||
The tests remain enabled, and the production STFT implementation is unchanged.
|
||||
|
||||
## Reproduction and focused checks
|
||||
|
||||
From this directory, apply the patch to a disposable SciPy 1.18.0 checkout:
|
||||
|
||||
```sh
|
||||
patch --fuzz=0 -d /path/to/scipy -p1 < stft-test-tolerances.patch
|
||||
```
|
||||
|
||||
Build and install that tree with SciPy's test dependencies. From outside the
|
||||
source directory, run the installed tests:
|
||||
|
||||
```sh
|
||||
python -m pytest --pyargs scipy.signal.tests.test_spectral \
|
||||
-k 'roundtrip_float32 or roundtrip_scaling' -q
|
||||
```
|
||||
|
||||
Use the same compiler flags and numerical libraries for before/after runs.
|
||||
The earlier reproduction called `TestSTFT.test_roundtrip_float32` and
|
||||
`TestSTFT.test_roundtrip_scaling` against the x86-64-v3 libraries, then loaded
|
||||
patched copies of the test modules. Both failed with the original bounds
|
||||
and passed with the adjusted bounds.
|
||||
|
||||
## Upstream status
|
||||
|
||||
[SciPy issue #25488](https://github.com/scipy/scipy/issues/25488) records
|
||||
related test failures with architecture-specific compiler flags. It is
|
||||
context for the local tolerance repair; this exact patch has not been
|
||||
submitted upstream during this work.
|
||||
|
||||
## Local NixOS integration and build results
|
||||
|
||||
[`../default.nix`](../default.nix) loads [`default.nix`](default.nix) through
|
||||
`pythonPackagesExtensions`, preserving the package's existing patches.
|
||||
The override also covers SciPy used to test other Python dependencies,
|
||||
including pgvector in portal's shared Python environment.
|
||||
|
||||
From the repository root:
|
||||
|
||||
```sh
|
||||
nix build --no-link -L .#nixosConfigurations.portal-1.pkgs.python314Packages.scipy
|
||||
```
|
||||
|
||||
The original remote build of patched SciPy 1.18.0 passed 87,723 tests, with
|
||||
8,342 skips, 300 expected failures, and 22 unexpected passes. The patch and
|
||||
override have been restored byte-for-byte from commit `24cbf74f`; those counts
|
||||
describe the earlier full build.
|
||||
|
||||
Restoration checks confirmed that the patch applies to the pinned source
|
||||
without fuzz, portal's evaluated SciPy retains its existing patch and install
|
||||
checks, and pgvector uses the patched SciPy. A full package or system rebuild
|
||||
was not repeated for this restoration.
|
||||
@@ -1,5 +0,0 @@
|
||||
{ scipy }:
|
||||
scipy.overridePythonAttrs (old: {
|
||||
# Keep the STFT tests enabled with tolerances for x86-64-v3 rounding.
|
||||
patches = (old.patches or [ ]) ++ [ ./stft-test-tolerances.patch ];
|
||||
})
|
||||
@@ -1,51 +0,0 @@
|
||||
Subject: [PATCH] signal: allow floating-point rounding in STFT tests
|
||||
|
||||
Keep the STFT tests enabled for x86-64-v3 builds. Allow two float32
|
||||
epsilons of relative error when comparing inverse-STFT implementations;
|
||||
float64 and the existing i686 override remain unchanged. Allow one
|
||||
float64 epsilon of absolute error in all three scaling round trips,
|
||||
which otherwise require exact zeros (observed residual: 4e-17 for a
|
||||
signal with amplitude 2).
|
||||
|
||||
Upstream issue: https://github.com/scipy/scipy/issues/25488
|
||||
|
||||
--- a/scipy/signal/tests/_scipy_spectral_test_shim.py
|
||||
+++ b/scipy/signal/tests/_scipy_spectral_test_shim.py
|
||||
@@ -294,7 +294,7 @@
|
||||
|
||||
# Adapted tolerances to account for resolution loss:
|
||||
atol = np.finfo(x.dtype).resolution*2 # instead of default atol = 0
|
||||
- rtol = 1e-7 # default for np.allclose()
|
||||
+ rtol = max(1e-7, 2 * np.finfo(x.dtype).eps)
|
||||
|
||||
# Relax atol on 32-Bit platforms a bit to pass CI tests.
|
||||
# - Not clear why there are discrepancies (in the FFT maybe?)
|
||||
--- a/scipy/signal/tests/test_spectral.py
|
||||
+++ b/scipy/signal/tests/test_spectral.py
|
||||
@@ -2044,7 +2044,7 @@
|
||||
|
||||
# Test round trip:
|
||||
x1 = istft(Zs, boundary=True, scaling='spectrum')[1]
|
||||
- assert_allclose(x1, x)
|
||||
+ assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
|
||||
|
||||
# For a Hann-windowed 256 sample length FFT, we expect a peak at
|
||||
# frequency 64 (since it is 1/4 the length of X) with a height of 1
|
||||
@@ -2074,7 +2074,7 @@
|
||||
|
||||
# Test round trip:
|
||||
x1 = istft(Zp, input_onesided=False, boundary=True, scaling='psd')[1]
|
||||
- assert_allclose(x1, x)
|
||||
+ assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
|
||||
|
||||
# The power of the one-sided psd-scaled STFT can be determined
|
||||
# analogously (note that the two sides are not of equal shape):
|
||||
@@ -2094,7 +2094,7 @@
|
||||
|
||||
# Test round trip:
|
||||
x1 = istft(Zp0, input_onesided=True, boundary=True, scaling='psd')[1]
|
||||
- assert_allclose(x1, x)
|
||||
+ assert_allclose(x1, x, atol=np.finfo(x.dtype).eps)
|
||||
|
||||
|
||||
class TestSampledSpectralRepresentations:
|
||||
@@ -1,58 +0,0 @@
|
||||
# Sentry SDK thread-metadata test isolation
|
||||
|
||||
The fallback tests globally mock `threading.current_thread` while a worker
|
||||
is running. Python 3.14's `Thread.join()` also calls that function. A one-use
|
||||
mock can therefore be consumed by the wrong caller or raise `StopIteration`
|
||||
when the main thread joins the worker.
|
||||
|
||||
## Scope and behavior
|
||||
|
||||
`isolate-threading-mocks.patch` changes three neighboring thread-metadata
|
||||
tests in `tests/test_utils.py`, including the formerly excluded
|
||||
`test_get_current_thread_meta_main_thread`.
|
||||
|
||||
Each test replaces only `sentry_sdk.utils.threading`, wraps the real module
|
||||
for unmocked operations, and sets the SDK lookup's return value. The real
|
||||
`Thread.join()` continues using Python's unmodified `threading` module.
|
||||
The fallback-result assertions remain; SDK production code is unchanged.
|
||||
|
||||
## Reproduction and focused checks
|
||||
|
||||
Use a disposable Sentry SDK 2.66.0 checkout and its Python test dependencies.
|
||||
From this directory:
|
||||
|
||||
```sh
|
||||
patch --fuzz=0 -d /path/to/sentry-python -p1 < isolate-threading-mocks.patch
|
||||
cd /path/to/sentry-python
|
||||
python -m pytest tests/test_utils.py -k get_current_thread_meta -q
|
||||
```
|
||||
|
||||
To reproduce the race, hold the worker inside its mock just after
|
||||
`get_current_thread_meta()` returns, signal that point to the main thread,
|
||||
and call `Thread.join()` before releasing the worker. Use an independent
|
||||
bounded release so the patched join can finish. The original test raises
|
||||
`StopIteration` in `join`; the patched test passes under the same schedule.
|
||||
Perform this scheduling instrumentation only in a disposable checkout.
|
||||
|
||||
## Upstream status
|
||||
|
||||
This is a standalone test patch for Sentry SDK 2.66.0. No upstream submission
|
||||
was made during this work. Recheck mock isolation and Python threading
|
||||
behavior when upgrading the SDK or interpreter.
|
||||
|
||||
## Local NixOS integration and build results
|
||||
|
||||
[`../default.nix`](../default.nix) loads `default.nix` through
|
||||
`pythonPackagesExtensions`. From the repository root:
|
||||
|
||||
```sh
|
||||
nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.python314Packages.sentry-sdk
|
||||
```
|
||||
|
||||
The patched package passed 2,356 tests with 116 existing skips on Python
|
||||
3.14.7. The controlled join reproduction failed before the fix and passed
|
||||
after it.
|
||||
|
||||
That package build used the preceding dependency set with this patch to avoid
|
||||
unrelated rebuilds after pytest-xdist changed. The integrated host derivation
|
||||
was evaluated; a complete NixOS rebuild was not performed.
|
||||
@@ -1,4 +0,0 @@
|
||||
{ sentry-sdk }:
|
||||
sentry-sdk.overridePythonAttrs (old: {
|
||||
patches = (old.patches or [ ]) ++ [ ./isolate-threading-mocks.patch ];
|
||||
})
|
||||
@@ -1,41 +0,0 @@
|
||||
Subject: [PATCH] tests: isolate SDK thread lookup mocks from Python threading
|
||||
|
||||
Thread.join also calls threading.current_thread on Python 3.14. A global
|
||||
single-use side effect can be consumed by join instead of the SDK, or
|
||||
raise StopIteration in join after the SDK consumes it. Patch the SDK's
|
||||
module binding and delegate unmocked operations to the real module.
|
||||
Apply the same isolation to the adjacent invalid-thread fallback tests.
|
||||
|
||||
--- a/tests/test_utils.py
|
||||
+++ b/tests/test_utils.py
|
||||
@@ -914,7 +914,8 @@
|
||||
results = Queue(maxsize=1)
|
||||
|
||||
def target():
|
||||
- with mock.patch("threading.current_thread", side_effect=["fake thread"]):
|
||||
+ with mock.patch("sentry_sdk.utils.threading", wraps=threading) as sdk_threading:
|
||||
+ sdk_threading.current_thread.return_value = "fake thread"
|
||||
results.put(get_current_thread_meta())
|
||||
|
||||
thread = threading.Thread(target=target)
|
||||
@@ -930,7 +931,9 @@
|
||||
|
||||
def target():
|
||||
# mock that somehow the current thread doesn't exist
|
||||
- with mock.patch("threading.current_thread", side_effect=[None]):
|
||||
+ # Keep the real threading module intact for concurrent Thread.join calls.
|
||||
+ with mock.patch("sentry_sdk.utils.threading", wraps=threading) as sdk_threading:
|
||||
+ sdk_threading.current_thread.return_value = None
|
||||
results.put(get_current_thread_meta())
|
||||
|
||||
main_thread = threading.main_thread()
|
||||
@@ -945,7 +948,8 @@
|
||||
results = Queue(maxsize=1)
|
||||
|
||||
def target():
|
||||
- with mock.patch("threading.current_thread", return_value="fake thread"):
|
||||
+ with mock.patch("sentry_sdk.utils.threading", wraps=threading) as sdk_threading:
|
||||
+ sdk_threading.current_thread.return_value = "fake thread"
|
||||
results.put(get_current_thread_meta())
|
||||
|
||||
main_thread = threading.main_thread()
|
||||
@@ -1,81 +0,0 @@
|
||||
[project]
|
||||
name = "system_tools"
|
||||
version = "0.1.0"
|
||||
description = ""
|
||||
authors = [{ name = "Richie Cahill", email = "richie@tmmworkshop.com" }]
|
||||
requires-python = "~=3.14.0"
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
|
||||
[tool.ruff]
|
||||
|
||||
target-version = "py314"
|
||||
|
||||
line-length = 120
|
||||
|
||||
lint.select = ["ALL"]
|
||||
lint.ignore = [
|
||||
"G004", # (PERM) This is a performers nit
|
||||
"COM812", # (TEMP) conflicts when used with the formatter
|
||||
"ISC001", # (TEMP) conflicts when used with the formatter
|
||||
"S603", # (PERM) This is known to cause a false positive
|
||||
"S607", # (PERM) This is becoming a consistent annoyance
|
||||
"CPY001", # (PERM) I don't include the license in every file
|
||||
]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
|
||||
"tests/**" = [
|
||||
"ANN", # (perm) type annotations not needed in tests
|
||||
"D", # (perm) docstrings not needed in tests
|
||||
"PLR2004", # (perm) magic values are fine in test assertions
|
||||
"S101", # (perm) pytest needs asserts
|
||||
]
|
||||
"python/stuff/**" = [
|
||||
"T201", # (perm) I don't care about print statements dir
|
||||
]
|
||||
"python/testing/**" = [
|
||||
"T201", # (perm) I don't care about print statements dir
|
||||
"ERA001", # (perm) I don't care about print statements dir
|
||||
]
|
||||
"python/splendor/**" = [
|
||||
"S311", # (perm) there is no security issue here
|
||||
"T201", # (perm) I don't care about print statements dir
|
||||
"PLR2004", # (temps) need to think about this
|
||||
]
|
||||
"python/orm/**" = [
|
||||
"TC003", # (perm) this creates issues because sqlalchemy uses these at runtime
|
||||
]
|
||||
"python/congress_tracker/**" = [
|
||||
"TC003", # (perm) this creates issues because sqlalchemy uses these at runtime
|
||||
]
|
||||
|
||||
"python/alembic/**" = [
|
||||
"INP001", # (perm) this creates LSP issues for alembic
|
||||
]
|
||||
|
||||
[tool.ruff.lint.pydocstyle]
|
||||
convention = "google"
|
||||
|
||||
[tool.ruff.lint.flake8-builtins]
|
||||
builtins-ignorelist = ["id"]
|
||||
|
||||
[tool.ruff.lint.pylint]
|
||||
max-args = 9
|
||||
|
||||
[tool.coverage.run]
|
||||
source = ["system_tools"]
|
||||
|
||||
[tool.coverage.report]
|
||||
exclude_lines = [
|
||||
"pragma: no cover",
|
||||
"if TYPE_CHECKING:",
|
||||
"raise NotImplementedError",
|
||||
"if __name__ == \"__main__\":",
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
addopts = "-n auto -ra --ignore=tests/ebook_search"
|
||||
asyncio_mode = "auto"
|
||||
testpaths = ["tests"]
|
||||
# --cov=system_tools --cov-report=term-missing --cov-report=xml --cov-report=html --cov-branch
|
||||
@@ -1 +0,0 @@
|
||||
"""Server Tools."""
|
||||
@@ -1,122 +0,0 @@
|
||||
"""Alembic."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Literal
|
||||
|
||||
from alembic import context
|
||||
from alembic.script import write_hooks
|
||||
from sqlalchemy.schema import CreateSchema
|
||||
|
||||
from python.common import bash_wrapper
|
||||
from python.orm.common import get_postgres_engine
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import MutableMapping
|
||||
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
config = context.config
|
||||
|
||||
base_class: type[DeclarativeBase] = config.attributes.get("base")
|
||||
if base_class is None:
|
||||
error = "No base class provided. Use the database CLI to run alembic commands."
|
||||
raise RuntimeError(error)
|
||||
|
||||
target_metadata = base_class.metadata
|
||||
logging.basicConfig(
|
||||
level="DEBUG",
|
||||
datefmt="%Y-%m-%dT%H:%M:%S%z",
|
||||
format="%(asctime)s %(levelname)s %(filename)s:%(lineno)d - %(message)s",
|
||||
handlers=[logging.StreamHandler(sys.stdout)],
|
||||
)
|
||||
|
||||
|
||||
@write_hooks.register("dynamic_schema")
|
||||
def dynamic_schema(filename: str, _options: dict[Any, Any]) -> None:
|
||||
"""Dynamic schema."""
|
||||
original_file = Path(filename).read_text()
|
||||
schema_name = base_class.schema_name
|
||||
dynamic_schema_file_part1 = original_file.replace(f"schema='{schema_name}'", "schema=schema")
|
||||
dynamic_schema_file = dynamic_schema_file_part1.replace(f"'{schema_name}.", "f'{schema}.")
|
||||
Path(filename).write_text(dynamic_schema_file)
|
||||
|
||||
|
||||
@write_hooks.register("import_postgresql")
|
||||
def import_postgresql(filename: str, _options: dict[Any, Any]) -> None:
|
||||
"""Add postgresql dialect import when postgresql types are used."""
|
||||
content = Path(filename).read_text()
|
||||
if "postgresql." in content and "from sqlalchemy.dialects import postgresql" not in content:
|
||||
content = content.replace(
|
||||
"import sqlalchemy as sa\n",
|
||||
"import sqlalchemy as sa\nfrom sqlalchemy.dialects import postgresql\n",
|
||||
)
|
||||
Path(filename).write_text(content)
|
||||
|
||||
|
||||
@write_hooks.register("ruff")
|
||||
def ruff_check_and_format(filename: str, _options: dict[Any, Any]) -> None:
|
||||
"""Docstring for ruff_check_and_format."""
|
||||
bash_wrapper(f"ruff check --fix {filename}")
|
||||
bash_wrapper(f"ruff format {filename}")
|
||||
|
||||
|
||||
def include_name(
|
||||
name: str | None,
|
||||
type_: Literal["schema", "table", "column", "index", "unique_constraint", "foreign_key_constraint"],
|
||||
_parent_names: MutableMapping[Literal["schema_name", "table_name", "schema_qualified_table_name"], str | None],
|
||||
) -> bool:
|
||||
"""Filter tables to be included in the migration.
|
||||
|
||||
Args:
|
||||
name (str): The name of the table.
|
||||
type_ (str): The type of the table.
|
||||
_parent_names (MutableMapping): The names of the parent tables.
|
||||
|
||||
Returns:
|
||||
bool: True if the table should be included, False otherwise.
|
||||
|
||||
"""
|
||||
if type_ == "schema":
|
||||
# allows a database with multiple schemas to have separate alembic revisions
|
||||
return name == target_metadata.schema
|
||||
return True
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations in 'online' mode.
|
||||
|
||||
In this scenario we need to create an Engine
|
||||
and associate a connection with the context.
|
||||
|
||||
"""
|
||||
env_prefix = config.attributes.get("env_prefix", "POSTGRES")
|
||||
connectable = get_postgres_engine(name=env_prefix)
|
||||
|
||||
with connectable.connect() as connection:
|
||||
schema = base_class.schema_name
|
||||
if not connectable.dialect.has_schema(connection, schema):
|
||||
answer = input(f"Schema {schema!r} does not exist. Create it? [y/N] ")
|
||||
if answer.lower() != "y":
|
||||
error = f"Schema {schema!r} does not exist. Exiting."
|
||||
raise SystemExit(error)
|
||||
connection.execute(CreateSchema(schema))
|
||||
connection.commit()
|
||||
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
include_schemas=True,
|
||||
version_table_schema=schema,
|
||||
include_name=include_name,
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
connection.commit()
|
||||
|
||||
|
||||
run_migrations_online()
|
||||
@@ -1,113 +0,0 @@
|
||||
"""created contact api.
|
||||
|
||||
Revision ID: edd7dd61a3d2
|
||||
Revises:
|
||||
Create Date: 2026-01-11 15:45:59.909266
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "edd7dd61a3d2"
|
||||
down_revision: str | None = None
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"contact",
|
||||
sa.Column("name", sa.String(), nullable=False),
|
||||
sa.Column("age", sa.Integer(), nullable=True),
|
||||
sa.Column("bio", sa.String(), nullable=True),
|
||||
sa.Column("current_job", sa.String(), nullable=True),
|
||||
sa.Column("gender", sa.String(), nullable=True),
|
||||
sa.Column("goals", sa.String(), nullable=True),
|
||||
sa.Column("legal_name", sa.String(), nullable=True),
|
||||
sa.Column("profile_pic", sa.String(), nullable=True),
|
||||
sa.Column("safe_conversation_starters", sa.String(), nullable=True),
|
||||
sa.Column("self_sufficiency_score", sa.Integer(), nullable=True),
|
||||
sa.Column("social_structure_style", sa.String(), nullable=True),
|
||||
sa.Column("ssn", sa.String(), nullable=True),
|
||||
sa.Column("suffix", sa.String(), nullable=True),
|
||||
sa.Column("timezone", sa.String(), nullable=True),
|
||||
sa.Column("topics_to_avoid", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_contact")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"need",
|
||||
sa.Column("name", sa.String(), nullable=False),
|
||||
sa.Column("description", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_need")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"contact_need",
|
||||
sa.Column("contact_id", sa.Integer(), nullable=False),
|
||||
sa.Column("need_id", sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["contact_id"],
|
||||
[f"{schema}.contact.id"],
|
||||
name=op.f("fk_contact_need_contact_id_contact"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["need_id"], [f"{schema}.need.id"], name=op.f("fk_contact_need_need_id_need"), ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("contact_id", "need_id", name=op.f("pk_contact_need")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"contact_relationship",
|
||||
sa.Column("contact_id", sa.Integer(), nullable=False),
|
||||
sa.Column("related_contact_id", sa.Integer(), nullable=False),
|
||||
sa.Column("relationship_type", sa.String(length=100), nullable=False),
|
||||
sa.Column("closeness_weight", sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["contact_id"],
|
||||
[f"{schema}.contact.id"],
|
||||
name=op.f("fk_contact_relationship_contact_id_contact"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["related_contact_id"],
|
||||
[f"{schema}.contact.id"],
|
||||
name=op.f("fk_contact_relationship_related_contact_id_contact"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("contact_id", "related_contact_id", name=op.f("pk_contact_relationship")),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("contact_relationship", schema=schema)
|
||||
op.drop_table("contact_need", schema=schema)
|
||||
op.drop_table("need", schema=schema)
|
||||
op.drop_table("contact", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
-135
@@ -1,135 +0,0 @@
|
||||
"""add congress tracker tables.
|
||||
|
||||
Revision ID: 3f71565e38de
|
||||
Revises: edd7dd61a3d2
|
||||
Create Date: 2026-02-12 16:36:09.457303
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "3f71565e38de"
|
||||
down_revision: str | None = "edd7dd61a3d2"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"bill",
|
||||
sa.Column("congress", sa.Integer(), nullable=False),
|
||||
sa.Column("bill_type", sa.String(), nullable=False),
|
||||
sa.Column("number", sa.Integer(), nullable=False),
|
||||
sa.Column("title", sa.String(), nullable=True),
|
||||
sa.Column("title_short", sa.String(), nullable=True),
|
||||
sa.Column("official_title", sa.String(), nullable=True),
|
||||
sa.Column("status", sa.String(), nullable=True),
|
||||
sa.Column("status_at", sa.Date(), nullable=True),
|
||||
sa.Column("sponsor_bioguide_id", sa.String(), nullable=True),
|
||||
sa.Column("subjects_top_term", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_bill")),
|
||||
sa.UniqueConstraint("congress", "bill_type", "number", name="uq_bill_congress_type_number"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index("ix_bill_congress", "bill", ["congress"], unique=False, schema=schema)
|
||||
op.create_table(
|
||||
"legislator",
|
||||
sa.Column("bioguide_id", sa.Text(), nullable=False),
|
||||
sa.Column("thomas_id", sa.String(), nullable=True),
|
||||
sa.Column("lis_id", sa.String(), nullable=True),
|
||||
sa.Column("govtrack_id", sa.Integer(), nullable=True),
|
||||
sa.Column("opensecrets_id", sa.String(), nullable=True),
|
||||
sa.Column("fec_ids", sa.String(), nullable=True),
|
||||
sa.Column("first_name", sa.String(), nullable=False),
|
||||
sa.Column("last_name", sa.String(), nullable=False),
|
||||
sa.Column("official_full_name", sa.String(), nullable=True),
|
||||
sa.Column("nickname", sa.String(), nullable=True),
|
||||
sa.Column("birthday", sa.Date(), nullable=True),
|
||||
sa.Column("gender", sa.String(), nullable=True),
|
||||
sa.Column("current_party", sa.String(), nullable=True),
|
||||
sa.Column("current_state", sa.String(), nullable=True),
|
||||
sa.Column("current_district", sa.Integer(), nullable=True),
|
||||
sa.Column("current_chamber", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_legislator")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(op.f("ix_legislator_bioguide_id"), "legislator", ["bioguide_id"], unique=True, schema=schema)
|
||||
op.create_table(
|
||||
"vote",
|
||||
sa.Column("congress", sa.Integer(), nullable=False),
|
||||
sa.Column("chamber", sa.String(), nullable=False),
|
||||
sa.Column("session", sa.Integer(), nullable=False),
|
||||
sa.Column("number", sa.Integer(), nullable=False),
|
||||
sa.Column("vote_type", sa.String(), nullable=True),
|
||||
sa.Column("question", sa.String(), nullable=True),
|
||||
sa.Column("result", sa.String(), nullable=True),
|
||||
sa.Column("result_text", sa.String(), nullable=True),
|
||||
sa.Column("vote_date", sa.Date(), nullable=False),
|
||||
sa.Column("yea_count", sa.Integer(), nullable=True),
|
||||
sa.Column("nay_count", sa.Integer(), nullable=True),
|
||||
sa.Column("not_voting_count", sa.Integer(), nullable=True),
|
||||
sa.Column("present_count", sa.Integer(), nullable=True),
|
||||
sa.Column("bill_id", sa.Integer(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(["bill_id"], [f"{schema}.bill.id"], name=op.f("fk_vote_bill_id_bill")),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_vote")),
|
||||
sa.UniqueConstraint("congress", "chamber", "session", "number", name="uq_vote_congress_chamber_session_number"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index("ix_vote_congress_chamber", "vote", ["congress", "chamber"], unique=False, schema=schema)
|
||||
op.create_index("ix_vote_date", "vote", ["vote_date"], unique=False, schema=schema)
|
||||
op.create_table(
|
||||
"vote_record",
|
||||
sa.Column("vote_id", sa.Integer(), nullable=False),
|
||||
sa.Column("legislator_id", sa.Integer(), nullable=False),
|
||||
sa.Column("position", sa.String(), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["legislator_id"],
|
||||
[f"{schema}.legislator.id"],
|
||||
name=op.f("fk_vote_record_legislator_id_legislator"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["vote_id"], [f"{schema}.vote.id"], name=op.f("fk_vote_record_vote_id_vote"), ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("vote_id", "legislator_id", name=op.f("pk_vote_record")),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("vote_record", schema=schema)
|
||||
op.drop_index("ix_vote_date", table_name="vote", schema=schema)
|
||||
op.drop_index("ix_vote_congress_chamber", table_name="vote", schema=schema)
|
||||
op.drop_table("vote", schema=schema)
|
||||
op.drop_index(op.f("ix_legislator_bioguide_id"), table_name="legislator", schema=schema)
|
||||
op.drop_table("legislator", schema=schema)
|
||||
op.drop_index("ix_bill_congress", table_name="bill", schema=schema)
|
||||
op.drop_table("bill", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
-58
@@ -1,58 +0,0 @@
|
||||
"""adding SignalDevice for DeviceRegistry for signal bot.
|
||||
|
||||
Revision ID: 4c410c16e39c
|
||||
Revises: 3f71565e38de
|
||||
Create Date: 2026-03-09 14:51:24.228976
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "4c410c16e39c"
|
||||
down_revision: str | None = "3f71565e38de"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"signal_device",
|
||||
sa.Column("phone_number", sa.String(length=50), nullable=False),
|
||||
sa.Column("safety_number", sa.String(), nullable=False),
|
||||
sa.Column(
|
||||
"trust_level",
|
||||
postgresql.ENUM("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", schema=schema),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("last_seen", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_signal_device")),
|
||||
sa.UniqueConstraint("phone_number", name=op.f("uq_signal_device_phone_number")),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("signal_device", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
@@ -1,41 +0,0 @@
|
||||
"""fixed safety number logic.
|
||||
|
||||
Revision ID: 99fec682516c
|
||||
Revises: 4c410c16e39c
|
||||
Create Date: 2026-03-09 16:25:25.085806
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "99fec682516c"
|
||||
down_revision: str | None = "4c410c16e39c"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.alter_column("signal_device", "safety_number", existing_type=sa.VARCHAR(), nullable=True, schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.alter_column("signal_device", "safety_number", existing_type=sa.VARCHAR(), nullable=False, schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
-54
@@ -1,54 +0,0 @@
|
||||
"""add dead_letter_message table.
|
||||
|
||||
Revision ID: a1b2c3d4e5f6
|
||||
Revises: 99fec682516c
|
||||
Create Date: 2026-03-10 12:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "a1b2c3d4e5f6"
|
||||
down_revision: str | None = "99fec682516c"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
op.create_table(
|
||||
"dead_letter_message",
|
||||
sa.Column("source", sa.String(), nullable=False),
|
||||
sa.Column("message", sa.Text(), nullable=False),
|
||||
sa.Column("received_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column(
|
||||
"status",
|
||||
postgresql.ENUM("UNPROCESSED", "PROCESSED", name="message_status", schema=schema),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_dead_letter_message")),
|
||||
schema=schema,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
op.drop_table("dead_letter_message", schema=schema)
|
||||
op.execute(sa.text(f"DROP TYPE IF EXISTS {schema}.message_status"))
|
||||
-66
@@ -1,66 +0,0 @@
|
||||
"""adding roles to signal devices.
|
||||
|
||||
Revision ID: 2ef7ba690159
|
||||
Revises: a1b2c3d4e5f6
|
||||
Create Date: 2026-03-16 19:22:38.020350
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "2ef7ba690159"
|
||||
down_revision: str | None = "a1b2c3d4e5f6"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"role",
|
||||
sa.Column("name", sa.String(length=50), nullable=False),
|
||||
sa.Column("id", sa.SmallInteger(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_role")),
|
||||
sa.UniqueConstraint("name", name=op.f("uq_role_name")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"device_role",
|
||||
sa.Column("device_id", sa.Integer(), nullable=False),
|
||||
sa.Column("role_id", sa.SmallInteger(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["device_id"], [f"{schema}.signal_device.id"], name=op.f("fk_device_role_device_id_signal_device")
|
||||
),
|
||||
sa.ForeignKeyConstraint(["role_id"], [f"{schema}.role.id"], name=op.f("fk_device_role_role_id_role")),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_device_role")),
|
||||
sa.UniqueConstraint("device_id", "role_id", name="uq_device_role_device_role"),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("device_role", schema=schema)
|
||||
op.drop_table("role", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
-171
@@ -1,171 +0,0 @@
|
||||
"""seprating signal_bot database.
|
||||
|
||||
Revision ID: 6b275323f435
|
||||
Revises: 2ef7ba690159
|
||||
Create Date: 2026-03-18 08:34:28.785885
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "6b275323f435"
|
||||
down_revision: str | None = "2ef7ba690159"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("device_role", schema=schema)
|
||||
op.drop_table("signal_device", schema=schema)
|
||||
op.drop_table("role", schema=schema)
|
||||
op.drop_table("dead_letter_message", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"dead_letter_message",
|
||||
sa.Column("source", sa.VARCHAR(), autoincrement=False, nullable=False),
|
||||
sa.Column("message", sa.TEXT(), autoincrement=False, nullable=False),
|
||||
sa.Column("received_at", postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False),
|
||||
sa.Column(
|
||||
"status",
|
||||
postgresql.ENUM("UNPROCESSED", "PROCESSED", name="message_status", schema=schema),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_dead_letter_message")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"role",
|
||||
sa.Column("name", sa.VARCHAR(length=50), autoincrement=False, nullable=False),
|
||||
sa.Column(
|
||||
"id",
|
||||
sa.SMALLINT(),
|
||||
server_default=sa.text(f"nextval('{schema}.role_id_seq'::regclass)"),
|
||||
autoincrement=True,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"created",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_role")),
|
||||
sa.UniqueConstraint(
|
||||
"name", name=op.f("uq_role_name"), postgresql_include=[], postgresql_nulls_not_distinct=False
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"signal_device",
|
||||
sa.Column("phone_number", sa.VARCHAR(length=50), autoincrement=False, nullable=False),
|
||||
sa.Column("safety_number", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column(
|
||||
"trust_level",
|
||||
postgresql.ENUM("VERIFIED", "UNVERIFIED", "BLOCKED", name="trust_level", schema=schema),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("last_seen", postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False),
|
||||
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_signal_device")),
|
||||
sa.UniqueConstraint(
|
||||
"phone_number",
|
||||
name=op.f("uq_signal_device_phone_number"),
|
||||
postgresql_include=[],
|
||||
postgresql_nulls_not_distinct=False,
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"device_role",
|
||||
sa.Column("device_id", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("role_id", sa.SMALLINT(), autoincrement=False, nullable=False),
|
||||
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["device_id"], [f"{schema}.signal_device.id"], name=op.f("fk_device_role_device_id_signal_device")
|
||||
),
|
||||
sa.ForeignKeyConstraint(["role_id"], [f"{schema}.role.id"], name=op.f("fk_device_role_role_id_role")),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_device_role")),
|
||||
sa.UniqueConstraint(
|
||||
"device_id",
|
||||
"role_id",
|
||||
name=op.f("uq_device_role_device_role"),
|
||||
postgresql_include=[],
|
||||
postgresql_nulls_not_distinct=False,
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
-187
@@ -1,187 +0,0 @@
|
||||
"""removed ds table from richie DB.
|
||||
|
||||
Revision ID: c8a794340928
|
||||
Revises: 6b275323f435
|
||||
Create Date: 2026-03-29 15:29:23.643146
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "c8a794340928"
|
||||
down_revision: str | None = "6b275323f435"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("vote_record", schema=schema)
|
||||
op.drop_index(op.f("ix_vote_congress_chamber"), table_name="vote", schema=schema)
|
||||
op.drop_index(op.f("ix_vote_date"), table_name="vote", schema=schema)
|
||||
op.drop_index(op.f("ix_legislator_bioguide_id"), table_name="legislator", schema=schema)
|
||||
op.drop_table("legislator", schema=schema)
|
||||
op.drop_table("vote", schema=schema)
|
||||
op.drop_index(op.f("ix_bill_congress"), table_name="bill", schema=schema)
|
||||
op.drop_table("bill", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"vote",
|
||||
sa.Column("congress", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("chamber", sa.VARCHAR(), autoincrement=False, nullable=False),
|
||||
sa.Column("session", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("number", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("vote_type", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("question", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("result", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("result_text", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("vote_date", sa.DATE(), autoincrement=False, nullable=False),
|
||||
sa.Column("yea_count", sa.INTEGER(), autoincrement=False, nullable=True),
|
||||
sa.Column("nay_count", sa.INTEGER(), autoincrement=False, nullable=True),
|
||||
sa.Column("not_voting_count", sa.INTEGER(), autoincrement=False, nullable=True),
|
||||
sa.Column("present_count", sa.INTEGER(), autoincrement=False, nullable=True),
|
||||
sa.Column("bill_id", sa.INTEGER(), autoincrement=False, nullable=True),
|
||||
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(["bill_id"], [f"{schema}.bill.id"], name=op.f("fk_vote_bill_id_bill")),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_vote")),
|
||||
sa.UniqueConstraint(
|
||||
"congress",
|
||||
"chamber",
|
||||
"session",
|
||||
"number",
|
||||
name=op.f("uq_vote_congress_chamber_session_number"),
|
||||
postgresql_include=[],
|
||||
postgresql_nulls_not_distinct=False,
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(op.f("ix_vote_date"), "vote", ["vote_date"], unique=False, schema=schema)
|
||||
op.create_index(op.f("ix_vote_congress_chamber"), "vote", ["congress", "chamber"], unique=False, schema=schema)
|
||||
op.create_table(
|
||||
"vote_record",
|
||||
sa.Column("vote_id", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("legislator_id", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("position", sa.VARCHAR(), autoincrement=False, nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["legislator_id"],
|
||||
[f"{schema}.legislator.id"],
|
||||
name=op.f("fk_vote_record_legislator_id_legislator"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["vote_id"], [f"{schema}.vote.id"], name=op.f("fk_vote_record_vote_id_vote"), ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("vote_id", "legislator_id", name=op.f("pk_vote_record")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"legislator",
|
||||
sa.Column("bioguide_id", sa.TEXT(), autoincrement=False, nullable=False),
|
||||
sa.Column("thomas_id", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("lis_id", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("govtrack_id", sa.INTEGER(), autoincrement=False, nullable=True),
|
||||
sa.Column("opensecrets_id", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("fec_ids", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("first_name", sa.VARCHAR(), autoincrement=False, nullable=False),
|
||||
sa.Column("last_name", sa.VARCHAR(), autoincrement=False, nullable=False),
|
||||
sa.Column("official_full_name", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("nickname", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("birthday", sa.DATE(), autoincrement=False, nullable=True),
|
||||
sa.Column("gender", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("current_party", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("current_state", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("current_district", sa.INTEGER(), autoincrement=False, nullable=True),
|
||||
sa.Column("current_chamber", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_legislator")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(op.f("ix_legislator_bioguide_id"), "legislator", ["bioguide_id"], unique=True, schema=schema)
|
||||
op.create_table(
|
||||
"bill",
|
||||
sa.Column("congress", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("bill_type", sa.VARCHAR(), autoincrement=False, nullable=False),
|
||||
sa.Column("number", sa.INTEGER(), autoincrement=False, nullable=False),
|
||||
sa.Column("title", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("title_short", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("official_title", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("status", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("status_at", sa.DATE(), autoincrement=False, nullable=True),
|
||||
sa.Column("sponsor_bioguide_id", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("subjects_top_term", sa.VARCHAR(), autoincrement=False, nullable=True),
|
||||
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
|
||||
sa.Column(
|
||||
"created",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated",
|
||||
postgresql.TIMESTAMP(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
autoincrement=False,
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_bill")),
|
||||
sa.UniqueConstraint(
|
||||
"congress",
|
||||
"bill_type",
|
||||
"number",
|
||||
name=op.f("uq_bill_congress_type_number"),
|
||||
postgresql_include=[],
|
||||
postgresql_nulls_not_distinct=False,
|
||||
),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(op.f("ix_bill_congress"), "bill", ["congress"], unique=False, schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
-93
@@ -1,93 +0,0 @@
|
||||
"""adding audiobook libreary metadata.
|
||||
|
||||
Revision ID: d7864d1ffc17
|
||||
Revises: c8a794340928
|
||||
Create Date: 2026-06-03 20:24:09.200837
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "d7864d1ffc17"
|
||||
down_revision: str | None = "c8a794340928"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"audiobook_author",
|
||||
sa.Column("name", sa.String(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_audiobook_author")),
|
||||
sa.UniqueConstraint("name", name=op.f("uq_audiobook_author_name")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"audiobook_series",
|
||||
sa.Column("name", sa.String(), nullable=False),
|
||||
sa.Column("author_id", sa.Integer(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["author_id"],
|
||||
[f"{schema}.audiobook_author.id"],
|
||||
name=op.f("fk_audiobook_series_author_id_audiobook_author"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_audiobook_series")),
|
||||
sa.UniqueConstraint("author_id", "name", name=op.f("uq_audiobook_series_author_id")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"audiobook",
|
||||
sa.Column("title", sa.String(), nullable=False),
|
||||
sa.Column("author_id", sa.Integer(), nullable=False),
|
||||
sa.Column("series_id", sa.Integer(), nullable=True),
|
||||
sa.Column("series_index", sa.Integer(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["author_id"],
|
||||
[f"{schema}.audiobook_author.id"],
|
||||
name=op.f("fk_audiobook_author_id_audiobook_author"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["series_id"],
|
||||
[f"{schema}.audiobook_series.id"],
|
||||
name=op.f("fk_audiobook_series_id_audiobook_series"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_audiobook")),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("audiobook", schema=schema)
|
||||
op.drop_table("audiobook_series", schema=schema)
|
||||
op.drop_table("audiobook_author", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
@@ -1,200 +0,0 @@
|
||||
"""add ebook search tables.
|
||||
|
||||
Revision ID: 2db132cace1a
|
||||
Revises: b3c60cc5beb5
|
||||
Create Date: 2026-06-10 22:10:54.379159
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pgvector
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "2db132cace1a"
|
||||
down_revision: str | None = "b3c60cc5beb5"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"ebook_embedding_model",
|
||||
sa.Column("name", sa.String(), nullable=False),
|
||||
sa.Column("dimension", sa.Integer(), nullable=False),
|
||||
sa.Column("is_default", sa.Boolean(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_embedding_model")),
|
||||
sa.UniqueConstraint("name", name=op.f("uq_ebook_embedding_model_name")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"ebook_source",
|
||||
sa.Column("title", sa.String(), nullable=False),
|
||||
sa.Column("author", sa.String(), nullable=True),
|
||||
sa.Column("language", sa.String(), nullable=True),
|
||||
sa.Column("publisher", sa.String(), nullable=True),
|
||||
sa.Column("identifier", sa.String(), nullable=True),
|
||||
sa.Column("file_path", sa.String(), nullable=False),
|
||||
sa.Column("file_sha256", sa.String(length=64), nullable=False),
|
||||
sa.Column("file_mtime", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("file_size", sa.BigInteger(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_source")),
|
||||
sa.UniqueConstraint("file_path", name=op.f("uq_ebook_source_file_path")),
|
||||
sa.UniqueConstraint("file_sha256", name=op.f("uq_ebook_source_file_sha256")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"ebook_chapter",
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("spine_index", sa.Integer(), nullable=False),
|
||||
sa.Column("title", sa.String(), nullable=True),
|
||||
sa.Column("href", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"],
|
||||
[f"{schema}.ebook_source.id"],
|
||||
name=op.f("fk_ebook_chapter_source_id_ebook_source"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chapter")),
|
||||
sa.UniqueConstraint("source_id", "spine_index", name=op.f("uq_ebook_chapter_source_id")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"ebook_chunk",
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("chapter_id", sa.Integer(), nullable=True),
|
||||
sa.Column("chunk_index", sa.Integer(), nullable=False),
|
||||
sa.Column("text", sa.String(), nullable=False),
|
||||
sa.Column("token_start", sa.Integer(), nullable=False),
|
||||
sa.Column("token_count", sa.Integer(), nullable=False),
|
||||
sa.Column("page_label", sa.String(), nullable=True),
|
||||
sa.Column("content_sha256", sa.String(length=64), nullable=False),
|
||||
sa.Column("search_text", sa.String(), nullable=False),
|
||||
sa.Column("id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["chapter_id"],
|
||||
[f"{schema}.ebook_chapter.id"],
|
||||
name=op.f("fk_ebook_chunk_chapter_id_ebook_chapter"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"],
|
||||
[f"{schema}.ebook_source.id"],
|
||||
name=op.f("fk_ebook_chunk_source_id_ebook_source"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk")),
|
||||
sa.UniqueConstraint("source_id", "chunk_index", name="uq_ebook_chunk_source_id_chunk_index"),
|
||||
sa.UniqueConstraint("source_id", "content_sha256", name="uq_ebook_chunk_source_id_content_sha256"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"ebook_chunk_embedding_1024",
|
||||
sa.Column("chunk_id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("model_id", sa.Integer(), nullable=False),
|
||||
sa.Column("embedding", pgvector.sqlalchemy.vector.VECTOR(dim=1024), nullable=False),
|
||||
sa.Column("id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["chunk_id"],
|
||||
[f"{schema}.ebook_chunk.id"],
|
||||
name=op.f("fk_ebook_chunk_embedding_1024_chunk_id_ebook_chunk"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["model_id"],
|
||||
[f"{schema}.ebook_embedding_model.id"],
|
||||
name=op.f("fk_ebook_chunk_embedding_1024_model_id_ebook_embedding_model"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk_embedding_1024")),
|
||||
sa.UniqueConstraint("chunk_id", "model_id", name=op.f("uq_ebook_chunk_embedding_1024_chunk_id")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"ebook_chunk_embedding_2560",
|
||||
sa.Column("chunk_id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("model_id", sa.Integer(), nullable=False),
|
||||
sa.Column("embedding", pgvector.sqlalchemy.vector.VECTOR(dim=2560), nullable=False),
|
||||
sa.Column("id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["chunk_id"],
|
||||
[f"{schema}.ebook_chunk.id"],
|
||||
name=op.f("fk_ebook_chunk_embedding_2560_chunk_id_ebook_chunk"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["model_id"],
|
||||
[f"{schema}.ebook_embedding_model.id"],
|
||||
name=op.f("fk_ebook_chunk_embedding_2560_model_id_ebook_embedding_model"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk_embedding_2560")),
|
||||
sa.UniqueConstraint("chunk_id", "model_id", name=op.f("uq_ebook_chunk_embedding_2560_chunk_id")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"ebook_chunk_embedding_4096",
|
||||
sa.Column("chunk_id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("model_id", sa.Integer(), nullable=False),
|
||||
sa.Column("embedding", pgvector.sqlalchemy.vector.VECTOR(dim=4096), nullable=False),
|
||||
sa.Column("id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["chunk_id"],
|
||||
[f"{schema}.ebook_chunk.id"],
|
||||
name=op.f("fk_ebook_chunk_embedding_4096_chunk_id_ebook_chunk"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["model_id"],
|
||||
[f"{schema}.ebook_embedding_model.id"],
|
||||
name=op.f("fk_ebook_chunk_embedding_4096_model_id_ebook_embedding_model"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_ebook_chunk_embedding_4096")),
|
||||
sa.UniqueConstraint("chunk_id", "model_id", name=op.f("uq_ebook_chunk_embedding_4096_chunk_id")),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("ebook_chunk_embedding_4096", schema=schema)
|
||||
op.drop_table("ebook_chunk_embedding_2560", schema=schema)
|
||||
op.drop_table("ebook_chunk_embedding_1024", schema=schema)
|
||||
op.drop_table("ebook_chunk", schema=schema)
|
||||
op.drop_table("ebook_chapter", schema=schema)
|
||||
op.drop_table("ebook_source", schema=schema)
|
||||
op.drop_table("ebook_embedding_model", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
-63
@@ -1,63 +0,0 @@
|
||||
"""updated series_index to float and added UniqueConstraint to audiobook and audiobook_author.
|
||||
|
||||
Revision ID: b3c60cc5beb5
|
||||
Revises: d7864d1ffc17
|
||||
Create Date: 2026-06-10 20:02:43.073725
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "b3c60cc5beb5"
|
||||
down_revision: str | None = "d7864d1ffc17"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.alter_column(
|
||||
"audiobook",
|
||||
"series_index",
|
||||
existing_type=sa.INTEGER(),
|
||||
type_=sa.Float(),
|
||||
existing_nullable=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_unique_constraint(
|
||||
op.f("uq_audiobook_author_id"),
|
||||
"audiobook",
|
||||
["author_id", "series_id", "title"],
|
||||
schema=schema,
|
||||
postgresql_nulls_not_distinct=True,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_constraint(op.f("uq_audiobook_author_id"), "audiobook", schema=schema, type_="unique")
|
||||
op.alter_column(
|
||||
"audiobook",
|
||||
"series_index",
|
||||
existing_type=sa.Float(),
|
||||
type_=sa.INTEGER(),
|
||||
existing_nullable=False,
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
-54
@@ -1,54 +0,0 @@
|
||||
"""add 1024 ebook embedding cosine index.
|
||||
|
||||
Revision ID: c460105682d2
|
||||
Revises: 2db132cace1a
|
||||
Create Date: 2026-06-13 19:53:45.680289
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "c460105682d2"
|
||||
down_revision: str | None = "2db132cace1a"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_index(
|
||||
"ix_ebook_chunk_embedding_1024_embedding_cosine",
|
||||
"ebook_chunk_embedding_1024",
|
||||
["embedding"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
postgresql_using="hnsw",
|
||||
postgresql_ops={"embedding": "vector_cosine_ops"},
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index(
|
||||
"ix_ebook_chunk_embedding_1024_embedding_cosine",
|
||||
table_name="ebook_chunk_embedding_1024",
|
||||
schema=schema,
|
||||
postgresql_using="hnsw",
|
||||
postgresql_ops={"embedding": "vector_cosine_ops"},
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
@@ -1,103 +0,0 @@
|
||||
"""adding haproxy data.
|
||||
|
||||
Revision ID: 96d72c748c24
|
||||
Revises: c460105682d2
|
||||
Create Date: 2026-06-23 16:37:17.768851
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "96d72c748c24"
|
||||
down_revision: str | None = "c460105682d2"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"haproxy_request",
|
||||
sa.Column("line_hash", sa.String(), nullable=False),
|
||||
sa.Column("requested_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("client_ip", sa.String(), nullable=False),
|
||||
sa.Column("client_port", sa.Integer(), nullable=False),
|
||||
sa.Column("frontend", sa.String(), nullable=False),
|
||||
sa.Column("ssl", sa.Boolean(), nullable=False),
|
||||
sa.Column("backend", sa.String(), nullable=False),
|
||||
sa.Column("server", sa.String(), nullable=False),
|
||||
sa.Column("time_request", sa.Integer(), nullable=False),
|
||||
sa.Column("time_queue", sa.Integer(), nullable=False),
|
||||
sa.Column("time_connect", sa.Integer(), nullable=False),
|
||||
sa.Column("time_response", sa.Integer(), nullable=False),
|
||||
sa.Column("time_total", sa.Integer(), nullable=False),
|
||||
sa.Column("status_code", sa.Integer(), nullable=False),
|
||||
sa.Column("bytes_read", sa.BigInteger(), nullable=False),
|
||||
sa.Column("termination_state", sa.String(), nullable=False),
|
||||
sa.Column("active_connections", sa.Integer(), nullable=False),
|
||||
sa.Column("frontend_connections", sa.Integer(), nullable=False),
|
||||
sa.Column("backend_connections", sa.Integer(), nullable=False),
|
||||
sa.Column("server_connections", sa.Integer(), nullable=False),
|
||||
sa.Column("retries", sa.Integer(), nullable=False),
|
||||
sa.Column("server_queue", sa.Integer(), nullable=False),
|
||||
sa.Column("backend_queue", sa.Integer(), nullable=False),
|
||||
sa.Column("host", sa.String(), nullable=True),
|
||||
sa.Column("user_agent", sa.String(), nullable=True),
|
||||
sa.Column("method", sa.String(), nullable=False),
|
||||
sa.Column("target", sa.String(), nullable=False),
|
||||
sa.Column("path", sa.String(), nullable=False),
|
||||
sa.Column("query", sa.String(), nullable=True),
|
||||
sa.Column("http_version", sa.String(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_haproxy_request")),
|
||||
sa.UniqueConstraint("line_hash", name=op.f("uq_haproxy_request_line_hash")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(op.f("ix_haproxy_request_backend"), "haproxy_request", ["backend"], unique=False, schema=schema)
|
||||
op.create_index(op.f("ix_haproxy_request_client_ip"), "haproxy_request", ["client_ip"], unique=False, schema=schema)
|
||||
op.create_index(op.f("ix_haproxy_request_host"), "haproxy_request", ["host"], unique=False, schema=schema)
|
||||
op.create_index(op.f("ix_haproxy_request_path"), "haproxy_request", ["path"], unique=False, schema=schema)
|
||||
op.create_index(
|
||||
op.f("ix_haproxy_request_requested_at"), "haproxy_request", ["requested_at"], unique=False, schema=schema
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_haproxy_request_status_code"), "haproxy_request", ["status_code"], unique=False, schema=schema
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_haproxy_request_time_response"), "haproxy_request", ["time_response"], unique=False, schema=schema
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_haproxy_request_user_agent"), "haproxy_request", ["user_agent"], unique=False, schema=schema
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index(op.f("ix_haproxy_request_user_agent"), table_name="haproxy_request", schema=schema)
|
||||
op.drop_index(op.f("ix_haproxy_request_time_response"), table_name="haproxy_request", schema=schema)
|
||||
op.drop_index(op.f("ix_haproxy_request_status_code"), table_name="haproxy_request", schema=schema)
|
||||
op.drop_index(op.f("ix_haproxy_request_requested_at"), table_name="haproxy_request", schema=schema)
|
||||
op.drop_index(op.f("ix_haproxy_request_path"), table_name="haproxy_request", schema=schema)
|
||||
op.drop_index(op.f("ix_haproxy_request_host"), table_name="haproxy_request", schema=schema)
|
||||
op.drop_index(op.f("ix_haproxy_request_client_ip"), table_name="haproxy_request", schema=schema)
|
||||
op.drop_index(op.f("ix_haproxy_request_backend"), table_name="haproxy_request", schema=schema)
|
||||
op.drop_table("haproxy_request", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
-206
@@ -1,206 +0,0 @@
|
||||
"""adding Phrase metadata tables.
|
||||
|
||||
Revision ID: dddee09eddcc
|
||||
Revises: 96d72c748c24
|
||||
Create Date: 2026-06-29 00:49:07.344159
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "dddee09eddcc"
|
||||
down_revision: str | None = "96d72c748c24"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"candidate_phrases",
|
||||
sa.Column("book_id", sa.Integer(), nullable=False),
|
||||
sa.Column("series_id", sa.Integer(), nullable=True),
|
||||
sa.Column("phrase_text", sa.Text(), nullable=False),
|
||||
sa.Column("phrase_norm", sa.Text(), nullable=False),
|
||||
sa.Column("token_count", sa.Integer(), nullable=False),
|
||||
sa.Column("source_raw_ngram", sa.Boolean(), nullable=False),
|
||||
sa.Column("source_yake", sa.Boolean(), nullable=False),
|
||||
sa.Column("source_spacy_ner", sa.Boolean(), nullable=False),
|
||||
sa.Column("source_spacy_noun_chunk", sa.Boolean(), nullable=False),
|
||||
sa.Column("source_capitalized", sa.Boolean(), nullable=False),
|
||||
sa.Column("source_metadata", sa.Boolean(), nullable=False),
|
||||
sa.Column("spacy_label", sa.String(), nullable=True),
|
||||
sa.Column("raw_count", sa.Integer(), nullable=False),
|
||||
sa.Column("chapter_count", sa.Integer(), nullable=False),
|
||||
sa.Column("yake_score", sa.Float(), nullable=True),
|
||||
sa.Column("candidate_score", sa.Float(), nullable=False),
|
||||
sa.Column(
|
||||
"sample_contexts",
|
||||
sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), "postgresql"),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column("llm_judged", sa.Boolean(), nullable=False),
|
||||
sa.Column("llm_keep", sa.Boolean(), nullable=True),
|
||||
sa.Column("llm_confidence", sa.Float(), nullable=True),
|
||||
sa.Column("llm_category", sa.String(), nullable=True),
|
||||
sa.Column("llm_reason", sa.Text(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["book_id"],
|
||||
[f"{schema}.ebook_source.id"],
|
||||
name=op.f("fk_candidate_phrases_book_id_ebook_source"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_candidate_phrases")),
|
||||
sa.UniqueConstraint("book_id", "phrase_norm", name="uq_candidate_phrases_book_id_phrase_norm"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"candidate_phrases_book_norm_idx", "candidate_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
|
||||
)
|
||||
op.create_index(
|
||||
"candidate_phrases_book_score_idx",
|
||||
"candidate_phrases",
|
||||
["book_id", "candidate_score"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"protected_phrases",
|
||||
sa.Column("book_id", sa.Integer(), nullable=True),
|
||||
sa.Column("series_id", sa.Integer(), nullable=True),
|
||||
sa.Column("phrase_text", sa.Text(), nullable=False),
|
||||
sa.Column("phrase_norm", sa.Text(), nullable=False),
|
||||
sa.Column("canonical_id", sa.String(), nullable=False),
|
||||
sa.Column("phrase_type", sa.String(), nullable=True),
|
||||
sa.Column("token_count", sa.Integer(), nullable=False),
|
||||
sa.Column("confidence", sa.Float(), nullable=False),
|
||||
sa.Column("importance", sa.Float(), nullable=False),
|
||||
sa.Column("allow_nested", sa.Boolean(), nullable=False),
|
||||
sa.Column("suppress_children", sa.Boolean(), nullable=False),
|
||||
sa.Column("source_candidate_id", sa.Integer(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["book_id"],
|
||||
[f"{schema}.ebook_source.id"],
|
||||
name=op.f("fk_protected_phrases_book_id_ebook_source"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_candidate_id"],
|
||||
[f"{schema}.candidate_phrases.id"],
|
||||
name=op.f("fk_protected_phrases_source_candidate_id_candidate_phrases"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_protected_phrases")),
|
||||
sa.UniqueConstraint("book_id", "phrase_norm", name="uq_protected_phrases_book_id_phrase_norm"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"protected_phrases_book_norm_idx", "protected_phrases", ["book_id", "phrase_norm"], unique=False, schema=schema
|
||||
)
|
||||
op.create_index("protected_phrases_norm_idx", "protected_phrases", ["phrase_norm"], unique=False, schema=schema)
|
||||
op.create_index(
|
||||
"protected_phrases_series_norm_idx",
|
||||
"protected_phrases",
|
||||
["series_id", "phrase_norm"],
|
||||
unique=False,
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"chunk_phrase_mentions",
|
||||
sa.Column("chunk_id", sa.BigInteger(), nullable=False),
|
||||
sa.Column("phrase_id", sa.Integer(), nullable=False),
|
||||
sa.Column("book_id", sa.Integer(), nullable=True),
|
||||
sa.Column("series_id", sa.Integer(), nullable=True),
|
||||
sa.Column("start_char", sa.Integer(), nullable=False),
|
||||
sa.Column("end_char", sa.Integer(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["book_id"],
|
||||
[f"{schema}.ebook_source.id"],
|
||||
name=op.f("fk_chunk_phrase_mentions_book_id_ebook_source"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["chunk_id"],
|
||||
[f"{schema}.ebook_chunk.id"],
|
||||
name=op.f("fk_chunk_phrase_mentions_chunk_id_ebook_chunk"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["phrase_id"],
|
||||
[f"{schema}.protected_phrases.id"],
|
||||
name=op.f("fk_chunk_phrase_mentions_phrase_id_protected_phrases"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_chunk_phrase_mentions")),
|
||||
sa.UniqueConstraint("chunk_id", "phrase_id", "start_char", name="uq_chunk_phrase_mentions_chunk_phrase_start"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index(
|
||||
"chunk_phrase_mentions_chunk_idx", "chunk_phrase_mentions", ["chunk_id"], unique=False, schema=schema
|
||||
)
|
||||
op.create_index(
|
||||
"chunk_phrase_mentions_phrase_idx", "chunk_phrase_mentions", ["phrase_id"], unique=False, schema=schema
|
||||
)
|
||||
op.create_table(
|
||||
"phrase_aliases",
|
||||
sa.Column("phrase_id", sa.Integer(), nullable=False),
|
||||
sa.Column("alias_text", sa.Text(), nullable=False),
|
||||
sa.Column("alias_norm", sa.Text(), nullable=False),
|
||||
sa.Column("confidence", sa.Float(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["phrase_id"],
|
||||
[f"{schema}.protected_phrases.id"],
|
||||
name=op.f("fk_phrase_aliases_phrase_id_protected_phrases"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_phrase_aliases")),
|
||||
sa.UniqueConstraint("phrase_id", "alias_norm", name="uq_phrase_aliases_phrase_id_alias_norm"),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_index("phrase_aliases_norm_idx", "phrase_aliases", ["alias_norm"], unique=False, schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index("phrase_aliases_norm_idx", table_name="phrase_aliases", schema=schema)
|
||||
op.drop_table("phrase_aliases", schema=schema)
|
||||
op.drop_index("chunk_phrase_mentions_phrase_idx", table_name="chunk_phrase_mentions", schema=schema)
|
||||
op.drop_index("chunk_phrase_mentions_chunk_idx", table_name="chunk_phrase_mentions", schema=schema)
|
||||
op.drop_table("chunk_phrase_mentions", schema=schema)
|
||||
op.drop_index("protected_phrases_series_norm_idx", table_name="protected_phrases", schema=schema)
|
||||
op.drop_index("protected_phrases_norm_idx", table_name="protected_phrases", schema=schema)
|
||||
op.drop_index("protected_phrases_book_norm_idx", table_name="protected_phrases", schema=schema)
|
||||
op.drop_table("protected_phrases", schema=schema)
|
||||
op.drop_index("candidate_phrases_book_score_idx", table_name="candidate_phrases", schema=schema)
|
||||
op.drop_index("candidate_phrases_book_norm_idx", table_name="candidate_phrases", schema=schema)
|
||||
op.drop_table("candidate_phrases", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
@@ -1,55 +0,0 @@
|
||||
"""remove spaCy-ner.
|
||||
|
||||
Revision ID: 751260fc3228
|
||||
Revises: dddee09eddcc
|
||||
Create Date: 2026-07-09 23:03:39.554083
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import RichieBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "751260fc3228"
|
||||
down_revision: str | None = "dddee09eddcc"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = RichieBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_column("candidate_phrases", "source_spacy_noun_chunk", schema=schema)
|
||||
op.drop_column("candidate_phrases", "source_spacy_ner", schema=schema)
|
||||
op.drop_column("candidate_phrases", "spacy_label", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.add_column(
|
||||
"candidate_phrases", sa.Column("spacy_label", sa.VARCHAR(), autoincrement=False, nullable=True), schema=schema
|
||||
)
|
||||
op.add_column(
|
||||
"candidate_phrases",
|
||||
sa.Column("source_spacy_ner", sa.BOOLEAN(), autoincrement=False, nullable=False),
|
||||
schema=schema,
|
||||
)
|
||||
op.add_column(
|
||||
"candidate_phrases",
|
||||
sa.Column("source_spacy_noun_chunk", sa.BOOLEAN(), autoincrement=False, nullable=False),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
@@ -1,36 +0,0 @@
|
||||
"""${message}.
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
from python.orm import ${config.attributes["base"].__name__}
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: str | None = ${repr(down_revision)}
|
||||
branch_labels: str | Sequence[str] | None = ${repr(branch_labels)}
|
||||
depends_on: str | Sequence[str] | None = ${repr(depends_on)}
|
||||
|
||||
schema=${config.attributes["base"].__name__}.schema_name
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
${downgrades if downgrades else "pass"}
|
||||
-80
@@ -1,80 +0,0 @@
|
||||
"""starting van invintory.
|
||||
|
||||
Revision ID: 15e733499804
|
||||
Revises:
|
||||
Create Date: 2026-03-08 00:18:20.759720
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from python.orm import VanInventoryBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "15e733499804"
|
||||
down_revision: str | None = None
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
schema = VanInventoryBase.schema_name
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"items",
|
||||
sa.Column("name", sa.String(), nullable=False),
|
||||
sa.Column("quantity", sa.Float(), nullable=False),
|
||||
sa.Column("unit", sa.String(), nullable=False),
|
||||
sa.Column("category", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_items")),
|
||||
sa.UniqueConstraint("name", name=op.f("uq_items_name")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"meals",
|
||||
sa.Column("name", sa.String(), nullable=False),
|
||||
sa.Column("instructions", sa.String(), nullable=True),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_meals")),
|
||||
sa.UniqueConstraint("name", name=op.f("uq_meals_name")),
|
||||
schema=schema,
|
||||
)
|
||||
op.create_table(
|
||||
"meal_ingredients",
|
||||
sa.Column("meal_id", sa.Integer(), nullable=False),
|
||||
sa.Column("item_id", sa.Integer(), nullable=False),
|
||||
sa.Column("quantity_needed", sa.Float(), nullable=False),
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("created", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.ForeignKeyConstraint(["item_id"], [f"{schema}.items.id"], name=op.f("fk_meal_ingredients_item_id_items")),
|
||||
sa.ForeignKeyConstraint(["meal_id"], [f"{schema}.meals.id"], name=op.f("fk_meal_ingredients_meal_id_meals")),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_meal_ingredients")),
|
||||
sa.UniqueConstraint("meal_id", "item_id", name=op.f("uq_meal_ingredients_meal_id")),
|
||||
schema=schema,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("meal_ingredients", schema=schema)
|
||||
op.drop_table("meals", schema=schema)
|
||||
op.drop_table("items", schema=schema)
|
||||
# ### end Alembic commands ###
|
||||
@@ -1,47 +0,0 @@
|
||||
"""common."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from subprocess import PIPE, Popen
|
||||
|
||||
from python.logging_config import configure_logger as _configure_logger
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_repo_dir() -> Path:
|
||||
"""Return the repository root directory."""
|
||||
return Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def configure_logger(level: str = "INFO") -> None:
|
||||
"""Configure the logger."""
|
||||
_configure_logger(level)
|
||||
|
||||
|
||||
def bash_wrapper(command: str) -> tuple[str, int]:
|
||||
"""Execute a bash command and capture the output.
|
||||
|
||||
Args:
|
||||
command (str): The bash command to be executed.
|
||||
|
||||
Returns:
|
||||
Tuple[str, int]: A tuple containing the output of the command (stdout) as a string,
|
||||
the error output (stderr) as a string (optional), and the return code as an integer.
|
||||
"""
|
||||
# This is a acceptable risk
|
||||
process = Popen(command.split(), stdout=PIPE, stderr=PIPE)
|
||||
output, error = process.communicate()
|
||||
if error:
|
||||
logger.error(f"{error=}")
|
||||
return error.decode(), process.returncode
|
||||
|
||||
return output.decode(), process.returncode
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
"""Get the current UTC time."""
|
||||
return datetime.now(tz=UTC)
|
||||
@@ -1,59 +0,0 @@
|
||||
"""database."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy.exc import NoInspectionAvailable
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def safe_insert(orm_objects: Sequence[object], session: Session) -> list[tuple[Exception, object]]:
|
||||
"""Safer insert at allows for partial rollbacks.
|
||||
|
||||
Args:
|
||||
orm_objects (Sequence[object]): Tables to insert.
|
||||
session (Session): Database session.
|
||||
"""
|
||||
if unmapped := [orm_object for orm_object in orm_objects if not _is_mapped_instance(orm_object)]:
|
||||
error = f"safe_insert expects ORM-mapped instances {unmapped}"
|
||||
raise TypeError(error)
|
||||
return _safe_insert(orm_objects, session)
|
||||
|
||||
|
||||
def _safe_insert(objects: Sequence[object], session: Session) -> list[tuple[Exception, object]]:
|
||||
exceptions: list[tuple[Exception, object]] = []
|
||||
try:
|
||||
session.add_all(objects)
|
||||
session.commit()
|
||||
|
||||
except Exception as error:
|
||||
session.rollback()
|
||||
|
||||
objects_len = len(objects)
|
||||
if objects_len == 1:
|
||||
logger.exception(objects)
|
||||
return [(error, objects[0])]
|
||||
|
||||
middle = objects_len // 2
|
||||
exceptions.extend(_safe_insert(objects=objects[:middle], session=session))
|
||||
exceptions.extend(_safe_insert(objects=objects[middle:], session=session))
|
||||
return exceptions
|
||||
|
||||
|
||||
def _is_mapped_instance(obj: object) -> bool:
|
||||
"""Return True if `obj` is a SQLAlchemy ORM-mapped instance."""
|
||||
try:
|
||||
inspect(obj) # raises NoInspectionAvailable if not mapped
|
||||
except NoInspectionAvailable:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
@@ -1,103 +0,0 @@
|
||||
"""CLI wrapper around alembic for multi-database support.
|
||||
|
||||
Usage:
|
||||
database <db_name> <command> [args...]
|
||||
|
||||
Examples:
|
||||
database richie check
|
||||
database richie upgrade head
|
||||
database richie downgrade head-1
|
||||
database richie revision --autogenerate -m "add meals table"
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from importlib import import_module
|
||||
from typing import TYPE_CHECKING, Annotated
|
||||
|
||||
import typer
|
||||
from alembic.config import CommandLine, Config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatabaseConfig:
|
||||
"""Configuration for a database."""
|
||||
|
||||
env_prefix: str
|
||||
version_location: str
|
||||
base_module: str
|
||||
base_class_name: str
|
||||
models_module: str
|
||||
script_location: str = "python/alembic"
|
||||
file_template: str = "%%(year)d_%%(month).2d_%%(day).2d-%%(slug)s_%%(rev)s"
|
||||
|
||||
def get_base(self) -> type[DeclarativeBase]:
|
||||
"""Import and return the Base class."""
|
||||
module = import_module(self.base_module)
|
||||
return getattr(module, self.base_class_name)
|
||||
|
||||
def import_models(self) -> None:
|
||||
"""Import ORM models so alembic autogenerate can detect them."""
|
||||
import_module(self.models_module)
|
||||
|
||||
def alembic_config(self) -> Config:
|
||||
"""Build an alembic Config for this database."""
|
||||
cfg = Config()
|
||||
cfg.set_main_option("script_location", self.script_location)
|
||||
cfg.set_main_option("file_template", self.file_template)
|
||||
cfg.set_main_option("prepend_sys_path", ".")
|
||||
cfg.set_main_option("version_path_separator", "os")
|
||||
cfg.set_main_option("version_locations", self.version_location)
|
||||
cfg.set_main_option("revision_environment", "true")
|
||||
cfg.set_section_option("post_write_hooks", "hooks", "dynamic_schema,import_postgresql,ruff")
|
||||
cfg.set_section_option("post_write_hooks", "dynamic_schema.type", "dynamic_schema")
|
||||
cfg.set_section_option("post_write_hooks", "import_postgresql.type", "import_postgresql")
|
||||
cfg.set_section_option("post_write_hooks", "ruff.type", "ruff")
|
||||
cfg.attributes["base"] = self.get_base()
|
||||
cfg.attributes["env_prefix"] = self.env_prefix
|
||||
self.import_models()
|
||||
return cfg
|
||||
|
||||
|
||||
DATABASES: dict[str, DatabaseConfig] = {
|
||||
"richie": DatabaseConfig(
|
||||
env_prefix="RICHIE",
|
||||
version_location="python/alembic/richie/versions",
|
||||
base_module="python.orm.richie.base",
|
||||
base_class_name="RichieBase",
|
||||
models_module="python.orm.richie",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
app = typer.Typer(help="Multi-database alembic wrapper.")
|
||||
|
||||
|
||||
@app.command(
|
||||
context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
|
||||
)
|
||||
def main(
|
||||
ctx: typer.Context,
|
||||
db_name: Annotated[str, typer.Argument(help=f"Database name. Options: {', '.join(DATABASES)}")],
|
||||
command: Annotated[str, typer.Argument(help="Alembic command (upgrade, downgrade, revision, check, etc.)")],
|
||||
) -> None:
|
||||
"""Run an alembic command against the specified database."""
|
||||
db_config = DATABASES.get(db_name)
|
||||
if not db_config:
|
||||
typer.echo(f"Unknown database: {db_name!r}. Available: {', '.join(DATABASES)}", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
alembic_cfg = db_config.alembic_config()
|
||||
|
||||
cmd_line = CommandLine()
|
||||
options = cmd_line.parser.parse_args([command, *ctx.args])
|
||||
cmd_line.run_cmd(alembic_cfg, options)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
"""EPUB search package."""
|
||||
@@ -1,50 +0,0 @@
|
||||
"""Grounded answer generation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from python.ebook_search.llm_interface import request_chat_completion
|
||||
from python.ebook_search.prompts import load_prompt
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import httpx
|
||||
|
||||
from python.ebook_search.config import EbookSearchConfig
|
||||
from python.ebook_search.search import SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def answer_query(
|
||||
client: httpx.AsyncClient,
|
||||
query: str,
|
||||
results: list[SearchResult],
|
||||
config: EbookSearchConfig,
|
||||
) -> str:
|
||||
"""Answer a question using only retrieved chunks."""
|
||||
if not config.answer_enabled:
|
||||
logger.info("ebook_answer_skipped_disabled")
|
||||
return "Answer generation is disabled. Source chunks are shown below."
|
||||
|
||||
if not results:
|
||||
logger.info("ebook_answer_skipped_no_results")
|
||||
return "No relevant sources were found."
|
||||
|
||||
logger.info(
|
||||
f"ebook_answer_request_start {config.vllm_base_url=} {config.chat_model=} sources={len(results)} "
|
||||
f"query_length={len(query)}"
|
||||
)
|
||||
context = "\n\n".join(
|
||||
f"[{index}] {result.source_title}{' - ' + result.chapter_title if result.chapter_title else ''}\n{result.text}"
|
||||
for index, result in enumerate(results, start=1)
|
||||
)
|
||||
content = await request_chat_completion(
|
||||
client,
|
||||
config,
|
||||
load_prompt("answer").messages(query=query, context=context),
|
||||
)
|
||||
|
||||
logger.info(f"ebook_answer_request_complete {config.chat_model=} answer_length={len(content)}")
|
||||
return content or "The model returned an empty answer."
|
||||
@@ -1 +0,0 @@
|
||||
"""Web and external API adapters for EPUB search."""
|
||||
@@ -1,70 +0,0 @@
|
||||
"""Background BM25 refresh tasks for the web app.
|
||||
|
||||
The refresh is scheduled on the event loop instead of a thread because the async psycopg
|
||||
driver only works from the loop; a bare thread cannot open a session on the async engine.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from python.ebook_search.bm25_corpus import load_bm25_corpus, refresh_bm25_corpus
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import FastAPI
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
|
||||
from python.ebook_search.config import EbookSearchConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def schedule_bm25_refresh(app: FastAPI) -> None:
|
||||
"""Schedule a delayed BM25 corpus refresh, replacing any pending refresh.
|
||||
|
||||
Only called from route handlers, so a running event loop is guaranteed.
|
||||
"""
|
||||
cancel_bm25_refresh(app)
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def start_refresh() -> None:
|
||||
app.state.bm25_refresh_task = loop.create_task(refresh_bm25_for_app(app))
|
||||
|
||||
app.state.bm25_refresh_timer = loop.call_later(app.state.config.bm25_refresh_delay_seconds, start_refresh)
|
||||
logger.info(f"ebook_bm25_refresh_scheduled {app.state.config.bm25_refresh_delay_seconds=}")
|
||||
|
||||
|
||||
def cancel_bm25_refresh(app: FastAPI) -> None:
|
||||
"""Cancel any pending BM25 corpus refresh timer and in-flight refresh task."""
|
||||
existing_timer = getattr(app.state, "bm25_refresh_timer", None)
|
||||
if existing_timer is not None:
|
||||
existing_timer.cancel()
|
||||
app.state.bm25_refresh_timer = None
|
||||
logger.info("ebook_bm25_refresh_cancelled")
|
||||
|
||||
existing_task = getattr(app.state, "bm25_refresh_task", None)
|
||||
if existing_task is not None:
|
||||
if not existing_task.done():
|
||||
existing_task.cancel()
|
||||
app.state.bm25_refresh_task = None
|
||||
|
||||
|
||||
async def refresh_bm25_for_app(app: FastAPI) -> None:
|
||||
"""Refresh the BM25 corpus using the app engine and config."""
|
||||
try:
|
||||
await refresh_bm25_for_engine(app.state.engine, app.state.config)
|
||||
except Exception:
|
||||
logger.exception("ebook_bm25_refresh_failed")
|
||||
|
||||
|
||||
async def refresh_bm25_for_engine(engine: AsyncEngine, config: EbookSearchConfig) -> None:
|
||||
"""Refresh the BM25 corpus using an async SQLAlchemy engine."""
|
||||
async with AsyncSession(engine) as session:
|
||||
await refresh_bm25_corpus(session, config)
|
||||
load_bm25_corpus.cache_clear()
|
||||
logger.info("ebook_bm25_corpus_cache_cleared_after_refresh")
|
||||
@@ -1,24 +0,0 @@
|
||||
"""FastAPI dependencies for the EPUB search app."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
import httpx
|
||||
from fastapi import Depends, Request
|
||||
|
||||
from python.ebook_search.config import EbookSearchConfig
|
||||
|
||||
|
||||
def get_config(request: Request) -> EbookSearchConfig:
|
||||
"""Get the loaded search config from app state."""
|
||||
return request.app.state.config
|
||||
|
||||
|
||||
def get_http_client(request: Request) -> httpx.AsyncClient:
|
||||
"""Get the shared LLM HTTP client from app state."""
|
||||
return request.app.state.http_client
|
||||
|
||||
|
||||
AppConfig = Annotated[EbookSearchConfig, Depends(get_config)]
|
||||
AppHttpClient = Annotated[httpx.AsyncClient, Depends(get_http_client)]
|
||||
@@ -1,127 +0,0 @@
|
||||
"""Background phrase-judging tasks for the web app.
|
||||
|
||||
Judging a book sends one LLM request per candidate phrase, which can take minutes, so it must
|
||||
not run inside the request where it would block the UI. Judgments run as async FastAPI
|
||||
background tasks, awaited on the event loop after the response is sent, and are tracked per
|
||||
book in app state so a second judge request for a book that is already being judged is
|
||||
rejected instead of doubling the work.
|
||||
|
||||
State is loop-confined: every read and mutation happens on the event loop (async route
|
||||
handlers and async background tasks) and no critical section contains an ``await``, so each
|
||||
mutation is atomic per loop iteration and no locking is needed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from python.ebook_search.protected_phrases.judge_ngrams import judge_candidate_phrases_for_books
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import BackgroundTasks, FastAPI
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class JudgeTaskState:
|
||||
"""Running book judgments and last outcome messages, keyed by book id."""
|
||||
|
||||
running_book_ids: set[int] = field(default_factory=set)
|
||||
outcome_messages: dict[int, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
def get_judge_task_state(app: FastAPI) -> JudgeTaskState:
|
||||
"""Return the app's judge task state, creating it on first use.
|
||||
|
||||
Args:
|
||||
app (FastAPI): App whose state holds the judge task registry.
|
||||
|
||||
Returns:
|
||||
JudgeTaskState: The shared judge task state for this app.
|
||||
"""
|
||||
state = getattr(app.state, "judge_tasks", None)
|
||||
if state is None:
|
||||
state = JudgeTaskState()
|
||||
app.state.judge_tasks = state
|
||||
return state
|
||||
|
||||
|
||||
def start_book_phrase_judgment(app: FastAPI, background_tasks: BackgroundTasks, source_id: int) -> bool:
|
||||
"""Queue judging of one book's candidate phrases as a FastAPI background task.
|
||||
|
||||
The book is claimed before the response returns, so a repeated judge request cannot queue
|
||||
a second run while one is pending or running.
|
||||
|
||||
Args:
|
||||
app (FastAPI): App supplying the engine, config, and judge task state.
|
||||
background_tasks (BackgroundTasks): Request's background tasks to queue the judgment on.
|
||||
source_id (int): Book to judge candidates for.
|
||||
|
||||
Returns:
|
||||
bool: True when a judgment was queued, False when one is already running for this book.
|
||||
"""
|
||||
state = get_judge_task_state(app)
|
||||
if source_id in state.running_book_ids:
|
||||
logger.info(f"ebook_book_phrase_judgment_already_running {source_id=}")
|
||||
return False
|
||||
state.running_book_ids.add(source_id)
|
||||
state.outcome_messages.pop(source_id, None)
|
||||
background_tasks.add_task(judge_book_phrases_for_app, app, source_id)
|
||||
logger.info(f"ebook_book_phrase_judgment_queued {source_id=}")
|
||||
return True
|
||||
|
||||
|
||||
async def judge_book_phrases_for_app(app: FastAPI, source_id: int) -> None:
|
||||
"""Judge one book using the app engine and config, recording the outcome message.
|
||||
|
||||
Args:
|
||||
app (FastAPI): App supplying the engine, config, and judge task state.
|
||||
source_id (int): Book to judge candidates for.
|
||||
"""
|
||||
state = get_judge_task_state(app)
|
||||
try:
|
||||
result = await judge_candidate_phrases_for_books(app.state.engine, app.state.config, source_ids=[source_id])
|
||||
logger.info(
|
||||
f"ebook_book_phrase_judgment_complete {source_id=} {result.candidates_judged=} {result.protected_phrases=} "
|
||||
f"{result.phrase_mentions=} {result.books_failed=}"
|
||||
)
|
||||
if result.books_failed:
|
||||
message = "Judging failed; see server logs for details"
|
||||
else:
|
||||
message = (
|
||||
f"Judged {result.candidates_judged} candidates; {result.protected_phrases} protected phrases promoted"
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(f"ebook_book_phrase_judgment_task_failed {source_id=}")
|
||||
message = "Judging failed; see server logs for details"
|
||||
state.running_book_ids.discard(source_id)
|
||||
state.outcome_messages[source_id] = message
|
||||
|
||||
|
||||
def is_judging_book(app: FastAPI, source_id: int) -> bool:
|
||||
"""Report whether a judgment is currently queued or running for one book.
|
||||
|
||||
Args:
|
||||
app (FastAPI): App supplying the judge task state.
|
||||
source_id (int): Book to check.
|
||||
|
||||
Returns:
|
||||
bool: True while the book's judgment is pending or running.
|
||||
"""
|
||||
return source_id in get_judge_task_state(app).running_book_ids
|
||||
|
||||
|
||||
def pop_book_judgment_outcome(app: FastAPI, source_id: int) -> str | None:
|
||||
"""Return and clear the outcome message from one book's last finished judgment.
|
||||
|
||||
Args:
|
||||
app (FastAPI): App supplying the judge task state.
|
||||
source_id (int): Book to fetch the outcome for.
|
||||
|
||||
Returns:
|
||||
str | None: The outcome message, or None when there is nothing new to report.
|
||||
"""
|
||||
return get_judge_task_state(app).outcome_messages.pop(source_id, None)
|
||||
@@ -1,91 +0,0 @@
|
||||
"""FastAPI HTMX app for EPUB search."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import TYPE_CHECKING, Annotated
|
||||
|
||||
import httpx
|
||||
import typer
|
||||
import uvicorn
|
||||
from fastapi import FastAPI
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from python.common import configure_logger
|
||||
from python.ebook_search.api.bm25_tasks import cancel_bm25_refresh
|
||||
from python.ebook_search.api.routes import admin_router, health_router, page_router, search_router
|
||||
from python.ebook_search.api.web import STATIC_DIR
|
||||
from python.ebook_search.bm25_corpus import ensure_bm25_corpus
|
||||
from python.ebook_search.config import load_config
|
||||
from python.ebook_search.protected_phrases.pool import shutdown_extraction_pool
|
||||
from python.fastapi_tools import ZstdMiddleware
|
||||
from python.orm.common import get_async_postgres_engine
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
"""Manage application startup and shutdown resources."""
|
||||
logger.info("ebook_search_startup")
|
||||
config = load_config()
|
||||
app.state.config = config
|
||||
logger.info(
|
||||
f"ebook_search_config_loaded {config.top_k=} {config.embedding_model=} {config.embedding_base_url=} "
|
||||
f"{config.vllm_base_url=} {config.rerank.enabled=} {config.phrase_matching_enabled=} {config.answer_enabled=} "
|
||||
f"library_paths={len(config.library_paths)}"
|
||||
)
|
||||
if not config.library_paths:
|
||||
logger.warning("ebook_search_no_library_paths_configured")
|
||||
# Concurrent phrase judging opens one session per book worker on this engine, so size the pool
|
||||
# to cover those plus headroom for ordinary web requests.
|
||||
app.state.engine = get_async_postgres_engine(
|
||||
name="RICHIE",
|
||||
vector_engine=True,
|
||||
pool_size=config.phrase_judge_book_workers + 10,
|
||||
)
|
||||
app.state.http_client = httpx.AsyncClient()
|
||||
async with AsyncSession(app.state.engine, expire_on_commit=False) as session:
|
||||
await ensure_bm25_corpus(session, config)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
logger.info("ebook_search_shutdown")
|
||||
cancel_bm25_refresh(app)
|
||||
shutdown_extraction_pool()
|
||||
await app.state.http_client.aclose()
|
||||
await app.state.engine.dispose()
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
"""Create the EPUB search web app."""
|
||||
app = FastAPI(title="EPUB Search", lifespan=lifespan)
|
||||
app.add_middleware(ZstdMiddleware)
|
||||
app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
|
||||
|
||||
app.include_router(admin_router)
|
||||
app.include_router(health_router)
|
||||
app.include_router(page_router)
|
||||
app.include_router(search_router)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def serve(
|
||||
host: Annotated[str, typer.Option("--host", "-h", help="Host to bind to")] = "127.0.0.1",
|
||||
port: Annotated[int, typer.Option("--port", "-p", help="Port to bind to")] = 8070,
|
||||
log_level: Annotated[str, typer.Option("--log-level", "-l", help="Log level")] = "INFO",
|
||||
) -> None:
|
||||
"""Start the EPUB search server."""
|
||||
configure_logger(log_level)
|
||||
uvicorn.run(create_app(), host=host, port=port)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
typer.run(serve)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user