Compare commits

..
1 Commits
Author SHA1 Message Date
gitea-actions[bot] 502a8e4ebb chore: update flake.lock 2026-08-14 20:02:39 -04:00
137 changed files with 2295 additions and 7365 deletions
-1
View File
@@ -18,7 +18,6 @@ jobs:
- "brain"
- "jeeves"
- "rhapsody-in-green"
- "portal-1"
continue-on-error: true
steps:
- uses: actions/checkout@v4
@@ -15,8 +15,9 @@ jobs:
steps:
- name: merge_flake_lock_update
run: >-
nix develop .#devShells.x86_64-linux.default -c
python -m python.gitea_flake_lock merge
--repo "${{ github.repository }}"
env:
JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
GITEA_URL: https://gitea.tmmworkshop.com
+19
View File
@@ -0,0 +1,19 @@
name: pytest
on:
workflow_dispatch:
push:
branches:
- main
pull_request:
branches:
- main
jobs:
pytest:
runs-on: self-hosted
steps:
- uses: actions/checkout@v4
- name: Run tests
run: nix develop .#devShells.x86_64-linux.default -c pytest tests
+2 -1
View File
@@ -18,8 +18,9 @@ jobs:
run: nix flake update
- name: Create or update flake.lock PR
env:
JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
GITEA_URL: https://gitea.tmmworkshop.com
run: >-
nix develop .#devShells.x86_64-linux.default -c
python -m python.gitea_flake_lock update
--repo "${{ github.repository }}"
-5
View File
@@ -1,9 +1,6 @@
# Generate AGE keys from SSH keys with:
# ssh-keygen -A
# nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
# update keys after addin/removing a key
# nix-shell -p sops --run "sops updatekeys users/secrets.yaml" users/secrets.yaml
keys:
- &admin_richie age1u8zj599elqqvcmhxn8zuwrufsz8w8w366d3ayrljjejljt2q45kq8mxw9c # cspell:disable-line
@@ -11,7 +8,6 @@ keys:
- &system_brain age1jhf7vm0005j60mjq63696frrmjhpy8kpc2d66mw044lqap5mjv4snmwvwm # cspell:disable-line
- &system_jeeves age13lmqgc3jvkyah5e3vcwmj4s5wsc2akctcga0lpc0x8v8du3fxprqp4ldkv # cspell:disable-line
- &system_rhapsody age1ufnewppysaq2wwcl4ugngjz8pfzc5a35yg7luq0qmuqvctajcycs5lf6k4 # cspell:disable-line
- &system_portal_1 age1vyav6kxtvt3z4vtnkkjj38eu8hlts5m7ygyckhskvalg2gpjk52su53d0a # cspell:disable-line
creation_rules:
- path_regex: users/secrets\.yaml$
@@ -22,4 +18,3 @@ creation_rules:
- *system_brain
- *system_jeeves
- *system_rhapsody
- *system_portal_1
+3
View File
@@ -10,6 +10,7 @@
"aiounifi",
"alsa",
"apiclient",
"apscheduler",
"archlinux",
"ashift",
"asrouter",
@@ -336,6 +337,8 @@
"yubioath",
"yzhang",
"zeroconf",
"zerotier",
"zerotierone",
"zoxide",
"zram",
"zstd"
Generated
-1686
View File
File diff suppressed because it is too large Load Diff
-4
View File
@@ -1,4 +0,0 @@
[workspace]
resolver = "2"
members = ["rust/*"]
+19 -1
View File
@@ -17,11 +17,16 @@
./nix.nix
./programs.nix
./ssh.nix
./snapshot_manager.nix
];
boot = {
tmp.useTmpfs = lib.mkDefault true;
tmp.useTmpfs = true;
kernelPackages = lib.mkDefault pkgs.linuxPackages_6_12;
zfs = {
package = lib.mkDefault pkgs.zfs_2_4;
forceImportRoot = lib.mkDefault false;
};
};
hardware.enableRedistributableFirmware = true;
@@ -37,6 +42,9 @@
overlays = builtins.attrValues outputs.overlays;
config = {
allowUnfree = true;
permittedInsecurePackages = [
"openssl-1.1.1w" # This is for discord-canary
];
};
};
@@ -45,6 +53,16 @@
# firmware update
fwupd.enable = true;
snapshot_manager = {
enable = lib.mkDefault true;
PYTHONPATH = "${inputs.self}/";
};
zfs = {
trim.enable = lib.mkDefault true;
autoScrub.enable = lib.mkDefault true;
};
};
powerManagement.powertop.enable = lib.mkDefault true;
-4
View File
@@ -31,10 +31,6 @@ in
"flakes"
"ca-derivations"
];
system-features = lib.mkAfter [
"gccarch-x86-64-v2"
"gccarch-x86-64-v3"
];
warn-dirty = false;
flake-registry = ""; # disable global flake registries
connect-timeout = 10;
+1
View File
@@ -2,5 +2,6 @@
{
environment.systemPackages = with pkgs; [
git
my_python
];
}
@@ -22,12 +22,6 @@ hourly = 0
daily = 0
monthly = 0
["root_pool/nix_build"]
15_min = 1
hourly = 0
daily = 0
monthly = 0
["root_pool/var"]
15_min = 8
hourly = 24
@@ -1,5 +1,4 @@
{
inputs,
pkgs,
lib,
config,
@@ -7,11 +6,6 @@
}:
let
cfg = config.services.snapshot_manager;
snapshotManagerPackages =
ps: with ps; [
httpx
typer
];
in
{
options = {
@@ -40,41 +34,6 @@ in
};
config = lib.mkIf cfg.enable {
nixpkgs.overlays = [
(final: _prev: {
snapshot_manager_python = final.python314.withPackages snapshotManagerPackages;
snapshot_manager_test_python = final.python314.withPackages (
ps:
snapshotManagerPackages ps
++ (with ps; [
pyfakefs
pytest
pytest-asyncio
pytest-mock
pytest-xdist
])
);
snapshot_manager_tests =
final.runCommand "snapshot-manager-tests"
{
nativeBuildInputs = [ final.snapshot_manager_test_python ];
}
''
export HOME="$TMPDIR"
cd ${inputs.self}
pytest \
-o cache_dir="$TMPDIR/pytest-cache" \
tests/test_common.py \
tests/test_signal_alert.py \
tests/test_snapshot_manager.py \
tests/test_zfs.py
touch "$out"
'';
})
];
system.checks = [ pkgs.snapshot_manager_tests ];
systemd = {
services.snapshot_manager = {
description = "ZFS Snapshot Manager";
@@ -86,7 +45,7 @@ in
};
serviceConfig = {
Type = "oneshot";
ExecStart = "${pkgs.snapshot_manager_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
ExecStart = "${pkgs.my_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
}
// lib.optionalAttrs (cfg.EnvironmentFile != null) {
EnvironmentFile = cfg.EnvironmentFile;
-1
View File
@@ -17,7 +17,6 @@
logDriver = "local";
storageDriver = "overlay2";
daemon.settings = {
live-restore = false;
experimental = true;
exec-opts = [ "native.cgroupdriver=systemd" ];
log-opts = {
+1 -1
View File
@@ -5,7 +5,7 @@
...
}:
let
monitoringInterface = "tailscale0";
monitoringInterface = "ztwfunumly";
nodeTextfileDir = "/var/lib/prometheus-node-exporter-textfile";
mkProcessNameTemplate =
-37
View File
@@ -1,37 +0,0 @@
{
config,
inputs,
...
}:
{
nix.settings = {
trusted-substituters = [ "http://jeeves:5000" ];
substituters = [ "http://jeeves:5000/?priority=1&want-mass-query=true" ];
trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
};
services.tailscale = {
enable = true;
openFirewall = true;
authKeyFile = config.sops.secrets.tailscale_auth_key.path;
# OAuth client secrets create ephemeral nodes by default. NixOS machines
# are persistent and should enroll without interactive device approval.
authKeyParameters = {
ephemeral = false;
preauthorized = true;
};
extraUpFlags = [ "--advertise-tags=tag:nixos" ];
};
sops = {
age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
secrets.tailscale_auth_key = {
sopsFile = "${inputs.self}/users/secrets.yaml";
owner = "root";
mode = "0400";
};
};
}
-9
View File
@@ -1,9 +0,0 @@
{
nixpkgs.hostPlatform = {
system = "x86_64-linux";
gcc = {
arch = "x86-64-v3";
tune = "generic";
};
};
}
+11
View File
@@ -0,0 +1,11 @@
{
services.zerotierone = {
enable = true;
joinNetworks = [ "e4da7455b2ae64ca" ];
};
nix.settings = {
trusted-substituters = [ "http://192.168.90.40:5000" ];
substituters = [ "http://192.168.90.40:5000/?priority=1&want-mass-query=true" ];
trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
};
}
-26
View File
@@ -1,26 +0,0 @@
{
inputs,
lib,
pkgs,
...
}:
{
imports = [ ./snapshot.nix ];
boot.zfs = {
package = lib.mkDefault pkgs.zfs_2_4;
forceImportRoot = lib.mkDefault false;
};
services = {
snapshot_manager = {
enable = lib.mkDefault true;
PYTHONPATH = "${inputs.self}/";
};
zfs = {
trim.enable = lib.mkDefault true;
autoScrub.enable = lib.mkDefault true;
};
};
}
Generated
+28 -26
View File
@@ -1,23 +1,25 @@
{
"nodes": {
"disko": {
"firefox-addons": {
"inputs": {
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1781152676,
"narHash": "sha256-RxWs5ND31KzTG7wvMM+PMfUjyNpmIEr999lqNARaM5o=",
"owner": "nix-community",
"repo": "disko",
"rev": "ff8702b4de27f72b4c78573dfb89ec74e36abdf1",
"type": "github"
"dir": "pkgs/firefox-addons",
"lastModified": 1786680168,
"narHash": "sha256-aQxQRegqqXdOy8eUI0WHYgvyuAhZIJYaddX39fPmrwQ=",
"owner": "rycee",
"repo": "nur-expressions",
"rev": "9bb332ed3a80406e60023485d66bd8473b5e6896",
"type": "gitlab"
},
"original": {
"owner": "nix-community",
"repo": "disko",
"type": "github"
"dir": "pkgs/firefox-addons",
"owner": "rycee",
"repo": "nur-expressions",
"type": "gitlab"
}
},
"home-manager": {
@@ -27,11 +29,11 @@
]
},
"locked": {
"lastModified": 1788651960,
"narHash": "sha256-v9wJd32eZ2bvhBzVOd7TIjLQd011P7nwOhjKtWlci5I=",
"lastModified": 1786719456,
"narHash": "sha256-B74DLQs/VjlqyhnnX3tguWwghqJHWSJX30TKZuAljIg=",
"owner": "nix-community",
"repo": "home-manager",
"rev": "2c0350c759688177331b8f5242311fae8877bdb3",
"rev": "83b7606dcf44abe3a94b86e8bb2b3355d22e8797",
"type": "github"
},
"original": {
@@ -45,11 +47,11 @@
"nixpkgs": "nixpkgs"
},
"locked": {
"lastModified": 1788860136,
"narHash": "sha256-MhPMOFV4pVkygWEbQ8t1De/uQ9cWF1u++tRe2L5tG48=",
"lastModified": 1786717298,
"narHash": "sha256-asdPlqtM0AweQ5YNLad6eY4tSYW55YNr1uyayf7mOMQ=",
"owner": "nixos",
"repo": "nixos-hardware",
"rev": "62173785b9a18c78b4a15aca2623d02bceb9d077",
"rev": "2dda192987ab6815e44df0130f03d6c907b79134",
"type": "github"
},
"original": {
@@ -74,11 +76,11 @@
},
"nixpkgs-master": {
"locked": {
"lastModified": 1788892992,
"narHash": "sha256-cIMFh9gyU4/aLeB3JCcsWM3tTAvD9pAq9Smr1Wa8aIU=",
"lastModified": 1786749337,
"narHash": "sha256-eVlLj0gExnn8cGNrQfehzFhntpOTACaXCKEuEVUhD5Y=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "dff6994123e257ec9901c271bc2b52e64d7c8f05",
"rev": "8ff36cc4da2caa4e7c81d7321dae5fa0cb3a4d0d",
"type": "github"
},
"original": {
@@ -106,11 +108,11 @@
},
"nixpkgs_2": {
"locked": {
"lastModified": 1788752844,
"narHash": "sha256-VaWGJ6+cIYN2erfSecbRV+4ljI185Ty2wUrXyvQbgOw=",
"lastModified": 1786599213,
"narHash": "sha256-yNJd40f11EzXBjSByCB7IPpeFFAdeoSKKM67dGkfFoU=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "dc5d91f840324650bac8c379428c7037a416959a",
"rev": "0e251e24a4f24e036a084b6b4b2d2491af4167f4",
"type": "github"
},
"original": {
@@ -122,7 +124,7 @@
},
"root": {
"inputs": {
"disko": "disko",
"firefox-addons": "firefox-addons",
"home-manager": "home-manager",
"nixos-hardware": "nixos-hardware",
"nixpkgs": "nixpkgs_2",
@@ -139,11 +141,11 @@
]
},
"locked": {
"lastModified": 1788337237,
"narHash": "sha256-gkSH8VUtCo6hnysNmb9DbTuDepH2t5pv+QWjP75xKAk=",
"lastModified": 1786629091,
"narHash": "sha256-gkig4nPi1CWc4Z50GBsjE4ygSE7hMpl/TwID2an2Cck=",
"owner": "Mic92",
"repo": "sops-nix",
"rev": "fbf759290e0cb0a98dfc813a4eb7d53ad1dacb57",
"rev": "a8627b21b9107c5711c96b84f32a9a4b3d45295f",
"type": "github"
},
"original": {
+4 -4
View File
@@ -26,13 +26,13 @@
inputs.nixpkgs.follows = "nixpkgs";
};
sops-nix = {
url = "github:Mic92/sops-nix";
firefox-addons = {
url = "gitlab:rycee/nur-expressions?dir=pkgs/firefox-addons";
inputs.nixpkgs.follows = "nixpkgs";
};
disko = {
url = "github:nix-community/disko";
sops-nix = {
url = "github:Mic92/sops-nix";
inputs.nixpkgs.follows = "nixpkgs";
};
};
+31 -2
View File
@@ -15,6 +15,35 @@
};
};
test-exclusions = import ./test-exclusions.nix;
x86-64-v3-workarounds = import ./x86-64-v3-workarounds.nix;
python-env = final: _prev: {
my_python = final.python314.withPackages (
ps: with ps; [
alembic
apprise
apscheduler
fastapi
fastapi-cli
httpx
jinja2
mypy
pgvector
psycopg
pydantic
pyfakefs
pytest
pytest-cov
pytest-mock
pytest-xdist
python-multipart
pydantic-settings
ruff
sqlalchemy
tenacity
tinytuya
typer
uvicorn
websockets
]
);
};
}
-133
View File
@@ -1,133 +0,0 @@
# Test exclusions for the locally rebuilt x86-64-v3 package set.
#
# Selecting x86-64-v3 changes every affected derivation, so the normal
# nixpkgs binary cache cannot be used and upstream test suites run locally.
# The jeeves builder uses /tmp/nix-builds so filesystem tests run on tmpfs
# instead of ZFS with normalization=formD and utf8only=on; those tests remain
# enabled. The remaining workarounds cover UDP readiness, resource-sensitive
# parser and nested-worker races, and mismatched timeout clocks, plus
# architecture-dependent floating-point differences whose risk we accept for
# our workloads. Keep these exceptions visible until their causes are fixed.
_final: prev: {
gnutls = prev.gnutls.overrideAttrs (old: {
# This test uses a fixed four-second sleep instead of checking UDP
# readiness; the client saw no listener in the x86-64-v3 build.
postPatch = (old.postPatch or "") + ''
sed '2iexit 77' -i tests/serv-udp.sh
'';
});
prometheus = prev.prometheus.overrideAttrs (
old:
let
assets = old.passthru.assets.overrideAttrs (assetsOld: {
# CodeMirror's bounded synchronous parser can return an incomplete tree
# when these cases run on a heavily loaded builder.
postPatch = (assetsOld.postPatch or "") + ''
substituteInPlace module/codemirror-promql/src/complete/hybrid.test.ts \
--replace-fail "it(value.title, () => {" \
"(value.title === 'autocomplete topk params 2' ? it.skip : it)(value.title, () => {"
substituteInPlace module/codemirror-promql/src/parser/vector.test.ts \
--replace-fail "it(value.binaryExpr, () => {" \
"(value.binaryExpr === 'foo * on(test,blub) bar' ? it.skip : it)(value.binaryExpr, () => {"
'';
});
in
{
postPatch = builtins.replaceStrings [ "${old.passthru.assets}" ] [ "${assets}" ] (
builtins.unsafeDiscardStringContext old.postPatch
);
passthru = old.passthru // {
inherit assets;
};
}
);
pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
(_pythonFinal: pythonPrev: {
backrefs = pythonPrev.backrefs.overridePythonAttrs (old: {
# regex measures its timeout in process CPU time, while this test used
# wall time and could miss the timeout when a busy builder descheduled it.
postPatch = (old.postPatch or "") + ''
substituteInPlace tests/test_bregex.py \
--replace-fail "time.time()" "time.process_time()"
'';
});
pytest-xdist = pythonPrev.pytest-xdist.overridePythonAttrs (old: {
# The suite exercises its own worker pools. Run the outer suite with one
# worker and allow inner workers more time on heavily loaded builders.
postPatch = (old.postPatch or "") + ''
substituteInPlace testing/test_remote.py \
--replace-fail "WAIT_TIMEOUT = 10.0" "WAIT_TIMEOUT = 60.0"
'';
preCheck = builtins.replaceStrings [ "--numprocesses=$NIX_BUILD_CORES" ] [ "--numprocesses=1" ] (
old.preCheck or ""
);
# This test deliberately crashes workers past the restart limit and
# races while checking which replacement message was emitted.
disabledTests = (old.disabledTests or [ ]) ++ [
"test_max_worker_restart_tests_queued"
];
});
jupyter-server = pythonPrev.jupyter-server.overridePythonAttrs (old: {
# The kernel reply arrived after the one-second outer deadline when the
# builder was heavily loaded. Keep the regression test but allow it the
# same margin as the other resource-sensitive tests.
postPatch = (old.postPatch or "") + ''
substituteInPlace tests/services/kernels/test_connection.py \
--replace-fail \
"await asyncio.wait_for(asyncio.wrap_future(conn2.request_kernel_info()), timeout=1.0)" \
"await asyncio.wait_for(asyncio.wrap_future(conn2.request_kernel_info()), timeout=10.0)"
'';
});
scipy = pythonPrev.scipy.overridePythonAttrs (old: {
# x86-64-v3 FFT implementations produce rounding differences outside
# these tests' strict tolerances. We accept the numerical-precision
# risk for our workloads.
disabledTests = (old.disabledTests or [ ]) ++ [
"test_roundtrip_float32"
"test_roundtrip_scaling"
];
});
sentry-sdk = pythonPrev.sentry-sdk.overridePythonAttrs (old: {
# This test globally mocks threading.current_thread while another
# thread is running. On Python 3.14, Thread.join can race with that
# mock and exhaust its single side effect before the worker removes it.
disabledTests = (old.disabledTests or [ ]) ++ [
"test_get_current_thread_meta_main_thread"
];
});
torchaudio = pythonPrev.torchaudio.overridePythonAttrs (old: {
# x86-64-v3 pitch shifting produces batch-versus-single-item numerical
# differences up to 2.9e-6. We accept that audio-precision risk for our
# workloads.
disabledTests = (old.disabledTests or [ ]) ++ [
"test_batch_pitch_shift"
];
});
torchcodec = pythonPrev.torchcodec.overridePythonAttrs (old: {
# For these 8 kHz MP3 cases, the x86-64-v3 API and CLI codec paths
# differ in 0.8% of decoded samples. Retain the original tolerance for
# 99% of samples and accept the localized audio-precision risk.
postPatch = (old.postPatch or "") + ''
substituteInPlace test/test_encoders.py \
--replace-fail \
'if sys.platform == "darwin":' \
'if sys.platform == "darwin" or (
format == "mp3"
and sample_rate == 8_000
and asset is SINE_MONO_S32
and bit_rate in (None, 0)
and num_channels in (None, 1)
):'
'';
});
})
];
}
-60
View File
@@ -1,60 +0,0 @@
# Compatibility fixes for packages rebuilt with x86-64-v3.
#
# The v3 baseline enables instructions that expose source assumptions hidden
# by the generic x86-64 build. Keep compile fixes here, separate from test
# exclusions, until upstream or nixpkgs incorporates them.
_final: prev:
let
patchAbseilBmi2Include =
package:
package.overrideAttrs (old: {
# GCC and Clang prohibit including their internal BMI2 header directly.
# The public umbrella provides the same intrinsics with the required
# compiler setup.
postPatch = (old.postPatch or "") + ''
substituteInPlace third_party/abseil-cpp/absl/container/internal/raw_hash_set.h \
--replace-fail "#include <bmi2intrin.h>" "#include <immintrin.h>"
'';
});
removeSiblingOutputChecks =
package:
package.overrideAttrs (old: {
# Nix 2.34 can validate a partial multi-output rebuild against only the
# outputs still being realised. PostgreSQL's checks then reject valid
# sibling names such as "out" and "lib". Keep the test suite and
# disallowed-requisite checks; accept the loss of cross-output checks.
outputChecks = builtins.mapAttrs (
_output: checks: builtins.removeAttrs checks [ "disallowedReferences" ]
) (old.outputChecks or { });
});
electron43Unwrapped = patchAbseilBmi2Include prev.electron_43.unwrapped;
electron43 = prev.electron_43.override {
electron-unwrapped = electron43Unwrapped;
};
signalCallPackage =
path: args:
let
package = prev.callPackage path args;
in
if builtins.baseNameOf path == "webrtc.nix" then patchAbseilBmi2Include package else package;
in
prev.lib.optionalAttrs ((prev.stdenv.hostPlatform.gcc.arch or null) == "x86-64-v3") {
deno =
let
librusty_v8 = patchAbseilBmi2Include prev.deno.passthru.librusty_v8;
in
prev.deno.override { inherit librusty_v8; };
electron_43 = electron43;
postgresql = removeSiblingOutputChecks prev.postgresql;
postgresql_18 = removeSiblingOutputChecks prev.postgresql_18;
signal-desktop = prev.signal-desktop.override {
electron_43 = electron43;
callPackage = signalCallPackage;
};
}
+45
View File
@@ -6,6 +6,51 @@ authors = [{ name = "Richie Cahill", email = "richie@tmmworkshop.com" }]
requires-python = "~=3.14.0"
readme = "README.md"
license = "MIT"
# these dependencies are a best effort and aren't guaranteed to work
# for up-to-date dependencies, see overlays/default.nix
dependencies = [
"alembic",
"apprise",
"apscheduler",
"beautifulsoup4",
"bm25s",
"ebooklib",
"fastapi",
"fastapi-cli",
"httpx",
"jinja2",
"pgvector",
"polars",
"psycopg[binary]",
"pydantic",
"pydantic-settings",
"python-multipart",
"sqlalchemy[asyncio]",
"tenacity",
"tiktoken",
"tinytuya",
"typer",
"uvicorn",
"websockets",
"yake",
]
[project.scripts]
database = "python.database_cli:app"
whisper-transcribe = "python.tools.whisper.transcribe:main"
[dependency-groups]
dev = [
"aiosqlite",
"mypy",
"pyfakefs",
"pytest-asyncio",
"pytest-cov",
"pytest-mock",
"pytest-xdist",
"pytest",
"ruff",
]
[tool.ruff]
+2 -2
View File
@@ -37,7 +37,7 @@ class RerankConfig(BaseSettings):
model_config = SettingsConfigDict(env_prefix="EBOOK_SEARCH_RERANK_", frozen=True, protected_namespaces=())
enabled: bool = True
base_url: str = "http://bob:8001"
base_url: str = "http://192.168.90.25:8001"
model: str = "qwen3-reranker-06b"
candidates: int = 24
timeout_seconds: float = 30.0
@@ -67,7 +67,7 @@ class EbookSearchConfig(BaseSettings):
)
chat_model: str = "deepseek-v4-flash"
answer_enabled: bool = True
embedding_base_url: str = "http://bob:8000/v1"
embedding_base_url: str = "http://192.168.90.25:8000/v1"
embedding_api_key: str = "not-needed"
embedding_model: str = "qwen3-embedding-0.6b"
embedding_batch_size: int = 32
@@ -9,6 +9,8 @@ services:
restart: unless-stopped
ports:
- "${EBOOK_SEARCH_PORT:-8070}:8070"
extra_hosts:
- "jeeves:192.168.90.40"
env_file:
- ../../../.env
environment:
+6 -6
View File
@@ -14,7 +14,7 @@ DEFAULT_BASE_BRANCH = "main"
DEFAULT_BRANCH = "automation/update-flake-lock"
DEFAULT_GITEA_URL = "https://gitea.tmmworkshop.com"
PR_LABELS = ["dependencies", "automated", "flake_lock_update"]
PR_CHECK_WORKFLOWS = ["build_systems.yml", "treefmt.yml"]
PR_CHECK_WORKFLOWS = ["build_systems.yml", "treefmt.yml", "pytest.yml"]
PR_TITLE = "Update flake.lock"
PR_BODY = "Automated flake.lock update."
@@ -84,13 +84,13 @@ def push_branch(*, branch: str) -> None:
run_cmd(["git", "push", "origin", f"HEAD:{branch}", "--force"])
def _required_jeeves_bot_token() -> str:
def _required_gitea_token() -> str:
"""Read the required Gitea token from the environment."""
token = getenv("JEEVES_BOT_TOKEN")
token = getenv("GITEA_TOKEN")
if token:
return token
msg = "JEEVES_BOT_TOKEN environment variable is required"
msg = "GITEA_TOKEN environment variable is required"
raise RuntimeError(msg)
@@ -111,7 +111,7 @@ def update(
owner, repo_name = split_repo_name(repo)
with GiteaClient(
base_url=getenv("GITEA_URL", DEFAULT_GITEA_URL),
token=_required_jeeves_bot_token(),
token=_required_gitea_token(),
) as client:
pull_request = ensure_flake_lock_pull_request(
client,
@@ -134,7 +134,7 @@ def merge(
owner, repo_name = split_repo_name(repo)
with GiteaClient(
base_url=getenv("GITEA_URL", DEFAULT_GITEA_URL),
token=_required_jeeves_bot_token(),
token=_required_gitea_token(),
) as client:
pull_request = find_flake_lock_pull_request(client, owner=owner, repo=repo_name)
if not pull_request:
-18
View File
@@ -176,21 +176,6 @@ def create_zfs_datasets() -> None:
run_output(("zfs", "create", "root_pool/home"))
run_output(("zfs", "create", "-o", "reservation=1G", "root_pool/var"))
run_output(("zfs", "create", "-o", "compression=zstd-9", "-o", "reservation=10G", "root_pool/nix"))
run_output(
(
"zfs",
"create",
"-o",
"sync=disabled",
"-o",
"redundant_metadata=some",
"-o",
"normalization=none",
"-o",
"utf8only=off",
"root_pool/nix_build",
)
)
datasets = run_output(("zfs", "list", "-o", "name"))
expected_datasets = {
@@ -198,7 +183,6 @@ def create_zfs_datasets() -> None:
"root_pool/home",
"root_pool/var",
"root_pool/nix",
"root_pool/nix_build",
}
missing_datasets = expected_datasets.difference(datasets.splitlines())
if missing_datasets:
@@ -264,7 +248,6 @@ def create_nix_hardware_file(mnt_dir: str, disks: Sequence[str], encrypt: str |
' "/home" = {\n device = "root_pool/home";\n fsType = "zfs";\n };\n\n'
' "/var" = {\n device = "root_pool/var";\n fsType = "zfs";\n };\n\n'
' "/nix" = {\n device = "root_pool/nix";\n fsType = "zfs";\n };\n\n'
' "/nix/var/nix/builds" = {\n device = "root_pool/nix_build";\n fsType = "zfs";\n };\n\n'
' "/boot" = {\n'
f' device = "/dev/disk/by-uuid/{get_boot_drive_id(disks[0])}";\n'
' fsType = "vfat";\n options = [\n "fmask=0077"\n'
@@ -287,7 +270,6 @@ def install_nixos(mnt_dir: str, disks: Sequence[str], encrypt: str | None) -> No
run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/home", f"{mnt_dir}/home"))
run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/var", f"{mnt_dir}/var"))
run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/nix", f"{mnt_dir}/nix"))
run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/nix_build", f"{mnt_dir}/nix/var/nix/builds"))
for disk in disks:
run_output(("mkfs.vfat", "-n", "EFI", f"{disk}-part1"))
+5 -25
View File
@@ -5,13 +5,10 @@ from __future__ import annotations
import logging
from os import getenv
import httpx
from apprise import Apprise
logger = logging.getLogger(__name__)
SIGNAL_API_URL = "http://localhost:8989/v2/send"
SIGNAL_API_TIMEOUT = 4.0
def signal_alert(body: str, title: str = "") -> None:
"""Send a signal alert.
@@ -20,31 +17,14 @@ def signal_alert(body: str, title: str = "") -> None:
body (str): The body of the alert.
title (str, optional): The title of the alert. Defaults to "".
"""
apprise_client = Apprise()
from_phone = getenv("SIGNAL_ALERT_FROM_PHONE")
to_phone = getenv("SIGNAL_ALERT_TO_PHONE")
if not from_phone or not to_phone:
logger.info("SIGNAL_ALERT_FROM_PHONE or SIGNAL_ALERT_TO_PHONE not set")
return
# Apprise's Signal integration did not support titles, so preserve that behavior.
if title:
logger.debug("Signal does not support notification titles; ignoring title")
apprise_client.add(f"signal://localhost:8989/{from_phone}/{to_phone}")
try:
response = httpx.post(
SIGNAL_API_URL,
json={
"message": body,
"number": from_phone,
"recipients": [to_phone],
"text_mode": "normal",
},
timeout=SIGNAL_API_TIMEOUT,
follow_redirects=True,
)
except httpx.HTTPError:
logger.exception("Unable to contact the Signal API")
return
if response.status_code not in {httpx.codes.OK, httpx.codes.CREATED}:
logger.error("Signal API returned HTTP status %d", response.status_code)
apprise_client.notify(title=title, body=body)
+1
View File
@@ -0,0 +1 @@
"""system_tests."""
+99
View File
@@ -0,0 +1,99 @@
"""Validate Jeeves."""
from __future__ import annotations
import logging
from copy import copy
from re import search
from time import sleep
from typing import TYPE_CHECKING
from python.common import bash_wrapper
from python.zfs import Zpool
if TYPE_CHECKING:
from collections.abc import Sequence
logger = logging.getLogger(__name__)
def zpool_tests(pool_names: Sequence[str], zpool_capacity_threshold: int = 90) -> list[str] | None:
"""Check the zpool health and capacity.
Args:
pool_names (Sequence[str]): A list of pool names to test.
zpool_capacity_threshold (int, optional): The threshold for the zpool capacity. Defaults to 90.
Returns:
list[str] | None: A list of errors if any.
"""
logger.info("Testing zpool")
errors: list[str] = []
for pool_name in pool_names:
pool = Zpool(pool_name)
if pool.health != "ONLINE":
errors.append(f"{pool.name} is {pool.health}")
if pool.capacity >= zpool_capacity_threshold:
errors.append(f"{pool.name} is low on space")
upgrade_status, _ = bash_wrapper("zpool upgrade")
if not search(r"Every feature flags pool has all supported and requested features enabled.", upgrade_status):
errors.append("ZPool out of date run `sudo zpool upgrade -a`")
return errors
def systemd_tests(
service_names: Sequence[str],
max_retries: int = 30,
retry_delay_secs: int = 1,
retryable_statuses: Sequence[str] | None = None,
valid_statuses: Sequence[str] | None = None,
) -> list[str] | None:
"""Tests a systemd services.
Args:
service_names (Sequence[str]): A list of service names to test.
max_retries (int, optional): The maximum number of retries. Defaults to 30.
minimum value is 1.
retry_delay_secs (int, optional): The delay between retries in seconds. Defaults to 1.
minimum value is 1.
retryable_statuses (Sequence[str] | None, optional): A list of retryable statuses. Defaults to None.
valid_statuses (Sequence[str] | None, optional): A list of valid statuses. Defaults to None.
Returns:
list[str] | None: A list of errors if any.
"""
logger.info("Testing systemd service")
max_retries = max(max_retries, 1)
retry_delay_secs = max(retry_delay_secs, 1)
last_try = max_retries - 1
if retryable_statuses is None:
retryable_statuses = ("inactive\n", "activating\n")
if valid_statuses is None:
valid_statuses = ("active\n",)
service_names_set = set(service_names)
errors: set[str] = set()
for retry in range(max_retries):
if not service_names_set:
break
logger.info(f"Testing systemd service in {retry + 1} of {max_retries}")
service_names_to_test = copy(service_names_set)
for service_name in service_names_to_test:
service_status, _ = bash_wrapper(f"systemctl is-active {service_name}")
if service_status in valid_statuses:
service_names_set.remove(service_name)
continue
if service_status in retryable_statuses and retry < last_try:
continue
errors.add(f"{service_name} is {service_status.strip()}")
sleep(retry_delay_secs)
return list(errors)
+67
View File
@@ -0,0 +1,67 @@
"""Validate {server_name}."""
import logging
import sys
import tomllib
from os import environ
from pathlib import Path # noqa: TC003 This is required for the typer CLI
from socket import gethostname
import typer
from python.common import configure_logger
from python.signal_alert import signal_alert
from python.system_tests.components import systemd_tests, zpool_tests
logger = logging.getLogger(__name__)
def load_config_data(config_file: Path) -> dict[str, list[str]]:
"""Load a TOML configuration file.
Args:
config_file (Path): The path to the configuration file.
Returns:
dict: The configuration data.
"""
return tomllib.loads(config_file.read_text())
def main(config_file: Path) -> None:
"""Main."""
configure_logger(level=environ.get("LOG_LEVEL", "INFO"))
server_name = gethostname()
logger.info(f"Starting {server_name} validation")
config_data = load_config_data(config_file)
errors: list[str] = []
try:
if config_data.get("zpools") and (zpool_errors := zpool_tests(config_data["zpools"])):
errors.extend(zpool_errors)
if config_data.get("services") and (systemd_errors := systemd_tests(config_data["services"])):
errors.extend(systemd_errors)
except Exception as error:
logger.exception(f"{server_name} validation failed")
errors.append(f"{server_name} validation failed: {error}")
if errors:
logger.error(f"{server_name} validation failed: \n{'\n'.join(errors)}")
signal_alert(f"{server_name} validation failed {errors}")
sys.exit(1)
logger.info(f"{server_name} validation passed")
def cli() -> None:
"""CLI."""
typer.run(main)
if __name__ == "__main__":
cli()
+1
View File
@@ -0,0 +1 @@
"""Van weather service - fetches weather with masked GPS location."""
+293
View File
@@ -0,0 +1,293 @@
"""Van weather service - fetches weather with masked GPS for privacy."""
import logging
from datetime import UTC, datetime
from typing import Annotated, Any
import httpx
import typer
from apscheduler.schedulers.blocking import BlockingScheduler
from tenacity import before_sleep_log, retry, stop_after_attempt, wait_fixed
from python.common import configure_logger
from python.van_weather.models import Config, DailyForecast, HourlyForecast, Weather
# Map Pirate Weather icons to Home Assistant conditions
CONDITION_MAP = {
"clear-day": "sunny",
"clear-night": "clear-night",
"rain": "rainy",
"snow": "snowy",
"sleet": "snowy-rainy",
"wind": "windy",
"fog": "fog",
"cloudy": "cloudy",
"partly-cloudy-day": "partlycloudy",
"partly-cloudy-night": "partlycloudy",
}
logger = logging.getLogger(__name__)
@retry(
stop=stop_after_attempt(3),
wait=wait_fixed(5),
before_sleep=before_sleep_log(logger, logging.WARNING),
reraise=True,
)
def get_ha_state(url: str, token: str, entity_id: str) -> float:
"""Get numeric state from Home Asasistant entity."""
response = httpx.get(
f"{url}/api/states/{entity_id}",
headers={"Authorization": f"Bearer {token}"},
timeout=30,
)
response.raise_for_status()
state = response.json()["state"]
if state in ("unavailable", "unknown"):
error = f"{entity_id} is {state}"
raise ValueError(error)
return float(state)
def parse_daily_forecast(data: dict[str, dict[str, Any]]) -> list[DailyForecast]:
"""Parse daily forecast from Pirate Weather API."""
daily = data.get("daily", {}).get("data", [])
daily_forecasts = []
for day in daily[:8]: # Up to 8 days
time_stamp = day.get("time")
if time_stamp:
date_time = datetime.fromtimestamp(time_stamp, tz=UTC).isoformat()
daily_forecasts.append(
DailyForecast(
date_time=date_time,
condition=CONDITION_MAP.get(day.get("icon", ""), "cloudy"),
temperature=day.get("temperatureHigh"),
templow=day.get("temperatureLow"),
precipitation_probability=day.get("precipProbability"),
moon_phase=day.get("moonPhase"),
wind_gust=day.get("windGust"),
cloud_cover=day.get("cloudCover"),
)
)
return daily_forecasts
def parse_hourly_forecast(data: dict[str, dict[str, Any]]) -> list[HourlyForecast]:
"""Parse hourly forecast from Pirate Weather API."""
hourly = data.get("hourly", {}).get("data", [])
hourly_forecasts = []
for hour in hourly[:48]: # Up to 48 hours
time_stamp = hour.get("time")
if time_stamp:
date_time = datetime.fromtimestamp(time_stamp, tz=UTC).isoformat()
hourly_forecasts.append(
HourlyForecast(
date_time=date_time,
condition=CONDITION_MAP.get(hour.get("icon", ""), "cloudy"),
temperature=hour.get("temperature"),
precipitation_probability=hour.get("precipProbability"),
)
)
return hourly_forecasts
@retry(
stop=stop_after_attempt(3),
wait=wait_fixed(5),
before_sleep=before_sleep_log(logger, logging.WARNING),
reraise=True,
)
def fetch_weather(api_key: str, lat: float, lon: float) -> Weather:
"""Fetch weather from Pirate Weather API."""
url = f"https://api.pirateweather.net/forecast/{api_key}/{lat},{lon}"
response = httpx.get(url, params={"units": "us"}, timeout=30)
response.raise_for_status()
data = response.json()
daily_forecasts = parse_daily_forecast(data)
hourly_forecasts = parse_hourly_forecast(data)
current = data.get("currently", {})
icon = current.get("icon", "")
return Weather(
temperature=current.get("temperature"),
feels_like=current.get("apparentTemperature"),
humidity=current.get("humidity"),
wind_speed=current.get("windSpeed"),
wind_bearing=current.get("windBearing"),
condition=CONDITION_MAP.get(icon, "cloudy"),
summary=current.get("summary"),
pressure=current.get("pressure"),
visibility=current.get("visibility"),
uv_index=current.get("uvIndex"),
ozone=current.get("ozone"),
nearest_storm_distance=current.get("nearestStormDistance"),
nearest_storm_bearing=current.get("nearestStormBearing"),
precip_probability=current.get("precipProbability"),
cloud_cover=current.get("cloudCover"),
daily_forecasts=daily_forecasts,
hourly_forecasts=hourly_forecasts,
)
@retry(
stop=stop_after_attempt(3),
wait=wait_fixed(5),
before_sleep=before_sleep_log(logger, logging.WARNING),
reraise=True,
)
def post_to_ha(url: str, token: str, weather: Weather) -> None:
"""Post weather data to Home Assistant as sensor entities."""
headers = {"Authorization": f"Bearer {token}"}
# Post current weather as individual sensors
sensors = {
"sensor.van_weather_condition": {
"state": weather.condition or "unknown",
"attributes": {"friendly_name": "Van Weather Condition"},
},
"sensor.van_weather_temperature": {
"state": weather.temperature,
"attributes": {"unit_of_measurement": "°F", "device_class": "temperature"},
},
"sensor.van_weather_apparent_temperature": {
"state": weather.feels_like,
"attributes": {"unit_of_measurement": "°F", "device_class": "temperature"},
},
"sensor.van_weather_humidity": {
"state": int((weather.humidity or 0) * 100),
"attributes": {"unit_of_measurement": "%", "device_class": "humidity"},
},
"sensor.van_weather_pressure": {
"state": weather.pressure,
"attributes": {"unit_of_measurement": "mbar", "device_class": "pressure"},
},
"sensor.van_weather_wind_speed": {
"state": weather.wind_speed,
"attributes": {"unit_of_measurement": "mph", "device_class": "wind_speed"},
},
"sensor.van_weather_wind_bearing": {
"state": weather.wind_bearing,
"attributes": {"unit_of_measurement": "°"},
},
"sensor.van_weather_visibility": {
"state": weather.visibility,
"attributes": {"unit_of_measurement": "mi"},
},
"sensor.van_weather_uv_index": {
"state": weather.uv_index,
"attributes": {"friendly_name": "Van Weather UV Index", "icon": "mdi:sun-wireless"},
},
"sensor.van_weather_ozone": {
"state": weather.ozone,
"attributes": {"unit_of_measurement": "DU", "icon": "mdi:earth"},
},
"sensor.van_weather_nearest_storm_distance": {
"state": weather.nearest_storm_distance,
"attributes": {"unit_of_measurement": "mi", "icon": "mdi:weather-lightning"},
},
"sensor.van_weather_nearest_storm_bearing": {
"state": weather.nearest_storm_bearing,
"attributes": {"unit_of_measurement": "°", "icon": "mdi:weather-lightning"},
},
"sensor.van_weather_precip_probability": {
"state": int((weather.precip_probability or 0) * 100),
"attributes": {"unit_of_measurement": "%", "icon": "mdi:weather-rainy"},
},
"sensor.van_weather_cloud_cover": {
"state": int((weather.cloud_cover or 0) * 100),
"attributes": {"unit_of_measurement": "%", "icon": "mdi:weather-cloudy"},
},
}
for entity_id, data in sensors.items():
if data["state"] is not None:
response = httpx.post(f"{url}/api/states/{entity_id}", headers=headers, json=data, timeout=30)
response.raise_for_status()
# Post daily forecast as JSON attribute sensor
daily_forecast = [
{
"datetime": daily_forecast.date_time.isoformat(),
"condition": daily_forecast.condition,
"temperature": daily_forecast.temperature,
"templow": daily_forecast.templow,
"precipitation_probability": int((daily_forecast.precipitation_probability or 0) * 100),
}
for daily_forecast in weather.daily_forecasts
]
response = httpx.post(
f"{url}/api/states/sensor.van_weather_forecast_daily",
headers=headers,
json={"state": len(daily_forecast), "attributes": {"forecast": daily_forecast}},
timeout=30,
)
response.raise_for_status()
# Post hourly forecast as JSON attribute sensor
hourly_forecast = [
{
"datetime": hourly_forecast.date_time.isoformat(),
"condition": hourly_forecast.condition,
"temperature": hourly_forecast.temperature,
"precipitation_probability": int((hourly_forecast.precipitation_probability or 0) * 100),
}
for hourly_forecast in weather.hourly_forecasts
]
response = httpx.post(
f"{url}/api/states/sensor.van_weather_forecast_hourly",
headers=headers,
json={"state": len(hourly_forecast), "attributes": {"forecast": hourly_forecast}},
timeout=30,
)
response.raise_for_status()
def update_weather(config: Config) -> None:
"""Fetch weather using last-known location, post to HA."""
lat = get_ha_state(config.ha_url, config.ha_token, config.lat_entity)
lon = get_ha_state(config.ha_url, config.ha_token, config.lon_entity)
masked_lat = round(lat, config.mask_decimals)
masked_lon = round(lon, config.mask_decimals)
logger.info(f"Masked location: {masked_lat}, {masked_lon}")
weather = fetch_weather(config.pirate_weather_api_key, masked_lat, masked_lon)
logger.info(f"Weather: {weather.temperature}°F, {weather.condition}")
post_to_ha(config.ha_url, config.ha_token, weather)
logger.info("Posted weather to HA")
def main(
ha_url: Annotated[str, typer.Option(envvar="HA_URL")],
ha_token: Annotated[str, typer.Option(envvar="HA_TOKEN")],
api_key: Annotated[str, typer.Option(envvar="PIRATE_WEATHER_API_KEY")],
interval: Annotated[int, typer.Option(help="Poll interval in seconds")] = 900,
log_level: Annotated[str, typer.Option()] = "INFO",
) -> None:
"""Fetch weather for van using masked GPS location."""
configure_logger(log_level)
config = Config(ha_url=ha_url, ha_token=ha_token, pirate_weather_api_key=api_key)
logger.info(f"Starting van weather service, polling every {interval}s")
scheduler = BlockingScheduler()
scheduler.add_job(
update_weather,
"interval",
seconds=interval,
args=[config],
next_run_time=datetime.now(UTC),
)
scheduler.start()
if __name__ == "__main__":
typer.run(main)
+72
View File
@@ -0,0 +1,72 @@
"""Models for van weather service."""
from __future__ import annotations
from datetime import datetime # noqa: TC003 This is required for pydantic
from pydantic import BaseModel, field_serializer
class Config(BaseModel):
"""Service configuration."""
ha_url: str
ha_token: str
pirate_weather_api_key: str
lat_entity: str = "sensor.van_last_known_latitude"
lon_entity: str = "sensor.van_last_known_longitude"
mask_decimals: int = 1 # ~11km accuracy
class DailyForecast(BaseModel):
"""Daily forecast entry."""
date_time: datetime
condition: str | None = None
temperature: float | None = None # High
templow: float | None = None # Low
precipitation_probability: float | None = None
moon_phase: float | None = None
wind_gust: float | None = None
cloud_cover: float | None = None
@field_serializer("date_time")
def serialize_date_time(self, date_time: datetime) -> str:
"""Serialize datetime to ISO format."""
return date_time.isoformat()
class HourlyForecast(BaseModel):
"""Hourly forecast entry."""
date_time: datetime
condition: str | None = None
temperature: float | None = None
precipitation_probability: float | None = None
@field_serializer("date_time")
def serialize_date_time(self, date_time: datetime) -> str:
"""Serialize datetime to ISO format."""
return date_time.isoformat()
class Weather(BaseModel):
"""Weather data from Pirate Weather."""
temperature: float | None = None
feels_like: float | None = None
humidity: float | None = None
wind_speed: float | None = None
wind_bearing: float | None = None
condition: str | None = None
summary: str | None = None
pressure: float | None = None
visibility: float | None = None
uv_index: float | None = None
ozone: float | None = None
nearest_storm_distance: float | None = None
nearest_storm_bearing: float | None = None
precip_probability: float | None = None
cloud_cover: float | None = None
daily_forecasts: list[DailyForecast] = []
hourly_forecasts: list[HourlyForecast] = []
-1685
View File
File diff suppressed because it is too large Load Diff
-21
View File
@@ -1,21 +0,0 @@
[package]
name = "van-weather"
version = "0.1.0"
edition = "2024"
description = "Fetch privacy-masked weather for a van and publish it to Home Assistant"
license = "MIT"
[dependencies]
anyhow = "1.0"
chrono = "0.4"
clap = { version = "4.5", features = ["derive", "env"] }
env_logger = "0.11"
log = "0.4"
reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
[lints.clippy]
all = "deny"
pedantic = "deny"
-13
View File
@@ -1,13 +0,0 @@
{ rustPlatform }:
rustPlatform.buildRustPackage {
pname = "van-weather";
version = "0.1.0";
src = ./.;
cargoLock.lockFile = ./Cargo.lock;
meta = {
description = "Privacy-masked van weather publisher for Home Assistant";
mainProgram = "van-weather";
};
}
-565
View File
@@ -1,565 +0,0 @@
use std::{thread, time::Duration};
use anyhow::{Context, Result, bail};
use chrono::{DateTime, Utc};
use clap::Parser;
use log::{error, info};
use reqwest::{
StatusCode, Url,
blocking::{Client, ClientBuilder},
header::{AUTHORIZATION, HeaderMap, HeaderValue},
retry,
};
use serde::Deserialize;
use serde_json::{Value, json};
const LAT_ENTITY: &str = "sensor.van_last_known_latitude";
const LON_ENTITY: &str = "sensor.van_last_known_longitude";
const PIRATE_WEATHER_HOST: &str = "api.pirateweather.net";
const MASK_DECIMALS: u32 = 1;
const MASK_FACTOR: f64 = decimal_factor(MASK_DECIMALS);
const RETRIES_PER_REQUEST: u32 = 2;
const fn decimal_factor(decimals: u32) -> f64 {
let mut factor = 1.0;
let mut remaining = decimals;
while remaining > 0 {
factor *= 10.0;
remaining -= 1;
}
factor
}
#[derive(Debug, Parser)]
#[command(about, version)]
struct Args {
#[arg(long, env = "HA_URL")]
ha_url: String,
#[arg(long, env = "HA_TOKEN", hide_env_values = true)]
ha_token: String,
#[arg(long, env = "PIRATE_WEATHER_API_KEY", hide_env_values = true)]
pirate_weather_api_key: String,
#[arg(
long,
default_value_t = 900,
value_parser = clap::value_parser!(u64).range(1..)
)]
interval: u64,
#[arg(long, default_value = "info", env = "RUST_LOG")]
log_level: String,
}
#[derive(Debug, Deserialize)]
struct HaState {
state: String,
}
#[derive(Debug)]
struct HttpClients {
home_assistant: Client,
pirate_weather: Client,
}
#[derive(Debug, Default, Deserialize)]
struct ApiResponse {
#[serde(default)]
currently: CurrentWeather,
#[serde(default)]
daily: ForecastBlock<DailyApiForecast>,
#[serde(default)]
hourly: ForecastBlock<HourlyApiForecast>,
}
#[derive(Debug, Default, Deserialize)]
#[serde(rename_all = "camelCase")]
struct CurrentWeather {
temperature: Option<f64>,
apparent_temperature: Option<f64>,
humidity: Option<f64>,
wind_speed: Option<f64>,
wind_bearing: Option<f64>,
icon: Option<String>,
pressure: Option<f64>,
visibility: Option<f64>,
uv_index: Option<f64>,
ozone: Option<f64>,
nearest_storm_distance: Option<f64>,
nearest_storm_bearing: Option<f64>,
precip_probability: Option<f64>,
cloud_cover: Option<f64>,
}
#[derive(Debug, Deserialize)]
struct ForecastBlock<T> {
#[serde(default)]
data: Vec<T>,
}
impl<T> Default for ForecastBlock<T> {
fn default() -> Self {
Self { data: Vec::new() }
}
}
#[derive(Debug, Default, Deserialize)]
#[serde(rename_all = "camelCase")]
struct DailyApiForecast {
time: Option<i64>,
icon: Option<String>,
temperature_high: Option<f64>,
temperature_low: Option<f64>,
precip_probability: Option<f64>,
}
#[derive(Debug, Default, Deserialize)]
#[serde(rename_all = "camelCase")]
struct HourlyApiForecast {
time: Option<i64>,
icon: Option<String>,
temperature: Option<f64>,
precip_probability: Option<f64>,
}
#[derive(Debug)]
struct Weather {
current: CurrentWeather,
daily: Vec<DailyForecast>,
hourly: Vec<HourlyForecast>,
}
#[derive(Debug)]
struct DailyForecast {
datetime: DateTime<Utc>,
condition: &'static str,
temperature: Option<f64>,
templow: Option<f64>,
precipitation_probability: Option<f64>,
}
#[derive(Debug)]
struct HourlyForecast {
datetime: DateTime<Utc>,
condition: &'static str,
temperature: Option<f64>,
precipitation_probability: Option<f64>,
}
fn main() -> Result<()> {
let Args {
ha_url,
ha_token,
pirate_weather_api_key,
interval,
log_level,
} = Args::parse();
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or(log_level.as_str()))
.init();
let ha_url = ha_url.trim_end_matches('/').to_owned();
let ha_host = Url::parse(&ha_url)
.context("HA_URL is not a valid URL")?
.host_str()
.context("HA_URL has no host")?
.to_owned();
let clients = HttpClients {
home_assistant: build_client(&ha_host, Some(&ha_token))?,
pirate_weather: build_client(PIRATE_WEATHER_HOST, None)?,
};
info!("Starting van weather service, polling every {interval}s");
loop {
if let Err(err) = update_weather(&clients, &ha_url, &pirate_weather_api_key) {
error!("Weather update failed: {err:#}");
}
thread::sleep(Duration::from_secs(interval));
}
}
fn build_client(host: &str, bearer_token: Option<&str>) -> Result<Client> {
let policy = retry::for_host(host.to_owned())
.max_retries_per_request(RETRIES_PER_REQUEST)
.classify_fn(|request| {
let retryable = request.error().is_some()
|| request.status().is_some_and(|status| {
status == StatusCode::REQUEST_TIMEOUT
|| status == StatusCode::TOO_MANY_REQUESTS
|| status.is_server_error()
});
if retryable {
request.retryable()
} else {
request.success()
}
});
let mut builder = ClientBuilder::new()
.timeout(Duration::from_secs(30))
.retry(policy);
if let Some(token) = bearer_token {
let mut authorization = HeaderValue::from_str(&format!("Bearer {token}"))
.context("HA_TOKEN contains invalid header characters")?;
authorization.set_sensitive(true);
let mut headers = HeaderMap::new();
headers.insert(AUTHORIZATION, authorization);
builder = builder.default_headers(headers);
}
builder
.build()
.with_context(|| format!("failed to create HTTP client for {host}"))
}
fn update_weather(clients: &HttpClients, ha_url: &str, api_key: &str) -> Result<()> {
let lat = get_ha_state(&clients.home_assistant, ha_url, LAT_ENTITY)?;
let lon = get_ha_state(&clients.home_assistant, ha_url, LON_ENTITY)?;
let masked_lat = mask_coordinate(lat);
let masked_lon = mask_coordinate(lon);
info!("Masked location: {masked_lat}, {masked_lon}");
let weather = fetch_weather(&clients.pirate_weather, api_key, masked_lat, masked_lon)?;
info!(
"Weather: {}°F, {}",
weather
.current
.temperature
.map_or_else(|| "unknown".to_owned(), |value| value.to_string()),
condition(weather.current.icon.as_deref())
);
post_to_ha(&clients.home_assistant, ha_url, &weather)?;
info!("Posted weather to Home Assistant");
Ok(())
}
fn mask_coordinate(value: f64) -> f64 {
(value * MASK_FACTOR).round() / MASK_FACTOR
}
fn get_ha_state(client: &Client, ha_url: &str, entity_id: &str) -> Result<f64> {
let HaState { state } = client
.get(format!("{ha_url}/api/states/{entity_id}"))
.send()
.with_context(|| format!("request for {entity_id} failed"))?
.error_for_status()
.with_context(|| format!("Home Assistant rejected {entity_id} request"))?
.json()
.context("Home Assistant returned invalid JSON")?;
if matches!(state.as_str(), "unavailable" | "unknown") {
bail!("{entity_id} is {state}");
}
state
.parse::<f64>()
.with_context(|| format!("{entity_id} state is not numeric: {state}"))
}
fn fetch_weather(client: &Client, api_key: &str, lat: f64, lon: f64) -> Result<Weather> {
let response = client
.get(format!(
"https://{PIRATE_WEATHER_HOST}/forecast/{api_key}/{lat},{lon}"
))
.query(&[("units", "us")])
.send()
.context("Pirate Weather request failed")?
.error_for_status()
.context("Pirate Weather rejected request")?;
let data = response
.json::<ApiResponse>()
.context("Pirate Weather returned invalid JSON")?;
Ok(parse_weather(data))
}
fn parse_weather(data: ApiResponse) -> Weather {
let daily = data
.daily
.data
.into_iter()
.take(8)
.filter_map(|day| {
timestamp(day.time).map(|datetime| DailyForecast {
datetime,
condition: condition(day.icon.as_deref()),
temperature: day.temperature_high,
templow: day.temperature_low,
precipitation_probability: day.precip_probability,
})
})
.collect();
let hourly = data
.hourly
.data
.into_iter()
.take(48)
.filter_map(|hour| {
timestamp(hour.time).map(|datetime| HourlyForecast {
datetime,
condition: condition(hour.icon.as_deref()),
temperature: hour.temperature,
precipitation_probability: hour.precip_probability,
})
})
.collect();
Weather {
current: data.currently,
daily,
hourly,
}
}
fn timestamp(value: Option<i64>) -> Option<DateTime<Utc>> {
value
.filter(|value| *value != 0)
.and_then(DateTime::from_timestamp_secs)
}
fn condition(icon: Option<&str>) -> &'static str {
match icon.unwrap_or_default() {
"clear-day" => "sunny",
"clear-night" => "clear-night",
"rain" => "rainy",
"snow" => "snowy",
"sleet" => "snowy-rainy",
"wind" => "windy",
"fog" => "fog",
"partly-cloudy-day" | "partly-cloudy-night" => "partlycloudy",
_ => "cloudy",
}
}
fn post_to_ha(client: &Client, ha_url: &str, weather: &Weather) -> Result<()> {
for (entity_id, payload) in weather_updates(weather) {
let response = client
.post(format!("{ha_url}/api/states/{entity_id}"))
.json(&payload)
.send()
.with_context(|| format!("failed to post {entity_id}"))?;
ensure_success(response.status(), &entity_id)?;
}
Ok(())
}
fn ensure_success(status: StatusCode, entity_id: &str) -> Result<()> {
if status.is_success() {
Ok(())
} else {
bail!("Home Assistant rejected {entity_id} update with {status}")
}
}
#[allow(clippy::too_many_lines)]
fn weather_updates(weather: &Weather) -> Vec<(String, Value)> {
let current = &weather.current;
let mut updates = vec![
sensor(
"sensor.van_weather_condition",
Some(json!(condition(current.icon.as_deref()))),
json!({"friendly_name": "Van Weather Condition"}),
),
sensor(
"sensor.van_weather_temperature",
current.temperature.map(|value| json!(value)),
json!({"unit_of_measurement": "°F", "device_class": "temperature"}),
),
sensor(
"sensor.van_weather_apparent_temperature",
current.apparent_temperature.map(|value| json!(value)),
json!({"unit_of_measurement": "°F", "device_class": "temperature"}),
),
sensor(
"sensor.van_weather_humidity",
Some(json!(percent(current.humidity))),
json!({"unit_of_measurement": "%", "device_class": "humidity"}),
),
sensor(
"sensor.van_weather_pressure",
current.pressure.map(|value| json!(value)),
json!({"unit_of_measurement": "mbar", "device_class": "pressure"}),
),
sensor(
"sensor.van_weather_wind_speed",
current.wind_speed.map(|value| json!(value)),
json!({"unit_of_measurement": "mph", "device_class": "wind_speed"}),
),
sensor(
"sensor.van_weather_wind_bearing",
current.wind_bearing.map(|value| json!(value)),
json!({"unit_of_measurement": "°"}),
),
sensor(
"sensor.van_weather_visibility",
current.visibility.map(|value| json!(value)),
json!({"unit_of_measurement": "mi"}),
),
sensor(
"sensor.van_weather_uv_index",
current.uv_index.map(|value| json!(value)),
json!({"friendly_name": "Van Weather UV Index", "icon": "mdi:sun-wireless"}),
),
sensor(
"sensor.van_weather_ozone",
current.ozone.map(|value| json!(value)),
json!({"unit_of_measurement": "DU", "icon": "mdi:earth"}),
),
sensor(
"sensor.van_weather_nearest_storm_distance",
current.nearest_storm_distance.map(|value| json!(value)),
json!({"unit_of_measurement": "mi", "icon": "mdi:weather-lightning"}),
),
sensor(
"sensor.van_weather_nearest_storm_bearing",
current.nearest_storm_bearing.map(|value| json!(value)),
json!({"unit_of_measurement": "°", "icon": "mdi:weather-lightning"}),
),
sensor(
"sensor.van_weather_precip_probability",
Some(json!(percent(current.precip_probability))),
json!({"unit_of_measurement": "%", "icon": "mdi:weather-rainy"}),
),
sensor(
"sensor.van_weather_cloud_cover",
Some(json!(percent(current.cloud_cover))),
json!({"unit_of_measurement": "%", "icon": "mdi:weather-cloudy"}),
),
]
.into_iter()
.flatten()
.collect::<Vec<_>>();
let daily = weather
.daily
.iter()
.map(|forecast| {
json!({
"datetime": forecast.datetime.to_rfc3339(),
"condition": forecast.condition,
"temperature": forecast.temperature,
"templow": forecast.templow,
"precipitation_probability": percent(forecast.precipitation_probability),
})
})
.collect::<Vec<_>>();
updates.push((
"sensor.van_weather_forecast_daily".to_owned(),
json!({"state": daily.len(), "attributes": {"forecast": daily}}),
));
let hourly = weather
.hourly
.iter()
.map(|forecast| {
json!({
"datetime": forecast.datetime.to_rfc3339(),
"condition": forecast.condition,
"temperature": forecast.temperature,
"precipitation_probability": percent(forecast.precipitation_probability),
})
})
.collect::<Vec<_>>();
updates.push((
"sensor.van_weather_forecast_hourly".to_owned(),
json!({"state": hourly.len(), "attributes": {"forecast": hourly}}),
));
updates
}
fn sensor(entity_id: &str, state: Option<Value>, attributes: Value) -> Option<(String, Value)> {
state.map(|state| {
let mut payload = serde_json::Map::new();
payload.insert("state".to_owned(), state);
payload.insert("attributes".to_owned(), attributes);
(entity_id.to_owned(), Value::Object(payload))
})
}
#[allow(clippy::cast_possible_truncation)]
fn percent(value: Option<f64>) -> i64 {
// Preserve Python's int(probability * 100) behavior for Home Assistant.
(value.unwrap_or_default() * 100.0) as i64
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn maps_conditions_for_home_assistant() {
assert_eq!(condition(Some("clear-day")), "sunny");
assert_eq!(condition(Some("sleet")), "snowy-rainy");
assert_eq!(condition(Some("partly-cloudy-night")), "partlycloudy");
assert_eq!(condition(Some("unexpected")), "cloudy");
assert_eq!(condition(None), "cloudy");
}
#[test]
fn parses_api_response_and_builds_compatible_payloads() {
let response: ApiResponse = serde_json::from_value(json!({
"currently": {
"temperature": 72.5,
"humidity": 0.67,
"icon": "clear-day",
"precipProbability": 0.129,
"cloudCover": 0.4,
"summary": "Fine"
},
"daily": {"data": [{
"time": 1_700_000_000,
"icon": "rain",
"temperatureHigh": 75.0,
"temperatureLow": 52.0,
"precipProbability": 0.8
}]},
"hourly": {"data": [{
"time": 1_700_000_000,
"icon": "fog",
"temperature": 61.0,
"precipProbability": 0.05
}]}
}))
.unwrap();
let weather = parse_weather(response);
let updates = weather_updates(&weather);
let find = |id: &str| updates.iter().find(|(entity, _)| entity == id).unwrap();
assert_eq!(find("sensor.van_weather_condition").1["state"], "sunny");
assert_eq!(find("sensor.van_weather_humidity").1["state"], 67);
assert_eq!(find("sensor.van_weather_precip_probability").1["state"], 12);
assert_eq!(find("sensor.van_weather_forecast_daily").1["state"], 1);
assert_eq!(
find("sensor.van_weather_forecast_daily").1["attributes"]["forecast"][0]["condition"],
"rainy"
);
assert_eq!(find("sensor.van_weather_forecast_hourly").1["state"], 1);
}
#[test]
fn omits_missing_optional_current_sensors_but_keeps_percentage_sensors() {
let weather = parse_weather(ApiResponse::default());
let updates = weather_updates(&weather);
assert!(
!updates
.iter()
.any(|(id, _)| id == "sensor.van_weather_temperature")
);
assert_eq!(
updates
.iter()
.find(|(id, _)| id == "sensor.van_weather_humidity")
.unwrap()
.1["state"],
0
);
}
#[test]
fn masks_coordinates_to_about_eleven_kilometres() {
assert!((mask_coordinate(37.7749) - 37.8).abs() < f64::EPSILON);
assert!((mask_coordinate(-122.4194) - (-122.4)).abs() < f64::EPSILON);
}
}
+2 -1
View File
@@ -8,7 +8,7 @@
# loader path, which NixOS does not provide globally.
ebook-search = pkgs.mkShell {
nativeBuildInputs = with pkgs; [
python314
my_python
uv
];
LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
@@ -23,6 +23,7 @@
nix
home-manager
git
my_python
ssh-to-age
gnupg
+5 -5
View File
@@ -3,18 +3,18 @@
imports = [
"${inputs.self}/users/math"
"${inputs.self}/users/richie"
"${inputs.self}/users/steve"
"${inputs.self}/common/global"
"${inputs.self}/common/optional/docker.nix"
"${inputs.self}/common/optional/monitoring-agent.nix"
"${inputs.self}/common/optional/nvidia.nix"
"${inputs.self}/common/optional/scanner.nix"
"${inputs.self}/common/optional/monitoring-agent.nix"
"${inputs.self}/common/optional/steam.nix"
"${inputs.self}/common/optional/syncthing_base.nix"
"${inputs.self}/common/optional/systemd-boot.nix"
"${inputs.self}/common/optional/tailscale.nix"
"${inputs.self}/common/optional/update.nix"
"${inputs.self}/common/optional/x86-64-v3.nix"
"${inputs.self}/common/optional/zfs"
"${inputs.self}/common/optional/yubikey.nix"
"${inputs.self}/common/optional/zerotier.nix"
"${inputs.self}/common/optional/nvidia.nix"
./hardware.nix
./syncthing.nix
./llms.nix
-5
View File
@@ -50,11 +50,6 @@
fsType = "zfs";
};
"/nix/var/nix/builds" = {
device = "root_pool/nix_build";
fsType = "zfs";
};
"/var" = {
device = "root_pool/var";
fsType = "zfs";
-6
View File
@@ -22,12 +22,6 @@ hourly = 0
daily = 0
monthly = 0
["root_pool/nix_build"]
15_min = 1
hourly = 0
daily = 0
monthly = 0
["root_pool/models"]
15_min = 4
hourly = 24
+1 -3
View File
@@ -7,10 +7,8 @@
"${inputs.self}/common/optional/ssh_decrypt.nix"
"${inputs.self}/common/optional/syncthing_base.nix"
"${inputs.self}/common/optional/systemd-boot.nix"
"${inputs.self}/common/optional/tailscale.nix"
"${inputs.self}/common/optional/update.nix"
"${inputs.self}/common/optional/x86-64-v3.nix"
"${inputs.self}/common/optional/zfs"
"${inputs.self}/common/optional/zerotier.nix"
./docker
./hardware.nix
./programs.nix
-5
View File
@@ -53,11 +53,6 @@
fsType = "zfs";
};
"/nix/var/nix/builds" = {
device = "root_pool/nix_build";
fsType = "zfs";
};
"/boot" = {
device = "/dev/disk/by-uuid/12CE-A600";
fsType = "vfat";
+1 -15
View File
@@ -4,20 +4,6 @@
...
}:
{
nixpkgs.overlays = [
(final: _prev: {
heater_python = final.python314.withPackages (
ps: with ps; [
fastapi
pydantic
tinytuya
typer
uvicorn
]
);
})
];
networking.firewall.allowedTCPPorts = [ 8124 ];
systemd.services.heater-api = {
@@ -31,7 +17,7 @@
serviceConfig = {
Type = "simple";
ExecStart = "${pkgs.heater_python}/bin/python -m python.heater.main --host 0.0.0.0 --port 8124";
ExecStart = "${pkgs.my_python}/bin/python -m python.heater.main --host 0.0.0.0 --port 8124";
EnvironmentFile = "/etc/heater.env";
Restart = "on-failure";
RestartSec = "5s";
+7 -10
View File
@@ -1,7 +1,5 @@
{ pkgs, ... }:
{
networking.firewall.allowedTCPPorts = [ 8123 ];
users = {
users.hass = {
isSystemUser = true;
@@ -13,7 +11,9 @@
services = {
home-assistant = {
enable = true;
openFirewall = true;
config = {
http.server_port = 8123;
homeassistant = {
time_zone = "America/New_York";
unit_system = "us_customary";
@@ -21,7 +21,6 @@
packages = {
victron_modbuss = "!include ${./home_assistant/victron_modbuss.yaml}";
battery_sensors = "!include ${./home_assistant/battery_sensors.yaml}";
energy_meters = "!include ${./home_assistant/energy_meters.yaml}";
gps_location = "!include ${./home_assistant/gps_location.yaml}";
heater = "!include ${./home_assistant/heater.yaml}";
van_weather = "!include ${./home_assistant/van_weather_template.yaml}";
@@ -68,26 +67,24 @@
paho-mqtt # for mqtt
psycopg2 # for postgresql
py-improv-ble-client # for esphome
pymodbus # for modbus
pyopenweathermap # for weather
pymetno # for met.no weather
uiprotect # for ubiquiti integration
unifi-discovery # for ubiquiti integration
jsonpath # for rest sensors
monarchmoneycommunity # for monarch
typedmonarchmoney # for monarch
];
extraComponents = [
"isal"
"modbus" # for victron modbus integration
];
extraComponents = [ "isal" ];
customComponents = with pkgs.home-assistant-custom-components; [
garmin_connect
pirate-weather
];
};
esphome = {
enable = true;
openFirewall = true;
address = "0.0.0.0";
address = "192.168.90.35";
};
};
}
@@ -1,71 +0,0 @@
template:
- sensor:
- name: "JK0 Energy In Clean"
unique_id: jk0_energy_in_clean
unit_of_measurement: "kWh"
device_class: energy
state_class: total_increasing
availability: >-
{{ is_number(states('sensor.jk0_energy_in')) }}
state: >-
{{ states('sensor.jk0_energy_in') | float }}
- name: "JK0 Energy Out Clean"
unique_id: jk0_energy_out_clean
unit_of_measurement: "kWh"
device_class: energy
state_class: total_increasing
availability: >-
{{ is_number(states('sensor.jk0_energy_out')) }}
state: >-
{{ states('sensor.jk0_energy_out') | float }}
- name: "JK1 Energy In Clean"
unique_id: jk1_energy_in_clean
unit_of_measurement: "kWh"
device_class: energy
state_class: total_increasing
availability: >-
{{ is_number(states('sensor.jk1_energy_in')) }}
state: >-
{{ states('sensor.jk1_energy_in') | float }}
- name: "JK1 Energy Out Clean"
unique_id: jk1_energy_out_clean
unit_of_measurement: "kWh"
device_class: energy
state_class: total_increasing
availability: >-
{{ is_number(states('sensor.jk1_energy_out')) }}
state: >-
{{ states('sensor.jk1_energy_out') | float }}
- name: "Solar Yield Clean"
unique_id: solar_yield_clean
unit_of_measurement: "kWh"
device_class: energy
state_class: total_increasing
availability: >-
{{ is_number(states('sensor.solar_yield_daily')) }}
state: >-
{{ states('sensor.solar_yield_daily') | float }}
- name: "DC Load Energy Clean"
unique_id: dc_load_energy_clean
unit_of_measurement: "kWh"
device_class: energy
state_class: total_increasing
availability: >-
{{ is_number(states('sensor.dc_load_energy_daily')) }}
state: >-
{{ states('sensor.dc_load_energy_daily') | float }}
- name: "DC Charger Energy Clean"
unique_id: dc_charger_energy_clean
unit_of_measurement: "kWh"
device_class: energy
state_class: total_increasing
availability: >-
{{ is_number(states('sensor.dc_charger_energy')) }}
state: >-
{{ states('sensor.dc_charger_energy') | float }}
+2 -2
View File
@@ -19,7 +19,7 @@
local all richie trust
host all richie 127.0.0.1/32 trust
host all richie ::1/128 trust
host all richie 100.64.0.0/10 trust
host all richie 192.168.90.1/24 trust
host all richie 192.168.99.1/24 trust
local vaninventory vaninventory trust
@@ -28,7 +28,7 @@
local hass hass trust
# ipv4
host hass hass 100.64.0.0/10 trust
host hass hass 192.168.90.1/24 trust
host hass hass 127.0.0.1/32 trust
# ipv6
+7 -4
View File
@@ -1,10 +1,8 @@
{
pkgs,
inputs,
...
}:
let
van-weather = pkgs.callPackage ../../../rust/van_weather/package.nix { };
in
{
systemd.services.van-weather = {
description = "Van Weather Service";
@@ -15,9 +13,13 @@ in
requires = [ "home-assistant.service" ];
wantedBy = [ "multi-user.target" ];
environment = {
PYTHONPATH = "${inputs.self}/";
};
serviceConfig = {
Type = "simple";
ExecStart = "${van-weather}/bin/van-weather";
ExecStart = "${pkgs.my_python}/bin/python -m python.van_weather.main";
EnvironmentFile = "/etc/van_weather.env";
Restart = "on-failure";
RestartSec = "5s";
@@ -27,6 +29,7 @@ in
ProtectSystem = "strict";
ProtectHome = "read-only";
PrivateTmp = true;
ReadOnlyPaths = [ "${inputs.self}" ];
};
};
}
+1 -1
View File
@@ -3,7 +3,7 @@
services.syncthing = {
overrideFolders = false;
guiAddress = "0.0.0.0:8384";
guiAddress = "192.168.90.35:8384";
settings = {
"dotfiles" = {
path = "/home/richie/dotfiles";
-71
View File
@@ -1,71 +0,0 @@
{ pkgs, ... }:
let
vars = import ./vars.nix;
in
{
virtualisation.docker.daemon.settings."data-root" = "${vars.containers}/docker";
# nixos-container hardcodes its state directory to /var/lib/nixos-containers,
# so route it to the shared container dataset with a bind mount.
fileSystems."/var/lib/nixos-containers" = {
device = "${vars.containers}/nixos-containers";
fsType = "none";
options = [ "bind" ];
};
systemd.services = {
prepare-docker-storage = {
description = "Create Docker storage directory";
requiredBy = [ "docker.service" ];
before = [ "docker.service" ];
requires = [ "zfs-mount.service" ];
after = [ "zfs-mount.service" ];
path = [
pkgs.coreutils
pkgs.util-linux
];
unitConfig = {
DefaultDependencies = false;
RequiresMountsFor = [ "/nix" ];
};
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
script = ''
mountpoint -q ${vars.containers}
install -d -m 0710 ${vars.containers}/docker
'';
};
prepare-nixos-container-storage = {
description = "Create NixOS container storage directory";
requiredBy = [ "var-lib-nixos\\x2dcontainers.mount" ];
before = [ "var-lib-nixos\\x2dcontainers.mount" ];
requires = [ "zfs-mount.service" ];
after = [ "zfs-mount.service" ];
path = [
pkgs.coreutils
pkgs.util-linux
];
unitConfig = {
DefaultDependencies = false;
RequiresMountsFor = [ "/nix" ];
};
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
script = ''
mountpoint -q ${vars.containers}
install -d -m 0755 ${vars.containers}/nixos-containers
'';
};
};
}
+7 -5
View File
@@ -4,23 +4,23 @@ let
in
{
imports = [
"${inputs.self}/users/dov"
"${inputs.self}/users/math"
"${inputs.self}/users/richie"
"${inputs.self}/users/steve"
"${inputs.self}/common/global"
"${inputs.self}/common/optional/docker.nix"
"${inputs.self}/common/optional/monitoring-agent.nix"
"${inputs.self}/common/optional/ssh_decrypt.nix"
"${inputs.self}/common/optional/syncthing_base.nix"
"${inputs.self}/common/optional/tailscale.nix"
"${inputs.self}/common/optional/update.nix"
"${inputs.self}/common/optional/x86-64-v3.nix"
"${inputs.self}/common/optional/zfs"
"${inputs.self}/common/optional/zerotier.nix"
./monitoring
./docker
./services
./containers.nix
./web_services
./hardware.nix
./networking
./networking.nix
./programs.nix
./runners
./syncthing.nix
@@ -35,6 +35,8 @@ in
path = ./snapshot_config.toml;
EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
};
zerotierone.joinNetworks = [ "a09acf02330d37b9" ];
};
users.groups = {
@@ -2,6 +2,9 @@ let
vars = import ../vars.nix;
in
{
networking.firewall.allowedTCPPorts = [
8989
];
virtualisation.oci-containers.containers.signal_cli_rest_api = {
image = "bbernhard/signal-cli-rest-api:0.199-dev";
ports = [
-5
View File
@@ -96,11 +96,6 @@ in
fsType = "zfs";
};
"/nix/var/nix/builds" = {
device = "root_pool/nix_build";
fsType = "zfs";
};
"/var" = {
device = "root_pool/var";
fsType = "zfs";
-33
View File
@@ -1,33 +0,0 @@
# Monitoring
## Vultr API metrics
The `vultr-exporter` service reads its API token from:
```text
/zfs/storage/secrets/services/vultr-exporter
```
Create the file on Jeeves as root with the following contents:
```text
API_KEY=<Vultr API token>
```
The token needs read access to the Vultr Account and Billing APIs. Unrelated
resource collectors are disabled in the packaged exporter.
Restrict the file to root and ensure the public egress IP used by Jeeves is
allowed for the token in the Vultr API settings:
```console
sudo chown root:root /zfs/storage/secrets/services/vultr-exporter
sudo chmod 600 /zfs/storage/secrets/services/vultr-exporter
```
The exporter listens on `127.0.0.1:9188`; it is scraped by the local
`prometheus-main` service every five minutes and is not exposed through the
host firewall.
Portal-1 exposes its node exporter only through `tailscale0` on port `9100`.
Jeeves reaches it using the Portal-1 Tailscale hostname.
@@ -1,798 +0,0 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": false,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
},
"mappings": [
{
"options": {
"0": {
"color": "red",
"index": 1,
"text": "Offline"
},
"1": {
"color": "green",
"index": 0,
"text": "Online"
}
},
"type": "value"
}
]
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 4,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "up{job=\"node\",instance=\"portal-1\"}",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Node Exporter",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 70
},
{
"color": "red",
"value": 90
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 5,
"x": 4,
"y": 0
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "100 * (1 - avg(rate(node_cpu_seconds_total{job=\"node\",instance=\"portal-1\",mode=\"idle\"}[5m])))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "CPU Used",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 70
},
{
"color": "red",
"value": 90
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 5,
"x": 9,
"y": 0
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "100 * (1 - (node_memory_MemAvailable_bytes{job=\"node\",instance=\"portal-1\"} / node_memory_MemTotal_bytes{job=\"node\",instance=\"portal-1\"}))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "RAM Used",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 70
},
{
"color": "red",
"value": 90
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 5,
"x": 14,
"y": 0
},
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "100 * (1 - (node_filesystem_avail_bytes{job=\"node\",instance=\"portal-1\",mountpoint=\"/\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node\",instance=\"portal-1\",mountpoint=\"/\",fstype!=\"\"}))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Root Filesystem Used",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 5,
"x": 19,
"y": 0
},
"id": 5,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "node_load1{job=\"node\",instance=\"portal-1\"}",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Load (1m)",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 70
},
{
"color": "red",
"value": 90
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 5
},
"id": 6,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "100 * (1 - avg(rate(node_cpu_seconds_total{job=\"node\",instance=\"portal-1\",mode=\"idle\"}[5m])))",
"instant": false,
"legendFormat": "CPU used",
"range": true,
"refId": "A"
}
],
"title": "CPU Usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 70
},
{
"color": "red",
"value": 90
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 5
},
"id": 7,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "100 * (1 - (node_memory_MemAvailable_bytes{job=\"node\",instance=\"portal-1\"} / node_memory_MemTotal_bytes{job=\"node\",instance=\"portal-1\"}))",
"instant": false,
"legendFormat": "RAM used",
"range": true,
"refId": "A"
}
],
"title": "RAM Usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 70
},
{
"color": "red",
"value": 90
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 13
},
"id": 8,
"options": {
"cellHeight": "sm",
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "Value"
}
]
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "100 * (1 - (node_filesystem_avail_bytes{job=\"node\",instance=\"portal-1\",fstype!~\"tmpfs|devtmpfs|overlay|squashfs\",mountpoint!~\"/run.*\"} / node_filesystem_size_bytes{job=\"node\",instance=\"portal-1\",fstype!~\"tmpfs|devtmpfs|overlay|squashfs\",mountpoint!~\"/run.*\"}))",
"format": "table",
"instant": true,
"legendFormat": "{{mountpoint}}",
"range": false,
"refId": "A"
}
],
"title": "Filesystem Usage",
"type": "table"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 13
},
"id": 9,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "sum(rate(node_disk_read_bytes_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
"instant": false,
"legendFormat": "read",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "sum(rate(node_disk_written_bytes_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
"instant": false,
"legendFormat": "write",
"range": true,
"refId": "B"
}
],
"title": "Disk Throughput",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "iops"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 21
},
"id": 10,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "sum(rate(node_disk_reads_completed_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
"instant": false,
"legendFormat": "reads",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "sum(rate(node_disk_writes_completed_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
"instant": false,
"legendFormat": "writes",
"range": true,
"refId": "B"
}
],
"title": "Disk Operations",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 21
},
"id": 11,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "sum(rate(node_network_receive_bytes_total{job=\"node\",instance=\"portal-1\",device!=\"lo\"}[5m]))",
"instant": false,
"legendFormat": "received",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "sum(rate(node_network_transmit_bytes_total{job=\"node\",instance=\"portal-1\",device!=\"lo\"}[5m]))",
"instant": false,
"legendFormat": "sent",
"range": true,
"refId": "B"
}
],
"title": "Network Traffic",
"type": "timeseries"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [
"monitoring",
"portal-1",
"vultr"
],
"templating": {
"list": []
},
"time": {
"from": "now-24h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Portal-1 Host",
"uid": "portal-1-host",
"version": 1,
"weekStart": ""
}
File diff suppressed because it is too large Load Diff
+10 -21
View File
@@ -41,36 +41,29 @@ let
{
job_name = "node";
static_configs = [
(mkTarget "jeeves" "jeeves.taile39c31.ts.net:9100")
(mkTarget "bob" "bob.taile39c31.ts.net:9100")
(mkTarget "portal-1" "portal-1.taile39c31.ts.net:9100")
(mkTarget "jeeves" "192.168.90.40:9100")
(mkTarget "bob" "192.168.90.25:9100")
];
}
{
job_name = "vultr";
scrape_interval = "5m";
scrape_timeout = "45s";
static_configs = [ (mkTarget "vultr" "127.0.0.1:9188") ];
}
{
job_name = "process_grouped";
static_configs = [
(mkTarget "jeeves" "jeeves.taile39c31.ts.net:9256")
(mkTarget "bob" "bob.taile39c31.ts.net:9256")
(mkTarget "jeeves" "192.168.90.40:9256")
(mkTarget "bob" "192.168.90.25:9256")
];
}
{
job_name = "smartctl";
static_configs = [
(mkTarget "jeeves" "jeeves.taile39c31.ts.net:9633")
(mkTarget "bob" "bob.taile39c31.ts.net:9633")
(mkTarget "jeeves" "192.168.90.40:9633")
(mkTarget "bob" "192.168.90.25:9633")
];
}
{
job_name = "zfs";
static_configs = [
(mkTarget "jeeves" "jeeves.taile39c31.ts.net:9134")
(mkTarget "bob" "bob.taile39c31.ts.net:9134")
(mkTarget "jeeves" "192.168.90.40:9134")
(mkTarget "bob" "192.168.90.25:9134")
];
}
];
@@ -86,8 +79,8 @@ let
{
job_name = "process_pid";
static_configs = [
(mkTarget "jeeves" "jeeves.taile39c31.ts.net:9257")
(mkTarget "bob" "bob.taile39c31.ts.net:9257")
(mkTarget "jeeves" "192.168.90.40:9257")
(mkTarget "bob" "192.168.90.25:9257")
];
}
];
@@ -104,9 +97,7 @@ let
after = [
"zfs-media-database-prometheus.mount"
"network.target"
"tailscaled-autoconnect.service"
];
wants = [ "tailscaled-autoconnect.service" ];
requires = [ "zfs-media-database-prometheus.mount" ];
wantedBy = [ "multi-user.target" ];
unitConfig.RequiresMountsFor = [ dataDir ];
@@ -160,8 +151,6 @@ let
};
in
{
imports = [ ./vultr-exporter.nix ];
users = {
groups.prometheus = { };
users.prometheus = {
@@ -1,96 +0,0 @@
{
lib,
pkgs,
...
}:
let
vars = import ../vars.nix;
vultrExporter = pkgs.buildGoModule rec {
pname = "vultr-exporter";
version = "0-unstable-2025-10-14";
src = pkgs.fetchFromGitHub {
owner = "DazWilkin";
repo = "vultr-exporter";
rev = "d8b5d7107c2eda31566e6e895c8a67db32fdaf68";
hash = "sha256-3LpXh+mFMBRJfiY2ftELpM3AIE0LiZ9c6NFok0OeH5I=";
};
vendorHash = "sha256-qCHWmg6GxLIjqlf5UqRcuDRd1xqRFI3sl4ZduLajx+Y=";
subPackages = [ "cmd/server" ];
postPatch = ''
substituteInPlace cmd/server/main.go \
--replace-fail 'registry.MustRegister(collector.NewBlockStorageCollector(s, client, log))' "" \
--replace-fail 'registry.MustRegister(collector.NewKubernetesCollector(s, client, log))' "" \
--replace-fail 'registry.MustRegister(collector.NewLoadBalancerCollector(s, client, log))' "" \
--replace-fail 'registry.MustRegister(collector.NewReservedIPsCollector(s, client, log))' ""
'';
ldflags = [
"-s"
"-w"
"-X main.GitCommit=${src.rev}"
"-X main.OSVersion=NixOS"
];
postInstall = ''
mv $out/bin/server $out/bin/vultr-exporter
'';
meta = {
description = "Prometheus exporter for Vultr account, billing, and bandwidth metrics";
homepage = "https://github.com/DazWilkin/vultr-exporter";
license = lib.licenses.asl20;
mainProgram = "vultr-exporter";
};
};
in
{
systemd.services.vultr-exporter = {
description = "Vultr Prometheus exporter";
documentation = [ "https://github.com/DazWilkin/vultr-exporter" ];
wants = [ "network-online.target" ];
after = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
ExecStart = "${lib.getExe vultrExporter} --endpoint=127.0.0.1:9188 --path=/metrics";
EnvironmentFile = "${vars.secrets}/services/vultr-exporter";
Restart = "on-failure";
RestartSec = "30s";
CapabilityBoundingSet = [ "" ];
DeviceAllow = [ "" ];
DynamicUser = true;
LockPersonality = true;
MemoryDenyWriteExecute = true;
NoNewPrivileges = true;
PrivateDevices = true;
PrivateTmp = true;
ProtectClock = true;
ProtectControlGroups = true;
ProtectHome = true;
ProtectHostname = true;
ProtectKernelLogs = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectSystem = "strict";
RemoveIPC = true;
RestrictAddressFamilies = [
"AF_INET"
"AF_INET6"
];
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
SystemCallArchitectures = "native";
SystemCallFilter = [
"@system-service"
"~@privileged"
];
UMask = "0077";
};
};
}
@@ -8,6 +8,27 @@
"net.bridge.bridge-nf-call-iptables" = 0;
};
networking = {
hostName = "jeeves";
hostId = "0e15ce35";
firewall = {
enable = true;
interfaces.br-nix-builder = {
allowedTCPPorts = [ ];
allowedUDPPorts = [ ];
};
allowedTCPPorts = [
8070
];
};
useNetworkd = true;
# Without this, unconfigured NICs get a catch-all DHCP network and
# join the LAN uninvited (how the dead atlantic card ended up
# ARP-poisoning 192.168.99.14). Every interface must be configured
# explicitly below.
useDHCP = false;
};
systemd.network = {
enable = true;
wait-online = {
-16
View File
@@ -1,16 +0,0 @@
{
imports = [
./firewall.nix
./interfaces.nix
];
networking = {
hostName = "jeeves";
hostId = "0e15ce35";
useNetworkd = true;
# Without this, unconfigured NICs get a catch-all DHCP network and
# join the LAN uninvited (how the dead atlantic card ended up
# ARP-poisoning 192.168.99.14). Every interface must be configured
# explicitly below.
useDHCP = false;
};
}
-47
View File
@@ -1,47 +0,0 @@
let
ports = {
audiobookshelf = 8000;
ebook_searcher = 8070;
gems = 8002;
gitea_http = 6443;
gitea_ssh = 2223;
grafana = 3000;
jellyfin_http = 8096;
nix_binary_cache = 5000;
nornsight = 8001;
ollama = 11434;
open_webui = 8080;
postgresql = 5432;
share = 8091;
signal_cli = 8989;
syncthing = 8384;
};
in
{
networking.firewall = {
enable = true;
interfaces.br-nix-builder = {
allowedTCPPorts = [ ];
allowedUDPPorts = [ ];
};
interfaces.tailscale0.allowedTCPPorts = [
ports.audiobookshelf
ports.gems
ports.jellyfin_http
ports.nix_binary_cache
ports.nornsight
ports.share
];
allowedTCPPorts = [
ports.ebook_searcher
ports.gitea_http
ports.gitea_ssh
ports.grafana
ports.ollama
ports.open_webui
ports.postgresql
ports.signal_cli
ports.syncthing
];
};
}
+89 -130
View File
@@ -1,7 +1,8 @@
{
config,
lib,
pkgs,
outputs,
utils,
...
}:
@@ -11,132 +12,6 @@ let
cfg = config.services.nix_builder;
runnerUsername = "gitea-runner";
runnerUserid = 601;
runnerLabels = [
"self-hosted:host"
"nixos:host"
];
containerConfig =
{
config,
pkgs,
lib,
...
}:
let
giteaAutomationPython = pkgs.python314.withPackages (
ps: with ps; [
httpx
pydantic
typer
]
);
runnerConfigFile = (pkgs.formats.yaml { }).generate "gitea-runner.yaml" { };
registerRunner = pkgs.writeShellApplication {
name = "register-gitea-runner";
runtimeInputs = [
pkgs.coreutils
pkgs.hostname
config.services.gitea-actions-runner.package
];
text = builtins.readFile ./register_gitea_runner.sh;
};
in
{
networking = {
hostName = "nix-builder";
useDHCP = lib.mkDefault true;
interfaces.eth0.useDHCP = true;
# Ensure containers don't inherit the host's stub resolver (127.0.0.53) which was causing issues
useHostResolvConf = false;
};
nix.settings = {
cores = 8;
max-jobs = 2;
system-features = lib.mkAfter [
"gccarch-x86-64-v2"
"gccarch-x86-64-v3"
];
trusted-substituters = [
"https://cache.nixos.org"
"https://cache.tmmworkshop.com"
"https://nix-community.cachix.org"
];
substituters = [
"https://cache.nixos.org/?priority=2&want-mass-query=true"
"https://cache.tmmworkshop.com/?priority=2&want-mass-query=true"
"https://nix-community.cachix.org/?priority=10&want-mass-query=true"
];
trusted-public-keys = [
"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
"cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA="
"nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
];
experimental-features = [
"flakes"
"nix-command"
];
sandbox = true;
allowed-users = [ runnerUsername ];
trusted-users = [
"root"
runnerUsername
];
};
users = {
users.${runnerUsername} = {
isSystemUser = true;
group = runnerUsername;
uid = runnerUserid;
};
groups.${runnerUsername}.gid = runnerUserid;
};
services.gitea-actions-runner.instances.builder = {
enable = true;
name = "jeeves-nix-builder";
url = "http://192.168.99.14:6443/";
labels = runnerLabels;
tokenFile = "/run/secrets/gitea-runners/registration-token";
settings.runner.timeout = "12h";
hostPackages = with pkgs; [
bash
coreutils
curl
gawk
gitMinimal
giteaAutomationPython
gnused
nix
nixfmt
nixos-rebuild
nodejs
ruff
treefmt
wget
];
};
systemd.services."gitea-runner-builder" = {
serviceConfig = {
DynamicUser = mkForce false;
User = mkForce runnerUsername;
Group = mkForce runnerUsername;
ExecStartPre = mkForce [
"${getExe registerRunner} builder http://192.168.99.14:6443/ ${runnerConfigFile} ${escapeShellArgs runnerLabels}"
];
};
};
system.stateVersion = "24.05";
};
sharedContainerPath =
(import "${pkgs.path}/nixos/lib/eval-config.nix" {
modules = [
{
boot.isNspawnContainer = true;
nixpkgs.pkgs = pkgs;
}
containerConfig
];
system = null;
}).config.system.build.toplevel;
in
{
options.services.nix_builder = {
@@ -173,11 +48,9 @@ in
containers = mapAttrs (
name: containerCfg:
mkIf containerCfg.enable {
path = sharedContainerPath;
autoStart = true;
privateNetwork = true;
hostBridge = cfg.bridgeName;
extraFlags = [ "--hostname=${name}" ];
bindMounts = {
host-nix = {
mountPoint = "/host-nix/var/nix/daemon-socket";
@@ -195,6 +68,93 @@ in
isReadOnly = false;
};
};
config =
{
config,
pkgs,
lib,
...
}:
{
networking = {
useDHCP = lib.mkDefault true;
interfaces.eth0.useDHCP = true;
# Ensure containers don't inherit the host's stub resolver (127.0.0.53) which was causing issues
useHostResolvConf = false;
};
nix.settings = {
trusted-substituters = [
"https://cache.nixos.org"
"https://cache.tmmworkshop.com"
"https://nix-community.cachix.org"
];
substituters = [
"https://cache.nixos.org/?priority=2&want-mass-query=true"
"https://cache.tmmworkshop.com/?priority=2&want-mass-query=true"
"https://nix-community.cachix.org/?priority=10&want-mass-query=true"
];
trusted-public-keys = [
"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
"cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA="
"nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
];
experimental-features = [
"flakes"
"nix-command"
];
sandbox = true;
allowed-users = [ "gitea-runner" ];
trusted-users = [
"root"
"gitea-runner"
];
};
nixpkgs = {
overlays = builtins.attrValues outputs.overlays;
config.allowUnfree = true;
};
users = {
users.${runnerUsername} = {
isSystemUser = true;
group = runnerUsername;
uid = runnerUserid;
};
groups.${runnerUsername}.gid = runnerUserid;
};
services.gitea-actions-runner.instances.${name} = {
enable = true;
name = "jeeves-${name}";
url = "http://192.168.99.14:6443/";
labels = [
"self-hosted:host"
"nixos:host"
];
tokenFile = "/run/secrets/gitea-runners/registration-token";
hostPackages = with pkgs; [
bash
coreutils
curl
gawk
gitMinimal
gnused
my_python
nix
nixfmt
nixos-rebuild
nodejs
treefmt
wget
];
};
systemd.services."gitea-runner-${utils.escapeSystemdPath name}" = {
serviceConfig = {
DynamicUser = mkForce false;
User = mkForce runnerUsername;
Group = mkForce runnerUsername;
};
};
system.stateVersion = "24.05";
};
}
) cfg.containers;
@@ -205,7 +165,6 @@ in
value = {
requires = [ "gitea.service" ];
after = [ "gitea.service" ];
serviceConfig.CPUQuota = "800%";
};
}) (builtins.attrNames (filterAttrs (_: c: c.enable) cfg.containers))
);
@@ -1,36 +0,0 @@
#!/usr/bin/env bash
legacy_instance_dir="$STATE_DIRECTORY/$(hostname)"
instance_dir="$STATE_DIRECTORY/$1"
runner_url="$2"
runner_config_file="$3"
shift 3
runner_labels="$(IFS=,; printf '%s' "$*")"
if [ ! -e "$instance_dir" ] && [ -d "$legacy_instance_dir" ]; then
mv "$legacy_instance_dir" "$instance_dir"
fi
mkdir -vp "$instance_dir"
cd "$instance_dir" || exit 1
token_hash_file="$instance_dir/.token-hash"
token_hash_current="$(printf '%s' "$TOKEN" | sha256sum | cut -d' ' -f1)"
token_hash_stored="$(cat "$token_hash_file" 2>/dev/null || echo "")"
labels_file="$instance_dir/.labels"
labels_wanted="$(printf '%s\n' "$@" | sort)"
labels_current="$(cat "$labels_file" 2>/dev/null || echo 0)"
if [ ! -e "$instance_dir/.runner" ] || [ "$labels_wanted" != "$labels_current" ] || [ "$token_hash_current" != "$token_hash_stored" ]; then
rm -v "$instance_dir/.runner" || true
gitea-runner register --no-interactive \
--instance "$runner_url" \
--token "$TOKEN" \
--name "jeeves-$(hostname)" \
--labels "$runner_labels" \
--config "$runner_config_file"
printf '%s' "$token_hash_current" > "$token_hash_file"
printf '%s\n' "$labels_wanted" > "$labels_file"
fi
+1 -1
View File
@@ -17,9 +17,9 @@ sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnode
# media datasets
sudo zfs create media/temp -o sync=disabled -o redundant_metadata=none
sudo zfs create media/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key
sudo zfs create media/secure/containers -o mountpoint=/zfs/media/containers -o compression=lz4 -o sync=disabled -o redundant_metadata=some -o normalization=none -o utf8only=off
sudo zfs create media/secure/docker -o compression=zstd-9
sudo zfs create media/secure/github-runners -o compression=zstd-9 -o sync=disabled
sudo zfs create media/secure/home_assistant -o compression=zstd-19
sudo zfs create media/secure/notes -o copies=2
sudo zfs create media/secure/postgres -o mountpoint=/zfs/media/database/postgres -o recordsize=16k -o primarycache=metadata
sudo zfs create media/secure/postgres-wal -o mountpoint=/zfs/media/database/postgres-wal -o recordsize=32k -o primarycache=metadata -o special_small_blocks=32K -o compression=lz4 -o secondarycache=none -o logbias=latency
@@ -6,7 +6,6 @@ in
services.audiobookshelf = {
enable = true;
port = 8000;
host = "0.0.0.0";
};
systemd.services.audiobookshelf.serviceConfig.WorkingDirectory =
lib.mkForce "${vars.docker_configs}/audiobookshelf";
@@ -0,0 +1,80 @@
{
...
}:
let
vars = import ../vars.nix;
in
{
systemd.tmpfiles.rules = [
"d ${vars.docker_configs}/camofox-browser 0750 root root - -"
];
containers.camofox-browser = {
autoStart = true;
privateNetwork = false;
bindMounts = {
camofox-browser = {
hostPath = "${vars.docker_configs}/camofox-browser";
mountPoint = "/var/lib/camofox-browser";
isReadOnly = false;
};
};
config =
{
pkgs,
lib,
...
}:
{
networking.hostName = "camofox-browser";
environment.systemPackages = with pkgs; [
ffmpeg
git
nodejs
python3Packages.yt-dlp
];
systemd.services.camofox-browser = {
description = "Camofox browser server";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
environment = {
CAMOFOX_HOST = "127.0.0.1";
CAMOFOX_PORT = "9377";
HOME = "/var/lib/camofox-browser";
};
path = with pkgs; [
bash
coreutils
git
nodejs
];
serviceConfig = {
Restart = "always";
RestartSec = "5s";
WorkingDirectory = "/var/lib/camofox-browser";
};
script = ''
set -eu
app_dir=/var/lib/camofox-browser/app
if [ ! -d "$app_dir/.git" ]; then
git clone --depth 1 https://github.com/jo-inc/camofox-browser "$app_dir"
fi
cd "$app_dir"
if [ ! -d node_modules ]; then
npm install
fi
exec npm start
'';
};
system.stateVersion = lib.mkDefault "24.05";
};
};
}
+1 -40
View File
@@ -6,47 +6,8 @@
let
vars = import ../vars.nix;
stateDir = "${vars.services}/gems";
gemsPackages =
ps: with ps; [
fastapi
jinja2
pydantic
pydantic-settings
python-multipart
typer
uvicorn
];
in
{
nixpkgs.overlays = [
(final: _prev: {
gems_python = final.python314.withPackages gemsPackages;
gems_test_python = final.python314.withPackages (
ps:
gemsPackages ps
++ (with ps; [
httpx
pytest
pytest-asyncio
pytest-xdist
])
);
gems_tests =
final.runCommand "gems-tests"
{
nativeBuildInputs = [ final.gems_test_python ];
}
''
export HOME="$TMPDIR"
cd ${inputs.self}
pytest -o cache_dir="$TMPDIR/pytest-cache" tests/gems
touch "$out"
'';
})
];
system.checks = [ pkgs.gems_tests ];
users.groups.gems = { };
users.users.gems = {
isSystemUser = true;
@@ -75,7 +36,7 @@ in
Type = "simple";
User = "gems";
Group = "gems";
ExecStart = "${pkgs.gems_python}/bin/python -m python.gems.main --host 0.0.0.0 --port 8002";
ExecStart = "${pkgs.my_python}/bin/python -m python.gems.main --host 127.0.0.1 --port 8002";
Restart = "on-failure";
RestartSec = "5s";
StandardOutput = "journal";
+6 -6
View File
@@ -2,6 +2,11 @@ let
vars = import ../vars.nix;
in
{
networking.firewall.allowedTCPPorts = [
6443
2223
];
services.gitea = {
enable = true;
appName = "TMM Workshop";
@@ -19,18 +24,13 @@ in
actions = {
ENABLED = true;
DEFAULT_ACTIONS_URL = "github";
ENDLESS_TASK_TIMEOUT = "12h";
};
service.DISABLE_REGISTRATION = true;
server = {
DOMAIN = "gitea.tmmworkshop.com";
DOMAIN = "tmmworkshop.com";
ROOT_URL = "https://gitea.tmmworkshop.com/";
HTTP_PORT = 6443;
BUILTIN_SSH_SERVER_USER = "gitea";
SSH_USER = "gitea";
SSH_DOMAIN = "gitea.tmmworkshop.com";
SSH_PORT = 2223;
SSH_LISTEN_HOST = "0.0.0.0";
SSH_LISTEN_PORT = 2223;
START_SSH_SERVER = true;
PUBLIC_URL_DETECTION = "auto";
+4 -2
View File
@@ -6,6 +6,8 @@ let
grafanaDataDir = "${vars.services}/grafana";
in
{
networking.firewall.allowedTCPPorts = [ 3000 ];
services.grafana = {
enable = true;
dataDir = grafanaDataDir;
@@ -17,9 +19,9 @@ in
secret_key = "$__file{${vars.secrets}/services/grafana/secret_key}";
};
server = {
http_addr = "0.0.0.0";
http_addr = "192.168.90.40";
http_port = 3000;
root_url = "http://jeeves:3000/";
root_url = "http://192.168.90.40:3000/";
};
};
provision = {
@@ -0,0 +1,79 @@
let
vars = import ../vars.nix;
in
{
users = {
users.hass = {
isSystemUser = true;
group = "hass";
};
groups.hass = { };
};
services = {
home-assistant = {
enable = true;
openFirewall = true;
configDir = vars.home_assistant;
config = {
http = {
server_port = 8123;
use_x_forwarded_for = true;
trusted_proxies = "127.0.0.1";
};
homeassistant = {
time_zone = "America/New_York";
unit_system = "us_customary";
temperature_unit = "F";
};
recorder = {
db_url = "postgresql://@/hass";
auto_purge = true;
purge_keep_days = 3650;
db_retry_wait = 15;
};
assist_pipeline = { };
backup = { };
bluetooth = { };
config = { };
dhcp = { };
energy = { };
history = { };
homeassistant_alerts = { };
image_upload = { };
logbook = { };
media_source = { };
mobile_app = { };
ssdp = { };
sun = { };
webhook = { };
zeroconf = { };
automation = "!include automations.yaml";
script = "!include scripts.yaml";
scene = "!include scenes.yaml";
group = "!include groups.yaml";
};
extraPackages =
python3Packages: with python3Packages; [
aioesphomeapi
aiounifi
bleak-esphome
esphome-dashboard-api
gtts
jellyfin-apiclient-python
psycopg2
pymetno
aio-ownet
rokuecp
uiprotect
wakeonlan
];
extraComponents = [ "isal" ];
};
esphome = {
enable = true;
openFirewall = true;
address = "192.168.90.40";
};
};
}
+1
View File
@@ -4,6 +4,7 @@ in
{
services.jellyfin = {
enable = true;
openFirewall = true;
dataDir = "${vars.services}/jellyfin";
cacheDir = "${vars.services}/jellyfin/cache";
};
+1
View File
@@ -38,6 +38,7 @@ in
"qwen3.5:35b"
];
modelsDir = vars.ollama;
openFirewall = true;
};
systemd.services = {
ollama.serviceConfig = {
+1
View File
@@ -7,5 +7,6 @@ in
package = pkgs.nix-serve-ng;
enable = true;
secretKeyFile = "${vars.secrets}/services/nix-cache/cache-priv-key.pem";
openFirewall = true;
};
}
+13
View File
@@ -0,0 +1,13 @@
{
services.open-webui = {
enable = true;
host = "0.0.0.0";
openFirewall = true;
environment = {
ANONYMIZED_TELEMETRY = "False";
DO_NOT_TRACK = "True";
SCARF_NO_ANALYTICS = "True";
OLLAMA_API_BASE_URL = "http://127.0.0.1:11434";
};
};
}
+18 -3
View File
@@ -3,6 +3,8 @@ let
vars = import ../vars.nix;
in
{
networking.firewall.allowedTCPPorts = [ 5432 ];
# Symlink pg_wal to a ZFS dataset on the special (metadata) vdev for fast WAL writes
# this is required for systemd sandboxing
systemd.services.postgresql.serviceConfig.ReadWritePaths = [ "/zfs/media/database/postgres-wal" ];
@@ -25,12 +27,13 @@ in
local all richie trust
host all richie 127.0.0.1/32 trust
host all richie ::1/128 trust
host all richie 100.64.0.0/10 trust
host all richie 192.168.90.1/24 trust
host all richie 192.168.99.1/24 trust
host all richie 172.16.0.0/12 trust
#type database DBuser origin-address auth-method
local hass hass trust
local gitea gitea trust
# signalbot
@@ -40,12 +43,12 @@ in
local postgres math trust
host postgres math 127.0.0.1/32 trust
host postgres math ::1/128 trust
host postgres math 100.64.0.0/10 trust
host postgres math 192.168.90.1/24 trust
local data_science_dev math trust
host data_science_dev math 127.0.0.1/32 trust
host data_science_dev math ::1/128 trust
host data_science_dev math 100.64.0.0/10 trust
host data_science_dev math 192.168.90.1/24 trust
'';
identMap = ''
@@ -54,6 +57,7 @@ in
superuser_map postgres postgres
# Let other names login as themselves
superuser_map richie postgres
superuser_map hass hass
'';
ensureUsers = [
{
@@ -77,6 +81,16 @@ in
replication = true;
};
}
{
name = "hass";
ensureDBOwnership = true;
ensureClauses = {
login = true;
createrole = true;
createdb = true;
replication = true;
};
}
{
name = "gitea";
ensureDBOwnership = true;
@@ -107,6 +121,7 @@ in
];
ensureDatabases = [
"data_science_dev"
"hass"
"gitea"
"math"
"n8n"
+26
View File
@@ -1,7 +1,11 @@
{
pkgs,
inputs,
...
}:
let
vars = import ../vars.nix;
in
{
systemd = {
services = {
@@ -26,6 +30,21 @@
ExecStart = "${pkgs.bash}/bin/bash -c 'echo 1 > /sys/bus/pci/devices/0000:61:00.0/remove'";
};
};
startup_validation = {
requires = [ "network-online.target" ];
after = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
description = "validates startup";
path = [ pkgs.zfs ];
environment = {
PYTHONPATH = "${inputs.self}/";
};
serviceConfig = {
EnvironmentFile = "${vars.secrets}/services/server-validation";
Type = "oneshot";
ExecStart = "${pkgs.my_python}/bin/python -m python.system_tests.validate_system '${./validate_system.toml}'";
};
};
};
timers = {
plex_permission = {
@@ -36,6 +55,13 @@
Unit = "plex_permission.service";
};
};
startup_validation = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnBootSec = "10min";
Unit = "startup_validation.service";
};
};
};
};
}
+2 -2
View File
@@ -21,11 +21,11 @@ in
message-level = 3;
peer-port = 51413;
rpc-bind-address = "0.0.0.0";
rpc-host-whitelist = "127.0.0.1,jeeves,jeeves.taile39c31.ts.net";
rpc-host-whitelist = "127.0.0.1,192.168.90.40";
rpc-host-whitelist-enabled = true;
rpc-port = 9091;
rpc-whitelist-enabled = true;
rpc-whitelist = "127.0.0.1,100.*.*.*";
rpc-whitelist = "127.0.0.1,192.168.90.49";
seed-queue-enabled = false;
};
};
@@ -0,0 +1,8 @@
zpool = ["root_pool", "storage", "media"]
services = [
"audiobookshelf",
"haproxy",
"docker",
"home-assistant",
"jellyfin",
]
-12
View File
@@ -23,12 +23,6 @@ hourly = 0
daily = 0
monthly = 0
["root_pool/nix_build"]
15_min = 1
hourly = 0
daily = 0
monthly = 0
["root_pool/var"]
15_min = 8
hourly = 24
@@ -66,12 +60,6 @@ daily = 30
monthly = 12
# media
["media/secure/containers"]
15_min = 2
hourly = 0
daily = 0
monthly = 0
["media/temp"]
15_min = 2
hourly = 0
+3 -1
View File
@@ -2,9 +2,11 @@ let
vars = import ./vars.nix;
in
{
networking.firewall.allowedTCPPorts = [ 8384 ];
services.syncthing = {
overrideFolders = false;
guiAddress = "0.0.0.0:8384";
guiAddress = "192.168.90.40:8384";
settings = {
devices.davids-server.id = "7GXTDGR-AOXFW2O-K6J7NM3-XYZNRRW-AKHAFWM-GBOWUPQ-OA6JIWD-ER7RDQL"; # cspell:disable-line
folders = {
+1 -1
View File
@@ -5,10 +5,10 @@ let
in
{
inherit zfs_media zfs_storage zfs_scratch;
containers = "${zfs_media}/containers";
database = "${zfs_media}/database";
docker = "${zfs_media}/docker";
docker_configs = "${zfs_media}/docker/configs";
home_assistant = "${zfs_media}/home_assistant";
notes = "${zfs_media}/notes";
secrets = "${zfs_storage}/secrets";
services = "${zfs_media}/services";
+75
View File
@@ -0,0 +1,75 @@
let
domains = [
"audiobookshelf"
"cache"
"gitea"
"gems"
"jellyfin"
"share"
];
extraDomains = [ "www.norn-sight.com" ];
makeCert = name: {
name = "${name}.tmmworkshop.com";
value = {
webroot = "/var/lib/acme/.challenges";
group = "acme";
reloadServices = [ "haproxy.service" ];
};
};
makeExtraCert = name: {
inherit name;
value = {
webroot = "/var/lib/acme/.challenges";
group = "acme";
reloadServices = [ "haproxy.service" ];
};
};
acmeServices =
map (domain: "acme-${domain}.tmmworkshop.com.service") domains
++ map (domain: "acme-${domain}.service") extraDomains;
in
{
users.users.haproxy.extraGroups = [ "acme" ];
security.acme = {
acceptTerms = true;
defaults.email = "Richie@tmmworkshop.com";
certs = builtins.listToAttrs ((map makeCert domains) ++ (map makeExtraCert extraDomains));
};
# Minimal nginx to serve ACME HTTP-01 challenge files for HAProxy
services.nginx = {
enable = true;
virtualHosts."acme-challenge" = {
listen = [
{
addr = "127.0.0.1";
port = 8402;
}
];
locations."/.well-known/acme-challenge/" = {
root = "/var/lib/acme/.challenges";
};
};
};
# Ensure the challenge directory exists with correct permissions
systemd.tmpfiles.rules = [
"d /var/lib/acme/.challenges 0750 acme acme - -"
"d /var/lib/acme/.challenges/.well-known 0750 acme acme - -"
"d /var/lib/acme/.challenges/.well-known/acme-challenge 0750 acme acme - -"
];
users.users.nginx.extraGroups = [ "acme" ];
# HAProxy needs certs to exist before it can bind :443.
# NixOS's acme module generates self-signed placeholders on first boot
# via acme-<domain>.service — just make HAProxy wait for them.
systemd.services.haproxy = {
after = acmeServices;
wants = acmeServices;
};
}
+9
View File
@@ -0,0 +1,9 @@
{ lib, ... }:
{
imports =
let
files = builtins.attrNames (builtins.readDir ./.);
nixFiles = builtins.filter (name: lib.hasSuffix ".nix" name && name != "default.nix") files;
in
map (file: ./. + "/${file}") nixFiles;
}
@@ -22,8 +22,8 @@ defaults
#Application Setup
frontend ContentSwitching
bind [::]:80 v4v6
bind [::]:443 v4v6 ssl crt /var/lib/acme/audiobookshelf.tmmworkshop.com/full.pem crt /var/lib/acme/cache.tmmworkshop.com/full.pem crt /var/lib/acme/gems.tmmworkshop.com/full.pem crt /var/lib/acme/jellyfin.tmmworkshop.com/full.pem crt /var/lib/acme/share.tmmworkshop.com/full.pem crt /var/lib/acme/gitea.tmmworkshop.com/full.pem crt /var/lib/acme/www.norn-sight.com/full.pem
bind *:80 v4v6
bind *:443 v4v6 ssl crt /var/lib/acme/audiobookshelf.tmmworkshop.com/full.pem crt /var/lib/acme/cache.tmmworkshop.com/full.pem crt /var/lib/acme/gems.tmmworkshop.com/full.pem crt /var/lib/acme/jellyfin.tmmworkshop.com/full.pem crt /var/lib/acme/share.tmmworkshop.com/full.pem crt /var/lib/acme/gitea.tmmworkshop.com/full.pem crt /var/lib/acme/www.norn-sight.com/full.pem
mode http
# ACME challenge routing (must be first)
@@ -110,14 +110,6 @@ frontend ContentSwitching
use_backend gems if host_gems
use_backend norn_sight if host_norn_sight
# Proxy Gitea's built-in SSH server over Tailscale to Jeeves. Port 22 remains
# available to Endlessh on Portal.
frontend GiteaSSH
bind [::]:2223 v4v6
mode tcp
option tcplog
default_backend gitea_ssh
# Stick-table only (no servers): tracks per-IP request rate to Gitea's compare
# endpoint over a 5-minute window so the frontend can cap it at 1 per 5 min.
backend st_compare
@@ -132,35 +124,31 @@ backend acme_challenge
backend audiobookshelf_nodes
mode http
server server jeeves:8000
server server 127.0.0.1:8000
backend cache_nodes
mode http
server server jeeves:5000
server server 127.0.0.1:5000
backend jellyfin
mode http
option forwardfor
server jellyfin jeeves:8096
server jellyfin 127.0.0.1:8096
backend share_nodes
mode http
server server jeeves:8091
server server 127.0.0.1:8091
backend gitea
mode http
server server jeeves:6443
backend gitea_ssh
mode tcp
server gitea jeeves:2223
server server 127.0.0.1:6443
backend gems
mode http
option forwardfor
timeout server 1h
server gems jeeves:8002
server gems 127.0.0.1:8002
backend norn_sight
mode http
server server jeeves:8001
server server 127.0.0.1:8001
+14
View File
@@ -0,0 +1,14 @@
{
networking.firewall.allowedTCPPorts = [
80
443
];
# Global robots.txt served by HAProxy for every vhost (see haproxy.cfg).
environment.etc."haproxy/robots.txt".source = ./robots.txt;
services.haproxy = {
enable = true;
config = builtins.readFile ./haproxy.cfg;
};
}
-83
View File
@@ -1,83 +0,0 @@
# portal_1
Minimal NixOS target for a Vultr VM, installed with nixos-anywhere. The Nix
flake target is `portal_1`; the machine hostname is `portal-1` because DNS
hostnames cannot contain underscores.
## Before deploying
1. Confirm the VM's system disk is `/dev/vda`. If it is not, update both
references in `disk-config.nix`.
2. Confirm the SSH public key in `default.nix` is the key that should have
administrator access.
3. Boot the VM into a NixOS installer or another nixos-anywhere-compatible
Linux rescue environment with root SSH access. Keep this environment
running while completing the SOPS bootstrap below.
## Bootstrap SOPS
Use the rescue environment's SSH host key as the permanent portal identity.
Replace `VM_IP` below:
```console
ssh root@VM_IP 'cat /etc/ssh/ssh_host_ed25519_key.pub' | \
nix shell nixpkgs#ssh-to-age --command ssh-to-age
```
This prints an `age1...` recipient; it does not copy the private key. Add the
recipient to `.sops.yaml`:
```yaml
- &system_portal_1 age1...
```
Then add `*system_portal_1` to the age recipients for
`users/secrets.yaml`. Re-encrypt the existing file for the new recipient and
add the Tailscale key:
```console
nix shell nixpkgs#sops --command sops updatekeys users/secrets.yaml
nix shell nixpkgs#sops --command sops users/secrets.yaml
```
Add the OAuth client secret from the `Auth Keys: Write` credential in the SOPS
editor and save it:
```yaml
tailscale_auth_key: tskey-client-...
```
## Deploy
From the repository root, replace `VM_IP` with the VM's public IP:
```console
nix run github:nix-community/nixos-anywhere -- \
--copy-host-keys --flake .#portal_1 root@VM_IP
```
This repartitions `/dev/vda`, so anything already on that disk is erased. The
layout reserves 8 GiB for swap and assigns the remaining space to the root
filesystem.
`--copy-host-keys` preserves the same private SSH host key at
`/etc/ssh/ssh_host_ed25519_key` on the installed system. SOPS-Nix converts that
key to an age identity during activation. After the reboot, connect as
`richie` and verify that automatic Tailscale enrollment succeeded:
```console
ssh -p 278 richie@VM_IP
sudo tailscale status
```
The installed OpenSSH service listens on port 278. Port 22 is served by
Endlessh and will not provide an SSH login.
HAProxy uses the same frontend, routing, and rate-limiting configuration as
Jeeves. Portal manages the ACME certificates for the existing public domains;
their DNS records must resolve to Portal for HTTP-01 issuance and renewal.
The application backends still use Jeeves' original `127.0.0.1` addresses.
Replace them with the corresponding Tailscale addresses before directing
application traffic through Portal. Ports 80 and 443 are allowed through the
firewall.
-56
View File
@@ -1,56 +0,0 @@
{
inputs,
lib,
pkgs,
...
}:
{
imports = [
inputs.disko.nixosModules.disko
"${inputs.self}/users/richie"
"${inputs.self}/common/global"
"${inputs.self}/common/optional/tailscale.nix"
"${inputs.self}/common/optional/x86-64-v3.nix"
./disk-config.nix
./haproxy
./monitoring.nix
];
boot = {
# Avoid consuming the VM's limited memory for /tmp.
tmp.useTmpfs = false;
# The Vultr system disk and NIC are exposed as virtio devices.
initrd.availableKernelModules = [
"virtio_pci"
"virtio_blk"
"virtio_scsi"
"sd_mod"
];
};
networking = {
hostName = "portal-1";
useDHCP = lib.mkDefault true;
firewall = {
enable = true;
allowedTCPPorts = [ 278 ];
};
};
services.openssh.ports = [ 278 ];
environment.systemPackages = with pkgs; [
curl
htop
vim
];
# The VM uses its regular swap instead of compressed RAM swap.
zramSwap.enable = false;
time.timeZone = "Etc/UTC";
system.stateVersion = "24.05";
}
-59
View File
@@ -1,59 +0,0 @@
{ ... }:
{
# Vultr's first virtio disk is normally /dev/vda. Change this before
# deployment if the selected image exposes its system disk differently.
disko.devices.disk.main = {
type = "disk";
device = "/dev/vda";
content = {
type = "gpt";
partitions = {
bios = {
size = "1M";
type = "EF02";
};
ESP = {
size = "512M";
type = "EF00";
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot";
mountOptions = [
"fmask=0077"
"dmask=0077"
];
};
};
swap = {
size = "8G";
content = {
type = "swap";
};
};
root = {
size = "100%";
content = {
type = "filesystem";
format = "ext4";
mountpoint = "/";
};
};
};
};
};
# Install GRUB for both legacy BIOS and EFI-capable Vultr plans.
boot.loader = {
grub = {
enable = true;
devices = [ "/dev/vda" ];
efiSupport = true;
efiInstallAsRemovable = true;
};
efi.canTouchEfiVariables = false;
};
}
-89
View File
@@ -1,89 +0,0 @@
let
domains = [
"audiobookshelf"
"cache"
"gitea"
"gems"
"jellyfin"
"share"
];
extraDomains = [ "www.norn-sight.com" ];
makeCert = name: {
name = "${name}.tmmworkshop.com";
value = {
webroot = "/var/lib/acme/.challenges";
group = "acme";
reloadServices = [ "haproxy.service" ];
};
};
makeExtraCert = name: {
inherit name;
value = {
webroot = "/var/lib/acme/.challenges";
group = "acme";
reloadServices = [ "haproxy.service" ];
};
};
acmeServices =
map (domain: "acme-${domain}.tmmworkshop.com.service") domains
++ map (domain: "acme-${domain}.service") extraDomains;
in
{
networking.firewall.allowedTCPPorts = [
80
443
2223
];
# Global robots.txt served by HAProxy for every vhost (see haproxy.cfg).
environment.etc."haproxy/robots.txt".source = ./robots.txt;
services = {
haproxy = {
enable = true;
config = builtins.readFile ./haproxy.cfg;
};
nginx = {
enable = true;
virtualHosts."acme-challenge" = {
listen = [
{
addr = "127.0.0.1";
port = 8402;
}
];
locations."/.well-known/acme-challenge/" = {
root = "/var/lib/acme/.challenges";
};
};
};
};
security.acme = {
acceptTerms = true;
defaults.email = "Richie@tmmworkshop.com";
certs = builtins.listToAttrs ((map makeCert domains) ++ (map makeExtraCert extraDomains));
};
systemd = {
services.haproxy = {
# HAProxy needs the initial certificates and Tailscale connectivity before
# it can serve requests to its backends.
after = acmeServices ++ [ "tailscaled-autoconnect.service" ];
wants = acmeServices ++ [ "tailscaled-autoconnect.service" ];
};
tmpfiles.rules = [
"d /var/lib/acme/.challenges 0750 acme acme - -"
"d /var/lib/acme/.challenges/.well-known 0750 acme acme - -"
"d /var/lib/acme/.challenges/.well-known/acme-challenge 0750 acme acme - -"
];
};
users.users = {
haproxy.extraGroups = [ "acme" ];
nginx.extraGroups = [ "acme" ];
};
}
-15
View File
@@ -1,15 +0,0 @@
{ config, ... }:
{
networking.firewall.interfaces.tailscale0.allowedTCPPorts = [
config.services.prometheus.exporters.node.port
];
services.prometheus.exporters.node = {
enable = true;
enabledCollectors = [
"pressure"
"processes"
"systemd"
];
};
}
@@ -0,0 +1,35 @@
{
pkgs,
inputs,
...
}:
{
systemd.services.agent-logger = {
description = "Unified agent logger";
after = [ "local-fs.target" ];
wantedBy = [ "multi-user.target" ];
environment = {
AGENT_LOG_DB = "/var/lib/agent-logger/agent_log.sqlite";
HOME = "/home/richie";
PYTHONPATH = "${inputs.self}";
};
serviceConfig = {
Type = "simple";
User = "richie";
WorkingDirectory = "/home/richie";
ExecStart = "${pkgs.my_python}/bin/python -m python.agent_logger.main";
StateDirectory = "agent-logger";
Restart = "on-failure";
RestartSec = "5s";
StandardOutput = "journal";
StandardError = "journal";
NoNewPrivileges = true;
ProtectSystem = "strict";
ProtectHome = "read-only";
PrivateTmp = true;
ReadOnlyPaths = [ "${inputs.self}" ];
};
};
}
+2 -3
View File
@@ -8,11 +8,10 @@
"${inputs.self}/common/optional/steam.nix"
"${inputs.self}/common/optional/syncthing_base.nix"
"${inputs.self}/common/optional/systemd-boot.nix"
"${inputs.self}/common/optional/tailscale.nix"
"${inputs.self}/common/optional/x86-64-v3.nix"
"${inputs.self}/common/optional/yubikey.nix"
"${inputs.self}/common/optional/zfs"
"${inputs.self}/common/optional/zerotier.nix"
./hardware.nix
./open_webui.nix
./programs.nix
./qmk.nix
./syncthing.nix
-8
View File
@@ -34,22 +34,14 @@
device = "root_pool/root";
fsType = "zfs";
};
"/home" = {
device = "root_pool/home";
fsType = "zfs";
};
"/nix/var/nix/builds" = {
device = "root_pool/nix_build";
fsType = "zfs";
};
"/var" = {
device = "root_pool/var";
fsType = "zfs";
};
"/boot" = {
device = "/dev/disk/by-uuid/12CE-A600";
fsType = "vfat";
+13
View File
@@ -0,0 +1,13 @@
{
services.open-webui = {
enable = true;
host = "0.0.0.0";
environment = {
ANONYMIZED_TELEMETRY = "False";
DO_NOT_TRACK = "True";
SCARF_NO_ANALYTICS = "True";
OLLAMA_API_BASE_URL = "https://ollama.com";
WEBUI_AUTH = "False";
};
};
}
+1 -1
View File
@@ -51,7 +51,7 @@ def test_config_defaults_enable_reranking(mocker: MockerFixture) -> None:
config = load_rerank_config()
assert config.enabled is True
assert config.base_url == "http://bob:8001"
assert config.base_url == "http://192.168.90.25:8001"
assert config.model == "qwen3-reranker-06b"
assert config.candidates == 24
assert config.timeout_seconds == 30
+104
View File
@@ -0,0 +1,104 @@
"""test_components."""
from pytest_mock import MockerFixture
from python.system_tests.components import systemd_tests, zpool_tests
from python.zfs import Zpool
temp = "Every feature flags pool has all supported and requested features enabled.\n"
SYSTEM_TESTS_COMPONENTS = "python.system_tests.components"
def test_zpool_tests(mocker: MockerFixture) -> None:
"""test_zpool_tests."""
mock_zpool = mocker.MagicMock(spec=Zpool)
mock_zpool.health = "ONLINE"
mock_zpool.capacity = 70
mock_zpool.name = "Main"
mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=(temp, ""))
errors = zpool_tests(("Main",))
assert errors == []
def test_zpool_tests_out_of_date(mocker: MockerFixture) -> None:
"""test_zpool_tests_out_of_date."""
mock_zpool = mocker.MagicMock(spec=Zpool)
mock_zpool.health = "ONLINE"
mock_zpool.capacity = 70
mock_zpool.name = "Main"
mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=("", ""))
errors = zpool_tests(("Main",))
assert errors == ["ZPool out of date run `sudo zpool upgrade -a`"]
def test_zpool_tests_out_of_space(mocker: MockerFixture) -> None:
"""test_zpool_tests_out_of_space."""
mock_zpool = mocker.MagicMock(spec=Zpool)
mock_zpool.health = "ONLINE"
mock_zpool.capacity = 100
mock_zpool.name = "Main"
mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=(temp, ""))
errors = zpool_tests(("Main",))
assert errors == ["Main is low on space"]
def test_zpool_tests_offline(mocker: MockerFixture) -> None:
"""test_zpool_tests_offline."""
mock_zpool = mocker.MagicMock(spec=Zpool)
mock_zpool.health = "OFFLINE"
mock_zpool.capacity = 70
mock_zpool.name = "Main"
mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=(temp, ""))
errors = zpool_tests(("Main",))
assert errors == ["Main is OFFLINE"]
def test_systemd_tests(mocker: MockerFixture) -> None:
"""test_systemd_tests."""
mocker.patch(
f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper",
side_effect=[
("inactive\n", ""),
("active\n", ""),
],
)
errors = systemd_tests(("docker",))
assert errors == []
"""test_systemd_tests."""
def test_systemd_tests_multiple_negative_retries(mocker: MockerFixture) -> None:
"""test_systemd_tests_fail."""
mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=("active\n", ""))
errors = systemd_tests(("docker",), max_retries=-1, retry_delay_secs=-1)
assert errors == []
def test_systemd_tests_multiple_pass(mocker: MockerFixture) -> None:
"""test_systemd_tests_fail."""
mocker.patch(
f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper",
side_effect=[
("inactive\n", ""),
("activating\n", ""),
("active\n", ""),
],
)
errors = systemd_tests(
("docker",),
retryable_statuses=("inactive\n", "activating\n"),
valid_statuses=("active\n",),
)
assert errors == []
def test_systemd_tests_fail(mocker: MockerFixture) -> None:
"""test_systemd_tests_fail."""
mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=("inactive\n", ""))
errors = systemd_tests(("docker",), max_retries=5)
assert errors == ["docker is inactive"]

Some files were not shown because too many files have changed in this diff Show More