feat(monitoring): add Portal-1 and Vultr dashboards

- scrape Portal-1 node metrics over Tailscale
- add a hardened Vultr billing and bandwidth exporter
- provision host health and infrastructure cost dashboards

.sops.yaml

@@ -1,6 +1,9 @@
 # Generate AGE keys from SSH keys with:
 #   ssh-keygen -A
 #   nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
+# update keys after addin/removing a key
+#   nix-shell -p sops --run "sops updatekeys users/secrets.yaml" users/secrets.yaml
+
 keys:
   - &admin_richie age1u8zj599elqqvcmhxn8zuwrufsz8w8w366d3ayrljjejljt2q45kq8mxw9c # cspell:disable-line
 
@@ -8,6 +11,7 @@ keys:
   - &system_brain age1jhf7vm0005j60mjq63696frrmjhpy8kpc2d66mw044lqap5mjv4snmwvwm # cspell:disable-line
   - &system_jeeves age13lmqgc3jvkyah5e3vcwmj4s5wsc2akctcga0lpc0x8v8du3fxprqp4ldkv # cspell:disable-line
   - &system_rhapsody age1ufnewppysaq2wwcl4ugngjz8pfzc5a35yg7luq0qmuqvctajcycs5lf6k4 # cspell:disable-line
+  - &system_portal_1 age1vyav6kxtvt3z4vtnkkjj38eu8hlts5m7ygyckhskvalg2gpjk52su53d0a # cspell:disable-line
 
 creation_rules:
   - path_regex: users/secrets\.yaml$
@@ -18,3 +22,4 @@ creation_rules:
           - *system_brain
           - *system_jeeves
           - *system_rhapsody
+          - *system_portal_1

Cargo.toml

@@ -0,0 +1,4 @@
+[workspace]
+resolver = "2"
+
+members = ["rust/*"]

common/global/default.nix

@@ -17,16 +17,11 @@
     ./nix.nix
     ./programs.nix
     ./ssh.nix
-    ./snapshot_manager.nix
   ];
 
   boot = {
-    tmp.useTmpfs = true;
+    tmp.useTmpfs = lib.mkDefault true;
     kernelPackages = lib.mkDefault pkgs.linuxPackages_6_12;
-    zfs = {
-      package = lib.mkDefault pkgs.zfs_2_4;
-      forceImportRoot = lib.mkDefault false;
-    };
   };
 
   hardware.enableRedistributableFirmware = true;
@@ -42,9 +37,6 @@
     overlays = builtins.attrValues outputs.overlays;
     config = {
       allowUnfree = true;
-      permittedInsecurePackages = [
-        "openssl-1.1.1w" # This is for discord-canary
-      ];
     };
   };
 
@@ -53,16 +45,6 @@
 
     # firmware update
     fwupd.enable = true;
-
-    snapshot_manager = {
-      enable = lib.mkDefault true;
-      PYTHONPATH = "${inputs.self}/";
-    };
-
-    zfs = {
-      trim.enable = lib.mkDefault true;
-      autoScrub.enable = lib.mkDefault true;
-    };
   };
 
   powerManagement.powertop.enable = lib.mkDefault true;

common/optional/monitoring-agent.nix

@@ -5,7 +5,7 @@
   ...
 }:
 let
-  monitoringInterface = "ztwfunumly";
+  monitoringInterface = "tailscale0";
   nodeTextfileDir = "/var/lib/prometheus-node-exporter-textfile";
 
   mkProcessNameTemplate =

common/optional/tailscale.nix

@@ -0,0 +1,37 @@
+{
+  config,
+  inputs,
+  ...
+}:
+{
+  nix.settings = {
+    trusted-substituters = [ "http://jeeves:5000" ];
+    substituters = [ "http://jeeves:5000/?priority=1&want-mass-query=true" ];
+    trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
+  };
+
+  services.tailscale = {
+    enable = true;
+    openFirewall = true;
+    authKeyFile = config.sops.secrets.tailscale_auth_key.path;
+
+    # OAuth client secrets create ephemeral nodes by default. NixOS machines
+    # are persistent and should enroll without interactive device approval.
+    authKeyParameters = {
+      ephemeral = false;
+      preauthorized = true;
+    };
+
+    extraUpFlags = [ "--advertise-tags=tag:nixos" ];
+  };
+
+  sops = {
+    age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
+
+    secrets.tailscale_auth_key = {
+      sopsFile = "${inputs.self}/users/secrets.yaml";
+      owner = "root";
+      mode = "0400";
+    };
+  };
+}

common/optional/zerotier.nix

@@ -1,11 +0,0 @@
-{
-  services.zerotierone = {
-    enable = true;
-    joinNetworks = [ "e4da7455b2ae64ca" ];
-  };
-  nix.settings = {
-    trusted-substituters = [ "http://192.168.90.40:5000" ];
-    substituters = [ "http://192.168.90.40:5000/?priority=1&want-mass-query=true" ];
-    trusted-public-keys = [ "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" ];
-  };
-}

common/optional/zfs/default.nix

@@ -0,0 +1,26 @@
+{
+  inputs,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  imports = [ ./snapshot.nix ];
+
+  boot.zfs = {
+    package = lib.mkDefault pkgs.zfs_2_4;
+    forceImportRoot = lib.mkDefault false;
+  };
+
+  services = {
+    snapshot_manager = {
+      enable = lib.mkDefault true;
+      PYTHONPATH = "${inputs.self}/";
+    };
+
+    zfs = {
+      trim.enable = lib.mkDefault true;
+      autoScrub.enable = lib.mkDefault true;
+    };
+  };
+}

flake.lock

@@ -1,5 +1,25 @@
 {
   "nodes": {
+    "disko": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1781152676,
+        "narHash": "sha256-RxWs5ND31KzTG7wvMM+PMfUjyNpmIEr999lqNARaM5o=",
+        "owner": "nix-community",
+        "repo": "disko",
+        "rev": "ff8702b4de27f72b4c78573dfb89ec74e36abdf1",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "disko",
+        "type": "github"
+      }
+    },
     "firefox-addons": {
       "inputs": {
         "nixpkgs": [
@@ -8,11 +28,11 @@
       },
       "locked": {
         "dir": "pkgs/firefox-addons",
-        "lastModified": 1784963334,
-        "narHash": "sha256-Xbj8fFapX+ufz8kx0sH33RAvF0Pauzfg5oihn38SWtE=",
+        "lastModified": 1787025780,
+        "narHash": "sha256-NhyLP9G4DFOn/7aYr7K/D7hWrzEGr5EgUBV+lpdmJ24=",
         "owner": "rycee",
         "repo": "nur-expressions",
-        "rev": "557935d63e75f4bfbd61ca3cede3a632edbed7e5",
+        "rev": "5ad360b6d3cb0aa1b61f9cb27fef113ca9117c37",
         "type": "gitlab"
       },
       "original": {
@@ -29,11 +49,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1784913159,
-        "narHash": "sha256-JWq0BfjO4ktpH5USfQNQzdvHpIDT8fSKD5K7LvdMRFs=",
+        "lastModified": 1786999651,
+        "narHash": "sha256-MTGMFlLDTklsXhCp4r5GXB4VAVadPdalXLvUjd/K7h0=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "079a3b5d1aa6a719920a51316253b7d6dd22738d",
+        "rev": "353742587cbaf079b3caee743115d037bc51fea6",
         "type": "github"
       },
       "original": {
@@ -47,11 +67,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1784723954,
-        "narHash": "sha256-1CfD8ZUjCkTgjsneLZ/lxCHhgDfqxxE7/GX0MmsgiqA=",
+        "lastModified": 1786867632,
+        "narHash": "sha256-ez+ubZlA1RtdjCB18a6zJ9M4u8qoPDy08EcnsW5M3Xw=",
         "owner": "nixos",
         "repo": "nixos-hardware",
-        "rev": "a017f5b72210026af5b3ac5949f08d94380a6fbd",
+        "rev": "ff17823245ab9ff7bcae6acf950bd89cba82c38c",
         "type": "github"
       },
       "original": {
@@ -76,11 +96,11 @@
     },
     "nixpkgs-master": {
       "locked": {
-        "lastModified": 1785001027,
-        "narHash": "sha256-Cc+7WLCh3sss1d+4DDLuEma4a2v3YdXGez/N+MJoomY=",
+        "lastModified": 1787081018,
+        "narHash": "sha256-K0uwZBtZsbBigHAMQW7YWti3gPe6a5ct5bcOw5F+Q9Y=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "b2a8a65ad069640cf1a0656b355338a65c41ac12",
+        "rev": "cacac5ac351a010599d9f9d106acfed25a8e4c77",
         "type": "github"
       },
       "original": {
@@ -108,11 +128,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1784796856,
-        "narHash": "sha256-wWFrV5/Qbm+lyt5x20E/bSbfJiGKMo4RCxZV8cl/WZI=",
+        "lastModified": 1787001381,
+        "narHash": "sha256-Ue1Yo8gfHdD4TMtNewhA4tkSYeFqXThju0nCyJc3ALo=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "e2587caef70cea85dd97d7daab492899902dbf5d",
+        "rev": "ec2d622de0773551768cf98f3fc50cbcc003b9c5",
         "type": "github"
       },
       "original": {
@@ -124,6 +144,7 @@
     },
     "root": {
       "inputs": {
+        "disko": "disko",
         "firefox-addons": "firefox-addons",
         "home-manager": "home-manager",
         "nixos-hardware": "nixos-hardware",
@@ -141,11 +162,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1783174389,
-        "narHash": "sha256-aCWC8ngycU7OdJrU2+Je3qf+1a2ykuBvpPhZT/9tXMc=",
+        "lastModified": 1786629091,
+        "narHash": "sha256-gkig4nPi1CWc4Z50GBsjE4ygSE7hMpl/TwID2an2Cck=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "f1406619a3884cd5c47992a70b8b35c9c0fcb4c9",
+        "rev": "a8627b21b9107c5711c96b84f32a9a4b3d45295f",
         "type": "github"
       },
       "original": {

flake.nix

@@ -35,6 +35,11 @@
       url = "github:Mic92/sops-nix";
       inputs.nixpkgs.follows = "nixpkgs";
     };
+
+    disko = {
+      url = "github:nix-community/disko";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
   };
 
   outputs =

pyproject.toml

@@ -65,6 +65,7 @@ lint.ignore = [
     "ISC001", # (TEMP) conflicts when used with the formatter
     "S603",   # (PERM) This is known to cause a false positive
     "S607",   # (PERM) This is becoming a consistent annoyance
+    "CPY001",  # (PERM) I don't include the license in every file
 ]
 
 [tool.ruff.lint.per-file-ignores]

python/ebook_search/config.py

@@ -37,7 +37,7 @@ class RerankConfig(BaseSettings):
     model_config = SettingsConfigDict(env_prefix="EBOOK_SEARCH_RERANK_", frozen=True, protected_namespaces=())
 
     enabled: bool = True
-    base_url: str = "http://192.168.90.25:8001"
+    base_url: str = "http://bob:8001"
     model: str = "qwen3-reranker-06b"
     candidates: int = 24
     timeout_seconds: float = 30.0
@@ -67,7 +67,7 @@ class EbookSearchConfig(BaseSettings):
     )
     chat_model: str = "deepseek-v4-flash"
     answer_enabled: bool = True
-    embedding_base_url: str = "http://192.168.90.25:8000/v1"
+    embedding_base_url: str = "http://bob:8000/v1"
     embedding_api_key: str = "not-needed"
     embedding_model: str = "qwen3-embedding-0.6b"
     embedding_batch_size: int = 32

python/ebook_search/docker/docker-compose.yml

@@ -9,8 +9,6 @@ services:
     restart: unless-stopped
     ports:
       - "${EBOOK_SEARCH_PORT:-8070}:8070"
-    extra_hosts:
-      - "jeeves:192.168.90.40"
     env_file:
       - ../../../.env
     environment:

python/zfs/__init__.py

@@ -1,11 +1,30 @@
 """init."""
 
-from python.zfs.dataset import Dataset, Snapshot, get_datasets
+# run_command is deliberately not re-exported here. It is available from
+# python.zfs.command when something genuinely needs another binary, but the
+# wrappers are what callers should reach for by default.
+from python.zfs.command import CommandResult, run_zfs, run_zpool
+from python.zfs.dataset import (
+    Dataset,
+    Snapshot,
+    create_dataset,
+    get_datasets,
+    get_properties,
+    list_dataset_names,
+    set_property,
+)
 from python.zfs.zpool import Zpool
 
 __all__ = [
+    "CommandResult",
     "Dataset",
     "Snapshot",
     "Zpool",
+    "create_dataset",
     "get_datasets",
+    "get_properties",
+    "list_dataset_names",
+    "run_zfs",
+    "run_zpool",
+    "set_property",
 ]

python/zfs/command.py

@@ -0,0 +1,90 @@
+"""Running zfs and zpool commands.
+
+One implementation shared by both, so the zpool side gets the same handling the
+zfs side does: arguments passed as a list, streams kept apart, and failures
+returned as data rather than guessed at by the caller.
+"""
+
+from __future__ import annotations
+
+import logging
+import subprocess
+from dataclasses import dataclass
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class CommandResult:
+    """The outcome of a single zfs or zpool invocation."""
+
+    args: tuple[str, ...]
+    stdout: str
+    stderr: str
+    return_code: int
+
+    @property
+    def ok(self) -> bool:
+        """Whether the command reported success."""
+        return self.return_code == 0
+
+    @property
+    def message(self) -> str:
+        """The most useful description of what went wrong."""
+        return (self.stderr or self.stdout).strip()
+
+
+def run_command(*args: str) -> CommandResult:
+    """Run a command, passing arguments as a list rather than a shell string.
+
+    Two things this buys over bash_wrapper. Arguments are never split on
+    whitespace, so a value containing a space arrives intact. And stdout stays
+    separate from stderr, so a warning on a successful command is never
+    mistaken for output, which bash_wrapper does whenever stderr is non-empty
+    regardless of the return code.
+
+    The encoding is pinned rather than using text=True, which would decode with
+    the locale encoding. These run from systemd units, where LANG is often
+    unset.
+
+    Args:
+        *args: The command and its arguments.
+
+    Returns:
+        CommandResult: The streams and return code, never raising on failure.
+    """
+    completed = subprocess.run(list(args), capture_output=True, encoding="utf-8", check=False)
+
+    if completed.returncode != 0:
+        logger.debug(f"{' '.join(args)} exited {completed.returncode}: {completed.stderr.strip()}")
+
+    return CommandResult(
+        args=tuple(args),
+        stdout=completed.stdout,
+        stderr=completed.stderr,
+        return_code=completed.returncode,
+    )
+
+
+def run_zfs(*args: str) -> CommandResult:
+    """Run a zfs command.
+
+    Args:
+        *args: The arguments to pass to zfs.
+
+    Returns:
+        CommandResult: The streams and return code.
+    """
+    return run_command("zfs", *args)
+
+
+def run_zpool(*args: str) -> CommandResult:
+    """Run a zpool command.
+
+    Args:
+        *args: The arguments to pass to zpool.
+
+    Returns:
+        CommandResult: The streams and return code.
+    """
+    return run_command("zpool", *args)

python/zfs/dataset.py

@@ -8,6 +8,7 @@ from datetime import UTC, datetime
 from typing import Any
 
 from python.common import bash_wrapper
+from python.zfs.command import run_zfs
 
 logger = logging.getLogger(__name__)
 
@@ -207,8 +208,91 @@ def get_datasets() -> list[Dataset]:
     """
     logger.info("Getting zfs list")
 
-    dataset_names, _ = bash_wrapper("zfs list -Hp -t filesystem -o name")
+    return [Dataset(dataset_name) for dataset_name in list_dataset_names() if "/" in dataset_name]
 
-    cleaned_datasets = dataset_names.strip().split("\n")
 
-    return [Dataset(dataset_name) for dataset_name in cleaned_datasets if "/" in dataset_name]
+def get_properties(name: str) -> dict[str, tuple[str, str]]:
+    """Get every property of a dataset along with where its value came from.
+
+    The source is what distinguishes a property that was deliberately set on
+    this dataset from one that is merely inherited or left at its default.
+
+    Args:
+        name (str): The name of the dataset.
+
+    Returns:
+        dict[str, tuple[str, str]]: A mapping of property name to (value, source).
+    """
+    result = run_zfs("get", "-Hp", "-o", "property,value,source", "all", name)
+    if not result.ok:
+        error = f"Failed to get properties for {name}: {result.message}"
+        raise RuntimeError(error)
+
+    properties = {}
+    for line in result.stdout.strip().splitlines():
+        if not line:
+            continue
+        prop, value, source = line.split("\t")
+        properties[prop] = (value, source)
+
+    return properties
+
+
+def create_dataset(name: str, properties: dict[str, str]) -> str | None:
+    """Create a dataset with the given properties.
+
+    Args:
+        name (str): The name of the dataset.
+        properties (dict[str, str]): The properties to create the dataset with.
+
+    Returns:
+        str | None: An error message on failure, None on success.
+    """
+    options = [argument for key, value in sorted(properties.items()) for argument in ("-o", f"{key}={value}")]
+    logger.debug(f"creating {name} with {properties}")
+
+    result = run_zfs("create", *options, name)
+    if not result.ok:
+        return f"Failed to create {name}: {result.message}"
+    return None
+
+
+def set_property(name: str, key: str, value: str) -> str | None:
+    """Set a single property on a dataset.
+
+    Args:
+        name (str): The name of the dataset.
+        key (str): The property to set.
+        value (str): The value to set the property to.
+
+    Returns:
+        str | None: An error message on failure, None on success.
+    """
+    logger.debug(f"setting {key}={value} on {name}")
+
+    result = run_zfs("set", f"{key}={value}", name)
+    if not result.ok:
+        return f"Failed to set {key}={value} on {name}: {result.message}"
+    return None
+
+
+def list_dataset_names() -> list[str]:
+    """List every zfs filesystem name, including pool root filesystems.
+
+    Unlike get_datasets this does not build Dataset objects and does not filter
+    out pool roots, which makes it usable for existence checks.
+
+    Returns:
+        list[str]: The names of every zfs filesystem.
+
+    Raises:
+        RuntimeError: If zfs list fails. Never returns a partial or error
+            derived list, since treating stderr as dataset names would make
+            the reconciler think every dataset is missing.
+    """
+    result = run_zfs("list", "-Hp", "-t", "filesystem", "-o", "name")
+    if not result.ok:
+        error = f"Failed to list ZFS datasets: {result.message}"
+        raise RuntimeError(error)
+
+    return [name for name in result.stdout.strip().splitlines() if name]

python/zfs/zpool.py

@@ -1,28 +1,42 @@
-"""test."""
+"""zpool."""
 
 from __future__ import annotations
 
 import json
 from typing import Any
 
-from python.common import bash_wrapper
+from python.zfs.command import run_zpool
 
 
-def _zpool_list(zfs_list: str) -> dict[str, Any]:
-    """Check the version of zfs."""
-    raw_zfs_list_data, _ = bash_wrapper(zfs_list)
+def _zpool_list(*args: str) -> dict[str, Any]:
+    """Run a zpool list and check the output is a format we understand.
 
-    zfs_list_data = json.loads(raw_zfs_list_data)
+    Args:
+        *args: The arguments to pass to zpool.
 
-    vers_major = zfs_list_data["output_version"]["vers_major"]
-    vers_minor = zfs_list_data["output_version"]["vers_minor"]
-    command = zfs_list_data["output_version"]["command"]
+    Returns:
+        dict[str, Any]: The decoded output.
+
+    Raises:
+        RuntimeError: If zpool fails, or reports a format this does not parse.
+            Never decodes a partial or error derived payload.
+    """
+    result = run_zpool(*args)
+    if not result.ok:
+        error = f"Failed to run zpool {' '.join(args)}: {result.message}"
+        raise RuntimeError(error)
+
+    zpool_data = json.loads(result.stdout)
+
+    vers_major = zpool_data["output_version"]["vers_major"]
+    vers_minor = zpool_data["output_version"]["vers_minor"]
+    command = zpool_data["output_version"]["command"]
 
     if vers_major != 0 or vers_minor != 1 or command != "zpool list":
         error = f"Datasets are not in the correct format {vers_major=} {vers_minor=} {command=}"
         raise RuntimeError(error)
 
-    return zfs_list_data
+    return zpool_data
 
 
 class Zpool:
@@ -33,7 +47,7 @@ class Zpool:
         name: str,
     ) -> None:
         """__init__."""
-        zpool_data = _zpool_list(f"zpool list {name} -pHj -o all")
+        zpool_data = _zpool_list("list", name, "-pHj", "-o", "all")
 
         properties = zpool_data["pools"][name]["properties"]
 

rust/van_weather/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "van-weather"
+version = "0.1.0"
+edition = "2024"
+description = "Fetch privacy-masked weather for a van and publish it to Home Assistant"
+license = "MIT"
+
+[dependencies]
+anyhow = "1.0"
+chrono = "0.4"
+clap = { version = "4.5", features = ["derive", "env"] }
+env_logger = "0.11"
+log = "0.4"
+reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+
+[lints.clippy]
+all = "deny"
+pedantic = "deny"
+

rust/van_weather/package.nix

@@ -0,0 +1,13 @@
+{ rustPlatform }:
+rustPlatform.buildRustPackage {
+  pname = "van-weather";
+  version = "0.1.0";
+  src = ./.;
+
+  cargoLock.lockFile = ./Cargo.lock;
+
+  meta = {
+    description = "Privacy-masked van weather publisher for Home Assistant";
+    mainProgram = "van-weather";
+  };
+}

rust/van_weather/src/main.rs

@@ -0,0 +1,565 @@
+use std::{thread, time::Duration};
+
+use anyhow::{Context, Result, bail};
+use chrono::{DateTime, Utc};
+use clap::Parser;
+use log::{error, info};
+use reqwest::{
+    StatusCode, Url,
+    blocking::{Client, ClientBuilder},
+    header::{AUTHORIZATION, HeaderMap, HeaderValue},
+    retry,
+};
+use serde::Deserialize;
+use serde_json::{Value, json};
+
+const LAT_ENTITY: &str = "sensor.van_last_known_latitude";
+const LON_ENTITY: &str = "sensor.van_last_known_longitude";
+const PIRATE_WEATHER_HOST: &str = "api.pirateweather.net";
+const MASK_DECIMALS: u32 = 1;
+const MASK_FACTOR: f64 = decimal_factor(MASK_DECIMALS);
+const RETRIES_PER_REQUEST: u32 = 2;
+
+const fn decimal_factor(decimals: u32) -> f64 {
+    let mut factor = 1.0;
+    let mut remaining = decimals;
+    while remaining > 0 {
+        factor *= 10.0;
+        remaining -= 1;
+    }
+    factor
+}
+
+#[derive(Debug, Parser)]
+#[command(about, version)]
+struct Args {
+    #[arg(long, env = "HA_URL")]
+    ha_url: String,
+
+    #[arg(long, env = "HA_TOKEN", hide_env_values = true)]
+    ha_token: String,
+
+    #[arg(long, env = "PIRATE_WEATHER_API_KEY", hide_env_values = true)]
+    pirate_weather_api_key: String,
+
+    #[arg(
+        long,
+        default_value_t = 900,
+        value_parser = clap::value_parser!(u64).range(1..)
+    )]
+    interval: u64,
+
+    #[arg(long, default_value = "info", env = "RUST_LOG")]
+    log_level: String,
+}
+
+#[derive(Debug, Deserialize)]
+struct HaState {
+    state: String,
+}
+
+#[derive(Debug)]
+struct HttpClients {
+    home_assistant: Client,
+    pirate_weather: Client,
+}
+
+#[derive(Debug, Default, Deserialize)]
+struct ApiResponse {
+    #[serde(default)]
+    currently: CurrentWeather,
+    #[serde(default)]
+    daily: ForecastBlock<DailyApiForecast>,
+    #[serde(default)]
+    hourly: ForecastBlock<HourlyApiForecast>,
+}
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct CurrentWeather {
+    temperature: Option<f64>,
+    apparent_temperature: Option<f64>,
+    humidity: Option<f64>,
+    wind_speed: Option<f64>,
+    wind_bearing: Option<f64>,
+    icon: Option<String>,
+    pressure: Option<f64>,
+    visibility: Option<f64>,
+    uv_index: Option<f64>,
+    ozone: Option<f64>,
+    nearest_storm_distance: Option<f64>,
+    nearest_storm_bearing: Option<f64>,
+    precip_probability: Option<f64>,
+    cloud_cover: Option<f64>,
+}
+
+#[derive(Debug, Deserialize)]
+struct ForecastBlock<T> {
+    #[serde(default)]
+    data: Vec<T>,
+}
+
+impl<T> Default for ForecastBlock<T> {
+    fn default() -> Self {
+        Self { data: Vec::new() }
+    }
+}
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct DailyApiForecast {
+    time: Option<i64>,
+    icon: Option<String>,
+    temperature_high: Option<f64>,
+    temperature_low: Option<f64>,
+    precip_probability: Option<f64>,
+}
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct HourlyApiForecast {
+    time: Option<i64>,
+    icon: Option<String>,
+    temperature: Option<f64>,
+    precip_probability: Option<f64>,
+}
+
+#[derive(Debug)]
+struct Weather {
+    current: CurrentWeather,
+    daily: Vec<DailyForecast>,
+    hourly: Vec<HourlyForecast>,
+}
+
+#[derive(Debug)]
+struct DailyForecast {
+    datetime: DateTime<Utc>,
+    condition: &'static str,
+    temperature: Option<f64>,
+    templow: Option<f64>,
+    precipitation_probability: Option<f64>,
+}
+
+#[derive(Debug)]
+struct HourlyForecast {
+    datetime: DateTime<Utc>,
+    condition: &'static str,
+    temperature: Option<f64>,
+    precipitation_probability: Option<f64>,
+}
+
+fn main() -> Result<()> {
+    let Args {
+        ha_url,
+        ha_token,
+        pirate_weather_api_key,
+        interval,
+        log_level,
+    } = Args::parse();
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or(log_level.as_str()))
+        .init();
+
+    let ha_url = ha_url.trim_end_matches('/').to_owned();
+    let ha_host = Url::parse(&ha_url)
+        .context("HA_URL is not a valid URL")?
+        .host_str()
+        .context("HA_URL has no host")?
+        .to_owned();
+    let clients = HttpClients {
+        home_assistant: build_client(&ha_host, Some(&ha_token))?,
+        pirate_weather: build_client(PIRATE_WEATHER_HOST, None)?,
+    };
+
+    info!("Starting van weather service, polling every {interval}s");
+    loop {
+        if let Err(err) = update_weather(&clients, &ha_url, &pirate_weather_api_key) {
+            error!("Weather update failed: {err:#}");
+        }
+        thread::sleep(Duration::from_secs(interval));
+    }
+}
+
+fn build_client(host: &str, bearer_token: Option<&str>) -> Result<Client> {
+    let policy = retry::for_host(host.to_owned())
+        .max_retries_per_request(RETRIES_PER_REQUEST)
+        .classify_fn(|request| {
+            let retryable = request.error().is_some()
+                || request.status().is_some_and(|status| {
+                    status == StatusCode::REQUEST_TIMEOUT
+                        || status == StatusCode::TOO_MANY_REQUESTS
+                        || status.is_server_error()
+                });
+            if retryable {
+                request.retryable()
+            } else {
+                request.success()
+            }
+        });
+
+    let mut builder = ClientBuilder::new()
+        .timeout(Duration::from_secs(30))
+        .retry(policy);
+
+    if let Some(token) = bearer_token {
+        let mut authorization = HeaderValue::from_str(&format!("Bearer {token}"))
+            .context("HA_TOKEN contains invalid header characters")?;
+        authorization.set_sensitive(true);
+        let mut headers = HeaderMap::new();
+        headers.insert(AUTHORIZATION, authorization);
+        builder = builder.default_headers(headers);
+    }
+
+    builder
+        .build()
+        .with_context(|| format!("failed to create HTTP client for {host}"))
+}
+
+fn update_weather(clients: &HttpClients, ha_url: &str, api_key: &str) -> Result<()> {
+    let lat = get_ha_state(&clients.home_assistant, ha_url, LAT_ENTITY)?;
+    let lon = get_ha_state(&clients.home_assistant, ha_url, LON_ENTITY)?;
+    let masked_lat = mask_coordinate(lat);
+    let masked_lon = mask_coordinate(lon);
+    info!("Masked location: {masked_lat}, {masked_lon}");
+
+    let weather = fetch_weather(&clients.pirate_weather, api_key, masked_lat, masked_lon)?;
+    info!(
+        "Weather: {}°F, {}",
+        weather
+            .current
+            .temperature
+            .map_or_else(|| "unknown".to_owned(), |value| value.to_string()),
+        condition(weather.current.icon.as_deref())
+    );
+    post_to_ha(&clients.home_assistant, ha_url, &weather)?;
+    info!("Posted weather to Home Assistant");
+    Ok(())
+}
+
+fn mask_coordinate(value: f64) -> f64 {
+    (value * MASK_FACTOR).round() / MASK_FACTOR
+}
+
+fn get_ha_state(client: &Client, ha_url: &str, entity_id: &str) -> Result<f64> {
+    let HaState { state } = client
+        .get(format!("{ha_url}/api/states/{entity_id}"))
+        .send()
+        .with_context(|| format!("request for {entity_id} failed"))?
+        .error_for_status()
+        .with_context(|| format!("Home Assistant rejected {entity_id} request"))?
+        .json()
+        .context("Home Assistant returned invalid JSON")?;
+
+    if matches!(state.as_str(), "unavailable" | "unknown") {
+        bail!("{entity_id} is {state}");
+    }
+
+    state
+        .parse::<f64>()
+        .with_context(|| format!("{entity_id} state is not numeric: {state}"))
+}
+
+fn fetch_weather(client: &Client, api_key: &str, lat: f64, lon: f64) -> Result<Weather> {
+    let response = client
+        .get(format!(
+            "https://{PIRATE_WEATHER_HOST}/forecast/{api_key}/{lat},{lon}"
+        ))
+        .query(&[("units", "us")])
+        .send()
+        .context("Pirate Weather request failed")?
+        .error_for_status()
+        .context("Pirate Weather rejected request")?;
+    let data = response
+        .json::<ApiResponse>()
+        .context("Pirate Weather returned invalid JSON")?;
+    Ok(parse_weather(data))
+}
+
+fn parse_weather(data: ApiResponse) -> Weather {
+    let daily = data
+        .daily
+        .data
+        .into_iter()
+        .take(8)
+        .filter_map(|day| {
+            timestamp(day.time).map(|datetime| DailyForecast {
+                datetime,
+                condition: condition(day.icon.as_deref()),
+                temperature: day.temperature_high,
+                templow: day.temperature_low,
+                precipitation_probability: day.precip_probability,
+            })
+        })
+        .collect();
+    let hourly = data
+        .hourly
+        .data
+        .into_iter()
+        .take(48)
+        .filter_map(|hour| {
+            timestamp(hour.time).map(|datetime| HourlyForecast {
+                datetime,
+                condition: condition(hour.icon.as_deref()),
+                temperature: hour.temperature,
+                precipitation_probability: hour.precip_probability,
+            })
+        })
+        .collect();
+    Weather {
+        current: data.currently,
+        daily,
+        hourly,
+    }
+}
+
+fn timestamp(value: Option<i64>) -> Option<DateTime<Utc>> {
+    value
+        .filter(|value| *value != 0)
+        .and_then(DateTime::from_timestamp_secs)
+}
+
+fn condition(icon: Option<&str>) -> &'static str {
+    match icon.unwrap_or_default() {
+        "clear-day" => "sunny",
+        "clear-night" => "clear-night",
+        "rain" => "rainy",
+        "snow" => "snowy",
+        "sleet" => "snowy-rainy",
+        "wind" => "windy",
+        "fog" => "fog",
+        "partly-cloudy-day" | "partly-cloudy-night" => "partlycloudy",
+        _ => "cloudy",
+    }
+}
+
+fn post_to_ha(client: &Client, ha_url: &str, weather: &Weather) -> Result<()> {
+    for (entity_id, payload) in weather_updates(weather) {
+        let response = client
+            .post(format!("{ha_url}/api/states/{entity_id}"))
+            .json(&payload)
+            .send()
+            .with_context(|| format!("failed to post {entity_id}"))?;
+        ensure_success(response.status(), &entity_id)?;
+    }
+    Ok(())
+}
+
+fn ensure_success(status: StatusCode, entity_id: &str) -> Result<()> {
+    if status.is_success() {
+        Ok(())
+    } else {
+        bail!("Home Assistant rejected {entity_id} update with {status}")
+    }
+}
+
+#[allow(clippy::too_many_lines)]
+fn weather_updates(weather: &Weather) -> Vec<(String, Value)> {
+    let current = &weather.current;
+    let mut updates = vec![
+        sensor(
+            "sensor.van_weather_condition",
+            Some(json!(condition(current.icon.as_deref()))),
+            json!({"friendly_name": "Van Weather Condition"}),
+        ),
+        sensor(
+            "sensor.van_weather_temperature",
+            current.temperature.map(|value| json!(value)),
+            json!({"unit_of_measurement": "°F", "device_class": "temperature"}),
+        ),
+        sensor(
+            "sensor.van_weather_apparent_temperature",
+            current.apparent_temperature.map(|value| json!(value)),
+            json!({"unit_of_measurement": "°F", "device_class": "temperature"}),
+        ),
+        sensor(
+            "sensor.van_weather_humidity",
+            Some(json!(percent(current.humidity))),
+            json!({"unit_of_measurement": "%", "device_class": "humidity"}),
+        ),
+        sensor(
+            "sensor.van_weather_pressure",
+            current.pressure.map(|value| json!(value)),
+            json!({"unit_of_measurement": "mbar", "device_class": "pressure"}),
+        ),
+        sensor(
+            "sensor.van_weather_wind_speed",
+            current.wind_speed.map(|value| json!(value)),
+            json!({"unit_of_measurement": "mph", "device_class": "wind_speed"}),
+        ),
+        sensor(
+            "sensor.van_weather_wind_bearing",
+            current.wind_bearing.map(|value| json!(value)),
+            json!({"unit_of_measurement": "°"}),
+        ),
+        sensor(
+            "sensor.van_weather_visibility",
+            current.visibility.map(|value| json!(value)),
+            json!({"unit_of_measurement": "mi"}),
+        ),
+        sensor(
+            "sensor.van_weather_uv_index",
+            current.uv_index.map(|value| json!(value)),
+            json!({"friendly_name": "Van Weather UV Index", "icon": "mdi:sun-wireless"}),
+        ),
+        sensor(
+            "sensor.van_weather_ozone",
+            current.ozone.map(|value| json!(value)),
+            json!({"unit_of_measurement": "DU", "icon": "mdi:earth"}),
+        ),
+        sensor(
+            "sensor.van_weather_nearest_storm_distance",
+            current.nearest_storm_distance.map(|value| json!(value)),
+            json!({"unit_of_measurement": "mi", "icon": "mdi:weather-lightning"}),
+        ),
+        sensor(
+            "sensor.van_weather_nearest_storm_bearing",
+            current.nearest_storm_bearing.map(|value| json!(value)),
+            json!({"unit_of_measurement": "°", "icon": "mdi:weather-lightning"}),
+        ),
+        sensor(
+            "sensor.van_weather_precip_probability",
+            Some(json!(percent(current.precip_probability))),
+            json!({"unit_of_measurement": "%", "icon": "mdi:weather-rainy"}),
+        ),
+        sensor(
+            "sensor.van_weather_cloud_cover",
+            Some(json!(percent(current.cloud_cover))),
+            json!({"unit_of_measurement": "%", "icon": "mdi:weather-cloudy"}),
+        ),
+    ]
+    .into_iter()
+    .flatten()
+    .collect::<Vec<_>>();
+
+    let daily = weather
+        .daily
+        .iter()
+        .map(|forecast| {
+            json!({
+                "datetime": forecast.datetime.to_rfc3339(),
+                "condition": forecast.condition,
+                "temperature": forecast.temperature,
+                "templow": forecast.templow,
+                "precipitation_probability": percent(forecast.precipitation_probability),
+            })
+        })
+        .collect::<Vec<_>>();
+    updates.push((
+        "sensor.van_weather_forecast_daily".to_owned(),
+        json!({"state": daily.len(), "attributes": {"forecast": daily}}),
+    ));
+
+    let hourly = weather
+        .hourly
+        .iter()
+        .map(|forecast| {
+            json!({
+                "datetime": forecast.datetime.to_rfc3339(),
+                "condition": forecast.condition,
+                "temperature": forecast.temperature,
+                "precipitation_probability": percent(forecast.precipitation_probability),
+            })
+        })
+        .collect::<Vec<_>>();
+    updates.push((
+        "sensor.van_weather_forecast_hourly".to_owned(),
+        json!({"state": hourly.len(), "attributes": {"forecast": hourly}}),
+    ));
+    updates
+}
+
+fn sensor(entity_id: &str, state: Option<Value>, attributes: Value) -> Option<(String, Value)> {
+    state.map(|state| {
+        let mut payload = serde_json::Map::new();
+        payload.insert("state".to_owned(), state);
+        payload.insert("attributes".to_owned(), attributes);
+        (entity_id.to_owned(), Value::Object(payload))
+    })
+}
+
+#[allow(clippy::cast_possible_truncation)]
+fn percent(value: Option<f64>) -> i64 {
+    // Preserve Python's int(probability * 100) behavior for Home Assistant.
+    (value.unwrap_or_default() * 100.0) as i64
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn maps_conditions_for_home_assistant() {
+        assert_eq!(condition(Some("clear-day")), "sunny");
+        assert_eq!(condition(Some("sleet")), "snowy-rainy");
+        assert_eq!(condition(Some("partly-cloudy-night")), "partlycloudy");
+        assert_eq!(condition(Some("unexpected")), "cloudy");
+        assert_eq!(condition(None), "cloudy");
+    }
+
+    #[test]
+    fn parses_api_response_and_builds_compatible_payloads() {
+        let response: ApiResponse = serde_json::from_value(json!({
+            "currently": {
+                "temperature": 72.5,
+                "humidity": 0.67,
+                "icon": "clear-day",
+                "precipProbability": 0.129,
+                "cloudCover": 0.4,
+                "summary": "Fine"
+            },
+            "daily": {"data": [{
+                "time": 1_700_000_000,
+                "icon": "rain",
+                "temperatureHigh": 75.0,
+                "temperatureLow": 52.0,
+                "precipProbability": 0.8
+            }]},
+            "hourly": {"data": [{
+                "time": 1_700_000_000,
+                "icon": "fog",
+                "temperature": 61.0,
+                "precipProbability": 0.05
+            }]}
+        }))
+        .unwrap();
+
+        let weather = parse_weather(response);
+        let updates = weather_updates(&weather);
+        let find = |id: &str| updates.iter().find(|(entity, _)| entity == id).unwrap();
+
+        assert_eq!(find("sensor.van_weather_condition").1["state"], "sunny");
+        assert_eq!(find("sensor.van_weather_humidity").1["state"], 67);
+        assert_eq!(find("sensor.van_weather_precip_probability").1["state"], 12);
+        assert_eq!(find("sensor.van_weather_forecast_daily").1["state"], 1);
+        assert_eq!(
+            find("sensor.van_weather_forecast_daily").1["attributes"]["forecast"][0]["condition"],
+            "rainy"
+        );
+        assert_eq!(find("sensor.van_weather_forecast_hourly").1["state"], 1);
+    }
+
+    #[test]
+    fn omits_missing_optional_current_sensors_but_keeps_percentage_sensors() {
+        let weather = parse_weather(ApiResponse::default());
+        let updates = weather_updates(&weather);
+
+        assert!(
+            !updates
+                .iter()
+                .any(|(id, _)| id == "sensor.van_weather_temperature")
+        );
+        assert_eq!(
+            updates
+                .iter()
+                .find(|(id, _)| id == "sensor.van_weather_humidity")
+                .unwrap()
+                .1["state"],
+            0
+        );
+    }
+
+    #[test]
+    fn masks_coordinates_to_about_eleven_kilometres() {
+        assert!((mask_coordinate(37.7749) - 37.8).abs() < f64::EPSILON);
+        assert!((mask_coordinate(-122.4194) - (-122.4)).abs() < f64::EPSILON);
+    }
+}

systems/bob/default.nix

@@ -6,15 +6,16 @@
     "${inputs.self}/users/steve"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/docker.nix"
-    "${inputs.self}/common/optional/scanner.nix"
     "${inputs.self}/common/optional/monitoring-agent.nix"
+    "${inputs.self}/common/optional/nvidia.nix"
+    "${inputs.self}/common/optional/scanner.nix"
     "${inputs.self}/common/optional/steam.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/systemd-boot.nix"
+    "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
     "${inputs.self}/common/optional/yubikey.nix"
-    "${inputs.self}/common/optional/zerotier.nix"
-    "${inputs.self}/common/optional/nvidia.nix"
+    "${inputs.self}/common/optional/zfs"
     ./hardware.nix
     ./syncthing.nix
     ./llms.nix

systems/bob/llms.nix

@@ -1,20 +1,8 @@
-{ pkgs, ... }:
-let
-  # CMake 4.3.4 requires nvcc to be present in CUDAToolkit_ROOT.
-  # Remove after https://github.com/NixOS/nixpkgs/pull/545542 reaches nixos-unstable.
-  ollama-cuda = pkgs.ollama-cuda.overrideAttrs (oldAttrs: {
-    preBuild = ''
-      export CUDAToolkit_ROOT="${pkgs.cudaPackages.cuda_nvcc};$CUDAToolkit_ROOT"
-    ''
-    + oldAttrs.preBuild;
-  });
-in
 {
   services.ollama = {
     user = "ollama";
     host = "0.0.0.0";
     enable = true;
-    package = ollama-cuda;
 
     syncModels = false;
     loadModels = [
@@ -61,7 +49,7 @@ in
       "translategemma:27b"
       "translategemma:4b"
     ];
-    models = "/zfs/storage/models";
+    modelsDir = "/zfs/storage/models";
     openFirewall = true;
   };
 }

systems/brain/default.nix

@@ -7,8 +7,9 @@
     "${inputs.self}/common/optional/ssh_decrypt.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/systemd-boot.nix"
+    "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/zerotier.nix"
+    "${inputs.self}/common/optional/zfs"
     ./docker
     ./hardware.nix
     ./programs.nix

systems/brain/services/home_assistant.nix

@@ -1,5 +1,7 @@
 { pkgs, ... }:
 {
+  networking.firewall.allowedTCPPorts = [ 8123 ];
+
   users = {
     users.hass = {
       isSystemUser = true;
@@ -11,9 +13,7 @@
   services = {
     home-assistant = {
       enable = true;
-      openFirewall = true;
       config = {
-        http.server_port = 8123;
         homeassistant = {
           time_zone = "America/New_York";
           unit_system = "us_customary";
@@ -21,6 +21,7 @@
           packages = {
             victron_modbuss = "!include ${./home_assistant/victron_modbuss.yaml}";
             battery_sensors = "!include ${./home_assistant/battery_sensors.yaml}";
+            energy_meters = "!include ${./home_assistant/energy_meters.yaml}";
             gps_location = "!include ${./home_assistant/gps_location.yaml}";
             heater = "!include ${./home_assistant/heater.yaml}";
             van_weather = "!include ${./home_assistant/van_weather_template.yaml}";
@@ -73,10 +74,11 @@
           uiprotect # for ubiquiti integration
           unifi-discovery # for ubiquiti integration
           jsonpath # for rest sensors
-          typedmonarchmoney # for monarch
+          monarchmoneycommunity # for monarch
         ];
       extraComponents = [ "isal" ];
       customComponents = with pkgs.home-assistant-custom-components; [
+        garmin_connect
         pirate-weather
       ];
 
@@ -84,7 +86,7 @@
     esphome = {
       enable = true;
       openFirewall = true;
-      address = "192.168.90.35";
+      address = "0.0.0.0";
     };
   };
 }

systems/brain/services/home_assistant/energy_meters.yaml

@@ -0,0 +1,71 @@
+template:
+  - sensor:
+      - name: "JK0 Energy In Clean"
+        unique_id: jk0_energy_in_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.jk0_energy_in')) }}
+        state: >-
+          {{ states('sensor.jk0_energy_in') | float }}
+
+      - name: "JK0 Energy Out Clean"
+        unique_id: jk0_energy_out_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.jk0_energy_out')) }}
+        state: >-
+          {{ states('sensor.jk0_energy_out') | float }}
+
+      - name: "JK1 Energy In Clean"
+        unique_id: jk1_energy_in_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.jk1_energy_in')) }}
+        state: >-
+          {{ states('sensor.jk1_energy_in') | float }}
+
+      - name: "JK1 Energy Out Clean"
+        unique_id: jk1_energy_out_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.jk1_energy_out')) }}
+        state: >-
+          {{ states('sensor.jk1_energy_out') | float }}
+
+      - name: "Solar Yield Clean"
+        unique_id: solar_yield_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.solar_yield_daily')) }}
+        state: >-
+          {{ states('sensor.solar_yield_daily') | float }}
+
+      - name: "DC Load Energy Clean"
+        unique_id: dc_load_energy_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.dc_load_energy_daily')) }}
+        state: >-
+          {{ states('sensor.dc_load_energy_daily') | float }}
+
+      - name: "DC Charger Energy Clean"
+        unique_id: dc_charger_energy_clean
+        unit_of_measurement: "kWh"
+        device_class: energy
+        state_class: total_increasing
+        availability: >-
+          {{ is_number(states('sensor.dc_charger_energy')) }}
+        state: >-
+          {{ states('sensor.dc_charger_energy') | float }}

systems/brain/services/postgress.nix

@@ -19,7 +19,7 @@
       local all  richie   trust
       host  all  richie   127.0.0.1/32    trust
       host  all  richie   ::1/128         trust
-      host  all  richie   192.168.90.1/24 trust
+      host  all  richie   100.64.0.0/10 trust
       host  all  richie   192.168.99.1/24 trust
 
       local vaninventory vaninventory trust
@@ -28,7 +28,7 @@
       local hass     hass     trust
 
       # ipv4
-      host  hass     hass     192.168.90.1/24 trust
+      host  hass     hass     100.64.0.0/10 trust
       host  hass     hass     127.0.0.1/32 trust
 
       # ipv6

systems/brain/services/van_weather.nix

@@ -1,8 +1,10 @@
 {
   pkgs,
-  inputs,
   ...
 }:
+let
+  van-weather = pkgs.callPackage ../../../rust/van_weather/package.nix { };
+in
 {
   systemd.services.van-weather = {
     description = "Van Weather Service";
@@ -13,13 +15,9 @@
     requires = [ "home-assistant.service" ];
     wantedBy = [ "multi-user.target" ];
 
-    environment = {
-      PYTHONPATH = "${inputs.self}/";
-    };
-
     serviceConfig = {
       Type = "simple";
-      ExecStart = "${pkgs.my_python}/bin/python -m python.van_weather.main";
+      ExecStart = "${van-weather}/bin/van-weather";
       EnvironmentFile = "/etc/van_weather.env";
       Restart = "on-failure";
       RestartSec = "5s";
@@ -29,7 +27,6 @@
       ProtectSystem = "strict";
       ProtectHome = "read-only";
       PrivateTmp = true;
-      ReadOnlyPaths = [ "${inputs.self}" ];
     };
   };
 }

systems/brain/syncthing.nix

@@ -3,7 +3,7 @@
 
   services.syncthing = {
     overrideFolders = false;
-    guiAddress = "192.168.90.35:8384";
+    guiAddress = "0.0.0.0:8384";
     settings = {
       "dotfiles" = {
         path = "/home/richie/dotfiles";

systems/jeeves/default.nix

@@ -13,14 +13,14 @@ in
     "${inputs.self}/common/optional/monitoring-agent.nix"
     "${inputs.self}/common/optional/ssh_decrypt.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
+    "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/zerotier.nix"
+    "${inputs.self}/common/optional/zfs"
     ./monitoring
     ./docker
     ./services
-    ./web_services
     ./hardware.nix
-    ./networking.nix
+    ./networking
     ./programs.nix
     ./runners
     ./syncthing.nix
@@ -36,7 +36,13 @@ in
       EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
     };
 
-    zerotierone.joinNetworks = [ "a09acf02330d37b9" ];
+    zerotierone = {
+      enable = true;
+      joinNetworks = [
+        "e4da7455b2ae64ca"
+        "a09acf02330d37b9"
+      ];
+    };
   };
 
   users.groups = {

systems/jeeves/docker/signal-cli-rest-api.nix

@@ -2,9 +2,6 @@ let
   vars = import ../vars.nix;
 in
 {
-  networking.firewall.allowedTCPPorts = [
-    8989
-  ];
   virtualisation.oci-containers.containers.signal_cli_rest_api = {
     image = "bbernhard/signal-cli-rest-api:0.199-dev";
     ports = [

systems/jeeves/monitoring/README.md

@@ -0,0 +1,33 @@
+# Monitoring
+
+## Vultr API metrics
+
+The `vultr-exporter` service reads its API token from:
+
+```text
+/zfs/storage/secrets/services/vultr-exporter
+```
+
+Create the file on Jeeves as root with the following contents:
+
+```text
+API_KEY=<Vultr API token>
+```
+
+The token needs read access to the Vultr Account and Billing APIs. Unrelated
+resource collectors are disabled in the packaged exporter.
+
+Restrict the file to root and ensure the public egress IP used by Jeeves is
+allowed for the token in the Vultr API settings:
+
+```console
+sudo chown root:root /zfs/storage/secrets/services/vultr-exporter
+sudo chmod 600 /zfs/storage/secrets/services/vultr-exporter
+```
+
+The exporter listens on `127.0.0.1:9188`; it is scraped by the local
+`prometheus-main` service every five minutes and is not exposed through the
+host firewall.
+
+Portal-1 exposes its node exporter only through `tailscale0` on port `9100`.
+Jeeves reaches it using the Portal-1 Tailscale hostname.

systems/jeeves/monitoring/dashboards/portal-1.json

@@ -0,0 +1,798 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": false,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "short",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "red",
+                  "index": 1,
+                  "text": "Offline"
+                },
+                "1": {
+                  "color": "green",
+                  "index": 0,
+                  "text": "Online"
+                }
+              },
+              "type": "value"
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 4,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "up{job=\"node\",instance=\"portal-1\"}",
+          "instant": true,
+          "legendFormat": "",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Node Exporter",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 5,
+        "x": 4,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{job=\"node\",instance=\"portal-1\",mode=\"idle\"}[5m])))",
+          "instant": true,
+          "legendFormat": "",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "CPU Used",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 5,
+        "x": 9,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - (node_memory_MemAvailable_bytes{job=\"node\",instance=\"portal-1\"} / node_memory_MemTotal_bytes{job=\"node\",instance=\"portal-1\"}))",
+          "instant": true,
+          "legendFormat": "",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "RAM Used",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 5,
+        "x": 14,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - (node_filesystem_avail_bytes{job=\"node\",instance=\"portal-1\",mountpoint=\"/\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node\",instance=\"portal-1\",mountpoint=\"/\",fstype!=\"\"}))",
+          "instant": true,
+          "legendFormat": "",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Root Filesystem Used",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 5,
+        "x": 19,
+        "y": 0
+      },
+      "id": 5,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "node_load1{job=\"node\",instance=\"portal-1\"}",
+          "instant": true,
+          "legendFormat": "",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Load (1m)",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 5
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{job=\"node\",instance=\"portal-1\",mode=\"idle\"}[5m])))",
+          "instant": false,
+          "legendFormat": "CPU used",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "CPU Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 5
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - (node_memory_MemAvailable_bytes{job=\"node\",instance=\"portal-1\"} / node_memory_MemTotal_bytes{job=\"node\",instance=\"portal-1\"}))",
+          "instant": false,
+          "legendFormat": "RAM used",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "RAM Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 13
+      },
+      "id": 8,
+      "options": {
+        "cellHeight": "sm",
+        "showHeader": true,
+        "sortBy": [
+          {
+            "desc": true,
+            "displayName": "Value"
+          }
+        ]
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "100 * (1 - (node_filesystem_avail_bytes{job=\"node\",instance=\"portal-1\",fstype!~\"tmpfs|devtmpfs|overlay|squashfs\",mountpoint!~\"/run.*\"} / node_filesystem_size_bytes{job=\"node\",instance=\"portal-1\",fstype!~\"tmpfs|devtmpfs|overlay|squashfs\",mountpoint!~\"/run.*\"}))",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "{{mountpoint}}",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Filesystem Usage",
+      "type": "table"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "Bps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 13
+      },
+      "id": 9,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(node_disk_read_bytes_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
+          "instant": false,
+          "legendFormat": "read",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(node_disk_written_bytes_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
+          "instant": false,
+          "legendFormat": "write",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Disk Throughput",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "iops"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 21
+      },
+      "id": 10,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(node_disk_reads_completed_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
+          "instant": false,
+          "legendFormat": "reads",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(node_disk_writes_completed_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
+          "instant": false,
+          "legendFormat": "writes",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Disk Operations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prom-main"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "unit": "Bps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 21
+      },
+      "id": 11,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(node_network_receive_bytes_total{job=\"node\",instance=\"portal-1\",device!=\"lo\"}[5m]))",
+          "instant": false,
+          "legendFormat": "received",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prom-main"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(node_network_transmit_bytes_total{job=\"node\",instance=\"portal-1\",device!=\"lo\"}[5m]))",
+          "instant": false,
+          "legendFormat": "sent",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Network Traffic",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 39,
+  "tags": [
+    "monitoring",
+    "portal-1",
+    "vultr"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Portal-1 Host",
+  "uid": "portal-1-host",
+  "version": 1,
+  "weekStart": ""
+}

systems/jeeves/monitoring/default.nix

@@ -41,29 +41,36 @@ let
       {
         job_name = "node";
         static_configs = [
-          (mkTarget "jeeves" "192.168.90.40:9100")
-          (mkTarget "bob" "192.168.90.25:9100")
+          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9100")
+          (mkTarget "bob" "bob.taile39c31.ts.net:9100")
+          (mkTarget "portal-1" "portal-1.taile39c31.ts.net:9100")
         ];
       }
+      {
+        job_name = "vultr";
+        scrape_interval = "5m";
+        scrape_timeout = "45s";
+        static_configs = [ (mkTarget "vultr" "127.0.0.1:9188") ];
+      }
       {
         job_name = "process_grouped";
         static_configs = [
-          (mkTarget "jeeves" "192.168.90.40:9256")
-          (mkTarget "bob" "192.168.90.25:9256")
+          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9256")
+          (mkTarget "bob" "bob.taile39c31.ts.net:9256")
         ];
       }
       {
         job_name = "smartctl";
         static_configs = [
-          (mkTarget "jeeves" "192.168.90.40:9633")
-          (mkTarget "bob" "192.168.90.25:9633")
+          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9633")
+          (mkTarget "bob" "bob.taile39c31.ts.net:9633")
         ];
       }
       {
         job_name = "zfs";
         static_configs = [
-          (mkTarget "jeeves" "192.168.90.40:9134")
-          (mkTarget "bob" "192.168.90.25:9134")
+          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9134")
+          (mkTarget "bob" "bob.taile39c31.ts.net:9134")
         ];
       }
     ];
@@ -79,8 +86,8 @@ let
       {
         job_name = "process_pid";
         static_configs = [
-          (mkTarget "jeeves" "192.168.90.40:9257")
-          (mkTarget "bob" "192.168.90.25:9257")
+          (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9257")
+          (mkTarget "bob" "bob.taile39c31.ts.net:9257")
         ];
       }
     ];
@@ -97,7 +104,9 @@ let
       after = [
         "zfs-media-database-prometheus.mount"
         "network.target"
+        "tailscaled-autoconnect.service"
       ];
+      wants = [ "tailscaled-autoconnect.service" ];
       requires = [ "zfs-media-database-prometheus.mount" ];
       wantedBy = [ "multi-user.target" ];
       unitConfig.RequiresMountsFor = [ dataDir ];
@@ -151,6 +160,8 @@ let
     };
 in
 {
+  imports = [ ./vultr-exporter.nix ];
+
   users = {
     groups.prometheus = { };
     users.prometheus = {

systems/jeeves/monitoring/vultr-exporter.nix

@@ -0,0 +1,96 @@
+{
+  lib,
+  pkgs,
+  ...
+}:
+let
+  vars = import ../vars.nix;
+
+  vultrExporter = pkgs.buildGoModule rec {
+    pname = "vultr-exporter";
+    version = "0-unstable-2025-10-14";
+
+    src = pkgs.fetchFromGitHub {
+      owner = "DazWilkin";
+      repo = "vultr-exporter";
+      rev = "d8b5d7107c2eda31566e6e895c8a67db32fdaf68";
+      hash = "sha256-3LpXh+mFMBRJfiY2ftELpM3AIE0LiZ9c6NFok0OeH5I=";
+    };
+
+    vendorHash = "sha256-qCHWmg6GxLIjqlf5UqRcuDRd1xqRFI3sl4ZduLajx+Y=";
+    subPackages = [ "cmd/server" ];
+
+    postPatch = ''
+      substituteInPlace cmd/server/main.go \
+        --replace-fail 'registry.MustRegister(collector.NewBlockStorageCollector(s, client, log))' "" \
+        --replace-fail 'registry.MustRegister(collector.NewKubernetesCollector(s, client, log))' "" \
+        --replace-fail 'registry.MustRegister(collector.NewLoadBalancerCollector(s, client, log))' "" \
+        --replace-fail 'registry.MustRegister(collector.NewReservedIPsCollector(s, client, log))' ""
+    '';
+
+    ldflags = [
+      "-s"
+      "-w"
+      "-X main.GitCommit=${src.rev}"
+      "-X main.OSVersion=NixOS"
+    ];
+
+    postInstall = ''
+      mv $out/bin/server $out/bin/vultr-exporter
+    '';
+
+    meta = {
+      description = "Prometheus exporter for Vultr account, billing, and bandwidth metrics";
+      homepage = "https://github.com/DazWilkin/vultr-exporter";
+      license = lib.licenses.asl20;
+      mainProgram = "vultr-exporter";
+    };
+  };
+in
+{
+  systemd.services.vultr-exporter = {
+    description = "Vultr Prometheus exporter";
+    documentation = [ "https://github.com/DazWilkin/vultr-exporter" ];
+    wants = [ "network-online.target" ];
+    after = [ "network-online.target" ];
+    wantedBy = [ "multi-user.target" ];
+
+    serviceConfig = {
+      ExecStart = "${lib.getExe vultrExporter} --endpoint=127.0.0.1:9188 --path=/metrics";
+      EnvironmentFile = "${vars.secrets}/services/vultr-exporter";
+      Restart = "on-failure";
+      RestartSec = "30s";
+
+      CapabilityBoundingSet = [ "" ];
+      DeviceAllow = [ "" ];
+      DynamicUser = true;
+      LockPersonality = true;
+      MemoryDenyWriteExecute = true;
+      NoNewPrivileges = true;
+      PrivateDevices = true;
+      PrivateTmp = true;
+      ProtectClock = true;
+      ProtectControlGroups = true;
+      ProtectHome = true;
+      ProtectHostname = true;
+      ProtectKernelLogs = true;
+      ProtectKernelModules = true;
+      ProtectKernelTunables = true;
+      ProtectSystem = "strict";
+      RemoveIPC = true;
+      RestrictAddressFamilies = [
+        "AF_INET"
+        "AF_INET6"
+      ];
+      RestrictNamespaces = true;
+      RestrictRealtime = true;
+      RestrictSUIDSGID = true;
+      SystemCallArchitectures = "native";
+      SystemCallFilter = [
+        "@system-service"
+        "~@privileged"
+      ];
+      UMask = "0077";
+    };
+  };
+}

systems/jeeves/networking/default.nix

@@ -0,0 +1,16 @@
+{
+  imports = [
+    ./firewall.nix
+    ./interfaces.nix
+  ];
+  networking = {
+    hostName = "jeeves";
+    hostId = "0e15ce35";
+    useNetworkd = true;
+    # Without this, unconfigured NICs get a catch-all DHCP network and
+    # join the LAN uninvited (how the dead atlantic card ended up
+    # ARP-poisoning 192.168.99.14). Every interface must be configured
+    # explicitly below.
+    useDHCP = false;
+  };
+}

systems/jeeves/networking/firewall.nix

@@ -0,0 +1,47 @@
+let
+  ports = {
+    audiobookshelf = 8000;
+    ebook_searcher = 8070;
+    gems = 8002;
+    gitea_http = 6443;
+    gitea_ssh = 2223;
+    grafana = 3000;
+    jellyfin_http = 8096;
+    nix_binary_cache = 5000;
+    nornsight = 8001;
+    ollama = 11434;
+    open_webui = 8080;
+    postgresql = 5432;
+    share = 8091;
+    signal_cli = 8989;
+    syncthing = 8384;
+  };
+in
+{
+  networking.firewall = {
+    enable = true;
+    interfaces.br-nix-builder = {
+      allowedTCPPorts = [ ];
+      allowedUDPPorts = [ ];
+    };
+    interfaces.tailscale0.allowedTCPPorts = [
+      ports.audiobookshelf
+      ports.gems
+      ports.jellyfin_http
+      ports.nix_binary_cache
+      ports.nornsight
+      ports.share
+    ];
+    allowedTCPPorts = [
+      ports.ebook_searcher
+      ports.gitea_http
+      ports.gitea_ssh
+      ports.grafana
+      ports.ollama
+      ports.open_webui
+      ports.postgresql
+      ports.signal_cli
+      ports.syncthing
+    ];
+  };
+}

systems/jeeves/networking/interfaces.nix

@@ -8,27 +8,6 @@
     "net.bridge.bridge-nf-call-iptables" = 0;
   };
 
-  networking = {
-    hostName = "jeeves";
-    hostId = "0e15ce35";
-    firewall = {
-      enable = true;
-      interfaces.br-nix-builder = {
-        allowedTCPPorts = [ ];
-        allowedUDPPorts = [ ];
-      };
-      allowedTCPPorts = [
-        8070
-      ];
-    };
-    useNetworkd = true;
-    # Without this, unconfigured NICs get a catch-all DHCP network and
-    # join the LAN uninvited (how the dead atlantic card ended up
-    # ARP-poisoning 192.168.99.14). Every interface must be configured
-    # explicitly below.
-    useDHCP = false;
-  };
-
   systemd.network = {
     enable = true;
     wait-online = {

systems/jeeves/scripts/zfs.sh

@@ -19,7 +19,6 @@ sudo zfs create media/temp -o sync=disabled -o redundant_metadata=none
 sudo zfs create media/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key
 sudo zfs create media/secure/docker -o compression=zstd-9
 sudo zfs create media/secure/github-runners -o compression=zstd-9 -o sync=disabled
-sudo zfs create media/secure/home_assistant -o compression=zstd-19
 sudo zfs create media/secure/notes -o copies=2
 sudo zfs create media/secure/postgres -o mountpoint=/zfs/media/database/postgres -o recordsize=16k -o primarycache=metadata
 sudo zfs create media/secure/postgres-wal -o mountpoint=/zfs/media/database/postgres-wal -o recordsize=32k -o primarycache=metadata -o special_small_blocks=32K -o compression=lz4 -o secondarycache=none -o logbias=latency

systems/jeeves/services/audiobookshelf.nix

@@ -6,6 +6,7 @@ in
   services.audiobookshelf = {
     enable = true;
     port = 8000;
+    host = "0.0.0.0";
   };
   systemd.services.audiobookshelf.serviceConfig.WorkingDirectory =
     lib.mkForce "${vars.docker_configs}/audiobookshelf";

systems/jeeves/services/gems.nix

@@ -36,7 +36,7 @@ in
       Type = "simple";
       User = "gems";
       Group = "gems";
-      ExecStart = "${pkgs.my_python}/bin/python -m python.gems.main --host 127.0.0.1 --port 8002";
+      ExecStart = "${pkgs.my_python}/bin/python -m python.gems.main --host 0.0.0.0 --port 8002";
       Restart = "on-failure";
       RestartSec = "5s";
       StandardOutput = "journal";

systems/jeeves/services/gitea.nix

@@ -2,11 +2,6 @@ let
   vars = import ../vars.nix;
 in
 {
-  networking.firewall.allowedTCPPorts = [
-    6443
-    2223
-  ];
-
   services.gitea = {
     enable = true;
     appName = "TMM Workshop";
@@ -27,10 +22,14 @@ in
       };
       service.DISABLE_REGISTRATION = true;
       server = {
-        DOMAIN = "tmmworkshop.com";
+        DOMAIN = "gitea.tmmworkshop.com";
         ROOT_URL = "https://gitea.tmmworkshop.com/";
         HTTP_PORT = 6443;
+        BUILTIN_SSH_SERVER_USER = "gitea";
+        SSH_USER = "gitea";
+        SSH_DOMAIN = "gitea.tmmworkshop.com";
         SSH_PORT = 2223;
+        SSH_LISTEN_HOST = "0.0.0.0";
         SSH_LISTEN_PORT = 2223;
         START_SSH_SERVER = true;
         PUBLIC_URL_DETECTION = "auto";

systems/jeeves/services/grafana.nix

@@ -6,8 +6,6 @@ let
   grafanaDataDir = "${vars.services}/grafana";
 in
 {
-  networking.firewall.allowedTCPPorts = [ 3000 ];
-
   services.grafana = {
     enable = true;
     dataDir = grafanaDataDir;
@@ -19,9 +17,9 @@ in
         secret_key = "$__file{${vars.secrets}/services/grafana/secret_key}";
       };
       server = {
-        http_addr = "192.168.90.40";
+        http_addr = "0.0.0.0";
         http_port = 3000;
-        root_url = "http://192.168.90.40:3000/";
+        root_url = "http://jeeves:3000/";
       };
     };
     provision = {

systems/jeeves/services/home_assistant.nix

@@ -1,79 +0,0 @@
-let
-  vars = import ../vars.nix;
-in
-{
-  users = {
-    users.hass = {
-      isSystemUser = true;
-      group = "hass";
-    };
-    groups.hass = { };
-  };
-
-  services = {
-    home-assistant = {
-      enable = true;
-      openFirewall = true;
-      configDir = vars.home_assistant;
-      config = {
-        http = {
-          server_port = 8123;
-          use_x_forwarded_for = true;
-          trusted_proxies = "127.0.0.1";
-        };
-        homeassistant = {
-          time_zone = "America/New_York";
-          unit_system = "us_customary";
-          temperature_unit = "F";
-        };
-        recorder = {
-          db_url = "postgresql://@/hass";
-          auto_purge = true;
-          purge_keep_days = 3650;
-          db_retry_wait = 15;
-        };
-        assist_pipeline = { };
-        backup = { };
-        bluetooth = { };
-        config = { };
-        dhcp = { };
-        energy = { };
-        history = { };
-        homeassistant_alerts = { };
-        image_upload = { };
-        logbook = { };
-        media_source = { };
-        mobile_app = { };
-        ssdp = { };
-        sun = { };
-        webhook = { };
-        zeroconf = { };
-        automation = "!include automations.yaml";
-        script = "!include scripts.yaml";
-        scene = "!include scenes.yaml";
-        group = "!include groups.yaml";
-      };
-      extraPackages =
-        python3Packages: with python3Packages; [
-          aioesphomeapi
-          aiounifi
-          bleak-esphome
-          esphome-dashboard-api
-          gtts
-          jellyfin-apiclient-python
-          psycopg2
-          pymetno
-          aio-ownet
-          rokuecp
-          uiprotect
-          wakeonlan
-        ];
-      extraComponents = [ "isal" ];
-    };
-    esphome = {
-      enable = true;
-      openFirewall = true;
-      address = "192.168.90.40";
-    };
-  };
-}

systems/jeeves/services/jellyfin.nix

@@ -4,7 +4,6 @@ in
 {
   services.jellyfin = {
     enable = true;
-    openFirewall = true;
     dataDir = "${vars.services}/jellyfin";
     cacheDir = "${vars.services}/jellyfin/cache";
   };

systems/jeeves/services/llms.nix

@@ -37,8 +37,7 @@ in
       "qwen3:14b"
       "qwen3.5:35b"
     ];
-    models = vars.ollama;
-    openFirewall = true;
+    modelsDir = vars.ollama;
   };
   systemd.services = {
     ollama.serviceConfig = {

systems/jeeves/services/nix_serve.nix

@@ -7,6 +7,5 @@ in
     package = pkgs.nix-serve-ng;
     enable = true;
     secretKeyFile = "${vars.secrets}/services/nix-cache/cache-priv-key.pem";
-    openFirewall = true;
   };
 }

systems/jeeves/services/open_webui.nix

@@ -2,7 +2,6 @@
   services.open-webui = {
     enable = true;
     host = "0.0.0.0";
-    openFirewall = true;
     environment = {
       ANONYMIZED_TELEMETRY = "False";
       DO_NOT_TRACK = "True";

systems/jeeves/services/postgress.nix

@@ -3,8 +3,6 @@ let
   vars = import ../vars.nix;
 in
 {
-  networking.firewall.allowedTCPPorts = [ 5432 ];
-
   # Symlink pg_wal to a ZFS dataset on the special (metadata) vdev for fast WAL writes
   # this is required for systemd sandboxing
   systemd.services.postgresql.serviceConfig.ReadWritePaths = [ "/zfs/media/database/postgres-wal" ];
@@ -27,13 +25,12 @@ in
       local all  richie   trust
       host  all  richie   127.0.0.1/32    trust
       host  all  richie   ::1/128         trust
-      host  all  richie   192.168.90.1/24 trust
+      host  all  richie   100.64.0.0/10 trust
       host  all  richie   192.168.99.1/24 trust
       host  all  richie   172.16.0.0/12   trust
 
 
       #type database DBuser origin-address auth-method
-      local hass     hass     trust
       local gitea    gitea    trust
 
       # signalbot
@@ -43,12 +40,12 @@ in
       local postgres  math   trust
       host  postgres  math   127.0.0.1/32    trust
       host  postgres  math   ::1/128         trust
-      host  postgres  math   192.168.90.1/24 trust
+      host  postgres  math   100.64.0.0/10 trust
 
       local data_science_dev  math   trust
       host  data_science_dev  math   127.0.0.1/32    trust
       host  data_science_dev  math   ::1/128         trust
-      host  data_science_dev  math   192.168.90.1/24 trust
+      host  data_science_dev  math   100.64.0.0/10 trust
     '';
 
     identMap = ''
@@ -57,7 +54,6 @@ in
         superuser_map      postgres  postgres
         # Let other names login as themselves
         superuser_map      richie    postgres
-        superuser_map      hass      hass
     '';
     ensureUsers = [
       {
@@ -81,16 +77,6 @@ in
           replication = true;
         };
       }
-      {
-        name = "hass";
-        ensureDBOwnership = true;
-        ensureClauses = {
-          login = true;
-          createrole = true;
-          createdb = true;
-          replication = true;
-        };
-      }
       {
         name = "gitea";
         ensureDBOwnership = true;
@@ -121,7 +107,6 @@ in
     ];
     ensureDatabases = [
       "data_science_dev"
-      "hass"
       "gitea"
       "math"
       "n8n"

systems/jeeves/services/transmission.nix

@@ -21,11 +21,11 @@ in
       message-level = 3;
       peer-port = 51413;
       rpc-bind-address = "0.0.0.0";
-      rpc-host-whitelist = "127.0.0.1,192.168.90.40";
+      rpc-host-whitelist = "127.0.0.1,jeeves,jeeves.taile39c31.ts.net";
       rpc-host-whitelist-enabled = true;
       rpc-port = 9091;
       rpc-whitelist-enabled = true;
-      rpc-whitelist = "127.0.0.1,192.168.90.49";
+      rpc-whitelist = "127.0.0.1,100.*.*.*";
       seed-queue-enabled = false;
     };
   };

systems/jeeves/services/validate_system.toml

@@ -1,8 +1,6 @@
 zpool = ["root_pool", "storage", "media"]
 services = [
     "audiobookshelf",
-    "haproxy",
     "docker",
-    "home-assistant",
     "jellyfin",
 ]

systems/jeeves/syncthing.nix

@@ -2,11 +2,9 @@ let
   vars = import ./vars.nix;
 in
 {
-  networking.firewall.allowedTCPPorts = [ 8384 ];
-
   services.syncthing = {
     overrideFolders = false;
-    guiAddress = "192.168.90.40:8384";
+    guiAddress = "0.0.0.0:8384";
     settings = {
       devices.davids-server.id = "7GXTDGR-AOXFW2O-K6J7NM3-XYZNRRW-AKHAFWM-GBOWUPQ-OA6JIWD-ER7RDQL"; # cspell:disable-line
       folders = {

systems/jeeves/vars.nix

@@ -8,7 +8,6 @@ in
   database = "${zfs_media}/database";
   docker = "${zfs_media}/docker";
   docker_configs = "${zfs_media}/docker/configs";
-  home_assistant = "${zfs_media}/home_assistant";
   notes = "${zfs_media}/notes";
   secrets = "${zfs_storage}/secrets";
   services = "${zfs_media}/services";

systems/jeeves/web_services/acme.nix

@@ -1,75 +0,0 @@
-let
-  domains = [
-    "audiobookshelf"
-    "cache"
-    "gitea"
-    "gems"
-    "jellyfin"
-    "share"
-  ];
-  extraDomains = [ "www.norn-sight.com" ];
-
-  makeCert = name: {
-    name = "${name}.tmmworkshop.com";
-    value = {
-      webroot = "/var/lib/acme/.challenges";
-      group = "acme";
-      reloadServices = [ "haproxy.service" ];
-    };
-  };
-
-  makeExtraCert = name: {
-    inherit name;
-    value = {
-      webroot = "/var/lib/acme/.challenges";
-      group = "acme";
-      reloadServices = [ "haproxy.service" ];
-    };
-  };
-
-  acmeServices =
-    map (domain: "acme-${domain}.tmmworkshop.com.service") domains
-    ++ map (domain: "acme-${domain}.service") extraDomains;
-in
-{
-  users.users.haproxy.extraGroups = [ "acme" ];
-
-  security.acme = {
-    acceptTerms = true;
-    defaults.email = "Richie@tmmworkshop.com";
-    certs = builtins.listToAttrs ((map makeCert domains) ++ (map makeExtraCert extraDomains));
-  };
-
-  # Minimal nginx to serve ACME HTTP-01 challenge files for HAProxy
-  services.nginx = {
-    enable = true;
-    virtualHosts."acme-challenge" = {
-      listen = [
-        {
-          addr = "127.0.0.1";
-          port = 8402;
-        }
-      ];
-      locations."/.well-known/acme-challenge/" = {
-        root = "/var/lib/acme/.challenges";
-      };
-    };
-  };
-
-  # Ensure the challenge directory exists with correct permissions
-  systemd.tmpfiles.rules = [
-    "d /var/lib/acme/.challenges 0750 acme acme - -"
-    "d /var/lib/acme/.challenges/.well-known 0750 acme acme - -"
-    "d /var/lib/acme/.challenges/.well-known/acme-challenge 0750 acme acme - -"
-  ];
-
-  users.users.nginx.extraGroups = [ "acme" ];
-
-  # HAProxy needs certs to exist before it can bind :443.
-  # NixOS's acme module generates self-signed placeholders on first boot
-  # via acme-<domain>.service — just make HAProxy wait for them.
-  systemd.services.haproxy = {
-    after = acmeServices;
-    wants = acmeServices;
-  };
-}

systems/jeeves/web_services/default.nix

@@ -1,9 +0,0 @@
-{ lib, ... }:
-{
-  imports =
-    let
-      files = builtins.attrNames (builtins.readDir ./.);
-      nixFiles = builtins.filter (name: lib.hasSuffix ".nix" name && name != "default.nix") files;
-    in
-    map (file: ./. + "/${file}") nixFiles;
-}

systems/jeeves/web_services/haproxy.nix

@@ -1,14 +0,0 @@
-{
-  networking.firewall.allowedTCPPorts = [
-    80
-    443
-  ];
-
-  # Global robots.txt served by HAProxy for every vhost (see haproxy.cfg).
-  environment.etc."haproxy/robots.txt".source = ./robots.txt;
-
-  services.haproxy = {
-    enable = true;
-    config = builtins.readFile ./haproxy.cfg;
-  };
-}

systems/portal-1/README.md

@@ -0,0 +1,83 @@
+# portal_1
+
+Minimal NixOS target for a Vultr VM, installed with nixos-anywhere. The Nix
+flake target is `portal_1`; the machine hostname is `portal-1` because DNS
+hostnames cannot contain underscores.
+
+## Before deploying
+
+1. Confirm the VM's system disk is `/dev/vda`. If it is not, update both
+   references in `disk-config.nix`.
+2. Confirm the SSH public key in `default.nix` is the key that should have
+   administrator access.
+3. Boot the VM into a NixOS installer or another nixos-anywhere-compatible
+   Linux rescue environment with root SSH access. Keep this environment
+   running while completing the SOPS bootstrap below.
+
+## Bootstrap SOPS
+
+Use the rescue environment's SSH host key as the permanent portal identity.
+Replace `VM_IP` below:
+
+```console
+ssh root@VM_IP 'cat /etc/ssh/ssh_host_ed25519_key.pub' | \
+  nix shell nixpkgs#ssh-to-age --command ssh-to-age
+```
+
+This prints an `age1...` recipient; it does not copy the private key. Add the
+recipient to `.sops.yaml`:
+
+```yaml
+- &system_portal_1 age1...
+```
+
+Then add `*system_portal_1` to the age recipients for
+`users/secrets.yaml`. Re-encrypt the existing file for the new recipient and
+add the Tailscale key:
+
+```console
+nix shell nixpkgs#sops --command sops updatekeys users/secrets.yaml
+nix shell nixpkgs#sops --command sops users/secrets.yaml
+```
+
+Add the OAuth client secret from the `Auth Keys: Write` credential in the SOPS
+editor and save it:
+
+```yaml
+tailscale_auth_key: tskey-client-...
+```
+
+## Deploy
+
+From the repository root, replace `VM_IP` with the VM's public IP:
+
+```console
+nix run github:nix-community/nixos-anywhere -- \
+  --copy-host-keys --flake .#portal_1 root@VM_IP
+```
+
+This repartitions `/dev/vda`, so anything already on that disk is erased. The
+layout reserves 8 GiB for swap and assigns the remaining space to the root
+filesystem.
+
+`--copy-host-keys` preserves the same private SSH host key at
+`/etc/ssh/ssh_host_ed25519_key` on the installed system. SOPS-Nix converts that
+key to an age identity during activation. After the reboot, connect as
+`richie` and verify that automatic Tailscale enrollment succeeded:
+
+```console
+ssh -p 278 richie@VM_IP
+sudo tailscale status
+```
+
+The installed OpenSSH service listens on port 278. Port 22 is served by
+Endlessh and will not provide an SSH login.
+
+HAProxy uses the same frontend, routing, and rate-limiting configuration as
+Jeeves. Portal manages the ACME certificates for the existing public domains;
+their DNS records must resolve to Portal for HTTP-01 issuance and renewal.
+
+The application backends still use Jeeves' original `127.0.0.1` addresses.
+Replace them with the corresponding Tailscale addresses before directing
+application traffic through Portal. Ports 80 and 443 are allowed through the
+firewall.

systems/portal-1/default.nix

@@ -0,0 +1,57 @@
+{
+  inputs,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  imports = [
+    inputs.disko.nixosModules.disko
+    "${inputs.self}/users/richie"
+    "${inputs.self}/common/global"
+    "${inputs.self}/common/optional/tailscale.nix"
+    ./disk-config.nix
+    ./haproxy
+    ./monitoring.nix
+  ];
+
+  nixpkgs.hostPlatform = "x86_64-linux";
+
+  boot = {
+    # Avoid consuming the VM's limited memory for /tmp.
+    tmp.useTmpfs = false;
+
+    # The Vultr system disk and NIC are exposed as virtio devices.
+    initrd.availableKernelModules = [
+      "virtio_pci"
+      "virtio_blk"
+      "virtio_scsi"
+      "sd_mod"
+    ];
+  };
+
+  networking = {
+    hostName = "portal-1";
+    useDHCP = lib.mkDefault true;
+
+    firewall = {
+      enable = true;
+      allowedTCPPorts = [ 278 ];
+    };
+  };
+
+  services.openssh.ports = [ 278 ];
+
+  environment.systemPackages = with pkgs; [
+    curl
+    htop
+    vim
+  ];
+
+  # The VM uses its regular swap instead of compressed RAM swap.
+  zramSwap.enable = false;
+
+  time.timeZone = "Etc/UTC";
+
+  system.stateVersion = "24.05";
+}

systems/portal-1/disk-config.nix

@@ -0,0 +1,59 @@
+{ ... }:
+{
+  # Vultr's first virtio disk is normally /dev/vda. Change this before
+  # deployment if the selected image exposes its system disk differently.
+  disko.devices.disk.main = {
+    type = "disk";
+    device = "/dev/vda";
+    content = {
+      type = "gpt";
+      partitions = {
+        bios = {
+          size = "1M";
+          type = "EF02";
+        };
+
+        ESP = {
+          size = "512M";
+          type = "EF00";
+          content = {
+            type = "filesystem";
+            format = "vfat";
+            mountpoint = "/boot";
+            mountOptions = [
+              "fmask=0077"
+              "dmask=0077"
+            ];
+          };
+        };
+
+        swap = {
+          size = "8G";
+          content = {
+            type = "swap";
+          };
+        };
+
+        root = {
+          size = "100%";
+          content = {
+            type = "filesystem";
+            format = "ext4";
+            mountpoint = "/";
+          };
+        };
+      };
+    };
+  };
+
+  # Install GRUB for both legacy BIOS and EFI-capable Vultr plans.
+  boot.loader = {
+    grub = {
+      enable = true;
+      devices = [ "/dev/vda" ];
+      efiSupport = true;
+      efiInstallAsRemovable = true;
+    };
+    efi.canTouchEfiVariables = false;
+  };
+}

systems/portal-1/haproxy/default.nix

@@ -0,0 +1,89 @@
+let
+  domains = [
+    "audiobookshelf"
+    "cache"
+    "gitea"
+    "gems"
+    "jellyfin"
+    "share"
+  ];
+  extraDomains = [ "www.norn-sight.com" ];
+
+  makeCert = name: {
+    name = "${name}.tmmworkshop.com";
+    value = {
+      webroot = "/var/lib/acme/.challenges";
+      group = "acme";
+      reloadServices = [ "haproxy.service" ];
+    };
+  };
+
+  makeExtraCert = name: {
+    inherit name;
+    value = {
+      webroot = "/var/lib/acme/.challenges";
+      group = "acme";
+      reloadServices = [ "haproxy.service" ];
+    };
+  };
+
+  acmeServices =
+    map (domain: "acme-${domain}.tmmworkshop.com.service") domains
+    ++ map (domain: "acme-${domain}.service") extraDomains;
+in
+{
+  networking.firewall.allowedTCPPorts = [
+    80
+    443
+    2223
+  ];
+
+  # Global robots.txt served by HAProxy for every vhost (see haproxy.cfg).
+  environment.etc."haproxy/robots.txt".source = ./robots.txt;
+
+  services = {
+    haproxy = {
+      enable = true;
+      config = builtins.readFile ./haproxy.cfg;
+    };
+    nginx = {
+      enable = true;
+      virtualHosts."acme-challenge" = {
+        listen = [
+          {
+            addr = "127.0.0.1";
+            port = 8402;
+          }
+        ];
+        locations."/.well-known/acme-challenge/" = {
+          root = "/var/lib/acme/.challenges";
+        };
+      };
+    };
+  };
+
+  security.acme = {
+    acceptTerms = true;
+    defaults.email = "Richie@tmmworkshop.com";
+    certs = builtins.listToAttrs ((map makeCert domains) ++ (map makeExtraCert extraDomains));
+  };
+
+  systemd = {
+    services.haproxy = {
+      # HAProxy needs the initial certificates and Tailscale connectivity before
+      # it can serve requests to its backends.
+      after = acmeServices ++ [ "tailscaled-autoconnect.service" ];
+      wants = acmeServices ++ [ "tailscaled-autoconnect.service" ];
+    };
+    tmpfiles.rules = [
+      "d /var/lib/acme/.challenges 0750 acme acme - -"
+      "d /var/lib/acme/.challenges/.well-known 0750 acme acme - -"
+      "d /var/lib/acme/.challenges/.well-known/acme-challenge 0750 acme acme - -"
+    ];
+  };
+
+  users.users = {
+    haproxy.extraGroups = [ "acme" ];
+    nginx.extraGroups = [ "acme" ];
+  };
+}

systems/portal-1/haproxy/haproxy.cfg

@@ -22,8 +22,8 @@ defaults
 
 #Application Setup
 frontend ContentSwitching
-  bind *:80 v4v6
-  bind *:443 v4v6 ssl crt /var/lib/acme/audiobookshelf.tmmworkshop.com/full.pem crt /var/lib/acme/cache.tmmworkshop.com/full.pem crt /var/lib/acme/gems.tmmworkshop.com/full.pem crt /var/lib/acme/jellyfin.tmmworkshop.com/full.pem crt /var/lib/acme/share.tmmworkshop.com/full.pem crt /var/lib/acme/gitea.tmmworkshop.com/full.pem crt /var/lib/acme/www.norn-sight.com/full.pem
+  bind [::]:80 v4v6
+  bind [::]:443 v4v6 ssl crt /var/lib/acme/audiobookshelf.tmmworkshop.com/full.pem crt /var/lib/acme/cache.tmmworkshop.com/full.pem crt /var/lib/acme/gems.tmmworkshop.com/full.pem crt /var/lib/acme/jellyfin.tmmworkshop.com/full.pem crt /var/lib/acme/share.tmmworkshop.com/full.pem crt /var/lib/acme/gitea.tmmworkshop.com/full.pem crt /var/lib/acme/www.norn-sight.com/full.pem
   mode  http
 
   # ACME challenge routing (must be first)
@@ -110,6 +110,14 @@ frontend ContentSwitching
   use_backend gems  if host_gems
   use_backend norn_sight  if host_norn_sight
 
+# Proxy Gitea's built-in SSH server over Tailscale to Jeeves. Port 22 remains
+# available to Endlessh on Portal.
+frontend GiteaSSH
+  bind [::]:2223 v4v6
+  mode tcp
+  option tcplog
+  default_backend gitea_ssh
+
 # Stick-table only (no servers): tracks per-IP request rate to Gitea's compare
 # endpoint over a 5-minute window so the frontend can cap it at 1 per 5 min.
 backend st_compare
@@ -124,31 +132,35 @@ backend acme_challenge
 
 backend audiobookshelf_nodes
   mode http
-  server server 127.0.0.1:8000
+  server server jeeves:8000
 
 backend cache_nodes
   mode http
-  server server 127.0.0.1:5000
+  server server jeeves:5000
 
 backend jellyfin
   mode http
   option forwardfor
-  server jellyfin 127.0.0.1:8096
+  server jellyfin jeeves:8096
 
 backend share_nodes
   mode http
-  server server 127.0.0.1:8091
+  server server jeeves:8091
 
 backend gitea
   mode http
-  server server 127.0.0.1:6443
+  server server jeeves:6443
+
+backend gitea_ssh
+  mode tcp
+  server gitea jeeves:2223
 
 backend gems
   mode http
   option forwardfor
   timeout server 1h
-  server gems 127.0.0.1:8002
+  server gems jeeves:8002
 
 backend norn_sight
   mode http
-  server server 127.0.0.1:8001
+  server server jeeves:8001

systems/portal-1/monitoring.nix

@@ -0,0 +1,15 @@
+{ config, ... }:
+{
+  networking.firewall.interfaces.tailscale0.allowedTCPPorts = [
+    config.services.prometheus.exporters.node.port
+  ];
+
+  services.prometheus.exporters.node = {
+    enable = true;
+    enabledCollectors = [
+      "pressure"
+      "processes"
+      "systemd"
+    ];
+  };
+}

systems/rhapsody-in-green/default.nix

@@ -8,8 +8,9 @@
     "${inputs.self}/common/optional/steam.nix"
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/systemd-boot.nix"
+    "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/yubikey.nix"
-    "${inputs.self}/common/optional/zerotier.nix"
+    "${inputs.self}/common/optional/zfs"
     ./hardware.nix
     ./open_webui.nix
     ./programs.nix

systems/rhapsody-in-green/open_webui.nix

@@ -6,7 +6,7 @@
       ANONYMIZED_TELEMETRY = "False";
       DO_NOT_TRACK = "True";
       SCARF_NO_ANALYTICS = "True";
-      OLLAMA_API_BASE_URL = "http://127.0.0.1:11434";
+      OLLAMA_API_BASE_URL = "https://ollama.com";
       WEBUI_AUTH = "False";
     };
   };

tests/ebook_search/test_rerank.py

@@ -51,7 +51,7 @@ def test_config_defaults_enable_reranking(mocker: MockerFixture) -> None:
     config = load_rerank_config()
 
     assert config.enabled is True
-    assert config.base_url == "http://192.168.90.25:8001"
+    assert config.base_url == "http://bob:8001"
     assert config.model == "qwen3-reranker-06b"
     assert config.candidates == 24
     assert config.timeout_seconds == 30

tests/test_zfs.py

@@ -2,15 +2,35 @@
 
 import json
 from datetime import UTC, datetime
+from subprocess import CompletedProcess
+from typing import TYPE_CHECKING
 from unittest.mock import call
 
 import pytest
-from pytest_mock import MockerFixture
 
-from python.zfs import Dataset, Snapshot, Zpool, get_datasets
+from python.zfs import (
+    CommandResult,
+    Dataset,
+    Snapshot,
+    Zpool,
+    create_dataset,
+    get_datasets,
+    get_properties,
+    list_dataset_names,
+    run_zfs,
+    run_zpool,
+    set_property,
+)
+
+# Not re-exported from python.zfs on purpose: the wrappers are the default.
+from python.zfs.command import run_command
 from python.zfs.dataset import _zfs_list
 from python.zfs.zpool import _zpool_list
 
+if TYPE_CHECKING:
+    from pytest_mock import MockerFixture
+
+COMMAND = "python.zfs.command"
 DATASET = "python.zfs.dataset"
 ZPOOL = "python.zfs.zpool"
 SAMPLE_SNAPSHOT_DATA = {
@@ -207,12 +227,20 @@ def test_zfs_list_version_check(mocker: MockerFixture) -> None:
 
 def test_get_datasets(mocker: MockerFixture) -> None:
     """Test get_datasets."""
-    mock_bash = mocker.patch(f"{DATASET}.bash_wrapper", return_value=("pool/dataset\npool/other\ninvalid", 0))
+    mock_run = mocker.patch(
+        f"{DATASET}.run_zfs",
+        return_value=CommandResult(
+            args=(),
+            stdout="pool/dataset\npool/other\ninvalid",
+            stderr="",
+            return_code=0,
+        ),
+    )
     mock_dataset = mocker.patch(f"{DATASET}.Dataset")
 
     get_datasets()
 
-    mock_bash.assert_called_once_with("zfs list -Hp -t filesystem -o name")
+    mock_run.assert_called_once_with("list", "-Hp", "-t", "filesystem", "-o", "name")
 
     calls = [call("pool/dataset"), call("pool/other")]
 
@@ -287,11 +315,16 @@ def test_zpool_repr(mocker: MockerFixture) -> None:
 def test_zpool_list(mocker: MockerFixture) -> None:
     """Test version validation in _zpool_list."""
     mocker.patch(
-        f"{ZPOOL}.bash_wrapper",
-        return_value=(json.dumps({"output_version": {"vers_major": 0, "vers_minor": 1, "command": "zpool list"}}), 0),
+        f"{ZPOOL}.run_zpool",
+        return_value=CommandResult(
+            args=(),
+            stdout=json.dumps({"output_version": {"vers_major": 0, "vers_minor": 1, "command": "zpool list"}}),
+            stderr="",
+            return_code=0,
+        ),
     )
 
-    result = _zpool_list("zpool list invalid -pHj -o all")
+    result = _zpool_list("list", "invalid", "-pHj", "-o", "all")
 
     assert result == {"output_version": {"command": "zpool list", "vers_major": 0, "vers_minor": 1}}
 
@@ -299,11 +332,352 @@ def test_zpool_list(mocker: MockerFixture) -> None:
 def test_zpool_list_version_check(mocker: MockerFixture) -> None:
     """Test version validation in _zpool_list."""
     mocker.patch(
-        f"{ZPOOL}.bash_wrapper",
-        return_value=(json.dumps({"output_version": {"vers_major": 1, "vers_minor": 0, "command": "zpool list"}}), 0),
+        f"{ZPOOL}.run_zpool",
+        return_value=CommandResult(
+            args=(),
+            stdout=json.dumps({"output_version": {"vers_major": 1, "vers_minor": 0, "command": "zpool list"}}),
+            stderr="",
+            return_code=0,
+        ),
     )
 
     with pytest.raises(RuntimeError) as excinfo:
-        _zpool_list("zpool list invalid -pHj -o all")
+        _zpool_list("list", "invalid", "-pHj", "-o", "all")
 
     assert "Datasets are not in the correct format" in str(excinfo.value)
+
+
+# -- run_zfs, the subprocess boundary -----------------------------------------
+
+
+def completed(returncode: int = 0, stdout: str = "", stderr: str = "") -> CompletedProcess:
+    """Build a CompletedProcess the way subprocess.run would return one."""
+    return CompletedProcess(args=["zfs"], returncode=returncode, stdout=stdout, stderr=stderr)
+
+
+def test_run_zfs_passes_arguments_as_a_list(mocker: MockerFixture) -> None:
+    """Arguments must never be joined into a string and re-split."""
+    mock_run = mocker.patch(f"{COMMAND}.subprocess.run", return_value=completed(stdout="ok\n"))
+
+    run_zfs("get", "-Hp", "all", "media/temp")
+
+    mock_run.assert_called_once_with(
+        ["zfs", "get", "-Hp", "all", "media/temp"],
+        capture_output=True,
+        encoding="utf-8",
+        check=False,
+    )
+
+
+def test_run_zfs_keeps_values_containing_spaces_intact(mocker: MockerFixture) -> None:
+    """A property value with a space must reach zfs as one argument.
+
+    bash_wrapper split on whitespace, so this silently became several
+    arguments and zfs was handed something it could not parse.
+    """
+    mock_run = mocker.patch(f"{COMMAND}.subprocess.run", return_value=completed())
+
+    run_zfs("set", "mountpoint=/zfs/two words", "media/temp")
+
+    assert mock_run.call_args.args[0] == ["zfs", "set", "mountpoint=/zfs/two words", "media/temp"]
+
+
+def test_run_zfs_reports_success(mocker: MockerFixture) -> None:
+    mocker.patch(f"{COMMAND}.subprocess.run", return_value=completed(stdout="output\n", stderr=""))
+
+    result = run_zfs("list")
+
+    assert result.ok
+    assert result.stdout == "output\n"
+    assert result.return_code == 0
+    assert result.args == ("zfs", "list")
+
+
+def test_run_zfs_returns_failures_rather_than_raising(mocker: MockerFixture) -> None:
+    """Callers decide what a failure means, so run_zfs never raises."""
+    mocker.patch(f"{COMMAND}.subprocess.run", return_value=completed(returncode=1, stderr="no such pool\n"))
+
+    result = run_zfs("list")
+
+    assert not result.ok
+    assert result.stderr == "no such pool\n"
+    assert result.message == "no such pool"
+
+
+def test_run_zfs_keeps_streams_separate(mocker: MockerFixture) -> None:
+    """A warning on stderr must not contaminate stdout.
+
+    bash_wrapper returned stderr in place of stdout whenever stderr was
+    non-empty, even on success, which is how a warning could be parsed as a
+    list of dataset names.
+    """
+    mocker.patch(
+        f"{COMMAND}.subprocess.run",
+        return_value=completed(stdout="pool/one\n", stderr="warning: something\n"),
+    )
+
+    result = run_zfs("list")
+
+    assert result.ok
+    assert result.stdout == "pool/one\n"
+    assert result.stderr == "warning: something\n"
+
+
+def test_command_result_message_falls_back_to_stdout() -> None:
+    """Some zfs errors land on stdout, so the message must not be empty."""
+    result = CommandResult(args=("list",), stdout="  something went wrong \n", stderr="", return_code=1)
+
+    assert result.message == "something went wrong"
+
+
+# -- list_dataset_names -------------------------------------------------------
+
+
+def patch_run(mocker: MockerFixture, **kwargs) -> object:
+    """Patch run_zfs with a single canned result."""
+    return mocker.patch(f"{DATASET}.run_zfs", return_value=CommandResult(args=(), **kwargs))
+
+
+def test_list_dataset_names_builds_the_right_command(mocker: MockerFixture) -> None:
+    mock_run = patch_run(mocker, stdout="pool\n", stderr="", return_code=0)
+
+    list_dataset_names()
+
+    mock_run.assert_called_once_with("list", "-Hp", "-t", "filesystem", "-o", "name")
+
+
+def test_list_dataset_names_includes_pool_roots(mocker: MockerFixture) -> None:
+    """Unlike get_datasets, nothing is filtered out, so existence checks work."""
+    patch_run(mocker, stdout="media\nmedia/temp\nmedia/secure/docker\n", stderr="", return_code=0)
+
+    assert list_dataset_names() == ["media", "media/temp", "media/secure/docker"]
+
+
+def test_list_dataset_names_drops_blank_lines(mocker: MockerFixture) -> None:
+    patch_run(mocker, stdout="media\n\nmedia/temp\n\n", stderr="", return_code=0)
+
+    assert list_dataset_names() == ["media", "media/temp"]
+
+
+def test_list_dataset_names_raises_on_failure(mocker: MockerFixture) -> None:
+    """Never return a partial list: the reconciler would create everything."""
+    patch_run(mocker, stdout="", stderr="cannot open 'media': no such pool\n", return_code=1)
+
+    with pytest.raises(RuntimeError) as excinfo:
+        list_dataset_names()
+
+    assert "Failed to list ZFS datasets" in str(excinfo.value)
+    assert "no such pool" in str(excinfo.value)
+
+
+def test_list_dataset_names_never_parses_stderr_as_names(mocker: MockerFixture) -> None:
+    """The regression this guards: error text read as a dataset list."""
+    patch_run(mocker, stdout="", stderr="permission denied\n", return_code=1)
+
+    with pytest.raises(RuntimeError):
+        list_dataset_names()
+
+
+# -- get_properties -----------------------------------------------------------
+
+
+def test_get_properties_builds_the_right_command(mocker: MockerFixture) -> None:
+    mock_run = patch_run(mocker, stdout="", stderr="", return_code=0)
+
+    get_properties("media/temp")
+
+    mock_run.assert_called_once_with("get", "-Hp", "-o", "property,value,source", "all", "media/temp")
+
+
+def test_get_properties_parses_value_and_source(mocker: MockerFixture) -> None:
+    patch_run(
+        mocker,
+        stdout="compression\tzstd-9\tlocal\natime\toff\tinherited from media\nexec\ton\tdefault\n",
+        stderr="",
+        return_code=0,
+    )
+
+    assert get_properties("media/temp") == {
+        "compression": ("zstd-9", "local"),
+        "atime": ("off", "inherited from media"),
+        "exec": ("on", "default"),
+    }
+
+
+def test_get_properties_skips_blank_lines(mocker: MockerFixture) -> None:
+    """A stray blank line in the middle must not raise on unpacking."""
+    patch_run(mocker, stdout="compression\tzstd\tlocal\n\natime\toff\tlocal\n", stderr="", return_code=0)
+
+    assert get_properties("media/temp") == {
+        "compression": ("zstd", "local"),
+        "atime": ("off", "local"),
+    }
+
+
+def test_get_properties_raises_on_failure(mocker: MockerFixture) -> None:
+    patch_run(mocker, stdout="", stderr="dataset does not exist\n", return_code=1)
+
+    with pytest.raises(RuntimeError) as excinfo:
+        get_properties("media/gone")
+
+    assert "Failed to get properties for media/gone" in str(excinfo.value)
+
+
+# -- create_dataset -----------------------------------------------------------
+
+
+def test_create_dataset_builds_sorted_option_flags(mocker: MockerFixture) -> None:
+    mock_run = patch_run(mocker, stdout="", stderr="", return_code=0)
+
+    assert create_dataset("media/temp", {"sync": "disabled", "compression": "zstd-9"}) is None
+
+    mock_run.assert_called_once_with(
+        "create",
+        "-o",
+        "compression=zstd-9",
+        "-o",
+        "sync=disabled",
+        "media/temp",
+    )
+
+
+def test_create_dataset_with_no_properties(mocker: MockerFixture) -> None:
+    mock_run = patch_run(mocker, stdout="", stderr="", return_code=0)
+
+    create_dataset("media/temp", {})
+
+    mock_run.assert_called_once_with("create", "media/temp")
+
+
+def test_create_dataset_keeps_a_value_with_spaces_together(mocker: MockerFixture) -> None:
+    mock_run = patch_run(mocker, stdout="", stderr="", return_code=0)
+
+    create_dataset("media/temp", {"mountpoint": "/zfs/two words"})
+
+    assert mock_run.call_args.args == ("create", "-o", "mountpoint=/zfs/two words", "media/temp")
+
+
+def test_create_dataset_returns_the_error_on_failure(mocker: MockerFixture) -> None:
+    patch_run(mocker, stdout="", stderr="cannot create 'media/temp': out of space\n", return_code=1)
+
+    error = create_dataset("media/temp", {})
+
+    assert error is not None
+    assert "Failed to create media/temp" in error
+    assert "out of space" in error
+
+
+# -- set_property -------------------------------------------------------------
+
+
+def test_set_property_builds_the_right_command(mocker: MockerFixture) -> None:
+    mock_run = patch_run(mocker, stdout="", stderr="", return_code=0)
+
+    assert set_property("media/temp", "compression", "zstd-9") is None
+
+    mock_run.assert_called_once_with("set", "compression=zstd-9", "media/temp")
+
+
+def test_set_property_keeps_a_value_with_spaces_together(mocker: MockerFixture) -> None:
+    mock_run = patch_run(mocker, stdout="", stderr="", return_code=0)
+
+    set_property("media/temp", "mountpoint", "/zfs/two words")
+
+    assert mock_run.call_args.args == ("set", "mountpoint=/zfs/two words", "media/temp")
+
+
+def test_set_property_returns_the_error_on_failure(mocker: MockerFixture) -> None:
+    patch_run(mocker, stdout="", stderr="permission denied\n", return_code=1)
+
+    error = set_property("media/temp", "compression", "zstd-9")
+
+    assert error is not None
+    assert "Failed to set compression=zstd-9 on media/temp" in error
+    assert "permission denied" in error
+
+
+# -- run_zpool, the same wrapper the zfs side uses ----------------------------
+
+
+def test_run_zpool_prefixes_the_binary(mocker: MockerFixture) -> None:
+    mock_run = mocker.patch(f"{COMMAND}.subprocess.run", return_value=completed(stdout="{}"))
+
+    run_zpool("list", "media", "-pHj", "-o", "all")
+
+    mock_run.assert_called_once_with(
+        ["zpool", "list", "media", "-pHj", "-o", "all"],
+        capture_output=True,
+        encoding="utf-8",
+        check=False,
+    )
+
+
+def test_run_zpool_reports_failures(mocker: MockerFixture) -> None:
+    mocker.patch(f"{COMMAND}.subprocess.run", return_value=completed(returncode=1, stderr="no such pool\n"))
+
+    result = run_zpool("list", "gone")
+
+    assert not result.ok
+    assert result.message == "no such pool"
+    assert result.args == ("zpool", "list", "gone")
+
+
+def test_run_zpool_keeps_streams_separate(mocker: MockerFixture) -> None:
+    """The same contamination bug the zfs side had must not exist here."""
+    mocker.patch(
+        f"{COMMAND}.subprocess.run",
+        return_value=completed(stdout="{}", stderr="warning: pool is degraded\n"),
+    )
+
+    result = run_zpool("list")
+
+    assert result.ok
+    assert result.stdout == "{}"
+
+
+def test_run_zpool_keeps_values_containing_spaces_intact(mocker: MockerFixture) -> None:
+    mock_run = mocker.patch(f"{COMMAND}.subprocess.run", return_value=completed())
+
+    run_zpool("set", "comment=two words", "media")
+
+    assert mock_run.call_args.args[0] == ["zpool", "set", "comment=two words", "media"]
+
+
+def test_run_command_runs_what_it_is_given(mocker: MockerFixture) -> None:
+    """zfs and zpool are the same function with a different first argument."""
+    mock_run = mocker.patch(f"{COMMAND}.subprocess.run", return_value=completed())
+
+    run_command("zpool", "status")
+
+    assert mock_run.call_args.args[0] == ["zpool", "status"]
+
+
+def test_zpool_list_raises_when_the_command_fails(mocker: MockerFixture) -> None:
+    """A failed zpool must never have its error text decoded as json."""
+    mocker.patch(
+        f"{ZPOOL}.run_zpool",
+        return_value=CommandResult(args=(), stdout="", stderr="no such pool: media\n", return_code=1),
+    )
+
+    with pytest.raises(RuntimeError) as excinfo:
+        _zpool_list("list", "media", "-pHj", "-o", "all")
+
+    assert "Failed to run zpool" in str(excinfo.value)
+    assert "no such pool" in str(excinfo.value)
+
+
+def test_zpool_builds_the_right_command(mocker: MockerFixture) -> None:
+    """Zpool passes argv through rather than a formatted string."""
+    mock_run = mocker.patch(
+        f"{ZPOOL}.run_zpool",
+        return_value=CommandResult(
+            args=(),
+            stdout=json.dumps(SAMPLE_ZPOOL_DATA),
+            stderr="",
+            return_code=0,
+        ),
+    )
+
+    Zpool("testpool")
+
+    mock_run.assert_called_once_with("list", "testpool", "-pHj", "-o", "all")

users/richie/home/app_image_path.nix

@@ -0,0 +1,5 @@
+{
+  home.sessionPath = [
+    "/home/richie/app_images/"
+  ];
+}

users/richie/home/global.nix

@@ -1,22 +1,7 @@
-{ config, ... }:
 {
   imports = [
-    ./cli
+    ./minimal.nix
     ./programs.nix
     ./ssh_config.nix
   ];
-
-  programs = {
-    home-manager.enable = true;
-    git.enable = true;
-  };
-
-  home = {
-    username = "richie";
-    homeDirectory = "/home/${config.home.username}";
-    stateVersion = "24.05";
-    sessionVariables = {
-      FLAKE = "$HOME/dotfiles";
-    };
-  };
 }

users/richie/home/gui/default.nix

@@ -7,6 +7,7 @@
     ./firefox
     ./kitty.nix
     ./llm_tools.nix
+    ./t3_code
     ./vscode
   ];
 
@@ -26,5 +27,8 @@
     gparted
     jetbrains.datagrip
     proxychains
+    # hardware tools
+    kicad-unstable
+    openscad
   ];
 }

users/richie/home/gui/t3_code/default.nix

@@ -0,0 +1,36 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  t3-code = pkgs.writeShellApplication {
+    name = "t3-code";
+    runtimeInputs = with pkgs; [
+      coreutils
+      kdePackages.kdialog
+    ];
+    text = builtins.readFile ./launch.sh;
+  };
+in
+{
+  home = {
+    # AppImages are runnable from a shell as well
+    sessionPath = [ "${config.home.homeDirectory}/app_images" ];
+    packages = [ t3-code ];
+  };
+
+  # KDE builds its menu from desktop entries, not from PATH
+  xdg.desktopEntries.t3-code = {
+    name = "T3 Code";
+    genericName = "Code Editor";
+    comment = "Newest T3 Code AppImage in ~/app_images";
+    exec = "${lib.getExe t3-code} %U";
+    icon = "${./icon.png}";
+    terminal = false;
+    categories = [ "Development" ];
+    startupNotify = true;
+    settings.StartupWMClass = "t3code";
+  };
+}

users/richie/home/gui/t3_code/launch.sh

@@ -0,0 +1,25 @@
+# Launch the newest T3 Code AppImage in the app image directory.
+#
+# The nightly builds carry the version in the file name, so the file to run is
+# resolved at launch time rather than baked into the desktop entry.
+# No shebang: this is wrapped by writeShellApplication, which supplies one.
+
+dir="${T3_CODE_DIR:-$HOME/app_images}"
+
+shopt -s nullglob
+images=("$dir"/T3-Code-*.AppImage)
+
+if [ "${#images[@]}" -eq 0 ]; then
+  msg="No T3 Code AppImage found in $dir"
+  echo "$msg" >&2
+  # launched from KDE there is no terminal to read, so say it on screen too
+  kdialog --error "$msg" || true
+  exit 1
+fi
+
+img="$(printf '%s\n' "${images[@]}" | sort -V | tail -n1)"
+[ -x "$img" ] || chmod +x "$img"
+
+# --no-sandbox matches the AppImage's own desktop entry; binfmt hands the
+# AppImage off to appimage-run.
+exec "$img" --no-sandbox "$@"

users/richie/home/gui/vscode/settings.json

@@ -35,7 +35,7 @@
   "sqltools.connections": [
     {
       "previewLimit": 50,
-      "server": "192.168.90.40",
+      "server": "jeeves.taile39c31.ts.net",
       "port": 5432,
       "askForPassword": true,
       "driver": "PostgreSQL",

users/richie/home/minimal.nix

@@ -0,0 +1,27 @@
+{ config, pkgs, ... }:
+{
+  imports = [
+    ./cli
+  ];
+
+  programs = {
+    home-manager.enable = true;
+  };
+
+  home = {
+    username = "richie";
+    homeDirectory = "/home/${config.home.username}";
+    stateVersion = "24.05";
+    sessionVariables.FLAKE = "$HOME/dotfiles";
+    packages = with pkgs; [
+      # cli
+      btop
+      eza
+      ripgrep
+      starship
+      tmux
+      # networking
+      wget
+    ];
+  };
+}

users/richie/home/programs.nix

@@ -3,8 +3,6 @@
   home.packages = with pkgs; [
     # cli
     bat
-    btop
-    eza
     fd
     ffmpegthumbnailer
     fzf
@@ -18,9 +16,6 @@
     p7zip
     poppler
     rar
-    ripgrep
-    starship
-    tmux
     unzip
     yazi
     zoxide
@@ -37,19 +32,25 @@
     # networking
     iperf3
     nmap
-    wget
     # python
     ruff
     uv
     # nodejs
     nodejs
     # Rust packages
+    bacon
+    cargo
+    cargo-audit
+    cargo-generate
+    cargo-machete
+    cargo-update
+    cargo-watch
+    clippy
+    rust-analyzer
+    rustc
+    rustfmt
     trunk
     wasm-pack
-    cargo-watch
-    cargo-generate
-    cargo-audit
-    cargo-update
     # cpp
     clang-tools
     clang_20

users/richie/home/ssh_config.nix

@@ -4,7 +4,7 @@
     enableDefaultConfig = false;
     settings = {
       jeeves = {
-        HostName = "192.168.90.40";
+        HostName = "jeeves";
         User = "richie";
         IdentityFile = "~/.ssh/id_ed25519";
         Port = 629;
@@ -18,7 +18,7 @@
         Port = 2222;
       };
       brain = {
-        HostName = "192.168.90.35";
+        HostName = "brain";
         User = "richie";
         IdentityFile = "~/.ssh/id_ed25519";
         Port = 129;
@@ -31,18 +31,30 @@
         Port = 2222;
       };
       bob = {
-        HostName = "192.168.90.25";
+        HostName = "bob";
         User = "richie";
         IdentityFile = "~/.ssh/id_ed25519";
         Port = 262;
         DynamicForward = [ { port = 9050; } ];
       };
       rhapsody-in-green = {
-        HostName = "192.168.90.221";
+        HostName = "rhapsody-in-green";
         User = "richie";
         IdentityFile = "~/.ssh/id_ed25519";
         Port = 922;
       };
+      portal-1 = {
+        HostName = "portal-1";
+        User = "richie";
+        IdentityFile = "~/.ssh/id_ed25519";
+        Port = 278;
+      };
+      "gitea.tmmworkshop.com" = {
+        HostName = "gitea.tmmworkshop.com";
+        User = "gitea";
+        IdentityFile = "~/.ssh/id_ed25519";
+        Port = 2223;
+      };
     };
   };
 }

users/richie/systems/portal-1.nix

@@ -0,0 +1,5 @@
+{
+  imports = [
+    ../home/minimal.nix
+  ];
+}

users/richie/systems/rhapsody-in-green.nix

@@ -1,5 +1,6 @@
 {
   imports = [
+    ../home/app_image_path.nix
     ../home/global.nix
     ../home/gui
   ];

users/secrets.yaml

@@ -1,62 +1,64 @@
 richie_password: ENC[AES256_GCM,data:DMi3M8aqrQ60APIofr8wJMh+VZ14hLRxz6jWZgzswr0pV/QVSX53ShBFr90ruO3mucOLYv0l+bI31covfqMAhXWBJp9wUgtC2Q==,iv:qgtn30hZfIL4dBnQSLkjbo7zPJA4m9TR0f52sTFc0v4=,tag:ydLbcGyXjv0fE+4b5ECX5w==,type:str]
 math_password: ENC[AES256_GCM,data:ykiSr3iBHrShJarEQSJ/zuXbCPcbW2oUpaAjblu1V15ufFKVSMZM94LlpMiCYtN9cYBLs98hcMeajJbvgbwT5emPHthy9+TJDw==,iv:1TJEUo0ishqFAZiUE1473yR3RT6Gbtqt4zM+C1a1KEk=,tag:pR6jyIj+bu3XaSx5yIHSmA==,type:str]
+tailscale_auth_key: ENC[AES256_GCM,data:SZEfpVnqA84AQLA/geGsPNZfsWBktlM3Zrm7OpuSS/hCWWEKtSIbRYyovH68h8tQJBQFpv18n4oXJe9eAr6Mog==,iv:x1m6XPS/h818qNRcO+uqYjgTfd/FkxHHaSHHJrLyrp4=,tag:eOpcJIhzUWFzXTuRdS9S6w==,type:str]
+tailscale_oauth_client_id: ENC[AES256_GCM,data:+cWxKlLIcJtsLhqmZUNLVDs=,iv:qrTjhLfu82pzbZnU6BrK1n+Ncbm2Cl3MMCSTKCJ6H8U=,tag:s/N5XpLwNH1gZ+LzbqRr2g==,type:str]
 sops:
     age:
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBObHhkaFlnaG4zaTZtbkIw
-            TkxSQnMxbDNwUVo4R1VYRDNKVFRDUE9kb0ZjCnpWWElJUVNuNFBsMzZod1ZQY0Fa
-            VXgxNjZ0TUJ3cEt2OTFWL2dGY0txVHMKLS0tIHNQU3Rsanh4dkc3ZElsai9YMFdH
-            cXUzVmFxTUVIOWZVR2Fpa2crdWsrdlkKwdGLfbKWc25qfBKyd/cawiUWv9iepKHN
-            EOp/LdH2GbCfnQSVbxi28ukLHxWqOLdqMm8xSni/Of2PXvMnpdyCyQ==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBEMjQvSzZSY08vRlI0dzBP
+            YytpVlpPSGFHaTI5b2xzUkFFdEY0bVhQdGgwClM3UmM1OElpcEZTcTB0NHpSZ2d6
+            aVBpcjVIaDk3Tko1cm4xYjdkYVQ1L1EKLS0tIE1aQW95U0JIVGc4MGloSXRXc3ZS
+            a3EvU1dJdmxOeVBIS28yT3dHd2ZEYWcKv5TIqRRQs3AXER1WHEx8wotyXr2Ktxsu
+            dwldwEHyDNqwf7UgGOZu7JueeZ7y2KC8KZ7ixIdrhVaoxuLEvCg7VA==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1u8zj599elqqvcmhxn8zuwrufsz8w8w366d3ayrljjejljt2q45kq8mxw9c
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6R0djMTArVmFySE5DMnFr
-            WFdBUERFbE1HRVVFd0oyaXJ2eU5HUStBUFN3CnR3ckZ2bkpGZFFScHQwTlBZYTMv
-            cTBXbGVoTXlWczhId05QVjZDRzFqRjQKLS0tIDNIcVBydHYwTGRnVU1mWWk4WTlR
-            eVlwQWgxSG5SdmFrWTlOcFo5eXZONWMKgx4huoSnbkRq0wQbsYgsWUKDTxDGNvYR
-            anVMQg+c7PwDlk1V4JQZ4WrYLx63Ep5qDjGlN/Ssf2Vo6rAuuKetcA==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSByaE5jb3p3VzZ1dE05Wmlx
+            c1ZoWGhTcTRHRE9oMWtkaCtBY1l1a2QyWWlvCk9MM2FmcEZubXR1REpOcHZ6aHVE
+            UEkzbllZdkxPYzlTMHNod3VWNnhoaTgKLS0tIGVCdEpqUzhFM0swRjVjb0Z6ckFN
+            UzlYb1U4UGJqWE9oczdlNVh1S1lqL3cKSKxaJFhun2ee45I1jC55YKNdfYB8MsS+
+            dgFOFCvn/4uEcUyKZbUf0STjbIZ5eMHl8NVRTupI4tftGiE559M5Wg==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1q47vup0tjhulkg7d6xwmdsgrw64h4ax3la3evzqpxyy4adsmk9fs56qz3y
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2QjZYejFYbCswQjdmaDA0
-            L2ZqUUhtYU12YlpISmxueHQzRG5YL0tQNXh3CndGamMwRzYvUzkvaE9DVnMwTkNC
-            cFMxczZuOUorS3dnVUpjaVdCUjhLZ2MKLS0tIGlDeGRGa3dRbWgzT3NVYjBaYUJ4
-            VW5yeFlvWUZ5MVpNZHA5M1VXR1hxU1kKqii08/MB2aabgP4RQs1ry8AxmFqB8Mn+
-            m7B0u64aziKXLSl0u471wqgD+YGRwNcajXT2pHCy8QWLznzvIMSrxA==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2U01jTVZQT1RRRU1nQmVv
+            Z0lnOGJLN1c0UkNLbnJjV29na3Rub0xzbXhNCjY0aFNuUnFDUnZheDRHT3R5L3Fv
+            ZkJ3QkF2RENKRVdUYjRzRXMrNklvN2cKLS0tIFJzK2o1MmVTWHZMWmVDWlgzeGY5
+            VDg4OGl0NGFYY2VNZGY5UkR1cVdWY0UK1qa3swdfgUw1tnnThRf1sl9f2zTP/d/d
+            iwcenOEryXatVMJjZVWrLC5C7MwtKz+pIBXa98yaSpCDLqjAMwOCPg==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1jhf7vm0005j60mjq63696frrmjhpy8kpc2d66mw044lqap5mjv4snmwvwm
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrSUc3VnllVDFZTm1jRnlP
-            ZDBqelhkeHliZ1VlcjVnblQyeFlWclZTWkZjCjhJQk5EWkVoQjdoMHg4Zko4OU1C
-            NWFnVTIwV3RYbTZSeWttaVRPbjNrMG8KLS0tIDZWUHJSbVVlR0lKUDRtc3VqOTc1
-            cGJ3NDBLem9FNUpnbStYRTlqQStHV2sKwxPe4nTULsU0mVeUh8mhr2KX9U0iT5dL
-            zvHldoQG6mZHgtHK6XI5AQJYf+zUW66OKqNSxAnn+BM20QkAQVZNVw==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTREI2UEZwS0Y0SmEvK1F4
+            Y1hZZjRVWWxMT29mc3ZXcHFDclRUYi96LzA4CkIwZ2h4T2JQRW1OL2hmRTFHWkFY
+            RHZqQUFqU3hvNlRBZHRiSThPVUE3TG8KLS0tIHh0Z2tlNXREczJsa1g2ekZyZjBt
+            dW41aVA3VEVqa1NDMHM0T3FKZFNTN0kKGQPRUMSRtibFVFRZR/igsjgxgDo4ogga
+            dUHBRU65r41er+Wk3lKdtbXJX1g83K8ay7540hNXheOhxFi57trrzA==
             -----END AGE ENCRYPTED FILE-----
           recipient: age13lmqgc3jvkyah5e3vcwmj4s5wsc2akctcga0lpc0x8v8du3fxprqp4ldkv
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0NU9ac2FuRHI1dkQycmc0
-            YlhGK29UeTdiZEZXcWtPUW4rMis4Z2NWYWpJCkVldEdMc3ZTaDFidHpaZk5mM283
-            bTd0RlN5SHVabGhjeXFnSThydnVoem8KLS0tIElWclYvTmtkb0wyNDlKT3ZEckwx
-            L2NObzZadlJ5d3MyeGRqKy95L3BOMFEKtoswi6r2TmCZzngUkiGQV5TTsuzisMFS
-            5QI0aQZwhexqUMvbPuajYKvcPj+D6a2xaxbL3TBRLjOrFmcp5J7/YA==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1l272y8udvg60z7edgje42fu49uwt4x2gxn5zvywssnv9h2krms8s094m4k
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSByUlVjM2xpc281bHZzOGVo
-            L3VEclhJZDd5SS9mazFiTk9DcmxMaWxPT213ClNPWERKQU03OWk0OEVIY05ib2VG
-            WXhiZDhuMDZ5ZmFPWHB5RTFKYmpkVzAKLS0tIGppUndCb25wb2dyV01YbENrWjdU
-            TmMvWVpobnl0eXBIOGQwMW5BSlhJTUkKzua1artJWbZlKfzv27xfZJeBpntBYwUf
-            c8i1gNlvRwkhFAlrWcKR65vgyxsO3rbkLJRkcwG/q4hHj9zBeC/K2A==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrV1Z2MGNPK28wTStRSUZW
+            dmFMaUt5MENHeUpVcDVxQnZhd1o5RmJwOWwwCittaGNrcW1hV25VaGJaTEhZRFBJ
+            Y2toWEpFWFZYT0plb2lkTlJCS2Mwb3MKLS0tIHlkVjVESWFsbWlVQ0xVQ0llclF2
+            L1FMbC91YjJsR3BGRjJ0QVQ1MDdBbEUKNnOI9+H1LJcvVcwl9uQMmFp35y5RB+Zl
+            Mo3z3unweQtS+jiUn7RyVs5daG1E2tfihu2CQDWaktuCuunMpt3Wew==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1ufnewppysaq2wwcl4ugngjz8pfzc5a35yg7luq0qmuqvctajcycs5lf6k4
-    lastmodified: "2026-06-29T20:19:44Z"
-    mac: ENC[AES256_GCM,data:GIvQxWt4tZGn0fyiXVtxGFQQoNcFUgilF+/PSz50exVrmzsS0XQUk/TIDFHaQR9jlJI50jqlyc1rBHgjnqC2oPHhPWaaVhgF18vQI55rGKdymNFjsHnaCkblFVdR1RJm0FSB2Ri6y5k8tfN3ywiwromJRz4NYzr1hbmr36azfg4=,iv:V3jspeYt/d2wy13gUrQmPGARm0hxwvSL/mocJAUofdw=,tag:vARoUzBWTJKkONDGoQdzNQ==,type:str]
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqR0hUT3hxTzRwTVhuTm9E
+            TUFtNFFCYWNUNUNVbHNaNXQ0aEVBa04vSEU0CkRPVzhuYUsrM0wxZjRDdkxvTVA2
+            UW9CSjlIQW5yenhaUi9mbEdLYlhWYk0KLS0tIGRXMm9Wa3l4UWxqcGl6ZlBZV3E0
+            Z3YrbUYrSmV0NHNTeTdhMkJEdytwSmMKGLpe5+p4FGJNHy6SVgnsDbEsiP0J8lW/
+            0+3X3LQoL0f1S2L5DjexF/H0NGvWjAPaGzDNB6V50VxGDabEIVqjvQ==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1vyav6kxtvt3z4vtnkkjj38eu8hlts5m7ygyckhskvalg2gpjk52su53d0a
+    lastmodified: "2026-08-25T15:35:53Z"
+    mac: ENC[AES256_GCM,data:4/WuSmUb2ONlB56dgbwwekZ8h7E1vvPzPxYCvNM8N68nt1nb9JxoMeAX9PktmljAarYLQJd1O5+49ZEph3Gzx9LrwvWnm9asq0UeeJi08x7wdQDcawNFlBFzXF2qlgxU1JssNHZZOB+5nYH4t8vFzPTq/9BqoifPR2+pLBYhUQI=,iv:N/gVgeWT2/1bfHuR6TanyDBijWmXAqyACXpnwfwDuVA=,tag:pQ4ct4G7SARz2wyjOc7AKQ==,type:str]
     unencrypted_suffix: _unencrypted
-    version: 3.13.1
+    version: 3.13.3