feat(jeeves): share one system closure across builders

Reuse a single NixOS container build for all runners while preserving
unique runtime hostnames and existing Gitea registrations.

.github/workflows/merge_flake_lock_update.yml

@@ -15,6 +15,7 @@ jobs:
     steps:
       - name: merge_flake_lock_update
         run: >-
+          nix develop .#devShells.x86_64-linux.default -c
           python -m python.gitea_flake_lock merge
           --repo "${{ github.repository }}"
         env:

.github/workflows/pytest.yml

@@ -0,0 +1,19 @@
+name: pytest
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  pytest:
+    runs-on: self-hosted
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run tests
+        run: nix develop .#devShells.x86_64-linux.default -c pytest tests

.github/workflows/update-flake-lock.yml

@@ -21,5 +21,6 @@ jobs:
           JEEVES_BOT_TOKEN: ${{ secrets.JEEVES_BOT_TOKEN }}
           GITEA_URL: https://gitea.tmmworkshop.com
         run: >-
+          nix develop .#devShells.x86_64-linux.default -c
           python -m python.gitea_flake_lock update
           --repo "${{ github.repository }}"

common/global/nix.nix

@@ -31,10 +31,6 @@ in
         "flakes"
         "ca-derivations"
       ];
-      system-features = lib.mkAfter [
-        "gccarch-x86-64-v2"
-        "gccarch-x86-64-v3"
-      ];
       warn-dirty = false;
       flake-registry = ""; # disable global flake registries
       connect-timeout = 10;

common/global/programs.nix

@@ -2,5 +2,6 @@
 {
   environment.systemPackages = with pkgs; [
     git
+    my_python
   ];
 }

common/optional/docker.nix

@@ -17,7 +17,6 @@
     logDriver = "local";
     storageDriver = "overlay2";
     daemon.settings = {
-      live-restore = false;
       experimental = true;
       exec-opts = [ "native.cgroupdriver=systemd" ];
       log-opts = {

common/optional/x86-64-v3.nix

@@ -1,9 +0,0 @@
-{
-  nixpkgs.hostPlatform = {
-    system = "x86_64-linux";
-    gcc = {
-      arch = "x86-64-v3";
-      tune = "generic";
-    };
-  };
-}

common/optional/zfs/snapshot.nix

@@ -1,5 +1,4 @@
 {
-  inputs,
   pkgs,
   lib,
   config,
@@ -7,11 +6,6 @@
 }:
 let
   cfg = config.services.snapshot_manager;
-  snapshotManagerPackages =
-    ps: with ps; [
-      httpx
-      typer
-    ];
 in
 {
   options = {
@@ -40,41 +34,6 @@ in
   };
 
   config = lib.mkIf cfg.enable {
-    nixpkgs.overlays = [
-      (final: _prev: {
-        snapshot_manager_python = final.python314.withPackages snapshotManagerPackages;
-        snapshot_manager_test_python = final.python314.withPackages (
-          ps:
-          snapshotManagerPackages ps
-          ++ (with ps; [
-            pyfakefs
-            pytest
-            pytest-asyncio
-            pytest-mock
-            pytest-xdist
-          ])
-        );
-        snapshot_manager_tests =
-          final.runCommand "snapshot-manager-tests"
-            {
-              nativeBuildInputs = [ final.snapshot_manager_test_python ];
-            }
-            ''
-              export HOME="$TMPDIR"
-              cd ${inputs.self}
-              pytest \
-                -o cache_dir="$TMPDIR/pytest-cache" \
-                tests/test_common.py \
-                tests/test_signal_alert.py \
-                tests/test_snapshot_manager.py \
-                tests/test_zfs.py
-              touch "$out"
-            '';
-      })
-    ];
-
-    system.checks = [ pkgs.snapshot_manager_tests ];
-
     systemd = {
       services.snapshot_manager = {
         description = "ZFS Snapshot Manager";
@@ -86,7 +45,7 @@ in
         };
         serviceConfig = {
           Type = "oneshot";
-          ExecStart = "${pkgs.snapshot_manager_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
+          ExecStart = "${pkgs.my_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}";
         }
         // lib.optionalAttrs (cfg.EnvironmentFile != null) {
           EnvironmentFile = cfg.EnvironmentFile;

common/optional/zfs/snapshot_config.toml

@@ -22,12 +22,6 @@ hourly = 0
 daily = 0
 monthly = 0
 
-["root_pool/nix_build"]
-15_min = 1
-hourly = 0
-daily = 0
-monthly = 0
-
 ["root_pool/var"]
 15_min = 8
 hourly = 24

flake.lock

@@ -20,6 +20,28 @@
         "type": "github"
       }
     },
+    "firefox-addons": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "dir": "pkgs/firefox-addons",
+        "lastModified": 1787025780,
+        "narHash": "sha256-NhyLP9G4DFOn/7aYr7K/D7hWrzEGr5EgUBV+lpdmJ24=",
+        "owner": "rycee",
+        "repo": "nur-expressions",
+        "rev": "5ad360b6d3cb0aa1b61f9cb27fef113ca9117c37",
+        "type": "gitlab"
+      },
+      "original": {
+        "dir": "pkgs/firefox-addons",
+        "owner": "rycee",
+        "repo": "nur-expressions",
+        "type": "gitlab"
+      }
+    },
     "home-manager": {
       "inputs": {
         "nixpkgs": [
@@ -27,11 +49,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1788651960,
+        "lastModified": 1786999651,
-        "narHash": "sha256-v9wJd32eZ2bvhBzVOd7TIjLQd011P7nwOhjKtWlci5I=",
+        "narHash": "sha256-MTGMFlLDTklsXhCp4r5GXB4VAVadPdalXLvUjd/K7h0=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "2c0350c759688177331b8f5242311fae8877bdb3",
+        "rev": "353742587cbaf079b3caee743115d037bc51fea6",
         "type": "github"
       },
       "original": {
@@ -45,11 +67,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1788860136,
+        "lastModified": 1786867632,
-        "narHash": "sha256-MhPMOFV4pVkygWEbQ8t1De/uQ9cWF1u++tRe2L5tG48=",
+        "narHash": "sha256-ez+ubZlA1RtdjCB18a6zJ9M4u8qoPDy08EcnsW5M3Xw=",
         "owner": "nixos",
         "repo": "nixos-hardware",
-        "rev": "62173785b9a18c78b4a15aca2623d02bceb9d077",
+        "rev": "ff17823245ab9ff7bcae6acf950bd89cba82c38c",
         "type": "github"
       },
       "original": {
@@ -74,11 +96,11 @@
     },
     "nixpkgs-master": {
       "locked": {
-        "lastModified": 1788892992,
+        "lastModified": 1787081018,
-        "narHash": "sha256-cIMFh9gyU4/aLeB3JCcsWM3tTAvD9pAq9Smr1Wa8aIU=",
+        "narHash": "sha256-K0uwZBtZsbBigHAMQW7YWti3gPe6a5ct5bcOw5F+Q9Y=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "dff6994123e257ec9901c271bc2b52e64d7c8f05",
+        "rev": "cacac5ac351a010599d9f9d106acfed25a8e4c77",
         "type": "github"
       },
       "original": {
@@ -106,11 +128,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1788752844,
+        "lastModified": 1787001381,
-        "narHash": "sha256-VaWGJ6+cIYN2erfSecbRV+4ljI185Ty2wUrXyvQbgOw=",
+        "narHash": "sha256-Ue1Yo8gfHdD4TMtNewhA4tkSYeFqXThju0nCyJc3ALo=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "dc5d91f840324650bac8c379428c7037a416959a",
+        "rev": "ec2d622de0773551768cf98f3fc50cbcc003b9c5",
         "type": "github"
       },
       "original": {
@@ -123,6 +145,7 @@
     "root": {
       "inputs": {
         "disko": "disko",
+        "firefox-addons": "firefox-addons",
         "home-manager": "home-manager",
         "nixos-hardware": "nixos-hardware",
         "nixpkgs": "nixpkgs_2",
@@ -139,11 +162,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1788337237,
+        "lastModified": 1786629091,
-        "narHash": "sha256-gkSH8VUtCo6hnysNmb9DbTuDepH2t5pv+QWjP75xKAk=",
+        "narHash": "sha256-gkig4nPi1CWc4Z50GBsjE4ygSE7hMpl/TwID2an2Cck=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "fbf759290e0cb0a98dfc813a4eb7d53ad1dacb57",
+        "rev": "a8627b21b9107c5711c96b84f32a9a4b3d45295f",
         "type": "github"
       },
       "original": {

flake.nix

@@ -26,6 +26,11 @@
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
+    firefox-addons = {
+      url = "gitlab:rycee/nur-expressions?dir=pkgs/firefox-addons";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
     sops-nix = {
       url = "github:Mic92/sops-nix";
       inputs.nixpkgs.follows = "nixpkgs";

overlays/default.nix

@@ -15,6 +15,34 @@
     };
   };
 
-  test-exclusions = import ./test-exclusions.nix;
+  python-env = final: _prev: {
-  x86-64-v3-workarounds = import ./x86-64-v3-workarounds.nix;
+    my_python = final.python314.withPackages (
+      ps: with ps; [
+        alembic
+        apprise
+        fastapi
+        fastapi-cli
+        httpx
+        jinja2
+        mypy
+        pgvector
+        psycopg
+        pydantic
+        pyfakefs
+        pytest
+        pytest-cov
+        pytest-mock
+        pytest-xdist
+        python-multipart
+        pydantic-settings
+        ruff
+        sqlalchemy
+        tenacity
+        tinytuya
+        typer
+        uvicorn
+        websockets
+      ]
+    );
+  };
 }

overlays/test-exclusions.nix

@@ -1,133 +0,0 @@
-# Test exclusions for the locally rebuilt x86-64-v3 package set.
-#
-# Selecting x86-64-v3 changes every affected derivation, so the normal
-# nixpkgs binary cache cannot be used and upstream test suites run locally.
-# The jeeves builder uses /tmp/nix-builds so filesystem tests run on tmpfs
-# instead of ZFS with normalization=formD and utf8only=on; those tests remain
-# enabled. The remaining workarounds cover UDP readiness, resource-sensitive
-# parser and nested-worker races, and mismatched timeout clocks, plus
-# architecture-dependent floating-point differences whose risk we accept for
-# our workloads. Keep these exceptions visible until their causes are fixed.
-_final: prev: {
-  gnutls = prev.gnutls.overrideAttrs (old: {
-    # This test uses a fixed four-second sleep instead of checking UDP
-    # readiness; the client saw no listener in the x86-64-v3 build.
-    postPatch = (old.postPatch or "") + ''
-      sed '2iexit 77' -i tests/serv-udp.sh
-    '';
-  });
-
-  prometheus = prev.prometheus.overrideAttrs (
-    old:
-    let
-      assets = old.passthru.assets.overrideAttrs (assetsOld: {
-        # CodeMirror's bounded synchronous parser can return an incomplete tree
-        # when these cases run on a heavily loaded builder.
-        postPatch = (assetsOld.postPatch or "") + ''
-          substituteInPlace module/codemirror-promql/src/complete/hybrid.test.ts \
-            --replace-fail "it(value.title, () => {" \
-            "(value.title === 'autocomplete topk params 2' ? it.skip : it)(value.title, () => {"
-          substituteInPlace module/codemirror-promql/src/parser/vector.test.ts \
-            --replace-fail "it(value.binaryExpr, () => {" \
-            "(value.binaryExpr === 'foo * on(test,blub) bar' ? it.skip : it)(value.binaryExpr, () => {"
-        '';
-      });
-    in
-    {
-      postPatch = builtins.replaceStrings [ "${old.passthru.assets}" ] [ "${assets}" ] (
-        builtins.unsafeDiscardStringContext old.postPatch
-      );
-      passthru = old.passthru // {
-        inherit assets;
-      };
-    }
-  );
-
-  pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
-    (_pythonFinal: pythonPrev: {
-      backrefs = pythonPrev.backrefs.overridePythonAttrs (old: {
-        # regex measures its timeout in process CPU time, while this test used
-        # wall time and could miss the timeout when a busy builder descheduled it.
-        postPatch = (old.postPatch or "") + ''
-          substituteInPlace tests/test_bregex.py \
-            --replace-fail "time.time()" "time.process_time()"
-        '';
-      });
-
-      pytest-xdist = pythonPrev.pytest-xdist.overridePythonAttrs (old: {
-        # The suite exercises its own worker pools. Run the outer suite with one
-        # worker and allow inner workers more time on heavily loaded builders.
-        postPatch = (old.postPatch or "") + ''
-          substituteInPlace testing/test_remote.py \
-            --replace-fail "WAIT_TIMEOUT = 10.0" "WAIT_TIMEOUT = 60.0"
-        '';
-        preCheck = builtins.replaceStrings [ "--numprocesses=$NIX_BUILD_CORES" ] [ "--numprocesses=1" ] (
-          old.preCheck or ""
-        );
-        # This test deliberately crashes workers past the restart limit and
-        # races while checking which replacement message was emitted.
-        disabledTests = (old.disabledTests or [ ]) ++ [
-          "test_max_worker_restart_tests_queued"
-        ];
-      });
-
-      jupyter-server = pythonPrev.jupyter-server.overridePythonAttrs (old: {
-        # The kernel reply arrived after the one-second outer deadline when the
-        # builder was heavily loaded. Keep the regression test but allow it the
-        # same margin as the other resource-sensitive tests.
-        postPatch = (old.postPatch or "") + ''
-          substituteInPlace tests/services/kernels/test_connection.py \
-            --replace-fail \
-            "await asyncio.wait_for(asyncio.wrap_future(conn2.request_kernel_info()), timeout=1.0)" \
-            "await asyncio.wait_for(asyncio.wrap_future(conn2.request_kernel_info()), timeout=10.0)"
-        '';
-      });
-
-      scipy = pythonPrev.scipy.overridePythonAttrs (old: {
-        # x86-64-v3 FFT implementations produce rounding differences outside
-        # these tests' strict tolerances. We accept the numerical-precision
-        # risk for our workloads.
-        disabledTests = (old.disabledTests or [ ]) ++ [
-          "test_roundtrip_float32"
-          "test_roundtrip_scaling"
-        ];
-      });
-
-      sentry-sdk = pythonPrev.sentry-sdk.overridePythonAttrs (old: {
-        # This test globally mocks threading.current_thread while another
-        # thread is running. On Python 3.14, Thread.join can race with that
-        # mock and exhaust its single side effect before the worker removes it.
-        disabledTests = (old.disabledTests or [ ]) ++ [
-          "test_get_current_thread_meta_main_thread"
-        ];
-      });
-
-      torchaudio = pythonPrev.torchaudio.overridePythonAttrs (old: {
-        # x86-64-v3 pitch shifting produces batch-versus-single-item numerical
-        # differences up to 2.9e-6. We accept that audio-precision risk for our
-        # workloads.
-        disabledTests = (old.disabledTests or [ ]) ++ [
-          "test_batch_pitch_shift"
-        ];
-      });
-
-      torchcodec = pythonPrev.torchcodec.overridePythonAttrs (old: {
-        # For these 8 kHz MP3 cases, the x86-64-v3 API and CLI codec paths
-        # differ in 0.8% of decoded samples. Retain the original tolerance for
-        # 99% of samples and accept the localized audio-precision risk.
-        postPatch = (old.postPatch or "") + ''
-          substituteInPlace test/test_encoders.py \
-            --replace-fail \
-            'if sys.platform == "darwin":' \
-            'if sys.platform == "darwin" or (
-                format == "mp3"
-                and sample_rate == 8_000
-                and asset is SINE_MONO_S32
-                and bit_rate in (None, 0)
-                and num_channels in (None, 1)
-            ):'
-        '';
-      });
-    })
-  ];
-}

overlays/x86-64-v3-workarounds.nix

@@ -1,60 +0,0 @@
-# Compatibility fixes for packages rebuilt with x86-64-v3.
-#
-# The v3 baseline enables instructions that expose source assumptions hidden
-# by the generic x86-64 build. Keep compile fixes here, separate from test
-# exclusions, until upstream or nixpkgs incorporates them.
-_final: prev:
-let
-  patchAbseilBmi2Include =
-    package:
-    package.overrideAttrs (old: {
-      # GCC and Clang prohibit including their internal BMI2 header directly.
-      # The public umbrella provides the same intrinsics with the required
-      # compiler setup.
-      postPatch = (old.postPatch or "") + ''
-        substituteInPlace third_party/abseil-cpp/absl/container/internal/raw_hash_set.h \
-          --replace-fail "#include <bmi2intrin.h>" "#include <immintrin.h>"
-      '';
-    });
-
-  removeSiblingOutputChecks =
-    package:
-    package.overrideAttrs (old: {
-      # Nix 2.34 can validate a partial multi-output rebuild against only the
-      # outputs still being realised. PostgreSQL's checks then reject valid
-      # sibling names such as "out" and "lib". Keep the test suite and
-      # disallowed-requisite checks; accept the loss of cross-output checks.
-      outputChecks = builtins.mapAttrs (
-        _output: checks: builtins.removeAttrs checks [ "disallowedReferences" ]
-      ) (old.outputChecks or { });
-    });
-
-  electron43Unwrapped = patchAbseilBmi2Include prev.electron_43.unwrapped;
-  electron43 = prev.electron_43.override {
-    electron-unwrapped = electron43Unwrapped;
-  };
-
-  signalCallPackage =
-    path: args:
-    let
-      package = prev.callPackage path args;
-    in
-    if builtins.baseNameOf path == "webrtc.nix" then patchAbseilBmi2Include package else package;
-in
-prev.lib.optionalAttrs ((prev.stdenv.hostPlatform.gcc.arch or null) == "x86-64-v3") {
-  deno =
-    let
-      librusty_v8 = patchAbseilBmi2Include prev.deno.passthru.librusty_v8;
-    in
-    prev.deno.override { inherit librusty_v8; };
-
-  electron_43 = electron43;
-
-  postgresql = removeSiblingOutputChecks prev.postgresql;
-  postgresql_18 = removeSiblingOutputChecks prev.postgresql_18;
-
-  signal-desktop = prev.signal-desktop.override {
-    electron_43 = electron43;
-    callPackage = signalCallPackage;
-  };
-}

pyproject.toml

@@ -6,6 +6,49 @@ authors = [{ name = "Richie Cahill", email = "richie@tmmworkshop.com" }]
 requires-python = "~=3.14.0"
 readme = "README.md"
 license = "MIT"
+# these dependencies are a best effort and aren't guaranteed to work
+# for up-to-date dependencies, see overlays/default.nix
+dependencies = [
+    "alembic",
+    "apprise",
+    "beautifulsoup4",
+    "bm25s",
+    "ebooklib",
+    "fastapi",
+    "fastapi-cli",
+    "httpx",
+    "jinja2",
+    "pgvector",
+    "psycopg[binary]",
+    "pydantic",
+    "pydantic-settings",
+    "python-multipart",
+    "sqlalchemy[asyncio]",
+    "tenacity",
+    "tiktoken",
+    "tinytuya",
+    "typer",
+    "uvicorn",
+    "websockets",
+    "yake",
+]
+
+[project.scripts]
+database = "python.database_cli:app"
+whisper-transcribe = "python.tools.whisper.transcribe:main"
+
+[dependency-groups]
+dev = [
+    "aiosqlite",
+    "mypy",
+    "pyfakefs",
+    "pytest-asyncio",
+    "pytest-cov",
+    "pytest-mock",
+    "pytest-xdist",
+    "pytest",
+    "ruff",
+]
 
 [tool.ruff]
 

python/gitea_flake_lock.py

@@ -14,7 +14,7 @@ DEFAULT_BASE_BRANCH = "main"
 DEFAULT_BRANCH = "automation/update-flake-lock"
 DEFAULT_GITEA_URL = "https://gitea.tmmworkshop.com"
 PR_LABELS = ["dependencies", "automated", "flake_lock_update"]
-PR_CHECK_WORKFLOWS = ["build_systems.yml", "treefmt.yml"]
+PR_CHECK_WORKFLOWS = ["build_systems.yml", "treefmt.yml", "pytest.yml"]
 PR_TITLE = "Update flake.lock"
 PR_BODY = "Automated flake.lock update."
 

python/installer/__main__.py

@@ -176,21 +176,6 @@ def create_zfs_datasets() -> None:
     run_output(("zfs", "create", "root_pool/home"))
     run_output(("zfs", "create", "-o", "reservation=1G", "root_pool/var"))
     run_output(("zfs", "create", "-o", "compression=zstd-9", "-o", "reservation=10G", "root_pool/nix"))
-    run_output(
-        (
-            "zfs",
-            "create",
-            "-o",
-            "sync=disabled",
-            "-o",
-            "redundant_metadata=some",
-            "-o",
-            "normalization=none",
-            "-o",
-            "utf8only=off",
-            "root_pool/nix_build",
-        )
-    )
     datasets = run_output(("zfs", "list", "-o", "name"))
 
     expected_datasets = {
@@ -198,7 +183,6 @@ def create_zfs_datasets() -> None:
         "root_pool/home",
         "root_pool/var",
         "root_pool/nix",
-        "root_pool/nix_build",
     }
     missing_datasets = expected_datasets.difference(datasets.splitlines())
     if missing_datasets:
@@ -264,7 +248,6 @@ def create_nix_hardware_file(mnt_dir: str, disks: Sequence[str], encrypt: str |
         '    "/home" = {\n      device = "root_pool/home";\n      fsType = "zfs";\n    };\n\n'
         '    "/var" = {\n      device = "root_pool/var";\n      fsType = "zfs";\n    };\n\n'
         '    "/nix" = {\n      device = "root_pool/nix";\n      fsType = "zfs";\n    };\n\n'
-        '    "/nix/var/nix/builds" = {\n      device = "root_pool/nix_build";\n      fsType = "zfs";\n    };\n\n'
         '    "/boot" = {\n'
         f'      device = "/dev/disk/by-uuid/{get_boot_drive_id(disks[0])}";\n'
         '      fsType = "vfat";\n      options = [\n        "fmask=0077"\n'
@@ -287,7 +270,6 @@ def install_nixos(mnt_dir: str, disks: Sequence[str], encrypt: str | None) -> No
     run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/home", f"{mnt_dir}/home"))
     run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/var", f"{mnt_dir}/var"))
     run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/nix", f"{mnt_dir}/nix"))
-    run_output(("mount", "-o", "X-mount.mkdir", "-t", "zfs", "root_pool/nix_build", f"{mnt_dir}/nix/var/nix/builds"))
 
     for disk in disks:
         run_output(("mkfs.vfat", "-n", "EFI", f"{disk}-part1"))

python/signal_alert.py

@@ -5,13 +5,10 @@ from __future__ import annotations
 import logging
 from os import getenv
 
-import httpx
+from apprise import Apprise
 
 logger = logging.getLogger(__name__)
 
-SIGNAL_API_URL = "http://localhost:8989/v2/send"
-SIGNAL_API_TIMEOUT = 4.0
-
 
 def signal_alert(body: str, title: str = "") -> None:
     """Send a signal alert.
@@ -20,31 +17,14 @@ def signal_alert(body: str, title: str = "") -> None:
         body (str): The body of the alert.
         title (str, optional): The title of the alert. Defaults to "".
     """
+    apprise_client = Apprise()
+
     from_phone = getenv("SIGNAL_ALERT_FROM_PHONE")
     to_phone = getenv("SIGNAL_ALERT_TO_PHONE")
     if not from_phone or not to_phone:
         logger.info("SIGNAL_ALERT_FROM_PHONE or SIGNAL_ALERT_TO_PHONE not set")
         return
 
-    # Apprise's Signal integration did not support titles, so preserve that behavior.
+    apprise_client.add(f"signal://localhost:8989/{from_phone}/{to_phone}")
-    if title:
-        logger.debug("Signal does not support notification titles; ignoring title")
 
-    try:
+    apprise_client.notify(title=title, body=body)
-        response = httpx.post(
-            SIGNAL_API_URL,
-            json={
-                "message": body,
-                "number": from_phone,
-                "recipients": [to_phone],
-                "text_mode": "normal",
-            },
-            timeout=SIGNAL_API_TIMEOUT,
-            follow_redirects=True,
-        )
-    except httpx.HTTPError:
-        logger.exception("Unable to contact the Signal API")
-        return
-
-    if response.status_code not in {httpx.codes.OK, httpx.codes.CREATED}:
-        logger.error("Signal API returned HTTP status %d", response.status_code)

python/system_tests/__init__.py

@@ -0,0 +1 @@
+"""system_tests."""

python/system_tests/components.py

@@ -0,0 +1,99 @@
+"""Validate Jeeves."""
+
+from __future__ import annotations
+
+import logging
+from copy import copy
+from re import search
+from time import sleep
+from typing import TYPE_CHECKING
+
+from python.common import bash_wrapper
+from python.zfs import Zpool
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+logger = logging.getLogger(__name__)
+
+
+def zpool_tests(pool_names: Sequence[str], zpool_capacity_threshold: int = 90) -> list[str] | None:
+    """Check the zpool health and capacity.
+
+    Args:
+        pool_names (Sequence[str]): A list of pool names to test.
+        zpool_capacity_threshold (int, optional): The threshold for the zpool capacity. Defaults to 90.
+
+    Returns:
+        list[str] | None: A list of errors if any.
+    """
+    logger.info("Testing zpool")
+
+    errors: list[str] = []
+    for pool_name in pool_names:
+        pool = Zpool(pool_name)
+        if pool.health != "ONLINE":
+            errors.append(f"{pool.name} is {pool.health}")
+        if pool.capacity >= zpool_capacity_threshold:
+            errors.append(f"{pool.name} is low on space")
+
+    upgrade_status, _ = bash_wrapper("zpool upgrade")
+    if not search(r"Every feature flags pool has all supported and requested features enabled.", upgrade_status):
+        errors.append("ZPool out of date run `sudo zpool upgrade -a`")
+
+    return errors
+
+
+def systemd_tests(
+    service_names: Sequence[str],
+    max_retries: int = 30,
+    retry_delay_secs: int = 1,
+    retryable_statuses: Sequence[str] | None = None,
+    valid_statuses: Sequence[str] | None = None,
+) -> list[str] | None:
+    """Tests a systemd services.
+
+    Args:
+        service_names (Sequence[str]): A list of service names to test.
+        max_retries (int, optional): The maximum number of retries. Defaults to 30.
+            minimum value is 1.
+        retry_delay_secs (int, optional): The delay between retries in seconds. Defaults to 1.
+            minimum value is 1.
+        retryable_statuses (Sequence[str] | None, optional): A list of retryable statuses. Defaults to None.
+        valid_statuses (Sequence[str] | None, optional): A list of valid statuses. Defaults to None.
+
+    Returns:
+        list[str] | None: A list of errors if any.
+    """
+    logger.info("Testing systemd service")
+
+    max_retries = max(max_retries, 1)
+    retry_delay_secs = max(retry_delay_secs, 1)
+    last_try = max_retries - 1
+
+    if retryable_statuses is None:
+        retryable_statuses = ("inactive\n", "activating\n")
+
+    if valid_statuses is None:
+        valid_statuses = ("active\n",)
+
+    service_names_set = set(service_names)
+
+    errors: set[str] = set()
+    for retry in range(max_retries):
+        if not service_names_set:
+            break
+        logger.info(f"Testing systemd service in {retry + 1} of {max_retries}")
+        service_names_to_test = copy(service_names_set)
+        for service_name in service_names_to_test:
+            service_status, _ = bash_wrapper(f"systemctl is-active {service_name}")
+            if service_status in valid_statuses:
+                service_names_set.remove(service_name)
+                continue
+            if service_status in retryable_statuses and retry < last_try:
+                continue
+            errors.add(f"{service_name} is {service_status.strip()}")
+
+        sleep(retry_delay_secs)
+
+    return list(errors)

python/system_tests/validate_system.py

@@ -0,0 +1,67 @@
+"""Validate {server_name}."""
+
+import logging
+import sys
+import tomllib
+from os import environ
+from pathlib import Path  # noqa: TC003 This is required for the typer CLI
+from socket import gethostname
+
+import typer
+
+from python.common import configure_logger
+from python.signal_alert import signal_alert
+from python.system_tests.components import systemd_tests, zpool_tests
+
+logger = logging.getLogger(__name__)
+
+
+def load_config_data(config_file: Path) -> dict[str, list[str]]:
+    """Load a TOML configuration file.
+
+    Args:
+        config_file (Path): The path to the configuration file.
+
+    Returns:
+        dict: The configuration data.
+    """
+    return tomllib.loads(config_file.read_text())
+
+
+def main(config_file: Path) -> None:
+    """Main."""
+    configure_logger(level=environ.get("LOG_LEVEL", "INFO"))
+
+    server_name = gethostname()
+    logger.info(f"Starting {server_name} validation")
+
+    config_data = load_config_data(config_file)
+
+    errors: list[str] = []
+    try:
+        if config_data.get("zpools") and (zpool_errors := zpool_tests(config_data["zpools"])):
+            errors.extend(zpool_errors)
+
+        if config_data.get("services") and (systemd_errors := systemd_tests(config_data["services"])):
+            errors.extend(systemd_errors)
+
+    except Exception as error:
+        logger.exception(f"{server_name} validation failed")
+        errors.append(f"{server_name} validation failed: {error}")
+
+    if errors:
+        logger.error(f"{server_name} validation failed: \n{'\n'.join(errors)}")
+        signal_alert(f"{server_name} validation failed {errors}")
+
+        sys.exit(1)
+
+    logger.info(f"{server_name} validation passed")
+
+
+def cli() -> None:
+    """CLI."""
+    typer.run(main)
+
+
+if __name__ == "__main__":
+    cli()

shell.nix

@@ -8,7 +8,7 @@
   # loader path, which NixOS does not provide globally.
   ebook-search = pkgs.mkShell {
     nativeBuildInputs = with pkgs; [
-      python314
+      my_python
       uv
     ];
     LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
@@ -23,6 +23,7 @@
       nix
       home-manager
       git
+      my_python
 
       ssh-to-age
       gnupg

systems/bob/default.nix

@@ -3,6 +3,7 @@
   imports = [
     "${inputs.self}/users/math"
     "${inputs.self}/users/richie"
+    "${inputs.self}/users/steve"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/docker.nix"
     "${inputs.self}/common/optional/monitoring-agent.nix"
@@ -13,7 +14,7 @@
     "${inputs.self}/common/optional/systemd-boot.nix"
     "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/x86-64-v3.nix"
+    "${inputs.self}/common/optional/yubikey.nix"
     "${inputs.self}/common/optional/zfs"
     ./hardware.nix
     ./syncthing.nix

systems/bob/hardware.nix

@@ -50,11 +50,6 @@
       fsType = "zfs";
     };
 
-    "/nix/var/nix/builds" = {
-      device = "root_pool/nix_build";
-      fsType = "zfs";
-    };
-
     "/var" = {
       device = "root_pool/var";
       fsType = "zfs";

systems/bob/snapshot_config.toml

@@ -22,12 +22,6 @@ hourly = 0
 daily = 0
 monthly = 0
 
-["root_pool/nix_build"]
-15_min = 1
-hourly = 0
-daily = 0
-monthly = 0
-
 ["root_pool/models"]
 15_min = 4
 hourly = 24

systems/brain/default.nix

@@ -9,7 +9,6 @@
     "${inputs.self}/common/optional/systemd-boot.nix"
     "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/x86-64-v3.nix"
     "${inputs.self}/common/optional/zfs"
     ./docker
     ./hardware.nix

systems/brain/hardware.nix

@@ -53,11 +53,6 @@
       fsType = "zfs";
     };
 
-    "/nix/var/nix/builds" = {
-      device = "root_pool/nix_build";
-      fsType = "zfs";
-    };
-
     "/boot" = {
       device = "/dev/disk/by-uuid/12CE-A600";
       fsType = "vfat";

systems/brain/services/heater.nix

@@ -4,20 +4,6 @@
   ...
 }:
 {
-  nixpkgs.overlays = [
-    (final: _prev: {
-      heater_python = final.python314.withPackages (
-        ps: with ps; [
-          fastapi
-          pydantic
-          tinytuya
-          typer
-          uvicorn
-        ]
-      );
-    })
-  ];
-
   networking.firewall.allowedTCPPorts = [ 8124 ];
 
   systemd.services.heater-api = {
@@ -31,7 +17,7 @@
 
     serviceConfig = {
       Type = "simple";
-      ExecStart = "${pkgs.heater_python}/bin/python -m python.heater.main --host 0.0.0.0 --port 8124";
+      ExecStart = "${pkgs.my_python}/bin/python -m python.heater.main --host 0.0.0.0 --port 8124";
       EnvironmentFile = "/etc/heater.env";
       Restart = "on-failure";
       RestartSec = "5s";

systems/brain/services/home_assistant.nix

@@ -68,6 +68,7 @@
           paho-mqtt # for mqtt
           psycopg2 # for postgresql
           py-improv-ble-client # for esphome
+          pymodbus # for modbus
           pyopenweathermap # for weather
           pymetno # for met.no weather
           uiprotect # for ubiquiti integration
@@ -75,10 +76,7 @@
           jsonpath # for rest sensors
           monarchmoneycommunity # for monarch
         ];
-      extraComponents = [
+      extraComponents = [ "isal" ];
-        "isal"
-        "modbus" # for victron modbus integration
-      ];
       customComponents = with pkgs.home-assistant-custom-components; [
         garmin_connect
       ];

systems/jeeves/containers.nix

@@ -1,71 +0,0 @@
-{ pkgs, ... }:
-let
-  vars = import ./vars.nix;
-in
-{
-  virtualisation.docker.daemon.settings."data-root" = "${vars.containers}/docker";
-
-  # nixos-container hardcodes its state directory to /var/lib/nixos-containers,
-  # so route it to the shared container dataset with a bind mount.
-  fileSystems."/var/lib/nixos-containers" = {
-    device = "${vars.containers}/nixos-containers";
-    fsType = "none";
-    options = [ "bind" ];
-  };
-
-  systemd.services = {
-    prepare-docker-storage = {
-      description = "Create Docker storage directory";
-      requiredBy = [ "docker.service" ];
-      before = [ "docker.service" ];
-      requires = [ "zfs-mount.service" ];
-      after = [ "zfs-mount.service" ];
-      path = [
-        pkgs.coreutils
-        pkgs.util-linux
-      ];
-
-      unitConfig = {
-        DefaultDependencies = false;
-        RequiresMountsFor = [ "/nix" ];
-      };
-
-      serviceConfig = {
-        Type = "oneshot";
-        RemainAfterExit = true;
-      };
-
-      script = ''
-        mountpoint -q ${vars.containers}
-        install -d -m 0710 ${vars.containers}/docker
-      '';
-    };
-
-    prepare-nixos-container-storage = {
-      description = "Create NixOS container storage directory";
-      requiredBy = [ "var-lib-nixos\\x2dcontainers.mount" ];
-      before = [ "var-lib-nixos\\x2dcontainers.mount" ];
-      requires = [ "zfs-mount.service" ];
-      after = [ "zfs-mount.service" ];
-      path = [
-        pkgs.coreutils
-        pkgs.util-linux
-      ];
-
-      unitConfig = {
-        DefaultDependencies = false;
-        RequiresMountsFor = [ "/nix" ];
-      };
-
-      serviceConfig = {
-        Type = "oneshot";
-        RemainAfterExit = true;
-      };
-
-      script = ''
-        mountpoint -q ${vars.containers}
-        install -d -m 0755 ${vars.containers}/nixos-containers
-      '';
-    };
-  };
-}

systems/jeeves/default.nix

@@ -4,8 +4,10 @@ let
 in
 {
   imports = [
+    "${inputs.self}/users/dov"
     "${inputs.self}/users/math"
     "${inputs.self}/users/richie"
+    "${inputs.self}/users/steve"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/docker.nix"
     "${inputs.self}/common/optional/monitoring-agent.nix"
@@ -13,12 +15,10 @@ in
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/tailscale.nix"
     "${inputs.self}/common/optional/update.nix"
-    "${inputs.self}/common/optional/x86-64-v3.nix"
     "${inputs.self}/common/optional/zfs"
     ./monitoring
     ./docker
     ./services
-    ./containers.nix
     ./hardware.nix
     ./networking
     ./programs.nix

systems/jeeves/hardware.nix

@@ -96,11 +96,6 @@ in
       fsType = "zfs";
     };
 
-    "/nix/var/nix/builds" = {
-      device = "root_pool/nix_build";
-      fsType = "zfs";
-    };
-
     "/var" = {
       device = "root_pool/var";
       fsType = "zfs";

systems/jeeves/runners/nix_builder.nix

@@ -23,13 +23,6 @@ let
       ...
     }:
     let
-      giteaAutomationPython = pkgs.python314.withPackages (
-        ps: with ps; [
-          httpx
-          pydantic
-          typer
-        ]
-      );
       runnerConfigFile = (pkgs.formats.yaml { }).generate "gitea-runner.yaml" { };
       registerRunner = pkgs.writeShellApplication {
         name = "register-gitea-runner";
@@ -50,8 +43,6 @@ let
         useHostResolvConf = false;
       };
       nix.settings = {
-        cores = 8;
-        max-jobs = 2;
         system-features = lib.mkAfter [
           "gccarch-x86-64-v2"
           "gccarch-x86-64-v3"
@@ -96,20 +87,18 @@ let
         url = "http://192.168.99.14:6443/";
         labels = runnerLabels;
         tokenFile = "/run/secrets/gitea-runners/registration-token";
-        settings.runner.timeout = "12h";
         hostPackages = with pkgs; [
           bash
           coreutils
           curl
           gawk
           gitMinimal
-          giteaAutomationPython
           gnused
+          my_python
           nix
           nixfmt
           nixos-rebuild
           nodejs
-          ruff
           treefmt
           wget
         ];
@@ -195,6 +184,93 @@ in
             isReadOnly = false;
           };
         };
+        config =
+          {
+            config,
+            pkgs,
+            lib,
+            ...
+          }:
+          {
+            networking = {
+              useDHCP = lib.mkDefault true;
+              interfaces.eth0.useDHCP = true;
+              # Ensure containers don't inherit the host's stub resolver (127.0.0.53) which was causing issues
+              useHostResolvConf = false;
+            };
+            nix.settings = {
+              trusted-substituters = [
+                "https://cache.nixos.org"
+                "https://cache.tmmworkshop.com"
+                "https://nix-community.cachix.org"
+              ];
+              substituters = [
+                "https://cache.nixos.org/?priority=2&want-mass-query=true"
+                "https://cache.tmmworkshop.com/?priority=2&want-mass-query=true"
+                "https://nix-community.cachix.org/?priority=10&want-mass-query=true"
+              ];
+              trusted-public-keys = [
+                "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
+                "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA="
+                "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
+              ];
+              experimental-features = [
+                "flakes"
+                "nix-command"
+              ];
+              sandbox = true;
+              allowed-users = [ "gitea-runner" ];
+              trusted-users = [
+                "root"
+                "gitea-runner"
+              ];
+            };
+            nixpkgs = {
+              overlays = builtins.attrValues outputs.overlays;
+              config.allowUnfree = true;
+            };
+            users = {
+              users.${runnerUsername} = {
+                isSystemUser = true;
+                group = runnerUsername;
+                uid = runnerUserid;
+              };
+              groups.${runnerUsername}.gid = runnerUserid;
+            };
+            services.gitea-actions-runner.instances.${name} = {
+              enable = true;
+              name = "jeeves-${name}";
+              url = "http://192.168.99.14:6443/";
+              labels = [
+                "self-hosted:host"
+                "nixos:host"
+              ];
+              tokenFile = "/run/secrets/gitea-runners/registration-token";
+              hostPackages = with pkgs; [
+                bash
+                coreutils
+                curl
+                gawk
+                gitMinimal
+                gnused
+                my_python
+                nix
+                nixfmt
+                nixos-rebuild
+                nodejs
+                treefmt
+                wget
+              ];
+            };
+            systemd.services."gitea-runner-${utils.escapeSystemdPath name}" = {
+              serviceConfig = {
+                DynamicUser = mkForce false;
+                User = mkForce runnerUsername;
+                Group = mkForce runnerUsername;
+              };
+            };
+            system.stateVersion = "24.05";
+          };
       }
     ) cfg.containers;
 
@@ -205,7 +281,6 @@ in
           value = {
             requires = [ "gitea.service" ];
             after = [ "gitea.service" ];
-            serviceConfig.CPUQuota = "800%";
           };
         }) (builtins.attrNames (filterAttrs (_: c: c.enable) cfg.containers))
       );

systems/jeeves/scripts/zfs.sh

@@ -17,7 +17,6 @@ sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnode
 # media datasets
 sudo zfs create media/temp -o sync=disabled -o redundant_metadata=none
 sudo zfs create media/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key
-sudo zfs create media/secure/containers -o mountpoint=/zfs/media/containers -o compression=lz4 -o sync=disabled -o redundant_metadata=some -o normalization=none -o utf8only=off
 sudo zfs create media/secure/docker -o compression=zstd-9
 sudo zfs create media/secure/github-runners -o compression=zstd-9 -o sync=disabled
 sudo zfs create media/secure/notes -o copies=2

systems/jeeves/services/camofox-browser.nix

@@ -0,0 +1,80 @@
+{
+  ...
+}:
+let
+  vars = import ../vars.nix;
+in
+{
+  systemd.tmpfiles.rules = [
+    "d ${vars.docker_configs}/camofox-browser 0750 root root - -"
+  ];
+
+  containers.camofox-browser = {
+    autoStart = true;
+    privateNetwork = false;
+    bindMounts = {
+      camofox-browser = {
+        hostPath = "${vars.docker_configs}/camofox-browser";
+        mountPoint = "/var/lib/camofox-browser";
+        isReadOnly = false;
+      };
+    };
+    config =
+      {
+        pkgs,
+        lib,
+        ...
+      }:
+      {
+        networking.hostName = "camofox-browser";
+
+        environment.systemPackages = with pkgs; [
+          ffmpeg
+          git
+          nodejs
+          python3Packages.yt-dlp
+        ];
+
+        systemd.services.camofox-browser = {
+          description = "Camofox browser server";
+          wantedBy = [ "multi-user.target" ];
+          after = [ "network.target" ];
+          environment = {
+            CAMOFOX_HOST = "127.0.0.1";
+            CAMOFOX_PORT = "9377";
+            HOME = "/var/lib/camofox-browser";
+          };
+          path = with pkgs; [
+            bash
+            coreutils
+            git
+            nodejs
+          ];
+          serviceConfig = {
+            Restart = "always";
+            RestartSec = "5s";
+            WorkingDirectory = "/var/lib/camofox-browser";
+          };
+          script = ''
+            set -eu
+
+            app_dir=/var/lib/camofox-browser/app
+
+            if [ ! -d "$app_dir/.git" ]; then
+              git clone --depth 1 https://github.com/jo-inc/camofox-browser "$app_dir"
+            fi
+
+            cd "$app_dir"
+
+            if [ ! -d node_modules ]; then
+              npm install
+            fi
+
+            exec npm start
+          '';
+        };
+
+        system.stateVersion = lib.mkDefault "24.05";
+      };
+  };
+}

systems/jeeves/services/gems.nix

@@ -6,47 +6,8 @@
 let
   vars = import ../vars.nix;
   stateDir = "${vars.services}/gems";
-  gemsPackages =
-    ps: with ps; [
-      fastapi
-      jinja2
-      pydantic
-      pydantic-settings
-      python-multipart
-      typer
-      uvicorn
-    ];
 in
 {
-  nixpkgs.overlays = [
-    (final: _prev: {
-      gems_python = final.python314.withPackages gemsPackages;
-      gems_test_python = final.python314.withPackages (
-        ps:
-        gemsPackages ps
-        ++ (with ps; [
-          httpx
-          pytest
-          pytest-asyncio
-          pytest-xdist
-        ])
-      );
-      gems_tests =
-        final.runCommand "gems-tests"
-          {
-            nativeBuildInputs = [ final.gems_test_python ];
-          }
-          ''
-            export HOME="$TMPDIR"
-            cd ${inputs.self}
-            pytest -o cache_dir="$TMPDIR/pytest-cache" tests/gems
-            touch "$out"
-          '';
-    })
-  ];
-
-  system.checks = [ pkgs.gems_tests ];
-
   users.groups.gems = { };
   users.users.gems = {
     isSystemUser = true;
@@ -75,7 +36,7 @@ in
       Type = "simple";
       User = "gems";
       Group = "gems";
-      ExecStart = "${pkgs.gems_python}/bin/python -m python.gems.main --host 0.0.0.0 --port 8002";
+      ExecStart = "${pkgs.my_python}/bin/python -m python.gems.main --host 0.0.0.0 --port 8002";
       Restart = "on-failure";
       RestartSec = "5s";
       StandardOutput = "journal";

systems/jeeves/services/gitea.nix

@@ -19,7 +19,6 @@ in
       actions = {
         ENABLED = true;
         DEFAULT_ACTIONS_URL = "github";
-        ENDLESS_TASK_TIMEOUT = "12h";
       };
       service.DISABLE_REGISTRATION = true;
       server = {

systems/jeeves/services/open_webui.nix

@@ -0,0 +1,12 @@
+{
+  services.open-webui = {
+    enable = true;
+    host = "0.0.0.0";
+    environment = {
+      ANONYMIZED_TELEMETRY = "False";
+      DO_NOT_TRACK = "True";
+      SCARF_NO_ANALYTICS = "True";
+      OLLAMA_API_BASE_URL = "http://127.0.0.1:11434";
+    };
+  };
+}

systems/jeeves/services/systemd.nix

@@ -1,7 +1,11 @@
 {
   pkgs,
+  inputs,
   ...
 }:
+let
+  vars = import ../vars.nix;
+in
 {
   systemd = {
     services = {
@@ -26,6 +30,21 @@
           ExecStart = "${pkgs.bash}/bin/bash -c 'echo 1 > /sys/bus/pci/devices/0000:61:00.0/remove'";
         };
       };
+      startup_validation = {
+        requires = [ "network-online.target" ];
+        after = [ "network-online.target" ];
+        wantedBy = [ "multi-user.target" ];
+        description = "validates startup";
+        path = [ pkgs.zfs ];
+        environment = {
+          PYTHONPATH = "${inputs.self}/";
+        };
+        serviceConfig = {
+          EnvironmentFile = "${vars.secrets}/services/server-validation";
+          Type = "oneshot";
+          ExecStart = "${pkgs.my_python}/bin/python -m python.system_tests.validate_system '${./validate_system.toml}'";
+        };
+      };
     };
     timers = {
       plex_permission = {
@@ -36,6 +55,13 @@
           Unit = "plex_permission.service";
         };
       };
+      startup_validation = {
+        wantedBy = [ "timers.target" ];
+        timerConfig = {
+          OnBootSec = "10min";
+          Unit = "startup_validation.service";
+        };
+      };
     };
   };
 }

systems/jeeves/services/validate_system.toml

@@ -0,0 +1,6 @@
+zpool = ["root_pool", "storage", "media"]
+services = [
+    "audiobookshelf",
+    "docker",
+    "jellyfin",
+]

systems/jeeves/snapshot_config.toml

@@ -23,12 +23,6 @@ hourly = 0
 daily = 0
 monthly = 0
 
-["root_pool/nix_build"]
-15_min = 1
-hourly = 0
-daily = 0
-monthly = 0
-
 ["root_pool/var"]
 15_min = 8
 hourly = 24
@@ -66,12 +60,6 @@ daily = 30
 monthly = 12
 
 # media
-["media/secure/containers"]
-15_min = 2
-hourly = 0
-daily = 0
-monthly = 0
-
 ["media/temp"]
 15_min = 2
 hourly = 0

systems/jeeves/vars.nix

@@ -5,7 +5,6 @@ let
 in
 {
   inherit zfs_media zfs_storage zfs_scratch;
-  containers = "${zfs_media}/containers";
   database = "${zfs_media}/database";
   docker = "${zfs_media}/docker";
   docker_configs = "${zfs_media}/docker/configs";

systems/portal-1/default.nix

@@ -10,12 +10,13 @@
     "${inputs.self}/users/richie"
     "${inputs.self}/common/global"
     "${inputs.self}/common/optional/tailscale.nix"
-    "${inputs.self}/common/optional/x86-64-v3.nix"
     ./disk-config.nix
     ./haproxy
     ./monitoring.nix
   ];
 
+  nixpkgs.hostPlatform = "x86_64-linux";
+
   boot = {
     # Avoid consuming the VM's limited memory for /tmp.
     tmp.useTmpfs = false;

systems/rhapsody-in-green/agent_logger.nix

@@ -0,0 +1,35 @@
+{
+  pkgs,
+  inputs,
+  ...
+}:
+{
+  systemd.services.agent-logger = {
+    description = "Unified agent logger";
+    after = [ "local-fs.target" ];
+    wantedBy = [ "multi-user.target" ];
+
+    environment = {
+      AGENT_LOG_DB = "/var/lib/agent-logger/agent_log.sqlite";
+      HOME = "/home/richie";
+      PYTHONPATH = "${inputs.self}";
+    };
+
+    serviceConfig = {
+      Type = "simple";
+      User = "richie";
+      WorkingDirectory = "/home/richie";
+      ExecStart = "${pkgs.my_python}/bin/python -m python.agent_logger.main";
+      StateDirectory = "agent-logger";
+      Restart = "on-failure";
+      RestartSec = "5s";
+      StandardOutput = "journal";
+      StandardError = "journal";
+      NoNewPrivileges = true;
+      ProtectSystem = "strict";
+      ProtectHome = "read-only";
+      PrivateTmp = true;
+      ReadOnlyPaths = [ "${inputs.self}" ];
+    };
+  };
+}

systems/rhapsody-in-green/default.nix

@@ -9,10 +9,10 @@
     "${inputs.self}/common/optional/syncthing_base.nix"
     "${inputs.self}/common/optional/systemd-boot.nix"
     "${inputs.self}/common/optional/tailscale.nix"
-    "${inputs.self}/common/optional/x86-64-v3.nix"
     "${inputs.self}/common/optional/yubikey.nix"
     "${inputs.self}/common/optional/zfs"
     ./hardware.nix
+    ./open_webui.nix
     ./programs.nix
     ./qmk.nix
     ./syncthing.nix

systems/rhapsody-in-green/hardware.nix

@@ -34,22 +34,14 @@
       device = "root_pool/root";
       fsType = "zfs";
     };
-
     "/home" = {
       device = "root_pool/home";
       fsType = "zfs";
     };
-
-    "/nix/var/nix/builds" = {
-      device = "root_pool/nix_build";
-      fsType = "zfs";
-    };
-
     "/var" = {
       device = "root_pool/var";
       fsType = "zfs";
     };
-
     "/boot" = {
       device = "/dev/disk/by-uuid/12CE-A600";
       fsType = "vfat";

systems/rhapsody-in-green/open_webui.nix

@@ -0,0 +1,13 @@
+{
+  services.open-webui = {
+    enable = true;
+    host = "0.0.0.0";
+    environment = {
+      ANONYMIZED_TELEMETRY = "False";
+      DO_NOT_TRACK = "True";
+      SCARF_NO_ANALYTICS = "True";
+      OLLAMA_API_BASE_URL = "https://ollama.com";
+      WEBUI_AUTH = "False";
+    };
+  };
+}

tests/test_components.py

@@ -0,0 +1,104 @@
+"""test_components."""
+
+from pytest_mock import MockerFixture
+
+from python.system_tests.components import systemd_tests, zpool_tests
+from python.zfs import Zpool
+
+temp = "Every feature flags pool has all supported and requested features enabled.\n"
+
+SYSTEM_TESTS_COMPONENTS = "python.system_tests.components"
+
+
+def test_zpool_tests(mocker: MockerFixture) -> None:
+    """test_zpool_tests."""
+    mock_zpool = mocker.MagicMock(spec=Zpool)
+    mock_zpool.health = "ONLINE"
+    mock_zpool.capacity = 70
+    mock_zpool.name = "Main"
+    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
+    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=(temp, ""))
+    errors = zpool_tests(("Main",))
+    assert errors == []
+
+
+def test_zpool_tests_out_of_date(mocker: MockerFixture) -> None:
+    """test_zpool_tests_out_of_date."""
+    mock_zpool = mocker.MagicMock(spec=Zpool)
+    mock_zpool.health = "ONLINE"
+    mock_zpool.capacity = 70
+    mock_zpool.name = "Main"
+    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
+    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=("", ""))
+    errors = zpool_tests(("Main",))
+    assert errors == ["ZPool out of date run `sudo zpool upgrade -a`"]
+
+
+def test_zpool_tests_out_of_space(mocker: MockerFixture) -> None:
+    """test_zpool_tests_out_of_space."""
+    mock_zpool = mocker.MagicMock(spec=Zpool)
+    mock_zpool.health = "ONLINE"
+    mock_zpool.capacity = 100
+    mock_zpool.name = "Main"
+    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
+    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=(temp, ""))
+    errors = zpool_tests(("Main",))
+    assert errors == ["Main is low on space"]
+
+
+def test_zpool_tests_offline(mocker: MockerFixture) -> None:
+    """test_zpool_tests_offline."""
+    mock_zpool = mocker.MagicMock(spec=Zpool)
+    mock_zpool.health = "OFFLINE"
+    mock_zpool.capacity = 70
+    mock_zpool.name = "Main"
+    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
+    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=(temp, ""))
+    errors = zpool_tests(("Main",))
+    assert errors == ["Main is OFFLINE"]
+
+
+def test_systemd_tests(mocker: MockerFixture) -> None:
+    """test_systemd_tests."""
+    mocker.patch(
+        f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper",
+        side_effect=[
+            ("inactive\n", ""),
+            ("active\n", ""),
+        ],
+    )
+    errors = systemd_tests(("docker",))
+    assert errors == []
+    """test_systemd_tests."""
+
+
+def test_systemd_tests_multiple_negative_retries(mocker: MockerFixture) -> None:
+    """test_systemd_tests_fail."""
+    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=("active\n", ""))
+    errors = systemd_tests(("docker",), max_retries=-1, retry_delay_secs=-1)
+    assert errors == []
+
+
+def test_systemd_tests_multiple_pass(mocker: MockerFixture) -> None:
+    """test_systemd_tests_fail."""
+    mocker.patch(
+        f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper",
+        side_effect=[
+            ("inactive\n", ""),
+            ("activating\n", ""),
+            ("active\n", ""),
+        ],
+    )
+    errors = systemd_tests(
+        ("docker",),
+        retryable_statuses=("inactive\n", "activating\n"),
+        valid_statuses=("active\n",),
+    )
+    assert errors == []
+
+
+def test_systemd_tests_fail(mocker: MockerFixture) -> None:
+    """test_systemd_tests_fail."""
+    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=("inactive\n", ""))
+    errors = systemd_tests(("docker",), max_retries=5)
+    assert errors == ["docker is inactive"]

tests/test_server_validate_scripts.py

@@ -0,0 +1,63 @@
+"""test_server_validate_scripts."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+from pytest_mock import MockerFixture
+
+from python.system_tests.validate_system import main
+
+if TYPE_CHECKING:
+    from pyfakefs.fake_filesystem import FakeFilesystem
+    from pytest_mock import MockerFixture
+
+VALIDATE_SYSTEM = "python.system_tests.validate_system"
+
+
+def test_validate_system(mocker: MockerFixture, fs: FakeFilesystem) -> None:
+    """test_validate_system."""
+    fs.create_file(
+        "/mock_snapshot_config.toml",
+        contents='zpools = ["root_pool", "storage", "media"]\nservices = ["docker"]\n',
+    )
+
+    mocker.patch(f"{VALIDATE_SYSTEM}.systemd_tests", return_value=None)
+    mocker.patch(f"{VALIDATE_SYSTEM}.zpool_tests", return_value=None)
+    main(Path("/mock_snapshot_config.toml"))
+
+
+def test_validate_system_errors(mocker: MockerFixture, fs: FakeFilesystem) -> None:
+    """test_validate_system_errors."""
+    fs.create_file(
+        "/mock_snapshot_config.toml",
+        contents='zpools = ["root_pool", "storage", "media"]\nservices = ["docker"]\n',
+    )
+
+    mocker.patch(f"{VALIDATE_SYSTEM}.signal_alert")
+    mocker.patch(f"{VALIDATE_SYSTEM}.systemd_tests", return_value=["systemd_tests error"])
+    mocker.patch(f"{VALIDATE_SYSTEM}.zpool_tests", return_value=["zpool_tests error"])
+
+    with pytest.raises(SystemExit) as exception_info:
+        main(Path("/mock_snapshot_config.toml"))
+
+    assert exception_info.value.code == 1
+
+
+def test_validate_system_execution(mocker: MockerFixture, fs: FakeFilesystem) -> None:
+    """test_validate_system_execution."""
+    fs.create_file(
+        "/mock_snapshot_config.toml",
+        contents='zpools = ["root_pool", "storage", "media"]\nservices = ["docker"]\n',
+    )
+
+    mocker.patch(f"{VALIDATE_SYSTEM}.signal_alert")
+    mocker.patch(f"{VALIDATE_SYSTEM}.systemd_tests", return_value=None)
+    mocker.patch(f"{VALIDATE_SYSTEM}.zpool_tests", side_effect=RuntimeError("zpool_tests error"))
+
+    with pytest.raises(SystemExit) as exception_info:
+        main(Path("/mock_snapshot_config.toml"))
+
+    assert exception_info.value.code == 1

tests/test_signal_alert.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 from os import environ
 from typing import TYPE_CHECKING
 
-import httpx
+from apprise import Apprise
 
 from python.signal_alert import signal_alert
 
@@ -19,23 +19,14 @@ def test_signal_alert(mocker: MockerFixture) -> None:
     environ["SIGNAL_ALERT_TO_PHONE"] = "0987654321"
 
     mock_logger = mocker.patch("python.signal_alert.logger")
-    mock_response = mocker.MagicMock(spec=httpx.Response, status_code=httpx.codes.CREATED)
+    mock_apprise_client = mocker.MagicMock(spec=Apprise)
-    mock_post = mocker.patch("python.signal_alert.httpx.post", return_value=mock_response)
+    mocker.patch("python.signal_alert.Apprise", return_value=mock_apprise_client)
 
     signal_alert("test")
 
     mock_logger.info.assert_not_called()
-    mock_post.assert_called_once_with(
+    mock_apprise_client.add.assert_called_once_with("signal://localhost:8989/1234567890/0987654321")
-        "http://localhost:8989/v2/send",
+    mock_apprise_client.notify.assert_called_once_with(title="", body="test")
-        json={
-            "message": "test",
-            "number": "1234567890",
-            "recipients": ["0987654321"],
-            "text_mode": "normal",
-        },
-        timeout=4.0,
-        follow_redirects=True,
-    )
 
 
 def test_signal_alert_no_phones(mocker: MockerFixture) -> None:
@@ -48,34 +39,3 @@ def test_signal_alert_no_phones(mocker: MockerFixture) -> None:
     signal_alert("test")
 
     mock_logger.info.assert_called_once_with("SIGNAL_ALERT_FROM_PHONE or SIGNAL_ALERT_TO_PHONE not set")
-
-
-def test_signal_alert_http_error(mocker: MockerFixture) -> None:
-    """HTTP errors are logged rather than propagated."""
-    environ["SIGNAL_ALERT_FROM_PHONE"] = "1234567890"
-    environ["SIGNAL_ALERT_TO_PHONE"] = "0987654321"
-
-    mock_logger = mocker.patch("python.signal_alert.logger")
-    request = httpx.Request("POST", "http://localhost:8989/v2/send")
-    mocker.patch(
-        "python.signal_alert.httpx.post",
-        side_effect=httpx.ConnectError("connection failed", request=request),
-    )
-
-    signal_alert("test")
-
-    mock_logger.exception.assert_called_once_with("Unable to contact the Signal API")
-
-
-def test_signal_alert_unsuccessful_response(mocker: MockerFixture) -> None:
-    """Unexpected response statuses are logged."""
-    environ["SIGNAL_ALERT_FROM_PHONE"] = "1234567890"
-    environ["SIGNAL_ALERT_TO_PHONE"] = "0987654321"
-
-    mock_logger = mocker.patch("python.signal_alert.logger")
-    mock_response = mocker.MagicMock(spec=httpx.Response, status_code=httpx.codes.BAD_GATEWAY)
-    mocker.patch("python.signal_alert.httpx.post", return_value=mock_response)
-
-    signal_alert("test")
-
-    mock_logger.error.assert_called_once_with("Signal API returned HTTP status %d", httpx.codes.BAD_GATEWAY)

users/dov/default.nix

@@ -0,0 +1,26 @@
+{
+  pkgs,
+  ...
+}:
+{
+
+  users = {
+    users.dov = {
+      isNormalUser = true;
+
+      shell = pkgs.zsh;
+      group = "dov";
+      openssh.authorizedKeys.keys = [
+        "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCoSBmMfvp6aThkrfkLQ4TDwZJD0HCC0lsTIRNREIAWeduNkVFdkO3T1SMLmCKJ+zlL9xziNBEqB8NFl4TabAhptIGlKgTOc8C9eDaPQMQV8PB/4DxZhCt7O2qb4Vfcz82IHPtdwwaIsGpssgh81mQ4tPtP8BI0TluRBy+2v095s20j+PYRGrAXJtOWpVY9zaXxEJ8dXDhHDF2xzxvUcwu9NsoY8t+4/ZJ1mrTTG+eGp5gLAKnSVhAGgsmqCY577Nkso1jSzKer1XcCiaGIedpLuTzmUbOFFEVhhTSy+Ki1NLEcjGW2e6Vyg5Pm8VGN75MqyDZsi5igv9Grxq62EHQ4mFea9rns48B74O/bNQ1OoLVO9u/HwdLCgBTZzptrCmNwML6kBWrmCogoH3ueVbHwDCW5kTTMBCoVV+HaZ+qDWu7xZVx49MuCK29QGZj/IrN0N7h78KL0NYajdei87R0mcmWGP0YaJTdCQ4iKi9c77BUYQp+Qpqt+mnIX0cgjQOU= dkruger@kzin" # cspell:disable-line
+      ];
+      extraGroups = [
+        "audio"
+        "video"
+        "users"
+      ];
+      uid = 1004;
+    };
+
+    groups.dov.gid = 1004;
+  };
+}

users/dov/home/cli/default.nix

@@ -0,0 +1,9 @@
+{
+  imports = [
+    ./direnv.nix
+    ./git.nix
+    ./zsh.nix
+  ];
+
+  programs.starship.enable = true;
+}

users/dov/home/cli/direnv.nix

@@ -0,0 +1,8 @@
+{
+
+  programs.direnv = {
+    enable = true;
+    enableZshIntegration = true;
+    nix-direnv.enable = true;
+  };
+}

users/dov/home/cli/git.nix

@@ -0,0 +1,15 @@
+{
+  programs.git = {
+    enable = true;
+    signing.format = null;
+    settings = {
+      user = {
+        email = "dov.kruger@gmail.com";
+        name = "Dov Kruger";
+      };
+      pull.rebase = true;
+      color.ui = true;
+    };
+    lfs.enable = true;
+  };
+}

users/dov/home/cli/zsh.nix

@@ -0,0 +1,27 @@
+{
+  programs.zsh = {
+    enable = true;
+    syntaxHighlighting.enable = true;
+    history.size = 10000;
+    oh-my-zsh = {
+      enable = true;
+      plugins = [
+        "git"
+        "docker"
+        "docker-compose"
+        "colored-man-pages"
+        "rust"
+        "systemd"
+        "tmux"
+        "ufw"
+        "z"
+      ];
+    };
+    shellAliases = {
+      "lrt" = "eza --icons -lsnew";
+      "ls" = "eza";
+      "ll" = "eza --long --group";
+      "la" = "eza --all";
+    };
+  };
+}

users/dov/home/global.nix

@@ -0,0 +1,22 @@
+{ config, ... }:
+{
+  imports = [
+    ./cli
+    ./programs.nix
+    ./ssh_config.nix
+  ];
+
+  programs = {
+    home-manager.enable = true;
+    git.enable = true;
+  };
+
+  home = {
+    username = "dov";
+    homeDirectory = "/home/${config.home.username}";
+    stateVersion = "24.05";
+    sessionVariables = {
+      FLAKE = "$HOME/dotfiles";
+    };
+  };
+}

users/dov/home/programs.nix

@@ -0,0 +1,56 @@
+{ pkgs, ... }:
+{
+  home.packages = with pkgs; [
+    # cli
+    bat
+    btop
+    eza
+    fd
+    ffmpegthumbnailer
+    fzf
+    git
+    gnupg
+    imagemagick
+    jq
+    ncdu
+    ouch
+    p7zip
+    poppler
+    rar
+    ripgrep
+    starship
+    tmux
+    unzip
+    yazi
+    zoxide
+    # system info
+    hwloc
+    lynis
+    pciutils
+    smartmontools
+    usbutils
+    # networking
+    iperf3
+    nmap
+    wget
+    # python
+    ruff
+    uv
+    # nodejs
+    nodejs
+    # Rust packages
+    trunk
+    wasm-pack
+    cargo-watch
+    cargo-generate
+    cargo-audit
+    cargo-update
+    # nix
+    nix-init
+    nix-output-monitor
+    nix-prefetch
+    nix-tree
+    nixfmt
+    treefmt
+  ];
+}

users/dov/home/ssh_config.nix

@@ -0,0 +1,6 @@
+{
+  programs.ssh = {
+    enable = true;
+    enableDefaultConfig = false;
+  };
+}

users/dov/systems/jeeves.nix

@@ -0,0 +1,5 @@
+{
+  imports = [
+    ../home/global.nix
+  ];
+}

users/math/home/programs.nix

@@ -3,17 +3,54 @@
   home.packages = with pkgs; [
     # cli
     bat
+    btop
+    eza
+    fd
+    ffmpegthumbnailer
     fzf
     git
+    gnupg
+    imagemagick
     jq
     ncdu
-    fastfetch
+    ouch
     p7zip
+    poppler
     rar
+    ripgrep
+    starship
+    tmux
     unzip
+    yazi
     zoxide
+    # system info
+    hwloc
+    lynis
+    pciutils
+    smartmontools
+    usbutils
+    # networking
+    iperf3
+    nmap
+    wget
     # python
     ruff
     uv
+    # nodejs
+    nodejs
+    # Rust packages
+    trunk
+    wasm-pack
+    cargo-watch
+    cargo-generate
+    cargo-audit
+    cargo-update
+    # nix
+    nix-init
+    nix-output-monitor
+    nix-prefetch
+    nix-tree
+    nixfmt
+    treefmt
   ];
 }

users/richie/home/gui/default.nix

@@ -1,11 +1,12 @@
-{ pkgs, ... }:
+{ inputs, pkgs, ... }:
 {
   imports = [
-    ./comms.nix
+    "${inputs.self}/users/shared/comms.nix"
-    ./games.nix
+    "${inputs.self}/users/shared/games.nix"
+    "${inputs.self}/users/shared/sweet.nix"
+    ./firefox
     ./kitty.nix
     ./llm_tools.nix
-    ./sweet.nix
     ./t3_code
     ./vscode
   ];

users/richie/home/gui/firefox/default.nix

@@ -0,0 +1,257 @@
+{ config, inputs, ... }:
+{
+  imports = [ ./search_engines.nix ];
+
+  programs.firefox = {
+    configPath = "${config.xdg.configHome}/mozilla/firefox";
+    enable = true;
+    profiles.richie = {
+      extensions.packages = with inputs.firefox-addons.packages.x86_64-linux; [
+        bitwarden
+        darkreader
+        dearrow
+        fastforwardteam
+        return-youtube-dislikes
+        sponsorblock
+        ublock-origin
+      ];
+      search = {
+        force = true;
+        default = "kagi";
+        order = [
+          "kagi"
+          "ddg"
+          "google"
+        ];
+      };
+      settings = {
+        # SECTION: FASTFOX
+        # GENERAL
+        "content.notify.interval" = 100000;
+
+        # GFX
+        "gfx.canvas.accelerated.cache-items" = 4096;
+        "gfx.canvas.accelerated.cache-size" = 512;
+        "gfx.content.skia-font-cache-size" = 20;
+
+        # DISK CACHE
+        "browser.cache.jsbc_compression_level" = 3;
+
+        # MEDIA CACHE
+        "media.memory_cache_max_size" = 65536;
+        "media.cache_readahead_limit" = 7200;
+        "media.cache_resume_threshold" = 3600;
+
+        # IMAGE CACHE
+        "image.mem.decode_bytes_at_a_time" = 32768;
+
+        # NETWORK
+        "network.buffer.cache.size" = 262144;
+        "network.buffer.cache.count" = 128;
+        "network.http.max-connections" = 1800;
+        "network.http.max-persistent-connections-per-server" = 10;
+        "network.http.max-urgent-start-excessive-connections-per-host" = 5;
+        "network.http.accept-encoding" = "gzip, deflate, br, zstd";
+        "network.http.pacing.requests.enabled" = false;
+        "network.dnsCacheExpiration" = 3600;
+        "network.dns.max_high_priority_threads" = 8;
+        "network.ssl_tokens_cache_capacity" = 10240;
+
+        # SPECULATIVE LOADING
+        "network.dns.disablePrefetch" = true;
+        "network.prefetch-next" = false;
+        "network.predictor.enabled" = false;
+
+        # EXPERIMENTAL
+        "layout.css.grid-template-masonry-value.enabled" = true;
+        "dom.enable_web_task_scheduling" = true;
+        "layout.css.has-selector.enabled" = true;
+        "dom.security.sanitizer.enabled" = true;
+
+        # SECTION: SECUREFOX
+        # TRACKING PROTECTION
+        "browser.contentblocking.category" = "strict";
+        "urlclassifier.trackingSkipURLs" = "*.reddit.com, *.twitter.com, *.twimg.com, *.tiktok.com";
+        "urlclassifier.features.socialtracking.skipURLs" = "*.instagram.com, *.twitter.com, *.twimg.com";
+        "network.cookie.sameSite.noneRequiresSecure" = true;
+        "browser.download.start_downloads_in_tmp_dir" = true;
+        "browser.helperApps.deleteTempFileOnExit" = true;
+        "browser.uitour.enabled" = false;
+        "privacy.globalprivacycontrol.enabled" = true;
+
+        # OCSP & CERTS / HPKP
+        "security.OCSP.enabled" = 0;
+        "security.remote_settings.crlite_filters.enabled" = true;
+        "security.pki.crlite_mode" = 2;
+
+        # SSL / TLS
+        "security.ssl.treat_unsafe_negotiation_as_broken" = true;
+        "browser.xul.error_pages.expert_bad_cert" = true;
+        "security.tls.enable_0rtt_data" = false;
+
+        # DISK AVOIDANCE
+        "browser.privatebrowsing.forceMediaMemoryCache" = true;
+        "browser.sessionstore.interval" = 60000;
+
+        # SHUTDOWN & SANITIZING
+        "privacy.history.custom" = true;
+
+        # SEARCH / URL BAR
+        "browser.search.separatePrivateDefault.ui.enabled" = true;
+        "browser.urlbar.update2.engineAliasRefresh" = true;
+        # PREF: restore search engine suggestions
+        "browser.search.suggest.enabled" = true;
+        "browser.urlbar.suggest.quicksuggest.sponsored" = false;
+        "browser.urlbar.suggest.quicksuggest.nonsponsored" = false;
+        "browser.formfill.enable" = false;
+        "security.insecure_connection_text.enabled" = true;
+        "security.insecure_connection_text.pbmode.enabled" = true;
+        "network.IDN_show_punycode" = true;
+
+        # HTTPS-FIRST POLICY
+        "dom.security.https_first" = true;
+        "dom.security.https_first_schemeless" = true;
+
+        # PASSWORDS
+        "signon.formlessCapture.enabled" = false;
+        "signon.rememberSignons" = false;
+        "signon.privateBrowsingCapture.enabled" = false;
+        "network.auth.subresource-http-auth-allow" = 1;
+        "editor.truncate_user_pastes" = false;
+
+        # MIXED CONTENT + CROSS-SITE
+        "security.mixed_content.block_display_content" = true;
+        "security.mixed_content.upgrade_display_content" = true;
+        "security.mixed_content.upgrade_display_content.image" = true;
+        "pdfjs.enableScripting" = false;
+        "extensions.postDownloadThirdPartyPrompt" = false;
+
+        # HEADERS / REFERERS
+        "network.http.referer.XOriginTrimmingPolicy" = 2;
+
+        # CONTAINERS
+        "privacy.userContext.ui.enabled" = true;
+
+        # WEBRTC
+        "media.peerconnection.ice.proxy_only_if_behind_proxy" = true;
+        "media.peerconnection.ice.default_address_only" = true;
+
+        # SAFE BROWSING
+        "browser.safebrowsing.downloads.remote.enabled" = false;
+
+        # MOZILLA
+        # PREF: allow websites to ask you to receive site notifications
+        "permissions.default.desktop-notification" = 0; # allow websites to ask
+        # PREF: allow websites to ask you for your location
+        "permissions.default.geo" = 0;
+        "geo.provider.network.url" =
+          "https://location.services.mozilla.com/v1/geolocate?key=%MOZILLA_API_KEY%";
+        "permissions.manager.defaultsUrl" = "";
+        "webchannel.allowObject.urlWhitelist" = "";
+
+        # TELEMETRY
+        "datareporting.policy.dataSubmissionEnabled" = false;
+        "datareporting.healthreport.uploadEnabled" = false;
+        "toolkit.telemetry.unified" = false;
+        "toolkit.telemetry.enabled" = false;
+        "toolkit.telemetry.server" = "data:,";
+        "toolkit.telemetry.archive.enabled" = false;
+        "toolkit.telemetry.newProfilePing.enabled" = false;
+        "toolkit.telemetry.shutdownPingSender.enabled" = false;
+        "toolkit.telemetry.updatePing.enabled" = false;
+        "toolkit.telemetry.bhrPing.enabled" = false;
+        "toolkit.telemetry.firstShutdownPing.enabled" = false;
+        "toolkit.telemetry.coverage.opt-out" = true;
+        "toolkit.coverage.opt-out" = true;
+        "toolkit.coverage.endpoint.base" = "";
+        "browser.ping-centre.telemetry" = false;
+        "browser.newtabpage.activity-stream.feeds.telemetry" = false;
+        "browser.newtabpage.activity-stream.telemetry" = false;
+
+        # EXPERIMENTS
+        "app.shield.optoutstudies.enabled" = false;
+        "app.normandy.enabled" = false;
+        "app.normandy.api_url" = "";
+
+        # CRASH REPORTS
+        "breakpad.reportURL" = "";
+        "browser.tabs.crashReporting.sendReport" = false;
+        "browser.crashReports.unsubmittedCheck.autoSubmit2" = false;
+
+        # DETECTION
+        "captivedetect.canonicalURL" = "";
+        "network.captive-portal-service.enabled" = false;
+        "network.connectivity-service.enabled" = false;
+
+        # SECTION: PESKYFOX
+        # MOZILLA UI
+        "browser.privatebrowsing.vpnpromourl" = "";
+        "extensions.getAddons.showPane" = false;
+        "extensions.htmlaboutaddons.recommendations.enabled" = false;
+        "browser.discovery.enabled" = false;
+        "browser.shell.checkDefaultBrowser" = false;
+        "browser.newtabpage.activity-stream.asrouter.userprefs.cfr.addons" = false;
+        "browser.newtabpage.activity-stream.asrouter.userprefs.cfr.features" = false;
+        "browser.preferences.moreFromMozilla" = false;
+        "browser.tabs.tabmanager.enabled" = false;
+        "browser.aboutConfig.showWarning" = false;
+        "browser.aboutwelcome.enabled" = false;
+
+        # THEME ADJUSTMENTS
+        "toolkit.legacyUserProfileCustomizations.stylesheets" = true;
+        "browser.compactmode.show" = true;
+        "browser.display.focus_ring_on_anything" = true;
+        "browser.display.focus_ring_style" = 0;
+        "browser.display.focus_ring_width" = 0;
+        "layout.css.prefers-color-scheme.content-override" = 2;
+
+        # COOKIE BANNER HANDLING
+        "cookiebanners.service.mode" = 1;
+        "cookiebanners.service.mode.privateBrowsing" = 1;
+
+        # FULLSCREEN NOTICE
+        "full-screen-api.transition-duration.enter" = "0 0";
+        "full-screen-api.transition-duration.leave" = "0 0";
+        "full-screen-api.warning.delay" = -1;
+        "full-screen-api.warning.timeout" = 0;
+
+        # URL BAR
+        "browser.urlbar.suggest.calculator" = true;
+        "browser.urlbar.unitConversion.enabled" = true;
+        "browser.urlbar.trending.featureGate" = false;
+
+        # NEW TAB PAGE
+        "browser.newtabpage.activity-stream.feeds.topsites" = false;
+        "browser.newtabpage.activity-stream.feeds.section.topstories" = false;
+
+        # POCKET
+        "extensions.pocket.enabled" = false;
+
+        # DOWNLOADS
+        "browser.download.always_ask_before_handling_new_types" = true;
+        "browser.download.manager.addToRecentDocs" = false;
+
+        # PDF
+        "browser.download.open_pdf_attachments_inline" = true;
+
+        # TAB BEHAVIOR
+        "browser.bookmarks.openInTabClosesMenu" = false;
+        "browser.menu.showViewImageInfo" = true;
+        "findbar.highlightAll" = true;
+        "layout.word_select.eat_space_to_next_word" = false;
+
+        # SECTION: MY OVERRIDES
+        "browser.startup.homepage" = "https://google.com";
+        "identity.fxaccounts.enabled" = false;
+
+        # SECTION SMOOTHFOX
+        # OPTION: SHARPEN SCROLLING                                                           *
+        "apz.overscroll.enabled" = true; # DEFAULT NON-LINUX
+        "mousewheel.min_line_scroll_amount" = 10; # 10-40; adjust this number to your liking; default=5
+        "general.smoothScroll.mouseWheel.durationMinMS" = 80; # default=50
+        "general.smoothScroll.currentVelocityWeighting" = "0.15"; # default=.25
+        "general.smoothScroll.stopDecelerationWeighting" = "0.6"; # default=.4
+      };
+    };
+  };
+}

users/richie/home/gui/firefox/github.svg

@@ -0,0 +1,3 @@
+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M16 0C7.16 0 0 7.16 0 16C0 23.08 4.58 29.06 10.94 31.18C11.74 31.32 12.04 30.84 12.04 30.42C12.04 30.04 12.02 28.78 12.02 27.44C8 28.18 6.96 26.46 6.64 25.56C6.46 25.1 5.68 23.68 5 23.3C4.44 23 3.64 22.26 4.98 22.24C6.24 22.22 7.14 23.4 7.44 23.88C8.88 26.3 11.18 25.62 12.1 25.2C12.24 24.16 12.66 23.46 13.12 23.06C9.56 22.66 5.84 21.28 5.84 15.16C5.84 13.42 6.46 11.98 7.48 10.86C7.32 10.46 6.76 8.82 7.64 6.62C7.64 6.62 8.98 6.2 12.04 8.26C13.32 7.9 14.68 7.72 16.04 7.72C17.4 7.72 18.76 7.9 20.04 8.26C23.1 6.18 24.44 6.62 24.44 6.62C25.32 8.82 24.76 10.46 24.6 10.86C25.62 11.98 26.24 13.4 26.24 15.16C26.24 21.3 22.5 22.66 18.94 23.06C19.52 23.56 20.02 24.52 20.02 26.02C20.02 28.16 20 29.88 20 30.42C20 30.84 20.3 31.34 21.1 31.18C27.42 29.06 32 23.06 32 16C32 7.16 24.84 0 16 0V0Z" fill="white"/>
+</svg>

users/richie/home/gui/firefox/search_engines.nix

@@ -0,0 +1,99 @@
+{ pkgs, ... }:
+{
+  programs.firefox.profiles.richie.search.engines = {
+    "Nix Options" = {
+      urls = [
+        {
+          template = "https://search.nixos.org/options";
+          params = [
+            {
+              name = "type";
+              value = "packages";
+            }
+            {
+              name = "channel";
+              value = "unstable";
+            }
+            {
+              name = "query";
+              value = "{searchTerms}";
+            }
+          ];
+        }
+      ];
+      icon = "${pkgs.nixos-icons}/share/icons/hicolor/scalable/apps/nix-snowflake.svg";
+      definedAliases = [ "@o" ];
+    };
+    "Nix Packages" = {
+      urls = [
+        {
+          template = "https://search.nixos.org/packages";
+          params = [
+            {
+              name = "type";
+              value = "packages";
+            }
+            {
+              name = "channel";
+              value = "unstable";
+            }
+            {
+              name = "query";
+              value = "{searchTerms}";
+            }
+          ];
+        }
+      ];
+      icon = "${pkgs.nixos-icons}/share/icons/hicolor/scalable/apps/nix-snowflake.svg";
+      definedAliases = [ "@n" ];
+    };
+    "Nix Packages pr-tracker" = {
+      urls = [
+        {
+          template = "https://nixpk.gs/pr-tracker.html?";
+          params = [
+            {
+              name = "pr";
+              value = "{searchTerms}";
+            }
+          ];
+        }
+      ];
+      icon = "${pkgs.nixos-icons}/share/icons/hicolor/scalable/apps/nix-snowflake.svg";
+      definedAliases = [ "@nprt" ];
+    };
+    "kagi" = {
+      urls = [
+        {
+          template = "https://kagi.com/search?";
+          params = [
+            {
+              name = "q";
+              value = "{searchTerms}";
+            }
+          ];
+        }
+      ];
+      icon = ./kagi.png;
+    };
+    github = {
+      urls = [
+        {
+          template = "https://github.com/search?";
+          params = [
+            {
+              name = "q";
+              value = "{searchTerms}";
+            }
+            {
+              name = "type";
+              value = "code";
+            }
+          ];
+        }
+      ];
+      icon = ./github.svg;
+      definedAliases = [ "@g" ];
+    };
+  };
+}

users/richie/home/programs.nix

@@ -3,14 +3,21 @@
   home.packages = with pkgs; [
     # cli
     bat
+    fd
+    ffmpegthumbnailer
     fzf
     git
+    gnupg
+    imagemagick
     jq
     ncdu
     fastfetch
+    ouch
     p7zip
+    poppler
     rar
     unzip
+    yazi
     zoxide
     # Home Assistant
     esphome
@@ -28,6 +35,8 @@
     # python
     ruff
     uv
+    # nodejs
+    nodejs
     # Rust packages
     bacon
     cargo
@@ -42,6 +51,9 @@
     rustfmt
     trunk
     wasm-pack
+    # cpp
+    clang-tools
+    clang_20
     # nix
     nix-init
     nix-output-monitor
@@ -49,5 +61,6 @@
     nix-tree
     nixfmt
     treefmt
+    codebase-memory-mcp
   ];
 }

users/steve/default.nix

@@ -0,0 +1,44 @@
+{
+  pkgs,
+  config,
+  ...
+}:
+let
+  ifTheyExist = groups: builtins.filter (group: builtins.hasAttr group config.users.groups) groups;
+in
+{
+
+  users = {
+    users.steve = {
+      isNormalUser = true;
+
+      shell = pkgs.zsh;
+      group = "steve";
+      openssh.authorizedKeys.keys = [
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJH03VzDbUhzfhvwD+OsYh6GobODYaI9jdNdzWQoqFsp matth@Jove" # cspell:disable-line
+      ];
+      extraGroups = [
+        "audio"
+        "video"
+        "wheel"
+        "users"
+      ]
+      ++ ifTheyExist [
+        "dialout"
+        "docker"
+        "hass"
+        "libvirtd"
+        "networkmanager"
+        "plugdev"
+        "scanner"
+        "transmission"
+        "uaccess"
+        "wireshark"
+      ];
+      uid = 1005;
+    };
+
+    groups.steve.gid = 1005;
+  };
+  home-manager.users.steve = import ./systems/${config.networking.hostName}.nix;
+}

users/steve/home/cli/default.nix

@@ -0,0 +1,9 @@
+{
+  imports = [
+    ./direnv.nix
+    ./git.nix
+    ./zsh.nix
+  ];
+
+  programs.starship.enable = true;
+}

users/steve/home/cli/direnv.nix

@@ -0,0 +1,8 @@
+{
+
+  programs.direnv = {
+    enable = true;
+    enableZshIntegration = true;
+    nix-direnv.enable = true;
+  };
+}

users/steve/home/cli/git.nix

@@ -0,0 +1,15 @@
+{
+  programs.git = {
+    enable = true;
+    signing.format = null;
+    settings = {
+      user = {
+        email = "matthew.michal11@gmail.com";
+        name = "Matthew Michal";
+      };
+      pull.rebase = true;
+      color.ui = true;
+    };
+    lfs.enable = true;
+  };
+}

users/steve/home/cli/zsh.nix

@@ -0,0 +1,28 @@
+{
+  programs.zsh = {
+    enable = true;
+    syntaxHighlighting.enable = true;
+    history.size = 10000;
+    oh-my-zsh = {
+      enable = true;
+      plugins = [
+        "git"
+        "docker"
+        "docker-compose"
+        "colored-man-pages"
+        "rust"
+        "systemd"
+        "tmux"
+        "ufw"
+        "z"
+      ];
+    };
+    shellAliases = {
+      "lrt" = "eza --icons -lsnew";
+      "ls" = "eza";
+      "ll" = "eza --long --group";
+      "la" = "eza --all";
+
+    };
+  };
+}

users/steve/home/global.nix

@@ -0,0 +1,22 @@
+{ config, ... }:
+{
+  imports = [
+    ./cli
+    ./programs.nix
+    ./ssh_config.nix
+  ];
+
+  programs = {
+    home-manager.enable = true;
+    git.enable = true;
+  };
+
+  home = {
+    username = "steve";
+    homeDirectory = "/home/${config.home.username}";
+    stateVersion = "24.05";
+    sessionVariables = {
+      FLAKE = "$HOME/dotfiles";
+    };
+  };
+}

users/steve/home/programs.nix

@@ -0,0 +1,56 @@
+{ pkgs, ... }:
+{
+  home.packages = with pkgs; [
+    # cli
+    bat
+    btop
+    eza
+    fd
+    ffmpegthumbnailer
+    fzf
+    git
+    gnupg
+    imagemagick
+    jq
+    ncdu
+    ouch
+    p7zip
+    poppler
+    rar
+    ripgrep
+    starship
+    tmux
+    unzip
+    yazi
+    zoxide
+    # system info
+    hwloc
+    lynis
+    pciutils
+    smartmontools
+    usbutils
+    # networking
+    iperf3
+    nmap
+    wget
+    # python
+    ruff
+    uv
+    # nodejs
+    nodejs
+    # Rust packages
+    trunk
+    wasm-pack
+    cargo-watch
+    cargo-generate
+    cargo-audit
+    cargo-update
+    # nix
+    nix-init
+    nix-output-monitor
+    nix-prefetch
+    nix-tree
+    nixfmt
+    treefmt
+  ];
+}

users/steve/home/ssh_config.nix

@@ -0,0 +1,6 @@
+{
+  programs.ssh = {
+    enable = true;
+    enableDefaultConfig = false;
+  };
+}

users/steve/systems/bob.nix

@@ -0,0 +1,5 @@
+{
+  imports = [
+    ../home/global.nix
+  ];
+}

users/steve/systems/jeeves.nix

@@ -0,0 +1,5 @@
+{
+  imports = [
+    ../home/global.nix
+  ];
+}