Richie/dotfiles
1
0
Fork 0
Code Issues 1 Pull Requests 3 Actions Packages Projects Releases Wiki Activity

feat(monitoring): add Portal-1 and Vultr dashboards
treefmt / nix fmt (pull_request) Successful in 5s
Details
pytest / pytest (pull_request) Successful in 30s
Details
test ebook search / test-ebook-search (pull_request) Successful in 34s
Details
build_systems / build-brain (pull_request) Successful in 49s
Details
build_systems / build-bob (pull_request) Successful in 50s
Details
build_systems / build-rhapsody-in-green (pull_request) Successful in 1m2s
Details
build_systems / build-jeeves (pull_request) Successful in 2m14s
Details

Browse Source 
- scrape Portal-1 node metrics over Tailscale
- add a hardened Vultr billing and bandwidth exporter
- provision host health and infrastructure cost dashboards
This commit is contained in:
Richie
2026-08-27 17:49:56 -04:00
parent d95b8b99ce
commit bdf67e96f0
7 changed files with 2005 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
systems/jeeves/monitoring/README.md
+33
View File
@@ -0,0 +1,33 @@
# Monitoring
## Vultr API metrics
The `vultr-exporter` service reads its API token from:
```text
/zfs/storage/secrets/services/vultr-exporter
```
Create the file on Jeeves as root with the following contents:
```text
API_KEY=<Vultr API token>
```
The token needs read access to the Vultr Account and Billing APIs. Unrelated
resource collectors are disabled in the packaged exporter.
Restrict the file to root and ensure the public egress IP used by Jeeves is
allowed for the token in the Vultr API settings:
```console
sudo chown root:root /zfs/storage/secrets/services/vultr-exporter
sudo chmod 600 /zfs/storage/secrets/services/vultr-exporter
```
The exporter listens on `127.0.0.1:9188`; it is scraped by the local
`prometheus-main` service every five minutes and is not exposed through the
host firewall.
Portal-1 exposes its node exporter only through `tailscale0` on port `9100`.
Jeeves reaches it using the Portal-1 Tailscale hostname.
systems/jeeves/monitoring/dashboards/portal-1.json
+798
View File
@@ -0,0 +1,798 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": false,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
},
"mappings": [
{
"options": {
"0": {
"color": "red",
"index": 1,
"text": "Offline"
},
"1": {
"color": "green",
"index": 0,
"text": "Online"
}
},
"type": "value"
}
]
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 4,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "up{job=\"node\",instance=\"portal-1\"}",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Node Exporter",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 70
},
{
"color": "red",
"value": 90
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 5,
"x": 4,
"y": 0
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "100 * (1 - avg(rate(node_cpu_seconds_total{job=\"node\",instance=\"portal-1\",mode=\"idle\"}[5m])))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "CPU Used",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 70
},
{
"color": "red",
"value": 90
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 5,
"x": 9,
"y": 0
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "100 * (1 - (node_memory_MemAvailable_bytes{job=\"node\",instance=\"portal-1\"} / node_memory_MemTotal_bytes{job=\"node\",instance=\"portal-1\"}))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "RAM Used",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 70
},
{
"color": "red",
"value": 90
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 5,
"x": 14,
"y": 0
},
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "100 * (1 - (node_filesystem_avail_bytes{job=\"node\",instance=\"portal-1\",mountpoint=\"/\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node\",instance=\"portal-1\",mountpoint=\"/\",fstype!=\"\"}))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Root Filesystem Used",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 5,
"x": 19,
"y": 0
},
"id": 5,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "node_load1{job=\"node\",instance=\"portal-1\"}",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Load (1m)",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 70
},
{
"color": "red",
"value": 90
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 5
},
"id": 6,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "100 * (1 - avg(rate(node_cpu_seconds_total{job=\"node\",instance=\"portal-1\",mode=\"idle\"}[5m])))",
"instant": false,
"legendFormat": "CPU used",
"range": true,
"refId": "A"
}
],
"title": "CPU Usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 70
},
{
"color": "red",
"value": 90
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 5
},
"id": 7,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "100 * (1 - (node_memory_MemAvailable_bytes{job=\"node\",instance=\"portal-1\"} / node_memory_MemTotal_bytes{job=\"node\",instance=\"portal-1\"}))",
"instant": false,
"legendFormat": "RAM used",
"range": true,
"refId": "A"
}
],
"title": "RAM Usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 70
},
{
"color": "red",
"value": 90
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 13
},
"id": 8,
"options": {
"cellHeight": "sm",
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "Value"
}
]
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "100 * (1 - (node_filesystem_avail_bytes{job=\"node\",instance=\"portal-1\",fstype!~\"tmpfs|devtmpfs|overlay|squashfs\",mountpoint!~\"/run.*\"} / node_filesystem_size_bytes{job=\"node\",instance=\"portal-1\",fstype!~\"tmpfs|devtmpfs|overlay|squashfs\",mountpoint!~\"/run.*\"}))",
"format": "table",
"instant": true,
"legendFormat": "{{mountpoint}}",
"range": false,
"refId": "A"
}
],
"title": "Filesystem Usage",
"type": "table"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 13
},
"id": 9,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "sum(rate(node_disk_read_bytes_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
"instant": false,
"legendFormat": "read",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "sum(rate(node_disk_written_bytes_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
"instant": false,
"legendFormat": "write",
"range": true,
"refId": "B"
}
],
"title": "Disk Throughput",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "iops"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 21
},
"id": 10,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "sum(rate(node_disk_reads_completed_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
"instant": false,
"legendFormat": "reads",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "sum(rate(node_disk_writes_completed_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))",
"instant": false,
"legendFormat": "writes",
"range": true,
"refId": "B"
}
],
"title": "Disk Operations",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 21
},
"id": 11,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "sum(rate(node_network_receive_bytes_total{job=\"node\",instance=\"portal-1\",device!=\"lo\"}[5m]))",
"instant": false,
"legendFormat": "received",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prom-main"
},
"editorMode": "code",
"expr": "sum(rate(node_network_transmit_bytes_total{job=\"node\",instance=\"portal-1\",device!=\"lo\"}[5m]))",
"instant": false,
"legendFormat": "sent",
"range": true,
"refId": "B"
}
],
"title": "Network Traffic",
"type": "timeseries"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [
"monitoring",
"portal-1",
"vultr"
],
"templating": {
"list": []
},
"time": {
"from": "now-24h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Portal-1 Host",
"uid": "portal-1-host",
"version": 1,
"weekStart": ""
}
systems/jeeves/monitoring/dashboards/vultr-infrastructure.json
+1053
View File
File diff suppressed because it is too large Load Diff
systems/jeeves/monitoring/default.nix
+9
View File
@@ -43,8 +43,15 @@ let
static_configs = [
(mkTarget "jeeves" "jeeves.taile39c31.ts.net:9100")
(mkTarget "bob" "bob.taile39c31.ts.net:9100")
(mkTarget "portal-1" "portal-1.taile39c31.ts.net:9100")
];
}
{
job_name = "vultr";
scrape_interval = "5m";
scrape_timeout = "45s";
static_configs = [ (mkTarget "vultr" "127.0.0.1:9188") ];
}
{
job_name = "process_grouped";
static_configs = [
@@ -153,6 +160,8 @@ let
};
in
{
imports = [ ./vultr-exporter.nix ];
users = {
groups.prometheus = { };
users.prometheus = {
systems/jeeves/monitoring/vultr-exporter.nix
+96
View File
@@ -0,0 +1,96 @@
{
lib,
pkgs,
...
}:
let
vars = import ../vars.nix;
vultrExporter = pkgs.buildGoModule rec {
pname = "vultr-exporter";
version = "0-unstable-2025-10-14";
src = pkgs.fetchFromGitHub {
owner = "DazWilkin";
repo = "vultr-exporter";
rev = "d8b5d7107c2eda31566e6e895c8a67db32fdaf68";
hash = "sha256-3LpXh+mFMBRJfiY2ftELpM3AIE0LiZ9c6NFok0OeH5I=";
};
vendorHash = "sha256-qCHWmg6GxLIjqlf5UqRcuDRd1xqRFI3sl4ZduLajx+Y=";
subPackages = [ "cmd/server" ];
postPatch = ''
substituteInPlace cmd/server/main.go \
--replace-fail 'registry.MustRegister(collector.NewBlockStorageCollector(s, client, log))' "" \
--replace-fail 'registry.MustRegister(collector.NewKubernetesCollector(s, client, log))' "" \
--replace-fail 'registry.MustRegister(collector.NewLoadBalancerCollector(s, client, log))' "" \
--replace-fail 'registry.MustRegister(collector.NewReservedIPsCollector(s, client, log))' ""
'';
ldflags = [
"-s"
"-w"
"-X main.GitCommit=${src.rev}"
"-X main.OSVersion=NixOS"
];
postInstall = ''
mv $out/bin/server $out/bin/vultr-exporter
'';
meta = {
description = "Prometheus exporter for Vultr account, billing, and bandwidth metrics";
homepage = "https://github.com/DazWilkin/vultr-exporter";
license = lib.licenses.asl20;
mainProgram = "vultr-exporter";
};
};
in
{
systemd.services.vultr-exporter = {
description = "Vultr Prometheus exporter";
documentation = [ "https://github.com/DazWilkin/vultr-exporter" ];
wants = [ "network-online.target" ];
after = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
ExecStart = "${lib.getExe vultrExporter} --endpoint=127.0.0.1:9188 --path=/metrics";
EnvironmentFile = "${vars.secrets}/services/vultr-exporter";
Restart = "on-failure";
RestartSec = "30s";
CapabilityBoundingSet = [ "" ];
DeviceAllow = [ "" ];
DynamicUser = true;
LockPersonality = true;
MemoryDenyWriteExecute = true;
NoNewPrivileges = true;
PrivateDevices = true;
PrivateTmp = true;
ProtectClock = true;
ProtectControlGroups = true;
ProtectHome = true;
ProtectHostname = true;
ProtectKernelLogs = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectSystem = "strict";
RemoveIPC = true;
RestrictAddressFamilies = [
"AF_INET"
"AF_INET6"
];
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
SystemCallArchitectures = "native";
SystemCallFilter = [
"@system-service"
"~@privileged"
];
UMask = "0077";
};
};
}
systems/portal-1/default.nix
+1
View File
@@ -12,6 +12,7 @@
"${inputs.self}/common/optional/tailscale.nix"
./disk-config.nix
./haproxy
./monitoring.nix
];
nixpkgs.hostPlatform = "x86_64-linux";
systems/portal-1/monitoring.nix
+15
View File
@@ -0,0 +1,15 @@
{ config, ... }:
{
networking.firewall.interfaces.tailscale0.allowedTCPPorts = [
config.services.prometheus.exporters.node.port
];
services.prometheus.exporters.node = {
enable = true;
enabledCollectors = [
"pressure"
"processes"
"systemd"
];
};
}