diff --git a/systems/jeeves/monitoring/README.md b/systems/jeeves/monitoring/README.md new file mode 100644 index 0000000..56df1fe --- /dev/null +++ b/systems/jeeves/monitoring/README.md @@ -0,0 +1,33 @@ +# Monitoring + +## Vultr API metrics + +The `vultr-exporter` service reads its API token from: + +```text +/zfs/storage/secrets/services/vultr-exporter +``` + +Create the file on Jeeves as root with the following contents: + +```text +API_KEY= +``` + +The token needs read access to the Vultr Account and Billing APIs. Unrelated +resource collectors are disabled in the packaged exporter. + +Restrict the file to root and ensure the public egress IP used by Jeeves is +allowed for the token in the Vultr API settings: + +```console +sudo chown root:root /zfs/storage/secrets/services/vultr-exporter +sudo chmod 600 /zfs/storage/secrets/services/vultr-exporter +``` + +The exporter listens on `127.0.0.1:9188`; it is scraped by the local +`prometheus-main` service every five minutes and is not exposed through the +host firewall. + +Portal-1 exposes its node exporter only through `tailscale0` on port `9100`. +Jeeves reaches it using the Portal-1 Tailscale hostname. diff --git a/systems/jeeves/monitoring/dashboards/portal-1.json b/systems/jeeves/monitoring/dashboards/portal-1.json new file mode 100644 index 0000000..278a4f7 --- /dev/null +++ b/systems/jeeves/monitoring/dashboards/portal-1.json @@ -0,0 +1,798 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Offline" + }, + "1": { + "color": "green", + "index": 0, + "text": "Online" + } + }, + "type": "value" + } + ] + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "up{job=\"node\",instance=\"portal-1\"}", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Node Exporter", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 4, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{job=\"node\",instance=\"portal-1\",mode=\"idle\"}[5m])))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "CPU Used", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 9, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "100 * (1 - (node_memory_MemAvailable_bytes{job=\"node\",instance=\"portal-1\"} / node_memory_MemTotal_bytes{job=\"node\",instance=\"portal-1\"}))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "RAM Used", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 14, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "100 * (1 - (node_filesystem_avail_bytes{job=\"node\",instance=\"portal-1\",mountpoint=\"/\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node\",instance=\"portal-1\",mountpoint=\"/\",fstype!=\"\"}))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Root Filesystem Used", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 19, + "y": 0 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "node_load1{job=\"node\",instance=\"portal-1\"}", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Load (1m)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{job=\"node\",instance=\"portal-1\",mode=\"idle\"}[5m])))", + "instant": false, + "legendFormat": "CPU used", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "100 * (1 - (node_memory_MemAvailable_bytes{job=\"node\",instance=\"portal-1\"} / node_memory_MemTotal_bytes{job=\"node\",instance=\"portal-1\"}))", + "instant": false, + "legendFormat": "RAM used", + "range": true, + "refId": "A" + } + ], + "title": "RAM Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 8, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Value" + } + ] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "100 * (1 - (node_filesystem_avail_bytes{job=\"node\",instance=\"portal-1\",fstype!~\"tmpfs|devtmpfs|overlay|squashfs\",mountpoint!~\"/run.*\"} / node_filesystem_size_bytes{job=\"node\",instance=\"portal-1\",fstype!~\"tmpfs|devtmpfs|overlay|squashfs\",mountpoint!~\"/run.*\"}))", + "format": "table", + "instant": true, + "legendFormat": "{{mountpoint}}", + "range": false, + "refId": "A" + } + ], + "title": "Filesystem Usage", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "sum(rate(node_disk_read_bytes_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))", + "instant": false, + "legendFormat": "read", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "sum(rate(node_disk_written_bytes_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))", + "instant": false, + "legendFormat": "write", + "range": true, + "refId": "B" + } + ], + "title": "Disk Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "sum(rate(node_disk_reads_completed_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))", + "instant": false, + "legendFormat": "reads", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "sum(rate(node_disk_writes_completed_total{job=\"node\",instance=\"portal-1\",device!~\"loop.*|ram.*|fd.*\"}[5m]))", + "instant": false, + "legendFormat": "writes", + "range": true, + "refId": "B" + } + ], + "title": "Disk Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "sum(rate(node_network_receive_bytes_total{job=\"node\",instance=\"portal-1\",device!=\"lo\"}[5m]))", + "instant": false, + "legendFormat": "received", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "sum(rate(node_network_transmit_bytes_total{job=\"node\",instance=\"portal-1\",device!=\"lo\"}[5m]))", + "instant": false, + "legendFormat": "sent", + "range": true, + "refId": "B" + } + ], + "title": "Network Traffic", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "monitoring", + "portal-1", + "vultr" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Portal-1 Host", + "uid": "portal-1-host", + "version": 1, + "weekStart": "" +} diff --git a/systems/jeeves/monitoring/dashboards/vultr-infrastructure.json b/systems/jeeves/monitoring/dashboards/vultr-infrastructure.json new file mode 100644 index 0000000..b979d18 --- /dev/null +++ b/systems/jeeves/monitoring/dashboards/vultr-infrastructure.json @@ -0,0 +1,1053 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Offline" + }, + "1": { + "color": "green", + "index": 0, + "text": "Online" + } + }, + "type": "value" + } + ] + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "up{job=\"vultr\"}", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Exporter", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "vultr_account_bandwidth_value{period=\"current\",type=\"total_instance_count\"}", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "VM Count", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "currencyUSD", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 25 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "sum(vultr_billing_cost_usd) or vector(0)", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Current-cycle Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "vultr_account_balance", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Account Balance", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "currencyUSD", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 25 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "vultr_account_pending_charges", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Pending Charges", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "currencyUSD", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 25 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "vultr_account_bandwidth_value{period=\"projected\",type=\"overage_cost\"}", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Projected Overage Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "vultr_account_bandwidth_value{period=\"current\",type=\"gb_out\"}", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Outbound Bandwidth", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "vultr_account_bandwidth_value{period=\"current\",type=\"gb_in\"}", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Inbound Bandwidth", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 5 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "vultr_account_bandwidth_value{period=\"current\",type=\"free_bandwidth_credits\"}", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Free Bandwidth Credits", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 5 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "sum(vultr_account_bandwidth_value{period=\"current\",type=~\"(instance|free|purchased)_bandwidth_credits\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Total Bandwidth Credits", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 11, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "100 * vultr_account_bandwidth_value{period=\"current\",type=\"gb_out\"} / clamp_min(sum(vultr_account_bandwidth_value{period=\"current\",type=~\"(instance|free|purchased)_bandwidth_credits\"}), 1)", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Outbound Allowance Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "vultr_account_bandwidth_value{period=\"current\",type=\"gb_out\"}", + "instant": false, + "legendFormat": "current outbound GB", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "vultr_account_bandwidth_value{period=\"current\",type=\"gb_in\"}", + "instant": false, + "legendFormat": "current inbound GB", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "vultr_account_bandwidth_value{period=\"projected\",type=\"gb_out\"}", + "instant": false, + "legendFormat": "projected outbound GB", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "vultr_account_bandwidth_value{period=\"projected\",type=\"gb_in\"}", + "instant": false, + "legendFormat": "projected inbound GB", + "range": true, + "refId": "D" + } + ], + "title": "Bandwidth: Current and Projected", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "vultr_account_bandwidth_value{period=\"current\",type=\"instance_bandwidth_credits\"}", + "instant": false, + "legendFormat": "instance credits", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "vultr_account_bandwidth_value{period=\"current\",type=\"free_bandwidth_credits\"}", + "instant": false, + "legendFormat": "free credits", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "vultr_account_bandwidth_value{period=\"current\",type=\"purchased_bandwidth_credits\"}", + "instant": false, + "legendFormat": "purchased credits", + "range": true, + "refId": "C" + } + ], + "title": "Bandwidth Credit Breakdown", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 14, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Value" + } + ] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "sum by (product, description) (vultr_billing_cost_usd)", + "format": "table", + "instant": true, + "legendFormat": "{{product}} — {{description}}", + "range": false, + "refId": "A" + } + ], + "title": "Infrastructure Cost by Product", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "vultr_account_bandwidth_value{period=\"current\",type=\"total_instance_count\"}", + "instant": false, + "legendFormat": "VMs", + "range": true, + "refId": "A" + } + ], + "title": "VM Count History", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "vultr_account_pending_charges", + "instant": false, + "legendFormat": "pending charges", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom-main" + }, + "editorMode": "code", + "expr": "sum(vultr_billing_cost_usd)", + "instant": false, + "legendFormat": "itemized cost", + "range": true, + "refId": "B" + } + ], + "title": "Account Charges", + "type": "timeseries" + } + ], + "refresh": "5m", + "schemaVersion": 39, + "tags": [ + "monitoring", + "vultr", + "billing" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Vultr Infrastructure", + "uid": "vultr-infrastructure", + "version": 1, + "weekStart": "" +} diff --git a/systems/jeeves/monitoring/default.nix b/systems/jeeves/monitoring/default.nix index acc91ec..3d886a2 100644 --- a/systems/jeeves/monitoring/default.nix +++ b/systems/jeeves/monitoring/default.nix @@ -43,8 +43,15 @@ let static_configs = [ (mkTarget "jeeves" "jeeves.taile39c31.ts.net:9100") (mkTarget "bob" "bob.taile39c31.ts.net:9100") + (mkTarget "portal-1" "portal-1.taile39c31.ts.net:9100") ]; } + { + job_name = "vultr"; + scrape_interval = "5m"; + scrape_timeout = "45s"; + static_configs = [ (mkTarget "vultr" "127.0.0.1:9188") ]; + } { job_name = "process_grouped"; static_configs = [ @@ -153,6 +160,8 @@ let }; in { + imports = [ ./vultr-exporter.nix ]; + users = { groups.prometheus = { }; users.prometheus = { diff --git a/systems/jeeves/monitoring/vultr-exporter.nix b/systems/jeeves/monitoring/vultr-exporter.nix new file mode 100644 index 0000000..37941a3 --- /dev/null +++ b/systems/jeeves/monitoring/vultr-exporter.nix @@ -0,0 +1,96 @@ +{ + lib, + pkgs, + ... +}: +let + vars = import ../vars.nix; + + vultrExporter = pkgs.buildGoModule rec { + pname = "vultr-exporter"; + version = "0-unstable-2025-10-14"; + + src = pkgs.fetchFromGitHub { + owner = "DazWilkin"; + repo = "vultr-exporter"; + rev = "d8b5d7107c2eda31566e6e895c8a67db32fdaf68"; + hash = "sha256-3LpXh+mFMBRJfiY2ftELpM3AIE0LiZ9c6NFok0OeH5I="; + }; + + vendorHash = "sha256-qCHWmg6GxLIjqlf5UqRcuDRd1xqRFI3sl4ZduLajx+Y="; + subPackages = [ "cmd/server" ]; + + postPatch = '' + substituteInPlace cmd/server/main.go \ + --replace-fail 'registry.MustRegister(collector.NewBlockStorageCollector(s, client, log))' "" \ + --replace-fail 'registry.MustRegister(collector.NewKubernetesCollector(s, client, log))' "" \ + --replace-fail 'registry.MustRegister(collector.NewLoadBalancerCollector(s, client, log))' "" \ + --replace-fail 'registry.MustRegister(collector.NewReservedIPsCollector(s, client, log))' "" + ''; + + ldflags = [ + "-s" + "-w" + "-X main.GitCommit=${src.rev}" + "-X main.OSVersion=NixOS" + ]; + + postInstall = '' + mv $out/bin/server $out/bin/vultr-exporter + ''; + + meta = { + description = "Prometheus exporter for Vultr account, billing, and bandwidth metrics"; + homepage = "https://github.com/DazWilkin/vultr-exporter"; + license = lib.licenses.asl20; + mainProgram = "vultr-exporter"; + }; + }; +in +{ + systemd.services.vultr-exporter = { + description = "Vultr Prometheus exporter"; + documentation = [ "https://github.com/DazWilkin/vultr-exporter" ]; + wants = [ "network-online.target" ]; + after = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + ExecStart = "${lib.getExe vultrExporter} --endpoint=127.0.0.1:9188 --path=/metrics"; + EnvironmentFile = "${vars.secrets}/services/vultr-exporter"; + Restart = "on-failure"; + RestartSec = "30s"; + + CapabilityBoundingSet = [ "" ]; + DeviceAllow = [ "" ]; + DynamicUser = true; + LockPersonality = true; + MemoryDenyWriteExecute = true; + NoNewPrivileges = true; + PrivateDevices = true; + PrivateTmp = true; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectSystem = "strict"; + RemoveIPC = true; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + ]; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ + "@system-service" + "~@privileged" + ]; + UMask = "0077"; + }; + }; +} diff --git a/systems/portal-1/default.nix b/systems/portal-1/default.nix index 4bf111e..1d96af5 100644 --- a/systems/portal-1/default.nix +++ b/systems/portal-1/default.nix @@ -12,6 +12,7 @@ "${inputs.self}/common/optional/tailscale.nix" ./disk-config.nix ./haproxy + ./monitoring.nix ]; nixpkgs.hostPlatform = "x86_64-linux"; diff --git a/systems/portal-1/monitoring.nix b/systems/portal-1/monitoring.nix new file mode 100644 index 0000000..133b140 --- /dev/null +++ b/systems/portal-1/monitoring.nix @@ -0,0 +1,15 @@ +{ config, ... }: +{ + networking.firewall.interfaces.tailscale0.allowedTCPPorts = [ + config.services.prometheus.exporters.node.port + ]; + + services.prometheus.exporters.node = { + enable = true; + enabledCollectors = [ + "pressure" + "processes" + "systemd" + ]; + }; +}