diff --git a/systems/jeeves/monitoring/dashboards/haproxy-requests.json b/systems/jeeves/monitoring/dashboards/haproxy-requests.json new file mode 100644 index 0000000..8c428e3 --- /dev/null +++ b/systems/jeeves/monitoring/dashboards/haproxy-requests.json @@ -0,0 +1,529 @@ +{ + "annotations": { + "list": [] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Total requests", + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT count(*) AS \"Total requests\" FROM main.haproxy_request WHERE $__timeFilter(requested_at)", + "refId": "A" + } + ] + }, + { + "id": 2, + "type": "stat", + "title": "Unique client IPs", + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT count(DISTINCT client_ip) AS \"Unique IPs\" FROM main.haproxy_request WHERE $__timeFilter(requested_at)", + "refId": "A" + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Distinct user-agents", + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT count(DISTINCT user_agent) AS \"User-Agents\" FROM main.haproxy_request WHERE $__timeFilter(requested_at)", + "refId": "A" + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "4xx/5xx responses", + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT count(*) FILTER (WHERE status_code >= 400) AS \"Errors\" FROM main.haproxy_request WHERE $__timeFilter(requested_at)", + "refId": "A" + } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Requests by backend", + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "showPoints": "never" + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT $__timeGroupAlias(requested_at, $__interval), backend AS metric, count(*) AS requests FROM main.haproxy_request WHERE $__timeFilter(requested_at) GROUP BY 1, backend ORDER BY 1", + "refId": "A" + } + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "Backend response time percentiles", + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "showPoints": "never" + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT $__timeGroupAlias(requested_at, $__interval), percentile_cont(0.50) WITHIN GROUP (ORDER BY time_response) AS \"p50\", percentile_cont(0.90) WITHIN GROUP (ORDER BY time_response) AS \"p90\", percentile_cont(0.95) WITHIN GROUP (ORDER BY time_response) AS \"p95\", percentile_cont(0.99) WITHIN GROUP (ORDER BY time_response) AS \"p99\" FROM main.haproxy_request WHERE $__timeFilter(requested_at) AND time_response >= 0 GROUP BY 1 ORDER BY 1", + "refId": "A" + } + ] + }, + { + "id": 7, + "type": "table", + "title": "Top user-agents (bots)", + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "fieldConfig": { + "defaults": { + "custom": { + "filterable": true + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "sortBy": [] + }, + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT user_agent AS \"User-Agent\", count(*) AS \"Requests\", count(DISTINCT client_ip) AS \"Distinct IPs\" FROM main.haproxy_request WHERE $__timeFilter(requested_at) GROUP BY user_agent ORDER BY \"Requests\" DESC LIMIT 25", + "refId": "A" + } + ] + }, + { + "id": 8, + "type": "table", + "title": "Top client IPs", + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "fieldConfig": { + "defaults": { + "custom": { + "filterable": true + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "sortBy": [] + }, + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT client_ip AS \"Client IP\", count(*) AS \"Requests\", max(user_agent) AS \"User-Agent (sample)\" FROM main.haproxy_request WHERE $__timeFilter(requested_at) GROUP BY client_ip ORDER BY \"Requests\" DESC LIMIT 25", + "refId": "A" + } + ] + }, + { + "id": 9, + "type": "table", + "title": "Top endpoints", + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 20 + }, + "fieldConfig": { + "defaults": { + "custom": { + "filterable": true + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "sortBy": [] + }, + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT host AS \"Host\", path AS \"Path\", count(*) AS \"Requests\", round(avg(time_response)) AS \"Avg ms\" FROM main.haproxy_request WHERE $__timeFilter(requested_at) AND time_response >= 0 GROUP BY host, path ORDER BY \"Requests\" DESC LIMIT 25", + "refId": "A" + } + ] + }, + { + "id": 10, + "type": "piechart", + "title": "Requests by status code", + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 20 + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "values": true + } + }, + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "richie-postgres" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT status_code::text AS metric, count(*) AS value FROM main.haproxy_request WHERE $__timeFilter(requested_at) GROUP BY status_code ORDER BY value DESC", + "refId": "A" + } + ] + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "haproxy", + "richie" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "HAProxy Requests", + "uid": "haproxy-requests", + "version": 1, + "weekStart": "" +} diff --git a/systems/jeeves/services/grafana.nix b/systems/jeeves/services/grafana.nix index 870dafd..8e925c9 100644 --- a/systems/jeeves/services/grafana.nix +++ b/systems/jeeves/services/grafana.nix @@ -62,6 +62,21 @@ in uid = "prom-pid-short"; url = "http://127.0.0.1:9092"; } + { + access = "proxy"; + editable = false; + name = "richie-postgres"; + type = "postgres"; + uid = "richie-postgres"; + url = "/run/postgresql"; + user = "richie"; + jsonData = { + database = "richie"; + sslmode = "disable"; + postgresVersion = 1700; + timescaledb = false; + }; + } ]; }; }; @@ -71,6 +86,7 @@ in services.grafana.after = [ "prometheus-main.service" "prometheus-pid-short.service" + "postgresql.service" ]; tmpfiles.rules = [