{ "uid": "incus-haproxy-traffic", "title": "HAProxy Traffic", "description": "HAProxy load balancer monitoring: request rates, response codes, backend health, traffic volume, sessions, and performance metrics.", "tags": ["incus", "haproxy", "loadbalancer"], "timezone": "browser", "schemaVersion": 39, "version": 2, "refresh": "30s", "time": { "from": "now-1h", "to": "now" }, "fiscalYearStartMonth": 0, "liveNow": false, "editable": true, "graphTooltip": 1, "links": [ { "asDropdown": true, "icon": "external link", "includeVars": true, "keepTime": true, "tags": ["incus"], "targetBlank": false, "title": "Incus Dashboards", "tooltip": "Navigate to other Incus dashboards", "type": "dashboards" } ], "templating": { "list": [ { "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(haproxy_backend_status, proxy)", "includeAll": true, "multi": true, "name": "proxy", "query": "label_values(haproxy_backend_status, proxy)", "refresh": 2, "regex": "", "sort": 1, "type": "query", "label": "Proxy" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(haproxy_server_status, server)", "includeAll": true, "multi": true, "name": "server", "query": "label_values(haproxy_server_status, server)", "refresh": 2, "regex": "", "sort": 1, "type": "query", "label": "Server" } ] }, "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "panels": [ { "type": "row", "title": "Overview", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 1, "panels": [] }, { "type": "stat", "title": "Active Backend Servers", "description": "Total number of active (healthy) servers across selected HAProxy backends.", "gridPos": { "h": 4, "w": 5, "x": 0, "y": 1 }, "id": 2, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { "expr": "sum(haproxy_backend_active_servers{proxy=~\"$proxy\"})", "legendFormat": "Active Servers", "refId": "A" } ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 1 }, { "color": "green", "value": 2 } ] }, "unit": "none", "color": { "mode": "thresholds" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto" } }, { "type": "stat", "title": "Current Sessions", "description": "Total current active sessions across selected HAProxy backends.", "gridPos": { "h": 4, "w": 5, "x": 5, "y": 1 }, "id": 3, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { "expr": "sum(haproxy_backend_current_sessions{proxy=~\"$proxy\"})", "legendFormat": "Sessions", "refId": "A" } ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 100 }, { "color": "red", "value": 500 } ] }, "unit": "none", "color": { "mode": "thresholds" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "area", "justifyMode": "auto" } }, { "type": "stat", "title": "HTTP Requests/s", "description": "Current rate of HTTP requests per second across selected HAProxy frontends.", "gridPos": { "h": 4, "w": 5, "x": 10, "y": 1 }, "id": 4, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { "expr": "sum(rate(haproxy_frontend_http_requests_total{proxy=~\"$proxy\"}[$__rate_interval]))", "legendFormat": "Requests/s", "refId": "A" } ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps", "color": { "mode": "thresholds" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "area", "justifyMode": "auto" } }, { "type": "stat", "title": "HTTP 4xx Rate", "description": "Rate of HTTP 4xx client error responses per second. Elevated rates may indicate bad requests or missing resources.", "gridPos": { "h": 4, "w": 4, "x": 15, "y": 1 }, "id": 14, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { "expr": "sum(rate(haproxy_frontend_http_responses_total{proxy=~\"$proxy\", code=\"4xx\"}[$__rate_interval]))", "legendFormat": "4xx/s", "refId": "A" } ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.1 }, { "color": "red", "value": 1 } ] }, "unit": "reqps", "color": { "mode": "thresholds" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "area", "justifyMode": "auto" } }, { "type": "stat", "title": "HTTP 5xx Rate", "description": "Rate of HTTP 5xx server error responses per second. Any sustained rate indicates backend failures requiring investigation.", "gridPos": { "h": 4, "w": 5, "x": 19, "y": 1 }, "id": 15, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { "expr": "sum(rate(haproxy_frontend_http_responses_total{proxy=~\"$proxy\", code=\"5xx\"}[$__rate_interval]))", "legendFormat": "5xx/s", "refId": "A" } ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.1 } ] }, "unit": "reqps", "color": { "mode": "thresholds" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "area", "justifyMode": "auto" } }, { "type": "row", "title": "HTTP Requests", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 5, "panels": [] }, { "type": "timeseries", "title": "HTTP Request Rate by Frontend", "description": "Rate of HTTP requests per second, broken down by HAProxy frontend/proxy. Useful for identifying which frontends carry the most traffic.", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "id": 6, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { "expr": "sum by (proxy) (rate(haproxy_frontend_http_requests_total{proxy=~\"$proxy\"}[$__rate_interval]))", "legendFormat": "{{proxy}}", "refId": "A" } ], "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 20, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 5, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "unit": "reqps", "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "tooltip": { "mode": "multi", "sort": "desc" }, "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } } }, { "type": "timeseries", "title": "Response Code Distribution", "description": "HTTP response code distribution across selected frontends, stacked by response class (2xx, 3xx, 4xx, 5xx). Helps identify error trends and traffic composition.", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, "id": 16, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { "expr": "sum by (code) (rate(haproxy_frontend_http_responses_total{proxy=~\"$proxy\"}[$__rate_interval]))", "legendFormat": "{{code}}", "refId": "A" } ], "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 50, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 5, "stacking": { "mode": "normal", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "unit": "reqps", "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "2xx" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] }, { "matcher": { "id": "byName", "options": "3xx" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } } ] }, { "matcher": { "id": "byName", "options": "4xx" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "yellow" } } ] }, { "matcher": { "id": "byName", "options": "5xx" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] } ] }, "options": { "tooltip": { "mode": "multi", "sort": "desc" }, "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } } }, { "type": "row", "title": "Backend Health", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, "id": 7, "panels": [] }, { "type": "table", "title": "Backend Status", "description": "Current status, active server count, and average response time for each HAProxy backend. Filterable by proxy and server template variables.", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, "id": 8, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { "expr": "haproxy_backend_status{proxy=~\"$proxy\"}", "legendFormat": "", "refId": "A", "format": "table", "instant": true }, { "expr": "haproxy_backend_active_servers{proxy=~\"$proxy\"}", "legendFormat": "", "refId": "B", "format": "table", "instant": true }, { "expr": "haproxy_server_response_time_average_seconds{proxy=~\"$proxy\", server=~\"$server\"}", "legendFormat": "", "refId": "C", "format": "table", "instant": true } ], "fieldConfig": { "defaults": { "custom": { "align": "auto", "displayMode": "auto", "inspect": false, "filterable": true }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "color": { "mode": "thresholds" } }, "overrides": [ { "matcher": { "id": "byName", "options": "Value #A" }, "properties": [ { "id": "displayName", "value": "Status" }, { "id": "mappings", "value": [ { "type": "value", "options": { "1": { "text": "UP", "color": "green" }, "0": { "text": "DOWN", "color": "red" } } } ] } ] }, { "matcher": { "id": "byName", "options": "Value #B" }, "properties": [ { "id": "displayName", "value": "Active Servers" } ] }, { "matcher": { "id": "byName", "options": "Value #C" }, "properties": [ { "id": "displayName", "value": "Avg Response Time" }, { "id": "unit", "value": "s" } ] }, { "matcher": { "id": "byName", "options": "proxy" }, "properties": [ { "id": "displayName", "value": "Backend" } ] } ] }, "options": { "showHeader": true, "sortBy": [], "footer": { "show": false, "reducer": ["sum"], "fields": "" } }, "transformations": [ { "id": "merge", "options": {} }, { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "instance": false, "job": true }, "indexByName": {}, "renameByName": {} } } ] }, { "type": "state-timeline", "title": "Server Status", "description": "Timeline view of individual HAProxy server health status. Green indicates UP, red indicates DOWN. Helps identify flapping servers and outage windows.", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, "id": 17, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { "expr": "haproxy_server_status{proxy=~\"$proxy\", server=~\"$server\"}", "legendFormat": "{{proxy}} / {{server}}", "refId": "A" } ], "fieldConfig": { "defaults": { "custom": { "lineWidth": 0, "fillOpacity": 80 }, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] }, "mappings": [ { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } ] }, "overrides": [] }, "options": { "showValue": "auto", "mergeValues": true, "alignValue": "left", "rowHeight": 0.9, "tooltip": { "mode": "multi", "sort": "desc" }, "legend": { "displayMode": "list", "placement": "bottom" } } }, { "type": "row", "title": "Traffic", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, "id": 9, "panels": [] }, { "type": "timeseries", "title": "Frontend Bytes In", "description": "Inbound traffic rate per HAProxy frontend. Shows how much data clients are sending to each frontend.", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, "id": 10, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { "expr": "sum by (proxy) (rate(haproxy_frontend_bytes_in_total{proxy=~\"$proxy\"}[$__rate_interval]))", "legendFormat": "{{proxy}} (in)", "refId": "A" } ], "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 20, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 5, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "unit": "Bps", "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "tooltip": { "mode": "multi", "sort": "desc" }, "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } } }, { "type": "timeseries", "title": "Frontend Bytes Out", "description": "Outbound traffic rate per HAProxy frontend. Shows how much data is being served to clients from each frontend.", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, "id": 11, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { "expr": "sum by (proxy) (rate(haproxy_frontend_bytes_out_total{proxy=~\"$proxy\"}[$__rate_interval]))", "legendFormat": "{{proxy}} (out)", "refId": "A" } ], "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 20, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 5, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "unit": "Bps", "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "tooltip": { "mode": "multi", "sort": "desc" }, "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } } }, { "type": "row", "title": "Sessions & Performance", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, "id": 12, "panels": [] }, { "type": "timeseries", "title": "Current Sessions by Proxy", "description": "Current active sessions for each HAProxy backend/proxy over time. Sustained high values may indicate slow backends or session buildup.", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, "id": 13, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { "expr": "haproxy_backend_current_sessions{proxy=~\"$proxy\"}", "legendFormat": "{{proxy}}", "refId": "A" } ], "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 20, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 5, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "unit": "none", "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "tooltip": { "mode": "multi", "sort": "desc" }, "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } } }, { "type": "timeseries", "title": "Backend Response Time", "description": "Average response time from backend servers. High or increasing values indicate backend performance degradation.", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 }, "id": 18, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { "expr": "haproxy_server_response_time_average_seconds{proxy=~\"$proxy\", server=~\"$server\"}", "legendFormat": "{{proxy}} / {{server}}", "refId": "A" } ], "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 15, "gradientMode": "scheme", "spanNulls": false, "showPoints": "never", "pointSize": 5, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "line" } }, "unit": "s", "color": { "mode": "continuous-GrYlRd" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "red", "value": 2 } ] } }, "overrides": [] }, "options": { "tooltip": { "mode": "multi", "sort": "desc" }, "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } } }, { "type": "timeseries", "title": "Queue Time", "description": "Average time requests spend in the backend queue before being assigned to a server. High values indicate insufficient backend capacity.", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 41 }, "id": 19, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { "expr": "haproxy_backend_queue_time_average_seconds{proxy=~\"$proxy\"}", "legendFormat": "{{proxy}}", "refId": "A" } ], "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 15, "gradientMode": "scheme", "spanNulls": false, "showPoints": "never", "pointSize": 5, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "line" } }, "unit": "s", "color": { "mode": "continuous-GrYlRd" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.1 }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "options": { "tooltip": { "mode": "multi", "sort": "desc" }, "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } } }, { "type": "timeseries", "title": "Connection Errors", "description": "Rate of backend connection errors and connection retries per proxy. Non-zero values indicate connectivity problems between HAProxy and backend servers.", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 41 }, "id": 20, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { "expr": "sum by (proxy) (rate(haproxy_backend_connection_errors_total{proxy=~\"$proxy\"}[$__rate_interval]))", "legendFormat": "{{proxy}} errors", "refId": "A" }, { "expr": "sum by (proxy) (rate(haproxy_backend_retry_warnings_total{proxy=~\"$proxy\"}[$__rate_interval]))", "legendFormat": "{{proxy}} retries", "refId": "B" } ], "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 15, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 5, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto", "axisLabel": "", "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "unit": "reqps", "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*errors$" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*retries$" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } ] } ] }, "options": { "tooltip": { "mode": "multi", "sort": "desc" }, "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } } } ] }