diff --git a/incusos/deploy-observability b/incusos/deploy-observability index 48bca3b..73b9f29 100755 --- a/incusos/deploy-observability +++ b/incusos/deploy-observability @@ -233,21 +233,41 @@ phase_deploy_monitoring() { -c limits.memory=2GiB \ -d root,size=20GiB + # Tell OVN the static IP we want — port security drops traffic from + # IPs that OVN didn't assign, so this must be set at the device level. + incus config device set "$(container_ref monitoring)" eth0 \ + ipv4.address="${MONITORING_IP}" + info "Waiting for container agent..." wait_for_container_agent monitoring 60 - # Configure static IP + # Discover upstream DNS from the UPLINK network config. + # OVN DHCP advertises this automatically, but we need it for the static + # networkd config (which bypasses DHCP). + local upstream_dns + upstream_dns=$(incus network get "${CLUSTER_REMOTE}:UPLINK" dns.nameservers 2>/dev/null) || true + upstream_dns="${upstream_dns:-192.168.100.1}" + detail "Using upstream DNS: ${upstream_dns}" + + # Build DNS lines (handles comma-separated list from UPLINK config) + local dns_lines="" + IFS=',' read -ra dns_addrs <<< "$upstream_dns" + for addr in "${dns_addrs[@]}"; do + dns_lines="${dns_lines}DNS=${addr}"$'\n' + done + + # Configure matching static IP inside the container (systemd-networkd + # bypasses DHCP, so we must specify DNS explicitly). container_exec monitoring bash -c " - cat > /etc/systemd/network/10-static.network << 'NETCFG' + cat > /etc/systemd/network/10-static.network << NETCFG [Match] Name=eth0 [Network] Address=${MONITORING_IP}/24 Gateway=10.10.10.1 -DNS=10.10.10.1 -NETCFG - systemctl restart systemd-networkd +${dns_lines}NETCFG + systemctl restart systemd-networkd systemd-resolved " sleep 3 @@ -281,7 +301,15 @@ phase_install_prometheus() { container_exec monitoring bash -c " export DEBIAN_FRONTEND=noninteractive + + # Add Grafana apt repository (provides grafana, loki, promtail) apt-get update -qq + apt-get install -y -qq ca-certificates curl gnupg apt-transport-https 2>&1 | tail -1 + curl -fsSL https://apt.grafana.com/gpg.key | gpg --dearmor -o /usr/share/keyrings/grafana-archive-keyring.gpg + echo 'deb [signed-by=/usr/share/keyrings/grafana-archive-keyring.gpg] https://apt.grafana.com stable main' \ + > /etc/apt/sources.list.d/grafana.list + apt-get update -qq + apt-get install -y -qq prometheus 2>&1 | tail -1 # Place Incus client certs where Prometheus can read them @@ -524,6 +552,10 @@ phase_deploy_node_exporters() { -c limits.memory=128MiB \ -c security.privileged=true + # Reserve this IP in OVN before switching to static inside the container + incus config device set "$(container_ref "$name")" eth0 \ + ipv4.address="${ip}" + wait_for_container_agent "$name" 30 # Mount host filesystems for full node metrics @@ -582,14 +614,13 @@ phase_create_acl() { # Create the ACL if it does not exist if ! incus network acl show "$(container_ref monitoring-allow)" &>/dev/null 2>&1; then - incus network acl create "$(container_ref monitoring-allow)" \ - description="Allow all traffic for observability stack" + incus network acl create "$(container_ref monitoring-allow)" # Add allow-all egress and ingress rules incus network acl rule add "$(container_ref monitoring-allow)" egress \ - action=allow description="Allow all egress" + action=allow incus network acl rule add "$(container_ref monitoring-allow)" ingress \ - action=allow description="Allow all ingress" + action=allow success "ACL 'monitoring-allow' created with allow-all rules" else success "ACL 'monitoring-allow' already exists" @@ -615,6 +646,19 @@ phase_create_acl() { idx=$((idx + 1)) done + # Changing security.acls can bring the NIC down. Bring all NICs back up. + sleep 2 + container_exec monitoring bash -c "ip link set eth0 up; systemctl restart systemd-networkd" 2>/dev/null || true + local nic_idx=1 + for ip in "${NODE_EXP_IPS[@]}"; do + local nname + nname=$(node_exp_name "$nic_idx") + if container_exists "$nname"; then + container_exec "$nname" sh -c "ip link set eth0 up; rc-service networking restart" 2>/dev/null || true + fi + nic_idx=$((nic_idx + 1)) + done + success "ACL attached to all observability containers" } @@ -815,6 +859,9 @@ phase_deploy_workload_web() { --target "oc-node-01" \ -c limits.memory=128MiB + incus config device set "$(container_ref workload-web)" eth0 \ + ipv4.address="${WORKLOAD_WEB_IP}" + wait_for_container_agent workload-web 30 container_exec workload-web sh -c " @@ -889,6 +936,9 @@ phase_deploy_workload_api() { --target "oc-node-03" \ -c limits.memory=128MiB + incus config device set "$(container_ref workload-api)" eth0 \ + ipv4.address="${WORKLOAD_API_IP}" + wait_for_container_agent workload-api 30 container_exec workload-api sh -c " diff --git a/incusos/manage-dashboards b/incusos/manage-dashboards index a1b21ed..1298971 100755 --- a/incusos/manage-dashboards +++ b/incusos/manage-dashboards @@ -190,7 +190,7 @@ for f in folders: " 2>/dev/null || echo "") if [[ -n "$folder_id" ]]; then - detail "Folder '${folder_title}' exists (ID: ${folder_id})" + detail "Folder '${folder_title}' exists (ID: ${folder_id})" >&2 echo "$folder_id" return 0 fi @@ -209,7 +209,7 @@ print(json.load(sys.stdin).get('id', '')) " 2>/dev/null || echo "") if [[ -n "$folder_id" ]]; then - detail "Created folder '${folder_title}' (ID: ${folder_id})" + detail "Created folder '${folder_title}' (ID: ${folder_id})" >&2 echo "$folder_id" else error "Failed to create folder '${folder_title}': ${result}" @@ -226,21 +226,54 @@ read_manifest() { fi python3 -c " -import yaml, sys +import sys, re + +entries = [] +current = {} with open('${MANIFEST_FILE}') as f: - data = yaml.safe_load(f) + for line in f: + stripped = line.rstrip() + # New entry starts with ' - file:' + m = re.match(r'^ - file:\s*(.*)', stripped) + if m: + if current: + entries.append(current) + current = {'file': m.group(1).strip()} + continue + # Key-value within an entry + m = re.match(r'^ (\w+):\s*(.*)', stripped) + if m and current: + key, val = m.group(1), m.group(2).strip() + # Strip YAML >- multiline indicators and quotes + if val in ('>-', '>', '|'): + val = '' + val = val.strip('\"').strip(\"'\") + if key in current and not val: + continue + current[key] = val + continue + # Continuation line (multiline description) + if current and stripped.startswith(' ') and stripped.strip(): + if 'description' in current: + sep = ' ' if current['description'] else '' + current['description'] = current['description'] + sep + stripped.strip() -for d in data.get('dashboards', []): - file_path = d.get('file', '') - uid = d.get('uid', '') - title = d.get('title', '') - folder = d.get('folder', 'General') - category = d.get('category', '') - version = d.get('version', '1') - description = d.get('description', '') - print(f'{file_path}\t{uid}\t{title}\t{folder}\t{category}\t{version}\t{description}') -" 2>/dev/null +if current: + entries.append(current) + +for d in entries: + parts = [ + d.get('file', ''), + d.get('uid', ''), + d.get('title', ''), + d.get('folder', 'General'), + d.get('category', ''), + d.get('version', '1'), + d.get('description', ''), + ] + print('\t'.join(parts)) +" } # Substitute datasource variables in JSON content @@ -279,7 +312,7 @@ action_install() { info "(dry-run) Would install dashboards from ${DASHBOARDS_DIR}" local count=0 - while IFS=$'\t' read -r file uid title folder category version description; do + while IFS=$'\t' read -r file uid title folder category version description <&3; do if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then continue fi @@ -288,7 +321,7 @@ action_install() { fi info "(dry-run) Would install: ${title} -> folder '${folder}'" count=$((count + 1)) - done < <(read_manifest) + done 3< <(read_manifest) info "(dry-run) ${count} dashboard(s) would be installed" return 0 @@ -299,7 +332,7 @@ action_install() { local installed=0 local failed=0 - while IFS=$'\t' read -r file uid title folder category version description; do + while IFS=$'\t' read -r file uid title folder category version description <&3; do if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then continue fi @@ -318,53 +351,24 @@ action_install() { local folder_id folder_id=$(ensure_folder "$folder") || { failed=$((failed + 1)); continue; } - # Read and process the dashboard JSON - local raw_json - raw_json=$(cat "$full_path") - - # Inject actual datasource UIDs - local processed_json - processed_json=$(inject_datasource_uids "$raw_json") - - # Wrap in Grafana API envelope + # Build Grafana API payload: inject datasource UIDs and wrap in envelope local api_payload - api_payload=$(python3 -c " -import json, sys - -dashboard = json.loads('''${processed_json//\'/\\\'}''') - -# Remove Grafana-internal fields that shouldn't be in the import -dashboard.pop('id', None) -dashboard.pop('version', None) -dashboard.pop('iteration', None) - -payload = { - 'dashboard': dashboard, - 'folderId': ${folder_id}, - 'overwrite': True, - 'message': 'Installed by manage-dashboards v${VERSION}' -} -print(json.dumps(payload)) -" 2>/dev/null) - - if [[ -z "$api_payload" ]]; then - # Fallback: use a simpler approach for large/complex JSON - api_payload=$(python3 << 'PYEOF' + api_payload=$(python3 - "$full_path" "$PROMETHEUS_UID" "${LOKI_UID:-}" "$folder_id" << 'PYEOF' import json, sys with open(sys.argv[1]) as f: dashboard = json.load(f) -# Inject datasource UIDs +# Inject actual datasource UIDs json_str = json.dumps(dashboard) json_str = json_str.replace('${DS_PROMETHEUS}', sys.argv[2]) -json_str = json_str.replace('${DS_LOKI}', sys.argv[3]) +if sys.argv[3]: + json_str = json_str.replace('${DS_LOKI}', sys.argv[3]) dashboard = json.loads(json_str) # Remove Grafana-internal fields -dashboard.pop('id', None) -dashboard.pop('version', None) -dashboard.pop('iteration', None) +for key in ('id', 'version', 'iteration'): + dashboard.pop(key, None) payload = { 'dashboard': dashboard, @@ -374,8 +378,7 @@ payload = { } print(json.dumps(payload)) PYEOF - "$full_path" "$PROMETHEUS_UID" "${LOKI_UID:-}" "$folder_id" 2>/dev/null) - fi + ) if [[ -z "$api_payload" ]]; then error "Failed to process dashboard JSON: ${file}" @@ -388,9 +391,10 @@ PYEOF tmpfile=$(mktemp /tmp/dashboard-payload.XXXXXX) echo "$api_payload" > "$tmpfile" + incus file push "$tmpfile" "${CLUSTER_REMOTE}:${MONITORING_CONTAINER}/tmp/dashboard-payload.json" >/dev/null 2>&1 + local result - result=$(incus file push "$tmpfile" "${CLUSTER_REMOTE}:${MONITORING_CONTAINER}/tmp/dashboard-payload.json" 2>/dev/null && \ - grafana_api POST "/api/dashboards/db" \ + result=$(grafana_api POST "/api/dashboards/db" \ -H "Content-Type: application/json" \ -d @/tmp/dashboard-payload.json 2>/dev/null) rm -f "$tmpfile" @@ -417,7 +421,7 @@ except: # Clean up temp file in container incus exec "${CLUSTER_REMOTE}:${MONITORING_CONTAINER}" -- rm -f /tmp/dashboard-payload.json 2>/dev/null || true - done < <(read_manifest) + done 3< <(read_manifest) echo if [[ $failed -eq 0 ]]; then @@ -446,7 +450,7 @@ action_export() { fi # Export all dashboards listed in the manifest - while IFS=$'\t' read -r file uid title folder category version description; do + while IFS=$'\t' read -r file uid title folder category version description <&3; do if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then continue fi @@ -454,7 +458,7 @@ action_export() { if export_single_dashboard "$uid" "${DASHBOARDS_DIR}/${file}"; then exported=$((exported + 1)) fi - done < <(read_manifest) + done 3< <(read_manifest) echo success "Exported ${exported} dashboard(s)" @@ -560,7 +564,7 @@ action_list() { printf " ${BOLD}%-35s %-30s %-20s %s${RESET}\n" "UID" "Title" "Folder" "Status" printf " %-35s %-30s %-20s %s\n" "---" "-----" "------" "------" - while IFS=$'\t' read -r file uid title folder category version description; do + while IFS=$'\t' read -r file uid title folder category version description <&3; do if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then continue fi @@ -601,7 +605,7 @@ except: printf " %-35s %-30s %-20s " "$uid" "$title" "$folder" echo -e "$status_display" - done < <(read_manifest) + done 3< <(read_manifest) } # --------------------------------------------------------------------------- @@ -617,7 +621,7 @@ action_validate() { local warnings=0 local errors=0 - while IFS=$'\t' read -r file uid title folder category version description; do + while IFS=$'\t' read -r file uid title folder category version description <&3; do if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then continue fi @@ -734,7 +738,7 @@ PYEOF success " ${basename_file}: valid" fi - done < <(read_manifest) + done 3< <(read_manifest) echo info "${total} dashboard(s) checked: ${passed} passed, ${errors} errors, ${warnings} warnings" @@ -758,7 +762,7 @@ action_status() { printf " ${BOLD}%-30s %-15s %-15s %s${RESET}\n" "Dashboard" "Local" "Grafana" "Match" printf " %-30s %-15s %-15s %s\n" "---------" "-----" "-------" "-----" - while IFS=$'\t' read -r file uid title folder category version description; do + while IFS=$'\t' read -r file uid title folder category version description <&3; do local full_path="${DASHBOARDS_DIR}/${file}" local local_ver="missing" @@ -794,7 +798,7 @@ except: printf " %-30s %-15s %-15s " "$title" "$local_ver" "$grafana_ver" echo -e "$match_str" - done < <(read_manifest) + done 3< <(read_manifest) } # --------------------------------------------------------------------------- diff --git a/incusos/observability-dashboards/core/cluster-overview.json b/incusos/observability-dashboards/core/cluster-overview.json index 28e9e7f..9635319 100644 --- a/incusos/observability-dashboards/core/cluster-overview.json +++ b/incusos/observability-dashboards/core/cluster-overview.json @@ -2,11 +2,18 @@ "uid": "incus-cluster-overview", "title": "Incus Cluster Overview", "description": "Overview of all Incus instances: CPU, memory, network, disk I/O, and instance status table with drill-down links.", - "tags": ["incus", "cluster", "containers"], + "tags": [ + "incus", + "cluster", + "containers" + ], "timezone": "browser", "schemaVersion": 39, "refresh": "30s", - "time": { "from": "now-1h", "to": "now" }, + "time": { + "from": "now-1h", + "to": "now" + }, "fiscalYearStartMonth": 0, "liveNow": false, "editable": true, @@ -17,7 +24,9 @@ "icon": "external link", "includeVars": true, "keepTime": true, - "tags": ["incus"], + "tags": [ + "incus" + ], "targetBlank": false, "title": "Incus Dashboards", "tooltip": "Navigate to other Incus dashboards", @@ -28,7 +37,10 @@ "list": [ { "current": {}, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "definition": "label_values(incus_uptime_seconds, name)", "includeAll": true, "multi": true, @@ -38,11 +50,15 @@ "regex": "", "sort": 1, "type": "query", - "label": "Instance" + "label": "Instance", + "allValue": ".*" }, { "current": {}, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "definition": "label_values(incus_uptime_seconds, instance)", "includeAll": true, "multi": true, @@ -52,7 +68,8 @@ "regex": "", "sort": 1, "type": "query", - "label": "Node" + "label": "Node", + "allValue": ".*" } ] }, @@ -60,7 +77,10 @@ "list": [ { "builtIn": 1, - "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -74,7 +94,12 @@ "type": "row", "title": "Cluster Status", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "id": 1, "panels": [] }, @@ -82,9 +107,17 @@ "type": "stat", "title": "Total Instances", "description": "Number of running Incus instances (reporting uptime).", - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, "id": 2, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "count(incus_uptime_seconds{instance=~\"$node\", name=~\"$instance\"})", @@ -97,18 +130,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 10 }, - { "color": "red", "value": 50 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 50 + } ] }, "unit": "none", - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -120,9 +170,17 @@ "type": "stat", "title": "Total CPUs", "description": "Sum of effective CPUs across all Incus instances.", - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, "id": 3, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum(incus_cpu_effective_total{instance=~\"$node\", name=~\"$instance\"})", @@ -134,15 +192,28 @@ "defaults": { "thresholds": { "mode": "absolute", - "steps": [{ "color": "green", "value": null }] + "steps": [ + { + "color": "green", + "value": null + } + ] }, "unit": "none", - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -154,9 +225,17 @@ "type": "stat", "title": "Total Memory", "description": "Sum of total memory across all Incus instances.", - "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, "id": 4, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum(incus_memory_MemTotal_bytes{instance=~\"$node\", name=~\"$instance\"})", @@ -168,15 +247,28 @@ "defaults": { "thresholds": { "mode": "absolute", - "steps": [{ "color": "green", "value": null }] + "steps": [ + { + "color": "green", + "value": null + } + ] }, "unit": "bytes", - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -188,9 +280,17 @@ "type": "gauge", "title": "Cluster Memory Utilization", "description": "Overall cluster memory utilization (RSS / Total) across all instances.", - "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, "id": 40, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "(sum(incus_memory_RSS_bytes{instance=~\"$node\", name=~\"$instance\"}) / sum(incus_memory_MemTotal_bytes{instance=~\"$node\", name=~\"$instance\"})) * 100", @@ -203,21 +303,41 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 60 }, - { "color": "orange", "value": 80 }, - { "color": "red", "value": 90 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 60 + }, + { + "color": "orange", + "value": 80 + }, + { + "color": "red", + "value": 90 + } ] }, "unit": "percent", "min": 0, "max": 100, - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "auto" @@ -227,7 +347,12 @@ "type": "row", "title": "Instance Status", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, "id": 50, "panels": [] }, @@ -235,9 +360,17 @@ "type": "table", "title": "Instance Status Table", "description": "Per-instance summary: uptime, CPU usage, memory RSS, total memory. Click instance name to drill down.", - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 6 + }, "id": 51, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "incus_uptime_seconds{instance=~\"$node\", name=~\"$instance\"}", @@ -278,16 +411,32 @@ }, "thresholds": { "mode": "absolute", - "steps": [{ "color": "green", "value": null }] + "steps": [ + { + "color": "green", + "value": null + } + ] }, - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [ { - "matcher": { "id": "byName", "options": "name" }, + "matcher": { + "id": "byName", + "options": "name" + }, "properties": [ - { "id": "displayName", "value": "Instance" }, - { "id": "custom.width", "value": 200 }, + { + "id": "displayName", + "value": "Instance" + }, + { + "id": "custom.width", + "value": 200 + }, { "id": "links", "value": [ @@ -301,54 +450,123 @@ ] }, { - "matcher": { "id": "byName", "options": "instance" }, - "properties": [{ "id": "displayName", "value": "Node" }] - }, - { - "matcher": { "id": "byName", "options": "Value #A" }, + "matcher": { + "id": "byName", + "options": "instance" + }, "properties": [ - { "id": "displayName", "value": "Uptime" }, - { "id": "unit", "value": "s" } + { + "id": "displayName", + "value": "Node" + } ] }, { - "matcher": { "id": "byName", "options": "Value #B" }, + "matcher": { + "id": "byName", + "options": "Value #A" + }, "properties": [ - { "id": "displayName", "value": "CPU/s" }, - { "id": "unit", "value": "short" }, - { "id": "decimals", "value": 3 } + { + "id": "displayName", + "value": "Uptime" + }, + { + "id": "unit", + "value": "s" + } ] }, { - "matcher": { "id": "byName", "options": "Value #C" }, + "matcher": { + "id": "byName", + "options": "Value #B" + }, "properties": [ - { "id": "displayName", "value": "Memory RSS" }, - { "id": "unit", "value": "bytes" } + { + "id": "displayName", + "value": "CPU/s" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 3 + } ] }, { - "matcher": { "id": "byName", "options": "Value #D" }, + "matcher": { + "id": "byName", + "options": "Value #C" + }, "properties": [ - { "id": "displayName", "value": "Total Memory" }, - { "id": "unit", "value": "bytes" } + { + "id": "displayName", + "value": "Memory RSS" + }, + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #D" + }, + "properties": [ + { + "id": "displayName", + "value": "Total Memory" + }, + { + "id": "unit", + "value": "bytes" + } ] } ] }, "options": { "showHeader": true, - "sortBy": [{ "displayName": "Instance", "desc": false }], - "footer": { "show": false, "reducer": ["sum"], "fields": "" } + "sortBy": [ + { + "displayName": "Instance", + "desc": false + } + ], + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "fields": "" + } }, "transformations": [ - { "id": "merge", "options": {} }, + { + "id": "merge", + "options": {} + }, { "id": "organize", "options": { - "excludeByName": { "Time": true, "__name__": true, "job": true }, + "excludeByName": { + "Time": true, + "__name__": true, + "job": true + }, "indexByName": { - "name": 0, "instance": 1, - "Value #A": 2, "Value #B": 3, "Value #C": 4, "Value #D": 5 + "name": 0, + "instance": 1, + "Value #A": 2, + "Value #B": 3, + "Value #C": 4, + "Value #D": 5 }, "renameByName": {} } @@ -359,7 +577,12 @@ "type": "row", "title": "Per-Instance CPU", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, "id": 5, "panels": [] }, @@ -367,9 +590,17 @@ "type": "timeseries", "title": "CPU Usage by Instance", "description": "Per-second CPU time consumed by each Incus instance.", - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 15 }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 15 + }, "id": 6, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum by (name) (rate(incus_cpu_seconds_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))", @@ -388,17 +619,31 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short", - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", - "steps": [{ "color": "green", "value": null }] + "steps": [ + { + "color": "green", + "value": null + } + ] }, "links": [ { @@ -411,15 +656,31 @@ "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "row", "title": "Per-Instance Memory", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, "id": 7, "panels": [] }, @@ -427,9 +688,17 @@ "type": "timeseries", "title": "Memory RSS by Instance", "description": "Resident set size (RSS) memory usage per Incus instance over time.", - "gridPos": { "h": 8, "w": 14, "x": 0, "y": 24 }, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 24 + }, "id": 8, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "incus_memory_RSS_bytes{instance=~\"$node\", name=~\"$instance\"}", @@ -448,33 +717,66 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "bytes", - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", - "steps": [{ "color": "green", "value": null }] + "steps": [ + { + "color": "green", + "value": null + } + ] } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "bargauge", "title": "Memory Usage % (RSS / Total)", "description": "Current memory utilization per instance as RSS divided by total memory.", - "gridPos": { "h": 8, "w": 10, "x": 14, "y": 24 }, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 24 + }, "id": 9, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "(incus_memory_RSS_bytes{instance=~\"$node\", name=~\"$instance\"} / incus_memory_MemTotal_bytes{instance=~\"$node\", name=~\"$instance\"}) * 100", @@ -487,21 +789,41 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 60 }, - { "color": "orange", "value": 80 }, - { "color": "red", "value": 90 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 60 + }, + { + "color": "orange", + "value": 80 + }, + { + "color": "red", + "value": 90 + } ] }, "unit": "percent", "min": 0, "max": 100, - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "horizontal", "displayMode": "gradient", "showUnfilled": true, @@ -514,7 +836,12 @@ "type": "row", "title": "Network I/O", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, "id": 10, "panels": [] }, @@ -522,9 +849,17 @@ "type": "timeseries", "title": "Network Receive Rate by Instance", "description": "Inbound network throughput per Incus instance.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, "id": 11, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum by (name) (rate(incus_network_receive_bytes_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))", @@ -535,32 +870,74 @@ "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, - "fillOpacity": 15, "gradientMode": "none", "spanNulls": false, - "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "Bps", - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "timeseries", "title": "Network Transmit Rate by Instance", "description": "Outbound network throughput per Incus instance.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, "id": 12, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum by (name) (rate(incus_network_transmit_bytes_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))", @@ -571,30 +948,69 @@ "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, - "fillOpacity": 15, "gradientMode": "none", "spanNulls": false, - "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "Bps", - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "row", "title": "Disk I/O", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, "id": 13, "panels": [] }, @@ -602,9 +1018,17 @@ "type": "timeseries", "title": "Disk Read Rate by Instance", "description": "Disk read throughput per Incus instance.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 42 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, "id": 14, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum by (name) (rate(incus_disk_read_bytes_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))", @@ -615,32 +1039,74 @@ "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, - "fillOpacity": 15, "gradientMode": "none", "spanNulls": false, - "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "Bps", - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "timeseries", "title": "Disk Write Rate by Instance", "description": "Disk write throughput per Incus instance.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 42 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, "id": 15, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum by (name) (rate(incus_disk_written_bytes_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))", @@ -651,23 +1117,57 @@ "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, - "fillOpacity": 15, "gradientMode": "none", "spanNulls": false, - "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "Bps", - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } } ] diff --git a/incusos/observability-dashboards/core/haproxy-traffic.json b/incusos/observability-dashboards/core/haproxy-traffic.json index 4d65a0b..b578306 100644 --- a/incusos/observability-dashboards/core/haproxy-traffic.json +++ b/incusos/observability-dashboards/core/haproxy-traffic.json @@ -2,7 +2,11 @@ "uid": "incus-haproxy-traffic", "title": "HAProxy Traffic", "description": "HAProxy load balancer monitoring: request rates, response codes, backend health, traffic volume, sessions, and performance metrics.", - "tags": ["incus", "haproxy", "loadbalancer"], + "tags": [ + "incus", + "haproxy", + "loadbalancer" + ], "timezone": "browser", "schemaVersion": 39, "version": 2, @@ -21,7 +25,9 @@ "icon": "external link", "includeVars": true, "keepTime": true, - "tags": ["incus"], + "tags": [ + "incus" + ], "targetBlank": false, "title": "Incus Dashboards", "tooltip": "Navigate to other Incus dashboards", @@ -32,7 +38,10 @@ "list": [ { "current": {}, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "definition": "label_values(haproxy_backend_status, proxy)", "includeAll": true, "multi": true, @@ -42,11 +51,15 @@ "regex": "", "sort": 1, "type": "query", - "label": "Proxy" + "label": "Proxy", + "allValue": ".*" }, { "current": {}, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "definition": "label_values(haproxy_server_status, server)", "includeAll": true, "multi": true, @@ -56,7 +69,8 @@ "regex": "", "sort": 1, "type": "query", - "label": "Server" + "label": "Server", + "allValue": ".*" } ] }, @@ -81,7 +95,12 @@ "type": "row", "title": "Overview", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "id": 1, "panels": [] }, @@ -89,9 +108,17 @@ "type": "stat", "title": "Active Backend Servers", "description": "Total number of active (healthy) servers across selected HAProxy backends.", - "gridPos": { "h": 4, "w": 5, "x": 0, "y": 1 }, + "gridPos": { + "h": 4, + "w": 5, + "x": 0, + "y": 1 + }, "id": 2, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum(haproxy_backend_active_servers{proxy=~\"$proxy\"})", @@ -104,19 +131,32 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "green", "value": 2 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "green", + "value": 2 + } ] }, "unit": "none", - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -131,9 +171,17 @@ "type": "stat", "title": "Current Sessions", "description": "Total current active sessions across selected HAProxy backends.", - "gridPos": { "h": 4, "w": 5, "x": 5, "y": 1 }, + "gridPos": { + "h": 4, + "w": 5, + "x": 5, + "y": 1 + }, "id": 3, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum(haproxy_backend_current_sessions{proxy=~\"$proxy\"})", @@ -146,19 +194,32 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 100 }, - { "color": "red", "value": 500 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 500 + } ] }, "unit": "none", - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -173,9 +234,17 @@ "type": "stat", "title": "HTTP Requests/s", "description": "Current rate of HTTP requests per second across selected HAProxy frontends.", - "gridPos": { "h": 4, "w": 5, "x": 10, "y": 1 }, + "gridPos": { + "h": 4, + "w": 5, + "x": 10, + "y": 1 + }, "id": 4, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum(rate(haproxy_frontend_http_requests_total{proxy=~\"$proxy\"}[$__rate_interval]))", @@ -188,17 +257,24 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "reqps", - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -213,9 +289,17 @@ "type": "stat", "title": "HTTP 4xx Rate", "description": "Rate of HTTP 4xx client error responses per second. Elevated rates may indicate bad requests or missing resources.", - "gridPos": { "h": 4, "w": 4, "x": 15, "y": 1 }, + "gridPos": { + "h": 4, + "w": 4, + "x": 15, + "y": 1 + }, "id": 14, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum(rate(haproxy_frontend_http_responses_total{proxy=~\"$proxy\", code=\"4xx\"}[$__rate_interval]))", @@ -228,19 +312,32 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.1 }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } ] }, "unit": "reqps", - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -255,9 +352,17 @@ "type": "stat", "title": "HTTP 5xx Rate", "description": "Rate of HTTP 5xx server error responses per second. Any sustained rate indicates backend failures requiring investigation.", - "gridPos": { "h": 4, "w": 5, "x": 19, "y": 1 }, + "gridPos": { + "h": 4, + "w": 5, + "x": 19, + "y": 1 + }, "id": 15, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum(rate(haproxy_frontend_http_responses_total{proxy=~\"$proxy\", code=\"5xx\"}[$__rate_interval]))", @@ -270,19 +375,32 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.01 }, - { "color": "red", "value": 0.1 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.01 + }, + { + "color": "red", + "value": 0.1 + } ] }, "unit": "reqps", - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -297,7 +415,12 @@ "type": "row", "title": "HTTP Requests", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, "id": 5, "panels": [] }, @@ -305,9 +428,17 @@ "type": "timeseries", "title": "HTTP Request Rate by Frontend", "description": "Rate of HTTP requests per second, broken down by HAProxy frontend/proxy. Useful for identifying which frontends carry the most traffic.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, "id": 6, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum by (proxy) (rate(haproxy_frontend_http_requests_total{proxy=~\"$proxy\"}[$__rate_interval]))", @@ -326,35 +457,66 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "reqps", - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "timeseries", "title": "Response Code Distribution", "description": "HTTP response code distribution across selected frontends, stacked by response class (2xx, 3xx, 4xx, 5xx). Helps identify error trends and traffic composition.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, "id": 16, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum by (code) (rate(haproxy_frontend_http_responses_total{proxy=~\"$proxy\"}[$__rate_interval]))", @@ -373,58 +535,122 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "normal", "group": "A" }, + "stacking": { + "mode": "normal", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "reqps", - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] } }, "overrides": [ { - "matcher": { "id": "byName", "options": "2xx" }, + "matcher": { + "id": "byName", + "options": "2xx" + }, "properties": [ - { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } ] }, { - "matcher": { "id": "byName", "options": "3xx" }, + "matcher": { + "id": "byName", + "options": "3xx" + }, "properties": [ - { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } } + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } ] }, { - "matcher": { "id": "byName", "options": "4xx" }, + "matcher": { + "id": "byName", + "options": "4xx" + }, "properties": [ - { "id": "color", "value": { "mode": "fixed", "fixedColor": "yellow" } } + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } ] }, { - "matcher": { "id": "byName", "options": "5xx" }, + "matcher": { + "id": "byName", + "options": "5xx" + }, "properties": [ - { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } ] } ] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "row", "title": "Backend Health", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, "id": 7, "panels": [] }, @@ -432,9 +658,17 @@ "type": "table", "title": "Backend Status", "description": "Current status, active server count, and average response time for each HAProxy backend. Filterable by proxy and server template variables.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, "id": 8, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "haproxy_backend_status{proxy=~\"$proxy\"}", @@ -469,41 +703,85 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [ { - "matcher": { "id": "byName", "options": "Value #A" }, + "matcher": { + "id": "byName", + "options": "Value #A" + }, "properties": [ - { "id": "displayName", "value": "Status" }, + { + "id": "displayName", + "value": "Status" + }, { "id": "mappings", "value": [ - { "type": "value", "options": { "1": { "text": "UP", "color": "green" }, "0": { "text": "DOWN", "color": "red" } } } + { + "type": "value", + "options": { + "1": { + "text": "UP", + "color": "green" + }, + "0": { + "text": "DOWN", + "color": "red" + } + } + } ] } ] }, { - "matcher": { "id": "byName", "options": "Value #B" }, + "matcher": { + "id": "byName", + "options": "Value #B" + }, "properties": [ - { "id": "displayName", "value": "Active Servers" } + { + "id": "displayName", + "value": "Active Servers" + } ] }, { - "matcher": { "id": "byName", "options": "Value #C" }, + "matcher": { + "id": "byName", + "options": "Value #C" + }, "properties": [ - { "id": "displayName", "value": "Avg Response Time" }, - { "id": "unit", "value": "s" } + { + "id": "displayName", + "value": "Avg Response Time" + }, + { + "id": "unit", + "value": "s" + } ] }, { - "matcher": { "id": "byName", "options": "proxy" }, + "matcher": { + "id": "byName", + "options": "proxy" + }, "properties": [ - { "id": "displayName", "value": "Backend" } + { + "id": "displayName", + "value": "Backend" + } ] } ] @@ -513,7 +791,9 @@ "sortBy": [], "footer": { "show": false, - "reducer": ["sum"], + "reducer": [ + "sum" + ], "fields": "" } }, @@ -541,9 +821,17 @@ "type": "state-timeline", "title": "Server Status", "description": "Timeline view of individual HAProxy server health status. Green indicates UP, red indicates DOWN. Helps identify flapping servers and outage windows.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, "id": 17, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "haproxy_server_status{proxy=~\"$proxy\", server=~\"$server\"}", @@ -557,20 +845,34 @@ "lineWidth": 0, "fillOpacity": 80 }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "green", "value": 1 } + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } ] }, "mappings": [ { "type": "value", "options": { - "0": { "text": "DOWN", "color": "red" }, - "1": { "text": "UP", "color": "green" } + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } } } ] @@ -582,15 +884,26 @@ "mergeValues": true, "alignValue": "left", "rowHeight": 0.9, - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom" } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom" + } } }, { "type": "row", "title": "Traffic", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, "id": 9, "panels": [] }, @@ -598,9 +911,17 @@ "type": "timeseries", "title": "Frontend Bytes In", "description": "Inbound traffic rate per HAProxy frontend. Shows how much data clients are sending to each frontend.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, "id": 10, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum by (proxy) (rate(haproxy_frontend_bytes_in_total{proxy=~\"$proxy\"}[$__rate_interval]))", @@ -619,35 +940,66 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "Bps", - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "timeseries", "title": "Frontend Bytes Out", "description": "Outbound traffic rate per HAProxy frontend. Shows how much data is being served to clients from each frontend.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, "id": 11, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum by (proxy) (rate(haproxy_frontend_bytes_out_total{proxy=~\"$proxy\"}[$__rate_interval]))", @@ -666,33 +1018,61 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "Bps", - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "row", "title": "Sessions & Performance", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, "id": 12, "panels": [] }, @@ -700,9 +1080,17 @@ "type": "timeseries", "title": "Current Sessions by Proxy", "description": "Current active sessions for each HAProxy backend/proxy over time. Sustained high values may indicate slow backends or session buildup.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, "id": 13, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "haproxy_backend_current_sessions{proxy=~\"$proxy\"}", @@ -721,35 +1109,66 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "none", - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "timeseries", "title": "Backend Response Time", "description": "Average response time from backend servers. High or increasing values indicate backend performance degradation.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, "id": 18, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "haproxy_server_response_time_average_seconds{proxy=~\"$proxy\", server=~\"$server\"}", @@ -768,37 +1187,74 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "line" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "line" + } }, "unit": "s", - "color": { "mode": "continuous-GrYlRd" }, + "color": { + "mode": "continuous-GrYlRd" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.5 }, - { "color": "red", "value": 2 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 2 + } ] } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "timeseries", "title": "Queue Time", "description": "Average time requests spend in the backend queue before being assigned to a server. High values indicate insufficient backend capacity.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 41 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, "id": 19, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "haproxy_backend_queue_time_average_seconds{proxy=~\"$proxy\"}", @@ -817,37 +1273,74 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "line" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "line" + } }, "unit": "s", - "color": { "mode": "continuous-GrYlRd" }, + "color": { + "mode": "continuous-GrYlRd" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.1 }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } ] } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "timeseries", "title": "Connection Errors", "description": "Rate of backend connection errors and connection retries per proxy. Non-zero values indicate connectivity problems between HAProxy and backend servers.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 41 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 41 + }, "id": 20, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum by (proxy) (rate(haproxy_backend_connection_errors_total{proxy=~\"$proxy\"}[$__rate_interval]))", @@ -871,39 +1364,80 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "reqps", - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] } }, "overrides": [ { - "matcher": { "id": "byRegexp", "options": ".*errors$" }, + "matcher": { + "id": "byRegexp", + "options": ".*errors$" + }, "properties": [ - { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } ] }, { - "matcher": { "id": "byRegexp", "options": ".*retries$" }, + "matcher": { + "id": "byRegexp", + "options": ".*retries$" + }, "properties": [ - { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } ] } ] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } } ] diff --git a/incusos/observability-dashboards/core/host-resources.json b/incusos/observability-dashboards/core/host-resources.json index 79b7eb7..bacb481 100644 --- a/incusos/observability-dashboards/core/host-resources.json +++ b/incusos/observability-dashboards/core/host-resources.json @@ -2,27 +2,45 @@ "uid": "incus-host-resources", "title": "Host Resources (IncusOS Nodes)", "description": "Physical host monitoring: CPU mode breakdown, load averages, memory breakdown, filesystem usage, disk IOPS, network errors.", - "tags": ["incus", "incusos", "host", "node-exporter"], + "tags": [ + "incus", + "incusos", + "host", + "node-exporter" + ], "timezone": "browser", "schemaVersion": 39, "refresh": "30s", - "time": { "from": "now-1h", "to": "now" }, + "time": { + "from": "now-1h", + "to": "now" + }, "fiscalYearStartMonth": 0, "liveNow": false, "editable": true, "graphTooltip": 1, "links": [ { - "asDropdown": true, "icon": "external link", "includeVars": true, - "keepTime": true, "tags": ["incus"], "targetBlank": false, - "title": "Incus Dashboards", "type": "dashboards" + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "incus" + ], + "targetBlank": false, + "title": "Incus Dashboards", + "type": "dashboards" } ], "templating": { "list": [ { "current": {}, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "definition": "label_values(node_uname_info, instance)", "includeAll": true, "multi": true, @@ -31,7 +49,8 @@ "refresh": 2, "sort": 1, "type": "query", - "label": "Node" + "label": "Node", + "allValue": ".*" } ] }, @@ -39,78 +58,232 @@ "list": [ { "builtIn": 1, - "datasource": { "type": "grafana", "uid": "-- Grafana --" }, - "enable": true, "hide": true, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", "type": "dashboard" + "name": "Annotations & Alerts", + "type": "dashboard" } ] }, "panels": [ { - "type": "row", "title": "System Info", + "type": "row", + "title": "System Info", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "id": 30, "panels": [] + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 30, + "panels": [] }, { "type": "table", "title": "System Information", "description": "Kernel version, uptime, total RAM, and CPU count per node.", - "gridPos": { "h": 5, "w": 24, "x": 0, "y": 1 }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 1 + }, "id": 31, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "node_uname_info{instance=~\"$node\"}", - "legendFormat": "", "refId": "A", "format": "table", "instant": true + "legendFormat": "", + "refId": "A", + "format": "table", + "instant": true }, { "expr": "node_boot_time_seconds{instance=~\"$node\"}", - "legendFormat": "", "refId": "B", "format": "table", "instant": true + "legendFormat": "", + "refId": "B", + "format": "table", + "instant": true }, { "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"}", - "legendFormat": "", "refId": "C", "format": "table", "instant": true + "legendFormat": "", + "refId": "C", + "format": "table", + "instant": true }, { "expr": "count by (instance) (node_cpu_seconds_total{instance=~\"$node\", mode=\"idle\"})", - "legendFormat": "", "refId": "D", "format": "table", "instant": true + "legendFormat": "", + "refId": "D", + "format": "table", + "instant": true } ], "fieldConfig": { "defaults": { - "custom": { "align": "auto", "displayMode": "auto", "inspect": false, "filterable": true }, - "color": { "mode": "thresholds" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false, + "filterable": true + }, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [ - { "matcher": { "id": "byName", "options": "instance" }, "properties": [{ "id": "displayName", "value": "Node" }] }, - { "matcher": { "id": "byName", "options": "release" }, "properties": [{ "id": "displayName", "value": "Kernel" }] }, - { "matcher": { "id": "byName", "options": "Value #B" }, "properties": [{ "id": "displayName", "value": "Boot Time" }, { "id": "unit", "value": "dateTimeFromNow" }] }, - { "matcher": { "id": "byName", "options": "Value #C" }, "properties": [{ "id": "displayName", "value": "Total RAM" }, { "id": "unit", "value": "bytes" }] }, - { "matcher": { "id": "byName", "options": "Value #D" }, "properties": [{ "id": "displayName", "value": "CPUs" }] } + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "displayName", + "value": "Node" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "release" + }, + "properties": [ + { + "id": "displayName", + "value": "Kernel" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "displayName", + "value": "Boot Time" + }, + { + "id": "unit", + "value": "dateTimeFromNow" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #C" + }, + "properties": [ + { + "id": "displayName", + "value": "Total RAM" + }, + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #D" + }, + "properties": [ + { + "id": "displayName", + "value": "CPUs" + } + ] + } ] }, - "options": { "showHeader": true, "sortBy": [], "footer": { "show": false, "reducer": ["sum"], "fields": "" } }, + "options": { + "showHeader": true, + "sortBy": [], + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "fields": "" + } + }, "transformations": [ - { "id": "merge", "options": {} }, - { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "job": true, "domainname": true, "machine": true, "nodename": true, "sysname": true, "version": true, "Value #A": true }, "renameByName": {} } } + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "job": true, + "domainname": true, + "machine": true, + "nodename": true, + "sysname": true, + "version": true, + "Value #A": true + }, + "renameByName": {} + } + } ] }, { - "type": "row", "title": "CPU", + "type": "row", + "title": "CPU", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }, - "id": 1, "panels": [] + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 1, + "panels": [] }, { "type": "timeseries", "title": "CPU Usage % by Node", "description": "Overall CPU utilization per host node.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, "id": 2, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$node\"}[$__rate_interval]))) * 100", @@ -121,111 +294,293 @@ "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, - "fillOpacity": 25, "gradientMode": "scheme", "spanNulls": false, - "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "line" } + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 25, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "line" + } }, - "unit": "percent", "min": 0, "max": 100, - "color": { "mode": "continuous-GrYlRd" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] } + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "continuous-GrYlRd" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "timeseries", "title": "CPU Mode Breakdown", "description": "CPU time by mode (user, system, iowait, idle) stacked per node.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, "id": 32, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ - { "expr": "avg by (instance, mode) (rate(node_cpu_seconds_total{instance=~\"$node\", mode=~\"user|system|iowait|idle\"}[$__rate_interval])) * 100", "legendFormat": "{{instance}} {{mode}}", "refId": "A" } + { + "expr": "avg by (instance, mode) (rate(node_cpu_seconds_total{instance=~\"$node\", mode=~\"user|system|iowait|idle\"}[$__rate_interval])) * 100", + "legendFormat": "{{instance}} {{mode}}", + "refId": "A" + } ], "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, - "fillOpacity": 50, "gradientMode": "none", "spanNulls": false, + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 50, + "gradientMode": "none", + "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "normal", "group": "A" }, - "axisPlacement": "auto", "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "unit": "percent", "min": 0, - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "unit": "percent", + "min": 0, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + } } }, { "type": "timeseries", "title": "Load Average", "description": "System load averages (1m, 5m, 15m) with CPU count threshold line.", - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 15 }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 15 + }, "id": 33, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ - { "expr": "node_load1{instance=~\"$node\"}", "legendFormat": "{{instance}} load1", "refId": "A" }, - { "expr": "node_load5{instance=~\"$node\"}", "legendFormat": "{{instance}} load5", "refId": "B" }, - { "expr": "node_load15{instance=~\"$node\"}", "legendFormat": "{{instance}} load15", "refId": "C" }, - { "expr": "count by (instance) (node_cpu_seconds_total{instance=~\"$node\", mode=\"idle\"})", "legendFormat": "{{instance}} CPUs", "refId": "D" } + { + "expr": "node_load1{instance=~\"$node\"}", + "legendFormat": "{{instance}} load1", + "refId": "A" + }, + { + "expr": "node_load5{instance=~\"$node\"}", + "legendFormat": "{{instance}} load5", + "refId": "B" + }, + { + "expr": "node_load15{instance=~\"$node\"}", + "legendFormat": "{{instance}} load15", + "refId": "C" + }, + { + "expr": "count by (instance) (node_cpu_seconds_total{instance=~\"$node\", mode=\"idle\"})", + "legendFormat": "{{instance}} CPUs", + "refId": "D" + } ], "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, - "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none", + "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short", - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [ { - "matcher": { "id": "byRegexp", "options": ".*CPUs$" }, + "matcher": { + "id": "byRegexp", + "options": ".*CPUs$" + }, "properties": [ - { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }, - { "id": "custom.lineWidth", "value": 2 }, - { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 10 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } ] } ] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { - "type": "row", "title": "Memory", + "type": "row", + "title": "Memory", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, - "id": 3, "panels": [] + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 3, + "panels": [] }, { "type": "timeseries", "title": "Memory Usage % by Node", "description": "Memory utilization percentage per host node.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, "id": 4, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "(1 - node_memory_MemAvailable_bytes{instance=~\"$node\"} / node_memory_MemTotal_bytes{instance=~\"$node\"}) * 100", @@ -236,71 +591,186 @@ "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, - "fillOpacity": 25, "gradientMode": "scheme", "spanNulls": false, + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 25, + "gradientMode": "scheme", + "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "line" } + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "line" + } }, - "unit": "percent", "min": 0, "max": 100, - "color": { "mode": "continuous-GrYlRd" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] } + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "continuous-GrYlRd" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "timeseries", "title": "Memory Breakdown by Node", "description": "Memory breakdown: used, buffers, cached, available (stacked).", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, "id": 34, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ - { "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"} - node_memory_MemAvailable_bytes{instance=~\"$node\"} - node_memory_Buffers_bytes{instance=~\"$node\"} - node_memory_Cached_bytes{instance=~\"$node\"}", "legendFormat": "{{instance}} used", "refId": "A" }, - { "expr": "node_memory_Buffers_bytes{instance=~\"$node\"}", "legendFormat": "{{instance}} buffers", "refId": "B" }, - { "expr": "node_memory_Cached_bytes{instance=~\"$node\"}", "legendFormat": "{{instance}} cached", "refId": "C" }, - { "expr": "node_memory_MemAvailable_bytes{instance=~\"$node\"}", "legendFormat": "{{instance}} available", "refId": "D" } + { + "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"} - node_memory_MemAvailable_bytes{instance=~\"$node\"} - node_memory_Buffers_bytes{instance=~\"$node\"} - node_memory_Cached_bytes{instance=~\"$node\"}", + "legendFormat": "{{instance}} used", + "refId": "A" + }, + { + "expr": "node_memory_Buffers_bytes{instance=~\"$node\"}", + "legendFormat": "{{instance}} buffers", + "refId": "B" + }, + { + "expr": "node_memory_Cached_bytes{instance=~\"$node\"}", + "legendFormat": "{{instance}} cached", + "refId": "C" + }, + { + "expr": "node_memory_MemAvailable_bytes{instance=~\"$node\"}", + "legendFormat": "{{instance}} available", + "refId": "D" + } ], "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, - "fillOpacity": 50, "gradientMode": "none", "spanNulls": false, + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 50, + "gradientMode": "none", + "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "normal", "group": "A" }, - "axisPlacement": "auto", "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "bytes", - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { - "type": "row", "title": "Disk", + "type": "row", + "title": "Disk", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, - "id": 5, "panels": [] + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 5, + "panels": [] }, { "type": "bargauge", "title": "Filesystem Usage %", "description": "Current filesystem usage percentage for important mount points.", - "gridPos": { "h": 6, "w": 8, "x": 0, "y": 33 }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 33 + }, "id": 35, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "(1 - node_filesystem_avail_bytes{instance=~\"$node\", fstype!~\"tmpfs|overlay\", mountpoint=~\"/.*\"} / node_filesystem_size_bytes{instance=~\"$node\", fstype!~\"tmpfs|overlay\", mountpoint=~\"/.*\"}) * 100", @@ -310,274 +780,694 @@ ], "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "orange", "value": 85 }, { "color": "red", "value": 95 }] }, - "unit": "percent", "min": 0, "max": 100, - "color": { "mode": "thresholds" } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "orange", + "value": 85 + }, + { + "color": "red", + "value": 95 + } + ] + }, + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "horizontal", "displayMode": "gradient", "showUnfilled": true, - "minVizWidth": 8, "minVizHeight": 16, "valueMode": "color" + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true, + "minVizWidth": 8, + "minVizHeight": 16, + "valueMode": "color" } }, { "type": "timeseries", "title": "Root Filesystem Available Space", "description": "Available disk space on the root filesystem per host node.", - "gridPos": { "h": 6, "w": 8, "x": 8, "y": 33 }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 33 + }, "id": 6, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ - { "expr": "node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=\"/\"}", "legendFormat": "{{instance}}", "refId": "A" } + { + "expr": "node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=\"/\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } ], "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, - "fillOpacity": 15, "gradientMode": "none", "spanNulls": false, + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "bytes", - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull", "min"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "lastNotNull", + "min" + ] + } } }, { "type": "timeseries", "title": "Disk IOPS by Node", "description": "Disk read and write operations per second by node.", - "gridPos": { "h": 6, "w": 8, "x": 16, "y": 33 }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 33 + }, "id": 36, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ - { "expr": "sum by (instance) (rate(node_disk_reads_completed_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} reads", "refId": "A" }, - { "expr": "sum by (instance) (rate(node_disk_writes_completed_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} writes", "refId": "B" } + { + "expr": "sum by (instance) (rate(node_disk_reads_completed_total{instance=~\"$node\"}[$__rate_interval]))", + "legendFormat": "{{instance}} reads", + "refId": "A" + }, + { + "expr": "sum by (instance) (rate(node_disk_writes_completed_total{instance=~\"$node\"}[$__rate_interval]))", + "legendFormat": "{{instance}} writes", + "refId": "B" + } ], "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, - "fillOpacity": 15, "gradientMode": "none", "spanNulls": false, + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "iops", - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + } } }, { "type": "timeseries", "title": "Disk Read Rate by Node", "description": "Disk read throughput aggregated per host node.", - "gridPos": { "h": 6, "w": 8, "x": 0, "y": 39 }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 39 + }, "id": 7, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ - { "expr": "sum by (instance) (rate(node_disk_read_bytes_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}}", "refId": "A" } + { + "expr": "sum by (instance) (rate(node_disk_read_bytes_total{instance=~\"$node\"}[$__rate_interval]))", + "legendFormat": "{{instance}}", + "refId": "A" + } ], "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, - "fillOpacity": 15, "gradientMode": "none", "spanNulls": false, + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "Bps", - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + } } }, { "type": "timeseries", "title": "Disk Write Rate by Node", "description": "Disk write throughput aggregated per host node.", - "gridPos": { "h": 6, "w": 8, "x": 8, "y": 39 }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 39 + }, "id": 8, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ - { "expr": "sum by (instance) (rate(node_disk_written_bytes_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}}", "refId": "A" } + { + "expr": "sum by (instance) (rate(node_disk_written_bytes_total{instance=~\"$node\"}[$__rate_interval]))", + "legendFormat": "{{instance}}", + "refId": "A" + } ], "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, - "fillOpacity": 15, "gradientMode": "none", "spanNulls": false, + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "Bps", - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + } } }, { "type": "timeseries", "title": "Disk I/O Time", "description": "Time spent on disk I/O operations (disk utilization indicator).", - "gridPos": { "h": 6, "w": 8, "x": 16, "y": 39 }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 39 + }, "id": 37, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ - { "expr": "sum by (instance) (rate(node_disk_io_time_seconds_total{instance=~\"$node\"}[$__rate_interval])) * 100", "legendFormat": "{{instance}}", "refId": "A" } + { + "expr": "sum by (instance) (rate(node_disk_io_time_seconds_total{instance=~\"$node\"}[$__rate_interval])) * 100", + "legendFormat": "{{instance}}", + "refId": "A" + } ], "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, - "fillOpacity": 15, "gradientMode": "scheme", "spanNulls": false, + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 15, + "gradientMode": "scheme", + "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "line" } + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "line" + } }, - "unit": "percent", "min": 0, "max": 100, - "color": { "mode": "continuous-GrYlRd" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 80 }] } + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "continuous-GrYlRd" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 80 + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + } } }, { - "type": "row", "title": "Network", + "type": "row", + "title": "Network", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 45 }, - "id": 9, "panels": [] + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 45 + }, + "id": 9, + "panels": [] }, { "type": "timeseries", "title": "Network Receive Rate by Device & Node", "description": "Inbound network throughput per network device and host node.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 46 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 46 + }, "id": 10, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ - { "expr": "rate(node_network_receive_bytes_total{instance=~\"$node\", device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])", "legendFormat": "{{instance}} {{device}} rx", "refId": "A" } + { + "expr": "rate(node_network_receive_bytes_total{instance=~\"$node\", device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])", + "legendFormat": "{{instance}} {{device}} rx", + "refId": "A" + } ], "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, - "fillOpacity": 15, "gradientMode": "none", "spanNulls": false, + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "Bps", - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + } } }, { "type": "timeseries", "title": "Network Transmit Rate by Device & Node", "description": "Outbound network throughput per network device and host node.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 46 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 46 + }, "id": 11, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ - { "expr": "rate(node_network_transmit_bytes_total{instance=~\"$node\", device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])", "legendFormat": "{{instance}} {{device}} tx", "refId": "A" } + { + "expr": "rate(node_network_transmit_bytes_total{instance=~\"$node\", device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])", + "legendFormat": "{{instance}} {{device}} tx", + "refId": "A" + } ], "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, - "fillOpacity": 15, "gradientMode": "none", "spanNulls": false, + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "Bps", - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + } } }, { "type": "timeseries", "title": "Network Errors & Drops", "description": "Network receive/transmit errors and dropped packets per node.", - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 54 }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 54 + }, "id": 38, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ - { "expr": "sum by (instance) (rate(node_network_receive_errs_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} rx errors", "refId": "A" }, - { "expr": "sum by (instance) (rate(node_network_transmit_errs_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} tx errors", "refId": "B" }, - { "expr": "sum by (instance) (rate(node_network_receive_drop_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} rx drops", "refId": "C" }, - { "expr": "sum by (instance) (rate(node_network_transmit_drop_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} tx drops", "refId": "D" } + { + "expr": "sum by (instance) (rate(node_network_receive_errs_total{instance=~\"$node\"}[$__rate_interval]))", + "legendFormat": "{{instance}} rx errors", + "refId": "A" + }, + { + "expr": "sum by (instance) (rate(node_network_transmit_errs_total{instance=~\"$node\"}[$__rate_interval]))", + "legendFormat": "{{instance}} tx errors", + "refId": "B" + }, + { + "expr": "sum by (instance) (rate(node_network_receive_drop_total{instance=~\"$node\"}[$__rate_interval]))", + "legendFormat": "{{instance}} rx drops", + "refId": "C" + }, + { + "expr": "sum by (instance) (rate(node_network_transmit_drop_total{instance=~\"$node\"}[$__rate_interval]))", + "legendFormat": "{{instance}} tx drops", + "refId": "D" + } ], "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, - "fillOpacity": 15, "gradientMode": "none", "spanNulls": false, + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "pps", - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } } ] diff --git a/incusos/observability-dashboards/core/logs-explorer.json b/incusos/observability-dashboards/core/logs-explorer.json index 5f4808c..2ac8ea0 100644 --- a/incusos/observability-dashboards/core/logs-explorer.json +++ b/incusos/observability-dashboards/core/logs-explorer.json @@ -2,11 +2,18 @@ "uid": "incus-logs-explorer", "title": "Logs Explorer", "description": "Log volume over time, full log stream with search, error and warning rates by systemd unit. Datasource: Loki.", - "tags": ["incus", "logs", "loki"], + "tags": [ + "incus", + "logs", + "loki" + ], "timezone": "browser", "schemaVersion": 39, "refresh": "30s", - "time": { "from": "now-1h", "to": "now" }, + "time": { + "from": "now-1h", + "to": "now" + }, "fiscalYearStartMonth": 0, "liveNow": false, "editable": true, @@ -17,7 +24,9 @@ "icon": "external link", "includeVars": true, "keepTime": true, - "tags": ["incus"], + "tags": [ + "incus" + ], "targetBlank": false, "title": "Incus Dashboards", "tooltip": "Navigate to other Incus dashboards", @@ -28,42 +37,57 @@ "list": [ { "type": "query", - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "query": "label_values(job)", "includeAll": true, "multi": true, "name": "job", "label": "Job", "refresh": 2, - "sort": 1 + "sort": 1, + "allValue": ".*" }, { "type": "query", - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "query": "label_values(host)", "includeAll": true, "multi": true, "name": "host", "label": "Host", "refresh": 2, - "sort": 1 + "sort": 1, + "allValue": ".*" }, { "type": "query", - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "query": "label_values(unit)", "includeAll": true, "multi": true, "name": "unit", "label": "Unit", "refresh": 2, - "sort": 1 + "sort": 1, + "allValue": ".*" }, { "type": "textbox", "name": "search", "label": "Search", - "current": { "text": "", "value": "" } + "current": { + "text": "", + "value": "" + } } ] }, @@ -71,7 +95,10 @@ "list": [ { "builtIn": 1, - "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -85,7 +112,12 @@ "type": "row", "title": "Log Volume", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "id": 1, "panels": [] }, @@ -93,9 +125,17 @@ "type": "timeseries", "title": "Log Volume Over Time", "description": "Total log line count over time, broken down by job. Shows the rate at which logs are being generated across all selected sources.", - "gridPos": { "h": 6, "w": 24, "x": 0, "y": 1 }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 1 + }, "id": 2, - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "targets": [ { "expr": "sum by (job) (count_over_time({job=~\"$job\", host=~\"$host\"} [$__auto]))", @@ -114,31 +154,61 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "normal", "group": "A" }, + "stacking": { + "mode": "normal", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short", - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", - "steps": [{ "color": "green", "value": null }] + "steps": [ + { + "color": "green", + "value": null + } + ] } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "sum", + "max", + "lastNotNull" + ] + } } }, { "type": "row", "title": "Log Stream", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, "id": 3, "panels": [] }, @@ -146,9 +216,17 @@ "type": "logs", "title": "Logs", "description": "Live log stream filtered by job, host, unit, and free-text search. Displays timestamps, labels, and full log messages with expand-on-click detail view.", - "gridPos": { "h": 16, "w": 24, "x": 0, "y": 8 }, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 8 + }, "id": 4, - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "targets": [ { "expr": "{job=~\"$job\", host=~\"$host\", unit=~\"$unit\"} |= \"$search\"", @@ -170,7 +248,12 @@ "type": "row", "title": "Error & Warning Rates", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, "id": 5, "panels": [] }, @@ -178,9 +261,17 @@ "type": "timeseries", "title": "Error Rate by Unit", "description": "Rate of log lines matching error, fail, or fatal (case-insensitive) over time, grouped by systemd unit. Spikes indicate services experiencing failures.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, "id": 6, - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "targets": [ { "expr": "sum by (unit) (count_over_time({job=~\"$job\", host=~\"$host\"} |~ \"(?i)error|fail|fatal\" [$__auto]))", @@ -199,33 +290,67 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short", - "color": { "mode": "fixed", "fixedColor": "red" }, + "color": { + "mode": "fixed", + "fixedColor": "red" + }, "thresholds": { "mode": "absolute", - "steps": [{ "color": "red", "value": null }] + "steps": [ + { + "color": "red", + "value": null + } + ] } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "sum", + "max", + "lastNotNull" + ] + } } }, { "type": "timeseries", "title": "Warning Rate by Unit", "description": "Rate of log lines matching warn (case-insensitive) over time, grouped by systemd unit. Helps identify services generating warnings that may precede errors.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 25 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, "id": 7, - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "targets": [ { "expr": "sum by (unit) (count_over_time({job=~\"$job\", host=~\"$host\"} |~ \"(?i)warn\" [$__auto]))", @@ -244,24 +369,50 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short", - "color": { "mode": "fixed", "fixedColor": "yellow" }, + "color": { + "mode": "fixed", + "fixedColor": "yellow" + }, "thresholds": { "mode": "absolute", - "steps": [{ "color": "yellow", "value": null }] + "steps": [ + { + "color": "yellow", + "value": null + } + ] } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "sum", + "max", + "lastNotNull" + ] + } } } ] diff --git a/incusos/observability-dashboards/core/network-deep-dive.json b/incusos/observability-dashboards/core/network-deep-dive.json index 89598bb..5f84a58 100644 --- a/incusos/observability-dashboards/core/network-deep-dive.json +++ b/incusos/observability-dashboards/core/network-deep-dive.json @@ -2,11 +2,19 @@ "uid": "incus-network-deep-dive", "title": "Network Deep Dive", "description": "Per-node and per-device network traffic, instance-level network metrics, receive/transmit errors, and packet drops.", - "tags": ["incus", "incusos", "network", "node-exporter"], + "tags": [ + "incus", + "incusos", + "network", + "node-exporter" + ], "timezone": "browser", "schemaVersion": 39, "refresh": "30s", - "time": { "from": "now-1h", "to": "now" }, + "time": { + "from": "now-1h", + "to": "now" + }, "fiscalYearStartMonth": 0, "liveNow": false, "editable": true, @@ -17,7 +25,9 @@ "icon": "external link", "includeVars": true, "keepTime": true, - "tags": ["incus"], + "tags": [ + "incus" + ], "targetBlank": false, "title": "Incus Dashboards", "tooltip": "Navigate to other Incus dashboards", @@ -28,7 +38,10 @@ "list": [ { "current": {}, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "definition": "label_values(node_network_receive_bytes_total, instance)", "includeAll": false, "multi": true, @@ -41,7 +54,10 @@ }, { "current": {}, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "definition": "label_values(node_network_receive_bytes_total{instance=~\"$node\"}, device)", "includeAll": true, "multi": true, @@ -51,7 +67,8 @@ "regex": "/^(?!lo$).*/", "sort": 1, "type": "query", - "label": "Device" + "label": "Device", + "allValue": ".*" } ] }, @@ -59,7 +76,10 @@ "list": [ { "builtIn": 1, - "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -73,7 +93,12 @@ "type": "row", "title": "Cluster Summary", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "id": 1, "panels": [] }, @@ -81,9 +106,17 @@ "type": "stat", "title": "Total Ingress", "description": "Aggregate inbound network throughput across all selected nodes and devices.", - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, "id": 2, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum(rate(node_network_receive_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval]))", @@ -95,15 +128,28 @@ "defaults": { "thresholds": { "mode": "absolute", - "steps": [{ "color": "green", "value": null }] + "steps": [ + { + "color": "green", + "value": null + } + ] }, "unit": "Bps", - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -115,9 +161,17 @@ "type": "stat", "title": "Total Egress", "description": "Aggregate outbound network throughput across all selected nodes and devices.", - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, "id": 3, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum(rate(node_network_transmit_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval]))", @@ -129,15 +183,28 @@ "defaults": { "thresholds": { "mode": "absolute", - "steps": [{ "color": "green", "value": null }] + "steps": [ + { + "color": "green", + "value": null + } + ] }, "unit": "Bps", - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -149,9 +216,17 @@ "type": "stat", "title": "Total Errors", "description": "Combined receive and transmit error rate across all selected nodes. Any non-zero value warrants investigation.", - "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, "id": 4, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum(rate(node_network_receive_errs_total{instance=~\"$node\"}[$__rate_interval])) + sum(rate(node_network_transmit_errs_total{instance=~\"$node\"}[$__rate_interval]))", @@ -164,18 +239,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.1 }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } ] }, "unit": "pps", - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -187,9 +279,17 @@ "type": "stat", "title": "Total Drops", "description": "Combined receive and transmit packet drop rate across all selected nodes. Drops indicate buffer or queue exhaustion.", - "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, "id": 5, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum(rate(node_network_receive_drop_total{instance=~\"$node\"}[$__rate_interval])) + sum(rate(node_network_transmit_drop_total{instance=~\"$node\"}[$__rate_interval]))", @@ -202,18 +302,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.1 }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } ] }, "unit": "pps", - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -225,7 +342,12 @@ "type": "row", "title": "Per-Node Traffic", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, "id": 10, "panels": [] }, @@ -233,9 +355,17 @@ "type": "timeseries", "title": "Receive Rate by Node & Device", "description": "Inbound network throughput broken down by node and network device. Identifies which interfaces are handling the most receive traffic.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, "id": 11, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "rate(node_network_receive_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])", @@ -254,33 +384,66 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "Bps", - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", - "steps": [{ "color": "green", "value": null }] + "steps": [ + { + "color": "green", + "value": null + } + ] } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "timeseries", "title": "Transmit Rate by Node & Device", "description": "Outbound network throughput broken down by node and network device. Identifies which interfaces are handling the most transmit traffic.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, "id": 12, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "rate(node_network_transmit_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])", @@ -299,31 +462,61 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "Bps", - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", - "steps": [{ "color": "green", "value": null }] + "steps": [ + { + "color": "green", + "value": null + } + ] } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "row", "title": "Instance Network (Incus metrics)", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, "id": 20, "panels": [] }, @@ -331,9 +524,17 @@ "type": "timeseries", "title": "Instance Receive Rate", "description": "Inbound network throughput per Incus instance, aggregated across all instance network devices. Uses Incus-native metrics for container and VM visibility.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, "id": 21, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum by (name) (rate(incus_network_receive_bytes_total{instance=~\"$node\"}[$__rate_interval]))", @@ -352,33 +553,66 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "Bps", - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", - "steps": [{ "color": "green", "value": null }] + "steps": [ + { + "color": "green", + "value": null + } + ] } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "timeseries", "title": "Instance Transmit Rate", "description": "Outbound network throughput per Incus instance, aggregated across all instance network devices. Uses Incus-native metrics for container and VM visibility.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, "id": 22, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum by (name) (rate(incus_network_transmit_bytes_total{instance=~\"$node\"}[$__rate_interval]))", @@ -397,31 +631,61 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "Bps", - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", - "steps": [{ "color": "green", "value": null }] + "steps": [ + { + "color": "green", + "value": null + } + ] } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "row", "title": "Errors & Drops", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, "id": 30, "panels": [] }, @@ -429,9 +693,17 @@ "type": "timeseries", "title": "Receive/Transmit Errors", "description": "Network interface error rates broken down by node and device. Receive errors may indicate CRC failures or framing issues; transmit errors may indicate collisions or driver problems.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, "id": 31, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "rate(node_network_receive_errs_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])", @@ -455,33 +727,66 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "pps", - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", - "steps": [{ "color": "green", "value": null }] + "steps": [ + { + "color": "green", + "value": null + } + ] } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "timeseries", "title": "Packet Drops", "description": "Network interface packet drop rates broken down by node and device. Drops indicate kernel buffer or queue exhaustion and may cause retransmissions or connection failures.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, "id": 32, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "rate(node_network_receive_drop_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])", @@ -505,24 +810,49 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "pps", - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", - "steps": [{ "color": "green", "value": null }] + "steps": [ + { + "color": "green", + "value": null + } + ] } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } } ] diff --git a/incusos/observability-dashboards/core/storage-filesystem.json b/incusos/observability-dashboards/core/storage-filesystem.json index 0a8cec0..60f91a9 100644 --- a/incusos/observability-dashboards/core/storage-filesystem.json +++ b/incusos/observability-dashboards/core/storage-filesystem.json @@ -2,27 +2,45 @@ "uid": "incus-storage-filesystem", "title": "Storage & Filesystem", "description": "Filesystem usage, available space trends, disk throughput, I/O latency, disk utilization, and inode usage per node.", - "tags": ["incus", "incusos", "storage", "node-exporter"], + "tags": [ + "incus", + "incusos", + "storage", + "node-exporter" + ], "timezone": "browser", "schemaVersion": 39, "refresh": "30s", - "time": { "from": "now-1h", "to": "now" }, + "time": { + "from": "now-1h", + "to": "now" + }, "fiscalYearStartMonth": 0, "liveNow": false, "editable": true, "graphTooltip": 1, "links": [ { - "asDropdown": true, "icon": "external link", "includeVars": true, - "keepTime": true, "tags": ["incus"], "targetBlank": false, - "title": "Incus Dashboards", "type": "dashboards" + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "incus" + ], + "targetBlank": false, + "title": "Incus Dashboards", + "type": "dashboards" } ], "templating": { "list": [ { "current": {}, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "definition": "label_values(node_filesystem_size_bytes, instance)", "includeAll": false, "multi": true, @@ -35,7 +53,10 @@ }, { "current": {}, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "definition": "label_values(node_filesystem_size_bytes{instance=~\"$node\"}, mountpoint)", "includeAll": true, "multi": true, @@ -44,7 +65,8 @@ "refresh": 2, "sort": 1, "type": "query", - "label": "Mountpoint" + "label": "Mountpoint", + "allValue": ".*" } ] }, @@ -52,83 +74,264 @@ "list": [ { "builtIn": 1, - "datasource": { "type": "grafana", "uid": "-- Grafana --" }, - "enable": true, "hide": true, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", "type": "dashboard" + "name": "Annotations & Alerts", + "type": "dashboard" } ] }, "panels": [ { - "type": "row", "title": "Filesystem Overview", + "type": "row", + "title": "Filesystem Overview", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "id": 1, "panels": [] + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [] }, { "type": "table", "title": "Filesystem Usage Table", "description": "Current filesystem size, available space, and usage percentage per node and mountpoint. High usage is highlighted with color thresholds.", - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 1 }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 1 + }, "id": 2, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "node_filesystem_size_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}", - "legendFormat": "", "refId": "A", "format": "table", "instant": true + "legendFormat": "", + "refId": "A", + "format": "table", + "instant": true }, { "expr": "node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}", - "legendFormat": "", "refId": "B", "format": "table", "instant": true + "legendFormat": "", + "refId": "B", + "format": "table", + "instant": true }, { "expr": "(1 - node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}) * 100", - "legendFormat": "", "refId": "C", "format": "table", "instant": true + "legendFormat": "", + "refId": "C", + "format": "table", + "instant": true } ], "fieldConfig": { "defaults": { - "custom": { "align": "auto", "displayMode": "auto", "inspect": false, "filterable": true }, - "color": { "mode": "thresholds" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false, + "filterable": true + }, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [ - { "matcher": { "id": "byName", "options": "instance" }, "properties": [{ "id": "displayName", "value": "Node" }] }, - { "matcher": { "id": "byName", "options": "mountpoint" }, "properties": [{ "id": "displayName", "value": "Mountpoint" }] }, - { "matcher": { "id": "byName", "options": "fstype" }, "properties": [{ "id": "displayName", "value": "FS Type" }] }, - { "matcher": { "id": "byName", "options": "Value #A" }, "properties": [{ "id": "displayName", "value": "Size" }, { "id": "unit", "value": "bytes" }] }, - { "matcher": { "id": "byName", "options": "Value #B" }, "properties": [{ "id": "displayName", "value": "Available" }, { "id": "unit", "value": "bytes" }] }, { - "matcher": { "id": "byName", "options": "Value #C" }, + "matcher": { + "id": "byName", + "options": "instance" + }, "properties": [ - { "id": "displayName", "value": "Used %" }, - { "id": "unit", "value": "percent" }, - { "id": "custom.displayMode", "value": "color-background" }, - { "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "orange", "value": 85 }, { "color": "red", "value": 95 }] } } + { + "id": "displayName", + "value": "Node" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "mountpoint" + }, + "properties": [ + { + "id": "displayName", + "value": "Mountpoint" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "fstype" + }, + "properties": [ + { + "id": "displayName", + "value": "FS Type" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "displayName", + "value": "Size" + }, + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "displayName", + "value": "Available" + }, + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #C" + }, + "properties": [ + { + "id": "displayName", + "value": "Used %" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "orange", + "value": 85 + }, + { + "color": "red", + "value": 95 + } + ] + } + } ] } ] }, - "options": { "showHeader": true, "sortBy": [], "footer": { "show": false, "reducer": ["sum"], "fields": "" } }, + "options": { + "showHeader": true, + "sortBy": [], + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "fields": "" + } + }, "transformations": [ - { "id": "merge", "options": {} }, - { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "job": true, "device": true }, "renameByName": {} } } + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "job": true, + "device": true + }, + "renameByName": {} + } + } ] }, { - "type": "row", "title": "Space Trends", + "type": "row", + "title": "Space Trends", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, - "id": 3, "panels": [] + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 3, + "panels": [] }, { "type": "timeseries", "title": "Available Space Over Time", "description": "Tracks available filesystem space over time per node and mountpoint. Use this to identify gradual space consumption trends before they become critical.", - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 10 }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 10 + }, "id": 4, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}", @@ -139,37 +342,86 @@ "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, - "fillOpacity": 15, "gradientMode": "none", "spanNulls": false, - "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "bytes", - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { - "type": "row", "title": "Disk Throughput", + "type": "row", + "title": "Disk Throughput", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 }, - "id": 5, "panels": [] + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 5, + "panels": [] }, { "type": "timeseries", "title": "Disk Read Throughput", "description": "Disk read throughput in bytes per second, broken down by instance and device. Identifies which disks are handling the most read traffic.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, "id": 6, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum by (instance, device) (rate(node_disk_read_bytes_total{instance=~\"$node\"}[$__rate_interval]))", @@ -180,31 +432,73 @@ "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, - "fillOpacity": 15, "gradientMode": "none", "spanNulls": false, - "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "Bps", - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "timeseries", "title": "Disk Write Throughput", "description": "Disk write throughput in bytes per second, broken down by instance and device. Identifies which disks are handling the most write traffic.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, "id": 7, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum by (instance, device) (rate(node_disk_written_bytes_total{instance=~\"$node\"}[$__rate_interval]))", @@ -215,37 +509,86 @@ "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, - "fillOpacity": 15, "gradientMode": "none", "spanNulls": false, - "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "Bps", - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { - "type": "row", "title": "I/O Performance", + "type": "row", + "title": "I/O Performance", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, - "id": 8, "panels": [] + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 8, + "panels": [] }, { "type": "timeseries", "title": "Average Read Latency", "description": "Average time per completed read operation. Calculated as total read time divided by completed reads. High values indicate slow disk response.", - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 28 }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 28 + }, "id": 9, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "rate(node_disk_read_time_seconds_total{instance=~\"$node\"}[$__rate_interval]) / rate(node_disk_reads_completed_total{instance=~\"$node\"}[$__rate_interval])", @@ -256,31 +599,73 @@ "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, - "fillOpacity": 15, "gradientMode": "none", "spanNulls": false, - "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "s", - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "timeseries", "title": "Average Write Latency", "description": "Average time per completed write operation. Calculated as total write time divided by completed writes. High values indicate slow disk response or write saturation.", - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 28 }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 28 + }, "id": 10, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "rate(node_disk_write_time_seconds_total{instance=~\"$node\"}[$__rate_interval]) / rate(node_disk_writes_completed_total{instance=~\"$node\"}[$__rate_interval])", @@ -291,31 +676,73 @@ "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, - "fillOpacity": 15, "gradientMode": "none", "spanNulls": false, - "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "s", - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "type": "timeseries", "title": "Disk Utilization %", "description": "Percentage of time the disk is busy processing I/O requests. Values near 100% indicate the disk is saturated and may be a bottleneck.", - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 28 }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 28 + }, "id": 11, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "sum by (instance) (rate(node_disk_io_time_seconds_total{instance=~\"$node\"}[$__rate_interval])) * 100", @@ -326,37 +753,96 @@ "fieldConfig": { "defaults": { "custom": { - "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, - "fillOpacity": 25, "gradientMode": "scheme", "spanNulls": false, - "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "line" } + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 25, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "line" + } }, - "unit": "percent", "min": 0, "max": 100, - "color": { "mode": "continuous-GrYlRd" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 80 }] } + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "continuous-GrYlRd" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 80 + } + ] + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { - "type": "row", "title": "Inodes", + "type": "row", + "title": "Inodes", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 }, - "id": 12, "panels": [] + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 12, + "panels": [] }, { "type": "bargauge", "title": "Inode Usage %", "description": "Percentage of used inodes per filesystem. Running out of inodes prevents creating new files even when disk space is available. Monitor for filesystems with many small files.", - "gridPos": { "h": 6, "w": 24, "x": 0, "y": 37 }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 37 + }, "id": 13, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "expr": "(1 - node_filesystem_files_free{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"} / node_filesystem_files{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}) * 100", @@ -366,17 +852,47 @@ ], "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] }, - "unit": "percent", "min": 0, "max": 100, - "color": { "mode": "thresholds" } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "horizontal", "displayMode": "gradient", "showUnfilled": true, - "minVizWidth": 8, "minVizHeight": 16, "valueMode": "color" + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true, + "minVizWidth": 8, + "minVizHeight": 16, + "valueMode": "color" } } ] -} \ No newline at end of file +} diff --git a/screenshots/grafana-capacity-planning.png b/screenshots/grafana-capacity-planning.png new file mode 100644 index 0000000..60a549d Binary files /dev/null and b/screenshots/grafana-capacity-planning.png differ diff --git a/screenshots/grafana-cluster-overview.png b/screenshots/grafana-cluster-overview.png new file mode 100644 index 0000000..2b4aa7d Binary files /dev/null and b/screenshots/grafana-cluster-overview.png differ diff --git a/screenshots/grafana-haproxy-traffic.png b/screenshots/grafana-haproxy-traffic.png new file mode 100644 index 0000000..a63d411 Binary files /dev/null and b/screenshots/grafana-haproxy-traffic.png differ diff --git a/screenshots/grafana-host-resources.png b/screenshots/grafana-host-resources.png new file mode 100644 index 0000000..d404dca Binary files /dev/null and b/screenshots/grafana-host-resources.png differ diff --git a/screenshots/grafana-instance-deep-dive.png b/screenshots/grafana-instance-deep-dive.png new file mode 100644 index 0000000..8130034 Binary files /dev/null and b/screenshots/grafana-instance-deep-dive.png differ diff --git a/screenshots/grafana-logs-explorer.png b/screenshots/grafana-logs-explorer.png new file mode 100644 index 0000000..f58907d Binary files /dev/null and b/screenshots/grafana-logs-explorer.png differ diff --git a/screenshots/grafana-network-deep-dive.png b/screenshots/grafana-network-deep-dive.png new file mode 100644 index 0000000..ccf63bb Binary files /dev/null and b/screenshots/grafana-network-deep-dive.png differ diff --git a/screenshots/grafana-prometheus-health.png b/screenshots/grafana-prometheus-health.png new file mode 100644 index 0000000..a085c3f Binary files /dev/null and b/screenshots/grafana-prometheus-health.png differ diff --git a/screenshots/grafana-storage-filesystem.png b/screenshots/grafana-storage-filesystem.png new file mode 100644 index 0000000..c3067ef Binary files /dev/null and b/screenshots/grafana-storage-filesystem.png differ