Fix deploy-observability and manage-dashboards issues found during live deployment
deploy-observability: - Discover DNS from UPLINK config instead of hardcoding 8.8.8.8 - Set ipv4.address on OVN NICs (port security drops unknown IPs) - Install ca-certificates/curl/gnupg before adding Grafana apt repo - Fix ACL create syntax (--description flag, not key=value) - Add NIC recovery after ACL changes (NICs go DOWN) manage-dashboards: - Rewrite read_manifest() without PyYAML dependency - Fix ensure_folder() mixing debug output with return value - Rewrite install payload generation with heredoc + file I/O - Separate incus file push from Grafana API call - Fix while-read loops consuming stdin from incus exec (fd3) Dashboard JSON: - Add allValue:".*" to all template variables with includeAll Screenshots: - Add Playwright screenshots of all 9 Grafana dashboards Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
6c7714459b
commit
f1f85d21ea
|
|
@ -233,21 +233,41 @@ phase_deploy_monitoring() {
|
|||
-c limits.memory=2GiB \
|
||||
-d root,size=20GiB
|
||||
|
||||
# Tell OVN the static IP we want — port security drops traffic from
|
||||
# IPs that OVN didn't assign, so this must be set at the device level.
|
||||
incus config device set "$(container_ref monitoring)" eth0 \
|
||||
ipv4.address="${MONITORING_IP}"
|
||||
|
||||
info "Waiting for container agent..."
|
||||
wait_for_container_agent monitoring 60
|
||||
|
||||
# Configure static IP
|
||||
# Discover upstream DNS from the UPLINK network config.
|
||||
# OVN DHCP advertises this automatically, but we need it for the static
|
||||
# networkd config (which bypasses DHCP).
|
||||
local upstream_dns
|
||||
upstream_dns=$(incus network get "${CLUSTER_REMOTE}:UPLINK" dns.nameservers 2>/dev/null) || true
|
||||
upstream_dns="${upstream_dns:-192.168.100.1}"
|
||||
detail "Using upstream DNS: ${upstream_dns}"
|
||||
|
||||
# Build DNS lines (handles comma-separated list from UPLINK config)
|
||||
local dns_lines=""
|
||||
IFS=',' read -ra dns_addrs <<< "$upstream_dns"
|
||||
for addr in "${dns_addrs[@]}"; do
|
||||
dns_lines="${dns_lines}DNS=${addr}"$'\n'
|
||||
done
|
||||
|
||||
# Configure matching static IP inside the container (systemd-networkd
|
||||
# bypasses DHCP, so we must specify DNS explicitly).
|
||||
container_exec monitoring bash -c "
|
||||
cat > /etc/systemd/network/10-static.network << 'NETCFG'
|
||||
cat > /etc/systemd/network/10-static.network << NETCFG
|
||||
[Match]
|
||||
Name=eth0
|
||||
|
||||
[Network]
|
||||
Address=${MONITORING_IP}/24
|
||||
Gateway=10.10.10.1
|
||||
DNS=10.10.10.1
|
||||
NETCFG
|
||||
systemctl restart systemd-networkd
|
||||
${dns_lines}NETCFG
|
||||
systemctl restart systemd-networkd systemd-resolved
|
||||
"
|
||||
|
||||
sleep 3
|
||||
|
|
@ -281,7 +301,15 @@ phase_install_prometheus() {
|
|||
|
||||
container_exec monitoring bash -c "
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Add Grafana apt repository (provides grafana, loki, promtail)
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq ca-certificates curl gnupg apt-transport-https 2>&1 | tail -1
|
||||
curl -fsSL https://apt.grafana.com/gpg.key | gpg --dearmor -o /usr/share/keyrings/grafana-archive-keyring.gpg
|
||||
echo 'deb [signed-by=/usr/share/keyrings/grafana-archive-keyring.gpg] https://apt.grafana.com stable main' \
|
||||
> /etc/apt/sources.list.d/grafana.list
|
||||
apt-get update -qq
|
||||
|
||||
apt-get install -y -qq prometheus 2>&1 | tail -1
|
||||
|
||||
# Place Incus client certs where Prometheus can read them
|
||||
|
|
@ -524,6 +552,10 @@ phase_deploy_node_exporters() {
|
|||
-c limits.memory=128MiB \
|
||||
-c security.privileged=true
|
||||
|
||||
# Reserve this IP in OVN before switching to static inside the container
|
||||
incus config device set "$(container_ref "$name")" eth0 \
|
||||
ipv4.address="${ip}"
|
||||
|
||||
wait_for_container_agent "$name" 30
|
||||
|
||||
# Mount host filesystems for full node metrics
|
||||
|
|
@ -582,14 +614,13 @@ phase_create_acl() {
|
|||
|
||||
# Create the ACL if it does not exist
|
||||
if ! incus network acl show "$(container_ref monitoring-allow)" &>/dev/null 2>&1; then
|
||||
incus network acl create "$(container_ref monitoring-allow)" \
|
||||
description="Allow all traffic for observability stack"
|
||||
incus network acl create "$(container_ref monitoring-allow)"
|
||||
|
||||
# Add allow-all egress and ingress rules
|
||||
incus network acl rule add "$(container_ref monitoring-allow)" egress \
|
||||
action=allow description="Allow all egress"
|
||||
action=allow
|
||||
incus network acl rule add "$(container_ref monitoring-allow)" ingress \
|
||||
action=allow description="Allow all ingress"
|
||||
action=allow
|
||||
success "ACL 'monitoring-allow' created with allow-all rules"
|
||||
else
|
||||
success "ACL 'monitoring-allow' already exists"
|
||||
|
|
@ -615,6 +646,19 @@ phase_create_acl() {
|
|||
idx=$((idx + 1))
|
||||
done
|
||||
|
||||
# Changing security.acls can bring the NIC down. Bring all NICs back up.
|
||||
sleep 2
|
||||
container_exec monitoring bash -c "ip link set eth0 up; systemctl restart systemd-networkd" 2>/dev/null || true
|
||||
local nic_idx=1
|
||||
for ip in "${NODE_EXP_IPS[@]}"; do
|
||||
local nname
|
||||
nname=$(node_exp_name "$nic_idx")
|
||||
if container_exists "$nname"; then
|
||||
container_exec "$nname" sh -c "ip link set eth0 up; rc-service networking restart" 2>/dev/null || true
|
||||
fi
|
||||
nic_idx=$((nic_idx + 1))
|
||||
done
|
||||
|
||||
success "ACL attached to all observability containers"
|
||||
}
|
||||
|
||||
|
|
@ -815,6 +859,9 @@ phase_deploy_workload_web() {
|
|||
--target "oc-node-01" \
|
||||
-c limits.memory=128MiB
|
||||
|
||||
incus config device set "$(container_ref workload-web)" eth0 \
|
||||
ipv4.address="${WORKLOAD_WEB_IP}"
|
||||
|
||||
wait_for_container_agent workload-web 30
|
||||
|
||||
container_exec workload-web sh -c "
|
||||
|
|
@ -889,6 +936,9 @@ phase_deploy_workload_api() {
|
|||
--target "oc-node-03" \
|
||||
-c limits.memory=128MiB
|
||||
|
||||
incus config device set "$(container_ref workload-api)" eth0 \
|
||||
ipv4.address="${WORKLOAD_API_IP}"
|
||||
|
||||
wait_for_container_agent workload-api 30
|
||||
|
||||
container_exec workload-api sh -c "
|
||||
|
|
|
|||
|
|
@ -190,7 +190,7 @@ for f in folders:
|
|||
" 2>/dev/null || echo "")
|
||||
|
||||
if [[ -n "$folder_id" ]]; then
|
||||
detail "Folder '${folder_title}' exists (ID: ${folder_id})"
|
||||
detail "Folder '${folder_title}' exists (ID: ${folder_id})" >&2
|
||||
echo "$folder_id"
|
||||
return 0
|
||||
fi
|
||||
|
|
@ -209,7 +209,7 @@ print(json.load(sys.stdin).get('id', ''))
|
|||
" 2>/dev/null || echo "")
|
||||
|
||||
if [[ -n "$folder_id" ]]; then
|
||||
detail "Created folder '${folder_title}' (ID: ${folder_id})"
|
||||
detail "Created folder '${folder_title}' (ID: ${folder_id})" >&2
|
||||
echo "$folder_id"
|
||||
else
|
||||
error "Failed to create folder '${folder_title}': ${result}"
|
||||
|
|
@ -226,21 +226,54 @@ read_manifest() {
|
|||
fi
|
||||
|
||||
python3 -c "
|
||||
import yaml, sys
|
||||
import sys, re
|
||||
|
||||
entries = []
|
||||
current = {}
|
||||
|
||||
with open('${MANIFEST_FILE}') as f:
|
||||
data = yaml.safe_load(f)
|
||||
for line in f:
|
||||
stripped = line.rstrip()
|
||||
# New entry starts with ' - file:'
|
||||
m = re.match(r'^ - file:\s*(.*)', stripped)
|
||||
if m:
|
||||
if current:
|
||||
entries.append(current)
|
||||
current = {'file': m.group(1).strip()}
|
||||
continue
|
||||
# Key-value within an entry
|
||||
m = re.match(r'^ (\w+):\s*(.*)', stripped)
|
||||
if m and current:
|
||||
key, val = m.group(1), m.group(2).strip()
|
||||
# Strip YAML >- multiline indicators and quotes
|
||||
if val in ('>-', '>', '|'):
|
||||
val = ''
|
||||
val = val.strip('\"').strip(\"'\")
|
||||
if key in current and not val:
|
||||
continue
|
||||
current[key] = val
|
||||
continue
|
||||
# Continuation line (multiline description)
|
||||
if current and stripped.startswith(' ') and stripped.strip():
|
||||
if 'description' in current:
|
||||
sep = ' ' if current['description'] else ''
|
||||
current['description'] = current['description'] + sep + stripped.strip()
|
||||
|
||||
for d in data.get('dashboards', []):
|
||||
file_path = d.get('file', '')
|
||||
uid = d.get('uid', '')
|
||||
title = d.get('title', '')
|
||||
folder = d.get('folder', 'General')
|
||||
category = d.get('category', '')
|
||||
version = d.get('version', '1')
|
||||
description = d.get('description', '')
|
||||
print(f'{file_path}\t{uid}\t{title}\t{folder}\t{category}\t{version}\t{description}')
|
||||
" 2>/dev/null
|
||||
if current:
|
||||
entries.append(current)
|
||||
|
||||
for d in entries:
|
||||
parts = [
|
||||
d.get('file', ''),
|
||||
d.get('uid', ''),
|
||||
d.get('title', ''),
|
||||
d.get('folder', 'General'),
|
||||
d.get('category', ''),
|
||||
d.get('version', '1'),
|
||||
d.get('description', ''),
|
||||
]
|
||||
print('\t'.join(parts))
|
||||
"
|
||||
}
|
||||
|
||||
# Substitute datasource variables in JSON content
|
||||
|
|
@ -279,7 +312,7 @@ action_install() {
|
|||
info "(dry-run) Would install dashboards from ${DASHBOARDS_DIR}"
|
||||
|
||||
local count=0
|
||||
while IFS=$'\t' read -r file uid title folder category version description; do
|
||||
while IFS=$'\t' read -r file uid title folder category version description <&3; do
|
||||
if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
|
@ -288,7 +321,7 @@ action_install() {
|
|||
fi
|
||||
info "(dry-run) Would install: ${title} -> folder '${folder}'"
|
||||
count=$((count + 1))
|
||||
done < <(read_manifest)
|
||||
done 3< <(read_manifest)
|
||||
|
||||
info "(dry-run) ${count} dashboard(s) would be installed"
|
||||
return 0
|
||||
|
|
@ -299,7 +332,7 @@ action_install() {
|
|||
local installed=0
|
||||
local failed=0
|
||||
|
||||
while IFS=$'\t' read -r file uid title folder category version description; do
|
||||
while IFS=$'\t' read -r file uid title folder category version description <&3; do
|
||||
if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
|
@ -318,53 +351,24 @@ action_install() {
|
|||
local folder_id
|
||||
folder_id=$(ensure_folder "$folder") || { failed=$((failed + 1)); continue; }
|
||||
|
||||
# Read and process the dashboard JSON
|
||||
local raw_json
|
||||
raw_json=$(cat "$full_path")
|
||||
|
||||
# Inject actual datasource UIDs
|
||||
local processed_json
|
||||
processed_json=$(inject_datasource_uids "$raw_json")
|
||||
|
||||
# Wrap in Grafana API envelope
|
||||
# Build Grafana API payload: inject datasource UIDs and wrap in envelope
|
||||
local api_payload
|
||||
api_payload=$(python3 -c "
|
||||
import json, sys
|
||||
|
||||
dashboard = json.loads('''${processed_json//\'/\\\'}''')
|
||||
|
||||
# Remove Grafana-internal fields that shouldn't be in the import
|
||||
dashboard.pop('id', None)
|
||||
dashboard.pop('version', None)
|
||||
dashboard.pop('iteration', None)
|
||||
|
||||
payload = {
|
||||
'dashboard': dashboard,
|
||||
'folderId': ${folder_id},
|
||||
'overwrite': True,
|
||||
'message': 'Installed by manage-dashboards v${VERSION}'
|
||||
}
|
||||
print(json.dumps(payload))
|
||||
" 2>/dev/null)
|
||||
|
||||
if [[ -z "$api_payload" ]]; then
|
||||
# Fallback: use a simpler approach for large/complex JSON
|
||||
api_payload=$(python3 << 'PYEOF'
|
||||
api_payload=$(python3 - "$full_path" "$PROMETHEUS_UID" "${LOKI_UID:-}" "$folder_id" << 'PYEOF'
|
||||
import json, sys
|
||||
|
||||
with open(sys.argv[1]) as f:
|
||||
dashboard = json.load(f)
|
||||
|
||||
# Inject datasource UIDs
|
||||
# Inject actual datasource UIDs
|
||||
json_str = json.dumps(dashboard)
|
||||
json_str = json_str.replace('${DS_PROMETHEUS}', sys.argv[2])
|
||||
json_str = json_str.replace('${DS_LOKI}', sys.argv[3])
|
||||
if sys.argv[3]:
|
||||
json_str = json_str.replace('${DS_LOKI}', sys.argv[3])
|
||||
dashboard = json.loads(json_str)
|
||||
|
||||
# Remove Grafana-internal fields
|
||||
dashboard.pop('id', None)
|
||||
dashboard.pop('version', None)
|
||||
dashboard.pop('iteration', None)
|
||||
for key in ('id', 'version', 'iteration'):
|
||||
dashboard.pop(key, None)
|
||||
|
||||
payload = {
|
||||
'dashboard': dashboard,
|
||||
|
|
@ -374,8 +378,7 @@ payload = {
|
|||
}
|
||||
print(json.dumps(payload))
|
||||
PYEOF
|
||||
"$full_path" "$PROMETHEUS_UID" "${LOKI_UID:-}" "$folder_id" 2>/dev/null)
|
||||
fi
|
||||
)
|
||||
|
||||
if [[ -z "$api_payload" ]]; then
|
||||
error "Failed to process dashboard JSON: ${file}"
|
||||
|
|
@ -388,9 +391,10 @@ PYEOF
|
|||
tmpfile=$(mktemp /tmp/dashboard-payload.XXXXXX)
|
||||
echo "$api_payload" > "$tmpfile"
|
||||
|
||||
incus file push "$tmpfile" "${CLUSTER_REMOTE}:${MONITORING_CONTAINER}/tmp/dashboard-payload.json" >/dev/null 2>&1
|
||||
|
||||
local result
|
||||
result=$(incus file push "$tmpfile" "${CLUSTER_REMOTE}:${MONITORING_CONTAINER}/tmp/dashboard-payload.json" 2>/dev/null && \
|
||||
grafana_api POST "/api/dashboards/db" \
|
||||
result=$(grafana_api POST "/api/dashboards/db" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d @/tmp/dashboard-payload.json 2>/dev/null)
|
||||
rm -f "$tmpfile"
|
||||
|
|
@ -417,7 +421,7 @@ except:
|
|||
# Clean up temp file in container
|
||||
incus exec "${CLUSTER_REMOTE}:${MONITORING_CONTAINER}" -- rm -f /tmp/dashboard-payload.json 2>/dev/null || true
|
||||
|
||||
done < <(read_manifest)
|
||||
done 3< <(read_manifest)
|
||||
|
||||
echo
|
||||
if [[ $failed -eq 0 ]]; then
|
||||
|
|
@ -446,7 +450,7 @@ action_export() {
|
|||
fi
|
||||
|
||||
# Export all dashboards listed in the manifest
|
||||
while IFS=$'\t' read -r file uid title folder category version description; do
|
||||
while IFS=$'\t' read -r file uid title folder category version description <&3; do
|
||||
if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
|
@ -454,7 +458,7 @@ action_export() {
|
|||
if export_single_dashboard "$uid" "${DASHBOARDS_DIR}/${file}"; then
|
||||
exported=$((exported + 1))
|
||||
fi
|
||||
done < <(read_manifest)
|
||||
done 3< <(read_manifest)
|
||||
|
||||
echo
|
||||
success "Exported ${exported} dashboard(s)"
|
||||
|
|
@ -560,7 +564,7 @@ action_list() {
|
|||
printf " ${BOLD}%-35s %-30s %-20s %s${RESET}\n" "UID" "Title" "Folder" "Status"
|
||||
printf " %-35s %-30s %-20s %s\n" "---" "-----" "------" "------"
|
||||
|
||||
while IFS=$'\t' read -r file uid title folder category version description; do
|
||||
while IFS=$'\t' read -r file uid title folder category version description <&3; do
|
||||
if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
|
@ -601,7 +605,7 @@ except:
|
|||
printf " %-35s %-30s %-20s " "$uid" "$title" "$folder"
|
||||
echo -e "$status_display"
|
||||
|
||||
done < <(read_manifest)
|
||||
done 3< <(read_manifest)
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -617,7 +621,7 @@ action_validate() {
|
|||
local warnings=0
|
||||
local errors=0
|
||||
|
||||
while IFS=$'\t' read -r file uid title folder category version description; do
|
||||
while IFS=$'\t' read -r file uid title folder category version description <&3; do
|
||||
if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
|
@ -734,7 +738,7 @@ PYEOF
|
|||
success " ${basename_file}: valid"
|
||||
fi
|
||||
|
||||
done < <(read_manifest)
|
||||
done 3< <(read_manifest)
|
||||
|
||||
echo
|
||||
info "${total} dashboard(s) checked: ${passed} passed, ${errors} errors, ${warnings} warnings"
|
||||
|
|
@ -758,7 +762,7 @@ action_status() {
|
|||
printf " ${BOLD}%-30s %-15s %-15s %s${RESET}\n" "Dashboard" "Local" "Grafana" "Match"
|
||||
printf " %-30s %-15s %-15s %s\n" "---------" "-----" "-------" "-----"
|
||||
|
||||
while IFS=$'\t' read -r file uid title folder category version description; do
|
||||
while IFS=$'\t' read -r file uid title folder category version description <&3; do
|
||||
local full_path="${DASHBOARDS_DIR}/${file}"
|
||||
|
||||
local local_ver="missing"
|
||||
|
|
@ -794,7 +798,7 @@ except:
|
|||
printf " %-30s %-15s %-15s " "$title" "$local_ver" "$grafana_ver"
|
||||
echo -e "$match_str"
|
||||
|
||||
done < <(read_manifest)
|
||||
done 3< <(read_manifest)
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -2,11 +2,18 @@
|
|||
"uid": "incus-logs-explorer",
|
||||
"title": "Logs Explorer",
|
||||
"description": "Log volume over time, full log stream with search, error and warning rates by systemd unit. Datasource: Loki.",
|
||||
"tags": ["incus", "logs", "loki"],
|
||||
"tags": [
|
||||
"incus",
|
||||
"logs",
|
||||
"loki"
|
||||
],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"fiscalYearStartMonth": 0,
|
||||
"liveNow": false,
|
||||
"editable": true,
|
||||
|
|
@ -17,7 +24,9 @@
|
|||
"icon": "external link",
|
||||
"includeVars": true,
|
||||
"keepTime": true,
|
||||
"tags": ["incus"],
|
||||
"tags": [
|
||||
"incus"
|
||||
],
|
||||
"targetBlank": false,
|
||||
"title": "Incus Dashboards",
|
||||
"tooltip": "Navigate to other Incus dashboards",
|
||||
|
|
@ -28,42 +37,57 @@
|
|||
"list": [
|
||||
{
|
||||
"type": "query",
|
||||
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "${DS_LOKI}"
|
||||
},
|
||||
"query": "label_values(job)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"name": "job",
|
||||
"label": "Job",
|
||||
"refresh": 2,
|
||||
"sort": 1
|
||||
"sort": 1,
|
||||
"allValue": ".*"
|
||||
},
|
||||
{
|
||||
"type": "query",
|
||||
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "${DS_LOKI}"
|
||||
},
|
||||
"query": "label_values(host)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"name": "host",
|
||||
"label": "Host",
|
||||
"refresh": 2,
|
||||
"sort": 1
|
||||
"sort": 1,
|
||||
"allValue": ".*"
|
||||
},
|
||||
{
|
||||
"type": "query",
|
||||
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "${DS_LOKI}"
|
||||
},
|
||||
"query": "label_values(unit)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"name": "unit",
|
||||
"label": "Unit",
|
||||
"refresh": 2,
|
||||
"sort": 1
|
||||
"sort": 1,
|
||||
"allValue": ".*"
|
||||
},
|
||||
{
|
||||
"type": "textbox",
|
||||
"name": "search",
|
||||
"label": "Search",
|
||||
"current": { "text": "", "value": "" }
|
||||
"current": {
|
||||
"text": "",
|
||||
"value": ""
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -71,7 +95,10 @@
|
|||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": { "type": "grafana", "uid": "-- Grafana --" },
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
|
|
@ -85,7 +112,12 @@
|
|||
"type": "row",
|
||||
"title": "Log Volume",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"panels": []
|
||||
},
|
||||
|
|
@ -93,9 +125,17 @@
|
|||
"type": "timeseries",
|
||||
"title": "Log Volume Over Time",
|
||||
"description": "Total log line count over time, broken down by job. Shows the rate at which logs are being generated across all selected sources.",
|
||||
"gridPos": { "h": 6, "w": 24, "x": 0, "y": 1 },
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"id": 2,
|
||||
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "${DS_LOKI}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (job) (count_over_time({job=~\"$job\", host=~\"$host\"} [$__auto]))",
|
||||
|
|
@ -114,31 +154,61 @@
|
|||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "normal", "group": "A" },
|
||||
"stacking": {
|
||||
"mode": "normal",
|
||||
"group": "A"
|
||||
},
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"unit": "short",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["sum", "max", "lastNotNull"] }
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
},
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"calcs": [
|
||||
"sum",
|
||||
"max",
|
||||
"lastNotNull"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Log Stream",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 },
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 7
|
||||
},
|
||||
"id": 3,
|
||||
"panels": []
|
||||
},
|
||||
|
|
@ -146,9 +216,17 @@
|
|||
"type": "logs",
|
||||
"title": "Logs",
|
||||
"description": "Live log stream filtered by job, host, unit, and free-text search. Displays timestamps, labels, and full log messages with expand-on-click detail view.",
|
||||
"gridPos": { "h": 16, "w": 24, "x": 0, "y": 8 },
|
||||
"gridPos": {
|
||||
"h": 16,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"id": 4,
|
||||
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "${DS_LOKI}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "{job=~\"$job\", host=~\"$host\", unit=~\"$unit\"} |= \"$search\"",
|
||||
|
|
@ -170,7 +248,12 @@
|
|||
"type": "row",
|
||||
"title": "Error & Warning Rates",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 },
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 24
|
||||
},
|
||||
"id": 5,
|
||||
"panels": []
|
||||
},
|
||||
|
|
@ -178,9 +261,17 @@
|
|||
"type": "timeseries",
|
||||
"title": "Error Rate by Unit",
|
||||
"description": "Rate of log lines matching error, fail, or fatal (case-insensitive) over time, grouped by systemd unit. Spikes indicate services experiencing failures.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 25
|
||||
},
|
||||
"id": 6,
|
||||
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "${DS_LOKI}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (unit) (count_over_time({job=~\"$job\", host=~\"$host\"} |~ \"(?i)error|fail|fatal\" [$__auto]))",
|
||||
|
|
@ -199,33 +290,67 @@
|
|||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"stacking": {
|
||||
"mode": "none",
|
||||
"group": "A"
|
||||
},
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"unit": "short",
|
||||
"color": { "mode": "fixed", "fixedColor": "red" },
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "red"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "red", "value": null }]
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["sum", "max", "lastNotNull"] }
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
},
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"calcs": [
|
||||
"sum",
|
||||
"max",
|
||||
"lastNotNull"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Warning Rate by Unit",
|
||||
"description": "Rate of log lines matching warn (case-insensitive) over time, grouped by systemd unit. Helps identify services generating warnings that may precede errors.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 25 },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 25
|
||||
},
|
||||
"id": 7,
|
||||
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "${DS_LOKI}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (unit) (count_over_time({job=~\"$job\", host=~\"$host\"} |~ \"(?i)warn\" [$__auto]))",
|
||||
|
|
@ -244,24 +369,50 @@
|
|||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"stacking": {
|
||||
"mode": "none",
|
||||
"group": "A"
|
||||
},
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"unit": "short",
|
||||
"color": { "mode": "fixed", "fixedColor": "yellow" },
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "yellow"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "yellow", "value": null }]
|
||||
"steps": [
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["sum", "max", "lastNotNull"] }
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
},
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"calcs": [
|
||||
"sum",
|
||||
"max",
|
||||
"lastNotNull"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -2,11 +2,19 @@
|
|||
"uid": "incus-network-deep-dive",
|
||||
"title": "Network Deep Dive",
|
||||
"description": "Per-node and per-device network traffic, instance-level network metrics, receive/transmit errors, and packet drops.",
|
||||
"tags": ["incus", "incusos", "network", "node-exporter"],
|
||||
"tags": [
|
||||
"incus",
|
||||
"incusos",
|
||||
"network",
|
||||
"node-exporter"
|
||||
],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"fiscalYearStartMonth": 0,
|
||||
"liveNow": false,
|
||||
"editable": true,
|
||||
|
|
@ -17,7 +25,9 @@
|
|||
"icon": "external link",
|
||||
"includeVars": true,
|
||||
"keepTime": true,
|
||||
"tags": ["incus"],
|
||||
"tags": [
|
||||
"incus"
|
||||
],
|
||||
"targetBlank": false,
|
||||
"title": "Incus Dashboards",
|
||||
"tooltip": "Navigate to other Incus dashboards",
|
||||
|
|
@ -28,7 +38,10 @@
|
|||
"list": [
|
||||
{
|
||||
"current": {},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"definition": "label_values(node_network_receive_bytes_total, instance)",
|
||||
"includeAll": false,
|
||||
"multi": true,
|
||||
|
|
@ -41,7 +54,10 @@
|
|||
},
|
||||
{
|
||||
"current": {},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"definition": "label_values(node_network_receive_bytes_total{instance=~\"$node\"}, device)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
|
|
@ -51,7 +67,8 @@
|
|||
"regex": "/^(?!lo$).*/",
|
||||
"sort": 1,
|
||||
"type": "query",
|
||||
"label": "Device"
|
||||
"label": "Device",
|
||||
"allValue": ".*"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -59,7 +76,10 @@
|
|||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": { "type": "grafana", "uid": "-- Grafana --" },
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
|
|
@ -73,7 +93,12 @@
|
|||
"type": "row",
|
||||
"title": "Cluster Summary",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"panels": []
|
||||
},
|
||||
|
|
@ -81,9 +106,17 @@
|
|||
"type": "stat",
|
||||
"title": "Total Ingress",
|
||||
"description": "Aggregate inbound network throughput across all selected nodes and devices.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"id": 2,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(node_network_receive_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval]))",
|
||||
|
|
@ -95,15 +128,28 @@
|
|||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "thresholds" }
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
|
|
@ -115,9 +161,17 @@
|
|||
"type": "stat",
|
||||
"title": "Total Egress",
|
||||
"description": "Aggregate outbound network throughput across all selected nodes and devices.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 1
|
||||
},
|
||||
"id": 3,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(node_network_transmit_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval]))",
|
||||
|
|
@ -129,15 +183,28 @@
|
|||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "thresholds" }
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
|
|
@ -149,9 +216,17 @@
|
|||
"type": "stat",
|
||||
"title": "Total Errors",
|
||||
"description": "Combined receive and transmit error rate across all selected nodes. Any non-zero value warrants investigation.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"y": 1
|
||||
},
|
||||
"id": 4,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(node_network_receive_errs_total{instance=~\"$node\"}[$__rate_interval])) + sum(rate(node_network_transmit_errs_total{instance=~\"$node\"}[$__rate_interval]))",
|
||||
|
|
@ -164,18 +239,35 @@
|
|||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 0.1 },
|
||||
{ "color": "red", "value": 1 }
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 0.1
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "pps",
|
||||
"color": { "mode": "thresholds" }
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
|
|
@ -187,9 +279,17 @@
|
|||
"type": "stat",
|
||||
"title": "Total Drops",
|
||||
"description": "Combined receive and transmit packet drop rate across all selected nodes. Drops indicate buffer or queue exhaustion.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"y": 1
|
||||
},
|
||||
"id": 5,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(node_network_receive_drop_total{instance=~\"$node\"}[$__rate_interval])) + sum(rate(node_network_transmit_drop_total{instance=~\"$node\"}[$__rate_interval]))",
|
||||
|
|
@ -202,18 +302,35 @@
|
|||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 0.1 },
|
||||
{ "color": "red", "value": 1 }
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 0.1
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "pps",
|
||||
"color": { "mode": "thresholds" }
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
|
|
@ -225,7 +342,12 @@
|
|||
"type": "row",
|
||||
"title": "Per-Node Traffic",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 5
|
||||
},
|
||||
"id": 10,
|
||||
"panels": []
|
||||
},
|
||||
|
|
@ -233,9 +355,17 @@
|
|||
"type": "timeseries",
|
||||
"title": "Receive Rate by Node & Device",
|
||||
"description": "Inbound network throughput broken down by node and network device. Identifies which interfaces are handling the most receive traffic.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 6
|
||||
},
|
||||
"id": 11,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(node_network_receive_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
|
||||
|
|
@ -254,33 +384,66 @@
|
|||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"stacking": {
|
||||
"mode": "none",
|
||||
"group": "A"
|
||||
},
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
},
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max",
|
||||
"lastNotNull"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Transmit Rate by Node & Device",
|
||||
"description": "Outbound network throughput broken down by node and network device. Identifies which interfaces are handling the most transmit traffic.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 6
|
||||
},
|
||||
"id": 12,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(node_network_transmit_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
|
||||
|
|
@ -299,31 +462,61 @@
|
|||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"stacking": {
|
||||
"mode": "none",
|
||||
"group": "A"
|
||||
},
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
},
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max",
|
||||
"lastNotNull"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Instance Network (Incus metrics)",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 14
|
||||
},
|
||||
"id": 20,
|
||||
"panels": []
|
||||
},
|
||||
|
|
@ -331,9 +524,17 @@
|
|||
"type": "timeseries",
|
||||
"title": "Instance Receive Rate",
|
||||
"description": "Inbound network throughput per Incus instance, aggregated across all instance network devices. Uses Incus-native metrics for container and VM visibility.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 15
|
||||
},
|
||||
"id": 21,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (name) (rate(incus_network_receive_bytes_total{instance=~\"$node\"}[$__rate_interval]))",
|
||||
|
|
@ -352,33 +553,66 @@
|
|||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"stacking": {
|
||||
"mode": "none",
|
||||
"group": "A"
|
||||
},
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
},
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max",
|
||||
"lastNotNull"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Instance Transmit Rate",
|
||||
"description": "Outbound network throughput per Incus instance, aggregated across all instance network devices. Uses Incus-native metrics for container and VM visibility.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 15
|
||||
},
|
||||
"id": 22,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (name) (rate(incus_network_transmit_bytes_total{instance=~\"$node\"}[$__rate_interval]))",
|
||||
|
|
@ -397,31 +631,61 @@
|
|||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"stacking": {
|
||||
"mode": "none",
|
||||
"group": "A"
|
||||
},
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
},
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max",
|
||||
"lastNotNull"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Errors & Drops",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 23
|
||||
},
|
||||
"id": 30,
|
||||
"panels": []
|
||||
},
|
||||
|
|
@ -429,9 +693,17 @@
|
|||
"type": "timeseries",
|
||||
"title": "Receive/Transmit Errors",
|
||||
"description": "Network interface error rates broken down by node and device. Receive errors may indicate CRC failures or framing issues; transmit errors may indicate collisions or driver problems.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 24
|
||||
},
|
||||
"id": 31,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(node_network_receive_errs_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
|
||||
|
|
@ -455,33 +727,66 @@
|
|||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"stacking": {
|
||||
"mode": "none",
|
||||
"group": "A"
|
||||
},
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"unit": "pps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
},
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max",
|
||||
"lastNotNull"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Packet Drops",
|
||||
"description": "Network interface packet drop rates broken down by node and device. Drops indicate kernel buffer or queue exhaustion and may cause retransmissions or connection failures.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 24
|
||||
},
|
||||
"id": 32,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(node_network_receive_drop_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
|
||||
|
|
@ -505,24 +810,49 @@
|
|||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"stacking": {
|
||||
"mode": "none",
|
||||
"group": "A"
|
||||
},
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"unit": "pps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
},
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max",
|
||||
"lastNotNull"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -2,27 +2,45 @@
|
|||
"uid": "incus-storage-filesystem",
|
||||
"title": "Storage & Filesystem",
|
||||
"description": "Filesystem usage, available space trends, disk throughput, I/O latency, disk utilization, and inode usage per node.",
|
||||
"tags": ["incus", "incusos", "storage", "node-exporter"],
|
||||
"tags": [
|
||||
"incus",
|
||||
"incusos",
|
||||
"storage",
|
||||
"node-exporter"
|
||||
],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"fiscalYearStartMonth": 0,
|
||||
"liveNow": false,
|
||||
"editable": true,
|
||||
"graphTooltip": 1,
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": true, "icon": "external link", "includeVars": true,
|
||||
"keepTime": true, "tags": ["incus"], "targetBlank": false,
|
||||
"title": "Incus Dashboards", "type": "dashboards"
|
||||
"asDropdown": true,
|
||||
"icon": "external link",
|
||||
"includeVars": true,
|
||||
"keepTime": true,
|
||||
"tags": [
|
||||
"incus"
|
||||
],
|
||||
"targetBlank": false,
|
||||
"title": "Incus Dashboards",
|
||||
"type": "dashboards"
|
||||
}
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"definition": "label_values(node_filesystem_size_bytes, instance)",
|
||||
"includeAll": false,
|
||||
"multi": true,
|
||||
|
|
@ -35,7 +53,10 @@
|
|||
},
|
||||
{
|
||||
"current": {},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"definition": "label_values(node_filesystem_size_bytes{instance=~\"$node\"}, mountpoint)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
|
|
@ -44,7 +65,8 @@
|
|||
"refresh": 2,
|
||||
"sort": 1,
|
||||
"type": "query",
|
||||
"label": "Mountpoint"
|
||||
"label": "Mountpoint",
|
||||
"allValue": ".*"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -52,83 +74,264 @@
|
|||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": { "type": "grafana", "uid": "-- Grafana --" },
|
||||
"enable": true, "hide": true,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts", "type": "dashboard"
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "row", "title": "Filesystem Overview",
|
||||
"type": "row",
|
||||
"title": "Filesystem Overview",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"id": 1, "panels": []
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "table",
|
||||
"title": "Filesystem Usage Table",
|
||||
"description": "Current filesystem size, available space, and usage percentage per node and mountpoint. High usage is highlighted with color thresholds.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 1 },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"id": 2,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "node_filesystem_size_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}",
|
||||
"legendFormat": "", "refId": "A", "format": "table", "instant": true
|
||||
"legendFormat": "",
|
||||
"refId": "A",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
},
|
||||
{
|
||||
"expr": "node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}",
|
||||
"legendFormat": "", "refId": "B", "format": "table", "instant": true
|
||||
"legendFormat": "",
|
||||
"refId": "B",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
},
|
||||
{
|
||||
"expr": "(1 - node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}) * 100",
|
||||
"legendFormat": "", "refId": "C", "format": "table", "instant": true
|
||||
"legendFormat": "",
|
||||
"refId": "C",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": { "align": "auto", "displayMode": "auto", "inspect": false, "filterable": true },
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
"custom": {
|
||||
"align": "auto",
|
||||
"displayMode": "auto",
|
||||
"inspect": false,
|
||||
"filterable": true
|
||||
},
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byName", "options": "instance" }, "properties": [{ "id": "displayName", "value": "Node" }] },
|
||||
{ "matcher": { "id": "byName", "options": "mountpoint" }, "properties": [{ "id": "displayName", "value": "Mountpoint" }] },
|
||||
{ "matcher": { "id": "byName", "options": "fstype" }, "properties": [{ "id": "displayName", "value": "FS Type" }] },
|
||||
{ "matcher": { "id": "byName", "options": "Value #A" }, "properties": [{ "id": "displayName", "value": "Size" }, { "id": "unit", "value": "bytes" }] },
|
||||
{ "matcher": { "id": "byName", "options": "Value #B" }, "properties": [{ "id": "displayName", "value": "Available" }, { "id": "unit", "value": "bytes" }] },
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Value #C" },
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "instance"
|
||||
},
|
||||
"properties": [
|
||||
{ "id": "displayName", "value": "Used %" },
|
||||
{ "id": "unit", "value": "percent" },
|
||||
{ "id": "custom.displayMode", "value": "color-background" },
|
||||
{ "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "orange", "value": 85 }, { "color": "red", "value": 95 }] } }
|
||||
{
|
||||
"id": "displayName",
|
||||
"value": "Node"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "mountpoint"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "displayName",
|
||||
"value": "Mountpoint"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "fstype"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "displayName",
|
||||
"value": "FS Type"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Value #A"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "displayName",
|
||||
"value": "Size"
|
||||
},
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "bytes"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Value #B"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "displayName",
|
||||
"value": "Available"
|
||||
},
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "bytes"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Value #C"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "displayName",
|
||||
"value": "Used %"
|
||||
},
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "percent"
|
||||
},
|
||||
{
|
||||
"id": "custom.displayMode",
|
||||
"value": "color-background"
|
||||
},
|
||||
{
|
||||
"id": "thresholds",
|
||||
"value": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 70
|
||||
},
|
||||
{
|
||||
"color": "orange",
|
||||
"value": 85
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 95
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": { "showHeader": true, "sortBy": [], "footer": { "show": false, "reducer": ["sum"], "fields": "" } },
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"sortBy": [],
|
||||
"footer": {
|
||||
"show": false,
|
||||
"reducer": [
|
||||
"sum"
|
||||
],
|
||||
"fields": ""
|
||||
}
|
||||
},
|
||||
"transformations": [
|
||||
{ "id": "merge", "options": {} },
|
||||
{ "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "job": true, "device": true }, "renameByName": {} } }
|
||||
{
|
||||
"id": "merge",
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {
|
||||
"Time": true,
|
||||
"__name__": true,
|
||||
"job": true,
|
||||
"device": true
|
||||
},
|
||||
"renameByName": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "row", "title": "Space Trends",
|
||||
"type": "row",
|
||||
"title": "Space Trends",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
|
||||
"id": 3, "panels": []
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
},
|
||||
"id": 3,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Available Space Over Time",
|
||||
"description": "Tracks available filesystem space over time per node and mountpoint. Use this to identify gradual space consumption trends before they become critical.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 10 },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 10
|
||||
},
|
||||
"id": 4,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}",
|
||||
|
|
@ -139,37 +342,86 @@
|
|||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never", "pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": {
|
||||
"mode": "none",
|
||||
"group": "A"
|
||||
},
|
||||
"axisPlacement": "auto",
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"unit": "bytes",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
},
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max",
|
||||
"lastNotNull"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row", "title": "Disk Throughput",
|
||||
"type": "row",
|
||||
"title": "Disk Throughput",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 },
|
||||
"id": 5, "panels": []
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 18
|
||||
},
|
||||
"id": 5,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Disk Read Throughput",
|
||||
"description": "Disk read throughput in bytes per second, broken down by instance and device. Identifies which disks are handling the most read traffic.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 19
|
||||
},
|
||||
"id": 6,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (instance, device) (rate(node_disk_read_bytes_total{instance=~\"$node\"}[$__rate_interval]))",
|
||||
|
|
@ -180,31 +432,73 @@
|
|||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never", "pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": {
|
||||
"mode": "none",
|
||||
"group": "A"
|
||||
},
|
||||
"axisPlacement": "auto",
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
},
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max",
|
||||
"lastNotNull"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Disk Write Throughput",
|
||||
"description": "Disk write throughput in bytes per second, broken down by instance and device. Identifies which disks are handling the most write traffic.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 19
|
||||
},
|
||||
"id": 7,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (instance, device) (rate(node_disk_written_bytes_total{instance=~\"$node\"}[$__rate_interval]))",
|
||||
|
|
@ -215,37 +509,86 @@
|
|||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never", "pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": {
|
||||
"mode": "none",
|
||||
"group": "A"
|
||||
},
|
||||
"axisPlacement": "auto",
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
},
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max",
|
||||
"lastNotNull"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row", "title": "I/O Performance",
|
||||
"type": "row",
|
||||
"title": "I/O Performance",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 },
|
||||
"id": 8, "panels": []
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 27
|
||||
},
|
||||
"id": 8,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Average Read Latency",
|
||||
"description": "Average time per completed read operation. Calculated as total read time divided by completed reads. High values indicate slow disk response.",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 28 },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 28
|
||||
},
|
||||
"id": 9,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(node_disk_read_time_seconds_total{instance=~\"$node\"}[$__rate_interval]) / rate(node_disk_reads_completed_total{instance=~\"$node\"}[$__rate_interval])",
|
||||
|
|
@ -256,31 +599,73 @@
|
|||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never", "pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": {
|
||||
"mode": "none",
|
||||
"group": "A"
|
||||
},
|
||||
"axisPlacement": "auto",
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"unit": "s",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
},
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max",
|
||||
"lastNotNull"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Average Write Latency",
|
||||
"description": "Average time per completed write operation. Calculated as total write time divided by completed writes. High values indicate slow disk response or write saturation.",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 28 },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 28
|
||||
},
|
||||
"id": 10,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(node_disk_write_time_seconds_total{instance=~\"$node\"}[$__rate_interval]) / rate(node_disk_writes_completed_total{instance=~\"$node\"}[$__rate_interval])",
|
||||
|
|
@ -291,31 +676,73 @@
|
|||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never", "pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": {
|
||||
"mode": "none",
|
||||
"group": "A"
|
||||
},
|
||||
"axisPlacement": "auto",
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"unit": "s",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
},
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max",
|
||||
"lastNotNull"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Disk Utilization %",
|
||||
"description": "Percentage of time the disk is busy processing I/O requests. Values near 100% indicate the disk is saturated and may be a bottleneck.",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 28 },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 28
|
||||
},
|
||||
"id": 11,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (instance) (rate(node_disk_io_time_seconds_total{instance=~\"$node\"}[$__rate_interval])) * 100",
|
||||
|
|
@ -326,37 +753,96 @@
|
|||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2,
|
||||
"fillOpacity": 25, "gradientMode": "scheme", "spanNulls": false,
|
||||
"showPoints": "never", "pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "line" }
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 25,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": {
|
||||
"mode": "none",
|
||||
"group": "A"
|
||||
},
|
||||
"axisPlacement": "auto",
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "line"
|
||||
}
|
||||
},
|
||||
"unit": "percent", "min": 0, "max": 100,
|
||||
"color": { "mode": "continuous-GrYlRd" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 80 }] }
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"color": {
|
||||
"mode": "continuous-GrYlRd"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 50
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
},
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max",
|
||||
"lastNotNull"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row", "title": "Inodes",
|
||||
"type": "row",
|
||||
"title": "Inodes",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 },
|
||||
"id": 12, "panels": []
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 36
|
||||
},
|
||||
"id": 12,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "bargauge",
|
||||
"title": "Inode Usage %",
|
||||
"description": "Percentage of used inodes per filesystem. Running out of inodes prevents creating new files even when disk space is available. Monitor for filesystems with many small files.",
|
||||
"gridPos": { "h": 6, "w": 24, "x": 0, "y": 37 },
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 37
|
||||
},
|
||||
"id": 13,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - node_filesystem_files_free{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"} / node_filesystem_files{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}) * 100",
|
||||
|
|
@ -366,16 +852,46 @@
|
|||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] },
|
||||
"unit": "percent", "min": 0, "max": 100,
|
||||
"color": { "mode": "thresholds" }
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 70
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "horizontal", "displayMode": "gradient", "showUnfilled": true,
|
||||
"minVizWidth": 8, "minVizHeight": 16, "valueMode": "color"
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"orientation": "horizontal",
|
||||
"displayMode": "gradient",
|
||||
"showUnfilled": true,
|
||||
"minVizWidth": 8,
|
||||
"minVizHeight": 16,
|
||||
"valueMode": "color"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
|
|||
Binary file not shown.
|
After Width: | Height: | Size: 213 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 197 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 109 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 266 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 139 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 72 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 147 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 163 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 191 KiB |
Loading…
Reference in New Issue