Add professional Grafana dashboard suite with management tooling
- New manage-dashboards script: install/export/list/validate/status
actions for Grafana dashboard lifecycle via HTTP API, with datasource
UID portability (${DS_PROMETHEUS}/${DS_LOKI} variables) and folder
organization
- Reorganize dashboards into core/ and community/ subdirectories with
dashboard.yaml manifest
- Enhance 3 existing dashboards: template variables (instance/node/
proxy/server), cross-dashboard navigation links, new panels (instance
status table, CPU mode breakdown, load averages, memory breakdown,
filesystem bar gauge, response code distribution, server state
timeline, connection errors)
- Add 6 new dashboards: Instance Deep Dive, Storage & Filesystem,
Network Deep Dive, Logs Explorer (Loki), Prometheus Health, Capacity
Planning (with predict_linear forecasting)
- Update deploy-observability: explicit datasource UIDs in provisioning,
phase 9 uses manage-dashboards --install, new --workloads action for
dummy metric generation (workload-web + workload-api containers)
- Update observability guide with all 9 dashboard descriptions
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
99ea7e4123
commit
6c7714459b
|
|
@ -88,6 +88,7 @@ ${BOLD}ACTIONS${RESET}
|
|||
--deploy Full deploy: monitoring container, Prometheus, Grafana,
|
||||
Loki, Promtail, node-exporters, ACLs, network forward,
|
||||
HAProxy exporter route, dashboards
|
||||
--workloads Deploy dummy workloads to generate varied metrics
|
||||
--status Show status of all observability components
|
||||
--cleanup Remove all observability infrastructure
|
||||
--doctor Diagnose common issues
|
||||
|
|
@ -462,13 +463,14 @@ phase_install_grafana() {
|
|||
apt-get install -y -qq grafana 2>&1 | tail -1
|
||||
"
|
||||
|
||||
# Provision datasources
|
||||
# Provision datasources with explicit UIDs for dashboard portability
|
||||
container_exec monitoring bash -c "mkdir -p /etc/grafana/provisioning/datasources
|
||||
cat > /etc/grafana/provisioning/datasources/observability.yaml << 'DSCFG'
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
access: proxy
|
||||
url: http://localhost:9090
|
||||
isDefault: true
|
||||
|
|
@ -476,28 +478,12 @@ datasources:
|
|||
|
||||
- name: Loki
|
||||
type: loki
|
||||
uid: loki
|
||||
access: proxy
|
||||
url: http://localhost:3100
|
||||
editable: true
|
||||
DSCFG"
|
||||
|
||||
# Provision dashboard directory
|
||||
container_exec monitoring bash -c "mkdir -p /etc/grafana/provisioning/dashboards
|
||||
cat > /etc/grafana/provisioning/dashboards/observability.yaml << 'DBCFG'
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: Observability
|
||||
orgId: 1
|
||||
folder: ''
|
||||
type: file
|
||||
disableDeletion: false
|
||||
editable: true
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
foldersFromFilesStructure: false
|
||||
DBCFG
|
||||
mkdir -p /var/lib/grafana/dashboards"
|
||||
|
||||
container_exec monitoring systemctl enable grafana-server
|
||||
container_exec monitoring systemctl restart grafana-server
|
||||
sleep 2
|
||||
|
|
@ -725,20 +711,24 @@ phase_create_network_forward() {
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
phase_upload_dashboards() {
|
||||
step "Phase 9: Upload Grafana dashboards"
|
||||
step "Phase 9: Install Grafana dashboards"
|
||||
|
||||
if ! dry_run_guard "Upload dashboards from ${DASHBOARDS_DIR}"; then
|
||||
if ! dry_run_guard "Install dashboards via manage-dashboards"; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local manage_script="${SCRIPT_DIR}/manage-dashboards"
|
||||
if [[ ! -x "$manage_script" ]]; then
|
||||
warn "manage-dashboards script not found at ${manage_script}"
|
||||
warn "Falling back to direct file upload."
|
||||
|
||||
if [[ ! -d "$DASHBOARDS_DIR" ]]; then
|
||||
warn "Dashboard directory not found: ${DASHBOARDS_DIR}"
|
||||
warn "Skipping dashboard upload -- create dashboards manually in Grafana."
|
||||
return 0
|
||||
fi
|
||||
|
||||
local count=0
|
||||
for dashboard in "${DASHBOARDS_DIR}"/*.json; do
|
||||
for dashboard in "${DASHBOARDS_DIR}"/core/*.json; do
|
||||
[[ -f "$dashboard" ]] || continue
|
||||
local basename
|
||||
basename=$(basename "$dashboard")
|
||||
|
|
@ -748,12 +738,242 @@ phase_upload_dashboards() {
|
|||
done
|
||||
|
||||
if [[ $count -gt 0 ]]; then
|
||||
# Restart Grafana to pick up new dashboards
|
||||
container_exec monitoring systemctl restart grafana-server 2>/dev/null || true
|
||||
success "${count} dashboard(s) uploaded"
|
||||
success "${count} dashboard(s) uploaded (direct push)"
|
||||
else
|
||||
info "No dashboard files found in ${DASHBOARDS_DIR}"
|
||||
info "No dashboard files found in ${DASHBOARDS_DIR}/core/"
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Wait for Grafana API to be ready
|
||||
info "Waiting for Grafana API..."
|
||||
local elapsed=0
|
||||
while true; do
|
||||
local code
|
||||
code=$(container_exec monitoring curl -s -o /dev/null -w '%{http_code}' \
|
||||
http://localhost:3000/api/health 2>/dev/null || echo "000")
|
||||
if [[ "$code" == "200" ]]; then
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
elapsed=$((elapsed + 2))
|
||||
if [[ $elapsed -ge 30 ]]; then
|
||||
warn "Grafana API not ready after 30s -- falling back to file push"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
# Use manage-dashboards to install with proper folder organization
|
||||
local flags="-r ${CLUSTER_REMOTE}"
|
||||
[[ "$VERBOSE" == true ]] && flags="${flags} -v"
|
||||
[[ "$QUIET" == true ]] && flags="${flags} -q"
|
||||
|
||||
"${manage_script}" --install ${flags}
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Workloads action
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
WORKLOAD_WEB_IP="10.10.10.80"
|
||||
WORKLOAD_API_IP="10.10.10.81"
|
||||
|
||||
action_workloads() {
|
||||
step "Deploying dummy workloads for metric generation"
|
||||
echo
|
||||
info " workload-web: ${WORKLOAD_WEB_IP} (nginx + CPU/disk/network crons)"
|
||||
info " workload-api: ${WORKLOAD_API_IP} (Python HTTP + memory/network crons)"
|
||||
echo
|
||||
|
||||
phase_deploy_workload_web
|
||||
phase_deploy_workload_api
|
||||
|
||||
echo
|
||||
success "Dummy workloads deployed!"
|
||||
info " Metrics will appear within 1-2 minutes"
|
||||
info " workload-web: nginx on ${WORKLOAD_WEB_IP}:80, CPU/disk/net crons"
|
||||
info " workload-api: Python on ${WORKLOAD_API_IP}:8080, memory sawtooth cron"
|
||||
}
|
||||
|
||||
phase_deploy_workload_web() {
|
||||
step "Deploy workload-web (nginx + varied load)"
|
||||
|
||||
if container_exists "workload-web"; then
|
||||
if container_running "workload-web"; then
|
||||
success "workload-web already running"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! dry_run_guard "Launch workload-web on ${OVN_NETWORK} at ${WORKLOAD_WEB_IP}"; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
incus launch images:alpine/3.20 "$(container_ref workload-web)" \
|
||||
-n "$OVN_NETWORK" \
|
||||
--target "oc-node-01" \
|
||||
-c limits.memory=128MiB
|
||||
|
||||
wait_for_container_agent workload-web 30
|
||||
|
||||
container_exec workload-web sh -c "
|
||||
apk add --no-cache nginx curl gzip coreutils > /dev/null 2>&1
|
||||
|
||||
# Configure nginx
|
||||
mkdir -p /var/www/html
|
||||
cat > /var/www/html/index.html << 'WEBHTML'
|
||||
<html><body><h1>workload-web</h1><p>Observability test workload</p></body></html>
|
||||
WEBHTML
|
||||
cat > /etc/nginx/http.d/default.conf << 'NGXCFG'
|
||||
server {
|
||||
listen 80 default_server;
|
||||
root /var/www/html;
|
||||
index index.html;
|
||||
location / { try_files \$uri \$uri/ =404; }
|
||||
}
|
||||
NGXCFG
|
||||
rc-update add nginx default 2>/dev/null || true
|
||||
rc-service nginx start 2>/dev/null || true
|
||||
|
||||
# Cron: every minute -> 10 HTTP requests (network traffic)
|
||||
# Cron: every 5 min -> gzip random data (CPU spike)
|
||||
# Cron: every 15 min -> dd 5 MiB write+delete (disk I/O burst)
|
||||
cat > /etc/crontabs/root << 'CRONCFG'
|
||||
* * * * * for i in \$(seq 1 10); do wget -q -O /dev/null http://localhost/ 2>/dev/null; done
|
||||
*/5 * * * * dd if=/dev/urandom bs=1M count=2 2>/dev/null | gzip > /dev/null 2>&1
|
||||
*/15 * * * * dd if=/dev/urandom of=/tmp/disktest bs=1M count=5 2>/dev/null && rm -f /tmp/disktest
|
||||
CRONCFG
|
||||
rc-update add crond default 2>/dev/null || true
|
||||
rc-service crond start 2>/dev/null || true
|
||||
"
|
||||
|
||||
# Configure static IP
|
||||
container_exec workload-web sh -c "
|
||||
cat > /etc/network/interfaces << NETCFG
|
||||
auto lo
|
||||
iface lo inet loopback
|
||||
|
||||
auto eth0
|
||||
iface eth0 inet static
|
||||
address ${WORKLOAD_WEB_IP}/24
|
||||
gateway 10.10.10.1
|
||||
NETCFG
|
||||
rc-service networking restart 2>/dev/null || true
|
||||
"
|
||||
|
||||
# Attach monitoring ACL
|
||||
incus config device set "$(container_ref workload-web)" eth0 \
|
||||
security.acls=monitoring-allow security.acls.default.ingress.action=allow \
|
||||
security.acls.default.egress.action=allow 2>/dev/null || true
|
||||
|
||||
success "workload-web deployed at ${WORKLOAD_WEB_IP}"
|
||||
}
|
||||
|
||||
phase_deploy_workload_api() {
|
||||
step "Deploy workload-api (Python HTTP + memory patterns)"
|
||||
|
||||
if container_exists "workload-api"; then
|
||||
if container_running "workload-api"; then
|
||||
success "workload-api already running"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! dry_run_guard "Launch workload-api on ${OVN_NETWORK} at ${WORKLOAD_API_IP}"; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
incus launch images:alpine/3.20 "$(container_ref workload-api)" \
|
||||
-n "$OVN_NETWORK" \
|
||||
--target "oc-node-03" \
|
||||
-c limits.memory=128MiB
|
||||
|
||||
wait_for_container_agent workload-api 30
|
||||
|
||||
container_exec workload-api sh -c "
|
||||
apk add --no-cache python3 curl coreutils > /dev/null 2>&1
|
||||
|
||||
# Create a simple JSON status endpoint
|
||||
mkdir -p /opt/api
|
||||
cat > /opt/api/server.py << 'PYAPI'
|
||||
import http.server
|
||||
import json
|
||||
import time
|
||||
import os
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.end_headers()
|
||||
data = {
|
||||
'status': 'ok',
|
||||
'uptime': time.time() - START_TIME,
|
||||
'hostname': os.uname().nodename,
|
||||
'pid': os.getpid()
|
||||
}
|
||||
self.wfile.write(json.dumps(data).encode())
|
||||
|
||||
def log_message(self, format, *args):
|
||||
pass # suppress access logs
|
||||
|
||||
START_TIME = time.time()
|
||||
http.server.HTTPServer(('0.0.0.0', 8080), Handler).serve_forever()
|
||||
PYAPI
|
||||
|
||||
# Create memory stress script
|
||||
cat > /opt/api/mem-stress.py << 'PYMEM'
|
||||
import time
|
||||
# Allocate ~20 MiB, hold for 30 seconds, then release
|
||||
data = bytearray(20 * 1024 * 1024)
|
||||
time.sleep(30)
|
||||
del data
|
||||
PYMEM
|
||||
|
||||
# Start the API server as a background service
|
||||
cat > /etc/init.d/api-server << 'INITSCRIPT'
|
||||
#!/sbin/openrc-run
|
||||
name=\"api-server\"
|
||||
command=\"/usr/bin/python3\"
|
||||
command_args=\"/opt/api/server.py\"
|
||||
command_background=true
|
||||
pidfile=\"/run/api-server.pid\"
|
||||
INITSCRIPT
|
||||
chmod +x /etc/init.d/api-server
|
||||
rc-update add api-server default 2>/dev/null || true
|
||||
rc-service api-server start 2>/dev/null || true
|
||||
|
||||
# Cron: every 2 min -> curl workload-web (inter-container traffic)
|
||||
# Cron: every 10 min -> allocate 20 MiB for 30s (memory sawtooth)
|
||||
cat > /etc/crontabs/root << 'CRONCFG'
|
||||
*/2 * * * * wget -q -O /dev/null http://${WORKLOAD_WEB_IP}/ 2>/dev/null
|
||||
*/10 * * * * python3 /opt/api/mem-stress.py 2>/dev/null &
|
||||
CRONCFG
|
||||
rc-update add crond default 2>/dev/null || true
|
||||
rc-service crond start 2>/dev/null || true
|
||||
"
|
||||
|
||||
# Configure static IP
|
||||
container_exec workload-api sh -c "
|
||||
cat > /etc/network/interfaces << NETCFG
|
||||
auto lo
|
||||
iface lo inet loopback
|
||||
|
||||
auto eth0
|
||||
iface eth0 inet static
|
||||
address ${WORKLOAD_API_IP}/24
|
||||
gateway 10.10.10.1
|
||||
NETCFG
|
||||
rc-service networking restart 2>/dev/null || true
|
||||
"
|
||||
|
||||
# Attach monitoring ACL
|
||||
incus config device set "$(container_ref workload-api)" eth0 \
|
||||
security.acls=monitoring-allow security.acls.default.ingress.action=allow \
|
||||
security.acls.default.egress.action=allow 2>/dev/null || true
|
||||
|
||||
success "workload-api deployed at ${WORKLOAD_API_IP}"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -798,6 +1018,17 @@ action_status() {
|
|||
idx=$((idx + 1))
|
||||
done
|
||||
|
||||
# Workload containers (optional)
|
||||
for wl in workload-web workload-api; do
|
||||
if container_exists "$wl"; then
|
||||
if container_running "$wl"; then
|
||||
success " ${wl}: running"
|
||||
else
|
||||
warn " ${wl}: stopped"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$all_ok" == false ]]; then
|
||||
echo
|
||||
error "Some containers are missing or stopped."
|
||||
|
|
@ -914,6 +1145,9 @@ action_cleanup() {
|
|||
if container_exists "$(node_exp_name "$idx")"; then has_anything=true; fi
|
||||
idx=$((idx + 1))
|
||||
done
|
||||
for wl in workload-web workload-api; do
|
||||
if container_exists "$wl"; then has_anything=true; fi
|
||||
done
|
||||
if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
|
||||
has_anything=true
|
||||
fi
|
||||
|
|
@ -936,6 +1170,11 @@ action_cleanup() {
|
|||
fi
|
||||
idx=$((idx + 1))
|
||||
done
|
||||
for wl in workload-web workload-api; do
|
||||
if container_exists "$wl"; then
|
||||
warn " - ${wl}"
|
||||
fi
|
||||
done
|
||||
if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
|
||||
warn " - network forward ${FORWARD_VIP}"
|
||||
fi
|
||||
|
|
@ -1003,6 +1242,15 @@ except: pass
|
|||
idx=$((idx + 1))
|
||||
done
|
||||
|
||||
# Delete workload containers
|
||||
for wl in workload-web workload-api; do
|
||||
if container_exists "$wl"; then
|
||||
info "Deleting ${wl}..."
|
||||
incus delete "$(container_ref "$wl")" --force 2>/dev/null || true
|
||||
success "Deleted ${wl}"
|
||||
fi
|
||||
done
|
||||
|
||||
# Delete network forward
|
||||
step "Removing network forward"
|
||||
if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
|
||||
|
|
@ -1153,6 +1401,7 @@ parse_args() {
|
|||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--deploy) ACTION="deploy";;
|
||||
--workloads) ACTION="workloads";;
|
||||
--status) ACTION="status";;
|
||||
--cleanup) ACTION="cleanup";;
|
||||
--doctor) ACTION="doctor";;
|
||||
|
|
@ -1188,6 +1437,7 @@ main() {
|
|||
echo
|
||||
case "$ACTION" in
|
||||
deploy) action_deploy;;
|
||||
workloads) action_workloads;;
|
||||
status) action_status;;
|
||||
cleanup) action_cleanup;;
|
||||
doctor) action_doctor;;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,854 @@
|
|||
#!/usr/bin/env bash
|
||||
# manage-dashboards - Manage Grafana dashboards for the Incus observability stack
|
||||
#
|
||||
# Push, export, validate, and inspect Grafana dashboards via the Grafana HTTP
|
||||
# API. Dashboards are organized into folders and use datasource UID variables
|
||||
# for portability across deployments.
|
||||
#
|
||||
# Usage: manage-dashboards [OPTIONS]
|
||||
# Run 'manage-dashboards --help' for full usage information.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
readonly VERSION="0.1.0"
|
||||
readonly SCRIPT_NAME="manage-dashboards"
|
||||
readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
readonly DASHBOARDS_DIR="${SCRIPT_DIR}/observability-dashboards"
|
||||
readonly MANIFEST_FILE="${DASHBOARDS_DIR}/dashboard.yaml"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Defaults (overridden by CLI flags)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
CLUSTER_REMOTE="oc-node-01"
|
||||
MONITORING_CONTAINER="monitoring"
|
||||
GRAFANA_URL="http://localhost:3000"
|
||||
GRAFANA_USER="admin"
|
||||
GRAFANA_PASSWORD="admin"
|
||||
|
||||
# Runtime flags
|
||||
DRY_RUN=false
|
||||
VERBOSE=false
|
||||
QUIET=false
|
||||
ACTION=""
|
||||
FILTER_CATEGORY=""
|
||||
FILTER_FILE=""
|
||||
FILTER_UID=""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Color and formatting
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
setup_colors() {
|
||||
if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[0;33m'
|
||||
BLUE='\033[0;34m'
|
||||
CYAN='\033[0;36m'
|
||||
BOLD='\033[1m'
|
||||
DIM='\033[2m'
|
||||
RESET='\033[0m'
|
||||
else
|
||||
RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET=''
|
||||
fi
|
||||
}
|
||||
|
||||
info() { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET} $*"; }
|
||||
success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET} $*"; }
|
||||
warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; }
|
||||
error() { echo -e "${RED}[error]${RESET} $*" >&2; }
|
||||
step() { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; }
|
||||
detail() { [[ "$VERBOSE" != true ]] && return; echo -e "${DIM} $*${RESET}"; }
|
||||
|
||||
dry_run_guard() {
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
info "(dry-run) $*"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Help
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
${BOLD}${SCRIPT_NAME}${RESET} v${VERSION} - Manage Grafana dashboards for the Incus observability stack
|
||||
|
||||
${BOLD}USAGE${RESET}
|
||||
${SCRIPT_NAME} [OPTIONS]
|
||||
|
||||
${BOLD}ACTIONS${RESET}
|
||||
--install Push dashboards to Grafana via API, organized into folders
|
||||
--export Export dashboards from Grafana to clean JSON files
|
||||
--list Show available dashboards with install status
|
||||
--validate Check JSON quality and portability
|
||||
--status Compare installed vs local versions
|
||||
|
||||
${BOLD}FILTERS${RESET}
|
||||
--category NAME Only process dashboards in this category/folder
|
||||
--file PATH Only process this specific dashboard file
|
||||
--uid UID Only process the dashboard with this UID (for --export)
|
||||
|
||||
${BOLD}OPTIONS${RESET}
|
||||
-r, --remote NAME Incus remote (default: ${CLUSTER_REMOTE})
|
||||
-n, --dry-run Preview actions without executing
|
||||
-v, --verbose Show detailed output
|
||||
-q, --quiet Suppress informational output
|
||||
-h, --help Show this help message
|
||||
|
||||
${BOLD}DATASOURCE PORTABILITY${RESET}
|
||||
Dashboard JSON files use \${DS_PROMETHEUS} and \${DS_LOKI} variables
|
||||
instead of hardcoded datasource UIDs. The --install action resolves
|
||||
these to the actual UIDs from the target Grafana instance. The --export
|
||||
action normalizes UIDs back to variables.
|
||||
|
||||
${BOLD}DASHBOARD MANIFEST${RESET}
|
||||
${MANIFEST_FILE}
|
||||
YAML file listing each dashboard with file path, UID, title, folder,
|
||||
category, and version. Used by --install and --list.
|
||||
|
||||
${BOLD}EXAMPLES${RESET}
|
||||
# Validate all dashboard JSON files
|
||||
${SCRIPT_NAME} --validate
|
||||
|
||||
# Install all dashboards into Grafana
|
||||
${SCRIPT_NAME} --install
|
||||
|
||||
# Install only dashboards in the "core" category
|
||||
${SCRIPT_NAME} --install --category core
|
||||
|
||||
# Export a single dashboard by UID
|
||||
${SCRIPT_NAME} --export --uid incus-cluster-overview
|
||||
|
||||
# Compare installed versions with local files
|
||||
${SCRIPT_NAME} --status
|
||||
|
||||
# Preview what would be installed
|
||||
${SCRIPT_NAME} --install --dry-run
|
||||
EOF
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
grafana_api() {
|
||||
local method="$1" path="$2"
|
||||
shift 2
|
||||
incus exec "${CLUSTER_REMOTE}:${MONITORING_CONTAINER}" -- \
|
||||
curl -s -u "${GRAFANA_USER}:${GRAFANA_PASSWORD}" \
|
||||
-X "$method" "${GRAFANA_URL}${path}" "$@"
|
||||
}
|
||||
|
||||
# Resolve datasource UID variables to actual UIDs from Grafana
|
||||
resolve_datasource_uids() {
|
||||
local ds_json
|
||||
ds_json=$(grafana_api GET "/api/datasources")
|
||||
|
||||
PROMETHEUS_UID=$(echo "$ds_json" | python3 -c "
|
||||
import sys, json
|
||||
ds = json.load(sys.stdin)
|
||||
for d in ds:
|
||||
if d.get('type') == 'prometheus':
|
||||
print(d['uid'])
|
||||
break
|
||||
" 2>/dev/null || echo "")
|
||||
|
||||
LOKI_UID=$(echo "$ds_json" | python3 -c "
|
||||
import sys, json
|
||||
ds = json.load(sys.stdin)
|
||||
for d in ds:
|
||||
if d.get('type') == 'loki':
|
||||
print(d['uid'])
|
||||
break
|
||||
" 2>/dev/null || echo "")
|
||||
|
||||
if [[ -z "$PROMETHEUS_UID" ]]; then
|
||||
error "Could not find Prometheus datasource in Grafana"
|
||||
return 1
|
||||
fi
|
||||
detail "Prometheus datasource UID: ${PROMETHEUS_UID}"
|
||||
detail "Loki datasource UID: ${LOKI_UID:-not found}"
|
||||
}
|
||||
|
||||
# Get or create a Grafana folder, returns folder ID
|
||||
ensure_folder() {
|
||||
local folder_title="$1"
|
||||
|
||||
# Check if folder exists
|
||||
local folder_id
|
||||
folder_id=$(grafana_api GET "/api/folders" | python3 -c "
|
||||
import sys, json
|
||||
folders = json.load(sys.stdin)
|
||||
for f in folders:
|
||||
if f.get('title') == '${folder_title}':
|
||||
print(f['id'])
|
||||
break
|
||||
" 2>/dev/null || echo "")
|
||||
|
||||
if [[ -n "$folder_id" ]]; then
|
||||
detail "Folder '${folder_title}' exists (ID: ${folder_id})"
|
||||
echo "$folder_id"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Create folder
|
||||
local folder_uid
|
||||
folder_uid=$(echo "$folder_title" | tr '[:upper:]' '[:lower:]' | tr ' ' '-')
|
||||
local result
|
||||
result=$(grafana_api POST "/api/folders" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"title\": \"${folder_title}\", \"uid\": \"${folder_uid}\"}")
|
||||
|
||||
folder_id=$(echo "$result" | python3 -c "
|
||||
import sys, json
|
||||
print(json.load(sys.stdin).get('id', ''))
|
||||
" 2>/dev/null || echo "")
|
||||
|
||||
if [[ -n "$folder_id" ]]; then
|
||||
detail "Created folder '${folder_title}' (ID: ${folder_id})"
|
||||
echo "$folder_id"
|
||||
else
|
||||
error "Failed to create folder '${folder_title}': ${result}"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Read manifest and output entries as tab-separated lines
|
||||
read_manifest() {
|
||||
if [[ ! -f "$MANIFEST_FILE" ]]; then
|
||||
error "Dashboard manifest not found: ${MANIFEST_FILE}"
|
||||
error "Expected a YAML file listing dashboard metadata."
|
||||
return 1
|
||||
fi
|
||||
|
||||
python3 -c "
|
||||
import yaml, sys
|
||||
|
||||
with open('${MANIFEST_FILE}') as f:
|
||||
data = yaml.safe_load(f)
|
||||
|
||||
for d in data.get('dashboards', []):
|
||||
file_path = d.get('file', '')
|
||||
uid = d.get('uid', '')
|
||||
title = d.get('title', '')
|
||||
folder = d.get('folder', 'General')
|
||||
category = d.get('category', '')
|
||||
version = d.get('version', '1')
|
||||
description = d.get('description', '')
|
||||
print(f'{file_path}\t{uid}\t{title}\t{folder}\t{category}\t{version}\t{description}')
|
||||
" 2>/dev/null
|
||||
}
|
||||
|
||||
# Substitute datasource variables in JSON content
|
||||
inject_datasource_uids() {
|
||||
local json_content="$1"
|
||||
echo "$json_content" \
|
||||
| sed "s/\\\${DS_PROMETHEUS}/${PROMETHEUS_UID}/g" \
|
||||
| sed "s/\\\${DS_LOKI}/${LOKI_UID}/g"
|
||||
}
|
||||
|
||||
# Normalize datasource UIDs back to variables for portability
|
||||
normalize_datasource_uids() {
|
||||
local json_content="$1"
|
||||
local prom_uid="$2"
|
||||
local loki_uid="$3"
|
||||
|
||||
local result="$json_content"
|
||||
if [[ -n "$prom_uid" ]]; then
|
||||
result=$(echo "$result" | sed "s/${prom_uid}/\\\${DS_PROMETHEUS}/g")
|
||||
fi
|
||||
if [[ -n "$loki_uid" ]]; then
|
||||
result=$(echo "$result" | sed "s/${loki_uid}/\\\${DS_LOKI}/g")
|
||||
fi
|
||||
echo "$result"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Install action
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
action_install() {
|
||||
step "Installing dashboards to Grafana"
|
||||
echo
|
||||
|
||||
if ! dry_run_guard "Resolve datasource UIDs and push dashboards"; then
|
||||
info "(dry-run) Would install dashboards from ${DASHBOARDS_DIR}"
|
||||
|
||||
local count=0
|
||||
while IFS=$'\t' read -r file uid title folder category version description; do
|
||||
if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
|
||||
continue
|
||||
fi
|
||||
if [[ -n "$FILTER_FILE" && "$file" != "$FILTER_FILE" ]]; then
|
||||
continue
|
||||
fi
|
||||
info "(dry-run) Would install: ${title} -> folder '${folder}'"
|
||||
count=$((count + 1))
|
||||
done < <(read_manifest)
|
||||
|
||||
info "(dry-run) ${count} dashboard(s) would be installed"
|
||||
return 0
|
||||
fi
|
||||
|
||||
resolve_datasource_uids
|
||||
|
||||
local installed=0
|
||||
local failed=0
|
||||
|
||||
while IFS=$'\t' read -r file uid title folder category version description; do
|
||||
if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
|
||||
continue
|
||||
fi
|
||||
if [[ -n "$FILTER_FILE" && "$file" != "$FILTER_FILE" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
local full_path="${DASHBOARDS_DIR}/${file}"
|
||||
if [[ ! -f "$full_path" ]]; then
|
||||
warn "Dashboard file not found: ${full_path}"
|
||||
failed=$((failed + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Get or create the target folder
|
||||
local folder_id
|
||||
folder_id=$(ensure_folder "$folder") || { failed=$((failed + 1)); continue; }
|
||||
|
||||
# Read and process the dashboard JSON
|
||||
local raw_json
|
||||
raw_json=$(cat "$full_path")
|
||||
|
||||
# Inject actual datasource UIDs
|
||||
local processed_json
|
||||
processed_json=$(inject_datasource_uids "$raw_json")
|
||||
|
||||
# Wrap in Grafana API envelope
|
||||
local api_payload
|
||||
api_payload=$(python3 -c "
|
||||
import json, sys
|
||||
|
||||
dashboard = json.loads('''${processed_json//\'/\\\'}''')
|
||||
|
||||
# Remove Grafana-internal fields that shouldn't be in the import
|
||||
dashboard.pop('id', None)
|
||||
dashboard.pop('version', None)
|
||||
dashboard.pop('iteration', None)
|
||||
|
||||
payload = {
|
||||
'dashboard': dashboard,
|
||||
'folderId': ${folder_id},
|
||||
'overwrite': True,
|
||||
'message': 'Installed by manage-dashboards v${VERSION}'
|
||||
}
|
||||
print(json.dumps(payload))
|
||||
" 2>/dev/null)
|
||||
|
||||
if [[ -z "$api_payload" ]]; then
|
||||
# Fallback: use a simpler approach for large/complex JSON
|
||||
api_payload=$(python3 << 'PYEOF'
|
||||
import json, sys
|
||||
|
||||
with open(sys.argv[1]) as f:
|
||||
dashboard = json.load(f)
|
||||
|
||||
# Inject datasource UIDs
|
||||
json_str = json.dumps(dashboard)
|
||||
json_str = json_str.replace('${DS_PROMETHEUS}', sys.argv[2])
|
||||
json_str = json_str.replace('${DS_LOKI}', sys.argv[3])
|
||||
dashboard = json.loads(json_str)
|
||||
|
||||
# Remove Grafana-internal fields
|
||||
dashboard.pop('id', None)
|
||||
dashboard.pop('version', None)
|
||||
dashboard.pop('iteration', None)
|
||||
|
||||
payload = {
|
||||
'dashboard': dashboard,
|
||||
'folderId': int(sys.argv[4]),
|
||||
'overwrite': True,
|
||||
'message': 'Installed by manage-dashboards'
|
||||
}
|
||||
print(json.dumps(payload))
|
||||
PYEOF
|
||||
"$full_path" "$PROMETHEUS_UID" "${LOKI_UID:-}" "$folder_id" 2>/dev/null)
|
||||
fi
|
||||
|
||||
if [[ -z "$api_payload" ]]; then
|
||||
error "Failed to process dashboard JSON: ${file}"
|
||||
failed=$((failed + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Push via API - write payload to temp file to avoid command-line size limits
|
||||
local tmpfile
|
||||
tmpfile=$(mktemp /tmp/dashboard-payload.XXXXXX)
|
||||
echo "$api_payload" > "$tmpfile"
|
||||
|
||||
local result
|
||||
result=$(incus file push "$tmpfile" "${CLUSTER_REMOTE}:${MONITORING_CONTAINER}/tmp/dashboard-payload.json" 2>/dev/null && \
|
||||
grafana_api POST "/api/dashboards/db" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d @/tmp/dashboard-payload.json 2>/dev/null)
|
||||
rm -f "$tmpfile"
|
||||
|
||||
local status_str
|
||||
status_str=$(echo "$result" | python3 -c "
|
||||
import sys, json
|
||||
try:
|
||||
d = json.load(sys.stdin)
|
||||
print(d.get('status', d.get('message', 'unknown')))
|
||||
except:
|
||||
print('error')
|
||||
" 2>/dev/null || echo "error")
|
||||
|
||||
if [[ "$status_str" == "success" ]]; then
|
||||
success " ${title} -> ${folder}"
|
||||
installed=$((installed + 1))
|
||||
else
|
||||
error " ${title}: ${status_str}"
|
||||
detail " Response: ${result}"
|
||||
failed=$((failed + 1))
|
||||
fi
|
||||
|
||||
# Clean up temp file in container
|
||||
incus exec "${CLUSTER_REMOTE}:${MONITORING_CONTAINER}" -- rm -f /tmp/dashboard-payload.json 2>/dev/null || true
|
||||
|
||||
done < <(read_manifest)
|
||||
|
||||
echo
|
||||
if [[ $failed -eq 0 ]]; then
|
||||
success "Installed ${installed} dashboard(s)"
|
||||
else
|
||||
warn "Installed ${installed}, failed ${failed}"
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Export action
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
action_export() {
|
||||
step "Exporting dashboards from Grafana"
|
||||
echo
|
||||
|
||||
resolve_datasource_uids
|
||||
|
||||
local exported=0
|
||||
|
||||
if [[ -n "$FILTER_UID" ]]; then
|
||||
# Export a single dashboard by UID
|
||||
export_single_dashboard "$FILTER_UID"
|
||||
return $?
|
||||
fi
|
||||
|
||||
# Export all dashboards listed in the manifest
|
||||
while IFS=$'\t' read -r file uid title folder category version description; do
|
||||
if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
if export_single_dashboard "$uid" "${DASHBOARDS_DIR}/${file}"; then
|
||||
exported=$((exported + 1))
|
||||
fi
|
||||
done < <(read_manifest)
|
||||
|
||||
echo
|
||||
success "Exported ${exported} dashboard(s)"
|
||||
}
|
||||
|
||||
export_single_dashboard() {
|
||||
local uid="$1"
|
||||
local output_file="${2:-}"
|
||||
|
||||
# Fetch from Grafana API
|
||||
local raw_response
|
||||
raw_response=$(grafana_api GET "/api/dashboards/uid/${uid}" 2>/dev/null)
|
||||
|
||||
local has_dashboard
|
||||
has_dashboard=$(echo "$raw_response" | python3 -c "
|
||||
import sys, json
|
||||
try:
|
||||
d = json.load(sys.stdin)
|
||||
if 'dashboard' in d:
|
||||
print('yes')
|
||||
else:
|
||||
print('no')
|
||||
except:
|
||||
print('no')
|
||||
" 2>/dev/null || echo "no")
|
||||
|
||||
if [[ "$has_dashboard" != "yes" ]]; then
|
||||
warn "Dashboard '${uid}' not found in Grafana"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# If no output file specified, find it in the manifest
|
||||
if [[ -z "$output_file" ]]; then
|
||||
output_file=$(read_manifest | while IFS=$'\t' read -r file muid title folder category version description; do
|
||||
if [[ "$muid" == "$uid" ]]; then
|
||||
echo "${DASHBOARDS_DIR}/${file}"
|
||||
break
|
||||
fi
|
||||
done)
|
||||
fi
|
||||
|
||||
if [[ -z "$output_file" ]]; then
|
||||
output_file="${DASHBOARDS_DIR}/core/${uid}.json"
|
||||
warn "Dashboard '${uid}' not in manifest, exporting to ${output_file}"
|
||||
fi
|
||||
|
||||
if ! dry_run_guard "Export ${uid} to ${output_file}"; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Clean and normalize the dashboard JSON
|
||||
python3 << PYEOF > "$output_file"
|
||||
import json, sys
|
||||
|
||||
raw = json.loads('''$(echo "$raw_response" | sed "s/'/\\\\'/g")''')
|
||||
dashboard = raw['dashboard']
|
||||
|
||||
# Remove Grafana-internal fields
|
||||
for key in ['id', 'version', 'iteration']:
|
||||
dashboard.pop(key, None)
|
||||
|
||||
# Normalize datasource UIDs to variables
|
||||
json_str = json.dumps(dashboard, indent=2)
|
||||
prom_uid = "${PROMETHEUS_UID}"
|
||||
loki_uid = "${LOKI_UID:-}"
|
||||
if prom_uid:
|
||||
json_str = json_str.replace(prom_uid, r'\${DS_PROMETHEUS}')
|
||||
if loki_uid:
|
||||
json_str = json_str.replace(loki_uid, r'\${DS_LOKI}')
|
||||
|
||||
# Re-parse to ensure valid JSON
|
||||
dashboard = json.loads(json_str.replace(r'\${DS_PROMETHEUS}', '\${DS_PROMETHEUS}').replace(r'\${DS_LOKI}', '\${DS_LOKI}'))
|
||||
|
||||
# Write with consistent formatting
|
||||
json_str = json.dumps(dashboard, indent=2)
|
||||
# Restore the variable syntax that json.dumps may have escaped
|
||||
json_str = json_str.replace(r'\${DS_PROMETHEUS}', '\${DS_PROMETHEUS}')
|
||||
json_str = json_str.replace(r'\${DS_LOKI}', '\${DS_LOKI}')
|
||||
print(json_str)
|
||||
PYEOF
|
||||
|
||||
if [[ $? -eq 0 ]]; then
|
||||
success " Exported: ${uid} -> $(basename "$output_file")"
|
||||
return 0
|
||||
else
|
||||
error " Failed to export: ${uid}"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# List action
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
action_list() {
|
||||
step "Dashboard Inventory"
|
||||
echo
|
||||
|
||||
# Try to get installed dashboards from Grafana
|
||||
local installed_json=""
|
||||
installed_json=$(grafana_api GET "/api/search?type=dash-db" 2>/dev/null || echo "[]")
|
||||
|
||||
printf " ${BOLD}%-35s %-30s %-20s %s${RESET}\n" "UID" "Title" "Folder" "Status"
|
||||
printf " %-35s %-30s %-20s %s\n" "---" "-----" "------" "------"
|
||||
|
||||
while IFS=$'\t' read -r file uid title folder category version description; do
|
||||
if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
local full_path="${DASHBOARDS_DIR}/${file}"
|
||||
local local_status="missing"
|
||||
if [[ -f "$full_path" ]]; then
|
||||
local_status="local"
|
||||
fi
|
||||
|
||||
# Check if installed in Grafana
|
||||
local grafana_status
|
||||
grafana_status=$(echo "$installed_json" | python3 -c "
|
||||
import sys, json
|
||||
try:
|
||||
dashboards = json.load(sys.stdin)
|
||||
for d in dashboards:
|
||||
if d.get('uid') == '${uid}':
|
||||
print('installed')
|
||||
break
|
||||
else:
|
||||
print('not installed')
|
||||
except:
|
||||
print('unknown')
|
||||
" 2>/dev/null || echo "unknown")
|
||||
|
||||
local status_display
|
||||
if [[ "$local_status" == "missing" ]]; then
|
||||
status_display="${RED}file missing${RESET}"
|
||||
elif [[ "$grafana_status" == "installed" ]]; then
|
||||
status_display="${GREEN}installed${RESET}"
|
||||
elif [[ "$grafana_status" == "not installed" ]]; then
|
||||
status_display="${YELLOW}not installed${RESET}"
|
||||
else
|
||||
status_display="${DIM}${local_status}${RESET}"
|
||||
fi
|
||||
|
||||
printf " %-35s %-30s %-20s " "$uid" "$title" "$folder"
|
||||
echo -e "$status_display"
|
||||
|
||||
done < <(read_manifest)
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validate action
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
action_validate() {
|
||||
step "Validating dashboard JSON files"
|
||||
echo
|
||||
|
||||
local total=0
|
||||
local passed=0
|
||||
local warnings=0
|
||||
local errors=0
|
||||
|
||||
while IFS=$'\t' read -r file uid title folder category version description; do
|
||||
if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
local full_path="${DASHBOARDS_DIR}/${file}"
|
||||
total=$((total + 1))
|
||||
|
||||
if [[ ! -f "$full_path" ]]; then
|
||||
error " ${file}: FILE NOT FOUND"
|
||||
errors=$((errors + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
local issues=0
|
||||
local warns=0
|
||||
local basename_file
|
||||
basename_file=$(basename "$full_path")
|
||||
|
||||
# Check 1: Valid JSON
|
||||
if ! python3 -c "import json; json.load(open('${full_path}'))" 2>/dev/null; then
|
||||
error " ${basename_file}: INVALID JSON"
|
||||
errors=$((errors + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Run all checks via python
|
||||
local check_result
|
||||
check_result=$(python3 << PYEOF
|
||||
import json
|
||||
|
||||
with open('${full_path}') as f:
|
||||
d = json.load(f)
|
||||
|
||||
issues = []
|
||||
warns = []
|
||||
|
||||
# Check 2: UID present and matches manifest
|
||||
json_uid = d.get('uid', '')
|
||||
if not json_uid:
|
||||
issues.append('missing uid field')
|
||||
elif json_uid != '${uid}':
|
||||
issues.append(f'uid mismatch: JSON has "{json_uid}", manifest has "${uid}"')
|
||||
|
||||
# Check 3: Title present
|
||||
if not d.get('title'):
|
||||
issues.append('missing title field')
|
||||
|
||||
# Check 4: Description present
|
||||
if not d.get('description'):
|
||||
warns.append('missing description field')
|
||||
|
||||
# Check 5: No hardcoded datasource UIDs (should use variables)
|
||||
json_str = json.dumps(d)
|
||||
# Known Grafana internal UIDs that are OK
|
||||
safe_uids = ['-- Grafana --']
|
||||
# Check for UIDs that look like Grafana-generated ones
|
||||
import re
|
||||
# Match patterns like "uid": "PBFA97CFB590B2093" (hex-like, not a variable)
|
||||
uid_pattern = re.compile(r'"uid":\s*"([A-Za-z0-9]{16,})"')
|
||||
for match in uid_pattern.finditer(json_str):
|
||||
found_uid = match.group(1)
|
||||
if found_uid not in safe_uids and not found_uid.startswith('\${'):
|
||||
issues.append(f'hardcoded datasource UID: {found_uid}')
|
||||
break
|
||||
|
||||
# Check 6: Datasource variable usage
|
||||
if '\${DS_PROMETHEUS}' not in json_str and '\${DS_LOKI}' not in json_str:
|
||||
# Check if it has any datasource references at all
|
||||
if '"type": "prometheus"' in json_str or '"type": "loki"' in json_str:
|
||||
warns.append('no datasource variables found (uses hardcoded UIDs?)')
|
||||
|
||||
# Check 7: Has panels
|
||||
panels = d.get('panels', [])
|
||||
if not panels:
|
||||
warns.append('no panels defined')
|
||||
|
||||
# Check 8: Check for id field (should not be present in stored files)
|
||||
if 'id' in d and d['id'] is not None:
|
||||
warns.append('contains "id" field (should be null or absent for portability)')
|
||||
|
||||
for i in issues:
|
||||
print(f'ERROR:{i}')
|
||||
for w in warns:
|
||||
print(f'WARN:{w}')
|
||||
if not issues:
|
||||
print('OK')
|
||||
PYEOF
|
||||
)
|
||||
|
||||
local has_error=false
|
||||
while IFS= read -r line; do
|
||||
case "$line" in
|
||||
ERROR:*)
|
||||
error " ${basename_file}: ${line#ERROR:}"
|
||||
issues=$((issues + 1))
|
||||
has_error=true
|
||||
;;
|
||||
WARN:*)
|
||||
warn " ${basename_file}: ${line#WARN:}"
|
||||
warns=$((warns + 1))
|
||||
;;
|
||||
OK)
|
||||
;;
|
||||
esac
|
||||
done <<< "$check_result"
|
||||
|
||||
if [[ "$has_error" == true ]]; then
|
||||
errors=$((errors + 1))
|
||||
else
|
||||
if [[ $warns -gt 0 ]]; then
|
||||
warnings=$((warnings + warns))
|
||||
fi
|
||||
passed=$((passed + 1))
|
||||
success " ${basename_file}: valid"
|
||||
fi
|
||||
|
||||
done < <(read_manifest)
|
||||
|
||||
echo
|
||||
info "${total} dashboard(s) checked: ${passed} passed, ${errors} errors, ${warnings} warnings"
|
||||
|
||||
if [[ $errors -gt 0 ]]; then
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Status action
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
action_status() {
|
||||
step "Dashboard Status (local vs installed)"
|
||||
echo
|
||||
|
||||
local installed_json
|
||||
installed_json=$(grafana_api GET "/api/search?type=dash-db" 2>/dev/null || echo "[]")
|
||||
|
||||
printf " ${BOLD}%-30s %-15s %-15s %s${RESET}\n" "Dashboard" "Local" "Grafana" "Match"
|
||||
printf " %-30s %-15s %-15s %s\n" "---------" "-----" "-------" "-----"
|
||||
|
||||
while IFS=$'\t' read -r file uid title folder category version description; do
|
||||
local full_path="${DASHBOARDS_DIR}/${file}"
|
||||
|
||||
local local_ver="missing"
|
||||
if [[ -f "$full_path" ]]; then
|
||||
local_ver="v${version}"
|
||||
fi
|
||||
|
||||
local grafana_ver
|
||||
grafana_ver=$(echo "$installed_json" | python3 -c "
|
||||
import sys, json
|
||||
try:
|
||||
for d in json.load(sys.stdin):
|
||||
if d.get('uid') == '${uid}':
|
||||
print('installed')
|
||||
break
|
||||
else:
|
||||
print('absent')
|
||||
except:
|
||||
print('unknown')
|
||||
" 2>/dev/null || echo "unknown")
|
||||
|
||||
local match_str
|
||||
if [[ "$local_ver" == "missing" ]]; then
|
||||
match_str="${RED}file missing${RESET}"
|
||||
elif [[ "$grafana_ver" == "absent" ]]; then
|
||||
match_str="${YELLOW}not installed${RESET}"
|
||||
elif [[ "$grafana_ver" == "installed" ]]; then
|
||||
match_str="${GREEN}synced${RESET}"
|
||||
else
|
||||
match_str="${DIM}unknown${RESET}"
|
||||
fi
|
||||
|
||||
printf " %-30s %-15s %-15s " "$title" "$local_ver" "$grafana_ver"
|
||||
echo -e "$match_str"
|
||||
|
||||
done < <(read_manifest)
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Argument parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--install) ACTION="install";;
|
||||
--export) ACTION="export";;
|
||||
--list) ACTION="list";;
|
||||
--validate) ACTION="validate";;
|
||||
--status) ACTION="status";;
|
||||
--category) FILTER_CATEGORY="$2"; shift;;
|
||||
--file) FILTER_FILE="$2"; shift;;
|
||||
--uid) FILTER_UID="$2"; shift;;
|
||||
-r|--remote) CLUSTER_REMOTE="$2"; shift;;
|
||||
-n|--dry-run) DRY_RUN=true;;
|
||||
-v|--verbose) VERBOSE=true;;
|
||||
-q|--quiet) QUIET=true;;
|
||||
-h|--help) usage; exit 0;;
|
||||
*)
|
||||
error "Unknown option: $1"
|
||||
echo "Run '${SCRIPT_NAME} --help' for usage."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
if [[ -z "$ACTION" ]]; then
|
||||
error "No action specified"
|
||||
echo "Run '${SCRIPT_NAME} --help' for usage."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
main() {
|
||||
setup_colors
|
||||
parse_args "$@"
|
||||
|
||||
echo
|
||||
case "$ACTION" in
|
||||
install) action_install;;
|
||||
export) action_export;;
|
||||
list) action_list;;
|
||||
validate) action_validate;;
|
||||
status) action_status;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
|
@ -1,31 +1,35 @@
|
|||
{
|
||||
"uid": "incus-host-resources",
|
||||
"title": "Host Resources (IncusOS Nodes)",
|
||||
"description": "Physical host monitoring for IncusOS cluster nodes: CPU, memory, disk, and network from node_exporter metrics.",
|
||||
"tags": ["incusos", "host", "node-exporter"],
|
||||
"uid": "incus-capacity-planning",
|
||||
"title": "Capacity Planning",
|
||||
"description": "CPU/memory headroom per node, resource trends, disk space forecast with predict_linear, instance count trends.",
|
||||
"tags": ["incus", "capacity", "operations"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "30s",
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"refresh": "5m",
|
||||
"time": { "from": "now-7d", "to": "now" },
|
||||
"fiscalYearStartMonth": 0,
|
||||
"liveNow": false,
|
||||
"editable": true,
|
||||
"graphTooltip": 1,
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": true,
|
||||
"icon": "external link",
|
||||
"includeVars": true,
|
||||
"keepTime": true,
|
||||
"tags": ["incus"],
|
||||
"targetBlank": false,
|
||||
"title": "Incus Dashboards",
|
||||
"tooltip": "Navigate to other Incus dashboards",
|
||||
"type": "dashboards"
|
||||
}
|
||||
],
|
||||
"templating": { "list": [] },
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"datasource": { "type": "grafana", "uid": "-- Grafana --" },
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
|
|
@ -37,25 +41,114 @@
|
|||
"panels": [
|
||||
{
|
||||
"type": "row",
|
||||
"title": "CPU",
|
||||
"title": "Current Headroom",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"id": 1,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "CPU Usage % by Node",
|
||||
"description": "Overall CPU utilization per host node, calculated as 1 minus the average idle CPU rate.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 1 },
|
||||
"type": "bargauge",
|
||||
"title": "CPU Headroom per Node",
|
||||
"description": "Available (idle) CPU percentage per host node. Higher values mean more headroom for additional workloads.",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 1 },
|
||||
"id": 2,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval]))) * 100",
|
||||
"expr": "avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])) * 100",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "orange", "value": 20 },
|
||||
{ "color": "yellow", "value": 40 },
|
||||
{ "color": "green", "value": 60 }
|
||||
]
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "horizontal",
|
||||
"displayMode": "gradient",
|
||||
"showUnfilled": true,
|
||||
"minVizWidth": 8,
|
||||
"minVizHeight": 16,
|
||||
"valueMode": "color"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "bargauge",
|
||||
"title": "Memory Headroom per Node",
|
||||
"description": "Available memory percentage per host node. Higher values mean more headroom for additional workloads.",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 1 },
|
||||
"id": 3,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "orange", "value": 20 },
|
||||
{ "color": "yellow", "value": 40 },
|
||||
{ "color": "green", "value": 60 }
|
||||
]
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "horizontal",
|
||||
"displayMode": "gradient",
|
||||
"showUnfilled": true,
|
||||
"minVizWidth": 8,
|
||||
"minVizHeight": 16,
|
||||
"valueMode": "color"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Resource Trends",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 },
|
||||
"id": 4,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "CPU Usage Trend (7-day)",
|
||||
"description": "CPU utilization percentage per host node over the past 7 days. Helps identify sustained usage patterns and growth trends.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
||||
"id": 5,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
|
|
@ -75,7 +168,7 @@
|
|||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "line" }
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
|
|
@ -97,24 +190,13 @@
|
|||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Memory",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
|
||||
"id": 3,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Memory Usage % by Node",
|
||||
"description": "Memory utilization percentage per host node, calculated from available vs total memory.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 10 },
|
||||
"id": 4,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"title": "Memory Usage Trend (7-day)",
|
||||
"description": "Memory utilization percentage per host node over the past 7 days. Helps identify sustained usage patterns and growth trends.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
||||
"id": 6,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100",
|
||||
|
|
@ -137,7 +219,7 @@
|
|||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "line" }
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
|
|
@ -161,27 +243,29 @@
|
|||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Disk",
|
||||
"title": "Disk Space Forecast",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 },
|
||||
"id": 5,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 16 },
|
||||
"id": 7,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Root Filesystem Available Space",
|
||||
"description": "Available disk space on the root filesystem (mountpoint='/') per host node.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 },
|
||||
"id": 6,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"title": "Disk Space Forecast (30-day projection)",
|
||||
"description": "Current available disk space on root filesystem with a 30-day linear projection based on the past 7 days of data. Dashed lines show the forecast; if a forecast line drops below zero, the disk will fill up within 30 days at current consumption rates.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 17 },
|
||||
"id": 8,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "node_filesystem_avail_bytes{mountpoint=\"/\"}",
|
||||
"legendFormat": "{{instance}}",
|
||||
"legendFormat": "{{instance}} actual",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "predict_linear(node_filesystem_avail_bytes{mountpoint=\"/\"}[7d], 30*24*3600)",
|
||||
"legendFormat": "{{instance}} forecast",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
|
|
@ -189,7 +273,7 @@
|
|||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
|
|
@ -205,140 +289,43 @@
|
|||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byRegexp", "options": ".*forecast$" },
|
||||
"properties": [
|
||||
{ "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } },
|
||||
{ "id": "custom.fillOpacity", "value": 0 }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull", "min"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Disk Read Rate by Node",
|
||||
"description": "Disk read throughput (bytes/sec) aggregated per host node.",
|
||||
"gridPos": { "h": 8, "w": 6, "x": 12, "y": 19 },
|
||||
"id": 7,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (instance) (rate(node_disk_read_bytes_total[$__rate_interval]))",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Disk Write Rate by Node",
|
||||
"description": "Disk write throughput (bytes/sec) aggregated per host node.",
|
||||
"gridPos": { "h": 8, "w": 6, "x": 18, "y": 19 },
|
||||
"id": 8,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (instance) (rate(node_disk_written_bytes_total[$__rate_interval]))",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Network",
|
||||
"title": "Instance Trends",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 },
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 },
|
||||
"id": 9,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Network Receive Rate by Device & Node",
|
||||
"description": "Inbound network throughput per network device and host node.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
|
||||
"title": "Instance Count Over Time",
|
||||
"description": "Total number of running Incus instances over time. Tracks growth and churn in the cluster workload count.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 },
|
||||
"id": 10,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(node_network_receive_bytes_total{device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])",
|
||||
"legendFormat": "{{instance}} - {{device}} (rx)",
|
||||
"expr": "count(incus_uptime_seconds)",
|
||||
"legendFormat": "Instances",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
|
|
@ -347,8 +334,8 @@
|
|||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 25,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
|
|
@ -359,37 +346,37 @@
|
|||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"unit": "short",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Network Transmit Rate by Device & Node",
|
||||
"description": "Outbound network throughput per network device and host node.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
|
||||
"title": "Memory Allocation vs Usage per Instance",
|
||||
"description": "Compares total allocated memory (MemTotal) against actual resident memory (RSS) for each Incus instance. A large gap between allocated and used indicates overprovisioning; converging lines indicate an instance may need more memory.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 },
|
||||
"id": 11,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])",
|
||||
"legendFormat": "{{instance}} - {{device}} (tx)",
|
||||
"expr": "incus_memory_MemTotal_bytes",
|
||||
"legendFormat": "{{name}} allocated",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "incus_memory_RSS_bytes",
|
||||
"legendFormat": "{{name}} used (RSS)",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
|
|
@ -398,7 +385,7 @@
|
|||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
|
|
@ -409,20 +396,32 @@
|
|||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"unit": "bytes",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byRegexp", "options": ".*allocated$" },
|
||||
"properties": [
|
||||
{ "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } },
|
||||
{ "id": "custom.lineWidth", "value": 2 }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byRegexp", "options": ".*used \\(RSS\\)$" },
|
||||
"properties": [
|
||||
{ "id": "custom.fillOpacity", "value": 15 }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,674 @@
|
|||
{
|
||||
"uid": "incus-cluster-overview",
|
||||
"title": "Incus Cluster Overview",
|
||||
"description": "Overview of all Incus instances: CPU, memory, network, disk I/O, and instance status table with drill-down links.",
|
||||
"tags": ["incus", "cluster", "containers"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"fiscalYearStartMonth": 0,
|
||||
"liveNow": false,
|
||||
"editable": true,
|
||||
"graphTooltip": 1,
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": true,
|
||||
"icon": "external link",
|
||||
"includeVars": true,
|
||||
"keepTime": true,
|
||||
"tags": ["incus"],
|
||||
"targetBlank": false,
|
||||
"title": "Incus Dashboards",
|
||||
"tooltip": "Navigate to other Incus dashboards",
|
||||
"type": "dashboards"
|
||||
}
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"definition": "label_values(incus_uptime_seconds, name)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"name": "instance",
|
||||
"query": "label_values(incus_uptime_seconds, name)",
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query",
|
||||
"label": "Instance"
|
||||
},
|
||||
{
|
||||
"current": {},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"definition": "label_values(incus_uptime_seconds, instance)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"name": "node",
|
||||
"query": "label_values(incus_uptime_seconds, instance)",
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query",
|
||||
"label": "Node"
|
||||
}
|
||||
]
|
||||
},
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": { "type": "grafana", "uid": "-- Grafana --" },
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Cluster Status",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"id": 1,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Total Instances",
|
||||
"description": "Number of running Incus instances (reporting uptime).",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
|
||||
"id": 2,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "count(incus_uptime_seconds{instance=~\"$node\", name=~\"$instance\"})",
|
||||
"legendFormat": "Instances",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 10 },
|
||||
{ "color": "red", "value": 50 }
|
||||
]
|
||||
},
|
||||
"unit": "none",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Total CPUs",
|
||||
"description": "Sum of effective CPUs across all Incus instances.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
|
||||
"id": 3,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(incus_cpu_effective_total{instance=~\"$node\", name=~\"$instance\"})",
|
||||
"legendFormat": "CPUs",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
},
|
||||
"unit": "none",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Total Memory",
|
||||
"description": "Sum of total memory across all Incus instances.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
|
||||
"id": 4,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(incus_memory_MemTotal_bytes{instance=~\"$node\", name=~\"$instance\"})",
|
||||
"legendFormat": "Total Memory",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
},
|
||||
"unit": "bytes",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "gauge",
|
||||
"title": "Cluster Memory Utilization",
|
||||
"description": "Overall cluster memory utilization (RSS / Total) across all instances.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
|
||||
"id": 40,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(sum(incus_memory_RSS_bytes{instance=~\"$node\", name=~\"$instance\"}) / sum(incus_memory_MemTotal_bytes{instance=~\"$node\", name=~\"$instance\"})) * 100",
|
||||
"legendFormat": "Utilization",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 60 },
|
||||
{ "color": "orange", "value": 80 },
|
||||
{ "color": "red", "value": 90 }
|
||||
]
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true,
|
||||
"orientation": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Instance Status",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
|
||||
"id": 50,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "table",
|
||||
"title": "Instance Status Table",
|
||||
"description": "Per-instance summary: uptime, CPU usage, memory RSS, total memory. Click instance name to drill down.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 },
|
||||
"id": 51,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "incus_uptime_seconds{instance=~\"$node\", name=~\"$instance\"}",
|
||||
"legendFormat": "",
|
||||
"refId": "A",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
},
|
||||
{
|
||||
"expr": "sum by (name, instance) (rate(incus_cpu_seconds_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))",
|
||||
"legendFormat": "",
|
||||
"refId": "B",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
},
|
||||
{
|
||||
"expr": "incus_memory_RSS_bytes{instance=~\"$node\", name=~\"$instance\"}",
|
||||
"legendFormat": "",
|
||||
"refId": "C",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
},
|
||||
{
|
||||
"expr": "incus_memory_MemTotal_bytes{instance=~\"$node\", name=~\"$instance\"}",
|
||||
"legendFormat": "",
|
||||
"refId": "D",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"align": "auto",
|
||||
"displayMode": "auto",
|
||||
"inspect": false,
|
||||
"filterable": true
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
},
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "name" },
|
||||
"properties": [
|
||||
{ "id": "displayName", "value": "Instance" },
|
||||
{ "id": "custom.width", "value": 200 },
|
||||
{
|
||||
"id": "links",
|
||||
"value": [
|
||||
{
|
||||
"title": "View instance details",
|
||||
"url": "/d/incus-instance-deep-dive?var-instance=${__value.text}&${__url_time_range}",
|
||||
"targetBlank": false
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "instance" },
|
||||
"properties": [{ "id": "displayName", "value": "Node" }]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Value #A" },
|
||||
"properties": [
|
||||
{ "id": "displayName", "value": "Uptime" },
|
||||
{ "id": "unit", "value": "s" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Value #B" },
|
||||
"properties": [
|
||||
{ "id": "displayName", "value": "CPU/s" },
|
||||
{ "id": "unit", "value": "short" },
|
||||
{ "id": "decimals", "value": 3 }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Value #C" },
|
||||
"properties": [
|
||||
{ "id": "displayName", "value": "Memory RSS" },
|
||||
{ "id": "unit", "value": "bytes" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Value #D" },
|
||||
"properties": [
|
||||
{ "id": "displayName", "value": "Total Memory" },
|
||||
{ "id": "unit", "value": "bytes" }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"sortBy": [{ "displayName": "Instance", "desc": false }],
|
||||
"footer": { "show": false, "reducer": ["sum"], "fields": "" }
|
||||
},
|
||||
"transformations": [
|
||||
{ "id": "merge", "options": {} },
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": { "Time": true, "__name__": true, "job": true },
|
||||
"indexByName": {
|
||||
"name": 0, "instance": 1,
|
||||
"Value #A": 2, "Value #B": 3, "Value #C": 4, "Value #D": 5
|
||||
},
|
||||
"renameByName": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Per-Instance CPU",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
|
||||
"id": 5,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "CPU Usage by Instance",
|
||||
"description": "Per-second CPU time consumed by each Incus instance.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 15 },
|
||||
"id": 6,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (name) (rate(incus_cpu_seconds_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "short",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
},
|
||||
"links": [
|
||||
{
|
||||
"title": "Drill down to ${__field.labels.name}",
|
||||
"url": "/d/incus-instance-deep-dive?var-instance=${__field.labels.name}&${__url_time_range}",
|
||||
"targetBlank": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Per-Instance Memory",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
|
||||
"id": 7,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Memory RSS by Instance",
|
||||
"description": "Resident set size (RSS) memory usage per Incus instance over time.",
|
||||
"gridPos": { "h": 8, "w": 14, "x": 0, "y": 24 },
|
||||
"id": 8,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "incus_memory_RSS_bytes{instance=~\"$node\", name=~\"$instance\"}",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "bytes",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "bargauge",
|
||||
"title": "Memory Usage % (RSS / Total)",
|
||||
"description": "Current memory utilization per instance as RSS divided by total memory.",
|
||||
"gridPos": { "h": 8, "w": 10, "x": 14, "y": 24 },
|
||||
"id": 9,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(incus_memory_RSS_bytes{instance=~\"$node\", name=~\"$instance\"} / incus_memory_MemTotal_bytes{instance=~\"$node\", name=~\"$instance\"}) * 100",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 60 },
|
||||
{ "color": "orange", "value": 80 },
|
||||
{ "color": "red", "value": 90 }
|
||||
]
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "horizontal",
|
||||
"displayMode": "gradient",
|
||||
"showUnfilled": true,
|
||||
"minVizWidth": 8,
|
||||
"minVizHeight": 16,
|
||||
"valueMode": "color"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Network I/O",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 },
|
||||
"id": 10,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Network Receive Rate by Instance",
|
||||
"description": "Inbound network throughput per Incus instance.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 },
|
||||
"id": 11,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (name) (rate(incus_network_receive_bytes_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never", "pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Network Transmit Rate by Instance",
|
||||
"description": "Outbound network throughput per Incus instance.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 },
|
||||
"id": 12,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (name) (rate(incus_network_transmit_bytes_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never", "pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Disk I/O",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 },
|
||||
"id": 13,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Disk Read Rate by Instance",
|
||||
"description": "Disk read throughput per Incus instance.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 42 },
|
||||
"id": 14,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (name) (rate(incus_disk_read_bytes_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never", "pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Disk Write Rate by Instance",
|
||||
"description": "Disk write throughput per Incus instance.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 42 },
|
||||
"id": 15,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (name) (rate(incus_disk_written_bytes_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never", "pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,910 @@
|
|||
{
|
||||
"uid": "incus-haproxy-traffic",
|
||||
"title": "HAProxy Traffic",
|
||||
"description": "HAProxy load balancer monitoring: request rates, response codes, backend health, traffic volume, sessions, and performance metrics.",
|
||||
"tags": ["incus", "haproxy", "loadbalancer"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 2,
|
||||
"refresh": "30s",
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"fiscalYearStartMonth": 0,
|
||||
"liveNow": false,
|
||||
"editable": true,
|
||||
"graphTooltip": 1,
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": true,
|
||||
"icon": "external link",
|
||||
"includeVars": true,
|
||||
"keepTime": true,
|
||||
"tags": ["incus"],
|
||||
"targetBlank": false,
|
||||
"title": "Incus Dashboards",
|
||||
"tooltip": "Navigate to other Incus dashboards",
|
||||
"type": "dashboards"
|
||||
}
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"definition": "label_values(haproxy_backend_status, proxy)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"name": "proxy",
|
||||
"query": "label_values(haproxy_backend_status, proxy)",
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query",
|
||||
"label": "Proxy"
|
||||
},
|
||||
{
|
||||
"current": {},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"definition": "label_values(haproxy_server_status, server)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"name": "server",
|
||||
"query": "label_values(haproxy_server_status, server)",
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query",
|
||||
"label": "Server"
|
||||
}
|
||||
]
|
||||
},
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Overview",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"id": 1,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Active Backend Servers",
|
||||
"description": "Total number of active (healthy) servers across selected HAProxy backends.",
|
||||
"gridPos": { "h": 4, "w": 5, "x": 0, "y": 1 },
|
||||
"id": 2,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(haproxy_backend_active_servers{proxy=~\"$proxy\"})",
|
||||
"legendFormat": "Active Servers",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "yellow", "value": 1 },
|
||||
{ "color": "green", "value": 2 }
|
||||
]
|
||||
},
|
||||
"unit": "none",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Current Sessions",
|
||||
"description": "Total current active sessions across selected HAProxy backends.",
|
||||
"gridPos": { "h": 4, "w": 5, "x": 5, "y": 1 },
|
||||
"id": 3,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(haproxy_backend_current_sessions{proxy=~\"$proxy\"})",
|
||||
"legendFormat": "Sessions",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 100 },
|
||||
{ "color": "red", "value": 500 }
|
||||
]
|
||||
},
|
||||
"unit": "none",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "HTTP Requests/s",
|
||||
"description": "Current rate of HTTP requests per second across selected HAProxy frontends.",
|
||||
"gridPos": { "h": 4, "w": 5, "x": 10, "y": 1 },
|
||||
"id": 4,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(haproxy_frontend_http_requests_total{proxy=~\"$proxy\"}[$__rate_interval]))",
|
||||
"legendFormat": "Requests/s",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
},
|
||||
"unit": "reqps",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "HTTP 4xx Rate",
|
||||
"description": "Rate of HTTP 4xx client error responses per second. Elevated rates may indicate bad requests or missing resources.",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 15, "y": 1 },
|
||||
"id": 14,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(haproxy_frontend_http_responses_total{proxy=~\"$proxy\", code=\"4xx\"}[$__rate_interval]))",
|
||||
"legendFormat": "4xx/s",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 0.1 },
|
||||
{ "color": "red", "value": 1 }
|
||||
]
|
||||
},
|
||||
"unit": "reqps",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "HTTP 5xx Rate",
|
||||
"description": "Rate of HTTP 5xx server error responses per second. Any sustained rate indicates backend failures requiring investigation.",
|
||||
"gridPos": { "h": 4, "w": 5, "x": 19, "y": 1 },
|
||||
"id": 15,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(haproxy_frontend_http_responses_total{proxy=~\"$proxy\", code=\"5xx\"}[$__rate_interval]))",
|
||||
"legendFormat": "5xx/s",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 0.01 },
|
||||
{ "color": "red", "value": 0.1 }
|
||||
]
|
||||
},
|
||||
"unit": "reqps",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "HTTP Requests",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
|
||||
"id": 5,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "HTTP Request Rate by Frontend",
|
||||
"description": "Rate of HTTP requests per second, broken down by HAProxy frontend/proxy. Useful for identifying which frontends carry the most traffic.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
|
||||
"id": 6,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (proxy) (rate(haproxy_frontend_http_requests_total{proxy=~\"$proxy\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{proxy}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "reqps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Response Code Distribution",
|
||||
"description": "HTTP response code distribution across selected frontends, stacked by response class (2xx, 3xx, 4xx, 5xx). Helps identify error trends and traffic composition.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
|
||||
"id": 16,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (code) (rate(haproxy_frontend_http_responses_total{proxy=~\"$proxy\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{code}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 50,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "normal", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "reqps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "2xx" },
|
||||
"properties": [
|
||||
{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "3xx" },
|
||||
"properties": [
|
||||
{ "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "4xx" },
|
||||
"properties": [
|
||||
{ "id": "color", "value": { "mode": "fixed", "fixedColor": "yellow" } }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "5xx" },
|
||||
"properties": [
|
||||
{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Backend Health",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
|
||||
"id": 7,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "table",
|
||||
"title": "Backend Status",
|
||||
"description": "Current status, active server count, and average response time for each HAProxy backend. Filterable by proxy and server template variables.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 },
|
||||
"id": 8,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "haproxy_backend_status{proxy=~\"$proxy\"}",
|
||||
"legendFormat": "",
|
||||
"refId": "A",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
},
|
||||
{
|
||||
"expr": "haproxy_backend_active_servers{proxy=~\"$proxy\"}",
|
||||
"legendFormat": "",
|
||||
"refId": "B",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
},
|
||||
{
|
||||
"expr": "haproxy_server_response_time_average_seconds{proxy=~\"$proxy\", server=~\"$server\"}",
|
||||
"legendFormat": "",
|
||||
"refId": "C",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"align": "auto",
|
||||
"displayMode": "auto",
|
||||
"inspect": false,
|
||||
"filterable": true
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
},
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Value #A" },
|
||||
"properties": [
|
||||
{ "id": "displayName", "value": "Status" },
|
||||
{
|
||||
"id": "mappings",
|
||||
"value": [
|
||||
{ "type": "value", "options": { "1": { "text": "UP", "color": "green" }, "0": { "text": "DOWN", "color": "red" } } }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Value #B" },
|
||||
"properties": [
|
||||
{ "id": "displayName", "value": "Active Servers" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Value #C" },
|
||||
"properties": [
|
||||
{ "id": "displayName", "value": "Avg Response Time" },
|
||||
{ "id": "unit", "value": "s" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "proxy" },
|
||||
"properties": [
|
||||
{ "id": "displayName", "value": "Backend" }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"sortBy": [],
|
||||
"footer": {
|
||||
"show": false,
|
||||
"reducer": ["sum"],
|
||||
"fields": ""
|
||||
}
|
||||
},
|
||||
"transformations": [
|
||||
{
|
||||
"id": "merge",
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {
|
||||
"Time": true,
|
||||
"__name__": true,
|
||||
"instance": false,
|
||||
"job": true
|
||||
},
|
||||
"indexByName": {},
|
||||
"renameByName": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "state-timeline",
|
||||
"title": "Server Status",
|
||||
"description": "Timeline view of individual HAProxy server health status. Green indicates UP, red indicates DOWN. Helps identify flapping servers and outage windows.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 },
|
||||
"id": 17,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "haproxy_server_status{proxy=~\"$proxy\", server=~\"$server\"}",
|
||||
"legendFormat": "{{proxy}} / {{server}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"lineWidth": 0,
|
||||
"fillOpacity": 80
|
||||
},
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"0": { "text": "DOWN", "color": "red" },
|
||||
"1": { "text": "UP", "color": "green" }
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"showValue": "auto",
|
||||
"mergeValues": true,
|
||||
"alignValue": "left",
|
||||
"rowHeight": 0.9,
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Traffic",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
|
||||
"id": 9,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Frontend Bytes In",
|
||||
"description": "Inbound traffic rate per HAProxy frontend. Shows how much data clients are sending to each frontend.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
|
||||
"id": 10,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (proxy) (rate(haproxy_frontend_bytes_in_total{proxy=~\"$proxy\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{proxy}} (in)",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Frontend Bytes Out",
|
||||
"description": "Outbound traffic rate per HAProxy frontend. Shows how much data is being served to clients from each frontend.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
|
||||
"id": 11,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (proxy) (rate(haproxy_frontend_bytes_out_total{proxy=~\"$proxy\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{proxy}} (out)",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Sessions & Performance",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 },
|
||||
"id": 12,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Current Sessions by Proxy",
|
||||
"description": "Current active sessions for each HAProxy backend/proxy over time. Sustained high values may indicate slow backends or session buildup.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 },
|
||||
"id": 13,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "haproxy_backend_current_sessions{proxy=~\"$proxy\"}",
|
||||
"legendFormat": "{{proxy}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "none",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Backend Response Time",
|
||||
"description": "Average response time from backend servers. High or increasing values indicate backend performance degradation.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 },
|
||||
"id": 18,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "haproxy_server_response_time_average_seconds{proxy=~\"$proxy\", server=~\"$server\"}",
|
||||
"legendFormat": "{{proxy}} / {{server}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "line" }
|
||||
},
|
||||
"unit": "s",
|
||||
"color": { "mode": "continuous-GrYlRd" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 0.5 },
|
||||
{ "color": "red", "value": 2 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Queue Time",
|
||||
"description": "Average time requests spend in the backend queue before being assigned to a server. High values indicate insufficient backend capacity.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 41 },
|
||||
"id": 19,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "haproxy_backend_queue_time_average_seconds{proxy=~\"$proxy\"}",
|
||||
"legendFormat": "{{proxy}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "line" }
|
||||
},
|
||||
"unit": "s",
|
||||
"color": { "mode": "continuous-GrYlRd" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 0.1 },
|
||||
{ "color": "red", "value": 1 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Connection Errors",
|
||||
"description": "Rate of backend connection errors and connection retries per proxy. Non-zero values indicate connectivity problems between HAProxy and backend servers.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 41 },
|
||||
"id": 20,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (proxy) (rate(haproxy_backend_connection_errors_total{proxy=~\"$proxy\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{proxy}} errors",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (proxy) (rate(haproxy_backend_retry_warnings_total{proxy=~\"$proxy\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{proxy}} retries",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "reqps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byRegexp", "options": ".*errors$" },
|
||||
"properties": [
|
||||
{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byRegexp", "options": ".*retries$" },
|
||||
"properties": [
|
||||
{ "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,584 @@
|
|||
{
|
||||
"uid": "incus-host-resources",
|
||||
"title": "Host Resources (IncusOS Nodes)",
|
||||
"description": "Physical host monitoring: CPU mode breakdown, load averages, memory breakdown, filesystem usage, disk IOPS, network errors.",
|
||||
"tags": ["incus", "incusos", "host", "node-exporter"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"fiscalYearStartMonth": 0,
|
||||
"liveNow": false,
|
||||
"editable": true,
|
||||
"graphTooltip": 1,
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": true, "icon": "external link", "includeVars": true,
|
||||
"keepTime": true, "tags": ["incus"], "targetBlank": false,
|
||||
"title": "Incus Dashboards", "type": "dashboards"
|
||||
}
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"definition": "label_values(node_uname_info, instance)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"name": "node",
|
||||
"query": "label_values(node_uname_info, instance)",
|
||||
"refresh": 2,
|
||||
"sort": 1,
|
||||
"type": "query",
|
||||
"label": "Node"
|
||||
}
|
||||
]
|
||||
},
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": { "type": "grafana", "uid": "-- Grafana --" },
|
||||
"enable": true, "hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts", "type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "row", "title": "System Info",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"id": 30, "panels": []
|
||||
},
|
||||
{
|
||||
"type": "table",
|
||||
"title": "System Information",
|
||||
"description": "Kernel version, uptime, total RAM, and CPU count per node.",
|
||||
"gridPos": { "h": 5, "w": 24, "x": 0, "y": 1 },
|
||||
"id": 31,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "node_uname_info{instance=~\"$node\"}",
|
||||
"legendFormat": "", "refId": "A", "format": "table", "instant": true
|
||||
},
|
||||
{
|
||||
"expr": "node_boot_time_seconds{instance=~\"$node\"}",
|
||||
"legendFormat": "", "refId": "B", "format": "table", "instant": true
|
||||
},
|
||||
{
|
||||
"expr": "node_memory_MemTotal_bytes{instance=~\"$node\"}",
|
||||
"legendFormat": "", "refId": "C", "format": "table", "instant": true
|
||||
},
|
||||
{
|
||||
"expr": "count by (instance) (node_cpu_seconds_total{instance=~\"$node\", mode=\"idle\"})",
|
||||
"legendFormat": "", "refId": "D", "format": "table", "instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": { "align": "auto", "displayMode": "auto", "inspect": false, "filterable": true },
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byName", "options": "instance" }, "properties": [{ "id": "displayName", "value": "Node" }] },
|
||||
{ "matcher": { "id": "byName", "options": "release" }, "properties": [{ "id": "displayName", "value": "Kernel" }] },
|
||||
{ "matcher": { "id": "byName", "options": "Value #B" }, "properties": [{ "id": "displayName", "value": "Boot Time" }, { "id": "unit", "value": "dateTimeFromNow" }] },
|
||||
{ "matcher": { "id": "byName", "options": "Value #C" }, "properties": [{ "id": "displayName", "value": "Total RAM" }, { "id": "unit", "value": "bytes" }] },
|
||||
{ "matcher": { "id": "byName", "options": "Value #D" }, "properties": [{ "id": "displayName", "value": "CPUs" }] }
|
||||
]
|
||||
},
|
||||
"options": { "showHeader": true, "sortBy": [], "footer": { "show": false, "reducer": ["sum"], "fields": "" } },
|
||||
"transformations": [
|
||||
{ "id": "merge", "options": {} },
|
||||
{ "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "job": true, "domainname": true, "machine": true, "nodename": true, "sysname": true, "version": true, "Value #A": true }, "renameByName": {} } }
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "row", "title": "CPU",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 },
|
||||
"id": 1, "panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "CPU Usage % by Node",
|
||||
"description": "Overall CPU utilization per host node.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 },
|
||||
"id": 2,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$node\"}[$__rate_interval]))) * 100",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2,
|
||||
"fillOpacity": 25, "gradientMode": "scheme", "spanNulls": false,
|
||||
"showPoints": "never", "pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "line" }
|
||||
},
|
||||
"unit": "percent", "min": 0, "max": 100,
|
||||
"color": { "mode": "continuous-GrYlRd" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "CPU Mode Breakdown",
|
||||
"description": "CPU time by mode (user, system, iowait, idle) stacked per node.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 },
|
||||
"id": 32,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{ "expr": "avg by (instance, mode) (rate(node_cpu_seconds_total{instance=~\"$node\", mode=~\"user|system|iowait|idle\"}[$__rate_interval])) * 100", "legendFormat": "{{instance}} {{mode}}", "refId": "A" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 50, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "normal", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "percent", "min": 0,
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Load Average",
|
||||
"description": "System load averages (1m, 5m, 15m) with CPU count threshold line.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 15 },
|
||||
"id": 33,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{ "expr": "node_load1{instance=~\"$node\"}", "legendFormat": "{{instance}} load1", "refId": "A" },
|
||||
{ "expr": "node_load5{instance=~\"$node\"}", "legendFormat": "{{instance}} load5", "refId": "B" },
|
||||
{ "expr": "node_load15{instance=~\"$node\"}", "legendFormat": "{{instance}} load15", "refId": "C" },
|
||||
{ "expr": "count by (instance) (node_cpu_seconds_total{instance=~\"$node\", mode=\"idle\"})", "legendFormat": "{{instance}} CPUs", "refId": "D" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 0, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "short",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byRegexp", "options": ".*CPUs$" },
|
||||
"properties": [
|
||||
{ "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } },
|
||||
{ "id": "custom.lineWidth", "value": 2 },
|
||||
{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row", "title": "Memory",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
|
||||
"id": 3, "panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Memory Usage % by Node",
|
||||
"description": "Memory utilization percentage per host node.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
|
||||
"id": 4,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - node_memory_MemAvailable_bytes{instance=~\"$node\"} / node_memory_MemTotal_bytes{instance=~\"$node\"}) * 100",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2,
|
||||
"fillOpacity": 25, "gradientMode": "scheme", "spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "line" }
|
||||
},
|
||||
"unit": "percent", "min": 0, "max": 100,
|
||||
"color": { "mode": "continuous-GrYlRd" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Memory Breakdown by Node",
|
||||
"description": "Memory breakdown: used, buffers, cached, available (stacked).",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
|
||||
"id": 34,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{ "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"} - node_memory_MemAvailable_bytes{instance=~\"$node\"} - node_memory_Buffers_bytes{instance=~\"$node\"} - node_memory_Cached_bytes{instance=~\"$node\"}", "legendFormat": "{{instance}} used", "refId": "A" },
|
||||
{ "expr": "node_memory_Buffers_bytes{instance=~\"$node\"}", "legendFormat": "{{instance}} buffers", "refId": "B" },
|
||||
{ "expr": "node_memory_Cached_bytes{instance=~\"$node\"}", "legendFormat": "{{instance}} cached", "refId": "C" },
|
||||
{ "expr": "node_memory_MemAvailable_bytes{instance=~\"$node\"}", "legendFormat": "{{instance}} available", "refId": "D" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 50, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "normal", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "bytes",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row", "title": "Disk",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 },
|
||||
"id": 5, "panels": []
|
||||
},
|
||||
{
|
||||
"type": "bargauge",
|
||||
"title": "Filesystem Usage %",
|
||||
"description": "Current filesystem usage percentage for important mount points.",
|
||||
"gridPos": { "h": 6, "w": 8, "x": 0, "y": 33 },
|
||||
"id": 35,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - node_filesystem_avail_bytes{instance=~\"$node\", fstype!~\"tmpfs|overlay\", mountpoint=~\"/.*\"} / node_filesystem_size_bytes{instance=~\"$node\", fstype!~\"tmpfs|overlay\", mountpoint=~\"/.*\"}) * 100",
|
||||
"legendFormat": "{{instance}} {{mountpoint}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "orange", "value": 85 }, { "color": "red", "value": 95 }] },
|
||||
"unit": "percent", "min": 0, "max": 100,
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "horizontal", "displayMode": "gradient", "showUnfilled": true,
|
||||
"minVizWidth": 8, "minVizHeight": 16, "valueMode": "color"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Root Filesystem Available Space",
|
||||
"description": "Available disk space on the root filesystem per host node.",
|
||||
"gridPos": { "h": 6, "w": 8, "x": 8, "y": 33 },
|
||||
"id": 6,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{ "expr": "node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=\"/\"}", "legendFormat": "{{instance}}", "refId": "A" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "bytes",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull", "min"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Disk IOPS by Node",
|
||||
"description": "Disk read and write operations per second by node.",
|
||||
"gridPos": { "h": 6, "w": 8, "x": 16, "y": 33 },
|
||||
"id": 36,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{ "expr": "sum by (instance) (rate(node_disk_reads_completed_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} reads", "refId": "A" },
|
||||
{ "expr": "sum by (instance) (rate(node_disk_writes_completed_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} writes", "refId": "B" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "iops",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Disk Read Rate by Node",
|
||||
"description": "Disk read throughput aggregated per host node.",
|
||||
"gridPos": { "h": 6, "w": 8, "x": 0, "y": 39 },
|
||||
"id": 7,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{ "expr": "sum by (instance) (rate(node_disk_read_bytes_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}}", "refId": "A" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Disk Write Rate by Node",
|
||||
"description": "Disk write throughput aggregated per host node.",
|
||||
"gridPos": { "h": 6, "w": 8, "x": 8, "y": 39 },
|
||||
"id": 8,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{ "expr": "sum by (instance) (rate(node_disk_written_bytes_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}}", "refId": "A" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Disk I/O Time",
|
||||
"description": "Time spent on disk I/O operations (disk utilization indicator).",
|
||||
"gridPos": { "h": 6, "w": 8, "x": 16, "y": 39 },
|
||||
"id": 37,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{ "expr": "sum by (instance) (rate(node_disk_io_time_seconds_total{instance=~\"$node\"}[$__rate_interval])) * 100", "legendFormat": "{{instance}}", "refId": "A" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "scheme", "spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "line" }
|
||||
},
|
||||
"unit": "percent", "min": 0, "max": 100,
|
||||
"color": { "mode": "continuous-GrYlRd" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 80 }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row", "title": "Network",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 45 },
|
||||
"id": 9, "panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Network Receive Rate by Device & Node",
|
||||
"description": "Inbound network throughput per network device and host node.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 46 },
|
||||
"id": 10,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{ "expr": "rate(node_network_receive_bytes_total{instance=~\"$node\", device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])", "legendFormat": "{{instance}} {{device}} rx", "refId": "A" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Network Transmit Rate by Device & Node",
|
||||
"description": "Outbound network throughput per network device and host node.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 46 },
|
||||
"id": 11,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{ "expr": "rate(node_network_transmit_bytes_total{instance=~\"$node\", device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])", "legendFormat": "{{instance}} {{device}} tx", "refId": "A" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Network Errors & Drops",
|
||||
"description": "Network receive/transmit errors and dropped packets per node.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 54 },
|
||||
"id": 38,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{ "expr": "sum by (instance) (rate(node_network_receive_errs_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} rx errors", "refId": "A" },
|
||||
{ "expr": "sum by (instance) (rate(node_network_transmit_errs_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} tx errors", "refId": "B" },
|
||||
{ "expr": "sum by (instance) (rate(node_network_receive_drop_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} rx drops", "refId": "C" },
|
||||
{ "expr": "sum by (instance) (rate(node_network_transmit_drop_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} tx drops", "refId": "D" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "pps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,31 +1,59 @@
|
|||
{
|
||||
"uid": "incus-cluster-overview",
|
||||
"title": "Incus Cluster Overview",
|
||||
"description": "Overview of all Incus containers: CPU, memory, network, and disk I/O per instance.",
|
||||
"uid": "incus-instance-deep-dive",
|
||||
"title": "Instance Deep Dive",
|
||||
"description": "Single-instance drill-down: CPU, memory, network, and disk metrics with detailed timeseries and utilization gauges.",
|
||||
"tags": ["incus", "cluster", "containers"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "30s",
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"fiscalYearStartMonth": 0,
|
||||
"liveNow": false,
|
||||
"editable": true,
|
||||
"graphTooltip": 1,
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": true,
|
||||
"icon": "external link",
|
||||
"includeVars": true,
|
||||
"keepTime": true,
|
||||
"tags": ["incus"],
|
||||
"targetBlank": false,
|
||||
"title": "Incus Dashboards",
|
||||
"tooltip": "Navigate to other Incus dashboards",
|
||||
"type": "dashboards"
|
||||
},
|
||||
{
|
||||
"title": "Back to Cluster Overview",
|
||||
"url": "/d/incus-cluster-overview?${__url_time_range}",
|
||||
"targetBlank": false,
|
||||
"type": "link",
|
||||
"icon": "arrow-left"
|
||||
}
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
"list": [
|
||||
{
|
||||
"current": {},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"definition": "label_values(incus_uptime_seconds, name)",
|
||||
"includeAll": false,
|
||||
"multi": false,
|
||||
"name": "instance",
|
||||
"query": "label_values(incus_uptime_seconds, name)",
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query",
|
||||
"label": "Instance"
|
||||
}
|
||||
]
|
||||
},
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"datasource": { "type": "grafana", "uid": "-- Grafana --" },
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
|
|
@ -37,7 +65,7 @@
|
|||
"panels": [
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Cluster Status",
|
||||
"title": "Instance Status",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"id": 1,
|
||||
|
|
@ -45,18 +73,15 @@
|
|||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Total Containers",
|
||||
"description": "Number of running Incus containers (instances reporting uptime).",
|
||||
"gridPos": { "h": 4, "w": 8, "x": 0, "y": 1 },
|
||||
"title": "Uptime",
|
||||
"description": "How long this instance has been running.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
|
||||
"id": 2,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "count(incus_uptime_seconds)",
|
||||
"legendFormat": "Containers",
|
||||
"expr": "incus_uptime_seconds{name=~\"$instance\"}",
|
||||
"legendFormat": "Uptime",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
|
|
@ -64,43 +89,32 @@
|
|||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 10 },
|
||||
{ "color": "red", "value": 50 }
|
||||
]
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
},
|
||||
"unit": "none",
|
||||
"unit": "s",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Total CPUs",
|
||||
"description": "Sum of effective CPUs across all Incus instances.",
|
||||
"gridPos": { "h": 4, "w": 8, "x": 8, "y": 1 },
|
||||
"title": "Effective CPUs",
|
||||
"description": "Number of effective CPUs allocated to this instance.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
|
||||
"id": 3,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(incus_cpu_effective_total)",
|
||||
"expr": "incus_cpu_effective_total{name=~\"$instance\"}",
|
||||
"legendFormat": "CPUs",
|
||||
"refId": "A"
|
||||
}
|
||||
|
|
@ -109,9 +123,7 @@
|
|||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
},
|
||||
"unit": "none",
|
||||
"color": { "mode": "thresholds" }
|
||||
|
|
@ -119,31 +131,24 @@
|
|||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Total Memory",
|
||||
"description": "Sum of total memory across all Incus instances.",
|
||||
"gridPos": { "h": 4, "w": 8, "x": 16, "y": 1 },
|
||||
"description": "Total memory allocated to this instance.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
|
||||
"id": 4,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(incus_memory_MemTotal_bytes)",
|
||||
"expr": "incus_memory_MemTotal_bytes{name=~\"$instance\"}",
|
||||
"legendFormat": "Total Memory",
|
||||
"refId": "A"
|
||||
}
|
||||
|
|
@ -152,9 +157,7 @@
|
|||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
},
|
||||
"unit": "bytes",
|
||||
"color": { "mode": "thresholds" }
|
||||
|
|
@ -162,148 +165,25 @@
|
|||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Per-Instance CPU",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
|
||||
"type": "gauge",
|
||||
"title": "Memory Utilization",
|
||||
"description": "Current memory utilization (RSS / Total) for this instance.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
|
||||
"id": 5,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "CPU Usage by Instance",
|
||||
"description": "Per-second CPU time consumed by each Incus instance.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 },
|
||||
"id": 6,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (name) (rate(incus_cpu_seconds_total[$__rate_interval]))",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "short",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Per-Instance Memory",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
|
||||
"id": 7,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Memory RSS by Instance",
|
||||
"description": "Resident set size (RSS) memory usage per Incus instance over time.",
|
||||
"gridPos": { "h": 8, "w": 14, "x": 0, "y": 15 },
|
||||
"id": 8,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "incus_memory_RSS_bytes",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "bytes",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "bargauge",
|
||||
"title": "Memory Usage % (RSS / Total)",
|
||||
"description": "Current memory utilization per instance as RSS divided by total memory.",
|
||||
"gridPos": { "h": 8, "w": 10, "x": 14, "y": 15 },
|
||||
"id": 9,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(incus_memory_RSS_bytes / incus_memory_MemTotal_bytes) * 100",
|
||||
"legendFormat": "{{name}}",
|
||||
"expr": "(incus_memory_RSS_bytes{name=~\"$instance\"} / incus_memory_MemTotal_bytes{name=~\"$instance\"}) * 100",
|
||||
"legendFormat": "Utilization",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
|
|
@ -326,40 +206,83 @@
|
|||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"orientation": "horizontal",
|
||||
"displayMode": "gradient",
|
||||
"showUnfilled": true,
|
||||
"minVizWidth": 8,
|
||||
"minVizHeight": 16,
|
||||
"valueMode": "color"
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true,
|
||||
"orientation": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Network I/O",
|
||||
"title": "CPU",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
|
||||
"id": 10,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Network Receive Rate by Instance",
|
||||
"description": "Inbound network throughput per Incus instance.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
|
||||
"title": "CPU Usage Over Time",
|
||||
"description": "Per-second CPU time consumed by this instance over time.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 },
|
||||
"id": 11,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (name) (rate(incus_network_receive_bytes_total[$__rate_interval]))",
|
||||
"expr": "sum by (name) (rate(incus_cpu_seconds_total{name=~\"$instance\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 25,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "short",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Memory",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
|
||||
"id": 20,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Memory RSS Over Time",
|
||||
"description": "Resident set size (RSS) memory usage for this instance over time.",
|
||||
"gridPos": { "h": 8, "w": 14, "x": 0, "y": 15 },
|
||||
"id": 21,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "incus_memory_RSS_bytes{name=~\"$instance\"}",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
|
|
@ -381,39 +304,99 @@
|
|||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"unit": "bytes",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Network Transmit Rate by Instance",
|
||||
"description": "Outbound network throughput per Incus instance.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
|
||||
"id": 12,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"title": "Memory Utilization %",
|
||||
"description": "Memory utilization percentage (RSS / Total) for this instance over time.",
|
||||
"gridPos": { "h": 8, "w": 10, "x": 14, "y": 15 },
|
||||
"id": 22,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (name) (rate(incus_network_transmit_bytes_total[$__rate_interval]))",
|
||||
"expr": "(incus_memory_RSS_bytes{name=~\"$instance\"} / incus_memory_MemTotal_bytes{name=~\"$instance\"}) * 100",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 25,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "line" }
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"color": { "mode": "continuous-GrYlRd" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 60 },
|
||||
{ "color": "orange", "value": 80 },
|
||||
{ "color": "red", "value": 90 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Network",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
|
||||
"id": 30,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Network Traffic",
|
||||
"description": "Network receive and transmit throughput for this instance. Transmit is shown as negative (below axis).",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 24 },
|
||||
"id": 31,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (name) (rate(incus_network_receive_bytes_total{name=~\"$instance\"}[$__rate_interval]))",
|
||||
"legendFormat": "Receive",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "-sum by (name) (rate(incus_network_transmit_bytes_total{name=~\"$instance\"}[$__rate_interval]))",
|
||||
"legendFormat": "Transmit",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
|
|
@ -435,16 +418,27 @@
|
|||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Receive" },
|
||||
"properties": [
|
||||
{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Transmit" },
|
||||
"properties": [
|
||||
{ "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -452,24 +446,26 @@
|
|||
"title": "Disk I/O",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 },
|
||||
"id": 13,
|
||||
"id": 40,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Disk Read Rate by Instance",
|
||||
"description": "Disk read throughput per Incus instance.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 },
|
||||
"id": 14,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"title": "Disk Read/Write Rate",
|
||||
"description": "Disk read and write throughput for this instance.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 33 },
|
||||
"id": 41,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (name) (rate(incus_disk_read_bytes_total[$__rate_interval]))",
|
||||
"legendFormat": "{{name}}",
|
||||
"expr": "sum by (name) (rate(incus_disk_read_bytes_total{name=~\"$instance\"}[$__rate_interval]))",
|
||||
"legendFormat": "Read",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (name) (rate(incus_disk_written_bytes_total{name=~\"$instance\"}[$__rate_interval]))",
|
||||
"legendFormat": "Write",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
|
|
@ -493,66 +489,14 @@
|
|||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Disk Write Rate by Instance",
|
||||
"description": "Disk write throughput per Incus instance.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 },
|
||||
"id": 15,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (name) (rate(incus_disk_written_bytes_total[$__rate_interval]))",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,268 @@
|
|||
{
|
||||
"uid": "incus-logs-explorer",
|
||||
"title": "Logs Explorer",
|
||||
"description": "Log volume over time, full log stream with search, error and warning rates by systemd unit. Datasource: Loki.",
|
||||
"tags": ["incus", "logs", "loki"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"fiscalYearStartMonth": 0,
|
||||
"liveNow": false,
|
||||
"editable": true,
|
||||
"graphTooltip": 1,
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": true,
|
||||
"icon": "external link",
|
||||
"includeVars": true,
|
||||
"keepTime": true,
|
||||
"tags": ["incus"],
|
||||
"targetBlank": false,
|
||||
"title": "Incus Dashboards",
|
||||
"tooltip": "Navigate to other Incus dashboards",
|
||||
"type": "dashboards"
|
||||
}
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"type": "query",
|
||||
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
||||
"query": "label_values(job)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"name": "job",
|
||||
"label": "Job",
|
||||
"refresh": 2,
|
||||
"sort": 1
|
||||
},
|
||||
{
|
||||
"type": "query",
|
||||
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
||||
"query": "label_values(host)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"name": "host",
|
||||
"label": "Host",
|
||||
"refresh": 2,
|
||||
"sort": 1
|
||||
},
|
||||
{
|
||||
"type": "query",
|
||||
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
||||
"query": "label_values(unit)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"name": "unit",
|
||||
"label": "Unit",
|
||||
"refresh": 2,
|
||||
"sort": 1
|
||||
},
|
||||
{
|
||||
"type": "textbox",
|
||||
"name": "search",
|
||||
"label": "Search",
|
||||
"current": { "text": "", "value": "" }
|
||||
}
|
||||
]
|
||||
},
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": { "type": "grafana", "uid": "-- Grafana --" },
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Log Volume",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"id": 1,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Log Volume Over Time",
|
||||
"description": "Total log line count over time, broken down by job. Shows the rate at which logs are being generated across all selected sources.",
|
||||
"gridPos": { "h": 6, "w": 24, "x": 0, "y": 1 },
|
||||
"id": 2,
|
||||
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (job) (count_over_time({job=~\"$job\", host=~\"$host\"} [$__auto]))",
|
||||
"legendFormat": "{{job}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "bars",
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 80,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "normal", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "short",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["sum", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Log Stream",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 },
|
||||
"id": 3,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "logs",
|
||||
"title": "Logs",
|
||||
"description": "Live log stream filtered by job, host, unit, and free-text search. Displays timestamps, labels, and full log messages with expand-on-click detail view.",
|
||||
"gridPos": { "h": 16, "w": 24, "x": 0, "y": 8 },
|
||||
"id": 4,
|
||||
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "{job=~\"$job\", host=~\"$host\", unit=~\"$unit\"} |= \"$search\"",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": true,
|
||||
"showCommonLabels": false,
|
||||
"wrapLogMessage": true,
|
||||
"prettifyLogMessage": false,
|
||||
"enableLogDetails": true,
|
||||
"sortOrder": "Descending",
|
||||
"dedupStrategy": "none"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Error & Warning Rates",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 },
|
||||
"id": 5,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Error Rate by Unit",
|
||||
"description": "Rate of log lines matching error, fail, or fatal (case-insensitive) over time, grouped by systemd unit. Spikes indicate services experiencing failures.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 },
|
||||
"id": 6,
|
||||
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (unit) (count_over_time({job=~\"$job\", host=~\"$host\"} |~ \"(?i)error|fail|fatal\" [$__auto]))",
|
||||
"legendFormat": "{{unit}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "short",
|
||||
"color": { "mode": "fixed", "fixedColor": "red" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "red", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["sum", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Warning Rate by Unit",
|
||||
"description": "Rate of log lines matching warn (case-insensitive) over time, grouped by systemd unit. Helps identify services generating warnings that may precede errors.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 25 },
|
||||
"id": 7,
|
||||
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (unit) (count_over_time({job=~\"$job\", host=~\"$host\"} |~ \"(?i)warn\" [$__auto]))",
|
||||
"legendFormat": "{{unit}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "short",
|
||||
"color": { "mode": "fixed", "fixedColor": "yellow" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "yellow", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["sum", "max", "lastNotNull"] }
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,529 @@
|
|||
{
|
||||
"uid": "incus-network-deep-dive",
|
||||
"title": "Network Deep Dive",
|
||||
"description": "Per-node and per-device network traffic, instance-level network metrics, receive/transmit errors, and packet drops.",
|
||||
"tags": ["incus", "incusos", "network", "node-exporter"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"fiscalYearStartMonth": 0,
|
||||
"liveNow": false,
|
||||
"editable": true,
|
||||
"graphTooltip": 1,
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": true,
|
||||
"icon": "external link",
|
||||
"includeVars": true,
|
||||
"keepTime": true,
|
||||
"tags": ["incus"],
|
||||
"targetBlank": false,
|
||||
"title": "Incus Dashboards",
|
||||
"tooltip": "Navigate to other Incus dashboards",
|
||||
"type": "dashboards"
|
||||
}
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"definition": "label_values(node_network_receive_bytes_total, instance)",
|
||||
"includeAll": false,
|
||||
"multi": true,
|
||||
"name": "node",
|
||||
"query": "label_values(node_network_receive_bytes_total, instance)",
|
||||
"refresh": 2,
|
||||
"sort": 1,
|
||||
"type": "query",
|
||||
"label": "Node"
|
||||
},
|
||||
{
|
||||
"current": {},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"definition": "label_values(node_network_receive_bytes_total{instance=~\"$node\"}, device)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"name": "device",
|
||||
"query": "label_values(node_network_receive_bytes_total{instance=~\"$node\"}, device)",
|
||||
"refresh": 2,
|
||||
"regex": "/^(?!lo$).*/",
|
||||
"sort": 1,
|
||||
"type": "query",
|
||||
"label": "Device"
|
||||
}
|
||||
]
|
||||
},
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": { "type": "grafana", "uid": "-- Grafana --" },
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Cluster Summary",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"id": 1,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Total Ingress",
|
||||
"description": "Aggregate inbound network throughput across all selected nodes and devices.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
|
||||
"id": 2,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(node_network_receive_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval]))",
|
||||
"legendFormat": "Ingress",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Total Egress",
|
||||
"description": "Aggregate outbound network throughput across all selected nodes and devices.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
|
||||
"id": 3,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(node_network_transmit_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval]))",
|
||||
"legendFormat": "Egress",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Total Errors",
|
||||
"description": "Combined receive and transmit error rate across all selected nodes. Any non-zero value warrants investigation.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
|
||||
"id": 4,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(node_network_receive_errs_total{instance=~\"$node\"}[$__rate_interval])) + sum(rate(node_network_transmit_errs_total{instance=~\"$node\"}[$__rate_interval]))",
|
||||
"legendFormat": "Errors",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 0.1 },
|
||||
{ "color": "red", "value": 1 }
|
||||
]
|
||||
},
|
||||
"unit": "pps",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Total Drops",
|
||||
"description": "Combined receive and transmit packet drop rate across all selected nodes. Drops indicate buffer or queue exhaustion.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
|
||||
"id": 5,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(node_network_receive_drop_total{instance=~\"$node\"}[$__rate_interval])) + sum(rate(node_network_transmit_drop_total{instance=~\"$node\"}[$__rate_interval]))",
|
||||
"legendFormat": "Drops",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 0.1 },
|
||||
{ "color": "red", "value": 1 }
|
||||
]
|
||||
},
|
||||
"unit": "pps",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Per-Node Traffic",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
|
||||
"id": 10,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Receive Rate by Node & Device",
|
||||
"description": "Inbound network throughput broken down by node and network device. Identifies which interfaces are handling the most receive traffic.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
|
||||
"id": 11,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(node_network_receive_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
|
||||
"legendFormat": "{{instance}} {{device}} rx",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Transmit Rate by Node & Device",
|
||||
"description": "Outbound network throughput broken down by node and network device. Identifies which interfaces are handling the most transmit traffic.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
|
||||
"id": 12,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(node_network_transmit_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
|
||||
"legendFormat": "{{instance}} {{device}} tx",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Instance Network (Incus metrics)",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
|
||||
"id": 20,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Instance Receive Rate",
|
||||
"description": "Inbound network throughput per Incus instance, aggregated across all instance network devices. Uses Incus-native metrics for container and VM visibility.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 },
|
||||
"id": 21,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (name) (rate(incus_network_receive_bytes_total{instance=~\"$node\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Instance Transmit Rate",
|
||||
"description": "Outbound network throughput per Incus instance, aggregated across all instance network devices. Uses Incus-native metrics for container and VM visibility.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 },
|
||||
"id": 22,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (name) (rate(incus_network_transmit_bytes_total{instance=~\"$node\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Errors & Drops",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
|
||||
"id": 30,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Receive/Transmit Errors",
|
||||
"description": "Network interface error rates broken down by node and device. Receive errors may indicate CRC failures or framing issues; transmit errors may indicate collisions or driver problems.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
|
||||
"id": 31,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(node_network_receive_errs_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
|
||||
"legendFormat": "{{instance}} {{device}} rx errors",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "rate(node_network_transmit_errs_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
|
||||
"legendFormat": "{{instance}} {{device}} tx errors",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "pps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Packet Drops",
|
||||
"description": "Network interface packet drop rates broken down by node and device. Drops indicate kernel buffer or queue exhaustion and may cause retransmissions or connection failures.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
|
||||
"id": 32,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(node_network_receive_drop_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
|
||||
"legendFormat": "{{instance}} {{device}} rx drops",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "rate(node_network_transmit_drop_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
|
||||
"legendFormat": "{{instance}} {{device}} tx drops",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "pps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,748 @@
|
|||
{
|
||||
"uid": "incus-prometheus-health",
|
||||
"title": "Prometheus Health",
|
||||
"description": "Prometheus self-monitoring: targets up/down, scrape duration, samples ingested, TSDB size, head series, WAL stats.",
|
||||
"tags": ["incus", "prometheus", "operations"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"fiscalYearStartMonth": 0,
|
||||
"liveNow": false,
|
||||
"editable": true,
|
||||
"graphTooltip": 1,
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": true,
|
||||
"icon": "external link",
|
||||
"includeVars": true,
|
||||
"keepTime": true,
|
||||
"tags": ["incus"],
|
||||
"targetBlank": false,
|
||||
"title": "Incus Dashboards",
|
||||
"tooltip": "Navigate to other Incus dashboards",
|
||||
"type": "dashboards"
|
||||
}
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": { "type": "grafana", "uid": "-- Grafana --" },
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Targets",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"id": 1,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Targets Up",
|
||||
"description": "Number of scrape targets currently reporting as healthy (up == 1).",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
|
||||
"id": 2,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "count(up == 1)",
|
||||
"legendFormat": "Up",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
},
|
||||
"unit": "none",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Targets Down",
|
||||
"description": "Number of scrape targets currently unreachable or failing (up == 0). Any value above 0 requires investigation.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
|
||||
"id": 3,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "count(up == 0)",
|
||||
"legendFormat": "Down",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "red", "value": 1 }
|
||||
]
|
||||
},
|
||||
"unit": "none",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Avg Scrape Duration",
|
||||
"description": "Average time Prometheus takes to scrape all targets. High values may indicate slow or overloaded exporters.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
|
||||
"id": 4,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "avg(scrape_duration_seconds)",
|
||||
"legendFormat": "Avg Duration",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 1 },
|
||||
{ "color": "red", "value": 5 }
|
||||
]
|
||||
},
|
||||
"unit": "s",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Samples Ingested/s",
|
||||
"description": "Total rate of samples being scraped and ingested across all targets per second.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
|
||||
"id": 5,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(scrape_samples_scraped[$__rate_interval]))",
|
||||
"legendFormat": "Samples/s",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "table",
|
||||
"title": "Target Health",
|
||||
"description": "Per-target health status showing each scrape target's current state. UP (1) means healthy, DOWN (0) means unreachable or failing.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 5 },
|
||||
"id": 6,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "up",
|
||||
"legendFormat": "",
|
||||
"refId": "A",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"align": "auto",
|
||||
"displayMode": "auto",
|
||||
"inspect": false,
|
||||
"filterable": true
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
},
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "instance" },
|
||||
"properties": [
|
||||
{ "id": "displayName", "value": "Instance" },
|
||||
{ "id": "custom.width", "value": 300 }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "job" },
|
||||
"properties": [
|
||||
{ "id": "displayName", "value": "Job" },
|
||||
{ "id": "custom.width", "value": 200 }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Value" },
|
||||
"properties": [
|
||||
{ "id": "displayName", "value": "Status" },
|
||||
{ "id": "custom.displayMode", "value": "color-background" },
|
||||
{
|
||||
"id": "mappings",
|
||||
"value": [
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"1": { "text": "UP", "color": "green" },
|
||||
"0": { "text": "DOWN", "color": "red" }
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"sortBy": [{ "displayName": "Job", "desc": false }],
|
||||
"footer": { "show": false, "reducer": ["sum"], "fields": "" }
|
||||
},
|
||||
"transformations": [
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": { "Time": true, "__name__": true },
|
||||
"indexByName": {
|
||||
"instance": 0,
|
||||
"job": 1,
|
||||
"Value": 2
|
||||
},
|
||||
"renameByName": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Scrape Performance",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 },
|
||||
"id": 7,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Scrape Duration by Job",
|
||||
"description": "How long each scrape takes per job. Spikes indicate slow exporters or network issues between Prometheus and targets.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 },
|
||||
"id": 8,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "scrape_duration_seconds",
|
||||
"legendFormat": "{{job}} - {{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "s",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Samples Scraped by Job",
|
||||
"description": "Rate of samples scraped per job. Sudden drops may indicate target failures; spikes may indicate new metrics being exposed.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 },
|
||||
"id": 9,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(scrape_samples_scraped[$__rate_interval])",
|
||||
"legendFormat": "{{job}} - {{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "short",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "TSDB",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 },
|
||||
"id": 10,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "TSDB Size",
|
||||
"description": "Current on-disk size of the Prometheus TSDB storage directory including all blocks and WAL.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 23 },
|
||||
"id": 11,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "prometheus_tsdb_storage_size_bytes",
|
||||
"legendFormat": "TSDB Size",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 1073741824 },
|
||||
{ "color": "red", "value": 5368709120 }
|
||||
]
|
||||
},
|
||||
"unit": "bytes",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Head Series",
|
||||
"description": "Number of active time series in the TSDB head block. High cardinality here impacts query performance and memory usage.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 23 },
|
||||
"id": 12,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "prometheus_tsdb_head_series",
|
||||
"legendFormat": "Head Series",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 100000 },
|
||||
{ "color": "red", "value": 500000 }
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Head Chunks",
|
||||
"description": "Number of chunks in the TSDB head block. Growth correlates with active series count and scrape frequency.",
|
||||
"gridPos": { "h": 8, "w": 6, "x": 12, "y": 23 },
|
||||
"id": 13,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "prometheus_tsdb_head_chunks",
|
||||
"legendFormat": "Head Chunks",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "short",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "WAL Size",
|
||||
"description": "Size of the Prometheus write-ahead log (WAL). The WAL buffers recent writes before compaction into blocks.",
|
||||
"gridPos": { "h": 8, "w": 6, "x": 18, "y": 23 },
|
||||
"id": 14,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "prometheus_tsdb_wal_storage_size_bytes",
|
||||
"legendFormat": "WAL Size",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "bytes",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "TSDB Trends",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 },
|
||||
"id": 15,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "TSDB Size Over Time",
|
||||
"description": "Prometheus TSDB total storage size trend. Useful for capacity planning and detecting unexpected growth.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 },
|
||||
"id": 16,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "prometheus_tsdb_storage_size_bytes",
|
||||
"legendFormat": "TSDB Size",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "bytes",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Samples Ingested Trend",
|
||||
"description": "Rate of new samples appended to the TSDB head block. Correlates with scrape frequency and number of active series.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 },
|
||||
"id": 17,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(prometheus_tsdb_head_samples_appended_total[$__rate_interval])",
|
||||
"legendFormat": "Samples/s",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "short",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Compactions",
|
||||
"description": "Rate of TSDB compaction operations. Compactions merge head block data into persistent blocks on disk for efficient storage.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 },
|
||||
"id": 18,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(prometheus_tsdb_compactions_total[$__rate_interval])",
|
||||
"legendFormat": "Compactions/s",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "short",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Head Series Count",
|
||||
"description": "Number of active time series in the TSDB head block over time. Trend helps identify cardinality growth or label explosion.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 },
|
||||
"id": 19,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "prometheus_tsdb_head_series",
|
||||
"legendFormat": "Head Series",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "short",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,382 @@
|
|||
{
|
||||
"uid": "incus-storage-filesystem",
|
||||
"title": "Storage & Filesystem",
|
||||
"description": "Filesystem usage, available space trends, disk throughput, I/O latency, disk utilization, and inode usage per node.",
|
||||
"tags": ["incus", "incusos", "storage", "node-exporter"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"fiscalYearStartMonth": 0,
|
||||
"liveNow": false,
|
||||
"editable": true,
|
||||
"graphTooltip": 1,
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": true, "icon": "external link", "includeVars": true,
|
||||
"keepTime": true, "tags": ["incus"], "targetBlank": false,
|
||||
"title": "Incus Dashboards", "type": "dashboards"
|
||||
}
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"definition": "label_values(node_filesystem_size_bytes, instance)",
|
||||
"includeAll": false,
|
||||
"multi": true,
|
||||
"name": "node",
|
||||
"query": "label_values(node_filesystem_size_bytes, instance)",
|
||||
"refresh": 2,
|
||||
"sort": 1,
|
||||
"type": "query",
|
||||
"label": "Node"
|
||||
},
|
||||
{
|
||||
"current": {},
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"definition": "label_values(node_filesystem_size_bytes{instance=~\"$node\"}, mountpoint)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"name": "mountpoint",
|
||||
"query": "label_values(node_filesystem_size_bytes{instance=~\"$node\"}, mountpoint)",
|
||||
"refresh": 2,
|
||||
"sort": 1,
|
||||
"type": "query",
|
||||
"label": "Mountpoint"
|
||||
}
|
||||
]
|
||||
},
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": { "type": "grafana", "uid": "-- Grafana --" },
|
||||
"enable": true, "hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts", "type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "row", "title": "Filesystem Overview",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"id": 1, "panels": []
|
||||
},
|
||||
{
|
||||
"type": "table",
|
||||
"title": "Filesystem Usage Table",
|
||||
"description": "Current filesystem size, available space, and usage percentage per node and mountpoint. High usage is highlighted with color thresholds.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 1 },
|
||||
"id": 2,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "node_filesystem_size_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}",
|
||||
"legendFormat": "", "refId": "A", "format": "table", "instant": true
|
||||
},
|
||||
{
|
||||
"expr": "node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}",
|
||||
"legendFormat": "", "refId": "B", "format": "table", "instant": true
|
||||
},
|
||||
{
|
||||
"expr": "(1 - node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}) * 100",
|
||||
"legendFormat": "", "refId": "C", "format": "table", "instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": { "align": "auto", "displayMode": "auto", "inspect": false, "filterable": true },
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byName", "options": "instance" }, "properties": [{ "id": "displayName", "value": "Node" }] },
|
||||
{ "matcher": { "id": "byName", "options": "mountpoint" }, "properties": [{ "id": "displayName", "value": "Mountpoint" }] },
|
||||
{ "matcher": { "id": "byName", "options": "fstype" }, "properties": [{ "id": "displayName", "value": "FS Type" }] },
|
||||
{ "matcher": { "id": "byName", "options": "Value #A" }, "properties": [{ "id": "displayName", "value": "Size" }, { "id": "unit", "value": "bytes" }] },
|
||||
{ "matcher": { "id": "byName", "options": "Value #B" }, "properties": [{ "id": "displayName", "value": "Available" }, { "id": "unit", "value": "bytes" }] },
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Value #C" },
|
||||
"properties": [
|
||||
{ "id": "displayName", "value": "Used %" },
|
||||
{ "id": "unit", "value": "percent" },
|
||||
{ "id": "custom.displayMode", "value": "color-background" },
|
||||
{ "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "orange", "value": 85 }, { "color": "red", "value": 95 }] } }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": { "showHeader": true, "sortBy": [], "footer": { "show": false, "reducer": ["sum"], "fields": "" } },
|
||||
"transformations": [
|
||||
{ "id": "merge", "options": {} },
|
||||
{ "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "job": true, "device": true }, "renameByName": {} } }
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "row", "title": "Space Trends",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
|
||||
"id": 3, "panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Available Space Over Time",
|
||||
"description": "Tracks available filesystem space over time per node and mountpoint. Use this to identify gradual space consumption trends before they become critical.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 10 },
|
||||
"id": 4,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}",
|
||||
"legendFormat": "{{instance}} {{mountpoint}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never", "pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "bytes",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row", "title": "Disk Throughput",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 },
|
||||
"id": 5, "panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Disk Read Throughput",
|
||||
"description": "Disk read throughput in bytes per second, broken down by instance and device. Identifies which disks are handling the most read traffic.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 },
|
||||
"id": 6,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (instance, device) (rate(node_disk_read_bytes_total{instance=~\"$node\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{instance}} {{device}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never", "pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Disk Write Throughput",
|
||||
"description": "Disk write throughput in bytes per second, broken down by instance and device. Identifies which disks are handling the most write traffic.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 },
|
||||
"id": 7,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (instance, device) (rate(node_disk_written_bytes_total{instance=~\"$node\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{instance}} {{device}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never", "pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row", "title": "I/O Performance",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 },
|
||||
"id": 8, "panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Average Read Latency",
|
||||
"description": "Average time per completed read operation. Calculated as total read time divided by completed reads. High values indicate slow disk response.",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 28 },
|
||||
"id": 9,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(node_disk_read_time_seconds_total{instance=~\"$node\"}[$__rate_interval]) / rate(node_disk_reads_completed_total{instance=~\"$node\"}[$__rate_interval])",
|
||||
"legendFormat": "{{instance}} {{device}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never", "pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "s",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Average Write Latency",
|
||||
"description": "Average time per completed write operation. Calculated as total write time divided by completed writes. High values indicate slow disk response or write saturation.",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 28 },
|
||||
"id": 10,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(node_disk_write_time_seconds_total{instance=~\"$node\"}[$__rate_interval]) / rate(node_disk_writes_completed_total{instance=~\"$node\"}[$__rate_interval])",
|
||||
"legendFormat": "{{instance}} {{device}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
|
||||
"fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
|
||||
"showPoints": "never", "pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "s",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Disk Utilization %",
|
||||
"description": "Percentage of time the disk is busy processing I/O requests. Values near 100% indicate the disk is saturated and may be a bottleneck.",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 28 },
|
||||
"id": 11,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (instance) (rate(node_disk_io_time_seconds_total{instance=~\"$node\"}[$__rate_interval])) * 100",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2,
|
||||
"fillOpacity": 25, "gradientMode": "scheme", "spanNulls": false,
|
||||
"showPoints": "never", "pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "line" }
|
||||
},
|
||||
"unit": "percent", "min": 0, "max": 100,
|
||||
"color": { "mode": "continuous-GrYlRd" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 80 }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row", "title": "Inodes",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 },
|
||||
"id": 12, "panels": []
|
||||
},
|
||||
{
|
||||
"type": "bargauge",
|
||||
"title": "Inode Usage %",
|
||||
"description": "Percentage of used inodes per filesystem. Running out of inodes prevents creating new files even when disk space is available. Monitor for filesystems with many small files.",
|
||||
"gridPos": { "h": 6, "w": 24, "x": 0, "y": 37 },
|
||||
"id": 13,
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - node_filesystem_files_free{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"} / node_filesystem_files{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}) * 100",
|
||||
"legendFormat": "{{instance}} {{mountpoint}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] },
|
||||
"unit": "percent", "min": 0, "max": 100,
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "horizontal", "displayMode": "gradient", "showUnfilled": true,
|
||||
"minVizWidth": 8, "minVizHeight": 16, "valueMode": "color"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
# Dashboard manifest for the Incus observability stack
|
||||
#
|
||||
# Each entry maps a dashboard JSON file to its Grafana folder, UID, and metadata.
|
||||
# Used by manage-dashboards for install, export, list, and validate operations.
|
||||
|
||||
dashboards:
|
||||
# --- Core dashboards (ship with product) ---
|
||||
|
||||
- file: core/cluster-overview.json
|
||||
uid: incus-cluster-overview
|
||||
title: Incus Cluster Overview
|
||||
folder: Incus Cluster
|
||||
category: core
|
||||
version: "2"
|
||||
description: >-
|
||||
Overview of all Incus instances: CPU, memory, network, disk I/O,
|
||||
and instance status table with drill-down links.
|
||||
|
||||
- file: core/host-resources.json
|
||||
uid: incus-host-resources
|
||||
title: Host Resources (IncusOS Nodes)
|
||||
folder: Incus Infrastructure
|
||||
category: core
|
||||
version: "2"
|
||||
description: >-
|
||||
Physical host monitoring: CPU mode breakdown, load averages,
|
||||
memory breakdown, filesystem usage, disk IOPS, network errors.
|
||||
|
||||
- file: core/haproxy-traffic.json
|
||||
uid: incus-haproxy-traffic
|
||||
title: HAProxy Traffic
|
||||
folder: Incus Services
|
||||
category: core
|
||||
version: "2"
|
||||
description: >-
|
||||
HAProxy load balancer monitoring: request rates, response codes,
|
||||
backend health, server status timeline, connection errors.
|
||||
|
||||
- file: core/instance-deep-dive.json
|
||||
uid: incus-instance-deep-dive
|
||||
title: Instance Deep Dive
|
||||
folder: Incus Cluster
|
||||
category: core
|
||||
version: "1"
|
||||
description: >-
|
||||
Single-instance drill-down: CPU, memory, network, and disk
|
||||
metrics with detailed timeseries and utilization gauges.
|
||||
|
||||
- file: core/storage-filesystem.json
|
||||
uid: incus-storage-filesystem
|
||||
title: Storage & Filesystem
|
||||
folder: Incus Infrastructure
|
||||
category: core
|
||||
version: "1"
|
||||
description: >-
|
||||
Filesystem usage, available space trends, disk throughput,
|
||||
I/O latency, disk utilization, and inode usage per node.
|
||||
|
||||
- file: core/network-deep-dive.json
|
||||
uid: incus-network-deep-dive
|
||||
title: Network Deep Dive
|
||||
folder: Incus Infrastructure
|
||||
category: core
|
||||
version: "1"
|
||||
description: >-
|
||||
Per-node and per-device network traffic, instance-level network
|
||||
metrics, receive/transmit errors, and packet drops.
|
||||
|
||||
- file: core/logs-explorer.json
|
||||
uid: incus-logs-explorer
|
||||
title: Logs Explorer
|
||||
folder: Incus Operations
|
||||
category: core
|
||||
version: "1"
|
||||
description: >-
|
||||
Log volume over time, full log stream with search, error
|
||||
and warning rates by systemd unit. Datasource: Loki.
|
||||
|
||||
- file: core/prometheus-health.json
|
||||
uid: incus-prometheus-health
|
||||
title: Prometheus Health
|
||||
folder: Incus Operations
|
||||
category: core
|
||||
version: "1"
|
||||
description: >-
|
||||
Prometheus self-monitoring: targets up/down, scrape duration,
|
||||
samples ingested, TSDB size, head series, WAL stats.
|
||||
|
||||
- file: core/capacity-planning.json
|
||||
uid: incus-capacity-planning
|
||||
title: Capacity Planning
|
||||
folder: Incus Operations
|
||||
category: core
|
||||
version: "1"
|
||||
description: >-
|
||||
CPU/memory headroom per node, resource trends, disk space
|
||||
forecast with predict_linear, instance count trends.
|
||||
|
|
@ -1,518 +0,0 @@
|
|||
{
|
||||
"uid": "incus-haproxy-traffic",
|
||||
"title": "HAProxy Traffic",
|
||||
"description": "HAProxy load balancer monitoring: request rates, backend health, traffic volume, and sessions.",
|
||||
"tags": ["haproxy", "loadbalancer", "incus"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "30s",
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"fiscalYearStartMonth": 0,
|
||||
"liveNow": false,
|
||||
"editable": true,
|
||||
"graphTooltip": 1,
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Overview",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"id": 1,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Active Backend Servers",
|
||||
"description": "Total number of active servers across all HAProxy backends.",
|
||||
"gridPos": { "h": 4, "w": 8, "x": 0, "y": 1 },
|
||||
"id": 2,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(haproxy_backend_active_servers)",
|
||||
"legendFormat": "Active Servers",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "yellow", "value": 1 },
|
||||
{ "color": "green", "value": 2 }
|
||||
]
|
||||
},
|
||||
"unit": "none",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Current Sessions",
|
||||
"description": "Total current sessions across all HAProxy backends.",
|
||||
"gridPos": { "h": 4, "w": 8, "x": 8, "y": 1 },
|
||||
"id": 3,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(haproxy_backend_current_sessions)",
|
||||
"legendFormat": "Sessions",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 100 },
|
||||
{ "color": "red", "value": 500 }
|
||||
]
|
||||
},
|
||||
"unit": "none",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Total HTTP Requests",
|
||||
"description": "Total HTTP requests across all HAProxy frontends (counter).",
|
||||
"gridPos": { "h": 4, "w": 8, "x": 16, "y": 1 },
|
||||
"id": 4,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(haproxy_frontend_http_requests_total[$__rate_interval]))",
|
||||
"legendFormat": "Requests/s",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
},
|
||||
"unit": "reqps",
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "HTTP Requests",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
|
||||
"id": 5,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "HTTP Request Rate by Frontend",
|
||||
"description": "Rate of HTTP requests per second, broken down by HAProxy frontend/proxy.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 },
|
||||
"id": 6,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (proxy) (rate(haproxy_frontend_http_requests_total[$__rate_interval]))",
|
||||
"legendFormat": "{{proxy}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "reqps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Backend Health",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
|
||||
"id": 7,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "table",
|
||||
"title": "Backend Status",
|
||||
"description": "Current status, active server count, and average response time for each HAProxy backend.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 15 },
|
||||
"id": 8,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "haproxy_backend_status",
|
||||
"legendFormat": "",
|
||||
"refId": "A",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
},
|
||||
{
|
||||
"expr": "haproxy_backend_active_servers",
|
||||
"legendFormat": "",
|
||||
"refId": "B",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
},
|
||||
{
|
||||
"expr": "haproxy_server_response_time_average_seconds",
|
||||
"legendFormat": "",
|
||||
"refId": "C",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"align": "auto",
|
||||
"displayMode": "auto",
|
||||
"inspect": false,
|
||||
"filterable": true
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
},
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Value #A" },
|
||||
"properties": [
|
||||
{ "id": "displayName", "value": "Status" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Value #B" },
|
||||
"properties": [
|
||||
{ "id": "displayName", "value": "Active Servers" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Value #C" },
|
||||
"properties": [
|
||||
{ "id": "displayName", "value": "Avg Response Time" },
|
||||
{ "id": "unit", "value": "s" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "proxy" },
|
||||
"properties": [
|
||||
{ "id": "displayName", "value": "Backend" }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"sortBy": [],
|
||||
"footer": {
|
||||
"show": false,
|
||||
"reducer": ["sum"],
|
||||
"fields": ""
|
||||
}
|
||||
},
|
||||
"transformations": [
|
||||
{
|
||||
"id": "merge",
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {
|
||||
"Time": true,
|
||||
"__name__": true,
|
||||
"instance": false,
|
||||
"job": true
|
||||
},
|
||||
"indexByName": {},
|
||||
"renameByName": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Traffic",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
|
||||
"id": 9,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Frontend Bytes In",
|
||||
"description": "Inbound traffic rate per HAProxy frontend.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
|
||||
"id": 10,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (proxy) (rate(haproxy_frontend_bytes_in_total[$__rate_interval]))",
|
||||
"legendFormat": "{{proxy}} (in)",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Frontend Bytes Out",
|
||||
"description": "Outbound traffic rate per HAProxy frontend.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
|
||||
"id": 11,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (proxy) (rate(haproxy_frontend_bytes_out_total[$__rate_interval]))",
|
||||
"legendFormat": "{{proxy}} (out)",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "Bps",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Sessions",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 },
|
||||
"id": 12,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Current Sessions by Proxy",
|
||||
"description": "Current active sessions for each HAProxy backend/proxy over time.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 33 },
|
||||
"id": 13,
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "haproxy_backend_current_sessions",
|
||||
"legendFormat": "{{proxy}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"pointSize": 5,
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"axisLabel": "",
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "none",
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -122,40 +122,106 @@ datasource proxy.
|
|||
|
||||
## Dashboards
|
||||
|
||||
Three dashboards are provisioned automatically during deployment.
|
||||
Nine dashboards are provisioned automatically during deployment, organized
|
||||
into four Grafana folders. All dashboards support template variables for
|
||||
filtering and include cross-dashboard navigation links.
|
||||
|
||||
### Incus Cluster Overview
|
||||
Use `manage-dashboards` to install, export, validate, or list dashboards:
|
||||
|
||||
Visualizes metrics scraped from the Incus `/1.0/metrics` endpoint on each
|
||||
cluster node.
|
||||
```bash
|
||||
incusos/manage-dashboards --install # push all dashboards to Grafana
|
||||
incusos/manage-dashboards --list # show install status
|
||||
incusos/manage-dashboards --validate # check JSON quality
|
||||
incusos/manage-dashboards --export # export from Grafana to clean JSON
|
||||
incusos/manage-dashboards --status # compare local vs installed
|
||||
```
|
||||
|
||||
- Instance CPU usage (per-instance, per-node)
|
||||
- Memory usage and allocation
|
||||
- Network I/O (bytes in/out per interface)
|
||||
- Disk I/O (reads/writes, latency)
|
||||
- Instance count and state
|
||||
### Incus Cluster — Incus Cluster Overview
|
||||
|
||||
### HAProxy Traffic
|
||||
Overview of all instances with instance/node template variables.
|
||||
|
||||
Visualizes metrics from the HAProxy stats endpoint (`:8404/metrics`).
|
||||
- Instance count, total CPUs, total memory, cluster memory utilization gauge
|
||||
- Instance status table with drill-down links to Instance Deep Dive
|
||||
- Per-instance CPU usage, memory RSS, memory usage % bar gauge
|
||||
- Network receive/transmit rates, disk read/write rates
|
||||
|
||||
- Request rate (frontend and backend)
|
||||
- Backend health status (UP/DOWN per server)
|
||||
- Active sessions and session rate
|
||||
- Traffic volume (bytes in/out)
|
||||
- HTTP response codes (2xx, 4xx, 5xx)
|
||||
- Connection errors and retries
|
||||
### Incus Cluster — Instance Deep Dive
|
||||
|
||||
### Host Resources
|
||||
Single-instance drill-down selected via template variable.
|
||||
|
||||
Visualizes metrics from node_exporter on each cluster node.
|
||||
- Uptime, effective CPUs, total memory, memory utilization gauge
|
||||
- CPU usage over time with gradient color scheme
|
||||
- Memory RSS and memory utilization % over time
|
||||
- Network traffic (receive above axis, transmit below on same panel)
|
||||
- Disk read/write rates
|
||||
|
||||
- CPU utilization (%)
|
||||
- Memory utilization (%)
|
||||
- Disk usage and I/O
|
||||
- Network throughput per interface
|
||||
- System load averages
|
||||
- Filesystem free space
|
||||
### Incus Services — HAProxy Traffic
|
||||
|
||||
HAProxy monitoring with proxy/server template variables.
|
||||
|
||||
- Active servers, current sessions, HTTP request rate
|
||||
- HTTP 4xx/5xx rate stat panels with warning thresholds
|
||||
- Response code distribution (stacked, color-coded by class)
|
||||
- Backend status table and server status state timeline
|
||||
- Frontend traffic, sessions by proxy, response times
|
||||
- Queue time and connection errors
|
||||
|
||||
### Incus Infrastructure — Host Resources (IncusOS Nodes)
|
||||
|
||||
Physical host monitoring with node template variable.
|
||||
|
||||
- System information table (kernel, uptime, RAM, CPU count)
|
||||
- CPU usage % and CPU mode breakdown (user/system/iowait/idle stacked)
|
||||
- Load average (1m/5m/15m) with CPU count threshold line
|
||||
- Memory usage % and memory breakdown (used/buffers/cached/available)
|
||||
- Filesystem usage % bar gauge, IOPS, disk I/O time
|
||||
- Network traffic by device, network errors and drops
|
||||
|
||||
### Incus Infrastructure — Storage & Filesystem
|
||||
|
||||
Filesystem and disk I/O with node/mountpoint template variables.
|
||||
|
||||
- Filesystem usage table with color-coded Used % column
|
||||
- Available space trend over time
|
||||
- Disk read/write throughput by device
|
||||
- Average read/write latency, disk utilization %
|
||||
- Inode usage % bar gauge
|
||||
|
||||
### Incus Infrastructure — Network Deep Dive
|
||||
|
||||
Network metrics with node/device template variables.
|
||||
|
||||
- Cluster ingress/egress/errors/drops summary stats
|
||||
- Per-node per-device receive/transmit rates
|
||||
- Instance-level network traffic (Incus `incus_network_*` metrics)
|
||||
- Receive/transmit errors and packet drops
|
||||
|
||||
### Incus Operations — Logs Explorer
|
||||
|
||||
Log exploration powered by Loki with job/host/unit/search variables.
|
||||
|
||||
- Log volume over time (stacked by job)
|
||||
- Full log stream with search filtering
|
||||
- Error rate and warning rate by systemd unit
|
||||
|
||||
### Incus Operations — Prometheus Health
|
||||
|
||||
Prometheus self-monitoring (no template variables).
|
||||
|
||||
- Targets up/down, average scrape duration, samples ingested/s
|
||||
- Target health table with UP/DOWN status
|
||||
- Scrape duration and samples scraped by job
|
||||
- TSDB size, head series, head chunks, WAL size
|
||||
- TSDB size trend, samples ingested trend, compactions
|
||||
|
||||
### Incus Operations — Capacity Planning
|
||||
|
||||
Resource planning with 7-day default time range (no template variables).
|
||||
|
||||
- CPU and memory headroom per node (inverted bar gauges)
|
||||
- CPU and memory usage trends (7-day)
|
||||
- Disk space forecast with 30-day `predict_linear` projection
|
||||
- Instance count trend, memory allocation vs usage per instance
|
||||
|
||||
## ACL Configuration
|
||||
|
||||
|
|
@ -208,11 +274,18 @@ manually or restart the container.
|
|||
| node-exp-02 | Alpine | 128 MiB | — | oc-node-02 |
|
||||
| node-exp-03 | Alpine | 128 MiB | — | oc-node-03 |
|
||||
|
||||
**Dummy workloads** (optional, deployed via `--workloads`):
|
||||
|
||||
| Container | Image | RAM | Placement | Purpose |
|
||||
|-----------|-------|-----|-----------|---------|
|
||||
| workload-web | Alpine | 128 MiB | oc-node-01 | nginx + CPU/disk/network crons |
|
||||
| workload-api | Alpine | 128 MiB | oc-node-03 | Python HTTP + memory sawtooth |
|
||||
|
||||
**Totals:**
|
||||
- RAM: ~2.4 GiB (2 GiB + 3 x 128 MiB)
|
||||
- RAM: ~2.4 GiB core (2 GiB + 3 x 128 MiB), +256 MiB with workloads
|
||||
- Disk: 20 GiB (only the monitoring container needs significant storage)
|
||||
- OVN forward IPs: 1 (192.168.103.201)
|
||||
- OVN network IPs: 4 (10.10.10.70-73)
|
||||
- OVN network IPs: 4 core (10.10.10.70-73), +2 with workloads (.80, .81)
|
||||
|
||||
## Management
|
||||
|
||||
|
|
@ -229,14 +302,30 @@ installs and configures Prometheus, Grafana, Loki, and Promtail,
|
|||
provisions dashboards, sets up the OVN network forward, and configures
|
||||
ACLs.
|
||||
|
||||
### Deploy dummy workloads
|
||||
|
||||
```bash
|
||||
incusos/deploy-observability --workloads
|
||||
```
|
||||
|
||||
Deploys two lightweight containers that generate varied metric patterns:
|
||||
|
||||
- **workload-web** (10.10.10.80, oc-node-01): nginx serving static
|
||||
content, cron jobs generating HTTP traffic (every minute), CPU spikes
|
||||
via gzip (every 5 min), and disk I/O bursts via dd (every 15 min).
|
||||
- **workload-api** (10.10.10.81, oc-node-03): Python HTTP server on
|
||||
port 8080, cron jobs for inter-container traffic (every 2 min) and
|
||||
memory sawtooth via 20 MiB allocation/release (every 10 min).
|
||||
|
||||
### Check status
|
||||
|
||||
```bash
|
||||
incusos/deploy-observability --status
|
||||
```
|
||||
|
||||
Shows the state of all containers, scrape target health, Grafana
|
||||
accessibility, and OVN forward configuration.
|
||||
Shows the state of all containers (including workloads if deployed),
|
||||
scrape target health, Grafana accessibility, and OVN forward
|
||||
configuration.
|
||||
|
||||
### Clean up
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue