diff --git a/incusos/deploy-observability b/incusos/deploy-observability
index 88533a5..48bca3b 100755
--- a/incusos/deploy-observability
+++ b/incusos/deploy-observability
@@ -88,6 +88,7 @@ ${BOLD}ACTIONS${RESET}
--deploy Full deploy: monitoring container, Prometheus, Grafana,
Loki, Promtail, node-exporters, ACLs, network forward,
HAProxy exporter route, dashboards
+ --workloads Deploy dummy workloads to generate varied metrics
--status Show status of all observability components
--cleanup Remove all observability infrastructure
--doctor Diagnose common issues
@@ -462,13 +463,14 @@ phase_install_grafana() {
apt-get install -y -qq grafana 2>&1 | tail -1
"
- # Provision datasources
+ # Provision datasources with explicit UIDs for dashboard portability
container_exec monitoring bash -c "mkdir -p /etc/grafana/provisioning/datasources
cat > /etc/grafana/provisioning/datasources/observability.yaml << 'DSCFG'
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
+ uid: prometheus
access: proxy
url: http://localhost:9090
isDefault: true
@@ -476,28 +478,12 @@ datasources:
- name: Loki
type: loki
+ uid: loki
access: proxy
url: http://localhost:3100
editable: true
DSCFG"
- # Provision dashboard directory
- container_exec monitoring bash -c "mkdir -p /etc/grafana/provisioning/dashboards
-cat > /etc/grafana/provisioning/dashboards/observability.yaml << 'DBCFG'
-apiVersion: 1
-providers:
- - name: Observability
- orgId: 1
- folder: ''
- type: file
- disableDeletion: false
- editable: true
- options:
- path: /var/lib/grafana/dashboards
- foldersFromFilesStructure: false
-DBCFG
-mkdir -p /var/lib/grafana/dashboards"
-
container_exec monitoring systemctl enable grafana-server
container_exec monitoring systemctl restart grafana-server
sleep 2
@@ -725,35 +711,269 @@ phase_create_network_forward() {
# ---------------------------------------------------------------------------
phase_upload_dashboards() {
- step "Phase 9: Upload Grafana dashboards"
+ step "Phase 9: Install Grafana dashboards"
- if ! dry_run_guard "Upload dashboards from ${DASHBOARDS_DIR}"; then
+ if ! dry_run_guard "Install dashboards via manage-dashboards"; then
return 0
fi
- if [[ ! -d "$DASHBOARDS_DIR" ]]; then
- warn "Dashboard directory not found: ${DASHBOARDS_DIR}"
- warn "Skipping dashboard upload -- create dashboards manually in Grafana."
+ local manage_script="${SCRIPT_DIR}/manage-dashboards"
+ if [[ ! -x "$manage_script" ]]; then
+ warn "manage-dashboards script not found at ${manage_script}"
+ warn "Falling back to direct file upload."
+
+ if [[ ! -d "$DASHBOARDS_DIR" ]]; then
+ warn "Dashboard directory not found: ${DASHBOARDS_DIR}"
+ return 0
+ fi
+
+ local count=0
+ for dashboard in "${DASHBOARDS_DIR}"/core/*.json; do
+ [[ -f "$dashboard" ]] || continue
+ local basename
+ basename=$(basename "$dashboard")
+ incus file push "$dashboard" "$(container_ref monitoring)/var/lib/grafana/dashboards/${basename}"
+ detail "Uploaded ${basename}"
+ count=$((count + 1))
+ done
+
+ if [[ $count -gt 0 ]]; then
+ container_exec monitoring systemctl restart grafana-server 2>/dev/null || true
+ success "${count} dashboard(s) uploaded (direct push)"
+ else
+ info "No dashboard files found in ${DASHBOARDS_DIR}/core/"
+ fi
return 0
fi
- local count=0
- for dashboard in "${DASHBOARDS_DIR}"/*.json; do
- [[ -f "$dashboard" ]] || continue
- local basename
- basename=$(basename "$dashboard")
- incus file push "$dashboard" "$(container_ref monitoring)/var/lib/grafana/dashboards/${basename}"
- detail "Uploaded ${basename}"
- count=$((count + 1))
+ # Wait for Grafana API to be ready
+ info "Waiting for Grafana API..."
+ local elapsed=0
+ while true; do
+ local code
+ code=$(container_exec monitoring curl -s -o /dev/null -w '%{http_code}' \
+ http://localhost:3000/api/health 2>/dev/null || echo "000")
+ if [[ "$code" == "200" ]]; then
+ break
+ fi
+ sleep 2
+ elapsed=$((elapsed + 2))
+ if [[ $elapsed -ge 30 ]]; then
+ warn "Grafana API not ready after 30s -- falling back to file push"
+ return 0
+ fi
done
- if [[ $count -gt 0 ]]; then
- # Restart Grafana to pick up new dashboards
- container_exec monitoring systemctl restart grafana-server 2>/dev/null || true
- success "${count} dashboard(s) uploaded"
- else
- info "No dashboard files found in ${DASHBOARDS_DIR}"
+ # Use manage-dashboards to install with proper folder organization
+ local flags="-r ${CLUSTER_REMOTE}"
+ [[ "$VERBOSE" == true ]] && flags="${flags} -v"
+ [[ "$QUIET" == true ]] && flags="${flags} -q"
+
+ "${manage_script}" --install ${flags}
+}
+
+# ---------------------------------------------------------------------------
+# Workloads action
+# ---------------------------------------------------------------------------
+
+WORKLOAD_WEB_IP="10.10.10.80"
+WORKLOAD_API_IP="10.10.10.81"
+
+action_workloads() {
+ step "Deploying dummy workloads for metric generation"
+ echo
+ info " workload-web: ${WORKLOAD_WEB_IP} (nginx + CPU/disk/network crons)"
+ info " workload-api: ${WORKLOAD_API_IP} (Python HTTP + memory/network crons)"
+ echo
+
+ phase_deploy_workload_web
+ phase_deploy_workload_api
+
+ echo
+ success "Dummy workloads deployed!"
+ info " Metrics will appear within 1-2 minutes"
+ info " workload-web: nginx on ${WORKLOAD_WEB_IP}:80, CPU/disk/net crons"
+ info " workload-api: Python on ${WORKLOAD_API_IP}:8080, memory sawtooth cron"
+}
+
+phase_deploy_workload_web() {
+ step "Deploy workload-web (nginx + varied load)"
+
+ if container_exists "workload-web"; then
+ if container_running "workload-web"; then
+ success "workload-web already running"
+ return 0
+ fi
fi
+
+ if ! dry_run_guard "Launch workload-web on ${OVN_NETWORK} at ${WORKLOAD_WEB_IP}"; then
+ return 0
+ fi
+
+ incus launch images:alpine/3.20 "$(container_ref workload-web)" \
+ -n "$OVN_NETWORK" \
+ --target "oc-node-01" \
+ -c limits.memory=128MiB
+
+ wait_for_container_agent workload-web 30
+
+ container_exec workload-web sh -c "
+ apk add --no-cache nginx curl gzip coreutils > /dev/null 2>&1
+
+ # Configure nginx
+ mkdir -p /var/www/html
+ cat > /var/www/html/index.html << 'WEBHTML'
+workload-web
Observability test workload
+WEBHTML
+ cat > /etc/nginx/http.d/default.conf << 'NGXCFG'
+server {
+ listen 80 default_server;
+ root /var/www/html;
+ index index.html;
+ location / { try_files \$uri \$uri/ =404; }
+}
+NGXCFG
+ rc-update add nginx default 2>/dev/null || true
+ rc-service nginx start 2>/dev/null || true
+
+ # Cron: every minute -> 10 HTTP requests (network traffic)
+ # Cron: every 5 min -> gzip random data (CPU spike)
+ # Cron: every 15 min -> dd 5 MiB write+delete (disk I/O burst)
+ cat > /etc/crontabs/root << 'CRONCFG'
+* * * * * for i in \$(seq 1 10); do wget -q -O /dev/null http://localhost/ 2>/dev/null; done
+*/5 * * * * dd if=/dev/urandom bs=1M count=2 2>/dev/null | gzip > /dev/null 2>&1
+*/15 * * * * dd if=/dev/urandom of=/tmp/disktest bs=1M count=5 2>/dev/null && rm -f /tmp/disktest
+CRONCFG
+ rc-update add crond default 2>/dev/null || true
+ rc-service crond start 2>/dev/null || true
+ "
+
+ # Configure static IP
+ container_exec workload-web sh -c "
+ cat > /etc/network/interfaces << NETCFG
+auto lo
+iface lo inet loopback
+
+auto eth0
+iface eth0 inet static
+ address ${WORKLOAD_WEB_IP}/24
+ gateway 10.10.10.1
+NETCFG
+ rc-service networking restart 2>/dev/null || true
+ "
+
+ # Attach monitoring ACL
+ incus config device set "$(container_ref workload-web)" eth0 \
+ security.acls=monitoring-allow security.acls.default.ingress.action=allow \
+ security.acls.default.egress.action=allow 2>/dev/null || true
+
+ success "workload-web deployed at ${WORKLOAD_WEB_IP}"
+}
+
+phase_deploy_workload_api() {
+ step "Deploy workload-api (Python HTTP + memory patterns)"
+
+ if container_exists "workload-api"; then
+ if container_running "workload-api"; then
+ success "workload-api already running"
+ return 0
+ fi
+ fi
+
+ if ! dry_run_guard "Launch workload-api on ${OVN_NETWORK} at ${WORKLOAD_API_IP}"; then
+ return 0
+ fi
+
+ incus launch images:alpine/3.20 "$(container_ref workload-api)" \
+ -n "$OVN_NETWORK" \
+ --target "oc-node-03" \
+ -c limits.memory=128MiB
+
+ wait_for_container_agent workload-api 30
+
+ container_exec workload-api sh -c "
+ apk add --no-cache python3 curl coreutils > /dev/null 2>&1
+
+ # Create a simple JSON status endpoint
+ mkdir -p /opt/api
+ cat > /opt/api/server.py << 'PYAPI'
+import http.server
+import json
+import time
+import os
+
+class Handler(http.server.BaseHTTPRequestHandler):
+ def do_GET(self):
+ self.send_response(200)
+ self.send_header('Content-Type', 'application/json')
+ self.end_headers()
+ data = {
+ 'status': 'ok',
+ 'uptime': time.time() - START_TIME,
+ 'hostname': os.uname().nodename,
+ 'pid': os.getpid()
+ }
+ self.wfile.write(json.dumps(data).encode())
+
+ def log_message(self, format, *args):
+ pass # suppress access logs
+
+START_TIME = time.time()
+http.server.HTTPServer(('0.0.0.0', 8080), Handler).serve_forever()
+PYAPI
+
+ # Create memory stress script
+ cat > /opt/api/mem-stress.py << 'PYMEM'
+import time
+# Allocate ~20 MiB, hold for 30 seconds, then release
+data = bytearray(20 * 1024 * 1024)
+time.sleep(30)
+del data
+PYMEM
+
+ # Start the API server as a background service
+ cat > /etc/init.d/api-server << 'INITSCRIPT'
+#!/sbin/openrc-run
+name=\"api-server\"
+command=\"/usr/bin/python3\"
+command_args=\"/opt/api/server.py\"
+command_background=true
+pidfile=\"/run/api-server.pid\"
+INITSCRIPT
+ chmod +x /etc/init.d/api-server
+ rc-update add api-server default 2>/dev/null || true
+ rc-service api-server start 2>/dev/null || true
+
+ # Cron: every 2 min -> curl workload-web (inter-container traffic)
+ # Cron: every 10 min -> allocate 20 MiB for 30s (memory sawtooth)
+ cat > /etc/crontabs/root << 'CRONCFG'
+*/2 * * * * wget -q -O /dev/null http://${WORKLOAD_WEB_IP}/ 2>/dev/null
+*/10 * * * * python3 /opt/api/mem-stress.py 2>/dev/null &
+CRONCFG
+ rc-update add crond default 2>/dev/null || true
+ rc-service crond start 2>/dev/null || true
+ "
+
+ # Configure static IP
+ container_exec workload-api sh -c "
+ cat > /etc/network/interfaces << NETCFG
+auto lo
+iface lo inet loopback
+
+auto eth0
+iface eth0 inet static
+ address ${WORKLOAD_API_IP}/24
+ gateway 10.10.10.1
+NETCFG
+ rc-service networking restart 2>/dev/null || true
+ "
+
+ # Attach monitoring ACL
+ incus config device set "$(container_ref workload-api)" eth0 \
+ security.acls=monitoring-allow security.acls.default.ingress.action=allow \
+ security.acls.default.egress.action=allow 2>/dev/null || true
+
+ success "workload-api deployed at ${WORKLOAD_API_IP}"
}
# ---------------------------------------------------------------------------
@@ -798,6 +1018,17 @@ action_status() {
idx=$((idx + 1))
done
+ # Workload containers (optional)
+ for wl in workload-web workload-api; do
+ if container_exists "$wl"; then
+ if container_running "$wl"; then
+ success " ${wl}: running"
+ else
+ warn " ${wl}: stopped"
+ fi
+ fi
+ done
+
if [[ "$all_ok" == false ]]; then
echo
error "Some containers are missing or stopped."
@@ -914,6 +1145,9 @@ action_cleanup() {
if container_exists "$(node_exp_name "$idx")"; then has_anything=true; fi
idx=$((idx + 1))
done
+ for wl in workload-web workload-api; do
+ if container_exists "$wl"; then has_anything=true; fi
+ done
if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
has_anything=true
fi
@@ -936,6 +1170,11 @@ action_cleanup() {
fi
idx=$((idx + 1))
done
+ for wl in workload-web workload-api; do
+ if container_exists "$wl"; then
+ warn " - ${wl}"
+ fi
+ done
if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
warn " - network forward ${FORWARD_VIP}"
fi
@@ -1003,6 +1242,15 @@ except: pass
idx=$((idx + 1))
done
+ # Delete workload containers
+ for wl in workload-web workload-api; do
+ if container_exists "$wl"; then
+ info "Deleting ${wl}..."
+ incus delete "$(container_ref "$wl")" --force 2>/dev/null || true
+ success "Deleted ${wl}"
+ fi
+ done
+
# Delete network forward
step "Removing network forward"
if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
@@ -1153,6 +1401,7 @@ parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--deploy) ACTION="deploy";;
+ --workloads) ACTION="workloads";;
--status) ACTION="status";;
--cleanup) ACTION="cleanup";;
--doctor) ACTION="doctor";;
@@ -1187,10 +1436,11 @@ main() {
echo
case "$ACTION" in
- deploy) action_deploy;;
- status) action_status;;
- cleanup) action_cleanup;;
- doctor) action_doctor;;
+ deploy) action_deploy;;
+ workloads) action_workloads;;
+ status) action_status;;
+ cleanup) action_cleanup;;
+ doctor) action_doctor;;
esac
}
diff --git a/incusos/manage-dashboards b/incusos/manage-dashboards
new file mode 100755
index 0000000..a1b21ed
--- /dev/null
+++ b/incusos/manage-dashboards
@@ -0,0 +1,854 @@
+#!/usr/bin/env bash
+# manage-dashboards - Manage Grafana dashboards for the Incus observability stack
+#
+# Push, export, validate, and inspect Grafana dashboards via the Grafana HTTP
+# API. Dashboards are organized into folders and use datasource UID variables
+# for portability across deployments.
+#
+# Usage: manage-dashboards [OPTIONS]
+# Run 'manage-dashboards --help' for full usage information.
+
+set -euo pipefail
+
+readonly VERSION="0.1.0"
+readonly SCRIPT_NAME="manage-dashboards"
+readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly DASHBOARDS_DIR="${SCRIPT_DIR}/observability-dashboards"
+readonly MANIFEST_FILE="${DASHBOARDS_DIR}/dashboard.yaml"
+
+# ---------------------------------------------------------------------------
+# Defaults (overridden by CLI flags)
+# ---------------------------------------------------------------------------
+
+CLUSTER_REMOTE="oc-node-01"
+MONITORING_CONTAINER="monitoring"
+GRAFANA_URL="http://localhost:3000"
+GRAFANA_USER="admin"
+GRAFANA_PASSWORD="admin"
+
+# Runtime flags
+DRY_RUN=false
+VERBOSE=false
+QUIET=false
+ACTION=""
+FILTER_CATEGORY=""
+FILTER_FILE=""
+FILTER_UID=""
+
+# ---------------------------------------------------------------------------
+# Color and formatting
+# ---------------------------------------------------------------------------
+
+setup_colors() {
+ if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
+ RED='\033[0;31m'
+ GREEN='\033[0;32m'
+ YELLOW='\033[0;33m'
+ BLUE='\033[0;34m'
+ CYAN='\033[0;36m'
+ BOLD='\033[1m'
+ DIM='\033[2m'
+ RESET='\033[0m'
+ else
+ RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET=''
+ fi
+}
+
+info() { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET} $*"; }
+success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET} $*"; }
+warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; }
+error() { echo -e "${RED}[error]${RESET} $*" >&2; }
+step() { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; }
+detail() { [[ "$VERBOSE" != true ]] && return; echo -e "${DIM} $*${RESET}"; }
+
+dry_run_guard() {
+ if [[ "$DRY_RUN" == true ]]; then
+ info "(dry-run) $*"
+ return 1
+ fi
+ return 0
+}
+
+# ---------------------------------------------------------------------------
+# Help
+# ---------------------------------------------------------------------------
+
+usage() {
+ cat </dev/null || echo "")
+
+ LOKI_UID=$(echo "$ds_json" | python3 -c "
+import sys, json
+ds = json.load(sys.stdin)
+for d in ds:
+ if d.get('type') == 'loki':
+ print(d['uid'])
+ break
+" 2>/dev/null || echo "")
+
+ if [[ -z "$PROMETHEUS_UID" ]]; then
+ error "Could not find Prometheus datasource in Grafana"
+ return 1
+ fi
+ detail "Prometheus datasource UID: ${PROMETHEUS_UID}"
+ detail "Loki datasource UID: ${LOKI_UID:-not found}"
+}
+
+# Get or create a Grafana folder, returns folder ID
+ensure_folder() {
+ local folder_title="$1"
+
+ # Check if folder exists
+ local folder_id
+ folder_id=$(grafana_api GET "/api/folders" | python3 -c "
+import sys, json
+folders = json.load(sys.stdin)
+for f in folders:
+ if f.get('title') == '${folder_title}':
+ print(f['id'])
+ break
+" 2>/dev/null || echo "")
+
+ if [[ -n "$folder_id" ]]; then
+ detail "Folder '${folder_title}' exists (ID: ${folder_id})"
+ echo "$folder_id"
+ return 0
+ fi
+
+ # Create folder
+ local folder_uid
+ folder_uid=$(echo "$folder_title" | tr '[:upper:]' '[:lower:]' | tr ' ' '-')
+ local result
+ result=$(grafana_api POST "/api/folders" \
+ -H "Content-Type: application/json" \
+ -d "{\"title\": \"${folder_title}\", \"uid\": \"${folder_uid}\"}")
+
+ folder_id=$(echo "$result" | python3 -c "
+import sys, json
+print(json.load(sys.stdin).get('id', ''))
+" 2>/dev/null || echo "")
+
+ if [[ -n "$folder_id" ]]; then
+ detail "Created folder '${folder_title}' (ID: ${folder_id})"
+ echo "$folder_id"
+ else
+ error "Failed to create folder '${folder_title}': ${result}"
+ return 1
+ fi
+}
+
+# Read manifest and output entries as tab-separated lines
+read_manifest() {
+ if [[ ! -f "$MANIFEST_FILE" ]]; then
+ error "Dashboard manifest not found: ${MANIFEST_FILE}"
+ error "Expected a YAML file listing dashboard metadata."
+ return 1
+ fi
+
+ python3 -c "
+import yaml, sys
+
+with open('${MANIFEST_FILE}') as f:
+ data = yaml.safe_load(f)
+
+for d in data.get('dashboards', []):
+ file_path = d.get('file', '')
+ uid = d.get('uid', '')
+ title = d.get('title', '')
+ folder = d.get('folder', 'General')
+ category = d.get('category', '')
+ version = d.get('version', '1')
+ description = d.get('description', '')
+ print(f'{file_path}\t{uid}\t{title}\t{folder}\t{category}\t{version}\t{description}')
+" 2>/dev/null
+}
+
+# Substitute datasource variables in JSON content
+inject_datasource_uids() {
+ local json_content="$1"
+ echo "$json_content" \
+ | sed "s/\\\${DS_PROMETHEUS}/${PROMETHEUS_UID}/g" \
+ | sed "s/\\\${DS_LOKI}/${LOKI_UID}/g"
+}
+
+# Normalize datasource UIDs back to variables for portability
+normalize_datasource_uids() {
+ local json_content="$1"
+ local prom_uid="$2"
+ local loki_uid="$3"
+
+ local result="$json_content"
+ if [[ -n "$prom_uid" ]]; then
+ result=$(echo "$result" | sed "s/${prom_uid}/\\\${DS_PROMETHEUS}/g")
+ fi
+ if [[ -n "$loki_uid" ]]; then
+ result=$(echo "$result" | sed "s/${loki_uid}/\\\${DS_LOKI}/g")
+ fi
+ echo "$result"
+}
+
+# ---------------------------------------------------------------------------
+# Install action
+# ---------------------------------------------------------------------------
+
+action_install() {
+ step "Installing dashboards to Grafana"
+ echo
+
+ if ! dry_run_guard "Resolve datasource UIDs and push dashboards"; then
+ info "(dry-run) Would install dashboards from ${DASHBOARDS_DIR}"
+
+ local count=0
+ while IFS=$'\t' read -r file uid title folder category version description; do
+ if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
+ continue
+ fi
+ if [[ -n "$FILTER_FILE" && "$file" != "$FILTER_FILE" ]]; then
+ continue
+ fi
+ info "(dry-run) Would install: ${title} -> folder '${folder}'"
+ count=$((count + 1))
+ done < <(read_manifest)
+
+ info "(dry-run) ${count} dashboard(s) would be installed"
+ return 0
+ fi
+
+ resolve_datasource_uids
+
+ local installed=0
+ local failed=0
+
+ while IFS=$'\t' read -r file uid title folder category version description; do
+ if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
+ continue
+ fi
+ if [[ -n "$FILTER_FILE" && "$file" != "$FILTER_FILE" ]]; then
+ continue
+ fi
+
+ local full_path="${DASHBOARDS_DIR}/${file}"
+ if [[ ! -f "$full_path" ]]; then
+ warn "Dashboard file not found: ${full_path}"
+ failed=$((failed + 1))
+ continue
+ fi
+
+ # Get or create the target folder
+ local folder_id
+ folder_id=$(ensure_folder "$folder") || { failed=$((failed + 1)); continue; }
+
+ # Read and process the dashboard JSON
+ local raw_json
+ raw_json=$(cat "$full_path")
+
+ # Inject actual datasource UIDs
+ local processed_json
+ processed_json=$(inject_datasource_uids "$raw_json")
+
+ # Wrap in Grafana API envelope
+ local api_payload
+ api_payload=$(python3 -c "
+import json, sys
+
+dashboard = json.loads('''${processed_json//\'/\\\'}''')
+
+# Remove Grafana-internal fields that shouldn't be in the import
+dashboard.pop('id', None)
+dashboard.pop('version', None)
+dashboard.pop('iteration', None)
+
+payload = {
+ 'dashboard': dashboard,
+ 'folderId': ${folder_id},
+ 'overwrite': True,
+ 'message': 'Installed by manage-dashboards v${VERSION}'
+}
+print(json.dumps(payload))
+" 2>/dev/null)
+
+ if [[ -z "$api_payload" ]]; then
+ # Fallback: use a simpler approach for large/complex JSON
+ api_payload=$(python3 << 'PYEOF'
+import json, sys
+
+with open(sys.argv[1]) as f:
+ dashboard = json.load(f)
+
+# Inject datasource UIDs
+json_str = json.dumps(dashboard)
+json_str = json_str.replace('${DS_PROMETHEUS}', sys.argv[2])
+json_str = json_str.replace('${DS_LOKI}', sys.argv[3])
+dashboard = json.loads(json_str)
+
+# Remove Grafana-internal fields
+dashboard.pop('id', None)
+dashboard.pop('version', None)
+dashboard.pop('iteration', None)
+
+payload = {
+ 'dashboard': dashboard,
+ 'folderId': int(sys.argv[4]),
+ 'overwrite': True,
+ 'message': 'Installed by manage-dashboards'
+}
+print(json.dumps(payload))
+PYEOF
+ "$full_path" "$PROMETHEUS_UID" "${LOKI_UID:-}" "$folder_id" 2>/dev/null)
+ fi
+
+ if [[ -z "$api_payload" ]]; then
+ error "Failed to process dashboard JSON: ${file}"
+ failed=$((failed + 1))
+ continue
+ fi
+
+ # Push via API - write payload to temp file to avoid command-line size limits
+ local tmpfile
+ tmpfile=$(mktemp /tmp/dashboard-payload.XXXXXX)
+ echo "$api_payload" > "$tmpfile"
+
+ local result
+ result=$(incus file push "$tmpfile" "${CLUSTER_REMOTE}:${MONITORING_CONTAINER}/tmp/dashboard-payload.json" 2>/dev/null && \
+ grafana_api POST "/api/dashboards/db" \
+ -H "Content-Type: application/json" \
+ -d @/tmp/dashboard-payload.json 2>/dev/null)
+ rm -f "$tmpfile"
+
+ local status_str
+ status_str=$(echo "$result" | python3 -c "
+import sys, json
+try:
+ d = json.load(sys.stdin)
+ print(d.get('status', d.get('message', 'unknown')))
+except:
+ print('error')
+" 2>/dev/null || echo "error")
+
+ if [[ "$status_str" == "success" ]]; then
+ success " ${title} -> ${folder}"
+ installed=$((installed + 1))
+ else
+ error " ${title}: ${status_str}"
+ detail " Response: ${result}"
+ failed=$((failed + 1))
+ fi
+
+ # Clean up temp file in container
+ incus exec "${CLUSTER_REMOTE}:${MONITORING_CONTAINER}" -- rm -f /tmp/dashboard-payload.json 2>/dev/null || true
+
+ done < <(read_manifest)
+
+ echo
+ if [[ $failed -eq 0 ]]; then
+ success "Installed ${installed} dashboard(s)"
+ else
+ warn "Installed ${installed}, failed ${failed}"
+ fi
+}
+
+# ---------------------------------------------------------------------------
+# Export action
+# ---------------------------------------------------------------------------
+
+action_export() {
+ step "Exporting dashboards from Grafana"
+ echo
+
+ resolve_datasource_uids
+
+ local exported=0
+
+ if [[ -n "$FILTER_UID" ]]; then
+ # Export a single dashboard by UID
+ export_single_dashboard "$FILTER_UID"
+ return $?
+ fi
+
+ # Export all dashboards listed in the manifest
+ while IFS=$'\t' read -r file uid title folder category version description; do
+ if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
+ continue
+ fi
+
+ if export_single_dashboard "$uid" "${DASHBOARDS_DIR}/${file}"; then
+ exported=$((exported + 1))
+ fi
+ done < <(read_manifest)
+
+ echo
+ success "Exported ${exported} dashboard(s)"
+}
+
+export_single_dashboard() {
+ local uid="$1"
+ local output_file="${2:-}"
+
+ # Fetch from Grafana API
+ local raw_response
+ raw_response=$(grafana_api GET "/api/dashboards/uid/${uid}" 2>/dev/null)
+
+ local has_dashboard
+ has_dashboard=$(echo "$raw_response" | python3 -c "
+import sys, json
+try:
+ d = json.load(sys.stdin)
+ if 'dashboard' in d:
+ print('yes')
+ else:
+ print('no')
+except:
+ print('no')
+" 2>/dev/null || echo "no")
+
+ if [[ "$has_dashboard" != "yes" ]]; then
+ warn "Dashboard '${uid}' not found in Grafana"
+ return 1
+ fi
+
+ # If no output file specified, find it in the manifest
+ if [[ -z "$output_file" ]]; then
+ output_file=$(read_manifest | while IFS=$'\t' read -r file muid title folder category version description; do
+ if [[ "$muid" == "$uid" ]]; then
+ echo "${DASHBOARDS_DIR}/${file}"
+ break
+ fi
+ done)
+ fi
+
+ if [[ -z "$output_file" ]]; then
+ output_file="${DASHBOARDS_DIR}/core/${uid}.json"
+ warn "Dashboard '${uid}' not in manifest, exporting to ${output_file}"
+ fi
+
+ if ! dry_run_guard "Export ${uid} to ${output_file}"; then
+ return 0
+ fi
+
+ # Clean and normalize the dashboard JSON
+ python3 << PYEOF > "$output_file"
+import json, sys
+
+raw = json.loads('''$(echo "$raw_response" | sed "s/'/\\\\'/g")''')
+dashboard = raw['dashboard']
+
+# Remove Grafana-internal fields
+for key in ['id', 'version', 'iteration']:
+ dashboard.pop(key, None)
+
+# Normalize datasource UIDs to variables
+json_str = json.dumps(dashboard, indent=2)
+prom_uid = "${PROMETHEUS_UID}"
+loki_uid = "${LOKI_UID:-}"
+if prom_uid:
+ json_str = json_str.replace(prom_uid, r'\${DS_PROMETHEUS}')
+if loki_uid:
+ json_str = json_str.replace(loki_uid, r'\${DS_LOKI}')
+
+# Re-parse to ensure valid JSON
+dashboard = json.loads(json_str.replace(r'\${DS_PROMETHEUS}', '\${DS_PROMETHEUS}').replace(r'\${DS_LOKI}', '\${DS_LOKI}'))
+
+# Write with consistent formatting
+json_str = json.dumps(dashboard, indent=2)
+# Restore the variable syntax that json.dumps may have escaped
+json_str = json_str.replace(r'\${DS_PROMETHEUS}', '\${DS_PROMETHEUS}')
+json_str = json_str.replace(r'\${DS_LOKI}', '\${DS_LOKI}')
+print(json_str)
+PYEOF
+
+ if [[ $? -eq 0 ]]; then
+ success " Exported: ${uid} -> $(basename "$output_file")"
+ return 0
+ else
+ error " Failed to export: ${uid}"
+ return 1
+ fi
+}
+
+# ---------------------------------------------------------------------------
+# List action
+# ---------------------------------------------------------------------------
+
+action_list() {
+ step "Dashboard Inventory"
+ echo
+
+ # Try to get installed dashboards from Grafana
+ local installed_json=""
+ installed_json=$(grafana_api GET "/api/search?type=dash-db" 2>/dev/null || echo "[]")
+
+ printf " ${BOLD}%-35s %-30s %-20s %s${RESET}\n" "UID" "Title" "Folder" "Status"
+ printf " %-35s %-30s %-20s %s\n" "---" "-----" "------" "------"
+
+ while IFS=$'\t' read -r file uid title folder category version description; do
+ if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
+ continue
+ fi
+
+ local full_path="${DASHBOARDS_DIR}/${file}"
+ local local_status="missing"
+ if [[ -f "$full_path" ]]; then
+ local_status="local"
+ fi
+
+ # Check if installed in Grafana
+ local grafana_status
+ grafana_status=$(echo "$installed_json" | python3 -c "
+import sys, json
+try:
+ dashboards = json.load(sys.stdin)
+ for d in dashboards:
+ if d.get('uid') == '${uid}':
+ print('installed')
+ break
+ else:
+ print('not installed')
+except:
+ print('unknown')
+" 2>/dev/null || echo "unknown")
+
+ local status_display
+ if [[ "$local_status" == "missing" ]]; then
+ status_display="${RED}file missing${RESET}"
+ elif [[ "$grafana_status" == "installed" ]]; then
+ status_display="${GREEN}installed${RESET}"
+ elif [[ "$grafana_status" == "not installed" ]]; then
+ status_display="${YELLOW}not installed${RESET}"
+ else
+ status_display="${DIM}${local_status}${RESET}"
+ fi
+
+ printf " %-35s %-30s %-20s " "$uid" "$title" "$folder"
+ echo -e "$status_display"
+
+ done < <(read_manifest)
+}
+
+# ---------------------------------------------------------------------------
+# Validate action
+# ---------------------------------------------------------------------------
+
+action_validate() {
+ step "Validating dashboard JSON files"
+ echo
+
+ local total=0
+ local passed=0
+ local warnings=0
+ local errors=0
+
+ while IFS=$'\t' read -r file uid title folder category version description; do
+ if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
+ continue
+ fi
+
+ local full_path="${DASHBOARDS_DIR}/${file}"
+ total=$((total + 1))
+
+ if [[ ! -f "$full_path" ]]; then
+ error " ${file}: FILE NOT FOUND"
+ errors=$((errors + 1))
+ continue
+ fi
+
+ local issues=0
+ local warns=0
+ local basename_file
+ basename_file=$(basename "$full_path")
+
+ # Check 1: Valid JSON
+ if ! python3 -c "import json; json.load(open('${full_path}'))" 2>/dev/null; then
+ error " ${basename_file}: INVALID JSON"
+ errors=$((errors + 1))
+ continue
+ fi
+
+ # Run all checks via python
+ local check_result
+ check_result=$(python3 << PYEOF
+import json
+
+with open('${full_path}') as f:
+ d = json.load(f)
+
+issues = []
+warns = []
+
+# Check 2: UID present and matches manifest
+json_uid = d.get('uid', '')
+if not json_uid:
+ issues.append('missing uid field')
+elif json_uid != '${uid}':
+ issues.append(f'uid mismatch: JSON has "{json_uid}", manifest has "${uid}"')
+
+# Check 3: Title present
+if not d.get('title'):
+ issues.append('missing title field')
+
+# Check 4: Description present
+if not d.get('description'):
+ warns.append('missing description field')
+
+# Check 5: No hardcoded datasource UIDs (should use variables)
+json_str = json.dumps(d)
+# Known Grafana internal UIDs that are OK
+safe_uids = ['-- Grafana --']
+# Check for UIDs that look like Grafana-generated ones
+import re
+# Match patterns like "uid": "PBFA97CFB590B2093" (hex-like, not a variable)
+uid_pattern = re.compile(r'"uid":\s*"([A-Za-z0-9]{16,})"')
+for match in uid_pattern.finditer(json_str):
+ found_uid = match.group(1)
+ if found_uid not in safe_uids and not found_uid.startswith('\${'):
+ issues.append(f'hardcoded datasource UID: {found_uid}')
+ break
+
+# Check 6: Datasource variable usage
+if '\${DS_PROMETHEUS}' not in json_str and '\${DS_LOKI}' not in json_str:
+ # Check if it has any datasource references at all
+ if '"type": "prometheus"' in json_str or '"type": "loki"' in json_str:
+ warns.append('no datasource variables found (uses hardcoded UIDs?)')
+
+# Check 7: Has panels
+panels = d.get('panels', [])
+if not panels:
+ warns.append('no panels defined')
+
+# Check 8: Check for id field (should not be present in stored files)
+if 'id' in d and d['id'] is not None:
+ warns.append('contains "id" field (should be null or absent for portability)')
+
+for i in issues:
+ print(f'ERROR:{i}')
+for w in warns:
+ print(f'WARN:{w}')
+if not issues:
+ print('OK')
+PYEOF
+ )
+
+ local has_error=false
+ while IFS= read -r line; do
+ case "$line" in
+ ERROR:*)
+ error " ${basename_file}: ${line#ERROR:}"
+ issues=$((issues + 1))
+ has_error=true
+ ;;
+ WARN:*)
+ warn " ${basename_file}: ${line#WARN:}"
+ warns=$((warns + 1))
+ ;;
+ OK)
+ ;;
+ esac
+ done <<< "$check_result"
+
+ if [[ "$has_error" == true ]]; then
+ errors=$((errors + 1))
+ else
+ if [[ $warns -gt 0 ]]; then
+ warnings=$((warnings + warns))
+ fi
+ passed=$((passed + 1))
+ success " ${basename_file}: valid"
+ fi
+
+ done < <(read_manifest)
+
+ echo
+ info "${total} dashboard(s) checked: ${passed} passed, ${errors} errors, ${warnings} warnings"
+
+ if [[ $errors -gt 0 ]]; then
+ return 1
+ fi
+}
+
+# ---------------------------------------------------------------------------
+# Status action
+# ---------------------------------------------------------------------------
+
+action_status() {
+ step "Dashboard Status (local vs installed)"
+ echo
+
+ local installed_json
+ installed_json=$(grafana_api GET "/api/search?type=dash-db" 2>/dev/null || echo "[]")
+
+ printf " ${BOLD}%-30s %-15s %-15s %s${RESET}\n" "Dashboard" "Local" "Grafana" "Match"
+ printf " %-30s %-15s %-15s %s\n" "---------" "-----" "-------" "-----"
+
+ while IFS=$'\t' read -r file uid title folder category version description; do
+ local full_path="${DASHBOARDS_DIR}/${file}"
+
+ local local_ver="missing"
+ if [[ -f "$full_path" ]]; then
+ local_ver="v${version}"
+ fi
+
+ local grafana_ver
+ grafana_ver=$(echo "$installed_json" | python3 -c "
+import sys, json
+try:
+ for d in json.load(sys.stdin):
+ if d.get('uid') == '${uid}':
+ print('installed')
+ break
+ else:
+ print('absent')
+except:
+ print('unknown')
+" 2>/dev/null || echo "unknown")
+
+ local match_str
+ if [[ "$local_ver" == "missing" ]]; then
+ match_str="${RED}file missing${RESET}"
+ elif [[ "$grafana_ver" == "absent" ]]; then
+ match_str="${YELLOW}not installed${RESET}"
+ elif [[ "$grafana_ver" == "installed" ]]; then
+ match_str="${GREEN}synced${RESET}"
+ else
+ match_str="${DIM}unknown${RESET}"
+ fi
+
+ printf " %-30s %-15s %-15s " "$title" "$local_ver" "$grafana_ver"
+ echo -e "$match_str"
+
+ done < <(read_manifest)
+}
+
+# ---------------------------------------------------------------------------
+# Argument parsing
+# ---------------------------------------------------------------------------
+
+parse_args() {
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --install) ACTION="install";;
+ --export) ACTION="export";;
+ --list) ACTION="list";;
+ --validate) ACTION="validate";;
+ --status) ACTION="status";;
+ --category) FILTER_CATEGORY="$2"; shift;;
+ --file) FILTER_FILE="$2"; shift;;
+ --uid) FILTER_UID="$2"; shift;;
+ -r|--remote) CLUSTER_REMOTE="$2"; shift;;
+ -n|--dry-run) DRY_RUN=true;;
+ -v|--verbose) VERBOSE=true;;
+ -q|--quiet) QUIET=true;;
+ -h|--help) usage; exit 0;;
+ *)
+ error "Unknown option: $1"
+ echo "Run '${SCRIPT_NAME} --help' for usage."
+ exit 1
+ ;;
+ esac
+ shift
+ done
+
+ if [[ -z "$ACTION" ]]; then
+ error "No action specified"
+ echo "Run '${SCRIPT_NAME} --help' for usage."
+ exit 1
+ fi
+}
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+main() {
+ setup_colors
+ parse_args "$@"
+
+ echo
+ case "$ACTION" in
+ install) action_install;;
+ export) action_export;;
+ list) action_list;;
+ validate) action_validate;;
+ status) action_status;;
+ esac
+}
+
+main "$@"
diff --git a/incusos/observability-dashboards/community/.gitkeep b/incusos/observability-dashboards/community/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/incusos/observability-dashboards/host-resources.json b/incusos/observability-dashboards/core/capacity-planning.json
similarity index 53%
rename from incusos/observability-dashboards/host-resources.json
rename to incusos/observability-dashboards/core/capacity-planning.json
index 282f7fa..2356711 100644
--- a/incusos/observability-dashboards/host-resources.json
+++ b/incusos/observability-dashboards/core/capacity-planning.json
@@ -1,31 +1,35 @@
{
- "uid": "incus-host-resources",
- "title": "Host Resources (IncusOS Nodes)",
- "description": "Physical host monitoring for IncusOS cluster nodes: CPU, memory, disk, and network from node_exporter metrics.",
- "tags": ["incusos", "host", "node-exporter"],
+ "uid": "incus-capacity-planning",
+ "title": "Capacity Planning",
+ "description": "CPU/memory headroom per node, resource trends, disk space forecast with predict_linear, instance count trends.",
+ "tags": ["incus", "capacity", "operations"],
"timezone": "browser",
"schemaVersion": 39,
- "version": 1,
- "refresh": "30s",
- "time": {
- "from": "now-1h",
- "to": "now"
- },
+ "refresh": "5m",
+ "time": { "from": "now-7d", "to": "now" },
"fiscalYearStartMonth": 0,
"liveNow": false,
"editable": true,
"graphTooltip": 1,
- "templating": {
- "list": []
- },
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": ["incus"],
+ "targetBlank": false,
+ "title": "Incus Dashboards",
+ "tooltip": "Navigate to other Incus dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "templating": { "list": [] },
"annotations": {
"list": [
{
"builtIn": 1,
- "datasource": {
- "type": "grafana",
- "uid": "-- Grafana --"
- },
+ "datasource": { "type": "grafana", "uid": "-- Grafana --" },
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
@@ -37,25 +41,114 @@
"panels": [
{
"type": "row",
- "title": "CPU",
+ "title": "Current Headroom",
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
"id": 1,
"panels": []
},
{
- "type": "timeseries",
- "title": "CPU Usage % by Node",
- "description": "Overall CPU utilization per host node, calculated as 1 minus the average idle CPU rate.",
- "gridPos": { "h": 8, "w": 24, "x": 0, "y": 1 },
+ "type": "bargauge",
+ "title": "CPU Headroom per Node",
+ "description": "Available (idle) CPU percentage per host node. Higher values mean more headroom for additional workloads.",
+ "gridPos": { "h": 6, "w": 12, "x": 0, "y": 1 },
"id": 2,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"targets": [
{
- "expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval]))) * 100",
+ "expr": "avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])) * 100",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "red", "value": null },
+ { "color": "orange", "value": 20 },
+ { "color": "yellow", "value": 40 },
+ { "color": "green", "value": 60 }
+ ]
+ },
+ "unit": "percent",
+ "min": 0,
+ "max": 100,
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "horizontal",
+ "displayMode": "gradient",
+ "showUnfilled": true,
+ "minVizWidth": 8,
+ "minVizHeight": 16,
+ "valueMode": "color"
+ }
+ },
+ {
+ "type": "bargauge",
+ "title": "Memory Headroom per Node",
+ "description": "Available memory percentage per host node. Higher values mean more headroom for additional workloads.",
+ "gridPos": { "h": 6, "w": 12, "x": 12, "y": 1 },
+ "id": 3,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "(node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "red", "value": null },
+ { "color": "orange", "value": 20 },
+ { "color": "yellow", "value": 40 },
+ { "color": "green", "value": 60 }
+ ]
+ },
+ "unit": "percent",
+ "min": 0,
+ "max": 100,
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "horizontal",
+ "displayMode": "gradient",
+ "showUnfilled": true,
+ "minVizWidth": 8,
+ "minVizHeight": 16,
+ "valueMode": "color"
+ }
+ },
+ {
+ "type": "row",
+ "title": "Resource Trends",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 },
+ "id": 4,
+ "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "CPU Usage Trend (7-day)",
+ "description": "CPU utilization percentage per host node over the past 7 days. Helps identify sustained usage patterns and growth trends.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+ "id": 5,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100",
"legendFormat": "{{instance}}",
"refId": "A"
}
@@ -75,7 +168,7 @@
"axisPlacement": "auto",
"axisLabel": "",
"scaleDistribution": { "type": "linear" },
- "thresholdsStyle": { "mode": "line" }
+ "thresholdsStyle": { "mode": "off" }
},
"unit": "percent",
"min": 0,
@@ -97,24 +190,13 @@
"legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
}
},
- {
- "type": "row",
- "title": "Memory",
- "collapsed": false,
- "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
- "id": 3,
- "panels": []
- },
{
"type": "timeseries",
- "title": "Memory Usage % by Node",
- "description": "Memory utilization percentage per host node, calculated from available vs total memory.",
- "gridPos": { "h": 8, "w": 24, "x": 0, "y": 10 },
- "id": 4,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
+ "title": "Memory Usage Trend (7-day)",
+ "description": "Memory utilization percentage per host node over the past 7 days. Helps identify sustained usage patterns and growth trends.",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+ "id": 6,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"targets": [
{
"expr": "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100",
@@ -137,7 +219,7 @@
"axisPlacement": "auto",
"axisLabel": "",
"scaleDistribution": { "type": "linear" },
- "thresholdsStyle": { "mode": "line" }
+ "thresholdsStyle": { "mode": "off" }
},
"unit": "percent",
"min": 0,
@@ -161,27 +243,29 @@
},
{
"type": "row",
- "title": "Disk",
+ "title": "Disk Space Forecast",
"collapsed": false,
- "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 },
- "id": 5,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 16 },
+ "id": 7,
"panels": []
},
{
"type": "timeseries",
- "title": "Root Filesystem Available Space",
- "description": "Available disk space on the root filesystem (mountpoint='/') per host node.",
- "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 },
- "id": 6,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
+ "title": "Disk Space Forecast (30-day projection)",
+ "description": "Current available disk space on root filesystem with a 30-day linear projection based on the past 7 days of data. Dashed lines show the forecast; if a forecast line drops below zero, the disk will fill up within 30 days at current consumption rates.",
+ "gridPos": { "h": 8, "w": 24, "x": 0, "y": 17 },
+ "id": 8,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"targets": [
{
"expr": "node_filesystem_avail_bytes{mountpoint=\"/\"}",
- "legendFormat": "{{instance}}",
+ "legendFormat": "{{instance}} actual",
"refId": "A"
+ },
+ {
+ "expr": "predict_linear(node_filesystem_avail_bytes{mountpoint=\"/\"}[7d], 30*24*3600)",
+ "legendFormat": "{{instance}} forecast",
+ "refId": "B"
}
],
"fieldConfig": {
@@ -189,7 +273,7 @@
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
- "lineWidth": 1,
+ "lineWidth": 2,
"fillOpacity": 15,
"gradientMode": "none",
"spanNulls": false,
@@ -205,140 +289,43 @@
"color": { "mode": "palette-classic" },
"thresholds": {
"mode": "absolute",
- "steps": [
- { "color": "green", "value": null }
- ]
+ "steps": [{ "color": "green", "value": null }]
}
},
- "overrides": []
+ "overrides": [
+ {
+ "matcher": { "id": "byRegexp", "options": ".*forecast$" },
+ "properties": [
+ { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } },
+ { "id": "custom.fillOpacity", "value": 0 }
+ ]
+ }
+ ]
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
- "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull", "min"] }
- }
- },
- {
- "type": "timeseries",
- "title": "Disk Read Rate by Node",
- "description": "Disk read throughput (bytes/sec) aggregated per host node.",
- "gridPos": { "h": 8, "w": 6, "x": 12, "y": 19 },
- "id": 7,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "targets": [
- {
- "expr": "sum by (instance) (rate(node_disk_read_bytes_total[$__rate_interval]))",
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "custom": {
- "drawStyle": "line",
- "lineInterpolation": "smooth",
- "lineWidth": 1,
- "fillOpacity": 15,
- "gradientMode": "none",
- "spanNulls": false,
- "showPoints": "never",
- "pointSize": 5,
- "stacking": { "mode": "none", "group": "A" },
- "axisPlacement": "auto",
- "axisLabel": "",
- "scaleDistribution": { "type": "linear" },
- "thresholdsStyle": { "mode": "off" }
- },
- "unit": "Bps",
- "color": { "mode": "palette-classic" },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- { "color": "green", "value": null }
- ]
- }
- },
- "overrides": []
- },
- "options": {
- "tooltip": { "mode": "multi", "sort": "desc" },
- "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
- }
- },
- {
- "type": "timeseries",
- "title": "Disk Write Rate by Node",
- "description": "Disk write throughput (bytes/sec) aggregated per host node.",
- "gridPos": { "h": 8, "w": 6, "x": 18, "y": 19 },
- "id": 8,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "targets": [
- {
- "expr": "sum by (instance) (rate(node_disk_written_bytes_total[$__rate_interval]))",
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "custom": {
- "drawStyle": "line",
- "lineInterpolation": "smooth",
- "lineWidth": 1,
- "fillOpacity": 15,
- "gradientMode": "none",
- "spanNulls": false,
- "showPoints": "never",
- "pointSize": 5,
- "stacking": { "mode": "none", "group": "A" },
- "axisPlacement": "auto",
- "axisLabel": "",
- "scaleDistribution": { "type": "linear" },
- "thresholdsStyle": { "mode": "off" }
- },
- "unit": "Bps",
- "color": { "mode": "palette-classic" },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- { "color": "green", "value": null }
- ]
- }
- },
- "overrides": []
- },
- "options": {
- "tooltip": { "mode": "multi", "sort": "desc" },
- "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
}
},
{
"type": "row",
- "title": "Network",
+ "title": "Instance Trends",
"collapsed": false,
- "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 },
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 },
"id": 9,
"panels": []
},
{
"type": "timeseries",
- "title": "Network Receive Rate by Device & Node",
- "description": "Inbound network throughput per network device and host node.",
- "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
+ "title": "Instance Count Over Time",
+ "description": "Total number of running Incus instances over time. Tracks growth and churn in the cluster workload count.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 },
"id": 10,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"targets": [
{
- "expr": "rate(node_network_receive_bytes_total{device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])",
- "legendFormat": "{{instance}} - {{device}} (rx)",
+ "expr": "count(incus_uptime_seconds)",
+ "legendFormat": "Instances",
"refId": "A"
}
],
@@ -347,8 +334,8 @@
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
- "lineWidth": 1,
- "fillOpacity": 15,
+ "lineWidth": 2,
+ "fillOpacity": 25,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
@@ -359,37 +346,37 @@
"scaleDistribution": { "type": "linear" },
"thresholdsStyle": { "mode": "off" }
},
- "unit": "Bps",
+ "unit": "short",
"color": { "mode": "palette-classic" },
"thresholds": {
"mode": "absolute",
- "steps": [
- { "color": "green", "value": null }
- ]
+ "steps": [{ "color": "green", "value": null }]
}
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
- "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
}
},
{
"type": "timeseries",
- "title": "Network Transmit Rate by Device & Node",
- "description": "Outbound network throughput per network device and host node.",
- "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
+ "title": "Memory Allocation vs Usage per Instance",
+ "description": "Compares total allocated memory (MemTotal) against actual resident memory (RSS) for each Incus instance. A large gap between allocated and used indicates overprovisioning; converging lines indicate an instance may need more memory.",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 },
"id": 11,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"targets": [
{
- "expr": "rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])",
- "legendFormat": "{{instance}} - {{device}} (tx)",
+ "expr": "incus_memory_MemTotal_bytes",
+ "legendFormat": "{{name}} allocated",
"refId": "A"
+ },
+ {
+ "expr": "incus_memory_RSS_bytes",
+ "legendFormat": "{{name}} used (RSS)",
+ "refId": "B"
}
],
"fieldConfig": {
@@ -398,7 +385,7 @@
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 1,
- "fillOpacity": 15,
+ "fillOpacity": 0,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
@@ -409,20 +396,32 @@
"scaleDistribution": { "type": "linear" },
"thresholdsStyle": { "mode": "off" }
},
- "unit": "Bps",
+ "unit": "bytes",
"color": { "mode": "palette-classic" },
"thresholds": {
"mode": "absolute",
- "steps": [
- { "color": "green", "value": null }
- ]
+ "steps": [{ "color": "green", "value": null }]
}
},
- "overrides": []
+ "overrides": [
+ {
+ "matcher": { "id": "byRegexp", "options": ".*allocated$" },
+ "properties": [
+ { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } },
+ { "id": "custom.lineWidth", "value": 2 }
+ ]
+ },
+ {
+ "matcher": { "id": "byRegexp", "options": ".*used \\(RSS\\)$" },
+ "properties": [
+ { "id": "custom.fillOpacity", "value": 15 }
+ ]
+ }
+ ]
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
- "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
}
}
]
diff --git a/incusos/observability-dashboards/core/cluster-overview.json b/incusos/observability-dashboards/core/cluster-overview.json
new file mode 100644
index 0000000..28e9e7f
--- /dev/null
+++ b/incusos/observability-dashboards/core/cluster-overview.json
@@ -0,0 +1,674 @@
+{
+ "uid": "incus-cluster-overview",
+ "title": "Incus Cluster Overview",
+ "description": "Overview of all Incus instances: CPU, memory, network, disk I/O, and instance status table with drill-down links.",
+ "tags": ["incus", "cluster", "containers"],
+ "timezone": "browser",
+ "schemaVersion": 39,
+ "refresh": "30s",
+ "time": { "from": "now-1h", "to": "now" },
+ "fiscalYearStartMonth": 0,
+ "liveNow": false,
+ "editable": true,
+ "graphTooltip": 1,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": ["incus"],
+ "targetBlank": false,
+ "title": "Incus Dashboards",
+ "tooltip": "Navigate to other Incus dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {},
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "definition": "label_values(incus_uptime_seconds, name)",
+ "includeAll": true,
+ "multi": true,
+ "name": "instance",
+ "query": "label_values(incus_uptime_seconds, name)",
+ "refresh": 2,
+ "regex": "",
+ "sort": 1,
+ "type": "query",
+ "label": "Instance"
+ },
+ {
+ "current": {},
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "definition": "label_values(incus_uptime_seconds, instance)",
+ "includeAll": true,
+ "multi": true,
+ "name": "node",
+ "query": "label_values(incus_uptime_seconds, instance)",
+ "refresh": 2,
+ "regex": "",
+ "sort": 1,
+ "type": "query",
+ "label": "Node"
+ }
+ ]
+ },
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": { "type": "grafana", "uid": "-- Grafana --" },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "panels": [
+ {
+ "type": "row",
+ "title": "Cluster Status",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+ "id": 1,
+ "panels": []
+ },
+ {
+ "type": "stat",
+ "title": "Total Instances",
+ "description": "Number of running Incus instances (reporting uptime).",
+ "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
+ "id": 2,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "count(incus_uptime_seconds{instance=~\"$node\", name=~\"$instance\"})",
+ "legendFormat": "Instances",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null },
+ { "color": "yellow", "value": 10 },
+ { "color": "red", "value": 50 }
+ ]
+ },
+ "unit": "none",
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "auto",
+ "textMode": "auto",
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto"
+ }
+ },
+ {
+ "type": "stat",
+ "title": "Total CPUs",
+ "description": "Sum of effective CPUs across all Incus instances.",
+ "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
+ "id": 3,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum(incus_cpu_effective_total{instance=~\"$node\", name=~\"$instance\"})",
+ "legendFormat": "CPUs",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ },
+ "unit": "none",
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "auto",
+ "textMode": "auto",
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto"
+ }
+ },
+ {
+ "type": "stat",
+ "title": "Total Memory",
+ "description": "Sum of total memory across all Incus instances.",
+ "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
+ "id": 4,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum(incus_memory_MemTotal_bytes{instance=~\"$node\", name=~\"$instance\"})",
+ "legendFormat": "Total Memory",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ },
+ "unit": "bytes",
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "auto",
+ "textMode": "auto",
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto"
+ }
+ },
+ {
+ "type": "gauge",
+ "title": "Cluster Memory Utilization",
+ "description": "Overall cluster memory utilization (RSS / Total) across all instances.",
+ "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
+ "id": 40,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "(sum(incus_memory_RSS_bytes{instance=~\"$node\", name=~\"$instance\"}) / sum(incus_memory_MemTotal_bytes{instance=~\"$node\", name=~\"$instance\"})) * 100",
+ "legendFormat": "Utilization",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null },
+ { "color": "yellow", "value": 60 },
+ { "color": "orange", "value": 80 },
+ { "color": "red", "value": 90 }
+ ]
+ },
+ "unit": "percent",
+ "min": 0,
+ "max": 100,
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true,
+ "orientation": "auto"
+ }
+ },
+ {
+ "type": "row",
+ "title": "Instance Status",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
+ "id": 50,
+ "panels": []
+ },
+ {
+ "type": "table",
+ "title": "Instance Status Table",
+ "description": "Per-instance summary: uptime, CPU usage, memory RSS, total memory. Click instance name to drill down.",
+ "gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 },
+ "id": 51,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "incus_uptime_seconds{instance=~\"$node\", name=~\"$instance\"}",
+ "legendFormat": "",
+ "refId": "A",
+ "format": "table",
+ "instant": true
+ },
+ {
+ "expr": "sum by (name, instance) (rate(incus_cpu_seconds_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "",
+ "refId": "B",
+ "format": "table",
+ "instant": true
+ },
+ {
+ "expr": "incus_memory_RSS_bytes{instance=~\"$node\", name=~\"$instance\"}",
+ "legendFormat": "",
+ "refId": "C",
+ "format": "table",
+ "instant": true
+ },
+ {
+ "expr": "incus_memory_MemTotal_bytes{instance=~\"$node\", name=~\"$instance\"}",
+ "legendFormat": "",
+ "refId": "D",
+ "format": "table",
+ "instant": true
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto",
+ "inspect": false,
+ "filterable": true
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ },
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": [
+ {
+ "matcher": { "id": "byName", "options": "name" },
+ "properties": [
+ { "id": "displayName", "value": "Instance" },
+ { "id": "custom.width", "value": 200 },
+ {
+ "id": "links",
+ "value": [
+ {
+ "title": "View instance details",
+ "url": "/d/incus-instance-deep-dive?var-instance=${__value.text}&${__url_time_range}",
+ "targetBlank": false
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "matcher": { "id": "byName", "options": "instance" },
+ "properties": [{ "id": "displayName", "value": "Node" }]
+ },
+ {
+ "matcher": { "id": "byName", "options": "Value #A" },
+ "properties": [
+ { "id": "displayName", "value": "Uptime" },
+ { "id": "unit", "value": "s" }
+ ]
+ },
+ {
+ "matcher": { "id": "byName", "options": "Value #B" },
+ "properties": [
+ { "id": "displayName", "value": "CPU/s" },
+ { "id": "unit", "value": "short" },
+ { "id": "decimals", "value": 3 }
+ ]
+ },
+ {
+ "matcher": { "id": "byName", "options": "Value #C" },
+ "properties": [
+ { "id": "displayName", "value": "Memory RSS" },
+ { "id": "unit", "value": "bytes" }
+ ]
+ },
+ {
+ "matcher": { "id": "byName", "options": "Value #D" },
+ "properties": [
+ { "id": "displayName", "value": "Total Memory" },
+ { "id": "unit", "value": "bytes" }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "showHeader": true,
+ "sortBy": [{ "displayName": "Instance", "desc": false }],
+ "footer": { "show": false, "reducer": ["sum"], "fields": "" }
+ },
+ "transformations": [
+ { "id": "merge", "options": {} },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": { "Time": true, "__name__": true, "job": true },
+ "indexByName": {
+ "name": 0, "instance": 1,
+ "Value #A": 2, "Value #B": 3, "Value #C": 4, "Value #D": 5
+ },
+ "renameByName": {}
+ }
+ }
+ ]
+ },
+ {
+ "type": "row",
+ "title": "Per-Instance CPU",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
+ "id": 5,
+ "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "CPU Usage by Instance",
+ "description": "Per-second CPU time consumed by each Incus instance.",
+ "gridPos": { "h": 8, "w": 24, "x": 0, "y": 15 },
+ "id": 6,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum by (name) (rate(incus_cpu_seconds_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "scheme",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "short",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ },
+ "links": [
+ {
+ "title": "Drill down to ${__field.labels.name}",
+ "url": "/d/incus-instance-deep-dive?var-instance=${__field.labels.name}&${__url_time_range}",
+ "targetBlank": false
+ }
+ ]
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "row",
+ "title": "Per-Instance Memory",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
+ "id": 7,
+ "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Memory RSS by Instance",
+ "description": "Resident set size (RSS) memory usage per Incus instance over time.",
+ "gridPos": { "h": 8, "w": 14, "x": 0, "y": 24 },
+ "id": 8,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "incus_memory_RSS_bytes{instance=~\"$node\", name=~\"$instance\"}",
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "bytes",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "bargauge",
+ "title": "Memory Usage % (RSS / Total)",
+ "description": "Current memory utilization per instance as RSS divided by total memory.",
+ "gridPos": { "h": 8, "w": 10, "x": 14, "y": 24 },
+ "id": 9,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "(incus_memory_RSS_bytes{instance=~\"$node\", name=~\"$instance\"} / incus_memory_MemTotal_bytes{instance=~\"$node\", name=~\"$instance\"}) * 100",
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null },
+ { "color": "yellow", "value": 60 },
+ { "color": "orange", "value": 80 },
+ { "color": "red", "value": 90 }
+ ]
+ },
+ "unit": "percent",
+ "min": 0,
+ "max": 100,
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "horizontal",
+ "displayMode": "gradient",
+ "showUnfilled": true,
+ "minVizWidth": 8,
+ "minVizHeight": 16,
+ "valueMode": "color"
+ }
+ },
+ {
+ "type": "row",
+ "title": "Network I/O",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 },
+ "id": 10,
+ "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Network Receive Rate by Instance",
+ "description": "Inbound network throughput per Incus instance.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 },
+ "id": 11,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum by (name) (rate(incus_network_receive_bytes_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+ "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+ "showPoints": "never", "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "Bps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Network Transmit Rate by Instance",
+ "description": "Outbound network throughput per Incus instance.",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 },
+ "id": 12,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum by (name) (rate(incus_network_transmit_bytes_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+ "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+ "showPoints": "never", "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "Bps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "row",
+ "title": "Disk I/O",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 },
+ "id": 13,
+ "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Disk Read Rate by Instance",
+ "description": "Disk read throughput per Incus instance.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 42 },
+ "id": 14,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum by (name) (rate(incus_disk_read_bytes_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+ "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+ "showPoints": "never", "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "Bps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Disk Write Rate by Instance",
+ "description": "Disk write throughput per Incus instance.",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 42 },
+ "id": 15,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum by (name) (rate(incus_disk_written_bytes_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+ "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+ "showPoints": "never", "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "Bps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ }
+ ]
+}
diff --git a/incusos/observability-dashboards/core/haproxy-traffic.json b/incusos/observability-dashboards/core/haproxy-traffic.json
new file mode 100644
index 0000000..4d65a0b
--- /dev/null
+++ b/incusos/observability-dashboards/core/haproxy-traffic.json
@@ -0,0 +1,910 @@
+{
+ "uid": "incus-haproxy-traffic",
+ "title": "HAProxy Traffic",
+ "description": "HAProxy load balancer monitoring: request rates, response codes, backend health, traffic volume, sessions, and performance metrics.",
+ "tags": ["incus", "haproxy", "loadbalancer"],
+ "timezone": "browser",
+ "schemaVersion": 39,
+ "version": 2,
+ "refresh": "30s",
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "fiscalYearStartMonth": 0,
+ "liveNow": false,
+ "editable": true,
+ "graphTooltip": 1,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": ["incus"],
+ "targetBlank": false,
+ "title": "Incus Dashboards",
+ "tooltip": "Navigate to other Incus dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {},
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "definition": "label_values(haproxy_backend_status, proxy)",
+ "includeAll": true,
+ "multi": true,
+ "name": "proxy",
+ "query": "label_values(haproxy_backend_status, proxy)",
+ "refresh": 2,
+ "regex": "",
+ "sort": 1,
+ "type": "query",
+ "label": "Proxy"
+ },
+ {
+ "current": {},
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "definition": "label_values(haproxy_server_status, server)",
+ "includeAll": true,
+ "multi": true,
+ "name": "server",
+ "query": "label_values(haproxy_server_status, server)",
+ "refresh": 2,
+ "regex": "",
+ "sort": 1,
+ "type": "query",
+ "label": "Server"
+ }
+ ]
+ },
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "grafana",
+ "uid": "-- Grafana --"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "panels": [
+ {
+ "type": "row",
+ "title": "Overview",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+ "id": 1,
+ "panels": []
+ },
+ {
+ "type": "stat",
+ "title": "Active Backend Servers",
+ "description": "Total number of active (healthy) servers across selected HAProxy backends.",
+ "gridPos": { "h": 4, "w": 5, "x": 0, "y": 1 },
+ "id": 2,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum(haproxy_backend_active_servers{proxy=~\"$proxy\"})",
+ "legendFormat": "Active Servers",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "red", "value": null },
+ { "color": "yellow", "value": 1 },
+ { "color": "green", "value": 2 }
+ ]
+ },
+ "unit": "none",
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": {
+ "calcs": ["lastNotNull"],
+ "fields": "",
+ "values": false
+ },
+ "orientation": "auto",
+ "textMode": "auto",
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto"
+ }
+ },
+ {
+ "type": "stat",
+ "title": "Current Sessions",
+ "description": "Total current active sessions across selected HAProxy backends.",
+ "gridPos": { "h": 4, "w": 5, "x": 5, "y": 1 },
+ "id": 3,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum(haproxy_backend_current_sessions{proxy=~\"$proxy\"})",
+ "legendFormat": "Sessions",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null },
+ { "color": "yellow", "value": 100 },
+ { "color": "red", "value": 500 }
+ ]
+ },
+ "unit": "none",
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": {
+ "calcs": ["lastNotNull"],
+ "fields": "",
+ "values": false
+ },
+ "orientation": "auto",
+ "textMode": "auto",
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto"
+ }
+ },
+ {
+ "type": "stat",
+ "title": "HTTP Requests/s",
+ "description": "Current rate of HTTP requests per second across selected HAProxy frontends.",
+ "gridPos": { "h": 4, "w": 5, "x": 10, "y": 1 },
+ "id": 4,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum(rate(haproxy_frontend_http_requests_total{proxy=~\"$proxy\"}[$__rate_interval]))",
+ "legendFormat": "Requests/s",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null }
+ ]
+ },
+ "unit": "reqps",
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": {
+ "calcs": ["lastNotNull"],
+ "fields": "",
+ "values": false
+ },
+ "orientation": "auto",
+ "textMode": "auto",
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto"
+ }
+ },
+ {
+ "type": "stat",
+ "title": "HTTP 4xx Rate",
+ "description": "Rate of HTTP 4xx client error responses per second. Elevated rates may indicate bad requests or missing resources.",
+ "gridPos": { "h": 4, "w": 4, "x": 15, "y": 1 },
+ "id": 14,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum(rate(haproxy_frontend_http_responses_total{proxy=~\"$proxy\", code=\"4xx\"}[$__rate_interval]))",
+ "legendFormat": "4xx/s",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null },
+ { "color": "yellow", "value": 0.1 },
+ { "color": "red", "value": 1 }
+ ]
+ },
+ "unit": "reqps",
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": {
+ "calcs": ["lastNotNull"],
+ "fields": "",
+ "values": false
+ },
+ "orientation": "auto",
+ "textMode": "auto",
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto"
+ }
+ },
+ {
+ "type": "stat",
+ "title": "HTTP 5xx Rate",
+ "description": "Rate of HTTP 5xx server error responses per second. Any sustained rate indicates backend failures requiring investigation.",
+ "gridPos": { "h": 4, "w": 5, "x": 19, "y": 1 },
+ "id": 15,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum(rate(haproxy_frontend_http_responses_total{proxy=~\"$proxy\", code=\"5xx\"}[$__rate_interval]))",
+ "legendFormat": "5xx/s",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null },
+ { "color": "yellow", "value": 0.01 },
+ { "color": "red", "value": 0.1 }
+ ]
+ },
+ "unit": "reqps",
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": {
+ "calcs": ["lastNotNull"],
+ "fields": "",
+ "values": false
+ },
+ "orientation": "auto",
+ "textMode": "auto",
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto"
+ }
+ },
+ {
+ "type": "row",
+ "title": "HTTP Requests",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
+ "id": 5,
+ "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "HTTP Request Rate by Frontend",
+ "description": "Rate of HTTP requests per second, broken down by HAProxy frontend/proxy. Useful for identifying which frontends carry the most traffic.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
+ "id": 6,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum by (proxy) (rate(haproxy_frontend_http_requests_total{proxy=~\"$proxy\"}[$__rate_interval]))",
+ "legendFormat": "{{proxy}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 20,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "reqps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Response Code Distribution",
+ "description": "HTTP response code distribution across selected frontends, stacked by response class (2xx, 3xx, 4xx, 5xx). Helps identify error trends and traffic composition.",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
+ "id": 16,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum by (code) (rate(haproxy_frontend_http_responses_total{proxy=~\"$proxy\"}[$__rate_interval]))",
+ "legendFormat": "{{code}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 50,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "normal", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "reqps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": { "id": "byName", "options": "2xx" },
+ "properties": [
+ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }
+ ]
+ },
+ {
+ "matcher": { "id": "byName", "options": "3xx" },
+ "properties": [
+ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }
+ ]
+ },
+ {
+ "matcher": { "id": "byName", "options": "4xx" },
+ "properties": [
+ { "id": "color", "value": { "mode": "fixed", "fixedColor": "yellow" } }
+ ]
+ },
+ {
+ "matcher": { "id": "byName", "options": "5xx" },
+ "properties": [
+ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "row",
+ "title": "Backend Health",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
+ "id": 7,
+ "panels": []
+ },
+ {
+ "type": "table",
+ "title": "Backend Status",
+ "description": "Current status, active server count, and average response time for each HAProxy backend. Filterable by proxy and server template variables.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 },
+ "id": 8,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "haproxy_backend_status{proxy=~\"$proxy\"}",
+ "legendFormat": "",
+ "refId": "A",
+ "format": "table",
+ "instant": true
+ },
+ {
+ "expr": "haproxy_backend_active_servers{proxy=~\"$proxy\"}",
+ "legendFormat": "",
+ "refId": "B",
+ "format": "table",
+ "instant": true
+ },
+ {
+ "expr": "haproxy_server_response_time_average_seconds{proxy=~\"$proxy\", server=~\"$server\"}",
+ "legendFormat": "",
+ "refId": "C",
+ "format": "table",
+ "instant": true
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto",
+ "inspect": false,
+ "filterable": true
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null }
+ ]
+ },
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": [
+ {
+ "matcher": { "id": "byName", "options": "Value #A" },
+ "properties": [
+ { "id": "displayName", "value": "Status" },
+ {
+ "id": "mappings",
+ "value": [
+ { "type": "value", "options": { "1": { "text": "UP", "color": "green" }, "0": { "text": "DOWN", "color": "red" } } }
+ ]
+ }
+ ]
+ },
+ {
+ "matcher": { "id": "byName", "options": "Value #B" },
+ "properties": [
+ { "id": "displayName", "value": "Active Servers" }
+ ]
+ },
+ {
+ "matcher": { "id": "byName", "options": "Value #C" },
+ "properties": [
+ { "id": "displayName", "value": "Avg Response Time" },
+ { "id": "unit", "value": "s" }
+ ]
+ },
+ {
+ "matcher": { "id": "byName", "options": "proxy" },
+ "properties": [
+ { "id": "displayName", "value": "Backend" }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "showHeader": true,
+ "sortBy": [],
+ "footer": {
+ "show": false,
+ "reducer": ["sum"],
+ "fields": ""
+ }
+ },
+ "transformations": [
+ {
+ "id": "merge",
+ "options": {}
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {
+ "Time": true,
+ "__name__": true,
+ "instance": false,
+ "job": true
+ },
+ "indexByName": {},
+ "renameByName": {}
+ }
+ }
+ ]
+ },
+ {
+ "type": "state-timeline",
+ "title": "Server Status",
+ "description": "Timeline view of individual HAProxy server health status. Green indicates UP, red indicates DOWN. Helps identify flapping servers and outage windows.",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 },
+ "id": 17,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "haproxy_server_status{proxy=~\"$proxy\", server=~\"$server\"}",
+ "legendFormat": "{{proxy}} / {{server}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "lineWidth": 0,
+ "fillOpacity": 80
+ },
+ "color": { "mode": "thresholds" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "red", "value": null },
+ { "color": "green", "value": 1 }
+ ]
+ },
+ "mappings": [
+ {
+ "type": "value",
+ "options": {
+ "0": { "text": "DOWN", "color": "red" },
+ "1": { "text": "UP", "color": "green" }
+ }
+ }
+ ]
+ },
+ "overrides": []
+ },
+ "options": {
+ "showValue": "auto",
+ "mergeValues": true,
+ "alignValue": "left",
+ "rowHeight": 0.9,
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "list", "placement": "bottom" }
+ }
+ },
+ {
+ "type": "row",
+ "title": "Traffic",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
+ "id": 9,
+ "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Frontend Bytes In",
+ "description": "Inbound traffic rate per HAProxy frontend. Shows how much data clients are sending to each frontend.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
+ "id": 10,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum by (proxy) (rate(haproxy_frontend_bytes_in_total{proxy=~\"$proxy\"}[$__rate_interval]))",
+ "legendFormat": "{{proxy}} (in)",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 20,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "Bps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Frontend Bytes Out",
+ "description": "Outbound traffic rate per HAProxy frontend. Shows how much data is being served to clients from each frontend.",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
+ "id": 11,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum by (proxy) (rate(haproxy_frontend_bytes_out_total{proxy=~\"$proxy\"}[$__rate_interval]))",
+ "legendFormat": "{{proxy}} (out)",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 20,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "Bps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "row",
+ "title": "Sessions & Performance",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 },
+ "id": 12,
+ "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Current Sessions by Proxy",
+ "description": "Current active sessions for each HAProxy backend/proxy over time. Sustained high values may indicate slow backends or session buildup.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 },
+ "id": 13,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "haproxy_backend_current_sessions{proxy=~\"$proxy\"}",
+ "legendFormat": "{{proxy}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 20,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "none",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Backend Response Time",
+ "description": "Average response time from backend servers. High or increasing values indicate backend performance degradation.",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 },
+ "id": 18,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "haproxy_server_response_time_average_seconds{proxy=~\"$proxy\", server=~\"$server\"}",
+ "legendFormat": "{{proxy}} / {{server}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "scheme",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "line" }
+ },
+ "unit": "s",
+ "color": { "mode": "continuous-GrYlRd" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null },
+ { "color": "yellow", "value": 0.5 },
+ { "color": "red", "value": 2 }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Queue Time",
+ "description": "Average time requests spend in the backend queue before being assigned to a server. High values indicate insufficient backend capacity.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 41 },
+ "id": 19,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "haproxy_backend_queue_time_average_seconds{proxy=~\"$proxy\"}",
+ "legendFormat": "{{proxy}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "scheme",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "line" }
+ },
+ "unit": "s",
+ "color": { "mode": "continuous-GrYlRd" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null },
+ { "color": "yellow", "value": 0.1 },
+ { "color": "red", "value": 1 }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Connection Errors",
+ "description": "Rate of backend connection errors and connection retries per proxy. Non-zero values indicate connectivity problems between HAProxy and backend servers.",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 41 },
+ "id": 20,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum by (proxy) (rate(haproxy_backend_connection_errors_total{proxy=~\"$proxy\"}[$__rate_interval]))",
+ "legendFormat": "{{proxy}} errors",
+ "refId": "A"
+ },
+ {
+ "expr": "sum by (proxy) (rate(haproxy_backend_retry_warnings_total{proxy=~\"$proxy\"}[$__rate_interval]))",
+ "legendFormat": "{{proxy}} retries",
+ "refId": "B"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "reqps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": { "id": "byRegexp", "options": ".*errors$" },
+ "properties": [
+ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }
+ ]
+ },
+ {
+ "matcher": { "id": "byRegexp", "options": ".*retries$" },
+ "properties": [
+ { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ }
+ ]
+}
diff --git a/incusos/observability-dashboards/core/host-resources.json b/incusos/observability-dashboards/core/host-resources.json
new file mode 100644
index 0000000..79b7eb7
--- /dev/null
+++ b/incusos/observability-dashboards/core/host-resources.json
@@ -0,0 +1,584 @@
+{
+ "uid": "incus-host-resources",
+ "title": "Host Resources (IncusOS Nodes)",
+ "description": "Physical host monitoring: CPU mode breakdown, load averages, memory breakdown, filesystem usage, disk IOPS, network errors.",
+ "tags": ["incus", "incusos", "host", "node-exporter"],
+ "timezone": "browser",
+ "schemaVersion": 39,
+ "refresh": "30s",
+ "time": { "from": "now-1h", "to": "now" },
+ "fiscalYearStartMonth": 0,
+ "liveNow": false,
+ "editable": true,
+ "graphTooltip": 1,
+ "links": [
+ {
+ "asDropdown": true, "icon": "external link", "includeVars": true,
+ "keepTime": true, "tags": ["incus"], "targetBlank": false,
+ "title": "Incus Dashboards", "type": "dashboards"
+ }
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {},
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "definition": "label_values(node_uname_info, instance)",
+ "includeAll": true,
+ "multi": true,
+ "name": "node",
+ "query": "label_values(node_uname_info, instance)",
+ "refresh": 2,
+ "sort": 1,
+ "type": "query",
+ "label": "Node"
+ }
+ ]
+ },
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": { "type": "grafana", "uid": "-- Grafana --" },
+ "enable": true, "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts", "type": "dashboard"
+ }
+ ]
+ },
+ "panels": [
+ {
+ "type": "row", "title": "System Info",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+ "id": 30, "panels": []
+ },
+ {
+ "type": "table",
+ "title": "System Information",
+ "description": "Kernel version, uptime, total RAM, and CPU count per node.",
+ "gridPos": { "h": 5, "w": 24, "x": 0, "y": 1 },
+ "id": 31,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "node_uname_info{instance=~\"$node\"}",
+ "legendFormat": "", "refId": "A", "format": "table", "instant": true
+ },
+ {
+ "expr": "node_boot_time_seconds{instance=~\"$node\"}",
+ "legendFormat": "", "refId": "B", "format": "table", "instant": true
+ },
+ {
+ "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"}",
+ "legendFormat": "", "refId": "C", "format": "table", "instant": true
+ },
+ {
+ "expr": "count by (instance) (node_cpu_seconds_total{instance=~\"$node\", mode=\"idle\"})",
+ "legendFormat": "", "refId": "D", "format": "table", "instant": true
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": { "align": "auto", "displayMode": "auto", "inspect": false, "filterable": true },
+ "color": { "mode": "thresholds" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": [
+ { "matcher": { "id": "byName", "options": "instance" }, "properties": [{ "id": "displayName", "value": "Node" }] },
+ { "matcher": { "id": "byName", "options": "release" }, "properties": [{ "id": "displayName", "value": "Kernel" }] },
+ { "matcher": { "id": "byName", "options": "Value #B" }, "properties": [{ "id": "displayName", "value": "Boot Time" }, { "id": "unit", "value": "dateTimeFromNow" }] },
+ { "matcher": { "id": "byName", "options": "Value #C" }, "properties": [{ "id": "displayName", "value": "Total RAM" }, { "id": "unit", "value": "bytes" }] },
+ { "matcher": { "id": "byName", "options": "Value #D" }, "properties": [{ "id": "displayName", "value": "CPUs" }] }
+ ]
+ },
+ "options": { "showHeader": true, "sortBy": [], "footer": { "show": false, "reducer": ["sum"], "fields": "" } },
+ "transformations": [
+ { "id": "merge", "options": {} },
+ { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "job": true, "domainname": true, "machine": true, "nodename": true, "sysname": true, "version": true, "Value #A": true }, "renameByName": {} } }
+ ]
+ },
+ {
+ "type": "row", "title": "CPU",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 },
+ "id": 1, "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "CPU Usage % by Node",
+ "description": "Overall CPU utilization per host node.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 },
+ "id": 2,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$node\"}[$__rate_interval]))) * 100",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2,
+ "fillOpacity": 25, "gradientMode": "scheme", "spanNulls": false,
+ "showPoints": "never", "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "line" }
+ },
+ "unit": "percent", "min": 0, "max": 100,
+ "color": { "mode": "continuous-GrYlRd" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "CPU Mode Breakdown",
+ "description": "CPU time by mode (user, system, iowait, idle) stacked per node.",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 },
+ "id": 32,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ { "expr": "avg by (instance, mode) (rate(node_cpu_seconds_total{instance=~\"$node\", mode=~\"user|system|iowait|idle\"}[$__rate_interval])) * 100", "legendFormat": "{{instance}} {{mode}}", "refId": "A" }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+ "fillOpacity": 50, "gradientMode": "none", "spanNulls": false,
+ "showPoints": "never",
+ "stacking": { "mode": "normal", "group": "A" },
+ "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "percent", "min": 0,
+ "color": { "mode": "palette-classic" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Load Average",
+ "description": "System load averages (1m, 5m, 15m) with CPU count threshold line.",
+ "gridPos": { "h": 8, "w": 24, "x": 0, "y": 15 },
+ "id": 33,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ { "expr": "node_load1{instance=~\"$node\"}", "legendFormat": "{{instance}} load1", "refId": "A" },
+ { "expr": "node_load5{instance=~\"$node\"}", "legendFormat": "{{instance}} load5", "refId": "B" },
+ { "expr": "node_load15{instance=~\"$node\"}", "legendFormat": "{{instance}} load15", "refId": "C" },
+ { "expr": "count by (instance) (node_cpu_seconds_total{instance=~\"$node\", mode=\"idle\"})", "legendFormat": "{{instance}} CPUs", "refId": "D" }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+ "fillOpacity": 0, "gradientMode": "none", "spanNulls": false,
+ "showPoints": "never",
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "short",
+ "color": { "mode": "palette-classic" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": [
+ {
+ "matcher": { "id": "byRegexp", "options": ".*CPUs$" },
+ "properties": [
+ { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } },
+ { "id": "custom.lineWidth", "value": 2 },
+ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "row", "title": "Memory",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
+ "id": 3, "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Memory Usage % by Node",
+ "description": "Memory utilization percentage per host node.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
+ "id": 4,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "(1 - node_memory_MemAvailable_bytes{instance=~\"$node\"} / node_memory_MemTotal_bytes{instance=~\"$node\"}) * 100",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2,
+ "fillOpacity": 25, "gradientMode": "scheme", "spanNulls": false,
+ "showPoints": "never",
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "line" }
+ },
+ "unit": "percent", "min": 0, "max": 100,
+ "color": { "mode": "continuous-GrYlRd" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Memory Breakdown by Node",
+ "description": "Memory breakdown: used, buffers, cached, available (stacked).",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
+ "id": 34,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ { "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"} - node_memory_MemAvailable_bytes{instance=~\"$node\"} - node_memory_Buffers_bytes{instance=~\"$node\"} - node_memory_Cached_bytes{instance=~\"$node\"}", "legendFormat": "{{instance}} used", "refId": "A" },
+ { "expr": "node_memory_Buffers_bytes{instance=~\"$node\"}", "legendFormat": "{{instance}} buffers", "refId": "B" },
+ { "expr": "node_memory_Cached_bytes{instance=~\"$node\"}", "legendFormat": "{{instance}} cached", "refId": "C" },
+ { "expr": "node_memory_MemAvailable_bytes{instance=~\"$node\"}", "legendFormat": "{{instance}} available", "refId": "D" }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+ "fillOpacity": 50, "gradientMode": "none", "spanNulls": false,
+ "showPoints": "never",
+ "stacking": { "mode": "normal", "group": "A" },
+ "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "bytes",
+ "color": { "mode": "palette-classic" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "row", "title": "Disk",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 },
+ "id": 5, "panels": []
+ },
+ {
+ "type": "bargauge",
+ "title": "Filesystem Usage %",
+ "description": "Current filesystem usage percentage for important mount points.",
+ "gridPos": { "h": 6, "w": 8, "x": 0, "y": 33 },
+ "id": 35,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "(1 - node_filesystem_avail_bytes{instance=~\"$node\", fstype!~\"tmpfs|overlay\", mountpoint=~\"/.*\"} / node_filesystem_size_bytes{instance=~\"$node\", fstype!~\"tmpfs|overlay\", mountpoint=~\"/.*\"}) * 100",
+ "legendFormat": "{{instance}} {{mountpoint}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "orange", "value": 85 }, { "color": "red", "value": 95 }] },
+ "unit": "percent", "min": 0, "max": 100,
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "horizontal", "displayMode": "gradient", "showUnfilled": true,
+ "minVizWidth": 8, "minVizHeight": 16, "valueMode": "color"
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Root Filesystem Available Space",
+ "description": "Available disk space on the root filesystem per host node.",
+ "gridPos": { "h": 6, "w": 8, "x": 8, "y": 33 },
+ "id": 6,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ { "expr": "node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=\"/\"}", "legendFormat": "{{instance}}", "refId": "A" }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+ "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+ "showPoints": "never",
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "bytes",
+ "color": { "mode": "palette-classic" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull", "min"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Disk IOPS by Node",
+ "description": "Disk read and write operations per second by node.",
+ "gridPos": { "h": 6, "w": 8, "x": 16, "y": 33 },
+ "id": 36,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ { "expr": "sum by (instance) (rate(node_disk_reads_completed_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} reads", "refId": "A" },
+ { "expr": "sum by (instance) (rate(node_disk_writes_completed_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} writes", "refId": "B" }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+ "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+ "showPoints": "never",
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "iops",
+ "color": { "mode": "palette-classic" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Disk Read Rate by Node",
+ "description": "Disk read throughput aggregated per host node.",
+ "gridPos": { "h": 6, "w": 8, "x": 0, "y": 39 },
+ "id": 7,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ { "expr": "sum by (instance) (rate(node_disk_read_bytes_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}}", "refId": "A" }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+ "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+ "showPoints": "never",
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "Bps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Disk Write Rate by Node",
+ "description": "Disk write throughput aggregated per host node.",
+ "gridPos": { "h": 6, "w": 8, "x": 8, "y": 39 },
+ "id": 8,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ { "expr": "sum by (instance) (rate(node_disk_written_bytes_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}}", "refId": "A" }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+ "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+ "showPoints": "never",
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "Bps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Disk I/O Time",
+ "description": "Time spent on disk I/O operations (disk utilization indicator).",
+ "gridPos": { "h": 6, "w": 8, "x": 16, "y": 39 },
+ "id": 37,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ { "expr": "sum by (instance) (rate(node_disk_io_time_seconds_total{instance=~\"$node\"}[$__rate_interval])) * 100", "legendFormat": "{{instance}}", "refId": "A" }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+ "fillOpacity": 15, "gradientMode": "scheme", "spanNulls": false,
+ "showPoints": "never",
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "line" }
+ },
+ "unit": "percent", "min": 0, "max": 100,
+ "color": { "mode": "continuous-GrYlRd" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 80 }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+ }
+ },
+ {
+ "type": "row", "title": "Network",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 45 },
+ "id": 9, "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Network Receive Rate by Device & Node",
+ "description": "Inbound network throughput per network device and host node.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 46 },
+ "id": 10,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ { "expr": "rate(node_network_receive_bytes_total{instance=~\"$node\", device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])", "legendFormat": "{{instance}} {{device}} rx", "refId": "A" }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+ "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+ "showPoints": "never",
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "Bps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Network Transmit Rate by Device & Node",
+ "description": "Outbound network throughput per network device and host node.",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 46 },
+ "id": 11,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ { "expr": "rate(node_network_transmit_bytes_total{instance=~\"$node\", device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])", "legendFormat": "{{instance}} {{device}} tx", "refId": "A" }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+ "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+ "showPoints": "never",
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "Bps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Network Errors & Drops",
+ "description": "Network receive/transmit errors and dropped packets per node.",
+ "gridPos": { "h": 8, "w": 24, "x": 0, "y": 54 },
+ "id": 38,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ { "expr": "sum by (instance) (rate(node_network_receive_errs_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} rx errors", "refId": "A" },
+ { "expr": "sum by (instance) (rate(node_network_transmit_errs_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} tx errors", "refId": "B" },
+ { "expr": "sum by (instance) (rate(node_network_receive_drop_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} rx drops", "refId": "C" },
+ { "expr": "sum by (instance) (rate(node_network_transmit_drop_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} tx drops", "refId": "D" }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+ "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+ "showPoints": "never",
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "pps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ }
+ ]
+}
diff --git a/incusos/observability-dashboards/cluster-overview.json b/incusos/observability-dashboards/core/instance-deep-dive.json
similarity index 56%
rename from incusos/observability-dashboards/cluster-overview.json
rename to incusos/observability-dashboards/core/instance-deep-dive.json
index 059d736..fcfa693 100644
--- a/incusos/observability-dashboards/cluster-overview.json
+++ b/incusos/observability-dashboards/core/instance-deep-dive.json
@@ -1,31 +1,59 @@
{
- "uid": "incus-cluster-overview",
- "title": "Incus Cluster Overview",
- "description": "Overview of all Incus containers: CPU, memory, network, and disk I/O per instance.",
+ "uid": "incus-instance-deep-dive",
+ "title": "Instance Deep Dive",
+ "description": "Single-instance drill-down: CPU, memory, network, and disk metrics with detailed timeseries and utilization gauges.",
"tags": ["incus", "cluster", "containers"],
"timezone": "browser",
"schemaVersion": 39,
- "version": 1,
"refresh": "30s",
- "time": {
- "from": "now-1h",
- "to": "now"
- },
+ "time": { "from": "now-1h", "to": "now" },
"fiscalYearStartMonth": 0,
"liveNow": false,
"editable": true,
"graphTooltip": 1,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": ["incus"],
+ "targetBlank": false,
+ "title": "Incus Dashboards",
+ "tooltip": "Navigate to other Incus dashboards",
+ "type": "dashboards"
+ },
+ {
+ "title": "Back to Cluster Overview",
+ "url": "/d/incus-cluster-overview?${__url_time_range}",
+ "targetBlank": false,
+ "type": "link",
+ "icon": "arrow-left"
+ }
+ ],
"templating": {
- "list": []
+ "list": [
+ {
+ "current": {},
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "definition": "label_values(incus_uptime_seconds, name)",
+ "includeAll": false,
+ "multi": false,
+ "name": "instance",
+ "query": "label_values(incus_uptime_seconds, name)",
+ "refresh": 2,
+ "regex": "",
+ "sort": 1,
+ "type": "query",
+ "label": "Instance"
+ }
+ ]
},
"annotations": {
"list": [
{
"builtIn": 1,
- "datasource": {
- "type": "grafana",
- "uid": "-- Grafana --"
- },
+ "datasource": { "type": "grafana", "uid": "-- Grafana --" },
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
@@ -37,7 +65,7 @@
"panels": [
{
"type": "row",
- "title": "Cluster Status",
+ "title": "Instance Status",
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
"id": 1,
@@ -45,18 +73,15 @@
},
{
"type": "stat",
- "title": "Total Containers",
- "description": "Number of running Incus containers (instances reporting uptime).",
- "gridPos": { "h": 4, "w": 8, "x": 0, "y": 1 },
+ "title": "Uptime",
+ "description": "How long this instance has been running.",
+ "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
"id": 2,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"targets": [
{
- "expr": "count(incus_uptime_seconds)",
- "legendFormat": "Containers",
+ "expr": "incus_uptime_seconds{name=~\"$instance\"}",
+ "legendFormat": "Uptime",
"refId": "A"
}
],
@@ -64,43 +89,32 @@
"defaults": {
"thresholds": {
"mode": "absolute",
- "steps": [
- { "color": "green", "value": null },
- { "color": "yellow", "value": 10 },
- { "color": "red", "value": 50 }
- ]
+ "steps": [{ "color": "green", "value": null }]
},
- "unit": "none",
+ "unit": "s",
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
- "reduceOptions": {
- "calcs": ["lastNotNull"],
- "fields": "",
- "values": false
- },
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
- "graphMode": "none",
+ "graphMode": "area",
"justifyMode": "auto"
}
},
{
"type": "stat",
- "title": "Total CPUs",
- "description": "Sum of effective CPUs across all Incus instances.",
- "gridPos": { "h": 4, "w": 8, "x": 8, "y": 1 },
+ "title": "Effective CPUs",
+ "description": "Number of effective CPUs allocated to this instance.",
+ "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
"id": 3,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"targets": [
{
- "expr": "sum(incus_cpu_effective_total)",
+ "expr": "incus_cpu_effective_total{name=~\"$instance\"}",
"legendFormat": "CPUs",
"refId": "A"
}
@@ -109,9 +123,7 @@
"defaults": {
"thresholds": {
"mode": "absolute",
- "steps": [
- { "color": "green", "value": null }
- ]
+ "steps": [{ "color": "green", "value": null }]
},
"unit": "none",
"color": { "mode": "thresholds" }
@@ -119,31 +131,24 @@
"overrides": []
},
"options": {
- "reduceOptions": {
- "calcs": ["lastNotNull"],
- "fields": "",
- "values": false
- },
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
- "graphMode": "none",
+ "graphMode": "area",
"justifyMode": "auto"
}
},
{
"type": "stat",
"title": "Total Memory",
- "description": "Sum of total memory across all Incus instances.",
- "gridPos": { "h": 4, "w": 8, "x": 16, "y": 1 },
+ "description": "Total memory allocated to this instance.",
+ "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
"id": 4,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"targets": [
{
- "expr": "sum(incus_memory_MemTotal_bytes)",
+ "expr": "incus_memory_MemTotal_bytes{name=~\"$instance\"}",
"legendFormat": "Total Memory",
"refId": "A"
}
@@ -152,9 +157,7 @@
"defaults": {
"thresholds": {
"mode": "absolute",
- "steps": [
- { "color": "green", "value": null }
- ]
+ "steps": [{ "color": "green", "value": null }]
},
"unit": "bytes",
"color": { "mode": "thresholds" }
@@ -162,148 +165,25 @@
"overrides": []
},
"options": {
- "reduceOptions": {
- "calcs": ["lastNotNull"],
- "fields": "",
- "values": false
- },
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
- "graphMode": "none",
+ "graphMode": "area",
"justifyMode": "auto"
}
},
{
- "type": "row",
- "title": "Per-Instance CPU",
- "collapsed": false,
- "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
+ "type": "gauge",
+ "title": "Memory Utilization",
+ "description": "Current memory utilization (RSS / Total) for this instance.",
+ "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
"id": 5,
- "panels": []
- },
- {
- "type": "timeseries",
- "title": "CPU Usage by Instance",
- "description": "Per-second CPU time consumed by each Incus instance.",
- "gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 },
- "id": 6,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"targets": [
{
- "expr": "sum by (name) (rate(incus_cpu_seconds_total[$__rate_interval]))",
- "legendFormat": "{{name}}",
- "refId": "A"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "custom": {
- "drawStyle": "line",
- "lineInterpolation": "smooth",
- "lineWidth": 1,
- "fillOpacity": 15,
- "gradientMode": "none",
- "spanNulls": false,
- "showPoints": "never",
- "pointSize": 5,
- "stacking": { "mode": "none", "group": "A" },
- "axisPlacement": "auto",
- "axisLabel": "",
- "scaleDistribution": { "type": "linear" },
- "thresholdsStyle": { "mode": "off" }
- },
- "unit": "short",
- "color": { "mode": "palette-classic" },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- { "color": "green", "value": null }
- ]
- }
- },
- "overrides": []
- },
- "options": {
- "tooltip": { "mode": "multi", "sort": "desc" },
- "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
- }
- },
- {
- "type": "row",
- "title": "Per-Instance Memory",
- "collapsed": false,
- "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
- "id": 7,
- "panels": []
- },
- {
- "type": "timeseries",
- "title": "Memory RSS by Instance",
- "description": "Resident set size (RSS) memory usage per Incus instance over time.",
- "gridPos": { "h": 8, "w": 14, "x": 0, "y": 15 },
- "id": 8,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "targets": [
- {
- "expr": "incus_memory_RSS_bytes",
- "legendFormat": "{{name}}",
- "refId": "A"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "custom": {
- "drawStyle": "line",
- "lineInterpolation": "smooth",
- "lineWidth": 1,
- "fillOpacity": 15,
- "gradientMode": "none",
- "spanNulls": false,
- "showPoints": "never",
- "pointSize": 5,
- "stacking": { "mode": "none", "group": "A" },
- "axisPlacement": "auto",
- "axisLabel": "",
- "scaleDistribution": { "type": "linear" },
- "thresholdsStyle": { "mode": "off" }
- },
- "unit": "bytes",
- "color": { "mode": "palette-classic" },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- { "color": "green", "value": null }
- ]
- }
- },
- "overrides": []
- },
- "options": {
- "tooltip": { "mode": "multi", "sort": "desc" },
- "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
- }
- },
- {
- "type": "bargauge",
- "title": "Memory Usage % (RSS / Total)",
- "description": "Current memory utilization per instance as RSS divided by total memory.",
- "gridPos": { "h": 8, "w": 10, "x": 14, "y": 15 },
- "id": 9,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "targets": [
- {
- "expr": "(incus_memory_RSS_bytes / incus_memory_MemTotal_bytes) * 100",
- "legendFormat": "{{name}}",
+ "expr": "(incus_memory_RSS_bytes{name=~\"$instance\"} / incus_memory_MemTotal_bytes{name=~\"$instance\"}) * 100",
+ "legendFormat": "Utilization",
"refId": "A"
}
],
@@ -326,40 +206,83 @@
"overrides": []
},
"options": {
- "reduceOptions": {
- "calcs": ["lastNotNull"],
- "fields": "",
- "values": false
- },
- "orientation": "horizontal",
- "displayMode": "gradient",
- "showUnfilled": true,
- "minVizWidth": 8,
- "minVizHeight": 16,
- "valueMode": "color"
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true,
+ "orientation": "auto"
}
},
{
"type": "row",
- "title": "Network I/O",
+ "title": "CPU",
"collapsed": false,
- "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
"id": 10,
"panels": []
},
{
"type": "timeseries",
- "title": "Network Receive Rate by Instance",
- "description": "Inbound network throughput per Incus instance.",
- "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
+ "title": "CPU Usage Over Time",
+ "description": "Per-second CPU time consumed by this instance over time.",
+ "gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 },
"id": 11,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"targets": [
{
- "expr": "sum by (name) (rate(incus_network_receive_bytes_total[$__rate_interval]))",
+ "expr": "sum by (name) (rate(incus_cpu_seconds_total{name=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 25,
+ "gradientMode": "scheme",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "short",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "row",
+ "title": "Memory",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
+ "id": 20,
+ "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Memory RSS Over Time",
+ "description": "Resident set size (RSS) memory usage for this instance over time.",
+ "gridPos": { "h": 8, "w": 14, "x": 0, "y": 15 },
+ "id": 21,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "incus_memory_RSS_bytes{name=~\"$instance\"}",
"legendFormat": "{{name}}",
"refId": "A"
}
@@ -381,39 +304,99 @@
"scaleDistribution": { "type": "linear" },
"thresholdsStyle": { "mode": "off" }
},
- "unit": "Bps",
+ "unit": "bytes",
"color": { "mode": "palette-classic" },
"thresholds": {
"mode": "absolute",
- "steps": [
- { "color": "green", "value": null }
- ]
+ "steps": [{ "color": "green", "value": null }]
}
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
- "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
}
},
{
"type": "timeseries",
- "title": "Network Transmit Rate by Instance",
- "description": "Outbound network throughput per Incus instance.",
- "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
- "id": 12,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
+ "title": "Memory Utilization %",
+ "description": "Memory utilization percentage (RSS / Total) for this instance over time.",
+ "gridPos": { "h": 8, "w": 10, "x": 14, "y": 15 },
+ "id": 22,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"targets": [
{
- "expr": "sum by (name) (rate(incus_network_transmit_bytes_total[$__rate_interval]))",
+ "expr": "(incus_memory_RSS_bytes{name=~\"$instance\"} / incus_memory_MemTotal_bytes{name=~\"$instance\"}) * 100",
"legendFormat": "{{name}}",
"refId": "A"
}
],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 2,
+ "fillOpacity": 25,
+ "gradientMode": "scheme",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "line" }
+ },
+ "unit": "percent",
+ "min": 0,
+ "max": 100,
+ "color": { "mode": "continuous-GrYlRd" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null },
+ { "color": "yellow", "value": 60 },
+ { "color": "orange", "value": 80 },
+ { "color": "red", "value": 90 }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "row",
+ "title": "Network",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
+ "id": 30,
+ "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Network Traffic",
+ "description": "Network receive and transmit throughput for this instance. Transmit is shown as negative (below axis).",
+ "gridPos": { "h": 8, "w": 24, "x": 0, "y": 24 },
+ "id": 31,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum by (name) (rate(incus_network_receive_bytes_total{name=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "Receive",
+ "refId": "A"
+ },
+ {
+ "expr": "-sum by (name) (rate(incus_network_transmit_bytes_total{name=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "Transmit",
+ "refId": "B"
+ }
+ ],
"fieldConfig": {
"defaults": {
"custom": {
@@ -435,16 +418,27 @@
"color": { "mode": "palette-classic" },
"thresholds": {
"mode": "absolute",
- "steps": [
- { "color": "green", "value": null }
- ]
+ "steps": [{ "color": "green", "value": null }]
}
},
- "overrides": []
+ "overrides": [
+ {
+ "matcher": { "id": "byName", "options": "Receive" },
+ "properties": [
+ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }
+ ]
+ },
+ {
+ "matcher": { "id": "byName", "options": "Transmit" },
+ "properties": [
+ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }
+ ]
+ }
+ ]
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
- "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
}
},
{
@@ -452,24 +446,26 @@
"title": "Disk I/O",
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 },
- "id": 13,
+ "id": 40,
"panels": []
},
{
"type": "timeseries",
- "title": "Disk Read Rate by Instance",
- "description": "Disk read throughput per Incus instance.",
- "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 },
- "id": 14,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
+ "title": "Disk Read/Write Rate",
+ "description": "Disk read and write throughput for this instance.",
+ "gridPos": { "h": 8, "w": 24, "x": 0, "y": 33 },
+ "id": 41,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"targets": [
{
- "expr": "sum by (name) (rate(incus_disk_read_bytes_total[$__rate_interval]))",
- "legendFormat": "{{name}}",
+ "expr": "sum by (name) (rate(incus_disk_read_bytes_total{name=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "Read",
"refId": "A"
+ },
+ {
+ "expr": "sum by (name) (rate(incus_disk_written_bytes_total{name=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "Write",
+ "refId": "B"
}
],
"fieldConfig": {
@@ -493,66 +489,14 @@
"color": { "mode": "palette-classic" },
"thresholds": {
"mode": "absolute",
- "steps": [
- { "color": "green", "value": null }
- ]
+ "steps": [{ "color": "green", "value": null }]
}
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
- "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
- }
- },
- {
- "type": "timeseries",
- "title": "Disk Write Rate by Instance",
- "description": "Disk write throughput per Incus instance.",
- "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 },
- "id": 15,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "targets": [
- {
- "expr": "sum by (name) (rate(incus_disk_written_bytes_total[$__rate_interval]))",
- "legendFormat": "{{name}}",
- "refId": "A"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "custom": {
- "drawStyle": "line",
- "lineInterpolation": "smooth",
- "lineWidth": 1,
- "fillOpacity": 15,
- "gradientMode": "none",
- "spanNulls": false,
- "showPoints": "never",
- "pointSize": 5,
- "stacking": { "mode": "none", "group": "A" },
- "axisPlacement": "auto",
- "axisLabel": "",
- "scaleDistribution": { "type": "linear" },
- "thresholdsStyle": { "mode": "off" }
- },
- "unit": "Bps",
- "color": { "mode": "palette-classic" },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- { "color": "green", "value": null }
- ]
- }
- },
- "overrides": []
- },
- "options": {
- "tooltip": { "mode": "multi", "sort": "desc" },
- "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
}
}
]
diff --git a/incusos/observability-dashboards/core/logs-explorer.json b/incusos/observability-dashboards/core/logs-explorer.json
new file mode 100644
index 0000000..5f4808c
--- /dev/null
+++ b/incusos/observability-dashboards/core/logs-explorer.json
@@ -0,0 +1,268 @@
+{
+ "uid": "incus-logs-explorer",
+ "title": "Logs Explorer",
+ "description": "Log volume over time, full log stream with search, error and warning rates by systemd unit. Datasource: Loki.",
+ "tags": ["incus", "logs", "loki"],
+ "timezone": "browser",
+ "schemaVersion": 39,
+ "refresh": "30s",
+ "time": { "from": "now-1h", "to": "now" },
+ "fiscalYearStartMonth": 0,
+ "liveNow": false,
+ "editable": true,
+ "graphTooltip": 1,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": ["incus"],
+ "targetBlank": false,
+ "title": "Incus Dashboards",
+ "tooltip": "Navigate to other Incus dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "templating": {
+ "list": [
+ {
+ "type": "query",
+ "datasource": { "type": "loki", "uid": "${DS_LOKI}" },
+ "query": "label_values(job)",
+ "includeAll": true,
+ "multi": true,
+ "name": "job",
+ "label": "Job",
+ "refresh": 2,
+ "sort": 1
+ },
+ {
+ "type": "query",
+ "datasource": { "type": "loki", "uid": "${DS_LOKI}" },
+ "query": "label_values(host)",
+ "includeAll": true,
+ "multi": true,
+ "name": "host",
+ "label": "Host",
+ "refresh": 2,
+ "sort": 1
+ },
+ {
+ "type": "query",
+ "datasource": { "type": "loki", "uid": "${DS_LOKI}" },
+ "query": "label_values(unit)",
+ "includeAll": true,
+ "multi": true,
+ "name": "unit",
+ "label": "Unit",
+ "refresh": 2,
+ "sort": 1
+ },
+ {
+ "type": "textbox",
+ "name": "search",
+ "label": "Search",
+ "current": { "text": "", "value": "" }
+ }
+ ]
+ },
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": { "type": "grafana", "uid": "-- Grafana --" },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "panels": [
+ {
+ "type": "row",
+ "title": "Log Volume",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+ "id": 1,
+ "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Log Volume Over Time",
+ "description": "Total log line count over time, broken down by job. Shows the rate at which logs are being generated across all selected sources.",
+ "gridPos": { "h": 6, "w": 24, "x": 0, "y": 1 },
+ "id": 2,
+ "datasource": { "type": "loki", "uid": "${DS_LOKI}" },
+ "targets": [
+ {
+ "expr": "sum by (job) (count_over_time({job=~\"$job\", host=~\"$host\"} [$__auto]))",
+ "legendFormat": "{{job}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "bars",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 80,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "normal", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "short",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "row",
+ "title": "Log Stream",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 },
+ "id": 3,
+ "panels": []
+ },
+ {
+ "type": "logs",
+ "title": "Logs",
+ "description": "Live log stream filtered by job, host, unit, and free-text search. Displays timestamps, labels, and full log messages with expand-on-click detail view.",
+ "gridPos": { "h": 16, "w": 24, "x": 0, "y": 8 },
+ "id": 4,
+ "datasource": { "type": "loki", "uid": "${DS_LOKI}" },
+ "targets": [
+ {
+ "expr": "{job=~\"$job\", host=~\"$host\", unit=~\"$unit\"} |= \"$search\"",
+ "refId": "A"
+ }
+ ],
+ "options": {
+ "showTime": true,
+ "showLabels": true,
+ "showCommonLabels": false,
+ "wrapLogMessage": true,
+ "prettifyLogMessage": false,
+ "enableLogDetails": true,
+ "sortOrder": "Descending",
+ "dedupStrategy": "none"
+ }
+ },
+ {
+ "type": "row",
+ "title": "Error & Warning Rates",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 },
+ "id": 5,
+ "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Error Rate by Unit",
+ "description": "Rate of log lines matching error, fail, or fatal (case-insensitive) over time, grouped by systemd unit. Spikes indicate services experiencing failures.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 },
+ "id": 6,
+ "datasource": { "type": "loki", "uid": "${DS_LOKI}" },
+ "targets": [
+ {
+ "expr": "sum by (unit) (count_over_time({job=~\"$job\", host=~\"$host\"} |~ \"(?i)error|fail|fatal\" [$__auto]))",
+ "legendFormat": "{{unit}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "short",
+ "color": { "mode": "fixed", "fixedColor": "red" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "red", "value": null }]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Warning Rate by Unit",
+ "description": "Rate of log lines matching warn (case-insensitive) over time, grouped by systemd unit. Helps identify services generating warnings that may precede errors.",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 25 },
+ "id": 7,
+ "datasource": { "type": "loki", "uid": "${DS_LOKI}" },
+ "targets": [
+ {
+ "expr": "sum by (unit) (count_over_time({job=~\"$job\", host=~\"$host\"} |~ \"(?i)warn\" [$__auto]))",
+ "legendFormat": "{{unit}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "short",
+ "color": { "mode": "fixed", "fixedColor": "yellow" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "yellow", "value": null }]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum", "max", "lastNotNull"] }
+ }
+ }
+ ]
+}
diff --git a/incusos/observability-dashboards/core/network-deep-dive.json b/incusos/observability-dashboards/core/network-deep-dive.json
new file mode 100644
index 0000000..89598bb
--- /dev/null
+++ b/incusos/observability-dashboards/core/network-deep-dive.json
@@ -0,0 +1,529 @@
+{
+ "uid": "incus-network-deep-dive",
+ "title": "Network Deep Dive",
+ "description": "Per-node and per-device network traffic, instance-level network metrics, receive/transmit errors, and packet drops.",
+ "tags": ["incus", "incusos", "network", "node-exporter"],
+ "timezone": "browser",
+ "schemaVersion": 39,
+ "refresh": "30s",
+ "time": { "from": "now-1h", "to": "now" },
+ "fiscalYearStartMonth": 0,
+ "liveNow": false,
+ "editable": true,
+ "graphTooltip": 1,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": ["incus"],
+ "targetBlank": false,
+ "title": "Incus Dashboards",
+ "tooltip": "Navigate to other Incus dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {},
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "definition": "label_values(node_network_receive_bytes_total, instance)",
+ "includeAll": false,
+ "multi": true,
+ "name": "node",
+ "query": "label_values(node_network_receive_bytes_total, instance)",
+ "refresh": 2,
+ "sort": 1,
+ "type": "query",
+ "label": "Node"
+ },
+ {
+ "current": {},
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "definition": "label_values(node_network_receive_bytes_total{instance=~\"$node\"}, device)",
+ "includeAll": true,
+ "multi": true,
+ "name": "device",
+ "query": "label_values(node_network_receive_bytes_total{instance=~\"$node\"}, device)",
+ "refresh": 2,
+ "regex": "/^(?!lo$).*/",
+ "sort": 1,
+ "type": "query",
+ "label": "Device"
+ }
+ ]
+ },
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": { "type": "grafana", "uid": "-- Grafana --" },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "panels": [
+ {
+ "type": "row",
+ "title": "Cluster Summary",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+ "id": 1,
+ "panels": []
+ },
+ {
+ "type": "stat",
+ "title": "Total Ingress",
+ "description": "Aggregate inbound network throughput across all selected nodes and devices.",
+ "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
+ "id": 2,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum(rate(node_network_receive_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval]))",
+ "legendFormat": "Ingress",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ },
+ "unit": "Bps",
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "auto",
+ "textMode": "auto",
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto"
+ }
+ },
+ {
+ "type": "stat",
+ "title": "Total Egress",
+ "description": "Aggregate outbound network throughput across all selected nodes and devices.",
+ "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
+ "id": 3,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum(rate(node_network_transmit_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval]))",
+ "legendFormat": "Egress",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ },
+ "unit": "Bps",
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "auto",
+ "textMode": "auto",
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto"
+ }
+ },
+ {
+ "type": "stat",
+ "title": "Total Errors",
+ "description": "Combined receive and transmit error rate across all selected nodes. Any non-zero value warrants investigation.",
+ "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
+ "id": 4,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum(rate(node_network_receive_errs_total{instance=~\"$node\"}[$__rate_interval])) + sum(rate(node_network_transmit_errs_total{instance=~\"$node\"}[$__rate_interval]))",
+ "legendFormat": "Errors",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null },
+ { "color": "yellow", "value": 0.1 },
+ { "color": "red", "value": 1 }
+ ]
+ },
+ "unit": "pps",
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "auto",
+ "textMode": "auto",
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto"
+ }
+ },
+ {
+ "type": "stat",
+ "title": "Total Drops",
+ "description": "Combined receive and transmit packet drop rate across all selected nodes. Drops indicate buffer or queue exhaustion.",
+ "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
+ "id": 5,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum(rate(node_network_receive_drop_total{instance=~\"$node\"}[$__rate_interval])) + sum(rate(node_network_transmit_drop_total{instance=~\"$node\"}[$__rate_interval]))",
+ "legendFormat": "Drops",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null },
+ { "color": "yellow", "value": 0.1 },
+ { "color": "red", "value": 1 }
+ ]
+ },
+ "unit": "pps",
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "auto",
+ "textMode": "auto",
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto"
+ }
+ },
+ {
+ "type": "row",
+ "title": "Per-Node Traffic",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
+ "id": 10,
+ "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Receive Rate by Node & Device",
+ "description": "Inbound network throughput broken down by node and network device. Identifies which interfaces are handling the most receive traffic.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
+ "id": 11,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "rate(node_network_receive_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
+ "legendFormat": "{{instance}} {{device}} rx",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "Bps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Transmit Rate by Node & Device",
+ "description": "Outbound network throughput broken down by node and network device. Identifies which interfaces are handling the most transmit traffic.",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
+ "id": 12,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "rate(node_network_transmit_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
+ "legendFormat": "{{instance}} {{device}} tx",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "Bps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "row",
+ "title": "Instance Network (Incus metrics)",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
+ "id": 20,
+ "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Instance Receive Rate",
+ "description": "Inbound network throughput per Incus instance, aggregated across all instance network devices. Uses Incus-native metrics for container and VM visibility.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 },
+ "id": 21,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum by (name) (rate(incus_network_receive_bytes_total{instance=~\"$node\"}[$__rate_interval]))",
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "Bps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Instance Transmit Rate",
+ "description": "Outbound network throughput per Incus instance, aggregated across all instance network devices. Uses Incus-native metrics for container and VM visibility.",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 },
+ "id": 22,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum by (name) (rate(incus_network_transmit_bytes_total{instance=~\"$node\"}[$__rate_interval]))",
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "Bps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "row",
+ "title": "Errors & Drops",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
+ "id": 30,
+ "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Receive/Transmit Errors",
+ "description": "Network interface error rates broken down by node and device. Receive errors may indicate CRC failures or framing issues; transmit errors may indicate collisions or driver problems.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
+ "id": 31,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "rate(node_network_receive_errs_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
+ "legendFormat": "{{instance}} {{device}} rx errors",
+ "refId": "A"
+ },
+ {
+ "expr": "rate(node_network_transmit_errs_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
+ "legendFormat": "{{instance}} {{device}} tx errors",
+ "refId": "B"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "pps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Packet Drops",
+ "description": "Network interface packet drop rates broken down by node and device. Drops indicate kernel buffer or queue exhaustion and may cause retransmissions or connection failures.",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
+ "id": 32,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "rate(node_network_receive_drop_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
+ "legendFormat": "{{instance}} {{device}} rx drops",
+ "refId": "A"
+ },
+ {
+ "expr": "rate(node_network_transmit_drop_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
+ "legendFormat": "{{instance}} {{device}} tx drops",
+ "refId": "B"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "pps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ }
+ ]
+}
diff --git a/incusos/observability-dashboards/core/prometheus-health.json b/incusos/observability-dashboards/core/prometheus-health.json
new file mode 100644
index 0000000..f8f3eee
--- /dev/null
+++ b/incusos/observability-dashboards/core/prometheus-health.json
@@ -0,0 +1,748 @@
+{
+ "uid": "incus-prometheus-health",
+ "title": "Prometheus Health",
+ "description": "Prometheus self-monitoring: targets up/down, scrape duration, samples ingested, TSDB size, head series, WAL stats.",
+ "tags": ["incus", "prometheus", "operations"],
+ "timezone": "browser",
+ "schemaVersion": 39,
+ "refresh": "30s",
+ "time": { "from": "now-1h", "to": "now" },
+ "fiscalYearStartMonth": 0,
+ "liveNow": false,
+ "editable": true,
+ "graphTooltip": 1,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": ["incus"],
+ "targetBlank": false,
+ "title": "Incus Dashboards",
+ "tooltip": "Navigate to other Incus dashboards",
+ "type": "dashboards"
+ }
+ ],
+ "templating": {
+ "list": []
+ },
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": { "type": "grafana", "uid": "-- Grafana --" },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "panels": [
+ {
+ "type": "row",
+ "title": "Targets",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+ "id": 1,
+ "panels": []
+ },
+ {
+ "type": "stat",
+ "title": "Targets Up",
+ "description": "Number of scrape targets currently reporting as healthy (up == 1).",
+ "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
+ "id": 2,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "count(up == 1)",
+ "legendFormat": "Up",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null }
+ ]
+ },
+ "unit": "none",
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "auto",
+ "textMode": "auto",
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto"
+ }
+ },
+ {
+ "type": "stat",
+ "title": "Targets Down",
+ "description": "Number of scrape targets currently unreachable or failing (up == 0). Any value above 0 requires investigation.",
+ "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
+ "id": 3,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "count(up == 0)",
+ "legendFormat": "Down",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null },
+ { "color": "red", "value": 1 }
+ ]
+ },
+ "unit": "none",
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "auto",
+ "textMode": "auto",
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto"
+ }
+ },
+ {
+ "type": "stat",
+ "title": "Avg Scrape Duration",
+ "description": "Average time Prometheus takes to scrape all targets. High values may indicate slow or overloaded exporters.",
+ "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
+ "id": 4,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "avg(scrape_duration_seconds)",
+ "legendFormat": "Avg Duration",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null },
+ { "color": "yellow", "value": 1 },
+ { "color": "red", "value": 5 }
+ ]
+ },
+ "unit": "s",
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "auto",
+ "textMode": "auto",
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto"
+ }
+ },
+ {
+ "type": "stat",
+ "title": "Samples Ingested/s",
+ "description": "Total rate of samples being scraped and ingested across all targets per second.",
+ "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
+ "id": 5,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum(rate(scrape_samples_scraped[$__rate_interval]))",
+ "legendFormat": "Samples/s",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null }
+ ]
+ },
+ "unit": "short",
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "auto",
+ "textMode": "auto",
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto"
+ }
+ },
+ {
+ "type": "table",
+ "title": "Target Health",
+ "description": "Per-target health status showing each scrape target's current state. UP (1) means healthy, DOWN (0) means unreachable or failing.",
+ "gridPos": { "h": 8, "w": 24, "x": 0, "y": 5 },
+ "id": 6,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "up",
+ "legendFormat": "",
+ "refId": "A",
+ "format": "table",
+ "instant": true
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto",
+ "inspect": false,
+ "filterable": true
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null }
+ ]
+ },
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": [
+ {
+ "matcher": { "id": "byName", "options": "instance" },
+ "properties": [
+ { "id": "displayName", "value": "Instance" },
+ { "id": "custom.width", "value": 300 }
+ ]
+ },
+ {
+ "matcher": { "id": "byName", "options": "job" },
+ "properties": [
+ { "id": "displayName", "value": "Job" },
+ { "id": "custom.width", "value": 200 }
+ ]
+ },
+ {
+ "matcher": { "id": "byName", "options": "Value" },
+ "properties": [
+ { "id": "displayName", "value": "Status" },
+ { "id": "custom.displayMode", "value": "color-background" },
+ {
+ "id": "mappings",
+ "value": [
+ {
+ "type": "value",
+ "options": {
+ "1": { "text": "UP", "color": "green" },
+ "0": { "text": "DOWN", "color": "red" }
+ }
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "showHeader": true,
+ "sortBy": [{ "displayName": "Job", "desc": false }],
+ "footer": { "show": false, "reducer": ["sum"], "fields": "" }
+ },
+ "transformations": [
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": { "Time": true, "__name__": true },
+ "indexByName": {
+ "instance": 0,
+ "job": 1,
+ "Value": 2
+ },
+ "renameByName": {}
+ }
+ }
+ ]
+ },
+ {
+ "type": "row",
+ "title": "Scrape Performance",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 },
+ "id": 7,
+ "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Scrape Duration by Job",
+ "description": "How long each scrape takes per job. Spikes indicate slow exporters or network issues between Prometheus and targets.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 },
+ "id": 8,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "scrape_duration_seconds",
+ "legendFormat": "{{job}} - {{instance}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "s",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Samples Scraped by Job",
+ "description": "Rate of samples scraped per job. Sudden drops may indicate target failures; spikes may indicate new metrics being exposed.",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 },
+ "id": 9,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "rate(scrape_samples_scraped[$__rate_interval])",
+ "legendFormat": "{{job}} - {{instance}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "short",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "row",
+ "title": "TSDB",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 },
+ "id": 10,
+ "panels": []
+ },
+ {
+ "type": "stat",
+ "title": "TSDB Size",
+ "description": "Current on-disk size of the Prometheus TSDB storage directory including all blocks and WAL.",
+ "gridPos": { "h": 4, "w": 6, "x": 0, "y": 23 },
+ "id": 11,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "prometheus_tsdb_storage_size_bytes",
+ "legendFormat": "TSDB Size",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null },
+ { "color": "yellow", "value": 1073741824 },
+ { "color": "red", "value": 5368709120 }
+ ]
+ },
+ "unit": "bytes",
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "auto",
+ "textMode": "auto",
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto"
+ }
+ },
+ {
+ "type": "stat",
+ "title": "Head Series",
+ "description": "Number of active time series in the TSDB head block. High cardinality here impacts query performance and memory usage.",
+ "gridPos": { "h": 4, "w": 6, "x": 6, "y": 23 },
+ "id": 12,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "prometheus_tsdb_head_series",
+ "legendFormat": "Head Series",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null },
+ { "color": "yellow", "value": 100000 },
+ { "color": "red", "value": 500000 }
+ ]
+ },
+ "unit": "short",
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "auto",
+ "textMode": "auto",
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto"
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Head Chunks",
+ "description": "Number of chunks in the TSDB head block. Growth correlates with active series count and scrape frequency.",
+ "gridPos": { "h": 8, "w": 6, "x": 12, "y": 23 },
+ "id": 13,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "prometheus_tsdb_head_chunks",
+ "legendFormat": "Head Chunks",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "short",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "WAL Size",
+ "description": "Size of the Prometheus write-ahead log (WAL). The WAL buffers recent writes before compaction into blocks.",
+ "gridPos": { "h": 8, "w": 6, "x": 18, "y": 23 },
+ "id": 14,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "prometheus_tsdb_wal_storage_size_bytes",
+ "legendFormat": "WAL Size",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "bytes",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "row",
+ "title": "TSDB Trends",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 },
+ "id": 15,
+ "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "TSDB Size Over Time",
+ "description": "Prometheus TSDB total storage size trend. Useful for capacity planning and detecting unexpected growth.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 },
+ "id": 16,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "prometheus_tsdb_storage_size_bytes",
+ "legendFormat": "TSDB Size",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "bytes",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Samples Ingested Trend",
+ "description": "Rate of new samples appended to the TSDB head block. Correlates with scrape frequency and number of active series.",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 },
+ "id": 17,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_head_samples_appended_total[$__rate_interval])",
+ "legendFormat": "Samples/s",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "short",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Compactions",
+ "description": "Rate of TSDB compaction operations. Compactions merge head block data into persistent blocks on disk for efficient storage.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 },
+ "id": 18,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_compactions_total[$__rate_interval])",
+ "legendFormat": "Compactions/s",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "short",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Head Series Count",
+ "description": "Number of active time series in the TSDB head block over time. Trend helps identify cardinality growth or label explosion.",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 },
+ "id": 19,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "prometheus_tsdb_head_series",
+ "legendFormat": "Head Series",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "gradientMode": "none",
+ "spanNulls": false,
+ "showPoints": "never",
+ "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "short",
+ "color": { "mode": "palette-classic" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{ "color": "green", "value": null }]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ }
+ ]
+}
diff --git a/incusos/observability-dashboards/core/storage-filesystem.json b/incusos/observability-dashboards/core/storage-filesystem.json
new file mode 100644
index 0000000..0a8cec0
--- /dev/null
+++ b/incusos/observability-dashboards/core/storage-filesystem.json
@@ -0,0 +1,382 @@
+{
+ "uid": "incus-storage-filesystem",
+ "title": "Storage & Filesystem",
+ "description": "Filesystem usage, available space trends, disk throughput, I/O latency, disk utilization, and inode usage per node.",
+ "tags": ["incus", "incusos", "storage", "node-exporter"],
+ "timezone": "browser",
+ "schemaVersion": 39,
+ "refresh": "30s",
+ "time": { "from": "now-1h", "to": "now" },
+ "fiscalYearStartMonth": 0,
+ "liveNow": false,
+ "editable": true,
+ "graphTooltip": 1,
+ "links": [
+ {
+ "asDropdown": true, "icon": "external link", "includeVars": true,
+ "keepTime": true, "tags": ["incus"], "targetBlank": false,
+ "title": "Incus Dashboards", "type": "dashboards"
+ }
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {},
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "definition": "label_values(node_filesystem_size_bytes, instance)",
+ "includeAll": false,
+ "multi": true,
+ "name": "node",
+ "query": "label_values(node_filesystem_size_bytes, instance)",
+ "refresh": 2,
+ "sort": 1,
+ "type": "query",
+ "label": "Node"
+ },
+ {
+ "current": {},
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "definition": "label_values(node_filesystem_size_bytes{instance=~\"$node\"}, mountpoint)",
+ "includeAll": true,
+ "multi": true,
+ "name": "mountpoint",
+ "query": "label_values(node_filesystem_size_bytes{instance=~\"$node\"}, mountpoint)",
+ "refresh": 2,
+ "sort": 1,
+ "type": "query",
+ "label": "Mountpoint"
+ }
+ ]
+ },
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": { "type": "grafana", "uid": "-- Grafana --" },
+ "enable": true, "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts", "type": "dashboard"
+ }
+ ]
+ },
+ "panels": [
+ {
+ "type": "row", "title": "Filesystem Overview",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+ "id": 1, "panels": []
+ },
+ {
+ "type": "table",
+ "title": "Filesystem Usage Table",
+ "description": "Current filesystem size, available space, and usage percentage per node and mountpoint. High usage is highlighted with color thresholds.",
+ "gridPos": { "h": 8, "w": 24, "x": 0, "y": 1 },
+ "id": 2,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "node_filesystem_size_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}",
+ "legendFormat": "", "refId": "A", "format": "table", "instant": true
+ },
+ {
+ "expr": "node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}",
+ "legendFormat": "", "refId": "B", "format": "table", "instant": true
+ },
+ {
+ "expr": "(1 - node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}) * 100",
+ "legendFormat": "", "refId": "C", "format": "table", "instant": true
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": { "align": "auto", "displayMode": "auto", "inspect": false, "filterable": true },
+ "color": { "mode": "thresholds" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": [
+ { "matcher": { "id": "byName", "options": "instance" }, "properties": [{ "id": "displayName", "value": "Node" }] },
+ { "matcher": { "id": "byName", "options": "mountpoint" }, "properties": [{ "id": "displayName", "value": "Mountpoint" }] },
+ { "matcher": { "id": "byName", "options": "fstype" }, "properties": [{ "id": "displayName", "value": "FS Type" }] },
+ { "matcher": { "id": "byName", "options": "Value #A" }, "properties": [{ "id": "displayName", "value": "Size" }, { "id": "unit", "value": "bytes" }] },
+ { "matcher": { "id": "byName", "options": "Value #B" }, "properties": [{ "id": "displayName", "value": "Available" }, { "id": "unit", "value": "bytes" }] },
+ {
+ "matcher": { "id": "byName", "options": "Value #C" },
+ "properties": [
+ { "id": "displayName", "value": "Used %" },
+ { "id": "unit", "value": "percent" },
+ { "id": "custom.displayMode", "value": "color-background" },
+ { "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "orange", "value": 85 }, { "color": "red", "value": 95 }] } }
+ ]
+ }
+ ]
+ },
+ "options": { "showHeader": true, "sortBy": [], "footer": { "show": false, "reducer": ["sum"], "fields": "" } },
+ "transformations": [
+ { "id": "merge", "options": {} },
+ { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "job": true, "device": true }, "renameByName": {} } }
+ ]
+ },
+ {
+ "type": "row", "title": "Space Trends",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
+ "id": 3, "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Available Space Over Time",
+ "description": "Tracks available filesystem space over time per node and mountpoint. Use this to identify gradual space consumption trends before they become critical.",
+ "gridPos": { "h": 8, "w": 24, "x": 0, "y": 10 },
+ "id": 4,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}",
+ "legendFormat": "{{instance}} {{mountpoint}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2,
+ "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+ "showPoints": "never", "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "bytes",
+ "color": { "mode": "palette-classic" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "row", "title": "Disk Throughput",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 },
+ "id": 5, "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Disk Read Throughput",
+ "description": "Disk read throughput in bytes per second, broken down by instance and device. Identifies which disks are handling the most read traffic.",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 },
+ "id": 6,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum by (instance, device) (rate(node_disk_read_bytes_total{instance=~\"$node\"}[$__rate_interval]))",
+ "legendFormat": "{{instance}} {{device}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+ "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+ "showPoints": "never", "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "Bps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Disk Write Throughput",
+ "description": "Disk write throughput in bytes per second, broken down by instance and device. Identifies which disks are handling the most write traffic.",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 },
+ "id": 7,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum by (instance, device) (rate(node_disk_written_bytes_total{instance=~\"$node\"}[$__rate_interval]))",
+ "legendFormat": "{{instance}} {{device}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+ "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+ "showPoints": "never", "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "Bps",
+ "color": { "mode": "palette-classic" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "row", "title": "I/O Performance",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 },
+ "id": 8, "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Average Read Latency",
+ "description": "Average time per completed read operation. Calculated as total read time divided by completed reads. High values indicate slow disk response.",
+ "gridPos": { "h": 8, "w": 8, "x": 0, "y": 28 },
+ "id": 9,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "rate(node_disk_read_time_seconds_total{instance=~\"$node\"}[$__rate_interval]) / rate(node_disk_reads_completed_total{instance=~\"$node\"}[$__rate_interval])",
+ "legendFormat": "{{instance}} {{device}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+ "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+ "showPoints": "never", "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "s",
+ "color": { "mode": "palette-classic" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Average Write Latency",
+ "description": "Average time per completed write operation. Calculated as total write time divided by completed writes. High values indicate slow disk response or write saturation.",
+ "gridPos": { "h": 8, "w": 8, "x": 8, "y": 28 },
+ "id": 10,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "rate(node_disk_write_time_seconds_total{instance=~\"$node\"}[$__rate_interval]) / rate(node_disk_writes_completed_total{instance=~\"$node\"}[$__rate_interval])",
+ "legendFormat": "{{instance}} {{device}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+ "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+ "showPoints": "never", "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "off" }
+ },
+ "unit": "s",
+ "color": { "mode": "palette-classic" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "timeseries",
+ "title": "Disk Utilization %",
+ "description": "Percentage of time the disk is busy processing I/O requests. Values near 100% indicate the disk is saturated and may be a bottleneck.",
+ "gridPos": { "h": 8, "w": 8, "x": 16, "y": 28 },
+ "id": 11,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "sum by (instance) (rate(node_disk_io_time_seconds_total{instance=~\"$node\"}[$__rate_interval])) * 100",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2,
+ "fillOpacity": 25, "gradientMode": "scheme", "spanNulls": false,
+ "showPoints": "never", "pointSize": 5,
+ "stacking": { "mode": "none", "group": "A" },
+ "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+ "thresholdsStyle": { "mode": "line" }
+ },
+ "unit": "percent", "min": 0, "max": 100,
+ "color": { "mode": "continuous-GrYlRd" },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 80 }] }
+ },
+ "overrides": []
+ },
+ "options": {
+ "tooltip": { "mode": "multi", "sort": "desc" },
+ "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+ }
+ },
+ {
+ "type": "row", "title": "Inodes",
+ "collapsed": false,
+ "gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 },
+ "id": 12, "panels": []
+ },
+ {
+ "type": "bargauge",
+ "title": "Inode Usage %",
+ "description": "Percentage of used inodes per filesystem. Running out of inodes prevents creating new files even when disk space is available. Monitor for filesystems with many small files.",
+ "gridPos": { "h": 6, "w": 24, "x": 0, "y": 37 },
+ "id": 13,
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "targets": [
+ {
+ "expr": "(1 - node_filesystem_files_free{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"} / node_filesystem_files{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}) * 100",
+ "legendFormat": "{{instance}} {{mountpoint}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] },
+ "unit": "percent", "min": 0, "max": 100,
+ "color": { "mode": "thresholds" }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "horizontal", "displayMode": "gradient", "showUnfilled": true,
+ "minVizWidth": 8, "minVizHeight": 16, "valueMode": "color"
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/incusos/observability-dashboards/dashboard.yaml b/incusos/observability-dashboards/dashboard.yaml
new file mode 100644
index 0000000..f97a7bf
--- /dev/null
+++ b/incusos/observability-dashboards/dashboard.yaml
@@ -0,0 +1,97 @@
+# Dashboard manifest for the Incus observability stack
+#
+# Each entry maps a dashboard JSON file to its Grafana folder, UID, and metadata.
+# Used by manage-dashboards for install, export, list, and validate operations.
+
+dashboards:
+ # --- Core dashboards (ship with product) ---
+
+ - file: core/cluster-overview.json
+ uid: incus-cluster-overview
+ title: Incus Cluster Overview
+ folder: Incus Cluster
+ category: core
+ version: "2"
+ description: >-
+ Overview of all Incus instances: CPU, memory, network, disk I/O,
+ and instance status table with drill-down links.
+
+ - file: core/host-resources.json
+ uid: incus-host-resources
+ title: Host Resources (IncusOS Nodes)
+ folder: Incus Infrastructure
+ category: core
+ version: "2"
+ description: >-
+ Physical host monitoring: CPU mode breakdown, load averages,
+ memory breakdown, filesystem usage, disk IOPS, network errors.
+
+ - file: core/haproxy-traffic.json
+ uid: incus-haproxy-traffic
+ title: HAProxy Traffic
+ folder: Incus Services
+ category: core
+ version: "2"
+ description: >-
+ HAProxy load balancer monitoring: request rates, response codes,
+ backend health, server status timeline, connection errors.
+
+ - file: core/instance-deep-dive.json
+ uid: incus-instance-deep-dive
+ title: Instance Deep Dive
+ folder: Incus Cluster
+ category: core
+ version: "1"
+ description: >-
+ Single-instance drill-down: CPU, memory, network, and disk
+ metrics with detailed timeseries and utilization gauges.
+
+ - file: core/storage-filesystem.json
+ uid: incus-storage-filesystem
+ title: Storage & Filesystem
+ folder: Incus Infrastructure
+ category: core
+ version: "1"
+ description: >-
+ Filesystem usage, available space trends, disk throughput,
+ I/O latency, disk utilization, and inode usage per node.
+
+ - file: core/network-deep-dive.json
+ uid: incus-network-deep-dive
+ title: Network Deep Dive
+ folder: Incus Infrastructure
+ category: core
+ version: "1"
+ description: >-
+ Per-node and per-device network traffic, instance-level network
+ metrics, receive/transmit errors, and packet drops.
+
+ - file: core/logs-explorer.json
+ uid: incus-logs-explorer
+ title: Logs Explorer
+ folder: Incus Operations
+ category: core
+ version: "1"
+ description: >-
+ Log volume over time, full log stream with search, error
+ and warning rates by systemd unit. Datasource: Loki.
+
+ - file: core/prometheus-health.json
+ uid: incus-prometheus-health
+ title: Prometheus Health
+ folder: Incus Operations
+ category: core
+ version: "1"
+ description: >-
+ Prometheus self-monitoring: targets up/down, scrape duration,
+ samples ingested, TSDB size, head series, WAL stats.
+
+ - file: core/capacity-planning.json
+ uid: incus-capacity-planning
+ title: Capacity Planning
+ folder: Incus Operations
+ category: core
+ version: "1"
+ description: >-
+ CPU/memory headroom per node, resource trends, disk space
+ forecast with predict_linear, instance count trends.
diff --git a/incusos/observability-dashboards/haproxy-traffic.json b/incusos/observability-dashboards/haproxy-traffic.json
deleted file mode 100644
index 8f89cbe..0000000
--- a/incusos/observability-dashboards/haproxy-traffic.json
+++ /dev/null
@@ -1,518 +0,0 @@
-{
- "uid": "incus-haproxy-traffic",
- "title": "HAProxy Traffic",
- "description": "HAProxy load balancer monitoring: request rates, backend health, traffic volume, and sessions.",
- "tags": ["haproxy", "loadbalancer", "incus"],
- "timezone": "browser",
- "schemaVersion": 39,
- "version": 1,
- "refresh": "30s",
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "fiscalYearStartMonth": 0,
- "liveNow": false,
- "editable": true,
- "graphTooltip": 1,
- "templating": {
- "list": []
- },
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": {
- "type": "grafana",
- "uid": "-- Grafana --"
- },
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "type": "dashboard"
- }
- ]
- },
- "panels": [
- {
- "type": "row",
- "title": "Overview",
- "collapsed": false,
- "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
- "id": 1,
- "panels": []
- },
- {
- "type": "stat",
- "title": "Active Backend Servers",
- "description": "Total number of active servers across all HAProxy backends.",
- "gridPos": { "h": 4, "w": 8, "x": 0, "y": 1 },
- "id": 2,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "targets": [
- {
- "expr": "sum(haproxy_backend_active_servers)",
- "legendFormat": "Active Servers",
- "refId": "A"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "thresholds": {
- "mode": "absolute",
- "steps": [
- { "color": "red", "value": null },
- { "color": "yellow", "value": 1 },
- { "color": "green", "value": 2 }
- ]
- },
- "unit": "none",
- "color": { "mode": "thresholds" }
- },
- "overrides": []
- },
- "options": {
- "reduceOptions": {
- "calcs": ["lastNotNull"],
- "fields": "",
- "values": false
- },
- "orientation": "auto",
- "textMode": "auto",
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto"
- }
- },
- {
- "type": "stat",
- "title": "Current Sessions",
- "description": "Total current sessions across all HAProxy backends.",
- "gridPos": { "h": 4, "w": 8, "x": 8, "y": 1 },
- "id": 3,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "targets": [
- {
- "expr": "sum(haproxy_backend_current_sessions)",
- "legendFormat": "Sessions",
- "refId": "A"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "thresholds": {
- "mode": "absolute",
- "steps": [
- { "color": "green", "value": null },
- { "color": "yellow", "value": 100 },
- { "color": "red", "value": 500 }
- ]
- },
- "unit": "none",
- "color": { "mode": "thresholds" }
- },
- "overrides": []
- },
- "options": {
- "reduceOptions": {
- "calcs": ["lastNotNull"],
- "fields": "",
- "values": false
- },
- "orientation": "auto",
- "textMode": "auto",
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto"
- }
- },
- {
- "type": "stat",
- "title": "Total HTTP Requests",
- "description": "Total HTTP requests across all HAProxy frontends (counter).",
- "gridPos": { "h": 4, "w": 8, "x": 16, "y": 1 },
- "id": 4,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "targets": [
- {
- "expr": "sum(rate(haproxy_frontend_http_requests_total[$__rate_interval]))",
- "legendFormat": "Requests/s",
- "refId": "A"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "thresholds": {
- "mode": "absolute",
- "steps": [
- { "color": "green", "value": null }
- ]
- },
- "unit": "reqps",
- "color": { "mode": "thresholds" }
- },
- "overrides": []
- },
- "options": {
- "reduceOptions": {
- "calcs": ["lastNotNull"],
- "fields": "",
- "values": false
- },
- "orientation": "auto",
- "textMode": "auto",
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto"
- }
- },
- {
- "type": "row",
- "title": "HTTP Requests",
- "collapsed": false,
- "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
- "id": 5,
- "panels": []
- },
- {
- "type": "timeseries",
- "title": "HTTP Request Rate by Frontend",
- "description": "Rate of HTTP requests per second, broken down by HAProxy frontend/proxy.",
- "gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 },
- "id": 6,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "targets": [
- {
- "expr": "sum by (proxy) (rate(haproxy_frontend_http_requests_total[$__rate_interval]))",
- "legendFormat": "{{proxy}}",
- "refId": "A"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "custom": {
- "drawStyle": "line",
- "lineInterpolation": "smooth",
- "lineWidth": 1,
- "fillOpacity": 20,
- "gradientMode": "none",
- "spanNulls": false,
- "showPoints": "never",
- "pointSize": 5,
- "stacking": { "mode": "none", "group": "A" },
- "axisPlacement": "auto",
- "axisLabel": "",
- "scaleDistribution": { "type": "linear" },
- "thresholdsStyle": { "mode": "off" }
- },
- "unit": "reqps",
- "color": { "mode": "palette-classic" },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- { "color": "green", "value": null }
- ]
- }
- },
- "overrides": []
- },
- "options": {
- "tooltip": { "mode": "multi", "sort": "desc" },
- "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
- }
- },
- {
- "type": "row",
- "title": "Backend Health",
- "collapsed": false,
- "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
- "id": 7,
- "panels": []
- },
- {
- "type": "table",
- "title": "Backend Status",
- "description": "Current status, active server count, and average response time for each HAProxy backend.",
- "gridPos": { "h": 8, "w": 24, "x": 0, "y": 15 },
- "id": 8,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "targets": [
- {
- "expr": "haproxy_backend_status",
- "legendFormat": "",
- "refId": "A",
- "format": "table",
- "instant": true
- },
- {
- "expr": "haproxy_backend_active_servers",
- "legendFormat": "",
- "refId": "B",
- "format": "table",
- "instant": true
- },
- {
- "expr": "haproxy_server_response_time_average_seconds",
- "legendFormat": "",
- "refId": "C",
- "format": "table",
- "instant": true
- }
- ],
- "fieldConfig": {
- "defaults": {
- "custom": {
- "align": "auto",
- "displayMode": "auto",
- "inspect": false,
- "filterable": true
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- { "color": "green", "value": null }
- ]
- },
- "color": { "mode": "thresholds" }
- },
- "overrides": [
- {
- "matcher": { "id": "byName", "options": "Value #A" },
- "properties": [
- { "id": "displayName", "value": "Status" }
- ]
- },
- {
- "matcher": { "id": "byName", "options": "Value #B" },
- "properties": [
- { "id": "displayName", "value": "Active Servers" }
- ]
- },
- {
- "matcher": { "id": "byName", "options": "Value #C" },
- "properties": [
- { "id": "displayName", "value": "Avg Response Time" },
- { "id": "unit", "value": "s" }
- ]
- },
- {
- "matcher": { "id": "byName", "options": "proxy" },
- "properties": [
- { "id": "displayName", "value": "Backend" }
- ]
- }
- ]
- },
- "options": {
- "showHeader": true,
- "sortBy": [],
- "footer": {
- "show": false,
- "reducer": ["sum"],
- "fields": ""
- }
- },
- "transformations": [
- {
- "id": "merge",
- "options": {}
- },
- {
- "id": "organize",
- "options": {
- "excludeByName": {
- "Time": true,
- "__name__": true,
- "instance": false,
- "job": true
- },
- "indexByName": {},
- "renameByName": {}
- }
- }
- ]
- },
- {
- "type": "row",
- "title": "Traffic",
- "collapsed": false,
- "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
- "id": 9,
- "panels": []
- },
- {
- "type": "timeseries",
- "title": "Frontend Bytes In",
- "description": "Inbound traffic rate per HAProxy frontend.",
- "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
- "id": 10,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "targets": [
- {
- "expr": "sum by (proxy) (rate(haproxy_frontend_bytes_in_total[$__rate_interval]))",
- "legendFormat": "{{proxy}} (in)",
- "refId": "A"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "custom": {
- "drawStyle": "line",
- "lineInterpolation": "smooth",
- "lineWidth": 1,
- "fillOpacity": 20,
- "gradientMode": "none",
- "spanNulls": false,
- "showPoints": "never",
- "pointSize": 5,
- "stacking": { "mode": "none", "group": "A" },
- "axisPlacement": "auto",
- "axisLabel": "",
- "scaleDistribution": { "type": "linear" },
- "thresholdsStyle": { "mode": "off" }
- },
- "unit": "Bps",
- "color": { "mode": "palette-classic" },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- { "color": "green", "value": null }
- ]
- }
- },
- "overrides": []
- },
- "options": {
- "tooltip": { "mode": "multi", "sort": "desc" },
- "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
- }
- },
- {
- "type": "timeseries",
- "title": "Frontend Bytes Out",
- "description": "Outbound traffic rate per HAProxy frontend.",
- "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
- "id": 11,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "targets": [
- {
- "expr": "sum by (proxy) (rate(haproxy_frontend_bytes_out_total[$__rate_interval]))",
- "legendFormat": "{{proxy}} (out)",
- "refId": "A"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "custom": {
- "drawStyle": "line",
- "lineInterpolation": "smooth",
- "lineWidth": 1,
- "fillOpacity": 20,
- "gradientMode": "none",
- "spanNulls": false,
- "showPoints": "never",
- "pointSize": 5,
- "stacking": { "mode": "none", "group": "A" },
- "axisPlacement": "auto",
- "axisLabel": "",
- "scaleDistribution": { "type": "linear" },
- "thresholdsStyle": { "mode": "off" }
- },
- "unit": "Bps",
- "color": { "mode": "palette-classic" },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- { "color": "green", "value": null }
- ]
- }
- },
- "overrides": []
- },
- "options": {
- "tooltip": { "mode": "multi", "sort": "desc" },
- "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
- }
- },
- {
- "type": "row",
- "title": "Sessions",
- "collapsed": false,
- "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 },
- "id": 12,
- "panels": []
- },
- {
- "type": "timeseries",
- "title": "Current Sessions by Proxy",
- "description": "Current active sessions for each HAProxy backend/proxy over time.",
- "gridPos": { "h": 8, "w": 24, "x": 0, "y": 33 },
- "id": 13,
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "targets": [
- {
- "expr": "haproxy_backend_current_sessions",
- "legendFormat": "{{proxy}}",
- "refId": "A"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "custom": {
- "drawStyle": "line",
- "lineInterpolation": "smooth",
- "lineWidth": 1,
- "fillOpacity": 20,
- "gradientMode": "none",
- "spanNulls": false,
- "showPoints": "never",
- "pointSize": 5,
- "stacking": { "mode": "none", "group": "A" },
- "axisPlacement": "auto",
- "axisLabel": "",
- "scaleDistribution": { "type": "linear" },
- "thresholdsStyle": { "mode": "off" }
- },
- "unit": "none",
- "color": { "mode": "palette-classic" },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- { "color": "green", "value": null }
- ]
- }
- },
- "overrides": []
- },
- "options": {
- "tooltip": { "mode": "multi", "sort": "desc" },
- "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
- }
- }
- ]
-}
diff --git a/notes/observability-guide.md b/notes/observability-guide.md
index a0e7dea..63fd180 100644
--- a/notes/observability-guide.md
+++ b/notes/observability-guide.md
@@ -122,40 +122,106 @@ datasource proxy.
## Dashboards
-Three dashboards are provisioned automatically during deployment.
+Nine dashboards are provisioned automatically during deployment, organized
+into four Grafana folders. All dashboards support template variables for
+filtering and include cross-dashboard navigation links.
-### Incus Cluster Overview
+Use `manage-dashboards` to install, export, validate, or list dashboards:
-Visualizes metrics scraped from the Incus `/1.0/metrics` endpoint on each
-cluster node.
+```bash
+incusos/manage-dashboards --install # push all dashboards to Grafana
+incusos/manage-dashboards --list # show install status
+incusos/manage-dashboards --validate # check JSON quality
+incusos/manage-dashboards --export # export from Grafana to clean JSON
+incusos/manage-dashboards --status # compare local vs installed
+```
-- Instance CPU usage (per-instance, per-node)
-- Memory usage and allocation
-- Network I/O (bytes in/out per interface)
-- Disk I/O (reads/writes, latency)
-- Instance count and state
+### Incus Cluster — Incus Cluster Overview
-### HAProxy Traffic
+Overview of all instances with instance/node template variables.
-Visualizes metrics from the HAProxy stats endpoint (`:8404/metrics`).
+- Instance count, total CPUs, total memory, cluster memory utilization gauge
+- Instance status table with drill-down links to Instance Deep Dive
+- Per-instance CPU usage, memory RSS, memory usage % bar gauge
+- Network receive/transmit rates, disk read/write rates
-- Request rate (frontend and backend)
-- Backend health status (UP/DOWN per server)
-- Active sessions and session rate
-- Traffic volume (bytes in/out)
-- HTTP response codes (2xx, 4xx, 5xx)
-- Connection errors and retries
+### Incus Cluster — Instance Deep Dive
-### Host Resources
+Single-instance drill-down selected via template variable.
-Visualizes metrics from node_exporter on each cluster node.
+- Uptime, effective CPUs, total memory, memory utilization gauge
+- CPU usage over time with gradient color scheme
+- Memory RSS and memory utilization % over time
+- Network traffic (receive above axis, transmit below on same panel)
+- Disk read/write rates
-- CPU utilization (%)
-- Memory utilization (%)
-- Disk usage and I/O
-- Network throughput per interface
-- System load averages
-- Filesystem free space
+### Incus Services — HAProxy Traffic
+
+HAProxy monitoring with proxy/server template variables.
+
+- Active servers, current sessions, HTTP request rate
+- HTTP 4xx/5xx rate stat panels with warning thresholds
+- Response code distribution (stacked, color-coded by class)
+- Backend status table and server status state timeline
+- Frontend traffic, sessions by proxy, response times
+- Queue time and connection errors
+
+### Incus Infrastructure — Host Resources (IncusOS Nodes)
+
+Physical host monitoring with node template variable.
+
+- System information table (kernel, uptime, RAM, CPU count)
+- CPU usage % and CPU mode breakdown (user/system/iowait/idle stacked)
+- Load average (1m/5m/15m) with CPU count threshold line
+- Memory usage % and memory breakdown (used/buffers/cached/available)
+- Filesystem usage % bar gauge, IOPS, disk I/O time
+- Network traffic by device, network errors and drops
+
+### Incus Infrastructure — Storage & Filesystem
+
+Filesystem and disk I/O with node/mountpoint template variables.
+
+- Filesystem usage table with color-coded Used % column
+- Available space trend over time
+- Disk read/write throughput by device
+- Average read/write latency, disk utilization %
+- Inode usage % bar gauge
+
+### Incus Infrastructure — Network Deep Dive
+
+Network metrics with node/device template variables.
+
+- Cluster ingress/egress/errors/drops summary stats
+- Per-node per-device receive/transmit rates
+- Instance-level network traffic (Incus `incus_network_*` metrics)
+- Receive/transmit errors and packet drops
+
+### Incus Operations — Logs Explorer
+
+Log exploration powered by Loki with job/host/unit/search variables.
+
+- Log volume over time (stacked by job)
+- Full log stream with search filtering
+- Error rate and warning rate by systemd unit
+
+### Incus Operations — Prometheus Health
+
+Prometheus self-monitoring (no template variables).
+
+- Targets up/down, average scrape duration, samples ingested/s
+- Target health table with UP/DOWN status
+- Scrape duration and samples scraped by job
+- TSDB size, head series, head chunks, WAL size
+- TSDB size trend, samples ingested trend, compactions
+
+### Incus Operations — Capacity Planning
+
+Resource planning with 7-day default time range (no template variables).
+
+- CPU and memory headroom per node (inverted bar gauges)
+- CPU and memory usage trends (7-day)
+- Disk space forecast with 30-day `predict_linear` projection
+- Instance count trend, memory allocation vs usage per instance
## ACL Configuration
@@ -208,11 +274,18 @@ manually or restart the container.
| node-exp-02 | Alpine | 128 MiB | — | oc-node-02 |
| node-exp-03 | Alpine | 128 MiB | — | oc-node-03 |
+**Dummy workloads** (optional, deployed via `--workloads`):
+
+| Container | Image | RAM | Placement | Purpose |
+|-----------|-------|-----|-----------|---------|
+| workload-web | Alpine | 128 MiB | oc-node-01 | nginx + CPU/disk/network crons |
+| workload-api | Alpine | 128 MiB | oc-node-03 | Python HTTP + memory sawtooth |
+
**Totals:**
-- RAM: ~2.4 GiB (2 GiB + 3 x 128 MiB)
+- RAM: ~2.4 GiB core (2 GiB + 3 x 128 MiB), +256 MiB with workloads
- Disk: 20 GiB (only the monitoring container needs significant storage)
- OVN forward IPs: 1 (192.168.103.201)
-- OVN network IPs: 4 (10.10.10.70-73)
+- OVN network IPs: 4 core (10.10.10.70-73), +2 with workloads (.80, .81)
## Management
@@ -229,14 +302,30 @@ installs and configures Prometheus, Grafana, Loki, and Promtail,
provisions dashboards, sets up the OVN network forward, and configures
ACLs.
+### Deploy dummy workloads
+
+```bash
+incusos/deploy-observability --workloads
+```
+
+Deploys two lightweight containers that generate varied metric patterns:
+
+- **workload-web** (10.10.10.80, oc-node-01): nginx serving static
+ content, cron jobs generating HTTP traffic (every minute), CPU spikes
+ via gzip (every 5 min), and disk I/O bursts via dd (every 15 min).
+- **workload-api** (10.10.10.81, oc-node-03): Python HTTP server on
+ port 8080, cron jobs for inter-container traffic (every 2 min) and
+ memory sawtooth via 20 MiB allocation/release (every 10 min).
+
### Check status
```bash
incusos/deploy-observability --status
```
-Shows the state of all containers, scrape target health, Grafana
-accessibility, and OVN forward configuration.
+Shows the state of all containers (including workloads if deployed),
+scrape target health, Grafana accessibility, and OVN forward
+configuration.
### Clean up