Add professional Grafana dashboard suite with management tooling

- New manage-dashboards script: install/export/list/validate/status
  actions for Grafana dashboard lifecycle via HTTP API, with datasource
  UID portability (${DS_PROMETHEUS}/${DS_LOKI} variables) and folder
  organization
- Reorganize dashboards into core/ and community/ subdirectories with
  dashboard.yaml manifest
- Enhance 3 existing dashboards: template variables (instance/node/
  proxy/server), cross-dashboard navigation links, new panels (instance
  status table, CPU mode breakdown, load averages, memory breakdown,
  filesystem bar gauge, response code distribution, server state
  timeline, connection errors)
- Add 6 new dashboards: Instance Deep Dive, Storage & Filesystem,
  Network Deep Dive, Logs Explorer (Loki), Prometheus Health, Capacity
  Planning (with predict_linear forecasting)
- Update deploy-observability: explicit datasource UIDs in provisioning,
  phase 9 uses manage-dashboards --install, new --workloads action for
  dummy metric generation (workload-web + workload-api containers)
- Update observability guide with all 9 dashboard descriptions

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

incusos/deploy-observability

@@ -88,6 +88,7 @@ ${BOLD}ACTIONS${RESET}
     --deploy          Full deploy: monitoring container, Prometheus, Grafana,
                       Loki, Promtail, node-exporters, ACLs, network forward,
                       HAProxy exporter route, dashboards
+    --workloads       Deploy dummy workloads to generate varied metrics
     --status          Show status of all observability components
     --cleanup         Remove all observability infrastructure
     --doctor          Diagnose common issues
@@ -462,13 +463,14 @@ phase_install_grafana() {
         apt-get install -y -qq grafana 2>&1 | tail -1
     "
 
-    # Provision datasources
+    # Provision datasources with explicit UIDs for dashboard portability
     container_exec monitoring bash -c "mkdir -p /etc/grafana/provisioning/datasources
 cat > /etc/grafana/provisioning/datasources/observability.yaml << 'DSCFG'
 apiVersion: 1
 datasources:
   - name: Prometheus
     type: prometheus
+    uid: prometheus
     access: proxy
     url: http://localhost:9090
     isDefault: true
@@ -476,28 +478,12 @@ datasources:
 
   - name: Loki
     type: loki
+    uid: loki
     access: proxy
     url: http://localhost:3100
     editable: true
 DSCFG"
 
-    # Provision dashboard directory
-    container_exec monitoring bash -c "mkdir -p /etc/grafana/provisioning/dashboards
-cat > /etc/grafana/provisioning/dashboards/observability.yaml << 'DBCFG'
-apiVersion: 1
-providers:
-  - name: Observability
-    orgId: 1
-    folder: ''
-    type: file
-    disableDeletion: false
-    editable: true
-    options:
-      path: /var/lib/grafana/dashboards
-      foldersFromFilesStructure: false
-DBCFG
-mkdir -p /var/lib/grafana/dashboards"
-
     container_exec monitoring systemctl enable grafana-server
     container_exec monitoring systemctl restart grafana-server
     sleep 2
@@ -725,35 +711,269 @@ phase_create_network_forward() {
 # ---------------------------------------------------------------------------
 
 phase_upload_dashboards() {
-    step "Phase 9: Upload Grafana dashboards"
+    step "Phase 9: Install Grafana dashboards"
 
-    if ! dry_run_guard "Upload dashboards from ${DASHBOARDS_DIR}"; then
+    if ! dry_run_guard "Install dashboards via manage-dashboards"; then
         return 0
     fi
 
-    if [[ ! -d "$DASHBOARDS_DIR" ]]; then
+    local manage_script="${SCRIPT_DIR}/manage-dashboards"
-        warn "Dashboard directory not found: ${DASHBOARDS_DIR}"
+    if [[ ! -x "$manage_script" ]]; then
-        warn "Skipping dashboard upload -- create dashboards manually in Grafana."
+        warn "manage-dashboards script not found at ${manage_script}"
+        warn "Falling back to direct file upload."
+
+        if [[ ! -d "$DASHBOARDS_DIR" ]]; then
+            warn "Dashboard directory not found: ${DASHBOARDS_DIR}"
+            return 0
+        fi
+
+        local count=0
+        for dashboard in "${DASHBOARDS_DIR}"/core/*.json; do
+            [[ -f "$dashboard" ]] || continue
+            local basename
+            basename=$(basename "$dashboard")
+            incus file push "$dashboard" "$(container_ref monitoring)/var/lib/grafana/dashboards/${basename}"
+            detail "Uploaded ${basename}"
+            count=$((count + 1))
+        done
+
+        if [[ $count -gt 0 ]]; then
+            container_exec monitoring systemctl restart grafana-server 2>/dev/null || true
+            success "${count} dashboard(s) uploaded (direct push)"
+        else
+            info "No dashboard files found in ${DASHBOARDS_DIR}/core/"
+        fi
         return 0
     fi
 
-    local count=0
+    # Wait for Grafana API to be ready
-    for dashboard in "${DASHBOARDS_DIR}"/*.json; do
+    info "Waiting for Grafana API..."
-        [[ -f "$dashboard" ]] || continue
+    local elapsed=0
-        local basename
+    while true; do
-        basename=$(basename "$dashboard")
+        local code
-        incus file push "$dashboard" "$(container_ref monitoring)/var/lib/grafana/dashboards/${basename}"
+        code=$(container_exec monitoring curl -s -o /dev/null -w '%{http_code}' \
-        detail "Uploaded ${basename}"
+            http://localhost:3000/api/health 2>/dev/null || echo "000")
-        count=$((count + 1))
+        if [[ "$code" == "200" ]]; then
+            break
+        fi
+        sleep 2
+        elapsed=$((elapsed + 2))
+        if [[ $elapsed -ge 30 ]]; then
+            warn "Grafana API not ready after 30s -- falling back to file push"
+            return 0
+        fi
     done
 
-    if [[ $count -gt 0 ]]; then
+    # Use manage-dashboards to install with proper folder organization
-        # Restart Grafana to pick up new dashboards
+    local flags="-r ${CLUSTER_REMOTE}"
-        container_exec monitoring systemctl restart grafana-server 2>/dev/null || true
+    [[ "$VERBOSE" == true ]] && flags="${flags} -v"
-        success "${count} dashboard(s) uploaded"
+    [[ "$QUIET" == true ]] && flags="${flags} -q"
-    else
+
-        info "No dashboard files found in ${DASHBOARDS_DIR}"
+    "${manage_script}" --install ${flags}
+}
+
+# ---------------------------------------------------------------------------
+# Workloads action
+# ---------------------------------------------------------------------------
+
+WORKLOAD_WEB_IP="10.10.10.80"
+WORKLOAD_API_IP="10.10.10.81"
+
+action_workloads() {
+    step "Deploying dummy workloads for metric generation"
+    echo
+    info "  workload-web: ${WORKLOAD_WEB_IP} (nginx + CPU/disk/network crons)"
+    info "  workload-api: ${WORKLOAD_API_IP} (Python HTTP + memory/network crons)"
+    echo
+
+    phase_deploy_workload_web
+    phase_deploy_workload_api
+
+    echo
+    success "Dummy workloads deployed!"
+    info "  Metrics will appear within 1-2 minutes"
+    info "  workload-web: nginx on ${WORKLOAD_WEB_IP}:80, CPU/disk/net crons"
+    info "  workload-api: Python on ${WORKLOAD_API_IP}:8080, memory sawtooth cron"
+}
+
+phase_deploy_workload_web() {
+    step "Deploy workload-web (nginx + varied load)"
+
+    if container_exists "workload-web"; then
+        if container_running "workload-web"; then
+            success "workload-web already running"
+            return 0
+        fi
     fi
+
+    if ! dry_run_guard "Launch workload-web on ${OVN_NETWORK} at ${WORKLOAD_WEB_IP}"; then
+        return 0
+    fi
+
+    incus launch images:alpine/3.20 "$(container_ref workload-web)" \
+        -n "$OVN_NETWORK" \
+        --target "oc-node-01" \
+        -c limits.memory=128MiB
+
+    wait_for_container_agent workload-web 30
+
+    container_exec workload-web sh -c "
+        apk add --no-cache nginx curl gzip coreutils > /dev/null 2>&1
+
+        # Configure nginx
+        mkdir -p /var/www/html
+        cat > /var/www/html/index.html << 'WEBHTML'
+<html><body><h1>workload-web</h1><p>Observability test workload</p></body></html>
+WEBHTML
+        cat > /etc/nginx/http.d/default.conf << 'NGXCFG'
+server {
+    listen 80 default_server;
+    root /var/www/html;
+    index index.html;
+    location / { try_files \$uri \$uri/ =404; }
+}
+NGXCFG
+        rc-update add nginx default 2>/dev/null || true
+        rc-service nginx start 2>/dev/null || true
+
+        # Cron: every minute -> 10 HTTP requests (network traffic)
+        # Cron: every 5 min -> gzip random data (CPU spike)
+        # Cron: every 15 min -> dd 5 MiB write+delete (disk I/O burst)
+        cat > /etc/crontabs/root << 'CRONCFG'
+* * * * * for i in \$(seq 1 10); do wget -q -O /dev/null http://localhost/ 2>/dev/null; done
+*/5 * * * * dd if=/dev/urandom bs=1M count=2 2>/dev/null | gzip > /dev/null 2>&1
+*/15 * * * * dd if=/dev/urandom of=/tmp/disktest bs=1M count=5 2>/dev/null && rm -f /tmp/disktest
+CRONCFG
+        rc-update add crond default 2>/dev/null || true
+        rc-service crond start 2>/dev/null || true
+    "
+
+    # Configure static IP
+    container_exec workload-web sh -c "
+        cat > /etc/network/interfaces << NETCFG
+auto lo
+iface lo inet loopback
+
+auto eth0
+iface eth0 inet static
+    address ${WORKLOAD_WEB_IP}/24
+    gateway 10.10.10.1
+NETCFG
+        rc-service networking restart 2>/dev/null || true
+    "
+
+    # Attach monitoring ACL
+    incus config device set "$(container_ref workload-web)" eth0 \
+        security.acls=monitoring-allow security.acls.default.ingress.action=allow \
+        security.acls.default.egress.action=allow 2>/dev/null || true
+
+    success "workload-web deployed at ${WORKLOAD_WEB_IP}"
+}
+
+phase_deploy_workload_api() {
+    step "Deploy workload-api (Python HTTP + memory patterns)"
+
+    if container_exists "workload-api"; then
+        if container_running "workload-api"; then
+            success "workload-api already running"
+            return 0
+        fi
+    fi
+
+    if ! dry_run_guard "Launch workload-api on ${OVN_NETWORK} at ${WORKLOAD_API_IP}"; then
+        return 0
+    fi
+
+    incus launch images:alpine/3.20 "$(container_ref workload-api)" \
+        -n "$OVN_NETWORK" \
+        --target "oc-node-03" \
+        -c limits.memory=128MiB
+
+    wait_for_container_agent workload-api 30
+
+    container_exec workload-api sh -c "
+        apk add --no-cache python3 curl coreutils > /dev/null 2>&1
+
+        # Create a simple JSON status endpoint
+        mkdir -p /opt/api
+        cat > /opt/api/server.py << 'PYAPI'
+import http.server
+import json
+import time
+import os
+
+class Handler(http.server.BaseHTTPRequestHandler):
+    def do_GET(self):
+        self.send_response(200)
+        self.send_header('Content-Type', 'application/json')
+        self.end_headers()
+        data = {
+            'status': 'ok',
+            'uptime': time.time() - START_TIME,
+            'hostname': os.uname().nodename,
+            'pid': os.getpid()
+        }
+        self.wfile.write(json.dumps(data).encode())
+
+    def log_message(self, format, *args):
+        pass  # suppress access logs
+
+START_TIME = time.time()
+http.server.HTTPServer(('0.0.0.0', 8080), Handler).serve_forever()
+PYAPI
+
+        # Create memory stress script
+        cat > /opt/api/mem-stress.py << 'PYMEM'
+import time
+# Allocate ~20 MiB, hold for 30 seconds, then release
+data = bytearray(20 * 1024 * 1024)
+time.sleep(30)
+del data
+PYMEM
+
+        # Start the API server as a background service
+        cat > /etc/init.d/api-server << 'INITSCRIPT'
+#!/sbin/openrc-run
+name=\"api-server\"
+command=\"/usr/bin/python3\"
+command_args=\"/opt/api/server.py\"
+command_background=true
+pidfile=\"/run/api-server.pid\"
+INITSCRIPT
+        chmod +x /etc/init.d/api-server
+        rc-update add api-server default 2>/dev/null || true
+        rc-service api-server start 2>/dev/null || true
+
+        # Cron: every 2 min -> curl workload-web (inter-container traffic)
+        # Cron: every 10 min -> allocate 20 MiB for 30s (memory sawtooth)
+        cat > /etc/crontabs/root << 'CRONCFG'
+*/2 * * * * wget -q -O /dev/null http://${WORKLOAD_WEB_IP}/ 2>/dev/null
+*/10 * * * * python3 /opt/api/mem-stress.py 2>/dev/null &
+CRONCFG
+        rc-update add crond default 2>/dev/null || true
+        rc-service crond start 2>/dev/null || true
+    "
+
+    # Configure static IP
+    container_exec workload-api sh -c "
+        cat > /etc/network/interfaces << NETCFG
+auto lo
+iface lo inet loopback
+
+auto eth0
+iface eth0 inet static
+    address ${WORKLOAD_API_IP}/24
+    gateway 10.10.10.1
+NETCFG
+        rc-service networking restart 2>/dev/null || true
+    "
+
+    # Attach monitoring ACL
+    incus config device set "$(container_ref workload-api)" eth0 \
+        security.acls=monitoring-allow security.acls.default.ingress.action=allow \
+        security.acls.default.egress.action=allow 2>/dev/null || true
+
+    success "workload-api deployed at ${WORKLOAD_API_IP}"
 }
 
 # ---------------------------------------------------------------------------
@@ -798,6 +1018,17 @@ action_status() {
         idx=$((idx + 1))
     done
 
+    # Workload containers (optional)
+    for wl in workload-web workload-api; do
+        if container_exists "$wl"; then
+            if container_running "$wl"; then
+                success "  ${wl}: running"
+            else
+                warn "  ${wl}: stopped"
+            fi
+        fi
+    done
+
     if [[ "$all_ok" == false ]]; then
         echo
         error "Some containers are missing or stopped."
@@ -914,6 +1145,9 @@ action_cleanup() {
         if container_exists "$(node_exp_name "$idx")"; then has_anything=true; fi
         idx=$((idx + 1))
     done
+    for wl in workload-web workload-api; do
+        if container_exists "$wl"; then has_anything=true; fi
+    done
     if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
         has_anything=true
     fi
@@ -936,6 +1170,11 @@ action_cleanup() {
         fi
         idx=$((idx + 1))
     done
+    for wl in workload-web workload-api; do
+        if container_exists "$wl"; then
+            warn "  - ${wl}"
+        fi
+    done
     if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
         warn "  - network forward ${FORWARD_VIP}"
     fi
@@ -1003,6 +1242,15 @@ except: pass
         idx=$((idx + 1))
     done
 
+    # Delete workload containers
+    for wl in workload-web workload-api; do
+        if container_exists "$wl"; then
+            info "Deleting ${wl}..."
+            incus delete "$(container_ref "$wl")" --force 2>/dev/null || true
+            success "Deleted ${wl}"
+        fi
+    done
+
     # Delete network forward
     step "Removing network forward"
     if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
@@ -1153,6 +1401,7 @@ parse_args() {
     while [[ $# -gt 0 ]]; do
         case "$1" in
             --deploy)      ACTION="deploy";;
+            --workloads)   ACTION="workloads";;
             --status)      ACTION="status";;
             --cleanup)     ACTION="cleanup";;
             --doctor)      ACTION="doctor";;
@@ -1187,10 +1436,11 @@ main() {
 
     echo
     case "$ACTION" in
-        deploy)  action_deploy;;
+        deploy)    action_deploy;;
-        status)  action_status;;
+        workloads) action_workloads;;
-        cleanup) action_cleanup;;
+        status)    action_status;;
-        doctor)  action_doctor;;
+        cleanup)   action_cleanup;;
+        doctor)    action_doctor;;
     esac
 }
 

incusos/manage-dashboards

@@ -0,0 +1,854 @@
+#!/usr/bin/env bash
+# manage-dashboards - Manage Grafana dashboards for the Incus observability stack
+#
+# Push, export, validate, and inspect Grafana dashboards via the Grafana HTTP
+# API. Dashboards are organized into folders and use datasource UID variables
+# for portability across deployments.
+#
+# Usage: manage-dashboards [OPTIONS]
+# Run 'manage-dashboards --help' for full usage information.
+
+set -euo pipefail
+
+readonly VERSION="0.1.0"
+readonly SCRIPT_NAME="manage-dashboards"
+readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly DASHBOARDS_DIR="${SCRIPT_DIR}/observability-dashboards"
+readonly MANIFEST_FILE="${DASHBOARDS_DIR}/dashboard.yaml"
+
+# ---------------------------------------------------------------------------
+# Defaults (overridden by CLI flags)
+# ---------------------------------------------------------------------------
+
+CLUSTER_REMOTE="oc-node-01"
+MONITORING_CONTAINER="monitoring"
+GRAFANA_URL="http://localhost:3000"
+GRAFANA_USER="admin"
+GRAFANA_PASSWORD="admin"
+
+# Runtime flags
+DRY_RUN=false
+VERBOSE=false
+QUIET=false
+ACTION=""
+FILTER_CATEGORY=""
+FILTER_FILE=""
+FILTER_UID=""
+
+# ---------------------------------------------------------------------------
+# Color and formatting
+# ---------------------------------------------------------------------------
+
+setup_colors() {
+    if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
+        RED='\033[0;31m'
+        GREEN='\033[0;32m'
+        YELLOW='\033[0;33m'
+        BLUE='\033[0;34m'
+        CYAN='\033[0;36m'
+        BOLD='\033[1m'
+        DIM='\033[2m'
+        RESET='\033[0m'
+    else
+        RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET=''
+    fi
+}
+
+info()    { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET}    $*"; }
+success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET}      $*"; }
+warn()    { echo -e "${YELLOW}[warn]${RESET}    $*" >&2; }
+error()   { echo -e "${RED}[error]${RESET}   $*" >&2; }
+step()    { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET}    ${BOLD}$*${RESET}"; }
+detail()  { [[ "$VERBOSE" != true ]] && return; echo -e "${DIM}          $*${RESET}"; }
+
+dry_run_guard() {
+    if [[ "$DRY_RUN" == true ]]; then
+        info "(dry-run) $*"
+        return 1
+    fi
+    return 0
+}
+
+# ---------------------------------------------------------------------------
+# Help
+# ---------------------------------------------------------------------------
+
+usage() {
+    cat <<EOF
+${BOLD}${SCRIPT_NAME}${RESET} v${VERSION} - Manage Grafana dashboards for the Incus observability stack
+
+${BOLD}USAGE${RESET}
+    ${SCRIPT_NAME} [OPTIONS]
+
+${BOLD}ACTIONS${RESET}
+    --install             Push dashboards to Grafana via API, organized into folders
+    --export              Export dashboards from Grafana to clean JSON files
+    --list                Show available dashboards with install status
+    --validate            Check JSON quality and portability
+    --status              Compare installed vs local versions
+
+${BOLD}FILTERS${RESET}
+    --category NAME       Only process dashboards in this category/folder
+    --file PATH           Only process this specific dashboard file
+    --uid UID             Only process the dashboard with this UID (for --export)
+
+${BOLD}OPTIONS${RESET}
+    -r, --remote NAME     Incus remote (default: ${CLUSTER_REMOTE})
+    -n, --dry-run         Preview actions without executing
+    -v, --verbose         Show detailed output
+    -q, --quiet           Suppress informational output
+    -h, --help            Show this help message
+
+${BOLD}DATASOURCE PORTABILITY${RESET}
+    Dashboard JSON files use \${DS_PROMETHEUS} and \${DS_LOKI} variables
+    instead of hardcoded datasource UIDs. The --install action resolves
+    these to the actual UIDs from the target Grafana instance. The --export
+    action normalizes UIDs back to variables.
+
+${BOLD}DASHBOARD MANIFEST${RESET}
+    ${MANIFEST_FILE}
+    YAML file listing each dashboard with file path, UID, title, folder,
+    category, and version. Used by --install and --list.
+
+${BOLD}EXAMPLES${RESET}
+    # Validate all dashboard JSON files
+    ${SCRIPT_NAME} --validate
+
+    # Install all dashboards into Grafana
+    ${SCRIPT_NAME} --install
+
+    # Install only dashboards in the "core" category
+    ${SCRIPT_NAME} --install --category core
+
+    # Export a single dashboard by UID
+    ${SCRIPT_NAME} --export --uid incus-cluster-overview
+
+    # Compare installed versions with local files
+    ${SCRIPT_NAME} --status
+
+    # Preview what would be installed
+    ${SCRIPT_NAME} --install --dry-run
+EOF
+}
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+grafana_api() {
+    local method="$1" path="$2"
+    shift 2
+    incus exec "${CLUSTER_REMOTE}:${MONITORING_CONTAINER}" -- \
+        curl -s -u "${GRAFANA_USER}:${GRAFANA_PASSWORD}" \
+        -X "$method" "${GRAFANA_URL}${path}" "$@"
+}
+
+# Resolve datasource UID variables to actual UIDs from Grafana
+resolve_datasource_uids() {
+    local ds_json
+    ds_json=$(grafana_api GET "/api/datasources")
+
+    PROMETHEUS_UID=$(echo "$ds_json" | python3 -c "
+import sys, json
+ds = json.load(sys.stdin)
+for d in ds:
+    if d.get('type') == 'prometheus':
+        print(d['uid'])
+        break
+" 2>/dev/null || echo "")
+
+    LOKI_UID=$(echo "$ds_json" | python3 -c "
+import sys, json
+ds = json.load(sys.stdin)
+for d in ds:
+    if d.get('type') == 'loki':
+        print(d['uid'])
+        break
+" 2>/dev/null || echo "")
+
+    if [[ -z "$PROMETHEUS_UID" ]]; then
+        error "Could not find Prometheus datasource in Grafana"
+        return 1
+    fi
+    detail "Prometheus datasource UID: ${PROMETHEUS_UID}"
+    detail "Loki datasource UID: ${LOKI_UID:-not found}"
+}
+
+# Get or create a Grafana folder, returns folder ID
+ensure_folder() {
+    local folder_title="$1"
+
+    # Check if folder exists
+    local folder_id
+    folder_id=$(grafana_api GET "/api/folders" | python3 -c "
+import sys, json
+folders = json.load(sys.stdin)
+for f in folders:
+    if f.get('title') == '${folder_title}':
+        print(f['id'])
+        break
+" 2>/dev/null || echo "")
+
+    if [[ -n "$folder_id" ]]; then
+        detail "Folder '${folder_title}' exists (ID: ${folder_id})"
+        echo "$folder_id"
+        return 0
+    fi
+
+    # Create folder
+    local folder_uid
+    folder_uid=$(echo "$folder_title" | tr '[:upper:]' '[:lower:]' | tr ' ' '-')
+    local result
+    result=$(grafana_api POST "/api/folders" \
+        -H "Content-Type: application/json" \
+        -d "{\"title\": \"${folder_title}\", \"uid\": \"${folder_uid}\"}")
+
+    folder_id=$(echo "$result" | python3 -c "
+import sys, json
+print(json.load(sys.stdin).get('id', ''))
+" 2>/dev/null || echo "")
+
+    if [[ -n "$folder_id" ]]; then
+        detail "Created folder '${folder_title}' (ID: ${folder_id})"
+        echo "$folder_id"
+    else
+        error "Failed to create folder '${folder_title}': ${result}"
+        return 1
+    fi
+}
+
+# Read manifest and output entries as tab-separated lines
+read_manifest() {
+    if [[ ! -f "$MANIFEST_FILE" ]]; then
+        error "Dashboard manifest not found: ${MANIFEST_FILE}"
+        error "Expected a YAML file listing dashboard metadata."
+        return 1
+    fi
+
+    python3 -c "
+import yaml, sys
+
+with open('${MANIFEST_FILE}') as f:
+    data = yaml.safe_load(f)
+
+for d in data.get('dashboards', []):
+    file_path = d.get('file', '')
+    uid = d.get('uid', '')
+    title = d.get('title', '')
+    folder = d.get('folder', 'General')
+    category = d.get('category', '')
+    version = d.get('version', '1')
+    description = d.get('description', '')
+    print(f'{file_path}\t{uid}\t{title}\t{folder}\t{category}\t{version}\t{description}')
+" 2>/dev/null
+}
+
+# Substitute datasource variables in JSON content
+inject_datasource_uids() {
+    local json_content="$1"
+    echo "$json_content" \
+        | sed "s/\\\${DS_PROMETHEUS}/${PROMETHEUS_UID}/g" \
+        | sed "s/\\\${DS_LOKI}/${LOKI_UID}/g"
+}
+
+# Normalize datasource UIDs back to variables for portability
+normalize_datasource_uids() {
+    local json_content="$1"
+    local prom_uid="$2"
+    local loki_uid="$3"
+
+    local result="$json_content"
+    if [[ -n "$prom_uid" ]]; then
+        result=$(echo "$result" | sed "s/${prom_uid}/\\\${DS_PROMETHEUS}/g")
+    fi
+    if [[ -n "$loki_uid" ]]; then
+        result=$(echo "$result" | sed "s/${loki_uid}/\\\${DS_LOKI}/g")
+    fi
+    echo "$result"
+}
+
+# ---------------------------------------------------------------------------
+# Install action
+# ---------------------------------------------------------------------------
+
+action_install() {
+    step "Installing dashboards to Grafana"
+    echo
+
+    if ! dry_run_guard "Resolve datasource UIDs and push dashboards"; then
+        info "(dry-run) Would install dashboards from ${DASHBOARDS_DIR}"
+
+        local count=0
+        while IFS=$'\t' read -r file uid title folder category version description; do
+            if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
+                continue
+            fi
+            if [[ -n "$FILTER_FILE" && "$file" != "$FILTER_FILE" ]]; then
+                continue
+            fi
+            info "(dry-run) Would install: ${title} -> folder '${folder}'"
+            count=$((count + 1))
+        done < <(read_manifest)
+
+        info "(dry-run) ${count} dashboard(s) would be installed"
+        return 0
+    fi
+
+    resolve_datasource_uids
+
+    local installed=0
+    local failed=0
+
+    while IFS=$'\t' read -r file uid title folder category version description; do
+        if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
+            continue
+        fi
+        if [[ -n "$FILTER_FILE" && "$file" != "$FILTER_FILE" ]]; then
+            continue
+        fi
+
+        local full_path="${DASHBOARDS_DIR}/${file}"
+        if [[ ! -f "$full_path" ]]; then
+            warn "Dashboard file not found: ${full_path}"
+            failed=$((failed + 1))
+            continue
+        fi
+
+        # Get or create the target folder
+        local folder_id
+        folder_id=$(ensure_folder "$folder") || { failed=$((failed + 1)); continue; }
+
+        # Read and process the dashboard JSON
+        local raw_json
+        raw_json=$(cat "$full_path")
+
+        # Inject actual datasource UIDs
+        local processed_json
+        processed_json=$(inject_datasource_uids "$raw_json")
+
+        # Wrap in Grafana API envelope
+        local api_payload
+        api_payload=$(python3 -c "
+import json, sys
+
+dashboard = json.loads('''${processed_json//\'/\\\'}''')
+
+# Remove Grafana-internal fields that shouldn't be in the import
+dashboard.pop('id', None)
+dashboard.pop('version', None)
+dashboard.pop('iteration', None)
+
+payload = {
+    'dashboard': dashboard,
+    'folderId': ${folder_id},
+    'overwrite': True,
+    'message': 'Installed by manage-dashboards v${VERSION}'
+}
+print(json.dumps(payload))
+" 2>/dev/null)
+
+        if [[ -z "$api_payload" ]]; then
+            # Fallback: use a simpler approach for large/complex JSON
+            api_payload=$(python3 << 'PYEOF'
+import json, sys
+
+with open(sys.argv[1]) as f:
+    dashboard = json.load(f)
+
+# Inject datasource UIDs
+json_str = json.dumps(dashboard)
+json_str = json_str.replace('${DS_PROMETHEUS}', sys.argv[2])
+json_str = json_str.replace('${DS_LOKI}', sys.argv[3])
+dashboard = json.loads(json_str)
+
+# Remove Grafana-internal fields
+dashboard.pop('id', None)
+dashboard.pop('version', None)
+dashboard.pop('iteration', None)
+
+payload = {
+    'dashboard': dashboard,
+    'folderId': int(sys.argv[4]),
+    'overwrite': True,
+    'message': 'Installed by manage-dashboards'
+}
+print(json.dumps(payload))
+PYEOF
+            "$full_path" "$PROMETHEUS_UID" "${LOKI_UID:-}" "$folder_id" 2>/dev/null)
+        fi
+
+        if [[ -z "$api_payload" ]]; then
+            error "Failed to process dashboard JSON: ${file}"
+            failed=$((failed + 1))
+            continue
+        fi
+
+        # Push via API - write payload to temp file to avoid command-line size limits
+        local tmpfile
+        tmpfile=$(mktemp /tmp/dashboard-payload.XXXXXX)
+        echo "$api_payload" > "$tmpfile"
+
+        local result
+        result=$(incus file push "$tmpfile" "${CLUSTER_REMOTE}:${MONITORING_CONTAINER}/tmp/dashboard-payload.json" 2>/dev/null && \
+            grafana_api POST "/api/dashboards/db" \
+                -H "Content-Type: application/json" \
+                -d @/tmp/dashboard-payload.json 2>/dev/null)
+        rm -f "$tmpfile"
+
+        local status_str
+        status_str=$(echo "$result" | python3 -c "
+import sys, json
+try:
+    d = json.load(sys.stdin)
+    print(d.get('status', d.get('message', 'unknown')))
+except:
+    print('error')
+" 2>/dev/null || echo "error")
+
+        if [[ "$status_str" == "success" ]]; then
+            success "  ${title} -> ${folder}"
+            installed=$((installed + 1))
+        else
+            error "  ${title}: ${status_str}"
+            detail "  Response: ${result}"
+            failed=$((failed + 1))
+        fi
+
+        # Clean up temp file in container
+        incus exec "${CLUSTER_REMOTE}:${MONITORING_CONTAINER}" -- rm -f /tmp/dashboard-payload.json 2>/dev/null || true
+
+    done < <(read_manifest)
+
+    echo
+    if [[ $failed -eq 0 ]]; then
+        success "Installed ${installed} dashboard(s)"
+    else
+        warn "Installed ${installed}, failed ${failed}"
+    fi
+}
+
+# ---------------------------------------------------------------------------
+# Export action
+# ---------------------------------------------------------------------------
+
+action_export() {
+    step "Exporting dashboards from Grafana"
+    echo
+
+    resolve_datasource_uids
+
+    local exported=0
+
+    if [[ -n "$FILTER_UID" ]]; then
+        # Export a single dashboard by UID
+        export_single_dashboard "$FILTER_UID"
+        return $?
+    fi
+
+    # Export all dashboards listed in the manifest
+    while IFS=$'\t' read -r file uid title folder category version description; do
+        if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
+            continue
+        fi
+
+        if export_single_dashboard "$uid" "${DASHBOARDS_DIR}/${file}"; then
+            exported=$((exported + 1))
+        fi
+    done < <(read_manifest)
+
+    echo
+    success "Exported ${exported} dashboard(s)"
+}
+
+export_single_dashboard() {
+    local uid="$1"
+    local output_file="${2:-}"
+
+    # Fetch from Grafana API
+    local raw_response
+    raw_response=$(grafana_api GET "/api/dashboards/uid/${uid}" 2>/dev/null)
+
+    local has_dashboard
+    has_dashboard=$(echo "$raw_response" | python3 -c "
+import sys, json
+try:
+    d = json.load(sys.stdin)
+    if 'dashboard' in d:
+        print('yes')
+    else:
+        print('no')
+except:
+    print('no')
+" 2>/dev/null || echo "no")
+
+    if [[ "$has_dashboard" != "yes" ]]; then
+        warn "Dashboard '${uid}' not found in Grafana"
+        return 1
+    fi
+
+    # If no output file specified, find it in the manifest
+    if [[ -z "$output_file" ]]; then
+        output_file=$(read_manifest | while IFS=$'\t' read -r file muid title folder category version description; do
+            if [[ "$muid" == "$uid" ]]; then
+                echo "${DASHBOARDS_DIR}/${file}"
+                break
+            fi
+        done)
+    fi
+
+    if [[ -z "$output_file" ]]; then
+        output_file="${DASHBOARDS_DIR}/core/${uid}.json"
+        warn "Dashboard '${uid}' not in manifest, exporting to ${output_file}"
+    fi
+
+    if ! dry_run_guard "Export ${uid} to ${output_file}"; then
+        return 0
+    fi
+
+    # Clean and normalize the dashboard JSON
+    python3 << PYEOF > "$output_file"
+import json, sys
+
+raw = json.loads('''$(echo "$raw_response" | sed "s/'/\\\\'/g")''')
+dashboard = raw['dashboard']
+
+# Remove Grafana-internal fields
+for key in ['id', 'version', 'iteration']:
+    dashboard.pop(key, None)
+
+# Normalize datasource UIDs to variables
+json_str = json.dumps(dashboard, indent=2)
+prom_uid = "${PROMETHEUS_UID}"
+loki_uid = "${LOKI_UID:-}"
+if prom_uid:
+    json_str = json_str.replace(prom_uid, r'\${DS_PROMETHEUS}')
+if loki_uid:
+    json_str = json_str.replace(loki_uid, r'\${DS_LOKI}')
+
+# Re-parse to ensure valid JSON
+dashboard = json.loads(json_str.replace(r'\${DS_PROMETHEUS}', '\${DS_PROMETHEUS}').replace(r'\${DS_LOKI}', '\${DS_LOKI}'))
+
+# Write with consistent formatting
+json_str = json.dumps(dashboard, indent=2)
+# Restore the variable syntax that json.dumps may have escaped
+json_str = json_str.replace(r'\${DS_PROMETHEUS}', '\${DS_PROMETHEUS}')
+json_str = json_str.replace(r'\${DS_LOKI}', '\${DS_LOKI}')
+print(json_str)
+PYEOF
+
+    if [[ $? -eq 0 ]]; then
+        success "  Exported: ${uid} -> $(basename "$output_file")"
+        return 0
+    else
+        error "  Failed to export: ${uid}"
+        return 1
+    fi
+}
+
+# ---------------------------------------------------------------------------
+# List action
+# ---------------------------------------------------------------------------
+
+action_list() {
+    step "Dashboard Inventory"
+    echo
+
+    # Try to get installed dashboards from Grafana
+    local installed_json=""
+    installed_json=$(grafana_api GET "/api/search?type=dash-db" 2>/dev/null || echo "[]")
+
+    printf "  ${BOLD}%-35s %-30s %-20s %s${RESET}\n" "UID" "Title" "Folder" "Status"
+    printf "  %-35s %-30s %-20s %s\n" "---" "-----" "------" "------"
+
+    while IFS=$'\t' read -r file uid title folder category version description; do
+        if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
+            continue
+        fi
+
+        local full_path="${DASHBOARDS_DIR}/${file}"
+        local local_status="missing"
+        if [[ -f "$full_path" ]]; then
+            local_status="local"
+        fi
+
+        # Check if installed in Grafana
+        local grafana_status
+        grafana_status=$(echo "$installed_json" | python3 -c "
+import sys, json
+try:
+    dashboards = json.load(sys.stdin)
+    for d in dashboards:
+        if d.get('uid') == '${uid}':
+            print('installed')
+            break
+    else:
+        print('not installed')
+except:
+    print('unknown')
+" 2>/dev/null || echo "unknown")
+
+        local status_display
+        if [[ "$local_status" == "missing" ]]; then
+            status_display="${RED}file missing${RESET}"
+        elif [[ "$grafana_status" == "installed" ]]; then
+            status_display="${GREEN}installed${RESET}"
+        elif [[ "$grafana_status" == "not installed" ]]; then
+            status_display="${YELLOW}not installed${RESET}"
+        else
+            status_display="${DIM}${local_status}${RESET}"
+        fi
+
+        printf "  %-35s %-30s %-20s " "$uid" "$title" "$folder"
+        echo -e "$status_display"
+
+    done < <(read_manifest)
+}
+
+# ---------------------------------------------------------------------------
+# Validate action
+# ---------------------------------------------------------------------------
+
+action_validate() {
+    step "Validating dashboard JSON files"
+    echo
+
+    local total=0
+    local passed=0
+    local warnings=0
+    local errors=0
+
+    while IFS=$'\t' read -r file uid title folder category version description; do
+        if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then
+            continue
+        fi
+
+        local full_path="${DASHBOARDS_DIR}/${file}"
+        total=$((total + 1))
+
+        if [[ ! -f "$full_path" ]]; then
+            error "  ${file}: FILE NOT FOUND"
+            errors=$((errors + 1))
+            continue
+        fi
+
+        local issues=0
+        local warns=0
+        local basename_file
+        basename_file=$(basename "$full_path")
+
+        # Check 1: Valid JSON
+        if ! python3 -c "import json; json.load(open('${full_path}'))" 2>/dev/null; then
+            error "  ${basename_file}: INVALID JSON"
+            errors=$((errors + 1))
+            continue
+        fi
+
+        # Run all checks via python
+        local check_result
+        check_result=$(python3 << PYEOF
+import json
+
+with open('${full_path}') as f:
+    d = json.load(f)
+
+issues = []
+warns = []
+
+# Check 2: UID present and matches manifest
+json_uid = d.get('uid', '')
+if not json_uid:
+    issues.append('missing uid field')
+elif json_uid != '${uid}':
+    issues.append(f'uid mismatch: JSON has "{json_uid}", manifest has "${uid}"')
+
+# Check 3: Title present
+if not d.get('title'):
+    issues.append('missing title field')
+
+# Check 4: Description present
+if not d.get('description'):
+    warns.append('missing description field')
+
+# Check 5: No hardcoded datasource UIDs (should use variables)
+json_str = json.dumps(d)
+# Known Grafana internal UIDs that are OK
+safe_uids = ['-- Grafana --']
+# Check for UIDs that look like Grafana-generated ones
+import re
+# Match patterns like "uid": "PBFA97CFB590B2093" (hex-like, not a variable)
+uid_pattern = re.compile(r'"uid":\s*"([A-Za-z0-9]{16,})"')
+for match in uid_pattern.finditer(json_str):
+    found_uid = match.group(1)
+    if found_uid not in safe_uids and not found_uid.startswith('\${'):
+        issues.append(f'hardcoded datasource UID: {found_uid}')
+        break
+
+# Check 6: Datasource variable usage
+if '\${DS_PROMETHEUS}' not in json_str and '\${DS_LOKI}' not in json_str:
+    # Check if it has any datasource references at all
+    if '"type": "prometheus"' in json_str or '"type": "loki"' in json_str:
+        warns.append('no datasource variables found (uses hardcoded UIDs?)')
+
+# Check 7: Has panels
+panels = d.get('panels', [])
+if not panels:
+    warns.append('no panels defined')
+
+# Check 8: Check for id field (should not be present in stored files)
+if 'id' in d and d['id'] is not None:
+    warns.append('contains "id" field (should be null or absent for portability)')
+
+for i in issues:
+    print(f'ERROR:{i}')
+for w in warns:
+    print(f'WARN:{w}')
+if not issues:
+    print('OK')
+PYEOF
+        )
+
+        local has_error=false
+        while IFS= read -r line; do
+            case "$line" in
+                ERROR:*)
+                    error "  ${basename_file}: ${line#ERROR:}"
+                    issues=$((issues + 1))
+                    has_error=true
+                    ;;
+                WARN:*)
+                    warn "  ${basename_file}: ${line#WARN:}"
+                    warns=$((warns + 1))
+                    ;;
+                OK)
+                    ;;
+            esac
+        done <<< "$check_result"
+
+        if [[ "$has_error" == true ]]; then
+            errors=$((errors + 1))
+        else
+            if [[ $warns -gt 0 ]]; then
+                warnings=$((warnings + warns))
+            fi
+            passed=$((passed + 1))
+            success "  ${basename_file}: valid"
+        fi
+
+    done < <(read_manifest)
+
+    echo
+    info "${total} dashboard(s) checked: ${passed} passed, ${errors} errors, ${warnings} warnings"
+
+    if [[ $errors -gt 0 ]]; then
+        return 1
+    fi
+}
+
+# ---------------------------------------------------------------------------
+# Status action
+# ---------------------------------------------------------------------------
+
+action_status() {
+    step "Dashboard Status (local vs installed)"
+    echo
+
+    local installed_json
+    installed_json=$(grafana_api GET "/api/search?type=dash-db" 2>/dev/null || echo "[]")
+
+    printf "  ${BOLD}%-30s %-15s %-15s %s${RESET}\n" "Dashboard" "Local" "Grafana" "Match"
+    printf "  %-30s %-15s %-15s %s\n" "---------" "-----" "-------" "-----"
+
+    while IFS=$'\t' read -r file uid title folder category version description; do
+        local full_path="${DASHBOARDS_DIR}/${file}"
+
+        local local_ver="missing"
+        if [[ -f "$full_path" ]]; then
+            local_ver="v${version}"
+        fi
+
+        local grafana_ver
+        grafana_ver=$(echo "$installed_json" | python3 -c "
+import sys, json
+try:
+    for d in json.load(sys.stdin):
+        if d.get('uid') == '${uid}':
+            print('installed')
+            break
+    else:
+        print('absent')
+except:
+    print('unknown')
+" 2>/dev/null || echo "unknown")
+
+        local match_str
+        if [[ "$local_ver" == "missing" ]]; then
+            match_str="${RED}file missing${RESET}"
+        elif [[ "$grafana_ver" == "absent" ]]; then
+            match_str="${YELLOW}not installed${RESET}"
+        elif [[ "$grafana_ver" == "installed" ]]; then
+            match_str="${GREEN}synced${RESET}"
+        else
+            match_str="${DIM}unknown${RESET}"
+        fi
+
+        printf "  %-30s %-15s %-15s " "$title" "$local_ver" "$grafana_ver"
+        echo -e "$match_str"
+
+    done < <(read_manifest)
+}
+
+# ---------------------------------------------------------------------------
+# Argument parsing
+# ---------------------------------------------------------------------------
+
+parse_args() {
+    while [[ $# -gt 0 ]]; do
+        case "$1" in
+            --install)     ACTION="install";;
+            --export)      ACTION="export";;
+            --list)        ACTION="list";;
+            --validate)    ACTION="validate";;
+            --status)      ACTION="status";;
+            --category)    FILTER_CATEGORY="$2"; shift;;
+            --file)        FILTER_FILE="$2"; shift;;
+            --uid)         FILTER_UID="$2"; shift;;
+            -r|--remote)   CLUSTER_REMOTE="$2"; shift;;
+            -n|--dry-run)  DRY_RUN=true;;
+            -v|--verbose)  VERBOSE=true;;
+            -q|--quiet)    QUIET=true;;
+            -h|--help)     usage; exit 0;;
+            *)
+                error "Unknown option: $1"
+                echo "Run '${SCRIPT_NAME} --help' for usage."
+                exit 1
+                ;;
+        esac
+        shift
+    done
+
+    if [[ -z "$ACTION" ]]; then
+        error "No action specified"
+        echo "Run '${SCRIPT_NAME} --help' for usage."
+        exit 1
+    fi
+}
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+main() {
+    setup_colors
+    parse_args "$@"
+
+    echo
+    case "$ACTION" in
+        install)   action_install;;
+        export)    action_export;;
+        list)      action_list;;
+        validate)  action_validate;;
+        status)    action_status;;
+    esac
+}
+
+main "$@"

incusos/observability-dashboards/core/capacity-planning.json

@@ -1,31 +1,35 @@
 {
-  "uid": "incus-host-resources",
+  "uid": "incus-capacity-planning",
-  "title": "Host Resources (IncusOS Nodes)",
+  "title": "Capacity Planning",
-  "description": "Physical host monitoring for IncusOS cluster nodes: CPU, memory, disk, and network from node_exporter metrics.",
+  "description": "CPU/memory headroom per node, resource trends, disk space forecast with predict_linear, instance count trends.",
-  "tags": ["incusos", "host", "node-exporter"],
+  "tags": ["incus", "capacity", "operations"],
   "timezone": "browser",
   "schemaVersion": 39,
-  "version": 1,
+  "refresh": "5m",
-  "refresh": "30s",
+  "time": { "from": "now-7d", "to": "now" },
-  "time": {
-    "from": "now-1h",
-    "to": "now"
-  },
   "fiscalYearStartMonth": 0,
   "liveNow": false,
   "editable": true,
   "graphTooltip": 1,
-  "templating": {
+  "links": [
-    "list": []
+    {
-  },
+      "asDropdown": true,
+      "icon": "external link",
+      "includeVars": true,
+      "keepTime": true,
+      "tags": ["incus"],
+      "targetBlank": false,
+      "title": "Incus Dashboards",
+      "tooltip": "Navigate to other Incus dashboards",
+      "type": "dashboards"
+    }
+  ],
+  "templating": { "list": [] },
   "annotations": {
     "list": [
       {
         "builtIn": 1,
-        "datasource": {
+        "datasource": { "type": "grafana", "uid": "-- Grafana --" },
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
         "enable": true,
         "hide": true,
         "iconColor": "rgba(0, 211, 255, 1)",
@@ -37,25 +41,114 @@
   "panels": [
     {
       "type": "row",
-      "title": "CPU",
+      "title": "Current Headroom",
       "collapsed": false,
       "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
       "id": 1,
       "panels": []
     },
     {
-      "type": "timeseries",
+      "type": "bargauge",
-      "title": "CPU Usage % by Node",
+      "title": "CPU Headroom per Node",
-      "description": "Overall CPU utilization per host node, calculated as 1 minus the average idle CPU rate.",
+      "description": "Available (idle) CPU percentage per host node. Higher values mean more headroom for additional workloads.",
-      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 1 },
+      "gridPos": { "h": 6, "w": 12, "x": 0, "y": 1 },
       "id": 2,
-      "datasource": {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
       "targets": [
         {
-          "expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval]))) * 100",
+          "expr": "avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])) * 100",
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red", "value": null },
+              { "color": "orange", "value": 20 },
+              { "color": "yellow", "value": 40 },
+              { "color": "green", "value": 60 }
+            ]
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "orientation": "horizontal",
+        "displayMode": "gradient",
+        "showUnfilled": true,
+        "minVizWidth": 8,
+        "minVizHeight": 16,
+        "valueMode": "color"
+      }
+    },
+    {
+      "type": "bargauge",
+      "title": "Memory Headroom per Node",
+      "description": "Available memory percentage per host node. Higher values mean more headroom for additional workloads.",
+      "gridPos": { "h": 6, "w": 12, "x": 12, "y": 1 },
+      "id": 3,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "(node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100",
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red", "value": null },
+              { "color": "orange", "value": 20 },
+              { "color": "yellow", "value": 40 },
+              { "color": "green", "value": 60 }
+            ]
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "orientation": "horizontal",
+        "displayMode": "gradient",
+        "showUnfilled": true,
+        "minVizWidth": 8,
+        "minVizHeight": 16,
+        "valueMode": "color"
+      }
+    },
+    {
+      "type": "row",
+      "title": "Resource Trends",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 },
+      "id": 4,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "CPU Usage Trend (7-day)",
+      "description": "CPU utilization percentage per host node over the past 7 days. Helps identify sustained usage patterns and growth trends.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+      "id": 5,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100",
           "legendFormat": "{{instance}}",
           "refId": "A"
         }
@@ -75,7 +168,7 @@
             "axisPlacement": "auto",
             "axisLabel": "",
             "scaleDistribution": { "type": "linear" },
-            "thresholdsStyle": { "mode": "line" }
+            "thresholdsStyle": { "mode": "off" }
           },
           "unit": "percent",
           "min": 0,
@@ -97,24 +190,13 @@
         "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
       }
     },
-    {
-      "type": "row",
-      "title": "Memory",
-      "collapsed": false,
-      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
-      "id": 3,
-      "panels": []
-    },
     {
       "type": "timeseries",
-      "title": "Memory Usage % by Node",
+      "title": "Memory Usage Trend (7-day)",
-      "description": "Memory utilization percentage per host node, calculated from available vs total memory.",
+      "description": "Memory utilization percentage per host node over the past 7 days. Helps identify sustained usage patterns and growth trends.",
-      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 10 },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
-      "id": 4,
+      "id": 6,
-      "datasource": {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
       "targets": [
         {
           "expr": "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100",
@@ -137,7 +219,7 @@
             "axisPlacement": "auto",
             "axisLabel": "",
             "scaleDistribution": { "type": "linear" },
-            "thresholdsStyle": { "mode": "line" }
+            "thresholdsStyle": { "mode": "off" }
           },
           "unit": "percent",
           "min": 0,
@@ -161,27 +243,29 @@
     },
     {
       "type": "row",
-      "title": "Disk",
+      "title": "Disk Space Forecast",
       "collapsed": false,
-      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 },
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 16 },
-      "id": 5,
+      "id": 7,
       "panels": []
     },
     {
       "type": "timeseries",
-      "title": "Root Filesystem Available Space",
+      "title": "Disk Space Forecast (30-day projection)",
-      "description": "Available disk space on the root filesystem (mountpoint='/') per host node.",
+      "description": "Current available disk space on root filesystem with a 30-day linear projection based on the past 7 days of data. Dashed lines show the forecast; if a forecast line drops below zero, the disk will fill up within 30 days at current consumption rates.",
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 17 },
-      "id": 6,
+      "id": 8,
-      "datasource": {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
       "targets": [
         {
           "expr": "node_filesystem_avail_bytes{mountpoint=\"/\"}",
-          "legendFormat": "{{instance}}",
+          "legendFormat": "{{instance}} actual",
           "refId": "A"
+        },
+        {
+          "expr": "predict_linear(node_filesystem_avail_bytes{mountpoint=\"/\"}[7d], 30*24*3600)",
+          "legendFormat": "{{instance}} forecast",
+          "refId": "B"
         }
       ],
       "fieldConfig": {
@@ -189,7 +273,7 @@
           "custom": {
             "drawStyle": "line",
             "lineInterpolation": "smooth",
-            "lineWidth": 1,
+            "lineWidth": 2,
             "fillOpacity": 15,
             "gradientMode": "none",
             "spanNulls": false,
@@ -205,140 +289,43 @@
           "color": { "mode": "palette-classic" },
           "thresholds": {
             "mode": "absolute",
-            "steps": [
+            "steps": [{ "color": "green", "value": null }]
-              { "color": "green", "value": null }
-            ]
           }
         },
-        "overrides": []
+        "overrides": [
+          {
+            "matcher": { "id": "byRegexp", "options": ".*forecast$" },
+            "properties": [
+              { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } },
+              { "id": "custom.fillOpacity", "value": 0 }
+            ]
+          }
+        ]
       },
       "options": {
         "tooltip": { "mode": "multi", "sort": "desc" },
-        "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull", "min"] }
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
-      }
-    },
-    {
-      "type": "timeseries",
-      "title": "Disk Read Rate by Node",
-      "description": "Disk read throughput (bytes/sec) aggregated per host node.",
-      "gridPos": { "h": 8, "w": 6, "x": 12, "y": 19 },
-      "id": 7,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "targets": [
-        {
-          "expr": "sum by (instance) (rate(node_disk_read_bytes_total[$__rate_interval]))",
-          "legendFormat": "{{instance}}",
-          "refId": "A"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "smooth",
-            "lineWidth": 1,
-            "fillOpacity": 15,
-            "gradientMode": "none",
-            "spanNulls": false,
-            "showPoints": "never",
-            "pointSize": 5,
-            "stacking": { "mode": "none", "group": "A" },
-            "axisPlacement": "auto",
-            "axisLabel": "",
-            "scaleDistribution": { "type": "linear" },
-            "thresholdsStyle": { "mode": "off" }
-          },
-          "unit": "Bps",
-          "color": { "mode": "palette-classic" },
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              { "color": "green", "value": null }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "options": {
-        "tooltip": { "mode": "multi", "sort": "desc" },
-        "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
-      }
-    },
-    {
-      "type": "timeseries",
-      "title": "Disk Write Rate by Node",
-      "description": "Disk write throughput (bytes/sec) aggregated per host node.",
-      "gridPos": { "h": 8, "w": 6, "x": 18, "y": 19 },
-      "id": 8,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "targets": [
-        {
-          "expr": "sum by (instance) (rate(node_disk_written_bytes_total[$__rate_interval]))",
-          "legendFormat": "{{instance}}",
-          "refId": "A"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "smooth",
-            "lineWidth": 1,
-            "fillOpacity": 15,
-            "gradientMode": "none",
-            "spanNulls": false,
-            "showPoints": "never",
-            "pointSize": 5,
-            "stacking": { "mode": "none", "group": "A" },
-            "axisPlacement": "auto",
-            "axisLabel": "",
-            "scaleDistribution": { "type": "linear" },
-            "thresholdsStyle": { "mode": "off" }
-          },
-          "unit": "Bps",
-          "color": { "mode": "palette-classic" },
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              { "color": "green", "value": null }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "options": {
-        "tooltip": { "mode": "multi", "sort": "desc" },
-        "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
       }
     },
     {
       "type": "row",
-      "title": "Network",
+      "title": "Instance Trends",
       "collapsed": false,
-      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 },
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 },
       "id": 9,
       "panels": []
     },
     {
       "type": "timeseries",
-      "title": "Network Receive Rate by Device & Node",
+      "title": "Instance Count Over Time",
-      "description": "Inbound network throughput per network device and host node.",
+      "description": "Total number of running Incus instances over time. Tracks growth and churn in the cluster workload count.",
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 },
       "id": 10,
-      "datasource": {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
       "targets": [
         {
-          "expr": "rate(node_network_receive_bytes_total{device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])",
+          "expr": "count(incus_uptime_seconds)",
-          "legendFormat": "{{instance}} - {{device}} (rx)",
+          "legendFormat": "Instances",
           "refId": "A"
         }
       ],
@@ -347,8 +334,8 @@
           "custom": {
             "drawStyle": "line",
             "lineInterpolation": "smooth",
-            "lineWidth": 1,
+            "lineWidth": 2,
-            "fillOpacity": 15,
+            "fillOpacity": 25,
             "gradientMode": "none",
             "spanNulls": false,
             "showPoints": "never",
@@ -359,37 +346,37 @@
             "scaleDistribution": { "type": "linear" },
             "thresholdsStyle": { "mode": "off" }
           },
-          "unit": "Bps",
+          "unit": "short",
           "color": { "mode": "palette-classic" },
           "thresholds": {
             "mode": "absolute",
-            "steps": [
+            "steps": [{ "color": "green", "value": null }]
-              { "color": "green", "value": null }
-            ]
           }
         },
         "overrides": []
       },
       "options": {
         "tooltip": { "mode": "multi", "sort": "desc" },
-        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
       }
     },
     {
       "type": "timeseries",
-      "title": "Network Transmit Rate by Device & Node",
+      "title": "Memory Allocation vs Usage per Instance",
-      "description": "Outbound network throughput per network device and host node.",
+      "description": "Compares total allocated memory (MemTotal) against actual resident memory (RSS) for each Incus instance. A large gap between allocated and used indicates overprovisioning; converging lines indicate an instance may need more memory.",
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 },
       "id": 11,
-      "datasource": {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
       "targets": [
         {
-          "expr": "rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])",
+          "expr": "incus_memory_MemTotal_bytes",
-          "legendFormat": "{{instance}} - {{device}} (tx)",
+          "legendFormat": "{{name}} allocated",
           "refId": "A"
+        },
+        {
+          "expr": "incus_memory_RSS_bytes",
+          "legendFormat": "{{name}} used (RSS)",
+          "refId": "B"
         }
       ],
       "fieldConfig": {
@@ -398,7 +385,7 @@
             "drawStyle": "line",
             "lineInterpolation": "smooth",
             "lineWidth": 1,
-            "fillOpacity": 15,
+            "fillOpacity": 0,
             "gradientMode": "none",
             "spanNulls": false,
             "showPoints": "never",
@@ -409,20 +396,32 @@
             "scaleDistribution": { "type": "linear" },
             "thresholdsStyle": { "mode": "off" }
           },
-          "unit": "Bps",
+          "unit": "bytes",
           "color": { "mode": "palette-classic" },
           "thresholds": {
             "mode": "absolute",
-            "steps": [
+            "steps": [{ "color": "green", "value": null }]
-              { "color": "green", "value": null }
-            ]
           }
         },
-        "overrides": []
+        "overrides": [
+          {
+            "matcher": { "id": "byRegexp", "options": ".*allocated$" },
+            "properties": [
+              { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } },
+              { "id": "custom.lineWidth", "value": 2 }
+            ]
+          },
+          {
+            "matcher": { "id": "byRegexp", "options": ".*used \\(RSS\\)$" },
+            "properties": [
+              { "id": "custom.fillOpacity", "value": 15 }
+            ]
+          }
+        ]
       },
       "options": {
         "tooltip": { "mode": "multi", "sort": "desc" },
-        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
       }
     }
   ]

incusos/observability-dashboards/core/cluster-overview.json

@@ -0,0 +1,674 @@
+{
+  "uid": "incus-cluster-overview",
+  "title": "Incus Cluster Overview",
+  "description": "Overview of all Incus instances: CPU, memory, network, disk I/O, and instance status table with drill-down links.",
+  "tags": ["incus", "cluster", "containers"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "refresh": "30s",
+  "time": { "from": "now-1h", "to": "now" },
+  "fiscalYearStartMonth": 0,
+  "liveNow": false,
+  "editable": true,
+  "graphTooltip": 1,
+  "links": [
+    {
+      "asDropdown": true,
+      "icon": "external link",
+      "includeVars": true,
+      "keepTime": true,
+      "tags": ["incus"],
+      "targetBlank": false,
+      "title": "Incus Dashboards",
+      "tooltip": "Navigate to other Incus dashboards",
+      "type": "dashboards"
+    }
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {},
+        "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+        "definition": "label_values(incus_uptime_seconds, name)",
+        "includeAll": true,
+        "multi": true,
+        "name": "instance",
+        "query": "label_values(incus_uptime_seconds, name)",
+        "refresh": 2,
+        "regex": "",
+        "sort": 1,
+        "type": "query",
+        "label": "Instance"
+      },
+      {
+        "current": {},
+        "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+        "definition": "label_values(incus_uptime_seconds, instance)",
+        "includeAll": true,
+        "multi": true,
+        "name": "node",
+        "query": "label_values(incus_uptime_seconds, instance)",
+        "refresh": 2,
+        "regex": "",
+        "sort": 1,
+        "type": "query",
+        "label": "Node"
+      }
+    ]
+  },
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": { "type": "grafana", "uid": "-- Grafana --" },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "panels": [
+    {
+      "type": "row",
+      "title": "Cluster Status",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+      "id": 1,
+      "panels": []
+    },
+    {
+      "type": "stat",
+      "title": "Total Instances",
+      "description": "Number of running Incus instances (reporting uptime).",
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
+      "id": 2,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "count(incus_uptime_seconds{instance=~\"$node\", name=~\"$instance\"})",
+          "legendFormat": "Instances",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 10 },
+              { "color": "red", "value": 50 }
+            ]
+          },
+          "unit": "none",
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "orientation": "auto",
+        "textMode": "auto",
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto"
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Total CPUs",
+      "description": "Sum of effective CPUs across all Incus instances.",
+      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
+      "id": 3,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum(incus_cpu_effective_total{instance=~\"$node\", name=~\"$instance\"})",
+          "legendFormat": "CPUs",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
+          "unit": "none",
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "orientation": "auto",
+        "textMode": "auto",
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto"
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Total Memory",
+      "description": "Sum of total memory across all Incus instances.",
+      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
+      "id": 4,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum(incus_memory_MemTotal_bytes{instance=~\"$node\", name=~\"$instance\"})",
+          "legendFormat": "Total Memory",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
+          "unit": "bytes",
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "orientation": "auto",
+        "textMode": "auto",
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto"
+      }
+    },
+    {
+      "type": "gauge",
+      "title": "Cluster Memory Utilization",
+      "description": "Overall cluster memory utilization (RSS / Total) across all instances.",
+      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
+      "id": 40,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "(sum(incus_memory_RSS_bytes{instance=~\"$node\", name=~\"$instance\"}) / sum(incus_memory_MemTotal_bytes{instance=~\"$node\", name=~\"$instance\"})) * 100",
+          "legendFormat": "Utilization",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 60 },
+              { "color": "orange", "value": 80 },
+              { "color": "red", "value": 90 }
+            ]
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true,
+        "orientation": "auto"
+      }
+    },
+    {
+      "type": "row",
+      "title": "Instance Status",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
+      "id": 50,
+      "panels": []
+    },
+    {
+      "type": "table",
+      "title": "Instance Status Table",
+      "description": "Per-instance summary: uptime, CPU usage, memory RSS, total memory. Click instance name to drill down.",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 },
+      "id": 51,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "incus_uptime_seconds{instance=~\"$node\", name=~\"$instance\"}",
+          "legendFormat": "",
+          "refId": "A",
+          "format": "table",
+          "instant": true
+        },
+        {
+          "expr": "sum by (name, instance) (rate(incus_cpu_seconds_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))",
+          "legendFormat": "",
+          "refId": "B",
+          "format": "table",
+          "instant": true
+        },
+        {
+          "expr": "incus_memory_RSS_bytes{instance=~\"$node\", name=~\"$instance\"}",
+          "legendFormat": "",
+          "refId": "C",
+          "format": "table",
+          "instant": true
+        },
+        {
+          "expr": "incus_memory_MemTotal_bytes{instance=~\"$node\", name=~\"$instance\"}",
+          "legendFormat": "",
+          "refId": "D",
+          "format": "table",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto",
+            "inspect": false,
+            "filterable": true
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "name" },
+            "properties": [
+              { "id": "displayName", "value": "Instance" },
+              { "id": "custom.width", "value": 200 },
+              {
+                "id": "links",
+                "value": [
+                  {
+                    "title": "View instance details",
+                    "url": "/d/incus-instance-deep-dive?var-instance=${__value.text}&${__url_time_range}",
+                    "targetBlank": false
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "instance" },
+            "properties": [{ "id": "displayName", "value": "Node" }]
+          },
+          {
+            "matcher": { "id": "byName", "options": "Value #A" },
+            "properties": [
+              { "id": "displayName", "value": "Uptime" },
+              { "id": "unit", "value": "s" }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "Value #B" },
+            "properties": [
+              { "id": "displayName", "value": "CPU/s" },
+              { "id": "unit", "value": "short" },
+              { "id": "decimals", "value": 3 }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "Value #C" },
+            "properties": [
+              { "id": "displayName", "value": "Memory RSS" },
+              { "id": "unit", "value": "bytes" }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "Value #D" },
+            "properties": [
+              { "id": "displayName", "value": "Total Memory" },
+              { "id": "unit", "value": "bytes" }
+            ]
+          }
+        ]
+      },
+      "options": {
+        "showHeader": true,
+        "sortBy": [{ "displayName": "Instance", "desc": false }],
+        "footer": { "show": false, "reducer": ["sum"], "fields": "" }
+      },
+      "transformations": [
+        { "id": "merge", "options": {} },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": { "Time": true, "__name__": true, "job": true },
+            "indexByName": {
+              "name": 0, "instance": 1,
+              "Value #A": 2, "Value #B": 3, "Value #C": 4, "Value #D": 5
+            },
+            "renameByName": {}
+          }
+        }
+      ]
+    },
+    {
+      "type": "row",
+      "title": "Per-Instance CPU",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
+      "id": 5,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "CPU Usage by Instance",
+      "description": "Per-second CPU time consumed by each Incus instance.",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 15 },
+      "id": 6,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum by (name) (rate(incus_cpu_seconds_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))",
+          "legendFormat": "{{name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "scheme",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "short",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
+          "links": [
+            {
+              "title": "Drill down to ${__field.labels.name}",
+              "url": "/d/incus-instance-deep-dive?var-instance=${__field.labels.name}&${__url_time_range}",
+              "targetBlank": false
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "row",
+      "title": "Per-Instance Memory",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
+      "id": 7,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Memory RSS by Instance",
+      "description": "Resident set size (RSS) memory usage per Incus instance over time.",
+      "gridPos": { "h": 8, "w": 14, "x": 0, "y": 24 },
+      "id": 8,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "incus_memory_RSS_bytes{instance=~\"$node\", name=~\"$instance\"}",
+          "legendFormat": "{{name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "bytes",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "bargauge",
+      "title": "Memory Usage % (RSS / Total)",
+      "description": "Current memory utilization per instance as RSS divided by total memory.",
+      "gridPos": { "h": 8, "w": 10, "x": 14, "y": 24 },
+      "id": 9,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "(incus_memory_RSS_bytes{instance=~\"$node\", name=~\"$instance\"} / incus_memory_MemTotal_bytes{instance=~\"$node\", name=~\"$instance\"}) * 100",
+          "legendFormat": "{{name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 60 },
+              { "color": "orange", "value": 80 },
+              { "color": "red", "value": 90 }
+            ]
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "orientation": "horizontal",
+        "displayMode": "gradient",
+        "showUnfilled": true,
+        "minVizWidth": 8,
+        "minVizHeight": 16,
+        "valueMode": "color"
+      }
+    },
+    {
+      "type": "row",
+      "title": "Network I/O",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 },
+      "id": 10,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Network Receive Rate by Instance",
+      "description": "Inbound network throughput per Incus instance.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 },
+      "id": 11,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum by (name) (rate(incus_network_receive_bytes_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))",
+          "legendFormat": "{{name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+            "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+            "showPoints": "never", "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "Bps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Network Transmit Rate by Instance",
+      "description": "Outbound network throughput per Incus instance.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 },
+      "id": 12,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum by (name) (rate(incus_network_transmit_bytes_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))",
+          "legendFormat": "{{name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+            "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+            "showPoints": "never", "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "Bps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "row",
+      "title": "Disk I/O",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 },
+      "id": 13,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Disk Read Rate by Instance",
+      "description": "Disk read throughput per Incus instance.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 42 },
+      "id": 14,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum by (name) (rate(incus_disk_read_bytes_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))",
+          "legendFormat": "{{name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+            "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+            "showPoints": "never", "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "Bps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Disk Write Rate by Instance",
+      "description": "Disk write throughput per Incus instance.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 42 },
+      "id": 15,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum by (name) (rate(incus_disk_written_bytes_total{instance=~\"$node\", name=~\"$instance\"}[$__rate_interval]))",
+          "legendFormat": "{{name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+            "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+            "showPoints": "never", "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "Bps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    }
+  ]
+}

incusos/observability-dashboards/core/haproxy-traffic.json

@@ -0,0 +1,910 @@
+{
+  "uid": "incus-haproxy-traffic",
+  "title": "HAProxy Traffic",
+  "description": "HAProxy load balancer monitoring: request rates, response codes, backend health, traffic volume, sessions, and performance metrics.",
+  "tags": ["incus", "haproxy", "loadbalancer"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 2,
+  "refresh": "30s",
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "fiscalYearStartMonth": 0,
+  "liveNow": false,
+  "editable": true,
+  "graphTooltip": 1,
+  "links": [
+    {
+      "asDropdown": true,
+      "icon": "external link",
+      "includeVars": true,
+      "keepTime": true,
+      "tags": ["incus"],
+      "targetBlank": false,
+      "title": "Incus Dashboards",
+      "tooltip": "Navigate to other Incus dashboards",
+      "type": "dashboards"
+    }
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {},
+        "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+        "definition": "label_values(haproxy_backend_status, proxy)",
+        "includeAll": true,
+        "multi": true,
+        "name": "proxy",
+        "query": "label_values(haproxy_backend_status, proxy)",
+        "refresh": 2,
+        "regex": "",
+        "sort": 1,
+        "type": "query",
+        "label": "Proxy"
+      },
+      {
+        "current": {},
+        "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+        "definition": "label_values(haproxy_server_status, server)",
+        "includeAll": true,
+        "multi": true,
+        "name": "server",
+        "query": "label_values(haproxy_server_status, server)",
+        "refresh": 2,
+        "regex": "",
+        "sort": 1,
+        "type": "query",
+        "label": "Server"
+      }
+    ]
+  },
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "panels": [
+    {
+      "type": "row",
+      "title": "Overview",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+      "id": 1,
+      "panels": []
+    },
+    {
+      "type": "stat",
+      "title": "Active Backend Servers",
+      "description": "Total number of active (healthy) servers across selected HAProxy backends.",
+      "gridPos": { "h": 4, "w": 5, "x": 0, "y": 1 },
+      "id": 2,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum(haproxy_backend_active_servers{proxy=~\"$proxy\"})",
+          "legendFormat": "Active Servers",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red", "value": null },
+              { "color": "yellow", "value": 1 },
+              { "color": "green", "value": 2 }
+            ]
+          },
+          "unit": "none",
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "orientation": "auto",
+        "textMode": "auto",
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto"
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Current Sessions",
+      "description": "Total current active sessions across selected HAProxy backends.",
+      "gridPos": { "h": 4, "w": 5, "x": 5, "y": 1 },
+      "id": 3,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum(haproxy_backend_current_sessions{proxy=~\"$proxy\"})",
+          "legendFormat": "Sessions",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 100 },
+              { "color": "red", "value": 500 }
+            ]
+          },
+          "unit": "none",
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "orientation": "auto",
+        "textMode": "auto",
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto"
+      }
+    },
+    {
+      "type": "stat",
+      "title": "HTTP Requests/s",
+      "description": "Current rate of HTTP requests per second across selected HAProxy frontends.",
+      "gridPos": { "h": 4, "w": 5, "x": 10, "y": 1 },
+      "id": 4,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum(rate(haproxy_frontend_http_requests_total{proxy=~\"$proxy\"}[$__rate_interval]))",
+          "legendFormat": "Requests/s",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "reqps",
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "orientation": "auto",
+        "textMode": "auto",
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto"
+      }
+    },
+    {
+      "type": "stat",
+      "title": "HTTP 4xx Rate",
+      "description": "Rate of HTTP 4xx client error responses per second. Elevated rates may indicate bad requests or missing resources.",
+      "gridPos": { "h": 4, "w": 4, "x": 15, "y": 1 },
+      "id": 14,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum(rate(haproxy_frontend_http_responses_total{proxy=~\"$proxy\", code=\"4xx\"}[$__rate_interval]))",
+          "legendFormat": "4xx/s",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 0.1 },
+              { "color": "red", "value": 1 }
+            ]
+          },
+          "unit": "reqps",
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "orientation": "auto",
+        "textMode": "auto",
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto"
+      }
+    },
+    {
+      "type": "stat",
+      "title": "HTTP 5xx Rate",
+      "description": "Rate of HTTP 5xx server error responses per second. Any sustained rate indicates backend failures requiring investigation.",
+      "gridPos": { "h": 4, "w": 5, "x": 19, "y": 1 },
+      "id": 15,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum(rate(haproxy_frontend_http_responses_total{proxy=~\"$proxy\", code=\"5xx\"}[$__rate_interval]))",
+          "legendFormat": "5xx/s",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 0.01 },
+              { "color": "red", "value": 0.1 }
+            ]
+          },
+          "unit": "reqps",
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "orientation": "auto",
+        "textMode": "auto",
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto"
+      }
+    },
+    {
+      "type": "row",
+      "title": "HTTP Requests",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
+      "id": 5,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "HTTP Request Rate by Frontend",
+      "description": "Rate of HTTP requests per second, broken down by HAProxy frontend/proxy. Useful for identifying which frontends carry the most traffic.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
+      "id": 6,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum by (proxy) (rate(haproxy_frontend_http_requests_total{proxy=~\"$proxy\"}[$__rate_interval]))",
+          "legendFormat": "{{proxy}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "reqps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Response Code Distribution",
+      "description": "HTTP response code distribution across selected frontends, stacked by response class (2xx, 3xx, 4xx, 5xx). Helps identify error trends and traffic composition.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
+      "id": 16,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum by (code) (rate(haproxy_frontend_http_responses_total{proxy=~\"$proxy\"}[$__rate_interval]))",
+          "legendFormat": "{{code}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 50,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "normal", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "reqps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "2xx" },
+            "properties": [
+              { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "3xx" },
+            "properties": [
+              { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "4xx" },
+            "properties": [
+              { "id": "color", "value": { "mode": "fixed", "fixedColor": "yellow" } }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "5xx" },
+            "properties": [
+              { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }
+            ]
+          }
+        ]
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "row",
+      "title": "Backend Health",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
+      "id": 7,
+      "panels": []
+    },
+    {
+      "type": "table",
+      "title": "Backend Status",
+      "description": "Current status, active server count, and average response time for each HAProxy backend. Filterable by proxy and server template variables.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 },
+      "id": 8,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "haproxy_backend_status{proxy=~\"$proxy\"}",
+          "legendFormat": "",
+          "refId": "A",
+          "format": "table",
+          "instant": true
+        },
+        {
+          "expr": "haproxy_backend_active_servers{proxy=~\"$proxy\"}",
+          "legendFormat": "",
+          "refId": "B",
+          "format": "table",
+          "instant": true
+        },
+        {
+          "expr": "haproxy_server_response_time_average_seconds{proxy=~\"$proxy\", server=~\"$server\"}",
+          "legendFormat": "",
+          "refId": "C",
+          "format": "table",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto",
+            "inspect": false,
+            "filterable": true
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "Value #A" },
+            "properties": [
+              { "id": "displayName", "value": "Status" },
+              {
+                "id": "mappings",
+                "value": [
+                  { "type": "value", "options": { "1": { "text": "UP", "color": "green" }, "0": { "text": "DOWN", "color": "red" } } }
+                ]
+              }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "Value #B" },
+            "properties": [
+              { "id": "displayName", "value": "Active Servers" }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "Value #C" },
+            "properties": [
+              { "id": "displayName", "value": "Avg Response Time" },
+              { "id": "unit", "value": "s" }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "proxy" },
+            "properties": [
+              { "id": "displayName", "value": "Backend" }
+            ]
+          }
+        ]
+      },
+      "options": {
+        "showHeader": true,
+        "sortBy": [],
+        "footer": {
+          "show": false,
+          "reducer": ["sum"],
+          "fields": ""
+        }
+      },
+      "transformations": [
+        {
+          "id": "merge",
+          "options": {}
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "__name__": true,
+              "instance": false,
+              "job": true
+            },
+            "indexByName": {},
+            "renameByName": {}
+          }
+        }
+      ]
+    },
+    {
+      "type": "state-timeline",
+      "title": "Server Status",
+      "description": "Timeline view of individual HAProxy server health status. Green indicates UP, red indicates DOWN. Helps identify flapping servers and outage windows.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 },
+      "id": 17,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "haproxy_server_status{proxy=~\"$proxy\", server=~\"$server\"}",
+          "legendFormat": "{{proxy}} / {{server}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "lineWidth": 0,
+            "fillOpacity": 80
+          },
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red", "value": null },
+              { "color": "green", "value": 1 }
+            ]
+          },
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": { "text": "DOWN", "color": "red" },
+                "1": { "text": "UP", "color": "green" }
+              }
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "options": {
+        "showValue": "auto",
+        "mergeValues": true,
+        "alignValue": "left",
+        "rowHeight": 0.9,
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "list", "placement": "bottom" }
+      }
+    },
+    {
+      "type": "row",
+      "title": "Traffic",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
+      "id": 9,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Frontend Bytes In",
+      "description": "Inbound traffic rate per HAProxy frontend. Shows how much data clients are sending to each frontend.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
+      "id": 10,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum by (proxy) (rate(haproxy_frontend_bytes_in_total{proxy=~\"$proxy\"}[$__rate_interval]))",
+          "legendFormat": "{{proxy}} (in)",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "Bps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Frontend Bytes Out",
+      "description": "Outbound traffic rate per HAProxy frontend. Shows how much data is being served to clients from each frontend.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
+      "id": 11,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum by (proxy) (rate(haproxy_frontend_bytes_out_total{proxy=~\"$proxy\"}[$__rate_interval]))",
+          "legendFormat": "{{proxy}} (out)",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "Bps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "row",
+      "title": "Sessions & Performance",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 },
+      "id": 12,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Current Sessions by Proxy",
+      "description": "Current active sessions for each HAProxy backend/proxy over time. Sustained high values may indicate slow backends or session buildup.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 },
+      "id": 13,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "haproxy_backend_current_sessions{proxy=~\"$proxy\"}",
+          "legendFormat": "{{proxy}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "none",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Backend Response Time",
+      "description": "Average response time from backend servers. High or increasing values indicate backend performance degradation.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 },
+      "id": 18,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "haproxy_server_response_time_average_seconds{proxy=~\"$proxy\", server=~\"$server\"}",
+          "legendFormat": "{{proxy}} / {{server}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "scheme",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "line" }
+          },
+          "unit": "s",
+          "color": { "mode": "continuous-GrYlRd" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 0.5 },
+              { "color": "red", "value": 2 }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Queue Time",
+      "description": "Average time requests spend in the backend queue before being assigned to a server. High values indicate insufficient backend capacity.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 41 },
+      "id": 19,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "haproxy_backend_queue_time_average_seconds{proxy=~\"$proxy\"}",
+          "legendFormat": "{{proxy}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "scheme",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "line" }
+          },
+          "unit": "s",
+          "color": { "mode": "continuous-GrYlRd" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 0.1 },
+              { "color": "red", "value": 1 }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Connection Errors",
+      "description": "Rate of backend connection errors and connection retries per proxy. Non-zero values indicate connectivity problems between HAProxy and backend servers.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 41 },
+      "id": 20,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum by (proxy) (rate(haproxy_backend_connection_errors_total{proxy=~\"$proxy\"}[$__rate_interval]))",
+          "legendFormat": "{{proxy}} errors",
+          "refId": "A"
+        },
+        {
+          "expr": "sum by (proxy) (rate(haproxy_backend_retry_warnings_total{proxy=~\"$proxy\"}[$__rate_interval]))",
+          "legendFormat": "{{proxy}} retries",
+          "refId": "B"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "reqps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byRegexp", "options": ".*errors$" },
+            "properties": [
+              { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }
+            ]
+          },
+          {
+            "matcher": { "id": "byRegexp", "options": ".*retries$" },
+            "properties": [
+              { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } }
+            ]
+          }
+        ]
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    }
+  ]
+}

incusos/observability-dashboards/core/host-resources.json

@@ -0,0 +1,584 @@
+{
+  "uid": "incus-host-resources",
+  "title": "Host Resources (IncusOS Nodes)",
+  "description": "Physical host monitoring: CPU mode breakdown, load averages, memory breakdown, filesystem usage, disk IOPS, network errors.",
+  "tags": ["incus", "incusos", "host", "node-exporter"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "refresh": "30s",
+  "time": { "from": "now-1h", "to": "now" },
+  "fiscalYearStartMonth": 0,
+  "liveNow": false,
+  "editable": true,
+  "graphTooltip": 1,
+  "links": [
+    {
+      "asDropdown": true, "icon": "external link", "includeVars": true,
+      "keepTime": true, "tags": ["incus"], "targetBlank": false,
+      "title": "Incus Dashboards", "type": "dashboards"
+    }
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {},
+        "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+        "definition": "label_values(node_uname_info, instance)",
+        "includeAll": true,
+        "multi": true,
+        "name": "node",
+        "query": "label_values(node_uname_info, instance)",
+        "refresh": 2,
+        "sort": 1,
+        "type": "query",
+        "label": "Node"
+      }
+    ]
+  },
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": { "type": "grafana", "uid": "-- Grafana --" },
+        "enable": true, "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts", "type": "dashboard"
+      }
+    ]
+  },
+  "panels": [
+    {
+      "type": "row", "title": "System Info",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+      "id": 30, "panels": []
+    },
+    {
+      "type": "table",
+      "title": "System Information",
+      "description": "Kernel version, uptime, total RAM, and CPU count per node.",
+      "gridPos": { "h": 5, "w": 24, "x": 0, "y": 1 },
+      "id": 31,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "node_uname_info{instance=~\"$node\"}",
+          "legendFormat": "", "refId": "A", "format": "table", "instant": true
+        },
+        {
+          "expr": "node_boot_time_seconds{instance=~\"$node\"}",
+          "legendFormat": "", "refId": "B", "format": "table", "instant": true
+        },
+        {
+          "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"}",
+          "legendFormat": "", "refId": "C", "format": "table", "instant": true
+        },
+        {
+          "expr": "count by (instance) (node_cpu_seconds_total{instance=~\"$node\", mode=\"idle\"})",
+          "legendFormat": "", "refId": "D", "format": "table", "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": { "align": "auto", "displayMode": "auto", "inspect": false, "filterable": true },
+          "color": { "mode": "thresholds" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": [
+          { "matcher": { "id": "byName", "options": "instance" }, "properties": [{ "id": "displayName", "value": "Node" }] },
+          { "matcher": { "id": "byName", "options": "release" }, "properties": [{ "id": "displayName", "value": "Kernel" }] },
+          { "matcher": { "id": "byName", "options": "Value #B" }, "properties": [{ "id": "displayName", "value": "Boot Time" }, { "id": "unit", "value": "dateTimeFromNow" }] },
+          { "matcher": { "id": "byName", "options": "Value #C" }, "properties": [{ "id": "displayName", "value": "Total RAM" }, { "id": "unit", "value": "bytes" }] },
+          { "matcher": { "id": "byName", "options": "Value #D" }, "properties": [{ "id": "displayName", "value": "CPUs" }] }
+        ]
+      },
+      "options": { "showHeader": true, "sortBy": [], "footer": { "show": false, "reducer": ["sum"], "fields": "" } },
+      "transformations": [
+        { "id": "merge", "options": {} },
+        { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "job": true, "domainname": true, "machine": true, "nodename": true, "sysname": true, "version": true, "Value #A": true }, "renameByName": {} } }
+      ]
+    },
+    {
+      "type": "row", "title": "CPU",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 },
+      "id": 1, "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "CPU Usage % by Node",
+      "description": "Overall CPU utilization per host node.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 },
+      "id": 2,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$node\"}[$__rate_interval]))) * 100",
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2,
+            "fillOpacity": 25, "gradientMode": "scheme", "spanNulls": false,
+            "showPoints": "never", "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "line" }
+          },
+          "unit": "percent", "min": 0, "max": 100,
+          "color": { "mode": "continuous-GrYlRd" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "CPU Mode Breakdown",
+      "description": "CPU time by mode (user, system, iowait, idle) stacked per node.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 },
+      "id": 32,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        { "expr": "avg by (instance, mode) (rate(node_cpu_seconds_total{instance=~\"$node\", mode=~\"user|system|iowait|idle\"}[$__rate_interval])) * 100", "legendFormat": "{{instance}} {{mode}}", "refId": "A" }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+            "fillOpacity": 50, "gradientMode": "none", "spanNulls": false,
+            "showPoints": "never",
+            "stacking": { "mode": "normal", "group": "A" },
+            "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "percent", "min": 0,
+          "color": { "mode": "palette-classic" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Load Average",
+      "description": "System load averages (1m, 5m, 15m) with CPU count threshold line.",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 15 },
+      "id": 33,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        { "expr": "node_load1{instance=~\"$node\"}", "legendFormat": "{{instance}} load1", "refId": "A" },
+        { "expr": "node_load5{instance=~\"$node\"}", "legendFormat": "{{instance}} load5", "refId": "B" },
+        { "expr": "node_load15{instance=~\"$node\"}", "legendFormat": "{{instance}} load15", "refId": "C" },
+        { "expr": "count by (instance) (node_cpu_seconds_total{instance=~\"$node\", mode=\"idle\"})", "legendFormat": "{{instance}} CPUs", "refId": "D" }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+            "fillOpacity": 0, "gradientMode": "none", "spanNulls": false,
+            "showPoints": "never",
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "short",
+          "color": { "mode": "palette-classic" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byRegexp", "options": ".*CPUs$" },
+            "properties": [
+              { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } },
+              { "id": "custom.lineWidth", "value": 2 },
+              { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }
+            ]
+          }
+        ]
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "row", "title": "Memory",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
+      "id": 3, "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Memory Usage % by Node",
+      "description": "Memory utilization percentage per host node.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
+      "id": 4,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "(1 - node_memory_MemAvailable_bytes{instance=~\"$node\"} / node_memory_MemTotal_bytes{instance=~\"$node\"}) * 100",
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2,
+            "fillOpacity": 25, "gradientMode": "scheme", "spanNulls": false,
+            "showPoints": "never",
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "line" }
+          },
+          "unit": "percent", "min": 0, "max": 100,
+          "color": { "mode": "continuous-GrYlRd" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Memory Breakdown by Node",
+      "description": "Memory breakdown: used, buffers, cached, available (stacked).",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
+      "id": 34,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        { "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"} - node_memory_MemAvailable_bytes{instance=~\"$node\"} - node_memory_Buffers_bytes{instance=~\"$node\"} - node_memory_Cached_bytes{instance=~\"$node\"}", "legendFormat": "{{instance}} used", "refId": "A" },
+        { "expr": "node_memory_Buffers_bytes{instance=~\"$node\"}", "legendFormat": "{{instance}} buffers", "refId": "B" },
+        { "expr": "node_memory_Cached_bytes{instance=~\"$node\"}", "legendFormat": "{{instance}} cached", "refId": "C" },
+        { "expr": "node_memory_MemAvailable_bytes{instance=~\"$node\"}", "legendFormat": "{{instance}} available", "refId": "D" }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+            "fillOpacity": 50, "gradientMode": "none", "spanNulls": false,
+            "showPoints": "never",
+            "stacking": { "mode": "normal", "group": "A" },
+            "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "bytes",
+          "color": { "mode": "palette-classic" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "row", "title": "Disk",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 },
+      "id": 5, "panels": []
+    },
+    {
+      "type": "bargauge",
+      "title": "Filesystem Usage %",
+      "description": "Current filesystem usage percentage for important mount points.",
+      "gridPos": { "h": 6, "w": 8, "x": 0, "y": 33 },
+      "id": 35,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "(1 - node_filesystem_avail_bytes{instance=~\"$node\", fstype!~\"tmpfs|overlay\", mountpoint=~\"/.*\"} / node_filesystem_size_bytes{instance=~\"$node\", fstype!~\"tmpfs|overlay\", mountpoint=~\"/.*\"}) * 100",
+          "legendFormat": "{{instance}} {{mountpoint}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "orange", "value": 85 }, { "color": "red", "value": 95 }] },
+          "unit": "percent", "min": 0, "max": 100,
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "orientation": "horizontal", "displayMode": "gradient", "showUnfilled": true,
+        "minVizWidth": 8, "minVizHeight": 16, "valueMode": "color"
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Root Filesystem Available Space",
+      "description": "Available disk space on the root filesystem per host node.",
+      "gridPos": { "h": 6, "w": 8, "x": 8, "y": 33 },
+      "id": 6,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        { "expr": "node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=\"/\"}", "legendFormat": "{{instance}}", "refId": "A" }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+            "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+            "showPoints": "never",
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "bytes",
+          "color": { "mode": "palette-classic" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull", "min"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Disk IOPS by Node",
+      "description": "Disk read and write operations per second by node.",
+      "gridPos": { "h": 6, "w": 8, "x": 16, "y": 33 },
+      "id": 36,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        { "expr": "sum by (instance) (rate(node_disk_reads_completed_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} reads", "refId": "A" },
+        { "expr": "sum by (instance) (rate(node_disk_writes_completed_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} writes", "refId": "B" }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+            "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+            "showPoints": "never",
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "iops",
+          "color": { "mode": "palette-classic" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Disk Read Rate by Node",
+      "description": "Disk read throughput aggregated per host node.",
+      "gridPos": { "h": 6, "w": 8, "x": 0, "y": 39 },
+      "id": 7,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        { "expr": "sum by (instance) (rate(node_disk_read_bytes_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}}", "refId": "A" }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+            "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+            "showPoints": "never",
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "Bps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Disk Write Rate by Node",
+      "description": "Disk write throughput aggregated per host node.",
+      "gridPos": { "h": 6, "w": 8, "x": 8, "y": 39 },
+      "id": 8,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        { "expr": "sum by (instance) (rate(node_disk_written_bytes_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}}", "refId": "A" }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+            "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+            "showPoints": "never",
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "Bps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Disk I/O Time",
+      "description": "Time spent on disk I/O operations (disk utilization indicator).",
+      "gridPos": { "h": 6, "w": 8, "x": 16, "y": 39 },
+      "id": 37,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        { "expr": "sum by (instance) (rate(node_disk_io_time_seconds_total{instance=~\"$node\"}[$__rate_interval])) * 100", "legendFormat": "{{instance}}", "refId": "A" }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+            "fillOpacity": 15, "gradientMode": "scheme", "spanNulls": false,
+            "showPoints": "never",
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "line" }
+          },
+          "unit": "percent", "min": 0, "max": 100,
+          "color": { "mode": "continuous-GrYlRd" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 80 }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+      }
+    },
+    {
+      "type": "row", "title": "Network",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 45 },
+      "id": 9, "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Network Receive Rate by Device & Node",
+      "description": "Inbound network throughput per network device and host node.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 46 },
+      "id": 10,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        { "expr": "rate(node_network_receive_bytes_total{instance=~\"$node\", device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])", "legendFormat": "{{instance}} {{device}} rx", "refId": "A" }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+            "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+            "showPoints": "never",
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "Bps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Network Transmit Rate by Device & Node",
+      "description": "Outbound network throughput per network device and host node.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 46 },
+      "id": 11,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        { "expr": "rate(node_network_transmit_bytes_total{instance=~\"$node\", device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])", "legendFormat": "{{instance}} {{device}} tx", "refId": "A" }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+            "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+            "showPoints": "never",
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "Bps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Network Errors & Drops",
+      "description": "Network receive/transmit errors and dropped packets per node.",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 54 },
+      "id": 38,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        { "expr": "sum by (instance) (rate(node_network_receive_errs_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} rx errors", "refId": "A" },
+        { "expr": "sum by (instance) (rate(node_network_transmit_errs_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} tx errors", "refId": "B" },
+        { "expr": "sum by (instance) (rate(node_network_receive_drop_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} rx drops", "refId": "C" },
+        { "expr": "sum by (instance) (rate(node_network_transmit_drop_total{instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} tx drops", "refId": "D" }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+            "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+            "showPoints": "never",
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "pps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    }
+  ]
+}

incusos/observability-dashboards/core/instance-deep-dive.json

@@ -1,31 +1,59 @@
 {
-  "uid": "incus-cluster-overview",
+  "uid": "incus-instance-deep-dive",
-  "title": "Incus Cluster Overview",
+  "title": "Instance Deep Dive",
-  "description": "Overview of all Incus containers: CPU, memory, network, and disk I/O per instance.",
+  "description": "Single-instance drill-down: CPU, memory, network, and disk metrics with detailed timeseries and utilization gauges.",
   "tags": ["incus", "cluster", "containers"],
   "timezone": "browser",
   "schemaVersion": 39,
-  "version": 1,
   "refresh": "30s",
-  "time": {
+  "time": { "from": "now-1h", "to": "now" },
-    "from": "now-1h",
-    "to": "now"
-  },
   "fiscalYearStartMonth": 0,
   "liveNow": false,
   "editable": true,
   "graphTooltip": 1,
+  "links": [
+    {
+      "asDropdown": true,
+      "icon": "external link",
+      "includeVars": true,
+      "keepTime": true,
+      "tags": ["incus"],
+      "targetBlank": false,
+      "title": "Incus Dashboards",
+      "tooltip": "Navigate to other Incus dashboards",
+      "type": "dashboards"
+    },
+    {
+      "title": "Back to Cluster Overview",
+      "url": "/d/incus-cluster-overview?${__url_time_range}",
+      "targetBlank": false,
+      "type": "link",
+      "icon": "arrow-left"
+    }
+  ],
   "templating": {
-    "list": []
+    "list": [
+      {
+        "current": {},
+        "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+        "definition": "label_values(incus_uptime_seconds, name)",
+        "includeAll": false,
+        "multi": false,
+        "name": "instance",
+        "query": "label_values(incus_uptime_seconds, name)",
+        "refresh": 2,
+        "regex": "",
+        "sort": 1,
+        "type": "query",
+        "label": "Instance"
+      }
+    ]
   },
   "annotations": {
     "list": [
       {
         "builtIn": 1,
-        "datasource": {
+        "datasource": { "type": "grafana", "uid": "-- Grafana --" },
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
         "enable": true,
         "hide": true,
         "iconColor": "rgba(0, 211, 255, 1)",
@@ -37,7 +65,7 @@
   "panels": [
     {
       "type": "row",
-      "title": "Cluster Status",
+      "title": "Instance Status",
       "collapsed": false,
       "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
       "id": 1,
@@ -45,18 +73,15 @@
     },
     {
       "type": "stat",
-      "title": "Total Containers",
+      "title": "Uptime",
-      "description": "Number of running Incus containers (instances reporting uptime).",
+      "description": "How long this instance has been running.",
-      "gridPos": { "h": 4, "w": 8, "x": 0, "y": 1 },
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
       "id": 2,
-      "datasource": {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
       "targets": [
         {
-          "expr": "count(incus_uptime_seconds)",
+          "expr": "incus_uptime_seconds{name=~\"$instance\"}",
-          "legendFormat": "Containers",
+          "legendFormat": "Uptime",
           "refId": "A"
         }
       ],
@@ -64,43 +89,32 @@
         "defaults": {
           "thresholds": {
             "mode": "absolute",
-            "steps": [
+            "steps": [{ "color": "green", "value": null }]
-              { "color": "green", "value": null },
-              { "color": "yellow", "value": 10 },
-              { "color": "red", "value": 50 }
-            ]
           },
-          "unit": "none",
+          "unit": "s",
           "color": { "mode": "thresholds" }
         },
         "overrides": []
       },
       "options": {
-        "reduceOptions": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
-          "calcs": ["lastNotNull"],
-          "fields": "",
-          "values": false
-        },
         "orientation": "auto",
         "textMode": "auto",
         "colorMode": "value",
-        "graphMode": "none",
+        "graphMode": "area",
         "justifyMode": "auto"
       }
     },
     {
       "type": "stat",
-      "title": "Total CPUs",
+      "title": "Effective CPUs",
-      "description": "Sum of effective CPUs across all Incus instances.",
+      "description": "Number of effective CPUs allocated to this instance.",
-      "gridPos": { "h": 4, "w": 8, "x": 8, "y": 1 },
+      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
       "id": 3,
-      "datasource": {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
       "targets": [
         {
-          "expr": "sum(incus_cpu_effective_total)",
+          "expr": "incus_cpu_effective_total{name=~\"$instance\"}",
           "legendFormat": "CPUs",
           "refId": "A"
         }
@@ -109,9 +123,7 @@
         "defaults": {
           "thresholds": {
             "mode": "absolute",
-            "steps": [
+            "steps": [{ "color": "green", "value": null }]
-              { "color": "green", "value": null }
-            ]
           },
           "unit": "none",
           "color": { "mode": "thresholds" }
@@ -119,31 +131,24 @@
         "overrides": []
       },
       "options": {
-        "reduceOptions": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
-          "calcs": ["lastNotNull"],
-          "fields": "",
-          "values": false
-        },
         "orientation": "auto",
         "textMode": "auto",
         "colorMode": "value",
-        "graphMode": "none",
+        "graphMode": "area",
         "justifyMode": "auto"
       }
     },
     {
       "type": "stat",
       "title": "Total Memory",
-      "description": "Sum of total memory across all Incus instances.",
+      "description": "Total memory allocated to this instance.",
-      "gridPos": { "h": 4, "w": 8, "x": 16, "y": 1 },
+      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
       "id": 4,
-      "datasource": {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
       "targets": [
         {
-          "expr": "sum(incus_memory_MemTotal_bytes)",
+          "expr": "incus_memory_MemTotal_bytes{name=~\"$instance\"}",
           "legendFormat": "Total Memory",
           "refId": "A"
         }
@@ -152,9 +157,7 @@
         "defaults": {
           "thresholds": {
             "mode": "absolute",
-            "steps": [
+            "steps": [{ "color": "green", "value": null }]
-              { "color": "green", "value": null }
-            ]
           },
           "unit": "bytes",
           "color": { "mode": "thresholds" }
@@ -162,148 +165,25 @@
         "overrides": []
       },
       "options": {
-        "reduceOptions": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
-          "calcs": ["lastNotNull"],
-          "fields": "",
-          "values": false
-        },
         "orientation": "auto",
         "textMode": "auto",
         "colorMode": "value",
-        "graphMode": "none",
+        "graphMode": "area",
         "justifyMode": "auto"
       }
     },
     {
-      "type": "row",
+      "type": "gauge",
-      "title": "Per-Instance CPU",
+      "title": "Memory Utilization",
-      "collapsed": false,
+      "description": "Current memory utilization (RSS / Total) for this instance.",
-      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
+      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
       "id": 5,
-      "panels": []
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
-    },
-    {
-      "type": "timeseries",
-      "title": "CPU Usage by Instance",
-      "description": "Per-second CPU time consumed by each Incus instance.",
-      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 },
-      "id": 6,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
       "targets": [
         {
-          "expr": "sum by (name) (rate(incus_cpu_seconds_total[$__rate_interval]))",
+          "expr": "(incus_memory_RSS_bytes{name=~\"$instance\"} / incus_memory_MemTotal_bytes{name=~\"$instance\"}) * 100",
-          "legendFormat": "{{name}}",
+          "legendFormat": "Utilization",
-          "refId": "A"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "smooth",
-            "lineWidth": 1,
-            "fillOpacity": 15,
-            "gradientMode": "none",
-            "spanNulls": false,
-            "showPoints": "never",
-            "pointSize": 5,
-            "stacking": { "mode": "none", "group": "A" },
-            "axisPlacement": "auto",
-            "axisLabel": "",
-            "scaleDistribution": { "type": "linear" },
-            "thresholdsStyle": { "mode": "off" }
-          },
-          "unit": "short",
-          "color": { "mode": "palette-classic" },
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              { "color": "green", "value": null }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "options": {
-        "tooltip": { "mode": "multi", "sort": "desc" },
-        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
-      }
-    },
-    {
-      "type": "row",
-      "title": "Per-Instance Memory",
-      "collapsed": false,
-      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
-      "id": 7,
-      "panels": []
-    },
-    {
-      "type": "timeseries",
-      "title": "Memory RSS by Instance",
-      "description": "Resident set size (RSS) memory usage per Incus instance over time.",
-      "gridPos": { "h": 8, "w": 14, "x": 0, "y": 15 },
-      "id": 8,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "targets": [
-        {
-          "expr": "incus_memory_RSS_bytes",
-          "legendFormat": "{{name}}",
-          "refId": "A"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "smooth",
-            "lineWidth": 1,
-            "fillOpacity": 15,
-            "gradientMode": "none",
-            "spanNulls": false,
-            "showPoints": "never",
-            "pointSize": 5,
-            "stacking": { "mode": "none", "group": "A" },
-            "axisPlacement": "auto",
-            "axisLabel": "",
-            "scaleDistribution": { "type": "linear" },
-            "thresholdsStyle": { "mode": "off" }
-          },
-          "unit": "bytes",
-          "color": { "mode": "palette-classic" },
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              { "color": "green", "value": null }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "options": {
-        "tooltip": { "mode": "multi", "sort": "desc" },
-        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
-      }
-    },
-    {
-      "type": "bargauge",
-      "title": "Memory Usage % (RSS / Total)",
-      "description": "Current memory utilization per instance as RSS divided by total memory.",
-      "gridPos": { "h": 8, "w": 10, "x": 14, "y": 15 },
-      "id": 9,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "targets": [
-        {
-          "expr": "(incus_memory_RSS_bytes / incus_memory_MemTotal_bytes) * 100",
-          "legendFormat": "{{name}}",
           "refId": "A"
         }
       ],
@@ -326,40 +206,83 @@
         "overrides": []
       },
       "options": {
-        "reduceOptions": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
-          "calcs": ["lastNotNull"],
+        "showThresholdLabels": false,
-          "fields": "",
+        "showThresholdMarkers": true,
-          "values": false
+        "orientation": "auto"
-        },
-        "orientation": "horizontal",
-        "displayMode": "gradient",
-        "showUnfilled": true,
-        "minVizWidth": 8,
-        "minVizHeight": 16,
-        "valueMode": "color"
       }
     },
     {
       "type": "row",
-      "title": "Network I/O",
+      "title": "CPU",
       "collapsed": false,
-      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
       "id": 10,
       "panels": []
     },
     {
       "type": "timeseries",
-      "title": "Network Receive Rate by Instance",
+      "title": "CPU Usage Over Time",
-      "description": "Inbound network throughput per Incus instance.",
+      "description": "Per-second CPU time consumed by this instance over time.",
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 },
       "id": 11,
-      "datasource": {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
       "targets": [
         {
-          "expr": "sum by (name) (rate(incus_network_receive_bytes_total[$__rate_interval]))",
+          "expr": "sum by (name) (rate(incus_cpu_seconds_total{name=~\"$instance\"}[$__rate_interval]))",
+          "legendFormat": "{{name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 25,
+            "gradientMode": "scheme",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "short",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "row",
+      "title": "Memory",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
+      "id": 20,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Memory RSS Over Time",
+      "description": "Resident set size (RSS) memory usage for this instance over time.",
+      "gridPos": { "h": 8, "w": 14, "x": 0, "y": 15 },
+      "id": 21,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "incus_memory_RSS_bytes{name=~\"$instance\"}",
           "legendFormat": "{{name}}",
           "refId": "A"
         }
@@ -381,39 +304,99 @@
             "scaleDistribution": { "type": "linear" },
             "thresholdsStyle": { "mode": "off" }
           },
-          "unit": "Bps",
+          "unit": "bytes",
           "color": { "mode": "palette-classic" },
           "thresholds": {
             "mode": "absolute",
-            "steps": [
+            "steps": [{ "color": "green", "value": null }]
-              { "color": "green", "value": null }
-            ]
           }
         },
         "overrides": []
       },
       "options": {
         "tooltip": { "mode": "multi", "sort": "desc" },
-        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
       }
     },
     {
       "type": "timeseries",
-      "title": "Network Transmit Rate by Instance",
+      "title": "Memory Utilization %",
-      "description": "Outbound network throughput per Incus instance.",
+      "description": "Memory utilization percentage (RSS / Total) for this instance over time.",
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
+      "gridPos": { "h": 8, "w": 10, "x": 14, "y": 15 },
-      "id": 12,
+      "id": 22,
-      "datasource": {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
       "targets": [
         {
-          "expr": "sum by (name) (rate(incus_network_transmit_bytes_total[$__rate_interval]))",
+          "expr": "(incus_memory_RSS_bytes{name=~\"$instance\"} / incus_memory_MemTotal_bytes{name=~\"$instance\"}) * 100",
           "legendFormat": "{{name}}",
           "refId": "A"
         }
       ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "fillOpacity": 25,
+            "gradientMode": "scheme",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "line" }
+          },
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "color": { "mode": "continuous-GrYlRd" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 60 },
+              { "color": "orange", "value": 80 },
+              { "color": "red", "value": 90 }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "row",
+      "title": "Network",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
+      "id": 30,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Network Traffic",
+      "description": "Network receive and transmit throughput for this instance. Transmit is shown as negative (below axis).",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 24 },
+      "id": 31,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum by (name) (rate(incus_network_receive_bytes_total{name=~\"$instance\"}[$__rate_interval]))",
+          "legendFormat": "Receive",
+          "refId": "A"
+        },
+        {
+          "expr": "-sum by (name) (rate(incus_network_transmit_bytes_total{name=~\"$instance\"}[$__rate_interval]))",
+          "legendFormat": "Transmit",
+          "refId": "B"
+        }
+      ],
       "fieldConfig": {
         "defaults": {
           "custom": {
@@ -435,16 +418,27 @@
           "color": { "mode": "palette-classic" },
           "thresholds": {
             "mode": "absolute",
-            "steps": [
+            "steps": [{ "color": "green", "value": null }]
-              { "color": "green", "value": null }
-            ]
           }
         },
-        "overrides": []
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "Receive" },
+            "properties": [
+              { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "Transmit" },
+            "properties": [
+              { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }
+            ]
+          }
+        ]
       },
       "options": {
         "tooltip": { "mode": "multi", "sort": "desc" },
-        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
       }
     },
     {
@@ -452,24 +446,26 @@
       "title": "Disk I/O",
       "collapsed": false,
       "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 },
-      "id": 13,
+      "id": 40,
       "panels": []
     },
     {
       "type": "timeseries",
-      "title": "Disk Read Rate by Instance",
+      "title": "Disk Read/Write Rate",
-      "description": "Disk read throughput per Incus instance.",
+      "description": "Disk read and write throughput for this instance.",
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 33 },
-      "id": 14,
+      "id": 41,
-      "datasource": {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
       "targets": [
         {
-          "expr": "sum by (name) (rate(incus_disk_read_bytes_total[$__rate_interval]))",
+          "expr": "sum by (name) (rate(incus_disk_read_bytes_total{name=~\"$instance\"}[$__rate_interval]))",
-          "legendFormat": "{{name}}",
+          "legendFormat": "Read",
           "refId": "A"
+        },
+        {
+          "expr": "sum by (name) (rate(incus_disk_written_bytes_total{name=~\"$instance\"}[$__rate_interval]))",
+          "legendFormat": "Write",
+          "refId": "B"
         }
       ],
       "fieldConfig": {
@@ -493,66 +489,14 @@
           "color": { "mode": "palette-classic" },
           "thresholds": {
             "mode": "absolute",
-            "steps": [
+            "steps": [{ "color": "green", "value": null }]
-              { "color": "green", "value": null }
-            ]
           }
         },
         "overrides": []
       },
       "options": {
         "tooltip": { "mode": "multi", "sort": "desc" },
-        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
-      }
-    },
-    {
-      "type": "timeseries",
-      "title": "Disk Write Rate by Instance",
-      "description": "Disk write throughput per Incus instance.",
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 },
-      "id": 15,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "targets": [
-        {
-          "expr": "sum by (name) (rate(incus_disk_written_bytes_total[$__rate_interval]))",
-          "legendFormat": "{{name}}",
-          "refId": "A"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "smooth",
-            "lineWidth": 1,
-            "fillOpacity": 15,
-            "gradientMode": "none",
-            "spanNulls": false,
-            "showPoints": "never",
-            "pointSize": 5,
-            "stacking": { "mode": "none", "group": "A" },
-            "axisPlacement": "auto",
-            "axisLabel": "",
-            "scaleDistribution": { "type": "linear" },
-            "thresholdsStyle": { "mode": "off" }
-          },
-          "unit": "Bps",
-          "color": { "mode": "palette-classic" },
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              { "color": "green", "value": null }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "options": {
-        "tooltip": { "mode": "multi", "sort": "desc" },
-        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
       }
     }
   ]

incusos/observability-dashboards/core/logs-explorer.json

@@ -0,0 +1,268 @@
+{
+  "uid": "incus-logs-explorer",
+  "title": "Logs Explorer",
+  "description": "Log volume over time, full log stream with search, error and warning rates by systemd unit. Datasource: Loki.",
+  "tags": ["incus", "logs", "loki"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "refresh": "30s",
+  "time": { "from": "now-1h", "to": "now" },
+  "fiscalYearStartMonth": 0,
+  "liveNow": false,
+  "editable": true,
+  "graphTooltip": 1,
+  "links": [
+    {
+      "asDropdown": true,
+      "icon": "external link",
+      "includeVars": true,
+      "keepTime": true,
+      "tags": ["incus"],
+      "targetBlank": false,
+      "title": "Incus Dashboards",
+      "tooltip": "Navigate to other Incus dashboards",
+      "type": "dashboards"
+    }
+  ],
+  "templating": {
+    "list": [
+      {
+        "type": "query",
+        "datasource": { "type": "loki", "uid": "${DS_LOKI}" },
+        "query": "label_values(job)",
+        "includeAll": true,
+        "multi": true,
+        "name": "job",
+        "label": "Job",
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "type": "query",
+        "datasource": { "type": "loki", "uid": "${DS_LOKI}" },
+        "query": "label_values(host)",
+        "includeAll": true,
+        "multi": true,
+        "name": "host",
+        "label": "Host",
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "type": "query",
+        "datasource": { "type": "loki", "uid": "${DS_LOKI}" },
+        "query": "label_values(unit)",
+        "includeAll": true,
+        "multi": true,
+        "name": "unit",
+        "label": "Unit",
+        "refresh": 2,
+        "sort": 1
+      },
+      {
+        "type": "textbox",
+        "name": "search",
+        "label": "Search",
+        "current": { "text": "", "value": "" }
+      }
+    ]
+  },
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": { "type": "grafana", "uid": "-- Grafana --" },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "panels": [
+    {
+      "type": "row",
+      "title": "Log Volume",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+      "id": 1,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Log Volume Over Time",
+      "description": "Total log line count over time, broken down by job. Shows the rate at which logs are being generated across all selected sources.",
+      "gridPos": { "h": 6, "w": 24, "x": 0, "y": 1 },
+      "id": 2,
+      "datasource": { "type": "loki", "uid": "${DS_LOKI}" },
+      "targets": [
+        {
+          "expr": "sum by (job) (count_over_time({job=~\"$job\", host=~\"$host\"} [$__auto]))",
+          "legendFormat": "{{job}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "bars",
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "fillOpacity": 80,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "normal", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "short",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "row",
+      "title": "Log Stream",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 },
+      "id": 3,
+      "panels": []
+    },
+    {
+      "type": "logs",
+      "title": "Logs",
+      "description": "Live log stream filtered by job, host, unit, and free-text search. Displays timestamps, labels, and full log messages with expand-on-click detail view.",
+      "gridPos": { "h": 16, "w": 24, "x": 0, "y": 8 },
+      "id": 4,
+      "datasource": { "type": "loki", "uid": "${DS_LOKI}" },
+      "targets": [
+        {
+          "expr": "{job=~\"$job\", host=~\"$host\", unit=~\"$unit\"} |= \"$search\"",
+          "refId": "A"
+        }
+      ],
+      "options": {
+        "showTime": true,
+        "showLabels": true,
+        "showCommonLabels": false,
+        "wrapLogMessage": true,
+        "prettifyLogMessage": false,
+        "enableLogDetails": true,
+        "sortOrder": "Descending",
+        "dedupStrategy": "none"
+      }
+    },
+    {
+      "type": "row",
+      "title": "Error & Warning Rates",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 },
+      "id": 5,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Error Rate by Unit",
+      "description": "Rate of log lines matching error, fail, or fatal (case-insensitive) over time, grouped by systemd unit. Spikes indicate services experiencing failures.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 },
+      "id": 6,
+      "datasource": { "type": "loki", "uid": "${DS_LOKI}" },
+      "targets": [
+        {
+          "expr": "sum by (unit) (count_over_time({job=~\"$job\", host=~\"$host\"} |~ \"(?i)error|fail|fatal\" [$__auto]))",
+          "legendFormat": "{{unit}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "short",
+          "color": { "mode": "fixed", "fixedColor": "red" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "red", "value": null }]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Warning Rate by Unit",
+      "description": "Rate of log lines matching warn (case-insensitive) over time, grouped by systemd unit. Helps identify services generating warnings that may precede errors.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 25 },
+      "id": 7,
+      "datasource": { "type": "loki", "uid": "${DS_LOKI}" },
+      "targets": [
+        {
+          "expr": "sum by (unit) (count_over_time({job=~\"$job\", host=~\"$host\"} |~ \"(?i)warn\" [$__auto]))",
+          "legendFormat": "{{unit}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "short",
+          "color": { "mode": "fixed", "fixedColor": "yellow" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "yellow", "value": null }]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum", "max", "lastNotNull"] }
+      }
+    }
+  ]
+}

incusos/observability-dashboards/core/network-deep-dive.json

@@ -0,0 +1,529 @@
+{
+  "uid": "incus-network-deep-dive",
+  "title": "Network Deep Dive",
+  "description": "Per-node and per-device network traffic, instance-level network metrics, receive/transmit errors, and packet drops.",
+  "tags": ["incus", "incusos", "network", "node-exporter"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "refresh": "30s",
+  "time": { "from": "now-1h", "to": "now" },
+  "fiscalYearStartMonth": 0,
+  "liveNow": false,
+  "editable": true,
+  "graphTooltip": 1,
+  "links": [
+    {
+      "asDropdown": true,
+      "icon": "external link",
+      "includeVars": true,
+      "keepTime": true,
+      "tags": ["incus"],
+      "targetBlank": false,
+      "title": "Incus Dashboards",
+      "tooltip": "Navigate to other Incus dashboards",
+      "type": "dashboards"
+    }
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {},
+        "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+        "definition": "label_values(node_network_receive_bytes_total, instance)",
+        "includeAll": false,
+        "multi": true,
+        "name": "node",
+        "query": "label_values(node_network_receive_bytes_total, instance)",
+        "refresh": 2,
+        "sort": 1,
+        "type": "query",
+        "label": "Node"
+      },
+      {
+        "current": {},
+        "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+        "definition": "label_values(node_network_receive_bytes_total{instance=~\"$node\"}, device)",
+        "includeAll": true,
+        "multi": true,
+        "name": "device",
+        "query": "label_values(node_network_receive_bytes_total{instance=~\"$node\"}, device)",
+        "refresh": 2,
+        "regex": "/^(?!lo$).*/",
+        "sort": 1,
+        "type": "query",
+        "label": "Device"
+      }
+    ]
+  },
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": { "type": "grafana", "uid": "-- Grafana --" },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "panels": [
+    {
+      "type": "row",
+      "title": "Cluster Summary",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+      "id": 1,
+      "panels": []
+    },
+    {
+      "type": "stat",
+      "title": "Total Ingress",
+      "description": "Aggregate inbound network throughput across all selected nodes and devices.",
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
+      "id": 2,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum(rate(node_network_receive_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval]))",
+          "legendFormat": "Ingress",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
+          "unit": "Bps",
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "orientation": "auto",
+        "textMode": "auto",
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto"
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Total Egress",
+      "description": "Aggregate outbound network throughput across all selected nodes and devices.",
+      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
+      "id": 3,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum(rate(node_network_transmit_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval]))",
+          "legendFormat": "Egress",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
+          "unit": "Bps",
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "orientation": "auto",
+        "textMode": "auto",
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto"
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Total Errors",
+      "description": "Combined receive and transmit error rate across all selected nodes. Any non-zero value warrants investigation.",
+      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
+      "id": 4,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum(rate(node_network_receive_errs_total{instance=~\"$node\"}[$__rate_interval])) + sum(rate(node_network_transmit_errs_total{instance=~\"$node\"}[$__rate_interval]))",
+          "legendFormat": "Errors",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 0.1 },
+              { "color": "red", "value": 1 }
+            ]
+          },
+          "unit": "pps",
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "orientation": "auto",
+        "textMode": "auto",
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto"
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Total Drops",
+      "description": "Combined receive and transmit packet drop rate across all selected nodes. Drops indicate buffer or queue exhaustion.",
+      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
+      "id": 5,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum(rate(node_network_receive_drop_total{instance=~\"$node\"}[$__rate_interval])) + sum(rate(node_network_transmit_drop_total{instance=~\"$node\"}[$__rate_interval]))",
+          "legendFormat": "Drops",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 0.1 },
+              { "color": "red", "value": 1 }
+            ]
+          },
+          "unit": "pps",
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "orientation": "auto",
+        "textMode": "auto",
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto"
+      }
+    },
+    {
+      "type": "row",
+      "title": "Per-Node Traffic",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
+      "id": 10,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Receive Rate by Node & Device",
+      "description": "Inbound network throughput broken down by node and network device. Identifies which interfaces are handling the most receive traffic.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
+      "id": 11,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "rate(node_network_receive_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
+          "legendFormat": "{{instance}} {{device}} rx",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "Bps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Transmit Rate by Node & Device",
+      "description": "Outbound network throughput broken down by node and network device. Identifies which interfaces are handling the most transmit traffic.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
+      "id": 12,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "rate(node_network_transmit_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
+          "legendFormat": "{{instance}} {{device}} tx",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "Bps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "row",
+      "title": "Instance Network (Incus metrics)",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
+      "id": 20,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Instance Receive Rate",
+      "description": "Inbound network throughput per Incus instance, aggregated across all instance network devices. Uses Incus-native metrics for container and VM visibility.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 },
+      "id": 21,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum by (name) (rate(incus_network_receive_bytes_total{instance=~\"$node\"}[$__rate_interval]))",
+          "legendFormat": "{{name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "Bps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Instance Transmit Rate",
+      "description": "Outbound network throughput per Incus instance, aggregated across all instance network devices. Uses Incus-native metrics for container and VM visibility.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 },
+      "id": 22,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum by (name) (rate(incus_network_transmit_bytes_total{instance=~\"$node\"}[$__rate_interval]))",
+          "legendFormat": "{{name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "Bps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "row",
+      "title": "Errors & Drops",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
+      "id": 30,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Receive/Transmit Errors",
+      "description": "Network interface error rates broken down by node and device. Receive errors may indicate CRC failures or framing issues; transmit errors may indicate collisions or driver problems.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
+      "id": 31,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "rate(node_network_receive_errs_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
+          "legendFormat": "{{instance}} {{device}} rx errors",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(node_network_transmit_errs_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
+          "legendFormat": "{{instance}} {{device}} tx errors",
+          "refId": "B"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "pps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Packet Drops",
+      "description": "Network interface packet drop rates broken down by node and device. Drops indicate kernel buffer or queue exhaustion and may cause retransmissions or connection failures.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
+      "id": 32,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "rate(node_network_receive_drop_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
+          "legendFormat": "{{instance}} {{device}} rx drops",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(node_network_transmit_drop_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])",
+          "legendFormat": "{{instance}} {{device}} tx drops",
+          "refId": "B"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "pps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    }
+  ]
+}

incusos/observability-dashboards/core/prometheus-health.json

@@ -0,0 +1,748 @@
+{
+  "uid": "incus-prometheus-health",
+  "title": "Prometheus Health",
+  "description": "Prometheus self-monitoring: targets up/down, scrape duration, samples ingested, TSDB size, head series, WAL stats.",
+  "tags": ["incus", "prometheus", "operations"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "refresh": "30s",
+  "time": { "from": "now-1h", "to": "now" },
+  "fiscalYearStartMonth": 0,
+  "liveNow": false,
+  "editable": true,
+  "graphTooltip": 1,
+  "links": [
+    {
+      "asDropdown": true,
+      "icon": "external link",
+      "includeVars": true,
+      "keepTime": true,
+      "tags": ["incus"],
+      "targetBlank": false,
+      "title": "Incus Dashboards",
+      "tooltip": "Navigate to other Incus dashboards",
+      "type": "dashboards"
+    }
+  ],
+  "templating": {
+    "list": []
+  },
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": { "type": "grafana", "uid": "-- Grafana --" },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "panels": [
+    {
+      "type": "row",
+      "title": "Targets",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+      "id": 1,
+      "panels": []
+    },
+    {
+      "type": "stat",
+      "title": "Targets Up",
+      "description": "Number of scrape targets currently reporting as healthy (up == 1).",
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
+      "id": 2,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "count(up == 1)",
+          "legendFormat": "Up",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "none",
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "orientation": "auto",
+        "textMode": "auto",
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto"
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Targets Down",
+      "description": "Number of scrape targets currently unreachable or failing (up == 0). Any value above 0 requires investigation.",
+      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
+      "id": 3,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "count(up == 0)",
+          "legendFormat": "Down",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "red", "value": 1 }
+            ]
+          },
+          "unit": "none",
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "orientation": "auto",
+        "textMode": "auto",
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto"
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Avg Scrape Duration",
+      "description": "Average time Prometheus takes to scrape all targets. High values may indicate slow or overloaded exporters.",
+      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
+      "id": 4,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "avg(scrape_duration_seconds)",
+          "legendFormat": "Avg Duration",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 1 },
+              { "color": "red", "value": 5 }
+            ]
+          },
+          "unit": "s",
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "orientation": "auto",
+        "textMode": "auto",
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto"
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Samples Ingested/s",
+      "description": "Total rate of samples being scraped and ingested across all targets per second.",
+      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
+      "id": 5,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum(rate(scrape_samples_scraped[$__rate_interval]))",
+          "legendFormat": "Samples/s",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "short",
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "orientation": "auto",
+        "textMode": "auto",
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto"
+      }
+    },
+    {
+      "type": "table",
+      "title": "Target Health",
+      "description": "Per-target health status showing each scrape target's current state. UP (1) means healthy, DOWN (0) means unreachable or failing.",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 5 },
+      "id": 6,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "up",
+          "legendFormat": "",
+          "refId": "A",
+          "format": "table",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto",
+            "inspect": false,
+            "filterable": true
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "instance" },
+            "properties": [
+              { "id": "displayName", "value": "Instance" },
+              { "id": "custom.width", "value": 300 }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "job" },
+            "properties": [
+              { "id": "displayName", "value": "Job" },
+              { "id": "custom.width", "value": 200 }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "Value" },
+            "properties": [
+              { "id": "displayName", "value": "Status" },
+              { "id": "custom.displayMode", "value": "color-background" },
+              {
+                "id": "mappings",
+                "value": [
+                  {
+                    "type": "value",
+                    "options": {
+                      "1": { "text": "UP", "color": "green" },
+                      "0": { "text": "DOWN", "color": "red" }
+                    }
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      "options": {
+        "showHeader": true,
+        "sortBy": [{ "displayName": "Job", "desc": false }],
+        "footer": { "show": false, "reducer": ["sum"], "fields": "" }
+      },
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": { "Time": true, "__name__": true },
+            "indexByName": {
+              "instance": 0,
+              "job": 1,
+              "Value": 2
+            },
+            "renameByName": {}
+          }
+        }
+      ]
+    },
+    {
+      "type": "row",
+      "title": "Scrape Performance",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 },
+      "id": 7,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Scrape Duration by Job",
+      "description": "How long each scrape takes per job. Spikes indicate slow exporters or network issues between Prometheus and targets.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 },
+      "id": 8,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "scrape_duration_seconds",
+          "legendFormat": "{{job}} - {{instance}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "s",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Samples Scraped by Job",
+      "description": "Rate of samples scraped per job. Sudden drops may indicate target failures; spikes may indicate new metrics being exposed.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 },
+      "id": 9,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "rate(scrape_samples_scraped[$__rate_interval])",
+          "legendFormat": "{{job}} - {{instance}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "short",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "row",
+      "title": "TSDB",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 },
+      "id": 10,
+      "panels": []
+    },
+    {
+      "type": "stat",
+      "title": "TSDB Size",
+      "description": "Current on-disk size of the Prometheus TSDB storage directory including all blocks and WAL.",
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 23 },
+      "id": 11,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "prometheus_tsdb_storage_size_bytes",
+          "legendFormat": "TSDB Size",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 1073741824 },
+              { "color": "red", "value": 5368709120 }
+            ]
+          },
+          "unit": "bytes",
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "orientation": "auto",
+        "textMode": "auto",
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto"
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Head Series",
+      "description": "Number of active time series in the TSDB head block. High cardinality here impacts query performance and memory usage.",
+      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 23 },
+      "id": 12,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "prometheus_tsdb_head_series",
+          "legendFormat": "Head Series",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 100000 },
+              { "color": "red", "value": 500000 }
+            ]
+          },
+          "unit": "short",
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "orientation": "auto",
+        "textMode": "auto",
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto"
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Head Chunks",
+      "description": "Number of chunks in the TSDB head block. Growth correlates with active series count and scrape frequency.",
+      "gridPos": { "h": 8, "w": 6, "x": 12, "y": 23 },
+      "id": 13,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "prometheus_tsdb_head_chunks",
+          "legendFormat": "Head Chunks",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "short",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "WAL Size",
+      "description": "Size of the Prometheus write-ahead log (WAL). The WAL buffers recent writes before compaction into blocks.",
+      "gridPos": { "h": 8, "w": 6, "x": 18, "y": 23 },
+      "id": 14,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "prometheus_tsdb_wal_storage_size_bytes",
+          "legendFormat": "WAL Size",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "bytes",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "row",
+      "title": "TSDB Trends",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 },
+      "id": 15,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "TSDB Size Over Time",
+      "description": "Prometheus TSDB total storage size trend. Useful for capacity planning and detecting unexpected growth.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 },
+      "id": 16,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "prometheus_tsdb_storage_size_bytes",
+          "legendFormat": "TSDB Size",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "bytes",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Samples Ingested Trend",
+      "description": "Rate of new samples appended to the TSDB head block. Correlates with scrape frequency and number of active series.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 },
+      "id": 17,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "rate(prometheus_tsdb_head_samples_appended_total[$__rate_interval])",
+          "legendFormat": "Samples/s",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "short",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Compactions",
+      "description": "Rate of TSDB compaction operations. Compactions merge head block data into persistent blocks on disk for efficient storage.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 },
+      "id": 18,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "rate(prometheus_tsdb_compactions_total[$__rate_interval])",
+          "legendFormat": "Compactions/s",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "short",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Head Series Count",
+      "description": "Number of active time series in the TSDB head block over time. Trend helps identify cardinality growth or label explosion.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 },
+      "id": 19,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "prometheus_tsdb_head_series",
+          "legendFormat": "Head Series",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "spanNulls": false,
+            "showPoints": "never",
+            "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto",
+            "axisLabel": "",
+            "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "short",
+          "color": { "mode": "palette-classic" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    }
+  ]
+}

incusos/observability-dashboards/core/storage-filesystem.json

@@ -0,0 +1,382 @@
+{
+  "uid": "incus-storage-filesystem",
+  "title": "Storage & Filesystem",
+  "description": "Filesystem usage, available space trends, disk throughput, I/O latency, disk utilization, and inode usage per node.",
+  "tags": ["incus", "incusos", "storage", "node-exporter"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "refresh": "30s",
+  "time": { "from": "now-1h", "to": "now" },
+  "fiscalYearStartMonth": 0,
+  "liveNow": false,
+  "editable": true,
+  "graphTooltip": 1,
+  "links": [
+    {
+      "asDropdown": true, "icon": "external link", "includeVars": true,
+      "keepTime": true, "tags": ["incus"], "targetBlank": false,
+      "title": "Incus Dashboards", "type": "dashboards"
+    }
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {},
+        "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+        "definition": "label_values(node_filesystem_size_bytes, instance)",
+        "includeAll": false,
+        "multi": true,
+        "name": "node",
+        "query": "label_values(node_filesystem_size_bytes, instance)",
+        "refresh": 2,
+        "sort": 1,
+        "type": "query",
+        "label": "Node"
+      },
+      {
+        "current": {},
+        "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+        "definition": "label_values(node_filesystem_size_bytes{instance=~\"$node\"}, mountpoint)",
+        "includeAll": true,
+        "multi": true,
+        "name": "mountpoint",
+        "query": "label_values(node_filesystem_size_bytes{instance=~\"$node\"}, mountpoint)",
+        "refresh": 2,
+        "sort": 1,
+        "type": "query",
+        "label": "Mountpoint"
+      }
+    ]
+  },
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": { "type": "grafana", "uid": "-- Grafana --" },
+        "enable": true, "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts", "type": "dashboard"
+      }
+    ]
+  },
+  "panels": [
+    {
+      "type": "row", "title": "Filesystem Overview",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+      "id": 1, "panels": []
+    },
+    {
+      "type": "table",
+      "title": "Filesystem Usage Table",
+      "description": "Current filesystem size, available space, and usage percentage per node and mountpoint. High usage is highlighted with color thresholds.",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 1 },
+      "id": 2,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "node_filesystem_size_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}",
+          "legendFormat": "", "refId": "A", "format": "table", "instant": true
+        },
+        {
+          "expr": "node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}",
+          "legendFormat": "", "refId": "B", "format": "table", "instant": true
+        },
+        {
+          "expr": "(1 - node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}) * 100",
+          "legendFormat": "", "refId": "C", "format": "table", "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": { "align": "auto", "displayMode": "auto", "inspect": false, "filterable": true },
+          "color": { "mode": "thresholds" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": [
+          { "matcher": { "id": "byName", "options": "instance" }, "properties": [{ "id": "displayName", "value": "Node" }] },
+          { "matcher": { "id": "byName", "options": "mountpoint" }, "properties": [{ "id": "displayName", "value": "Mountpoint" }] },
+          { "matcher": { "id": "byName", "options": "fstype" }, "properties": [{ "id": "displayName", "value": "FS Type" }] },
+          { "matcher": { "id": "byName", "options": "Value #A" }, "properties": [{ "id": "displayName", "value": "Size" }, { "id": "unit", "value": "bytes" }] },
+          { "matcher": { "id": "byName", "options": "Value #B" }, "properties": [{ "id": "displayName", "value": "Available" }, { "id": "unit", "value": "bytes" }] },
+          {
+            "matcher": { "id": "byName", "options": "Value #C" },
+            "properties": [
+              { "id": "displayName", "value": "Used %" },
+              { "id": "unit", "value": "percent" },
+              { "id": "custom.displayMode", "value": "color-background" },
+              { "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "orange", "value": 85 }, { "color": "red", "value": 95 }] } }
+            ]
+          }
+        ]
+      },
+      "options": { "showHeader": true, "sortBy": [], "footer": { "show": false, "reducer": ["sum"], "fields": "" } },
+      "transformations": [
+        { "id": "merge", "options": {} },
+        { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "job": true, "device": true }, "renameByName": {} } }
+      ]
+    },
+    {
+      "type": "row", "title": "Space Trends",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
+      "id": 3, "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Available Space Over Time",
+      "description": "Tracks available filesystem space over time per node and mountpoint. Use this to identify gradual space consumption trends before they become critical.",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 10 },
+      "id": 4,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}",
+          "legendFormat": "{{instance}} {{mountpoint}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2,
+            "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+            "showPoints": "never", "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "bytes",
+          "color": { "mode": "palette-classic" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "row", "title": "Disk Throughput",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 },
+      "id": 5, "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Disk Read Throughput",
+      "description": "Disk read throughput in bytes per second, broken down by instance and device. Identifies which disks are handling the most read traffic.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 },
+      "id": 6,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum by (instance, device) (rate(node_disk_read_bytes_total{instance=~\"$node\"}[$__rate_interval]))",
+          "legendFormat": "{{instance}} {{device}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+            "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+            "showPoints": "never", "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "Bps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Disk Write Throughput",
+      "description": "Disk write throughput in bytes per second, broken down by instance and device. Identifies which disks are handling the most write traffic.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 },
+      "id": 7,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum by (instance, device) (rate(node_disk_written_bytes_total{instance=~\"$node\"}[$__rate_interval]))",
+          "legendFormat": "{{instance}} {{device}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+            "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+            "showPoints": "never", "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "Bps",
+          "color": { "mode": "palette-classic" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "row", "title": "I/O Performance",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 },
+      "id": 8, "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Average Read Latency",
+      "description": "Average time per completed read operation. Calculated as total read time divided by completed reads. High values indicate slow disk response.",
+      "gridPos": { "h": 8, "w": 8, "x": 0, "y": 28 },
+      "id": 9,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "rate(node_disk_read_time_seconds_total{instance=~\"$node\"}[$__rate_interval]) / rate(node_disk_reads_completed_total{instance=~\"$node\"}[$__rate_interval])",
+          "legendFormat": "{{instance}} {{device}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+            "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+            "showPoints": "never", "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "s",
+          "color": { "mode": "palette-classic" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Average Write Latency",
+      "description": "Average time per completed write operation. Calculated as total write time divided by completed writes. High values indicate slow disk response or write saturation.",
+      "gridPos": { "h": 8, "w": 8, "x": 8, "y": 28 },
+      "id": 10,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "rate(node_disk_write_time_seconds_total{instance=~\"$node\"}[$__rate_interval]) / rate(node_disk_writes_completed_total{instance=~\"$node\"}[$__rate_interval])",
+          "legendFormat": "{{instance}} {{device}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1,
+            "fillOpacity": 15, "gradientMode": "none", "spanNulls": false,
+            "showPoints": "never", "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "unit": "s",
+          "color": { "mode": "palette-classic" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Disk Utilization %",
+      "description": "Percentage of time the disk is busy processing I/O requests. Values near 100% indicate the disk is saturated and may be a bottleneck.",
+      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 28 },
+      "id": 11,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "sum by (instance) (rate(node_disk_io_time_seconds_total{instance=~\"$node\"}[$__rate_interval])) * 100",
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2,
+            "fillOpacity": 25, "gradientMode": "scheme", "spanNulls": false,
+            "showPoints": "never", "pointSize": 5,
+            "stacking": { "mode": "none", "group": "A" },
+            "axisPlacement": "auto", "scaleDistribution": { "type": "linear" },
+            "thresholdsStyle": { "mode": "line" }
+          },
+          "unit": "percent", "min": 0, "max": 100,
+          "color": { "mode": "continuous-GrYlRd" },
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 80 }] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "tooltip": { "mode": "multi", "sort": "desc" },
+        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
+      }
+    },
+    {
+      "type": "row", "title": "Inodes",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 },
+      "id": 12, "panels": []
+    },
+    {
+      "type": "bargauge",
+      "title": "Inode Usage %",
+      "description": "Percentage of used inodes per filesystem. Running out of inodes prevents creating new files even when disk space is available. Monitor for filesystems with many small files.",
+      "gridPos": { "h": 6, "w": 24, "x": 0, "y": 37 },
+      "id": 13,
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "targets": [
+        {
+          "expr": "(1 - node_filesystem_files_free{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"} / node_filesystem_files{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}) * 100",
+          "legendFormat": "{{instance}} {{mountpoint}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] },
+          "unit": "percent", "min": 0, "max": 100,
+          "color": { "mode": "thresholds" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "orientation": "horizontal", "displayMode": "gradient", "showUnfilled": true,
+        "minVizWidth": 8, "minVizHeight": 16, "valueMode": "color"
+      }
+    }
+  ]
+}

incusos/observability-dashboards/dashboard.yaml

@@ -0,0 +1,97 @@
+# Dashboard manifest for the Incus observability stack
+#
+# Each entry maps a dashboard JSON file to its Grafana folder, UID, and metadata.
+# Used by manage-dashboards for install, export, list, and validate operations.
+
+dashboards:
+  # --- Core dashboards (ship with product) ---
+
+  - file: core/cluster-overview.json
+    uid: incus-cluster-overview
+    title: Incus Cluster Overview
+    folder: Incus Cluster
+    category: core
+    version: "2"
+    description: >-
+      Overview of all Incus instances: CPU, memory, network, disk I/O,
+      and instance status table with drill-down links.
+
+  - file: core/host-resources.json
+    uid: incus-host-resources
+    title: Host Resources (IncusOS Nodes)
+    folder: Incus Infrastructure
+    category: core
+    version: "2"
+    description: >-
+      Physical host monitoring: CPU mode breakdown, load averages,
+      memory breakdown, filesystem usage, disk IOPS, network errors.
+
+  - file: core/haproxy-traffic.json
+    uid: incus-haproxy-traffic
+    title: HAProxy Traffic
+    folder: Incus Services
+    category: core
+    version: "2"
+    description: >-
+      HAProxy load balancer monitoring: request rates, response codes,
+      backend health, server status timeline, connection errors.
+
+  - file: core/instance-deep-dive.json
+    uid: incus-instance-deep-dive
+    title: Instance Deep Dive
+    folder: Incus Cluster
+    category: core
+    version: "1"
+    description: >-
+      Single-instance drill-down: CPU, memory, network, and disk
+      metrics with detailed timeseries and utilization gauges.
+
+  - file: core/storage-filesystem.json
+    uid: incus-storage-filesystem
+    title: Storage & Filesystem
+    folder: Incus Infrastructure
+    category: core
+    version: "1"
+    description: >-
+      Filesystem usage, available space trends, disk throughput,
+      I/O latency, disk utilization, and inode usage per node.
+
+  - file: core/network-deep-dive.json
+    uid: incus-network-deep-dive
+    title: Network Deep Dive
+    folder: Incus Infrastructure
+    category: core
+    version: "1"
+    description: >-
+      Per-node and per-device network traffic, instance-level network
+      metrics, receive/transmit errors, and packet drops.
+
+  - file: core/logs-explorer.json
+    uid: incus-logs-explorer
+    title: Logs Explorer
+    folder: Incus Operations
+    category: core
+    version: "1"
+    description: >-
+      Log volume over time, full log stream with search, error
+      and warning rates by systemd unit. Datasource: Loki.
+
+  - file: core/prometheus-health.json
+    uid: incus-prometheus-health
+    title: Prometheus Health
+    folder: Incus Operations
+    category: core
+    version: "1"
+    description: >-
+      Prometheus self-monitoring: targets up/down, scrape duration,
+      samples ingested, TSDB size, head series, WAL stats.
+
+  - file: core/capacity-planning.json
+    uid: incus-capacity-planning
+    title: Capacity Planning
+    folder: Incus Operations
+    category: core
+    version: "1"
+    description: >-
+      CPU/memory headroom per node, resource trends, disk space
+      forecast with predict_linear, instance count trends.

incusos/observability-dashboards/haproxy-traffic.json

@@ -1,518 +0,0 @@
-{
-  "uid": "incus-haproxy-traffic",
-  "title": "HAProxy Traffic",
-  "description": "HAProxy load balancer monitoring: request rates, backend health, traffic volume, and sessions.",
-  "tags": ["haproxy", "loadbalancer", "incus"],
-  "timezone": "browser",
-  "schemaVersion": 39,
-  "version": 1,
-  "refresh": "30s",
-  "time": {
-    "from": "now-1h",
-    "to": "now"
-  },
-  "fiscalYearStartMonth": 0,
-  "liveNow": false,
-  "editable": true,
-  "graphTooltip": 1,
-  "templating": {
-    "list": []
-  },
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "panels": [
-    {
-      "type": "row",
-      "title": "Overview",
-      "collapsed": false,
-      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
-      "id": 1,
-      "panels": []
-    },
-    {
-      "type": "stat",
-      "title": "Active Backend Servers",
-      "description": "Total number of active servers across all HAProxy backends.",
-      "gridPos": { "h": 4, "w": 8, "x": 0, "y": 1 },
-      "id": 2,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "targets": [
-        {
-          "expr": "sum(haproxy_backend_active_servers)",
-          "legendFormat": "Active Servers",
-          "refId": "A"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              { "color": "red", "value": null },
-              { "color": "yellow", "value": 1 },
-              { "color": "green", "value": 2 }
-            ]
-          },
-          "unit": "none",
-          "color": { "mode": "thresholds" }
-        },
-        "overrides": []
-      },
-      "options": {
-        "reduceOptions": {
-          "calcs": ["lastNotNull"],
-          "fields": "",
-          "values": false
-        },
-        "orientation": "auto",
-        "textMode": "auto",
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto"
-      }
-    },
-    {
-      "type": "stat",
-      "title": "Current Sessions",
-      "description": "Total current sessions across all HAProxy backends.",
-      "gridPos": { "h": 4, "w": 8, "x": 8, "y": 1 },
-      "id": 3,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "targets": [
-        {
-          "expr": "sum(haproxy_backend_current_sessions)",
-          "legendFormat": "Sessions",
-          "refId": "A"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              { "color": "green", "value": null },
-              { "color": "yellow", "value": 100 },
-              { "color": "red", "value": 500 }
-            ]
-          },
-          "unit": "none",
-          "color": { "mode": "thresholds" }
-        },
-        "overrides": []
-      },
-      "options": {
-        "reduceOptions": {
-          "calcs": ["lastNotNull"],
-          "fields": "",
-          "values": false
-        },
-        "orientation": "auto",
-        "textMode": "auto",
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto"
-      }
-    },
-    {
-      "type": "stat",
-      "title": "Total HTTP Requests",
-      "description": "Total HTTP requests across all HAProxy frontends (counter).",
-      "gridPos": { "h": 4, "w": 8, "x": 16, "y": 1 },
-      "id": 4,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "targets": [
-        {
-          "expr": "sum(rate(haproxy_frontend_http_requests_total[$__rate_interval]))",
-          "legendFormat": "Requests/s",
-          "refId": "A"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              { "color": "green", "value": null }
-            ]
-          },
-          "unit": "reqps",
-          "color": { "mode": "thresholds" }
-        },
-        "overrides": []
-      },
-      "options": {
-        "reduceOptions": {
-          "calcs": ["lastNotNull"],
-          "fields": "",
-          "values": false
-        },
-        "orientation": "auto",
-        "textMode": "auto",
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto"
-      }
-    },
-    {
-      "type": "row",
-      "title": "HTTP Requests",
-      "collapsed": false,
-      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
-      "id": 5,
-      "panels": []
-    },
-    {
-      "type": "timeseries",
-      "title": "HTTP Request Rate by Frontend",
-      "description": "Rate of HTTP requests per second, broken down by HAProxy frontend/proxy.",
-      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 },
-      "id": 6,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "targets": [
-        {
-          "expr": "sum by (proxy) (rate(haproxy_frontend_http_requests_total[$__rate_interval]))",
-          "legendFormat": "{{proxy}}",
-          "refId": "A"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "smooth",
-            "lineWidth": 1,
-            "fillOpacity": 20,
-            "gradientMode": "none",
-            "spanNulls": false,
-            "showPoints": "never",
-            "pointSize": 5,
-            "stacking": { "mode": "none", "group": "A" },
-            "axisPlacement": "auto",
-            "axisLabel": "",
-            "scaleDistribution": { "type": "linear" },
-            "thresholdsStyle": { "mode": "off" }
-          },
-          "unit": "reqps",
-          "color": { "mode": "palette-classic" },
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              { "color": "green", "value": null }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "options": {
-        "tooltip": { "mode": "multi", "sort": "desc" },
-        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
-      }
-    },
-    {
-      "type": "row",
-      "title": "Backend Health",
-      "collapsed": false,
-      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
-      "id": 7,
-      "panels": []
-    },
-    {
-      "type": "table",
-      "title": "Backend Status",
-      "description": "Current status, active server count, and average response time for each HAProxy backend.",
-      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 15 },
-      "id": 8,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "targets": [
-        {
-          "expr": "haproxy_backend_status",
-          "legendFormat": "",
-          "refId": "A",
-          "format": "table",
-          "instant": true
-        },
-        {
-          "expr": "haproxy_backend_active_servers",
-          "legendFormat": "",
-          "refId": "B",
-          "format": "table",
-          "instant": true
-        },
-        {
-          "expr": "haproxy_server_response_time_average_seconds",
-          "legendFormat": "",
-          "refId": "C",
-          "format": "table",
-          "instant": true
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "custom": {
-            "align": "auto",
-            "displayMode": "auto",
-            "inspect": false,
-            "filterable": true
-          },
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              { "color": "green", "value": null }
-            ]
-          },
-          "color": { "mode": "thresholds" }
-        },
-        "overrides": [
-          {
-            "matcher": { "id": "byName", "options": "Value #A" },
-            "properties": [
-              { "id": "displayName", "value": "Status" }
-            ]
-          },
-          {
-            "matcher": { "id": "byName", "options": "Value #B" },
-            "properties": [
-              { "id": "displayName", "value": "Active Servers" }
-            ]
-          },
-          {
-            "matcher": { "id": "byName", "options": "Value #C" },
-            "properties": [
-              { "id": "displayName", "value": "Avg Response Time" },
-              { "id": "unit", "value": "s" }
-            ]
-          },
-          {
-            "matcher": { "id": "byName", "options": "proxy" },
-            "properties": [
-              { "id": "displayName", "value": "Backend" }
-            ]
-          }
-        ]
-      },
-      "options": {
-        "showHeader": true,
-        "sortBy": [],
-        "footer": {
-          "show": false,
-          "reducer": ["sum"],
-          "fields": ""
-        }
-      },
-      "transformations": [
-        {
-          "id": "merge",
-          "options": {}
-        },
-        {
-          "id": "organize",
-          "options": {
-            "excludeByName": {
-              "Time": true,
-              "__name__": true,
-              "instance": false,
-              "job": true
-            },
-            "indexByName": {},
-            "renameByName": {}
-          }
-        }
-      ]
-    },
-    {
-      "type": "row",
-      "title": "Traffic",
-      "collapsed": false,
-      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
-      "id": 9,
-      "panels": []
-    },
-    {
-      "type": "timeseries",
-      "title": "Frontend Bytes In",
-      "description": "Inbound traffic rate per HAProxy frontend.",
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
-      "id": 10,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "targets": [
-        {
-          "expr": "sum by (proxy) (rate(haproxy_frontend_bytes_in_total[$__rate_interval]))",
-          "legendFormat": "{{proxy}} (in)",
-          "refId": "A"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "smooth",
-            "lineWidth": 1,
-            "fillOpacity": 20,
-            "gradientMode": "none",
-            "spanNulls": false,
-            "showPoints": "never",
-            "pointSize": 5,
-            "stacking": { "mode": "none", "group": "A" },
-            "axisPlacement": "auto",
-            "axisLabel": "",
-            "scaleDistribution": { "type": "linear" },
-            "thresholdsStyle": { "mode": "off" }
-          },
-          "unit": "Bps",
-          "color": { "mode": "palette-classic" },
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              { "color": "green", "value": null }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "options": {
-        "tooltip": { "mode": "multi", "sort": "desc" },
-        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
-      }
-    },
-    {
-      "type": "timeseries",
-      "title": "Frontend Bytes Out",
-      "description": "Outbound traffic rate per HAProxy frontend.",
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
-      "id": 11,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "targets": [
-        {
-          "expr": "sum by (proxy) (rate(haproxy_frontend_bytes_out_total[$__rate_interval]))",
-          "legendFormat": "{{proxy}} (out)",
-          "refId": "A"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "smooth",
-            "lineWidth": 1,
-            "fillOpacity": 20,
-            "gradientMode": "none",
-            "spanNulls": false,
-            "showPoints": "never",
-            "pointSize": 5,
-            "stacking": { "mode": "none", "group": "A" },
-            "axisPlacement": "auto",
-            "axisLabel": "",
-            "scaleDistribution": { "type": "linear" },
-            "thresholdsStyle": { "mode": "off" }
-          },
-          "unit": "Bps",
-          "color": { "mode": "palette-classic" },
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              { "color": "green", "value": null }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "options": {
-        "tooltip": { "mode": "multi", "sort": "desc" },
-        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }
-      }
-    },
-    {
-      "type": "row",
-      "title": "Sessions",
-      "collapsed": false,
-      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 },
-      "id": 12,
-      "panels": []
-    },
-    {
-      "type": "timeseries",
-      "title": "Current Sessions by Proxy",
-      "description": "Current active sessions for each HAProxy backend/proxy over time.",
-      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 33 },
-      "id": 13,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "targets": [
-        {
-          "expr": "haproxy_backend_current_sessions",
-          "legendFormat": "{{proxy}}",
-          "refId": "A"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "smooth",
-            "lineWidth": 1,
-            "fillOpacity": 20,
-            "gradientMode": "none",
-            "spanNulls": false,
-            "showPoints": "never",
-            "pointSize": 5,
-            "stacking": { "mode": "none", "group": "A" },
-            "axisPlacement": "auto",
-            "axisLabel": "",
-            "scaleDistribution": { "type": "linear" },
-            "thresholdsStyle": { "mode": "off" }
-          },
-          "unit": "none",
-          "color": { "mode": "palette-classic" },
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              { "color": "green", "value": null }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "options": {
-        "tooltip": { "mode": "multi", "sort": "desc" },
-        "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }
-      }
-    }
-  ]
-}

notes/observability-guide.md

@@ -122,40 +122,106 @@ datasource proxy.
 
 ## Dashboards
 
-Three dashboards are provisioned automatically during deployment.
+Nine dashboards are provisioned automatically during deployment, organized
+into four Grafana folders. All dashboards support template variables for
+filtering and include cross-dashboard navigation links.
 
-### Incus Cluster Overview
+Use `manage-dashboards` to install, export, validate, or list dashboards:
 
-Visualizes metrics scraped from the Incus `/1.0/metrics` endpoint on each
+```bash
-cluster node.
+incusos/manage-dashboards --install     # push all dashboards to Grafana
+incusos/manage-dashboards --list        # show install status
+incusos/manage-dashboards --validate    # check JSON quality
+incusos/manage-dashboards --export      # export from Grafana to clean JSON
+incusos/manage-dashboards --status      # compare local vs installed
+```
 
-- Instance CPU usage (per-instance, per-node)
+### Incus Cluster — Incus Cluster Overview
-- Memory usage and allocation
-- Network I/O (bytes in/out per interface)
-- Disk I/O (reads/writes, latency)
-- Instance count and state
 
-### HAProxy Traffic
+Overview of all instances with instance/node template variables.
 
-Visualizes metrics from the HAProxy stats endpoint (`:8404/metrics`).
+- Instance count, total CPUs, total memory, cluster memory utilization gauge
+- Instance status table with drill-down links to Instance Deep Dive
+- Per-instance CPU usage, memory RSS, memory usage % bar gauge
+- Network receive/transmit rates, disk read/write rates
 
-- Request rate (frontend and backend)
+### Incus Cluster — Instance Deep Dive
-- Backend health status (UP/DOWN per server)
-- Active sessions and session rate
-- Traffic volume (bytes in/out)
-- HTTP response codes (2xx, 4xx, 5xx)
-- Connection errors and retries
 
-### Host Resources
+Single-instance drill-down selected via template variable.
 
-Visualizes metrics from node_exporter on each cluster node.
+- Uptime, effective CPUs, total memory, memory utilization gauge
+- CPU usage over time with gradient color scheme
+- Memory RSS and memory utilization % over time
+- Network traffic (receive above axis, transmit below on same panel)
+- Disk read/write rates
 
-- CPU utilization (%)
+### Incus Services — HAProxy Traffic
-- Memory utilization (%)
+
-- Disk usage and I/O
+HAProxy monitoring with proxy/server template variables.
-- Network throughput per interface
+
-- System load averages
+- Active servers, current sessions, HTTP request rate
-- Filesystem free space
+- HTTP 4xx/5xx rate stat panels with warning thresholds
+- Response code distribution (stacked, color-coded by class)
+- Backend status table and server status state timeline
+- Frontend traffic, sessions by proxy, response times
+- Queue time and connection errors
+
+### Incus Infrastructure — Host Resources (IncusOS Nodes)
+
+Physical host monitoring with node template variable.
+
+- System information table (kernel, uptime, RAM, CPU count)
+- CPU usage % and CPU mode breakdown (user/system/iowait/idle stacked)
+- Load average (1m/5m/15m) with CPU count threshold line
+- Memory usage % and memory breakdown (used/buffers/cached/available)
+- Filesystem usage % bar gauge, IOPS, disk I/O time
+- Network traffic by device, network errors and drops
+
+### Incus Infrastructure — Storage & Filesystem
+
+Filesystem and disk I/O with node/mountpoint template variables.
+
+- Filesystem usage table with color-coded Used % column
+- Available space trend over time
+- Disk read/write throughput by device
+- Average read/write latency, disk utilization %
+- Inode usage % bar gauge
+
+### Incus Infrastructure — Network Deep Dive
+
+Network metrics with node/device template variables.
+
+- Cluster ingress/egress/errors/drops summary stats
+- Per-node per-device receive/transmit rates
+- Instance-level network traffic (Incus `incus_network_*` metrics)
+- Receive/transmit errors and packet drops
+
+### Incus Operations — Logs Explorer
+
+Log exploration powered by Loki with job/host/unit/search variables.
+
+- Log volume over time (stacked by job)
+- Full log stream with search filtering
+- Error rate and warning rate by systemd unit
+
+### Incus Operations — Prometheus Health
+
+Prometheus self-monitoring (no template variables).
+
+- Targets up/down, average scrape duration, samples ingested/s
+- Target health table with UP/DOWN status
+- Scrape duration and samples scraped by job
+- TSDB size, head series, head chunks, WAL size
+- TSDB size trend, samples ingested trend, compactions
+
+### Incus Operations — Capacity Planning
+
+Resource planning with 7-day default time range (no template variables).
+
+- CPU and memory headroom per node (inverted bar gauges)
+- CPU and memory usage trends (7-day)
+- Disk space forecast with 30-day `predict_linear` projection
+- Instance count trend, memory allocation vs usage per instance
 
 ## ACL Configuration
 
@@ -208,11 +274,18 @@ manually or restart the container.
 | node-exp-02 | Alpine | 128 MiB | — | oc-node-02 |
 | node-exp-03 | Alpine | 128 MiB | — | oc-node-03 |
 
+**Dummy workloads** (optional, deployed via `--workloads`):
+
+| Container | Image | RAM | Placement | Purpose |
+|-----------|-------|-----|-----------|---------|
+| workload-web | Alpine | 128 MiB | oc-node-01 | nginx + CPU/disk/network crons |
+| workload-api | Alpine | 128 MiB | oc-node-03 | Python HTTP + memory sawtooth |
+
 **Totals:**
-- RAM: ~2.4 GiB (2 GiB + 3 x 128 MiB)
+- RAM: ~2.4 GiB core (2 GiB + 3 x 128 MiB), +256 MiB with workloads
 - Disk: 20 GiB (only the monitoring container needs significant storage)
 - OVN forward IPs: 1 (192.168.103.201)
-- OVN network IPs: 4 (10.10.10.70-73)
+- OVN network IPs: 4 core (10.10.10.70-73), +2 with workloads (.80, .81)
 
 ## Management
 
@@ -229,14 +302,30 @@ installs and configures Prometheus, Grafana, Loki, and Promtail,
 provisions dashboards, sets up the OVN network forward, and configures
 ACLs.
 
+### Deploy dummy workloads
+
+```bash
+incusos/deploy-observability --workloads
+```
+
+Deploys two lightweight containers that generate varied metric patterns:
+
+- **workload-web** (10.10.10.80, oc-node-01): nginx serving static
+  content, cron jobs generating HTTP traffic (every minute), CPU spikes
+  via gzip (every 5 min), and disk I/O bursts via dd (every 15 min).
+- **workload-api** (10.10.10.81, oc-node-03): Python HTTP server on
+  port 8080, cron jobs for inter-container traffic (every 2 min) and
+  memory sawtooth via 20 MiB allocation/release (every 10 min).
+
 ### Check status
 
 ```bash
 incusos/deploy-observability --status
 ```
 
-Shows the state of all containers, scrape target health, Grafana
+Shows the state of all containers (including workloads if deployed),
-accessibility, and OVN forward configuration.
+scrape target health, Grafana accessibility, and OVN forward
+configuration.
 
 ### Clean up