diff --git a/incusos/deploy-observability b/incusos/deploy-observability index 73b9f29..8b3155c 100755 --- a/incusos/deploy-observability +++ b/incusos/deploy-observability @@ -465,6 +465,9 @@ scrape_configs: target_label: unit PTCFG" + # Ensure data directory exists with correct ownership + container_exec monitoring bash -c "mkdir -p /var/lib/loki && chown loki:nogroup /var/lib/loki" + container_exec monitoring systemctl enable loki container_exec monitoring systemctl restart loki sleep 2 diff --git a/incusos/observability-dashboards/core/capacity-planning.json b/incusos/observability-dashboards/core/capacity-planning.json index 2356711..dca049b 100644 --- a/incusos/observability-dashboards/core/capacity-planning.json +++ b/incusos/observability-dashboards/core/capacity-planning.json @@ -324,7 +324,7 @@ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { - "expr": "count(incus_uptime_seconds)", + "expr": "count(incus_cpu_effective_total)", "legendFormat": "Instances", "refId": "A" } @@ -363,7 +363,7 @@ { "type": "timeseries", "title": "Memory Allocation vs Usage per Instance", - "description": "Compares total allocated memory (MemTotal) against actual resident memory (RSS) for each Incus instance. A large gap between allocated and used indicates overprovisioning; converging lines indicate an instance may need more memory.", + "description": "Compares total allocated memory (MemTotal) against active memory usage for each Incus instance. A large gap between allocated and used indicates overprovisioning; converging lines indicate an instance may need more memory.", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 }, "id": 11, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, @@ -374,8 +374,8 @@ "refId": "A" }, { - "expr": "incus_memory_RSS_bytes", - "legendFormat": "{{name}} used (RSS)", + "expr": "incus_memory_Active_bytes", + "legendFormat": "{{name}} used (active)", "refId": "B" } ], @@ -412,7 +412,7 @@ ] }, { - "matcher": { "id": "byRegexp", "options": ".*used \\(RSS\\)$" }, + "matcher": { "id": "byRegexp", "options": ".*used \\(active\\)$" }, "properties": [ { "id": "custom.fillOpacity", "value": 15 } ] diff --git a/incusos/observability-dashboards/core/cluster-overview.json b/incusos/observability-dashboards/core/cluster-overview.json index 9635319..0be4ba1 100644 --- a/incusos/observability-dashboards/core/cluster-overview.json +++ b/incusos/observability-dashboards/core/cluster-overview.json @@ -41,11 +41,11 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(incus_uptime_seconds, name)", + "definition": "label_values(incus_cpu_effective_total, name)", "includeAll": true, "multi": true, "name": "instance", - "query": "label_values(incus_uptime_seconds, name)", + "query": "label_values(incus_cpu_effective_total, name)", "refresh": 2, "regex": "", "sort": 1, @@ -59,11 +59,11 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(incus_uptime_seconds, instance)", + "definition": "label_values(incus_cpu_effective_total, instance)", "includeAll": true, "multi": true, "name": "node", - "query": "label_values(incus_uptime_seconds, instance)", + "query": "label_values(incus_cpu_effective_total, instance)", "refresh": 2, "regex": "", "sort": 1, @@ -106,7 +106,7 @@ { "type": "stat", "title": "Total Instances", - "description": "Number of running Incus instances (reporting uptime).", + "description": "Number of running Incus instances (reporting CPU metrics).", "gridPos": { "h": 4, "w": 6, @@ -120,7 +120,7 @@ }, "targets": [ { - "expr": "count(incus_uptime_seconds{instance=~\"$node\", name=~\"$instance\"})", + "expr": "count(incus_cpu_effective_total{instance=~\"$node\", name=~\"$instance\"})", "legendFormat": "Instances", "refId": "A" } @@ -279,7 +279,7 @@ { "type": "gauge", "title": "Cluster Memory Utilization", - "description": "Overall cluster memory utilization (RSS / Total) across all instances.", + "description": "Overall cluster memory utilization (Active / Total) across all instances.", "gridPos": { "h": 4, "w": 6, @@ -293,7 +293,7 @@ }, "targets": [ { - "expr": "(sum(incus_memory_RSS_bytes{instance=~\"$node\", name=~\"$instance\"}) / sum(incus_memory_MemTotal_bytes{instance=~\"$node\", name=~\"$instance\"})) * 100", + "expr": "(sum(incus_memory_Active_bytes{instance=~\"$node\", name=~\"$instance\"}) / sum(incus_memory_MemTotal_bytes{instance=~\"$node\", name=~\"$instance\"})) * 100", "legendFormat": "Utilization", "refId": "A" } @@ -359,7 +359,7 @@ { "type": "table", "title": "Instance Status Table", - "description": "Per-instance summary: uptime, CPU usage, memory RSS, total memory. Click instance name to drill down.", + "description": "Per-instance summary: processes, CPU usage, active memory, total memory. Click instance name to drill down.", "gridPos": { "h": 8, "w": 24, @@ -373,7 +373,7 @@ }, "targets": [ { - "expr": "incus_uptime_seconds{instance=~\"$node\", name=~\"$instance\"}", + "expr": "incus_procs_total{instance=~\"$node\", name=~\"$instance\"}", "legendFormat": "", "refId": "A", "format": "table", @@ -387,7 +387,7 @@ "instant": true }, { - "expr": "incus_memory_RSS_bytes{instance=~\"$node\", name=~\"$instance\"}", + "expr": "incus_memory_Active_bytes{instance=~\"$node\", name=~\"$instance\"}", "legendFormat": "", "refId": "C", "format": "table", @@ -469,11 +469,11 @@ "properties": [ { "id": "displayName", - "value": "Uptime" + "value": "Processes" }, { "id": "unit", - "value": "s" + "value": "short" } ] }, @@ -505,7 +505,7 @@ "properties": [ { "id": "displayName", - "value": "Memory RSS" + "value": "Active Memory" }, { "id": "unit", @@ -686,8 +686,8 @@ }, { "type": "timeseries", - "title": "Memory RSS by Instance", - "description": "Resident set size (RSS) memory usage per Incus instance over time.", + "title": "Active Memory by Instance", + "description": "Active memory usage per Incus instance over time.", "gridPos": { "h": 8, "w": 14, @@ -701,7 +701,7 @@ }, "targets": [ { - "expr": "incus_memory_RSS_bytes{instance=~\"$node\", name=~\"$instance\"}", + "expr": "incus_memory_Active_bytes{instance=~\"$node\", name=~\"$instance\"}", "legendFormat": "{{name}}", "refId": "A" } @@ -764,8 +764,8 @@ }, { "type": "bargauge", - "title": "Memory Usage % (RSS / Total)", - "description": "Current memory utilization per instance as RSS divided by total memory.", + "title": "Memory Usage % (Active / Total)", + "description": "Current memory utilization per instance as active memory divided by total memory.", "gridPos": { "h": 8, "w": 10, @@ -779,7 +779,7 @@ }, "targets": [ { - "expr": "(incus_memory_RSS_bytes{instance=~\"$node\", name=~\"$instance\"} / incus_memory_MemTotal_bytes{instance=~\"$node\", name=~\"$instance\"}) * 100", + "expr": "(incus_memory_Active_bytes{instance=~\"$node\", name=~\"$instance\"} / incus_memory_MemTotal_bytes{instance=~\"$node\", name=~\"$instance\"}) * 100", "legendFormat": "{{name}}", "refId": "A" } diff --git a/incusos/observability-dashboards/core/instance-deep-dive.json b/incusos/observability-dashboards/core/instance-deep-dive.json index fcfa693..e4d9b02 100644 --- a/incusos/observability-dashboards/core/instance-deep-dive.json +++ b/incusos/observability-dashboards/core/instance-deep-dive.json @@ -36,11 +36,11 @@ { "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(incus_uptime_seconds, name)", + "definition": "label_values(incus_cpu_effective_total, name)", "includeAll": false, "multi": false, "name": "instance", - "query": "label_values(incus_uptime_seconds, name)", + "query": "label_values(incus_cpu_effective_total, name)", "refresh": 2, "regex": "", "sort": 1, @@ -73,15 +73,15 @@ }, { "type": "stat", - "title": "Uptime", - "description": "How long this instance has been running.", + "title": "Processes", + "description": "Number of running processes in this instance.", "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, "id": 2, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { - "expr": "incus_uptime_seconds{name=~\"$instance\"}", - "legendFormat": "Uptime", + "expr": "incus_procs_total{name=~\"$instance\"}", + "legendFormat": "Processes", "refId": "A" } ], @@ -91,7 +91,7 @@ "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "unit": "s", + "unit": "short", "color": { "mode": "thresholds" } }, "overrides": [] @@ -176,13 +176,13 @@ { "type": "gauge", "title": "Memory Utilization", - "description": "Current memory utilization (RSS / Total) for this instance.", + "description": "Current memory utilization (Active / Total) for this instance.", "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, "id": 5, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { - "expr": "(incus_memory_RSS_bytes{name=~\"$instance\"} / incus_memory_MemTotal_bytes{name=~\"$instance\"}) * 100", + "expr": "(incus_memory_Active_bytes{name=~\"$instance\"} / incus_memory_MemTotal_bytes{name=~\"$instance\"}) * 100", "legendFormat": "Utilization", "refId": "A" } @@ -275,14 +275,14 @@ }, { "type": "timeseries", - "title": "Memory RSS Over Time", - "description": "Resident set size (RSS) memory usage for this instance over time.", + "title": "Active Memory Over Time", + "description": "Active memory usage for this instance over time.", "gridPos": { "h": 8, "w": 14, "x": 0, "y": 15 }, "id": 21, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { - "expr": "incus_memory_RSS_bytes{name=~\"$instance\"}", + "expr": "incus_memory_Active_bytes{name=~\"$instance\"}", "legendFormat": "{{name}}", "refId": "A" } @@ -321,13 +321,13 @@ { "type": "timeseries", "title": "Memory Utilization %", - "description": "Memory utilization percentage (RSS / Total) for this instance over time.", + "description": "Memory utilization percentage (Active / Total) for this instance over time.", "gridPos": { "h": 8, "w": 10, "x": 14, "y": 15 }, "id": 22, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { - "expr": "(incus_memory_RSS_bytes{name=~\"$instance\"} / incus_memory_MemTotal_bytes{name=~\"$instance\"}) * 100", + "expr": "(incus_memory_Active_bytes{name=~\"$instance\"} / incus_memory_MemTotal_bytes{name=~\"$instance\"}) * 100", "legendFormat": "{{name}}", "refId": "A" } diff --git a/incusos/observability-dashboards/core/network-deep-dive.json b/incusos/observability-dashboards/core/network-deep-dive.json index 5f84a58..ee3fcef 100644 --- a/incusos/observability-dashboards/core/network-deep-dive.json +++ b/incusos/observability-dashboards/core/network-deep-dive.json @@ -537,7 +537,7 @@ }, "targets": [ { - "expr": "sum by (name) (rate(incus_network_receive_bytes_total{instance=~\"$node\"}[$__rate_interval]))", + "expr": "sum by (name) (rate(incus_network_receive_bytes_total[$__rate_interval]))", "legendFormat": "{{name}}", "refId": "A" } @@ -615,7 +615,7 @@ }, "targets": [ { - "expr": "sum by (name) (rate(incus_network_transmit_bytes_total{instance=~\"$node\"}[$__rate_interval]))", + "expr": "sum by (name) (rate(incus_network_transmit_bytes_total[$__rate_interval]))", "legendFormat": "{{name}}", "refId": "A" } diff --git a/notes/observability-guide.md b/notes/observability-guide.md index 63fd180..6d2033c 100644 --- a/notes/observability-guide.md +++ b/notes/observability-guide.md @@ -73,7 +73,7 @@ Dashboard and visualization frontend. - Default credentials: `admin` / `admin` - Pre-provisioned datasources for Prometheus and Loki -- Three dashboards deployed automatically (see Dashboards section) +- Nine dashboards deployed automatically (see Dashboards section) - Accessible from LAN via OVN network forward ### Loki 3.6 @@ -142,19 +142,27 @@ Overview of all instances with instance/node template variables. - Instance count, total CPUs, total memory, cluster memory utilization gauge - Instance status table with drill-down links to Instance Deep Dive -- Per-instance CPU usage, memory RSS, memory usage % bar gauge +- Per-instance CPU usage, active memory, memory usage % bar gauge - Network receive/transmit rates, disk read/write rates +![Cluster Overview](../screenshots/cluster-overview.png) +*Cluster-wide view showing 8 instances across 3 nodes with 28 total CPUs, +active memory utilization, and the instance status table.* + ### Incus Cluster — Instance Deep Dive Single-instance drill-down selected via template variable. -- Uptime, effective CPUs, total memory, memory utilization gauge +- Processes, effective CPUs, total memory, memory utilization gauge - CPU usage over time with gradient color scheme -- Memory RSS and memory utilization % over time +- Active memory and memory utilization % over time - Network traffic (receive above axis, transmit below on same panel) - Disk read/write rates +![Instance Deep Dive](../screenshots/instance-deep-dive.png) +*Deep dive into workload-web showing 11 processes, 4 effective CPUs, 128 MiB +total memory, and per-metric time series for CPU, memory, network, and disk.* + ### Incus Services — HAProxy Traffic HAProxy monitoring with proxy/server template variables. @@ -166,6 +174,10 @@ HAProxy monitoring with proxy/server template variables. - Frontend traffic, sessions by proxy, response times - Queue time and connection errors +![HAProxy Traffic](../screenshots/haproxy-traffic.png) +*HAProxy dashboard showing panel layout. Panels display "No data" when +HAProxy instances are not deployed or their stats endpoints are unreachable.* + ### Incus Infrastructure — Host Resources (IncusOS Nodes) Physical host monitoring with node template variable. @@ -177,6 +189,10 @@ Physical host monitoring with node template variable. - Filesystem usage % bar gauge, IOPS, disk I/O time - Network traffic by device, network errors and drops +![Host Resources](../screenshots/host-resources.png) +*Host-level metrics from node-exporter showing system info, CPU/memory +breakdown, filesystem usage, and network traffic for all 3 IncusOS nodes.* + ### Incus Infrastructure — Storage & Filesystem Filesystem and disk I/O with node/mountpoint template variables. @@ -187,6 +203,10 @@ Filesystem and disk I/O with node/mountpoint template variables. - Average read/write latency, disk utilization % - Inode usage % bar gauge +![Storage & Filesystem](../screenshots/storage-filesystem.png) +*Storage dashboard showing filesystem usage table, available space trends, +disk throughput, and I/O latency across all cluster nodes.* + ### Incus Infrastructure — Network Deep Dive Network metrics with node/device template variables. @@ -196,6 +216,10 @@ Network metrics with node/device template variables. - Instance-level network traffic (Incus `incus_network_*` metrics) - Receive/transmit errors and packet drops +![Network Deep Dive](../screenshots/network-deep-dive.png) +*Network dashboard with cluster-wide ingress/egress stats, per-node traffic +breakdown by device, and instance-level network rates for all 8 instances.* + ### Incus Operations — Logs Explorer Log exploration powered by Loki with job/host/unit/search variables. @@ -204,6 +228,10 @@ Log exploration powered by Loki with job/host/unit/search variables. - Full log stream with search filtering - Error rate and warning rate by systemd unit +![Logs Explorer](../screenshots/logs-explorer.png) +*Log exploration dashboard with volume timeline, log stream, and error/warning +rate panels. Requires Promtail to be shipping logs to Loki for data.* + ### Incus Operations — Prometheus Health Prometheus self-monitoring (no template variables). @@ -214,6 +242,10 @@ Prometheus self-monitoring (no template variables). - TSDB size, head series, head chunks, WAL size - TSDB size trend, samples ingested trend, compactions +![Prometheus Health](../screenshots/prometheus-health.png) +*Prometheus self-monitoring showing 7 targets UP (3 Incus, 3 node-exporter, +1 self), 2 DOWN (HAProxy), scrape performance, and TSDB stats.* + ### Incus Operations — Capacity Planning Resource planning with 7-day default time range (no template variables). @@ -221,7 +253,11 @@ Resource planning with 7-day default time range (no template variables). - CPU and memory headroom per node (inverted bar gauges) - CPU and memory usage trends (7-day) - Disk space forecast with 30-day `predict_linear` projection -- Instance count trend, memory allocation vs usage per instance +- Instance count trend, memory allocation vs active memory per instance + +![Capacity Planning](../screenshots/capacity-planning.png) +*Capacity planning with CPU/memory headroom bars, 7-day usage trends, 30-day +disk space forecast using predict_linear, and instance count over time.* ## ACL Configuration diff --git a/screenshots/capacity-planning.png b/screenshots/capacity-planning.png new file mode 100644 index 0000000..1f808b5 Binary files /dev/null and b/screenshots/capacity-planning.png differ diff --git a/screenshots/cluster-overview.png b/screenshots/cluster-overview.png new file mode 100644 index 0000000..87ae773 Binary files /dev/null and b/screenshots/cluster-overview.png differ diff --git a/screenshots/haproxy-traffic.png b/screenshots/haproxy-traffic.png new file mode 100644 index 0000000..fe03fb8 Binary files /dev/null and b/screenshots/haproxy-traffic.png differ diff --git a/screenshots/host-resources.png b/screenshots/host-resources.png new file mode 100644 index 0000000..8304bbc Binary files /dev/null and b/screenshots/host-resources.png differ diff --git a/screenshots/instance-deep-dive.png b/screenshots/instance-deep-dive.png new file mode 100644 index 0000000..d53bbf1 Binary files /dev/null and b/screenshots/instance-deep-dive.png differ diff --git a/screenshots/logs-explorer.png b/screenshots/logs-explorer.png new file mode 100644 index 0000000..1eca9d6 Binary files /dev/null and b/screenshots/logs-explorer.png differ diff --git a/screenshots/network-deep-dive.png b/screenshots/network-deep-dive.png new file mode 100644 index 0000000..325ddb9 Binary files /dev/null and b/screenshots/network-deep-dive.png differ diff --git a/screenshots/prometheus-health.png b/screenshots/prometheus-health.png new file mode 100644 index 0000000..37f679a Binary files /dev/null and b/screenshots/prometheus-health.png differ diff --git a/screenshots/storage-filesystem.png b/screenshots/storage-filesystem.png new file mode 100644 index 0000000..724d4a7 Binary files /dev/null and b/screenshots/storage-filesystem.png differ