diff --git a/incusos/deploy-observability b/incusos/deploy-observability index 0e99ad0..a4356dd 100755 --- a/incusos/deploy-observability +++ b/incusos/deploy-observability @@ -337,11 +337,14 @@ phase_install_prometheus() { idx=$((idx + 1)) done - local node_exp_targets="" + local node_exp_config="" idx=0 for ip in "${NODE_EXP_IPS[@]}"; do - if [[ $idx -gt 0 ]]; then node_exp_targets+=", "; fi - node_exp_targets+="'${ip}:9100'" + local target_name="${NODE_EXP_TARGETS[$idx]}" + node_exp_config+=" + - targets: ['${ip}:9100'] + labels: + nodename: '${target_name}'" idx=$((idx + 1)) done @@ -364,14 +367,20 @@ scrape_configs: metrics_path: /1.0/metrics static_configs: - targets: [${incus_targets}] + metric_relabel_configs: + - source_labels: [__name__] + regex: 'node_.*' + action: drop - job_name: 'haproxy' static_configs: - targets: [${haproxy_targets}] - job_name: 'node-exporter' - static_configs: - - targets: [${node_exp_targets}] + static_configs:${node_exp_config} + relabel_configs: + - source_labels: [nodename] + target_label: instance PROMCFG" container_exec monitoring systemctl enable prometheus @@ -463,6 +472,10 @@ scrape_configs: relabel_configs: - source_labels: ['__journal__systemd_unit'] target_label: unit + - source_labels: ['__journal_priority_keyword'] + target_label: level + - source_labels: ['__journal_syslog_identifier'] + target_label: service_name PTCFG" # Ensure data directories exist with correct ownership @@ -498,6 +511,9 @@ phase_install_grafana() { apt-get install -y -qq grafana 2>&1 | tail -1 " + # Install Logs Drilldown plugin for interactive log exploration + container_exec monitoring grafana-cli plugins install grafana-lokiexplore-app 2>&1 | tail -1 + # Provision datasources with explicit UIDs for dashboard portability container_exec monitoring bash -c "mkdir -p /etc/grafana/provisioning/datasources cat > /etc/grafana/provisioning/datasources/observability.yaml << 'DSCFG' diff --git a/incusos/observability-dashboards/core/capacity-planning.json b/incusos/observability-dashboards/core/capacity-planning.json index dca049b..9470f89 100644 --- a/incusos/observability-dashboards/core/capacity-planning.json +++ b/incusos/observability-dashboards/core/capacity-planning.json @@ -56,7 +56,7 @@ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { - "expr": "avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])) * 100", + "expr": "avg by (instance) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\"}[$__rate_interval])) * 100", "legendFormat": "{{instance}}", "refId": "A" } @@ -98,7 +98,7 @@ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { - "expr": "(node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100", + "expr": "(node_memory_MemAvailable_bytes{job=\"node-exporter\"} / node_memory_MemTotal_bytes{job=\"node-exporter\"}) * 100", "legendFormat": "{{instance}}", "refId": "A" } @@ -148,7 +148,7 @@ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { - "expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100", + "expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\"}[5m]))) * 100", "legendFormat": "{{instance}}", "refId": "A" } @@ -199,7 +199,7 @@ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { - "expr": "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100", + "expr": "(1 - node_memory_MemAvailable_bytes{job=\"node-exporter\"} / node_memory_MemTotal_bytes{job=\"node-exporter\"}) * 100", "legendFormat": "{{instance}}", "refId": "A" } @@ -258,12 +258,12 @@ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { - "expr": "node_filesystem_avail_bytes{mountpoint=\"/\"}", + "expr": "node_filesystem_avail_bytes{job=\"node-exporter\", mountpoint=\"/\"}", "legendFormat": "{{instance}} actual", "refId": "A" }, { - "expr": "predict_linear(node_filesystem_avail_bytes{mountpoint=\"/\"}[7d], 30*24*3600)", + "expr": "predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\", mountpoint=\"/\"}[7d], 30*24*3600)", "legendFormat": "{{instance}} forecast", "refId": "B" } diff --git a/incusos/observability-dashboards/core/host-resources.json b/incusos/observability-dashboards/core/host-resources.json index bacb481..322b0e3 100644 --- a/incusos/observability-dashboards/core/host-resources.json +++ b/incusos/observability-dashboards/core/host-resources.json @@ -41,11 +41,11 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(node_uname_info, instance)", + "definition": "label_values(node_uname_info{job=\"node-exporter\"}, instance)", "includeAll": true, "multi": true, "name": "node", - "query": "label_values(node_uname_info, instance)", + "query": "label_values(node_uname_info{job=\"node-exporter\"}, instance)", "refresh": 2, "sort": 1, "type": "query", @@ -101,28 +101,28 @@ }, "targets": [ { - "expr": "node_uname_info{instance=~\"$node\"}", + "expr": "node_uname_info{job=\"node-exporter\", instance=~\"$node\"}", "legendFormat": "", "refId": "A", "format": "table", "instant": true }, { - "expr": "node_boot_time_seconds{instance=~\"$node\"}", + "expr": "node_boot_time_seconds{job=\"node-exporter\", instance=~\"$node\"}", "legendFormat": "", "refId": "B", "format": "table", "instant": true }, { - "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"}", + "expr": "node_memory_MemTotal_bytes{job=\"node-exporter\", instance=~\"$node\"}", "legendFormat": "", "refId": "C", "format": "table", "instant": true }, { - "expr": "count by (instance) (node_cpu_seconds_total{instance=~\"$node\", mode=\"idle\"})", + "expr": "count by (instance) (node_cpu_seconds_total{job=\"node-exporter\", instance=~\"$node\", mode=\"idle\"})", "legendFormat": "", "refId": "D", "format": "table", @@ -286,7 +286,7 @@ }, "targets": [ { - "expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$node\"}[$__rate_interval]))) * 100", + "expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=~\"$node\"}[$__rate_interval]))) * 100", "legendFormat": "{{instance}}", "refId": "A" } @@ -373,7 +373,7 @@ }, "targets": [ { - "expr": "avg by (instance, mode) (rate(node_cpu_seconds_total{instance=~\"$node\", mode=~\"user|system|iowait|idle\"}[$__rate_interval])) * 100", + "expr": "avg by (instance, mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", instance=~\"$node\", mode=~\"user|system|iowait|idle\"}[$__rate_interval])) * 100", "legendFormat": "{{instance}} {{mode}}", "refId": "A" } @@ -449,22 +449,22 @@ }, "targets": [ { - "expr": "node_load1{instance=~\"$node\"}", + "expr": "node_load1{job=\"node-exporter\", instance=~\"$node\"}", "legendFormat": "{{instance}} load1", "refId": "A" }, { - "expr": "node_load5{instance=~\"$node\"}", + "expr": "node_load5{job=\"node-exporter\", instance=~\"$node\"}", "legendFormat": "{{instance}} load5", "refId": "B" }, { - "expr": "node_load15{instance=~\"$node\"}", + "expr": "node_load15{job=\"node-exporter\", instance=~\"$node\"}", "legendFormat": "{{instance}} load15", "refId": "C" }, { - "expr": "count by (instance) (node_cpu_seconds_total{instance=~\"$node\", mode=\"idle\"})", + "expr": "count by (instance) (node_cpu_seconds_total{job=\"node-exporter\", instance=~\"$node\", mode=\"idle\"})", "legendFormat": "{{instance}} CPUs", "refId": "D" } @@ -583,7 +583,7 @@ }, "targets": [ { - "expr": "(1 - node_memory_MemAvailable_bytes{instance=~\"$node\"} / node_memory_MemTotal_bytes{instance=~\"$node\"}) * 100", + "expr": "(1 - node_memory_MemAvailable_bytes{job=\"node-exporter\", instance=~\"$node\"} / node_memory_MemTotal_bytes{job=\"node-exporter\", instance=~\"$node\"}) * 100", "legendFormat": "{{instance}}", "refId": "A" } @@ -669,22 +669,22 @@ }, "targets": [ { - "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"} - node_memory_MemAvailable_bytes{instance=~\"$node\"} - node_memory_Buffers_bytes{instance=~\"$node\"} - node_memory_Cached_bytes{instance=~\"$node\"}", + "expr": "node_memory_MemTotal_bytes{job=\"node-exporter\", instance=~\"$node\"} - node_memory_MemAvailable_bytes{job=\"node-exporter\", instance=~\"$node\"} - node_memory_Buffers_bytes{job=\"node-exporter\", instance=~\"$node\"} - node_memory_Cached_bytes{job=\"node-exporter\", instance=~\"$node\"}", "legendFormat": "{{instance}} used", "refId": "A" }, { - "expr": "node_memory_Buffers_bytes{instance=~\"$node\"}", + "expr": "node_memory_Buffers_bytes{job=\"node-exporter\", instance=~\"$node\"}", "legendFormat": "{{instance}} buffers", "refId": "B" }, { - "expr": "node_memory_Cached_bytes{instance=~\"$node\"}", + "expr": "node_memory_Cached_bytes{job=\"node-exporter\", instance=~\"$node\"}", "legendFormat": "{{instance}} cached", "refId": "C" }, { - "expr": "node_memory_MemAvailable_bytes{instance=~\"$node\"}", + "expr": "node_memory_MemAvailable_bytes{job=\"node-exporter\", instance=~\"$node\"}", "legendFormat": "{{instance}} available", "refId": "D" } @@ -773,7 +773,7 @@ }, "targets": [ { - "expr": "(1 - node_filesystem_avail_bytes{instance=~\"$node\", fstype!~\"tmpfs|overlay\", mountpoint=~\"/.*\"} / node_filesystem_size_bytes{instance=~\"$node\", fstype!~\"tmpfs|overlay\", mountpoint=~\"/.*\"}) * 100", + "expr": "(1 - node_filesystem_avail_bytes{job=\"node-exporter\", instance=~\"$node\", fstype!~\"tmpfs|overlay\", mountpoint=~\"/.*\"} / node_filesystem_size_bytes{job=\"node-exporter\", instance=~\"$node\", fstype!~\"tmpfs|overlay\", mountpoint=~\"/.*\"}) * 100", "legendFormat": "{{instance}} {{mountpoint}}", "refId": "A" } @@ -843,7 +843,7 @@ }, "targets": [ { - "expr": "node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=\"/\"}", + "expr": "node_filesystem_avail_bytes{job=\"node-exporter\", instance=~\"$node\", mountpoint=\"/\"}", "legendFormat": "{{instance}}", "refId": "A" } @@ -918,12 +918,12 @@ }, "targets": [ { - "expr": "sum by (instance) (rate(node_disk_reads_completed_total{instance=~\"$node\"}[$__rate_interval]))", + "expr": "sum by (instance) (rate(node_disk_reads_completed_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} reads", "refId": "A" }, { - "expr": "sum by (instance) (rate(node_disk_writes_completed_total{instance=~\"$node\"}[$__rate_interval]))", + "expr": "sum by (instance) (rate(node_disk_writes_completed_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} writes", "refId": "B" } @@ -998,7 +998,7 @@ }, "targets": [ { - "expr": "sum by (instance) (rate(node_disk_read_bytes_total{instance=~\"$node\"}[$__rate_interval]))", + "expr": "sum by (instance) (rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}}", "refId": "A" } @@ -1073,7 +1073,7 @@ }, "targets": [ { - "expr": "sum by (instance) (rate(node_disk_written_bytes_total{instance=~\"$node\"}[$__rate_interval]))", + "expr": "sum by (instance) (rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}}", "refId": "A" } @@ -1148,7 +1148,7 @@ }, "targets": [ { - "expr": "sum by (instance) (rate(node_disk_io_time_seconds_total{instance=~\"$node\"}[$__rate_interval])) * 100", + "expr": "sum by (instance) (rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval])) * 100", "legendFormat": "{{instance}}", "refId": "A" } @@ -1246,7 +1246,7 @@ }, "targets": [ { - "expr": "rate(node_network_receive_bytes_total{instance=~\"$node\", device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])", + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=~\"$node\", device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])", "legendFormat": "{{instance}} {{device}} rx", "refId": "A" } @@ -1321,7 +1321,7 @@ }, "targets": [ { - "expr": "rate(node_network_transmit_bytes_total{instance=~\"$node\", device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])", + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=~\"$node\", device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval])", "legendFormat": "{{instance}} {{device}} tx", "refId": "A" } @@ -1396,22 +1396,22 @@ }, "targets": [ { - "expr": "sum by (instance) (rate(node_network_receive_errs_total{instance=~\"$node\"}[$__rate_interval]))", + "expr": "sum by (instance) (rate(node_network_receive_errs_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} rx errors", "refId": "A" }, { - "expr": "sum by (instance) (rate(node_network_transmit_errs_total{instance=~\"$node\"}[$__rate_interval]))", + "expr": "sum by (instance) (rate(node_network_transmit_errs_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} tx errors", "refId": "B" }, { - "expr": "sum by (instance) (rate(node_network_receive_drop_total{instance=~\"$node\"}[$__rate_interval]))", + "expr": "sum by (instance) (rate(node_network_receive_drop_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} rx drops", "refId": "C" }, { - "expr": "sum by (instance) (rate(node_network_transmit_drop_total{instance=~\"$node\"}[$__rate_interval]))", + "expr": "sum by (instance) (rate(node_network_transmit_drop_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} tx drops", "refId": "D" } diff --git a/incusos/observability-dashboards/core/logs-explorer.json b/incusos/observability-dashboards/core/logs-explorer.json index 3a213ee..6832aaa 100644 --- a/incusos/observability-dashboards/core/logs-explorer.json +++ b/incusos/observability-dashboards/core/logs-explorer.json @@ -31,6 +31,17 @@ "title": "Incus Dashboards", "tooltip": "Navigate to other Incus dashboards", "type": "dashboards" + }, + { + "asDropdown": false, + "icon": "doc", + "includeVars": false, + "keepTime": true, + "targetBlank": false, + "title": "Logs Drilldown", + "tooltip": "Interactive log exploration with automatic field discovery and pattern detection", + "type": "link", + "url": "/a/grafana-lokiexplore-app/explore" } ], "templating": { diff --git a/incusos/observability-dashboards/core/network-deep-dive.json b/incusos/observability-dashboards/core/network-deep-dive.json index ee3fcef..6ede30f 100644 --- a/incusos/observability-dashboards/core/network-deep-dive.json +++ b/incusos/observability-dashboards/core/network-deep-dive.json @@ -42,11 +42,11 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(node_network_receive_bytes_total, instance)", + "definition": "label_values(node_network_receive_bytes_total{job=\"node-exporter\"}, instance)", "includeAll": false, "multi": true, "name": "node", - "query": "label_values(node_network_receive_bytes_total, instance)", + "query": "label_values(node_network_receive_bytes_total{job=\"node-exporter\"}, instance)", "refresh": 2, "sort": 1, "type": "query", @@ -58,11 +58,11 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(node_network_receive_bytes_total{instance=~\"$node\"}, device)", + "definition": "label_values(node_network_receive_bytes_total{job=\"node-exporter\", instance=~\"$node\"}, device)", "includeAll": true, "multi": true, "name": "device", - "query": "label_values(node_network_receive_bytes_total{instance=~\"$node\"}, device)", + "query": "label_values(node_network_receive_bytes_total{job=\"node-exporter\", instance=~\"$node\"}, device)", "refresh": 2, "regex": "/^(?!lo$).*/", "sort": 1, @@ -119,7 +119,7 @@ }, "targets": [ { - "expr": "sum(rate(node_network_receive_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval]))", + "expr": "sum(rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=~\"$node\", device=~\"$device\"}[$__rate_interval]))", "legendFormat": "Ingress", "refId": "A" } @@ -174,7 +174,7 @@ }, "targets": [ { - "expr": "sum(rate(node_network_transmit_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval]))", + "expr": "sum(rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=~\"$node\", device=~\"$device\"}[$__rate_interval]))", "legendFormat": "Egress", "refId": "A" } @@ -229,7 +229,7 @@ }, "targets": [ { - "expr": "sum(rate(node_network_receive_errs_total{instance=~\"$node\"}[$__rate_interval])) + sum(rate(node_network_transmit_errs_total{instance=~\"$node\"}[$__rate_interval]))", + "expr": "sum(rate(node_network_receive_errs_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval])) + sum(rate(node_network_transmit_errs_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "Errors", "refId": "A" } @@ -292,7 +292,7 @@ }, "targets": [ { - "expr": "sum(rate(node_network_receive_drop_total{instance=~\"$node\"}[$__rate_interval])) + sum(rate(node_network_transmit_drop_total{instance=~\"$node\"}[$__rate_interval]))", + "expr": "sum(rate(node_network_receive_drop_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval])) + sum(rate(node_network_transmit_drop_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "Drops", "refId": "A" } @@ -368,7 +368,7 @@ }, "targets": [ { - "expr": "rate(node_network_receive_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])", + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=~\"$node\", device=~\"$device\"}[$__rate_interval])", "legendFormat": "{{instance}} {{device}} rx", "refId": "A" } @@ -446,7 +446,7 @@ }, "targets": [ { - "expr": "rate(node_network_transmit_bytes_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])", + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=~\"$node\", device=~\"$device\"}[$__rate_interval])", "legendFormat": "{{instance}} {{device}} tx", "refId": "A" } @@ -706,12 +706,12 @@ }, "targets": [ { - "expr": "rate(node_network_receive_errs_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])", + "expr": "rate(node_network_receive_errs_total{job=\"node-exporter\", instance=~\"$node\", device=~\"$device\"}[$__rate_interval])", "legendFormat": "{{instance}} {{device}} rx errors", "refId": "A" }, { - "expr": "rate(node_network_transmit_errs_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])", + "expr": "rate(node_network_transmit_errs_total{job=\"node-exporter\", instance=~\"$node\", device=~\"$device\"}[$__rate_interval])", "legendFormat": "{{instance}} {{device}} tx errors", "refId": "B" } @@ -789,12 +789,12 @@ }, "targets": [ { - "expr": "rate(node_network_receive_drop_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])", + "expr": "rate(node_network_receive_drop_total{job=\"node-exporter\", instance=~\"$node\", device=~\"$device\"}[$__rate_interval])", "legendFormat": "{{instance}} {{device}} rx drops", "refId": "A" }, { - "expr": "rate(node_network_transmit_drop_total{instance=~\"$node\", device=~\"$device\"}[$__rate_interval])", + "expr": "rate(node_network_transmit_drop_total{job=\"node-exporter\", instance=~\"$node\", device=~\"$device\"}[$__rate_interval])", "legendFormat": "{{instance}} {{device}} tx drops", "refId": "B" } diff --git a/incusos/observability-dashboards/core/storage-filesystem.json b/incusos/observability-dashboards/core/storage-filesystem.json index 60f91a9..abe64d7 100644 --- a/incusos/observability-dashboards/core/storage-filesystem.json +++ b/incusos/observability-dashboards/core/storage-filesystem.json @@ -41,11 +41,11 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(node_filesystem_size_bytes, instance)", + "definition": "label_values(node_filesystem_size_bytes{job=\"node-exporter\"}, instance)", "includeAll": false, "multi": true, "name": "node", - "query": "label_values(node_filesystem_size_bytes, instance)", + "query": "label_values(node_filesystem_size_bytes{job=\"node-exporter\"}, instance)", "refresh": 2, "sort": 1, "type": "query", @@ -57,11 +57,11 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(node_filesystem_size_bytes{instance=~\"$node\"}, mountpoint)", + "definition": "label_values(node_filesystem_size_bytes{job=\"node-exporter\", instance=~\"$node\"}, mountpoint)", "includeAll": true, "multi": true, "name": "mountpoint", - "query": "label_values(node_filesystem_size_bytes{instance=~\"$node\"}, mountpoint)", + "query": "label_values(node_filesystem_size_bytes{job=\"node-exporter\", instance=~\"$node\"}, mountpoint)", "refresh": 2, "sort": 1, "type": "query", @@ -117,21 +117,21 @@ }, "targets": [ { - "expr": "node_filesystem_size_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}", + "expr": "node_filesystem_size_bytes{job=\"node-exporter\", instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}", "legendFormat": "", "refId": "A", "format": "table", "instant": true }, { - "expr": "node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}", + "expr": "node_filesystem_avail_bytes{job=\"node-exporter\", instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}", "legendFormat": "", "refId": "B", "format": "table", "instant": true }, { - "expr": "(1 - node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}) * 100", + "expr": "(1 - node_filesystem_avail_bytes{job=\"node-exporter\", instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{job=\"node-exporter\", instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}) * 100", "legendFormat": "", "refId": "C", "format": "table", @@ -334,7 +334,7 @@ }, "targets": [ { - "expr": "node_filesystem_avail_bytes{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}", + "expr": "node_filesystem_avail_bytes{job=\"node-exporter\", instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}", "legendFormat": "{{instance}} {{mountpoint}}", "refId": "A" } @@ -424,7 +424,7 @@ }, "targets": [ { - "expr": "sum by (instance, device) (rate(node_disk_read_bytes_total{instance=~\"$node\"}[$__rate_interval]))", + "expr": "sum by (instance, device) (rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} {{device}}", "refId": "A" } @@ -501,7 +501,7 @@ }, "targets": [ { - "expr": "sum by (instance, device) (rate(node_disk_written_bytes_total{instance=~\"$node\"}[$__rate_interval]))", + "expr": "sum by (instance, device) (rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{instance}} {{device}}", "refId": "A" } @@ -591,7 +591,7 @@ }, "targets": [ { - "expr": "rate(node_disk_read_time_seconds_total{instance=~\"$node\"}[$__rate_interval]) / rate(node_disk_reads_completed_total{instance=~\"$node\"}[$__rate_interval])", + "expr": "rate(node_disk_read_time_seconds_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval]) / rate(node_disk_reads_completed_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval])", "legendFormat": "{{instance}} {{device}}", "refId": "A" } @@ -668,7 +668,7 @@ }, "targets": [ { - "expr": "rate(node_disk_write_time_seconds_total{instance=~\"$node\"}[$__rate_interval]) / rate(node_disk_writes_completed_total{instance=~\"$node\"}[$__rate_interval])", + "expr": "rate(node_disk_write_time_seconds_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval]) / rate(node_disk_writes_completed_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval])", "legendFormat": "{{instance}} {{device}}", "refId": "A" } @@ -745,7 +745,7 @@ }, "targets": [ { - "expr": "sum by (instance) (rate(node_disk_io_time_seconds_total{instance=~\"$node\"}[$__rate_interval])) * 100", + "expr": "sum by (instance) (rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=~\"$node\"}[$__rate_interval])) * 100", "legendFormat": "{{instance}}", "refId": "A" } @@ -845,7 +845,7 @@ }, "targets": [ { - "expr": "(1 - node_filesystem_files_free{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"} / node_filesystem_files{instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}) * 100", + "expr": "(1 - node_filesystem_files_free{job=\"node-exporter\", instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"} / node_filesystem_files{job=\"node-exporter\", instance=~\"$node\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|overlay\"}) * 100", "legendFormat": "{{instance}} {{mountpoint}}", "refId": "A" } diff --git a/notes/observability-guide.md b/notes/observability-guide.md index 94c095d..6e8fe16 100644 --- a/notes/observability-guide.md +++ b/notes/observability-guide.md @@ -232,6 +232,43 @@ Log exploration powered by Loki with job/host/unit/search variables. *Log exploration showing systemd journal and varlog streams, live log entries with labels and severity highlighting, and error/warning rates by unit.* +### Interactive Log Exploration (Logs Drilldown) + +For deeper investigation beyond the Logs Explorer dashboard, Grafana's +**Logs Drilldown** plugin provides a VMware LogInsight-style interactive +experience — no LogQL knowledge required. + +Access it from the **Drilldown > Logs** menu in Grafana's left sidebar, or +via the **Logs Drilldown** link at the top of the Logs Explorer dashboard. + +**What it provides:** + +- **Service-based browsing** — automatically discovers all log-producing + services and shows them with volume sparklines and log previews +- **Automatic field discovery** — surfaces all label fields (service, level, + unit, host) as interactive facets for filtering +- **Volume-based drill-down** — click into time ranges on the volume chart + to narrow focus to specific incidents +- **Pattern detection** — groups log lines by structure, revealing recurring + patterns and anomalies without writing queries +- **Severity filtering** — filter by log level (info, warn, error) using the + enriched `level` label from Promtail's journal scrape + +**Logs Explorer dashboard vs Logs Drilldown:** + +| | Logs Explorer Dashboard | Logs Drilldown | +|---|---|---| +| Purpose | Overview and monitoring | Investigation and debugging | +| Interface | Pre-built panels with template variables | Interactive, queryless exploration | +| Filtering | Select job/host/unit from dropdowns | Click any field value to filter | +| Pattern analysis | Manual (requires LogQL) | Automatic pattern grouping | +| Best for | "What's happening now?" | "Why did this break?" | + +![Logs Drilldown](../screenshots/logs-drilldown.png) +*Logs Drilldown showing automatic service discovery with volume sparklines, +log previews with severity highlighting, and the "Add label" faceted filter +bar for queryless exploration.* + ### Incus Operations — Prometheus Health Prometheus self-monitoring (no template variables). diff --git a/screenshots/host-resources-fixed.png b/screenshots/host-resources-fixed.png new file mode 100644 index 0000000..a228633 Binary files /dev/null and b/screenshots/host-resources-fixed.png differ diff --git a/screenshots/logs-drilldown.png b/screenshots/logs-drilldown.png new file mode 100644 index 0000000..bacda47 Binary files /dev/null and b/screenshots/logs-drilldown.png differ