diff --git a/incusos/deploy-observability b/incusos/deploy-observability index 8b3155c..0e99ad0 100755 --- a/incusos/deploy-observability +++ b/incusos/deploy-observability @@ -465,8 +465,12 @@ scrape_configs: target_label: unit PTCFG" - # Ensure data directory exists with correct ownership + # Ensure data directories exist with correct ownership container_exec monitoring bash -c "mkdir -p /var/lib/loki && chown loki:nogroup /var/lib/loki" + container_exec monitoring bash -c "mkdir -p /var/lib/promtail && chown promtail:nogroup /var/lib/promtail" + + # Promtail needs adm group for /var/log access, systemd-journal for journal + container_exec monitoring bash -c "usermod -aG adm,systemd-journal promtail" container_exec monitoring systemctl enable loki container_exec monitoring systemctl restart loki diff --git a/incusos/observability-dashboards/core/logs-explorer.json b/incusos/observability-dashboards/core/logs-explorer.json index 2ac8ea0..3a213ee 100644 --- a/incusos/observability-dashboards/core/logs-explorer.json +++ b/incusos/observability-dashboards/core/logs-explorer.json @@ -48,7 +48,7 @@ "label": "Job", "refresh": 2, "sort": 1, - "allValue": ".*" + "allValue": ".+" }, { "type": "query", @@ -63,7 +63,7 @@ "label": "Host", "refresh": 2, "sort": 1, - "allValue": ".*" + "allValue": ".+" }, { "type": "query", @@ -78,7 +78,7 @@ "label": "Unit", "refresh": 2, "sort": 1, - "allValue": ".*" + "allValue": ".+" }, { "type": "textbox", @@ -229,7 +229,7 @@ }, "targets": [ { - "expr": "{job=~\"$job\", host=~\"$host\", unit=~\"$unit\"} |= \"$search\"", + "expr": "{job=~\"$job\", host=~\"$host\"} |= \"$search\"", "refId": "A" } ], diff --git a/notes/observability-guide.md b/notes/observability-guide.md index 6d2033c..94c095d 100644 --- a/notes/observability-guide.md +++ b/notes/observability-guide.md @@ -229,8 +229,8 @@ Log exploration powered by Loki with job/host/unit/search variables. - Error rate and warning rate by systemd unit ![Logs Explorer](../screenshots/logs-explorer.png) -*Log exploration dashboard with volume timeline, log stream, and error/warning -rate panels. Requires Promtail to be shipping logs to Loki for data.* +*Log exploration showing systemd journal and varlog streams, live log entries +with labels and severity highlighting, and error/warning rates by unit.* ### Incus Operations — Prometheus Health diff --git a/screenshots/logs-explorer.png b/screenshots/logs-explorer.png index 1eca9d6..d6c429f 100644 Binary files a/screenshots/logs-explorer.png and b/screenshots/logs-explorer.png differ