#!/usr/bin/env bash # manage-dashboards - Manage Grafana dashboards for the Incus observability stack # # Push, export, validate, and inspect Grafana dashboards via the Grafana HTTP # API. Dashboards are organized into folders and use datasource UID variables # for portability across deployments. # # Usage: manage-dashboards [OPTIONS] # Run 'manage-dashboards --help' for full usage information. set -euo pipefail readonly VERSION="0.1.0" readonly SCRIPT_NAME="manage-dashboards" readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly DASHBOARDS_DIR="${SCRIPT_DIR}/observability-dashboards" readonly MANIFEST_FILE="${DASHBOARDS_DIR}/dashboard.yaml" # --------------------------------------------------------------------------- # Defaults (overridden by CLI flags) # --------------------------------------------------------------------------- CLUSTER_REMOTE="oc-node-01" MONITORING_CONTAINER="monitoring" GRAFANA_URL="http://localhost:3000" GRAFANA_USER="admin" GRAFANA_PASSWORD="admin" # Runtime flags DRY_RUN=false VERBOSE=false QUIET=false ACTION="" FILTER_CATEGORY="" FILTER_FILE="" FILTER_UID="" # --------------------------------------------------------------------------- # Color and formatting # --------------------------------------------------------------------------- setup_colors() { if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' BOLD='\033[1m' DIM='\033[2m' RESET='\033[0m' else RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET='' fi } info() { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET} $*"; } success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET} $*"; } warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; } error() { echo -e "${RED}[error]${RESET} $*" >&2; } step() { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; } detail() { [[ "$VERBOSE" != true ]] && return; echo -e "${DIM} $*${RESET}"; } dry_run_guard() { if [[ "$DRY_RUN" == true ]]; then info "(dry-run) $*" return 1 fi return 0 } # --------------------------------------------------------------------------- # Help # --------------------------------------------------------------------------- usage() { cat </dev/null || echo "") LOKI_UID=$(echo "$ds_json" | python3 -c " import sys, json ds = json.load(sys.stdin) for d in ds: if d.get('type') == 'loki': print(d['uid']) break " 2>/dev/null || echo "") if [[ -z "$PROMETHEUS_UID" ]]; then error "Could not find Prometheus datasource in Grafana" return 1 fi detail "Prometheus datasource UID: ${PROMETHEUS_UID}" detail "Loki datasource UID: ${LOKI_UID:-not found}" } # Get or create a Grafana folder, returns folder ID ensure_folder() { local folder_title="$1" # Check if folder exists local folder_id folder_id=$(grafana_api GET "/api/folders" | python3 -c " import sys, json folders = json.load(sys.stdin) for f in folders: if f.get('title') == '${folder_title}': print(f['id']) break " 2>/dev/null || echo "") if [[ -n "$folder_id" ]]; then detail "Folder '${folder_title}' exists (ID: ${folder_id})" >&2 echo "$folder_id" return 0 fi # Create folder local folder_uid folder_uid=$(echo "$folder_title" | tr '[:upper:]' '[:lower:]' | tr ' ' '-') local result result=$(grafana_api POST "/api/folders" \ -H "Content-Type: application/json" \ -d "{\"title\": \"${folder_title}\", \"uid\": \"${folder_uid}\"}") folder_id=$(echo "$result" | python3 -c " import sys, json print(json.load(sys.stdin).get('id', '')) " 2>/dev/null || echo "") if [[ -n "$folder_id" ]]; then detail "Created folder '${folder_title}' (ID: ${folder_id})" >&2 echo "$folder_id" else error "Failed to create folder '${folder_title}': ${result}" return 1 fi } # Read manifest and output entries as tab-separated lines read_manifest() { if [[ ! -f "$MANIFEST_FILE" ]]; then error "Dashboard manifest not found: ${MANIFEST_FILE}" error "Expected a YAML file listing dashboard metadata." return 1 fi python3 -c " import sys, re entries = [] current = {} with open('${MANIFEST_FILE}') as f: for line in f: stripped = line.rstrip() # New entry starts with ' - file:' m = re.match(r'^ - file:\s*(.*)', stripped) if m: if current: entries.append(current) current = {'file': m.group(1).strip()} continue # Key-value within an entry m = re.match(r'^ (\w+):\s*(.*)', stripped) if m and current: key, val = m.group(1), m.group(2).strip() # Strip YAML >- multiline indicators and quotes if val in ('>-', '>', '|'): val = '' val = val.strip('\"').strip(\"'\") if key in current and not val: continue current[key] = val continue # Continuation line (multiline description) if current and stripped.startswith(' ') and stripped.strip(): if 'description' in current: sep = ' ' if current['description'] else '' current['description'] = current['description'] + sep + stripped.strip() if current: entries.append(current) for d in entries: parts = [ d.get('file', ''), d.get('uid', ''), d.get('title', ''), d.get('folder', 'General'), d.get('category', ''), d.get('version', '1'), d.get('description', ''), ] print('\t'.join(parts)) " } # Substitute datasource variables in JSON content inject_datasource_uids() { local json_content="$1" echo "$json_content" \ | sed "s/\\\${DS_PROMETHEUS}/${PROMETHEUS_UID}/g" \ | sed "s/\\\${DS_LOKI}/${LOKI_UID}/g" } # Normalize datasource UIDs back to variables for portability normalize_datasource_uids() { local json_content="$1" local prom_uid="$2" local loki_uid="$3" local result="$json_content" if [[ -n "$prom_uid" ]]; then result=$(echo "$result" | sed "s/${prom_uid}/\\\${DS_PROMETHEUS}/g") fi if [[ -n "$loki_uid" ]]; then result=$(echo "$result" | sed "s/${loki_uid}/\\\${DS_LOKI}/g") fi echo "$result" } # --------------------------------------------------------------------------- # Install action # --------------------------------------------------------------------------- action_install() { step "Installing dashboards to Grafana" echo if ! dry_run_guard "Resolve datasource UIDs and push dashboards"; then info "(dry-run) Would install dashboards from ${DASHBOARDS_DIR}" local count=0 while IFS=$'\t' read -r file uid title folder category version description <&3; do if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then continue fi if [[ -n "$FILTER_FILE" && "$file" != "$FILTER_FILE" ]]; then continue fi info "(dry-run) Would install: ${title} -> folder '${folder}'" count=$((count + 1)) done 3< <(read_manifest) info "(dry-run) ${count} dashboard(s) would be installed" return 0 fi resolve_datasource_uids local installed=0 local failed=0 while IFS=$'\t' read -r file uid title folder category version description <&3; do if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then continue fi if [[ -n "$FILTER_FILE" && "$file" != "$FILTER_FILE" ]]; then continue fi local full_path="${DASHBOARDS_DIR}/${file}" if [[ ! -f "$full_path" ]]; then warn "Dashboard file not found: ${full_path}" failed=$((failed + 1)) continue fi # Get or create the target folder local folder_id folder_id=$(ensure_folder "$folder") || { failed=$((failed + 1)); continue; } # Build Grafana API payload: inject datasource UIDs and wrap in envelope local api_payload api_payload=$(python3 - "$full_path" "$PROMETHEUS_UID" "${LOKI_UID:-}" "$folder_id" << 'PYEOF' import json, sys with open(sys.argv[1]) as f: dashboard = json.load(f) # Inject actual datasource UIDs json_str = json.dumps(dashboard) json_str = json_str.replace('${DS_PROMETHEUS}', sys.argv[2]) if sys.argv[3]: json_str = json_str.replace('${DS_LOKI}', sys.argv[3]) dashboard = json.loads(json_str) # Remove Grafana-internal fields for key in ('id', 'version', 'iteration'): dashboard.pop(key, None) payload = { 'dashboard': dashboard, 'folderId': int(sys.argv[4]), 'overwrite': True, 'message': 'Installed by manage-dashboards' } print(json.dumps(payload)) PYEOF ) if [[ -z "$api_payload" ]]; then error "Failed to process dashboard JSON: ${file}" failed=$((failed + 1)) continue fi # Push via API - write payload to temp file to avoid command-line size limits local tmpfile tmpfile=$(mktemp /tmp/dashboard-payload.XXXXXX) echo "$api_payload" > "$tmpfile" incus file push "$tmpfile" "${CLUSTER_REMOTE}:${MONITORING_CONTAINER}/tmp/dashboard-payload.json" >/dev/null 2>&1 local result result=$(grafana_api POST "/api/dashboards/db" \ -H "Content-Type: application/json" \ -d @/tmp/dashboard-payload.json 2>/dev/null) rm -f "$tmpfile" local status_str status_str=$(echo "$result" | python3 -c " import sys, json try: d = json.load(sys.stdin) print(d.get('status', d.get('message', 'unknown'))) except: print('error') " 2>/dev/null || echo "error") if [[ "$status_str" == "success" ]]; then success " ${title} -> ${folder}" installed=$((installed + 1)) else error " ${title}: ${status_str}" detail " Response: ${result}" failed=$((failed + 1)) fi # Clean up temp file in container incus exec "${CLUSTER_REMOTE}:${MONITORING_CONTAINER}" -- rm -f /tmp/dashboard-payload.json 2>/dev/null || true done 3< <(read_manifest) echo if [[ $failed -eq 0 ]]; then success "Installed ${installed} dashboard(s)" else warn "Installed ${installed}, failed ${failed}" fi } # --------------------------------------------------------------------------- # Export action # --------------------------------------------------------------------------- action_export() { step "Exporting dashboards from Grafana" echo resolve_datasource_uids local exported=0 if [[ -n "$FILTER_UID" ]]; then # Export a single dashboard by UID export_single_dashboard "$FILTER_UID" return $? fi # Export all dashboards listed in the manifest while IFS=$'\t' read -r file uid title folder category version description <&3; do if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then continue fi if export_single_dashboard "$uid" "${DASHBOARDS_DIR}/${file}"; then exported=$((exported + 1)) fi done 3< <(read_manifest) echo success "Exported ${exported} dashboard(s)" } export_single_dashboard() { local uid="$1" local output_file="${2:-}" # Fetch from Grafana API local raw_response raw_response=$(grafana_api GET "/api/dashboards/uid/${uid}" 2>/dev/null) local has_dashboard has_dashboard=$(echo "$raw_response" | python3 -c " import sys, json try: d = json.load(sys.stdin) if 'dashboard' in d: print('yes') else: print('no') except: print('no') " 2>/dev/null || echo "no") if [[ "$has_dashboard" != "yes" ]]; then warn "Dashboard '${uid}' not found in Grafana" return 1 fi # If no output file specified, find it in the manifest if [[ -z "$output_file" ]]; then output_file=$(read_manifest | while IFS=$'\t' read -r file muid title folder category version description; do if [[ "$muid" == "$uid" ]]; then echo "${DASHBOARDS_DIR}/${file}" break fi done) fi if [[ -z "$output_file" ]]; then output_file="${DASHBOARDS_DIR}/core/${uid}.json" warn "Dashboard '${uid}' not in manifest, exporting to ${output_file}" fi if ! dry_run_guard "Export ${uid} to ${output_file}"; then return 0 fi # Clean and normalize the dashboard JSON python3 << PYEOF > "$output_file" import json, sys raw = json.loads('''$(echo "$raw_response" | sed "s/'/\\\\'/g")''') dashboard = raw['dashboard'] # Remove Grafana-internal fields for key in ['id', 'version', 'iteration']: dashboard.pop(key, None) # Normalize datasource UIDs to variables json_str = json.dumps(dashboard, indent=2) prom_uid = "${PROMETHEUS_UID}" loki_uid = "${LOKI_UID:-}" if prom_uid: json_str = json_str.replace(prom_uid, r'\${DS_PROMETHEUS}') if loki_uid: json_str = json_str.replace(loki_uid, r'\${DS_LOKI}') # Re-parse to ensure valid JSON dashboard = json.loads(json_str.replace(r'\${DS_PROMETHEUS}', '\${DS_PROMETHEUS}').replace(r'\${DS_LOKI}', '\${DS_LOKI}')) # Write with consistent formatting json_str = json.dumps(dashboard, indent=2) # Restore the variable syntax that json.dumps may have escaped json_str = json_str.replace(r'\${DS_PROMETHEUS}', '\${DS_PROMETHEUS}') json_str = json_str.replace(r'\${DS_LOKI}', '\${DS_LOKI}') print(json_str) PYEOF if [[ $? -eq 0 ]]; then success " Exported: ${uid} -> $(basename "$output_file")" return 0 else error " Failed to export: ${uid}" return 1 fi } # --------------------------------------------------------------------------- # List action # --------------------------------------------------------------------------- action_list() { step "Dashboard Inventory" echo # Try to get installed dashboards from Grafana local installed_json="" installed_json=$(grafana_api GET "/api/search?type=dash-db" 2>/dev/null || echo "[]") printf " ${BOLD}%-35s %-30s %-20s %s${RESET}\n" "UID" "Title" "Folder" "Status" printf " %-35s %-30s %-20s %s\n" "---" "-----" "------" "------" while IFS=$'\t' read -r file uid title folder category version description <&3; do if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then continue fi local full_path="${DASHBOARDS_DIR}/${file}" local local_status="missing" if [[ -f "$full_path" ]]; then local_status="local" fi # Check if installed in Grafana local grafana_status grafana_status=$(echo "$installed_json" | python3 -c " import sys, json try: dashboards = json.load(sys.stdin) for d in dashboards: if d.get('uid') == '${uid}': print('installed') break else: print('not installed') except: print('unknown') " 2>/dev/null || echo "unknown") local status_display if [[ "$local_status" == "missing" ]]; then status_display="${RED}file missing${RESET}" elif [[ "$grafana_status" == "installed" ]]; then status_display="${GREEN}installed${RESET}" elif [[ "$grafana_status" == "not installed" ]]; then status_display="${YELLOW}not installed${RESET}" else status_display="${DIM}${local_status}${RESET}" fi printf " %-35s %-30s %-20s " "$uid" "$title" "$folder" echo -e "$status_display" done 3< <(read_manifest) } # --------------------------------------------------------------------------- # Validate action # --------------------------------------------------------------------------- action_validate() { step "Validating dashboard JSON files" echo local total=0 local passed=0 local warnings=0 local errors=0 while IFS=$'\t' read -r file uid title folder category version description <&3; do if [[ -n "$FILTER_CATEGORY" && "$category" != "$FILTER_CATEGORY" ]]; then continue fi local full_path="${DASHBOARDS_DIR}/${file}" total=$((total + 1)) if [[ ! -f "$full_path" ]]; then error " ${file}: FILE NOT FOUND" errors=$((errors + 1)) continue fi local issues=0 local warns=0 local basename_file basename_file=$(basename "$full_path") # Check 1: Valid JSON if ! python3 -c "import json; json.load(open('${full_path}'))" 2>/dev/null; then error " ${basename_file}: INVALID JSON" errors=$((errors + 1)) continue fi # Run all checks via python local check_result check_result=$(python3 << PYEOF import json with open('${full_path}') as f: d = json.load(f) issues = [] warns = [] # Check 2: UID present and matches manifest json_uid = d.get('uid', '') if not json_uid: issues.append('missing uid field') elif json_uid != '${uid}': issues.append(f'uid mismatch: JSON has "{json_uid}", manifest has "${uid}"') # Check 3: Title present if not d.get('title'): issues.append('missing title field') # Check 4: Description present if not d.get('description'): warns.append('missing description field') # Check 5: No hardcoded datasource UIDs (should use variables) json_str = json.dumps(d) # Known Grafana internal UIDs that are OK safe_uids = ['-- Grafana --'] # Check for UIDs that look like Grafana-generated ones import re # Match patterns like "uid": "PBFA97CFB590B2093" (hex-like, not a variable) uid_pattern = re.compile(r'"uid":\s*"([A-Za-z0-9]{16,})"') for match in uid_pattern.finditer(json_str): found_uid = match.group(1) if found_uid not in safe_uids and not found_uid.startswith('\${'): issues.append(f'hardcoded datasource UID: {found_uid}') break # Check 6: Datasource variable usage if '\${DS_PROMETHEUS}' not in json_str and '\${DS_LOKI}' not in json_str: # Check if it has any datasource references at all if '"type": "prometheus"' in json_str or '"type": "loki"' in json_str: warns.append('no datasource variables found (uses hardcoded UIDs?)') # Check 7: Has panels panels = d.get('panels', []) if not panels: warns.append('no panels defined') # Check 8: Check for id field (should not be present in stored files) if 'id' in d and d['id'] is not None: warns.append('contains "id" field (should be null or absent for portability)') for i in issues: print(f'ERROR:{i}') for w in warns: print(f'WARN:{w}') if not issues: print('OK') PYEOF ) local has_error=false while IFS= read -r line; do case "$line" in ERROR:*) error " ${basename_file}: ${line#ERROR:}" issues=$((issues + 1)) has_error=true ;; WARN:*) warn " ${basename_file}: ${line#WARN:}" warns=$((warns + 1)) ;; OK) ;; esac done <<< "$check_result" if [[ "$has_error" == true ]]; then errors=$((errors + 1)) else if [[ $warns -gt 0 ]]; then warnings=$((warnings + warns)) fi passed=$((passed + 1)) success " ${basename_file}: valid" fi done 3< <(read_manifest) echo info "${total} dashboard(s) checked: ${passed} passed, ${errors} errors, ${warnings} warnings" if [[ $errors -gt 0 ]]; then return 1 fi } # --------------------------------------------------------------------------- # Status action # --------------------------------------------------------------------------- action_status() { step "Dashboard Status (local vs installed)" echo local installed_json installed_json=$(grafana_api GET "/api/search?type=dash-db" 2>/dev/null || echo "[]") printf " ${BOLD}%-30s %-15s %-15s %s${RESET}\n" "Dashboard" "Local" "Grafana" "Match" printf " %-30s %-15s %-15s %s\n" "---------" "-----" "-------" "-----" while IFS=$'\t' read -r file uid title folder category version description <&3; do local full_path="${DASHBOARDS_DIR}/${file}" local local_ver="missing" if [[ -f "$full_path" ]]; then local_ver="v${version}" fi local grafana_ver grafana_ver=$(echo "$installed_json" | python3 -c " import sys, json try: for d in json.load(sys.stdin): if d.get('uid') == '${uid}': print('installed') break else: print('absent') except: print('unknown') " 2>/dev/null || echo "unknown") local match_str if [[ "$local_ver" == "missing" ]]; then match_str="${RED}file missing${RESET}" elif [[ "$grafana_ver" == "absent" ]]; then match_str="${YELLOW}not installed${RESET}" elif [[ "$grafana_ver" == "installed" ]]; then match_str="${GREEN}synced${RESET}" else match_str="${DIM}unknown${RESET}" fi printf " %-30s %-15s %-15s " "$title" "$local_ver" "$grafana_ver" echo -e "$match_str" done 3< <(read_manifest) } # --------------------------------------------------------------------------- # Argument parsing # --------------------------------------------------------------------------- parse_args() { while [[ $# -gt 0 ]]; do case "$1" in --install) ACTION="install";; --export) ACTION="export";; --list) ACTION="list";; --validate) ACTION="validate";; --status) ACTION="status";; --category) FILTER_CATEGORY="$2"; shift;; --file) FILTER_FILE="$2"; shift;; --uid) FILTER_UID="$2"; shift;; -r|--remote) CLUSTER_REMOTE="$2"; shift;; -n|--dry-run) DRY_RUN=true;; -v|--verbose) VERBOSE=true;; -q|--quiet) QUIET=true;; -h|--help) usage; exit 0;; *) error "Unknown option: $1" echo "Run '${SCRIPT_NAME} --help' for usage." exit 1 ;; esac shift done if [[ -z "$ACTION" ]]; then error "No action specified" echo "Run '${SCRIPT_NAME} --help' for usage." exit 1 fi } # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- main() { setup_colors parse_args "$@" echo case "$ACTION" in install) action_install;; export) action_export;; list) action_list;; validate) action_validate;; status) action_status;; esac } main "$@"