#!/usr/bin/env bash # deploy-observability - Deploy and manage the Incus observability stack # # Deploys Prometheus, Grafana, Loki, and Promtail in a monitoring container, # plus node-exporter containers on each cluster node. Configures Prometheus # to scrape Incus metrics (with client certs), HAProxy stats, and node # exporters. Sets up Grafana with datasources and dashboards. # # Usage: deploy-observability [OPTIONS] # Run 'deploy-observability --help' for full usage information. set -euo pipefail readonly VERSION="0.1.0" readonly SCRIPT_NAME="deploy-observability" readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly DASHBOARDS_DIR="${SCRIPT_DIR}/observability-dashboards" # --------------------------------------------------------------------------- # Defaults (overridden by CLI flags) # --------------------------------------------------------------------------- CLUSTER_REMOTE="oc-node-01" OVN_NETWORK="net-prod" MONITORING_IP="10.10.10.70" MONITORING_TARGET="oc-node-02" FORWARD_VIP="192.168.103.201" INCUS_NODES=("192.168.102.140" "192.168.102.141" "192.168.102.142") HAPROXY_IPS=("10.10.10.50" "10.10.10.51") NODE_EXP_IPS=("10.10.10.71" "10.10.10.72" "10.10.10.73") NODE_EXP_TARGETS=("oc-node-01" "oc-node-02" "oc-node-03") HAPROXY_CLUSTER_ID="52" # Runtime flags DRY_RUN=false VERBOSE=false QUIET=false ACTION="" # --------------------------------------------------------------------------- # Color and formatting # --------------------------------------------------------------------------- setup_colors() { if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' BOLD='\033[1m' DIM='\033[2m' RESET='\033[0m' else RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET='' fi } info() { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET} $*"; } success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET} $*"; } warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; } error() { echo -e "${RED}[error]${RESET} $*" >&2; } step() { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; } detail() { [[ "$VERBOSE" != true ]] && return; echo -e "${DIM} $*${RESET}"; } dry_run_guard() { if [[ "$DRY_RUN" == true ]]; then info "(dry-run) $*" return 1 fi return 0 } # --------------------------------------------------------------------------- # Help # --------------------------------------------------------------------------- usage() { cat < Grafana (:3000), Prometheus (:9090) ${BOLD}EXAMPLES${RESET} # Check prerequisites and connectivity ${SCRIPT_NAME} --doctor # Preview what would be deployed ${SCRIPT_NAME} --deploy --dry-run # Full deployment ${SCRIPT_NAME} --deploy # Check health of all components ${SCRIPT_NAME} --status # Tear down everything ${SCRIPT_NAME} --cleanup EOF } # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- container_ref() { echo "${CLUSTER_REMOTE}:$1" } container_exists() { incus info "$(container_ref "$1")" &>/dev/null 2>&1 } container_running() { local state state=$(incus info "$(container_ref "$1")" 2>/dev/null | grep -i '^status:' | awk '{print $2}') [[ "$state" == "RUNNING" || "$state" == "Running" ]] } container_exec() { local name="$1" shift incus exec "$(container_ref "$name")" -- "$@" } wait_for_container_agent() { local name="$1" local timeout="${2:-60}" local elapsed=0 while ! container_exec "$name" true &>/dev/null 2>&1; do sleep 2 elapsed=$((elapsed + 2)) if [[ $elapsed -ge $timeout ]]; then error "Container agent for '${name}' not available after ${timeout}s" return 1 fi done } node_exp_name() { local idx="$1" printf "node-exp-%02d" "$idx" } # --------------------------------------------------------------------------- # Deploy action # --------------------------------------------------------------------------- action_deploy() { step "Deploying observability stack on ${CLUSTER_REMOTE}" echo info " Monitoring: ${MONITORING_IP} (on ${MONITORING_TARGET})" info " Node-exporters: ${NODE_EXP_IPS[*]}" info " Incus nodes: ${INCUS_NODES[*]}" info " HAProxy stats: ${HAPROXY_IPS[*]}" info " Network fwd: ${FORWARD_VIP} -> Grafana/Prometheus" echo if container_exists "monitoring"; then error "Monitoring container already exists" error "Use --status to check health, or --cleanup first." return 1 fi phase_deploy_monitoring phase_install_prometheus phase_install_loki phase_install_grafana phase_deploy_node_exporters phase_create_acl phase_configure_haproxy_exporter phase_create_network_forward phase_upload_dashboards echo success "Observability stack deployed!" echo info " Grafana: http://${FORWARD_VIP}:3000 (admin / admin)" info " Prometheus: http://${FORWARD_VIP}:9090" info " Loki: http://${MONITORING_IP}:3100 (internal)" echo info "Next steps:" info " ${SCRIPT_NAME} --status # verify all targets are up" info " Open Grafana and explore the pre-loaded dashboards" } # --------------------------------------------------------------------------- # Phase 1: Create monitoring container # --------------------------------------------------------------------------- phase_deploy_monitoring() { step "Phase 1: Create monitoring container" if ! dry_run_guard "Launch monitoring on ${OVN_NETWORK} at ${MONITORING_IP}"; then return 0 fi incus launch images:debian/12 "$(container_ref monitoring)" \ -n "$OVN_NETWORK" \ --target "$MONITORING_TARGET" \ -c limits.memory=2GiB \ -d root,size=20GiB # Tell OVN the static IP we want — port security drops traffic from # IPs that OVN didn't assign, so this must be set at the device level. incus config device set "$(container_ref monitoring)" eth0 \ ipv4.address="${MONITORING_IP}" info "Waiting for container agent..." wait_for_container_agent monitoring 60 # Discover upstream DNS from the UPLINK network config. # OVN DHCP advertises this automatically, but we need it for the static # networkd config (which bypasses DHCP). local upstream_dns upstream_dns=$(incus network get "${CLUSTER_REMOTE}:UPLINK" dns.nameservers 2>/dev/null) || true upstream_dns="${upstream_dns:-192.168.100.1}" detail "Using upstream DNS: ${upstream_dns}" # Build DNS lines (handles comma-separated list from UPLINK config) local dns_lines="" IFS=',' read -ra dns_addrs <<< "$upstream_dns" for addr in "${dns_addrs[@]}"; do dns_lines="${dns_lines}DNS=${addr}"$'\n' done # Configure matching static IP inside the container (systemd-networkd # bypasses DHCP, so we must specify DNS explicitly). container_exec monitoring bash -c " cat > /etc/systemd/network/10-static.network << NETCFG [Match] Name=eth0 [Network] Address=${MONITORING_IP}/24 Gateway=10.10.10.1 ${dns_lines}NETCFG systemctl restart systemd-networkd systemd-resolved " sleep 3 success "Monitoring container created at ${MONITORING_IP}" } # --------------------------------------------------------------------------- # Phase 2: Install and configure Prometheus # --------------------------------------------------------------------------- phase_install_prometheus() { step "Phase 2: Install and configure Prometheus" if ! dry_run_guard "Install Prometheus in monitoring container"; then return 0 fi # Copy Incus client certs into the container for metrics scraping local incus_cert="${HOME}/.config/incus/client.crt" local incus_key="${HOME}/.config/incus/client.key" if [[ ! -f "$incus_cert" || ! -f "$incus_key" ]]; then error "Incus client certificates not found at ~/.config/incus/" error "Prometheus needs these to scrape Incus metrics endpoints." error "Install the Incus client and run 'incus remote add' first." return 1 fi incus file push "$incus_cert" "$(container_ref monitoring)/tmp/incus-client.crt" incus file push "$incus_key" "$(container_ref monitoring)/tmp/incus-client.key" container_exec monitoring bash -c " export DEBIAN_FRONTEND=noninteractive # Add Grafana apt repository (provides grafana, loki, promtail) apt-get update -qq apt-get install -y -qq ca-certificates curl gnupg apt-transport-https 2>&1 | tail -1 curl -fsSL https://apt.grafana.com/gpg.key | gpg --dearmor -o /usr/share/keyrings/grafana-archive-keyring.gpg echo 'deb [signed-by=/usr/share/keyrings/grafana-archive-keyring.gpg] https://apt.grafana.com stable main' \ > /etc/apt/sources.list.d/grafana.list apt-get update -qq apt-get install -y -qq prometheus 2>&1 | tail -1 # Place Incus client certs where Prometheus can read them mkdir -p /etc/prometheus/certs cp /tmp/incus-client.crt /etc/prometheus/certs/ cp /tmp/incus-client.key /etc/prometheus/certs/ chown -R prometheus:prometheus /etc/prometheus/certs chmod 600 /etc/prometheus/certs/incus-client.key " # Build scrape targets local incus_targets="" local idx=0 for node in "${INCUS_NODES[@]}"; do if [[ $idx -gt 0 ]]; then incus_targets+=", "; fi incus_targets+="'${node}:8443'" idx=$((idx + 1)) done local haproxy_targets="" idx=0 for ip in "${HAPROXY_IPS[@]}"; do if [[ $idx -gt 0 ]]; then haproxy_targets+=", "; fi haproxy_targets+="'${ip}:8404'" idx=$((idx + 1)) done local node_exp_targets="" idx=0 for ip in "${NODE_EXP_IPS[@]}"; do if [[ $idx -gt 0 ]]; then node_exp_targets+=", "; fi node_exp_targets+="'${ip}:9100'" idx=$((idx + 1)) done container_exec monitoring bash -c "cat > /etc/prometheus/prometheus.yml << 'PROMCFG' global: scrape_interval: 15s evaluation_interval: 15s scrape_configs: - job_name: 'prometheus' static_configs: - targets: ['localhost:9090'] - job_name: 'incus' scheme: https tls_config: cert_file: /etc/prometheus/certs/incus-client.crt key_file: /etc/prometheus/certs/incus-client.key insecure_skip_verify: true metrics_path: /1.0/metrics static_configs: - targets: [${incus_targets}] - job_name: 'haproxy' static_configs: - targets: [${haproxy_targets}] - job_name: 'node-exporter' static_configs: - targets: [${node_exp_targets}] PROMCFG" container_exec monitoring systemctl enable prometheus container_exec monitoring systemctl restart prometheus sleep 2 success "Prometheus installed and configured" } # --------------------------------------------------------------------------- # Phase 3: Install and configure Loki + Promtail # --------------------------------------------------------------------------- phase_install_loki() { step "Phase 3: Install Loki and Promtail" if ! dry_run_guard "Install Loki + Promtail in monitoring container"; then return 0 fi container_exec monitoring bash -c " export DEBIAN_FRONTEND=noninteractive apt-get install -y -qq loki promtail 2>&1 | tail -1 " # Configure Loki container_exec monitoring bash -c "cat > /etc/loki/config.yml << 'LOKICFG' auth_enabled: false server: http_listen_port: 3100 grpc_listen_port: 9096 common: path_prefix: /var/lib/loki storage: filesystem: chunks_directory: /var/lib/loki/chunks rules_directory: /var/lib/loki/rules replication_factor: 1 ring: kvstore: store: inmemory schema_config: configs: - from: 2020-10-24 store: tsdb object_store: filesystem schema: v13 index: prefix: index_ period: 24h limits_config: reject_old_samples: true reject_old_samples_max_age: 168h allow_structured_metadata: false LOKICFG" # Configure Promtail to ship local logs container_exec monitoring bash -c "cat > /etc/promtail/config.yml << 'PTCFG' server: http_listen_port: 9080 grpc_listen_port: 0 positions: filename: /var/lib/promtail/positions.yaml clients: - url: http://localhost:3100/loki/api/v1/push scrape_configs: - job_name: system static_configs: - targets: - localhost labels: job: varlogs host: monitoring __path__: /var/log/**/*.log - job_name: journal journal: max_age: 12h labels: job: systemd-journal host: monitoring relabel_configs: - source_labels: ['__journal__systemd_unit'] target_label: unit PTCFG" # Ensure data directories exist with correct ownership container_exec monitoring bash -c "mkdir -p /var/lib/loki && chown loki:nogroup /var/lib/loki" container_exec monitoring bash -c "mkdir -p /var/lib/promtail && chown promtail:nogroup /var/lib/promtail" # Promtail needs adm group for /var/log access, systemd-journal for journal container_exec monitoring bash -c "usermod -aG adm,systemd-journal promtail" container_exec monitoring systemctl enable loki container_exec monitoring systemctl restart loki sleep 2 container_exec monitoring systemctl enable promtail container_exec monitoring systemctl restart promtail success "Loki and Promtail installed and configured" } # --------------------------------------------------------------------------- # Phase 4: Install and configure Grafana # --------------------------------------------------------------------------- phase_install_grafana() { step "Phase 4: Install and configure Grafana" if ! dry_run_guard "Install Grafana in monitoring container"; then return 0 fi container_exec monitoring bash -c " export DEBIAN_FRONTEND=noninteractive apt-get install -y -qq grafana 2>&1 | tail -1 " # Provision datasources with explicit UIDs for dashboard portability container_exec monitoring bash -c "mkdir -p /etc/grafana/provisioning/datasources cat > /etc/grafana/provisioning/datasources/observability.yaml << 'DSCFG' apiVersion: 1 datasources: - name: Prometheus type: prometheus uid: prometheus access: proxy url: http://localhost:9090 isDefault: true editable: true - name: Loki type: loki uid: loki access: proxy url: http://localhost:3100 editable: true DSCFG" container_exec monitoring systemctl enable grafana-server container_exec monitoring systemctl restart grafana-server sleep 2 success "Grafana installed with Prometheus + Loki datasources" } # --------------------------------------------------------------------------- # Phase 5: Deploy node-exporter containers # --------------------------------------------------------------------------- phase_deploy_node_exporters() { step "Phase 5: Deploy node-exporter containers" local idx=1 for ip in "${NODE_EXP_IPS[@]}"; do local name name=$(node_exp_name "$idx") local target="${NODE_EXP_TARGETS[$((idx - 1))]}" if container_exists "$name"; then if container_running "$name"; then success " ${name} already running" idx=$((idx + 1)) continue fi fi if ! dry_run_guard "Launch ${name} on ${target} at ${ip}"; then idx=$((idx + 1)) continue fi info "Launching ${name} (${ip}) on ${target}..." incus launch images:alpine/3.20 "$(container_ref "$name")" \ -n "$OVN_NETWORK" \ --target "$target" \ -c limits.memory=128MiB \ -c security.privileged=true # Reserve this IP in OVN before switching to static inside the container incus config device set "$(container_ref "$name")" eth0 \ ipv4.address="${ip}" wait_for_container_agent "$name" 30 # Mount host filesystems for full node metrics incus config device add "$(container_ref "$name")" host-proc disk \ source=/proc path=/host/proc readonly=true incus config device add "$(container_ref "$name")" host-sys disk \ source=/sys path=/host/sys readonly=true incus config device add "$(container_ref "$name")" host-root disk \ source=/ path=/host/root readonly=true # Install and configure node_exporter container_exec "$name" sh -c " apk add --no-cache prometheus-node-exporter > /dev/null 2>&1 # Configure to use host mount paths mkdir -p /etc/conf.d cat > /etc/conf.d/node-exporter << 'NEXPCFG' NODE_EXPORTER_OPTS=\"--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/host/root\" NEXPCFG rc-update add node-exporter default 2>/dev/null || true rc-service node-exporter start 2>/dev/null || true " # Configure static IP container_exec "$name" sh -c " cat > /etc/network/interfaces << NETCFG auto lo iface lo inet loopback auto eth0 iface eth0 inet static address ${ip}/24 gateway 10.10.10.1 NETCFG rc-service networking restart 2>/dev/null || true " success " ${name} deployed at ${ip} on ${target}" idx=$((idx + 1)) done success "Node-exporter containers deployed" } # --------------------------------------------------------------------------- # Phase 6: Create ACL and attach to containers # --------------------------------------------------------------------------- phase_create_acl() { step "Phase 6: Create monitoring ACL" if ! dry_run_guard "Create monitoring-allow ACL"; then return 0 fi # Create the ACL if it does not exist if ! incus network acl show "$(container_ref monitoring-allow)" &>/dev/null 2>&1; then incus network acl create "$(container_ref monitoring-allow)" # Add allow-all egress and ingress rules incus network acl rule add "$(container_ref monitoring-allow)" egress \ action=allow incus network acl rule add "$(container_ref monitoring-allow)" ingress \ action=allow success "ACL 'monitoring-allow' created with allow-all rules" else success "ACL 'monitoring-allow' already exists" fi # Attach ACL to monitoring container incus config device set "$(container_ref monitoring)" eth0 \ security.acls=monitoring-allow security.acls.default.ingress.action=allow \ security.acls.default.egress.action=allow 2>/dev/null || true detail "ACL attached to monitoring" # Attach ACL to node-exporter containers local idx=1 for ip in "${NODE_EXP_IPS[@]}"; do local name name=$(node_exp_name "$idx") if container_exists "$name"; then incus config device set "$(container_ref "$name")" eth0 \ security.acls=monitoring-allow security.acls.default.ingress.action=allow \ security.acls.default.egress.action=allow 2>/dev/null || true detail "ACL attached to ${name}" fi idx=$((idx + 1)) done # Changing security.acls can bring the NIC down. Bring all NICs back up. sleep 2 container_exec monitoring bash -c "ip link set eth0 up; systemctl restart systemd-networkd" 2>/dev/null || true local nic_idx=1 for ip in "${NODE_EXP_IPS[@]}"; do local nname nname=$(node_exp_name "$nic_idx") if container_exists "$nname"; then container_exec "$nname" sh -c "ip link set eth0 up; rc-service networking restart" 2>/dev/null || true fi nic_idx=$((nic_idx + 1)) done success "ACL attached to all observability containers" } # --------------------------------------------------------------------------- # Phase 7: Configure HAProxy prometheus exporter # --------------------------------------------------------------------------- phase_configure_haproxy_exporter() { step "Phase 7: Configure HAProxy prometheus exporter + ACL rules" if ! dry_run_guard "Add prometheus exporter to HAProxy containers"; then return 0 fi local haproxy_remote="${CLUSTER_REMOTE}" for n in 01 02; do local cname="ffsdn-haproxy-${HAPROXY_CLUSTER_ID}-${n}" if ! container_exists "$cname"; then warn "HAProxy container ${cname} not found -- skipping" continue fi # Add prometheus exporter line to the stats frontend in haproxy.cfg # Only add if not already present local has_exporter has_exporter=$(container_exec "$cname" grep -c 'prometheus-exporter' /etc/haproxy/haproxy.cfg 2>/dev/null || echo "0") if [[ "$has_exporter" == "0" ]]; then container_exec "$cname" bash -c " # Insert the prometheus exporter line into the stats section sed -i '/^[[:space:]]*stats enable/a\\ http-request use-service prometheus-exporter if { path /metrics }' /etc/haproxy/haproxy.cfg haproxy -c -f /etc/haproxy/haproxy.cfg 2>/dev/null && systemctl reload haproxy || true " success " ${cname}: prometheus exporter route added" else success " ${cname}: prometheus exporter already configured" fi # Add monitoring ingress rules to Aether ACL local acl_name="${HAPROXY_CLUSTER_ID}-${cname}-aether-acl" local has_monitoring_rule has_monitoring_rule=$(incus network acl show "${haproxy_remote}:${acl_name}" 2>/dev/null \ | grep -c "monitoring-prometheus" || echo "0") if [[ "$has_monitoring_rule" == "0" ]]; then incus network acl rule add "${haproxy_remote}:${acl_name}" ingress \ action=allow protocol=tcp source="${MONITORING_IP}/32" destination_port=8404 \ description="monitoring-prometheus" 2>/dev/null || true incus network acl rule add "${haproxy_remote}:${acl_name}" ingress \ action=allow protocol=icmp source="${MONITORING_IP}/32" \ description="monitoring-icmp" 2>/dev/null || true success " ${cname}: monitoring ACL rules added" else success " ${cname}: monitoring ACL rules already present" fi done success "HAProxy prometheus exporter configured" } # --------------------------------------------------------------------------- # Phase 8: Create OVN network forward # --------------------------------------------------------------------------- phase_create_network_forward() { step "Phase 8: Create network forward ${FORWARD_VIP}" if ! dry_run_guard "Create network forward ${FORWARD_VIP}"; then return 0 fi # Check if forward already exists if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then success "Network forward ${FORWARD_VIP} already exists" return 0 fi incus network forward create "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" # Grafana port incus network forward port add "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" \ tcp 3000 "${MONITORING_IP}" 3000 # Prometheus port incus network forward port add "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" \ tcp 9090 "${MONITORING_IP}" 9090 success "Network forward created: ${FORWARD_VIP} -> Grafana(:3000), Prometheus(:9090)" } # --------------------------------------------------------------------------- # Phase 9: Upload Grafana dashboards # --------------------------------------------------------------------------- phase_upload_dashboards() { step "Phase 9: Install Grafana dashboards" if ! dry_run_guard "Install dashboards via manage-dashboards"; then return 0 fi local manage_script="${SCRIPT_DIR}/manage-dashboards" if [[ ! -x "$manage_script" ]]; then warn "manage-dashboards script not found at ${manage_script}" warn "Falling back to direct file upload." if [[ ! -d "$DASHBOARDS_DIR" ]]; then warn "Dashboard directory not found: ${DASHBOARDS_DIR}" return 0 fi local count=0 for dashboard in "${DASHBOARDS_DIR}"/core/*.json; do [[ -f "$dashboard" ]] || continue local basename basename=$(basename "$dashboard") incus file push "$dashboard" "$(container_ref monitoring)/var/lib/grafana/dashboards/${basename}" detail "Uploaded ${basename}" count=$((count + 1)) done if [[ $count -gt 0 ]]; then container_exec monitoring systemctl restart grafana-server 2>/dev/null || true success "${count} dashboard(s) uploaded (direct push)" else info "No dashboard files found in ${DASHBOARDS_DIR}/core/" fi return 0 fi # Wait for Grafana API to be ready info "Waiting for Grafana API..." local elapsed=0 while true; do local code code=$(container_exec monitoring curl -s -o /dev/null -w '%{http_code}' \ http://localhost:3000/api/health 2>/dev/null || echo "000") if [[ "$code" == "200" ]]; then break fi sleep 2 elapsed=$((elapsed + 2)) if [[ $elapsed -ge 30 ]]; then warn "Grafana API not ready after 30s -- falling back to file push" return 0 fi done # Use manage-dashboards to install with proper folder organization local flags="-r ${CLUSTER_REMOTE}" [[ "$VERBOSE" == true ]] && flags="${flags} -v" [[ "$QUIET" == true ]] && flags="${flags} -q" "${manage_script}" --install ${flags} } # --------------------------------------------------------------------------- # Workloads action # --------------------------------------------------------------------------- WORKLOAD_WEB_IP="10.10.10.80" WORKLOAD_API_IP="10.10.10.81" action_workloads() { step "Deploying dummy workloads for metric generation" echo info " workload-web: ${WORKLOAD_WEB_IP} (nginx + CPU/disk/network crons)" info " workload-api: ${WORKLOAD_API_IP} (Python HTTP + memory/network crons)" echo phase_deploy_workload_web phase_deploy_workload_api echo success "Dummy workloads deployed!" info " Metrics will appear within 1-2 minutes" info " workload-web: nginx on ${WORKLOAD_WEB_IP}:80, CPU/disk/net crons" info " workload-api: Python on ${WORKLOAD_API_IP}:8080, memory sawtooth cron" } phase_deploy_workload_web() { step "Deploy workload-web (nginx + varied load)" if container_exists "workload-web"; then if container_running "workload-web"; then success "workload-web already running" return 0 fi fi if ! dry_run_guard "Launch workload-web on ${OVN_NETWORK} at ${WORKLOAD_WEB_IP}"; then return 0 fi incus launch images:alpine/3.20 "$(container_ref workload-web)" \ -n "$OVN_NETWORK" \ --target "oc-node-01" \ -c limits.memory=128MiB incus config device set "$(container_ref workload-web)" eth0 \ ipv4.address="${WORKLOAD_WEB_IP}" wait_for_container_agent workload-web 30 container_exec workload-web sh -c " apk add --no-cache nginx curl gzip coreutils > /dev/null 2>&1 # Configure nginx mkdir -p /var/www/html cat > /var/www/html/index.html << 'WEBHTML'

workload-web

Observability test workload

WEBHTML cat > /etc/nginx/http.d/default.conf << 'NGXCFG' server { listen 80 default_server; root /var/www/html; index index.html; location / { try_files \$uri \$uri/ =404; } } NGXCFG rc-update add nginx default 2>/dev/null || true rc-service nginx start 2>/dev/null || true # Cron: every minute -> 10 HTTP requests (network traffic) # Cron: every 5 min -> gzip random data (CPU spike) # Cron: every 15 min -> dd 5 MiB write+delete (disk I/O burst) cat > /etc/crontabs/root << 'CRONCFG' * * * * * for i in \$(seq 1 10); do wget -q -O /dev/null http://localhost/ 2>/dev/null; done */5 * * * * dd if=/dev/urandom bs=1M count=2 2>/dev/null | gzip > /dev/null 2>&1 */15 * * * * dd if=/dev/urandom of=/tmp/disktest bs=1M count=5 2>/dev/null && rm -f /tmp/disktest CRONCFG rc-update add crond default 2>/dev/null || true rc-service crond start 2>/dev/null || true " # Configure static IP container_exec workload-web sh -c " cat > /etc/network/interfaces << NETCFG auto lo iface lo inet loopback auto eth0 iface eth0 inet static address ${WORKLOAD_WEB_IP}/24 gateway 10.10.10.1 NETCFG rc-service networking restart 2>/dev/null || true " # Attach monitoring ACL incus config device set "$(container_ref workload-web)" eth0 \ security.acls=monitoring-allow security.acls.default.ingress.action=allow \ security.acls.default.egress.action=allow 2>/dev/null || true success "workload-web deployed at ${WORKLOAD_WEB_IP}" } phase_deploy_workload_api() { step "Deploy workload-api (Python HTTP + memory patterns)" if container_exists "workload-api"; then if container_running "workload-api"; then success "workload-api already running" return 0 fi fi if ! dry_run_guard "Launch workload-api on ${OVN_NETWORK} at ${WORKLOAD_API_IP}"; then return 0 fi incus launch images:alpine/3.20 "$(container_ref workload-api)" \ -n "$OVN_NETWORK" \ --target "oc-node-03" \ -c limits.memory=128MiB incus config device set "$(container_ref workload-api)" eth0 \ ipv4.address="${WORKLOAD_API_IP}" wait_for_container_agent workload-api 30 container_exec workload-api sh -c " apk add --no-cache python3 curl coreutils > /dev/null 2>&1 # Create a simple JSON status endpoint mkdir -p /opt/api cat > /opt/api/server.py << 'PYAPI' import http.server import json import time import os class Handler(http.server.BaseHTTPRequestHandler): def do_GET(self): self.send_response(200) self.send_header('Content-Type', 'application/json') self.end_headers() data = { 'status': 'ok', 'uptime': time.time() - START_TIME, 'hostname': os.uname().nodename, 'pid': os.getpid() } self.wfile.write(json.dumps(data).encode()) def log_message(self, format, *args): pass # suppress access logs START_TIME = time.time() http.server.HTTPServer(('0.0.0.0', 8080), Handler).serve_forever() PYAPI # Create memory stress script cat > /opt/api/mem-stress.py << 'PYMEM' import time # Allocate ~20 MiB, hold for 30 seconds, then release data = bytearray(20 * 1024 * 1024) time.sleep(30) del data PYMEM # Start the API server as a background service cat > /etc/init.d/api-server << 'INITSCRIPT' #!/sbin/openrc-run name=\"api-server\" command=\"/usr/bin/python3\" command_args=\"/opt/api/server.py\" command_background=true pidfile=\"/run/api-server.pid\" INITSCRIPT chmod +x /etc/init.d/api-server rc-update add api-server default 2>/dev/null || true rc-service api-server start 2>/dev/null || true # Cron: every 2 min -> curl workload-web (inter-container traffic) # Cron: every 10 min -> allocate 20 MiB for 30s (memory sawtooth) cat > /etc/crontabs/root << 'CRONCFG' */2 * * * * wget -q -O /dev/null http://${WORKLOAD_WEB_IP}/ 2>/dev/null */10 * * * * python3 /opt/api/mem-stress.py 2>/dev/null & CRONCFG rc-update add crond default 2>/dev/null || true rc-service crond start 2>/dev/null || true " # Configure static IP container_exec workload-api sh -c " cat > /etc/network/interfaces << NETCFG auto lo iface lo inet loopback auto eth0 iface eth0 inet static address ${WORKLOAD_API_IP}/24 gateway 10.10.10.1 NETCFG rc-service networking restart 2>/dev/null || true " # Attach monitoring ACL incus config device set "$(container_ref workload-api)" eth0 \ security.acls=monitoring-allow security.acls.default.ingress.action=allow \ security.acls.default.egress.action=allow 2>/dev/null || true success "workload-api deployed at ${WORKLOAD_API_IP}" } # --------------------------------------------------------------------------- # Status action # --------------------------------------------------------------------------- action_status() { step "Observability Stack Status" echo # --- Container status --- info "Containers:" local all_ok=true for cname in monitoring; do if container_exists "$cname"; then if container_running "$cname"; then success " ${cname}: running" else error " ${cname}: stopped" all_ok=false fi else error " ${cname}: not found" all_ok=false fi done local idx=1 for ip in "${NODE_EXP_IPS[@]}"; do local name name=$(node_exp_name "$idx") if container_exists "$name"; then if container_running "$name"; then success " ${name} (${ip}): running" else warn " ${name} (${ip}): stopped" fi else warn " ${name} (${ip}): not found" fi idx=$((idx + 1)) done # Workload containers (optional) for wl in workload-web workload-api; do if container_exists "$wl"; then if container_running "$wl"; then success " ${wl}: running" else warn " ${wl}: stopped" fi fi done if [[ "$all_ok" == false ]]; then echo error "Some containers are missing or stopped." echo fi # --- Service status --- if container_exists monitoring && container_running monitoring; then echo info "Services:" for svc in prometheus grafana-server loki promtail; do local status status=$(container_exec monitoring systemctl is-active "$svc" 2>/dev/null || echo "inactive") if [[ "$status" == "active" ]]; then success " ${svc}: active" else error " ${svc}: ${status}" fi done # --- Prometheus targets --- echo info "Prometheus targets:" local targets_json targets_json=$(curl -s --connect-timeout 5 "http://${MONITORING_IP}:9090/api/v1/targets" 2>/dev/null || echo "") if [[ -n "$targets_json" ]]; then echo "$targets_json" | python3 -c " import sys, json try: data = json.load(sys.stdin) targets = data.get('data', {}).get('activeTargets', []) for t in targets: job = t.get('labels', {}).get('job', 'unknown') instance = t.get('labels', {}).get('instance', 'unknown') health = t.get('health', 'unknown') icon = 'UP' if health == 'up' else 'DOWN' print(f' {job:20s} {instance:30s} {icon}') except Exception as e: print(f' (error parsing targets: {e})') " 2>/dev/null || warn " Could not parse Prometheus targets" else warn " Prometheus not reachable at http://${MONITORING_IP}:9090" fi # --- Grafana health --- echo info "Grafana health:" local grafana_code grafana_code=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 5 \ "http://${MONITORING_IP}:3000/api/health" 2>/dev/null || echo "000") if [[ "$grafana_code" == "200" ]]; then success " Grafana API: healthy (HTTP ${grafana_code})" else error " Grafana API: HTTP ${grafana_code}" fi # --- Loki labels --- echo info "Loki status:" local loki_labels loki_labels=$(curl -s --connect-timeout 5 "http://${MONITORING_IP}:3100/loki/api/v1/labels" 2>/dev/null || echo "") if [[ -n "$loki_labels" ]]; then local label_count label_count=$(echo "$loki_labels" | python3 -c " import sys, json try: data = json.load(sys.stdin) labels = data.get('data', []) print(len(labels)) except: print(0) " 2>/dev/null || echo "0") if [[ "$label_count" -gt 0 ]]; then success " Loki: responding (${label_count} label(s))" else info " Loki: responding (no labels yet -- logs may not have been ingested)" fi else warn " Loki not reachable at http://${MONITORING_IP}:3100" fi fi # --- Network forward --- echo info "Network forward:" if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then success " ${FORWARD_VIP}: configured" local grafana_fwd grafana_fwd=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 5 \ "http://${FORWARD_VIP}:3000/api/health" 2>/dev/null || echo "000") if [[ "$grafana_fwd" == "200" ]]; then success " Grafana via forward: reachable (HTTP ${grafana_fwd})" else warn " Grafana via forward: HTTP ${grafana_fwd}" fi else warn " ${FORWARD_VIP}: not found" fi } # --------------------------------------------------------------------------- # Cleanup action # --------------------------------------------------------------------------- action_cleanup() { step "Cleaning up observability stack" echo local has_anything=false if container_exists monitoring; then has_anything=true; fi local idx=1 for ip in "${NODE_EXP_IPS[@]}"; do if container_exists "$(node_exp_name "$idx")"; then has_anything=true; fi idx=$((idx + 1)) done for wl in workload-web workload-api; do if container_exists "$wl"; then has_anything=true; fi done if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then has_anything=true fi if [[ "$has_anything" == false ]]; then info "Nothing to clean up" return 0 fi warn "This will delete:" if container_exists monitoring; then warn " - monitoring container (Prometheus, Grafana, Loki data)" fi idx=1 for ip in "${NODE_EXP_IPS[@]}"; do local name name=$(node_exp_name "$idx") if container_exists "$name"; then warn " - ${name}" fi idx=$((idx + 1)) done for wl in workload-web workload-api; do if container_exists "$wl"; then warn " - ${wl}" fi done if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then warn " - network forward ${FORWARD_VIP}" fi if incus network acl show "$(container_ref monitoring-allow)" &>/dev/null 2>&1; then warn " - ACL monitoring-allow" fi echo if [[ "$DRY_RUN" == true ]]; then info "(dry-run) Would delete all observability infrastructure" return 0 fi read -r -p "Type 'yes' to confirm: " confirm if [[ "$confirm" != "yes" ]]; then info "Aborted" return 0 fi # Remove monitoring ACL rules from HAProxy ACLs step "Removing monitoring rules from HAProxy ACLs" for n in 01 02; do local cname="ffsdn-haproxy-${HAPROXY_CLUSTER_ID}-${n}" local acl_name="${HAPROXY_CLUSTER_ID}-${cname}-aether-acl" # Remove rules with monitoring descriptions local rule_ids rule_ids=$(incus network acl show "${CLUSTER_REMOTE}:${acl_name}" 2>/dev/null \ | python3 -c " import sys, yaml try: data = yaml.safe_load(sys.stdin) for rule in data.get('ingress', []): desc = rule.get('description', '') if desc.startswith('monitoring-'): # ACL rules are removed by matching all fields, not by ID print(desc) except: pass " 2>/dev/null || true) if [[ -n "$rule_ids" ]]; then for desc in $rule_ids; do incus network acl rule remove "${CLUSTER_REMOTE}:${acl_name}" ingress \ description="$desc" 2>/dev/null || true done info " Removed monitoring rules from ${acl_name}" fi done # Delete containers step "Deleting containers" if container_exists monitoring; then info "Deleting monitoring..." incus delete "$(container_ref monitoring)" --force 2>/dev/null || true success "Deleted monitoring" fi idx=1 for ip in "${NODE_EXP_IPS[@]}"; do local name name=$(node_exp_name "$idx") if container_exists "$name"; then info "Deleting ${name}..." incus delete "$(container_ref "$name")" --force 2>/dev/null || true success "Deleted ${name}" fi idx=$((idx + 1)) done # Delete workload containers for wl in workload-web workload-api; do if container_exists "$wl"; then info "Deleting ${wl}..." incus delete "$(container_ref "$wl")" --force 2>/dev/null || true success "Deleted ${wl}" fi done # Delete network forward step "Removing network forward" if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then incus network forward delete "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" success "Deleted network forward ${FORWARD_VIP}" fi # Delete ACL step "Removing ACL" if incus network acl show "$(container_ref monitoring-allow)" &>/dev/null 2>&1; then incus network acl delete "$(container_ref monitoring-allow)" 2>/dev/null || true success "Deleted ACL monitoring-allow" fi echo success "Cleanup complete" } # --------------------------------------------------------------------------- # Doctor action # --------------------------------------------------------------------------- action_doctor() { step "Diagnosing observability stack" echo if ! container_exists monitoring || ! container_running monitoring; then error "Monitoring container is not running" error "Deploy with: ${SCRIPT_NAME} --deploy" return 1 fi local issues=0 # Check container networking info "Container networking:" if container_exec monitoring ping -c 1 -W 3 10.10.10.1 &>/dev/null; then success " Gateway (10.10.10.1): reachable" else error " Gateway (10.10.10.1): unreachable" issues=$((issues + 1)) fi local idx=1 for ip in "${NODE_EXP_IPS[@]}"; do local name name=$(node_exp_name "$idx") if container_exec monitoring ping -c 1 -W 3 "$ip" &>/dev/null; then success " ${name} (${ip}): reachable" else warn " ${name} (${ip}): unreachable" issues=$((issues + 1)) fi idx=$((idx + 1)) done for ip in "${HAPROXY_IPS[@]}"; do if container_exec monitoring ping -c 1 -W 3 "$ip" &>/dev/null; then success " HAProxy (${ip}): reachable" else warn " HAProxy (${ip}): unreachable" issues=$((issues + 1)) fi done # Check Prometheus scrape errors echo info "Prometheus scrape health:" local targets_json targets_json=$(curl -s --connect-timeout 5 "http://${MONITORING_IP}:9090/api/v1/targets" 2>/dev/null || echo "") if [[ -n "$targets_json" ]]; then echo "$targets_json" | python3 -c " import sys, json try: data = json.load(sys.stdin) targets = data.get('data', {}).get('activeTargets', []) down = [t for t in targets if t.get('health') != 'up'] if down: for t in down: job = t.get('labels', {}).get('job', '?') inst = t.get('labels', {}).get('instance', '?') err = t.get('lastError', 'unknown error') print(f' DOWN: {job} {inst}') print(f' {err}') else: print(f' All {len(targets)} target(s) are UP') except Exception as e: print(f' (parse error: {e})') " 2>/dev/null || warn " Could not query Prometheus" else error " Prometheus not reachable" issues=$((issues + 1)) fi # Check systemd service failures echo info "Service health:" for svc in prometheus grafana-server loki promtail; do local status status=$(container_exec monitoring systemctl is-active "$svc" 2>/dev/null || echo "inactive") if [[ "$status" == "active" ]]; then success " ${svc}: active" else error " ${svc}: ${status}" # Show recent logs local logs logs=$(container_exec monitoring journalctl -u "$svc" --no-pager -n 5 --no-hostname 2>/dev/null || true) if [[ -n "$logs" ]]; then echo "$logs" | while IFS= read -r line; do detail " ${line}" done fi issues=$((issues + 1)) fi done # Check disk space echo info "Disk space (monitoring container):" local disk_usage disk_usage=$(container_exec monitoring df -h / 2>/dev/null | tail -1 || echo "") if [[ -n "$disk_usage" ]]; then local use_pct use_pct=$(echo "$disk_usage" | awk '{print $5}' | tr -d '%') if [[ -n "$use_pct" ]] && [[ "$use_pct" -gt 85 ]]; then warn " Disk usage: ${use_pct}% -- consider increasing disk size" issues=$((issues + 1)) else success " Disk usage: ${use_pct}%" fi else warn " Could not check disk usage" fi echo if [[ $issues -eq 0 ]]; then success "No issues found" else warn "${issues} issue(s) found -- review the output above" fi } # --------------------------------------------------------------------------- # Argument parsing # --------------------------------------------------------------------------- parse_args() { while [[ $# -gt 0 ]]; do case "$1" in --deploy) ACTION="deploy";; --workloads) ACTION="workloads";; --status) ACTION="status";; --cleanup) ACTION="cleanup";; --doctor) ACTION="doctor";; -r|--remote) CLUSTER_REMOTE="$2"; shift;; -n|--dry-run) DRY_RUN=true;; -v|--verbose) VERBOSE=true;; -q|--quiet) QUIET=true;; -h|--help) usage; exit 0;; *) error "Unknown option: $1" echo "Run '${SCRIPT_NAME} --help' for usage." exit 1 ;; esac shift done if [[ -z "$ACTION" ]]; then error "No action specified" echo "Run '${SCRIPT_NAME} --help' for usage." exit 1 fi } # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- main() { setup_colors parse_args "$@" echo case "$ACTION" in deploy) action_deploy;; workloads) action_workloads;; status) action_status;; cleanup) action_cleanup;; doctor) action_doctor;; esac } main "$@"