diff --git a/incusos/deploy-observability b/incusos/deploy-observability index a4356dd..396893c 100755 --- a/incusos/deploy-observability +++ b/incusos/deploy-observability @@ -26,18 +26,25 @@ MONITORING_IP="10.10.10.70" MONITORING_TARGET="oc-node-02" FORWARD_VIP="192.168.103.201" -INCUS_NODES=("192.168.102.140" "192.168.102.141" "192.168.102.142") +# Fallback defaults — used when cluster is unreachable (e.g., --dry-run) +DEFAULT_INCUS_NODES=("192.168.102.140" "192.168.102.141" "192.168.102.142") +DEFAULT_NODE_EXP_TARGETS=("oc-node-01" "oc-node-02" "oc-node-03") +DEFAULT_NODE_EXP_IPS=("10.10.10.71" "10.10.10.72" "10.10.10.73") + +INCUS_NODES=() +NODE_EXP_TARGETS=() +NODE_EXP_IPS=() HAPROXY_IPS=("10.10.10.50" "10.10.10.51") -NODE_EXP_IPS=("10.10.10.71" "10.10.10.72" "10.10.10.73") -NODE_EXP_TARGETS=("oc-node-01" "oc-node-02" "oc-node-03") HAPROXY_CLUSTER_ID="52" +OC_SERVER_IP="" # Runtime flags DRY_RUN=false VERBOSE=false QUIET=false ACTION="" +MANUAL_NODES="" # --------------------------------------------------------------------------- # Color and formatting @@ -95,6 +102,8 @@ ${BOLD}ACTIONS${RESET} ${BOLD}OPTIONS${RESET} -r, --remote NAME Incus remote (default: ${CLUSTER_REMOTE}) + --nodes NODE1,NODE2 Manual node list (skip auto-discovery) + --oc-server IP Monitor oc-server at this IP (proxy device for 9100) -n, --dry-run Preview actions without executing -v, --verbose Show detailed output -q, --quiet Suppress informational output @@ -108,7 +117,7 @@ ${BOLD}COMPONENTS${RESET} Promtail - log shipping agent Node-exporter containers (Alpine, 128 MiB, privileged): - node-exp-01/02/03 on cluster nodes, :9100 + Auto-discovered from cluster (or --nodes override), :9100 Network forward: ${FORWARD_VIP} -> Grafana (:3000), Prometheus (:9090) @@ -120,9 +129,15 @@ ${BOLD}EXAMPLES${RESET} # Preview what would be deployed ${SCRIPT_NAME} --deploy --dry-run - # Full deployment + # Full deployment (auto-discovers cluster nodes) ${SCRIPT_NAME} --deploy + # Deploy with oc-server monitoring + ${SCRIPT_NAME} --deploy --oc-server 192.168.102.120 + + # Deploy with manual node list + ${SCRIPT_NAME} --deploy --nodes oc-node-01,oc-node-02,oc-node-03 + # Check health of all components ${SCRIPT_NAME} --status @@ -174,6 +189,99 @@ node_exp_name() { printf "node-exp-%02d" "$idx" } +# --------------------------------------------------------------------------- +# Cluster auto-discovery +# --------------------------------------------------------------------------- + +discover_cluster_nodes() { + # Manual override via --nodes takes priority + if [[ -n "$MANUAL_NODES" ]]; then + info "Using manually specified nodes: ${MANUAL_NODES}" + INCUS_NODES=() + NODE_EXP_TARGETS=() + NODE_EXP_IPS=() + local exp_octet=71 + IFS=',' read -ra names <<< "$MANUAL_NODES" + for name in "${names[@]}"; do + # Look up address from Incus remote + local addr + addr=$(incus cluster list "${CLUSTER_REMOTE}:" --format csv 2>/dev/null \ + | grep "^${name}," | head -1 | cut -d',' -f2 | sed 's|https://||;s|:.*||') || true + if [[ -z "$addr" ]]; then + warn "Could not resolve address for node '${name}' — skipping" + continue + fi + INCUS_NODES+=("$addr") + NODE_EXP_TARGETS+=("$name") + NODE_EXP_IPS+=("10.10.10.${exp_octet}") + exp_octet=$((exp_octet + 1)) + done + return 0 + fi + + # Auto-discover from cluster + local csv + csv=$(incus cluster list "${CLUSTER_REMOTE}:" --format csv 2>/dev/null) || true + + if [[ -z "$csv" ]]; then + if [[ "$DRY_RUN" == true ]]; then + info "Cluster not reachable — using fallback defaults for dry-run" + INCUS_NODES=("${DEFAULT_INCUS_NODES[@]}") + NODE_EXP_TARGETS=("${DEFAULT_NODE_EXP_TARGETS[@]}") + NODE_EXP_IPS=("${DEFAULT_NODE_EXP_IPS[@]}") + return 0 + fi + error "Cannot reach cluster via remote '${CLUSTER_REMOTE}'" + error "Check that 'incus remote list' includes '${CLUSTER_REMOTE}'." + return 1 + fi + + INCUS_NODES=() + NODE_EXP_TARGETS=() + NODE_EXP_IPS=() + + local exp_octet=71 + while IFS=',' read -r name url roles _rest; do + # Extract IP from URL: https://192.168.102.140:8443 → 192.168.102.140 + local addr="${url#https://}" + addr="${addr%%:*}" + INCUS_NODES+=("$addr") + NODE_EXP_TARGETS+=("$name") + NODE_EXP_IPS+=("10.10.10.${exp_octet}") + exp_octet=$((exp_octet + 1)) + done <<< "$csv" + + info "Discovered ${#INCUS_NODES[@]} cluster node(s): ${NODE_EXP_TARGETS[*]}" +} + +# --------------------------------------------------------------------------- +# OC server helpers +# --------------------------------------------------------------------------- + +oc_server_remote_exists() { + incus remote list --format csv 2>/dev/null | grep -q "^oc-server," +} + +oc_server_container_ref() { + echo "oc-server:$1" +} + +oc_server_container_exists() { + incus info "$(oc_server_container_ref "$1")" &>/dev/null 2>&1 +} + +oc_server_container_running() { + local state + state=$(incus info "$(oc_server_container_ref "$1")" 2>/dev/null | grep -i '^status:' | awk '{print $2}') + [[ "$state" == "RUNNING" || "$state" == "Running" ]] +} + +oc_server_container_exec() { + local name="$1" + shift + incus exec "$(oc_server_container_ref "$name")" -- "$@" +} + # --------------------------------------------------------------------------- # Deploy action # --------------------------------------------------------------------------- @@ -186,6 +294,7 @@ action_deploy() { info " Incus nodes: ${INCUS_NODES[*]}" info " HAProxy stats: ${HAPROXY_IPS[*]}" info " Network fwd: ${FORWARD_VIP} -> Grafana/Prometheus" + [[ -n "$OC_SERVER_IP" ]] && info " OC server: ${OC_SERVER_IP} (proxy :9100)" echo if container_exists "monitoring"; then @@ -199,6 +308,7 @@ action_deploy() { phase_install_loki phase_install_grafana phase_deploy_node_exporters + phase_deploy_oc_server_exporter phase_create_acl phase_configure_haproxy_exporter phase_create_network_forward @@ -328,6 +438,11 @@ phase_install_prometheus() { incus_targets+="'${node}:8443'" idx=$((idx + 1)) done + # Add oc-server to Incus API scraping + if [[ -n "$OC_SERVER_IP" ]]; then + if [[ $idx -gt 0 ]]; then incus_targets+=", "; fi + incus_targets+="'${OC_SERVER_IP}:8443'" + fi local haproxy_targets="" idx=0 @@ -347,6 +462,13 @@ phase_install_prometheus() { nodename: '${target_name}'" idx=$((idx + 1)) done + # Add oc-server node-exporter (via proxy device on host IP) + if [[ -n "$OC_SERVER_IP" ]]; then + node_exp_config+=" + - targets: ['${OC_SERVER_IP}:9100'] + labels: + nodename: 'oc-server'" + fi container_exec monitoring bash -c "cat > /etc/prometheus/prometheus.yml << 'PROMCFG' global: @@ -596,7 +718,7 @@ phase_deploy_node_exporters() { # Configure to use host mount paths mkdir -p /etc/conf.d cat > /etc/conf.d/node-exporter << 'NEXPCFG' -NODE_EXPORTER_OPTS=\"--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/host/root\" +ARGS=\"--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/host/root\" NEXPCFG rc-update add node-exporter default 2>/dev/null || true @@ -624,6 +746,86 @@ NETCFG success "Node-exporter containers deployed" } +# --------------------------------------------------------------------------- +# Phase 5b: Deploy node-exporter on oc-server (if --oc-server set) +# --------------------------------------------------------------------------- + +phase_deploy_oc_server_exporter() { + [[ -z "$OC_SERVER_IP" ]] && return 0 + + step "Phase 5b: Deploy node-exporter on oc-server (${OC_SERVER_IP})" + + # Validate oc-server remote exists + if ! oc_server_remote_exists; then + error "Incus remote 'oc-server' not found." + error "Add it with:" + error " incus remote add oc-server https://${OC_SERVER_IP}:8443 --accept-certificate" + return 1 + fi + + local name="node-exp-oc-server" + + if oc_server_container_exists "$name"; then + if oc_server_container_running "$name"; then + success " ${name} already running on oc-server" + return 0 + fi + fi + + if ! dry_run_guard "Launch ${name} on oc-server at ${OC_SERVER_IP}"; then + return 0 + fi + + info "Launching ${name} on oc-server..." + incus launch images:alpine/3.20 "$(oc_server_container_ref "$name")" \ + -c limits.memory=128MiB \ + -c security.privileged=true + + wait_for_container_agent_oc_server "$name" 30 + + # Mount host filesystems for full node metrics + incus config device add "$(oc_server_container_ref "$name")" host-proc disk \ + source=/proc path=/host/proc readonly=true + incus config device add "$(oc_server_container_ref "$name")" host-sys disk \ + source=/sys path=/host/sys readonly=true + incus config device add "$(oc_server_container_ref "$name")" host-root disk \ + source=/ path=/host/root readonly=true + + # Expose 9100 on the host IP via proxy device (oc-server is on incusbr0, + # not net-prod, so Prometheus reaches it via the host address) + incus config device add "$(oc_server_container_ref "$name")" proxy9100 proxy \ + listen=tcp:${OC_SERVER_IP}:9100 connect=tcp:127.0.0.1:9100 + + # Install and configure node_exporter + oc_server_container_exec "$name" sh -c " + apk add --no-cache prometheus-node-exporter > /dev/null 2>&1 + + mkdir -p /etc/conf.d + cat > /etc/conf.d/node-exporter << 'NEXPCFG' +ARGS=\"--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/host/root\" +NEXPCFG + + rc-update add node-exporter default 2>/dev/null || true + rc-service node-exporter start 2>/dev/null || true + " + + success " ${name} deployed on oc-server with proxy ${OC_SERVER_IP}:9100" +} + +wait_for_container_agent_oc_server() { + local name="$1" + local timeout="${2:-60}" + local elapsed=0 + while ! oc_server_container_exec "$name" true &>/dev/null 2>&1; do + sleep 2 + elapsed=$((elapsed + 2)) + if [[ $elapsed -ge $timeout ]]; then + error "Container agent for '${name}' on oc-server not available after ${timeout}s" + return 1 + fi + done +} + # --------------------------------------------------------------------------- # Phase 6: Create ACL and attach to containers # --------------------------------------------------------------------------- @@ -1091,6 +1293,19 @@ action_status() { idx=$((idx + 1)) done + # OC server node-exporter (optional) + if [[ -n "$OC_SERVER_IP" ]] && oc_server_remote_exists; then + if oc_server_container_exists "node-exp-oc-server"; then + if oc_server_container_running "node-exp-oc-server"; then + success " node-exp-oc-server (${OC_SERVER_IP}:9100): running" + else + warn " node-exp-oc-server (${OC_SERVER_IP}:9100): stopped" + fi + else + warn " node-exp-oc-server: not found on oc-server" + fi + fi + # Workload containers (optional) for wl in workload-web workload-api; do if container_exists "$wl"; then @@ -1221,6 +1436,9 @@ action_cleanup() { for wl in workload-web workload-api; do if container_exists "$wl"; then has_anything=true; fi done + if [[ -n "$OC_SERVER_IP" ]] && oc_server_remote_exists && oc_server_container_exists "node-exp-oc-server"; then + has_anything=true + fi if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then has_anything=true fi @@ -1243,6 +1461,9 @@ action_cleanup() { fi idx=$((idx + 1)) done + if [[ -n "$OC_SERVER_IP" ]] && oc_server_remote_exists && oc_server_container_exists "node-exp-oc-server"; then + warn " - node-exp-oc-server (on oc-server)" + fi for wl in workload-web workload-api; do if container_exists "$wl"; then warn " - ${wl}" @@ -1315,6 +1536,13 @@ except: pass idx=$((idx + 1)) done + # Delete oc-server node-exporter + if [[ -n "$OC_SERVER_IP" ]] && oc_server_remote_exists && oc_server_container_exists "node-exp-oc-server"; then + info "Deleting node-exp-oc-server on oc-server..." + incus delete "$(oc_server_container_ref "node-exp-oc-server")" --force 2>/dev/null || true + success "Deleted node-exp-oc-server" + fi + # Delete workload containers for wl in workload-web workload-api; do if container_exists "$wl"; then @@ -1479,6 +1707,8 @@ parse_args() { --cleanup) ACTION="cleanup";; --doctor) ACTION="doctor";; -r|--remote) CLUSTER_REMOTE="$2"; shift;; + --nodes) MANUAL_NODES="$2"; shift;; + --oc-server) OC_SERVER_IP="$2"; shift;; -n|--dry-run) DRY_RUN=true;; -v|--verbose) VERBOSE=true;; -q|--quiet) QUIET=true;; @@ -1507,6 +1737,9 @@ main() { setup_colors parse_args "$@" + # Discover cluster nodes (populates INCUS_NODES, NODE_EXP_TARGETS, NODE_EXP_IPS) + discover_cluster_nodes + echo case "$ACTION" in deploy) action_deploy;;