Add cluster auto-discovery, oc-server monitoring, fix node-exporter RAM bug
Three changes to deploy-observability: 1. Fix node-exporter showing 128 MiB RAM: the conf.d template wrote NODE_EXPORTER_OPTS but Alpine's init script reads ARGS, so node-exporter started with no flags and read container /proc instead of host /proc. Changed to ARGS. 2. Auto-discover cluster nodes from `incus cluster list` instead of hardcoded arrays. Old values become fallback defaults for --dry-run. Added --nodes flag for manual override. 3. Add --oc-server flag to monitor the standalone OC management VM. Deploys node-exporter with host mounts and a proxy device to expose 9100 on the host IP (oc-server uses incusbr0, not OVN). Adds to both Incus API and node-exporter Prometheus targets. Updated status and cleanup to handle the oc-server container. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
03cad2dc93
commit
be3f989312
|
|
@ -26,18 +26,25 @@ MONITORING_IP="10.10.10.70"
|
|||
MONITORING_TARGET="oc-node-02"
|
||||
FORWARD_VIP="192.168.103.201"
|
||||
|
||||
INCUS_NODES=("192.168.102.140" "192.168.102.141" "192.168.102.142")
|
||||
# Fallback defaults — used when cluster is unreachable (e.g., --dry-run)
|
||||
DEFAULT_INCUS_NODES=("192.168.102.140" "192.168.102.141" "192.168.102.142")
|
||||
DEFAULT_NODE_EXP_TARGETS=("oc-node-01" "oc-node-02" "oc-node-03")
|
||||
DEFAULT_NODE_EXP_IPS=("10.10.10.71" "10.10.10.72" "10.10.10.73")
|
||||
|
||||
INCUS_NODES=()
|
||||
NODE_EXP_TARGETS=()
|
||||
NODE_EXP_IPS=()
|
||||
HAPROXY_IPS=("10.10.10.50" "10.10.10.51")
|
||||
NODE_EXP_IPS=("10.10.10.71" "10.10.10.72" "10.10.10.73")
|
||||
NODE_EXP_TARGETS=("oc-node-01" "oc-node-02" "oc-node-03")
|
||||
|
||||
HAPROXY_CLUSTER_ID="52"
|
||||
OC_SERVER_IP=""
|
||||
|
||||
# Runtime flags
|
||||
DRY_RUN=false
|
||||
VERBOSE=false
|
||||
QUIET=false
|
||||
ACTION=""
|
||||
MANUAL_NODES=""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Color and formatting
|
||||
|
|
@ -95,6 +102,8 @@ ${BOLD}ACTIONS${RESET}
|
|||
|
||||
${BOLD}OPTIONS${RESET}
|
||||
-r, --remote NAME Incus remote (default: ${CLUSTER_REMOTE})
|
||||
--nodes NODE1,NODE2 Manual node list (skip auto-discovery)
|
||||
--oc-server IP Monitor oc-server at this IP (proxy device for 9100)
|
||||
-n, --dry-run Preview actions without executing
|
||||
-v, --verbose Show detailed output
|
||||
-q, --quiet Suppress informational output
|
||||
|
|
@ -108,7 +117,7 @@ ${BOLD}COMPONENTS${RESET}
|
|||
Promtail - log shipping agent
|
||||
|
||||
Node-exporter containers (Alpine, 128 MiB, privileged):
|
||||
node-exp-01/02/03 on cluster nodes, :9100
|
||||
Auto-discovered from cluster (or --nodes override), :9100
|
||||
|
||||
Network forward:
|
||||
${FORWARD_VIP} -> Grafana (:3000), Prometheus (:9090)
|
||||
|
|
@ -120,9 +129,15 @@ ${BOLD}EXAMPLES${RESET}
|
|||
# Preview what would be deployed
|
||||
${SCRIPT_NAME} --deploy --dry-run
|
||||
|
||||
# Full deployment
|
||||
# Full deployment (auto-discovers cluster nodes)
|
||||
${SCRIPT_NAME} --deploy
|
||||
|
||||
# Deploy with oc-server monitoring
|
||||
${SCRIPT_NAME} --deploy --oc-server 192.168.102.120
|
||||
|
||||
# Deploy with manual node list
|
||||
${SCRIPT_NAME} --deploy --nodes oc-node-01,oc-node-02,oc-node-03
|
||||
|
||||
# Check health of all components
|
||||
${SCRIPT_NAME} --status
|
||||
|
||||
|
|
@ -174,6 +189,99 @@ node_exp_name() {
|
|||
printf "node-exp-%02d" "$idx"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cluster auto-discovery
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
discover_cluster_nodes() {
|
||||
# Manual override via --nodes takes priority
|
||||
if [[ -n "$MANUAL_NODES" ]]; then
|
||||
info "Using manually specified nodes: ${MANUAL_NODES}"
|
||||
INCUS_NODES=()
|
||||
NODE_EXP_TARGETS=()
|
||||
NODE_EXP_IPS=()
|
||||
local exp_octet=71
|
||||
IFS=',' read -ra names <<< "$MANUAL_NODES"
|
||||
for name in "${names[@]}"; do
|
||||
# Look up address from Incus remote
|
||||
local addr
|
||||
addr=$(incus cluster list "${CLUSTER_REMOTE}:" --format csv 2>/dev/null \
|
||||
| grep "^${name}," | head -1 | cut -d',' -f2 | sed 's|https://||;s|:.*||') || true
|
||||
if [[ -z "$addr" ]]; then
|
||||
warn "Could not resolve address for node '${name}' — skipping"
|
||||
continue
|
||||
fi
|
||||
INCUS_NODES+=("$addr")
|
||||
NODE_EXP_TARGETS+=("$name")
|
||||
NODE_EXP_IPS+=("10.10.10.${exp_octet}")
|
||||
exp_octet=$((exp_octet + 1))
|
||||
done
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Auto-discover from cluster
|
||||
local csv
|
||||
csv=$(incus cluster list "${CLUSTER_REMOTE}:" --format csv 2>/dev/null) || true
|
||||
|
||||
if [[ -z "$csv" ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
info "Cluster not reachable — using fallback defaults for dry-run"
|
||||
INCUS_NODES=("${DEFAULT_INCUS_NODES[@]}")
|
||||
NODE_EXP_TARGETS=("${DEFAULT_NODE_EXP_TARGETS[@]}")
|
||||
NODE_EXP_IPS=("${DEFAULT_NODE_EXP_IPS[@]}")
|
||||
return 0
|
||||
fi
|
||||
error "Cannot reach cluster via remote '${CLUSTER_REMOTE}'"
|
||||
error "Check that 'incus remote list' includes '${CLUSTER_REMOTE}'."
|
||||
return 1
|
||||
fi
|
||||
|
||||
INCUS_NODES=()
|
||||
NODE_EXP_TARGETS=()
|
||||
NODE_EXP_IPS=()
|
||||
|
||||
local exp_octet=71
|
||||
while IFS=',' read -r name url roles _rest; do
|
||||
# Extract IP from URL: https://192.168.102.140:8443 → 192.168.102.140
|
||||
local addr="${url#https://}"
|
||||
addr="${addr%%:*}"
|
||||
INCUS_NODES+=("$addr")
|
||||
NODE_EXP_TARGETS+=("$name")
|
||||
NODE_EXP_IPS+=("10.10.10.${exp_octet}")
|
||||
exp_octet=$((exp_octet + 1))
|
||||
done <<< "$csv"
|
||||
|
||||
info "Discovered ${#INCUS_NODES[@]} cluster node(s): ${NODE_EXP_TARGETS[*]}"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OC server helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
oc_server_remote_exists() {
|
||||
incus remote list --format csv 2>/dev/null | grep -q "^oc-server,"
|
||||
}
|
||||
|
||||
oc_server_container_ref() {
|
||||
echo "oc-server:$1"
|
||||
}
|
||||
|
||||
oc_server_container_exists() {
|
||||
incus info "$(oc_server_container_ref "$1")" &>/dev/null 2>&1
|
||||
}
|
||||
|
||||
oc_server_container_running() {
|
||||
local state
|
||||
state=$(incus info "$(oc_server_container_ref "$1")" 2>/dev/null | grep -i '^status:' | awk '{print $2}')
|
||||
[[ "$state" == "RUNNING" || "$state" == "Running" ]]
|
||||
}
|
||||
|
||||
oc_server_container_exec() {
|
||||
local name="$1"
|
||||
shift
|
||||
incus exec "$(oc_server_container_ref "$name")" -- "$@"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Deploy action
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -186,6 +294,7 @@ action_deploy() {
|
|||
info " Incus nodes: ${INCUS_NODES[*]}"
|
||||
info " HAProxy stats: ${HAPROXY_IPS[*]}"
|
||||
info " Network fwd: ${FORWARD_VIP} -> Grafana/Prometheus"
|
||||
[[ -n "$OC_SERVER_IP" ]] && info " OC server: ${OC_SERVER_IP} (proxy :9100)"
|
||||
echo
|
||||
|
||||
if container_exists "monitoring"; then
|
||||
|
|
@ -199,6 +308,7 @@ action_deploy() {
|
|||
phase_install_loki
|
||||
phase_install_grafana
|
||||
phase_deploy_node_exporters
|
||||
phase_deploy_oc_server_exporter
|
||||
phase_create_acl
|
||||
phase_configure_haproxy_exporter
|
||||
phase_create_network_forward
|
||||
|
|
@ -328,6 +438,11 @@ phase_install_prometheus() {
|
|||
incus_targets+="'${node}:8443'"
|
||||
idx=$((idx + 1))
|
||||
done
|
||||
# Add oc-server to Incus API scraping
|
||||
if [[ -n "$OC_SERVER_IP" ]]; then
|
||||
if [[ $idx -gt 0 ]]; then incus_targets+=", "; fi
|
||||
incus_targets+="'${OC_SERVER_IP}:8443'"
|
||||
fi
|
||||
|
||||
local haproxy_targets=""
|
||||
idx=0
|
||||
|
|
@ -347,6 +462,13 @@ phase_install_prometheus() {
|
|||
nodename: '${target_name}'"
|
||||
idx=$((idx + 1))
|
||||
done
|
||||
# Add oc-server node-exporter (via proxy device on host IP)
|
||||
if [[ -n "$OC_SERVER_IP" ]]; then
|
||||
node_exp_config+="
|
||||
- targets: ['${OC_SERVER_IP}:9100']
|
||||
labels:
|
||||
nodename: 'oc-server'"
|
||||
fi
|
||||
|
||||
container_exec monitoring bash -c "cat > /etc/prometheus/prometheus.yml << 'PROMCFG'
|
||||
global:
|
||||
|
|
@ -596,7 +718,7 @@ phase_deploy_node_exporters() {
|
|||
# Configure to use host mount paths
|
||||
mkdir -p /etc/conf.d
|
||||
cat > /etc/conf.d/node-exporter << 'NEXPCFG'
|
||||
NODE_EXPORTER_OPTS=\"--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/host/root\"
|
||||
ARGS=\"--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/host/root\"
|
||||
NEXPCFG
|
||||
|
||||
rc-update add node-exporter default 2>/dev/null || true
|
||||
|
|
@ -624,6 +746,86 @@ NETCFG
|
|||
success "Node-exporter containers deployed"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 5b: Deploy node-exporter on oc-server (if --oc-server set)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
phase_deploy_oc_server_exporter() {
|
||||
[[ -z "$OC_SERVER_IP" ]] && return 0
|
||||
|
||||
step "Phase 5b: Deploy node-exporter on oc-server (${OC_SERVER_IP})"
|
||||
|
||||
# Validate oc-server remote exists
|
||||
if ! oc_server_remote_exists; then
|
||||
error "Incus remote 'oc-server' not found."
|
||||
error "Add it with:"
|
||||
error " incus remote add oc-server https://${OC_SERVER_IP}:8443 --accept-certificate"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local name="node-exp-oc-server"
|
||||
|
||||
if oc_server_container_exists "$name"; then
|
||||
if oc_server_container_running "$name"; then
|
||||
success " ${name} already running on oc-server"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! dry_run_guard "Launch ${name} on oc-server at ${OC_SERVER_IP}"; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
info "Launching ${name} on oc-server..."
|
||||
incus launch images:alpine/3.20 "$(oc_server_container_ref "$name")" \
|
||||
-c limits.memory=128MiB \
|
||||
-c security.privileged=true
|
||||
|
||||
wait_for_container_agent_oc_server "$name" 30
|
||||
|
||||
# Mount host filesystems for full node metrics
|
||||
incus config device add "$(oc_server_container_ref "$name")" host-proc disk \
|
||||
source=/proc path=/host/proc readonly=true
|
||||
incus config device add "$(oc_server_container_ref "$name")" host-sys disk \
|
||||
source=/sys path=/host/sys readonly=true
|
||||
incus config device add "$(oc_server_container_ref "$name")" host-root disk \
|
||||
source=/ path=/host/root readonly=true
|
||||
|
||||
# Expose 9100 on the host IP via proxy device (oc-server is on incusbr0,
|
||||
# not net-prod, so Prometheus reaches it via the host address)
|
||||
incus config device add "$(oc_server_container_ref "$name")" proxy9100 proxy \
|
||||
listen=tcp:${OC_SERVER_IP}:9100 connect=tcp:127.0.0.1:9100
|
||||
|
||||
# Install and configure node_exporter
|
||||
oc_server_container_exec "$name" sh -c "
|
||||
apk add --no-cache prometheus-node-exporter > /dev/null 2>&1
|
||||
|
||||
mkdir -p /etc/conf.d
|
||||
cat > /etc/conf.d/node-exporter << 'NEXPCFG'
|
||||
ARGS=\"--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/host/root\"
|
||||
NEXPCFG
|
||||
|
||||
rc-update add node-exporter default 2>/dev/null || true
|
||||
rc-service node-exporter start 2>/dev/null || true
|
||||
"
|
||||
|
||||
success " ${name} deployed on oc-server with proxy ${OC_SERVER_IP}:9100"
|
||||
}
|
||||
|
||||
wait_for_container_agent_oc_server() {
|
||||
local name="$1"
|
||||
local timeout="${2:-60}"
|
||||
local elapsed=0
|
||||
while ! oc_server_container_exec "$name" true &>/dev/null 2>&1; do
|
||||
sleep 2
|
||||
elapsed=$((elapsed + 2))
|
||||
if [[ $elapsed -ge $timeout ]]; then
|
||||
error "Container agent for '${name}' on oc-server not available after ${timeout}s"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 6: Create ACL and attach to containers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -1091,6 +1293,19 @@ action_status() {
|
|||
idx=$((idx + 1))
|
||||
done
|
||||
|
||||
# OC server node-exporter (optional)
|
||||
if [[ -n "$OC_SERVER_IP" ]] && oc_server_remote_exists; then
|
||||
if oc_server_container_exists "node-exp-oc-server"; then
|
||||
if oc_server_container_running "node-exp-oc-server"; then
|
||||
success " node-exp-oc-server (${OC_SERVER_IP}:9100): running"
|
||||
else
|
||||
warn " node-exp-oc-server (${OC_SERVER_IP}:9100): stopped"
|
||||
fi
|
||||
else
|
||||
warn " node-exp-oc-server: not found on oc-server"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Workload containers (optional)
|
||||
for wl in workload-web workload-api; do
|
||||
if container_exists "$wl"; then
|
||||
|
|
@ -1221,6 +1436,9 @@ action_cleanup() {
|
|||
for wl in workload-web workload-api; do
|
||||
if container_exists "$wl"; then has_anything=true; fi
|
||||
done
|
||||
if [[ -n "$OC_SERVER_IP" ]] && oc_server_remote_exists && oc_server_container_exists "node-exp-oc-server"; then
|
||||
has_anything=true
|
||||
fi
|
||||
if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
|
||||
has_anything=true
|
||||
fi
|
||||
|
|
@ -1243,6 +1461,9 @@ action_cleanup() {
|
|||
fi
|
||||
idx=$((idx + 1))
|
||||
done
|
||||
if [[ -n "$OC_SERVER_IP" ]] && oc_server_remote_exists && oc_server_container_exists "node-exp-oc-server"; then
|
||||
warn " - node-exp-oc-server (on oc-server)"
|
||||
fi
|
||||
for wl in workload-web workload-api; do
|
||||
if container_exists "$wl"; then
|
||||
warn " - ${wl}"
|
||||
|
|
@ -1315,6 +1536,13 @@ except: pass
|
|||
idx=$((idx + 1))
|
||||
done
|
||||
|
||||
# Delete oc-server node-exporter
|
||||
if [[ -n "$OC_SERVER_IP" ]] && oc_server_remote_exists && oc_server_container_exists "node-exp-oc-server"; then
|
||||
info "Deleting node-exp-oc-server on oc-server..."
|
||||
incus delete "$(oc_server_container_ref "node-exp-oc-server")" --force 2>/dev/null || true
|
||||
success "Deleted node-exp-oc-server"
|
||||
fi
|
||||
|
||||
# Delete workload containers
|
||||
for wl in workload-web workload-api; do
|
||||
if container_exists "$wl"; then
|
||||
|
|
@ -1479,6 +1707,8 @@ parse_args() {
|
|||
--cleanup) ACTION="cleanup";;
|
||||
--doctor) ACTION="doctor";;
|
||||
-r|--remote) CLUSTER_REMOTE="$2"; shift;;
|
||||
--nodes) MANUAL_NODES="$2"; shift;;
|
||||
--oc-server) OC_SERVER_IP="$2"; shift;;
|
||||
-n|--dry-run) DRY_RUN=true;;
|
||||
-v|--verbose) VERBOSE=true;;
|
||||
-q|--quiet) QUIET=true;;
|
||||
|
|
@ -1507,6 +1737,9 @@ main() {
|
|||
setup_colors
|
||||
parse_args "$@"
|
||||
|
||||
# Discover cluster nodes (populates INCUS_NODES, NODE_EXP_TARGETS, NODE_EXP_IPS)
|
||||
discover_cluster_nodes
|
||||
|
||||
echo
|
||||
case "$ACTION" in
|
||||
deploy) action_deploy;;
|
||||
|
|
|
|||
Loading…
Reference in New Issue