1505 lines
48 KiB
Bash
Executable File
1505 lines
48 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# deploy-observability - Deploy and manage the Incus observability stack
|
|
#
|
|
# Deploys Prometheus, Grafana, Loki, and Promtail in a monitoring container,
|
|
# plus node-exporter containers on each cluster node. Configures Prometheus
|
|
# to scrape Incus metrics (with client certs), HAProxy stats, and node
|
|
# exporters. Sets up Grafana with datasources and dashboards.
|
|
#
|
|
# Usage: deploy-observability [OPTIONS]
|
|
# Run 'deploy-observability --help' for full usage information.
|
|
|
|
set -euo pipefail
|
|
|
|
readonly VERSION="0.1.0"
|
|
readonly SCRIPT_NAME="deploy-observability"
|
|
readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
readonly DASHBOARDS_DIR="${SCRIPT_DIR}/observability-dashboards"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Defaults (overridden by CLI flags)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
CLUSTER_REMOTE="oc-node-01"
|
|
OVN_NETWORK="net-prod"
|
|
MONITORING_IP="10.10.10.70"
|
|
MONITORING_TARGET="oc-node-02"
|
|
FORWARD_VIP="192.168.103.201"
|
|
|
|
INCUS_NODES=("192.168.102.140" "192.168.102.141" "192.168.102.142")
|
|
HAPROXY_IPS=("10.10.10.50" "10.10.10.51")
|
|
NODE_EXP_IPS=("10.10.10.71" "10.10.10.72" "10.10.10.73")
|
|
NODE_EXP_TARGETS=("oc-node-01" "oc-node-02" "oc-node-03")
|
|
|
|
HAPROXY_CLUSTER_ID="52"
|
|
|
|
# Runtime flags
|
|
DRY_RUN=false
|
|
VERBOSE=false
|
|
QUIET=false
|
|
ACTION=""
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Color and formatting
|
|
# ---------------------------------------------------------------------------
|
|
|
|
setup_colors() {
|
|
if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[0;33m'
|
|
BLUE='\033[0;34m'
|
|
CYAN='\033[0;36m'
|
|
BOLD='\033[1m'
|
|
DIM='\033[2m'
|
|
RESET='\033[0m'
|
|
else
|
|
RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET=''
|
|
fi
|
|
}
|
|
|
|
info() { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET} $*"; }
|
|
success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET} $*"; }
|
|
warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; }
|
|
error() { echo -e "${RED}[error]${RESET} $*" >&2; }
|
|
step() { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; }
|
|
detail() { [[ "$VERBOSE" != true ]] && return; echo -e "${DIM} $*${RESET}"; }
|
|
|
|
dry_run_guard() {
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "(dry-run) $*"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Help
|
|
# ---------------------------------------------------------------------------
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
${BOLD}${SCRIPT_NAME}${RESET} v${VERSION} - Deploy the Incus observability stack
|
|
|
|
${BOLD}USAGE${RESET}
|
|
${SCRIPT_NAME} [OPTIONS]
|
|
|
|
${BOLD}ACTIONS${RESET}
|
|
--deploy Full deploy: monitoring container, Prometheus, Grafana,
|
|
Loki, Promtail, node-exporters, ACLs, network forward,
|
|
HAProxy exporter route, dashboards
|
|
--workloads Deploy dummy workloads to generate varied metrics
|
|
--status Show status of all observability components
|
|
--cleanup Remove all observability infrastructure
|
|
--doctor Diagnose common issues
|
|
|
|
${BOLD}OPTIONS${RESET}
|
|
-r, --remote NAME Incus remote (default: ${CLUSTER_REMOTE})
|
|
-n, --dry-run Preview actions without executing
|
|
-v, --verbose Show detailed output
|
|
-q, --quiet Suppress informational output
|
|
-h, --help Show this help message
|
|
|
|
${BOLD}COMPONENTS${RESET}
|
|
Monitoring container (Debian 12, 2 GiB RAM, 20 GiB disk):
|
|
Prometheus :9090 - metrics collection and alerting
|
|
Grafana :3000 - dashboards and visualization
|
|
Loki :3100 - log aggregation
|
|
Promtail - log shipping agent
|
|
|
|
Node-exporter containers (Alpine, 128 MiB, privileged):
|
|
node-exp-01/02/03 on cluster nodes, :9100
|
|
|
|
Network forward:
|
|
${FORWARD_VIP} -> Grafana (:3000), Prometheus (:9090)
|
|
|
|
${BOLD}EXAMPLES${RESET}
|
|
# Check prerequisites and connectivity
|
|
${SCRIPT_NAME} --doctor
|
|
|
|
# Preview what would be deployed
|
|
${SCRIPT_NAME} --deploy --dry-run
|
|
|
|
# Full deployment
|
|
${SCRIPT_NAME} --deploy
|
|
|
|
# Check health of all components
|
|
${SCRIPT_NAME} --status
|
|
|
|
# Tear down everything
|
|
${SCRIPT_NAME} --cleanup
|
|
EOF
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
container_ref() {
|
|
echo "${CLUSTER_REMOTE}:$1"
|
|
}
|
|
|
|
container_exists() {
|
|
incus info "$(container_ref "$1")" &>/dev/null 2>&1
|
|
}
|
|
|
|
container_running() {
|
|
local state
|
|
state=$(incus info "$(container_ref "$1")" 2>/dev/null | grep -i '^status:' | awk '{print $2}')
|
|
[[ "$state" == "RUNNING" || "$state" == "Running" ]]
|
|
}
|
|
|
|
container_exec() {
|
|
local name="$1"
|
|
shift
|
|
incus exec "$(container_ref "$name")" -- "$@"
|
|
}
|
|
|
|
wait_for_container_agent() {
|
|
local name="$1"
|
|
local timeout="${2:-60}"
|
|
local elapsed=0
|
|
while ! container_exec "$name" true &>/dev/null 2>&1; do
|
|
sleep 2
|
|
elapsed=$((elapsed + 2))
|
|
if [[ $elapsed -ge $timeout ]]; then
|
|
error "Container agent for '${name}' not available after ${timeout}s"
|
|
return 1
|
|
fi
|
|
done
|
|
}
|
|
|
|
node_exp_name() {
|
|
local idx="$1"
|
|
printf "node-exp-%02d" "$idx"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Deploy action
|
|
# ---------------------------------------------------------------------------
|
|
|
|
action_deploy() {
|
|
step "Deploying observability stack on ${CLUSTER_REMOTE}"
|
|
echo
|
|
info " Monitoring: ${MONITORING_IP} (on ${MONITORING_TARGET})"
|
|
info " Node-exporters: ${NODE_EXP_IPS[*]}"
|
|
info " Incus nodes: ${INCUS_NODES[*]}"
|
|
info " HAProxy stats: ${HAPROXY_IPS[*]}"
|
|
info " Network fwd: ${FORWARD_VIP} -> Grafana/Prometheus"
|
|
echo
|
|
|
|
if container_exists "monitoring"; then
|
|
error "Monitoring container already exists"
|
|
error "Use --status to check health, or --cleanup first."
|
|
return 1
|
|
fi
|
|
|
|
phase_deploy_monitoring
|
|
phase_install_prometheus
|
|
phase_install_loki
|
|
phase_install_grafana
|
|
phase_deploy_node_exporters
|
|
phase_create_acl
|
|
phase_configure_haproxy_exporter
|
|
phase_create_network_forward
|
|
phase_upload_dashboards
|
|
|
|
echo
|
|
success "Observability stack deployed!"
|
|
echo
|
|
info " Grafana: http://${FORWARD_VIP}:3000 (admin / admin)"
|
|
info " Prometheus: http://${FORWARD_VIP}:9090"
|
|
info " Loki: http://${MONITORING_IP}:3100 (internal)"
|
|
echo
|
|
info "Next steps:"
|
|
info " ${SCRIPT_NAME} --status # verify all targets are up"
|
|
info " Open Grafana and explore the pre-loaded dashboards"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 1: Create monitoring container
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_deploy_monitoring() {
|
|
step "Phase 1: Create monitoring container"
|
|
|
|
if ! dry_run_guard "Launch monitoring on ${OVN_NETWORK} at ${MONITORING_IP}"; then
|
|
return 0
|
|
fi
|
|
|
|
incus launch images:debian/12 "$(container_ref monitoring)" \
|
|
-n "$OVN_NETWORK" \
|
|
--target "$MONITORING_TARGET" \
|
|
-c limits.memory=2GiB \
|
|
-d root,size=20GiB
|
|
|
|
# Tell OVN the static IP we want — port security drops traffic from
|
|
# IPs that OVN didn't assign, so this must be set at the device level.
|
|
incus config device set "$(container_ref monitoring)" eth0 \
|
|
ipv4.address="${MONITORING_IP}"
|
|
|
|
info "Waiting for container agent..."
|
|
wait_for_container_agent monitoring 60
|
|
|
|
# Discover upstream DNS from the UPLINK network config.
|
|
# OVN DHCP advertises this automatically, but we need it for the static
|
|
# networkd config (which bypasses DHCP).
|
|
local upstream_dns
|
|
upstream_dns=$(incus network get "${CLUSTER_REMOTE}:UPLINK" dns.nameservers 2>/dev/null) || true
|
|
upstream_dns="${upstream_dns:-192.168.100.1}"
|
|
detail "Using upstream DNS: ${upstream_dns}"
|
|
|
|
# Build DNS lines (handles comma-separated list from UPLINK config)
|
|
local dns_lines=""
|
|
IFS=',' read -ra dns_addrs <<< "$upstream_dns"
|
|
for addr in "${dns_addrs[@]}"; do
|
|
dns_lines="${dns_lines}DNS=${addr}"$'\n'
|
|
done
|
|
|
|
# Configure matching static IP inside the container (systemd-networkd
|
|
# bypasses DHCP, so we must specify DNS explicitly).
|
|
container_exec monitoring bash -c "
|
|
cat > /etc/systemd/network/10-static.network << NETCFG
|
|
[Match]
|
|
Name=eth0
|
|
|
|
[Network]
|
|
Address=${MONITORING_IP}/24
|
|
Gateway=10.10.10.1
|
|
${dns_lines}NETCFG
|
|
systemctl restart systemd-networkd systemd-resolved
|
|
"
|
|
|
|
sleep 3
|
|
success "Monitoring container created at ${MONITORING_IP}"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 2: Install and configure Prometheus
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_install_prometheus() {
|
|
step "Phase 2: Install and configure Prometheus"
|
|
|
|
if ! dry_run_guard "Install Prometheus in monitoring container"; then
|
|
return 0
|
|
fi
|
|
|
|
# Copy Incus client certs into the container for metrics scraping
|
|
local incus_cert="${HOME}/.config/incus/client.crt"
|
|
local incus_key="${HOME}/.config/incus/client.key"
|
|
|
|
if [[ ! -f "$incus_cert" || ! -f "$incus_key" ]]; then
|
|
error "Incus client certificates not found at ~/.config/incus/"
|
|
error "Prometheus needs these to scrape Incus metrics endpoints."
|
|
error "Install the Incus client and run 'incus remote add' first."
|
|
return 1
|
|
fi
|
|
|
|
incus file push "$incus_cert" "$(container_ref monitoring)/tmp/incus-client.crt"
|
|
incus file push "$incus_key" "$(container_ref monitoring)/tmp/incus-client.key"
|
|
|
|
container_exec monitoring bash -c "
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
|
|
# Add Grafana apt repository (provides grafana, loki, promtail)
|
|
apt-get update -qq
|
|
apt-get install -y -qq ca-certificates curl gnupg apt-transport-https 2>&1 | tail -1
|
|
curl -fsSL https://apt.grafana.com/gpg.key | gpg --dearmor -o /usr/share/keyrings/grafana-archive-keyring.gpg
|
|
echo 'deb [signed-by=/usr/share/keyrings/grafana-archive-keyring.gpg] https://apt.grafana.com stable main' \
|
|
> /etc/apt/sources.list.d/grafana.list
|
|
apt-get update -qq
|
|
|
|
apt-get install -y -qq prometheus 2>&1 | tail -1
|
|
|
|
# Place Incus client certs where Prometheus can read them
|
|
mkdir -p /etc/prometheus/certs
|
|
cp /tmp/incus-client.crt /etc/prometheus/certs/
|
|
cp /tmp/incus-client.key /etc/prometheus/certs/
|
|
chown -R prometheus:prometheus /etc/prometheus/certs
|
|
chmod 600 /etc/prometheus/certs/incus-client.key
|
|
"
|
|
|
|
# Build scrape targets
|
|
local incus_targets=""
|
|
local idx=0
|
|
for node in "${INCUS_NODES[@]}"; do
|
|
if [[ $idx -gt 0 ]]; then incus_targets+=", "; fi
|
|
incus_targets+="'${node}:8443'"
|
|
idx=$((idx + 1))
|
|
done
|
|
|
|
local haproxy_targets=""
|
|
idx=0
|
|
for ip in "${HAPROXY_IPS[@]}"; do
|
|
if [[ $idx -gt 0 ]]; then haproxy_targets+=", "; fi
|
|
haproxy_targets+="'${ip}:8404'"
|
|
idx=$((idx + 1))
|
|
done
|
|
|
|
local node_exp_targets=""
|
|
idx=0
|
|
for ip in "${NODE_EXP_IPS[@]}"; do
|
|
if [[ $idx -gt 0 ]]; then node_exp_targets+=", "; fi
|
|
node_exp_targets+="'${ip}:9100'"
|
|
idx=$((idx + 1))
|
|
done
|
|
|
|
container_exec monitoring bash -c "cat > /etc/prometheus/prometheus.yml << 'PROMCFG'
|
|
global:
|
|
scrape_interval: 15s
|
|
evaluation_interval: 15s
|
|
|
|
scrape_configs:
|
|
- job_name: 'prometheus'
|
|
static_configs:
|
|
- targets: ['localhost:9090']
|
|
|
|
- job_name: 'incus'
|
|
scheme: https
|
|
tls_config:
|
|
cert_file: /etc/prometheus/certs/incus-client.crt
|
|
key_file: /etc/prometheus/certs/incus-client.key
|
|
insecure_skip_verify: true
|
|
metrics_path: /1.0/metrics
|
|
static_configs:
|
|
- targets: [${incus_targets}]
|
|
|
|
- job_name: 'haproxy'
|
|
static_configs:
|
|
- targets: [${haproxy_targets}]
|
|
|
|
- job_name: 'node-exporter'
|
|
static_configs:
|
|
- targets: [${node_exp_targets}]
|
|
PROMCFG"
|
|
|
|
container_exec monitoring systemctl enable prometheus
|
|
container_exec monitoring systemctl restart prometheus
|
|
sleep 2
|
|
|
|
success "Prometheus installed and configured"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 3: Install and configure Loki + Promtail
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_install_loki() {
|
|
step "Phase 3: Install Loki and Promtail"
|
|
|
|
if ! dry_run_guard "Install Loki + Promtail in monitoring container"; then
|
|
return 0
|
|
fi
|
|
|
|
container_exec monitoring bash -c "
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
apt-get install -y -qq loki promtail 2>&1 | tail -1
|
|
"
|
|
|
|
# Configure Loki
|
|
container_exec monitoring bash -c "cat > /etc/loki/config.yml << 'LOKICFG'
|
|
auth_enabled: false
|
|
|
|
server:
|
|
http_listen_port: 3100
|
|
grpc_listen_port: 9096
|
|
|
|
common:
|
|
path_prefix: /var/lib/loki
|
|
storage:
|
|
filesystem:
|
|
chunks_directory: /var/lib/loki/chunks
|
|
rules_directory: /var/lib/loki/rules
|
|
replication_factor: 1
|
|
ring:
|
|
kvstore:
|
|
store: inmemory
|
|
|
|
schema_config:
|
|
configs:
|
|
- from: 2020-10-24
|
|
store: tsdb
|
|
object_store: filesystem
|
|
schema: v13
|
|
index:
|
|
prefix: index_
|
|
period: 24h
|
|
|
|
limits_config:
|
|
reject_old_samples: true
|
|
reject_old_samples_max_age: 168h
|
|
allow_structured_metadata: false
|
|
LOKICFG"
|
|
|
|
# Configure Promtail to ship local logs
|
|
container_exec monitoring bash -c "cat > /etc/promtail/config.yml << 'PTCFG'
|
|
server:
|
|
http_listen_port: 9080
|
|
grpc_listen_port: 0
|
|
|
|
positions:
|
|
filename: /var/lib/promtail/positions.yaml
|
|
|
|
clients:
|
|
- url: http://localhost:3100/loki/api/v1/push
|
|
|
|
scrape_configs:
|
|
- job_name: system
|
|
static_configs:
|
|
- targets:
|
|
- localhost
|
|
labels:
|
|
job: varlogs
|
|
host: monitoring
|
|
__path__: /var/log/**/*.log
|
|
|
|
- job_name: journal
|
|
journal:
|
|
max_age: 12h
|
|
labels:
|
|
job: systemd-journal
|
|
host: monitoring
|
|
relabel_configs:
|
|
- source_labels: ['__journal__systemd_unit']
|
|
target_label: unit
|
|
PTCFG"
|
|
|
|
# Ensure data directories exist with correct ownership
|
|
container_exec monitoring bash -c "mkdir -p /var/lib/loki && chown loki:nogroup /var/lib/loki"
|
|
container_exec monitoring bash -c "mkdir -p /var/lib/promtail && chown promtail:nogroup /var/lib/promtail"
|
|
|
|
# Promtail needs adm group for /var/log access, systemd-journal for journal
|
|
container_exec monitoring bash -c "usermod -aG adm,systemd-journal promtail"
|
|
|
|
container_exec monitoring systemctl enable loki
|
|
container_exec monitoring systemctl restart loki
|
|
sleep 2
|
|
|
|
container_exec monitoring systemctl enable promtail
|
|
container_exec monitoring systemctl restart promtail
|
|
|
|
success "Loki and Promtail installed and configured"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 4: Install and configure Grafana
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_install_grafana() {
|
|
step "Phase 4: Install and configure Grafana"
|
|
|
|
if ! dry_run_guard "Install Grafana in monitoring container"; then
|
|
return 0
|
|
fi
|
|
|
|
container_exec monitoring bash -c "
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
apt-get install -y -qq grafana 2>&1 | tail -1
|
|
"
|
|
|
|
# Provision datasources with explicit UIDs for dashboard portability
|
|
container_exec monitoring bash -c "mkdir -p /etc/grafana/provisioning/datasources
|
|
cat > /etc/grafana/provisioning/datasources/observability.yaml << 'DSCFG'
|
|
apiVersion: 1
|
|
datasources:
|
|
- name: Prometheus
|
|
type: prometheus
|
|
uid: prometheus
|
|
access: proxy
|
|
url: http://localhost:9090
|
|
isDefault: true
|
|
editable: true
|
|
|
|
- name: Loki
|
|
type: loki
|
|
uid: loki
|
|
access: proxy
|
|
url: http://localhost:3100
|
|
editable: true
|
|
DSCFG"
|
|
|
|
container_exec monitoring systemctl enable grafana-server
|
|
container_exec monitoring systemctl restart grafana-server
|
|
sleep 2
|
|
|
|
success "Grafana installed with Prometheus + Loki datasources"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 5: Deploy node-exporter containers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_deploy_node_exporters() {
|
|
step "Phase 5: Deploy node-exporter containers"
|
|
|
|
local idx=1
|
|
for ip in "${NODE_EXP_IPS[@]}"; do
|
|
local name
|
|
name=$(node_exp_name "$idx")
|
|
local target="${NODE_EXP_TARGETS[$((idx - 1))]}"
|
|
|
|
if container_exists "$name"; then
|
|
if container_running "$name"; then
|
|
success " ${name} already running"
|
|
idx=$((idx + 1))
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
if ! dry_run_guard "Launch ${name} on ${target} at ${ip}"; then
|
|
idx=$((idx + 1))
|
|
continue
|
|
fi
|
|
|
|
info "Launching ${name} (${ip}) on ${target}..."
|
|
incus launch images:alpine/3.20 "$(container_ref "$name")" \
|
|
-n "$OVN_NETWORK" \
|
|
--target "$target" \
|
|
-c limits.memory=128MiB \
|
|
-c security.privileged=true
|
|
|
|
# Reserve this IP in OVN before switching to static inside the container
|
|
incus config device set "$(container_ref "$name")" eth0 \
|
|
ipv4.address="${ip}"
|
|
|
|
wait_for_container_agent "$name" 30
|
|
|
|
# Mount host filesystems for full node metrics
|
|
incus config device add "$(container_ref "$name")" host-proc disk \
|
|
source=/proc path=/host/proc readonly=true
|
|
incus config device add "$(container_ref "$name")" host-sys disk \
|
|
source=/sys path=/host/sys readonly=true
|
|
incus config device add "$(container_ref "$name")" host-root disk \
|
|
source=/ path=/host/root readonly=true
|
|
|
|
# Install and configure node_exporter
|
|
container_exec "$name" sh -c "
|
|
apk add --no-cache prometheus-node-exporter > /dev/null 2>&1
|
|
|
|
# Configure to use host mount paths
|
|
mkdir -p /etc/conf.d
|
|
cat > /etc/conf.d/node-exporter << 'NEXPCFG'
|
|
NODE_EXPORTER_OPTS=\"--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/host/root\"
|
|
NEXPCFG
|
|
|
|
rc-update add node-exporter default 2>/dev/null || true
|
|
rc-service node-exporter start 2>/dev/null || true
|
|
"
|
|
|
|
# Configure static IP
|
|
container_exec "$name" sh -c "
|
|
cat > /etc/network/interfaces << NETCFG
|
|
auto lo
|
|
iface lo inet loopback
|
|
|
|
auto eth0
|
|
iface eth0 inet static
|
|
address ${ip}/24
|
|
gateway 10.10.10.1
|
|
NETCFG
|
|
rc-service networking restart 2>/dev/null || true
|
|
"
|
|
|
|
success " ${name} deployed at ${ip} on ${target}"
|
|
idx=$((idx + 1))
|
|
done
|
|
|
|
success "Node-exporter containers deployed"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 6: Create ACL and attach to containers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_create_acl() {
|
|
step "Phase 6: Create monitoring ACL"
|
|
|
|
if ! dry_run_guard "Create monitoring-allow ACL"; then
|
|
return 0
|
|
fi
|
|
|
|
# Create the ACL if it does not exist
|
|
if ! incus network acl show "$(container_ref monitoring-allow)" &>/dev/null 2>&1; then
|
|
incus network acl create "$(container_ref monitoring-allow)"
|
|
|
|
# Add allow-all egress and ingress rules
|
|
incus network acl rule add "$(container_ref monitoring-allow)" egress \
|
|
action=allow
|
|
incus network acl rule add "$(container_ref monitoring-allow)" ingress \
|
|
action=allow
|
|
success "ACL 'monitoring-allow' created with allow-all rules"
|
|
else
|
|
success "ACL 'monitoring-allow' already exists"
|
|
fi
|
|
|
|
# Attach ACL to monitoring container
|
|
incus config device set "$(container_ref monitoring)" eth0 \
|
|
security.acls=monitoring-allow security.acls.default.ingress.action=allow \
|
|
security.acls.default.egress.action=allow 2>/dev/null || true
|
|
detail "ACL attached to monitoring"
|
|
|
|
# Attach ACL to node-exporter containers
|
|
local idx=1
|
|
for ip in "${NODE_EXP_IPS[@]}"; do
|
|
local name
|
|
name=$(node_exp_name "$idx")
|
|
if container_exists "$name"; then
|
|
incus config device set "$(container_ref "$name")" eth0 \
|
|
security.acls=monitoring-allow security.acls.default.ingress.action=allow \
|
|
security.acls.default.egress.action=allow 2>/dev/null || true
|
|
detail "ACL attached to ${name}"
|
|
fi
|
|
idx=$((idx + 1))
|
|
done
|
|
|
|
# Changing security.acls can bring the NIC down. Bring all NICs back up.
|
|
sleep 2
|
|
container_exec monitoring bash -c "ip link set eth0 up; systemctl restart systemd-networkd" 2>/dev/null || true
|
|
local nic_idx=1
|
|
for ip in "${NODE_EXP_IPS[@]}"; do
|
|
local nname
|
|
nname=$(node_exp_name "$nic_idx")
|
|
if container_exists "$nname"; then
|
|
container_exec "$nname" sh -c "ip link set eth0 up; rc-service networking restart" 2>/dev/null || true
|
|
fi
|
|
nic_idx=$((nic_idx + 1))
|
|
done
|
|
|
|
success "ACL attached to all observability containers"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 7: Configure HAProxy prometheus exporter
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_configure_haproxy_exporter() {
|
|
step "Phase 7: Configure HAProxy prometheus exporter + ACL rules"
|
|
|
|
if ! dry_run_guard "Add prometheus exporter to HAProxy containers"; then
|
|
return 0
|
|
fi
|
|
|
|
local haproxy_remote="${CLUSTER_REMOTE}"
|
|
|
|
for n in 01 02; do
|
|
local cname="ffsdn-haproxy-${HAPROXY_CLUSTER_ID}-${n}"
|
|
|
|
if ! container_exists "$cname"; then
|
|
warn "HAProxy container ${cname} not found -- skipping"
|
|
continue
|
|
fi
|
|
|
|
# Add prometheus exporter line to the stats frontend in haproxy.cfg
|
|
# Only add if not already present
|
|
local has_exporter
|
|
has_exporter=$(container_exec "$cname" grep -c 'prometheus-exporter' /etc/haproxy/haproxy.cfg 2>/dev/null || echo "0")
|
|
if [[ "$has_exporter" == "0" ]]; then
|
|
container_exec "$cname" bash -c "
|
|
# Insert the prometheus exporter line into the stats section
|
|
sed -i '/^[[:space:]]*stats enable/a\\ http-request use-service prometheus-exporter if { path /metrics }' /etc/haproxy/haproxy.cfg
|
|
haproxy -c -f /etc/haproxy/haproxy.cfg 2>/dev/null && systemctl reload haproxy || true
|
|
"
|
|
success " ${cname}: prometheus exporter route added"
|
|
else
|
|
success " ${cname}: prometheus exporter already configured"
|
|
fi
|
|
|
|
# Add monitoring ingress rules to Aether ACL
|
|
local acl_name="${HAPROXY_CLUSTER_ID}-${cname}-aether-acl"
|
|
local has_monitoring_rule
|
|
has_monitoring_rule=$(incus network acl show "${haproxy_remote}:${acl_name}" 2>/dev/null \
|
|
| grep -c "monitoring-prometheus" || echo "0")
|
|
|
|
if [[ "$has_monitoring_rule" == "0" ]]; then
|
|
incus network acl rule add "${haproxy_remote}:${acl_name}" ingress \
|
|
action=allow protocol=tcp source="${MONITORING_IP}/32" destination_port=8404 \
|
|
description="monitoring-prometheus" 2>/dev/null || true
|
|
incus network acl rule add "${haproxy_remote}:${acl_name}" ingress \
|
|
action=allow protocol=icmp source="${MONITORING_IP}/32" \
|
|
description="monitoring-icmp" 2>/dev/null || true
|
|
success " ${cname}: monitoring ACL rules added"
|
|
else
|
|
success " ${cname}: monitoring ACL rules already present"
|
|
fi
|
|
done
|
|
|
|
success "HAProxy prometheus exporter configured"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 8: Create OVN network forward
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_create_network_forward() {
|
|
step "Phase 8: Create network forward ${FORWARD_VIP}"
|
|
|
|
if ! dry_run_guard "Create network forward ${FORWARD_VIP}"; then
|
|
return 0
|
|
fi
|
|
|
|
# Check if forward already exists
|
|
if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
|
|
success "Network forward ${FORWARD_VIP} already exists"
|
|
return 0
|
|
fi
|
|
|
|
incus network forward create "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP"
|
|
|
|
# Grafana port
|
|
incus network forward port add "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" \
|
|
tcp 3000 "${MONITORING_IP}" 3000
|
|
|
|
# Prometheus port
|
|
incus network forward port add "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" \
|
|
tcp 9090 "${MONITORING_IP}" 9090
|
|
|
|
success "Network forward created: ${FORWARD_VIP} -> Grafana(:3000), Prometheus(:9090)"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 9: Upload Grafana dashboards
|
|
# ---------------------------------------------------------------------------
|
|
|
|
phase_upload_dashboards() {
|
|
step "Phase 9: Install Grafana dashboards"
|
|
|
|
if ! dry_run_guard "Install dashboards via manage-dashboards"; then
|
|
return 0
|
|
fi
|
|
|
|
local manage_script="${SCRIPT_DIR}/manage-dashboards"
|
|
if [[ ! -x "$manage_script" ]]; then
|
|
warn "manage-dashboards script not found at ${manage_script}"
|
|
warn "Falling back to direct file upload."
|
|
|
|
if [[ ! -d "$DASHBOARDS_DIR" ]]; then
|
|
warn "Dashboard directory not found: ${DASHBOARDS_DIR}"
|
|
return 0
|
|
fi
|
|
|
|
local count=0
|
|
for dashboard in "${DASHBOARDS_DIR}"/core/*.json; do
|
|
[[ -f "$dashboard" ]] || continue
|
|
local basename
|
|
basename=$(basename "$dashboard")
|
|
incus file push "$dashboard" "$(container_ref monitoring)/var/lib/grafana/dashboards/${basename}"
|
|
detail "Uploaded ${basename}"
|
|
count=$((count + 1))
|
|
done
|
|
|
|
if [[ $count -gt 0 ]]; then
|
|
container_exec monitoring systemctl restart grafana-server 2>/dev/null || true
|
|
success "${count} dashboard(s) uploaded (direct push)"
|
|
else
|
|
info "No dashboard files found in ${DASHBOARDS_DIR}/core/"
|
|
fi
|
|
return 0
|
|
fi
|
|
|
|
# Wait for Grafana API to be ready
|
|
info "Waiting for Grafana API..."
|
|
local elapsed=0
|
|
while true; do
|
|
local code
|
|
code=$(container_exec monitoring curl -s -o /dev/null -w '%{http_code}' \
|
|
http://localhost:3000/api/health 2>/dev/null || echo "000")
|
|
if [[ "$code" == "200" ]]; then
|
|
break
|
|
fi
|
|
sleep 2
|
|
elapsed=$((elapsed + 2))
|
|
if [[ $elapsed -ge 30 ]]; then
|
|
warn "Grafana API not ready after 30s -- falling back to file push"
|
|
return 0
|
|
fi
|
|
done
|
|
|
|
# Use manage-dashboards to install with proper folder organization
|
|
local flags="-r ${CLUSTER_REMOTE}"
|
|
[[ "$VERBOSE" == true ]] && flags="${flags} -v"
|
|
[[ "$QUIET" == true ]] && flags="${flags} -q"
|
|
|
|
"${manage_script}" --install ${flags}
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Workloads action
|
|
# ---------------------------------------------------------------------------
|
|
|
|
WORKLOAD_WEB_IP="10.10.10.80"
|
|
WORKLOAD_API_IP="10.10.10.81"
|
|
|
|
action_workloads() {
|
|
step "Deploying dummy workloads for metric generation"
|
|
echo
|
|
info " workload-web: ${WORKLOAD_WEB_IP} (nginx + CPU/disk/network crons)"
|
|
info " workload-api: ${WORKLOAD_API_IP} (Python HTTP + memory/network crons)"
|
|
echo
|
|
|
|
phase_deploy_workload_web
|
|
phase_deploy_workload_api
|
|
|
|
echo
|
|
success "Dummy workloads deployed!"
|
|
info " Metrics will appear within 1-2 minutes"
|
|
info " workload-web: nginx on ${WORKLOAD_WEB_IP}:80, CPU/disk/net crons"
|
|
info " workload-api: Python on ${WORKLOAD_API_IP}:8080, memory sawtooth cron"
|
|
}
|
|
|
|
phase_deploy_workload_web() {
|
|
step "Deploy workload-web (nginx + varied load)"
|
|
|
|
if container_exists "workload-web"; then
|
|
if container_running "workload-web"; then
|
|
success "workload-web already running"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if ! dry_run_guard "Launch workload-web on ${OVN_NETWORK} at ${WORKLOAD_WEB_IP}"; then
|
|
return 0
|
|
fi
|
|
|
|
incus launch images:alpine/3.20 "$(container_ref workload-web)" \
|
|
-n "$OVN_NETWORK" \
|
|
--target "oc-node-01" \
|
|
-c limits.memory=128MiB
|
|
|
|
incus config device set "$(container_ref workload-web)" eth0 \
|
|
ipv4.address="${WORKLOAD_WEB_IP}"
|
|
|
|
wait_for_container_agent workload-web 30
|
|
|
|
container_exec workload-web sh -c "
|
|
apk add --no-cache nginx curl gzip coreutils > /dev/null 2>&1
|
|
|
|
# Configure nginx
|
|
mkdir -p /var/www/html
|
|
cat > /var/www/html/index.html << 'WEBHTML'
|
|
<html><body><h1>workload-web</h1><p>Observability test workload</p></body></html>
|
|
WEBHTML
|
|
cat > /etc/nginx/http.d/default.conf << 'NGXCFG'
|
|
server {
|
|
listen 80 default_server;
|
|
root /var/www/html;
|
|
index index.html;
|
|
location / { try_files \$uri \$uri/ =404; }
|
|
}
|
|
NGXCFG
|
|
rc-update add nginx default 2>/dev/null || true
|
|
rc-service nginx start 2>/dev/null || true
|
|
|
|
# Cron: every minute -> 10 HTTP requests (network traffic)
|
|
# Cron: every 5 min -> gzip random data (CPU spike)
|
|
# Cron: every 15 min -> dd 5 MiB write+delete (disk I/O burst)
|
|
cat > /etc/crontabs/root << 'CRONCFG'
|
|
* * * * * for i in \$(seq 1 10); do wget -q -O /dev/null http://localhost/ 2>/dev/null; done
|
|
*/5 * * * * dd if=/dev/urandom bs=1M count=2 2>/dev/null | gzip > /dev/null 2>&1
|
|
*/15 * * * * dd if=/dev/urandom of=/tmp/disktest bs=1M count=5 2>/dev/null && rm -f /tmp/disktest
|
|
CRONCFG
|
|
rc-update add crond default 2>/dev/null || true
|
|
rc-service crond start 2>/dev/null || true
|
|
"
|
|
|
|
# Configure static IP
|
|
container_exec workload-web sh -c "
|
|
cat > /etc/network/interfaces << NETCFG
|
|
auto lo
|
|
iface lo inet loopback
|
|
|
|
auto eth0
|
|
iface eth0 inet static
|
|
address ${WORKLOAD_WEB_IP}/24
|
|
gateway 10.10.10.1
|
|
NETCFG
|
|
rc-service networking restart 2>/dev/null || true
|
|
"
|
|
|
|
# Attach monitoring ACL
|
|
incus config device set "$(container_ref workload-web)" eth0 \
|
|
security.acls=monitoring-allow security.acls.default.ingress.action=allow \
|
|
security.acls.default.egress.action=allow 2>/dev/null || true
|
|
|
|
success "workload-web deployed at ${WORKLOAD_WEB_IP}"
|
|
}
|
|
|
|
phase_deploy_workload_api() {
|
|
step "Deploy workload-api (Python HTTP + memory patterns)"
|
|
|
|
if container_exists "workload-api"; then
|
|
if container_running "workload-api"; then
|
|
success "workload-api already running"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if ! dry_run_guard "Launch workload-api on ${OVN_NETWORK} at ${WORKLOAD_API_IP}"; then
|
|
return 0
|
|
fi
|
|
|
|
incus launch images:alpine/3.20 "$(container_ref workload-api)" \
|
|
-n "$OVN_NETWORK" \
|
|
--target "oc-node-03" \
|
|
-c limits.memory=128MiB
|
|
|
|
incus config device set "$(container_ref workload-api)" eth0 \
|
|
ipv4.address="${WORKLOAD_API_IP}"
|
|
|
|
wait_for_container_agent workload-api 30
|
|
|
|
container_exec workload-api sh -c "
|
|
apk add --no-cache python3 curl coreutils > /dev/null 2>&1
|
|
|
|
# Create a simple JSON status endpoint
|
|
mkdir -p /opt/api
|
|
cat > /opt/api/server.py << 'PYAPI'
|
|
import http.server
|
|
import json
|
|
import time
|
|
import os
|
|
|
|
class Handler(http.server.BaseHTTPRequestHandler):
|
|
def do_GET(self):
|
|
self.send_response(200)
|
|
self.send_header('Content-Type', 'application/json')
|
|
self.end_headers()
|
|
data = {
|
|
'status': 'ok',
|
|
'uptime': time.time() - START_TIME,
|
|
'hostname': os.uname().nodename,
|
|
'pid': os.getpid()
|
|
}
|
|
self.wfile.write(json.dumps(data).encode())
|
|
|
|
def log_message(self, format, *args):
|
|
pass # suppress access logs
|
|
|
|
START_TIME = time.time()
|
|
http.server.HTTPServer(('0.0.0.0', 8080), Handler).serve_forever()
|
|
PYAPI
|
|
|
|
# Create memory stress script
|
|
cat > /opt/api/mem-stress.py << 'PYMEM'
|
|
import time
|
|
# Allocate ~20 MiB, hold for 30 seconds, then release
|
|
data = bytearray(20 * 1024 * 1024)
|
|
time.sleep(30)
|
|
del data
|
|
PYMEM
|
|
|
|
# Start the API server as a background service
|
|
cat > /etc/init.d/api-server << 'INITSCRIPT'
|
|
#!/sbin/openrc-run
|
|
name=\"api-server\"
|
|
command=\"/usr/bin/python3\"
|
|
command_args=\"/opt/api/server.py\"
|
|
command_background=true
|
|
pidfile=\"/run/api-server.pid\"
|
|
INITSCRIPT
|
|
chmod +x /etc/init.d/api-server
|
|
rc-update add api-server default 2>/dev/null || true
|
|
rc-service api-server start 2>/dev/null || true
|
|
|
|
# Cron: every 2 min -> curl workload-web (inter-container traffic)
|
|
# Cron: every 10 min -> allocate 20 MiB for 30s (memory sawtooth)
|
|
cat > /etc/crontabs/root << 'CRONCFG'
|
|
*/2 * * * * wget -q -O /dev/null http://${WORKLOAD_WEB_IP}/ 2>/dev/null
|
|
*/10 * * * * python3 /opt/api/mem-stress.py 2>/dev/null &
|
|
CRONCFG
|
|
rc-update add crond default 2>/dev/null || true
|
|
rc-service crond start 2>/dev/null || true
|
|
"
|
|
|
|
# Configure static IP
|
|
container_exec workload-api sh -c "
|
|
cat > /etc/network/interfaces << NETCFG
|
|
auto lo
|
|
iface lo inet loopback
|
|
|
|
auto eth0
|
|
iface eth0 inet static
|
|
address ${WORKLOAD_API_IP}/24
|
|
gateway 10.10.10.1
|
|
NETCFG
|
|
rc-service networking restart 2>/dev/null || true
|
|
"
|
|
|
|
# Attach monitoring ACL
|
|
incus config device set "$(container_ref workload-api)" eth0 \
|
|
security.acls=monitoring-allow security.acls.default.ingress.action=allow \
|
|
security.acls.default.egress.action=allow 2>/dev/null || true
|
|
|
|
success "workload-api deployed at ${WORKLOAD_API_IP}"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Status action
|
|
# ---------------------------------------------------------------------------
|
|
|
|
action_status() {
|
|
step "Observability Stack Status"
|
|
echo
|
|
|
|
# --- Container status ---
|
|
info "Containers:"
|
|
local all_ok=true
|
|
|
|
for cname in monitoring; do
|
|
if container_exists "$cname"; then
|
|
if container_running "$cname"; then
|
|
success " ${cname}: running"
|
|
else
|
|
error " ${cname}: stopped"
|
|
all_ok=false
|
|
fi
|
|
else
|
|
error " ${cname}: not found"
|
|
all_ok=false
|
|
fi
|
|
done
|
|
|
|
local idx=1
|
|
for ip in "${NODE_EXP_IPS[@]}"; do
|
|
local name
|
|
name=$(node_exp_name "$idx")
|
|
if container_exists "$name"; then
|
|
if container_running "$name"; then
|
|
success " ${name} (${ip}): running"
|
|
else
|
|
warn " ${name} (${ip}): stopped"
|
|
fi
|
|
else
|
|
warn " ${name} (${ip}): not found"
|
|
fi
|
|
idx=$((idx + 1))
|
|
done
|
|
|
|
# Workload containers (optional)
|
|
for wl in workload-web workload-api; do
|
|
if container_exists "$wl"; then
|
|
if container_running "$wl"; then
|
|
success " ${wl}: running"
|
|
else
|
|
warn " ${wl}: stopped"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if [[ "$all_ok" == false ]]; then
|
|
echo
|
|
error "Some containers are missing or stopped."
|
|
echo
|
|
fi
|
|
|
|
# --- Service status ---
|
|
if container_exists monitoring && container_running monitoring; then
|
|
echo
|
|
info "Services:"
|
|
for svc in prometheus grafana-server loki promtail; do
|
|
local status
|
|
status=$(container_exec monitoring systemctl is-active "$svc" 2>/dev/null || echo "inactive")
|
|
if [[ "$status" == "active" ]]; then
|
|
success " ${svc}: active"
|
|
else
|
|
error " ${svc}: ${status}"
|
|
fi
|
|
done
|
|
|
|
# --- Prometheus targets ---
|
|
echo
|
|
info "Prometheus targets:"
|
|
local targets_json
|
|
targets_json=$(curl -s --connect-timeout 5 "http://${MONITORING_IP}:9090/api/v1/targets" 2>/dev/null || echo "")
|
|
if [[ -n "$targets_json" ]]; then
|
|
echo "$targets_json" | python3 -c "
|
|
import sys, json
|
|
try:
|
|
data = json.load(sys.stdin)
|
|
targets = data.get('data', {}).get('activeTargets', [])
|
|
for t in targets:
|
|
job = t.get('labels', {}).get('job', 'unknown')
|
|
instance = t.get('labels', {}).get('instance', 'unknown')
|
|
health = t.get('health', 'unknown')
|
|
icon = 'UP' if health == 'up' else 'DOWN'
|
|
print(f' {job:20s} {instance:30s} {icon}')
|
|
except Exception as e:
|
|
print(f' (error parsing targets: {e})')
|
|
" 2>/dev/null || warn " Could not parse Prometheus targets"
|
|
else
|
|
warn " Prometheus not reachable at http://${MONITORING_IP}:9090"
|
|
fi
|
|
|
|
# --- Grafana health ---
|
|
echo
|
|
info "Grafana health:"
|
|
local grafana_code
|
|
grafana_code=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 5 \
|
|
"http://${MONITORING_IP}:3000/api/health" 2>/dev/null || echo "000")
|
|
if [[ "$grafana_code" == "200" ]]; then
|
|
success " Grafana API: healthy (HTTP ${grafana_code})"
|
|
else
|
|
error " Grafana API: HTTP ${grafana_code}"
|
|
fi
|
|
|
|
# --- Loki labels ---
|
|
echo
|
|
info "Loki status:"
|
|
local loki_labels
|
|
loki_labels=$(curl -s --connect-timeout 5 "http://${MONITORING_IP}:3100/loki/api/v1/labels" 2>/dev/null || echo "")
|
|
if [[ -n "$loki_labels" ]]; then
|
|
local label_count
|
|
label_count=$(echo "$loki_labels" | python3 -c "
|
|
import sys, json
|
|
try:
|
|
data = json.load(sys.stdin)
|
|
labels = data.get('data', [])
|
|
print(len(labels))
|
|
except:
|
|
print(0)
|
|
" 2>/dev/null || echo "0")
|
|
if [[ "$label_count" -gt 0 ]]; then
|
|
success " Loki: responding (${label_count} label(s))"
|
|
else
|
|
info " Loki: responding (no labels yet -- logs may not have been ingested)"
|
|
fi
|
|
else
|
|
warn " Loki not reachable at http://${MONITORING_IP}:3100"
|
|
fi
|
|
fi
|
|
|
|
# --- Network forward ---
|
|
echo
|
|
info "Network forward:"
|
|
if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
|
|
success " ${FORWARD_VIP}: configured"
|
|
local grafana_fwd
|
|
grafana_fwd=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 5 \
|
|
"http://${FORWARD_VIP}:3000/api/health" 2>/dev/null || echo "000")
|
|
if [[ "$grafana_fwd" == "200" ]]; then
|
|
success " Grafana via forward: reachable (HTTP ${grafana_fwd})"
|
|
else
|
|
warn " Grafana via forward: HTTP ${grafana_fwd}"
|
|
fi
|
|
else
|
|
warn " ${FORWARD_VIP}: not found"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cleanup action
|
|
# ---------------------------------------------------------------------------
|
|
|
|
action_cleanup() {
|
|
step "Cleaning up observability stack"
|
|
echo
|
|
|
|
local has_anything=false
|
|
|
|
if container_exists monitoring; then has_anything=true; fi
|
|
local idx=1
|
|
for ip in "${NODE_EXP_IPS[@]}"; do
|
|
if container_exists "$(node_exp_name "$idx")"; then has_anything=true; fi
|
|
idx=$((idx + 1))
|
|
done
|
|
for wl in workload-web workload-api; do
|
|
if container_exists "$wl"; then has_anything=true; fi
|
|
done
|
|
if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
|
|
has_anything=true
|
|
fi
|
|
|
|
if [[ "$has_anything" == false ]]; then
|
|
info "Nothing to clean up"
|
|
return 0
|
|
fi
|
|
|
|
warn "This will delete:"
|
|
if container_exists monitoring; then
|
|
warn " - monitoring container (Prometheus, Grafana, Loki data)"
|
|
fi
|
|
idx=1
|
|
for ip in "${NODE_EXP_IPS[@]}"; do
|
|
local name
|
|
name=$(node_exp_name "$idx")
|
|
if container_exists "$name"; then
|
|
warn " - ${name}"
|
|
fi
|
|
idx=$((idx + 1))
|
|
done
|
|
for wl in workload-web workload-api; do
|
|
if container_exists "$wl"; then
|
|
warn " - ${wl}"
|
|
fi
|
|
done
|
|
if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
|
|
warn " - network forward ${FORWARD_VIP}"
|
|
fi
|
|
if incus network acl show "$(container_ref monitoring-allow)" &>/dev/null 2>&1; then
|
|
warn " - ACL monitoring-allow"
|
|
fi
|
|
echo
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "(dry-run) Would delete all observability infrastructure"
|
|
return 0
|
|
fi
|
|
|
|
read -r -p "Type 'yes' to confirm: " confirm
|
|
if [[ "$confirm" != "yes" ]]; then
|
|
info "Aborted"
|
|
return 0
|
|
fi
|
|
|
|
# Remove monitoring ACL rules from HAProxy ACLs
|
|
step "Removing monitoring rules from HAProxy ACLs"
|
|
for n in 01 02; do
|
|
local cname="ffsdn-haproxy-${HAPROXY_CLUSTER_ID}-${n}"
|
|
local acl_name="${HAPROXY_CLUSTER_ID}-${cname}-aether-acl"
|
|
# Remove rules with monitoring descriptions
|
|
local rule_ids
|
|
rule_ids=$(incus network acl show "${CLUSTER_REMOTE}:${acl_name}" 2>/dev/null \
|
|
| python3 -c "
|
|
import sys, yaml
|
|
try:
|
|
data = yaml.safe_load(sys.stdin)
|
|
for rule in data.get('ingress', []):
|
|
desc = rule.get('description', '')
|
|
if desc.startswith('monitoring-'):
|
|
# ACL rules are removed by matching all fields, not by ID
|
|
print(desc)
|
|
except: pass
|
|
" 2>/dev/null || true)
|
|
if [[ -n "$rule_ids" ]]; then
|
|
for desc in $rule_ids; do
|
|
incus network acl rule remove "${CLUSTER_REMOTE}:${acl_name}" ingress \
|
|
description="$desc" 2>/dev/null || true
|
|
done
|
|
info " Removed monitoring rules from ${acl_name}"
|
|
fi
|
|
done
|
|
|
|
# Delete containers
|
|
step "Deleting containers"
|
|
if container_exists monitoring; then
|
|
info "Deleting monitoring..."
|
|
incus delete "$(container_ref monitoring)" --force 2>/dev/null || true
|
|
success "Deleted monitoring"
|
|
fi
|
|
|
|
idx=1
|
|
for ip in "${NODE_EXP_IPS[@]}"; do
|
|
local name
|
|
name=$(node_exp_name "$idx")
|
|
if container_exists "$name"; then
|
|
info "Deleting ${name}..."
|
|
incus delete "$(container_ref "$name")" --force 2>/dev/null || true
|
|
success "Deleted ${name}"
|
|
fi
|
|
idx=$((idx + 1))
|
|
done
|
|
|
|
# Delete workload containers
|
|
for wl in workload-web workload-api; do
|
|
if container_exists "$wl"; then
|
|
info "Deleting ${wl}..."
|
|
incus delete "$(container_ref "$wl")" --force 2>/dev/null || true
|
|
success "Deleted ${wl}"
|
|
fi
|
|
done
|
|
|
|
# Delete network forward
|
|
step "Removing network forward"
|
|
if incus network forward show "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP" &>/dev/null 2>&1; then
|
|
incus network forward delete "${CLUSTER_REMOTE}:${OVN_NETWORK}" "$FORWARD_VIP"
|
|
success "Deleted network forward ${FORWARD_VIP}"
|
|
fi
|
|
|
|
# Delete ACL
|
|
step "Removing ACL"
|
|
if incus network acl show "$(container_ref monitoring-allow)" &>/dev/null 2>&1; then
|
|
incus network acl delete "$(container_ref monitoring-allow)" 2>/dev/null || true
|
|
success "Deleted ACL monitoring-allow"
|
|
fi
|
|
|
|
echo
|
|
success "Cleanup complete"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Doctor action
|
|
# ---------------------------------------------------------------------------
|
|
|
|
action_doctor() {
|
|
step "Diagnosing observability stack"
|
|
echo
|
|
|
|
if ! container_exists monitoring || ! container_running monitoring; then
|
|
error "Monitoring container is not running"
|
|
error "Deploy with: ${SCRIPT_NAME} --deploy"
|
|
return 1
|
|
fi
|
|
|
|
local issues=0
|
|
|
|
# Check container networking
|
|
info "Container networking:"
|
|
if container_exec monitoring ping -c 1 -W 3 10.10.10.1 &>/dev/null; then
|
|
success " Gateway (10.10.10.1): reachable"
|
|
else
|
|
error " Gateway (10.10.10.1): unreachable"
|
|
issues=$((issues + 1))
|
|
fi
|
|
|
|
local idx=1
|
|
for ip in "${NODE_EXP_IPS[@]}"; do
|
|
local name
|
|
name=$(node_exp_name "$idx")
|
|
if container_exec monitoring ping -c 1 -W 3 "$ip" &>/dev/null; then
|
|
success " ${name} (${ip}): reachable"
|
|
else
|
|
warn " ${name} (${ip}): unreachable"
|
|
issues=$((issues + 1))
|
|
fi
|
|
idx=$((idx + 1))
|
|
done
|
|
|
|
for ip in "${HAPROXY_IPS[@]}"; do
|
|
if container_exec monitoring ping -c 1 -W 3 "$ip" &>/dev/null; then
|
|
success " HAProxy (${ip}): reachable"
|
|
else
|
|
warn " HAProxy (${ip}): unreachable"
|
|
issues=$((issues + 1))
|
|
fi
|
|
done
|
|
|
|
# Check Prometheus scrape errors
|
|
echo
|
|
info "Prometheus scrape health:"
|
|
local targets_json
|
|
targets_json=$(curl -s --connect-timeout 5 "http://${MONITORING_IP}:9090/api/v1/targets" 2>/dev/null || echo "")
|
|
if [[ -n "$targets_json" ]]; then
|
|
echo "$targets_json" | python3 -c "
|
|
import sys, json
|
|
try:
|
|
data = json.load(sys.stdin)
|
|
targets = data.get('data', {}).get('activeTargets', [])
|
|
down = [t for t in targets if t.get('health') != 'up']
|
|
if down:
|
|
for t in down:
|
|
job = t.get('labels', {}).get('job', '?')
|
|
inst = t.get('labels', {}).get('instance', '?')
|
|
err = t.get('lastError', 'unknown error')
|
|
print(f' DOWN: {job} {inst}')
|
|
print(f' {err}')
|
|
else:
|
|
print(f' All {len(targets)} target(s) are UP')
|
|
except Exception as e:
|
|
print(f' (parse error: {e})')
|
|
" 2>/dev/null || warn " Could not query Prometheus"
|
|
else
|
|
error " Prometheus not reachable"
|
|
issues=$((issues + 1))
|
|
fi
|
|
|
|
# Check systemd service failures
|
|
echo
|
|
info "Service health:"
|
|
for svc in prometheus grafana-server loki promtail; do
|
|
local status
|
|
status=$(container_exec monitoring systemctl is-active "$svc" 2>/dev/null || echo "inactive")
|
|
if [[ "$status" == "active" ]]; then
|
|
success " ${svc}: active"
|
|
else
|
|
error " ${svc}: ${status}"
|
|
# Show recent logs
|
|
local logs
|
|
logs=$(container_exec monitoring journalctl -u "$svc" --no-pager -n 5 --no-hostname 2>/dev/null || true)
|
|
if [[ -n "$logs" ]]; then
|
|
echo "$logs" | while IFS= read -r line; do
|
|
detail " ${line}"
|
|
done
|
|
fi
|
|
issues=$((issues + 1))
|
|
fi
|
|
done
|
|
|
|
# Check disk space
|
|
echo
|
|
info "Disk space (monitoring container):"
|
|
local disk_usage
|
|
disk_usage=$(container_exec monitoring df -h / 2>/dev/null | tail -1 || echo "")
|
|
if [[ -n "$disk_usage" ]]; then
|
|
local use_pct
|
|
use_pct=$(echo "$disk_usage" | awk '{print $5}' | tr -d '%')
|
|
if [[ -n "$use_pct" ]] && [[ "$use_pct" -gt 85 ]]; then
|
|
warn " Disk usage: ${use_pct}% -- consider increasing disk size"
|
|
issues=$((issues + 1))
|
|
else
|
|
success " Disk usage: ${use_pct}%"
|
|
fi
|
|
else
|
|
warn " Could not check disk usage"
|
|
fi
|
|
|
|
echo
|
|
if [[ $issues -eq 0 ]]; then
|
|
success "No issues found"
|
|
else
|
|
warn "${issues} issue(s) found -- review the output above"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Argument parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
parse_args() {
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--deploy) ACTION="deploy";;
|
|
--workloads) ACTION="workloads";;
|
|
--status) ACTION="status";;
|
|
--cleanup) ACTION="cleanup";;
|
|
--doctor) ACTION="doctor";;
|
|
-r|--remote) CLUSTER_REMOTE="$2"; shift;;
|
|
-n|--dry-run) DRY_RUN=true;;
|
|
-v|--verbose) VERBOSE=true;;
|
|
-q|--quiet) QUIET=true;;
|
|
-h|--help) usage; exit 0;;
|
|
*)
|
|
error "Unknown option: $1"
|
|
echo "Run '${SCRIPT_NAME} --help' for usage."
|
|
exit 1
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
if [[ -z "$ACTION" ]]; then
|
|
error "No action specified"
|
|
echo "Run '${SCRIPT_NAME} --help' for usage."
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
main() {
|
|
setup_colors
|
|
parse_args "$@"
|
|
|
|
echo
|
|
case "$ACTION" in
|
|
deploy) action_deploy;;
|
|
workloads) action_workloads;;
|
|
status) action_status;;
|
|
cleanup) action_cleanup;;
|
|
doctor) action_doctor;;
|
|
esac
|
|
}
|
|
|
|
main "$@"
|