484 lines
16 KiB
Bash
Executable File
484 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# ovn-inspect -- Inspect OVN/OVS topology for an Incus cluster
|
|
#
|
|
# Usage: ovn-inspect [ACTION] [OPTIONS]
|
|
#
|
|
# Actions: --nb, --sb, --ovs, --full, --trace <VIP>
|
|
# Requires: incus CLI with cluster remote configured
|
|
#
|
|
# Outputs formatted topology information on stdout.
|
|
|
|
set -euo pipefail
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Colors
|
|
# ---------------------------------------------------------------------------
|
|
|
|
setup_colors() {
|
|
if [[ "${NO_COLOR:-}" == "1" || "${TERM:-}" == "dumb" ]]; then
|
|
BOLD="" DIM="" RESET="" CYAN="" GREEN="" YELLOW="" RED="" BLUE=""
|
|
else
|
|
BOLD=$'\033[1m' DIM=$'\033[2m' RESET=$'\033[0m'
|
|
CYAN=$'\033[36m' GREEN=$'\033[32m' YELLOW=$'\033[33m'
|
|
RED=$'\033[31m' BLUE=$'\033[34m'
|
|
fi
|
|
}
|
|
setup_colors
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Usage
|
|
# ---------------------------------------------------------------------------
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage: ovn-inspect [ACTION] [OPTIONS]
|
|
|
|
Inspect the OVN/OVS topology of an Incus cluster.
|
|
|
|
Actions:
|
|
--nb Show OVN Northbound database (logical topology)
|
|
--sb Show OVN Southbound database (physical bindings)
|
|
--ovs Show OVS bridges, ports, and tunnels per node
|
|
--full Show all of the above
|
|
--trace <VIP> Trace packet path from LAN through OVN LB to backends
|
|
|
|
Options:
|
|
--remote NAME Incus remote name (default: oc-node-01)
|
|
--ovn-container Name of ovn-central container (default: ovn-central)
|
|
--json Output raw JSON/OVSDB data instead of formatted text
|
|
--dry-run Show commands that would be executed
|
|
--help, -h Show this help
|
|
|
|
Examples:
|
|
ovn-inspect --nb
|
|
ovn-inspect --full
|
|
ovn-inspect --trace 192.168.103.200
|
|
ovn-inspect --ovs --remote my-cluster
|
|
EOF
|
|
exit 0
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Defaults and argument parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
ACTION=""
|
|
REMOTE="oc-node-01"
|
|
OVN_CONTAINER="ovn-central"
|
|
RAW_JSON=false
|
|
DRY_RUN=false
|
|
TRACE_VIP=""
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--nb) ACTION="nb"; shift ;;
|
|
--sb) ACTION="sb"; shift ;;
|
|
--ovs) ACTION="ovs"; shift ;;
|
|
--full) ACTION="full"; shift ;;
|
|
--trace)
|
|
ACTION="trace"
|
|
TRACE_VIP="${2:-}"
|
|
if [[ -z "$TRACE_VIP" ]]; then
|
|
echo "Error: --trace requires a VIP address" >&2
|
|
exit 1
|
|
fi
|
|
shift 2
|
|
;;
|
|
--remote) REMOTE="${2:?--remote requires a name}"; shift 2 ;;
|
|
--ovn-container) OVN_CONTAINER="${2:?--ovn-container requires a name}"; shift 2 ;;
|
|
--json) RAW_JSON=true; shift ;;
|
|
--dry-run) DRY_RUN=true; shift ;;
|
|
-h|--help) usage ;;
|
|
*)
|
|
echo "Error: unknown option: $1" >&2
|
|
echo "Run 'ovn-inspect --help' for usage." >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [[ -z "$ACTION" ]]; then
|
|
usage
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
section() { echo; echo "${BOLD}${CYAN}=== $1 ===${RESET}"; }
|
|
subsection() { echo; echo " ${BOLD}${GREEN}--- $1 ---${RESET}"; }
|
|
kv() { printf " ${DIM}%-30s${RESET} %s\n" "$1" "$2"; }
|
|
warn() { echo " ${YELLOW}WARNING: $1${RESET}" >&2; }
|
|
|
|
# Execute a command inside the OVN container
|
|
ovn_exec() {
|
|
local cmd="$*"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo " ${DIM}[dry-run] incus exec ${REMOTE}:${OVN_CONTAINER} -- $cmd${RESET}"
|
|
return 0
|
|
fi
|
|
incus exec "${REMOTE}:${OVN_CONTAINER}" -- $cmd 2>&1
|
|
}
|
|
|
|
# Execute in any container
|
|
container_exec() {
|
|
local container="$1"; shift
|
|
local cmd="$*"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo " ${DIM}[dry-run] incus exec ${REMOTE}:${container} -- $cmd${RESET}"
|
|
return 0
|
|
fi
|
|
incus exec "${REMOTE}:${container}" -- $cmd 2>&1
|
|
}
|
|
|
|
# Map OVN port name to Incus instance name by matching IPs
|
|
build_port_map() {
|
|
# Get instance list with IPs
|
|
INSTANCE_LIST=$(incus list "${REMOTE}:" -f csv -c n4 2>/dev/null || echo "")
|
|
}
|
|
|
|
port_to_instance() {
|
|
local port_name="$1"
|
|
# Extract UUID from port name: incus-net8-instance-<UUID>-eth0
|
|
local uuid
|
|
uuid=$(echo "$port_name" | sed -n 's/.*instance-\(.*\)-eth0/\1/p')
|
|
if [[ -z "$uuid" ]]; then
|
|
echo "$port_name"
|
|
return
|
|
fi
|
|
# Try to find instance by checking instance details
|
|
# Fallback: use the DNS records which we can query
|
|
echo "$port_name"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# NB: Northbound database
|
|
# ---------------------------------------------------------------------------
|
|
|
|
show_nb() {
|
|
section "OVN Northbound Database (Logical Topology)"
|
|
|
|
subsection "Logical Switches"
|
|
local switches
|
|
switches=$(ovn_exec ovn-nbctl --format=table ls-list)
|
|
echo "$switches" | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
|
|
subsection "Logical Routers"
|
|
local routers
|
|
routers=$(ovn_exec ovn-nbctl --format=table lr-list)
|
|
echo "$routers" | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
|
|
# Get all switch names for detailed inspection
|
|
local switch_names
|
|
switch_names=$(ovn_exec ovn-nbctl ls-list 2>/dev/null | sed 's/.*(\(.*\))/\1/' || echo "")
|
|
|
|
while IFS= read -r sw; do
|
|
[[ -z "$sw" ]] && continue
|
|
subsection "Switch: $sw — Ports"
|
|
local ports
|
|
ports=$(ovn_exec ovn-nbctl lsp-list "$sw" < /dev/null)
|
|
echo "$ports" | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
done <<< "$switch_names"
|
|
|
|
# Router details
|
|
local router_names
|
|
router_names=$(ovn_exec ovn-nbctl lr-list 2>/dev/null | sed 's/.*(\(.*\))/\1/' || echo "")
|
|
|
|
while IFS= read -r lr; do
|
|
[[ -z "$lr" ]] && continue
|
|
|
|
subsection "Router: $lr — Ports"
|
|
ovn_exec ovn-nbctl lrp-list "$lr" < /dev/null | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
|
|
subsection "Router: $lr — NAT Rules"
|
|
ovn_exec ovn-nbctl lr-nat-list "$lr" < /dev/null | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
|
|
subsection "Router: $lr — Static Routes"
|
|
ovn_exec ovn-nbctl lr-route-list "$lr" < /dev/null | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
done <<< "$router_names"
|
|
|
|
subsection "Load Balancers"
|
|
ovn_exec ovn-nbctl lb-list | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
|
|
subsection "DHCP Options"
|
|
ovn_exec ovn-nbctl find DHCP_Options | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
|
|
subsection "DNS Records"
|
|
local dns_output
|
|
dns_output=$(ovn_exec ovn-nbctl find DNS)
|
|
# Extract just hostname→IP mappings for readability
|
|
echo "$dns_output" | grep -o '[a-z].*\.incus="[^"]*"' | sort | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
|
|
subsection "ACLs (Internal Switch)"
|
|
local int_switch
|
|
int_switch=$(echo "$switch_names" | grep -- "ls-int" | head -1)
|
|
if [[ -n "$int_switch" ]]; then
|
|
ovn_exec ovn-nbctl acl-list "$int_switch" | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
else
|
|
echo " ${DIM}(no internal switch found)${RESET}"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# SB: Southbound database
|
|
# ---------------------------------------------------------------------------
|
|
|
|
show_sb() {
|
|
section "OVN Southbound Database (Physical Bindings)"
|
|
|
|
subsection "Chassis (Nodes)"
|
|
ovn_exec ovn-sbctl show | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
|
|
subsection "Encapsulation (Geneve Tunnels)"
|
|
ovn_exec ovn-sbctl list Encap | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
|
|
subsection "Datapath Bindings (Tunnel Keys)"
|
|
ovn_exec ovn-sbctl list Datapath_Binding | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
|
|
subsection "HA Chassis Groups (Gateway Scheduling)"
|
|
ovn_exec ovn-sbctl list HA_Chassis_Group | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
ovn_exec ovn-sbctl list HA_Chassis | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
|
|
subsection "Port Bindings Summary"
|
|
local bindings
|
|
bindings=$(ovn_exec ovn-sbctl --format=csv --no-headings --columns=logical_port,type,chassis,tunnel_key list Port_Binding 2>/dev/null || echo "")
|
|
printf " ${DIM}%-65s %-18s %-40s %s${RESET}\n" "LOGICAL_PORT" "TYPE" "CHASSIS" "TUNNEL_KEY"
|
|
echo "$bindings" | sort | while IFS=',' read -r port type chassis tkey; do
|
|
port="${port//\"/}"
|
|
type="${type//\"/}"
|
|
printf " %-65s %-18s %-40s %s\n" "$port" "${type:-(VIF)}" "$chassis" "$tkey"
|
|
done
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# OVS: Physical switch layer
|
|
# ---------------------------------------------------------------------------
|
|
|
|
show_ovs() {
|
|
section "OVS Physical Layer"
|
|
|
|
# Get cluster members
|
|
local members
|
|
members=$(incus cluster list "${REMOTE}:" -f csv -c n 2>/dev/null || echo "")
|
|
|
|
if [[ -z "$members" ]]; then
|
|
warn "Could not list cluster members"
|
|
return
|
|
fi
|
|
|
|
echo
|
|
echo " ${DIM}This action deploys temporary privileged Alpine containers on net-prod"
|
|
echo " with /run/openvswitch mounted to access OVS on each node.${RESET}"
|
|
echo
|
|
|
|
local containers_created=()
|
|
|
|
while IFS= read -r node; do
|
|
[[ -z "$node" ]] && continue
|
|
local inspect_name="ovs-inspect-${node}"
|
|
|
|
subsection "Node: $node"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo " [dry-run] Would deploy ${inspect_name} on ${node}"
|
|
echo " [dry-run] Would run: ovs-vsctl show"
|
|
echo " [dry-run] Would run: ovs-ofctl dump-flows br-int"
|
|
echo " [dry-run] Would delete ${inspect_name}"
|
|
continue
|
|
fi
|
|
|
|
# Deploy temp container
|
|
echo " ${DIM}Deploying ${inspect_name}...${RESET}"
|
|
if ! incus launch "images:alpine/3.21" "${REMOTE}:${inspect_name}" \
|
|
--target "$node" -c security.privileged=true \
|
|
-c limits.memory=128MiB -n net-prod 2>/dev/null; then
|
|
warn "Failed to deploy ${inspect_name} on ${node}"
|
|
continue
|
|
fi
|
|
containers_created+=("$inspect_name")
|
|
|
|
# Add OVS socket mount
|
|
incus config device add "${REMOTE}:${inspect_name}" ovs-run \
|
|
disk source=/run/openvswitch path=/run/openvswitch 2>/dev/null || true
|
|
|
|
# Wait for container + install OVS
|
|
local retries=0
|
|
while ! incus exec "${REMOTE}:${inspect_name}" -- true 2>/dev/null; do
|
|
retries=$((retries + 1))
|
|
if [[ $retries -gt 15 ]]; then
|
|
warn "Container ${inspect_name} not ready after 15s"
|
|
continue 2
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
incus exec "${REMOTE}:${inspect_name}" -- \
|
|
apk add --no-cache openvswitch 2>/dev/null >/dev/null || {
|
|
warn "Failed to install openvswitch in ${inspect_name}"
|
|
continue
|
|
}
|
|
|
|
# Gather data
|
|
echo
|
|
echo " ${BOLD}Bridges:${RESET}"
|
|
container_exec "$inspect_name" ovs-vsctl show | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
|
|
echo
|
|
echo " ${BOLD}br-int port → OVN mapping:${RESET}"
|
|
local ports
|
|
ports=$(container_exec "$inspect_name" ovs-vsctl list-ports br-int 2>/dev/null || echo "")
|
|
while IFS= read -r port; do
|
|
[[ -z "$port" ]] && continue
|
|
local ext_ids
|
|
ext_ids=$(container_exec "$inspect_name" ovs-vsctl get Interface "$port" external_ids 2>/dev/null || echo "{}")
|
|
local iface_type
|
|
iface_type=$(container_exec "$inspect_name" ovs-vsctl get Interface "$port" type 2>/dev/null || echo "")
|
|
iface_type="${iface_type//\"/}"
|
|
printf " %-20s %-10s %s\n" "$port" "${iface_type:-(veth)}" "$ext_ids"
|
|
done <<< "$ports"
|
|
|
|
done <<< "$members"
|
|
|
|
# Cleanup
|
|
if [[ ${#containers_created[@]} -gt 0 ]]; then
|
|
echo
|
|
echo " ${DIM}Cleaning up temporary containers...${RESET}"
|
|
for c in "${containers_created[@]}"; do
|
|
incus delete "${REMOTE}:${c}" --force 2>/dev/null || true
|
|
done
|
|
echo " ${GREEN}Done.${RESET}"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Trace: Packet path through OVN
|
|
# ---------------------------------------------------------------------------
|
|
|
|
show_trace() {
|
|
local vip="$TRACE_VIP"
|
|
section "Packet Trace: LAN → ${vip}"
|
|
|
|
# Find the LB for this VIP
|
|
subsection "Load Balancer for ${vip}"
|
|
local lb_info
|
|
lb_info=$(ovn_exec ovn-nbctl lb-list)
|
|
local lb_line
|
|
lb_line=$(echo "$lb_info" | grep "$vip" || echo "")
|
|
if [[ -z "$lb_line" ]]; then
|
|
warn "No load balancer found for VIP ${vip}"
|
|
echo " Available LBs:"
|
|
echo "$lb_info" | while IFS= read -r line; do echo " $line"; done
|
|
return
|
|
fi
|
|
echo " $lb_line"
|
|
|
|
# Extract backends
|
|
local backends
|
|
backends=$(echo "$lb_line" | grep -oP '(?<=IPs\s{4})[^\n]+' || echo "")
|
|
if [[ -z "$backends" ]]; then
|
|
# Try different column format
|
|
backends=$(echo "$lb_line" | awk '{print $NF}')
|
|
fi
|
|
|
|
# Find the router
|
|
subsection "Router NAT (SNAT)"
|
|
ovn_exec ovn-nbctl lr-nat-list "$(ovn_exec ovn-nbctl lr-list 2>/dev/null | sed 's/.*(\(.*\))/\1/' | head -1)" \
|
|
| while IFS= read -r line; do echo " $line"; done
|
|
|
|
# Find gateway chassis
|
|
subsection "Gateway Chassis"
|
|
local gw_binding
|
|
gw_binding=$(ovn_exec ovn-sbctl find Port_Binding type=chassisredirect 2>/dev/null | grep -A2 "chassis" || echo "")
|
|
ovn_exec ovn-sbctl show | grep -B1 "cr-" | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
|
|
# Build the trace narrative
|
|
subsection "Packet Path"
|
|
local router_name
|
|
router_name=$(ovn_exec ovn-nbctl lr-list 2>/dev/null | sed 's/.*(\(.*\))/\1/' | head -1)
|
|
local ext_port
|
|
ext_port=$(ovn_exec ovn-nbctl lrp-list "$router_name" | grep "lrp-ext" | awk '{print $2}' | tr -d '()')
|
|
local int_port
|
|
int_port=$(ovn_exec ovn-nbctl lrp-list "$router_name" | grep "lrp-int" | awk '{print $2}' | tr -d '()')
|
|
local gw_node
|
|
gw_node=$(ovn_exec ovn-sbctl show | grep -B5 "cr-" | grep "hostname:" | tail -1 | awk '{print $2}')
|
|
|
|
echo
|
|
echo " ${BOLD}1. Ingress (LAN → Provider Bridge)${RESET}"
|
|
echo " Client sends packet to ${vip}:80"
|
|
echo " → Arrives on gateway chassis ${GREEN}${gw_node}${RESET}"
|
|
echo " → Physical NIC → incusovn7 (provider bridge) → patch port → br-int"
|
|
echo
|
|
echo " ${BOLD}2. OVN Router (External → Internal)${RESET}"
|
|
echo " → br-int delivers to external switch (ls-ext)"
|
|
echo " → Patch port to router ${router_name}"
|
|
echo " → Router external port: ${ext_port}"
|
|
echo " → ${YELLOW}LB DNAT: ${vip}:80 → backend IPs${RESET}"
|
|
echo
|
|
echo " ${BOLD}3. Load Balancing${RESET}"
|
|
echo " → OVN conntrack LB selects backend from:"
|
|
echo " $lb_line" | awk '{print " " $0}'
|
|
echo
|
|
echo " ${BOLD}4. Forwarding to Backend${RESET}"
|
|
echo " → Router internal port: ${int_port} (10.10.10.1)"
|
|
echo " → Internal switch (ls-int)"
|
|
echo " → If backend on same chassis: deliver via local veth"
|
|
echo " → If backend on remote chassis: Geneve encap → tunnel → remote br-int → veth"
|
|
echo
|
|
echo " ${BOLD}5. Return Path${RESET}"
|
|
echo " → Backend replies to client"
|
|
echo " → ls-int → router → ${YELLOW}SNAT: 10.10.10.0/24 → ${vip}${RESET}"
|
|
echo " → ls-ext → provider bridge → physical NIC → LAN"
|
|
echo
|
|
echo " ${BOLD}Tunnel Details:${RESET}"
|
|
echo " Protocol: Geneve (UDP 6081), BFD enabled"
|
|
ovn_exec ovn-sbctl list Encap | grep -E "ip|type" | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
case "$ACTION" in
|
|
nb) show_nb ;;
|
|
sb) show_sb ;;
|
|
ovs) show_ovs ;;
|
|
full) show_nb; show_sb; show_ovs ;;
|
|
trace) show_trace ;;
|
|
esac
|