#!/usr/bin/env bash # ovn-inspect -- Inspect OVN/OVS topology for an Incus cluster # # Usage: ovn-inspect [ACTION] [OPTIONS] # # Actions: --nb, --sb, --ovs, --full, --trace # Requires: incus CLI with cluster remote configured # # Outputs formatted topology information on stdout. set -euo pipefail # --------------------------------------------------------------------------- # Colors # --------------------------------------------------------------------------- setup_colors() { if [[ "${NO_COLOR:-}" == "1" || "${TERM:-}" == "dumb" ]]; then BOLD="" DIM="" RESET="" CYAN="" GREEN="" YELLOW="" RED="" BLUE="" else BOLD=$'\033[1m' DIM=$'\033[2m' RESET=$'\033[0m' CYAN=$'\033[36m' GREEN=$'\033[32m' YELLOW=$'\033[33m' RED=$'\033[31m' BLUE=$'\033[34m' fi } setup_colors # --------------------------------------------------------------------------- # Usage # --------------------------------------------------------------------------- usage() { cat <<'EOF' Usage: ovn-inspect [ACTION] [OPTIONS] Inspect the OVN/OVS topology of an Incus cluster. Actions: --nb Show OVN Northbound database (logical topology) --sb Show OVN Southbound database (physical bindings) --ovs Show OVS bridges, ports, and tunnels per node --full Show all of the above --trace Trace packet path from LAN through OVN LB to backends Options: --remote NAME Incus remote name (default: oc-node-01) --ovn-container Name of ovn-central container (default: ovn-central) --json Output raw JSON/OVSDB data instead of formatted text --dry-run Show commands that would be executed --help, -h Show this help Examples: ovn-inspect --nb ovn-inspect --full ovn-inspect --trace 192.168.103.200 ovn-inspect --ovs --remote my-cluster EOF exit 0 } # --------------------------------------------------------------------------- # Defaults and argument parsing # --------------------------------------------------------------------------- ACTION="" REMOTE="oc-node-01" OVN_CONTAINER="ovn-central" RAW_JSON=false DRY_RUN=false TRACE_VIP="" while [[ $# -gt 0 ]]; do case "$1" in --nb) ACTION="nb"; shift ;; --sb) ACTION="sb"; shift ;; --ovs) ACTION="ovs"; shift ;; --full) ACTION="full"; shift ;; --trace) ACTION="trace" TRACE_VIP="${2:-}" if [[ -z "$TRACE_VIP" ]]; then echo "Error: --trace requires a VIP address" >&2 exit 1 fi shift 2 ;; --remote) REMOTE="${2:?--remote requires a name}"; shift 2 ;; --ovn-container) OVN_CONTAINER="${2:?--ovn-container requires a name}"; shift 2 ;; --json) RAW_JSON=true; shift ;; --dry-run) DRY_RUN=true; shift ;; -h|--help) usage ;; *) echo "Error: unknown option: $1" >&2 echo "Run 'ovn-inspect --help' for usage." >&2 exit 1 ;; esac done if [[ -z "$ACTION" ]]; then usage fi # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- section() { echo; echo "${BOLD}${CYAN}=== $1 ===${RESET}"; } subsection() { echo; echo " ${BOLD}${GREEN}--- $1 ---${RESET}"; } kv() { printf " ${DIM}%-30s${RESET} %s\n" "$1" "$2"; } warn() { echo " ${YELLOW}WARNING: $1${RESET}" >&2; } # Execute a command inside the OVN container ovn_exec() { local cmd="$*" if [[ "$DRY_RUN" == true ]]; then echo " ${DIM}[dry-run] incus exec ${REMOTE}:${OVN_CONTAINER} -- $cmd${RESET}" return 0 fi incus exec "${REMOTE}:${OVN_CONTAINER}" -- $cmd 2>&1 } # Execute in any container container_exec() { local container="$1"; shift local cmd="$*" if [[ "$DRY_RUN" == true ]]; then echo " ${DIM}[dry-run] incus exec ${REMOTE}:${container} -- $cmd${RESET}" return 0 fi incus exec "${REMOTE}:${container}" -- $cmd 2>&1 } # Map OVN port name to Incus instance name by matching IPs build_port_map() { # Get instance list with IPs INSTANCE_LIST=$(incus list "${REMOTE}:" -f csv -c n4 2>/dev/null || echo "") } port_to_instance() { local port_name="$1" # Extract UUID from port name: incus-net8-instance--eth0 local uuid uuid=$(echo "$port_name" | sed -n 's/.*instance-\(.*\)-eth0/\1/p') if [[ -z "$uuid" ]]; then echo "$port_name" return fi # Try to find instance by checking instance details # Fallback: use the DNS records which we can query echo "$port_name" } # --------------------------------------------------------------------------- # NB: Northbound database # --------------------------------------------------------------------------- show_nb() { section "OVN Northbound Database (Logical Topology)" subsection "Logical Switches" local switches switches=$(ovn_exec ovn-nbctl --format=table ls-list) echo "$switches" | while IFS= read -r line; do echo " $line" done subsection "Logical Routers" local routers routers=$(ovn_exec ovn-nbctl --format=table lr-list) echo "$routers" | while IFS= read -r line; do echo " $line" done # Get all switch names for detailed inspection local switch_names switch_names=$(ovn_exec ovn-nbctl ls-list 2>/dev/null | sed 's/.*(\(.*\))/\1/' || echo "") while IFS= read -r sw; do [[ -z "$sw" ]] && continue subsection "Switch: $sw — Ports" local ports ports=$(ovn_exec ovn-nbctl lsp-list "$sw" < /dev/null) echo "$ports" | while IFS= read -r line; do echo " $line" done done <<< "$switch_names" # Router details local router_names router_names=$(ovn_exec ovn-nbctl lr-list 2>/dev/null | sed 's/.*(\(.*\))/\1/' || echo "") while IFS= read -r lr; do [[ -z "$lr" ]] && continue subsection "Router: $lr — Ports" ovn_exec ovn-nbctl lrp-list "$lr" < /dev/null | while IFS= read -r line; do echo " $line" done subsection "Router: $lr — NAT Rules" ovn_exec ovn-nbctl lr-nat-list "$lr" < /dev/null | while IFS= read -r line; do echo " $line" done subsection "Router: $lr — Static Routes" ovn_exec ovn-nbctl lr-route-list "$lr" < /dev/null | while IFS= read -r line; do echo " $line" done done <<< "$router_names" subsection "Load Balancers" ovn_exec ovn-nbctl lb-list | while IFS= read -r line; do echo " $line" done subsection "DHCP Options" ovn_exec ovn-nbctl find DHCP_Options | while IFS= read -r line; do echo " $line" done subsection "DNS Records" local dns_output dns_output=$(ovn_exec ovn-nbctl find DNS) # Extract just hostname→IP mappings for readability echo "$dns_output" | grep -o '[a-z].*\.incus="[^"]*"' | sort | while IFS= read -r line; do echo " $line" done subsection "ACLs (Internal Switch)" local int_switch int_switch=$(echo "$switch_names" | grep -- "ls-int" | head -1) if [[ -n "$int_switch" ]]; then ovn_exec ovn-nbctl acl-list "$int_switch" | while IFS= read -r line; do echo " $line" done else echo " ${DIM}(no internal switch found)${RESET}" fi } # --------------------------------------------------------------------------- # SB: Southbound database # --------------------------------------------------------------------------- show_sb() { section "OVN Southbound Database (Physical Bindings)" subsection "Chassis (Nodes)" ovn_exec ovn-sbctl show | while IFS= read -r line; do echo " $line" done subsection "Encapsulation (Geneve Tunnels)" ovn_exec ovn-sbctl list Encap | while IFS= read -r line; do echo " $line" done subsection "Datapath Bindings (Tunnel Keys)" ovn_exec ovn-sbctl list Datapath_Binding | while IFS= read -r line; do echo " $line" done subsection "HA Chassis Groups (Gateway Scheduling)" ovn_exec ovn-sbctl list HA_Chassis_Group | while IFS= read -r line; do echo " $line" done ovn_exec ovn-sbctl list HA_Chassis | while IFS= read -r line; do echo " $line" done subsection "Port Bindings Summary" local bindings bindings=$(ovn_exec ovn-sbctl --format=csv --no-headings --columns=logical_port,type,chassis,tunnel_key list Port_Binding 2>/dev/null || echo "") printf " ${DIM}%-65s %-18s %-40s %s${RESET}\n" "LOGICAL_PORT" "TYPE" "CHASSIS" "TUNNEL_KEY" echo "$bindings" | sort | while IFS=',' read -r port type chassis tkey; do port="${port//\"/}" type="${type//\"/}" printf " %-65s %-18s %-40s %s\n" "$port" "${type:-(VIF)}" "$chassis" "$tkey" done } # --------------------------------------------------------------------------- # OVS: Physical switch layer # --------------------------------------------------------------------------- show_ovs() { section "OVS Physical Layer" # Get cluster members local members members=$(incus cluster list "${REMOTE}:" -f csv -c n 2>/dev/null || echo "") if [[ -z "$members" ]]; then warn "Could not list cluster members" return fi echo echo " ${DIM}This action deploys temporary privileged Alpine containers on net-prod" echo " with /run/openvswitch mounted to access OVS on each node.${RESET}" echo local containers_created=() while IFS= read -r node; do [[ -z "$node" ]] && continue local inspect_name="ovs-inspect-${node}" subsection "Node: $node" if [[ "$DRY_RUN" == true ]]; then echo " [dry-run] Would deploy ${inspect_name} on ${node}" echo " [dry-run] Would run: ovs-vsctl show" echo " [dry-run] Would run: ovs-ofctl dump-flows br-int" echo " [dry-run] Would delete ${inspect_name}" continue fi # Deploy temp container echo " ${DIM}Deploying ${inspect_name}...${RESET}" if ! incus launch "images:alpine/3.21" "${REMOTE}:${inspect_name}" \ --target "$node" -c security.privileged=true \ -c limits.memory=128MiB -n net-prod 2>/dev/null; then warn "Failed to deploy ${inspect_name} on ${node}" continue fi containers_created+=("$inspect_name") # Add OVS socket mount incus config device add "${REMOTE}:${inspect_name}" ovs-run \ disk source=/run/openvswitch path=/run/openvswitch 2>/dev/null || true # Wait for container + install OVS local retries=0 while ! incus exec "${REMOTE}:${inspect_name}" -- true 2>/dev/null; do retries=$((retries + 1)) if [[ $retries -gt 15 ]]; then warn "Container ${inspect_name} not ready after 15s" continue 2 fi sleep 1 done incus exec "${REMOTE}:${inspect_name}" -- \ apk add --no-cache openvswitch 2>/dev/null >/dev/null || { warn "Failed to install openvswitch in ${inspect_name}" continue } # Gather data echo echo " ${BOLD}Bridges:${RESET}" container_exec "$inspect_name" ovs-vsctl show | while IFS= read -r line; do echo " $line" done echo echo " ${BOLD}br-int port → OVN mapping:${RESET}" local ports ports=$(container_exec "$inspect_name" ovs-vsctl list-ports br-int 2>/dev/null || echo "") while IFS= read -r port; do [[ -z "$port" ]] && continue local ext_ids ext_ids=$(container_exec "$inspect_name" ovs-vsctl get Interface "$port" external_ids 2>/dev/null || echo "{}") local iface_type iface_type=$(container_exec "$inspect_name" ovs-vsctl get Interface "$port" type 2>/dev/null || echo "") iface_type="${iface_type//\"/}" printf " %-20s %-10s %s\n" "$port" "${iface_type:-(veth)}" "$ext_ids" done <<< "$ports" done <<< "$members" # Cleanup if [[ ${#containers_created[@]} -gt 0 ]]; then echo echo " ${DIM}Cleaning up temporary containers...${RESET}" for c in "${containers_created[@]}"; do incus delete "${REMOTE}:${c}" --force 2>/dev/null || true done echo " ${GREEN}Done.${RESET}" fi } # --------------------------------------------------------------------------- # Trace: Packet path through OVN # --------------------------------------------------------------------------- show_trace() { local vip="$TRACE_VIP" section "Packet Trace: LAN → ${vip}" # Find the LB for this VIP subsection "Load Balancer for ${vip}" local lb_info lb_info=$(ovn_exec ovn-nbctl lb-list) local lb_line lb_line=$(echo "$lb_info" | grep "$vip" || echo "") if [[ -z "$lb_line" ]]; then warn "No load balancer found for VIP ${vip}" echo " Available LBs:" echo "$lb_info" | while IFS= read -r line; do echo " $line"; done return fi echo " $lb_line" # Extract backends local backends backends=$(echo "$lb_line" | grep -oP '(?<=IPs\s{4})[^\n]+' || echo "") if [[ -z "$backends" ]]; then # Try different column format backends=$(echo "$lb_line" | awk '{print $NF}') fi # Find the router subsection "Router NAT (SNAT)" ovn_exec ovn-nbctl lr-nat-list "$(ovn_exec ovn-nbctl lr-list 2>/dev/null | sed 's/.*(\(.*\))/\1/' | head -1)" \ | while IFS= read -r line; do echo " $line"; done # Find gateway chassis subsection "Gateway Chassis" local gw_binding gw_binding=$(ovn_exec ovn-sbctl find Port_Binding type=chassisredirect 2>/dev/null | grep -A2 "chassis" || echo "") ovn_exec ovn-sbctl show | grep -B1 "cr-" | while IFS= read -r line; do echo " $line" done # Build the trace narrative subsection "Packet Path" local router_name router_name=$(ovn_exec ovn-nbctl lr-list 2>/dev/null | sed 's/.*(\(.*\))/\1/' | head -1) local ext_port ext_port=$(ovn_exec ovn-nbctl lrp-list "$router_name" | grep "lrp-ext" | awk '{print $2}' | tr -d '()') local int_port int_port=$(ovn_exec ovn-nbctl lrp-list "$router_name" | grep "lrp-int" | awk '{print $2}' | tr -d '()') local gw_node gw_node=$(ovn_exec ovn-sbctl show | grep -B5 "cr-" | grep "hostname:" | tail -1 | awk '{print $2}') echo echo " ${BOLD}1. Ingress (LAN → Provider Bridge)${RESET}" echo " Client sends packet to ${vip}:80" echo " → Arrives on gateway chassis ${GREEN}${gw_node}${RESET}" echo " → Physical NIC → incusovn7 (provider bridge) → patch port → br-int" echo echo " ${BOLD}2. OVN Router (External → Internal)${RESET}" echo " → br-int delivers to external switch (ls-ext)" echo " → Patch port to router ${router_name}" echo " → Router external port: ${ext_port}" echo " → ${YELLOW}LB DNAT: ${vip}:80 → backend IPs${RESET}" echo echo " ${BOLD}3. Load Balancing${RESET}" echo " → OVN conntrack LB selects backend from:" echo " $lb_line" | awk '{print " " $0}' echo echo " ${BOLD}4. Forwarding to Backend${RESET}" echo " → Router internal port: ${int_port} (10.10.10.1)" echo " → Internal switch (ls-int)" echo " → If backend on same chassis: deliver via local veth" echo " → If backend on remote chassis: Geneve encap → tunnel → remote br-int → veth" echo echo " ${BOLD}5. Return Path${RESET}" echo " → Backend replies to client" echo " → ls-int → router → ${YELLOW}SNAT: 10.10.10.0/24 → ${vip}${RESET}" echo " → ls-ext → provider bridge → physical NIC → LAN" echo echo " ${BOLD}Tunnel Details:${RESET}" echo " Protocol: Geneve (UDP 6081), BFD enabled" ovn_exec ovn-sbctl list Encap | grep -E "ip|type" | while IFS= read -r line; do echo " $line" done } # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- case "$ACTION" in nb) show_nb ;; sb) show_sb ;; ovs) show_ovs ;; full) show_nb; show_sb; show_ovs ;; trace) show_trace ;; esac