diff --git a/.claude/rules/ovn-internals.md b/.claude/rules/ovn-internals.md new file mode 100644 index 0000000..11826f9 --- /dev/null +++ b/.claude/rules/ovn-internals.md @@ -0,0 +1,38 @@ +# OVN Internals Context + +paths: + - ovn-inspect + - ovn-deep-dive + +## OVN Architecture in this Cluster + +3-node IncusOS cluster with OVN overlay network `net-prod` (10.10.10.0/24). +Incus manages OVN objects using a `net8` prefix convention. + +### Key OVN Objects + +- **Logical Router**: `incus-net8-lr` — gateway at 192.168.103.200/22 + - SNAT: 10.10.10.0/24 → 192.168.103.200 + - LB: 192.168.103.200:80 → 10.10.10.50:80, 10.10.10.51:80 +- **Internal Switch**: `incus-net8-ls-int` — 10.10.10.0/24 overlay +- **External Switch**: `incus-net8-ls-ext` — bridges to UPLINK +- **Gateway Chassis**: oc-node-03 (HA group, highest priority) +- **Geneve Tunnels**: Full mesh on 192.168.102.140-142, BFD enabled +- **Provider Bridge**: `incusovn7` on each node maps to UPLINK physical network +- **MTU**: 1442 (1500 - 58 Geneve overhead) + +### Accessing OVN + +- NB/SB commands: `incus exec oc-node-01:ovn-central -- ovn-nbctl ...` +- OVS commands: Need privileged container with `/run/openvswitch` mounted + (deploy Alpine on net-prod, `apk add openvswitch`) +- Helper script: `incusos/helpers/ovn-inspect --nb|--sb|--ovs|--full|--trace` + +### Important Details + +- `incusbr0` has no working NAT — containers needing internet must use `net-prod` +- The `ovn-central` container runs on oc-node-03 (on incusbr0 network) +- OVS version: 3.6.1 +- All OVN names start with `incus-net8-` for the net-prod network +- Instance OVN ports contain the Incus instance UUID +- LB is conntrack-based (ct_lb_mark), stateful diff --git a/CLAUDE.md b/CLAUDE.md index cb7d47d..0afd4f7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,7 +31,8 @@ incus-contrib/ │ ├── helpers/ │ │ ├── proxmox-screenshot # VMID -> PNG console screenshot │ │ ├── proxmox-api # Authenticated API calls (handles ! in token) -│ │ └── aether-browser # Playwright browser automation for Aether web UI +│ │ ├── aether-browser # Playwright browser automation for Aether web UI +│ │ └── ovn-inspect # OVN/OVS topology inspection (--nb/--sb/--ovs/--trace) │ ├── lab-test # Guided lab validation (12 test phases) │ ├── observe-deploy # Single-VM deploy with console screenshots │ ├── proxmox.yaml # Proxmox connection (gitignored) @@ -185,6 +186,7 @@ which files are being edited: | `networking-storage.md` | networking, storage, migration, UTM guides | | `awx-integration.md` | ansible/, deploy-awx, awx-manifests, AWX guide | | `haproxy-lb.md` | deploy-haproxy, HAProxy guide | +| `ovn-internals.md` | ovn-inspect, ovn-deep-dive | | `proxmox-ssh-rules.md` | **Always loaded** (no paths filter) | | `lab-infrastructure.md` | incusos-proxmox, lab-test, examples | diff --git a/incusos/helpers/ovn-inspect b/incusos/helpers/ovn-inspect new file mode 100755 index 0000000..376c65d --- /dev/null +++ b/incusos/helpers/ovn-inspect @@ -0,0 +1,483 @@ +#!/usr/bin/env bash +# ovn-inspect -- Inspect OVN/OVS topology for an Incus cluster +# +# Usage: ovn-inspect [ACTION] [OPTIONS] +# +# Actions: --nb, --sb, --ovs, --full, --trace +# Requires: incus CLI with cluster remote configured +# +# Outputs formatted topology information on stdout. + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Colors +# --------------------------------------------------------------------------- + +setup_colors() { + if [[ "${NO_COLOR:-}" == "1" || "${TERM:-}" == "dumb" ]]; then + BOLD="" DIM="" RESET="" CYAN="" GREEN="" YELLOW="" RED="" BLUE="" + else + BOLD=$'\033[1m' DIM=$'\033[2m' RESET=$'\033[0m' + CYAN=$'\033[36m' GREEN=$'\033[32m' YELLOW=$'\033[33m' + RED=$'\033[31m' BLUE=$'\033[34m' + fi +} +setup_colors + +# --------------------------------------------------------------------------- +# Usage +# --------------------------------------------------------------------------- + +usage() { + cat <<'EOF' +Usage: ovn-inspect [ACTION] [OPTIONS] + +Inspect the OVN/OVS topology of an Incus cluster. + +Actions: + --nb Show OVN Northbound database (logical topology) + --sb Show OVN Southbound database (physical bindings) + --ovs Show OVS bridges, ports, and tunnels per node + --full Show all of the above + --trace Trace packet path from LAN through OVN LB to backends + +Options: + --remote NAME Incus remote name (default: oc-node-01) + --ovn-container Name of ovn-central container (default: ovn-central) + --json Output raw JSON/OVSDB data instead of formatted text + --dry-run Show commands that would be executed + --help, -h Show this help + +Examples: + ovn-inspect --nb + ovn-inspect --full + ovn-inspect --trace 192.168.103.200 + ovn-inspect --ovs --remote my-cluster +EOF + exit 0 +} + +# --------------------------------------------------------------------------- +# Defaults and argument parsing +# --------------------------------------------------------------------------- + +ACTION="" +REMOTE="oc-node-01" +OVN_CONTAINER="ovn-central" +RAW_JSON=false +DRY_RUN=false +TRACE_VIP="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --nb) ACTION="nb"; shift ;; + --sb) ACTION="sb"; shift ;; + --ovs) ACTION="ovs"; shift ;; + --full) ACTION="full"; shift ;; + --trace) + ACTION="trace" + TRACE_VIP="${2:-}" + if [[ -z "$TRACE_VIP" ]]; then + echo "Error: --trace requires a VIP address" >&2 + exit 1 + fi + shift 2 + ;; + --remote) REMOTE="${2:?--remote requires a name}"; shift 2 ;; + --ovn-container) OVN_CONTAINER="${2:?--ovn-container requires a name}"; shift 2 ;; + --json) RAW_JSON=true; shift ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) usage ;; + *) + echo "Error: unknown option: $1" >&2 + echo "Run 'ovn-inspect --help' for usage." >&2 + exit 1 + ;; + esac +done + +if [[ -z "$ACTION" ]]; then + usage +fi + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +section() { echo; echo "${BOLD}${CYAN}=== $1 ===${RESET}"; } +subsection() { echo; echo " ${BOLD}${GREEN}--- $1 ---${RESET}"; } +kv() { printf " ${DIM}%-30s${RESET} %s\n" "$1" "$2"; } +warn() { echo " ${YELLOW}WARNING: $1${RESET}" >&2; } + +# Execute a command inside the OVN container +ovn_exec() { + local cmd="$*" + if [[ "$DRY_RUN" == true ]]; then + echo " ${DIM}[dry-run] incus exec ${REMOTE}:${OVN_CONTAINER} -- $cmd${RESET}" + return 0 + fi + incus exec "${REMOTE}:${OVN_CONTAINER}" -- $cmd 2>&1 +} + +# Execute in any container +container_exec() { + local container="$1"; shift + local cmd="$*" + if [[ "$DRY_RUN" == true ]]; then + echo " ${DIM}[dry-run] incus exec ${REMOTE}:${container} -- $cmd${RESET}" + return 0 + fi + incus exec "${REMOTE}:${container}" -- $cmd 2>&1 +} + +# Map OVN port name to Incus instance name by matching IPs +build_port_map() { + # Get instance list with IPs + INSTANCE_LIST=$(incus list "${REMOTE}:" -f csv -c n4 2>/dev/null || echo "") +} + +port_to_instance() { + local port_name="$1" + # Extract UUID from port name: incus-net8-instance--eth0 + local uuid + uuid=$(echo "$port_name" | sed -n 's/.*instance-\(.*\)-eth0/\1/p') + if [[ -z "$uuid" ]]; then + echo "$port_name" + return + fi + # Try to find instance by checking instance details + # Fallback: use the DNS records which we can query + echo "$port_name" +} + +# --------------------------------------------------------------------------- +# NB: Northbound database +# --------------------------------------------------------------------------- + +show_nb() { + section "OVN Northbound Database (Logical Topology)" + + subsection "Logical Switches" + local switches + switches=$(ovn_exec ovn-nbctl --format=table ls-list) + echo "$switches" | while IFS= read -r line; do + echo " $line" + done + + subsection "Logical Routers" + local routers + routers=$(ovn_exec ovn-nbctl --format=table lr-list) + echo "$routers" | while IFS= read -r line; do + echo " $line" + done + + # Get all switch names for detailed inspection + local switch_names + switch_names=$(ovn_exec ovn-nbctl ls-list 2>/dev/null | sed 's/.*(\(.*\))/\1/' || echo "") + + while IFS= read -r sw; do + [[ -z "$sw" ]] && continue + subsection "Switch: $sw — Ports" + local ports + ports=$(ovn_exec ovn-nbctl lsp-list "$sw" < /dev/null) + echo "$ports" | while IFS= read -r line; do + echo " $line" + done + done <<< "$switch_names" + + # Router details + local router_names + router_names=$(ovn_exec ovn-nbctl lr-list 2>/dev/null | sed 's/.*(\(.*\))/\1/' || echo "") + + while IFS= read -r lr; do + [[ -z "$lr" ]] && continue + + subsection "Router: $lr — Ports" + ovn_exec ovn-nbctl lrp-list "$lr" < /dev/null | while IFS= read -r line; do + echo " $line" + done + + subsection "Router: $lr — NAT Rules" + ovn_exec ovn-nbctl lr-nat-list "$lr" < /dev/null | while IFS= read -r line; do + echo " $line" + done + + subsection "Router: $lr — Static Routes" + ovn_exec ovn-nbctl lr-route-list "$lr" < /dev/null | while IFS= read -r line; do + echo " $line" + done + done <<< "$router_names" + + subsection "Load Balancers" + ovn_exec ovn-nbctl lb-list | while IFS= read -r line; do + echo " $line" + done + + subsection "DHCP Options" + ovn_exec ovn-nbctl find DHCP_Options | while IFS= read -r line; do + echo " $line" + done + + subsection "DNS Records" + local dns_output + dns_output=$(ovn_exec ovn-nbctl find DNS) + # Extract just hostname→IP mappings for readability + echo "$dns_output" | grep -o '[a-z].*\.incus="[^"]*"' | sort | while IFS= read -r line; do + echo " $line" + done + + subsection "ACLs (Internal Switch)" + local int_switch + int_switch=$(echo "$switch_names" | grep -- "ls-int" | head -1) + if [[ -n "$int_switch" ]]; then + ovn_exec ovn-nbctl acl-list "$int_switch" | while IFS= read -r line; do + echo " $line" + done + else + echo " ${DIM}(no internal switch found)${RESET}" + fi +} + +# --------------------------------------------------------------------------- +# SB: Southbound database +# --------------------------------------------------------------------------- + +show_sb() { + section "OVN Southbound Database (Physical Bindings)" + + subsection "Chassis (Nodes)" + ovn_exec ovn-sbctl show | while IFS= read -r line; do + echo " $line" + done + + subsection "Encapsulation (Geneve Tunnels)" + ovn_exec ovn-sbctl list Encap | while IFS= read -r line; do + echo " $line" + done + + subsection "Datapath Bindings (Tunnel Keys)" + ovn_exec ovn-sbctl list Datapath_Binding | while IFS= read -r line; do + echo " $line" + done + + subsection "HA Chassis Groups (Gateway Scheduling)" + ovn_exec ovn-sbctl list HA_Chassis_Group | while IFS= read -r line; do + echo " $line" + done + ovn_exec ovn-sbctl list HA_Chassis | while IFS= read -r line; do + echo " $line" + done + + subsection "Port Bindings Summary" + local bindings + bindings=$(ovn_exec ovn-sbctl --format=csv --no-headings --columns=logical_port,type,chassis,tunnel_key list Port_Binding 2>/dev/null || echo "") + printf " ${DIM}%-65s %-18s %-40s %s${RESET}\n" "LOGICAL_PORT" "TYPE" "CHASSIS" "TUNNEL_KEY" + echo "$bindings" | sort | while IFS=',' read -r port type chassis tkey; do + port="${port//\"/}" + type="${type//\"/}" + printf " %-65s %-18s %-40s %s\n" "$port" "${type:-(VIF)}" "$chassis" "$tkey" + done +} + +# --------------------------------------------------------------------------- +# OVS: Physical switch layer +# --------------------------------------------------------------------------- + +show_ovs() { + section "OVS Physical Layer" + + # Get cluster members + local members + members=$(incus cluster list "${REMOTE}:" -f csv -c n 2>/dev/null || echo "") + + if [[ -z "$members" ]]; then + warn "Could not list cluster members" + return + fi + + echo + echo " ${DIM}This action deploys temporary privileged Alpine containers on net-prod" + echo " with /run/openvswitch mounted to access OVS on each node.${RESET}" + echo + + local containers_created=() + + while IFS= read -r node; do + [[ -z "$node" ]] && continue + local inspect_name="ovs-inspect-${node}" + + subsection "Node: $node" + + if [[ "$DRY_RUN" == true ]]; then + echo " [dry-run] Would deploy ${inspect_name} on ${node}" + echo " [dry-run] Would run: ovs-vsctl show" + echo " [dry-run] Would run: ovs-ofctl dump-flows br-int" + echo " [dry-run] Would delete ${inspect_name}" + continue + fi + + # Deploy temp container + echo " ${DIM}Deploying ${inspect_name}...${RESET}" + if ! incus launch "images:alpine/3.21" "${REMOTE}:${inspect_name}" \ + --target "$node" -c security.privileged=true \ + -c limits.memory=128MiB -n net-prod 2>/dev/null; then + warn "Failed to deploy ${inspect_name} on ${node}" + continue + fi + containers_created+=("$inspect_name") + + # Add OVS socket mount + incus config device add "${REMOTE}:${inspect_name}" ovs-run \ + disk source=/run/openvswitch path=/run/openvswitch 2>/dev/null || true + + # Wait for container + install OVS + local retries=0 + while ! incus exec "${REMOTE}:${inspect_name}" -- true 2>/dev/null; do + retries=$((retries + 1)) + if [[ $retries -gt 15 ]]; then + warn "Container ${inspect_name} not ready after 15s" + continue 2 + fi + sleep 1 + done + + incus exec "${REMOTE}:${inspect_name}" -- \ + apk add --no-cache openvswitch 2>/dev/null >/dev/null || { + warn "Failed to install openvswitch in ${inspect_name}" + continue + } + + # Gather data + echo + echo " ${BOLD}Bridges:${RESET}" + container_exec "$inspect_name" ovs-vsctl show | while IFS= read -r line; do + echo " $line" + done + + echo + echo " ${BOLD}br-int port → OVN mapping:${RESET}" + local ports + ports=$(container_exec "$inspect_name" ovs-vsctl list-ports br-int 2>/dev/null || echo "") + while IFS= read -r port; do + [[ -z "$port" ]] && continue + local ext_ids + ext_ids=$(container_exec "$inspect_name" ovs-vsctl get Interface "$port" external_ids 2>/dev/null || echo "{}") + local iface_type + iface_type=$(container_exec "$inspect_name" ovs-vsctl get Interface "$port" type 2>/dev/null || echo "") + iface_type="${iface_type//\"/}" + printf " %-20s %-10s %s\n" "$port" "${iface_type:-(veth)}" "$ext_ids" + done <<< "$ports" + + done <<< "$members" + + # Cleanup + if [[ ${#containers_created[@]} -gt 0 ]]; then + echo + echo " ${DIM}Cleaning up temporary containers...${RESET}" + for c in "${containers_created[@]}"; do + incus delete "${REMOTE}:${c}" --force 2>/dev/null || true + done + echo " ${GREEN}Done.${RESET}" + fi +} + +# --------------------------------------------------------------------------- +# Trace: Packet path through OVN +# --------------------------------------------------------------------------- + +show_trace() { + local vip="$TRACE_VIP" + section "Packet Trace: LAN → ${vip}" + + # Find the LB for this VIP + subsection "Load Balancer for ${vip}" + local lb_info + lb_info=$(ovn_exec ovn-nbctl lb-list) + local lb_line + lb_line=$(echo "$lb_info" | grep "$vip" || echo "") + if [[ -z "$lb_line" ]]; then + warn "No load balancer found for VIP ${vip}" + echo " Available LBs:" + echo "$lb_info" | while IFS= read -r line; do echo " $line"; done + return + fi + echo " $lb_line" + + # Extract backends + local backends + backends=$(echo "$lb_line" | grep -oP '(?<=IPs\s{4})[^\n]+' || echo "") + if [[ -z "$backends" ]]; then + # Try different column format + backends=$(echo "$lb_line" | awk '{print $NF}') + fi + + # Find the router + subsection "Router NAT (SNAT)" + ovn_exec ovn-nbctl lr-nat-list "$(ovn_exec ovn-nbctl lr-list 2>/dev/null | sed 's/.*(\(.*\))/\1/' | head -1)" \ + | while IFS= read -r line; do echo " $line"; done + + # Find gateway chassis + subsection "Gateway Chassis" + local gw_binding + gw_binding=$(ovn_exec ovn-sbctl find Port_Binding type=chassisredirect 2>/dev/null | grep -A2 "chassis" || echo "") + ovn_exec ovn-sbctl show | grep -B1 "cr-" | while IFS= read -r line; do + echo " $line" + done + + # Build the trace narrative + subsection "Packet Path" + local router_name + router_name=$(ovn_exec ovn-nbctl lr-list 2>/dev/null | sed 's/.*(\(.*\))/\1/' | head -1) + local ext_port + ext_port=$(ovn_exec ovn-nbctl lrp-list "$router_name" | grep "lrp-ext" | awk '{print $2}' | tr -d '()') + local int_port + int_port=$(ovn_exec ovn-nbctl lrp-list "$router_name" | grep "lrp-int" | awk '{print $2}' | tr -d '()') + local gw_node + gw_node=$(ovn_exec ovn-sbctl show | grep -B5 "cr-" | grep "hostname:" | tail -1 | awk '{print $2}') + + echo + echo " ${BOLD}1. Ingress (LAN → Provider Bridge)${RESET}" + echo " Client sends packet to ${vip}:80" + echo " → Arrives on gateway chassis ${GREEN}${gw_node}${RESET}" + echo " → Physical NIC → incusovn7 (provider bridge) → patch port → br-int" + echo + echo " ${BOLD}2. OVN Router (External → Internal)${RESET}" + echo " → br-int delivers to external switch (ls-ext)" + echo " → Patch port to router ${router_name}" + echo " → Router external port: ${ext_port}" + echo " → ${YELLOW}LB DNAT: ${vip}:80 → backend IPs${RESET}" + echo + echo " ${BOLD}3. Load Balancing${RESET}" + echo " → OVN conntrack LB selects backend from:" + echo " $lb_line" | awk '{print " " $0}' + echo + echo " ${BOLD}4. Forwarding to Backend${RESET}" + echo " → Router internal port: ${int_port} (10.10.10.1)" + echo " → Internal switch (ls-int)" + echo " → If backend on same chassis: deliver via local veth" + echo " → If backend on remote chassis: Geneve encap → tunnel → remote br-int → veth" + echo + echo " ${BOLD}5. Return Path${RESET}" + echo " → Backend replies to client" + echo " → ls-int → router → ${YELLOW}SNAT: 10.10.10.0/24 → ${vip}${RESET}" + echo " → ls-ext → provider bridge → physical NIC → LAN" + echo + echo " ${BOLD}Tunnel Details:${RESET}" + echo " Protocol: Geneve (UDP 6081), BFD enabled" + ovn_exec ovn-sbctl list Encap | grep -E "ip|type" | while IFS= read -r line; do + echo " $line" + done +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +case "$ACTION" in + nb) show_nb ;; + sb) show_sb ;; + ovs) show_ovs ;; + full) show_nb; show_sb; show_ovs ;; + trace) show_trace ;; +esac diff --git a/notes/ovn-deep-dive.md b/notes/ovn-deep-dive.md new file mode 100644 index 0000000..eb422a9 --- /dev/null +++ b/notes/ovn-deep-dive.md @@ -0,0 +1,604 @@ +# OVN Deep-Dive: Incus Cluster Networking Internals + +How Incus translates its network abstractions into OVN logical objects, +OVS physical flows, and Geneve tunnels. Based on a live 3-node IncusOS +cluster running OVS 3.6.1 with an OVN overlay network. + +## Lab Topology + +``` +LAN (192.168.100.0/22) +│ +├── oc-node-01 192.168.102.140 (Geneve endpoint) +├── oc-node-02 192.168.102.141 (Geneve endpoint) +├── oc-node-03 192.168.102.142 (Geneve endpoint, gateway chassis) +│ +└── UPLINK physical network + └── OVN range: 192.168.103.200-210 + └── net-prod (10.10.10.0/24) ← OVN overlay +``` + +**Instances on net-prod:** + +| Instance | IP | Node | Role | +|---|---|---|---| +| ffsdn-haproxy-52-01 | 10.10.10.50 | oc-node-01 | HAProxy LB | +| ffsdn-haproxy-52-02 | 10.10.10.51 | oc-node-02 | HAProxy LB | +| nginx-lb-01 | 10.10.10.60 | oc-node-01 | Nginx backend | +| nginx-lb-02 | 10.10.10.61 | oc-node-02 | Nginx backend | +| nginx-lb-03 | 10.10.10.62 | oc-node-03 | Nginx backend | +| test-app-manual-web-tier-app-1 | 10.10.10.4 | oc-node-03 | App server | +| test-app-manual-web-tier-web-1 | 10.10.10.2 | oc-node-01 | Web server | +| test-app-manual-web-tier-web-2 | 10.10.10.3 | oc-node-02 | Web server | + +## OVN Northbound Database (Logical Layer) + +The NB database describes the desired network topology. Incus creates and +manages all objects here — users never touch OVN directly. + +### Incus → OVN Naming Convention + +Incus uses a consistent naming scheme. For a network named `net-prod` +with internal ID `net8`: + +| Incus Concept | OVN Object | OVN Name | +|---|---|---| +| OVN network | Logical Switch (internal) | `incus-net8-ls-int` | +| OVN uplink | Logical Switch (external) | `incus-net8-ls-ext` | +| OVN gateway | Logical Router | `incus-net8-lr` | +| Instance NIC | Logical Switch Port | `incus-net8-instance--eth0` | +| Router↔int switch | Router Port + Switch Port | `incus-net8-lr-lrp-int` / `incus-net8-ls-int-lsp-router` | +| Router↔ext switch | Router Port + Switch Port | `incus-net8-lr-lrp-ext` / `incus-net8-ls-ext-lsp-router` | +| Provider connection | Localnet Port | `incus-net8-ls-ext-lsp-provider` | +| Network forward LB | Load Balancer | `incus-net8-lb--` | + +The numeric `net8` suffix is an internal Incus identifier for the network. +It increments as networks are created. + +### Logical Switches + +**Internal switch** (`incus-net8-ls-int`): The overlay network where all +instances connect. Subnet 10.10.10.0/24, IPv6 fd42:842b:1e5f:b27::/64. + +``` +incus-net8-ls-int +├── 8 instance ports (one per container on net-prod) +├── 1 router port (incus-net8-ls-int-lsp-router) +├── 1 load balancer attached (VIP 192.168.103.200:80) +├── 15 ACL rules (Incus baseline security) +├── 8 DNS records (forward + reverse per instance) +└── DHCP options (IPv4 lease=3600, MTU=1442, DNS=192.168.100.1) +``` + +Key config in `other_config`: +- `subnet=10.10.10.0/24` — Incus IPAM range +- `exclude_ips=10.10.10.1 10.10.10.254 ...` — reserved IPs (gateway, broadcast, static assignments) +- `ipv6_prefix=fd42:842b:1e5f:b27::/64` — SLAAC prefix + +**External switch** (`incus-net8-ls-ext`): Bridges the logical router to +the physical UPLINK network. + +``` +incus-net8-ls-ext +├── 1 localnet port (incus-net8-ls-ext-lsp-provider) +│ └── options: network_name=UPLINK +└── 1 router port (incus-net8-ls-ext-lsp-router) + └── nat_addresses: 10:66:6a:d4:30:c7 192.168.103.200 + is_chassis_resident("cr-incus-net8-lr-lrp-ext") +``` + +The `localnet` port type is special — it tells OVN to bridge this logical +switch to a physical network via `ovn-bridge-mappings` on each chassis. +The mapping `UPLINK:incusovn7` connects it to the OVS provider bridge. + +### Logical Router + +**Router** (`incus-net8-lr`): Connects internal overlay to external UPLINK. + +``` +incus-net8-lr +├── lrp-int: 10.10.10.1/24, fd42:842b:1e5f:b27::1/64 +│ MAC: 10:66:6a:d4:30:c7, MTU: 1442 +│ IPv6 RA: periodic, DHCPv6 stateless, DNSSL=incus +│ +├── lrp-ext: 192.168.103.200/22 +│ MAC: 10:66:6a:d4:30:c7, MTU: 1442 +│ HA Chassis Group: incus-net8 (gateway on oc-node-03) +│ +├── NAT: +│ └── SNAT: 10.10.10.0/24 → 192.168.103.200 (stateful) +│ +├── Static Routes: +│ ├── 0.0.0.0/0 → 192.168.100.1 via lrp-ext (default gateway) +│ └── 192.168.103.200/32 → 10.10.10.1 (hairpin VIP route) +│ +├── Load Balancer: (same LB as on ls-int) +│ └── 192.168.103.200:80 → 10.10.10.50:80, 10.10.10.51:80 +│ +├── Policies: +│ ├── priority 600: allow ip4.src == $incus_net8_routes_ip4 +│ ├── priority 600: allow ip6.src == $incus_net8_routes_ip6 +│ └── priority 500: drop (inport == lrp-int) ← deny-by-default +│ +└── Options: + ├── always_learn_from_arp_request=false + └── dynamic_neigh_routers=true +``` + +**Router policies explained:** The priority-500 drop rule blocks all +traffic from the internal network by default. The priority-600 allow rules +create exceptions for the network's own IP ranges. This prevents instances +from spoofing source IPs outside their assigned ranges. + +**Hairpin VIP route:** The `192.168.103.200/32 → 10.10.10.1` route +handles the case where an instance on net-prod accesses the VIP from +inside the overlay. Without this, the router would try to route the VIP +out the external port, but the packet originated internally. + +### Load Balancer + +``` +Name: incus-net8-lb-192.168.103.200-tcp +Protocol: tcp +VIPs: 192.168.103.200:80 → 10.10.10.50:80, 10.10.10.51:80 +``` + +Incus creates this LB from `incus network forward` or from Aether's +HAProxy deployment. The LB is attached to both the logical switch +(ls-int) and the logical router (lr). This dual attachment ensures +the LB intercepts traffic regardless of where it enters: + +- **On the router**: Catches external traffic (LAN → VIP) +- **On the switch**: Catches internal traffic (instance → VIP) + +### DHCP and DNS + +**DHCP Options** (IPv4): +- Server: 10.10.10.1, MAC: 10:66:6a:d4:30:c7 +- Lease time: 3600s, MTU: 1442 (Geneve overhead: 1500 - 58 = 1442) +- DNS: 192.168.100.1 (upstream resolver from UPLINK config) +- Domain: incus, search list: incus + +**DHCP Options** (IPv6): +- Stateless DHCPv6 (SLAAC for addresses, DHCPv6 for DNS) +- DNS: fd42:842b:1e5f:b27::1 + +**DNS Records**: Incus creates forward and reverse DNS entries for every +instance. The DNS server runs inside the OVN router (lrp-int port). + +``` +ffsdn-haproxy-52-01.incus → 10.10.10.50, fd42:842b:1e5f:b27:1266:6aff:fe67:3482 +ffsdn-haproxy-52-02.incus → 10.10.10.51, fd42:842b:1e5f:b27:1266:6aff:fe5f:418f +nginx-lb-01.incus → 10.10.10.60, fd42:842b:1e5f:b27:1266:6aff:fe2a:da10 +nginx-lb-02.incus → 10.10.10.61, fd42:842b:1e5f:b27:1266:6aff:fe69:888e +nginx-lb-03.incus → 10.10.10.62, fd42:842b:1e5f:b27:1266:6aff:fe90:4ab8 +``` + +### ACLs (Baseline Security) + +Incus creates a default ACL set on the internal switch. These are not +user-configurable — they form the baseline security policy: + +| Priority | Direction | Match | Action | Purpose | +|---|---|---|---|---| +| 200 | to-lport | `arp \|\| nd` | allow | ARP/ND always allowed | +| 200 | to-lport | `icmp4.type == {3,11,12} && ip.ttl == 255` | allow | ICMP errors | +| 200 | to-lport | `igmp && ip.ttl == 1 && ip4.mcast` | allow | IGMP | +| 200 | to-lport | Router port ping echo reply | allow | Router ping | +| 200 | to-lport | DHCP to router port | allow | DHCP relay | +| 200 | to-lport | DNS to router port | allow | DNS queries | +| 200 | to-lport | `nd_ra` from router | allow | IPv6 RA | +| 200 | to-lport | `nd_rs` to router | allow | IPv6 RS | +| 200 | to-lport | `tcp.flags == 0x014` | allow | TCP RST+ACK | + +All ACLs are `to-lport` (ingress to the logical port), which means they +filter traffic arriving at a port. There are no explicit `from-lport` +rules — the default for egress is allow. + +## OVN Southbound Database (Physical Layer) + +The SB database maps logical topology to physical infrastructure. It's +computed by `ovn-northd` from the NB database and consumed by +`ovn-controller` on each chassis. + +### Chassis + +Each Incus cluster member registers as a chassis: + +| Chassis | Hostname | Geneve IP | OVS UUID | +|---|---|---|---| +| b840b5b2-... | oc-node-01 | 192.168.102.140 | 298b76a9-... | +| 4652b51f-... | oc-node-02 | 192.168.102.141 | 5ee7906c-... | +| 3f7400f9-... | oc-node-03 | 192.168.102.142 | fd0c3d36-... | + +All chassis use Geneve encapsulation with checksum enabled (`csum=true`). +Each chassis advertises `ovn-bridge-mappings=UPLINK:incusovn7`. + +Key `other_config` values: +- `datapath-type=system` — kernel datapath (not DPDK) +- `ovn-bridge-mappings=UPLINK:incusovn7` — maps logical "UPLINK" network to OVS bridge +- `ovn-monitor-all=false` — each chassis only gets flows relevant to its local ports +- `ct-commit-nat-v2=true`, `ct-next-zone=true` — modern conntrack features + +### Datapath Bindings (Tunnel Keys) + +Each logical switch/router gets a unique tunnel key used inside Geneve: + +| Tunnel Key | Datapath | OVN Object | +|---|---|---| +| 1 | incus-net8-lr | Logical Router | +| 2 | incus-net8-ls-ext | External Switch | +| 3 | incus-net8-ls-int | Internal Switch | + +When a packet traverses a Geneve tunnel, the tunnel key (VNI) identifies +which datapath it belongs to. + +### Port Bindings + +Port bindings map logical ports to physical chassis: + +| Logical Port | Type | Chassis | Tunnel Key | +|---|---|---|---| +| Instance ports (8 total) | VIF | Respective chassis | 2-10 | +| cr-incus-net8-lr-lrp-ext | chassisredirect | oc-node-03 | 2 | +| incus-net8-lr-lrp-ext | patch | (distributed) | 1 | +| incus-net8-lr-lrp-int | patch | (distributed) | 3 | +| incus-net8-ls-ext-lsp-provider | localnet | (all chassis) | 2 | +| incus-net8-ls-ext-lsp-router | patch | (distributed) | 1 | + +**Key port types:** + +- **VIF (Virtual Interface)**: Regular instance ports, bound to a specific + chassis. Each has a MAC+IP pair for the instance. +- **patch**: Connects two OVN datapaths (switch↔router). Distributed — + processed locally on every chassis. +- **chassisredirect**: The gateway port. Centralizes external-facing + traffic on a single chassis for SNAT/DNAT. Bound to oc-node-03. +- **localnet**: Maps to a physical network. Present on every chassis + via `ovn-bridge-mappings`. + +### HA Chassis Group (Gateway Failover) + +The gateway port uses an HA chassis group for failover: + +``` +HA Chassis Group: incus-net8 +├── oc-node-03 priority 22372 ← active gateway +├── oc-node-02 priority 18011 ← first failover +└── oc-node-01 priority 12213 ← second failover +``` + +Incus assigns random-looking priorities (likely based on a hash). If +oc-node-03 goes down, the `cr-incus-net8-lr-lrp-ext` binding migrates +to oc-node-02, then oc-node-01 if needed. BFD (Bidirectional Forwarding +Detection) between chassis enables fast failure detection. + +### Logical Flows (Compiled Pipeline) + +The lflow pipeline is the compiled form of all NB rules. For ls-int ingress: + +| Table | Name | Key Rules | +|---|---|---| +| 0 | ls_in_check_port_sec | Drop multicast src, VLAN tagged | +| 1 | ls_in_apply_port_sec | Enforce port security | +| 4 | ls_in_pre_acl | Skip ACL for router port traffic | +| 5 | ls_in_pre_lb | Pre-process for LB (set reg0[2] for CT) | +| 6 | ls_in_pre_stateful | **LB intercept**: dst=192.168.103.200:80 → `ct_lb_mark` | +| 7 | ls_in_acl_hint | Compute ACL hints from conntrack state | +| 8 | ls_in_acl | Apply ACLs, track connections | + +The critical LB flow in table 6: +``` +priority=120, match=(reg0[2] == 1 && ip4.dst == 192.168.103.200 && tcp.dst == 80) +action=(reg1 = 192.168.103.200; reg2[0..15] = 80; ct_lb_mark;) +``` + +This intercepts packets to the VIP and sends them through conntrack +load balancing, which selects a backend and rewrites the destination. + +## OVS Physical Layer + +Each IncusOS node runs OVS 3.6.1 with two bridges. + +### Bridge Architecture + +``` +Per-node OVS layout: + +incusovn7 (provider bridge) br-int (integration bridge) +├── incusovn7 ← physical NIC ├── veth* ← instance NICs +├── incusovn7b ← internal port ├── ovn-* ← Geneve tunnels +└── patch-...-to-br-int ←──patch──→ ├── patch-br-int-to-... + └── br-int ← internal port +``` + +**`br-int` (integration bridge)**: +- `fail_mode: secure` — drops all traffic if OVN controller disconnects +- Contains all instance veth ports, Geneve tunnel ports, and the patch port + to the provider bridge +- All OVN logical processing (ACLs, NAT, LB, routing) happens here via + OpenFlow rules installed by `ovn-controller` + +**`incusovn7` (provider bridge)**: +- Named after the Incus-managed OVS bridge (`incusovn` + network ID) +- Contains the physical NIC port, an internal port, and a patch port to br-int +- Single OpenFlow rule: `priority=0 actions=NORMAL` (standard L2 switching) +- This bridge is the on-ramp/off-ramp between OVN and the physical network + +### Geneve Tunnels (Full Mesh) + +Every pair of chassis has a Geneve tunnel with BFD health monitoring: + +``` +oc-node-01 ←── Geneve (UDP 6081) ──→ oc-node-02 + ↑ ↑ + └──────── Geneve (UDP 6081) ──────────┘ + ↓ + oc-node-03 +``` + +| Source | Destination | OVS Port Name | BFD State | +|---|---|---|---| +| .140 (node-01) | .141 (node-02) | ovn-4652b5-0 | forwarding=true | +| .140 (node-01) | .142 (node-03) | ovn-3f7400-0 | forwarding=true | +| .141 (node-02) | .140 (node-01) | ovn-b840b5-0 | forwarding=true | +| .141 (node-02) | .142 (node-03) | ovn-3f7400-0 | forwarding=true | +| .142 (node-03) | .140 (node-01) | ovn-b840b5-0 | forwarding=true | +| .142 (node-03) | .141 (node-02) | ovn-4652b5-0 | forwarding=true | + +Tunnel port names use the first 6 hex chars of the remote chassis name. +Options: `key=flow` (tunnel key set per-packet from OVN datapath), `csum=true`. + +### Veth Port Mapping + +Each instance on net-prod gets a veth pair: one end in the container's +network namespace, the other plugged into br-int. The OVS `external_ids` +field links the veth to its OVN logical port. + +**oc-node-01:** +| veth | OVN Port (iface-id) | Instance | OVS ofport | +|---|---|---|---| +| veth8bd9abf3 | ...c446bd6a...-eth0 | nginx-lb-01 (10.10.10.60) | 8 | +| vethfc99b1ec | ...b90abddd...-eth0 | test-web-tier-web-1 (10.10.10.2) | 5 | +| veth07cf17cf | ...0880f911...-eth0 | ffsdn-haproxy-52-01 (10.10.10.50) | 9 | + +**oc-node-02:** +| veth | OVN Port (iface-id) | Instance | +|---|---|---| +| veth3352cb4f | ...a2cad635...-eth0 | ffsdn-haproxy-52-02 (10.10.10.51) | +| vethc0b4d30a | ...07270515...-eth0 | nginx-lb-02 (10.10.10.61) | +| vethd2307cca | ...b2cbf869...-eth0 | test-web-tier-web-2 (10.10.10.3) | + +**oc-node-03:** +| veth | OVN Port (iface-id) | Instance | +|---|---|---| +| veth9306f6ef | ...292ce9ce...-eth0 | nginx-lb-03 (10.10.10.62) | +| veth4701597a | ...1f056d1a...-eth0 | test-web-tier-app-1 (10.10.10.4) | + +## Packet Trace: LAN → VIP → HAProxy → Nginx → Return + +Complete path for an HTTP request from a LAN client to +`http://192.168.103.200/` (the HAProxy VIP). + +### 1. Client → Gateway Chassis + +``` +Client (192.168.1.x) sends TCP SYN to 192.168.103.200:80 +→ LAN switch forwards to oc-node-03 (gateway chassis) + 192.168.103.200 is announced by oc-node-03 via the + cr-incus-net8-lr-lrp-ext chassisredirect port +→ Packet enters physical NIC → incusovn7 bridge +→ OVS NORMAL action → patch port → br-int +``` + +Why oc-node-03? The external router port (`lrp-ext`) uses HA chassis +scheduling, and oc-node-03 has the highest priority (22372). OVN makes +oc-node-03's OVS respond to ARP for 192.168.103.200, directing all +external traffic to this node. + +### 2. br-int → OVN Router (External Processing) + +``` +br-int receives packet on patch port +→ OpenFlow table 0: classify as localnet traffic (metadata=0x2, ls-ext) +→ Pipeline: ls-ext ingress → router pipeline +→ Router receives on lrp-ext (192.168.103.200/22) +``` + +### 3. OVN Load Balancer DNAT + +``` +Router detects dst=192.168.103.200:80 matches LB +→ ct_lb_mark: conntrack creates new entry +→ DNAT: rewrite dst to 10.10.10.50:80 or 10.10.10.51:80 + (round-robin selection, conntrack-aware) +→ Packet now has dst=10.10.10.50:80 (say, haproxy-01) +``` + +The LB is processed in the router pipeline because it's attached to both +the router and the internal switch. For external traffic, the router +processes it first. + +### 4. Router → Internal Switch + +``` +Router forwards via lrp-int (10.10.10.1) +→ Enters ls-int pipeline +→ ACL check (baseline allows established connections) +→ Destination lookup: 10.10.10.50 → ffsdn-haproxy-52-01 +→ Port binding: haproxy-01 is on oc-node-01 +``` + +### 5. Geneve Tunnel (Cross-Chassis) + +Since the gateway is on oc-node-03 but the destination (haproxy-01) is +on oc-node-01: + +``` +br-int on oc-node-03: +→ Output action: tunnel to oc-node-01 +→ Geneve encapsulate: + - Outer: src=192.168.102.142, dst=192.168.102.140, UDP:6081 + - VNI (tunnel key): 3 (ls-int datapath) + - TUN_METADATA0: encodes reg14 (destination port) + reg15 +→ Physical NIC sends to LAN + +oc-node-01 receives Geneve packet: +→ br-int decapsulates +→ Restores metadata from tunnel headers +→ Delivers to veth07cf17cf (haproxy-01's veth, ofport 9) +→ Packet enters container's network namespace +``` + +### 6. HAProxy Processing + +``` +HAProxy receives HTTP request on 10.10.10.50:80 +→ HAProxy selects backend: nginx-lb-01 (10.10.10.60) +→ Opens new TCP connection to 10.10.10.60:80 +→ Proxies request +``` + +### 7. HAProxy → Nginx (Same or Different Chassis) + +If nginx-lb-01 is on the same node (oc-node-01): +``` +→ br-int local delivery (no tunnel needed) +→ veth8bd9abf3 (nginx-lb-01, ofport 8) +``` + +If HAProxy chose nginx-lb-02 (oc-node-02) or nginx-lb-03 (oc-node-03): +``` +→ Geneve tunnel to remote node +→ Same encap/decap as step 5 +``` + +### 8. Return Path + +``` +Nginx response → HAProxy (reverse of step 7) +→ HAProxy response → OVN (enters ls-int as src=10.10.10.50) +→ ls-int → router (dst is LAN client, matches default route) +→ Router SNAT: src 10.10.10.50 → 192.168.103.200 +→ lrp-ext → ls-ext → provider bridge → physical NIC → LAN +→ Client receives response from 192.168.103.200 +``` + +**Important**: The return path from HAProxy to the LAN client goes through +the gateway chassis (oc-node-03) because SNAT is centralized there. If +HAProxy is on oc-node-01, the reply tunnels to oc-node-03 for SNAT, then +exits to the LAN. + +## MTU: Why 1442? + +Standard Ethernet MTU is 1500 bytes. Geneve adds 58 bytes of overhead: + +``` +Outer Ethernet: 14 bytes +Outer IP: 20 bytes +Outer UDP: 8 bytes +Geneve header: 16 bytes (8 base + 8 metadata) + ───────── +Total overhead: 58 bytes +Inner MTU: 1442 bytes (1500 - 58) +``` + +Incus sets `bridge.mtu=1442` on net-prod and propagates this via: +- DHCP option: `mtu=1442` +- Router port option: `gateway_mtu=1442` +- IPv6 RA: `mtu=1442` + +## Inspection Tools + +### Using ovn-inspect + +The `incusos/helpers/ovn-inspect` script provides structured inspection: + +```bash +# NB database (logical topology) +incusos/helpers/ovn-inspect --nb + +# SB database (physical bindings) +incusos/helpers/ovn-inspect --sb + +# OVS on each node (deploys temp containers, cleans up after) +incusos/helpers/ovn-inspect --ovs + +# Everything +incusos/helpers/ovn-inspect --full + +# Trace a VIP's packet path +incusos/helpers/ovn-inspect --trace 192.168.103.200 + +# Dry run (show commands without executing) +incusos/helpers/ovn-inspect --nb --dry-run +``` + +### Manual Commands + +All NB/SB commands go through the `ovn-central` container: + +```bash +# NB: Logical topology +incus exec oc-node-01:ovn-central -- ovn-nbctl ls-list +incus exec oc-node-01:ovn-central -- ovn-nbctl lr-list +incus exec oc-node-01:ovn-central -- ovn-nbctl lsp-list incus-net8-ls-int +incus exec oc-node-01:ovn-central -- ovn-nbctl lr-nat-list incus-net8-lr +incus exec oc-node-01:ovn-central -- ovn-nbctl lb-list +incus exec oc-node-01:ovn-central -- ovn-nbctl acl-list incus-net8-ls-int + +# SB: Physical bindings +incus exec oc-node-01:ovn-central -- ovn-sbctl show +incus exec oc-node-01:ovn-central -- ovn-sbctl list Chassis +incus exec oc-node-01:ovn-central -- ovn-sbctl list Port_Binding +incus exec oc-node-01:ovn-central -- ovn-sbctl lflow-list incus-net8-ls-int + +# OVS: Physical layer (requires privileged container with /run/openvswitch) +ovs-vsctl show +ovs-ofctl dump-flows br-int +ovs-vsctl get Interface external_ids +``` + +### Incus CLI Cross-Reference + +```bash +# See OVN network config +incus network show oc-node-01:net-prod --target oc-node-01 + +# See UPLINK config +incus network show oc-node-01:UPLINK --target oc-node-01 + +# See which instances use net-prod +incus list oc-node-01: -f csv -c n4l | grep "10.10.10" + +# Network forwards (LB VIPs managed via Incus) +incus network forward list oc-node-01:net-prod +``` + +## Key Architectural Insights + +1. **Incus fully manages OVN**: Users never interact with OVN directly. + `incus network create`, `incus network forward`, and instance NIC + configs translate to OVN objects automatically. + +2. **Single gateway chassis**: All external traffic (SNAT, DNAT, LB for + external VIPs) is centralized on one node. This is a potential + bottleneck but simplifies state management. HA failover handles node + failures. + +3. **Distributed routing for internal traffic**: East-west traffic + between instances on the same switch is fully distributed. No traffic + goes through the gateway unless it needs SNAT/DNAT. + +4. **LB is in OVN, not HAProxy**: The VIP load balancing between HAProxy + instances is done by OVN's built-in L4 LB (conntrack-based). HAProxy + then does L7 load balancing to nginx backends. This is a two-tier LB + architecture. + +5. **BFD for fast failover**: All Geneve tunnels have BFD enabled, which + detects chassis failures in ~3×detection-interval (typically <1s), + much faster than relying on OVN cluster heartbeats. + +6. **MTU must be consistent**: The 1442 byte MTU is critical. If any + path (physical switch, hypervisor NIC) has MTU < 1500, Geneve + encapsulated packets will be fragmented or dropped.