diff --git a/notes/api-interception-guide.md b/notes/api-interception-guide.md index 487fcca..01564a9 100644 --- a/notes/api-interception-guide.md +++ b/notes/api-interception-guide.md @@ -5,27 +5,33 @@ Aether (SDN controller) and the Incus cluster at the wire level. ## Architecture -``` -Aether (192.168.102.160) - │ Client cert: CN=root@oc-server - │ Subscribes to: /1.0/events (lifecycle stream) - │ - ├──→ Incus API (https://192.168.102.140-142:8443) - │ All instance/network/storage operations - │ - └──→ Aether's own REST API (/api/*) - Auth, clusters, ACLs only - JWT-authenticated, separate from Incus +```mermaid +flowchart LR + subgraph callers["API Callers"] + aether["Aether
192.168.102.160
CN=root@oc-server"] + cli["CLI
dev-vm-beelink
CN=maarten@dev-vm-beelink"] + nodes["Cluster Nodes
oc-node-01/02/03
CN=root@oc-node-*"] + end -CLI (dev-vm-beelink) - │ Client cert: CN=maarten@dev-vm-beelink - │ - └──→ Incus API (same endpoints) + incus["Incus API
:8443 on .140-.142"] + aether_api["Aether REST API
/api/* (JWT auth)"] -Cluster nodes (oc-node-01/02/03) - │ Client certs: CN=root@oc-node-* - │ - └──→ Inter-node API (internal cluster operations) + aether -->|"client cert
+ /1.0/events"| incus + aether --> aether_api + cli -->|"client cert"| incus + nodes -->|"inter-node"| incus + + classDef mgmt fill:#CC79A7,color:#fff,stroke:#a36088 + classDef instance fill:#56B4E9,color:#fff,stroke:#3a8fbf + classDef network fill:#0072B2,color:#fff,stroke:#005a8e + classDef node fill:#009E73,color:#fff,stroke:#007a5e + + class aether mgmt + class cli instance + class nodes node + class incus,aether_api network + + style callers fill:#f5f5f5,stroke:#999 ``` ## Capture Methods diff --git a/notes/haproxy-guide.md b/notes/haproxy-guide.md index 30c9e0c..35d4f18 100644 --- a/notes/haproxy-guide.md +++ b/notes/haproxy-guide.md @@ -10,37 +10,31 @@ creating test backends, configuring services, and verifying load balancing. ## Architecture -``` - Client / LAN traffic - | - v - +---------------------------+ - | UPLINK VIP Address | - | 192.168.103.200 | - +---------------------------+ - | - v - +---------------------------+ - | OVN Load Balancer | - | (distributes to HA | - | pair for failover) | - +---------------------------+ - / \ - / \ - v v - +-----------------+ +-----------------+ - | HAProxy 01 | | HAProxy 02 | - | 10.10.10.50 | | 10.10.10.51 | - | (container) | | (container) | - +-----------------+ +-----------------+ - \ | / - \ | / - v v v - +----------+ +----------+ +----------+ - | nginx-01 | | nginx-02 | | nginx-03 | - | .60 :80 | | .61 :80 | | .62 :80 | - +----------+ +----------+ +----------+ - (backend servers on net-prod) +```mermaid +flowchart TD + client(["Client / LAN"]) + vip["VIP 192.168.103.200"] + ovnlb("OVN Load Balancer") + ha1["HAProxy 01
10.10.10.50"] + ha2["HAProxy 02
10.10.10.51"] + ng1["nginx-01 · .60"] + ng2["nginx-02 · .61"] + ng3["nginx-03 · .62"] + + client --> vip + vip --> ovnlb + ovnlb --> ha1 & ha2 + ha1 & ha2 --> ng1 & ng2 & ng3 + + classDef external fill:#f5f5f5,color:#333,stroke:#999 + classDef network fill:#0072B2,color:#fff,stroke:#005a8e + classDef lb fill:#E69F00,color:#fff,stroke:#b87d00 + classDef instance fill:#56B4E9,color:#fff,stroke:#3a8fbf + + class client external + class vip,ovnlb network + class ha1,ha2 lb + class ng1,ng2,ng3 instance ``` ### OVN native LB vs HAProxy diff --git a/notes/incusos-break-fix.md b/notes/incusos-break-fix.md index d5aebd2..c6b35d2 100644 --- a/notes/incusos-break-fix.md +++ b/notes/incusos-break-fix.md @@ -50,16 +50,22 @@ node-01 was already on this version; node-02 and node-03 were on Each node has a single SCSI disk (QEMU HARDDISK) with this layout: -``` -+------------------------------------------------------+ -| SCSI Disk (64-100 GiB, QEMU HARDDISK) | -|------------------------------------------------------| -| Partition 1 EFI System Partition | -| Partition A Root filesystem (active or standby) | -| Partition B Root filesystem (standby or active) | -| Partition X Swap (encrypted, TPM-unlocked) | -| Partition 11 ZFS data pool ("local") | -+------------------------------------------------------+ +```mermaid +block-beta + columns 1 + disk["SCSI Disk · 64-100 GiB · QEMU HARDDISK"] + p1["Partition 1 — EFI System Partition"] + pA["Partition A — Root filesystem (active or standby)"] + pB["Partition B — Root filesystem (standby or active)"] + pX["Partition X — Swap (encrypted, TPM-unlocked)"] + p11["Partition 11 — ZFS data pool 'local'"] + + style disk fill:#f5f5f5,color:#333,stroke:#999 + style p1 fill:#E69F00,color:#fff,stroke:#b87d00 + style pA fill:#009E73,color:#fff,stroke:#007a5e + style pB fill:#56B4E9,color:#fff,stroke:#3a8fbf + style pX fill:#CC79A7,color:#fff,stroke:#a36088 + style p11 fill:#0072B2,color:#fff,stroke:#005a8e ``` Key observations from the storage API: @@ -74,15 +80,28 @@ Key observations from the storage API: The boot security chain, as validated through the API: -``` -UEFI firmware - --> Secure Boot certificate validation (PK -> KEK -> db certs) - --> Signed kernel + initrd loaded - --> TPM measured boot (PCR measurements recorded) - --> TPM validates measurements match expected policy - --> Root partition decrypted and mounted - --> Swap partition decrypted and activated - --> System boots into trusted state +```mermaid +flowchart TD + uefi["UEFI Firmware"] + sb["Secure Boot validation
PK → KEK → db certs"] + kernel["Signed kernel + initrd loaded"] + tpm["TPM measured boot
PCR measurements recorded"] + validate["TPM validates measurements"] + root["Root partition decrypted"] + swap["Swap partition decrypted"] + boot["System boots into trusted state"] + + uefi --> sb --> kernel --> tpm --> validate + validate --> root & swap --> boot + + classDef security fill:#CC79A7,color:#fff,stroke:#a36088 + classDef node fill:#009E73,color:#fff,stroke:#007a5e + classDef network fill:#0072B2,color:#fff,stroke:#005a8e + + class uefi,sb,kernel security + class tpm,validate security + class root,swap network + class boot node ``` API-reported security state: @@ -173,19 +192,34 @@ updates. Observations from querying the update API and live testing: ### Update lifecycle (observed) -``` -Config: check_frequency set via PUT /os/1.0/system/update - --> Timer fires (or boot-time check runs) - --> Query stable channel for new version - --> Download new rootfs to standby partition - --> os_version_next updated, needs_reboot = true - --> status = "IncusOS has been updated to version YYYYMMDDHHMI" - --> (if auto_reboot) Reboot automatically - --> (if !auto_reboot) Wait for manual reboot - --> On reboot: boot from new partition - --> TPM re-measures, validates new boot chain - --> If valid: new partition becomes active - --> If invalid: rollback to previous partition +```mermaid +flowchart TD + config["check_frequency set via
PUT /os/1.0/system/update"] + timer["Timer fires
(or boot-time check)"] + query["Query stable channel"] + download["Download rootfs to
standby partition"] + ready["os_version_next updated
needs_reboot = true"] + + config --> timer --> query --> download --> ready + + ready -->|auto_reboot| reboot + ready -->|manual| wait["Wait for
manual reboot"] + wait --> reboot["Reboot from
new partition"] + + reboot --> tpm["TPM re-measures
validates boot chain"] + tpm -->|valid| active["New partition
becomes active"] + tpm -->|invalid| rollback["Rollback to
previous partition"] + + classDef mgmt fill:#CC79A7,color:#fff,stroke:#a36088 + classDef node fill:#009E73,color:#fff,stroke:#007a5e + classDef network fill:#0072B2,color:#fff,stroke:#005a8e + classDef lb fill:#E69F00,color:#fff,stroke:#b87d00 + + class config,timer,query mgmt + class download,ready network + class reboot,tpm,wait lb + class active node + class rollback fill:#D55E00,color:#fff,stroke:#a34a00 ``` --- diff --git a/notes/mermaid-test.md b/notes/mermaid-test.md index a9e1808..c65217f 100644 --- a/notes/mermaid-test.md +++ b/notes/mermaid-test.md @@ -1,7 +1,7 @@ # Mermaid Diagram Rendering Test -Test file to verify Gitea renders mermaid diagrams correctly. -Delete this file after confirming. +Style reference for all mermaid diagrams in this repository. +See individual guide files for the actual diagrams in use. --- @@ -31,6 +31,7 @@ and minimal text per node. - Use `classDef` for styling, never per-node `style` statements. - No emojis or unicode symbols in labels (renderer compatibility). - Edge labels: 1-3 words max, only when relationship isn't obvious. +- Use `
` for line breaks in node labels (not `\n` — Gitea renders that literally). --- @@ -43,8 +44,8 @@ flowchart TD client(["Client / LAN"]) vip["VIP 192.168.103.200"] ovnlb("OVN Load Balancer") - ha1["HAProxy 01\n10.10.10.50"] - ha2["HAProxy 02\n10.10.10.51"] + ha1["HAProxy 01
10.10.10.50"] + ha2["HAProxy 02
10.10.10.51"] ng1["nginx-01 · .60"] ng2["nginx-02 · .61"] ng3["nginx-03 · .62"] @@ -74,16 +75,16 @@ Original: `notes/networking-guide.md` ```mermaid flowchart TD subgraph node1["Node 1 · net-node-01"] - c1["c1 · .202"] & c2["c2 · .40"] --- br1("incusbr0\n10.0.0.1/24") + c1["c1 · .202"] & c2["c2 · .40"] --- br1("incusbr0
10.0.0.1/24") br1 --> nat1["NAT · 192.168.1.209"] end subgraph node2["Node 2 · net-node-02"] - c3["c3 · .52"] --- br2("incusbr0\n10.0.0.1/24") + c3["c3 · .52"] --- br2("incusbr0
10.0.0.1/24") br2 --> nat2["NAT · 192.168.1.150"] end - nat1 & nat2 --- lan(("LAN\n192.168.1.0/24")) + nat1 & nat2 --- lan(("LAN
192.168.1.0/24")) classDef instance fill:#56B4E9,color:#fff,stroke:#3a8fbf classDef network fill:#0072B2,color:#fff,stroke:#005a8e @@ -109,22 +110,22 @@ Original: `notes/networking-guide.md` ```mermaid flowchart TD subgraph cp["OVN Control Plane"] - ovnc["ovn-central\nNB :6641 · SB :6642"] + ovnc["ovn-central
NB :6641 · SB :6642"] end subgraph n1["Node 1 · net-node-01"] ctrl1["ovn-controller"] ~~~ ls1 - c1["c1 · .2"] & c2["c2 · .3"] --- ls1("logical switch\n10.10.10.0/24") + c1["c1 · .2"] & c2["c2 · .3"] --- ls1("logical switch
10.10.10.0/24") end subgraph n2["Node 2 · net-node-02"] ctrl2["ovn-controller"] ~~~ ls2 - c3["c3 · .4"] --- ls2("logical switch\n10.10.10.0/24") + c3["c3 · .4"] --- ls2("logical switch
10.10.10.0/24") end subgraph n3["Node 3 · net-node-03"] ctrl3["ovn-controller"] ~~~ ls3 - c4["c4 · .5"] --- ls3("logical switch\n10.10.10.0/24") + c4["c4 · .5"] --- ls3("logical switch
10.10.10.0/24") end ovnc -.-> ctrl1 & ctrl2 & ctrl3 @@ -133,7 +134,7 @@ flowchart TD ls2 <-->|Geneve| ls3 ls1 <-->|Geneve| ls3 - n1 & n2 & n3 --- lan(("LAN\n192.168.1.0/24")) + n1 & n2 & n3 --- lan(("LAN
192.168.1.0/24")) classDef mgmt fill:#CC79A7,color:#fff,stroke:#a36088 classDef instance fill:#56B4E9,color:#fff,stroke:#3a8fbf @@ -162,9 +163,9 @@ Original: `notes/ovn-deep-dive.md` ```mermaid graph LR - n1(("oc-node-01\n.140")) - n2(("oc-node-02\n.141")) - n3(("oc-node-03\n.142")) + n1(("oc-node-01
.140")) + n2(("oc-node-02
.141")) + n3(("oc-node-03
.142")) n1 <-->|"Geneve 6081"| n2 n2 <-->|"Geneve 6081"| n3 @@ -224,15 +225,15 @@ Original: `notes/operations-center-guide.md` flowchart TD subgraph proxmox["Proxmox VE Host · i9-13900HK · 64 GiB"] subgraph cluster["Incus Cluster · net-prod 10.10.10.0/24"] - n1["oc-node-01\nVMID 400 · .140"] - n2["oc-node-02\nVMID 401 · .141"] - n3["oc-node-03\nVMID 402 · .142"] + n1["oc-node-01
VMID 400 · .140"] + n2["oc-node-02
VMID 401 · .141"] + n3["oc-node-03
VMID 402 · .142"] end - oc["Operations Center\nVMID 920 · .120"] + oc["Operations Center
VMID 920 · .120"] end - vmbr0(("VLAN 69\n192.168.100.0/22")) - ext["OVN external IPs\n192.168.103.200-210"] + vmbr0(("VLAN 69
192.168.100.0/22")) + ext["OVN external IPs
192.168.103.200-210"] proxmox --- vmbr0 cluster -.->|"external gateway"| ext diff --git a/notes/networking-guide.md b/notes/networking-guide.md index 8421162..81399d6 100644 --- a/notes/networking-guide.md +++ b/notes/networking-guide.md @@ -78,25 +78,30 @@ works without any additional infrastructure. ### Bridge topology -``` -Node 1 (net-node-01) Node 2 (net-node-02) -┌─────────────────────┐ ┌─────────────────────┐ -│ incusbr0 │ │ incusbr0 │ -│ 10.0.0.1/24 │ │ 10.0.0.1/24 │ -│ ┌───┐ ┌───┐ │ │ ┌───┐ │ -│ │c1 │ │c2 │ │ │ │c3 │ │ -│ │.202│ │.40│ │ │ │.52│ │ -│ └───┘ └───┘ │ │ └───┘ │ -│ │ │ │ │ │ │ -│ ──┴──────┴── │ │ ──┴── │ -│ (bridge) │ │ (bridge) │ -│ │ │ │ │ │ -│ NAT ──> ens18 │ │ NAT ──> ens18 │ -│ 192.168.1.209 │ │ 192.168.1.150 │ -└─────────────────────┘ └─────────────────────┘ - │ │ - ────┴──────────────────────────────────┴──── - LAN (192.168.1.0/24) +```mermaid +flowchart TD + subgraph node1["Node 1 · net-node-01"] + c1["c1 · .202"] & c2["c2 · .40"] --- br1("incusbr0
10.0.0.1/24") + br1 --> nat1["NAT · 192.168.1.209"] + end + + subgraph node2["Node 2 · net-node-02"] + c3["c3 · .52"] --- br2("incusbr0
10.0.0.1/24") + br2 --> nat2["NAT · 192.168.1.150"] + end + + nat1 & nat2 --- lan(("LAN
192.168.1.0/24")) + + classDef instance fill:#56B4E9,color:#fff,stroke:#3a8fbf + classDef network fill:#0072B2,color:#fff,stroke:#005a8e + classDef node fill:#009E73,color:#fff,stroke:#007a5e + + class c1,c2,c3 instance + class br1,br2,lan network + class nat1,nat2 node + + style node1 fill:#e6f5f0,stroke:#009E73 + style node2 fill:#e6f5f0,stroke:#009E73 ``` **Key point:** Each node has its own bridge with the same subnet (10.0.0.1/24). @@ -193,37 +198,49 @@ Options for deploying the control plane: ### OVN topology -``` - ┌──────────────────────────────────┐ - │ OVN Control Plane │ - │ (container: ovn-central) │ - │ ovn-northd + ovsdb-server │ - │ NB: tcp:192.168.1.209:6641 │ - │ SB: tcp:192.168.1.209:6642 │ - └──────────┬───────────────────────┘ - │ (proxy devices) - ┌─────────────────────────┼─────────────────────────┐ - │ │ │ -Node 1 (net-node-01) Node 2 (net-node-02) Node 3 (net-node-03) -┌───────────────────┐ ┌───────────────────┐ ┌───────────────────┐ -│ ovn-controller │ │ ovn-controller │ │ ovn-controller │ -│ ovs-vswitchd │ │ ovs-vswitchd │ │ ovs-vswitchd │ -│ │ │ │ │ │ -│ ┌───┐ ┌───┐ │ │ ┌───┐ │ │ ┌───┐ │ -│ │c1 │ │c2 │ │ │ │c3 │ │ │ │c4 │ │ -│ │.2 │ │.3 │ │ │ │.4 │ │ │ │.5 │ │ -│ └─┬─┘ └─┬─┘ │ │ └─┬─┘ │ │ └─┬─┘ │ -│ └──┬───┘ │ │ │ │ │ │ │ -│ OVN logical │ │ OVN logical │ │ OVN logical │ -│ switch │ │ switch │ │ switch │ -│ 10.10.10.0/24 │ │ 10.10.10.0/24 │ │ 10.10.10.0/24 │ -│ │ │ │ │ │ │ │ │ -│ Geneve tunnel ──┼──┼── Geneve tunnel ──┼──┼── Geneve tunnel │ -│ 192.168.1.209 │ │ 192.168.1.150 │ │ 192.168.1.13 │ -└───────────────────┘ └───────────────────┘ └───────────────────┘ - │ │ │ - ────┴──────────────────────┴──────────────────────┴──── - LAN (192.168.1.0/24) +```mermaid +flowchart TD + subgraph cp["OVN Control Plane"] + ovnc["ovn-central
NB :6641 · SB :6642"] + end + + subgraph n1["Node 1 · net-node-01"] + ctrl1["ovn-controller"] ~~~ ls1 + c1["c1 · .2"] & c2["c2 · .3"] --- ls1("logical switch
10.10.10.0/24") + end + + subgraph n2["Node 2 · net-node-02"] + ctrl2["ovn-controller"] ~~~ ls2 + c3["c3 · .4"] --- ls2("logical switch
10.10.10.0/24") + end + + subgraph n3["Node 3 · net-node-03"] + ctrl3["ovn-controller"] ~~~ ls3 + c4["c4 · .5"] --- ls3("logical switch
10.10.10.0/24") + end + + ovnc -.->|"proxy devices"| ctrl1 & ctrl2 & ctrl3 + + ls1 <-->|Geneve| ls2 + ls2 <-->|Geneve| ls3 + ls1 <-->|Geneve| ls3 + + n1 & n2 & n3 --- lan(("LAN
192.168.1.0/24")) + + classDef mgmt fill:#CC79A7,color:#fff,stroke:#a36088 + classDef instance fill:#56B4E9,color:#fff,stroke:#3a8fbf + classDef network fill:#0072B2,color:#fff,stroke:#005a8e + classDef node fill:#009E73,color:#fff,stroke:#007a5e + + class ovnc mgmt + class ctrl1,ctrl2,ctrl3 node + class c1,c2,c3,c4 instance + class ls1,ls2,ls3,lan network + + style cp fill:#f5e6f0,stroke:#CC79A7 + style n1 fill:#e6f5f0,stroke:#009E73 + style n2 fill:#e6f5f0,stroke:#009E73 + style n3 fill:#e6f5f0,stroke:#009E73 ``` **Key difference from bridge:** All instances share a single logical switch diff --git a/notes/observability-guide.md b/notes/observability-guide.md index 15de7c8..a0e7dea 100644 --- a/notes/observability-guide.md +++ b/notes/observability-guide.md @@ -11,57 +11,48 @@ traffic on the shared network. ## Architecture -``` - LAN (192.168.103.0/24) - | - | OVN network forward (192.168.103.201) - | :3000 -> 10.10.10.70:3000 (Grafana) - | :9090 -> 10.10.10.70:9090 (Prometheus) - | - ═══════╪══════════════════════════════════════════════ - | OVN net-prod (10.10.10.0/24) - | - v - +----------------------------------------------+ - | monitoring (10.10.10.70, oc-node-02) | - | Debian/12 container — 2 GiB RAM, 20 GiB | - | | - | +-------------+ +-----------+ | - | | Prometheus | | Grafana | | - | | :9090 | | :3000 | | - | +-------------+ +-----------+ | - | +-------------+ +-----------+ | - | | Loki | | Promtail | | - | | :3100 | | (agent) | | - | | gRPC :9096 | | | | - | +-------------+ +-----------+ | - +----------------------------------------------+ - | - | Scrape targets - | - +-------> Incus nodes (client cert auth, :8443) - | 192.168.102.140 /1.0/metrics - | 192.168.102.141 /1.0/metrics - | 192.168.102.142 /1.0/metrics - | - +-------> HAProxy (:8404) - | 10.10.10.50 /metrics - | 10.10.10.51 /metrics - | - +-------> node-exporters (:9100) - 10.10.10.71 node-exp-01 (oc-node-01) - 10.10.10.72 node-exp-02 (oc-node-02) - 10.10.10.73 node-exp-03 (oc-node-03) +```mermaid +flowchart TD + lan(["LAN 192.168.103.0/24"]) - +------------------+ +------------------+ +------------------+ - | node-exp-01 | | node-exp-02 | | node-exp-03 | - | 10.10.10.71 | | 10.10.10.72 | | 10.10.10.73 | - | Alpine, privil. | | Alpine, privil. | | Alpine, privil. | - | :9100 | | :9100 | | :9100 | - | host /proc,/sys | | host /proc,/sys | | host /proc,/sys | - | 128 MiB RAM | | 128 MiB RAM | | 128 MiB RAM | - | pinned to node | | pinned to node | | pinned to node | - +------------------+ +------------------+ +------------------+ + subgraph fwd["OVN Forward · 192.168.103.201"] + fwd_grafana[":3000 → Grafana"] + fwd_prom[":9090 → Prometheus"] + end + + subgraph mon["monitoring · 10.10.10.70 · oc-node-02"] + prometheus["Prometheus :9090"] + grafana["Grafana :3000"] + loki["Loki :3100
gRPC :9096"] + promtail["Promtail"] + end + + subgraph targets["Scrape Targets"] + incus["Incus nodes :8443
.140 · .141 · .142"] + haproxy["HAProxy :8404
.50 · .51"] + ne["node-exporters :9100
.71 · .72 · .73"] + end + + lan --> fwd + fwd --> mon + prometheus -->|"client cert"| incus + prometheus --> haproxy + prometheus --> ne + + classDef external fill:#f5f5f5,color:#333,stroke:#999 + classDef network fill:#0072B2,color:#fff,stroke:#005a8e + classDef mgmt fill:#CC79A7,color:#fff,stroke:#a36088 + classDef instance fill:#56B4E9,color:#fff,stroke:#3a8fbf + classDef node fill:#009E73,color:#fff,stroke:#007a5e + + class lan external + class fwd_grafana,fwd_prom network + class prometheus,grafana,loki,promtail mgmt + class incus,haproxy,ne instance + + style fwd fill:#e0eef8,stroke:#0072B2 + style mon fill:#f5e6f0,stroke:#CC79A7 + style targets fill:#e0f2fe,stroke:#56B4E9 ``` ## Components diff --git a/notes/operations-center-guide.md b/notes/operations-center-guide.md index d3a7cdd..ea254a4 100644 --- a/notes/operations-center-guide.md +++ b/notes/operations-center-guide.md @@ -9,39 +9,33 @@ build 202602230420 on Proxmox VE 9.1.5 (nested virtualization on Intel). ## Section 0: Architecture Overview -``` - ┌──────────────────────────────────────────┐ - │ Proxmox VE Host (pve) │ - │ Intel i9-13900HK 64 GiB 20 cores │ - │ │ - │ ┌──────────┐ ┌──────────┐ ┌──────────┐│ - │ │oc-node-01│ │oc-node-02│ │oc-node-03││ - │ │ VMID 400 │ │ VMID 401 │ │ VMID 402 ││ - │ │ 4c/8G/64G│ │ 4c/8G/50G│ │ 4c/8G/50G││ - │ │ .140 │ │ .141 │ │ .142 ││ - │ │ │ │ │ │ ││ - │ │ovn-central│ │ │ │ ││ - │ │ha-web-01 │ │ha-web-02 │ │ha-web-03 ││ - │ └──────────┘ └──────────┘ └──────────┘│ - │ │ │ │ │ - │ └────────────┼────────────┘ │ - │ OVN Geneve tunnels │ - │ net-prod 10.10.10.0/24 │ - │ │ - │ ┌──────────┐ │ - │ │oc-server │ OC Management │ - │ │ VMID 920 │ https://192.168.102.120 │ - │ │ 2c/4G/50G│ Web UI + CLI + REST API │ - │ │ .120 │ │ - │ └──────────┘ │ - │ │ - │ vmbr0 (VLAN 69) ── 192.168.100.0/22 │ - └──────────────────────────────────────────┘ - │ - │ gateway 192.168.100.1 - │ - OVN external IPs: 192.168.103.200-210 - LB: 192.168.103.201 Fwd: 192.168.103.202 +```mermaid +flowchart TD + subgraph proxmox["Proxmox VE Host · i9-13900HK · 64 GiB"] + subgraph cluster["Incus Cluster · net-prod 10.10.10.0/24"] + n1["oc-node-01
VMID 400 · .140"] + n2["oc-node-02
VMID 401 · .141"] + n3["oc-node-03
VMID 402 · .142"] + end + oc["Operations Center
VMID 920 · .120"] + end + + vmbr0(("VLAN 69
192.168.100.0/22")) + ext["OVN external IPs
192.168.103.200-210"] + + proxmox --- vmbr0 + cluster -.->|"external gateway"| ext + + classDef node fill:#009E73,color:#fff,stroke:#007a5e + classDef mgmt fill:#CC79A7,color:#fff,stroke:#a36088 + classDef network fill:#0072B2,color:#fff,stroke:#005a8e + + class n1,n2,n3 node + class oc mgmt + class vmbr0,ext network + + style proxmox fill:#f5f5f5,stroke:#999 + style cluster fill:#e6f5f0,stroke:#009E73 ``` ### Infrastructure diff --git a/notes/ovn-deep-dive.md b/notes/ovn-deep-dive.md index eb422a9..4978ba4 100644 --- a/notes/ovn-deep-dive.md +++ b/notes/ovn-deep-dive.md @@ -6,16 +6,25 @@ cluster running OVS 3.6.1 with an OVN overlay network. ## Lab Topology -``` -LAN (192.168.100.0/22) -│ -├── oc-node-01 192.168.102.140 (Geneve endpoint) -├── oc-node-02 192.168.102.141 (Geneve endpoint) -├── oc-node-03 192.168.102.142 (Geneve endpoint, gateway chassis) -│ -└── UPLINK physical network - └── OVN range: 192.168.103.200-210 - └── net-prod (10.10.10.0/24) ← OVN overlay +```mermaid +flowchart TD + lan(("LAN
192.168.100.0/22")) + + n1["oc-node-01 · .140
Geneve endpoint"] + n2["oc-node-02 · .141
Geneve endpoint"] + n3["oc-node-03 · .142
Geneve + gateway"] + + uplink("UPLINK
OVN range .103.200-210") + netprod("net-prod
10.10.10.0/24") + + lan --- n1 & n2 & n3 + lan --- uplink --> netprod + + classDef node fill:#009E73,color:#fff,stroke:#007a5e + classDef network fill:#0072B2,color:#fff,stroke:#005a8e + + class n1,n2,n3 node + class lan,uplink,netprod network ``` **Instances on net-prod:** @@ -302,14 +311,30 @@ Each IncusOS node runs OVS 3.6.1 with two bridges. ### Bridge Architecture -``` -Per-node OVS layout: +```mermaid +flowchart LR + subgraph provider["incusovn7 · provider bridge"] + nic["physical NIC"] + intport["internal port"] + patch1["patch to br-int"] + end -incusovn7 (provider bridge) br-int (integration bridge) -├── incusovn7 ← physical NIC ├── veth* ← instance NICs -├── incusovn7b ← internal port ├── ovn-* ← Geneve tunnels -└── patch-...-to-br-int ←──patch──→ ├── patch-br-int-to-... - └── br-int ← internal port + subgraph integration["br-int · integration bridge"] + veth["instance veth ports"] + ovntun["Geneve tunnels"] + patch2["patch to incusovn7"] + end + + patch1 <-->|"patch port"| patch2 + + classDef prov fill:#0072B2,color:#fff,stroke:#005a8e + classDef integ fill:#009E73,color:#fff,stroke:#007a5e + + class nic,intport,patch1 prov + class veth,ovntun,patch2 integ + + style provider fill:#e0eef8,stroke:#0072B2 + style integration fill:#e6f5f0,stroke:#009E73 ``` **`br-int` (integration bridge)**: @@ -329,12 +354,19 @@ incusovn7 (provider bridge) br-int (integration bridge) Every pair of chassis has a Geneve tunnel with BFD health monitoring: -``` -oc-node-01 ←── Geneve (UDP 6081) ──→ oc-node-02 - ↑ ↑ - └──────── Geneve (UDP 6081) ──────────┘ - ↓ - oc-node-03 +```mermaid +graph LR + n1(("oc-node-01
.140")) + n2(("oc-node-02
.141")) + n3(("oc-node-03
.142")) + + n1 <-->|"Geneve 6081"| n2 + n2 <-->|"Geneve 6081"| n3 + n1 <-->|"Geneve 6081"| n3 + + classDef chassis fill:#009E73,color:#fff,stroke:#007a5e + + class n1,n2,n3 chassis ``` | Source | Destination | OVS Port Name | BFD State | diff --git a/notes/production-lab-guide.md b/notes/production-lab-guide.md index 15cfa63..c36e53c 100644 --- a/notes/production-lab-guide.md +++ b/notes/production-lab-guide.md @@ -14,35 +14,40 @@ client 6.21, Operations Center v0.3.0. ### Network Topology -``` - VLAN 69 (192.168.100.0/22) - Gateway: 192.168.100.1 - │ - ┌──────────────────┼──────────────────────────────────┐ - │ │ │ - ┌──────┴──────┐ ┌──────┴──────┐ ┌───────┴─────┐ ┌─────┴───────┐ - │ lab-oc │ │ lab-node-01 │ │ lab-node-02 │ │ lab-node-03 │ - │ VMID 910 │ │ VMID 911 │ │ VMID 912 │ │ VMID 913 │ - │ .102.110 │ │ .102.111 │ │ .102.112 │ │ .102.113 │ - │ OC server │ │ Init node │ │ Join node │ │ Join node │ - │ 2c/4G/50G │ │ 4c/8G/64G │ │ 4c/8G/50G │ │ 4c/8G/50G │ - └─────────────┘ │ │ │ │ │ │ - │ ovn-central │ │ │ │ │ - │ (container) │ │ │ │ │ - │ NB:6641 │ │ │ │ │ - │ SB:6642 │ │ │ │ │ - └─────────────┘ └─────────────┘ └─────────────┘ - │ │ │ - └────────────────┼────────────────┘ - Geneve tunnels (OVN) - │ - ┌────────────────┼────────────────┐ - │ │ │ - ┌──────┴──────┐ ┌──────┴──────┐ ┌─────┴───────┐ - │ net-prod │ │net-isolated │ │ UPLINK │ - │ 10.10.10/24 │ │ 10.10.20/24 │ │ .103.200-210│ - │ OVN L2 │ │ OVN L2 │ │ Physical │ - └─────────────┘ └─────────────┘ └─────────────┘ +```mermaid +flowchart TD + vlan(("VLAN 69
192.168.100.0/22")) + + subgraph mgmt["Management"] + oc["lab-oc
VMID 910 · .110
OC server"] + end + + subgraph cluster["Incus Cluster"] + n1["lab-node-01
VMID 911 · .111
init + ovn-central"] + n2["lab-node-02
VMID 912 · .112"] + n3["lab-node-03
VMID 913 · .113"] + end + + subgraph networks["OVN Networks"] + prod("net-prod
10.10.10.0/24") + iso("net-isolated
10.10.20.0/24") + uplink("UPLINK
.103.200-210") + end + + vlan --- mgmt & cluster + cluster -->|"Geneve tunnels"| networks + + classDef nodeClass fill:#009E73,color:#fff,stroke:#007a5e + classDef mgmtClass fill:#CC79A7,color:#fff,stroke:#a36088 + classDef networkClass fill:#0072B2,color:#fff,stroke:#005a8e + + class n1,n2,n3 nodeClass + class oc mgmtClass + class prod,iso,uplink,vlan networkClass + + style mgmt fill:#f5e6f0,stroke:#CC79A7 + style cluster fill:#e6f5f0,stroke:#009E73 + style networks fill:#e0eef8,stroke:#0072B2 ``` ### Infrastructure diff --git a/notes/shared-storage-guide.md b/notes/shared-storage-guide.md index 3623bbb..c50726f 100644 --- a/notes/shared-storage-guide.md +++ b/notes/shared-storage-guide.md @@ -43,21 +43,31 @@ on local ZFS. ### How iSCSI + lvmcluster works -``` -┌──────────────────┐ iSCSI (1GbE) ┌──────────────────┐ -│ iSCSI Target │◄───────────────────────►│ oc-node-01 │ -│ (QNAP / VM / │◄───────────────────────►│ oc-node-02 │ -│ any target) │◄───────────────────────►│ oc-node-03 │ -│ │ │ │ -│ Exposes LUN(s) │ │ iSCSI initiator │ -│ as block device │ │ (IncusOS built-in)│ -└──────────────────┘ │ │ - │ lvmlockd+sanlock│ - │ (IncusOS built-in)│ - │ │ - │ Incus lvmcluster│ - │ storage pool │ - └──────────────────┘ +```mermaid +flowchart LR + target["iSCSI Target
QNAP / VM / any target
Exposes LUN as block device"] + + subgraph cluster["Incus Cluster Nodes"] + n1["oc-node-01"] + n2["oc-node-02"] + n3["oc-node-03"] + stack["iSCSI initiator (built-in)
lvmlockd + sanlock (built-in)
Incus lvmcluster pool"] + end + + target <-->|"iSCSI (1GbE)"| n1 + target <-->|"iSCSI (1GbE)"| n2 + target <-->|"iSCSI (1GbE)"| n3 + n1 & n2 & n3 ~~~ stack + + classDef network fill:#0072B2,color:#fff,stroke:#005a8e + classDef node fill:#009E73,color:#fff,stroke:#007a5e + classDef instance fill:#56B4E9,color:#fff,stroke:#3a8fbf + + class target network + class n1,n2,n3 node + class stack instance + + style cluster fill:#e6f5f0,stroke:#009E73 ``` 1. **iSCSI target** exports a block device (LUN) over the network @@ -97,29 +107,33 @@ The recommended setup uses **both** pools: ### Network topology (tested) -``` -┌─ Proxmox host ──────────────────────────────────────────────────────┐ -│ │ -│ ┌──────────────────────────────────────────────────────────────┐ │ -│ │ oc-node-01 (VMID 400, 192.168.102.140) │ │ -│ │ ┌──────────────┐ │ │ -│ │ │iscsi-target │ Debian container running tgt │ │ -│ │ │10.207.217.19 │ incusbr0 (bridge, same-node access) │ │ -│ │ │.102.150 │ macvlan on mgmt (cross-node access) │ │ -│ │ │LUN: 20 GiB │◄─── iSCSI initiator (oc-node-01 via bridge)│ │ -│ │ └──────────────┘◄─── iSCSI initiator (oc-node-02 via macvlan)│ │ -│ │ ◄─── iSCSI initiator (oc-node-03 via macvlan)│ │ -│ └──────────────────────────────────────────────────────────────┘ │ -│ │ -│ ┌──────────┐ ┌──────────┐ │ -│ │oc-node-02│ │oc-node-03│ │ -│ │ VMID 401 │ │ VMID 402 │ │ -│ │ .102.141 │ │ .102.142 │ │ -│ │ lvmclust │ │ lvmclust │ │ -│ └──────────┘ └──────────┘ │ -│ │ -│ vmbr0 (VLAN 69) ── 192.168.100.0/22 │ -└─────────────────────────────────────────────────────────────────────┘ +```mermaid +flowchart TD + subgraph proxmox["Proxmox Host"] + subgraph n1["oc-node-01 · VMID 400 · .140"] + target["iscsi-target
10.207.217.19 (bridge)
192.168.102.150 (macvlan)
LUN: 20 GiB"] + end + + n2["oc-node-02
VMID 401 · .141
lvmcluster"] + n3["oc-node-03
VMID 402 · .142
lvmcluster"] + end + + target <-->|"bridge"| n1 + target <-->|"macvlan"| n2 + target <-->|"macvlan"| n3 + vlan(("VLAN 69
192.168.100.0/22")) + proxmox --- vlan + + classDef network fill:#0072B2,color:#fff,stroke:#005a8e + classDef node fill:#009E73,color:#fff,stroke:#007a5e + classDef lb fill:#E69F00,color:#fff,stroke:#b87d00 + + class target network + class n2,n3 node + class vlan network + + style proxmox fill:#f5f5f5,stroke:#999 + style n1 fill:#e6f5f0,stroke:#009E73 ``` **Lab target container** (Option A, tested): a Debian container on oc-node-01 @@ -1365,19 +1379,31 @@ For large VMs, this can take time depending on disk size and pool speeds. With shared iSCSI storage, adding a second Proxmox host enables true cross- host live migration: -``` -┌─ Proxmox Host A ────────┐ ┌─ Proxmox Host B ────────┐ -│ oc-node-01, oc-node-02 │ │ oc-node-03, oc-node-04 │ -│ iSCSI initiators │ │ iSCSI initiators │ -└──────────┬───────────────┘ └──────────┬───────────────┘ - │ │ - └────────── iSCSI ───────────────┘ - │ - ┌─────────┴──────────┐ - │ QNAP NAS │ - │ iSCSI target │ - │ Shared LUN │ - └────────────────────┘ +```mermaid +flowchart TD + subgraph hostA["Proxmox Host A"] + a1["oc-node-01"] + a2["oc-node-02"] + end + + subgraph hostB["Proxmox Host B"] + b1["oc-node-03"] + b2["oc-node-04"] + end + + qnap["QNAP NAS
iSCSI target
Shared LUN"] + + hostA <-->|"iSCSI"| qnap + hostB <-->|"iSCSI"| qnap + + classDef node fill:#009E73,color:#fff,stroke:#007a5e + classDef network fill:#0072B2,color:#fff,stroke:#005a8e + + class a1,a2,b1,b2 node + class qnap network + + style hostA fill:#f5f5f5,stroke:#999 + style hostB fill:#f5f5f5,stroke:#999 ``` Requirements: