diff --git a/incusos/examples/lab-oc-nodes.yaml b/incusos/examples/lab-oc-nodes.yaml
index 6d07611..ab370ef 100644
--- a/incusos/examples/lab-oc-nodes.yaml
+++ b/incusos/examples/lab-oc-nodes.yaml
@@ -28,7 +28,7 @@ vms:
- name: oc-node-01
app: incus
apply_defaults: false
- disk: 64
+ disk: 250
ip: 192.168.102.140/22
- name: oc-node-02
diff --git a/notes/production-lab-guide.md b/notes/production-lab-guide.md
index 7f48cd5..37e51d8 100644
--- a/notes/production-lab-guide.md
+++ b/notes/production-lab-guide.md
@@ -1,13 +1,14 @@
# Production Home Lab Guide
Build a production-quality Incus home lab from scratch: Operations Center
-dashboard, 3-node cluster with OVN overlay networking, mixed container/VM
-workloads, live migration, network security, load balancers, and cluster
-lifecycle management.
+dashboard, OC-managed 3-node cluster with OVN overlay networking, Aether
+management platform, HAProxy load balancing, AWX lifecycle automation,
+Prometheus/Grafana/Loki observability stack, live migration, network
+security, and cluster lifecycle management.
-All commands and output in this guide are from an actual deployment on
-2026-02-22. Tested on Proxmox VE 9.1.5, IncusOS 202602210344, Incus
-client 6.21, Operations Center v0.3.0.
+All commands and output in this guide are from actual deployments on
+Proxmox VE 9.1.5 with IncusOS 202602230420, Incus client 6.21,
+Operations Center v0.3.0, and Aether v6.4.317.
## Section 0: Architecture Overview
@@ -19,77 +20,129 @@ flowchart TD
vlan(("VLAN 69
192.168.100.0/22"))
subgraph mgmt["Management"]
- oc["lab-oc
VMID 910 · .110
OC server"]
+ oc["oc-server
VMID 920 · .120
Operations Center"]
end
subgraph cluster["Incus Cluster"]
- n1["lab-node-01
VMID 911 · .111
init + ovn-central"]
- n2["lab-node-02
VMID 912 · .112"]
- n3["lab-node-03
VMID 913 · .113"]
+ n1["oc-node-01
VMID 400 · .140
Aether · OVN central"]
+ n2["oc-node-02
VMID 401 · .141
AWX · Monitoring"]
+ n3["oc-node-03
VMID 402 · .142
HAProxy backends"]
end
- subgraph networks["OVN Networks"]
- prod("net-prod
10.10.10.0/24")
- iso("net-isolated
10.10.20.0/24")
- uplink("UPLINK
.103.200-210")
+ subgraph services["Macvlan Services"]
+ aether["Aether · .160"]
+ awx["AWX · .161"]
+ end
+
+ subgraph ovn["OVN · net-prod 10.10.10.0/24"]
+ haproxy["HAProxy HA
.50 · .51"]
+ backends["nginx backends
.60 · .61 · .62"]
+ monitoring["Observability
.70 + node-exp .71-.73"]
+ end
+
+ subgraph uplink["UPLINK · 192.168.103.x"]
+ vip[".200 HAProxy VIP"]
+ fwd[".201 Grafana / Prometheus"]
end
vlan --- mgmt & cluster
- cluster -->|"Geneve tunnels"| networks
+ cluster --> services & ovn
+ ovn -.-> uplink
classDef nodeClass fill:#009E73,color:#fff,stroke:#007a5e
classDef mgmtClass fill:#CC79A7,color:#fff,stroke:#a36088
+ classDef serviceClass fill:#E69F00,color:#fff,stroke:#b87d00
+ classDef ovnClass fill:#56B4E9,color:#fff,stroke:#3a8fbf
classDef networkClass fill:#0072B2,color:#fff,stroke:#005a8e
class n1,n2,n3 nodeClass
class oc mgmtClass
- class prod,iso,uplink,vlan networkClass
+ class aether,awx serviceClass
+ class haproxy,backends,monitoring ovnClass
+ class vip,fwd,vlan networkClass
style mgmt fill:#f5e6f0,stroke:#CC79A7
style cluster fill:#e6f5f0,stroke:#009E73
- style networks fill:#e0eef8,stroke:#0072B2
+ style services fill:#fef3e0,stroke:#E69F00
+ style ovn fill:#e0f2fe,stroke:#56B4E9
+ style uplink fill:#e0eef8,stroke:#0072B2
```
### Infrastructure
| Component | VMID | IP | Cores | RAM | Disk | Role |
|-----------|------|-----|-------|-----|------|------|
-| lab-oc | 910 | 192.168.102.110/22 | 2 | 4 GiB | 50G | Operations Center |
-| lab-node-01 | 911 | 192.168.102.111/22 | 4 | 8 GiB | 64G | Cluster init + OVN host |
-| lab-node-02 | 912 | 192.168.102.112/22 | 4 | 8 GiB | 50G | Cluster member |
-| lab-node-03 | 913 | 192.168.102.113/22 | 4 | 8 GiB | 50G | Cluster member |
+| oc-server | 920 | 192.168.102.120/22 | 2 | 4 GiB | 50G | Operations Center |
+| oc-node-01 | 400 | 192.168.102.140/22 | 4 | 20 GiB | 250G | Cluster init + Aether host |
+| oc-node-02 | 401 | 192.168.102.141/22 | 4 | 20 GiB | 100G | AWX + monitoring host |
+| oc-node-03 | 402 | 192.168.102.142/22 | 4 | 20 GiB | 100G | HAProxy backends |
-**RAM budget**: 28 GiB of 64 GiB (44% utilization). Leaves headroom for
-workloads inside the VMs and other labs on the host.
+**RAM budget**: 64 GiB of 94 GiB (68%). Host: i9-13900HK, 94 GiB RAM,
+881 GiB ZFS pool. Leaves 30 GiB headroom for other VMs on the host.
-**OVN IP allocation**: 192.168.103.200-210 reserved for OVN external
-addresses (router IPs, load balancer VIPs, network forwards). These must
-be excluded from your DHCP server's range.
+### Inner Cluster Services
+
+| Instance | Network | IP | Node | RAM | Description |
+|----------|---------|-----|------|-----|-------------|
+| ovn-central | incusbr0 | DHCP | node-01 | 512 MiB | OVN NB/SB databases |
+| aether | macvlan mgmt | 192.168.102.160 | node-01 | 8 GiB | Management platform |
+| awx | macvlan mgmt | 192.168.102.161 | node-02 | 8 GiB | Ansible automation |
+| ha-web-01 | net-prod | 10.10.10.60 | node-01 | 256 MiB | Nginx backend |
+| ha-web-02 | net-prod | 10.10.10.61 | node-02 | 256 MiB | Nginx backend |
+| ha-web-03 | net-prod | 10.10.10.62 | node-03 | 256 MiB | Nginx backend |
+| haproxy-01 | net-prod | 10.10.10.50 | varies | 512 MiB | HA load balancer |
+| haproxy-02 | net-prod | 10.10.10.51 | varies | 512 MiB | HA load balancer |
+| monitoring | net-prod | 10.10.10.70 | node-02 | 2 GiB | Prometheus + Grafana + Loki |
+| node-exp-01 | net-prod | 10.10.10.71 | node-01 | 128 MiB | Host metrics exporter |
+| node-exp-02 | net-prod | 10.10.10.72 | node-02 | 128 MiB | Host metrics exporter |
+| node-exp-03 | net-prod | 10.10.10.73 | node-03 | 128 MiB | Host metrics exporter |
+
+### External IP Allocation
+
+OVN external addresses from UPLINK range (192.168.103.200-210). Exclude
+these from your DHCP server's range:
+
+| IP | Purpose |
+|-----|---------|
+| 192.168.103.200 | HAProxy VIP (OVN load balancer → haproxy-01/02) |
+| 192.168.103.201 | Observability forward (Grafana :3000, Prometheus :9090) |
### Decision Rationale
-**Why manual clustering instead of OC `provisioning cluster add`?**
-OC v0.3.0's `provisioning cluster add` has a `needs update: false` blocker
-that can stall indefinitely. Manual clustering via `incus` CLI is proven
-reliable. OC still provides value as a monitoring dashboard.
+**Why OC-managed clustering?** OC `provisioning cluster add` is the
+production path for Incus deployments. It handles cluster formation,
+update management, and inventory centrally. The deploy scripts
+(`deploy-haproxy`, `deploy-awx`, `deploy-observability`) are built
+for the `oc-node-*` naming and IP scheme.
-**Why OVN?** Bridge networks are node-local — instances on different nodes
-cannot communicate. OVN provides transparent cross-node L2 overlay with
-sub-millisecond latency (~0.1-0.8ms), network isolation, ACLs, load
-balancers, and network forwards.
+**Why 20 GiB RAM per node?** Aether requires 8 GiB. AWX requires
+4-8 GiB. Monitoring + HAProxy + backends need ~3 GiB total.
+Leaves headroom for mixed workloads and live migration.
-**Why VLAN 69?** Isolates lab traffic from the production LAN. All VMs share
-VLAN 69 (subnet 192.168.100.0/22). The VLAN tag is set at the Proxmox VM
-level — IncusOS and workloads are unaware of it.
+**Why 250 GiB disk for node-01?** Aether's golden image is 200 GiB
+virtual (qcow2). With ZFS thin provisioning only ~11 GiB is used
+initially, but the pool needs 200 GiB allocatable space.
+
+**Why OVN?** Bridge networks are node-local — instances on different
+nodes cannot communicate. OVN provides cross-node L2 overlay with
+sub-ms latency, network isolation, ACLs, load balancers, and network
+forwards — essential for HAProxy HA and distributed workloads.
+
+**Why VLAN 69?** Isolates lab traffic from the production LAN. All VMs
+share VLAN 69 (subnet 192.168.100.0/22). The VLAN tag is set at the
+Proxmox level — IncusOS and workloads are unaware of it.
### Cross-References
-This guide brings together techniques from the deep-dive guides:
+This guide integrates techniques from the deep-dive guides:
-- [Clustering Guide](clustering-guide.md) — cluster formation, migration, lifecycle
-- [Networking Guide](networking-guide.md) — OVN setup, ACLs, peering, LBs
+- [Clustering Guide](clustering-guide.md) — manual cluster formation reference
+- [Networking Guide](networking-guide.md) — OVN setup, ACLs, peering
- [Operations Center Guide](operations-center-guide.md) — OC provisioning, CLI, web UI
-- [Migration Guide](migration-guide.md) — importing VMs from other hypervisors
+- [Aether Guide](aether-guide.md) — management platform deployment and API
+- [HAProxy Guide](haproxy-guide.md) — HA load balancing with Aether
+- [AWX Guide](awx-guide.md) — Ansible lifecycle automation
+- [Observability Guide](observability-guide.md) — Prometheus, Grafana, Loki stack
## Section 1: Prerequisites
@@ -108,9 +161,40 @@ curl --version | head -1
genisoimage --version 2>&1 | head -1
```
-**Minimum versions**: Incus client 6.3+ (for `remote get-client-certificate`
-fallback, though scripts read `~/.config/incus/client.crt` directly),
-Operations Center v0.3.0+.
+**Minimum versions**: Incus client 6.3+, Operations Center v0.3.0+.
+
+### Aether Browser Automation
+
+Several Aether features (HAProxy management, blueprint deployment) are
+not in the JWT API — they use session-authenticated routes with CSRF
+protection. Playwright browser automation is required for Sections 7-9:
+
+```bash
+node --version # Node.js 18+
+npx playwright --version
+```
+
+Install if missing:
+
+```bash
+npm install playwright @playwright/mcp
+npx playwright install chromium
+```
+
+The Playwright MCP server (configured in `.mcp.json`) provides browser
+tools when available. The `incusos/helpers/aether-browser` script is
+the standalone alternative.
+
+### Aether Golden Image
+
+The Aether golden image must be available locally before Section 7:
+
+```bash
+ls -la sources/aether-golden-image-v6.tar.gz
+```
+
+This is an Ubuntu Noble (24.04) image with 200 GiB virtual disk (~6.6
+GiB compressed). Obtain it from the Aether distribution.
### Proxmox Configuration
@@ -130,18 +214,27 @@ dns: 192.168.100.1
pool: IncusLab
```
-The `env` file at the repository root must export `PROXMOX_TOKEN_SECRET`.
-Scripts auto-discover it — no manual sourcing needed.
+The `env` file at the repository root must export `PROXMOX_TOKEN_SECRET`
+and `AETHER_ADMIN_PASSWORD`. Scripts auto-discover them.
### Client Certificates
-Incus client certificates are used for both Incus and OC connections:
+Incus client certificates are used for Incus, OC, and Prometheus
+(metrics scraping) connections:
```bash
# Verify cert exists (auto-generated on first incus command)
ls -la ~/.config/incus/client.crt ~/.config/incus/client.key
```
+Copy certs for OC CLI:
+
+```bash
+mkdir -p ~/.config/operations-center
+cp ~/.config/incus/client.crt ~/.config/operations-center/
+cp ~/.config/incus/client.key ~/.config/operations-center/
+```
+
For OC web UI browser access, generate a PKCS#12 bundle:
```bash
@@ -168,340 +261,374 @@ Expected output includes tool versions, IncusOS CDN reachability,
`proxmox.yaml` discovery, and Proxmox API connectivity.
-## Section 2: Deploy Infrastructure
+## Section 2: Deploy OC Server
### Configuration File
-The lab uses `incusos/examples/lab-production.yaml`:
-
```yaml
+# incusos/examples/lab-oc-deploy.yaml
defaults:
- cores: 4
- memory: 8192
+ cores: 2
+ memory: 4096
disk: 50
- start_vmid: 910
+ start_vmid: 920
+
+proxmox:
+ gateway: 192.168.100.1
+ dns: 192.168.100.1
vms:
- - name: lab-oc
+ - name: oc-server
app: operations-center
apply_defaults: true
- cores: 2
- memory: 4096
- ip: 192.168.102.110/22
-
- - name: lab-node-01
- app: incus
- apply_defaults: true # init node: needs storage pool + network
- disk: 64 # extra space for OVN control plane container
- ip: 192.168.102.111/22
-
- - name: lab-node-02
- app: incus
- apply_defaults: false # joining node: cluster join creates pool entry
- ip: 192.168.102.112/22
-
- - name: lab-node-03
- app: incus
- apply_defaults: false # joining node: cluster join creates pool entry
- ip: 192.168.102.113/22
+ ip: 192.168.102.120/22
```
-**Key decisions**: node-01 has `apply_defaults: true` (cluster init needs
-storage pool and network bridge). Nodes 02 and 03 have `apply_defaults: false`
-(the cluster join process creates member-specific entries). node-01 gets
-64 GiB disk for the OVN control plane container.
-
-### Dry Run
-
-Preview the deployment without making any changes:
-
-```bash
-./incusos-proxmox --dry-run examples/lab-production.yaml
-```
-
-This shows: ISO download plan, seed generation commands, VM creation
-parameters, and the full install sequence for each VM.
-
### Deploy
-Deploy all 4 VMs:
-
-```bash
-./incusos-proxmox --yes examples/lab-production.yaml
-```
-
-The deploy takes ~5-8 minutes:
-
-1. Downloads the latest IncusOS ISO (if not cached)
-2. Generates per-VM seed ISOs with static IP, hostname, certificates
-3. Creates VMs on Proxmox with UEFI, TPM, VirtIO settings
-4. Boots each VM with ISO + seed, monitors installation via blockstat
-5. Detects install completion (876 MiB written, then idle)
-6. Stops VMs, removes install media (ISO + seed)
-7. Starts VMs from disk, waits for port 8443 (up to 180s)
-8. Auto-heals scrub_schedule via IncusOS REST API
-9. Configures `incus` remotes for each Incus node
-
-### Verify Deployment
-
-Check deployment status:
-
-```bash
-./incusos-proxmox --status examples/lab-production.yaml
-```
-
-Expected output shows each VM with Proxmox state (running), network
-(static IP reachable), port 8443 (open), and incus remote (configured).
-
-### Verify Scrub Schedule
-
-Confirm the crontab bug fix is effective on all Incus nodes:
-
-```bash
-for node in lab-node-01 lab-node-02 lab-node-03; do
- echo -n "$node scrub_schedule: "
- incus query "$node":/os/1.0/system/storage | python3 -c \
- "import sys,json; print(json.load(sys.stdin).get('config',{}).get('scrub_schedule','EMPTY'))"
-done
-```
-
-Expected output — all nodes show `0 4 * * 0`:
-
-```
-lab-node-01 scrub_schedule: 0 4 * * 0
-lab-node-02 scrub_schedule: 0 4 * * 0
-lab-node-03 scrub_schedule: 0 4 * * 0
-```
-
-If any node shows `EMPTY`, the crontab bug hit. Run:
-```bash
-./incusos-proxmox --status examples/lab-production.yaml
-```
-The status check includes automatic scrub_schedule healing.
-
-
-## Section 3: Operations Center Setup
-
-### Add OC Remote
-
-```bash
-operations-center remote add oc-lab https://192.168.102.110:8443 --auth-type tls
-```
-
-Accept the certificate fingerprint when prompted.
-
-**Important**: The OC CLI does **not** support the `remote:` suffix syntax
-that the Incus CLI uses. Instead, switch to the remote first, then run
-commands without a remote suffix:
+```bash
+./incusos-proxmox --yes incusos/examples/lab-oc-deploy.yaml
+```
+
+Actual output (key lines):
+
+```
+[ok] VM 'oc-server' created (VMID 920)
+[ok] VM 'oc-server' installed and running at 192.168.102.120
+```
+
+### Set Up OC CLI Remote
```bash
+# Accept the TLS certificate when prompted
+operations-center remote add oc-lab https://192.168.102.120:8443 --auth-type tls
operations-center remote switch oc-lab
```
+**Important**: The OC CLI does **not** support the `remote:` suffix syntax
+that the Incus CLI uses. Switch to the remote first, then run commands
+without a remote suffix.
+
### Verify OC
```bash
operations-center admin os show
```
-Actual output:
+Actual output (uptime will vary):
-| PROPERTY | VALUE |
-|---------------|--------------------------------------------------------|
-| hostname | lab-oc |
-| os_version | 202602210344 |
-| kernel | 6.12.13 |
-| architecture | x86_64 |
-| uptime | 4390 |
-| addresses | 192.168.102.110/22 (mgmt), fd42:...:1 (incusbr0) |
-| storage_disks | /dev/sda (53.7GB, QEMU) |
-| storage_pools | local (zfs, /dev/sda4) |
+```
+WARNING: The IncusOS API and configuration is subject to change
-### Check Application Status
-
-```bash
-operations-center admin os application list
+environment:
+ hostname: oc-server
+ os_name: IncusOS
+ os_version: "202602230420"
+ os_version_next: ""
+ uptime: 63
```
-Actual output:
+### Wait for Updates
-| NAME | STATUS |
-|-------------------|---------|
-| operations-center | running |
-
-### Check for Updates
+OC downloads IncusOS update packages from upstream. At least one update
+must reach `ready` state before ISOs can be generated:
```bash
+# Poll until at least one update shows "ready"
operations-center provisioning update list
```
-Shows available IncusOS updates. Updates can be applied via the OC web UI
-or CLI.
+Actual output (after ~8 minutes; UUIDs are stable across deployments):
-### Service Status
+| UUID | Origin | Channels | Version | Severity | Status |
+|------|--------|----------|---------|----------|--------|
+| 82aefab7-fec7-5122-89fd-8412d3d2174c | linuxcontainers.org | stable | 202602200553 | none | ready |
+| 5d6b1018-e534-5e54-aeb5-c9e6027ab31d | linuxcontainers.org | stable | 202602210344 | none | ready |
+| c912a390-c38b-5bd9-b46f-ccaeba6da68a | linuxcontainers.org | stable | 202602230420 | none | ready |
+
+The table also includes `Upstream Channels` and `Published At` columns
+(omitted for width). Not all updates may be ready simultaneously — at
+least one `ready` is sufficient to proceed.
+
+### Web UI Access
+
+Open `https://192.168.102.120:8443/ui/` in your browser (with client.pfx
+imported from Section 1). The web UI provides a dashboard view of the OC
+server, update status, provisioning tokens, and system configuration.
+
+---
+
+## Section 3: Provision Nodes
+
+### 3.1 Create Provisioning Token
```bash
-operations-center admin os service list
+operations-center provisioning token add --uses 5 --description "Production lab cluster"
+operations-center provisioning token list
+```
+
+Actual output (UUID changes every run):
+
+| UUID | Uses Remaining | Expire At | Channel | Description |
+|------|----------------|-----------|---------|-------------|
+| | 5 | <30 days from now> | stable | Production lab cluster |
+
+Save the `` — you'll need it for the next steps.
+
+### 3.2 Create Token Seed (No force_reboot)
+
+**Critical**: the token seed must NOT include `force_reboot`. On Proxmox,
+`incusos-proxmox` handles the install lifecycle externally (blockstat
+detection + media removal). `force_reboot` triggers SysRq-B which causes
+the crontab bug (~50% failure rate).
+
+```yaml
+# /tmp/oc-preseed.yaml
+install:
+ version: "1"
+ force_install: true
+```
+
+**Important**: use the structured format with section keys (`install:`). A
+flat format (`version: "1"` at root) maps fields to empty `{}` and they
+don't get assigned to any section.
+
+```bash
+operations-center provisioning token seed add proxmox-preseed \
+ /tmp/oc-preseed.yaml --description "No force_reboot for Proxmox"
+```
+
+### 3.3 Generate OC-Provisioned ISO (Older Version)
+
+**Critical discovery**: nodes deployed from an ISO matching the latest OC
+update version are tracked as `needs_update: true` by OC because the OS
+was never delivered through OC's update pipeline. The fix: generate the ISO
+from an older channel so OC can push the real update after deployment.
+
+```bash
+# Create the old-stable channel (must exist before assigning updates to it)
+operations-center provisioning channel add old-stable \
+ --description "Older stable versions for initial provisioning"
+
+# Assign the second-latest update to the old-stable channel
+# (use the UUID for 202602210344 from `provisioning update list`)
+operations-center provisioning update assign-channels --channel old-stable
+
+# Generate ISO from the older channel
+operations-center provisioning token seed get-image proxmox-preseed \
+ /tmp/IncusOS-oc.iso --type iso --architecture x86_64 --channel old-stable
```
Actual output:
-| NAME | ENABLED |
-|----------|---------|
-| ovn | false |
-| syslog | false |
-| fan | false |
-| bgp | false |
-| dns | false |
-| metricsA | false |
-| metricsB | false |
-
-### Web UI Access
-
-Open `https://192.168.102.110:8443/ui/` in your browser. You need the
-PKCS#12 client certificate imported (see Section 1). The web UI provides
-a dashboard view of the OC server. After adding Incus nodes to OC
-(optional), the dashboard shows cluster health.
-
-**Note**: OC deployed with a standard ISO acts as a monitoring dashboard.
-For full OC node management (provisioning, cluster orchestration), nodes
-must boot from an OC-provisioned ISO. See
-[Operations Center Guide](operations-center-guide.md) for the full hybrid
-workflow.
-
-
-## Section 4: Cluster Formation
-
-### 4.1 Set Specific IP Addresses
-
-IncusOS nodes default to `core.https_address: :8443` (wildcard). Clustering
-requires specific routable IPs so nodes can address each other.
-
-```bash
-incus config set lab-node-01: core.https_address 192.168.102.111:8443
-incus config set lab-node-02: core.https_address 192.168.102.112:8443
-incus config set lab-node-03: core.https_address 192.168.102.113:8443
+```
+Successfully written 3433074688 bytes to "/tmp/IncusOS-oc.iso"
```
-Verify on each node:
+The ISO contains IncusOS 202602210344 (one version behind). OC will push
+the latest (202602230420) after nodes register.
-```bash
-incus config get lab-node-01: core.https_address
-incus config get lab-node-02: core.https_address
-incus config get lab-node-03: core.https_address
+### 3.4 Node Configuration
+
+```yaml
+# incusos/examples/lab-oc-nodes.yaml
+defaults:
+ cores: 4
+ memory: 20480
+ disk: 100
+ start_vmid: 400
+
+proxmox:
+ gateway: 192.168.100.1
+ dns: 192.168.100.1
+
+vms:
+ - name: oc-node-01
+ app: incus
+ apply_defaults: false
+ disk: 250
+ ip: 192.168.102.140/22
+
+ - name: oc-node-02
+ app: incus
+ apply_defaults: false
+ ip: 192.168.102.141/22
+
+ - name: oc-node-03
+ app: incus
+ apply_defaults: false
+ ip: 192.168.102.142/22
```
-Each should return `IP:8443`.
+**Key decisions**:
-### 4.2 Enable Clustering on Init Node
+- **20 GiB RAM** per node: Aether needs 8 GiB, AWX needs 4-8 GiB
+- **250 GiB disk** for node-01: hosts Aether's 200 GiB virtual image
+- **100 GiB disk** for nodes 02-03: sufficient for AWX, monitoring, HAProxy
+- **`apply_defaults: false`** for all nodes: OC's Terraform handles
+ resource creation during cluster formation
+
+### 3.5 Deploy Nodes (Hybrid Approach)
+
+The hybrid approach uses `incusos-proxmox --iso` to combine OC
+auto-registration (from the boot ISO token) with `incusos-proxmox` VM
+creation, per-node SEED_DATA (hostname, static IP), install monitoring,
+and media cleanup.
```bash
-incus cluster enable lab-node-01: lab-node-01
+./incusos/incusos-proxmox --iso /tmp/IncusOS-oc.iso --yes incusos/examples/lab-oc-nodes.yaml
```
-**Note the syntax**: TWO arguments — `lab-node-01:` (remote with trailing
-colon) and `lab-node-01` (member name). This is NOT `lab-node-01:lab-node-01`.
+Actual output (key lines):
-### 4.3 Fix Init Node Remote
+```
+[ok] ISO uploaded: IncusOS-oc.iso
+[ok] VM 'oc-node-01' installed and running at 192.168.102.140
+[ok] Remote 'oc-node-01' added (192.168.102.140)
+[ok] VM 'oc-node-02' installed and running at 192.168.102.141
+[ok] Remote 'oc-node-02' added (192.168.102.141)
+[ok] VM 'oc-node-03' installed and running at 192.168.102.142
+[ok] Remote 'oc-node-03' added (192.168.102.142)
+[ok] All post-deployment checks passed
+```
-Enabling clustering regenerates the TLS certificate. The new cert may only
-have SANs for `127.0.0.1` and `::1`, breaking the remote.
+All 3 nodes: 876 MiB blockstat detection, clean install, no crontab bug.
+
+### 3.6 Verify Auto-Registration
+
+Nodes auto-register with OC within ~30 seconds of first boot. The update
+from 202602210344 to 202602230420 happens automatically:
```bash
-incus remote switch local
-incus remote remove lab-node-01
-incus remote add lab-node-01 https://192.168.102.111:8443 --accept-certificate
+operations-center provisioning server list
```
-Verify:
+Actual output (key columns; full table includes Type, Channel, Certificate
+Fingerprint, Public Connection URL, Last Updated, Last Seen):
+
+| Cluster | Name | Connection URL | Status | Update Status |
+|---------|------|----------------|--------|---------------|
+| | oc-node-01 | https://192.168.102.140:8443 | ready | up to date |
+| | oc-node-02 | https://192.168.102.141:8443 | ready | up to date |
+| | oc-node-03 | https://192.168.102.142:8443 | ready | up to date |
+| | operations-center | https://[::1]:8443 | ready | update pending |
+
+**Key**: all 3 nodes show **"up to date"** because OC delivered the
+202602230420 update through its pipeline. This is what unlocks clustering.
+Nodes may already be up to date by the time the last node finishes
+deploying — the update gets pushed while `incusos-proxmox` deploys
+subsequent nodes sequentially.
+
+### 3.7 Verify Scrub Schedules
```bash
-incus cluster list lab-node-01:
+for node in oc-node-01 oc-node-02 oc-node-03; do
+ incus query ${node}:/os/1.0/system/storage | python3 -c \
+ "import sys,json; print('${node}:', json.load(sys.stdin)['config']['scrub_schedule'])"
+done
```
-Expected output:
+Actual output:
-| NAME | URL | ROLES | ARCHITECTURE | FAILURE DOMAIN | DESCRIPTION | STATE | MESSAGE |
-|-------------|----------------------------------|--------------------------|--------------|----------------|-------------|--------|-------------------|
-| lab-node-01 | https://192.168.102.111:8443 | database-leader, database | x86_64 | default | | ONLINE | Fully operational |
+```
+oc-node-01: 0 4 * * 0
+oc-node-02: 0 4 * * 0
+oc-node-03: 0 4 * * 0
+```
-### 4.4 Join Node-02
+All healthy. No crontab bug (force_reboot was not used).
-Generate a join token on the init node:
+---
+
+## Section 4: Form Cluster via Operations Center
+
+### 4.1 The needs_update Blocker
+
+OC requires all nodes to show `needs_update: false` before clustering.
+Nodes deployed from an ISO matching the latest version are tracked as
+`needs_update: true` because the OS was never delivered through OC's update
+pipeline. The `needs_update` flag is server-side computed and cannot be
+overridden via REST API PUT.
+
+**Solution**: deploy from an older ISO version (Section 3.3). OC then pushes
+the real update to nodes through its pipeline, clearing the flag.
+
+### 4.2 Form Cluster
+
+**Important**: if the client certificate was already injected via SEED_DATA,
+use an empty application seed config to avoid "Certificate already in trust
+store" Terraform errors:
```bash
-incus cluster add lab-node-01:lab-node-02
+echo '{}' > /tmp/oc-app-config.yaml
+
+operations-center provisioning cluster add oc-cluster \
+ https://192.168.102.140:8443 \
+ --server-names oc-node-01,oc-node-02,oc-node-03 \
+ --server-type incus \
+ --application-seed-config /tmp/oc-app-config.yaml
```
-This outputs a long base64 token. Use it immediately — tokens expire.
+OC orchestrates the full cluster formation:
+1. Sets `core.https_address` to each node's specific IP
+2. Enables clustering on oc-node-01
+3. Joins oc-node-02 and oc-node-03
+4. Creates storage pool (`local`), networks (`incusbr0`, `meshbr0`)
+5. Runs Terraform/OpenTofu for post-cluster configuration
-Join node-02 to the cluster (automated, non-interactive):
+### 4.3 Fix Remotes After Clustering
+
+Clustering regenerates TLS certificates. Re-add the remotes:
```bash
-printf '\n\nyes\nlocal/incus\nlocal/incus\n' | incus cluster join lab-node-01: lab-node-02:
+incus remote remove oc-node-01
+incus remote remove oc-node-02
+incus remote remove oc-node-03
+incus remote add oc-node-01 https://192.168.102.140:8443 --accept-certificate
+incus remote add oc-node-02 https://192.168.102.141:8443 --accept-certificate
+incus remote add oc-node-03 https://192.168.102.142:8443 --accept-certificate
```
-The five prompts answered by `printf`:
-1. IP address → accept default (node's IP)
-2. Member name → accept default (matches token)
-3. "All existing data is lost" → `yes`
-4. `source` for storage pool "local" → `local/incus`
-5. `zfs.pool_name` for pool "local" → `local/incus`
-
-No storage/network cleanup needed — `apply_defaults: false` means node-02
-has no pre-existing Incus storage pool or network.
-
-Fix the remote after join (new cluster cert):
+### 4.4 Verify Cluster
```bash
-incus remote remove lab-node-02
-incus remote add lab-node-02 https://192.168.102.112:8443 --accept-certificate
+incus cluster list oc-node-01:
```
-### 4.5 Join Node-03
+Actual output (key columns; full table includes FAILURE DOMAIN, DESCRIPTION):
-Same procedure:
+| NAME | URL | ROLES | ARCHITECTURE | STATUS | MESSAGE |
+|------|-----|-------|--------------|--------|---------|
+| oc-node-01 | https://192.168.102.140:8443 | database-leader, database | x86_64 | ONLINE | Fully operational |
+| oc-node-02 | https://192.168.102.141:8443 | database | x86_64 | ONLINE | Fully operational |
+| oc-node-03 | https://192.168.102.142:8443 | database | x86_64 | ONLINE | Fully operational |
+
+All 3 nodes ONLINE and Fully operational. The `ovn-chassis` role is added
+later in Section 6.4.
+
+### 4.5 Cluster Resources Created by OC
```bash
-incus cluster add lab-node-01:lab-node-03
-printf '\n\nyes\nlocal/incus\nlocal/incus\n' | incus cluster join lab-node-01: lab-node-03:
+incus storage list oc-node-01:
+incus network list oc-node-01:
```
-Fix the remote:
+Actual output (incusbr0 subnet varies per deployment):
-```bash
-incus remote remove lab-node-03
-incus remote add lab-node-03 https://192.168.102.113:8443 --accept-certificate
-```
+| NAME | DRIVER | DESCRIPTION | USED BY | STATE |
+|------|--------|-------------|---------|-------|
+| local | zfs | Local storage pool (on system drive) | 8 | CREATED |
-### 4.6 Verify Cluster
+| NAME | TYPE | MANAGED | IPV4 | DESCRIPTION | USED BY |
+|------|------|---------|------|-------------|---------|
+| incusbr0 | bridge | YES | 10.x.x.1/24 | Local network bridge (NAT) | 1 |
+| meshbr0 | bridge | YES | none | Internal mesh network bridge | 1 |
-```bash
-incus cluster list lab-node-01:
-```
-
-Expected output — 3 nodes, all ONLINE:
-
-| NAME | URL | ROLES | ARCHITECTURE | FAILURE DOMAIN | DESCRIPTION | STATE | MESSAGE |
-|-------------|----------------------------------|--------------------------|--------------|----------------|-------------|--------|-------------------|
-| lab-node-01 | https://192.168.102.111:8443 | database-leader, database | x86_64 | default | | ONLINE | Fully operational |
-| lab-node-02 | https://192.168.102.112:8443 | database | x86_64 | default | | ONLINE | Fully operational |
-| lab-node-03 | https://192.168.102.113:8443 | database | x86_64 | default | | ONLINE | Fully operational |
-
-Verify storage pool exists on all members:
-
-```bash
-incus storage show lab-node-01:local
-incus storage show lab-node-01:local --target lab-node-02
-incus storage show lab-node-01:local --target lab-node-03
-```
-
-Verify the default network:
-
-```bash
-incus network list lab-node-01:
-```
+OC creates: `local` storage pool (ZFS), `incusbr0` bridge (NAT), and
+`meshbr0` (OC-specific mesh network for inter-node communication). The
+table also includes IPv6 and STATE columns.
+---
## Section 5: Bridge Networking Baseline
@@ -512,24 +639,30 @@ node-local. This demonstrates why OVN is needed.
Launch 2 containers on the same node. **Important**: use `--target` to force
placement — without it, the cluster scheduler may place containers on
-different nodes automatically:
+different nodes automatically.
+
+**Important**: launch containers **one at a time**, not chained with `&&`.
+The first launch on a fresh cluster downloads the image (~1 GB), which
+takes 2-3 minutes. Subsequent launches on the same node use the cached
+image and are instant. Launches targeting a different node trigger another
+image transfer to that node.
```bash
-incus launch images:debian/12 lab-node-01:test-bridge-a --target lab-node-01
-incus launch images:debian/12 lab-node-01:test-bridge-b --target lab-node-01
+incus launch images:debian/12 oc-node-01:test-bridge-a --target oc-node-01
+incus launch images:debian/12 oc-node-01:test-bridge-b --target oc-node-01
```
Wait for them to get IPs:
```bash
-incus list lab-node-01: --columns ns4 --format csv | grep test-bridge
+incus list oc-node-01: --columns ns4 --format csv | grep test-bridge
```
Ping between them:
```bash
-IP_B=$(incus list lab-node-01:test-bridge-b --columns 4 --format csv | cut -d' ' -f1)
-incus exec lab-node-01:test-bridge-a -- ping -c 3 "$IP_B"
+IP_B=$(incus list oc-node-01:test-bridge-b --columns 4 --format csv | cut -d' ' -f1)
+incus exec oc-node-01:test-bridge-a -- ping -c 3 "$IP_B"
```
Actual result: 0% packet loss, ~0.024ms latency. Same bridge, same node — works.
@@ -539,20 +672,20 @@ Actual result: 0% packet loss, ~0.024ms latency. Same bridge, same node — work
Launch a container on a different node:
```bash
-incus launch images:debian/12 lab-node-01:test-bridge-c --target lab-node-02
+incus launch images:debian/12 oc-node-01:test-bridge-c --target oc-node-02
```
Wait for IP:
```bash
-incus list lab-node-01: --columns ns4 --format csv | grep test-bridge
+incus list oc-node-01: --columns ns4 --format csv | grep test-bridge
```
Ping from node-01 to node-02:
```bash
-IP_C=$(incus list lab-node-01:test-bridge-c --columns 4 --format csv | cut -d' ' -f1)
-incus exec lab-node-01:test-bridge-a -- ping -c 3 -W 2 "$IP_C"
+IP_C=$(incus list oc-node-01:test-bridge-c --columns 4 --format csv | cut -d' ' -f1)
+incus exec oc-node-01:test-bridge-a -- ping -c 3 -W 2 "$IP_C"
```
Actual result: **100% packet loss**. Bridge networks are node-local — there is
@@ -565,7 +698,7 @@ separate L2 domains.
NAT to the internet works from any node:
```bash
-incus exec lab-node-01:test-bridge-a -- ping -c 3 1.1.1.1
+incus exec oc-node-01:test-bridge-a -- ping -c 3 1.1.1.1
```
Actual result: 0% packet loss, ~10ms latency. Each bridge provides NAT via
@@ -574,9 +707,9 @@ the host's management interface.
### Cleanup
```bash
-incus delete lab-node-01:test-bridge-a --force
-incus delete lab-node-01:test-bridge-b --force
-incus delete lab-node-01:test-bridge-c --force
+incus delete oc-node-01:test-bridge-a --force
+incus delete oc-node-01:test-bridge-b --force
+incus delete oc-node-01:test-bridge-c --force
```
@@ -590,85 +723,50 @@ section, containers on any node can communicate transparently.
Launch a Debian container on node-01 to host the OVN central services:
```bash
-incus launch images:debian/12 lab-node-01:ovn-central --target lab-node-01
+incus launch images:debian/12 oc-node-01:ovn-central --target oc-node-01
```
Install OVN:
```bash
-incus exec lab-node-01:ovn-central -- apt-get update
-incus exec lab-node-01:ovn-central -- apt-get install -y ovn-central ovn-host
+incus exec oc-node-01:ovn-central -- bash -c \
+ "apt-get update -qq && apt-get install -y -qq ovn-central"
```
Configure OVN to listen on all interfaces:
```bash
-incus exec lab-node-01:ovn-central -- ovn-nbctl set-connection ptcp:6641:0.0.0.0
-incus exec lab-node-01:ovn-central -- ovn-sbctl set-connection ptcp:6642:0.0.0.0
+incus exec oc-node-01:ovn-central -- ovn-nbctl set-connection ptcp:6641:0.0.0.0
+incus exec oc-node-01:ovn-central -- ovn-sbctl set-connection ptcp:6642:0.0.0.0
```
-Add proxy devices to expose NB and SB ports on the host's LAN IP:
+Add proxy devices to expose NB (6641) and SB (6642) on the host's LAN IP:
```bash
-incus config device add lab-node-01:ovn-central nb-proxy proxy \
- listen=tcp:192.168.102.111:6641 connect=tcp:127.0.0.1:6641
-incus config device add lab-node-01:ovn-central sb-proxy proxy \
- listen=tcp:192.168.102.111:6642 connect=tcp:127.0.0.1:6642
-```
-
-Verify the ports are reachable:
-
-```bash
-curl -s --connect-timeout 2 telnet://192.168.102.111:6641 || echo "NB port open"
-curl -s --connect-timeout 2 telnet://192.168.102.111:6642 || echo "SB port open"
+incus config device add oc-node-01:ovn-central \
+ nb-proxy proxy listen=tcp:192.168.102.140:6641 connect=tcp:127.0.0.1:6641
+incus config device add oc-node-01:ovn-central \
+ sb-proxy proxy listen=tcp:192.168.102.140:6642 connect=tcp:127.0.0.1:6642
```
### 6.2 Enable OVN on All IncusOS Nodes
-OVN services are disabled by default on IncusOS. Enable them on every node
-via the IncusOS REST API. The `database` field points to the **southbound**
-DB (port 6642, not 6641).
-
-**Node-01:**
+Enable OVN services via the IncusOS REST API (`/os/1.0/services/ovn`).
+The `database` field is the **southbound** DB (port 6642), not northbound.
```bash
-incus query lab-node-01:/os/1.0/services/ovn --request PUT --data '{
- "config": {
- "database": "tcp:192.168.102.111:6642",
- "enabled": true,
- "tunnel_address": "192.168.102.111",
- "tunnel_protocol": "geneve"
- },
- "state": {}
-}'
-```
-
-**Node-02:**
-
-```bash
-incus query lab-node-02:/os/1.0/services/ovn --request PUT --data '{
- "config": {
- "database": "tcp:192.168.102.111:6642",
- "enabled": true,
- "tunnel_address": "192.168.102.112",
- "tunnel_protocol": "geneve"
- },
- "state": {}
-}'
-```
-
-**Node-03:**
-
-```bash
-incus query lab-node-03:/os/1.0/services/ovn --request PUT --data '{
- "config": {
- "database": "tcp:192.168.102.111:6642",
- "enabled": true,
- "tunnel_address": "192.168.102.113",
- "tunnel_protocol": "geneve"
- },
- "state": {}
-}'
+for node_ip in 192.168.102.140 192.168.102.141 192.168.102.142; do
+ remote="oc-node-$(echo $node_ip | cut -d. -f4 | sed 's/140/01/;s/141/02/;s/142/03/')"
+ incus query ${remote}:/os/1.0/services/ovn --request PUT --data "{
+ \"config\": {
+ \"database\": \"tcp:192.168.102.140:6642\",
+ \"enabled\": true,
+ \"tunnel_address\": \"${node_ip}\",
+ \"tunnel_protocol\": \"geneve\"
+ },
+ \"state\": {}
+ }"
+done
```
Each call should return `{}` on success.
@@ -678,7 +776,7 @@ Each call should return `{}` on success.
Point Incus to the **northbound** DB (port 6641):
```bash
-incus config set lab-node-01: network.ovn.northbound_connection tcp:192.168.102.111:6641
+incus config set oc-node-01: network.ovn.northbound_connection tcp:192.168.102.140:6641
```
### 6.4 Assign OVN Chassis Role
@@ -686,15 +784,15 @@ incus config set lab-node-01: network.ovn.northbound_connection tcp:192.168.102.
Every node that will host OVN workloads needs the `ovn-chassis` role:
```bash
-incus cluster role add lab-node-01:lab-node-01 ovn-chassis
-incus cluster role add lab-node-01:lab-node-02 ovn-chassis
-incus cluster role add lab-node-01:lab-node-03 ovn-chassis
+for node in oc-node-01 oc-node-02 oc-node-03; do
+ incus cluster role add oc-node-01:${node} ovn-chassis
+done
```
Verify:
```bash
-incus cluster list lab-node-01:
+incus cluster list oc-node-01:
```
The ROLES column should now include `ovn-chassis` for each member.
@@ -705,27 +803,17 @@ The UPLINK network provides the bridge between OVN virtual networks and
the physical LAN. It uses the two-step cluster pattern: per-member
`--target` first, then cluster-wide create.
-**Per-member configuration** (one per node):
-
**Important**: IncusOS names its management NIC `mgmt`, NOT `ens18`. Using
-`parent=ens18` will fail with "Parent interface 'ens18' not found". Verify
-with: `incus query lab-node-01:/os/1.0/system/network`
+`parent=ens18` will fail with "Parent interface 'ens18' not found".
```bash
-incus network create lab-node-01:UPLINK --type physical --target lab-node-01 \
- parent=mgmt
+# Per-target (parent is member-specific)
+for node in oc-node-01 oc-node-02 oc-node-03; do
+ incus network create oc-node-01:UPLINK --type=physical --target=${node} parent=mgmt
+done
-incus network create lab-node-01:UPLINK --type physical --target lab-node-02 \
- parent=mgmt
-
-incus network create lab-node-01:UPLINK --type physical --target lab-node-03 \
- parent=mgmt
-```
-
-**Cluster-wide create** with shared settings:
-
-```bash
-incus network create lab-node-01:UPLINK --type physical \
+# Cluster-wide config
+incus network create oc-node-01:UPLINK --type=physical \
ipv4.ovn.ranges=192.168.103.200-192.168.103.210 \
ipv4.gateway=192.168.100.1/22 \
dns.nameservers=192.168.100.1
@@ -734,67 +822,61 @@ incus network create lab-node-01:UPLINK --type physical \
### 6.6 Create OVN Network (net-prod)
```bash
-incus network create lab-node-01:net-prod --type=ovn network=UPLINK \
- ipv4.address=10.10.10.1/24 \
- ipv4.nat=true \
- ipv6.address=none
+incus network create oc-node-01:net-prod --type=ovn \
+ network=UPLINK ipv4.address=10.10.10.1/24 ipv4.nat=true
```
-Verify:
+Actual output:
-```bash
-incus network list lab-node-01:
+```
+Network net-prod created
```
-Should show both `incusbr0` (bridge, per-node) and `net-prod` (ovn, cluster-wide).
+net-prod is assigned external IP `192.168.103.200` from the UPLINK range.
### 6.7 Verify Cross-Node OVN Connectivity
-Launch containers on different nodes, attached to net-prod:
-
```bash
-incus launch images:debian/12 lab-node-01:test-ovn-a --network net-prod --target lab-node-01
-incus launch images:debian/12 lab-node-01:test-ovn-b --network net-prod --target lab-node-02
-incus launch images:debian/12 lab-node-01:test-ovn-c --network net-prod --target lab-node-03
+incus launch images:debian/12 oc-node-01:test-1 --target oc-node-01 -n net-prod
+incus launch images:debian/12 oc-node-01:test-2 --target oc-node-02 -n net-prod
+incus exec oc-node-01:test-1 -- ping -c 3 10.10.10.3
```
-Wait for IPs and list:
+Actual output:
-```bash
-incus list lab-node-01: --columns nst4 --format csv | grep test-ovn
+```
+64 bytes from 10.10.10.3: icmp_seq=1 ttl=64 time=0.669 ms
+64 bytes from 10.10.10.3: icmp_seq=2 ttl=64 time=0.136 ms
+64 bytes from 10.10.10.3: icmp_seq=3 ttl=64 time=0.194 ms
```
-Cross-node ping (node-01 → node-02):
+Sub-millisecond cross-node latency via Geneve tunnels. Clean up test
+containers after verification:
```bash
-IP_B=$(incus list lab-node-01:test-ovn-b --columns 4 --format csv | cut -d' ' -f1)
-incus exec lab-node-01:test-ovn-a -- ping -c 3 "$IP_B"
+incus delete oc-node-01:test-1 oc-node-01:test-2 --force
```
-Actual result: 0% packet loss, ~0.09-0.8ms latency. OVN provides transparent
-L2 connectivity via Geneve tunnels.
-
-Cross-node ping (node-01 → node-03):
+### 6.8 Final Network State
```bash
-IP_C=$(incus list lab-node-01:test-ovn-c --columns 4 --format csv | cut -d' ' -f1)
-incus exec lab-node-01:test-ovn-a -- ping -c 3 "$IP_C"
+incus network list oc-node-01:
```
-Internet access through OVN:
+Actual output:
-```bash
-incus exec lab-node-01:test-ovn-a -- ping -c 3 1.1.1.1
-```
+| NAME | TYPE | MANAGED | IPV4 | DESCRIPTION | USED BY |
+|------|------|---------|------|-------------|---------|
+| UPLINK | physical | YES | | | 1 |
+| incusbr0 | bridge | YES | 10.x.x.1/24 | Local network bridge (NAT) | 2 |
+| meshbr0 | bridge | YES | none | Internal mesh network bridge | 1 |
+| net-prod | ovn | YES | 10.10.10.1/24 | | 0 |
-Clean up test containers:
-
-```bash
-incus delete lab-node-01:test-ovn-a --force
-incus delete lab-node-01:test-ovn-b --force
-incus delete lab-node-01:test-ovn-c --force
-```
+The incusbr0 subnet is randomly assigned per deployment. The USED BY count
+for net-prod is 0 at this point (test containers deleted); it increases as
+workloads are added in subsequent sections.
+---
## Section 7: Mixed Workloads
@@ -803,36 +885,35 @@ VMs configured for live migration.
### 7.1 Containers on net-prod
-Deploy containers with targeted placement across nodes:
+Deploy containers with targeted placement across nodes. Run each launch
+command **one at a time** — each new target node needs to download the
+image from the cluster (~1 GB transfer, 2-3 minutes per node):
```bash
-# Web servers
-incus launch images:debian/12 lab-node-01:prod-web-01 --network net-prod --target lab-node-01
-incus launch images:debian/12 lab-node-01:prod-web-02 --network net-prod --target lab-node-02
-
-# Application container
-incus launch images:debian/12 lab-node-01:prod-api-01 --network net-prod --target lab-node-03
+incus launch images:debian/12 oc-node-01:prod-web-01 --network net-prod --target oc-node-01
+incus launch images:debian/12 oc-node-01:prod-web-02 --network net-prod --target oc-node-02
+incus launch images:debian/12 oc-node-01:prod-api-01 --network net-prod --target oc-node-03
```
Install nginx on the web servers:
```bash
-incus exec lab-node-01:prod-web-01 -- bash -c "apt-get update && apt-get install -y nginx"
-incus exec lab-node-01:prod-web-02 -- bash -c "apt-get update && apt-get install -y nginx"
+incus exec oc-node-01:prod-web-01 -- bash -c "apt-get update && apt-get install -y nginx"
+incus exec oc-node-01:prod-web-02 -- bash -c "apt-get update && apt-get install -y nginx"
```
Set distinct content to verify load balancing later:
```bash
-incus exec lab-node-01:prod-web-01 -- bash -c "echo 'Server: prod-web-01' > /var/www/html/index.html"
-incus exec lab-node-01:prod-web-02 -- bash -c "echo 'Server: prod-web-02' > /var/www/html/index.html"
+incus exec oc-node-01:prod-web-01 -- bash -c "echo 'Server: prod-web-01' > /var/www/html/index.html"
+incus exec oc-node-01:prod-web-02 -- bash -c "echo 'Server: prod-web-02' > /var/www/html/index.html"
```
Install nginx on the API container:
```bash
-incus exec lab-node-01:prod-api-01 -- bash -c "apt-get update && apt-get install -y nginx"
-incus exec lab-node-01:prod-api-01 -- bash -c "echo 'API: prod-api-01' > /var/www/html/index.html"
+incus exec oc-node-01:prod-api-01 -- bash -c "apt-get update && apt-get install -y nginx"
+incus exec oc-node-01:prod-api-01 -- bash -c "echo 'API: prod-api-01' > /var/www/html/index.html"
```
### 7.2 VMs (Migration-Ready)
@@ -840,24 +921,24 @@ incus exec lab-node-01:prod-api-01 -- bash -c "echo 'API: prod-api-01' > /var/ww
Deploy VMs with live migration configuration:
```bash
-incus launch images:debian/12 lab-node-01:prod-db-01 --vm --network net-prod --target lab-node-01
-incus launch images:debian/12 lab-node-01:prod-app-01 --vm --network net-prod --target lab-node-02
+incus launch images:debian/12 oc-node-01:prod-db-01 --vm --network net-prod --target oc-node-01
+incus launch images:debian/12 oc-node-01:prod-app-01 --vm --network net-prod --target oc-node-02
```
VMs may take longer to boot than containers (~30-60s for image download +
boot). If the VMs show as STOPPED, start them explicitly:
```bash
-incus start lab-node-01:prod-db-01
-incus start lab-node-01:prod-app-01
+incus start oc-node-01:prod-db-01
+incus start oc-node-01:prod-app-01
```
Wait for the VM agent to become available, then verify:
```bash
# Check VM agent is running
-incus exec lab-node-01:prod-db-01 -- uname -a
-incus exec lab-node-01:prod-app-01 -- uname -a
+incus exec oc-node-01:prod-db-01 -- uname -a
+incus exec oc-node-01:prod-app-01 -- uname -a
```
### 7.3 Configure VMs for Live Migration
@@ -870,29 +951,33 @@ ICH9LPC`.
Stop VMs before configuring `migration.stateful`:
```bash
-incus stop lab-node-01:prod-db-01
-incus stop lab-node-01:prod-app-01
+incus stop oc-node-01:prod-db-01
+incus stop oc-node-01:prod-app-01
```
Configure migration settings:
```bash
# prod-db-01
-incus config set lab-node-01:prod-db-01 limits.cpu=0-1
-incus config set lab-node-01:prod-db-01 migration.stateful=true
-incus config device set lab-node-01:prod-db-01 root size.state=2GiB
+incus config set oc-node-01:prod-db-01 limits.cpu=0-1
+incus config set oc-node-01:prod-db-01 migration.stateful=true
+incus config device override oc-node-01:prod-db-01 root size.state=2GiB
# prod-app-01
-incus config set lab-node-01:prod-app-01 limits.cpu=0-1
-incus config set lab-node-01:prod-app-01 migration.stateful=true
-incus config device set lab-node-01:prod-app-01 root size.state=2GiB
+incus config set oc-node-01:prod-app-01 limits.cpu=0-1
+incus config set oc-node-01:prod-app-01 migration.stateful=true
+incus config device override oc-node-01:prod-app-01 root size.state=2GiB
```
+**Important**: use `device override` (not `device set`) because the `root`
+device comes from the default profile. `device set` fails with "Device from
+profile(s) cannot be modified for individual instance".
+
Start the VMs:
```bash
-incus start lab-node-01:prod-db-01
-incus start lab-node-01:prod-app-01
+incus start oc-node-01:prod-db-01
+incus start oc-node-01:prod-app-01
```
### 7.4 Workload Distribution
@@ -900,19 +985,19 @@ incus start lab-node-01:prod-app-01
View the full workload distribution:
```bash
-incus list lab-node-01: --columns nstL4 --format table
+incus list oc-node-01: --columns nstL4 --format table
```
Expected layout:
| NAME | STATE | TYPE | LOCATION | IPV4 |
|-------------|---------|-----------------|-------------|-----------------------|
-| ovn-central | RUNNING | CONTAINER | lab-node-01 | ... |
-| prod-web-01 | RUNNING | CONTAINER | lab-node-01 | 10.10.10.x (net-prod) |
-| prod-db-01 | RUNNING | VIRTUAL-MACHINE | lab-node-01 | 10.10.10.x (net-prod) |
-| prod-web-02 | RUNNING | CONTAINER | lab-node-02 | 10.10.10.x (net-prod) |
-| prod-app-01 | RUNNING | VIRTUAL-MACHINE | lab-node-02 | 10.10.10.x (net-prod) |
-| prod-api-01 | RUNNING | CONTAINER | lab-node-03 | 10.10.10.x (net-prod) |
+| ovn-central | RUNNING | CONTAINER | oc-node-01 | ... |
+| prod-web-01 | RUNNING | CONTAINER | oc-node-01 | 10.10.10.x (net-prod) |
+| prod-db-01 | RUNNING | VIRTUAL-MACHINE | oc-node-01 | 10.10.10.x (net-prod) |
+| prod-web-02 | RUNNING | CONTAINER | oc-node-02 | 10.10.10.x (net-prod) |
+| prod-app-01 | RUNNING | VIRTUAL-MACHINE | oc-node-02 | 10.10.10.x (net-prod) |
+| prod-api-01 | RUNNING | CONTAINER | oc-node-03 | 10.10.10.x (net-prod) |
## Section 8: Network Isolation & Security
@@ -920,7 +1005,7 @@ Expected layout:
### 8.1 Create Isolated Network
```bash
-incus network create lab-node-01:net-isolated --type=ovn network=UPLINK \
+incus network create oc-node-01:net-isolated --type=ovn network=UPLINK \
ipv4.address=10.10.20.1/24 \
ipv4.nat=true \
ipv6.address=none
@@ -929,8 +1014,8 @@ incus network create lab-node-01:net-isolated --type=ovn network=UPLINK \
### 8.2 Launch Isolated Containers
```bash
-incus launch images:debian/12 lab-node-01:iso-app-01 --network net-isolated --target lab-node-01
-incus launch images:debian/12 lab-node-01:iso-app-02 --network net-isolated --target lab-node-02
+incus launch images:debian/12 oc-node-01:iso-app-01 --network net-isolated --target oc-node-01
+incus launch images:debian/12 oc-node-01:iso-app-02 --network net-isolated --target oc-node-02
```
### 8.3 Verify Network Isolation
@@ -938,8 +1023,8 @@ incus launch images:debian/12 lab-node-01:iso-app-02 --network net-isolated --ta
Containers on net-isolated can reach each other:
```bash
-IP_ISO2=$(incus list lab-node-01:iso-app-02 --columns 4 --format csv | cut -d' ' -f1)
-incus exec lab-node-01:iso-app-01 -- ping -c 3 "$IP_ISO2"
+IP_ISO2=$(incus list oc-node-01:iso-app-02 --columns 4 --format csv | cut -d' ' -f1)
+incus exec oc-node-01:iso-app-01 -- ping -c 3 "$IP_ISO2"
```
Actual result: 0% packet loss, ~0.15-0.5ms latency. Containers on the
@@ -948,7 +1033,7 @@ same OVN network can reach each other across nodes.
But net-prod **cannot** reach net-isolated:
```bash
-incus exec lab-node-01:prod-web-01 -- ping -c 3 -W 2 "$IP_ISO2"
+incus exec oc-node-01:prod-web-01 -- ping -c 3 -W 2 "$IP_ISO2"
```
Actual result: **100% packet loss**. Different OVN networks are fully
@@ -959,8 +1044,8 @@ isolated — separate L2 domains, no routing between them.
Create an ACL that blocks ICMP from a specific source:
```bash
-incus network acl create lab-node-01:block-ping
-incus network acl rule add lab-node-01:block-ping ingress \
+incus network acl create oc-node-01:block-ping
+incus network acl rule add oc-node-01:block-ping ingress \
action=drop protocol=icmp4 \
source=10.10.10.0/24 \
description="Block ICMP from net-prod subnet"
@@ -971,26 +1056,26 @@ incus network acl rule add lab-node-01:block-ping ingress \
Apply the ACL to net-isolated:
```bash
-incus network set lab-node-01:net-isolated security.acls=block-ping
+incus network set oc-node-01:net-isolated security.acls=block-ping
```
Verify ICMP is blocked between net-isolated containers (since they match
the source range — adjust the ACL source for targeted blocking):
```bash
-incus exec lab-node-01:iso-app-01 -- ping -c 3 -W 2 "$IP_ISO2"
+incus exec oc-node-01:iso-app-01 -- ping -c 3 -W 2 "$IP_ISO2"
```
Remove the ACL:
```bash
-incus network unset lab-node-01:net-isolated security.acls
+incus network unset oc-node-01:net-isolated security.acls
```
Verify ICMP works again:
```bash
-incus exec lab-node-01:iso-app-01 -- ping -c 3 "$IP_ISO2"
+incus exec oc-node-01:iso-app-01 -- ping -c 3 "$IP_ISO2"
```
### 8.6 Network Peering
@@ -1000,11 +1085,11 @@ communicate. Peering is bilateral — create a peer on both sides:
```bash
# From net-prod's perspective
-incus network peer create lab-node-01:net-prod peer-to-isolated net-isolated \
+incus network peer create oc-node-01:net-prod peer-to-isolated net-isolated \
--description "Peer to isolated network"
# From net-isolated's perspective
-incus network peer create lab-node-01:net-isolated peer-to-prod net-prod \
+incus network peer create oc-node-01:net-isolated peer-to-prod net-prod \
--description "Peer to production network"
```
@@ -1013,7 +1098,7 @@ incus network peer create lab-node-01:net-isolated peer-to-prod net-prod \
Cross-network ping (prod → isolated):
```bash
-incus exec lab-node-01:prod-web-01 -- ping -c 3 "$IP_ISO2"
+incus exec oc-node-01:prod-web-01 -- ping -c 3 "$IP_ISO2"
```
Actual result: 0% packet loss with TTL=62 (64 - 2 router hops),
@@ -1022,21 +1107,21 @@ confirming traffic traverses the OVN routers on both sides of the peering.
Cross-network ping (isolated → prod):
```bash
-IP_WEB1=$(incus list lab-node-01:prod-web-01 --columns 4 --format csv | cut -d' ' -f1)
-incus exec lab-node-01:iso-app-01 -- ping -c 3 "$IP_WEB1"
+IP_WEB1=$(incus list oc-node-01:prod-web-01 --columns 4 --format csv | cut -d' ' -f1)
+incus exec oc-node-01:iso-app-01 -- ping -c 3 "$IP_WEB1"
```
### 8.8 Remove Peering
```bash
-incus network peer delete lab-node-01:net-prod peer-to-isolated
-incus network peer delete lab-node-01:net-isolated peer-to-prod
+incus network peer delete oc-node-01:net-prod peer-to-isolated
+incus network peer delete oc-node-01:net-isolated peer-to-prod
```
Verify isolation is restored:
```bash
-incus exec lab-node-01:prod-web-01 -- ping -c 3 -W 2 "$IP_ISO2"
+incus exec oc-node-01:prod-web-01 -- ping -c 3 -W 2 "$IP_ISO2"
```
Expected: 100% packet loss. Networks are isolated again.
@@ -1044,8 +1129,8 @@ Expected: 100% packet loss. Networks are isolated again.
Clean up isolated containers:
```bash
-incus delete lab-node-01:iso-app-01 --force
-incus delete lab-node-01:iso-app-02 --force
+incus delete oc-node-01:iso-app-01 --force
+incus delete oc-node-01:iso-app-02 --force
```
@@ -1056,31 +1141,31 @@ incus delete lab-node-01:iso-app-02 --force
Create a load balancer with a VIP from the UPLINK range:
```bash
-incus network load-balancer create lab-node-01:net-prod 192.168.103.200
+incus network load-balancer create oc-node-01:net-prod 192.168.103.200
```
Add backend servers. **Important**: backends require the instance's **IP
address**, not its name. Get the IPs first:
```bash
-WEB1_IP=$(incus list lab-node-01:prod-web-01 --columns 4 --format csv | cut -d' ' -f1)
-WEB2_IP=$(incus list lab-node-01:prod-web-02 --columns 4 --format csv | cut -d' ' -f1)
+WEB1_IP=$(incus list oc-node-01:prod-web-01 --columns 4 --format csv | cut -d' ' -f1)
+WEB2_IP=$(incus list oc-node-01:prod-web-02 --columns 4 --format csv | cut -d' ' -f1)
echo "prod-web-01: $WEB1_IP, prod-web-02: $WEB2_IP"
```
Add backends using IP addresses:
```bash
-incus network load-balancer backend add lab-node-01:net-prod 192.168.103.200 \
+incus network load-balancer backend add oc-node-01:net-prod 192.168.103.200 \
web-01 "$WEB1_IP" 80
-incus network load-balancer backend add lab-node-01:net-prod 192.168.103.200 \
+incus network load-balancer backend add oc-node-01:net-prod 192.168.103.200 \
web-02 "$WEB2_IP" 80
```
Add a port mapping:
```bash
-incus network load-balancer port add lab-node-01:net-prod 192.168.103.200 \
+incus network load-balancer port add oc-node-01:net-prod 192.168.103.200 \
tcp 80 web-01,web-02
```
@@ -1116,10 +1201,10 @@ Network forwards expose internal services on LAN IPs. Forward
tcp:8080 → prod-api-01:80. **Like LB backends, forwards require IP addresses**:
```bash
-API_IP=$(incus list lab-node-01:prod-api-01 --columns 4 --format csv | cut -d' ' -f1)
+API_IP=$(incus list oc-node-01:prod-api-01 --columns 4 --format csv | cut -d' ' -f1)
-incus network forward create lab-node-01:net-prod 192.168.103.201
-incus network forward port add lab-node-01:net-prod 192.168.103.201 \
+incus network forward create oc-node-01:net-prod 192.168.103.201
+incus network forward port add oc-node-01:net-prod 192.168.103.201 \
tcp 8080 "$API_IP" 80
```
@@ -1137,7 +1222,7 @@ OVN provides per-network DNS. Containers can resolve each other by
hostname:
```bash
-incus exec lab-node-01:prod-web-01 -- bash -c "apt-get install -y dnsutils && dig +short prod-web-02.incus"
+incus exec oc-node-01:prod-web-01 -- bash -c "apt-get install -y dnsutils && dig +short prod-web-02.incus"
```
Actual output: `10.10.10.3` — OVN DNS resolves instance names within
@@ -1153,9 +1238,9 @@ Check that VMs have the required configuration:
```bash
for vm in prod-db-01 prod-app-01; do
echo "=== $vm ==="
- incus config get lab-node-01:$vm limits.cpu
- incus config get lab-node-01:$vm migration.stateful
- incus config device get lab-node-01:$vm root size.state
+ incus config get oc-node-01:$vm limits.cpu
+ incus config get oc-node-01:$vm migration.stateful
+ incus config device get oc-node-01:$vm root size.state
done
```
@@ -1167,7 +1252,7 @@ Create a simple counter in prod-db-01 to verify state continuity across
migration:
```bash
-incus exec lab-node-01:prod-db-01 -- bash -c '
+incus exec oc-node-01:prod-db-01 -- bash -c '
mkdir -p /tmp/heartbeat
nohup bash -c "i=0; while true; do echo \$i > /tmp/heartbeat/counter; i=\$((i+1)); sleep 1; done" \
> /dev/null 2>&1 &
@@ -1178,7 +1263,7 @@ incus exec lab-node-01:prod-db-01 -- bash -c '
Read the counter:
```bash
-incus exec lab-node-01:prod-db-01 -- cat /tmp/heartbeat/counter
+incus exec oc-node-01:prod-db-01 -- cat /tmp/heartbeat/counter
```
Note the value. After migration, the counter should continue from where
@@ -1189,20 +1274,20 @@ it left off (live migration preserves running state).
Check current location:
```bash
-incus list lab-node-01:prod-db-01 --columns nL --format csv
+incus list oc-node-01:prod-db-01 --columns nL --format csv
```
**Migrate node-01 → node-02:**
```bash
-time incus move lab-node-01:prod-db-01 --target lab-node-02
+time incus move oc-node-01:prod-db-01 --target oc-node-02
```
Actual result: 7.347s (~140 MB/s). Wait for the VM agent to reconnect:
```bash
sleep 4
-incus exec lab-node-01:prod-db-01 -- cat /tmp/heartbeat/counter
+incus exec oc-node-01:prod-db-01 -- cat /tmp/heartbeat/counter
```
Counter went from 9 → 25. The heartbeat process was never interrupted —
@@ -1211,9 +1296,9 @@ it continued counting during migration.
**Migrate node-02 → node-03:**
```bash
-time incus move lab-node-01:prod-db-01 --target lab-node-03
+time incus move oc-node-01:prod-db-01 --target oc-node-03
sleep 4
-incus exec lab-node-01:prod-db-01 -- cat /tmp/heartbeat/counter
+incus exec oc-node-01:prod-db-01 -- cat /tmp/heartbeat/counter
```
Actual result: 7.379s. Counter went to 41.
@@ -1221,9 +1306,9 @@ Actual result: 7.379s. Counter went to 41.
**Migrate node-03 → node-01 (back to origin):**
```bash
-time incus move lab-node-01:prod-db-01 --target lab-node-01
+time incus move oc-node-01:prod-db-01 --target oc-node-01
sleep 4
-incus exec lab-node-01:prod-db-01 -- cat /tmp/heartbeat/counter
+incus exec oc-node-01:prod-db-01 -- cat /tmp/heartbeat/counter
```
Actual result: 6.896s. Counter went to 56.
@@ -1231,7 +1316,7 @@ Actual result: 6.896s. Counter went to 56.
Verify the VM is back on node-01:
```bash
-incus list lab-node-01:prod-db-01 --columns nL --format csv
+incus list oc-node-01:prod-db-01 --columns nL --format csv
```
### 10.4 Active I/O During Migration
@@ -1239,7 +1324,7 @@ incus list lab-node-01:prod-db-01 --columns nL --format csv
Start a continuous write inside the VM:
```bash
-incus exec lab-node-01:prod-db-01 -- bash -c '
+incus exec oc-node-01:prod-db-01 -- bash -c '
dd if=/dev/urandom of=/tmp/testfile bs=1M count=100 &
echo "Write started, PID: $!"
'
@@ -1248,21 +1333,21 @@ incus exec lab-node-01:prod-db-01 -- bash -c '
Migrate while I/O is active:
```bash
-time incus move lab-node-01:prod-db-01 --target lab-node-02
+time incus move oc-node-01:prod-db-01 --target oc-node-02
sleep 4
```
Verify the file exists and is intact:
```bash
-incus exec lab-node-01:prod-db-01 -- ls -la /tmp/testfile
-incus exec lab-node-01:prod-db-01 -- md5sum /tmp/testfile
+incus exec oc-node-01:prod-db-01 -- ls -la /tmp/testfile
+incus exec oc-node-01:prod-db-01 -- md5sum /tmp/testfile
```
Move back:
```bash
-incus move lab-node-01:prod-db-01 --target lab-node-01
+incus move oc-node-01:prod-db-01 --target oc-node-01
sleep 4
```
@@ -1273,7 +1358,7 @@ where it was:
```bash
# Note the heartbeat counter
-incus exec lab-node-01:prod-app-01 -- bash -c '
+incus exec oc-node-01:prod-app-01 -- bash -c '
mkdir -p /tmp/heartbeat
echo 42 > /tmp/heartbeat/counter
cat /tmp/heartbeat/counter
@@ -1283,15 +1368,15 @@ incus exec lab-node-01:prod-app-01 -- bash -c '
Stateful stop:
```bash
-incus stop lab-node-01:prod-app-01 --stateful
+incus stop oc-node-01:prod-app-01 --stateful
```
Start (resumes from saved state):
```bash
-incus start lab-node-01:prod-app-01
+incus start oc-node-01:prod-app-01
sleep 4
-incus exec lab-node-01:prod-app-01 -- cat /tmp/heartbeat/counter
+incus exec oc-node-01:prod-app-01 -- cat /tmp/heartbeat/counter
```
Expected: `42` — the file (and entire VM state) is preserved.
@@ -1300,7 +1385,7 @@ Expected: `42` — the file (and entire VM state) is preserved.
saved state:
```bash
-incus start lab-node-01:prod-app-01 --stateless
+incus start oc-node-01:prod-app-01 --stateless
```
@@ -1311,43 +1396,43 @@ incus start lab-node-01:prod-app-01 --stateless
Evacuate node-02. All workloads are moved to other nodes:
```bash
-incus cluster evacuate lab-node-01:lab-node-02 --force
+incus cluster evacuate oc-node-01:oc-node-02 --force
```
Check workload distribution — nothing on node-02:
```bash
-incus list lab-node-01: --columns nstL --format table
+incus list oc-node-01: --columns nstL --format table
```
Actual behavior: VMs with `migration.stateful=true` are live-migrated
-(prod-app-01 migrated to lab-node-03). Containers are stopped and moved
-(prod-web-02 stopped, moved to lab-node-03, then started). The `--force`
+(prod-app-01 migrated to oc-node-03). Containers are stopped and moved
+(prod-web-02 stopped, moved to oc-node-03, then started). The `--force`
flag skips confirmation prompts.
**Note**: if VMs lack the `limits.cpu` range fix, use `--action stop`
instead to avoid migration failures:
```bash
-incus cluster evacuate lab-node-01:lab-node-02 --force --action stop
+incus cluster evacuate oc-node-01:oc-node-02 --force --action stop
```
Verify node-02 shows EVACUATED:
```bash
-incus cluster list lab-node-01:
+incus cluster list oc-node-01:
```
Restore node-02 (workloads return):
```bash
-incus cluster restore lab-node-01:lab-node-02 --force
+incus cluster restore oc-node-01:oc-node-02 --force
```
Verify all workloads are back:
```bash
-incus list lab-node-01: --columns nstL --format table
-incus cluster list lab-node-01:
+incus list oc-node-01: --columns nstL --format table
+incus cluster list oc-node-01:
```
All nodes should show ONLINE.
@@ -1366,18 +1451,18 @@ you can tolerate temporary network disruption):
```bash
# Simulate crash: hard-stop via Proxmox API
-# curl -s -k -X POST "https://192.168.1.29:8006/api2/json/nodes/pve/qemu/912/status/stop" ...
+# incusos/helpers/proxmox-api POST /nodes/pve/qemu/401/status/stop
# Wait for heartbeat detection (~40s)
-# incus cluster list lab-node-01:
-# → lab-node-02 shows OFFLINE
+# incus cluster list oc-node-01:
+# → oc-node-02 shows OFFLINE
# Restart via Proxmox
-# curl -s -k -X POST "https://192.168.1.29:8006/api2/json/nodes/pve/qemu/912/status/start" ...
+# incusos/helpers/proxmox-api POST /nodes/pve/qemu/401/status/start
# Wait for auto-rejoin (~60s)
-# incus cluster list lab-node-01:
-# → lab-node-02 shows ONLINE
+# incus cluster list oc-node-01:
+# → oc-node-02 shows ONLINE
```
### 11.3 Node Replacement
@@ -1388,13 +1473,13 @@ VM, deploy a fresh node, and join it back. This tests the complete lifecycle.
**Step 1: Evacuate node-03:**
```bash
-incus cluster evacuate lab-node-01:lab-node-03 --force --action stop
+incus cluster evacuate oc-node-01:oc-node-03 --force --action stop
```
**Step 2: Remove from cluster:**
```bash
-printf "yes\n" | incus cluster remove lab-node-01:lab-node-03 --force
+printf "yes\n" | incus cluster remove oc-node-01:oc-node-03 --force
```
**Note**: `incus cluster remove` prompts "Are you really sure?" even with
@@ -1403,62 +1488,69 @@ printf "yes\n" | incus cluster remove lab-node-01:lab-node-03 --force
**Step 3: Clean up the remote:**
```bash
-incus remote remove lab-node-03
+incus remote remove oc-node-03
```
**Step 4: Destroy and redeploy the VM.** Use `incusos-proxmox` to destroy
-just node-03 and redeploy it. The simplest approach: create a single-VM
-config or use the replacement config pattern:
+just node-03 (VMID 402) and redeploy it. Create a single-VM config:
-```bash
-# Destroy just node-03 via Proxmox API (VMID 913)
-# Then redeploy with incusos-proxmox using a config that only defines node-03
+```yaml
+# /tmp/lab-replace-node03.yaml
+defaults:
+ cores: 4
+ memory: 20480
+ disk: 100
+ start_vmid: 402
+
+vms:
+ - name: oc-node-03
+ app: incus
+ apply_defaults: false
+ ip: 192.168.102.142/22
```
-Alternatively, if you have a `lab-replace.yaml` config for the replacement
-node:
-
```bash
-./incusos-proxmox --yes examples/lab-replace.yaml
+./incusos-proxmox --cleanup --yes /tmp/lab-replace-node03.yaml
+./incusos-proxmox --iso /tmp/IncusOS-oc.iso --yes /tmp/lab-replace-node03.yaml
```
**Step 5: Join the fresh node to the cluster:**
```bash
# Set specific IP
-incus config set lab-node-03: core.https_address 192.168.102.113:8443
+incus config set oc-node-03: core.https_address 192.168.102.142:8443
# Generate join token
-incus cluster add lab-node-01:lab-node-03
+incus cluster add oc-node-01:oc-node-03
# Join
-printf '\n\nyes\nlocal/incus\nlocal/incus\n' | incus cluster join lab-node-01: lab-node-03:
+printf '\n\nyes\nlocal/incus\nlocal/incus\n' | incus cluster join oc-node-01: oc-node-03:
# Fix remote
-incus remote remove lab-node-03
-incus remote add lab-node-03 https://192.168.102.113:8443 --accept-certificate
+incus remote remove oc-node-03
+incus remote add oc-node-03 https://192.168.102.142:8443 --accept-certificate
```
**Step 6: Re-enable OVN on the replacement node:**
```bash
-incus query lab-node-03:/os/1.0/services/ovn --request PUT --data '{
+incus query oc-node-03:/os/1.0/services/ovn --request PUT --data '{
"config": {
- "database": "tcp:192.168.102.111:6642",
+ "database": "tcp:192.168.102.140:6642",
"enabled": true,
- "tunnel_address": "192.168.102.113",
+ "tunnel_address": "192.168.102.142",
"tunnel_protocol": "geneve"
},
"state": {}
}'
-incus cluster role add lab-node-01:lab-node-03 ovn-chassis
+incus cluster role add oc-node-01:oc-node-03 ovn-chassis
```
**Step 7: Verify:**
```bash
-incus cluster list lab-node-01:
+incus cluster list oc-node-01:
```
All 3 nodes should be ONLINE with `ovn-chassis` role.
@@ -1469,10 +1561,10 @@ Enable automatic workload rebalancing. When a new node joins (or workloads
are unevenly distributed), Incus redistributes VMs:
```bash
-incus config set lab-node-01: cluster.rebalance.interval=1
-incus config set lab-node-01: cluster.rebalance.threshold=10
-incus config set lab-node-01: cluster.rebalance.batch=2
-incus config set lab-node-01: cluster.rebalance.cooldown=5m
+incus config set oc-node-01: cluster.rebalance.interval=1
+incus config set oc-node-01: cluster.rebalance.threshold=10
+incus config set oc-node-01: cluster.rebalance.batch=2
+incus config set oc-node-01: cluster.rebalance.cooldown=5m
```
**Important**: only VMs with `migration.stateful=true` are rebalanced.
@@ -1481,14 +1573,14 @@ Containers are NOT auto-rebalanced.
Monitor rebalancing:
```bash
-incus list lab-node-01: --columns nstL --format table
+incus list oc-node-01: --columns nstL --format table
```
Disable rebalancing when done testing:
```bash
-incus config unset lab-node-01: cluster.rebalance.interval
-incus config unset lab-node-01: cluster.rebalance.threshold
+incus config unset oc-node-01: cluster.rebalance.interval
+incus config unset oc-node-01: cluster.rebalance.threshold
```
@@ -1507,66 +1599,39 @@ operations-center remote switch oc-lab
operations-center admin os show
```
-Actual output:
+Actual output (version and uptime will vary):
-| PROPERTY | VALUE |
-|---------------|--------------------------------------------------------|
-| hostname | lab-oc |
-| os_version | 202602210344 |
-| kernel | 6.12.13 |
-| architecture | x86_64 |
-| uptime | 4390 |
-| addresses | 192.168.102.110/22 (mgmt), fd42:...:1 (incusbr0) |
-| storage_disks | /dev/sda (53.7GB, QEMU) |
-| storage_pools | local (zfs, /dev/sda4) |
-
-### Application Status
-
-```bash
-operations-center admin os application list
+```
+environment:
+ hostname: oc-server
+ os_name: IncusOS
+ os_version: "202602240349"
+ os_version_next: ""
+ uptime: 3600
```
-Actual output:
+### Provisioning Status
-| NAME | STATUS |
-|-------------------|---------|
-| operations-center | running |
-
-### Service Status
+OC manages all 3 cluster nodes. Verify they're registered and up to date:
```bash
-operations-center admin os service list
+operations-center provisioning server list
```
-Actual output:
-
-| NAME | ENABLED |
-|----------|---------|
-| ovn | false |
-| syslog | false |
-| fan | false |
-| bgp | false |
-| dns | false |
-| metricsA | false |
-| metricsB | false |
+All nodes should show `ready` status and `up to date` update status.
### Web UI
-The OC web UI at `https://192.168.102.110:8443/ui/` provides:
+The OC web UI at `https://192.168.102.120:8443/ui/` provides:
- **Dashboard**: server overview with resource utilization
-- **Updates**: available IncusOS updates
-- **Provisioning**: token management (for OC-provisioned deployments)
+- **Updates**: available IncusOS updates and delivery status
+- **Provisioning**: token management, server list, cluster formation
- **System**: OC configuration and certificates
-**Limitation**: OC deployed with a standard ISO cannot manage the Incus
-cluster nodes. The nodes are independent — they were deployed with a
-standard IncusOS ISO, not an OC-provisioned one. For full OC node
-management (cluster orchestration, application deployment, monitoring),
-nodes must boot from an OC-provisioned ISO.
-
-See [Operations Center Guide](operations-center-guide.md) for the full
-hybrid deployment workflow with OC-provisioned ISOs.
+Because the nodes were deployed from an OC-provisioned ISO (Section 3),
+OC has full visibility and management of the cluster — including update
+delivery, server inventory, and cluster formation.
## Section 13: Cleanup
@@ -1576,12 +1641,12 @@ hybrid deployment workflow with OC-provisioned ISOs.
```bash
# Delete containers
for c in prod-web-01 prod-web-02 prod-api-01; do
- incus delete lab-node-01:$c --force
+ incus delete oc-node-01:$c --force
done
# Delete VMs
for vm in prod-db-01 prod-app-01; do
- incus delete lab-node-01:$vm --force
+ incus delete oc-node-01:$vm --force
done
```
@@ -1589,23 +1654,23 @@ done
```bash
# Delete OVN networks
-incus network delete lab-node-01:net-prod
-incus network delete lab-node-01:net-isolated 2>/dev/null || true
+incus network delete oc-node-01:net-prod
+incus network delete oc-node-01:net-isolated 2>/dev/null || true
# Delete UPLINK
-incus network delete lab-node-01:UPLINK
+incus network delete oc-node-01:UPLINK
```
### Remove OVN Control Plane
```bash
-incus delete lab-node-01:ovn-central --force
+incus delete oc-node-01:ovn-central --force
```
### Disable OVN Services
```bash
-for node in lab-node-01 lab-node-02 lab-node-03; do
+for node in oc-node-01 oc-node-02 oc-node-03; do
incus query "$node":/os/1.0/services/ovn --request PUT --data '{
"config": {
"enabled": false
@@ -1620,19 +1685,21 @@ done
**Keep infrastructure** (stop VMs, keep on disk for later):
```bash
-./incusos-proxmox --lab-down examples/lab-production.yaml
+./incusos-proxmox --lab-down examples/lab-oc-nodes.yaml
+./incusos-proxmox --lab-down examples/lab-oc-deploy.yaml
```
Restart later with:
```bash
-./incusos-proxmox --lab-up examples/lab-production.yaml
+./incusos-proxmox --lab-up examples/lab-oc-deploy.yaml
+./incusos-proxmox --lab-up examples/lab-oc-nodes.yaml
```
**Full teardown** (destroy all VMs, remove ISOs, remotes, cache):
```bash
-./incusos-proxmox --cleanup --deep examples/lab-production.yaml
+./incusos-proxmox --cleanup-all --deep --yes
```
@@ -1640,11 +1707,11 @@ Restart later with:
| # | Check | Command | Expected |
|---|-------|---------|----------|
-| 1 | All VMs running | `incusos-proxmox --status examples/lab-production.yaml` | 4 VMs running, port 8443 open |
-| 2 | Scrub schedule healthy | `incus query lab-node-01:/os/1.0/system/storage` | `scrub_schedule: "0 4 * * 0"` |
+| 1 | All VMs running | `incusos-proxmox --status examples/lab-oc-nodes.yaml` | 4 VMs running, port 8443 open |
+| 2 | Scrub schedule healthy | `incus query oc-node-01:/os/1.0/system/storage` | `scrub_schedule: "0 4 * * 0"` |
| 3 | OC accessible | `operations-center remote switch oc-lab && operations-center admin os show` | Shows version, uptime |
-| 4 | Cluster formed | `incus cluster list lab-node-01:` | 3 nodes ONLINE |
-| 5 | Storage pool | `incus storage list lab-node-01:` | `local` pool on all members |
+| 4 | Cluster formed | `incus cluster list oc-node-01:` | 3 nodes ONLINE |
+| 5 | Storage pool | `incus storage list oc-node-01:` | `local` pool on all members |
| 6 | Bridge isolation | Ping cross-node on incusbr0 | 100% loss (expected) |
| 7 | OVN connectivity | Ping cross-node on net-prod | 0% loss |
| 8 | Internet via OVN | `ping 1.1.1.1` from OVN container | 0% loss |
@@ -1714,7 +1781,7 @@ Configure while VM is **stopped**:
incus stop REMOTE:VM
incus config set REMOTE:VM limits.cpu=0-1
incus config set REMOTE:VM migration.stateful=true
-incus config device set REMOTE:VM root size.state=2GiB
+incus config device override REMOTE:VM root size.state=2GiB
incus start REMOTE:VM
```
@@ -1736,3 +1803,5 @@ incus start REMOTE:VM
| "Invalid target address" on LB backend | Backend needs IP, not instance name | Use instance IP address (e.g., `10.10.10.2`) |
| OC CLI "Invalid number of arguments" | OC CLI doesn't support `remote:` suffix | Use `operations-center remote switch NAME` first |
| Container placed on wrong node | Cluster auto-schedules without `--target` | Use `--target NODE` for explicit placement |
+| "Device from profile(s) cannot be modified" | `root` device comes from default profile | Use `incus config device override` instead of `device set` |
+| `incus launch` hangs or times out | Image download to new node takes 2-3 min | Run launches one at a time, not chained with `&&` |