From 4617de8869c4bef4744d5854d92107ba98f261bc Mon Sep 17 00:00:00 2001 From: Maarten Date: Tue, 3 Mar 2026 18:52:36 +0100 Subject: [PATCH] Update all metadata and documentation for new repo structure - .gitignore: envs/*/env, envs/*/proxmox.yaml instead of old incusos/ paths - .claude/rules/*.md: all paths: directives updated from incusos/ to bin/ - .claude/skills/*.md: helper script paths updated - CLAUDE.md: structure diagram, capabilities paths, context loading table - README.md: structure, quick start examples with new paths - bin/TESTING.md, bin/README.md: command examples updated - envs/hetzner/setup/: all guide and script path references updated - bin/examples/proxmox.yaml.example: updated copy instructions Co-Authored-By: Claude Opus 4.6 --- .claude/rules/aether-api-payloads.md | 8 +-- .claude/rules/awx-integration.md | 4 +- .claude/rules/clustering.md | 4 +- .claude/rules/haproxy-lb.md | 4 +- .claude/rules/hetzner-setup.md | 6 +- .claude/rules/incusos-immutability.md | 2 +- .claude/rules/incusos-scripts.md | 10 +-- .claude/rules/lab-infrastructure.md | 6 +- .claude/rules/observability.md | 4 +- .claude/rules/operations-center.md | 4 +- .claude/rules/ovn-internals.md | 6 +- .claude/rules/proxmox-deployment.md | 10 +-- .gitignore | 11 ++- CLAUDE.md | 74 +++++++++++++------- README.md | 65 ++++++++++-------- bin/README.md | 16 ++--- bin/TESTING.md | 89 ++++++++++++------------- bin/deploy-observability | 1 + bin/examples/proxmox.yaml.example | 4 +- envs/hetzner/proxmox.yaml.example | 2 +- envs/hetzner/setup/README.md | 10 +-- envs/hetzner/setup/hetzner-lab-guide.md | 44 ++++++------ envs/hetzner/setup/hetzner-setup.md | 10 +-- envs/hetzner/setup/proxmox-setup | 10 +-- 24 files changed, 218 insertions(+), 186 deletions(-) diff --git a/.claude/rules/aether-api-payloads.md b/.claude/rules/aether-api-payloads.md index e9ea42e..c7cf7f0 100644 --- a/.claude/rules/aether-api-payloads.md +++ b/.claude/rules/aether-api-payloads.md @@ -1,7 +1,7 @@ # Aether API Payloads Context paths: - - incus-mitm + - bin/helpers/incus-mitm - api-interception ## How Aether Communicates with Incus @@ -20,11 +20,11 @@ prefix is `6cfd2a1949a7`. ```bash # Live stream with Aether filter -incusos/helpers/incus-mitm --live --filter aether +bin/helpers/incus-mitm --live --filter aether # Capture to file -incusos/helpers/incus-mitm --capture 120 -incusos/helpers/incus-mitm --analyze /tmp/incus-events-*.json --filter aether +bin/helpers/incus-mitm --capture 120 +bin/helpers/incus-mitm --analyze /tmp/incus-events-*.json --filter aether ``` ### API Call Patterns diff --git a/.claude/rules/awx-integration.md b/.claude/rules/awx-integration.md index 42171b5..a5fe401 100644 --- a/.claude/rules/awx-integration.md +++ b/.claude/rules/awx-integration.md @@ -1,8 +1,8 @@ --- paths: - "ansible/**" - - "incusos/deploy-awx" - - "incusos/awx-manifests/*" + - "bin/deploy-awx" + - "bin/data/awx-manifests/*" - "notes/awx-guide.md" --- diff --git a/.claude/rules/clustering.md b/.claude/rules/clustering.md index 0a3ff47..049bf2a 100644 --- a/.claude/rules/clustering.md +++ b/.claude/rules/clustering.md @@ -1,7 +1,7 @@ --- paths: - - "incusos/lab-test" - - "incusos/incusos-proxmox" + - "bin/lab-test" + - "bin/incusos-proxmox" - "notes/clustering-guide.md" - "notes/production-lab-guide.md" --- diff --git a/.claude/rules/haproxy-lb.md b/.claude/rules/haproxy-lb.md index d5efa5e..8044292 100644 --- a/.claude/rules/haproxy-lb.md +++ b/.claude/rules/haproxy-lb.md @@ -1,7 +1,7 @@ --- paths: - - "incusos/deploy-haproxy" - - "incusos/helpers/aether-browser" + - "bin/deploy-haproxy" + - "bin/helpers/aether-browser" - "notes/haproxy-guide.md" --- diff --git a/.claude/rules/hetzner-setup.md b/.claude/rules/hetzner-setup.md index ea0d947..8e2ea07 100644 --- a/.claude/rules/hetzner-setup.md +++ b/.claude/rules/hetzner-setup.md @@ -1,8 +1,8 @@ # Hetzner Setup Context paths: - - hetzner/* - - incusos/targets/hetzner/* + - envs/hetzner/* + - envs/hetzner/setup/* ## Network Topology @@ -35,7 +35,7 @@ VMs cannot use public IPs directly. ### proxmox-setup Script -`hetzner/proxmox-setup` runs from the workstation over SSH. +`envs/hetzner/setup/proxmox-setup` runs from the workstation over SSH. All commands executed via `ssh `. Follows same conventions as other scripts: setup_colors, info/warn/error, --dry-run, --yes flags. diff --git a/.claude/rules/incusos-immutability.md b/.claude/rules/incusos-immutability.md index 7498297..7bc01d8 100644 --- a/.claude/rules/incusos-immutability.md +++ b/.claude/rules/incusos-immutability.md @@ -1,6 +1,6 @@ --- paths: - - "incusos/helpers/incusos-health" + - "bin/helpers/incusos-health" - "notes/incusos-break-fix.md" --- diff --git a/.claude/rules/incusos-scripts.md b/.claude/rules/incusos-scripts.md index cff7f16..099beaf 100644 --- a/.claude/rules/incusos-scripts.md +++ b/.claude/rules/incusos-scripts.md @@ -1,10 +1,10 @@ --- paths: - - "incusos/incusos-iso" - - "incusos/incusos-seed" - - "incusos/incusos-proxmox" - - "incusos/helpers/*" - - "incusos/examples/*" + - "bin/incusos-iso" + - "bin/incusos-seed" + - "bin/incusos-proxmox" + - "bin/helpers/*" + - "bin/examples/*" --- # IncusOS Scripts Context diff --git a/.claude/rules/lab-infrastructure.md b/.claude/rules/lab-infrastructure.md index 63e996e..55a56a6 100644 --- a/.claude/rules/lab-infrastructure.md +++ b/.claude/rules/lab-infrastructure.md @@ -1,8 +1,8 @@ --- paths: - - "incusos/incusos-proxmox" - - "incusos/lab-test" - - "incusos/examples/*" + - "bin/incusos-proxmox" + - "bin/lab-test" + - "bin/examples/*" --- # Lab Infrastructure diff --git a/.claude/rules/observability.md b/.claude/rules/observability.md index df45ac2..970c0d1 100644 --- a/.claude/rules/observability.md +++ b/.claude/rules/observability.md @@ -1,7 +1,7 @@ --- paths: - - "incusos/deploy-observability" - - "incusos/observability-dashboards" + - "bin/deploy-observability" + - "bin/data/observability-dashboards" - "notes/observability-guide.md" --- diff --git a/.claude/rules/operations-center.md b/.claude/rules/operations-center.md index 669f1dc..8c2499f 100644 --- a/.claude/rules/operations-center.md +++ b/.claude/rules/operations-center.md @@ -1,8 +1,8 @@ --- paths: - "notes/operations-center-guide.md" - - "incusos/incusos-proxmox" - - "incusos/examples/*oc*" + - "bin/incusos-proxmox" + - "bin/examples/*oc*" --- # Operations Center diff --git a/.claude/rules/ovn-internals.md b/.claude/rules/ovn-internals.md index 11826f9..3cc4223 100644 --- a/.claude/rules/ovn-internals.md +++ b/.claude/rules/ovn-internals.md @@ -1,8 +1,8 @@ # OVN Internals Context paths: - - ovn-inspect - - ovn-deep-dive + - bin/helpers/ovn-inspect + - notes/ovn-deep-dive* ## OVN Architecture in this Cluster @@ -26,7 +26,7 @@ Incus manages OVN objects using a `net8` prefix convention. - NB/SB commands: `incus exec oc-node-01:ovn-central -- ovn-nbctl ...` - OVS commands: Need privileged container with `/run/openvswitch` mounted (deploy Alpine on net-prod, `apk add openvswitch`) -- Helper script: `incusos/helpers/ovn-inspect --nb|--sb|--ovs|--full|--trace` +- Helper script: `bin/helpers/ovn-inspect --nb|--sb|--ovs|--full|--trace` ### Important Details diff --git a/.claude/rules/proxmox-deployment.md b/.claude/rules/proxmox-deployment.md index c8683e2..5f8fef6 100644 --- a/.claude/rules/proxmox-deployment.md +++ b/.claude/rules/proxmox-deployment.md @@ -1,10 +1,10 @@ --- paths: - - "incusos/incusos-proxmox" - - "incusos/observe-deploy" - - "incusos/helpers/*" - - "incusos/examples/*" - - "incusos/TESTING.md" + - "bin/incusos-proxmox" + - "bin/diagnostic/observe-deploy" + - "bin/helpers/*" + - "bin/examples/*" + - "bin/TESTING.md" --- # Proxmox VE Deployment diff --git a/.gitignore b/.gitignore index 8801b88..13aa91e 100644 --- a/.gitignore +++ b/.gitignore @@ -26,14 +26,11 @@ Thumbs.db # Go binaries (flasher-tool) flasher-tool -# Proxmox connection configs (contain secrets, per-target and default) -incusos/proxmox.yaml -incusos/targets/*/proxmox.yaml +# Per-environment secrets and connection configs +envs/*/env +envs/*/proxmox.yaml -# Per-target env files -env-* - -# Environment secrets +# Legacy: root env file (backward compat) env # Observational deploy output (screenshots, logs) diff --git a/CLAUDE.md b/CLAUDE.md index c893988..3bb5051 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,16 +20,16 @@ incus-contrib/ │ └── playbooks/ │ ├── post-deploy.yml # Runs after Aether creates an instance │ └── decommission.yml # Runs before Aether deletes an instance -├── incusos/ # IncusOS installation tooling -│ ├── README.md +├── bin/ # ALL reusable scripts │ ├── incusos-iso # ISO/IMG builder (wraps flasher-tool) │ ├── incusos-seed # Seed archive generator (Linux + macOS) │ ├── incusos-proxmox # Declarative Proxmox VM deployment + lab lifecycle +│ ├── lab-test # Guided lab validation (12 test phases) │ ├── deploy-awx # AWX deployment + management on Incus cluster │ ├── deploy-haproxy # HAProxy LB deployment + management via Aether │ ├── deploy-observability # Prometheus + Grafana + Loki observability stack -│ ├── awx-manifests/ # K8s manifests for AWX Operator + instance -│ ├── observability-dashboards/ # Grafana dashboard JSON exports +│ ├── manage-dashboards # Grafana dashboard management +│ ├── pve-api # Python Proxmox API helper │ ├── helpers/ │ │ ├── proxmox-screenshot # VMID -> PNG console screenshot │ │ ├── proxmox-api # Authenticated API calls (handles ! in token) @@ -37,18 +37,41 @@ incus-contrib/ │ │ ├── ovn-inspect # OVN/OVS topology inspection (--nb/--sb/--ovs/--trace) │ │ ├── incus-mitm # API traffic capture and analysis (--capture/--live/--analyze) │ │ └── incusos-health # IncusOS health inspection (--status/--tpm/--partitions/--all) -│ ├── lab-test # Guided lab validation (12 test phases) -│ ├── observe-deploy # Single-VM deploy with console screenshots -│ ├── targets/ # Per-host configs (beelink/, hetzner/) -│ ├── proxmox.yaml # Proxmox connection (gitignored) -│ ├── TESTING.md -│ └── examples/ # Example seed + Proxmox YAML files -├── hetzner/ # Hetzner dedicated server setup -│ ├── README.md # Overview, prerequisites, quickstart -│ ├── hetzner-setup.md # Step-by-step Proxmox setup guide -│ └── proxmox-setup # Interactive setup helper (runs over SSH) -├── env # PROXMOX_TOKEN_SECRET, PROXMOX_ROOT_PASSWORD, AETHER_ADMIN_PASSWORD (gitignored) -└── notes/ # Reference guides (clustering, networking, OC, AWX, etc.) +│ ├── data/ # Static data consumed by scripts +│ │ ├── awx-manifests/ # K8s manifests for AWX Operator + instance +│ │ └── observability-dashboards/ # Grafana dashboard JSON exports +│ ├── diagnostic/ # Investigation/one-off tools +│ │ ├── observe-deploy # Single-VM deploy with console screenshots +│ │ ├── investigate-boot # Boot investigation +│ │ └── test-boot-reliability # Parallel boot testing +│ ├── examples/ # Example seed + Proxmox YAML files +│ └── TESTING.md +├── envs/ # Per-environment config + secrets +│ ├── README.md # How environments work +│ ├── beelink/ # Beelink mini PC target +│ │ ├── env # Secrets (gitignored) +│ │ ├── proxmox.yaml # Proxmox connection (gitignored) +│ │ ├── proxmox.yaml.example +│ │ ├── lab-cluster.yaml +│ │ ├── awx.yaml +│ │ ├── haproxy.yaml +│ │ └── observability.yaml +│ └── hetzner/ # Hetzner dedicated server target +│ ├── env # Secrets (gitignored) +│ ├── proxmox.yaml # Proxmox connection (gitignored) +│ ├── proxmox.yaml.example +│ ├── lab-cluster.yaml +│ ├── lab-production.yaml +│ ├── awx.yaml +│ ├── haproxy.yaml +│ ├── observability.yaml +│ └── setup/ # Host provisioning +│ ├── README.md +│ ├── hetzner-setup.md +│ ├── hetzner-lab-guide.md +│ └── proxmox-setup +├── notes/ # Reference guides (clustering, networking, OC, AWX, etc.) +└── sources/ # External software (gitignored) ``` ## Capabilities you have @@ -56,7 +79,7 @@ incus-contrib/ ### Proxmox VM screenshots Take console screenshots during deploys to see boot progress or errors: ``` -incusos/helpers/proxmox-screenshot VMID [/tmp/output.png] +bin/helpers/proxmox-screenshot VMID [/tmp/output.png] ``` Use the Read tool on the PNG to view it. **USE THIS PROACTIVELY** during deploy scenarios to monitor boot progress, diagnose hangs, and verify installs. @@ -64,9 +87,9 @@ deploy scenarios to monitor boot progress, diagnose hangs, and verify installs. ### Proxmox API calls Make authenticated API calls (handles the `!` in `automation@pve!deploy`): ``` -incusos/helpers/proxmox-api GET /nodes/pve/qemu/900/status/current --json -incusos/helpers/proxmox-api GET /nodes/pve/status --json -incusos/helpers/proxmox-api POST /nodes/pve/qemu/900/status/stop +bin/helpers/proxmox-api GET /nodes/pve/qemu/900/status/current --json +bin/helpers/proxmox-api GET /nodes/pve/status --json +bin/helpers/proxmox-api POST /nodes/pve/qemu/900/status/stop ``` ### Aether API @@ -75,7 +98,7 @@ Aether has a documented REST API at `https://192.168.102.160:8443/api/`. - **Swagger UI**: `https://192.168.102.160:8443/api/docs` - **OpenAPI spec**: `https://192.168.102.160:8443/api/swagger.yaml` - **Auth**: JWT via `POST /api/auth/token` with `{"username":"...","password":"..."}` - Credentials in `env` file (`AETHER_ADMIN_PASSWORD`). Token valid 24h. + Credentials in `envs/*/env` file (`AETHER_ADMIN_PASSWORD`). Token valid 24h. ```bash # Get JWT @@ -117,11 +140,11 @@ that curl cannot handle reliably. **USE PLAYWRIGHT** for these interactions. `browser_click`, `browser_fill`, `browser_screenshot` etc. as tools when the Playwright MCP server is running. Prefer MCP tools when available. -**Helper script**: `incusos/helpers/aether-browser` provides standalone +**Helper script**: `bin/helpers/aether-browser` provides standalone Playwright automation when MCP tools aren't loaded: ```bash -source env # loads AETHER_ADMIN_PASSWORD -NODE_PATH=~/node_modules node incusos/helpers/aether-browser [args] +source envs/beelink/env # loads AETHER_ADMIN_PASSWORD +NODE_PATH=~/node_modules node bin/helpers/aether-browser [args] ``` Actions: @@ -177,6 +200,7 @@ still UI-only. The API endpoint may have been added since the guide was written. - **Flags**: both short (`-d`) and long (`--defaults`) - **Defaults**: useful behavior with zero flags - **Dry run**: all scripts support `--dry-run` +- **Config**: deploy scripts use `--config FILE` for environment-specific settings - **Cert detection**: files on disk first (`~/.config/incus/client.crt`), CLI second - **Errors**: include actionable remediation steps - **No hardcoded package managers**: say "install the Incus client" with a link @@ -201,7 +225,7 @@ which files are being edited: | `incusos-immutability.md` | incusos-health, incusos-break-fix | | `proxmox-ssh-rules.md` | **Always loaded** (no paths filter) | | `lab-infrastructure.md` | incusos-proxmox, lab-test, examples | -| `hetzner-setup.md` | hetzner/, incusos/targets/hetzner/ | +| `hetzner-setup.md` | envs/hetzner/, setup scripts | For deep reference, see the guides in `notes/`. diff --git a/README.md b/README.md index 0d79f27..61e20f5 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# incu-contrib +# incus-contrib Peripheral tools, snippets, and scripts for working with [Incus](https://linuxcontainers.org/incus/), [IncusOS](https://linuxcontainers.org/incus-os/), and the broader ecosystem. @@ -7,9 +7,10 @@ Primarily aimed at home lab environments but useful in production as well. ## Contents -### [`incusos/`](incusos/) +### [`bin/`](bin/) -Scripts for building IncusOS installation media and seed configurations: +Reusable scripts for building IncusOS installation media, deploying labs, +and managing services: - **`incusos-iso`** -- Build customized IncusOS ISO/IMG images using the official flasher-tool. Supports both architectures (x86_64, aarch64), multiple applications (Incus, Operations Center), @@ -23,15 +24,29 @@ Scripts for building IncusOS installation media and seed configurations: YAML, and the script handles seed generation, ISO upload, VM creation with correct settings (UEFI, TPM 2.0, VirtIO-SCSI), and automated installation. -- **`targets/`** -- Per-host Proxmox connection configs and lab definitions. Each - subdirectory (beelink, hetzner) has a `proxmox.yaml.example` and lab YAML files - sized for that host. See [`targets/README.md`](incusos/targets/README.md). - - **`deploy-awx`** -- Deploy and manage AWX (Ansible automation) on the Incus cluster. Handles VM creation, K3s install, AWX Operator deployment, project/template - configuration, and Aether integration. + configuration, and Aether integration. Requires `--config` for environment settings. -See [`incusos/README.md`](incusos/README.md) for full documentation. +- **`deploy-haproxy`** -- Deploy HA HAProxy load balancing via Aether. Requires + `--config` for environment settings. + +- **`deploy-observability`** -- Deploy Prometheus + Grafana + Loki monitoring stack. + Requires `--config` for environment settings. + +See [`bin/README.md`](bin/README.md) for full documentation. + +### [`envs/`](envs/) + +Per-environment configuration, secrets, and deployment settings: + +- **`beelink/`** -- Beelink mini PC target (LAN, vmbr0, VLAN 69, local-lvm) +- **`hetzner/`** -- Hetzner dedicated server target (WireGuard, vmbr1, local-zfs) + - **`setup/`** -- Host provisioning guide and interactive setup script + +Each environment has: `env` (secrets), `proxmox.yaml` (connection), +lab YAML files (VM definitions), and service configs (`awx.yaml`, `haproxy.yaml`, +`observability.yaml`). See [`envs/README.md`](envs/README.md). ### [`ansible/`](ansible/) @@ -43,18 +58,6 @@ API (not SSH) for all instance operations — works regardless of network topolo - **`playbooks/decommission.yml`** -- Runs before Aether deletes an instance. Graceful service shutdown and decommission logging (best-effort). -### [`hetzner/`](hetzner/) - -Setup guide and automation for deploying IncusOS labs on a Hetzner dedicated server: - -- **`hetzner-setup.md`** -- Step-by-step guide: Proxmox install, private networking, - WireGuard tunnel, firewall lockdown, and API token creation. -- **`proxmox-setup`** -- Interactive helper script that automates the guide steps - over SSH. Configures repos, bridge, storage, WireGuard, firewall, and API tokens. - -See also [`incusos/targets/`](incusos/targets/) for per-host connection configs -and lab definitions. - ### [`notes/`](notes/) Guides and reference material (roughly in learning order): @@ -89,19 +92,27 @@ Guides and reference material (roughly in learning order): go install github.com/lxc/incus-os/incus-osd/cmd/flasher-tool@latest # Build a default IncusOS ISO with your client cert injected -./incusos/incusos-iso +bin/incusos-iso # Build for a specific architecture and application -./incusos/incusos-iso --arch aarch64 --app operations-center --defaults +bin/incusos-iso --arch aarch64 --app operations-center --defaults # Generate all common ISO variants at once -./incusos/incusos-iso --batch +bin/incusos-iso --batch # Generate a standalone seed archive -./incusos/incusos-seed --app incus --defaults --output my-seed.tar +bin/incusos-seed --app incus --defaults --output my-seed.tar -# Generate a SEED_DATA FAT image for external boot media -./incusos/incusos-seed --format fat --app incus --defaults --output seed.img +# Deploy a cluster on Hetzner +source envs/hetzner/env +bin/incusos-proxmox --proxmox envs/hetzner/proxmox.yaml envs/hetzner/lab-production.yaml + +# Deploy AWX on Hetzner +bin/deploy-awx --config envs/hetzner/awx.yaml --deploy + +# Deploy HAProxy on beelink +source envs/beelink/env +bin/deploy-haproxy --config envs/beelink/haproxy.yaml --deploy ``` ## Requirements diff --git a/bin/README.md b/bin/README.md index 4275c84..1fe214f 100644 --- a/bin/README.md +++ b/bin/README.md @@ -376,7 +376,7 @@ directory and current directory. Override with `--proxmox FILE`. Copy the template to get started: ```bash -cp incusos/examples/proxmox.yaml.example incusos/proxmox.yaml +cp bin/examples/proxmox.yaml.example envs/beelink/proxmox.yaml # Edit with your Proxmox host, credentials, and storage settings ``` @@ -557,25 +557,25 @@ default — prints what it will do, confirms, executes, and reports pass/fail. ```bash # Run all phases (deploy, single-node tests, cluster, workloads, migration) -./incusos/lab-test examples/lab-cluster.yaml +bin/lab-test examples/lab-cluster.yaml # Deploy and verify only -./incusos/lab-test --phase deploy examples/lab-cluster.yaml +bin/lab-test --phase deploy examples/lab-cluster.yaml # Single-node workload tests -./incusos/lab-test --phase single examples/lab-cluster.yaml +bin/lab-test --phase single examples/lab-cluster.yaml # Form cluster from deployed nodes -./incusos/lab-test --phase cluster examples/lab-cluster.yaml +bin/lab-test --phase cluster examples/lab-cluster.yaml # Test migration and evacuation -./incusos/lab-test --phase migrate examples/lab-cluster.yaml +bin/lab-test --phase migrate examples/lab-cluster.yaml # Full run without prompts -./incusos/lab-test --yes examples/lab-cluster.yaml +bin/lab-test --yes examples/lab-cluster.yaml # Clean up test instances (keeps VMs) -./incusos/lab-test --cleanup examples/lab-cluster.yaml +bin/lab-test --cleanup examples/lab-cluster.yaml ``` ### Phases diff --git a/bin/TESTING.md b/bin/TESTING.md index 727952d..0347fda 100644 --- a/bin/TESTING.md +++ b/bin/TESTING.md @@ -140,17 +140,16 @@ key auth first (`ssh-copy-id root@`). ### Step 2: Set Up Proxmox Connection ```bash -cp incusos/examples/proxmox.yaml.example incusos/proxmox.yaml +cp bin/examples/proxmox.yaml.example envs/beelink/proxmox.yaml ``` -Edit `incusos/proxmox.yaml` with your Proxmox host settings. This is -auto-discovered by the scripts and avoids repeating credentials in each -lab config. +Edit `envs/beelink/proxmox.yaml` with your Proxmox host settings, or +use `--proxmox envs/beelink/proxmox.yaml` to point at it explicitly. ### Step 3: Create a Test Config ```bash -cp incusos/examples/proxmox-minimal.yaml my-test.yaml +cp bin/examples/proxmox-minimal.yaml my-test.yaml ``` Edit `my-test.yaml` -- change only the host IP (or rely on proxmox.yaml): @@ -176,7 +175,7 @@ Using VMID 900+ avoids clashing with existing VMs. ### Step 4: Dry Run ```bash -./incusos/incusos-proxmox --dry-run my-test.yaml +bin/incusos-proxmox --dry-run my-test.yaml ``` **Verify in the output:** @@ -193,7 +192,7 @@ Using VMID 900+ avoids clashing with existing VMs. ### Step 5: Deploy (Single VM) ```bash -./incusos/incusos-proxmox my-test.yaml +bin/incusos-proxmox my-test.yaml ``` **Watch the phases:** @@ -219,7 +218,7 @@ Using VMID 900+ avoids clashing with existing VMs. After the deploy completes, verify with the built-in status command: ```bash -./incusos/incusos-proxmox --status my-test.yaml +bin/incusos-proxmox --status my-test.yaml ``` **Verify in the output:** @@ -256,7 +255,7 @@ check the Proxmox console — stale ARP entries can be misleading. Re-run the deploy to verify the reconcile menu: ```bash -./incusos/incusos-proxmox my-test.yaml +bin/incusos-proxmox my-test.yaml ``` **Verify:** The script detects the existing VM and shows the interactive menu. @@ -265,7 +264,7 @@ Choose option 1 to run status checks without modifying anything. Also test with `--yes` to verify it defaults to safe behavior: ```bash -./incusos/incusos-proxmox --yes my-test.yaml +bin/incusos-proxmox --yes my-test.yaml ``` **Verify:** Runs post-deployment checks, does NOT destroy. @@ -273,7 +272,7 @@ Also test with `--yes` to verify it defaults to safe behavior: ### Step 9: Clean Up the Test VM ```bash -./incusos/incusos-proxmox --cleanup my-test.yaml +bin/incusos-proxmox --cleanup my-test.yaml ``` - Confirms before destroying @@ -346,9 +345,9 @@ proxmox: Deploy and verify: ```bash -./incusos/incusos-proxmox --dry-run my-test.yaml # Should show pool in plan -./incusos/incusos-proxmox my-test.yaml # VM created in pool -./incusos/incusos-proxmox --status my-test.yaml # Status scoped to pool +bin/incusos-proxmox --dry-run my-test.yaml # Should show pool in plan +bin/incusos-proxmox my-test.yaml # VM created in pool +bin/incusos-proxmox --status my-test.yaml # Status scoped to pool ``` ### Step 12: (Optional) Full Lab Deployment @@ -356,10 +355,10 @@ Deploy and verify: Once single-VM tests pass, deploy the full 4-VM lab: ```bash -cp incusos/examples/proxmox-lab.yaml my-lab.yaml +cp bin/examples/proxmox-lab.yaml my-lab.yaml # Edit: change host IP, adjust start_vmid if needed -./incusos/incusos-proxmox --dry-run my-lab.yaml -./incusos/incusos-proxmox my-lab.yaml +bin/incusos-proxmox --dry-run my-lab.yaml +bin/incusos-proxmox my-lab.yaml ``` This deploys 1 Operations Center + 3 Incus nodes. After deployment: @@ -379,44 +378,44 @@ VMs: single-node workloads, cluster formation, migration, and evacuation. ```bash # 1. Check environment -./incusos/incusos-proxmox --doctor +bin/incusos-proxmox --doctor # 2. Deploy 3 nodes -./incusos/incusos-proxmox --yes examples/lab-cluster.yaml +bin/incusos-proxmox --yes examples/lab-cluster.yaml # 3. Add incus remotes for each node (use IPs from --status) -./incusos/incusos-proxmox --status examples/lab-cluster.yaml +bin/incusos-proxmox --status examples/lab-cluster.yaml incus remote add incus-lab-01 --accept-certificate incus remote add incus-lab-02 --accept-certificate incus remote add incus-lab-03 --accept-certificate # 4. Run full lab validation -./incusos/lab-test --yes examples/lab-cluster.yaml +bin/lab-test --yes examples/lab-cluster.yaml ``` ### Phase-by-Phase Testing ```bash # Single-node workloads only (container + VM launch/exec) -./incusos/lab-test --phase single --skip-deploy examples/lab-cluster.yaml +bin/lab-test --phase single --skip-deploy examples/lab-cluster.yaml # Cluster formation only -./incusos/lab-test --phase cluster --skip-deploy examples/lab-cluster.yaml +bin/lab-test --phase cluster --skip-deploy examples/lab-cluster.yaml # Cluster workloads (scheduler placement + targeted placement) -./incusos/lab-test --phase workload --skip-deploy examples/lab-cluster.yaml +bin/lab-test --phase workload --skip-deploy examples/lab-cluster.yaml # Migration and evacuation tests -./incusos/lab-test --phase migrate --skip-deploy examples/lab-cluster.yaml +bin/lab-test --phase migrate --skip-deploy examples/lab-cluster.yaml # Extended phases (require a working cluster): -./incusos/lab-test --phase storage --skip-deploy examples/lab-cluster.yaml -./incusos/lab-test --phase network --skip-deploy examples/lab-cluster.yaml -./incusos/lab-test --phase projects --skip-deploy examples/lab-cluster.yaml -./incusos/lab-test --phase backup --skip-deploy examples/lab-cluster.yaml -./incusos/lab-test --phase limits --skip-deploy examples/lab-cluster.yaml -./incusos/lab-test --phase security --skip-deploy examples/lab-cluster.yaml -./incusos/lab-test --phase resilience --skip-deploy examples/lab-cluster.yaml +bin/lab-test --phase storage --skip-deploy examples/lab-cluster.yaml +bin/lab-test --phase network --skip-deploy examples/lab-cluster.yaml +bin/lab-test --phase projects --skip-deploy examples/lab-cluster.yaml +bin/lab-test --phase backup --skip-deploy examples/lab-cluster.yaml +bin/lab-test --phase limits --skip-deploy examples/lab-cluster.yaml +bin/lab-test --phase security --skip-deploy examples/lab-cluster.yaml +bin/lab-test --phase resilience --skip-deploy examples/lab-cluster.yaml ``` ### Extended Test Phases @@ -660,17 +659,17 @@ and detailed test results. ```bash # Remove test instances only (keeps VMs and cluster) -./incusos/lab-test --cleanup examples/lab-cluster.yaml +bin/lab-test --cleanup examples/lab-cluster.yaml # Destroy VMs only -./incusos/incusos-proxmox --cleanup examples/lab-cluster.yaml +bin/incusos-proxmox --cleanup examples/lab-cluster.yaml # Full cleanup: VMs + deployment ISO + seeds + remotes + cache -./incusos/incusos-proxmox --cleanup --deep --yes examples/lab-cluster.yaml +bin/incusos-proxmox --cleanup --deep --yes examples/lab-cluster.yaml # Pool-wide: destroy ALL managed VMs (no config file needed) -./incusos/incusos-proxmox --cleanup-all --yes -./incusos/incusos-proxmox --cleanup-all --deep --yes # nuke everything +bin/incusos-proxmox --cleanup-all --yes +bin/incusos-proxmox --cleanup-all --deep --yes # nuke everything ``` --- @@ -707,14 +706,14 @@ openssl pkcs12 -export \ -out ~/.config/incus/client.pfx # 4. Verify -./incusos/incusos-proxmox --doctor +bin/incusos-proxmox --doctor ``` ### Step 1: Deploy OC Server ```bash -./incusos/incusos-proxmox --yes incusos/examples/lab-oc-deploy.yaml -./incusos/incusos-proxmox --status incusos/examples/lab-oc-deploy.yaml +bin/incusos-proxmox --yes bin/examples/lab-oc-deploy.yaml +bin/incusos-proxmox --status bin/examples/lab-oc-deploy.yaml # Note OC_IP from output ``` @@ -763,9 +762,9 @@ ls -lh /tmp/IncusOS-oc.iso # ~3.4 GB ### Step 6: Deploy Nodes (Hybrid) ```bash -./incusos/incusos-proxmox --iso /tmp/IncusOS-oc.iso --yes \ - incusos/examples/lab-oc-nodes.yaml -./incusos/incusos-proxmox --status incusos/examples/lab-oc-nodes.yaml +bin/incusos-proxmox --iso /tmp/IncusOS-oc.iso --yes \ + bin/examples/lab-oc-nodes.yaml +bin/incusos-proxmox --status bin/examples/lab-oc-nodes.yaml ``` `incusos-proxmox` creates VMs, uploads the OC ISO to ide2, generates per-node @@ -858,10 +857,10 @@ incus delete oc-node-01:test-ct --force 2>/dev/null incus delete oc-node-01:test-vm --force 2>/dev/null # Nodes -./incusos/incusos-proxmox --cleanup --deep --yes incusos/examples/lab-oc-nodes.yaml +bin/incusos-proxmox --cleanup --deep --yes bin/examples/lab-oc-nodes.yaml # OC server -./incusos/incusos-proxmox --cleanup --deep --yes incusos/examples/lab-oc-deploy.yaml +bin/incusos-proxmox --cleanup --deep --yes bin/examples/lab-oc-deploy.yaml # Remotes operations-center remote remove oc-lab 2>/dev/null diff --git a/bin/deploy-observability b/bin/deploy-observability index d9f2ba8..3887339 100755 --- a/bin/deploy-observability +++ b/bin/deploy-observability @@ -103,6 +103,7 @@ ${BOLD}ACTIONS${RESET} --doctor Diagnose common issues ${BOLD}OPTIONS${RESET} + -c, --config FILE Load environment config from YAML file (see envs/) -r, --remote NAME Incus remote (default: ${CLUSTER_REMOTE}) --nodes NODE1,NODE2 Manual node list (skip auto-discovery) --monitoring-target NODE Node to deploy monitoring container on (default: ${MONITORING_TARGET}) diff --git a/bin/examples/proxmox.yaml.example b/bin/examples/proxmox.yaml.example index d1398b0..cbc1680 100644 --- a/bin/examples/proxmox.yaml.example +++ b/bin/examples/proxmox.yaml.example @@ -1,7 +1,7 @@ # proxmox.yaml.example - Proxmox connection config template # -# Copy to incusos/proxmox.yaml and edit with your settings. -# This file is auto-discovered by incusos-proxmox and lab-test. +# Copy to envs//proxmox.yaml and edit with your settings. +# Use with: bin/incusos-proxmox --proxmox envs//proxmox.yaml # # Lab config files (lab-cluster.yaml, lab-oc.yaml, etc.) reference # this for connection settings, so you only need to change credentials diff --git a/envs/hetzner/proxmox.yaml.example b/envs/hetzner/proxmox.yaml.example index 6985492..fdaf7ce 100644 --- a/envs/hetzner/proxmox.yaml.example +++ b/envs/hetzner/proxmox.yaml.example @@ -9,7 +9,7 @@ # # Storage: local-zfs (ZFS pool on dedicated disks). # -# See hetzner/hetzner-setup.md for the full setup guide. +# See envs/hetzner/setup/hetzner-setup.md for the full setup guide. host: 10.10.0.1 # Proxmox host IP (via WireGuard, on vmbr1) method: api # api recommended (WireGuard may add SSH latency) diff --git a/envs/hetzner/setup/README.md b/envs/hetzner/setup/README.md index 5f62061..dd7621e 100644 --- a/envs/hetzner/setup/README.md +++ b/envs/hetzner/setup/README.md @@ -51,12 +51,12 @@ and API tokens for `incusos-proxmox`. ### 4. Deploy IncusOS VMs ```bash -cp incusos/targets/hetzner/proxmox.yaml.example incusos/targets/hetzner/proxmox.yaml +cp envs/hetzner/proxmox.yaml.example envs/hetzner/proxmox.yaml # Fill in the API token from step 3 export PROXMOX_TOKEN_SECRET="" -incusos-proxmox --proxmox incusos/targets/hetzner/proxmox.yaml \ - incusos/targets/hetzner/lab-cluster.yaml +bin/incusos-proxmox --proxmox envs/hetzner/proxmox.yaml \ + envs/hetzner/lab-cluster.yaml ``` ## What's in this directory @@ -92,5 +92,5 @@ Internet ## Related files -- [`incusos/targets/hetzner/`](../incusos/targets/hetzner/) -- Proxmox connection configs and lab definitions -- [`incusos/targets/README.md`](../incusos/targets/README.md) -- Multi-target concept explanation +- [`envs/hetzner/`](../) -- Proxmox connection configs and lab definitions +- [`envs/README.md`](../../README.md) -- Multi-target concept explanation diff --git a/envs/hetzner/setup/hetzner-lab-guide.md b/envs/hetzner/setup/hetzner-lab-guide.md index 11e073a..89fbf84 100644 --- a/envs/hetzner/setup/hetzner-lab-guide.md +++ b/envs/hetzner/setup/hetzner-lab-guide.md @@ -41,7 +41,7 @@ Hetzner dedicated (256 GB RAM, 32+ cores) - WireGuard tunnel active (10.10.99.x → 10.10.0.x accessible) - This repository cloned locally - `env` file populated (see below) -- `incusos/targets/hetzner/proxmox.yaml` filled in +- `envs/hetzner/proxmox.yaml` filled in - Incus client installed on workstation ### env file @@ -56,8 +56,8 @@ export AETHER_ADMIN_PASSWORD="admin1234" ### proxmox.yaml ```bash -cp incusos/targets/hetzner/proxmox.yaml.example \ - incusos/targets/hetzner/proxmox.yaml +cp envs/hetzner/proxmox.yaml.example \ + envs/hetzner/proxmox.yaml # Fill in: host, node name, token_id, storage ``` @@ -68,9 +68,9 @@ cp incusos/targets/hetzner/proxmox.yaml.example \ ```bash source env -incusos/incusos-proxmox \ - --proxmox incusos/targets/hetzner/proxmox.yaml \ - incusos/targets/hetzner/lab-production.yaml +bin/incusos-proxmox \ + --proxmox envs/hetzner/proxmox.yaml \ + envs/hetzner/lab-production.yaml ``` **What gets deployed:** @@ -88,8 +88,8 @@ interrupt. Monitor progress: ```bash # Screenshot VM console (requires PROXMOX_ROOT_PASSWORD in env) source env -incusos/helpers/proxmox-screenshot 910 # hz-oc -incusos/helpers/proxmox-screenshot 911 # hz-node-01 +bin/helpers/proxmox-screenshot 910 # hz-oc +bin/helpers/proxmox-screenshot 911 # hz-node-01 ``` Wait until all VMs are reachable: @@ -324,7 +324,7 @@ export AETHER_ADMIN_PASSWORD="admin1234" ### 6.1 Create config file -Save as `incusos/targets/hetzner/hz-haproxy.yaml` (or `/tmp/hz-deploy.yaml`): +Save as `envs/hetzner/hz-haproxy.yaml` (or `/tmp/hz-deploy.yaml`): ```yaml haproxy: @@ -360,14 +360,14 @@ Alternatively, use the script (requires the UI steps above for first build): ```bash source env -incusos/deploy-haproxy -c /tmp/hz-deploy.yaml --deploy --skip-image +bin/deploy-haproxy -c /tmp/hz-deploy.yaml --deploy --skip-image ``` ### 6.3 Full deploy ```bash source env -incusos/deploy-haproxy -c /tmp/hz-deploy.yaml --deploy +bin/deploy-haproxy -c /tmp/hz-deploy.yaml --deploy ``` This: @@ -378,7 +378,7 @@ This: On subsequent runs (if backends already exist): ```bash -incusos/deploy-haproxy -c /tmp/hz-deploy.yaml --deploy \ +bin/deploy-haproxy -c /tmp/hz-deploy.yaml --deploy \ --skip-backends --skip-image ``` @@ -398,7 +398,7 @@ for i in $(seq 1 6); do curl -s http://10.10.0.200/ | grep 'Backend:'; done ### 7.1 Create config file -Save as `incusos/targets/hetzner/awx.yaml`: +Save as `envs/hetzner/awx.yaml`: ```yaml awx: @@ -424,7 +424,7 @@ awx: ```bash source env -incusos/deploy-awx -c incusos/targets/hetzner/awx.yaml --deploy +bin/deploy-awx -c envs/hetzner/awx.yaml --deploy ``` This deploys a Debian 12 VM with K3s + AWX Operator. Takes ~20 minutes. @@ -433,7 +433,7 @@ AWX will be at `http://10.10.0.122:30080`. On partial failure (e.g. network issue mid-deploy), resume: ```bash -incusos/deploy-awx -c incusos/targets/hetzner/awx.yaml --deploy --resume +bin/deploy-awx -c envs/hetzner/awx.yaml --deploy --resume # Skips VM creation and K3s install phases, jumps to AWX Operator phase ``` @@ -459,7 +459,7 @@ incus exec hz-cluster:awx -- rm -f \ ```bash source env -incusos/deploy-awx -c incusos/targets/hetzner/awx.yaml --configure +bin/deploy-awx -c envs/hetzner/awx.yaml --configure ``` This creates: @@ -470,7 +470,7 @@ This creates: Get AWX admin password: ```bash -incusos/deploy-awx -c /tmp/hz-awx.yaml --status +bin/deploy-awx -c /tmp/hz-awx.yaml --status ``` ### 7.4 Join Aether @@ -479,7 +479,7 @@ Generate an AWX API token for Aether: ```bash source env -incusos/deploy-awx -c incusos/targets/hetzner/awx.yaml --join-aether +bin/deploy-awx -c envs/hetzner/awx.yaml --join-aether # Prints the PAT token and template IDs ``` @@ -533,7 +533,7 @@ the AWX endpoint, post-deploy, and decommission templates. ```bash source env -incusos/deploy-observability \ +bin/deploy-observability \ --remote hz-cluster \ --monitoring-target hz-node-01 \ --monitoring-ip 10.10.10.70 \ @@ -649,7 +649,7 @@ On partial failure (phases 6-9 only), resume: ```bash source env -incusos/deploy-observability \ +bin/deploy-observability \ --remote hz-cluster \ --monitoring-target hz-node-01 \ --monitoring-ip 10.10.10.70 \ @@ -836,11 +836,11 @@ incus exec hz-cluster:nginx-lb-01 -- ip a # Force destroy all VMs (no nice cleanup needed for test lab) source env for vmid in 910 911 912 913; do - incusos/helpers/proxmox-api POST \ + bin/helpers/proxmox-api POST \ "/nodes/aether-lab/qemu/${vmid}/status/stop" \ '{"forceStop":1}' 2>/dev/null || true sleep 3 - incusos/helpers/proxmox-api DELETE \ + bin/helpers/proxmox-api DELETE \ "/nodes/aether-lab/qemu/${vmid}?purge=1&destroy-unreferenced-disks=1" done diff --git a/envs/hetzner/setup/hetzner-setup.md b/envs/hetzner/setup/hetzner-setup.md index 6d15119..fd89233 100644 --- a/envs/hetzner/setup/hetzner-setup.md +++ b/envs/hetzner/setup/hetzner-setup.md @@ -295,7 +295,7 @@ Each step prompts for confirmation. Use `--yes` to skip prompts, or `--skip-repos`, `--skip-storage`, etc. to skip individual steps. At the end, the script prints: -- A ready-to-use `proxmox.yaml` config for `incusos/targets/hetzner/` +- A ready-to-use `proxmox.yaml` config for `envs/hetzner/` - An `env` file snippet with the API token - A WireGuard client config to save on your workstation @@ -330,15 +330,15 @@ Host hetzner-lab-wg ```bash # Copy and fill in the connection config -cp incusos/targets/hetzner/proxmox.yaml.example incusos/targets/hetzner/proxmox.yaml +cp envs/hetzner/proxmox.yaml.example envs/hetzner/proxmox.yaml # Set the API token export PROXMOX_TOKEN_SECRET="" # Dry run -- verify correct bridge, IPs, storage -incusos-proxmox --dry-run \ - --proxmox incusos/targets/hetzner/proxmox.yaml \ - incusos/targets/hetzner/lab-cluster.yaml +bin/incusos-proxmox --dry-run \ + --proxmox envs/hetzner/proxmox.yaml \ + envs/hetzner/lab-cluster.yaml ``` Confirm the output shows: `vmbr1` bridge, `10.10.0.x` IPs, `local-zfs` diff --git a/envs/hetzner/setup/proxmox-setup b/envs/hetzner/setup/proxmox-setup index 0cf0f8f..50ba1b0 100755 --- a/envs/hetzner/setup/proxmox-setup +++ b/envs/hetzner/setup/proxmox-setup @@ -247,7 +247,7 @@ ${BOLD}PREREQUISITES${RESET} ${BOLD}SEE ALSO${RESET} hetzner/hetzner-setup.md Manual installation guide - incusos/targets/hetzner/ Target config files + envs/hetzner/ Target config files EOF exit 0 } @@ -1302,7 +1302,7 @@ show_summary() { step "Summary" echo "" - echo -e "${BOLD}proxmox.yaml for incusos/targets/hetzner/:${RESET}" + echo -e "${BOLD}proxmox.yaml for envs/hetzner/:${RESET}" echo "" echo " host: ${SUBNET_GW}" echo " method: api" @@ -1327,11 +1327,11 @@ show_summary() { echo "" echo " 1. Save WireGuard client config and connect" echo " 2. Copy proxmox.yaml.example and fill in credentials:" - echo " cp incusos/targets/hetzner/proxmox.yaml.example incusos/targets/hetzner/proxmox.yaml" + echo " cp envs/hetzner/proxmox.yaml.example envs/hetzner/proxmox.yaml" echo " 3. Test with dry run:" echo " export PROXMOX_TOKEN_SECRET=\"\$HETZNER_PROXMOX_TOKEN_SECRET\"" - echo " incusos-proxmox --dry-run --proxmox incusos/targets/hetzner/proxmox.yaml \\" - echo " incusos/targets/hetzner/lab-cluster.yaml" + echo " bin/incusos-proxmox --dry-run --proxmox envs/hetzner/proxmox.yaml \\" + echo " envs/hetzner/lab-cluster.yaml" echo "" }