diff --git a/.claude/skills/proxmox-api.md b/.claude/skills/proxmox-api.md index 82a5e61..21d0cb0 100644 --- a/.claude/skills/proxmox-api.md +++ b/.claude/skills/proxmox-api.md @@ -16,7 +16,7 @@ Make an authenticated Proxmox API call with proper token handling. 1. Run the helper script: ```bash - /home/maarten/dev/incus-contrib/incusos/helpers/proxmox-api {{args}} + /home/maarten/dev/incus-contrib/bin/helpers/proxmox-api {{args}} ``` 2. Parse the JSON response 3. Present the results in a readable format diff --git a/.claude/skills/screenshot.md b/.claude/skills/screenshot.md index ad74984..d2ec062 100644 --- a/.claude/skills/screenshot.md +++ b/.claude/skills/screenshot.md @@ -16,7 +16,7 @@ Take a console screenshot of a Proxmox VM and display it. 1. Run the helper script to capture the screenshot: ```bash - /home/maarten/dev/incus-contrib/incusos/helpers/proxmox-screenshot {{vmid}} + /home/maarten/dev/incus-contrib/bin/helpers/proxmox-screenshot {{vmid}} ``` 2. Read the output path from stdout 3. Use the Read tool to view the PNG image diff --git a/bin/deploy-awx b/bin/deploy-awx index f5224c0..34f0e2c 100755 --- a/bin/deploy-awx +++ b/bin/deploy-awx @@ -13,7 +13,7 @@ set -euo pipefail readonly VERSION="0.1.0" readonly SCRIPT_NAME="deploy-awx" readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly MANIFESTS_DIR="${SCRIPT_DIR}/awx-manifests" +readonly MANIFESTS_DIR="${SCRIPT_DIR}/data/awx-manifests" readonly AWX_NAMESPACE="awx" # --------------------------------------------------------------------------- @@ -21,20 +21,20 @@ readonly AWX_NAMESPACE="awx" # --------------------------------------------------------------------------- VM_NAME="awx" -TARGET_REMOTE="oc-node-02" +TARGET_REMOTE="" TARGET_NODE="" # Node within cluster for --target; defaults to TARGET_REMOTE if empty -AWX_IP="192.168.102.161" +AWX_IP="" AWX_PREFIX="22" -AWX_GATEWAY="192.168.100.1" -AWX_DNS="192.168.100.1" +AWX_GATEWAY="" +AWX_DNS="" AWX_CPU="4" AWX_MEMORY="8GiB" AWX_DISK="40GiB" -GIT_REPO="ssh://git@192.168.1.200:2222/maarten/incus-contrib.git" +GIT_REPO="" GIT_BRANCH="master" PLAYBOOK_DIR="playbooks" -AETHER_URL="https://192.168.102.160:8443" -AETHER_CLUSTER_ID="52" +AETHER_URL="" +AETHER_CLUSTER_ID="" # Runtime flags DRY_RUN=false @@ -1196,6 +1196,21 @@ parse_args() { # Main # --------------------------------------------------------------------------- +require_config() { + local missing=() + [[ -z "$TARGET_REMOTE" ]] && missing+=("target_remote") + [[ -z "$AWX_IP" ]] && missing+=("ip") + [[ -z "$AWX_GATEWAY" ]] && missing+=("gateway") + [[ -z "$AWX_DNS" ]] && missing+=("dns") + [[ -z "$AETHER_URL" ]] && missing+=("aether_url") + [[ -z "$AETHER_CLUSTER_ID" ]] && missing+=("aether_cluster_id") + if [[ ${#missing[@]} -gt 0 ]]; then + error "Required config missing: ${missing[*]}" + error "Use --config FILE to load environment settings (see envs/)" + exit 1 + fi +} + main() { setup_colors parse_args "$@" @@ -1205,6 +1220,11 @@ main() { parse_config "$CONFIG_FILE" fi + # Validate required fields (must come from --config or CLI flags) + if [[ "$ACTION" != "doctor" ]]; then + require_config + fi + echo case "$ACTION" in deploy) action_deploy;; diff --git a/bin/deploy-haproxy b/bin/deploy-haproxy index b9b1bff..faa37bd 100755 --- a/bin/deploy-haproxy +++ b/bin/deploy-haproxy @@ -16,27 +16,26 @@ set -euo pipefail readonly VERSION="0.1.0" readonly SCRIPT_NAME="deploy-haproxy" readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly ENV_FILE="${SCRIPT_DIR}/../env" # --------------------------------------------------------------------------- # Defaults (overridden by config file or CLI flags) # --------------------------------------------------------------------------- -CLUSTER_REMOTE="oc-node-01" -CLUSTER_ID="52" +CLUSTER_REMOTE="" +CLUSTER_ID="" OVN_NETWORK="net-prod" -VIP="192.168.103.200" -HAPROXY_01_IP="10.10.10.50" -HAPROXY_02_IP="10.10.10.51" +VIP="" +HAPROXY_01_IP="" +HAPROXY_02_IP="" HAPROXY_CPU="2" HAPROXY_MEMORY="1GB" -BACKEND_IPS="10.10.10.60,10.10.10.61,10.10.10.62" +BACKEND_IPS="" BACKEND_PORT="80" SERVICE_NAME="web-test" SERVICE_MODE="http" SERVICE_BALANCE="roundrobin" IMAGE_VERSION="1.0.0" -AETHER_URL="https://192.168.102.160:8443" +AETHER_URL="" AETHER_USER="admin" AETHER_PASSWORD="" @@ -197,18 +196,29 @@ load_password() { return 0 fi - if [[ -f "$ENV_FILE" ]]; then - local val - val=$(grep 'AETHER_ADMIN_PASSWORD=' "$ENV_FILE" | head -1 | sed 's/^.*AETHER_ADMIN_PASSWORD=//' | tr -d '"' | tr -d "'" || true) - if [[ -n "$val" ]]; then - AETHER_PASSWORD="$val" - detail "Password loaded from env file" - return 0 - fi + # Check if already exported (user sourced envs/*/env) + if [[ -n "${AETHER_ADMIN_PASSWORD:-}" ]]; then + AETHER_PASSWORD="$AETHER_ADMIN_PASSWORD" + return 0 fi + # Search for env file: walk up from script dir + local dir="$SCRIPT_DIR" + while [[ "$dir" != "/" ]]; do + if [[ -f "${dir}/env" ]]; then + local val + val=$(grep 'AETHER_ADMIN_PASSWORD=' "${dir}/env" | head -1 | sed 's/^.*AETHER_ADMIN_PASSWORD=//' | tr -d '"' | tr -d "'" || true) + if [[ -n "$val" ]]; then + AETHER_PASSWORD="$val" + detail "Password loaded from ${dir}/env" + return 0 + fi + fi + dir="$(dirname "$dir")" + done + error "Aether password not found" - error "Set AETHER_ADMIN_PASSWORD in ${ENV_FILE} or use --password" + error "Source envs//env or use --password" return 1 } @@ -1190,6 +1200,22 @@ parse_args() { # Main # --------------------------------------------------------------------------- +require_config() { + local missing=() + [[ -z "$CLUSTER_REMOTE" ]] && missing+=("cluster_remote") + [[ -z "$CLUSTER_ID" ]] && missing+=("cluster_id") + [[ -z "$VIP" ]] && missing+=("vip") + [[ -z "$HAPROXY_01_IP" ]] && missing+=("haproxy_01_ip") + [[ -z "$HAPROXY_02_IP" ]] && missing+=("haproxy_02_ip") + [[ -z "$BACKEND_IPS" ]] && missing+=("backend_ips") + [[ -z "$AETHER_URL" ]] && missing+=("aether_url") + if [[ ${#missing[@]} -gt 0 ]]; then + error "Required config missing: ${missing[*]}" + error "Use --config FILE to load environment settings (see envs/)" + exit 1 + fi +} + main() { setup_colors parse_args "$@" @@ -1199,6 +1225,11 @@ main() { parse_config "$CONFIG_FILE" fi + # Validate required fields (must come from --config or CLI flags) + if [[ "$ACTION" != "doctor" ]]; then + require_config + fi + echo case "$ACTION" in deploy) action_deploy;; diff --git a/bin/deploy-observability b/bin/deploy-observability index 89632a2..d9f2ba8 100755 --- a/bin/deploy-observability +++ b/bin/deploy-observability @@ -14,29 +14,29 @@ set -euo pipefail readonly VERSION="0.1.0" readonly SCRIPT_NAME="deploy-observability" readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly DASHBOARDS_DIR="${SCRIPT_DIR}/observability-dashboards" +readonly DASHBOARDS_DIR="${SCRIPT_DIR}/data/observability-dashboards" # --------------------------------------------------------------------------- -# Defaults (overridden by CLI flags) +# Defaults (overridden by config file or CLI flags) # --------------------------------------------------------------------------- -CLUSTER_REMOTE="oc-node-01" +CLUSTER_REMOTE="" OVN_NETWORK="net-prod" MONITORING_IP="10.10.10.70" -MONITORING_TARGET="oc-node-02" -FORWARD_VIP="192.168.103.201" +MONITORING_TARGET="" +FORWARD_VIP="" # Fallback defaults — used when cluster is unreachable (e.g., --dry-run) -DEFAULT_INCUS_NODES=("192.168.102.140" "192.168.102.141" "192.168.102.142") -DEFAULT_NODE_EXP_TARGETS=("oc-node-01" "oc-node-02" "oc-node-03") -DEFAULT_NODE_EXP_IPS=("10.10.10.71" "10.10.10.72" "10.10.10.73") +DEFAULT_INCUS_NODES=() +DEFAULT_NODE_EXP_TARGETS=() +DEFAULT_NODE_EXP_IPS=() INCUS_NODES=() NODE_EXP_TARGETS=() NODE_EXP_IPS=() -HAPROXY_IPS=("10.10.10.50" "10.10.10.51") +HAPROXY_IPS=() -HAPROXY_CLUSTER_ID="52" +HAPROXY_CLUSTER_ID="" OC_SERVER_IP="" # Runtime flags @@ -44,6 +44,7 @@ DRY_RUN=false VERBOSE=false QUIET=false RESUME=false +CONFIG_FILE="" ACTION="" MANUAL_NODES="" @@ -1704,6 +1705,48 @@ except Exception as e: fi } +# --------------------------------------------------------------------------- +# Config file parsing +# --------------------------------------------------------------------------- + +parse_config() { + local file="$1" + if [[ ! -f "$file" ]]; then + error "Config file not found: $file" + exit 1 + fi + + local val + val=$(grep -A50 '^observability:' "$file" | grep '^ *cluster_remote:' | head -1 | sed 's/.*: *//' | tr -d '"' || true) + [[ -n "$val" ]] && CLUSTER_REMOTE="$val" + val=$(grep -A50 '^observability:' "$file" | grep '^ *ovn_network:' | head -1 | sed 's/.*: *//' | tr -d '"' || true) + [[ -n "$val" ]] && OVN_NETWORK="$val" + val=$(grep -A50 '^observability:' "$file" | grep '^ *monitoring_ip:' | head -1 | sed 's/.*: *//' | tr -d '"' || true) + [[ -n "$val" ]] && MONITORING_IP="$val" + val=$(grep -A50 '^observability:' "$file" | grep '^ *monitoring_target:' | head -1 | sed 's/.*: *//' | tr -d '"' || true) + [[ -n "$val" ]] && MONITORING_TARGET="$val" + val=$(grep -A50 '^observability:' "$file" | grep '^ *forward_vip:' | head -1 | sed 's/.*: *//' | tr -d '"' || true) + [[ -n "$val" ]] && FORWARD_VIP="$val" + val=$(grep -A50 '^observability:' "$file" | grep '^ *incus_nodes:' | head -1 | sed 's/.*: *//' | tr -d '"' || true) + if [[ -n "$val" ]]; then + IFS=',' read -ra DEFAULT_INCUS_NODES <<< "$val" + fi + val=$(grep -A50 '^observability:' "$file" | grep '^ *node_exp_targets:' | head -1 | sed 's/.*: *//' | tr -d '"' || true) + if [[ -n "$val" ]]; then + IFS=',' read -ra DEFAULT_NODE_EXP_TARGETS <<< "$val" + fi + val=$(grep -A50 '^observability:' "$file" | grep '^ *node_exp_ips:' | head -1 | sed 's/.*: *//' | tr -d '"' || true) + if [[ -n "$val" ]]; then + IFS=',' read -ra DEFAULT_NODE_EXP_IPS <<< "$val" + fi + val=$(grep -A50 '^observability:' "$file" | grep '^ *haproxy_ips:' | head -1 | sed 's/.*: *//' | tr -d '"' || true) + if [[ -n "$val" ]]; then + IFS=',' read -ra HAPROXY_IPS <<< "$val" + fi + val=$(grep -A50 '^observability:' "$file" | grep '^ *haproxy_cluster_id:' | head -1 | sed 's/.*: *//' | tr -d '"' || true) + [[ -n "$val" ]] && HAPROXY_CLUSTER_ID="$val" +} + # --------------------------------------------------------------------------- # Argument parsing # --------------------------------------------------------------------------- @@ -1716,6 +1759,7 @@ parse_args() { --status) ACTION="status";; --cleanup) ACTION="cleanup";; --doctor) ACTION="doctor";; + -c|--config) CONFIG_FILE="$2"; shift;; -r|--remote) CLUSTER_REMOTE="$2"; shift;; --nodes) MANUAL_NODES="$2"; shift;; --monitoring-target) MONITORING_TARGET="$2"; shift;; @@ -1743,6 +1787,18 @@ parse_args() { fi } +require_config() { + local missing=() + [[ -z "$CLUSTER_REMOTE" ]] && missing+=("cluster_remote") + [[ -z "$FORWARD_VIP" ]] && missing+=("forward_vip") + [[ -z "$MONITORING_TARGET" ]] && missing+=("monitoring_target") + if [[ ${#missing[@]} -gt 0 ]]; then + error "Required config missing: ${missing[*]}" + error "Use --config FILE to load environment settings (see envs/)" + exit 1 + fi +} + # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- @@ -1751,6 +1807,16 @@ main() { setup_colors parse_args "$@" + # Load config file if provided + if [[ -n "$CONFIG_FILE" ]]; then + parse_config "$CONFIG_FILE" + fi + + # Validate required fields (must come from --config or CLI flags) + if [[ "$ACTION" != "doctor" ]]; then + require_config + fi + # Discover cluster nodes (populates INCUS_NODES, NODE_EXP_TARGETS, NODE_EXP_IPS) discover_cluster_nodes diff --git a/bin/diagnostic/investigate-boot b/bin/diagnostic/investigate-boot index 5e25efa..e29cafe 100755 --- a/bin/diagnostic/investigate-boot +++ b/bin/diagnostic/investigate-boot @@ -163,6 +163,20 @@ done # --------------------------------------------------------------------------- load_env() { + [[ -n "${PROXMOX_TOKEN_SECRET:-}" ]] && return + + # Check envs/*/env in repo root + local repo_root + repo_root=$(cd "$SCRIPT_DIR/../.." && pwd) + for env_file in "${repo_root}"/envs/*/env; do + if [[ -f "$env_file" ]]; then + # shellcheck disable=SC1091 + source "$env_file" + [[ -n "${PROXMOX_TOKEN_SECRET:-}" ]] && return + fi + done + + # Walk up from script dir (backward compat) local dir="$SCRIPT_DIR" while [[ "$dir" != "/" ]]; do if [[ -f "${dir}/env" ]]; then diff --git a/bin/diagnostic/investigate-parallel b/bin/diagnostic/investigate-parallel index 083d44e..4d4469d 100755 --- a/bin/diagnostic/investigate-parallel +++ b/bin/diagnostic/investigate-parallel @@ -86,6 +86,16 @@ done # --------------------------------------------------------------------------- load_env() { + [[ -n "${PROXMOX_TOKEN_SECRET:-}" ]] && return + + # Check envs/*/env in repo root + local repo_root + repo_root=$(cd "$SCRIPT_DIR/../.." && pwd) + for env_file in "${repo_root}"/envs/*/env; do + [[ -f "$env_file" ]] && { source "$env_file"; [[ -n "${PROXMOX_TOKEN_SECRET:-}" ]] && return; } + done + + # Walk up from script dir (backward compat) local dir="$SCRIPT_DIR" while [[ "$dir" != "/" ]]; do [[ -f "${dir}/env" ]] && { source "${dir}/env"; return; } @@ -97,9 +107,21 @@ load_env [[ -z "${PROXMOX_TOKEN_SECRET:-}" ]] && { error "PROXMOX_TOKEN_SECRET not set"; exit 1; } [[ -z "${PROXMOX_ROOT_PASSWORD:-}" ]] && { error "PROXMOX_ROOT_PASSWORD not set"; exit 1; } +find_proxmox_yaml() { + local repo_root + repo_root=$(cd "$SCRIPT_DIR/../.." && pwd) + for f in "${repo_root}"/envs/*/proxmox.yaml; do + [[ -f "$f" ]] && { echo "$f"; return; } + done + [[ -f "${repo_root}/proxmox.yaml" ]] && { echo "${repo_root}/proxmox.yaml"; return; } + echo "" +} +PROXMOX_YAML=$(find_proxmox_yaml) +[[ -z "$PROXMOX_YAML" ]] && { error "proxmox.yaml not found"; exit 1; } + yaml_get() { local val - val=$(awk -F': ' -v k="$1" '$1 == k {print $2; exit}' "${SCRIPT_DIR}/proxmox.yaml" 2>/dev/null) || val="" + val=$(awk -F': ' -v k="$1" '$1 == k {print $2; exit}' "$PROXMOX_YAML" 2>/dev/null) || val="" val="${val#\"}" ; val="${val%\"}" ; val="${val#\'}" ; val="${val%\'}" echo "${val:-${2:-}}" } @@ -448,7 +470,7 @@ for ((i=0; i repo), script parent, cwd + local repo_root + repo_root=$(cd "$SCRIPT_DIR/../.." && pwd) + + # Check envs/*/proxmox.yaml in repo root + for f in "${repo_root}"/envs/*/proxmox.yaml; do + if [[ -f "$f" ]]; then + echo "$f" + return + fi + done + + # Walk up from script dir (backward compat) local search_dirs=( "${SCRIPT_DIR}/.." - "${SCRIPT_DIR}/../.." + "${repo_root}" "$(pwd)" ) for dir in "${search_dirs[@]}"; do @@ -113,10 +139,6 @@ find_proxmox_yaml() { echo "${dir}/proxmox.yaml" return fi - if [[ -f "${dir}/incusos/proxmox.yaml" ]]; then - echo "${dir}/incusos/proxmox.yaml" - return - fi done echo "" } diff --git a/bin/incusos-proxmox b/bin/incusos-proxmox index e3f1322..ff81d8e 100755 --- a/bin/incusos-proxmox +++ b/bin/incusos-proxmox @@ -22,10 +22,10 @@ readonly MANAGED_MARKER="[incusos-lab:managed]" # --------------------------------------------------------------------------- # Auto-load env file (secrets like PROXMOX_TOKEN_SECRET) # --------------------------------------------------------------------------- -# Searches for an 'env' file: first next to the script, then walking up from -# the current directory. Sources it if found and not already loaded. This is -# a convenience -- users can still export variables manually or source the -# file themselves. +# Searches for an 'env' file. Sources it if found and not already loaded. +# Priority: already exported > envs/*/env in repo > walk up from cwd. +# This is a convenience -- users can still export variables manually or +# source the file themselves (e.g., source envs/hetzner/env). load_env_file() { # Skip if the secret is already set (user exported it, or env already sourced) @@ -35,18 +35,24 @@ load_env_file() { local script_real script_real=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + local repo_root + repo_root=$(cd "$script_real/.." && pwd) - # Check: script directory's parent (repo root when script is in incusos/) - if [[ -f "${script_real}/../env" ]]; then - # shellcheck source=/dev/null - source "${script_real}/../env" - return 0 - fi + # Check: envs/*/env in repo root (per-environment secrets) + for env_file in "${repo_root}"/envs/*/env; do + if [[ -f "$env_file" ]]; then + # shellcheck source=/dev/null + source "$env_file" + if [[ -n "${PROXMOX_TOKEN_SECRET:-}" ]]; then + return 0 + fi + fi + done - # Check: script directory itself - if [[ -f "${script_real}/env" ]]; then + # Check: repo root env file (backward compat) + if [[ -f "${repo_root}/env" ]]; then # shellcheck source=/dev/null - source "${script_real}/env" + source "${repo_root}/env" return 0 fi diff --git a/bin/manage-dashboards b/bin/manage-dashboards index 1298971..99903b8 100755 --- a/bin/manage-dashboards +++ b/bin/manage-dashboards @@ -13,7 +13,7 @@ set -euo pipefail readonly VERSION="0.1.0" readonly SCRIPT_NAME="manage-dashboards" readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly DASHBOARDS_DIR="${SCRIPT_DIR}/observability-dashboards" +readonly DASHBOARDS_DIR="${SCRIPT_DIR}/data/observability-dashboards" readonly MANIFEST_FILE="${DASHBOARDS_DIR}/dashboard.yaml" # ---------------------------------------------------------------------------