incus-contrib/incusos/deploy-awx

1134 lines
39 KiB
Bash
Executable File

#!/usr/bin/env bash
# deploy-awx - Deploy and manage AWX on an Incus cluster
#
# Deploys a Debian 12 VM with K3s + AWX Operator, configures AWX for
# Aether integration (project, inventory, job templates), and provides
# ongoing health checks and healing.
#
# Usage: deploy-awx [OPTIONS]
# Run 'deploy-awx --help' for full usage information.
set -euo pipefail
readonly VERSION="0.1.0"
readonly SCRIPT_NAME="deploy-awx"
readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly MANIFESTS_DIR="${SCRIPT_DIR}/awx-manifests"
readonly AWX_NAMESPACE="awx"
# ---------------------------------------------------------------------------
# Defaults (overridden by config file or CLI flags)
# ---------------------------------------------------------------------------
VM_NAME="awx"
TARGET_REMOTE="oc-node-02"
AWX_IP="192.168.102.161"
AWX_PREFIX="22"
AWX_GATEWAY="192.168.100.1"
AWX_DNS="192.168.100.1"
AWX_CPU="4"
AWX_MEMORY="8GiB"
AWX_DISK="40GiB"
GIT_REPO="ssh://git@192.168.1.200:2222/maarten/incus-contrib.git"
GIT_BRANCH="master"
PLAYBOOK_DIR="playbooks"
AETHER_URL="https://192.168.102.160:8443"
AETHER_CLUSTER_ID="52"
# Runtime flags
DRY_RUN=false
VERBOSE=false
QUIET=false
CONFIG_FILE=""
ACTION=""
# ---------------------------------------------------------------------------
# Color and formatting
# ---------------------------------------------------------------------------
setup_colors() {
if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]] && [[ "${TERM:-}" != "dumb" ]]; then
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
BOLD='\033[1m'
DIM='\033[2m'
RESET='\033[0m'
else
RED='' GREEN='' YELLOW='' BLUE='' CYAN='' BOLD='' DIM='' RESET=''
fi
}
info() { [[ "$QUIET" == true ]] && return; echo -e "${BLUE}[info]${RESET} $*"; }
success() { [[ "$QUIET" == true ]] && return; echo -e "${GREEN}[ok]${RESET} $*"; }
warn() { echo -e "${YELLOW}[warn]${RESET} $*" >&2; }
error() { echo -e "${RED}[error]${RESET} $*" >&2; }
step() { [[ "$QUIET" == true ]] && return; echo -e "${CYAN}[step]${RESET} ${BOLD}$*${RESET}"; }
detail() { [[ "$VERBOSE" != true ]] && return; echo -e "${DIM} $*${RESET}"; }
dry_run_guard() {
if [[ "$DRY_RUN" == true ]]; then
info "(dry-run) $*"
return 1
fi
return 0
}
# ---------------------------------------------------------------------------
# Help
# ---------------------------------------------------------------------------
usage() {
cat <<EOF
${BOLD}${SCRIPT_NAME}${RESET} v${VERSION} - Deploy and manage AWX on an Incus cluster
${BOLD}USAGE${RESET}
${SCRIPT_NAME} [OPTIONS]
${BOLD}ACTIONS${RESET}
--deploy Full deploy: create VM, install K3s, deploy AWX, configure
--status Check AWX health, K3s pods, Aether connection
--heal Restart failed pods, re-sync project, re-validate templates
--configure (Re-)configure AWX: project, inventory, templates
--join-aether Register AWX endpoint in Aether + bind to cluster
--cleanup Destroy the AWX VM
--doctor Check prerequisites (incus CLI, curl, python3, kubectl)
${BOLD}OPTIONS${RESET}
-c, --config FILE Configuration file (YAML with awx: section)
-n, --dry-run Preview actions without executing
-v, --verbose Show detailed output
-q, --quiet Suppress informational output
-h, --help Show this help message
${BOLD}CONFIGURATION${RESET}
The script uses built-in defaults for the lab environment. Override with
a YAML config file containing an 'awx:' section:
awx:
vm_name: awx
target_node: oc-node-02
ip: 192.168.102.161/22
gateway: 192.168.100.1
dns: 192.168.100.1
cpu: 4
memory: 8GiB
disk: 40GiB
git_repo: ssh://git@192.168.1.200:2222/maarten/incus-contrib.git
git_branch: master
aether_url: https://192.168.102.160:8443
aether_cluster_id: 52
${BOLD}EXAMPLES${RESET}
# Check prerequisites
${SCRIPT_NAME} --doctor
# Full deployment with defaults
${SCRIPT_NAME} --deploy
# Preview deployment without executing
${SCRIPT_NAME} --deploy --dry-run
# Check AWX health
${SCRIPT_NAME} --status
# Heal broken pods
${SCRIPT_NAME} --heal
# Reconfigure AWX (project, templates)
${SCRIPT_NAME} --configure
# Register with Aether
${SCRIPT_NAME} --join-aether
# Destroy AWX VM
${SCRIPT_NAME} --cleanup
EOF
}
# ---------------------------------------------------------------------------
# Config parsing (simple YAML — same pattern as other scripts)
# ---------------------------------------------------------------------------
parse_config() {
local file="$1"
if [[ ! -f "$file" ]]; then
error "Config file not found: $file"
exit 1
fi
# Extract awx: section values using grep + sed (no yq dependency)
local val
val=$(grep -A50 '^awx:' "$file" | grep '^ *vm_name:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && VM_NAME="$val"
val=$(grep -A50 '^awx:' "$file" | grep '^ *target_node:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && TARGET_REMOTE="$val"
val=$(grep -A50 '^awx:' "$file" | grep '^ *ip:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
if [[ -n "$val" ]]; then
AWX_IP="${val%%/*}"
[[ "$val" == */* ]] && AWX_PREFIX="${val#*/}"
fi
val=$(grep -A50 '^awx:' "$file" | grep '^ *gateway:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && AWX_GATEWAY="$val"
val=$(grep -A50 '^awx:' "$file" | grep '^ *dns:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && AWX_DNS="$val"
val=$(grep -A50 '^awx:' "$file" | grep '^ *cpu:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && AWX_CPU="$val"
val=$(grep -A50 '^awx:' "$file" | grep '^ *memory:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && AWX_MEMORY="$val"
val=$(grep -A50 '^awx:' "$file" | grep '^ *disk:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && AWX_DISK="$val"
val=$(grep -A50 '^awx:' "$file" | grep '^ *git_repo:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && GIT_REPO="$val"
val=$(grep -A50 '^awx:' "$file" | grep '^ *git_branch:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && GIT_BRANCH="$val"
val=$(grep -A50 '^awx:' "$file" | grep '^ *aether_url:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && AETHER_URL="$val"
val=$(grep -A50 '^awx:' "$file" | grep '^ *aether_cluster_id:' | head -1 | sed 's/.*: *//' | tr -d '"' || true)
[[ -n "$val" ]] && AETHER_CLUSTER_ID="$val"
}
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
vm_ref() {
echo "${TARGET_REMOTE}:${VM_NAME}"
}
vm_exists() {
incus info "$(vm_ref)" &>/dev/null 2>&1
}
vm_running() {
local state
state=$(incus info "$(vm_ref)" 2>/dev/null | grep -i '^status:' | awk '{print $2}')
[[ "$state" == "RUNNING" || "$state" == "Running" ]]
}
vm_exec() {
incus exec "$(vm_ref)" -- "$@"
}
awx_url() {
# AWX is exposed via K3s NodePort on port 30080 (HTTP)
# Traefik ingress returns 404 for IP-based access
echo "http://${AWX_IP}:30080"
}
awx_api() {
local method="$1" path="$2"
shift 2
curl -sk -X "$method" "$(awx_url)${path}" \
-H "Content-Type: application/json" \
"$@"
}
awx_api_auth() {
local method="$1" path="$2"
shift 2
local token
token=$(get_awx_admin_password)
curl -sk -X "$method" "$(awx_url)${path}" \
--user "admin:${token}" \
-H "Content-Type: application/json" \
"$@"
}
get_awx_admin_password() {
vm_exec kubectl -n "$AWX_NAMESPACE" get secret awx-admin-password \
-o jsonpath='{.data.password}' 2>/dev/null | base64 -d
}
wait_for_vm_agent() {
local timeout="${1:-120}"
local elapsed=0
info "Waiting for VM agent (up to ${timeout}s)..."
while ! vm_exec true &>/dev/null 2>&1; do
sleep 5
elapsed=$((elapsed + 5))
if [[ $elapsed -ge $timeout ]]; then
error "VM agent not available after ${timeout}s"
return 1
fi
done
success "VM agent connected (${elapsed}s)"
}
wait_for_awx_web() {
local timeout="${1:-600}"
local elapsed=0
info "Waiting for AWX web interface (up to ${timeout}s)..."
while ! curl -sk -o /dev/null -w '%{http_code}' "$(awx_url)/api/v2/ping/" 2>/dev/null | grep -q '200'; do
sleep 10
elapsed=$((elapsed + 10))
if [[ $elapsed -ge $timeout ]]; then
error "AWX web not responding after ${timeout}s"
return 1
fi
[[ "$VERBOSE" == true ]] && detail "Still waiting... (${elapsed}s)"
done
success "AWX web interface ready (${elapsed}s)"
}
# ---------------------------------------------------------------------------
# Doctor — prerequisite checks
# ---------------------------------------------------------------------------
action_doctor() {
step "Checking prerequisites"
local ok=true
# incus CLI
if command -v incus &>/dev/null; then
success "incus CLI: $(incus version 2>/dev/null || echo 'installed')"
else
error "incus CLI not found — install the Incus client"
ok=false
fi
# curl
if command -v curl &>/dev/null; then
success "curl: $(curl --version | head -1 | awk '{print $2}')"
else
error "curl not found"
ok=false
fi
# python3
if command -v python3 &>/dev/null; then
success "python3: $(python3 --version 2>&1 | awk '{print $2}')"
else
error "python3 not found"
ok=false
fi
# base64
if command -v base64 &>/dev/null; then
success "base64: available"
else
error "base64 not found"
ok=false
fi
# Manifest files
if [[ -d "$MANIFESTS_DIR" ]]; then
success "AWX manifests: $MANIFESTS_DIR"
else
error "AWX manifests not found at $MANIFESTS_DIR"
ok=false
fi
# Target remote
if incus remote list -f csv 2>/dev/null | grep -q "^${TARGET_REMOTE},"; then
success "Incus remote '${TARGET_REMOTE}': reachable"
else
warn "Incus remote '${TARGET_REMOTE}' not found (needed for deployment)"
fi
# Existing VM
if vm_exists; then
info "AWX VM '$(vm_ref)' exists"
if vm_running; then
success " VM is running"
# Check K3s
if vm_exec kubectl get nodes &>/dev/null 2>&1; then
success " K3s is responding"
else
warn " K3s not responding"
fi
# Check AWX pods
local pod_count
pod_count=$(vm_exec kubectl -n "$AWX_NAMESPACE" get pods --no-headers 2>/dev/null | wc -l || echo 0)
info " AWX pods: ${pod_count}"
else
warn " VM is not running"
fi
else
info "AWX VM '$(vm_ref)' does not exist (--deploy to create)"
fi
echo
if [[ "$ok" == true ]]; then
success "All prerequisites satisfied"
else
error "Some prerequisites missing — fix the issues above"
return 1
fi
}
# ---------------------------------------------------------------------------
# Deploy — full deployment pipeline
# ---------------------------------------------------------------------------
action_deploy() {
step "Deploying AWX on $(vm_ref)"
echo
info " Target: ${TARGET_REMOTE}"
info " VM: ${VM_NAME}"
info " IP: ${AWX_IP}/${AWX_PREFIX}"
info " CPU: ${AWX_CPU}"
info " Memory: ${AWX_MEMORY}"
info " Disk: ${AWX_DISK}"
echo
if vm_exists; then
if vm_running; then
error "AWX VM already exists and is running. Use --status, --heal, or --cleanup first."
return 1
else
warn "AWX VM exists but is not running — starting it"
if dry_run_guard "incus start $(vm_ref)"; then
incus start "$(vm_ref)"
fi
return 0
fi
fi
phase_create_vm
phase_configure_network
phase_install_k3s
phase_deploy_awx_operator
phase_deploy_awx_instance
phase_verify
echo
success "AWX deployment complete!"
info " Web UI: $(awx_url)/"
info " API: $(awx_url)/api/v2/ping/"
local pw
pw=$(get_awx_admin_password 2>/dev/null || echo "<retrieve with --status>")
info " Admin: admin / ${pw}"
echo
info "Next steps:"
info " ${SCRIPT_NAME} --configure # set up project, inventory, templates"
info " ${SCRIPT_NAME} --join-aether # register with Aether"
}
phase_create_vm() {
step "Phase 1: Create VM"
if ! dry_run_guard "incus launch images:debian/12 $(vm_ref) --vm --target ${TARGET_REMOTE} ..."; then
return 0
fi
incus launch images:debian/12 "$(vm_ref)" --vm \
--target "$TARGET_REMOTE" \
-c "limits.cpu=${AWX_CPU}" \
-c "limits.memory=${AWX_MEMORY}" \
-d "root,size=${AWX_DISK}"
success "VM created"
detail "Waiting 10s for initial boot..."
sleep 10
# Switch to macvlan for direct VLAN access
info "Configuring macvlan NIC (parent=mgmt)"
incus config device remove "$(vm_ref)" eth0 2>/dev/null || true
incus config device add "$(vm_ref)" eth0 nic \
nictype=macvlan parent=mgmt
success "NIC switched to macvlan — VM will restart networking"
}
phase_configure_network() {
step "Phase 2: Configure static IP + hostname"
wait_for_vm_agent 120
if ! dry_run_guard "Configure networking on $(vm_ref)"; then
return 0
fi
# Debian 12 images use systemd-networkd (not netplan)
# The macvlan NIC appears as enp5s0 in Incus VMs
vm_exec bash -c "
hostnamectl set-hostname '${VM_NAME}'
# Disable any existing DHCP config
rm -f /etc/network/interfaces.d/* 2>/dev/null || true
# Configure static IP via systemd-networkd
mkdir -p /etc/systemd/network
cat > /etc/systemd/network/10-static.network << 'NETWORKD'
[Match]
Name=enp5s0
[Network]
Address=${AWX_IP}/${AWX_PREFIX}
Gateway=${AWX_GATEWAY}
DNS=${AWX_DNS}
NETWORKD
systemctl enable systemd-networkd
systemctl restart systemd-networkd
"
success "Static IP ${AWX_IP}/${AWX_PREFIX} configured"
# Verify connectivity
detail "Verifying network connectivity..."
sleep 3
if vm_exec ping -c 1 -W 3 "${AWX_GATEWAY}" &>/dev/null; then
success "Gateway reachable from VM"
else
warn "Cannot reach gateway — check VLAN/network config"
fi
}
phase_install_k3s() {
step "Phase 3: Install K3s"
if ! dry_run_guard "Install K3s on $(vm_ref)"; then
return 0
fi
# Ensure curl is available (minimal Debian 12 may not have it)
info "Installing prerequisites..."
vm_exec bash -c "
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq
apt-get install -y -qq curl python3 2>&1 | tail -1
"
vm_exec bash -c "
curl -sfL https://get.k3s.io | sh -s - --write-kubeconfig-mode 644
"
success "K3s installed"
info "Waiting for K3s node to be ready..."
vm_exec kubectl wait --for=condition=Ready node --all --timeout=120s
success "K3s node ready"
}
phase_deploy_awx_operator() {
step "Phase 4: Deploy AWX Operator"
if ! dry_run_guard "Deploy AWX Operator on $(vm_ref)"; then
return 0
fi
# Create namespace
vm_exec kubectl create namespace "$AWX_NAMESPACE" 2>/dev/null || true
# Push kustomization files
info "Pushing operator manifests..."
incus file push "${MANIFESTS_DIR}/operator/kustomization.yaml" \
"$(vm_ref)/tmp/awx-operator-kustomization.yaml"
vm_exec bash -c "
mkdir -p /opt/awx/operator
cp /tmp/awx-operator-kustomization.yaml /opt/awx/operator/kustomization.yaml
kubectl apply -k /opt/awx/operator/
"
info "Waiting for AWX Operator (up to 5 min)..."
vm_exec kubectl -n "$AWX_NAMESPACE" wait \
--for=condition=Available \
deployment/awx-operator-controller-manager \
--timeout=300s
success "AWX Operator deployed"
}
phase_deploy_awx_instance() {
step "Phase 5: Deploy AWX instance"
if ! dry_run_guard "Deploy AWX instance on $(vm_ref)"; then
return 0
fi
# Push AWX CR manifests
info "Pushing AWX instance manifests..."
incus file push "${MANIFESTS_DIR}/base/awx.yaml" \
"$(vm_ref)/tmp/awx-cr.yaml"
incus file push "${MANIFESTS_DIR}/base/kustomization.yaml" \
"$(vm_ref)/tmp/awx-kustomization.yaml"
vm_exec bash -c "
mkdir -p /opt/awx/base
cp /tmp/awx-cr.yaml /opt/awx/base/awx.yaml
cp /tmp/awx-kustomization.yaml /opt/awx/base/kustomization.yaml
kubectl apply -k /opt/awx/base/
"
info "AWX instance deploying — this takes 5-10 minutes..."
info "Waiting for AWX pods to start..."
# Wait for AWX deployment to exist and become available
local elapsed=0
local timeout=600
while ! vm_exec kubectl -n "$AWX_NAMESPACE" get deployment awx-web &>/dev/null 2>&1; do
sleep 15
elapsed=$((elapsed + 15))
if [[ $elapsed -ge $timeout ]]; then
error "AWX web deployment not created after ${timeout}s"
warn "Check pod status: incus exec $(vm_ref) -- kubectl -n awx get pods"
return 1
fi
[[ "$VERBOSE" == true ]] && detail "Waiting for AWX web deployment... (${elapsed}s)"
done
vm_exec kubectl -n "$AWX_NAMESPACE" wait \
--for=condition=Available \
deployment/awx-web \
--timeout=300s || {
warn "AWX web deployment not fully available yet — checking pods"
vm_exec kubectl -n "$AWX_NAMESPACE" get pods
}
success "AWX instance deployed"
}
phase_verify() {
step "Phase 6: Verify AWX"
wait_for_awx_web 300
local admin_pw
admin_pw=$(get_awx_admin_password)
success "Admin password retrieved"
# Test API ping (NodePort 30080)
local ping_result
ping_result=$(curl -sk "$(awx_url)/api/v2/ping/" 2>/dev/null)
if echo "$ping_result" | python3 -c "import sys,json; json.load(sys.stdin)" &>/dev/null; then
success "AWX API responding"
else
warn "AWX API returned unexpected response"
fi
}
# ---------------------------------------------------------------------------
# Status — health checks
# ---------------------------------------------------------------------------
action_status() {
step "AWX Status for $(vm_ref)"
echo
# VM existence and state
if ! vm_exists; then
error "AWX VM does not exist — use --deploy to create"
return 1
fi
if ! vm_running; then
error "AWX VM exists but is not running"
return 1
fi
success "VM: running"
# K3s
if vm_exec kubectl get nodes &>/dev/null 2>&1; then
success "K3s: healthy"
if [[ "$VERBOSE" == true ]]; then
vm_exec kubectl get nodes -o wide
fi
else
error "K3s: not responding"
return 1
fi
# AWX pods
echo
info "AWX pods:"
vm_exec kubectl -n "$AWX_NAMESPACE" get pods -o wide 2>/dev/null || warn "Cannot list pods"
# Pod health summary
local total running
total=$(vm_exec kubectl -n "$AWX_NAMESPACE" get pods --no-headers 2>/dev/null | wc -l || echo 0)
running=$(vm_exec kubectl -n "$AWX_NAMESPACE" get pods --no-headers 2>/dev/null | grep -c 'Running' || echo 0)
echo
if [[ "$total" -gt 0 && "$running" -eq "$total" ]]; then
success "All ${total} pods running"
else
warn "Pods: ${running}/${total} running"
fi
# AWX web (NodePort 30080)
local http_code
http_code=$(curl -sk -o /dev/null -w '%{http_code}' "$(awx_url)/api/v2/ping/" 2>/dev/null || echo "000")
if [[ "$http_code" == "200" ]]; then
success "AWX API: responding at $(awx_url) (HTTP ${http_code})"
else
error "AWX API: not responding (HTTP ${http_code})"
fi
# Admin password
local admin_pw
admin_pw=$(get_awx_admin_password 2>/dev/null || echo "")
if [[ -n "$admin_pw" ]]; then
success "Admin password: ${admin_pw}"
else
warn "Cannot retrieve admin password"
fi
# AWX version
local version
version=$(curl -sk "$(awx_url)/api/v2/ping/" 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('version','unknown'))" 2>/dev/null || echo "unknown")
info "AWX version: ${version}"
# Aether connectivity
echo
local aether_code
aether_code=$(curl -sk -o /dev/null -w '%{http_code}' "${AETHER_URL}/api/health" 2>/dev/null || echo "000")
if [[ "$aether_code" == "200" || "$aether_code" == "401" || "$aether_code" == "302" ]]; then
success "Aether: reachable at ${AETHER_URL}"
else
warn "Aether: not reachable (HTTP ${aether_code})"
fi
}
# ---------------------------------------------------------------------------
# Heal — fix common issues
# ---------------------------------------------------------------------------
action_heal() {
step "Healing AWX on $(vm_ref)"
echo
if ! vm_exists || ! vm_running; then
error "AWX VM is not running"
return 1
fi
# 1. K3s service
info "Checking K3s service..."
if vm_exec systemctl is-active k3s &>/dev/null; then
success "K3s service: active"
else
warn "K3s service not active — restarting"
if dry_run_guard "systemctl restart k3s"; then
vm_exec systemctl restart k3s
sleep 10
fi
fi
# 2. Unhealthy pods
info "Checking for unhealthy pods..."
local unhealthy
unhealthy=$(vm_exec kubectl -n "$AWX_NAMESPACE" get pods --no-headers 2>/dev/null \
| grep -vE 'Running|Completed' | awk '{print $1}' || true)
if [[ -n "$unhealthy" ]]; then
warn "Unhealthy pods found — deleting for operator to recreate"
for pod in $unhealthy; do
info " Deleting pod: $pod"
if dry_run_guard "kubectl delete pod $pod"; then
vm_exec kubectl -n "$AWX_NAMESPACE" delete pod "$pod" --grace-period=30
fi
done
info "Waiting 30s for pods to restart..."
sleep 30
else
success "All pods healthy"
fi
# 3. AWX web responsiveness (NodePort 30080)
info "Checking AWX web..."
local http_code
http_code=$(curl -sk -o /dev/null -w '%{http_code}' "$(awx_url)/api/v2/ping/" 2>/dev/null || echo "000")
if [[ "$http_code" == "200" ]]; then
success "AWX web: responding"
else
warn "AWX web not responding — restarting deployment"
if dry_run_guard "kubectl rollout restart deployment/awx-web"; then
vm_exec kubectl -n "$AWX_NAMESPACE" rollout restart deployment/awx-web 2>/dev/null || true
vm_exec kubectl -n "$AWX_NAMESPACE" rollout restart deployment/awx-task 2>/dev/null || true
wait_for_awx_web 300
fi
fi
# 4. Project sync
info "Triggering project sync..."
local project_id
project_id=$(awx_api_auth GET "/api/v2/projects/?name=incus-contrib" 2>/dev/null \
| python3 -c "import sys,json; r=json.load(sys.stdin); print(r['results'][0]['id'] if r.get('count',0)>0 else '')" 2>/dev/null || true)
if [[ -n "$project_id" ]]; then
if dry_run_guard "POST /api/v2/projects/${project_id}/update/"; then
awx_api_auth POST "/api/v2/projects/${project_id}/update/" 2>/dev/null || true
success "Project sync triggered (ID: ${project_id})"
fi
else
info "No project configured yet — use --configure"
fi
echo
success "Heal complete"
}
# ---------------------------------------------------------------------------
# Configure — set up AWX project, inventory, templates
# ---------------------------------------------------------------------------
action_configure() {
step "Configuring AWX"
echo
if ! vm_running; then
error "AWX VM is not running"
return 1
fi
wait_for_awx_web 60
local admin_pw
admin_pw=$(get_awx_admin_password)
detail "Admin password: ${admin_pw}"
# Helper for authenticated API calls with password
awx_auth() {
local method="$1" path="$2"
shift 2
curl -sk -X "$method" "$(awx_url)${path}" \
--user "admin:${admin_pw}" \
-H "Content-Type: application/json" \
"$@"
}
# 1. Push Incus client cert/key to AWX project directory
# Playbooks use Incus REST API (not SSH) — cert is required.
# AWX EE mounts project dir at /runner/project/ during job execution.
step "Pushing Incus client certificate to AWX"
local incus_cert="${HOME}/.config/incus/client.crt"
local incus_key="${HOME}/.config/incus/client.key"
if [[ -f "$incus_cert" && -f "$incus_key" ]]; then
local task_pod
task_pod=$(vm_exec kubectl -n "$AWX_NAMESPACE" get pods \
-l app.kubernetes.io/component=awx-task -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
if [[ -n "$task_pod" ]]; then
local project_dir="/var/lib/awx/projects/incus-contrib"
vm_exec kubectl -n "$AWX_NAMESPACE" exec "$task_pod" -- mkdir -p "$project_dir"
# Push cert via base64 to avoid file push permission issues
local cert_b64 key_b64
cert_b64=$(base64 -w0 "$incus_cert")
key_b64=$(base64 -w0 "$incus_key")
vm_exec kubectl -n "$AWX_NAMESPACE" exec "$task_pod" -- \
bash -c "echo '${cert_b64}' | base64 -d > ${project_dir}/incus-client.crt"
vm_exec kubectl -n "$AWX_NAMESPACE" exec "$task_pod" -- \
bash -c "echo '${key_b64}' | base64 -d > ${project_dir}/incus-client.key"
vm_exec kubectl -n "$AWX_NAMESPACE" exec "$task_pod" -- \
chmod 600 "${project_dir}/incus-client.key"
success "Incus client cert/key pushed to AWX task pod"
else
warn "Cannot find AWX task pod — push cert manually later"
fi
else
warn "Incus client cert not found at ${incus_cert}"
warn "Playbooks need the cert to talk to Incus API — push it manually"
fi
# 2. Create manual project (local_path)
# AWX EE containers cannot reach the private git repo, so we use a
# manual project with files pushed directly to the task pod.
step "Creating project"
local project_id
project_id=$(awx_auth GET "/api/v2/projects/?name=incus-contrib" 2>/dev/null \
| python3 -c "import sys,json; r=json.load(sys.stdin); print(r['results'][0]['id'] if r.get('count',0)>0 else '')" 2>/dev/null || true)
if [[ -z "$project_id" ]]; then
project_id=$(awx_auth POST "/api/v2/projects/" \
-d '{"name":"incus-contrib","organization":1,"scm_type":"","local_path":"incus-contrib"}' 2>/dev/null \
| python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))" 2>/dev/null || true)
[[ -n "$project_id" ]] && success "Project created (ID: ${project_id}, local_path)" || warn "Failed to create project"
else
success "Project exists (ID: ${project_id})"
fi
# Push playbooks to AWX task pod project directory
if [[ -n "$project_id" ]]; then
local task_pod
task_pod=$(vm_exec kubectl -n "$AWX_NAMESPACE" get pods \
-l app.kubernetes.io/component=awx-task -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
if [[ -n "$task_pod" ]]; then
local project_dir="/var/lib/awx/projects/incus-contrib"
vm_exec kubectl -n "$AWX_NAMESPACE" exec "$task_pod" -- mkdir -p "${project_dir}/playbooks"
for playbook in post-deploy.yml decommission.yml; do
local pb_path="${SCRIPT_DIR}/../ansible/playbooks/${playbook}"
if [[ -f "$pb_path" ]]; then
local pb_b64
pb_b64=$(base64 -w0 "$pb_path")
vm_exec kubectl -n "$AWX_NAMESPACE" exec "$task_pod" -- \
bash -c "echo '${pb_b64}' | base64 -d > ${project_dir}/playbooks/${playbook}"
detail "Pushed ${playbook}"
else
warn "Playbook not found: ${pb_path}"
fi
done
success "Playbooks pushed to AWX project directory"
else
warn "Cannot find AWX task pod — push playbooks manually"
fi
fi
# 3. Create inventory
step "Creating inventory"
local inventory_id
inventory_id=$(awx_auth GET "/api/v2/inventories/?name=incus-instances" 2>/dev/null \
| python3 -c "import sys,json; r=json.load(sys.stdin); print(r['results'][0]['id'] if r.get('count',0)>0 else '')" 2>/dev/null || true)
if [[ -z "$inventory_id" ]]; then
inventory_id=$(awx_auth POST "/api/v2/inventories/" \
-d '{"name":"incus-instances","organization":1}' 2>/dev/null \
| python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))" 2>/dev/null || true)
[[ -n "$inventory_id" ]] && success "Inventory created (ID: ${inventory_id})" || warn "Failed to create inventory"
else
success "Inventory exists (ID: ${inventory_id})"
fi
# 4. Create machine credential (required by AWX templates, though
# playbooks use Incus API not SSH)
step "Creating machine credential"
local machine_cred_id
machine_cred_id=$(awx_auth GET "/api/v2/credentials/?name=incus-instances" 2>/dev/null \
| python3 -c "import sys,json; r=json.load(sys.stdin); print(r['results'][0]['id'] if r.get('count',0)>0 else '')" 2>/dev/null || true)
if [[ -z "$machine_cred_id" ]]; then
machine_cred_id=$(awx_auth POST "/api/v2/credentials/" \
-d '{"name":"incus-instances","organization":1,"credential_type":1,"inputs":{"username":"root"}}' 2>/dev/null \
| python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))" 2>/dev/null || true)
[[ -n "$machine_cred_id" ]] && success "Machine credential created (ID: ${machine_cred_id})" || warn "Failed to create machine credential"
else
success "Machine credential exists (ID: ${machine_cred_id})"
fi
# 5. Create job templates
step "Creating job templates"
for tmpl_name in post-deploy decommission; do
local tmpl_id
tmpl_id=$(awx_auth GET "/api/v2/job_templates/?name=${tmpl_name}" 2>/dev/null \
| python3 -c "import sys,json; r=json.load(sys.stdin); print(r['results'][0]['id'] if r.get('count',0)>0 else '')" 2>/dev/null || true)
if [[ -z "$tmpl_id" ]]; then
local tmpl_payload
tmpl_payload=$(python3 -c "
import json
d = {
'name': '${tmpl_name}',
'organization': 1,
'job_type': 'run',
'ask_variables_on_launch': True,
'playbook': '${PLAYBOOK_DIR}/${tmpl_name}.yml'
}
pid = '${project_id}'
iid = '${inventory_id}'
cid = '${machine_cred_id}'
if pid: d['project'] = int(pid)
if iid: d['inventory'] = int(iid)
print(json.dumps(d))")
tmpl_id=$(awx_auth POST "/api/v2/job_templates/" -d "$tmpl_payload" 2>/dev/null \
| python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))" 2>/dev/null || true)
# Associate credential separately (templates use credential list)
if [[ -n "$tmpl_id" && -n "$machine_cred_id" ]]; then
awx_auth POST "/api/v2/job_templates/${tmpl_id}/credentials/" \
-d "{\"id\":${machine_cred_id}}" 2>/dev/null || true
fi
[[ -n "$tmpl_id" ]] && success "Template '${tmpl_name}' created (ID: ${tmpl_id})" || warn "Failed to create template '${tmpl_name}'"
else
success "Template '${tmpl_name}' exists (ID: ${tmpl_id})"
fi
done
echo
success "AWX configuration complete"
echo
info "Template IDs (for Aether binding):"
for tmpl_name in post-deploy decommission; do
local tid
tid=$(awx_auth GET "/api/v2/job_templates/?name=${tmpl_name}" 2>/dev/null \
| python3 -c "import sys,json; r=json.load(sys.stdin); print(r['results'][0]['id'] if r.get('count',0)>0 else '')" 2>/dev/null || true)
info " ${tmpl_name}: ${tid:-unknown}"
done
echo
info "Next: ${SCRIPT_NAME} --join-aether"
}
# ---------------------------------------------------------------------------
# Join Aether — register AWX endpoint + bind to cluster
# ---------------------------------------------------------------------------
action_join_aether() {
step "Registering AWX with Aether"
echo
if ! vm_running; then
error "AWX VM is not running"
return 1
fi
# 1. Create Personal Access Token
step "Creating AWX Personal Access Token"
local admin_pw
admin_pw=$(get_awx_admin_password)
local pat_response
pat_response=$(curl -sk -X POST "$(awx_url)/api/v2/users/1/personal_tokens/" \
--user "admin:${admin_pw}" \
-H "Content-Type: application/json" \
-d '{"description": "Aether integration", "scope": "write"}' 2>/dev/null)
local awx_token
awx_token=$(echo "$pat_response" | python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null || true)
if [[ -z "$awx_token" ]]; then
error "Failed to create PAT — check AWX admin credentials"
detail "Response: ${pat_response}"
return 1
fi
success "PAT created"
# 2. Get template IDs
local post_deploy_id decommission_id
post_deploy_id=$(curl -sk "$(awx_url)/api/v2/job_templates/?name=post-deploy" \
--user "admin:${admin_pw}" 2>/dev/null \
| python3 -c "import sys,json; r=json.load(sys.stdin); print(r['results'][0]['id'] if r.get('count',0)>0 else '')" 2>/dev/null || true)
decommission_id=$(curl -sk "$(awx_url)/api/v2/job_templates/?name=decommission" \
--user "admin:${admin_pw}" 2>/dev/null \
| python3 -c "import sys,json; r=json.load(sys.stdin); print(r['results'][0]['id'] if r.get('count',0)>0 else '')" 2>/dev/null || true)
info "Post-deploy template ID: ${post_deploy_id:-not found}"
info "Decommission template ID: ${decommission_id:-not found}"
echo
info "To complete Aether integration:"
echo
info "1. Log into Aether at ${AETHER_URL}"
info "2. Navigate to Settings → Ansible Automation (/awx-endpoints)"
info "3. Add Endpoint:"
info " Name: lab-awx"
info " URL: $(awx_url)"
info " Token: ${awx_token}"
info " Verify SSL: unchecked"
info "4. Configure cluster AWX binding:"
info " Post-deploy template ID: ${post_deploy_id:-<configure first>}"
info " Decommission template ID: ${decommission_id:-<configure first>}"
info " Job timeout: 600 seconds"
echo
# Print API commands for scripted registration
if [[ "$VERBOSE" == true ]]; then
detail "API commands for scripted registration:"
detail ""
detail "# Register endpoint:"
detail "curl -sSk -b cookies.txt -H 'X-CSRF-Token: \$CSRF' \\"
detail " -H 'Referer: ${AETHER_URL}/' \\"
detail " -X POST ${AETHER_URL}/api/awx/endpoints \\"
detail " -H 'Content-Type: application/json' \\"
detail " -d '{\"name\":\"lab-awx\",\"url\":\"http://${AWX_IP}:30080\",\"token\":\"${awx_token}\",\"verify_ssl\":false}'"
detail ""
detail "# Bind cluster (replace ENDPOINT_ID):"
detail "curl -sSk -b cookies.txt -H 'X-CSRF-Token: \$CSRF' \\"
detail " -H 'Referer: ${AETHER_URL}/' \\"
detail " -X PUT ${AETHER_URL}/api/clusters/${AETHER_CLUSTER_ID}/awx-config \\"
detail " -H 'Content-Type: application/json' \\"
detail " -d '{\"cluster_id\":${AETHER_CLUSTER_ID},\"awx_endpoint_id\":ENDPOINT_ID,\"awx_post_deploy_template_id\":${post_deploy_id:-0},\"awx_decommission_template_id\":${decommission_id:-0},\"awx_job_timeout_seconds\":600}'"
fi
}
# ---------------------------------------------------------------------------
# Cleanup — destroy AWX VM
# ---------------------------------------------------------------------------
action_cleanup() {
step "Cleaning up AWX VM"
if ! vm_exists; then
info "AWX VM does not exist — nothing to clean up"
return 0
fi
echo
warn "This will permanently destroy the AWX VM '$(vm_ref)'"
warn "All AWX data (jobs, templates, credentials) will be lost."
echo
if [[ "$DRY_RUN" == true ]]; then
info "(dry-run) Would destroy $(vm_ref)"
return 0
fi
read -r -p "Type 'yes' to confirm: " confirm
if [[ "$confirm" != "yes" ]]; then
info "Aborted"
return 0
fi
if vm_running; then
info "Stopping VM..."
incus stop "$(vm_ref)" --force 2>/dev/null || true
sleep 3
fi
info "Deleting VM..."
incus delete "$(vm_ref)" --force 2>/dev/null || true
success "AWX VM destroyed"
}
# ---------------------------------------------------------------------------
# Argument parsing
# ---------------------------------------------------------------------------
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--deploy) ACTION="deploy";;
--status) ACTION="status";;
--heal) ACTION="heal";;
--configure) ACTION="configure";;
--join-aether) ACTION="join-aether";;
--cleanup) ACTION="cleanup";;
--doctor) ACTION="doctor";;
-c|--config) CONFIG_FILE="$2"; shift;;
-n|--dry-run) DRY_RUN=true;;
-v|--verbose) VERBOSE=true;;
-q|--quiet) QUIET=true;;
-h|--help) usage; exit 0;;
*)
error "Unknown option: $1"
echo "Run '${SCRIPT_NAME} --help' for usage."
exit 1
;;
esac
shift
done
if [[ -z "$ACTION" ]]; then
error "No action specified"
echo "Run '${SCRIPT_NAME} --help' for usage."
exit 1
fi
}
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
main() {
setup_colors
parse_args "$@"
# Load config file if provided
if [[ -n "$CONFIG_FILE" ]]; then
parse_config "$CONFIG_FILE"
fi
echo
case "$ACTION" in
deploy) action_deploy;;
status) action_status;;
heal) action_heal;;
configure) action_configure;;
join-aether) action_join_aether;;
cleanup) action_cleanup;;
doctor) action_doctor;;
esac
}
main "$@"