incus-contrib/ansible/playbooks/showcase/k3s-worker.yml

---
# showcase/k3s-worker.yml — K3s agent (worker node) setup
#
# Discovers the control plane instance via Incus API, reads the join
# token from the control instance, and installs K3s agent joining
# the discovered control plane.
#
# Note: The Aether blueprint profile should include:
#   security.nesting: "true"

# --- Step 1: Find the control plane from the deployment manifest ---

- name: Find control plane instances from deployment
  ansible.builtin.set_fact:
    control_instances: >-
      {{ ffsdn_instances | selectattr('component_name', 'equalto', 'control') | list }}      

- name: Validate control plane found
  ansible.builtin.assert:
    that:
      - control_instances | length > 0
    fail_msg: >-
      No control component found in ffsdn_instances. The blueprint must
      include a control component for worker nodes to join.      

- name: Select control plane instance
  ansible.builtin.set_fact:
    control_instance:
      name: "{{ control_instances[0].name }}"
      ip: "{{ control_instances[0].ip }}"

- name: Display control plane target
  ansible.builtin.debug:
    msg: "Joining control plane: {{ control_instance.name }} at {{ control_instance.ip }}"

# --- Step 2: Wait for and read the join token ---

- name: Wait for K3s join token to be available (up to 10 minutes)
  ansible.builtin.uri:
    url: "{{ incus_api }}/1.0/instances/{{ control_instance.name }}/files?path=/var/lib/rancher/k3s/server/token"
    method: GET
    client_cert: "{{ incus_cert }}"
    client_key: "{{ incus_key }}"
    validate_certs: false
    return_content: true
    status_code: [200, 404]
  register: k3s_token_response
  until: k3s_token_response.status == 200
  retries: 60
  delay: 10

- name: Extract join token
  ansible.builtin.set_fact:
    k3s_server_url: "https://{{ control_instance.ip }}:6443"
    k3s_token: "{{ k3s_token_response.content | trim }}"

# --- Step 2b: Add ACL rules for K3s communication ---

- name: Build control ACL name
  ansible.builtin.set_fact:
    control_acl_name: "52-{{ control_instance.name }}-aether-acl"

- name: Read current control ACL
  ansible.builtin.uri:
    url: "{{ incus_api }}/1.0/network-acls/{{ control_acl_name }}"
    method: GET
    client_cert: "{{ incus_cert }}"
    client_key: "{{ incus_key }}"
    validate_certs: false
    return_content: true
    status_code: [200, 404]
  register: control_acl_response

- name: Build updated control ingress rules (allow worker to reach K3s API)
  ansible.builtin.set_fact:
    updated_control_ingress: >-
      {{ (control_acl_response.json.metadata.ingress | default([])) + [
        {
          'action': 'allow',
          'state': 'enabled',
          'source': (ffsdn_instance_ip | default('')) + '/32',
          'destination': control_instance.ip + '/32',
          'protocol': 'tcp',
          'destination_port': '6443,10250',
          'description': 'Allow ' + ffsdn_instance_name + ' to control K3s TCP ports'
        },
        {
          'action': 'allow',
          'state': 'enabled',
          'source': (ffsdn_instance_ip | default('')) + '/32',
          'destination': control_instance.ip + '/32',
          'protocol': 'udp',
          'destination_port': '8472',
          'description': 'Allow ' + ffsdn_instance_name + ' to control VXLAN'
        }
      ] }}      
  when: control_acl_response.status == 200

- name: Update control ACL with worker access rule
  ansible.builtin.uri:
    url: "{{ incus_api }}/1.0/network-acls/{{ control_acl_name }}"
    method: PATCH
    client_cert: "{{ incus_cert }}"
    client_key: "{{ incus_key }}"
    validate_certs: false
    body_format: json
    body:
      ingress: "{{ updated_control_ingress }}"
    status_code: [200]
  when:
    - control_acl_response.status == 200
    - updated_control_ingress is defined

- name: Build worker ACL name
  ansible.builtin.set_fact:
    worker_acl_name: "52-{{ ffsdn_instance_name }}-aether-acl"

- name: Read current worker ACL
  ansible.builtin.uri:
    url: "{{ incus_api }}/1.0/network-acls/{{ worker_acl_name }}"
    method: GET
    client_cert: "{{ incus_cert }}"
    client_key: "{{ incus_key }}"
    validate_certs: false
    return_content: true
    status_code: [200, 404]
  register: worker_acl_response

- name: Build updated worker ingress rules (allow control to reach kubelet)
  ansible.builtin.set_fact:
    updated_worker_ingress: >-
      {{ (worker_acl_response.json.metadata.ingress | default([])) + [
        {
          'action': 'allow',
          'state': 'enabled',
          'source': control_instance.ip + '/32',
          'destination': (ffsdn_instance_ip | default('')) + '/32',
          'protocol': 'tcp',
          'destination_port': '10250',
          'description': 'Allow control to reach ' + ffsdn_instance_name + ' kubelet'
        },
        {
          'action': 'allow',
          'state': 'enabled',
          'source': control_instance.ip + '/32',
          'destination': (ffsdn_instance_ip | default('')) + '/32',
          'protocol': 'udp',
          'destination_port': '8472',
          'description': 'Allow control VXLAN to ' + ffsdn_instance_name
        }
      ] }}      
  when: worker_acl_response.status == 200

- name: Update worker ACL with control access rule
  ansible.builtin.uri:
    url: "{{ incus_api }}/1.0/network-acls/{{ worker_acl_name }}"
    method: PATCH
    client_cert: "{{ incus_cert }}"
    client_key: "{{ incus_key }}"
    validate_certs: false
    body_format: json
    body:
      ingress: "{{ updated_worker_ingress }}"
    status_code: [200]
  when:
    - worker_acl_response.status == 200
    - updated_worker_ingress is defined

- name: Report ACL updates
  ansible.builtin.debug:
    msg: >-
      ACL rules added: {{ ffsdn_instance_name }} ↔ {{ control_instance.name }}
      (K3s API 6443, kubelet 10250, VXLAN 8472)      

# --- Step 3: Install K3s agent ---

- name: Generate K3s agent setup script
  ansible.builtin.set_fact:
    k3s_agent_script: |
      #!/bin/bash
      exec > >(tee /tmp/k3s-agent-setup.log) 2>&1
      set -e

      echo "=== K3s agent setup starting ==="
      export DEBIAN_FRONTEND=noninteractive

      # Container workaround: create /dev/kmsg if missing
      [ -e /dev/kmsg ] || ln -s /dev/console /dev/kmsg

      # Install prerequisites
      apt-get install -y -qq curl iptables 2>&1 | tail -1
      echo "[OK] curl + iptables installed"

      # Test connectivity to control plane before installing
      echo "Testing connectivity to {{ k3s_server_url }}..."
      if curl -sk --connect-timeout 5 "{{ k3s_server_url }}" >/dev/null 2>&1; then
        echo "[OK] Control plane reachable"
      else
        echo "[WARN] Control plane not reachable via curl (may still work via K3s)"
      fi

      # Download and install K3s binary (skip service start)
      echo "Downloading K3s install script..."
      curl -sfL https://get.k3s.io -o /tmp/k3s-install.sh
      chmod +x /tmp/k3s-install.sh
      echo "[OK] Install script downloaded"

      echo "Installing K3s agent binary (skip start)..."
      K3S_URL="{{ k3s_server_url }}" \
        K3S_TOKEN="{{ k3s_token }}" \
        INSTALL_K3S_EXEC="--snapshotter=native --kubelet-arg=feature-gates=KubeletInUserNamespace=true --node-ip={{ ffsdn_instance_ip | default('127.0.0.1') }}" \
        INSTALL_K3S_SKIP_START=true \
        /tmp/k3s-install.sh
      echo "[OK] K3s agent binary installed"

      # Show resource limits
      echo "--- container limits ---"
      echo "Memory limit: $(cat /sys/fs/cgroup/memory.max 2>/dev/null || echo unknown)"
      echo "PIDs max: $(cat /sys/fs/cgroup/pids.max 2>/dev/null || echo unknown)"
      free -m 2>/dev/null || true

      # Write config file (K3s re-execs itself and loses CLI args)
      mkdir -p /etc/rancher/k3s
      cat > /etc/rancher/k3s/config.yaml <<'CFGEOF'
      server: "{{ k3s_server_url }}"
      token: "{{ k3s_token }}"
      snapshotter: native
      kubelet-arg:
        - "feature-gates=KubeletInUserNamespace=true"
      node-ip: "{{ ffsdn_instance_ip | default('127.0.0.1') }}"
      CFGEOF
      echo "[OK] K3s agent config written to /etc/rancher/k3s/config.yaml"

      # Start K3s agent directly (bypass systemd/D-Bus to avoid container crash)
      echo "Starting K3s agent directly..."
      nohup /usr/local/bin/k3s agent \
        > /var/log/k3s-agent.log 2>&1 &
      K3S_PID=$!
      echo "[OK] K3s agent started (PID $K3S_PID)"

      # Wait for agent to initialize (check process is still alive)
      echo "Waiting for K3s agent to initialize..."
      for i in $(seq 1 12); do
        sleep 5
        if ! kill -0 $K3S_PID 2>/dev/null; then
          echo "[ERROR] K3s agent process died"
          echo "--- last 30 lines of agent log ---"
          tail -30 /var/log/k3s-agent.log 2>/dev/null || true
          exit 1
        fi
        echo "  ... alive after ${i}x5s (PID $K3S_PID)"
      done
      echo "[OK] K3s agent running for 60s"

      # Verify node is registered with the cluster
      echo "--- K3s agent log (last 10 lines) ---"
      tail -10 /var/log/k3s-agent.log 2>/dev/null || true

      echo "=== K3s agent setup complete ==="      

- name: Push K3s agent setup script
  ansible.builtin.uri:
    url: "{{ incus_api }}/1.0/instances/{{ ffsdn_instance_name }}/files?path=/tmp/k3s-agent-setup.sh"
    method: POST
    client_cert: "{{ incus_cert }}"
    client_key: "{{ incus_key }}"
    validate_certs: false
    body: "{{ k3s_agent_script }}"
    headers:
      Content-Type: "application/octet-stream"
      X-Incus-type: "file"
      X-Incus-mode: "0755"
    status_code: [200]

- name: Execute K3s agent setup script
  ansible.builtin.uri:
    url: "{{ incus_api }}/1.0/instances/{{ ffsdn_instance_name }}/exec"
    method: POST
    client_cert: "{{ incus_cert }}"
    client_key: "{{ incus_key }}"
    validate_certs: false
    body_format: json
    body:
      command: ["/bin/bash", "/tmp/k3s-agent-setup.sh"]
      record-output: true
      interactive: false
      wait-for-websocket: false
      environment:
        DEBIAN_FRONTEND: "noninteractive"
    status_code: [202]
    return_content: true
  register: k3s_agent_exec

- name: Wait for K3s agent setup to complete (up to 10 minutes)
  ansible.builtin.uri:
    url: "{{ incus_api }}{{ k3s_agent_exec.json.operation }}/wait?timeout=600"
    method: GET
    client_cert: "{{ incus_cert }}"
    client_key: "{{ incus_key }}"
    validate_certs: false
    return_content: true
    timeout: 620
  register: k3s_agent_wait
  failed_when: false

- name: Retrieve K3s agent setup log from container
  ansible.builtin.uri:
    url: "{{ incus_api }}/1.0/instances/{{ ffsdn_instance_name }}/files?path=/tmp/k3s-agent-setup.log"
    method: GET
    client_cert: "{{ incus_cert }}"
    client_key: "{{ incus_key }}"
    validate_certs: false
    return_content: true
    status_code: [200, 404]
  register: k3s_agent_log
  ignore_errors: true

- name: Display K3s agent setup log
  ansible.builtin.debug:
    msg: "{{ k3s_agent_log.content | default('No log captured') }}"
  when: k3s_agent_log is defined and k3s_agent_log.status == 200
  ignore_errors: true

- name: Verify K3s agent setup exit code
  ansible.builtin.assert:
    that:
      - k3s_agent_wait.json.metadata.metadata.return | int == 0
    fail_msg: "K3s agent setup exited with code {{ k3s_agent_wait.json.metadata.metadata.return }}"
    success_msg: "K3s agent joined control plane at {{ k3s_server_url }}"

- name: Clean up setup script
  ansible.builtin.uri:
    url: "{{ incus_api }}/1.0/instances/{{ ffsdn_instance_name }}/exec"
    method: POST
    client_cert: "{{ incus_cert }}"
    client_key: "{{ incus_key }}"
    validate_certs: false
    body_format: json
    body:
      command: ["/bin/rm", "-f", "/tmp/k3s-agent-setup.sh"]
      record-output: false
      interactive: false
      wait-for-websocket: false
    status_code: [202]
  ignore_errors: true