fix(k3s-server): serialize master joins and capture diagnostics

- validate host-only connectivity before cluster bootstrap\n- materialize per-host join arguments and wait for sequential registration\n- collect bounded network diagnostics when k3s initialization fails

.ansible-lint

@@ -3,6 +3,7 @@ profile: production
 exclude_paths:
   # default paths
   - .cache/
+  - .ansible/
   - .github/
   - test/fixtures/formatting-before/
   - test/fixtures/formatting-prettier/

.github/actionlint.yaml

@@ -0,0 +1,6 @@
+---
+self-hosted-runner:
+  labels:
+    - k3s-ci
+    - virtualbox
+    - nested-virt

.github/download-boxes.sh

@@ -9,14 +9,12 @@ set -euo pipefail
 GIT_ROOT=$(git rev-parse --show-toplevel)
 PROVIDER=virtualbox
 
-yq --version
-
 # Define the path to the molecule.yml files
-MOLECULE_YML_PATH="${GIT_ROOT}/molecule/*/molecule.yml"
+MOLECULE_YML_PATH=("${GIT_ROOT}"/molecule/*/molecule.yml)
 
 # Extract and sort unique boxes from all molecule.yml files
-all_boxes=$(for file in $MOLECULE_YML_PATH; do
-    yq eval '.platforms[].box' "$file"
+all_boxes=$(for file in "${MOLECULE_YML_PATH[@]}"; do
+    yq -r '.platforms[].box' "$file"
 done | sort -u)
 
 echo all_boxes: "$all_boxes"

.github/scripts/cleanup-runner-resources.sh

@@ -0,0 +1,249 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+usage() {
+  printf '%s\n' \
+    'Usage: cleanup-runner-resources.sh [--snapshot|--dry-run|--apply]' \
+    '' \
+    'Discover and, with --apply, remove only VirtualBox resources referenced by' \
+    'repository-owned Molecule Vagrant state. The default is --dry-run.'
+}
+
+mode="dry-run"
+case "${1:-}" in
+  "") ;;
+  --snapshot) mode="snapshot" ;;
+  --dry-run) mode="dry-run" ;;
+  --apply) mode="apply" ;;
+  --help|-h) usage; exit 0 ;;
+  *) usage >&2; exit 2 ;;
+esac
+
+home_dir="${HOME:?HOME must be set}"
+molecule_root="${K3S_CI_MOLECULE_ROOT:-${home_dir}/.cache/molecule}"
+repository_name="${K3S_CI_MOLECULE_PROJECT:-k3s-ansible}"
+virtualbox_root="${K3S_CI_VIRTUALBOX_ROOT:-${home_dir}/VirtualBox VMs}"
+hostonly_marker="${K3S_CI_HOSTONLY_MARKER:-${home_dir}/.cache/k3s-ci/hostonly-interfaces}"
+
+record_hostonly() {
+  local marker_dir="${hostonly_marker%/*}"
+  local marker_tmp="${hostonly_marker}.tmp"
+  local hostonly_inventory
+  if ! hostonly_inventory="$(VBoxManage list hostonlyifs)"; then
+    fail_closed 'unable to inventory VirtualBox host-only interfaces'
+  fi
+  mkdir -p -- "$marker_dir"
+  awk -F': ' '
+    /^Name:/ { name=$2 }
+    /^IPAddress:/ { print name "|" $2 }
+  ' <<< "$hostonly_inventory" > "$marker_tmp"
+  mv -- "$marker_tmp" "$hostonly_marker"
+  chmod 600 "$hostonly_marker"
+  printf 'Recorded host-only interface baseline: %s\n' "$hostonly_marker"
+}
+
+cleanup_hostonly() {
+  local hostonly_inventory
+  if [[ ! -f "$hostonly_marker" ]]; then
+    printf 'No host-only interface baseline found; leaving interfaces unchanged.\n'
+    return 0
+  fi
+
+  if ! hostonly_inventory="$(VBoxManage list hostonlyifs)"; then
+    fail_closed 'unable to inventory VirtualBox host-only interfaces'
+  fi
+
+  while IFS='|' read -r interface_name interface_ip; do
+    [[ "$interface_name" == vboxnet* ]] || continue
+    [[ "$interface_ip" == 192.168.30.* || "$interface_ip" == fdad:bad:ba55:* ]] || continue
+    if grep -Fqx "${interface_name}|${interface_ip}" "$hostonly_marker"; then
+      continue
+    fi
+    if [[ "$mode" == apply ]]; then
+      VBoxManage hostonlyif remove "$interface_name"
+      printf 'Removed host-only interface %s (%s)\n' "$interface_name" "$interface_ip"
+    else
+      printf 'Would remove host-only interface %s (%s)\n' "$interface_name" "$interface_ip"
+    fi
+  done < <(awk -F': ' '
+    /^Name:/ { name=$2 }
+    /^IPAddress:/ { print name "|" $2 }
+  ' <<< "$hostonly_inventory")
+}
+
+if [[ "$mode" == snapshot ]]; then
+  record_hostonly
+  exit 0
+fi
+
+resolve_existing_dir() {
+  local candidate="$1"
+  if [[ ! -d "$candidate" ]]; then
+    return 1
+  fi
+  readlink -f -- "$candidate"
+}
+
+root_contains() {
+  local root="$1"
+  local path="$2"
+  [[ "$path" == "$root"/* ]]
+}
+
+is_supported_scenario() {
+  case "$1" in
+    default|single_node|calico|cilium|kube-vip|ipv6) return 0 ;;
+    *) return 1 ;;
+  esac
+}
+
+is_unregistered_vm_error() {
+  grep -Eq 'Could not find a registered machine|VBOX_E_OBJECT_NOT_FOUND'
+}
+
+fail_closed() {
+  printf 'cleanup refused: %s\n' "$1" >&2
+  exit 3
+}
+
+if [[ ! "$repository_name" =~ ^[A-Za-z0-9._-]+$ ]]; then
+  fail_closed 'invalid Molecule repository name'
+fi
+
+print_inventory() {
+  local phase="$1"
+  printf '%s VirtualBox inventory:\n' "$phase"
+  VBoxManage list vms || true
+  VBoxManage list hdds || true
+  VBoxManage list hostonlyifs || true
+}
+
+molecule_root_real="$(resolve_existing_dir "$molecule_root" || true)"
+if [[ -z "$molecule_root_real" ]]; then
+  printf 'No Molecule root exists: %s\n' "$molecule_root"
+  cleanup_hostonly
+  exit 0
+fi
+
+print_inventory before
+
+repository_root_real="$(resolve_existing_dir "$molecule_root_real/$repository_name" || true)"
+if [[ -z "$repository_root_real" ]]; then
+  printf 'No repository Molecule state root exists: %s\n' "$molecule_root_real/$repository_name"
+  cleanup_hostonly
+  print_inventory after
+  exit 0
+fi
+if ! root_contains "$molecule_root_real" "$repository_root_real"; then
+  fail_closed "repository Molecule state root is outside Molecule root: $repository_root_real"
+fi
+
+declare -a state_files=()
+while IFS= read -r -d '' state_file; do
+  state_files+=("$state_file")
+done < <(find "$repository_root_real" -mindepth 6 -maxdepth 6 -type f \
+  -path '*/.vagrant/machines/*/virtualbox/id' -print0 2>/dev/null)
+
+if ((${#state_files[@]} == 0)); then
+  printf 'No repository-owned Molecule Vagrant state found under %s\n' "$repository_root_real"
+  cleanup_hostonly
+  print_inventory after
+  exit 0
+fi
+
+virtualbox_root_real="$(resolve_existing_dir "$virtualbox_root" || true)"
+
+declare -a vm_records=()
+for state_file in "${state_files[@]}"; do
+  if [[ ! -f "$state_file" ]]; then
+    printf 'Skipping Vagrant state removed with its stale scenario directory: %s\n' "$state_file"
+    continue
+  fi
+  state_file_real="$(readlink -f -- "$state_file")"
+  state_dir="${state_file_real%/.vagrant/machines/*/virtualbox/id}"
+  machine_dir="${state_file_real%/virtualbox/id}"
+  machine_name="${machine_dir##*/}"
+  scenario_name="${state_dir##*/}"
+
+  if ! root_contains "$repository_root_real" "$state_dir"; then
+    fail_closed "state path is outside the repository Molecule root: $state_file_real"
+  fi
+  if ! is_supported_scenario "$scenario_name"; then
+    fail_closed "unexpected Molecule scenario: $scenario_name"
+  fi
+  if [[ "$machine_name" != control* && "$machine_name" != node* ]]; then
+    fail_closed "unexpected Molecule machine name: $machine_name"
+  fi
+
+  vm_uuid="$(tr -d '[:space:]' < "$state_file_real")"
+  if [[ ! "$vm_uuid" =~ ^[0-9a-fA-F-]{36}$ ]]; then
+    fail_closed "invalid VirtualBox UUID in $state_file_real"
+  fi
+
+  if ! vm_info="$(VBoxManage showvminfo "$vm_uuid" --machinereadable 2>&1)"; then
+    if is_unregistered_vm_error <<< "$vm_info"; then
+      printf 'Stale Vagrant state without a registered VM: %s (%s)\n' "$machine_name" "$vm_uuid"
+      if [[ "$mode" == apply ]]; then
+        rm -rf -- "${state_dir}/.vagrant"
+        printf 'Removed stale Vagrant state: %s\n' "${state_dir}/.vagrant"
+      fi
+      continue
+    fi
+    fail_closed "unable to inspect VirtualBox VM $vm_uuid: $vm_info"
+  fi
+
+  if [[ -z "$virtualbox_root_real" ]]; then
+    fail_closed "VirtualBox VM root does not exist: $virtualbox_root"
+  fi
+
+  cfg_file="$(awk -F= '$1 == "CfgFile" {gsub(/"/, "", $2); print $2; exit}' <<< "$vm_info")"
+  if [[ -z "$cfg_file" ]]; then
+    fail_closed "VirtualBox configuration path missing for $vm_uuid"
+  fi
+  cfg_file_real="$(readlink -f -- "$cfg_file")"
+  if ! root_contains "$virtualbox_root_real" "$cfg_file_real"; then
+    fail_closed "VM configuration is outside VirtualBox root: $cfg_file_real"
+  fi
+
+  while IFS= read -r disk_path; do
+    [[ -z "$disk_path" ]] && continue
+    disk_path_real="$(readlink -f -- "$disk_path" 2>/dev/null || true)"
+    if [[ -z "$disk_path_real" ]] || ! root_contains "$virtualbox_root_real" "$disk_path_real"; then
+      fail_closed "attached disk is outside VirtualBox root: $disk_path"
+    fi
+  done < <(awk -F= '$1 ~ /^(SATA|IDE|SCSI|SAS|VirtioSCSI|NVMe)-[0-9]+-[0-9]+$/ {gsub(/"/, "", $2); print $2}' <<< "$vm_info")
+
+  vm_records+=("$vm_uuid|$machine_name|$cfg_file_real")
+done
+
+if ((${#vm_records[@]} == 0)); then
+  printf 'No live repository-owned VirtualBox resources found\n'
+  cleanup_hostonly
+  print_inventory after
+  exit 0
+fi
+
+for record in "${vm_records[@]}"; do
+  IFS='|' read -r vm_uuid machine_name cfg_file_real <<< "$record"
+  if [[ "$mode" == dry-run ]]; then
+    printf 'Would remove VM %s (%s) config=%s\n' "$machine_name" "$vm_uuid" "$cfg_file_real"
+    continue
+  fi
+
+  vm_state="$(VBoxManage showvminfo "$vm_uuid" --machinereadable | awk -F= '$1 == "VMState" {gsub(/"/, "", $2); print $2; exit}')"
+  if [[ "$vm_state" != poweroff && "$vm_state" != saved ]]; then
+    VBoxManage controlvm "$vm_uuid" poweroff
+  fi
+  VBoxManage unregistervm "$vm_uuid" --delete
+  printf 'Removed VM %s (%s)\n' "$machine_name" "$vm_uuid"
+done
+
+cleanup_hostonly
+print_inventory after
+
+if [[ "$mode" == apply ]]; then
+  printf 'Repository-owned VM and host-only interface cleanup complete.\n'
+else
+  printf 'Dry run complete. No resources were modified.\n'
+fi

.github/scripts/collect-runner-diagnostics.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+output_dir="${1:-${RUNNER_TEMP:-/tmp}/k3s-ci-diagnostics}"
+mkdir -p -- "$output_dir"
+umask 077
+
+run_capture() {
+  local output_file="$1"
+  shift
+  {
+    printf '$'
+    printf ' %q' "$@"
+    printf '\n'
+    "$@"
+  } > "$output_dir/$output_file" 2>&1 || true
+}
+
+run_capture system.txt uname -a
+run_capture runner-user.txt id
+run_capture memory.txt free -h
+run_capture disk.txt df -h
+run_capture virtualbox-version VBoxManage --version
+run_capture virtualbox-vms VBoxManage list vms
+run_capture virtualbox-running-vms VBoxManage list runningvms
+run_capture virtualbox-disks VBoxManage list hdds
+run_capture virtualbox-hostonlyifs VBoxManage list hostonlyifs
+run_capture vagrant-status vagrant global-status
+run_capture molecule-state find "${HOME}/.cache/molecule" -maxdepth 6 -type f -path '*/.vagrant/machines/*/virtualbox/id' -print
+
+if [[ -r /etc/vbox/networks.conf ]]; then
+  cp -- /etc/vbox/networks.conf "$output_dir/virtualbox-networks.conf"
+fi
+
+printf 'Diagnostics written to %s\n' "$output_dir"

.github/scripts/test-cleanup-runner-resources.sh

@@ -0,0 +1,129 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC2016
+
+set -Eeuo pipefail
+
+repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
+test_root="$(mktemp -d "${TMPDIR:-/tmp}/k3s-ci-cleanup-test.XXXXXX")"
+trap 'rm -rf -- "$test_root"' EXIT
+
+molecule_root="$test_root/molecule"
+virtualbox_root="$test_root/VirtualBox VMs"
+fake_bin="$test_root/bin"
+mkdir -p -- "$molecule_root/k3s-ansible/single_node/.vagrant/machines/control1/virtualbox" \
+  "$molecule_root/k3s-ansible/single_node/.vagrant/machines/control2/virtualbox" \
+  "$virtualbox_root/control1" "$virtualbox_root/unmarked" "$fake_bin"
+printf '%s\n' '11111111-1111-1111-1111-111111111111' \
+  > "$molecule_root/k3s-ansible/single_node/.vagrant/machines/control1/virtualbox/id"
+printf '%s\n' '22222222-2222-2222-2222-222222222222' \
+  > "$molecule_root/k3s-ansible/single_node/.vagrant/machines/control2/virtualbox/id"
+touch "$virtualbox_root/control1/control1.vbox" "$virtualbox_root/control1/disk.vdi" \
+  "$virtualbox_root/unmarked/unmarked.vbox"
+
+printf '%s\n' \
+  '#!/usr/bin/env bash' \
+  'set -Eeuo pipefail' \
+  'case "${1:-}" in' \
+  '  list)' \
+  '    if [[ "${2:-}" == hostonlyifs && "${FAKE_HOSTONLY_FAIL:-false}" == true ]]; then exit 1; fi' \
+  '    exit 0' \
+  '    ;;' \
+  '  showvminfo)' \
+  '    if [[ "${FAKE_VM_MODE:-normal}" == missing ]]; then' \
+  '      printf '\''VBoxManage: error: Could not find a registered machine named "missing"\n'\'' >&2' \
+  '      exit 1' \
+  '    fi' \
+  '    if [[ "${FAKE_VM_MODE:-normal}" == fault ]]; then' \
+  '      printf '\''VBoxManage: error: VirtualBox service is unavailable\n'\'' >&2' \
+  '      exit 1' \
+  '    fi' \
+  '    printf '\''CfgFile="%s"\n'\'' "${FAKE_VBOX_ROOT}/control1/control1.vbox"' \
+  '    printf '\''SATA-0-0="%s"\n'\'' "${FAKE_VBOX_ROOT}/control1/disk.vdi"' \
+  '    printf '\''VMState="running"\n'\''' \
+  '    ;;' \
+  '  controlvm) printf '\''controlvm %s\n'\'' "$*" >> "${FAKE_LOG}" ;;' \
+  '  unregistervm)' \
+  '    printf '\''unregistervm %s\n'\'' "$*" >> "${FAKE_LOG}"' \
+  '    rm -f -- "${FAKE_VBOX_ROOT}/control1/control1.vbox" "${FAKE_VBOX_ROOT}/control1/disk.vdi"' \
+  '    ;;' \
+  '  *) : ;;' \
+  'esac' > "$fake_bin/VBoxManage"
+chmod 700 "$fake_bin/VBoxManage"
+
+output="$test_root/output.txt"
+if PATH="$fake_bin:$PATH" \
+  HOME="$test_root/home" \
+  K3S_CI_MOLECULE_ROOT="$molecule_root" \
+  K3S_CI_MOLECULE_PROJECT=k3s-ansible \
+  K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
+  K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
+  FAKE_VM_MODE=fault \
+  FAKE_VBOX_ROOT="$virtualbox_root" \
+  FAKE_LOG="$test_root/vbox.log" \
+  bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --apply > "$output" 2>&1; then
+  printf '%s\n' 'cleanup unexpectedly accepted a VirtualBox inspection failure' >&2
+  exit 1
+fi
+grep -Fq 'cleanup refused: unable to inspect VirtualBox VM' "$output"
+[[ -f "$molecule_root/k3s-ansible/single_node/.vagrant/machines/control1/virtualbox/id" ]]
+
+printf '%s\n' 'vboxnet0|192.168.30.1' > "$test_root/hostonly-baseline"
+if PATH="$fake_bin:$PATH" \
+  HOME="$test_root/home" \
+  K3S_CI_MOLECULE_ROOT="$molecule_root" \
+  K3S_CI_MOLECULE_PROJECT=k3s-ansible \
+  K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
+  K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
+  FAKE_HOSTONLY_FAIL=true \
+  FAKE_VBOX_ROOT="$virtualbox_root" \
+  FAKE_LOG="$test_root/vbox.log" \
+  bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --dry-run > "$output" 2>&1; then
+  printf '%s\n' 'cleanup unexpectedly accepted a host-only inventory failure' >&2
+  exit 1
+fi
+grep -Fq 'cleanup refused: unable to inventory VirtualBox host-only interfaces' "$output"
+
+PATH="$fake_bin:$PATH" \
+  HOME="$test_root/home" \
+  K3S_CI_MOLECULE_ROOT="$molecule_root" \
+  K3S_CI_MOLECULE_PROJECT=k3s-ansible \
+  K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
+  K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
+  FAKE_VBOX_ROOT="$virtualbox_root" \
+  FAKE_LOG="$test_root/vbox.log" \
+  bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --dry-run > "$output"
+
+grep -Fq 'Would remove VM control1 (11111111-1111-1111-1111-111111111111)' "$output"
+[[ ! -e "$test_root/vbox.log" ]]
+
+PATH="$fake_bin:$PATH" \
+  HOME="$test_root/home" \
+  K3S_CI_MOLECULE_ROOT="$molecule_root" \
+  K3S_CI_MOLECULE_PROJECT=k3s-ansible \
+  K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
+  K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
+  FAKE_VBOX_ROOT="$virtualbox_root" \
+  FAKE_LOG="$test_root/vbox.log" \
+  bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --apply > "$output"
+
+grep -Fq 'controlvm 11111111-1111-1111-1111-111111111111 poweroff' "$test_root/vbox.log"
+grep -Fq 'unregistervm unregistervm 11111111-1111-1111-1111-111111111111 --delete' "$test_root/vbox.log"
+[[ ! -e "$virtualbox_root/control1/control1.vbox" ]]
+[[ -e "$virtualbox_root/unmarked/unmarked.vbox" ]]
+
+PATH="$fake_bin:$PATH" \
+  HOME="$test_root/home" \
+  K3S_CI_MOLECULE_ROOT="$molecule_root" \
+  K3S_CI_MOLECULE_PROJECT=k3s-ansible \
+  K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
+  K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
+  FAKE_VM_MODE=missing \
+  FAKE_VBOX_ROOT="$virtualbox_root" \
+  FAKE_LOG="$test_root/vbox.log" \
+  bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --apply > "$output"
+
+grep -Fq 'Stale Vagrant state without a registered VM' "$output"
+[[ ! -d "$molecule_root/k3s-ansible/single_node/.vagrant" ]]
+
+printf 'cleanup-runner-resources fixture test passed\n'

.github/workflows/cache.yml

@@ -5,32 +5,47 @@ on:
 jobs:
   molecule:
     name: cache
-    runs-on: self-hosted
+    runs-on: [self-hosted, linux, x64, k3s-ci, virtualbox, nested-virt]
     env:
       PYTHON_VERSION: "3.11"
+      VAGRANT_DEFAULT_PROVIDER: virtualbox
+      VAGRANT_HOME: ${{ github.workspace }}/.vagrant-home
 
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # 7.0.1
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - name: Check nested VirtualBox platform
+        run: |
+          set -Eeuo pipefail
+          grep -Eq 'vmx|svm' /proc/cpuinfo
+          test -c /dev/kvm
+          test -c /dev/vboxdrv
+          VBoxManage --version
+          vagrant --version
+          test -r /etc/vbox/networks.conf
+          test "$(stat -c '%u' /etc/vbox/networks.conf)" -eq 0
+          free -h
+          df -Pk "${RUNNER_TEMP}"
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # 5.3.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # 7.0.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies
 
       - name: Cache Vagrant boxes
         id: cache-vagrant
-        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # 4.1.2
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # 6.1.0
         with:
-          lookup-only: true #if it exists, we don't need to restore and can skip the next step
+          lookup-only: true # if it exists, we don't need to restore and can skip the next step
           path: |
-            ~/.vagrant.d/boxes
-          key: vagrant-boxes-${{ hashFiles('**/molecule.yml') }}
+            .vagrant-home/boxes
+          key: vagrant-boxes-${{ runner.os }}-${{ runner.arch }}-virtualbox-7.2-vagrant-2.4-${{ hashFiles('**/molecule.yml') }} # yamllint disable-line rule:line-length
           restore-keys: |
-            vagrant-boxes
+            vagrant-boxes-${{ runner.os }}-${{ runner.arch }}-virtualbox-7.2-vagrant-2.4-
 
       - name: Download Vagrant boxes for all scenarios
         # To save some cache space, all scenarios share the same cache key.

.github/workflows/ci.yml

@@ -5,6 +5,8 @@ on:
     types:
       - opened
       - synchronize
+      - reopened
+      - ready_for_review
     paths-ignore:
       - '**/.gitignore'
       - '**/FUNDING.yml'
@@ -16,12 +18,21 @@ on:
       - '**/LICENSE'
       - '**/reboot.sh'
       - '**/reset.sh'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   pre:
     uses: ./.github/workflows/cache.yml
+    needs: [lint]
   lint:
     uses: ./.github/workflows/lint.yml
-    needs: [pre]
   test:
     uses: ./.github/workflows/test.yml
     needs: [pre, lint]

.github/workflows/lint.yml

@@ -5,24 +5,25 @@ on:
 jobs:
   pre-commit-ci:
     name: Pre-Commit
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     env:
-      PYTHON_VERSION: "3.11"
+      PYTHON_VERSION: "3.12"
 
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # 7.0.1
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # 5.3.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # 7.0.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies
 
       - name: Restore Ansible cache
-        uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # 4.1.2
+        id: cache-ansible
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # 6.1.0
         with:
           path: ~/.ansible/collections
           key: ansible-${{ hashFiles('collections/requirements.yml') }}
@@ -37,17 +38,37 @@ jobs:
           python3 -m pip install -r requirements.txt
           echo "::endgroup::"
 
+      - name: Install Ansible collections with retries
+        if: steps.cache-ansible.outputs.cache-hit != 'true'
+        run: |
+          set -Eeuo pipefail
+          for attempt in 1 2 3 4 5; do
+            if ansible-galaxy collection install -r collections/requirements.yml; then
+              exit 0
+            fi
+            echo "Ansible Galaxy attempt ${attempt} failed; retrying."
+            sleep $((attempt * 10))
+          done
+          exit 1
+
+      - name: Save Ansible collection cache
+        if: steps.cache-ansible.outputs.cache-hit != 'true'
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # 6.1.0
+        with:
+          path: ~/.ansible/collections
+          key: ansible-${{ hashFiles('collections/requirements.yml') }}
+
       - name: Run pre-commit
         uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # 3.0.1
 
   ensure-pinned-actions:
     name: Ensure SHA Pinned Actions
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # 7.0.1
       - name: Ensure SHA pinned actions
-        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@38608ef4fb69adae7f1eac6eeb88e67b7d083bfd # 3.0.16
+        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@3db98c0363e2fa5df3e1c4c471777a7c10b24cc9 # 5.0.5
         with:
           allowlist: |
             aws-actions/

.github/workflows/test.yml

@@ -5,70 +5,59 @@ on:
 jobs:
   molecule:
     name: Molecule
-    runs-on: self-hosted
+    runs-on: [self-hosted, linux, x64, k3s-ci, virtualbox, nested-virt]
     strategy:
       matrix:
         scenario:
           - default
-          # - ipv6
           - single_node
           - calico
           - cilium
           - kube-vip
-      fail-fast: false
+          # - ipv6
+      fail-fast: true
+      max-parallel: 1
     env:
       PYTHON_VERSION: "3.11"
+      VAGRANT_DEFAULT_PROVIDER: virtualbox
+      VAGRANT_HOME: ${{ github.workspace }}/.vagrant-home
 
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # 7.0.1
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
 
-      # these steps are necessary if not using ephemeral nodes
-      - name: Delete old Vagrant box versions
-        if: always() # do this even if a step before has failed
-        run: vagrant box prune --force
+      - name: Clean repository-owned resources before testing
+        run: ./.github/scripts/cleanup-runner-resources.sh --apply
 
-      - name: Remove all local Vagrant boxes
-        if: always() # do this even if a step before has failed
-        run: if vagrant box list 2>/dev/null; then vagrant box list | cut -f 1 -d ' ' | xargs -L 1 vagrant box remove -f 2>/dev/null && echo "All Vagrant boxes removed." || echo "No Vagrant boxes found."; else echo "No Vagrant boxes found."; fi
+      - name: Record host-only network baseline
+        run: ./.github/scripts/cleanup-runner-resources.sh --snapshot
 
-      - name: Remove all Virtualbox VMs
-        if: always() # do this even if a step before has failed
-        run: VBoxManage list vms | awk -F'"' '{print $2}' | xargs -I {} VBoxManage unregistervm --delete "{}"
-
-      - name: Remove all Virtualbox HDs
-        if: always() # do this even if a step before has failed
-        run: VBoxManage list hdds | awk -F':' '/^UUID:/ {print $2}' | xargs -I {} VBoxManage closemedium disk "{}" --delete
-
-      - name: Remove all Virtualbox Networks
-        if: always() # do this even if a step before has failed
-        run: VBoxManage list hostonlyifs | grep '^Name:' | awk '{print $2}' | grep '^vboxnet' | xargs -I {} VBoxManage hostonlyif remove {}
-
-      - name: Remove Virtualbox network config
-        if: always() # do this even if a step before has failed
-        run: sudo rm /etc/vbox/networks.conf || true
-
-      - name: Configure VirtualBox
-        run: |-
-          sudo mkdir -p /etc/vbox
-          cat <<EOF | sudo tee -a /etc/vbox/networks.conf > /dev/null
-          * 192.168.30.0/24
-          * fdad:bad:ba55::/64
-          EOF
+      - name: Check nested VirtualBox platform
+        run: |
+          set -Eeuo pipefail
+          grep -Eq 'vmx|svm' /proc/cpuinfo
+          test -c /dev/kvm
+          test -c /dev/vboxdrv
+          VBoxManage --version
+          vagrant --version
+          test -r /etc/vbox/networks.conf
+          test "$(stat -c '%u' /etc/vbox/networks.conf)" -eq 0
+          free -h
+          df -Pk "${RUNNER_TEMP}"
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # 5.3.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # 7.0.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies
 
       - name: Restore vagrant Boxes cache
-        uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # 4.1.2
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # 6.1.0
         with:
-          path: ~/.vagrant.d/boxes
-          key: vagrant-boxes-${{ hashFiles('**/molecule.yml') }}
+          path: .vagrant-home/boxes
+          key: vagrant-boxes-${{ runner.os }}-${{ runner.arch }}-virtualbox-7.2-vagrant-2.4-${{ hashFiles('**/molecule.yml') }} # yamllint disable-line rule:line-length
           fail-on-cache-miss: true
 
       - name: Install dependencies
@@ -91,36 +80,20 @@ jobs:
           PY_COLORS: 1
           ANSIBLE_FORCE_COLOR: 1
 
-      # these steps are necessary if not using ephemeral nodes
-      - name: Delete old Vagrant box versions
-        if: always() # do this even if a step before has failed
-        run: vagrant box prune --force
+      - name: Collect runner diagnostics
+        if: always()
+        run: ./.github/scripts/collect-runner-diagnostics.sh "${RUNNER_TEMP}/logs/runner"
 
-      - name: Remove all local Vagrant boxes
-        if: always() # do this even if a step before has failed
-        run: if vagrant box list 2>/dev/null; then vagrant box list | cut -f 1 -d ' ' | xargs -L 1 vagrant box remove -f 2>/dev/null && echo "All Vagrant boxes removed." || echo "No Vagrant boxes found."; else echo "No Vagrant boxes found."; fi
-
-      - name: Remove all Virtualbox VMs
-        if: always() # do this even if a step before has failed
-        run: VBoxManage list vms | awk -F'"' '{print $2}' | xargs -I {} VBoxManage unregistervm --delete "{}"
-
-      - name: Remove all Virtualbox HDs
-        if: always() # do this even if a step before has failed
-        run: VBoxManage list hdds | awk -F':' '/^UUID:/ {print $2}' | xargs -I {} VBoxManage closemedium disk "{}" --delete
-
-      - name: Remove all Virtualbox Networks
-        if: always() # do this even if a step before has failed
-        run: VBoxManage list hostonlyifs | grep '^Name:' | awk '{print $2}' | grep '^vboxnet' | xargs -I {} VBoxManage hostonlyif remove {}
-
-      - name: Remove Virtualbox network config
-        if: always() # do this even if a step before has failed
-        run: sudo rm /etc/vbox/networks.conf || true
+      - name: Clean repository-owned resources after testing
+        if: always()
+        run: ./.github/scripts/cleanup-runner-resources.sh --apply
 
       - name: Upload log files
         if: always() # do this even if a step before has failed
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # 4.4.3
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # 7.0.1
         with:
-          name: logs
+          name: logs-${{ matrix.scenario }}-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
             ${{ runner.temp }}/logs
-          overwrite: true
+          if-no-files-found: warn
+          retention-days: 14

.gitignore

@@ -1,4 +1,6 @@
 .env/
 *.log
 ansible.cfg
+.ansible/
 kubeconfig
+zIgnore/

.pre-commit-config.yaml

@@ -20,6 +20,9 @@ repos:
     rev: v6.22.2
     hooks:
       - id: ansible-lint
+        additional_dependencies: [ansible-core==2.18.0]
+        language_version: python3.12
+        args: [--offline]
   - repo: https://github.com/shellcheck-py/shellcheck-py
     rev: v0.9.0.6
     hooks:
@@ -33,3 +36,11 @@ repos:
     rev: 0.6.4
     hooks:
       - id: fix-smartquotes
+  - repo: local
+    hooks:
+      - id: cleanup-runner-resources-test
+        name: cleanup runner resources test
+        entry: .github/scripts/test-cleanup-runner-resources.sh
+        language: system
+        pass_filenames: false
+        files: ^\.github/scripts/(cleanup-runner-resources|test-cleanup-runner-resources)\.sh$

molecule/default/prepare.yml

@@ -20,3 +20,53 @@
         enabled: false
         state: stopped
       become: true
+
+    - name: Verify the private cluster interface
+      ansible.builtin.assert:
+        that:
+          - flannel_iface in ansible_facts
+          - ansible_facts[flannel_iface].ipv4 is defined
+          - ansible_facts[flannel_iface].ipv4.address is defined
+        fail_msg: >-
+          The Vagrant private interface {{ flannel_iface }} does not have an
+          IPv4 address on {{ inventory_hostname }}.
+
+    - name: Verify guest-to-guest cluster network reachability
+      ansible.builtin.command:
+        argv:
+          - ping
+          - -c
+          - "1"
+          - -W
+          - "1"
+          - >-
+            {{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip
+            | split(',') | first }}
+      register: primary_cluster_ping
+      until: primary_cluster_ping.rc == 0
+      retries: 6
+      delay: 2
+      changed_when: false
+
+    - name: Verify GitHub release host DNS
+      ansible.builtin.getent:
+        database: hosts
+        key: github.com
+      register: github_dns
+      retries: 6
+      delay: 5
+      until: github_dns is succeeded
+
+    - name: Verify k3s checksum URL is reachable
+      ansible.builtin.uri:
+        url: >-
+          https://github.com/k3s-io/k3s/releases/download/{{ k3s_version
+          }}/sha256sum-amd64.txt
+        method: HEAD
+        follow_redirects: safe
+        status_code: [200, 302]
+        timeout: 15
+      register: k3s_checksum_request
+      retries: 3
+      delay: 5
+      until: k3s_checksum_request.status in [200, 302]

roles/download/tasks/main.yml

@@ -7,6 +7,10 @@
     owner: root
     group: root
     mode: "0755"
+  register: k3s_download_x64
+  retries: 5
+  delay: 10
+  until: k3s_download_x64 is succeeded
   when: ansible_facts.architecture == "x86_64"
 
 - name: Download k3s binary arm64
@@ -17,6 +21,10 @@
     owner: root
     group: root
     mode: "0755"
+  register: k3s_download_arm64
+  retries: 5
+  delay: 10
+  until: k3s_download_arm64 is succeeded
   when:
     - ( ansible_facts.architecture is search("arm") and ansible_facts.userspace_bits == "64" )
       or ansible_facts.architecture is search("aarch64")
@@ -29,6 +37,10 @@
     owner: root
     group: root
     mode: "0755"
+  register: k3s_download_armhf
+  retries: 5
+  delay: 10
+  until: k3s_download_armhf is succeeded
   when:
     - ansible_facts.architecture is search("arm")
     - ansible_facts.userspace_bits == "32"

roles/k3s_server/tasks/fetch_k3s_init_logs.yml

@@ -4,10 +4,89 @@
 
 - name: Fetch k3s-init.service logs
   ansible.builtin.command:
-    cmd: journalctl --all --unit=k3s-init.service
+    cmd: >-
+      timeout --signal=TERM --kill-after=5s 30s journalctl --no-pager
+      --unit=k3s-init.service --since=-30min --lines=5000
   changed_when: false
+  failed_when: false
+  no_log: true
   register: k3s_init_log
 
+- name: Fetch k3s-init.service status
+  ansible.builtin.command:
+    cmd: >-
+      timeout --signal=TERM --kill-after=5s 15s systemctl status
+      k3s-init.service --no-pager --full
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_init_status
+
+- name: Fetch IP address state
+  ansible.builtin.command:
+    cmd: ip -br address
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_ip_address_state
+
+- name: Fetch IP route state
+  ansible.builtin.command:
+    cmd: ip route show
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_ip_route_state
+
+- name: Fetch IP neighbor state
+  ansible.builtin.command:
+    cmd: ip neigh show
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_ip_neighbor_state
+
+- name: Fetch IP rule state
+  ansible.builtin.command:
+    cmd: ip rule show
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_ip_rule_state
+
+- name: Fetch listening TCP sockets
+  ansible.builtin.command:
+    cmd: ss -ltn
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_tcp_listener_state
+
+- name: Ping the primary Kubernetes API address from {{ ansible_hostname }}
+  ansible.builtin.command:
+    argv:
+      - ping
+      - -c
+      - "1"
+      - -W
+      - "1"
+      - "{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(',') | first }}"
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_primary_api_ping
+
+- name: Probe primary Kubernetes API from {{ ansible_hostname }}
+  ansible.builtin.wait_for:
+    host: "{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(',') | first }}"
+    port: 6443
+    connect_timeout: 1
+    timeout: 2
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_primary_api_probe
+
 - name: Create {{ log_destination }}
   delegate_to: localhost
   run_once: true
@@ -25,4 +104,43 @@
     dest: "{{ log_destination }}/k3s-init@{{ ansible_hostname }}.log"
     mode: "0644"
   vars:
-    content: "{{ k3s_init_log.stdout }}"
+    content: |
+      === k3s-init.service status ===
+      {{ k3s_init_status.stdout | regex_replace('--token(?:=| +)[^ ]+', '--token ***') }}
+
+      === k3s-init.service journal ===
+      {{ k3s_init_log.stdout | regex_replace('--token(?:=| +)[^ ]+', '--token ***') }}
+
+- name: Store network diagnostics to {{ log_destination }}
+  delegate_to: localhost
+  become: false
+  ansible.builtin.template:
+    src: content.j2
+    dest: "{{ log_destination }}/network@{{ ansible_hostname }}.log"
+    mode: "0644"
+  vars:
+    content: |
+      === ip -br address ===
+      {{ k3s_ip_address_state.stdout }}
+
+      === ip route show ===
+      {{ k3s_ip_route_state.stdout }}
+
+      === ip neigh show ===
+      {{ k3s_ip_neighbor_state.stdout }}
+
+      === ip rule show ===
+      {{ k3s_ip_rule_state.stdout }}
+
+      === ss -ltn ===
+      {{ k3s_tcp_listener_state.stdout }}
+
+      === primary API ping ===
+      rc={{ k3s_primary_api_ping.rc }}
+      {{ k3s_primary_api_ping.stdout }}
+      {{ k3s_primary_api_ping.stderr }}
+
+      === primary API probe ===
+      failed={{ k3s_primary_api_probe.failed | default(false) }}
+      elapsed={{ k3s_primary_api_probe.elapsed | default('unknown') }}
+      msg={{ k3s_primary_api_probe.msg | default('connected') }}

roles/k3s_server/tasks/join_master.yml

@@ -0,0 +1,21 @@
+---
+- name: Join transient k3s-init service for {{ joining_master }}
+  ansible.builtin.command:
+    cmd: >-
+      systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init
+      k3s server {{ hostvars[joining_master].k3s_server_init_args }}
+    creates: "{{ systemd_dir }}/k3s-init.service"
+  delegate_to: "{{ joining_master }}"
+  no_log: true
+
+- name: Wait for primary registration of {{ joining_master }}
+  ansible.builtin.command:
+    cmd: >-
+      {{ k3s_kubectl_binary | default('k3s kubectl') }} get node
+      {{ hostvars[joining_master].ansible_hostname }}
+  delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
+  register: joined_master
+  until: joined_master.rc == 0
+  retries: "{{ retry_count | default(20) }}"
+  delay: 2
+  changed_when: false

roles/k3s_server/tasks/main.yml

@@ -34,22 +34,64 @@
   tags: kubevip
   when: kube_vip_lb_ip_range is defined
 
-- name: Init cluster inside the transient k3s-init service
-  ansible.builtin.command:
-    cmd: systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server {{ server_init_args }}
-    creates: "{{ systemd_dir }}/k3s-init.service"
-
-- name: Verification
+- name: Initialize and verify the K3s control plane
   when: not ansible_check_mode
   block:
+    - name: Materialize per-host server initialization arguments
+      ansible.builtin.set_fact:
+        k3s_server_init_args: "{{ server_init_args }}"
+      no_log: true
+      when: groups[group_name_master | default('master')] | length > 1
+
+    - name: Init the first master inside the transient k3s-init service
+      ansible.builtin.command:
+        cmd: >-
+          systemd-run -p RestartSec=2 -p Restart=on-failure
+          --unit=k3s-init k3s server {{ server_init_args }}
+        creates: "{{ systemd_dir }}/k3s-init.service"
+      no_log: true
+      when: inventory_hostname == groups[group_name_master | default('master')][0]
+
+    - name: Wait for the first master Kubernetes API
+      ansible.builtin.command:
+        cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get --raw=/readyz"
+      delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
+      register: first_master_api
+      until: first_master_api.rc == 0
+      retries: "{{ retry_count | default(20) }}"
+      delay: 2
+      changed_when: false
+      run_once: true
+
+    - name: Verify the first master API listener on its node address
+      ansible.builtin.wait_for:
+        host: >-
+          {{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip
+          | split(',') | first }}
+        port: 6443
+        connect_timeout: 2
+        timeout: "{{ retry_count | default(20) * 2 }}"
+      delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
+      run_once: true
+
+    - name: Join additional masters one at a time
+      ansible.builtin.include_tasks: join_master.yml
+      loop: "{{ groups[group_name_master | default('master')][1:] }}"
+      loop_control:
+        loop_var: joining_master
+      run_once: true
+      when: groups[group_name_master | default('master')] | length > 1
+
     - name: Verify that all nodes actually joined (check k3s-init.service if this fails)
       ansible.builtin.command:
         cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get nodes -l 'node-role.kubernetes.io/master=true' -o=jsonpath='{.items[*].metadata.name}'" # yamllint disable-line rule:line-length
+      delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
       register: nodes
       until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups[group_name_master | default('master')] | length) # yamllint disable-line rule:line-length
       retries: "{{ retry_count | default(20) }}"
       delay: 10
       changed_when: false
+      run_once: true
   always:
     - name: Save logs of k3s-init.service
       ansible.builtin.include_tasks: fetch_k3s_init_logs.yml