fix(k3s-server): stabilize single-server bootstrap

- delegate cgroups for transient K3s server units\n- verify inventory node registration without legacy role labels\n- wait for bootstrap CRDs before replacing the transient service

.ansible-lint

@@ -20,3 +20,8 @@ exclude_paths:
 
 skip_list:
   - var-naming[no-role-prefix]
+
+# The Molecule Vagrant driver injects this module at runtime. The custom create
+# playbook is syntax-checked separately against the exact pinned plugin module.
+mock_modules:
+  - vagrant

.github/download-boxes.sh

@@ -1,40 +1,91 @@
 #!/bin/bash
 
 # download-boxes.sh
-# Check all molecule.yml files for required Vagrant boxes and download the ones that are not
+# Validate the pinned Vagrant box set and download exact versions that are not
-# already present on the system.
+# already present in VAGRANT_HOME.
 
 set -euo pipefail
 
 GIT_ROOT=$(git rev-parse --show-toplevel)
 PROVIDER=virtualbox
+LOCK_FILE="${VAGRANT_BOX_LOCK_FILE:-${GIT_ROOT}/.github/vagrant-boxes.lock}"
 
-# Define the path to the molecule.yml files
 MOLECULE_YML_PATH=("${GIT_ROOT}"/molecule/*/molecule.yml)
 
-# Extract and sort unique boxes from all molecule.yml files
+# Extract the unique boxes referenced by the scenarios.
-all_boxes=$(for file in "${MOLECULE_YML_PATH[@]}"; do
+declared_boxes=$(for file in "${MOLECULE_YML_PATH[@]}"; do
     yq -r '.platforms[].box' "$file"
 done | sort -u)
 
-echo all_boxes: "$all_boxes"
+if [[ ! -r "$LOCK_FILE" ]]; then
+    printf 'Vagrant box lock file is missing or unreadable: %s\n' "$LOCK_FILE" >&2
+    exit 1
+fi
 
-# Read the boxes that are currently present on the system (for the current provider)
+lock_entries=$(awk '
+    /^[[:space:]]*#/ || NF == 0 { next }
+    NF != 3 {
+        printf "Invalid lock entry on line %d: expected box, version, architecture\n", NR > "/dev/stderr"
+        invalid = 1
+        next
+    }
+    { print $1 " " $2 " " $3 }
+    END { exit invalid }
+' "$LOCK_FILE")
+
+duplicate_boxes=$(printf '%s\n' "$lock_entries" | awk '{ print $1 }' | sort | uniq -d)
+if [[ -n "$duplicate_boxes" ]]; then
+    printf 'Duplicate Vagrant box lock entries:\n%s\n' "$duplicate_boxes" >&2
+    exit 1
+fi
+
+locked_boxes=$(printf '%s\n' "$lock_entries" | sort)
+locked_names=$(printf '%s\n' "$locked_boxes" | awk '{ print $1 }')
+missing_locks=$(comm -23 <(printf '%s\n' "$declared_boxes") <(printf '%s\n' "$locked_names"))
+unused_locks=$(comm -13 <(printf '%s\n' "$declared_boxes") <(printf '%s\n' "$locked_names"))
+
+if [[ -n "$missing_locks" || -n "$unused_locks" ]]; then
+    if [[ -n "$missing_locks" ]]; then
+        printf 'Scenario boxes missing from the lock file:\n%s\n' "$missing_locks" >&2
+    fi
+    if [[ -n "$unused_locks" ]]; then
+        printf 'Lock entries not referenced by a scenario:\n%s\n' "$unused_locks" >&2
+    fi
+    exit 1
+fi
+
+printf 'Pinned Vagrant boxes:\n%s\n' "$locked_boxes"
+
+# Read exact box, provider, version, and architecture tuples already present.
 present_boxes=$(
-    (vagrant box list |
+    vagrant box list --machine-readable |
-        grep "${PROVIDER}" | # Filter by boxes available for the current provider
+        awk -F, -v expected_provider="$PROVIDER" '
-        awk '{print $1;}' |  # The box name is the first word in each line
+            $3 == "box-name" { name = $4; next }
-        sort |
+            $3 == "box-provider" { provider = $4; next }
-        uniq) ||
+            $3 == "box-version" { version = $4; next }
-        echo "" # In case any of these commands fails, just use an empty list
+            $3 == "box-architecture" {
+                architecture = $4
+                if (provider == expected_provider) {
+                    print name " " version " " architecture
+                }
+                name = provider = version = architecture = ""
+            }
+        ' |
+        sort -u
 )
 
-# The boxes that we need to download are the ones present in $all_boxes, but not $present_boxes.
+download_boxes=$(comm -23 \
-download_boxes=$(comm -2 -3 <(echo "${all_boxes}") <(echo "${present_boxes}"))
+    <(printf '%s\n' "$locked_boxes") \
+    <(printf '%s\n' "$present_boxes"))
 
-# Actually download the necessary boxes
+if [[ -n "$download_boxes" ]]; then
-if [ -n "${download_boxes}" ]; then
+    printf '%s\n' "$download_boxes" | while read -r box version architecture; do
-    echo "${download_boxes}" | while IFS= read -r box; do
+        vagrant box add \
-        vagrant box add --provider "${PROVIDER}" "${box}"
+            --provider "$PROVIDER" \
+            --box-version "$version" \
+            --architecture "$architecture" \
+            "$box"
     done
+else
+    printf 'All pinned Vagrant boxes are already present.\n'
 fi

.github/scripts/collect-runner-diagnostics.sh

@@ -26,9 +26,20 @@ run_capture virtualbox-vms VBoxManage list vms
 run_capture virtualbox-running-vms VBoxManage list runningvms
 run_capture virtualbox-disks VBoxManage list hdds
 run_capture virtualbox-hostonlyifs VBoxManage list hostonlyifs
+run_capture virtualbox-groups VBoxManage list groups
 run_capture vagrant-status vagrant global-status
 run_capture molecule-state find "${HOME}/.cache/molecule" -maxdepth 6 -type f -path '*/.vagrant/machines/*/virtualbox/id' -print
 
+scenario_name="${K3S_CI_SCENARIO_NAME:-}"
+if [[ "$scenario_name" =~ ^[A-Za-z0-9_-]+$ ]]; then
+  molecule_state_dir="${HOME}/.cache/molecule/k3s-ansible/${scenario_name}"
+  for log_name in vagrant.out vagrant.err; do
+    if [[ -r "${molecule_state_dir}/${log_name}" ]]; then
+      cp -- "${molecule_state_dir}/${log_name}" "$output_dir/${scenario_name}-${log_name}"
+    fi
+  done
+fi
+
 if [[ -r /etc/vbox/networks.conf ]]; then
   cp -- /etc/vbox/networks.conf "$output_dir/virtualbox-networks.conf"
 fi

.github/scripts/monitor-runner-resources.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+output_dir="${1:?output directory is required}"
+interval="${2:-10}"
+[[ "$interval" =~ ^[1-9][0-9]*$ ]] || {
+  printf 'monitor interval must be a positive integer\n' >&2
+  exit 2
+}
+
+mkdir -p -- "$output_dir"
+umask 077
+
+free -h > "$output_dir/memory-before.txt"
+df -h > "$output_dir/disk-before.txt"
+vmstat -w "$interval" > "$output_dir/vmstat.txt" &
+vmstat_pid=$!
+
+iostat_pid=""
+if command -v iostat >/dev/null 2>&1; then
+  iostat -dx "$interval" > "$output_dir/iostat.txt" &
+  iostat_pid=$!
+else
+  printf 'iostat is not installed on this runner\n' > "$output_dir/iostat-unavailable.txt"
+fi
+
+cleanup() {
+  local rc=$?
+  trap - EXIT INT TERM
+  kill "$vmstat_pid" 2>/dev/null || true
+  [[ -z "$iostat_pid" ]] || kill "$iostat_pid" 2>/dev/null || true
+  wait "$vmstat_pid" 2>/dev/null || true
+  [[ -z "$iostat_pid" ]] || wait "$iostat_pid" 2>/dev/null || true
+  free -h > "$output_dir/memory-after.txt"
+  df -h > "$output_dir/disk-after.txt"
+  exit "$rc"
+}
+trap cleanup EXIT INT TERM
+
+while :; do
+  sleep 3600 &
+  wait $!
+done

.github/scripts/prepare-vagrant-box-masters.sh

@@ -0,0 +1,210 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+fail() {
+  printf 'Vagrant box master preparation refused: %s\n' "$1" >&2
+  exit 3
+}
+
+root_contains() {
+  local root="$1"
+  local path="$2"
+  [[ "$path" == "$root"/* ]]
+}
+
+read_machine_value() {
+  local machine_info="$1"
+  local key="$2"
+  awk -F= -v key="$key" '$1 == key {gsub(/"/, "", $2); print $2; exit}' <<< "$machine_info"
+}
+
+read_extra_data() {
+  local uuid="$1"
+  local key="$2"
+  local value
+  value="$(VBoxManage getextradata "$uuid" "$key" 2>/dev/null || true)"
+  [[ "$value" == 'Value: '* ]] || return 1
+  printf '%s\n' "${value#Value: }"
+}
+
+validate_owned_master() {
+  local uuid="$1"
+  local box="$2"
+  local version="$3"
+  local architecture="$4"
+  local machine_info cfg_file cfg_file_real vm_state groups disk_path disk_path_real
+
+  [[ "$uuid" =~ ^[0-9a-fA-F-]{36}$ ]] || return 1
+  machine_info="$(VBoxManage showvminfo "$uuid" --machinereadable 2>/dev/null)" || return 1
+  vm_state="$(read_machine_value "$machine_info" VMState)"
+  groups="$(read_machine_value "$machine_info" groups)"
+  cfg_file="$(read_machine_value "$machine_info" CfgFile)"
+  [[ "$vm_state" == poweroff ]] || return 1
+  [[ ",$groups," == *,/k3s-ansible/box-masters,* ]] || return 1
+  [[ -n "$cfg_file" ]] || return 1
+  cfg_file_real="$(readlink -f -- "$cfg_file" 2>/dev/null || true)"
+  [[ -n "$cfg_file_real" ]] || return 1
+  root_contains "$virtualbox_root_real" "$cfg_file_real" || return 1
+
+  [[ "$(read_extra_data "$uuid" k3s-ansible/owner || true)" == box-master ]] || return 1
+  [[ "$(read_extra_data "$uuid" k3s-ansible/box || true)" == "$box" ]] || return 1
+  [[ "$(read_extra_data "$uuid" k3s-ansible/version || true)" == "$version" ]] || return 1
+  [[ "$(read_extra_data "$uuid" k3s-ansible/architecture || true)" == "$architecture" ]] || return 1
+
+  while IFS= read -r disk_path; do
+    [[ -z "$disk_path" || "$disk_path" == none ]] && continue
+    disk_path_real="$(readlink -f -- "$disk_path" 2>/dev/null || true)"
+    [[ -n "$disk_path_real" ]] || return 1
+    root_contains "$virtualbox_root_real" "$disk_path_real" || return 1
+  done < <(awk -F= '$1 ~ /^(SATA|IDE|SCSI|SAS|VirtioSCSI|NVMe)-[0-9]+-[0-9]+$/ {
+    gsub(/"/, "", $2); print $2
+  }' <<< "$machine_info")
+}
+
+write_prewarm_vagrantfile() {
+  local destination="$1"
+  local box="$2"
+  local version="$3"
+  {
+    printf '%s\n' "Vagrant.configure('2') do |config|"
+    printf '  config.vm.box = "%s"\n' "$box"
+    printf '  config.vm.box_version = "%s"\n' "$version"
+    printf '%s\n' \
+      '  config.vm.synced_folder ".", "/vagrant", disabled: true' \
+      '  config.vm.hostname = "k3s-ansible-box-prewarm"' \
+      '  config.vm.provider "virtualbox" do |virtualbox|' \
+      '    virtualbox.linked_clone = true' \
+      '    virtualbox.memory = 512' \
+      '    virtualbox.cpus = 1' \
+      '  end' \
+      'end'
+  } > "$destination"
+}
+
+cleanup_prewarm() {
+  local rc=$?
+  trap - EXIT
+  if [[ -n "${prewarm_dir:-}" && -d "$prewarm_dir" ]]; then
+    VAGRANT_CWD="$prewarm_dir" vagrant destroy --force >/dev/null 2>&1 || true
+    rm -rf -- "$prewarm_dir"
+  fi
+  exit "$rc"
+}
+
+create_owned_master() {
+  local box="$1"
+  local version="$2"
+  local architecture="$3"
+  local master_id_file="$4"
+  local mapping_file="$5"
+  local uuid machine_info cfg_file cfg_file_real vm_state mapping_tmp
+
+  # A master_id restored from an immutable cache is only a hint. Without the
+  # runner-local ownership record and matching VirtualBox metadata it is not
+  # trusted, adopted, modified, or deleted.
+  rm -f -- "$master_id_file"
+
+  prewarm_dir="$(mktemp -d "${master_root}/prewarm.XXXXXX")"
+  trap cleanup_prewarm EXIT
+  write_prewarm_vagrantfile "$prewarm_dir/Vagrantfile" "$box" "$version"
+
+  printf 'Creating runner-owned linked-clone master for %s %s %s\n' \
+    "$box" "$version" "$architecture"
+  VAGRANT_CWD="$prewarm_dir" vagrant up --provider virtualbox --no-provision
+
+  [[ -r "$master_id_file" ]] || fail "Vagrant did not record a master UUID for $box"
+  uuid="$(tr -d '[:space:]' < "$master_id_file")"
+  [[ "$uuid" =~ ^[0-9a-fA-F-]{36}$ ]] || fail "Vagrant recorded an invalid master UUID for $box"
+
+  machine_info="$(VBoxManage showvminfo "$uuid" --machinereadable 2>/dev/null)" || \
+    fail "Vagrant master $uuid for $box is not registered"
+  vm_state="$(read_machine_value "$machine_info" VMState)"
+  cfg_file="$(read_machine_value "$machine_info" CfgFile)"
+  cfg_file_real="$(readlink -f -- "$cfg_file" 2>/dev/null || true)"
+  [[ "$vm_state" == poweroff ]] || fail "new Vagrant master $uuid is not powered off"
+  if [[ -z "$cfg_file_real" ]] || ! root_contains "$virtualbox_root_real" "$cfg_file_real"; then
+    fail "new Vagrant master $uuid is outside the runner VirtualBox root"
+  fi
+
+  VAGRANT_CWD="$prewarm_dir" vagrant destroy --force
+  VBoxManage modifyvm "$uuid" --groups /k3s-ansible/box-masters
+  VBoxManage setextradata "$uuid" k3s-ansible/owner box-master
+  VBoxManage setextradata "$uuid" k3s-ansible/box "$box"
+  VBoxManage setextradata "$uuid" k3s-ansible/version "$version"
+  VBoxManage setextradata "$uuid" k3s-ansible/architecture "$architecture"
+  validate_owned_master "$uuid" "$box" "$version" "$architecture" || \
+    fail "new Vagrant master $uuid failed ownership validation"
+
+  rm -rf -- "$prewarm_dir"
+  prewarm_dir=""
+  trap - EXIT
+
+  mapping_tmp="${mapping_file}.tmp"
+  printf '%s\n' "$uuid" > "$mapping_tmp"
+  chmod 600 "$mapping_tmp"
+  mv -- "$mapping_tmp" "$mapping_file"
+  printf '%s\n' "$uuid" > "$master_id_file"
+  chmod 600 "$master_id_file"
+  printf 'Created and recorded owned master %s for %s\n' "$uuid" "$box"
+}
+
+repository_root="${K3S_CI_REPOSITORY_ROOT:-$(git rev-parse --show-toplevel)}"
+lock_file="${VAGRANT_BOX_LOCK_FILE:-${repository_root}/.github/vagrant-boxes.lock}"
+vagrant_home="${VAGRANT_HOME:?VAGRANT_HOME must be set}"
+master_root="${K3S_CI_VAGRANT_MASTER_ROOT:-${HOME:?HOME must be set}/.cache/k3s-ci/vagrant-masters}"
+virtualbox_root="${K3S_CI_VIRTUALBOX_ROOT:-${HOME}/VirtualBox VMs}"
+
+[[ -r "$lock_file" ]] || fail "box lock file is missing or unreadable: $lock_file"
+[[ -d "$vagrant_home/boxes" ]] || fail "Vagrant box directory is missing: $vagrant_home/boxes"
+[[ -d "$virtualbox_root" ]] || fail "VirtualBox root is missing: $virtualbox_root"
+
+box_root_real="$(readlink -f -- "$vagrant_home/boxes")"
+virtualbox_root_real="$(readlink -f -- "$virtualbox_root")"
+mkdir -p -- "$master_root"
+chmod 700 "$master_root"
+
+exec 9> "${master_root}/prepare.lock"
+flock 9
+
+lock_entries="$(awk '
+  /^[[:space:]]*#/ || NF == 0 { next }
+  NF != 3 { invalid = 1; next }
+  { print $1 " " $2 " " $3 }
+  END { exit invalid }
+' "$lock_file")" || fail 'invalid Vagrant box lock entry'
+[[ -n "$lock_entries" ]] || fail 'Vagrant box lock is empty'
+
+while read -r box version architecture; do
+  [[ "$box" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || fail "invalid box name: $box"
+  [[ "$version" =~ ^[A-Za-z0-9._-]+$ ]] || fail "invalid box version: $version"
+  [[ "$architecture" =~ ^[A-Za-z0-9._-]+$ ]] || fail "invalid box architecture: $architecture"
+
+  box_slug="${box//\//-VAGRANTSLASH-}"
+  record_slug="${box//\//_}-${version}-${architecture}"
+  box_dir="${vagrant_home}/boxes/${box_slug}/${version}/${architecture}/virtualbox"
+  [[ -d "$box_dir" ]] || fail "pinned box is not installed: $box $version $architecture"
+  box_dir_real="$(readlink -f -- "$box_dir")"
+  root_contains "$box_root_real" "$box_dir_real" || fail "box directory is outside VAGRANT_HOME: $box_dir_real"
+
+  master_id_file="${box_dir_real}/master_id"
+  mapping_file="${master_root}/${record_slug}.uuid"
+  uuid=""
+  if [[ -r "$mapping_file" ]]; then
+    uuid="$(tr -d '[:space:]' < "$mapping_file")"
+  fi
+
+  if [[ -n "$uuid" ]] && validate_owned_master "$uuid" "$box" "$version" "$architecture"; then
+    printf '%s\n' "$uuid" > "$master_id_file"
+    chmod 600 "$master_id_file"
+    printf 'Reusing owned master %s for %s %s %s\n' "$uuid" "$box" "$version" "$architecture"
+    continue
+  fi
+
+  if [[ -e "$mapping_file" ]]; then
+    printf 'Owned master record is stale for %s; rebuilding without deleting any VM or disk.\n' "$box"
+  fi
+  create_owned_master "$box" "$version" "$architecture" "$master_id_file" "$mapping_file"
+done <<< "$lock_entries"
+
+printf 'All pinned Vagrant box masters are ready.\n'

.github/scripts/test-fixtures/mock-flock

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+[[ "${1:-}" =~ ^[0-9]+$ ]]

.github/scripts/test-fixtures/mock-vagrant

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+printf '%s\n' "$*" >> "$MOCK_VAGRANT_LOG"
+case "${1:-}" in
+  up)
+    counter_file="$MOCK_VBOX_STATE/counter"
+    counter=0
+    [[ ! -r "$counter_file" ]] || counter="$(cat "$counter_file")"
+    counter=$((counter + 1))
+    printf '%s\n' "$counter" > "$counter_file"
+    uuid="00000000-0000-4000-8000-$(printf '%012d' "$counter")"
+    vm_dir="$MOCK_VBOX_ROOT/master-$counter"
+    mkdir -p -- "$vm_dir"
+    : > "$vm_dir/master.vbox"
+    : > "$vm_dir/master.vdi"
+    printf '%s\n' \
+      'VMState="poweroff"' \
+      'groups="/"' \
+      "CfgFile=\"$vm_dir/master.vbox\"" \
+      "SATA-0-0=\"$vm_dir/master.vdi\"" > "$MOCK_VBOX_STATE/vm-$uuid"
+    printf '%s\n' "$uuid" > "$MOCK_BOX_DIR/master_id"
+    ;;
+  destroy) ;;
+  *) exit 2 ;;
+esac

.github/scripts/test-fixtures/mock-vboxmanage

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+printf '%s\n' "$*" >> "$MOCK_VBOX_LOG"
+command_name="${1:-}"
+uuid="${2:-}"
+
+case "$command_name" in
+  showvminfo)
+    [[ -r "$MOCK_VBOX_STATE/vm-$uuid" ]] || exit 1
+    cat "$MOCK_VBOX_STATE/vm-$uuid"
+    ;;
+  getextradata)
+    key_slug="${3//\//_}"
+    if [[ ! -r "$MOCK_VBOX_STATE/extra-$uuid-$key_slug" ]]; then
+      printf '%s\n' 'No value set!'
+      exit 0
+    fi
+    printf 'Value: '
+    cat "$MOCK_VBOX_STATE/extra-$uuid-$key_slug"
+    ;;
+  modifyvm)
+    [[ "${3:-}" == --groups ]]
+    awk -v groups="${4:-}" '
+      $1 !~ /^groups=/ { print }
+      END { printf "groups=\"%s\"\n", groups }
+    ' "$MOCK_VBOX_STATE/vm-$uuid" > "$MOCK_VBOX_STATE/vm-$uuid.tmp"
+    mv "$MOCK_VBOX_STATE/vm-$uuid.tmp" "$MOCK_VBOX_STATE/vm-$uuid"
+    ;;
+  setextradata)
+    key_slug="${3//\//_}"
+    printf '%s\n' "${4:-}" > "$MOCK_VBOX_STATE/extra-$uuid-$key_slug"
+    ;;
+  unregistervm|closemedium)
+    printf '%s\n' 'destructive VirtualBox command invoked' >&2
+    exit 99
+    ;;
+  *) exit 2 ;;
+esac

.github/scripts/test-k3s-server-bootstrap.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+repo_root="$(git rev-parse --show-toplevel)"
+main_tasks="$repo_root/roles/k3s_server/tasks/main.yml"
+join_tasks="$repo_root/roles/k3s_server/tasks/join_master.yml"
+
+for task_file in "$main_tasks" "$join_tasks"; do
+  for property in \
+    'Delegate=yes' \
+    'TasksMax=infinity' \
+    'KillMode=process' \
+    'LimitNOFILE=1048576' \
+    'LimitNPROC=infinity' \
+    'LimitCORE=infinity'; do
+    grep -Fq -- "$property" "$task_file" || {
+      printf '%s is missing transient K3s property %s\n' "$task_file" "$property" >&2
+      exit 1
+    }
+  done
+done
+
+if grep -Fq -- "node-role.kubernetes.io/master=true' -o=jsonpath" "$main_tasks"; then
+  printf 'control-plane registration still depends on the optional legacy master role key\n' >&2
+  exit 1
+fi
+
+grep -Fq -- "map('extract', hostvars, 'ansible_hostname')" "$main_tasks"
+grep -Fq -- 'difference(nodes.stdout.split())' "$main_tasks"
+grep -Fq -- 'crd/addons.k3s.cattle.io' "$main_tasks"
+grep -Fq -- 'crd/helmcharts.helm.cattle.io' "$main_tasks"
+grep -Fq -- 'crd/helmchartconfigs.helm.cattle.io' "$main_tasks"
+
+printf 'K3s transient bootstrap regression test passed\n'

.github/scripts/test-prepare-vagrant-box-masters.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+repo_root="$(git rev-parse --show-toplevel)"
+fixture="$(mktemp -d)"
+trap 'rm -rf -- "$fixture"' EXIT
+
+mock_bin="$fixture/bin"
+mock_state="$fixture/state"
+mock_home="$fixture/home"
+mock_vagrant_home="$fixture/vagrant-home"
+mock_box_dir="$mock_vagrant_home/boxes/generic-VAGRANTSLASH-ubuntu2204/4.3.12/amd64/virtualbox"
+mock_vbox_root="$mock_home/VirtualBox VMs"
+mock_master_root="$mock_home/.cache/k3s-ci/vagrant-masters"
+lock_file="$fixture/vagrant-boxes.lock"
+mkdir -p -- "$mock_bin" "$mock_state" "$mock_box_dir" "$mock_vbox_root"
+printf '%s\n' 'generic/ubuntu2204 4.3.12 amd64' > "$lock_file"
+ln -s "$repo_root/.github/scripts/test-fixtures/mock-vboxmanage" "$mock_bin/VBoxManage"
+ln -s "$repo_root/.github/scripts/test-fixtures/mock-vagrant" "$mock_bin/vagrant"
+ln -s "$repo_root/.github/scripts/test-fixtures/mock-flock" "$mock_bin/flock"
+
+export PATH="$mock_bin:$PATH"
+export HOME="$mock_home"
+export VAGRANT_HOME="$mock_vagrant_home"
+export VAGRANT_BOX_LOCK_FILE="$lock_file"
+export K3S_CI_REPOSITORY_ROOT="$repo_root"
+export K3S_CI_VAGRANT_MASTER_ROOT="$mock_master_root"
+export K3S_CI_VIRTUALBOX_ROOT="$mock_vbox_root"
+export MOCK_VBOX_STATE="$mock_state"
+export MOCK_VBOX_ROOT="$mock_vbox_root"
+export MOCK_BOX_DIR="$mock_box_dir"
+export MOCK_VBOX_LOG="$fixture/vbox.log"
+export MOCK_VAGRANT_LOG="$fixture/vagrant.log"
+: > "$MOCK_VBOX_LOG"
+: > "$MOCK_VAGRANT_LOG"
+
+unowned_uuid='99999999-9999-4999-8999-999999999999'
+printf '%s\n' "$unowned_uuid" > "$mock_box_dir/master_id"
+
+script="$repo_root/.github/scripts/prepare-vagrant-box-masters.sh"
+first_output="$fixture/first-output"
+second_output="$fixture/second-output"
+third_output="$fixture/third-output"
+
+"$script" > "$first_output"
+mapping_file="$mock_master_root/generic_ubuntu2204-4.3.12-amd64.uuid"
+test -s "$mapping_file"
+cmp -s "$mapping_file" "$mock_box_dir/master_id"
+grep -Fq 'Created and recorded owned master' "$first_output"
+grep -Fq 'modifyvm' "$MOCK_VBOX_LOG"
+grep -Fq 'setextradata' "$MOCK_VBOX_LOG"
+if grep -Fq "$unowned_uuid" "$MOCK_VBOX_LOG"; then
+  printf 'unowned cached master UUID was unexpectedly inspected or modified\n' >&2
+  exit 1
+fi
+if grep -Eq 'unregistervm|closemedium' "$MOCK_VBOX_LOG"; then
+  printf 'master preparation invoked a destructive VirtualBox command\n' >&2
+  exit 1
+fi
+
+: > "$MOCK_VAGRANT_LOG"
+"$script" > "$second_output"
+grep -Fq 'Reusing owned master' "$second_output"
+if grep -Fq 'up ' "$MOCK_VAGRANT_LOG"; then
+  printf 'valid owned master was unexpectedly rebuilt\n' >&2
+  exit 1
+fi
+
+stale_uuid="$(tr -d '[:space:]' < "$mapping_file")"
+rm -f -- "$mock_state/vm-$stale_uuid"
+: > "$MOCK_VAGRANT_LOG"
+"$script" > "$third_output"
+grep -Fq 'rebuilding without deleting any VM or disk' "$third_output"
+grep -Fq 'up ' "$MOCK_VAGRANT_LOG"
+if grep -Eq 'unregistervm|closemedium' "$MOCK_VBOX_LOG"; then
+  printf 'stale master recovery invoked a destructive VirtualBox command\n' >&2
+  exit 1
+fi
+
+printf 'Vagrant box master preparation fixture test passed\n'

.github/scripts/vagrant-up-timed.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+if (($# < 2)); then
+  printf 'Usage: vagrant-up-timed.sh WORKDIR MACHINE [MACHINE ...]\n' >&2
+  exit 2
+fi
+
+workdir="$1"
+shift
+timing_log="${K3S_CI_CREATE_TIMING_LOG:-${RUNNER_TEMP:-/tmp}/k3s-ci-create-timing.log}"
+mkdir -p -- "${timing_log%/*}"
+
+printf '%s batch-start machines=%s\n' "$(date --iso-8601=ns)" "$*" | tee -a "$timing_log"
+set +e
+VAGRANT_CWD="$workdir" vagrant up "$@" --provider virtualbox --no-provision 2>&1 |
+  while IFS= read -r line; do
+    printf '%s %s\n' "$(date --iso-8601=ns)" "$line"
+  done | tee -a "$timing_log"
+rc=${PIPESTATUS[0]}
+set -e
+printf '%s batch-end rc=%d machines=%s\n' "$(date --iso-8601=ns)" "$rc" "$*" | tee -a "$timing_log"
+exit "$rc"

.github/test-download-boxes.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+
+# The single-quoted expressions below are written into fake executables and
+# intentionally expand only when those executables run.
+# shellcheck disable=SC2016
+
+set -Eeuo pipefail
+
+repo_root=$(git rev-parse --show-toplevel)
+test_root=$(mktemp -d)
+fake_bin="$test_root/bin"
+fake_log="$test_root/vagrant.log"
+output="$test_root/output.txt"
+mkdir -p "$fake_bin"
+trap 'rm -rf "$test_root"' EXIT
+
+printf '%s\n' \
+    '#!/usr/bin/env bash' \
+    'set -Eeuo pipefail' \
+    'printf "%s\n" generic/debian12 generic/rocky9 generic/ubuntu2204' \
+    >"$fake_bin/yq"
+
+printf '%s\n' \
+    '#!/usr/bin/env bash' \
+    'set -Eeuo pipefail' \
+    'emit_box() {' \
+    '    printf "0,,box-name,%s\n" "$1"' \
+    '    printf "0,,box-provider,virtualbox\n"' \
+    '    printf "0,,box-version,4.3.12\n"' \
+    '    printf "0,,box-architecture,amd64\n"' \
+    '}' \
+    'if [[ "${1:-}" == box && "${2:-}" == list ]]; then' \
+    '    emit_box generic/debian12' \
+    '    emit_box generic/rocky9' \
+    '    if [[ "${FAKE_PRESENT_MODE:-all}" == all ]]; then' \
+    '        emit_box generic/ubuntu2204' \
+    '    fi' \
+    'elif [[ "${1:-}" == box && "${2:-}" == add ]]; then' \
+    '    printf "%s\n" "$*" >>"${FAKE_VAGRANT_LOG:?}"' \
+    'else' \
+    '    printf "Unexpected vagrant arguments: %s\n" "$*" >&2' \
+    '    exit 1' \
+    'fi' \
+    >"$fake_bin/vagrant"
+chmod +x "$fake_bin/yq" "$fake_bin/vagrant"
+
+PATH="$fake_bin:$PATH" \
+    FAKE_VAGRANT_LOG="$fake_log" \
+    "$repo_root/.github/download-boxes.sh" >"$output"
+grep -Fq 'All pinned Vagrant boxes are already present.' "$output"
+[[ ! -e "$fake_log" ]]
+
+PATH="$fake_bin:$PATH" \
+    FAKE_PRESENT_MODE=partial \
+    FAKE_VAGRANT_LOG="$fake_log" \
+    "$repo_root/.github/download-boxes.sh" >"$output"
+grep -Fxq \
+    'box add --provider virtualbox --box-version 4.3.12 --architecture amd64 generic/ubuntu2204' \
+    "$fake_log"
+
+incomplete_lock="$test_root/incomplete.lock"
+printf '%s\n' \
+    'generic/debian12 4.3.12 amd64' \
+    'generic/rocky9 4.3.12 amd64' \
+    >"$incomplete_lock"
+if PATH="$fake_bin:$PATH" \
+    VAGRANT_BOX_LOCK_FILE="$incomplete_lock" \
+    FAKE_VAGRANT_LOG="$fake_log" \
+    "$repo_root/.github/download-boxes.sh" >"$output" 2>&1; then
+    printf 'Download script accepted a lock missing a scenario box.\n' >&2
+    exit 1
+fi
+grep -Fq 'Scenario boxes missing from the lock file:' "$output"
+grep -Fq 'generic/ubuntu2204' "$output"
+
+duplicate_lock="$test_root/duplicate.lock"
+printf '%s\n' \
+    'generic/debian12 4.3.12 amd64' \
+    'generic/debian12 4.3.11 amd64' \
+    'generic/rocky9 4.3.12 amd64' \
+    'generic/ubuntu2204 4.3.12 amd64' \
+    >"$duplicate_lock"
+if PATH="$fake_bin:$PATH" \
+    VAGRANT_BOX_LOCK_FILE="$duplicate_lock" \
+    FAKE_VAGRANT_LOG="$fake_log" \
+    "$repo_root/.github/download-boxes.sh" >"$output" 2>&1; then
+    printf 'Download script accepted duplicate box lock entries.\n' >&2
+    exit 1
+fi
+grep -Fq 'Duplicate Vagrant box lock entries:' "$output"
+
+printf 'Vagrant box download tests passed.\n'

.github/vagrant-boxes.lock

@@ -0,0 +1,4 @@
+# box version architecture
+generic/debian12 4.3.12 amd64
+generic/rocky9 4.3.12 amd64
+generic/ubuntu2204 4.3.12 amd64

.github/workflows/cache.yml

@@ -40,18 +40,18 @@ jobs:
         id: cache-vagrant
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # 6.1.0
         with:
-          lookup-only: true # if it exists, we don't need to restore and can skip the next step
+          # This producer only needs to know whether the immutable cache exists.
+          # Molecule jobs restore it after this job completes.
+          lookup-only: true
           path: |
             .vagrant-home/boxes
-          key: vagrant-boxes-${{ runner.os }}-${{ runner.arch }}-virtualbox-7.2-vagrant-2.4-${{ hashFiles('**/molecule.yml') }} # yamllint disable-line rule:line-length
+          key: vagrant-boxes-${{ runner.name }}-${{ runner.os }}-${{ runner.arch }}-virtualbox-7.2-vagrant-2.4-${{ hashFiles('.github/vagrant-boxes.lock') }} # yamllint disable-line rule:line-length
-          restore-keys: |
-            vagrant-boxes-${{ runner.os }}-${{ runner.arch }}-virtualbox-7.2-vagrant-2.4-
 
       - name: Download Vagrant boxes for all scenarios
-        # To save some cache space, all scenarios share the same cache key.
+        # An exact hit skips both cache restoration and upstream downloads.
-        # On the other hand, this means that the cache contents should be
+        # A lock change builds and saves one clean, version-pinned cache.
-        # the same across all scenarios. This step ensures that.
+        if: steps.cache-vagrant.outputs.cache-hit != 'true'
-        if: steps.cache-vagrant.outputs.cache-hit != 'true' # only run if false since this is just a cache step
         run: |
           ./.github/download-boxes.sh
+          ./.github/scripts/prepare-vagrant-box-masters.sh
           vagrant box list

.github/workflows/test.yml

@@ -57,9 +57,12 @@ jobs:
         uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # 6.1.0
         with:
           path: .vagrant-home/boxes
-          key: vagrant-boxes-${{ runner.os }}-${{ runner.arch }}-virtualbox-7.2-vagrant-2.4-${{ hashFiles('**/molecule.yml') }} # yamllint disable-line rule:line-length
+          key: vagrant-boxes-${{ runner.name }}-${{ runner.os }}-${{ runner.arch }}-virtualbox-7.2-vagrant-2.4-${{ hashFiles('.github/vagrant-boxes.lock') }} # yamllint disable-line rule:line-length
           fail-on-cache-miss: true
 
+      - name: Prepare runner-owned Vagrant box masters
+        run: ./.github/scripts/prepare-vagrant-box-masters.sh
+
       - name: Install dependencies
         run: |
           echo "::group::Upgrade pip"
@@ -71,7 +74,20 @@ jobs:
           echo "::endgroup::"
 
       - name: Test with molecule
-        run: molecule test --scenario-name ${{ matrix.scenario }}
+        run: |
+          set -Eeuo pipefail
+          resource_dir="${RUNNER_TEMP}/logs/resources/${{ matrix.scenario }}"
+          timing_file="${RUNNER_TEMP}/logs/timing/${{ matrix.scenario }}.txt"
+          mkdir -p -- "${timing_file%/*}"
+          ./.github/scripts/monitor-runner-resources.sh "$resource_dir" 10 &
+          monitor_pid=$!
+          stop_monitor() {
+            kill -TERM "$monitor_pid" 2>/dev/null || true
+            wait "$monitor_pid" 2>/dev/null || true
+          }
+          trap stop_monitor EXIT
+          /usr/bin/time -v -o "$timing_file" \
+            molecule test --scenario-name ${{ matrix.scenario }}
         timeout-minutes: 90
         env:
           ANSIBLE_K3S_LOG_DIR: ${{ runner.temp }}/logs/k3s-ansible/${{ matrix.scenario }}
@@ -79,10 +95,13 @@ jobs:
           ANSIBLE_TIMEOUT: 120
           PY_COLORS: 1
           ANSIBLE_FORCE_COLOR: 1
+          K3S_CI_CREATE_TIMING_LOG: ${{ runner.temp }}/logs/timing/${{ matrix.scenario }}-create.log
 
       - name: Collect runner diagnostics
         if: always()
         run: ./.github/scripts/collect-runner-diagnostics.sh "${RUNNER_TEMP}/logs/runner"
+        env:
+          K3S_CI_SCENARIO_NAME: ${{ matrix.scenario }}
 
       - name: Clean repository-owned resources after testing
         if: always()

.pre-commit-config.yaml

@@ -44,3 +44,21 @@ repos:
         language: system
         pass_filenames: false
         files: ^\.github/scripts/(cleanup-runner-resources|test-cleanup-runner-resources)\.sh$
+      - id: download-vagrant-boxes-test
+        name: Vagrant box download test
+        entry: .github/test-download-boxes.sh
+        language: system
+        pass_filenames: false
+        files: ^\.github/(download-boxes|test-download-boxes)\.sh$|^\.github/vagrant-boxes\.lock$
+      - id: prepare-vagrant-box-masters-test
+        name: Vagrant box master preparation test
+        entry: .github/scripts/test-prepare-vagrant-box-masters.sh
+        language: system
+        pass_filenames: false
+        files: ^\.github/scripts/(prepare-vagrant-box-masters|test-prepare-vagrant-box-masters)\.sh$
+      - id: k3s-server-bootstrap-test
+        name: K3s transient bootstrap test
+        entry: .github/scripts/test-k3s-server-bootstrap.sh
+        language: system
+        pass_filenames: false
+        files: ^roles/k3s_server/tasks/(main|join_master)\.yml$|^\.github/scripts/test-k3s-server-bootstrap\.sh$

molecule/calico/overrides.yml

@@ -7,6 +7,7 @@
         # See:
         # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
         calico_iface: eth1
+        kube_vip_iface: eth1
 
         # The test VMs might be a bit slow, so we give them more time to join the cluster:
         retry_count: 45

molecule/cilium/overrides.yml

@@ -7,6 +7,7 @@
         # See:
         # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
         cilium_iface: eth1
+        kube_vip_iface: eth1
 
         # The test VMs might be a bit slow, so we give them more time to join the cluster:
         retry_count: 45

molecule/default/create.yml

@@ -0,0 +1,87 @@
+---
+- name: Create
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  no_log: "{{ molecule_no_log }}"
+  vars:
+    create_batches:
+      - [control1, control2]
+      - [control3, node1]
+      - [node2]
+  tasks:
+    - name: Verify that bounded batches cover the configured platforms exactly once
+      ansible.builtin.assert:
+        that:
+          - create_batches | flatten | sort == molecule_yml.platforms | map(attribute='name') | sort
+          - create_batches | flatten | length == create_batches | flatten | unique | length
+          - create_batches | map('length') | max <= 2
+        fail_msg: Bounded create batches do not match the configured default platforms.
+
+    - name: Check for existing Vagrant machine state
+      ansible.builtin.stat:
+        path: "{{ molecule_ephemeral_directory }}/.vagrant/machines"
+      register: vagrant_machine_state
+
+    - name: Generate the complete Vagrant configuration for a clean create # noqa fqcn[action]
+      vagrant:
+        instances: "{{ molecule_yml.platforms }}"
+        default_box: "{{ molecule_yml.driver.default_box | default('generic/alpine316') }}"
+        provider_name: "{{ molecule_yml.driver.provider.name | default(omit, true) }}"
+        provision: "{{ molecule_yml.driver.provision | default(omit) }}"
+        cachier: "{{ molecule_yml.driver.cachier | default(omit) }}"
+        parallel: false
+        state: halt
+      when: not vagrant_machine_state.stat.exists
+      changed_when: false
+
+    - name: Start clean Vagrant guests in bounded batches
+      ansible.builtin.command:
+        argv: >-
+          {{
+            [molecule_project_directory + '/.github/scripts/vagrant-up-timed.sh',
+             molecule_ephemeral_directory] + item
+          }}
+      loop: "{{ create_batches }}"
+      loop_control:
+        label: "{{ item | join(', ') }}"
+      when: not vagrant_machine_state.stat.exists
+      changed_when: true
+
+    - name: Reconcile all instances and collect their connection configuration # noqa fqcn[action]
+      vagrant:
+        instances: "{{ molecule_yml.platforms }}"
+        default_box: "{{ molecule_yml.driver.default_box | default('generic/alpine316') }}"
+        provider_name: "{{ molecule_yml.driver.provider.name | default(omit, true) }}"
+        provision: "{{ molecule_yml.driver.provision | default(omit) }}"
+        cachier: "{{ molecule_yml.driver.cachier | default(omit) }}"
+        parallel: false
+        state: up
+      register: server
+      no_log: false
+
+    - name: Populate instance configuration dictionaries
+      ansible.builtin.set_fact:
+        instance_conf_dict:
+          instance: "{{ item.Host }}"
+          address: "{{ item.HostName }}"
+          user: "{{ item.User }}"
+          port: "{{ item.Port }}"
+          identity_file: "{{ item.IdentityFile }}"
+      loop: "{{ server.results }}"
+      register: instance_config_dict
+
+    - name: Convert instance configuration dictionaries to a list
+      ansible.builtin.set_fact:
+        instance_conf: >-
+          {{
+            instance_config_dict.results
+            | map(attribute='ansible_facts.instance_conf_dict')
+            | list
+          }}
+
+    - name: Write Molecule instance configuration
+      ansible.builtin.copy:
+        content: "{{ instance_conf | to_json | from_json | to_yaml }}"
+        dest: "{{ molecule_instance_config }}"
+        mode: "0600"

molecule/default/molecule.yml

@@ -3,6 +3,9 @@ dependency:
   name: galaxy
 driver:
   name: vagrant
+  # The Vagrant driver warns that parallel VirtualBox creation can cause
+  # platform issues. Keep this five-node, mixed-distribution scenario serial.
+  parallel: false
 platforms:
   - name: control1
     box: generic/ubuntu2204
@@ -11,6 +14,10 @@ platforms:
     groups:
       - k3s_cluster
       - master
+    # Keep adapter 2 stable across linked-clone rebuilds so stale host-only
+    # neighbor state still identifies the current scenario guest.
+    provider_raw_config_args:
+      - "customize ['modifyvm', :id, '--mac-address2', '080027A13038']"
     interfaces:
       - network_name: private_network
         ip: 192.168.30.38
@@ -27,6 +34,8 @@ platforms:
     groups:
       - k3s_cluster
       - master
+    provider_raw_config_args:
+      - "customize ['modifyvm', :id, '--mac-address2', '080027A13039']"
     interfaces:
       - network_name: private_network
         ip: 192.168.30.39
@@ -38,6 +47,8 @@ platforms:
     groups:
       - k3s_cluster
       - master
+    provider_raw_config_args:
+      - "customize ['modifyvm', :id, '--mac-address2', '080027A13040']"
     interfaces:
       - network_name: private_network
         ip: 192.168.30.40
@@ -49,6 +60,8 @@ platforms:
     groups:
       - k3s_cluster
       - node
+    provider_raw_config_args:
+      - "customize ['modifyvm', :id, '--mac-address2', '080027A13041']"
     interfaces:
       - network_name: private_network
         ip: 192.168.30.41
@@ -65,6 +78,8 @@ platforms:
     groups:
       - k3s_cluster
       - node
+    provider_raw_config_args:
+      - "customize ['modifyvm', :id, '--mac-address2', '080027A13042']"
     interfaces:
       - network_name: private_network
         ip: 192.168.30.42
@@ -74,6 +89,7 @@ provisioner:
   env:
     ANSIBLE_VERBOSITY: 1
   playbooks:
+    create: create.yml
     converge: ../resources/converge.yml
     side_effect: ../resources/reset.yml
     verify: ../resources/verify.yml

molecule/default/overrides.yml

@@ -7,6 +7,9 @@
         # See:
         # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
         flannel_iface: eth1
+        # kube-vip cannot infer the cluster interface in these multi-NIC
+        # Vagrant guests because the default route is on eth0.
+        kube_vip_iface: eth1
 
         # The test VMs might be a bit slow, so we give them more time to join the cluster:
         retry_count: 45

molecule/default/prepare.yml

@@ -5,21 +5,39 @@
 
 - name: Network setup
   hosts: all
+  vars:
+    primary_master: "{{ groups[group_name_master | default('master')][0] }}"
+    primary_cluster_ip: >-
+      {{ hostvars[primary_master].k3s_node_ip | split(',') | first }}
+    cluster_interface: >-
+      {{ cilium_iface | default(calico_iface | default(flannel_iface)) }}
+    primary_cluster_interface: >-
+      {{ hostvars[primary_master].cilium_iface
+      | default(hostvars[primary_master].calico_iface
+      | default(hostvars[primary_master].flannel_iface)) }}
+    primary_cluster_mac: >-
+      {{ hostvars[primary_master].ansible_facts[primary_cluster_interface].macaddress }}
   tasks:
-    - name: Disable firewalld
+    - name: Gather service facts
-      when: ansible_distribution == "Rocky"
+      ansible.builtin.service_facts:
-      # Rocky Linux comes with firewalld enabled. It blocks some of the network
+
-      # connections needed for our k3s cluster. For our test setup, we just disable
+    - name: Disable guest firewall services
-      # it since the VM host's firewall is still active for connections to and from
+      # The disposable test guests use an isolated VirtualBox network. A distro
-      # the Internet.
+      # firewall can allow ICMP while silently blocking the inter-node Kubernetes
+      # API connection, so disable the known guest firewalls consistently.
       # When building your own cluster, please DO NOT blindly copy this. Instead,
       # please create a custom firewall configuration that fits your network design
       # and security needs.
       ansible.builtin.systemd:
-        name: firewalld
+        name: "{{ item }}"
         enabled: false
         state: stopped
       become: true
+      loop:
+        - firewalld.service
+        - nftables.service
+        - ufw.service
+      when: item in ansible_facts.services
 
     - name: Verify the private cluster interface
       ansible.builtin.assert:
@@ -31,6 +49,35 @@
           The Vagrant private interface {{ flannel_iface }} does not have an
           IPv4 address on {{ inventory_hostname }}.
 
+    - name: Pin disposable cluster peer neighbor entries
+      ansible.builtin.command:
+        argv:
+          - ip
+          - neigh
+          - replace
+          - "{{ peer_cluster_ip }}"
+          - lladdr
+          - "{{ peer_cluster_mac }}"
+          - nud
+          - permanent
+          - dev
+          - "{{ cluster_interface }}"
+      become: true
+      changed_when: false
+      loop: "{{ groups['k3s_cluster'] }}"
+      loop_control:
+        label: "{{ inventory_hostname }} -> {{ item }}"
+      vars:
+        peer_cluster_interface: >-
+          {{ hostvars[item].cilium_iface
+          | default(hostvars[item].calico_iface
+          | default(hostvars[item].flannel_iface)) }}
+        peer_cluster_ip: >-
+          {{ hostvars[item].k3s_node_ip | split(',') | first }}
+        peer_cluster_mac: >-
+          {{ hostvars[item].ansible_facts[peer_cluster_interface].macaddress }}
+      when: item != inventory_hostname
+
     - name: Verify guest-to-guest cluster network reachability
       ansible.builtin.command:
         argv:
@@ -39,15 +86,37 @@
           - "1"
           - -W
           - "1"
-          - >-
+          - "{{ primary_cluster_ip }}"
-            {{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip
-            | split(',') | first }}
       register: primary_cluster_ping
       until: primary_cluster_ping.rc == 0
       retries: 6
       delay: 2
       changed_when: false
 
+    - name: Read the primary neighbor entry
+      ansible.builtin.command:
+        argv:
+          - ip
+          - neigh
+          - show
+          - to
+          - "{{ primary_cluster_ip }}"
+          - dev
+          - "{{ cluster_interface }}"
+      register: primary_cluster_neighbor
+      changed_when: false
+      when: inventory_hostname != primary_master
+
+    - name: Verify the primary neighbor identity
+      ansible.builtin.assert:
+        that:
+          - (primary_cluster_mac | lower) in (primary_cluster_neighbor.stdout | lower)
+        fail_msg: >-
+          {{ inventory_hostname }} resolved primary {{ primary_cluster_ip }} to
+          an unexpected MAC on {{ cluster_interface }}. Expected
+          {{ primary_cluster_mac }}, got: {{ primary_cluster_neighbor.stdout }}
+      when: inventory_hostname != primary_master
+
     - name: Verify GitHub release host DNS
       ansible.builtin.getent:
         database: hosts

molecule/kube-vip/overrides.yml

@@ -7,6 +7,7 @@
         # See:
         # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
         flannel_iface: eth1
+        kube_vip_iface: eth1
 
         # The test VMs might be a bit slow, so we give them more time to join the cluster:
         retry_count: 45

molecule/single_node/overrides.yml

@@ -7,6 +7,7 @@
         # See:
         # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
         flannel_iface: eth1
+        kube_vip_iface: eth1
 
         # The test VMs might be a bit slow, so we give them more time to join the cluster:
         retry_count: 45

roles/k3s_server/tasks/fetch_k3s_init_logs.yml

@@ -30,6 +30,14 @@
   no_log: true
   register: k3s_ip_address_state
 
+- name: Fetch IP link state
+  ansible.builtin.command:
+    cmd: ip -br link
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_ip_link_state
+
 - name: Fetch IP route state
   ansible.builtin.command:
     cmd: ip route show
@@ -62,6 +70,19 @@
   no_log: true
   register: k3s_tcp_listener_state
 
+- name: Fetch guest firewall service state
+  ansible.builtin.service_facts:
+  no_log: true
+  register: k3s_service_facts
+
+- name: Fetch input firewall rules
+  ansible.builtin.command:
+    cmd: iptables -S INPUT
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_input_firewall_rules
+
 - name: Ping the primary Kubernetes API address from {{ ansible_hostname }}
   ansible.builtin.command:
     argv:
@@ -123,6 +144,9 @@
       === ip -br address ===
       {{ k3s_ip_address_state.stdout }}
 
+      === ip -br link ===
+      {{ k3s_ip_link_state.stdout }}
+
       === ip route show ===
       {{ k3s_ip_route_state.stdout }}
 
@@ -135,6 +159,16 @@
       === ss -ltn ===
       {{ k3s_tcp_listener_state.stdout }}
 
+      === guest firewall services ===
+      firewalld={{ k3s_service_facts.ansible_facts.services.get('firewalld.service', {}).get('state', 'not-found') }}
+      nftables={{ k3s_service_facts.ansible_facts.services.get('nftables.service', {}).get('state', 'not-found') }}
+      ufw={{ k3s_service_facts.ansible_facts.services.get('ufw.service', {}).get('state', 'not-found') }}
+
+      === iptables -S INPUT ===
+      rc={{ k3s_input_firewall_rules.rc }}
+      {{ k3s_input_firewall_rules.stdout }}
+      {{ k3s_input_firewall_rules.stderr }}
+
       === primary API ping ===
       rc={{ k3s_primary_api_ping.rc }}
       {{ k3s_primary_api_ping.stdout }}

roles/k3s_server/tasks/join_master.yml

@@ -1,8 +1,21 @@
 ---
+- name: Verify primary Kubernetes API reachability from {{ joining_master }}
+  ansible.builtin.wait_for:
+    host: >-
+      {{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip
+      | split(',') | first }}
+    port: 6443
+    connect_timeout: 2
+    timeout: 30
+  delegate_to: "{{ joining_master }}"
+
 - name: Join transient k3s-init service for {{ joining_master }}
   ansible.builtin.command:
     cmd: >-
-      systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init
+      systemd-run -p RestartSec=2 -p Restart=on-failure
+      -p Delegate=yes -p TasksMax=infinity -p KillMode=process
+      -p LimitNOFILE=1048576 -p LimitNPROC=infinity -p LimitCORE=infinity
+      --unit=k3s-init
       k3s server {{ hostvars[joining_master].k3s_server_init_args }}
     creates: "{{ systemd_dir }}/k3s-init.service"
   delegate_to: "{{ joining_master }}"

roles/k3s_server/tasks/main.yml

@@ -35,6 +35,7 @@
   when: kube_vip_lb_ip_range is defined
 
 - name: Initialize and verify the K3s control plane
+  any_errors_fatal: true
   when: not ansible_check_mode
   block:
     - name: Materialize per-host server initialization arguments
@@ -43,25 +44,27 @@
       no_log: true
       when: groups[group_name_master | default('master')] | length > 1
 
+    - name: Orchestrate control-plane initialization from the first master
+      when: inventory_hostname == groups[group_name_master | default('master')][0]
+      block:
         - name: Init the first master inside the transient k3s-init service
           ansible.builtin.command:
             cmd: >-
               systemd-run -p RestartSec=2 -p Restart=on-failure
+              -p Delegate=yes -p TasksMax=infinity -p KillMode=process
+              -p LimitNOFILE=1048576 -p LimitNPROC=infinity -p LimitCORE=infinity
               --unit=k3s-init k3s server {{ server_init_args }}
             creates: "{{ systemd_dir }}/k3s-init.service"
           no_log: true
-      when: inventory_hostname == groups[group_name_master | default('master')][0]
 
         - name: Wait for the first master Kubernetes API
           ansible.builtin.command:
             cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get --raw=/readyz"
-      delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
           register: first_master_api
           until: first_master_api.rc == 0
           retries: "{{ retry_count | default(20) }}"
           delay: 2
           changed_when: false
-      run_once: true
 
         - name: Verify the first master API listener on its node address
           ansible.builtin.wait_for:
@@ -71,27 +74,47 @@
             port: 6443
             connect_timeout: 2
             timeout: "{{ retry_count | default(20) * 2 }}"
-      delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
-      run_once: true
 
         - name: Join additional masters one at a time
           ansible.builtin.include_tasks: join_master.yml
           loop: "{{ groups[group_name_master | default('master')][1:] }}"
           loop_control:
             loop_var: joining_master
-      run_once: true
           when: groups[group_name_master | default('master')] | length > 1
 
-    - name: Verify that all nodes actually joined (check k3s-init.service if this fails)
+        - name: Verify that all expected control-plane nodes registered
           ansible.builtin.command:
-        cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get nodes -l 'node-role.kubernetes.io/master=true' -o=jsonpath='{.items[*].metadata.name}'" # yamllint disable-line rule:line-length
+            cmd: >-
-      delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
+              {{ k3s_kubectl_binary | default('k3s kubectl') }} get nodes
+              -o=jsonpath='{.items[*].metadata.name}'
           register: nodes
-      until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups[group_name_master | default('master')] | length) # yamllint disable-line rule:line-length
+          until:
+            - nodes.rc == 0
+            - expected_control_plane_names | difference(nodes.stdout.split()) | length == 0
           retries: "{{ retry_count | default(20) }}"
-      delay: 10
+          delay: 2
+          changed_when: false
+          vars:
+            expected_control_plane_names: >-
+              {{
+                groups[group_name_master | default('master')]
+                | map('extract', hostvars, 'ansible_hostname')
+                | list
+              }}
+
+        - name: Wait for K3s bootstrap CRDs to become established
+          ansible.builtin.command:
+            cmd: >-
+              {{ k3s_kubectl_binary | default('k3s kubectl') }} wait
+              --for=condition=Established --timeout=5s
+              crd/addons.k3s.cattle.io
+              crd/helmcharts.helm.cattle.io
+              crd/helmchartconfigs.helm.cattle.io
+          register: bootstrap_crds
+          until: bootstrap_crds.rc == 0
+          retries: "{{ retry_count | default(20) }}"
+          delay: 2
           changed_when: false
-      run_once: true
   always:
     - name: Save logs of k3s-init.service
       ansible.builtin.include_tasks: fetch_k3s_init_logs.yml