fix(ci): harden Molecule download and cleanup

- Retry transient k3s release downloads with bounded backoff.\n- Bound failure diagnostics and validate guest release connectivity.\n- Discover repository-owned Molecule state under the actual project root.
This commit is contained in:
Timothy Stewart
2026-07-31 10:51:44 -05:00
committed by Techno Tim
parent 5bc347aed4
commit 87ea8160c1
6 changed files with 69 additions and 10 deletions
+19 -3
View File
@@ -22,6 +22,7 @@ esac
home_dir="${HOME:?HOME must be set}" home_dir="${HOME:?HOME must be set}"
molecule_root="${K3S_CI_MOLECULE_ROOT:-${home_dir}/.cache/molecule}" molecule_root="${K3S_CI_MOLECULE_ROOT:-${home_dir}/.cache/molecule}"
repository_name="${K3S_CI_MOLECULE_PROJECT:-k3s-ansible}"
virtualbox_root="${K3S_CI_VIRTUALBOX_ROOT:-${home_dir}/VirtualBox VMs}" virtualbox_root="${K3S_CI_VIRTUALBOX_ROOT:-${home_dir}/VirtualBox VMs}"
hostonly_marker="${K3S_CI_HOSTONLY_MARKER:-${home_dir}/.cache/k3s-ci/hostonly-interfaces}" hostonly_marker="${K3S_CI_HOSTONLY_MARKER:-${home_dir}/.cache/k3s-ci/hostonly-interfaces}"
@@ -106,6 +107,10 @@ fail_closed() {
exit 3 exit 3
} }
if [[ ! "$repository_name" =~ ^[A-Za-z0-9._-]+$ ]]; then
fail_closed 'invalid Molecule repository name'
fi
print_inventory() { print_inventory() {
local phase="$1" local phase="$1"
printf '%s VirtualBox inventory:\n' "$phase" printf '%s VirtualBox inventory:\n' "$phase"
@@ -123,14 +128,25 @@ fi
print_inventory before print_inventory before
repository_root_real="$(resolve_existing_dir "$molecule_root_real/$repository_name" || true)"
if [[ -z "$repository_root_real" ]]; then
printf 'No repository Molecule state root exists: %s\n' "$molecule_root_real/$repository_name"
cleanup_hostonly
print_inventory after
exit 0
fi
if ! root_contains "$molecule_root_real" "$repository_root_real"; then
fail_closed "repository Molecule state root is outside Molecule root: $repository_root_real"
fi
declare -a state_files=() declare -a state_files=()
while IFS= read -r -d '' state_file; do while IFS= read -r -d '' state_file; do
state_files+=("$state_file") state_files+=("$state_file")
done < <(find "$molecule_root_real/repo" -mindepth 6 -maxdepth 6 -type f \ done < <(find "$repository_root_real" -mindepth 6 -maxdepth 6 -type f \
-path '*/.vagrant/machines/*/virtualbox/id' -print0 2>/dev/null) -path '*/.vagrant/machines/*/virtualbox/id' -print0 2>/dev/null)
if ((${#state_files[@]} == 0)); then if ((${#state_files[@]} == 0)); then
printf 'No repository-owned Molecule Vagrant state found under %s\n' "$molecule_root_real" printf 'No repository-owned Molecule Vagrant state found under %s\n' "$repository_root_real"
cleanup_hostonly cleanup_hostonly
print_inventory after print_inventory after
exit 0 exit 0
@@ -146,7 +162,7 @@ for state_file in "${state_files[@]}"; do
machine_name="${machine_dir##*/}" machine_name="${machine_dir##*/}"
scenario_name="${state_dir##*/}" scenario_name="${state_dir##*/}"
if ! root_contains "$molecule_root_real/repo" "$state_dir"; then if ! root_contains "$repository_root_real" "$state_dir"; then
fail_closed "state path is outside the repository Molecule root: $state_file_real" fail_closed "state path is outside the repository Molecule root: $state_file_real"
fi fi
if ! is_supported_scenario "$scenario_name"; then if ! is_supported_scenario "$scenario_name"; then
@@ -11,10 +11,10 @@ trap 'rm -rf -- "$test_root"' EXIT
molecule_root="$test_root/molecule" molecule_root="$test_root/molecule"
virtualbox_root="$test_root/VirtualBox VMs" virtualbox_root="$test_root/VirtualBox VMs"
fake_bin="$test_root/bin" fake_bin="$test_root/bin"
mkdir -p -- "$molecule_root/repo/single_node/.vagrant/machines/control1/virtualbox" \ mkdir -p -- "$molecule_root/k3s-ansible/single_node/.vagrant/machines/control1/virtualbox" \
"$virtualbox_root/control1" "$virtualbox_root/unmarked" "$fake_bin" "$virtualbox_root/control1" "$virtualbox_root/unmarked" "$fake_bin"
printf '%s\n' '11111111-1111-1111-1111-111111111111' \ printf '%s\n' '11111111-1111-1111-1111-111111111111' \
> "$molecule_root/repo/single_node/.vagrant/machines/control1/virtualbox/id" > "$molecule_root/k3s-ansible/single_node/.vagrant/machines/control1/virtualbox/id"
touch "$virtualbox_root/control1/control1.vbox" "$virtualbox_root/control1/disk.vdi" \ touch "$virtualbox_root/control1/control1.vbox" "$virtualbox_root/control1/disk.vdi" \
"$virtualbox_root/unmarked/unmarked.vbox" "$virtualbox_root/unmarked/unmarked.vbox"
@@ -52,6 +52,7 @@ output="$test_root/output.txt"
if PATH="$fake_bin:$PATH" \ if PATH="$fake_bin:$PATH" \
HOME="$test_root/home" \ HOME="$test_root/home" \
K3S_CI_MOLECULE_ROOT="$molecule_root" \ K3S_CI_MOLECULE_ROOT="$molecule_root" \
K3S_CI_MOLECULE_PROJECT=k3s-ansible \
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \ K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \ K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
FAKE_VM_MODE=fault \ FAKE_VM_MODE=fault \
@@ -62,12 +63,13 @@ if PATH="$fake_bin:$PATH" \
exit 1 exit 1
fi fi
grep -Fq 'cleanup refused: unable to inspect VirtualBox VM' "$output" grep -Fq 'cleanup refused: unable to inspect VirtualBox VM' "$output"
[[ -f "$molecule_root/repo/single_node/.vagrant/machines/control1/virtualbox/id" ]] [[ -f "$molecule_root/k3s-ansible/single_node/.vagrant/machines/control1/virtualbox/id" ]]
printf '%s\n' 'vboxnet0|192.168.30.1' > "$test_root/hostonly-baseline" printf '%s\n' 'vboxnet0|192.168.30.1' > "$test_root/hostonly-baseline"
if PATH="$fake_bin:$PATH" \ if PATH="$fake_bin:$PATH" \
HOME="$test_root/home" \ HOME="$test_root/home" \
K3S_CI_MOLECULE_ROOT="$molecule_root" \ K3S_CI_MOLECULE_ROOT="$molecule_root" \
K3S_CI_MOLECULE_PROJECT=k3s-ansible \
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \ K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \ K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
FAKE_HOSTONLY_FAIL=true \ FAKE_HOSTONLY_FAIL=true \
@@ -82,6 +84,7 @@ grep -Fq 'cleanup refused: unable to inventory VirtualBox host-only interfaces'
PATH="$fake_bin:$PATH" \ PATH="$fake_bin:$PATH" \
HOME="$test_root/home" \ HOME="$test_root/home" \
K3S_CI_MOLECULE_ROOT="$molecule_root" \ K3S_CI_MOLECULE_ROOT="$molecule_root" \
K3S_CI_MOLECULE_PROJECT=k3s-ansible \
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \ K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \ K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
FAKE_VBOX_ROOT="$virtualbox_root" \ FAKE_VBOX_ROOT="$virtualbox_root" \
@@ -94,6 +97,7 @@ grep -Fq 'Would remove VM control1 (11111111-1111-1111-1111-111111111111)' "$out
PATH="$fake_bin:$PATH" \ PATH="$fake_bin:$PATH" \
HOME="$test_root/home" \ HOME="$test_root/home" \
K3S_CI_MOLECULE_ROOT="$molecule_root" \ K3S_CI_MOLECULE_ROOT="$molecule_root" \
K3S_CI_MOLECULE_PROJECT=k3s-ansible \
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \ K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \ K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
FAKE_VBOX_ROOT="$virtualbox_root" \ FAKE_VBOX_ROOT="$virtualbox_root" \
@@ -108,6 +112,7 @@ grep -Fq 'unregistervm unregistervm 11111111-1111-1111-1111-111111111111 --delet
PATH="$fake_bin:$PATH" \ PATH="$fake_bin:$PATH" \
HOME="$test_root/home" \ HOME="$test_root/home" \
K3S_CI_MOLECULE_ROOT="$molecule_root" \ K3S_CI_MOLECULE_ROOT="$molecule_root" \
K3S_CI_MOLECULE_PROJECT=k3s-ansible \
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \ K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \ K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
FAKE_VM_MODE=missing \ FAKE_VM_MODE=missing \
@@ -116,6 +121,6 @@ PATH="$fake_bin:$PATH" \
bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --apply > "$output" bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --apply > "$output"
grep -Fq 'Stale Vagrant state without a registered VM' "$output" grep -Fq 'Stale Vagrant state without a registered VM' "$output"
[[ ! -d "$molecule_root/repo/single_node/.vagrant" ]] [[ ! -d "$molecule_root/k3s-ansible/single_node/.vagrant" ]]
printf 'cleanup-runner-resources fixture test passed\n' printf 'cleanup-runner-resources fixture test passed\n'
+2 -2
View File
@@ -9,12 +9,12 @@ jobs:
strategy: strategy:
matrix: matrix:
scenario: scenario:
- default
# - ipv6
- single_node - single_node
- calico - calico
- cilium - cilium
- kube-vip - kube-vip
- default
# - ipv6
fail-fast: false fail-fast: false
max-parallel: 1 max-parallel: 1
env: env:
+23
View File
@@ -20,3 +20,26 @@
enabled: false enabled: false
state: stopped state: stopped
become: true become: true
- name: Verify GitHub release host DNS
ansible.builtin.getent:
database: hosts
key: github.com
register: github_dns
retries: 6
delay: 5
until: github_dns is succeeded
- name: Verify k3s checksum URL is reachable
ansible.builtin.uri:
url: >-
https://github.com/k3s-io/k3s/releases/download/{{ k3s_version
}}/sha256sum-amd64.txt
method: HEAD
follow_redirects: safe
status_code: [200, 302]
timeout: 15
register: k3s_checksum_request
retries: 3
delay: 5
until: k3s_checksum_request.status in [200, 302]
+12
View File
@@ -7,6 +7,10 @@
owner: root owner: root
group: root group: root
mode: "0755" mode: "0755"
register: k3s_download_x64
retries: 5
delay: 10
until: k3s_download_x64 is succeeded
when: ansible_facts.architecture == "x86_64" when: ansible_facts.architecture == "x86_64"
- name: Download k3s binary arm64 - name: Download k3s binary arm64
@@ -17,6 +21,10 @@
owner: root owner: root
group: root group: root
mode: "0755" mode: "0755"
register: k3s_download_arm64
retries: 5
delay: 10
until: k3s_download_arm64 is succeeded
when: when:
- ( ansible_facts.architecture is search("arm") and ansible_facts.userspace_bits == "64" ) - ( ansible_facts.architecture is search("arm") and ansible_facts.userspace_bits == "64" )
or ansible_facts.architecture is search("aarch64") or ansible_facts.architecture is search("aarch64")
@@ -29,6 +37,10 @@
owner: root owner: root
group: root group: root
mode: "0755" mode: "0755"
register: k3s_download_armhf
retries: 5
delay: 10
until: k3s_download_armhf is succeeded
when: when:
- ansible_facts.architecture is search("arm") - ansible_facts.architecture is search("arm")
- ansible_facts.userspace_bits == "32" - ansible_facts.userspace_bits == "32"
@@ -4,8 +4,11 @@
- name: Fetch k3s-init.service logs - name: Fetch k3s-init.service logs
ansible.builtin.command: ansible.builtin.command:
cmd: journalctl --all --unit=k3s-init.service cmd: >-
timeout --signal=TERM 30s journalctl --no-pager
--unit=k3s-init.service --since=-30min --lines=5000
changed_when: false changed_when: false
failed_when: false
register: k3s_init_log register: k3s_init_log
- name: Create {{ log_destination }} - name: Create {{ log_destination }}