forked from tim/k3s-ansible
ci(runner): restore isolated Molecule CI
- Scope VirtualBox cleanup to repository-owned Molecule resources\n- Route lint to hosted runners and harden self-hosted workflow execution\n- Add diagnostics, fixture coverage, and reproducible lint validation
This commit is contained in:
Timothy Stewart
committed by
Techno Tim
Techno Tim
parent
2fad0a8db6
commit
29b7aa1b72
Executable
+229
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
usage() {
|
||||
printf '%s\n' \
|
||||
'Usage: cleanup-runner-resources.sh [--snapshot|--dry-run|--apply]' \
|
||||
'' \
|
||||
'Discover and, with --apply, remove only VirtualBox resources referenced by' \
|
||||
'repository-owned Molecule Vagrant state. The default is --dry-run.'
|
||||
}
|
||||
|
||||
mode="dry-run"
|
||||
case "${1:-}" in
|
||||
"") ;;
|
||||
--snapshot) mode="snapshot" ;;
|
||||
--dry-run) mode="dry-run" ;;
|
||||
--apply) mode="apply" ;;
|
||||
--help|-h) usage; exit 0 ;;
|
||||
*) usage >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
home_dir="${HOME:?HOME must be set}"
|
||||
molecule_root="${K3S_CI_MOLECULE_ROOT:-${home_dir}/.cache/molecule}"
|
||||
virtualbox_root="${K3S_CI_VIRTUALBOX_ROOT:-${home_dir}/VirtualBox VMs}"
|
||||
hostonly_marker="${K3S_CI_HOSTONLY_MARKER:-${home_dir}/.cache/k3s-ci/hostonly-interfaces}"
|
||||
|
||||
record_hostonly() {
|
||||
local marker_dir="${hostonly_marker%/*}"
|
||||
local marker_tmp="${hostonly_marker}.tmp"
|
||||
local hostonly_inventory
|
||||
if ! hostonly_inventory="$(VBoxManage list hostonlyifs)"; then
|
||||
fail_closed 'unable to inventory VirtualBox host-only interfaces'
|
||||
fi
|
||||
mkdir -p -- "$marker_dir"
|
||||
awk -F': ' '
|
||||
/^Name:/ { name=$2 }
|
||||
/^IPAddress:/ { print name "|" $2 }
|
||||
' <<< "$hostonly_inventory" > "$marker_tmp"
|
||||
mv -- "$marker_tmp" "$hostonly_marker"
|
||||
chmod 600 "$hostonly_marker"
|
||||
printf 'Recorded host-only interface baseline: %s\n' "$hostonly_marker"
|
||||
}
|
||||
|
||||
cleanup_hostonly() {
|
||||
local hostonly_inventory
|
||||
if [[ ! -f "$hostonly_marker" ]]; then
|
||||
printf 'No host-only interface baseline found; leaving interfaces unchanged.\n'
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! hostonly_inventory="$(VBoxManage list hostonlyifs)"; then
|
||||
fail_closed 'unable to inventory VirtualBox host-only interfaces'
|
||||
fi
|
||||
|
||||
while IFS='|' read -r interface_name interface_ip; do
|
||||
[[ "$interface_name" == vboxnet* ]] || continue
|
||||
[[ "$interface_ip" == 192.168.30.* || "$interface_ip" == fdad:bad:ba55:* ]] || continue
|
||||
if grep -Fqx "${interface_name}|${interface_ip}" "$hostonly_marker"; then
|
||||
continue
|
||||
fi
|
||||
if [[ "$mode" == apply ]]; then
|
||||
VBoxManage hostonlyif remove "$interface_name"
|
||||
printf 'Removed host-only interface %s (%s)\n' "$interface_name" "$interface_ip"
|
||||
else
|
||||
printf 'Would remove host-only interface %s (%s)\n' "$interface_name" "$interface_ip"
|
||||
fi
|
||||
done < <(awk -F': ' '
|
||||
/^Name:/ { name=$2 }
|
||||
/^IPAddress:/ { print name "|" $2 }
|
||||
' <<< "$hostonly_inventory")
|
||||
}
|
||||
|
||||
if [[ "$mode" == snapshot ]]; then
|
||||
record_hostonly
|
||||
exit 0
|
||||
fi
|
||||
|
||||
resolve_existing_dir() {
|
||||
local candidate="$1"
|
||||
if [[ ! -d "$candidate" ]]; then
|
||||
return 1
|
||||
fi
|
||||
readlink -f -- "$candidate"
|
||||
}
|
||||
|
||||
root_contains() {
|
||||
local root="$1"
|
||||
local path="$2"
|
||||
[[ "$path" == "$root"/* ]]
|
||||
}
|
||||
|
||||
is_supported_scenario() {
|
||||
case "$1" in
|
||||
default|single_node|calico|cilium|kube-vip|ipv6) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
is_unregistered_vm_error() {
|
||||
grep -Eq 'Could not find a registered machine|VBOX_E_OBJECT_NOT_FOUND'
|
||||
}
|
||||
|
||||
fail_closed() {
|
||||
printf 'cleanup refused: %s\n' "$1" >&2
|
||||
exit 3
|
||||
}
|
||||
|
||||
print_inventory() {
|
||||
local phase="$1"
|
||||
printf '%s VirtualBox inventory:\n' "$phase"
|
||||
VBoxManage list vms || true
|
||||
VBoxManage list hdds || true
|
||||
VBoxManage list hostonlyifs || true
|
||||
}
|
||||
|
||||
molecule_root_real="$(resolve_existing_dir "$molecule_root" || true)"
|
||||
if [[ -z "$molecule_root_real" ]]; then
|
||||
printf 'No Molecule root exists: %s\n' "$molecule_root"
|
||||
cleanup_hostonly
|
||||
exit 0
|
||||
fi
|
||||
|
||||
print_inventory before
|
||||
|
||||
declare -a state_files=()
|
||||
while IFS= read -r -d '' state_file; do
|
||||
state_files+=("$state_file")
|
||||
done < <(find "$molecule_root_real/repo" -mindepth 6 -maxdepth 6 -type f \
|
||||
-path '*/.vagrant/machines/*/virtualbox/id' -print0 2>/dev/null)
|
||||
|
||||
if ((${#state_files[@]} == 0)); then
|
||||
printf 'No repository-owned Molecule Vagrant state found under %s\n' "$molecule_root_real"
|
||||
cleanup_hostonly
|
||||
print_inventory after
|
||||
exit 0
|
||||
fi
|
||||
|
||||
virtualbox_root_real="$(resolve_existing_dir "$virtualbox_root" || true)"
|
||||
|
||||
declare -a vm_records=()
|
||||
for state_file in "${state_files[@]}"; do
|
||||
state_file_real="$(readlink -f -- "$state_file")"
|
||||
state_dir="${state_file_real%/.vagrant/machines/*/virtualbox/id}"
|
||||
machine_dir="${state_file_real%/virtualbox/id}"
|
||||
machine_name="${machine_dir##*/}"
|
||||
scenario_name="${state_dir##*/}"
|
||||
|
||||
if ! root_contains "$molecule_root_real/repo" "$state_dir"; then
|
||||
fail_closed "state path is outside the repository Molecule root: $state_file_real"
|
||||
fi
|
||||
if ! is_supported_scenario "$scenario_name"; then
|
||||
fail_closed "unexpected Molecule scenario: $scenario_name"
|
||||
fi
|
||||
if [[ "$machine_name" != control* && "$machine_name" != node* ]]; then
|
||||
fail_closed "unexpected Molecule machine name: $machine_name"
|
||||
fi
|
||||
|
||||
vm_uuid="$(tr -d '[:space:]' < "$state_file_real")"
|
||||
if [[ ! "$vm_uuid" =~ ^[0-9a-fA-F-]{36}$ ]]; then
|
||||
fail_closed "invalid VirtualBox UUID in $state_file_real"
|
||||
fi
|
||||
|
||||
if ! vm_info="$(VBoxManage showvminfo "$vm_uuid" --machinereadable 2>&1)"; then
|
||||
if is_unregistered_vm_error <<< "$vm_info"; then
|
||||
printf 'Stale Vagrant state without a registered VM: %s (%s)\n' "$machine_name" "$vm_uuid"
|
||||
if [[ "$mode" == apply ]]; then
|
||||
rm -rf -- "${state_dir}/.vagrant"
|
||||
printf 'Removed stale Vagrant state: %s\n' "${state_dir}/.vagrant"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
fail_closed "unable to inspect VirtualBox VM $vm_uuid: $vm_info"
|
||||
fi
|
||||
|
||||
if [[ -z "$virtualbox_root_real" ]]; then
|
||||
fail_closed "VirtualBox VM root does not exist: $virtualbox_root"
|
||||
fi
|
||||
|
||||
cfg_file="$(awk -F= '$1 == "CfgFile" {gsub(/"/, "", $2); print $2; exit}' <<< "$vm_info")"
|
||||
if [[ -z "$cfg_file" ]]; then
|
||||
fail_closed "VirtualBox configuration path missing for $vm_uuid"
|
||||
fi
|
||||
cfg_file_real="$(readlink -f -- "$cfg_file")"
|
||||
if ! root_contains "$virtualbox_root_real" "$cfg_file_real"; then
|
||||
fail_closed "VM configuration is outside VirtualBox root: $cfg_file_real"
|
||||
fi
|
||||
|
||||
while IFS= read -r disk_path; do
|
||||
[[ -z "$disk_path" ]] && continue
|
||||
disk_path_real="$(readlink -f -- "$disk_path" 2>/dev/null || true)"
|
||||
if [[ -z "$disk_path_real" ]] || ! root_contains "$virtualbox_root_real" "$disk_path_real"; then
|
||||
fail_closed "attached disk is outside VirtualBox root: $disk_path"
|
||||
fi
|
||||
done < <(awk -F= '$1 ~ /^(SATA|IDE|SCSI|SAS|VirtioSCSI|NVMe)-[0-9]+-[0-9]+$/ {gsub(/"/, "", $2); print $2}' <<< "$vm_info")
|
||||
|
||||
vm_records+=("$vm_uuid|$machine_name|$cfg_file_real")
|
||||
done
|
||||
|
||||
if ((${#vm_records[@]} == 0)); then
|
||||
printf 'No live repository-owned VirtualBox resources found\n'
|
||||
cleanup_hostonly
|
||||
print_inventory after
|
||||
exit 0
|
||||
fi
|
||||
|
||||
for record in "${vm_records[@]}"; do
|
||||
IFS='|' read -r vm_uuid machine_name cfg_file_real <<< "$record"
|
||||
if [[ "$mode" == dry-run ]]; then
|
||||
printf 'Would remove VM %s (%s) config=%s\n' "$machine_name" "$vm_uuid" "$cfg_file_real"
|
||||
continue
|
||||
fi
|
||||
|
||||
vm_state="$(VBoxManage showvminfo "$vm_uuid" --machinereadable | awk -F= '$1 == "VMState" {gsub(/"/, "", $2); print $2; exit}')"
|
||||
if [[ "$vm_state" != poweroff && "$vm_state" != saved ]]; then
|
||||
VBoxManage controlvm "$vm_uuid" poweroff
|
||||
fi
|
||||
VBoxManage unregistervm "$vm_uuid" --delete
|
||||
printf 'Removed VM %s (%s)\n' "$machine_name" "$vm_uuid"
|
||||
done
|
||||
|
||||
cleanup_hostonly
|
||||
print_inventory after
|
||||
|
||||
if [[ "$mode" == apply ]]; then
|
||||
printf 'Repository-owned VM and host-only interface cleanup complete.\n'
|
||||
else
|
||||
printf 'Dry run complete. No resources were modified.\n'
|
||||
fi
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
output_dir="${1:-${RUNNER_TEMP:-/tmp}/k3s-ci-diagnostics}"
|
||||
mkdir -p -- "$output_dir"
|
||||
umask 077
|
||||
|
||||
run_capture() {
|
||||
local output_file="$1"
|
||||
shift
|
||||
{
|
||||
printf '$'
|
||||
printf ' %q' "$@"
|
||||
printf '\n'
|
||||
"$@"
|
||||
} > "$output_dir/$output_file" 2>&1 || true
|
||||
}
|
||||
|
||||
run_capture system.txt uname -a
|
||||
run_capture runner-user.txt id
|
||||
run_capture memory.txt free -h
|
||||
run_capture disk.txt df -h
|
||||
run_capture virtualbox-version VBoxManage --version
|
||||
run_capture virtualbox-vms VBoxManage list vms
|
||||
run_capture virtualbox-running-vms VBoxManage list runningvms
|
||||
run_capture virtualbox-disks VBoxManage list hdds
|
||||
run_capture virtualbox-hostonlyifs VBoxManage list hostonlyifs
|
||||
run_capture vagrant-status vagrant global-status
|
||||
run_capture molecule-state find "${HOME}/.cache/molecule" -maxdepth 6 -type f -path '*/.vagrant/machines/*/virtualbox/id' -print
|
||||
|
||||
if [[ -r /etc/vbox/networks.conf ]]; then
|
||||
cp -- /etc/vbox/networks.conf "$output_dir/virtualbox-networks.conf"
|
||||
fi
|
||||
|
||||
printf 'Diagnostics written to %s\n' "$output_dir"
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
test_root="$(mktemp -d "${TMPDIR:-/tmp}/k3s-ci-cleanup-test.XXXXXX")"
|
||||
trap 'rm -rf -- "$test_root"' EXIT
|
||||
|
||||
molecule_root="$test_root/molecule"
|
||||
virtualbox_root="$test_root/VirtualBox VMs"
|
||||
fake_bin="$test_root/bin"
|
||||
mkdir -p -- "$molecule_root/repo/single_node/.vagrant/machines/control1/virtualbox" \
|
||||
"$virtualbox_root/control1" "$virtualbox_root/unmarked" "$fake_bin"
|
||||
printf '%s\n' '11111111-1111-1111-1111-111111111111' \
|
||||
> "$molecule_root/repo/single_node/.vagrant/machines/control1/virtualbox/id"
|
||||
touch "$virtualbox_root/control1/control1.vbox" "$virtualbox_root/control1/disk.vdi" \
|
||||
"$virtualbox_root/unmarked/unmarked.vbox"
|
||||
|
||||
printf '%s\n' \
|
||||
'#!/usr/bin/env bash' \
|
||||
'set -Eeuo pipefail' \
|
||||
'case "${1:-}" in' \
|
||||
' list)' \
|
||||
' if [[ "${2:-}" == hostonlyifs && "${FAKE_HOSTONLY_FAIL:-false}" == true ]]; then exit 1; fi' \
|
||||
' exit 0' \
|
||||
' ;;' \
|
||||
' showvminfo)' \
|
||||
' if [[ "${FAKE_VM_MODE:-normal}" == missing ]]; then' \
|
||||
' printf '\''VBoxManage: error: Could not find a registered machine named "missing"\n'\'' >&2' \
|
||||
' exit 1' \
|
||||
' fi' \
|
||||
' if [[ "${FAKE_VM_MODE:-normal}" == fault ]]; then' \
|
||||
' printf '\''VBoxManage: error: VirtualBox service is unavailable\n'\'' >&2' \
|
||||
' exit 1' \
|
||||
' fi' \
|
||||
' printf '\''CfgFile="%s"\n'\'' "${FAKE_VBOX_ROOT}/control1/control1.vbox"' \
|
||||
' printf '\''SATA-0-0="%s"\n'\'' "${FAKE_VBOX_ROOT}/control1/disk.vdi"' \
|
||||
' printf '\''VMState="running"\n'\''' \
|
||||
' ;;' \
|
||||
' controlvm) printf '\''controlvm %s\n'\'' "$*" >> "${FAKE_LOG}" ;;' \
|
||||
' unregistervm)' \
|
||||
' printf '\''unregistervm %s\n'\'' "$*" >> "${FAKE_LOG}"' \
|
||||
' rm -f -- "${FAKE_VBOX_ROOT}/control1/control1.vbox" "${FAKE_VBOX_ROOT}/control1/disk.vdi"' \
|
||||
' ;;' \
|
||||
' *) : ;;' \
|
||||
'esac' > "$fake_bin/VBoxManage"
|
||||
chmod 700 "$fake_bin/VBoxManage"
|
||||
|
||||
output="$test_root/output.txt"
|
||||
if PATH="$fake_bin:$PATH" \
|
||||
HOME="$test_root/home" \
|
||||
K3S_CI_MOLECULE_ROOT="$molecule_root" \
|
||||
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
|
||||
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
|
||||
FAKE_VM_MODE=fault \
|
||||
FAKE_VBOX_ROOT="$virtualbox_root" \
|
||||
FAKE_LOG="$test_root/vbox.log" \
|
||||
bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --apply > "$output" 2>&1; then
|
||||
printf '%s\n' 'cleanup unexpectedly accepted a VirtualBox inspection failure' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq 'cleanup refused: unable to inspect VirtualBox VM' "$output"
|
||||
[[ -f "$molecule_root/repo/single_node/.vagrant/machines/control1/virtualbox/id" ]]
|
||||
|
||||
printf '%s\n' 'vboxnet0|192.168.30.1' > "$test_root/hostonly-baseline"
|
||||
if PATH="$fake_bin:$PATH" \
|
||||
HOME="$test_root/home" \
|
||||
K3S_CI_MOLECULE_ROOT="$molecule_root" \
|
||||
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
|
||||
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
|
||||
FAKE_HOSTONLY_FAIL=true \
|
||||
FAKE_VBOX_ROOT="$virtualbox_root" \
|
||||
FAKE_LOG="$test_root/vbox.log" \
|
||||
bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --dry-run > "$output" 2>&1; then
|
||||
printf '%s\n' 'cleanup unexpectedly accepted a host-only inventory failure' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq 'cleanup refused: unable to inventory VirtualBox host-only interfaces' "$output"
|
||||
|
||||
PATH="$fake_bin:$PATH" \
|
||||
HOME="$test_root/home" \
|
||||
K3S_CI_MOLECULE_ROOT="$molecule_root" \
|
||||
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
|
||||
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
|
||||
FAKE_VBOX_ROOT="$virtualbox_root" \
|
||||
FAKE_LOG="$test_root/vbox.log" \
|
||||
bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --dry-run > "$output"
|
||||
|
||||
grep -Fq 'Would remove VM control1 (11111111-1111-1111-1111-111111111111)' "$output"
|
||||
[[ ! -e "$test_root/vbox.log" ]]
|
||||
|
||||
PATH="$fake_bin:$PATH" \
|
||||
HOME="$test_root/home" \
|
||||
K3S_CI_MOLECULE_ROOT="$molecule_root" \
|
||||
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
|
||||
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
|
||||
FAKE_VBOX_ROOT="$virtualbox_root" \
|
||||
FAKE_LOG="$test_root/vbox.log" \
|
||||
bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --apply > "$output"
|
||||
|
||||
grep -Fq 'controlvm 11111111-1111-1111-1111-111111111111 poweroff' "$test_root/vbox.log"
|
||||
grep -Fq 'unregistervm unregistervm 11111111-1111-1111-1111-111111111111 --delete' "$test_root/vbox.log"
|
||||
[[ ! -e "$virtualbox_root/control1/control1.vbox" ]]
|
||||
[[ -e "$virtualbox_root/unmarked/unmarked.vbox" ]]
|
||||
|
||||
PATH="$fake_bin:$PATH" \
|
||||
HOME="$test_root/home" \
|
||||
K3S_CI_MOLECULE_ROOT="$molecule_root" \
|
||||
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
|
||||
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
|
||||
FAKE_VM_MODE=missing \
|
||||
FAKE_VBOX_ROOT="$virtualbox_root" \
|
||||
FAKE_LOG="$test_root/vbox.log" \
|
||||
bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --apply > "$output"
|
||||
|
||||
grep -Fq 'Stale Vagrant state without a registered VM' "$output"
|
||||
[[ ! -d "$molecule_root/repo/single_node/.vagrant" ]]
|
||||
|
||||
printf 'cleanup-runner-resources fixture test passed\n'
|
||||
Reference in New Issue
Block a user