feat(ci): optimize Molecule VM creation

- reuse validated runner-owned Vagrant linked-clone masters\n- create the default guests in bounded two-machine batches\n- capture create timing and runner utilization diagnostics
This commit is contained in:
Timothy Stewart
2026-08-01 01:15:25 -05:00
committed by Techno Tim
parent 57f234c8da
commit 10bde4eff0
14 changed files with 563 additions and 3 deletions
@@ -26,9 +26,20 @@ run_capture virtualbox-vms VBoxManage list vms
run_capture virtualbox-running-vms VBoxManage list runningvms
run_capture virtualbox-disks VBoxManage list hdds
run_capture virtualbox-hostonlyifs VBoxManage list hostonlyifs
run_capture virtualbox-groups VBoxManage list groups
run_capture vagrant-status vagrant global-status
run_capture molecule-state find "${HOME}/.cache/molecule" -maxdepth 6 -type f -path '*/.vagrant/machines/*/virtualbox/id' -print
scenario_name="${K3S_CI_SCENARIO_NAME:-}"
if [[ "$scenario_name" =~ ^[A-Za-z0-9_-]+$ ]]; then
molecule_state_dir="${HOME}/.cache/molecule/k3s-ansible/${scenario_name}"
for log_name in vagrant.out vagrant.err; do
if [[ -r "${molecule_state_dir}/${log_name}" ]]; then
cp -- "${molecule_state_dir}/${log_name}" "$output_dir/${scenario_name}-${log_name}"
fi
done
fi
if [[ -r /etc/vbox/networks.conf ]]; then
cp -- /etc/vbox/networks.conf "$output_dir/virtualbox-networks.conf"
fi
+44
View File
@@ -0,0 +1,44 @@
#!/usr/bin/env bash
set -Eeuo pipefail
output_dir="${1:?output directory is required}"
interval="${2:-10}"
[[ "$interval" =~ ^[1-9][0-9]*$ ]] || {
printf 'monitor interval must be a positive integer\n' >&2
exit 2
}
mkdir -p -- "$output_dir"
umask 077
free -h > "$output_dir/memory-before.txt"
df -h > "$output_dir/disk-before.txt"
vmstat -w "$interval" > "$output_dir/vmstat.txt" &
vmstat_pid=$!
iostat_pid=""
if command -v iostat >/dev/null 2>&1; then
iostat -dx "$interval" > "$output_dir/iostat.txt" &
iostat_pid=$!
else
printf 'iostat is not installed on this runner\n' > "$output_dir/iostat-unavailable.txt"
fi
cleanup() {
local rc=$?
trap - EXIT INT TERM
kill "$vmstat_pid" 2>/dev/null || true
[[ -z "$iostat_pid" ]] || kill "$iostat_pid" 2>/dev/null || true
wait "$vmstat_pid" 2>/dev/null || true
[[ -z "$iostat_pid" ]] || wait "$iostat_pid" 2>/dev/null || true
free -h > "$output_dir/memory-after.txt"
df -h > "$output_dir/disk-after.txt"
exit "$rc"
}
trap cleanup EXIT INT TERM
while :; do
sleep 3600 &
wait $!
done
+210
View File
@@ -0,0 +1,210 @@
#!/usr/bin/env bash
set -Eeuo pipefail
fail() {
printf 'Vagrant box master preparation refused: %s\n' "$1" >&2
exit 3
}
root_contains() {
local root="$1"
local path="$2"
[[ "$path" == "$root"/* ]]
}
read_machine_value() {
local machine_info="$1"
local key="$2"
awk -F= -v key="$key" '$1 == key {gsub(/"/, "", $2); print $2; exit}' <<< "$machine_info"
}
read_extra_data() {
local uuid="$1"
local key="$2"
local value
value="$(VBoxManage getextradata "$uuid" "$key" 2>/dev/null || true)"
[[ "$value" == 'Value: '* ]] || return 1
printf '%s\n' "${value#Value: }"
}
validate_owned_master() {
local uuid="$1"
local box="$2"
local version="$3"
local architecture="$4"
local machine_info cfg_file cfg_file_real vm_state groups disk_path disk_path_real
[[ "$uuid" =~ ^[0-9a-fA-F-]{36}$ ]] || return 1
machine_info="$(VBoxManage showvminfo "$uuid" --machinereadable 2>/dev/null)" || return 1
vm_state="$(read_machine_value "$machine_info" VMState)"
groups="$(read_machine_value "$machine_info" groups)"
cfg_file="$(read_machine_value "$machine_info" CfgFile)"
[[ "$vm_state" == poweroff ]] || return 1
[[ ",$groups," == *,/k3s-ansible/box-masters,* ]] || return 1
[[ -n "$cfg_file" ]] || return 1
cfg_file_real="$(readlink -f -- "$cfg_file" 2>/dev/null || true)"
[[ -n "$cfg_file_real" ]] || return 1
root_contains "$virtualbox_root_real" "$cfg_file_real" || return 1
[[ "$(read_extra_data "$uuid" k3s-ansible/owner || true)" == box-master ]] || return 1
[[ "$(read_extra_data "$uuid" k3s-ansible/box || true)" == "$box" ]] || return 1
[[ "$(read_extra_data "$uuid" k3s-ansible/version || true)" == "$version" ]] || return 1
[[ "$(read_extra_data "$uuid" k3s-ansible/architecture || true)" == "$architecture" ]] || return 1
while IFS= read -r disk_path; do
[[ -z "$disk_path" || "$disk_path" == none ]] && continue
disk_path_real="$(readlink -f -- "$disk_path" 2>/dev/null || true)"
[[ -n "$disk_path_real" ]] || return 1
root_contains "$virtualbox_root_real" "$disk_path_real" || return 1
done < <(awk -F= '$1 ~ /^(SATA|IDE|SCSI|SAS|VirtioSCSI|NVMe)-[0-9]+-[0-9]+$/ {
gsub(/"/, "", $2); print $2
}' <<< "$machine_info")
}
write_prewarm_vagrantfile() {
local destination="$1"
local box="$2"
local version="$3"
{
printf '%s\n' "Vagrant.configure('2') do |config|"
printf ' config.vm.box = "%s"\n' "$box"
printf ' config.vm.box_version = "%s"\n' "$version"
printf '%s\n' \
' config.vm.synced_folder ".", "/vagrant", disabled: true' \
' config.vm.hostname = "k3s-ansible-box-prewarm"' \
' config.vm.provider "virtualbox" do |virtualbox|' \
' virtualbox.linked_clone = true' \
' virtualbox.memory = 512' \
' virtualbox.cpus = 1' \
' end' \
'end'
} > "$destination"
}
cleanup_prewarm() {
local rc=$?
trap - EXIT
if [[ -n "${prewarm_dir:-}" && -d "$prewarm_dir" ]]; then
VAGRANT_CWD="$prewarm_dir" vagrant destroy --force >/dev/null 2>&1 || true
rm -rf -- "$prewarm_dir"
fi
exit "$rc"
}
create_owned_master() {
local box="$1"
local version="$2"
local architecture="$3"
local master_id_file="$4"
local mapping_file="$5"
local uuid machine_info cfg_file cfg_file_real vm_state mapping_tmp
# A master_id restored from an immutable cache is only a hint. Without the
# runner-local ownership record and matching VirtualBox metadata it is not
# trusted, adopted, modified, or deleted.
rm -f -- "$master_id_file"
prewarm_dir="$(mktemp -d "${master_root}/prewarm.XXXXXX")"
trap cleanup_prewarm EXIT
write_prewarm_vagrantfile "$prewarm_dir/Vagrantfile" "$box" "$version"
printf 'Creating runner-owned linked-clone master for %s %s %s\n' \
"$box" "$version" "$architecture"
VAGRANT_CWD="$prewarm_dir" vagrant up --provider virtualbox --no-provision
[[ -r "$master_id_file" ]] || fail "Vagrant did not record a master UUID for $box"
uuid="$(tr -d '[:space:]' < "$master_id_file")"
[[ "$uuid" =~ ^[0-9a-fA-F-]{36}$ ]] || fail "Vagrant recorded an invalid master UUID for $box"
machine_info="$(VBoxManage showvminfo "$uuid" --machinereadable 2>/dev/null)" || \
fail "Vagrant master $uuid for $box is not registered"
vm_state="$(read_machine_value "$machine_info" VMState)"
cfg_file="$(read_machine_value "$machine_info" CfgFile)"
cfg_file_real="$(readlink -f -- "$cfg_file" 2>/dev/null || true)"
[[ "$vm_state" == poweroff ]] || fail "new Vagrant master $uuid is not powered off"
if [[ -z "$cfg_file_real" ]] || ! root_contains "$virtualbox_root_real" "$cfg_file_real"; then
fail "new Vagrant master $uuid is outside the runner VirtualBox root"
fi
VAGRANT_CWD="$prewarm_dir" vagrant destroy --force
VBoxManage modifyvm "$uuid" --groups /k3s-ansible/box-masters
VBoxManage setextradata "$uuid" k3s-ansible/owner box-master
VBoxManage setextradata "$uuid" k3s-ansible/box "$box"
VBoxManage setextradata "$uuid" k3s-ansible/version "$version"
VBoxManage setextradata "$uuid" k3s-ansible/architecture "$architecture"
validate_owned_master "$uuid" "$box" "$version" "$architecture" || \
fail "new Vagrant master $uuid failed ownership validation"
rm -rf -- "$prewarm_dir"
prewarm_dir=""
trap - EXIT
mapping_tmp="${mapping_file}.tmp"
printf '%s\n' "$uuid" > "$mapping_tmp"
chmod 600 "$mapping_tmp"
mv -- "$mapping_tmp" "$mapping_file"
printf '%s\n' "$uuid" > "$master_id_file"
chmod 600 "$master_id_file"
printf 'Created and recorded owned master %s for %s\n' "$uuid" "$box"
}
repository_root="${K3S_CI_REPOSITORY_ROOT:-$(git rev-parse --show-toplevel)}"
lock_file="${VAGRANT_BOX_LOCK_FILE:-${repository_root}/.github/vagrant-boxes.lock}"
vagrant_home="${VAGRANT_HOME:?VAGRANT_HOME must be set}"
master_root="${K3S_CI_VAGRANT_MASTER_ROOT:-${HOME:?HOME must be set}/.cache/k3s-ci/vagrant-masters}"
virtualbox_root="${K3S_CI_VIRTUALBOX_ROOT:-${HOME}/VirtualBox VMs}"
[[ -r "$lock_file" ]] || fail "box lock file is missing or unreadable: $lock_file"
[[ -d "$vagrant_home/boxes" ]] || fail "Vagrant box directory is missing: $vagrant_home/boxes"
[[ -d "$virtualbox_root" ]] || fail "VirtualBox root is missing: $virtualbox_root"
box_root_real="$(readlink -f -- "$vagrant_home/boxes")"
virtualbox_root_real="$(readlink -f -- "$virtualbox_root")"
mkdir -p -- "$master_root"
chmod 700 "$master_root"
exec 9> "${master_root}/prepare.lock"
flock 9
lock_entries="$(awk '
/^[[:space:]]*#/ || NF == 0 { next }
NF != 3 { invalid = 1; next }
{ print $1 " " $2 " " $3 }
END { exit invalid }
' "$lock_file")" || fail 'invalid Vagrant box lock entry'
[[ -n "$lock_entries" ]] || fail 'Vagrant box lock is empty'
while read -r box version architecture; do
[[ "$box" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || fail "invalid box name: $box"
[[ "$version" =~ ^[A-Za-z0-9._-]+$ ]] || fail "invalid box version: $version"
[[ "$architecture" =~ ^[A-Za-z0-9._-]+$ ]] || fail "invalid box architecture: $architecture"
box_slug="${box//\//-VAGRANTSLASH-}"
record_slug="${box//\//_}-${version}-${architecture}"
box_dir="${vagrant_home}/boxes/${box_slug}/${version}/${architecture}/virtualbox"
[[ -d "$box_dir" ]] || fail "pinned box is not installed: $box $version $architecture"
box_dir_real="$(readlink -f -- "$box_dir")"
root_contains "$box_root_real" "$box_dir_real" || fail "box directory is outside VAGRANT_HOME: $box_dir_real"
master_id_file="${box_dir_real}/master_id"
mapping_file="${master_root}/${record_slug}.uuid"
uuid=""
if [[ -r "$mapping_file" ]]; then
uuid="$(tr -d '[:space:]' < "$mapping_file")"
fi
if [[ -n "$uuid" ]] && validate_owned_master "$uuid" "$box" "$version" "$architecture"; then
printf '%s\n' "$uuid" > "$master_id_file"
chmod 600 "$master_id_file"
printf 'Reusing owned master %s for %s %s %s\n' "$uuid" "$box" "$version" "$architecture"
continue
fi
if [[ -e "$mapping_file" ]]; then
printf 'Owned master record is stale for %s; rebuilding without deleting any VM or disk.\n' "$box"
fi
create_owned_master "$box" "$version" "$architecture" "$master_id_file" "$mapping_file"
done <<< "$lock_entries"
printf 'All pinned Vagrant box masters are ready.\n'
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -Eeuo pipefail
[[ "${1:-}" =~ ^[0-9]+$ ]]
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env bash
set -Eeuo pipefail
printf '%s\n' "$*" >> "$MOCK_VAGRANT_LOG"
case "${1:-}" in
up)
counter_file="$MOCK_VBOX_STATE/counter"
counter=0
[[ ! -r "$counter_file" ]] || counter="$(cat "$counter_file")"
counter=$((counter + 1))
printf '%s\n' "$counter" > "$counter_file"
uuid="00000000-0000-4000-8000-$(printf '%012d' "$counter")"
vm_dir="$MOCK_VBOX_ROOT/master-$counter"
mkdir -p -- "$vm_dir"
: > "$vm_dir/master.vbox"
: > "$vm_dir/master.vdi"
printf '%s\n' \
'VMState="poweroff"' \
'groups="/"' \
"CfgFile=\"$vm_dir/master.vbox\"" \
"SATA-0-0=\"$vm_dir/master.vdi\"" > "$MOCK_VBOX_STATE/vm-$uuid"
printf '%s\n' "$uuid" > "$MOCK_BOX_DIR/master_id"
;;
destroy) ;;
*) exit 2 ;;
esac
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/env bash
set -Eeuo pipefail
printf '%s\n' "$*" >> "$MOCK_VBOX_LOG"
command_name="${1:-}"
uuid="${2:-}"
case "$command_name" in
showvminfo)
[[ -r "$MOCK_VBOX_STATE/vm-$uuid" ]] || exit 1
cat "$MOCK_VBOX_STATE/vm-$uuid"
;;
getextradata)
key_slug="${3//\//_}"
if [[ ! -r "$MOCK_VBOX_STATE/extra-$uuid-$key_slug" ]]; then
printf '%s\n' 'No value set!'
exit 0
fi
printf 'Value: '
cat "$MOCK_VBOX_STATE/extra-$uuid-$key_slug"
;;
modifyvm)
[[ "${3:-}" == --groups ]]
awk -v groups="${4:-}" '
$1 !~ /^groups=/ { print }
END { printf "groups=\"%s\"\n", groups }
' "$MOCK_VBOX_STATE/vm-$uuid" > "$MOCK_VBOX_STATE/vm-$uuid.tmp"
mv "$MOCK_VBOX_STATE/vm-$uuid.tmp" "$MOCK_VBOX_STATE/vm-$uuid"
;;
setextradata)
key_slug="${3//\//_}"
printf '%s\n' "${4:-}" > "$MOCK_VBOX_STATE/extra-$uuid-$key_slug"
;;
unregistervm|closemedium)
printf '%s\n' 'destructive VirtualBox command invoked' >&2
exit 99
;;
*) exit 2 ;;
esac
+81
View File
@@ -0,0 +1,81 @@
#!/usr/bin/env bash
set -Eeuo pipefail
repo_root="$(git rev-parse --show-toplevel)"
fixture="$(mktemp -d)"
trap 'rm -rf -- "$fixture"' EXIT
mock_bin="$fixture/bin"
mock_state="$fixture/state"
mock_home="$fixture/home"
mock_vagrant_home="$fixture/vagrant-home"
mock_box_dir="$mock_vagrant_home/boxes/generic-VAGRANTSLASH-ubuntu2204/4.3.12/amd64/virtualbox"
mock_vbox_root="$mock_home/VirtualBox VMs"
mock_master_root="$mock_home/.cache/k3s-ci/vagrant-masters"
lock_file="$fixture/vagrant-boxes.lock"
mkdir -p -- "$mock_bin" "$mock_state" "$mock_box_dir" "$mock_vbox_root"
printf '%s\n' 'generic/ubuntu2204 4.3.12 amd64' > "$lock_file"
ln -s "$repo_root/.github/scripts/test-fixtures/mock-vboxmanage" "$mock_bin/VBoxManage"
ln -s "$repo_root/.github/scripts/test-fixtures/mock-vagrant" "$mock_bin/vagrant"
ln -s "$repo_root/.github/scripts/test-fixtures/mock-flock" "$mock_bin/flock"
export PATH="$mock_bin:$PATH"
export HOME="$mock_home"
export VAGRANT_HOME="$mock_vagrant_home"
export VAGRANT_BOX_LOCK_FILE="$lock_file"
export K3S_CI_REPOSITORY_ROOT="$repo_root"
export K3S_CI_VAGRANT_MASTER_ROOT="$mock_master_root"
export K3S_CI_VIRTUALBOX_ROOT="$mock_vbox_root"
export MOCK_VBOX_STATE="$mock_state"
export MOCK_VBOX_ROOT="$mock_vbox_root"
export MOCK_BOX_DIR="$mock_box_dir"
export MOCK_VBOX_LOG="$fixture/vbox.log"
export MOCK_VAGRANT_LOG="$fixture/vagrant.log"
: > "$MOCK_VBOX_LOG"
: > "$MOCK_VAGRANT_LOG"
unowned_uuid='99999999-9999-4999-8999-999999999999'
printf '%s\n' "$unowned_uuid" > "$mock_box_dir/master_id"
script="$repo_root/.github/scripts/prepare-vagrant-box-masters.sh"
first_output="$fixture/first-output"
second_output="$fixture/second-output"
third_output="$fixture/third-output"
"$script" > "$first_output"
mapping_file="$mock_master_root/generic_ubuntu2204-4.3.12-amd64.uuid"
test -s "$mapping_file"
cmp -s "$mapping_file" "$mock_box_dir/master_id"
grep -Fq 'Created and recorded owned master' "$first_output"
grep -Fq 'modifyvm' "$MOCK_VBOX_LOG"
grep -Fq 'setextradata' "$MOCK_VBOX_LOG"
if grep -Fq "$unowned_uuid" "$MOCK_VBOX_LOG"; then
printf 'unowned cached master UUID was unexpectedly inspected or modified\n' >&2
exit 1
fi
if grep -Eq 'unregistervm|closemedium' "$MOCK_VBOX_LOG"; then
printf 'master preparation invoked a destructive VirtualBox command\n' >&2
exit 1
fi
: > "$MOCK_VAGRANT_LOG"
"$script" > "$second_output"
grep -Fq 'Reusing owned master' "$second_output"
if grep -Fq 'up ' "$MOCK_VAGRANT_LOG"; then
printf 'valid owned master was unexpectedly rebuilt\n' >&2
exit 1
fi
stale_uuid="$(tr -d '[:space:]' < "$mapping_file")"
rm -f -- "$mock_state/vm-$stale_uuid"
: > "$MOCK_VAGRANT_LOG"
"$script" > "$third_output"
grep -Fq 'rebuilding without deleting any VM or disk' "$third_output"
grep -Fq 'up ' "$MOCK_VAGRANT_LOG"
if grep -Eq 'unregistervm|closemedium' "$MOCK_VBOX_LOG"; then
printf 'stale master recovery invoked a destructive VirtualBox command\n' >&2
exit 1
fi
printf 'Vagrant box master preparation fixture test passed\n'
+24
View File
@@ -0,0 +1,24 @@
#!/usr/bin/env bash
set -Eeuo pipefail
if (($# < 2)); then
printf 'Usage: vagrant-up-timed.sh WORKDIR MACHINE [MACHINE ...]\n' >&2
exit 2
fi
workdir="$1"
shift
timing_log="${K3S_CI_CREATE_TIMING_LOG:-${RUNNER_TEMP:-/tmp}/k3s-ci-create-timing.log}"
mkdir -p -- "${timing_log%/*}"
printf '%s batch-start machines=%s\n' "$(date --iso-8601=ns)" "$*" | tee -a "$timing_log"
set +e
VAGRANT_CWD="$workdir" vagrant up "$@" --provider virtualbox --no-provision 2>&1 |
while IFS= read -r line; do
printf '%s %s\n' "$(date --iso-8601=ns)" "$line"
done | tee -a "$timing_log"
rc=${PIPESTATUS[0]}
set -e
printf '%s batch-end rc=%d machines=%s\n' "$(date --iso-8601=ns)" "$rc" "$*" | tee -a "$timing_log"
exit "$rc"