diff --git a/.ansible-lint b/.ansible-lint index 607c1ab2..c535afc1 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -20,3 +20,8 @@ exclude_paths: skip_list: - var-naming[no-role-prefix] + +# The Molecule Vagrant driver injects this module at runtime. The custom create +# playbook is syntax-checked separately against the exact pinned plugin module. +mock_modules: + - vagrant diff --git a/.github/scripts/collect-runner-diagnostics.sh b/.github/scripts/collect-runner-diagnostics.sh index 69f50097..0f6ec12d 100755 --- a/.github/scripts/collect-runner-diagnostics.sh +++ b/.github/scripts/collect-runner-diagnostics.sh @@ -26,9 +26,20 @@ run_capture virtualbox-vms VBoxManage list vms run_capture virtualbox-running-vms VBoxManage list runningvms run_capture virtualbox-disks VBoxManage list hdds run_capture virtualbox-hostonlyifs VBoxManage list hostonlyifs +run_capture virtualbox-groups VBoxManage list groups run_capture vagrant-status vagrant global-status run_capture molecule-state find "${HOME}/.cache/molecule" -maxdepth 6 -type f -path '*/.vagrant/machines/*/virtualbox/id' -print +scenario_name="${K3S_CI_SCENARIO_NAME:-}" +if [[ "$scenario_name" =~ ^[A-Za-z0-9_-]+$ ]]; then + molecule_state_dir="${HOME}/.cache/molecule/k3s-ansible/${scenario_name}" + for log_name in vagrant.out vagrant.err; do + if [[ -r "${molecule_state_dir}/${log_name}" ]]; then + cp -- "${molecule_state_dir}/${log_name}" "$output_dir/${scenario_name}-${log_name}" + fi + done +fi + if [[ -r /etc/vbox/networks.conf ]]; then cp -- /etc/vbox/networks.conf "$output_dir/virtualbox-networks.conf" fi diff --git a/.github/scripts/monitor-runner-resources.sh b/.github/scripts/monitor-runner-resources.sh new file mode 100755 index 00000000..777d6061 --- /dev/null +++ b/.github/scripts/monitor-runner-resources.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +output_dir="${1:?output directory is required}" +interval="${2:-10}" +[[ "$interval" =~ ^[1-9][0-9]*$ ]] || { + printf 'monitor interval must be a positive integer\n' >&2 + exit 2 +} + +mkdir -p -- "$output_dir" +umask 077 + +free -h > "$output_dir/memory-before.txt" +df -h > "$output_dir/disk-before.txt" +vmstat -w "$interval" > "$output_dir/vmstat.txt" & +vmstat_pid=$! + +iostat_pid="" +if command -v iostat >/dev/null 2>&1; then + iostat -dx "$interval" > "$output_dir/iostat.txt" & + iostat_pid=$! +else + printf 'iostat is not installed on this runner\n' > "$output_dir/iostat-unavailable.txt" +fi + +cleanup() { + local rc=$? + trap - EXIT INT TERM + kill "$vmstat_pid" 2>/dev/null || true + [[ -z "$iostat_pid" ]] || kill "$iostat_pid" 2>/dev/null || true + wait "$vmstat_pid" 2>/dev/null || true + [[ -z "$iostat_pid" ]] || wait "$iostat_pid" 2>/dev/null || true + free -h > "$output_dir/memory-after.txt" + df -h > "$output_dir/disk-after.txt" + exit "$rc" +} +trap cleanup EXIT INT TERM + +while :; do + sleep 3600 & + wait $! +done diff --git a/.github/scripts/prepare-vagrant-box-masters.sh b/.github/scripts/prepare-vagrant-box-masters.sh new file mode 100755 index 00000000..355405e3 --- /dev/null +++ b/.github/scripts/prepare-vagrant-box-masters.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +fail() { + printf 'Vagrant box master preparation refused: %s\n' "$1" >&2 + exit 3 +} + +root_contains() { + local root="$1" + local path="$2" + [[ "$path" == "$root"/* ]] +} + +read_machine_value() { + local machine_info="$1" + local key="$2" + awk -F= -v key="$key" '$1 == key {gsub(/"/, "", $2); print $2; exit}' <<< "$machine_info" +} + +read_extra_data() { + local uuid="$1" + local key="$2" + local value + value="$(VBoxManage getextradata "$uuid" "$key" 2>/dev/null || true)" + [[ "$value" == 'Value: '* ]] || return 1 + printf '%s\n' "${value#Value: }" +} + +validate_owned_master() { + local uuid="$1" + local box="$2" + local version="$3" + local architecture="$4" + local machine_info cfg_file cfg_file_real vm_state groups disk_path disk_path_real + + [[ "$uuid" =~ ^[0-9a-fA-F-]{36}$ ]] || return 1 + machine_info="$(VBoxManage showvminfo "$uuid" --machinereadable 2>/dev/null)" || return 1 + vm_state="$(read_machine_value "$machine_info" VMState)" + groups="$(read_machine_value "$machine_info" groups)" + cfg_file="$(read_machine_value "$machine_info" CfgFile)" + [[ "$vm_state" == poweroff ]] || return 1 + [[ ",$groups," == *,/k3s-ansible/box-masters,* ]] || return 1 + [[ -n "$cfg_file" ]] || return 1 + cfg_file_real="$(readlink -f -- "$cfg_file" 2>/dev/null || true)" + [[ -n "$cfg_file_real" ]] || return 1 + root_contains "$virtualbox_root_real" "$cfg_file_real" || return 1 + + [[ "$(read_extra_data "$uuid" k3s-ansible/owner || true)" == box-master ]] || return 1 + [[ "$(read_extra_data "$uuid" k3s-ansible/box || true)" == "$box" ]] || return 1 + [[ "$(read_extra_data "$uuid" k3s-ansible/version || true)" == "$version" ]] || return 1 + [[ "$(read_extra_data "$uuid" k3s-ansible/architecture || true)" == "$architecture" ]] || return 1 + + while IFS= read -r disk_path; do + [[ -z "$disk_path" || "$disk_path" == none ]] && continue + disk_path_real="$(readlink -f -- "$disk_path" 2>/dev/null || true)" + [[ -n "$disk_path_real" ]] || return 1 + root_contains "$virtualbox_root_real" "$disk_path_real" || return 1 + done < <(awk -F= '$1 ~ /^(SATA|IDE|SCSI|SAS|VirtioSCSI|NVMe)-[0-9]+-[0-9]+$/ { + gsub(/"/, "", $2); print $2 + }' <<< "$machine_info") +} + +write_prewarm_vagrantfile() { + local destination="$1" + local box="$2" + local version="$3" + { + printf '%s\n' "Vagrant.configure('2') do |config|" + printf ' config.vm.box = "%s"\n' "$box" + printf ' config.vm.box_version = "%s"\n' "$version" + printf '%s\n' \ + ' config.vm.synced_folder ".", "/vagrant", disabled: true' \ + ' config.vm.hostname = "k3s-ansible-box-prewarm"' \ + ' config.vm.provider "virtualbox" do |virtualbox|' \ + ' virtualbox.linked_clone = true' \ + ' virtualbox.memory = 512' \ + ' virtualbox.cpus = 1' \ + ' end' \ + 'end' + } > "$destination" +} + +cleanup_prewarm() { + local rc=$? + trap - EXIT + if [[ -n "${prewarm_dir:-}" && -d "$prewarm_dir" ]]; then + VAGRANT_CWD="$prewarm_dir" vagrant destroy --force >/dev/null 2>&1 || true + rm -rf -- "$prewarm_dir" + fi + exit "$rc" +} + +create_owned_master() { + local box="$1" + local version="$2" + local architecture="$3" + local master_id_file="$4" + local mapping_file="$5" + local uuid machine_info cfg_file cfg_file_real vm_state mapping_tmp + + # A master_id restored from an immutable cache is only a hint. Without the + # runner-local ownership record and matching VirtualBox metadata it is not + # trusted, adopted, modified, or deleted. + rm -f -- "$master_id_file" + + prewarm_dir="$(mktemp -d "${master_root}/prewarm.XXXXXX")" + trap cleanup_prewarm EXIT + write_prewarm_vagrantfile "$prewarm_dir/Vagrantfile" "$box" "$version" + + printf 'Creating runner-owned linked-clone master for %s %s %s\n' \ + "$box" "$version" "$architecture" + VAGRANT_CWD="$prewarm_dir" vagrant up --provider virtualbox --no-provision + + [[ -r "$master_id_file" ]] || fail "Vagrant did not record a master UUID for $box" + uuid="$(tr -d '[:space:]' < "$master_id_file")" + [[ "$uuid" =~ ^[0-9a-fA-F-]{36}$ ]] || fail "Vagrant recorded an invalid master UUID for $box" + + machine_info="$(VBoxManage showvminfo "$uuid" --machinereadable 2>/dev/null)" || \ + fail "Vagrant master $uuid for $box is not registered" + vm_state="$(read_machine_value "$machine_info" VMState)" + cfg_file="$(read_machine_value "$machine_info" CfgFile)" + cfg_file_real="$(readlink -f -- "$cfg_file" 2>/dev/null || true)" + [[ "$vm_state" == poweroff ]] || fail "new Vagrant master $uuid is not powered off" + if [[ -z "$cfg_file_real" ]] || ! root_contains "$virtualbox_root_real" "$cfg_file_real"; then + fail "new Vagrant master $uuid is outside the runner VirtualBox root" + fi + + VAGRANT_CWD="$prewarm_dir" vagrant destroy --force + VBoxManage modifyvm "$uuid" --groups /k3s-ansible/box-masters + VBoxManage setextradata "$uuid" k3s-ansible/owner box-master + VBoxManage setextradata "$uuid" k3s-ansible/box "$box" + VBoxManage setextradata "$uuid" k3s-ansible/version "$version" + VBoxManage setextradata "$uuid" k3s-ansible/architecture "$architecture" + validate_owned_master "$uuid" "$box" "$version" "$architecture" || \ + fail "new Vagrant master $uuid failed ownership validation" + + rm -rf -- "$prewarm_dir" + prewarm_dir="" + trap - EXIT + + mapping_tmp="${mapping_file}.tmp" + printf '%s\n' "$uuid" > "$mapping_tmp" + chmod 600 "$mapping_tmp" + mv -- "$mapping_tmp" "$mapping_file" + printf '%s\n' "$uuid" > "$master_id_file" + chmod 600 "$master_id_file" + printf 'Created and recorded owned master %s for %s\n' "$uuid" "$box" +} + +repository_root="${K3S_CI_REPOSITORY_ROOT:-$(git rev-parse --show-toplevel)}" +lock_file="${VAGRANT_BOX_LOCK_FILE:-${repository_root}/.github/vagrant-boxes.lock}" +vagrant_home="${VAGRANT_HOME:?VAGRANT_HOME must be set}" +master_root="${K3S_CI_VAGRANT_MASTER_ROOT:-${HOME:?HOME must be set}/.cache/k3s-ci/vagrant-masters}" +virtualbox_root="${K3S_CI_VIRTUALBOX_ROOT:-${HOME}/VirtualBox VMs}" + +[[ -r "$lock_file" ]] || fail "box lock file is missing or unreadable: $lock_file" +[[ -d "$vagrant_home/boxes" ]] || fail "Vagrant box directory is missing: $vagrant_home/boxes" +[[ -d "$virtualbox_root" ]] || fail "VirtualBox root is missing: $virtualbox_root" + +box_root_real="$(readlink -f -- "$vagrant_home/boxes")" +virtualbox_root_real="$(readlink -f -- "$virtualbox_root")" +mkdir -p -- "$master_root" +chmod 700 "$master_root" + +exec 9> "${master_root}/prepare.lock" +flock 9 + +lock_entries="$(awk ' + /^[[:space:]]*#/ || NF == 0 { next } + NF != 3 { invalid = 1; next } + { print $1 " " $2 " " $3 } + END { exit invalid } +' "$lock_file")" || fail 'invalid Vagrant box lock entry' +[[ -n "$lock_entries" ]] || fail 'Vagrant box lock is empty' + +while read -r box version architecture; do + [[ "$box" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || fail "invalid box name: $box" + [[ "$version" =~ ^[A-Za-z0-9._-]+$ ]] || fail "invalid box version: $version" + [[ "$architecture" =~ ^[A-Za-z0-9._-]+$ ]] || fail "invalid box architecture: $architecture" + + box_slug="${box//\//-VAGRANTSLASH-}" + record_slug="${box//\//_}-${version}-${architecture}" + box_dir="${vagrant_home}/boxes/${box_slug}/${version}/${architecture}/virtualbox" + [[ -d "$box_dir" ]] || fail "pinned box is not installed: $box $version $architecture" + box_dir_real="$(readlink -f -- "$box_dir")" + root_contains "$box_root_real" "$box_dir_real" || fail "box directory is outside VAGRANT_HOME: $box_dir_real" + + master_id_file="${box_dir_real}/master_id" + mapping_file="${master_root}/${record_slug}.uuid" + uuid="" + if [[ -r "$mapping_file" ]]; then + uuid="$(tr -d '[:space:]' < "$mapping_file")" + fi + + if [[ -n "$uuid" ]] && validate_owned_master "$uuid" "$box" "$version" "$architecture"; then + printf '%s\n' "$uuid" > "$master_id_file" + chmod 600 "$master_id_file" + printf 'Reusing owned master %s for %s %s %s\n' "$uuid" "$box" "$version" "$architecture" + continue + fi + + if [[ -e "$mapping_file" ]]; then + printf 'Owned master record is stale for %s; rebuilding without deleting any VM or disk.\n' "$box" + fi + create_owned_master "$box" "$version" "$architecture" "$master_id_file" "$mapping_file" +done <<< "$lock_entries" + +printf 'All pinned Vagrant box masters are ready.\n' diff --git a/.github/scripts/test-fixtures/mock-flock b/.github/scripts/test-fixtures/mock-flock new file mode 100755 index 00000000..a2828e3a --- /dev/null +++ b/.github/scripts/test-fixtures/mock-flock @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +[[ "${1:-}" =~ ^[0-9]+$ ]] diff --git a/.github/scripts/test-fixtures/mock-vagrant b/.github/scripts/test-fixtures/mock-vagrant new file mode 100755 index 00000000..c9a11c78 --- /dev/null +++ b/.github/scripts/test-fixtures/mock-vagrant @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +printf '%s\n' "$*" >> "$MOCK_VAGRANT_LOG" +case "${1:-}" in + up) + counter_file="$MOCK_VBOX_STATE/counter" + counter=0 + [[ ! -r "$counter_file" ]] || counter="$(cat "$counter_file")" + counter=$((counter + 1)) + printf '%s\n' "$counter" > "$counter_file" + uuid="00000000-0000-4000-8000-$(printf '%012d' "$counter")" + vm_dir="$MOCK_VBOX_ROOT/master-$counter" + mkdir -p -- "$vm_dir" + : > "$vm_dir/master.vbox" + : > "$vm_dir/master.vdi" + printf '%s\n' \ + 'VMState="poweroff"' \ + 'groups="/"' \ + "CfgFile=\"$vm_dir/master.vbox\"" \ + "SATA-0-0=\"$vm_dir/master.vdi\"" > "$MOCK_VBOX_STATE/vm-$uuid" + printf '%s\n' "$uuid" > "$MOCK_BOX_DIR/master_id" + ;; + destroy) ;; + *) exit 2 ;; +esac diff --git a/.github/scripts/test-fixtures/mock-vboxmanage b/.github/scripts/test-fixtures/mock-vboxmanage new file mode 100755 index 00000000..dc395cac --- /dev/null +++ b/.github/scripts/test-fixtures/mock-vboxmanage @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +printf '%s\n' "$*" >> "$MOCK_VBOX_LOG" +command_name="${1:-}" +uuid="${2:-}" + +case "$command_name" in + showvminfo) + [[ -r "$MOCK_VBOX_STATE/vm-$uuid" ]] || exit 1 + cat "$MOCK_VBOX_STATE/vm-$uuid" + ;; + getextradata) + key_slug="${3//\//_}" + if [[ ! -r "$MOCK_VBOX_STATE/extra-$uuid-$key_slug" ]]; then + printf '%s\n' 'No value set!' + exit 0 + fi + printf 'Value: ' + cat "$MOCK_VBOX_STATE/extra-$uuid-$key_slug" + ;; + modifyvm) + [[ "${3:-}" == --groups ]] + awk -v groups="${4:-}" ' + $1 !~ /^groups=/ { print } + END { printf "groups=\"%s\"\n", groups } + ' "$MOCK_VBOX_STATE/vm-$uuid" > "$MOCK_VBOX_STATE/vm-$uuid.tmp" + mv "$MOCK_VBOX_STATE/vm-$uuid.tmp" "$MOCK_VBOX_STATE/vm-$uuid" + ;; + setextradata) + key_slug="${3//\//_}" + printf '%s\n' "${4:-}" > "$MOCK_VBOX_STATE/extra-$uuid-$key_slug" + ;; + unregistervm|closemedium) + printf '%s\n' 'destructive VirtualBox command invoked' >&2 + exit 99 + ;; + *) exit 2 ;; +esac diff --git a/.github/scripts/test-prepare-vagrant-box-masters.sh b/.github/scripts/test-prepare-vagrant-box-masters.sh new file mode 100755 index 00000000..2aaa04d1 --- /dev/null +++ b/.github/scripts/test-prepare-vagrant-box-masters.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +repo_root="$(git rev-parse --show-toplevel)" +fixture="$(mktemp -d)" +trap 'rm -rf -- "$fixture"' EXIT + +mock_bin="$fixture/bin" +mock_state="$fixture/state" +mock_home="$fixture/home" +mock_vagrant_home="$fixture/vagrant-home" +mock_box_dir="$mock_vagrant_home/boxes/generic-VAGRANTSLASH-ubuntu2204/4.3.12/amd64/virtualbox" +mock_vbox_root="$mock_home/VirtualBox VMs" +mock_master_root="$mock_home/.cache/k3s-ci/vagrant-masters" +lock_file="$fixture/vagrant-boxes.lock" +mkdir -p -- "$mock_bin" "$mock_state" "$mock_box_dir" "$mock_vbox_root" +printf '%s\n' 'generic/ubuntu2204 4.3.12 amd64' > "$lock_file" +ln -s "$repo_root/.github/scripts/test-fixtures/mock-vboxmanage" "$mock_bin/VBoxManage" +ln -s "$repo_root/.github/scripts/test-fixtures/mock-vagrant" "$mock_bin/vagrant" +ln -s "$repo_root/.github/scripts/test-fixtures/mock-flock" "$mock_bin/flock" + +export PATH="$mock_bin:$PATH" +export HOME="$mock_home" +export VAGRANT_HOME="$mock_vagrant_home" +export VAGRANT_BOX_LOCK_FILE="$lock_file" +export K3S_CI_REPOSITORY_ROOT="$repo_root" +export K3S_CI_VAGRANT_MASTER_ROOT="$mock_master_root" +export K3S_CI_VIRTUALBOX_ROOT="$mock_vbox_root" +export MOCK_VBOX_STATE="$mock_state" +export MOCK_VBOX_ROOT="$mock_vbox_root" +export MOCK_BOX_DIR="$mock_box_dir" +export MOCK_VBOX_LOG="$fixture/vbox.log" +export MOCK_VAGRANT_LOG="$fixture/vagrant.log" +: > "$MOCK_VBOX_LOG" +: > "$MOCK_VAGRANT_LOG" + +unowned_uuid='99999999-9999-4999-8999-999999999999' +printf '%s\n' "$unowned_uuid" > "$mock_box_dir/master_id" + +script="$repo_root/.github/scripts/prepare-vagrant-box-masters.sh" +first_output="$fixture/first-output" +second_output="$fixture/second-output" +third_output="$fixture/third-output" + +"$script" > "$first_output" +mapping_file="$mock_master_root/generic_ubuntu2204-4.3.12-amd64.uuid" +test -s "$mapping_file" +cmp -s "$mapping_file" "$mock_box_dir/master_id" +grep -Fq 'Created and recorded owned master' "$first_output" +grep -Fq 'modifyvm' "$MOCK_VBOX_LOG" +grep -Fq 'setextradata' "$MOCK_VBOX_LOG" +if grep -Fq "$unowned_uuid" "$MOCK_VBOX_LOG"; then + printf 'unowned cached master UUID was unexpectedly inspected or modified\n' >&2 + exit 1 +fi +if grep -Eq 'unregistervm|closemedium' "$MOCK_VBOX_LOG"; then + printf 'master preparation invoked a destructive VirtualBox command\n' >&2 + exit 1 +fi + +: > "$MOCK_VAGRANT_LOG" +"$script" > "$second_output" +grep -Fq 'Reusing owned master' "$second_output" +if grep -Fq 'up ' "$MOCK_VAGRANT_LOG"; then + printf 'valid owned master was unexpectedly rebuilt\n' >&2 + exit 1 +fi + +stale_uuid="$(tr -d '[:space:]' < "$mapping_file")" +rm -f -- "$mock_state/vm-$stale_uuid" +: > "$MOCK_VAGRANT_LOG" +"$script" > "$third_output" +grep -Fq 'rebuilding without deleting any VM or disk' "$third_output" +grep -Fq 'up ' "$MOCK_VAGRANT_LOG" +if grep -Eq 'unregistervm|closemedium' "$MOCK_VBOX_LOG"; then + printf 'stale master recovery invoked a destructive VirtualBox command\n' >&2 + exit 1 +fi + +printf 'Vagrant box master preparation fixture test passed\n' diff --git a/.github/scripts/vagrant-up-timed.sh b/.github/scripts/vagrant-up-timed.sh new file mode 100755 index 00000000..2e0c0139 --- /dev/null +++ b/.github/scripts/vagrant-up-timed.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +if (($# < 2)); then + printf 'Usage: vagrant-up-timed.sh WORKDIR MACHINE [MACHINE ...]\n' >&2 + exit 2 +fi + +workdir="$1" +shift +timing_log="${K3S_CI_CREATE_TIMING_LOG:-${RUNNER_TEMP:-/tmp}/k3s-ci-create-timing.log}" +mkdir -p -- "${timing_log%/*}" + +printf '%s batch-start machines=%s\n' "$(date --iso-8601=ns)" "$*" | tee -a "$timing_log" +set +e +VAGRANT_CWD="$workdir" vagrant up "$@" --provider virtualbox --no-provision 2>&1 | + while IFS= read -r line; do + printf '%s %s\n' "$(date --iso-8601=ns)" "$line" + done | tee -a "$timing_log" +rc=${PIPESTATUS[0]} +set -e +printf '%s batch-end rc=%d machines=%s\n' "$(date --iso-8601=ns)" "$rc" "$*" | tee -a "$timing_log" +exit "$rc" diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml index fc2cedb9..ef7ce44a 100644 --- a/.github/workflows/cache.yml +++ b/.github/workflows/cache.yml @@ -45,7 +45,7 @@ jobs: lookup-only: true path: | .vagrant-home/boxes - key: vagrant-boxes-${{ runner.os }}-${{ runner.arch }}-virtualbox-7.2-vagrant-2.4-${{ hashFiles('.github/vagrant-boxes.lock') }} # yamllint disable-line rule:line-length + key: vagrant-boxes-${{ runner.name }}-${{ runner.os }}-${{ runner.arch }}-virtualbox-7.2-vagrant-2.4-${{ hashFiles('.github/vagrant-boxes.lock') }} # yamllint disable-line rule:line-length - name: Download Vagrant boxes for all scenarios # An exact hit skips both cache restoration and upstream downloads. @@ -53,4 +53,5 @@ jobs: if: steps.cache-vagrant.outputs.cache-hit != 'true' run: | ./.github/download-boxes.sh + ./.github/scripts/prepare-vagrant-box-masters.sh vagrant box list diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 068f98e9..1f1932d4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,9 +57,12 @@ jobs: uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # 6.1.0 with: path: .vagrant-home/boxes - key: vagrant-boxes-${{ runner.os }}-${{ runner.arch }}-virtualbox-7.2-vagrant-2.4-${{ hashFiles('.github/vagrant-boxes.lock') }} # yamllint disable-line rule:line-length + key: vagrant-boxes-${{ runner.name }}-${{ runner.os }}-${{ runner.arch }}-virtualbox-7.2-vagrant-2.4-${{ hashFiles('.github/vagrant-boxes.lock') }} # yamllint disable-line rule:line-length fail-on-cache-miss: true + - name: Prepare runner-owned Vagrant box masters + run: ./.github/scripts/prepare-vagrant-box-masters.sh + - name: Install dependencies run: | echo "::group::Upgrade pip" @@ -71,7 +74,20 @@ jobs: echo "::endgroup::" - name: Test with molecule - run: molecule test --scenario-name ${{ matrix.scenario }} + run: | + set -Eeuo pipefail + resource_dir="${RUNNER_TEMP}/logs/resources/${{ matrix.scenario }}" + timing_file="${RUNNER_TEMP}/logs/timing/${{ matrix.scenario }}.txt" + mkdir -p -- "${timing_file%/*}" + ./.github/scripts/monitor-runner-resources.sh "$resource_dir" 10 & + monitor_pid=$! + stop_monitor() { + kill -TERM "$monitor_pid" 2>/dev/null || true + wait "$monitor_pid" 2>/dev/null || true + } + trap stop_monitor EXIT + /usr/bin/time -v -o "$timing_file" \ + molecule test --scenario-name ${{ matrix.scenario }} timeout-minutes: 90 env: ANSIBLE_K3S_LOG_DIR: ${{ runner.temp }}/logs/k3s-ansible/${{ matrix.scenario }} @@ -79,10 +95,13 @@ jobs: ANSIBLE_TIMEOUT: 120 PY_COLORS: 1 ANSIBLE_FORCE_COLOR: 1 + K3S_CI_CREATE_TIMING_LOG: ${{ runner.temp }}/logs/timing/${{ matrix.scenario }}-create.log - name: Collect runner diagnostics if: always() run: ./.github/scripts/collect-runner-diagnostics.sh "${RUNNER_TEMP}/logs/runner" + env: + K3S_CI_SCENARIO_NAME: ${{ matrix.scenario }} - name: Clean repository-owned resources after testing if: always() diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8fde5469..d8528dfe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -50,3 +50,9 @@ repos: language: system pass_filenames: false files: ^\.github/(download-boxes|test-download-boxes)\.sh$|^\.github/vagrant-boxes\.lock$ + - id: prepare-vagrant-box-masters-test + name: Vagrant box master preparation test + entry: .github/scripts/test-prepare-vagrant-box-masters.sh + language: system + pass_filenames: false + files: ^\.github/scripts/(prepare-vagrant-box-masters|test-prepare-vagrant-box-masters)\.sh$ diff --git a/molecule/default/create.yml b/molecule/default/create.yml new file mode 100644 index 00000000..ca1de402 --- /dev/null +++ b/molecule/default/create.yml @@ -0,0 +1,87 @@ +--- +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + create_batches: + - [control1, control2] + - [control3, node1] + - [node2] + tasks: + - name: Verify that bounded batches cover the configured platforms exactly once + ansible.builtin.assert: + that: + - create_batches | flatten | sort == molecule_yml.platforms | map(attribute='name') | sort + - create_batches | flatten | length == create_batches | flatten | unique | length + - create_batches | map('length') | max <= 2 + fail_msg: Bounded create batches do not match the configured default platforms. + + - name: Check for existing Vagrant machine state + ansible.builtin.stat: + path: "{{ molecule_ephemeral_directory }}/.vagrant/machines" + register: vagrant_machine_state + + - name: Generate the complete Vagrant configuration for a clean create # noqa fqcn[action] + vagrant: + instances: "{{ molecule_yml.platforms }}" + default_box: "{{ molecule_yml.driver.default_box | default('generic/alpine316') }}" + provider_name: "{{ molecule_yml.driver.provider.name | default(omit, true) }}" + provision: "{{ molecule_yml.driver.provision | default(omit) }}" + cachier: "{{ molecule_yml.driver.cachier | default(omit) }}" + parallel: false + state: halt + when: not vagrant_machine_state.stat.exists + changed_when: false + + - name: Start clean Vagrant guests in bounded batches + ansible.builtin.command: + argv: >- + {{ + [molecule_project_directory + '/.github/scripts/vagrant-up-timed.sh', + molecule_ephemeral_directory] + item + }} + loop: "{{ create_batches }}" + loop_control: + label: "{{ item | join(', ') }}" + when: not vagrant_machine_state.stat.exists + changed_when: true + + - name: Reconcile all instances and collect their connection configuration # noqa fqcn[action] + vagrant: + instances: "{{ molecule_yml.platforms }}" + default_box: "{{ molecule_yml.driver.default_box | default('generic/alpine316') }}" + provider_name: "{{ molecule_yml.driver.provider.name | default(omit, true) }}" + provision: "{{ molecule_yml.driver.provision | default(omit) }}" + cachier: "{{ molecule_yml.driver.cachier | default(omit) }}" + parallel: false + state: up + register: server + no_log: false + + - name: Populate instance configuration dictionaries + ansible.builtin.set_fact: + instance_conf_dict: + instance: "{{ item.Host }}" + address: "{{ item.HostName }}" + user: "{{ item.User }}" + port: "{{ item.Port }}" + identity_file: "{{ item.IdentityFile }}" + loop: "{{ server.results }}" + register: instance_config_dict + + - name: Convert instance configuration dictionaries to a list + ansible.builtin.set_fact: + instance_conf: >- + {{ + instance_config_dict.results + | map(attribute='ansible_facts.instance_conf_dict') + | list + }} + + - name: Write Molecule instance configuration + ansible.builtin.copy: + content: "{{ instance_conf | to_json | from_json | to_yaml }}" + dest: "{{ molecule_instance_config }}" + mode: "0600" diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index 51f418db..2f79fc3c 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -89,6 +89,7 @@ provisioner: env: ANSIBLE_VERBOSITY: 1 playbooks: + create: create.yml converge: ../resources/converge.yml side_effect: ../resources/reset.yml verify: ../resources/verify.yml