1 Commits

Author SHA1 Message Date
dependabot[bot] 34f1032470 chore(deps): bump actions/checkout from 4.2.0 to 4.2.2
Bumps [actions/checkout](https://github.com/actions/checkout) from 4.2.0 to 4.2.2.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/d632683dd7b4114ad314bca15554477dd762a938...11bd71901bbe5b1630ceea73d27597364c9af683)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-10-26 17:13:58 +00:00
101 changed files with 508 additions and 4909 deletions
-10
View File
@@ -3,7 +3,6 @@ profile: production
exclude_paths: exclude_paths:
# default paths # default paths
- .cache/ - .cache/
- .ansible/
- .github/ - .github/
- test/fixtures/formatting-before/ - test/fixtures/formatting-before/
- test/fixtures/formatting-prettier/ - test/fixtures/formatting-prettier/
@@ -15,17 +14,8 @@ exclude_paths:
- molecule/**/prepare.yml - molecule/**/prepare.yml
- molecule/**/reset.yml - molecule/**/reset.yml
# Scenario verify inputs are plain variable files, not playbooks. They are
# loaded as vars, not executed, so ansible-lint must not treat them as plays.
- molecule/**/verify-vars.yml
# The file was generated by galaxy ansible - don't mess with it. # The file was generated by galaxy ansible - don't mess with it.
- galaxy.yml - galaxy.yml
skip_list: skip_list:
- var-naming[no-role-prefix] - var-naming[no-role-prefix]
# The Molecule Vagrant driver injects this module at runtime. The custom create
# playbook is syntax-checked separately against the exact pinned plugin module.
mock_modules:
- vagrant
+85
View File
@@ -0,0 +1,85 @@
<!-- It's a good idea to check this post first for general troubleshooting https://github.com/techno-tim/k3s-ansible/discussions/19 -->
<!--- Provide a general summary of the issue in the Title above -->
## Expected Behavior
<!--- Tell us what should happen -->
## Current Behavior
<!--- Tell us what happens instead of the expected behavior -->
## Steps to Reproduce
<!--- reproduce this bug. Include code to reproduce, if relevant -->
1.
2.
3.
4.
## Context (variables)
<!--- please include which OS, along with the variables used when running the playbook -->
Operating system:
Hardware:
### Variables Used
`all.yml`
```yml
k3s_version: ""
ansible_user: NA
systemd_dir: ""
flannel_iface: ""
#calico_iface: ""
calico_ebpf: ""
calico_cidr: ""
calico_tag: ""
apiserver_endpoint: ""
k3s_token: "NA"
extra_server_args: ""
extra_agent_args: ""
kube_vip_tag_version: ""
kube_vip_cloud_provider_tag_version: ""
kube_vip_lb_ip_range: ""
metal_lb_speaker_tag_version: ""
metal_lb_controller_tag_version: ""
metal_lb_ip_range: ""
```
### Hosts
`host.ini`
```ini
[master]
IP.ADDRESS.ONE
IP.ADDRESS.TWO
IP.ADDRESS.THREE
[node]
IP.ADDRESS.FOUR
IP.ADDRESS.FIVE
[k3s_cluster:children]
master
node
```
## Possible Solution
<!--- Not obligatory, but suggest a fix/reason for the bug, -->
- [ ] I've checked the [General Troubleshooting Guide](https://github.com/techno-tim/k3s-ansible/discussions/20)
-102
View File
@@ -1,102 +0,0 @@
---
name: Bug report
description: Report a reproducible problem with the playbooks, roles, or generated resources.
title: "[Bug]: "
labels:
- bug
body:
- type: markdown
attributes:
value: |
Thanks for reporting a problem. Search existing issues and review the troubleshooting link first.
Remove credentials, tokens, public IP addresses, and private hostnames from all fields and logs.
- type: checkboxes
id: prerequisites
attributes:
label: Prerequisites
options:
- label: I searched existing issues and discussions for this problem.
required: true
- label: I reviewed the troubleshooting guidance linked from the issue chooser.
required: true
- label: I removed secrets and identifying infrastructure details from this report.
required: true
- type: textarea
id: summary
attributes:
label: Problem summary
description: Describe what failed and its impact.
placeholder: A concise description of the problem and affected nodes or components.
validations:
required: true
- type: textarea
id: expected
attributes:
label: Expected behavior
description: What should have happened?
validations:
required: true
- type: textarea
id: reproduction
attributes:
label: Steps to reproduce
description: Provide the smallest reliable sequence that reproduces the issue.
placeholder: |
1. Configure ...
2. Run ...
3. Observe ...
validations:
required: true
- type: input
id: revision
attributes:
label: Repository revision
description: Release, tag, branch, or commit SHA used.
placeholder: v1.36.2+k3s1+tt1 or a commit SHA
validations:
required: true
- type: input
id: ansible-version
attributes:
label: Ansible version
description: Output of `ansible --version`, shortened to version and Python details.
placeholder: ansible-core 2.18.0, Python 3.12
validations:
required: true
- type: textarea
id: environment
attributes:
label: Environment
description: Include target OS and version, architecture, node counts, platform, and network provider.
placeholder: Debian 13, amd64, 3 control nodes and 2 agents, bare metal, Cilium
validations:
required: true
- type: textarea
id: configuration
attributes:
label: Relevant sanitized configuration
description: Include only variables and inventory groups needed to reproduce the problem.
render: yaml
validations:
required: true
- type: textarea
id: logs
attributes:
label: Relevant logs or task output
description: Include the failing task and surrounding output. Redact sensitive or identifying values.
render: shell
- type: textarea
id: context
attributes:
label: Additional context
description: Add attempted fixes, suspected causes, regressions, or other useful context.
-9
View File
@@ -1,9 +0,0 @@
---
blank_issues_enabled: false
contact_links:
- name: Troubleshooting and support
url: https://github.com/timothystewart6/k3s-ansible/discussions/20
about: Review common troubleshooting guidance and ask configuration or usage questions.
- name: General discussions
url: https://github.com/timothystewart6/k3s-ansible/discussions
about: Discuss ideas and questions that are not confirmed bugs or concrete feature requests.
@@ -1,55 +0,0 @@
---
name: Feature request
description: Propose a focused improvement to supported repository behavior.
title: "[Feature]: "
labels:
- enhancement
body:
- type: markdown
attributes:
value: |
Describe the use case before proposing an implementation. Search existing issues and discussions first.
- type: checkboxes
id: prerequisites
attributes:
label: Prerequisites
options:
- label: I searched existing issues and discussions for this request.
required: true
- label: This request is about reusable project behavior, not support for one private environment.
required: true
- type: textarea
id: problem
attributes:
label: Problem or use case
description: What limitation exists, who encounters it, and why does it matter?
validations:
required: true
- type: textarea
id: proposal
attributes:
label: Proposed behavior
description: Describe the desired user-visible result. Include example variables or commands when useful.
validations:
required: true
- type: textarea
id: alternatives
attributes:
label: Alternatives considered
description: Describe workarounds or other designs and their tradeoffs.
- type: textarea
id: compatibility
attributes:
label: Compatibility and operational impact
description: Note affected operating systems, architectures, CNIs, existing clusters, or reset behavior.
- type: textarea
id: context
attributes:
label: Additional context
description: Add relevant upstream documentation, examples, or prior discussion.
+11 -42
View File
@@ -1,46 +1,15 @@
## Summary # Proposed Changes
<!--- Provide a general summary of your changes -->
<!-- Explain the problem and the resulting behavior. Keep implementation details in the sections below. -->
## Changes
-
-
- -
## Related issues ## Checklist
<!-- Use "Fixes #123" when this pull request should close an issue. Write "None" when not applicable. --> - [ ] Tested locally
- [ ] Ran `site.yml` playbook
## Testing - [ ] Ran `reset.yml` playbook
- [ ] Did not add any unnecessary changes
<!-- List exact commands, scenarios, and relevant manual checks. Do not check a box for a test that was not run. --> - [ ] Ran pre-commit install at least once before committing
- [ ] 🚀
- [ ] `pre-commit run --all-files`
- [ ] Relevant Ansible syntax checks
- [ ] Relevant focused regression tests
- [ ] Relevant Molecule scenario
- [ ] Provisioning tested against a non-production cluster
- [ ] Reset behavior tested against a non-production cluster
Not run, with reason:
## Risk and compatibility
<!-- Cover existing clusters, upgrades, networking, supported platforms, security, and rollback. Write "None" when a
category is not affected. -->
- Existing cluster or upgrade impact:
- Networking or CNI impact:
- Security impact:
- Rollback plan:
## Documentation
<!-- Identify updated docs and sample configuration, or explain why no documentation change is needed. -->
## Final checklist
- [ ] The change is focused and contains no unrelated edits.
- [ ] Tests cover new behavior or a regression, where applicable.
- [ ] User-facing variables are documented in role defaults, sample inventory, and the README.
- [ ] Logs, examples, and configuration contain no secrets or identifying infrastructure details.
- [ ] Generated files and local environment files are not included.
-6
View File
@@ -1,6 +0,0 @@
---
self-hosted-runner:
labels:
- k3s-ci
- virtualbox
- nested-virt
-6
View File
@@ -1,6 +0,0 @@
# GitHub Copilot instructions
Read and follow the repository's root-level `AGENTS.md` before proposing or making changes. It is the canonical guide
for architecture, safety, implementation, validation, and documentation expectations.
Do not duplicate repository guidance here. If instructions need to change, update `AGENTS.md`.
+23 -72
View File
@@ -1,91 +1,42 @@
#!/bin/bash #!/bin/bash
# download-boxes.sh # download-boxes.sh
# Validate the pinned Vagrant box set and download exact versions that are not # Check all molecule.yml files for required Vagrant boxes and download the ones that are not
# already present in VAGRANT_HOME. # already present on the system.
set -euo pipefail set -euo pipefail
GIT_ROOT=$(git rev-parse --show-toplevel) GIT_ROOT=$(git rev-parse --show-toplevel)
PROVIDER=virtualbox PROVIDER=virtualbox
LOCK_FILE="${VAGRANT_BOX_LOCK_FILE:-${GIT_ROOT}/.github/vagrant-boxes.lock}"
MOLECULE_YML_PATH=("${GIT_ROOT}"/molecule/*/molecule.yml) yq --version
# Extract the unique boxes referenced by the scenarios. # Define the path to the molecule.yml files
declared_boxes=$(for file in "${MOLECULE_YML_PATH[@]}"; do MOLECULE_YML_PATH="${GIT_ROOT}/molecule/*/molecule.yml"
yq -r '.platforms[].box' "$file"
# Extract and sort unique boxes from all molecule.yml files
all_boxes=$(for file in $MOLECULE_YML_PATH; do
yq eval '.platforms[].box' "$file"
done | sort -u) done | sort -u)
if [[ ! -r "$LOCK_FILE" ]]; then echo all_boxes: "$all_boxes"
printf 'Vagrant box lock file is missing or unreadable: %s\n' "$LOCK_FILE" >&2
exit 1
fi
lock_entries=$(awk ' # Read the boxes that are currently present on the system (for the current provider)
/^[[:space:]]*#/ || NF == 0 { next }
NF != 3 {
printf "Invalid lock entry on line %d: expected box, version, architecture\n", NR > "/dev/stderr"
invalid = 1
next
}
{ print $1 " " $2 " " $3 }
END { exit invalid }
' "$LOCK_FILE")
duplicate_boxes=$(printf '%s\n' "$lock_entries" | awk '{ print $1 }' | sort | uniq -d)
if [[ -n "$duplicate_boxes" ]]; then
printf 'Duplicate Vagrant box lock entries:\n%s\n' "$duplicate_boxes" >&2
exit 1
fi
locked_boxes=$(printf '%s\n' "$lock_entries" | sort)
locked_names=$(printf '%s\n' "$locked_boxes" | awk '{ print $1 }')
missing_locks=$(comm -23 <(printf '%s\n' "$declared_boxes") <(printf '%s\n' "$locked_names"))
unused_locks=$(comm -13 <(printf '%s\n' "$declared_boxes") <(printf '%s\n' "$locked_names"))
if [[ -n "$missing_locks" || -n "$unused_locks" ]]; then
if [[ -n "$missing_locks" ]]; then
printf 'Scenario boxes missing from the lock file:\n%s\n' "$missing_locks" >&2
fi
if [[ -n "$unused_locks" ]]; then
printf 'Lock entries not referenced by a scenario:\n%s\n' "$unused_locks" >&2
fi
exit 1
fi
printf 'Pinned Vagrant boxes:\n%s\n' "$locked_boxes"
# Read exact box, provider, version, and architecture tuples already present.
present_boxes=$( present_boxes=$(
vagrant box list --machine-readable | (vagrant box list |
awk -F, -v expected_provider="$PROVIDER" ' grep "${PROVIDER}" | # Filter by boxes available for the current provider
$3 == "box-name" { name = $4; next } awk '{print $1;}' | # The box name is the first word in each line
$3 == "box-provider" { provider = $4; next } sort |
$3 == "box-version" { version = $4; next } uniq) ||
$3 == "box-architecture" { echo "" # In case any of these commands fails, just use an empty list
architecture = $4
if (provider == expected_provider) {
print name " " version " " architecture
}
name = provider = version = architecture = ""
}
' |
sort -u
) )
download_boxes=$(comm -23 \ # The boxes that we need to download are the ones present in $all_boxes, but not $present_boxes.
<(printf '%s\n' "$locked_boxes") \ download_boxes=$(comm -2 -3 <(echo "${all_boxes}") <(echo "${present_boxes}"))
<(printf '%s\n' "$present_boxes"))
if [[ -n "$download_boxes" ]]; then # Actually download the necessary boxes
printf '%s\n' "$download_boxes" | while read -r box version architecture; do if [ -n "${download_boxes}" ]; then
vagrant box add \ echo "${download_boxes}" | while IFS= read -r box; do
--provider "$PROVIDER" \ vagrant box add --provider "${PROVIDER}" "${box}"
--box-version "$version" \
--architecture "$architecture" \
"$box"
done done
else
printf 'All pinned Vagrant boxes are already present.\n'
fi fi
-249
View File
@@ -1,249 +0,0 @@
#!/usr/bin/env bash
set -Eeuo pipefail
usage() {
printf '%s\n' \
'Usage: cleanup-runner-resources.sh [--snapshot|--dry-run|--apply]' \
'' \
'Discover and, with --apply, remove only VirtualBox resources referenced by' \
'repository-owned Molecule Vagrant state. The default is --dry-run.'
}
mode="dry-run"
case "${1:-}" in
"") ;;
--snapshot) mode="snapshot" ;;
--dry-run) mode="dry-run" ;;
--apply) mode="apply" ;;
--help|-h) usage; exit 0 ;;
*) usage >&2; exit 2 ;;
esac
home_dir="${HOME:?HOME must be set}"
molecule_root="${K3S_CI_MOLECULE_ROOT:-${home_dir}/.cache/molecule}"
repository_name="${K3S_CI_MOLECULE_PROJECT:-k3s-ansible}"
virtualbox_root="${K3S_CI_VIRTUALBOX_ROOT:-${home_dir}/VirtualBox VMs}"
hostonly_marker="${K3S_CI_HOSTONLY_MARKER:-${home_dir}/.cache/k3s-ci/hostonly-interfaces}"
record_hostonly() {
local marker_dir="${hostonly_marker%/*}"
local marker_tmp="${hostonly_marker}.tmp"
local hostonly_inventory
if ! hostonly_inventory="$(VBoxManage list hostonlyifs)"; then
fail_closed 'unable to inventory VirtualBox host-only interfaces'
fi
mkdir -p -- "$marker_dir"
awk -F': ' '
/^Name:/ { name=$2 }
/^IPAddress:/ { print name "|" $2 }
' <<< "$hostonly_inventory" > "$marker_tmp"
mv -- "$marker_tmp" "$hostonly_marker"
chmod 600 "$hostonly_marker"
printf 'Recorded host-only interface baseline: %s\n' "$hostonly_marker"
}
cleanup_hostonly() {
local hostonly_inventory
if [[ ! -f "$hostonly_marker" ]]; then
printf 'No host-only interface baseline found; leaving interfaces unchanged.\n'
return 0
fi
if ! hostonly_inventory="$(VBoxManage list hostonlyifs)"; then
fail_closed 'unable to inventory VirtualBox host-only interfaces'
fi
while IFS='|' read -r interface_name interface_ip; do
[[ "$interface_name" == vboxnet* ]] || continue
[[ "$interface_ip" == 192.168.30.* || "$interface_ip" == fdad:bad:ba55:* ]] || continue
if grep -Fqx "${interface_name}|${interface_ip}" "$hostonly_marker"; then
continue
fi
if [[ "$mode" == apply ]]; then
VBoxManage hostonlyif remove "$interface_name"
printf 'Removed host-only interface %s (%s)\n' "$interface_name" "$interface_ip"
else
printf 'Would remove host-only interface %s (%s)\n' "$interface_name" "$interface_ip"
fi
done < <(awk -F': ' '
/^Name:/ { name=$2 }
/^IPAddress:/ { print name "|" $2 }
' <<< "$hostonly_inventory")
}
if [[ "$mode" == snapshot ]]; then
record_hostonly
exit 0
fi
resolve_existing_dir() {
local candidate="$1"
if [[ ! -d "$candidate" ]]; then
return 1
fi
readlink -f -- "$candidate"
}
root_contains() {
local root="$1"
local path="$2"
[[ "$path" == "$root"/* ]]
}
is_supported_scenario() {
case "$1" in
default|single_node|calico|cilium|kube-vip|ipv6) return 0 ;;
*) return 1 ;;
esac
}
is_unregistered_vm_error() {
grep -Eq 'Could not find a registered machine|VBOX_E_OBJECT_NOT_FOUND'
}
fail_closed() {
printf 'cleanup refused: %s\n' "$1" >&2
exit 3
}
if [[ ! "$repository_name" =~ ^[A-Za-z0-9._-]+$ ]]; then
fail_closed 'invalid Molecule repository name'
fi
print_inventory() {
local phase="$1"
printf '%s VirtualBox inventory:\n' "$phase"
VBoxManage list vms || true
VBoxManage list hdds || true
VBoxManage list hostonlyifs || true
}
molecule_root_real="$(resolve_existing_dir "$molecule_root" || true)"
if [[ -z "$molecule_root_real" ]]; then
printf 'No Molecule root exists: %s\n' "$molecule_root"
cleanup_hostonly
exit 0
fi
print_inventory before
repository_root_real="$(resolve_existing_dir "$molecule_root_real/$repository_name" || true)"
if [[ -z "$repository_root_real" ]]; then
printf 'No repository Molecule state root exists: %s\n' "$molecule_root_real/$repository_name"
cleanup_hostonly
print_inventory after
exit 0
fi
if ! root_contains "$molecule_root_real" "$repository_root_real"; then
fail_closed "repository Molecule state root is outside Molecule root: $repository_root_real"
fi
declare -a state_files=()
while IFS= read -r -d '' state_file; do
state_files+=("$state_file")
done < <(find "$repository_root_real" -mindepth 6 -maxdepth 6 -type f \
-path '*/.vagrant/machines/*/virtualbox/id' -print0 2>/dev/null)
if ((${#state_files[@]} == 0)); then
printf 'No repository-owned Molecule Vagrant state found under %s\n' "$repository_root_real"
cleanup_hostonly
print_inventory after
exit 0
fi
virtualbox_root_real="$(resolve_existing_dir "$virtualbox_root" || true)"
declare -a vm_records=()
for state_file in "${state_files[@]}"; do
if [[ ! -f "$state_file" ]]; then
printf 'Skipping Vagrant state removed with its stale scenario directory: %s\n' "$state_file"
continue
fi
state_file_real="$(readlink -f -- "$state_file")"
state_dir="${state_file_real%/.vagrant/machines/*/virtualbox/id}"
machine_dir="${state_file_real%/virtualbox/id}"
machine_name="${machine_dir##*/}"
scenario_name="${state_dir##*/}"
if ! root_contains "$repository_root_real" "$state_dir"; then
fail_closed "state path is outside the repository Molecule root: $state_file_real"
fi
if ! is_supported_scenario "$scenario_name"; then
fail_closed "unexpected Molecule scenario: $scenario_name"
fi
if [[ "$machine_name" != control* && "$machine_name" != node* ]]; then
fail_closed "unexpected Molecule machine name: $machine_name"
fi
vm_uuid="$(tr -d '[:space:]' < "$state_file_real")"
if [[ ! "$vm_uuid" =~ ^[0-9a-fA-F-]{36}$ ]]; then
fail_closed "invalid VirtualBox UUID in $state_file_real"
fi
if ! vm_info="$(VBoxManage showvminfo "$vm_uuid" --machinereadable 2>&1)"; then
if is_unregistered_vm_error <<< "$vm_info"; then
printf 'Stale Vagrant state without a registered VM: %s (%s)\n' "$machine_name" "$vm_uuid"
if [[ "$mode" == apply ]]; then
rm -rf -- "${state_dir}/.vagrant"
printf 'Removed stale Vagrant state: %s\n' "${state_dir}/.vagrant"
fi
continue
fi
fail_closed "unable to inspect VirtualBox VM $vm_uuid: $vm_info"
fi
if [[ -z "$virtualbox_root_real" ]]; then
fail_closed "VirtualBox VM root does not exist: $virtualbox_root"
fi
cfg_file="$(awk -F= '$1 == "CfgFile" {gsub(/"/, "", $2); print $2; exit}' <<< "$vm_info")"
if [[ -z "$cfg_file" ]]; then
fail_closed "VirtualBox configuration path missing for $vm_uuid"
fi
cfg_file_real="$(readlink -f -- "$cfg_file")"
if ! root_contains "$virtualbox_root_real" "$cfg_file_real"; then
fail_closed "VM configuration is outside VirtualBox root: $cfg_file_real"
fi
while IFS= read -r disk_path; do
[[ -z "$disk_path" ]] && continue
disk_path_real="$(readlink -f -- "$disk_path" 2>/dev/null || true)"
if [[ -z "$disk_path_real" ]] || ! root_contains "$virtualbox_root_real" "$disk_path_real"; then
fail_closed "attached disk is outside VirtualBox root: $disk_path"
fi
done < <(awk -F= '$1 ~ /^(SATA|IDE|SCSI|SAS|VirtioSCSI|NVMe)-[0-9]+-[0-9]+$/ {gsub(/"/, "", $2); print $2}' <<< "$vm_info")
vm_records+=("$vm_uuid|$machine_name|$cfg_file_real")
done
if ((${#vm_records[@]} == 0)); then
printf 'No live repository-owned VirtualBox resources found\n'
cleanup_hostonly
print_inventory after
exit 0
fi
for record in "${vm_records[@]}"; do
IFS='|' read -r vm_uuid machine_name cfg_file_real <<< "$record"
if [[ "$mode" == dry-run ]]; then
printf 'Would remove VM %s (%s) config=%s\n' "$machine_name" "$vm_uuid" "$cfg_file_real"
continue
fi
vm_state="$(VBoxManage showvminfo "$vm_uuid" --machinereadable | awk -F= '$1 == "VMState" {gsub(/"/, "", $2); print $2; exit}')"
if [[ "$vm_state" != poweroff && "$vm_state" != saved ]]; then
VBoxManage controlvm "$vm_uuid" poweroff
fi
VBoxManage unregistervm "$vm_uuid" --delete
printf 'Removed VM %s (%s)\n' "$machine_name" "$vm_uuid"
done
cleanup_hostonly
print_inventory after
if [[ "$mode" == apply ]]; then
printf 'Repository-owned VM and host-only interface cleanup complete.\n'
else
printf 'Dry run complete. No resources were modified.\n'
fi
@@ -1,47 +0,0 @@
#!/usr/bin/env bash
set -Eeuo pipefail
output_dir="${1:-${RUNNER_TEMP:-/tmp}/k3s-ci-diagnostics}"
mkdir -p -- "$output_dir"
umask 077
run_capture() {
local output_file="$1"
shift
{
printf '$'
printf ' %q' "$@"
printf '\n'
"$@"
} > "$output_dir/$output_file" 2>&1 || true
}
run_capture system.txt uname -a
run_capture runner-user.txt id
run_capture memory.txt free -h
run_capture disk.txt df -h
run_capture virtualbox-version VBoxManage --version
run_capture virtualbox-vms VBoxManage list vms
run_capture virtualbox-running-vms VBoxManage list runningvms
run_capture virtualbox-disks VBoxManage list hdds
run_capture virtualbox-hostonlyifs VBoxManage list hostonlyifs
run_capture virtualbox-groups VBoxManage list groups
run_capture vagrant-status vagrant global-status
run_capture molecule-state find "${HOME}/.cache/molecule" -maxdepth 6 -type f -path '*/.vagrant/machines/*/virtualbox/id' -print
scenario_name="${K3S_CI_SCENARIO_NAME:-}"
if [[ "$scenario_name" =~ ^[A-Za-z0-9_-]+$ ]]; then
molecule_state_dir="${HOME}/.cache/molecule/k3s-ansible/${scenario_name}"
for log_name in vagrant.out vagrant.err; do
if [[ -r "${molecule_state_dir}/${log_name}" ]]; then
cp -- "${molecule_state_dir}/${log_name}" "$output_dir/${scenario_name}-${log_name}"
fi
done
fi
if [[ -r /etc/vbox/networks.conf ]]; then
cp -- /etc/vbox/networks.conf "$output_dir/virtualbox-networks.conf"
fi
printf 'Diagnostics written to %s\n' "$output_dir"
@@ -1,44 +0,0 @@
#!/usr/bin/env bash
set -Eeuo pipefail
output_dir="${1:?output directory is required}"
interval="${2:-10}"
[[ "$interval" =~ ^[1-9][0-9]*$ ]] || {
printf 'monitor interval must be a positive integer\n' >&2
exit 2
}
mkdir -p -- "$output_dir"
umask 077
free -h > "$output_dir/memory-before.txt"
df -h > "$output_dir/disk-before.txt"
vmstat -w "$interval" > "$output_dir/vmstat.txt" &
vmstat_pid=$!
iostat_pid=""
if command -v iostat >/dev/null 2>&1; then
iostat -dx "$interval" > "$output_dir/iostat.txt" &
iostat_pid=$!
else
printf 'iostat is not installed on this runner\n' > "$output_dir/iostat-unavailable.txt"
fi
cleanup() {
local rc=$?
trap - EXIT INT TERM
kill "$vmstat_pid" 2>/dev/null || true
[[ -z "$iostat_pid" ]] || kill "$iostat_pid" 2>/dev/null || true
wait "$vmstat_pid" 2>/dev/null || true
[[ -z "$iostat_pid" ]] || wait "$iostat_pid" 2>/dev/null || true
free -h > "$output_dir/memory-after.txt"
df -h > "$output_dir/disk-after.txt"
exit "$rc"
}
trap cleanup EXIT INT TERM
while :; do
sleep 3600 &
wait $!
done
@@ -1,211 +0,0 @@
#!/usr/bin/env bash
set -Eeuo pipefail
fail() {
printf 'Vagrant box master preparation refused: %s\n' "$1" >&2
exit 3
}
root_contains() {
local root="$1"
local path="$2"
[[ "$path" == "$root"/* ]]
}
read_machine_value() {
local machine_info="$1"
local key="$2"
awk -F= -v key="$key" '$1 == key {gsub(/"/, "", $2); print $2; exit}' <<< "$machine_info"
}
read_extra_data() {
local uuid="$1"
local key="$2"
local value
value="$(VBoxManage getextradata "$uuid" "$key" 2>/dev/null || true)"
[[ "$value" == 'Value: '* ]] || return 1
printf '%s\n' "${value#Value: }"
}
validate_owned_master() {
local uuid="$1"
local box="$2"
local version="$3"
local architecture="$4"
local machine_info cfg_file cfg_file_real vm_state groups disk_path disk_path_real
[[ "$uuid" =~ ^[0-9a-fA-F-]{36}$ ]] || return 1
machine_info="$(VBoxManage showvminfo "$uuid" --machinereadable 2>/dev/null)" || return 1
vm_state="$(read_machine_value "$machine_info" VMState)"
groups="$(read_machine_value "$machine_info" groups)"
cfg_file="$(read_machine_value "$machine_info" CfgFile)"
[[ "$vm_state" == poweroff ]] || return 1
[[ ",$groups," == *,/k3s-ansible/box-masters,* ]] || return 1
[[ -n "$cfg_file" ]] || return 1
cfg_file_real="$(readlink -f -- "$cfg_file" 2>/dev/null || true)"
[[ -n "$cfg_file_real" ]] || return 1
root_contains "$virtualbox_root_real" "$cfg_file_real" || return 1
[[ "$(read_extra_data "$uuid" k3s-ansible/owner || true)" == box-master ]] || return 1
[[ "$(read_extra_data "$uuid" k3s-ansible/box || true)" == "$box" ]] || return 1
[[ "$(read_extra_data "$uuid" k3s-ansible/version || true)" == "$version" ]] || return 1
[[ "$(read_extra_data "$uuid" k3s-ansible/architecture || true)" == "$architecture" ]] || return 1
while IFS= read -r disk_path; do
[[ -z "$disk_path" || "$disk_path" == none ]] && continue
disk_path_real="$(readlink -f -- "$disk_path" 2>/dev/null || true)"
[[ -n "$disk_path_real" ]] || return 1
root_contains "$virtualbox_root_real" "$disk_path_real" || return 1
done < <(awk -F= '$1 ~ /^(SATA|IDE|SCSI|SAS|VirtioSCSI|NVMe)-[0-9]+-[0-9]+$/ {
gsub(/"/, "", $2); print $2
}' <<< "$machine_info")
}
write_prewarm_vagrantfile() {
local destination="$1"
local box="$2"
local version="$3"
{
printf '%s\n' "Vagrant.configure('2') do |config|"
printf ' config.vm.box = "%s"\n' "$box"
printf ' config.vm.box_version = "%s"\n' "$version"
printf '%s\n' \
' config.vm.synced_folder ".", "/vagrant", disabled: true' \
' config.vm.hostname = "k3s-ansible-box-prewarm"' \
' config.vm.boot_timeout = 600' \
' config.vm.provider "virtualbox" do |virtualbox|' \
' virtualbox.linked_clone = true' \
' virtualbox.memory = 1024' \
' virtualbox.cpus = 2' \
' end' \
'end'
} > "$destination"
}
cleanup_prewarm() {
local rc=$?
trap - EXIT
if [[ -n "${prewarm_dir:-}" && -d "$prewarm_dir" ]]; then
VAGRANT_CWD="$prewarm_dir" vagrant destroy --force >/dev/null 2>&1 || true
rm -rf -- "$prewarm_dir"
fi
exit "$rc"
}
create_owned_master() {
local box="$1"
local version="$2"
local architecture="$3"
local master_id_file="$4"
local mapping_file="$5"
local uuid machine_info cfg_file cfg_file_real vm_state mapping_tmp
# A master_id restored from an immutable cache is only a hint. Without the
# runner-local ownership record and matching VirtualBox metadata it is not
# trusted, adopted, modified, or deleted.
rm -f -- "$master_id_file"
prewarm_dir="$(mktemp -d "${master_root}/prewarm.XXXXXX")"
trap cleanup_prewarm EXIT
write_prewarm_vagrantfile "$prewarm_dir/Vagrantfile" "$box" "$version"
printf 'Creating runner-owned linked-clone master for %s %s %s\n' \
"$box" "$version" "$architecture"
VAGRANT_CWD="$prewarm_dir" vagrant up --provider virtualbox --no-provision
[[ -r "$master_id_file" ]] || fail "Vagrant did not record a master UUID for $box"
uuid="$(tr -d '[:space:]' < "$master_id_file")"
[[ "$uuid" =~ ^[0-9a-fA-F-]{36}$ ]] || fail "Vagrant recorded an invalid master UUID for $box"
machine_info="$(VBoxManage showvminfo "$uuid" --machinereadable 2>/dev/null)" || \
fail "Vagrant master $uuid for $box is not registered"
vm_state="$(read_machine_value "$machine_info" VMState)"
cfg_file="$(read_machine_value "$machine_info" CfgFile)"
cfg_file_real="$(readlink -f -- "$cfg_file" 2>/dev/null || true)"
[[ "$vm_state" == poweroff ]] || fail "new Vagrant master $uuid is not powered off"
if [[ -z "$cfg_file_real" ]] || ! root_contains "$virtualbox_root_real" "$cfg_file_real"; then
fail "new Vagrant master $uuid is outside the runner VirtualBox root"
fi
VAGRANT_CWD="$prewarm_dir" vagrant destroy --force
VBoxManage modifyvm "$uuid" --groups /k3s-ansible/box-masters
VBoxManage setextradata "$uuid" k3s-ansible/owner box-master
VBoxManage setextradata "$uuid" k3s-ansible/box "$box"
VBoxManage setextradata "$uuid" k3s-ansible/version "$version"
VBoxManage setextradata "$uuid" k3s-ansible/architecture "$architecture"
validate_owned_master "$uuid" "$box" "$version" "$architecture" || \
fail "new Vagrant master $uuid failed ownership validation"
rm -rf -- "$prewarm_dir"
prewarm_dir=""
trap - EXIT
mapping_tmp="${mapping_file}.tmp"
printf '%s\n' "$uuid" > "$mapping_tmp"
chmod 600 "$mapping_tmp"
mv -- "$mapping_tmp" "$mapping_file"
printf '%s\n' "$uuid" > "$master_id_file"
chmod 600 "$master_id_file"
printf 'Created and recorded owned master %s for %s\n' "$uuid" "$box"
}
repository_root="${K3S_CI_REPOSITORY_ROOT:-$(git rev-parse --show-toplevel)}"
lock_file="${VAGRANT_BOX_LOCK_FILE:-${repository_root}/.github/vagrant-boxes.lock}"
vagrant_home="${VAGRANT_HOME:?VAGRANT_HOME must be set}"
master_root="${K3S_CI_VAGRANT_MASTER_ROOT:-${HOME:?HOME must be set}/.cache/k3s-ci/vagrant-masters}"
virtualbox_root="${K3S_CI_VIRTUALBOX_ROOT:-${HOME}/VirtualBox VMs}"
[[ -r "$lock_file" ]] || fail "box lock file is missing or unreadable: $lock_file"
[[ -d "$vagrant_home/boxes" ]] || fail "Vagrant box directory is missing: $vagrant_home/boxes"
[[ -d "$virtualbox_root" ]] || fail "VirtualBox root is missing: $virtualbox_root"
box_root_real="$(readlink -f -- "$vagrant_home/boxes")"
virtualbox_root_real="$(readlink -f -- "$virtualbox_root")"
mkdir -p -- "$master_root"
chmod 700 "$master_root"
exec 9> "${master_root}/prepare.lock"
flock 9
lock_entries="$(awk '
/^[[:space:]]*#/ || NF == 0 { next }
NF != 3 { invalid = 1; next }
{ print $1 " " $2 " " $3 }
END { exit invalid }
' "$lock_file")" || fail 'invalid Vagrant box lock entry'
[[ -n "$lock_entries" ]] || fail 'Vagrant box lock is empty'
while read -r box version architecture; do
[[ "$box" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || fail "invalid box name: $box"
[[ "$version" =~ ^[A-Za-z0-9._-]+$ ]] || fail "invalid box version: $version"
[[ "$architecture" =~ ^[A-Za-z0-9._-]+$ ]] || fail "invalid box architecture: $architecture"
box_slug="${box//\//-VAGRANTSLASH-}"
record_slug="${box//\//_}-${version}-${architecture}"
box_dir="${vagrant_home}/boxes/${box_slug}/${version}/${architecture}/virtualbox"
[[ -d "$box_dir" ]] || fail "pinned box is not installed: $box $version $architecture"
box_dir_real="$(readlink -f -- "$box_dir")"
root_contains "$box_root_real" "$box_dir_real" || fail "box directory is outside VAGRANT_HOME: $box_dir_real"
master_id_file="${box_dir_real}/master_id"
mapping_file="${master_root}/${record_slug}.uuid"
uuid=""
if [[ -r "$mapping_file" ]]; then
uuid="$(tr -d '[:space:]' < "$mapping_file")"
fi
if [[ -n "$uuid" ]] && validate_owned_master "$uuid" "$box" "$version" "$architecture"; then
printf '%s\n' "$uuid" > "$master_id_file"
chmod 600 "$master_id_file"
printf 'Reusing owned master %s for %s %s %s\n' "$uuid" "$box" "$version" "$architecture"
continue
fi
if [[ -e "$mapping_file" ]]; then
printf 'Owned master record is stale for %s; rebuilding without deleting any VM or disk.\n' "$box"
fi
create_owned_master "$box" "$version" "$architecture" "$master_id_file" "$mapping_file"
done <<< "$lock_entries"
printf 'All pinned Vagrant box masters are ready.\n'
-134
View File
@@ -1,134 +0,0 @@
#!/usr/bin/env python3
"""Render the Cilium BGP CRD template and assert it uses the v2 API.
This is a manifest-only regression test used where no real BGP peer is
available. It renders roles/k3s_server_post/templates/cilium.crs.j2 with
zero, one, and multiple neighbors, then checks that the output:
- never contains CiliumBGPPeeringPolicy or cilium.io/v2alpha1
- emits the Cilium v2 BGP resources
- emits deterministic DNS-safe peer and instance names
- advertises Pod CIDRs only when cilium_exportPodCIDR is true
"""
from __future__ import print_function
import os
import subprocess
from jinja2 import Environment, FileSystemLoader, StrictUndefined
def repo_root():
return subprocess.check_output(
["git", "rev-parse", "--show-toplevel"], text=True
).strip()
def fail(message):
raise SystemExit("Cilium BGP manifest test failed: " + message)
def render(env, extra_vars):
base_vars = {
"cilium_bgp_my_asn": "64513",
"cilium_bgp_peer_asn": "64512",
"cilium_bgp_peer_address": "192.168.30.1",
"cilium_exportPodCIDR": True,
"cilium_bgp_lb_cidr": "192.168.31.0/24",
}
base_vars.update(extra_vars)
template = env.get_template("cilium.crs.j2")
return template.render(**base_vars)
def check_common(output):
if "cilium.io/v2alpha1" in output:
fail("rendered output still contains cilium.io/v2alpha1")
if "kind: CiliumBGPPeeringPolicy" in output:
fail("rendered output still contains CiliumBGPPeeringPolicy")
for kind in (
"CiliumBGPPeerConfig",
"CiliumBGPClusterConfig",
"CiliumBGPAdvertisement",
"CiliumLoadBalancerIPPool",
):
if ("kind: " + kind) not in output:
fail("rendered output is missing kind: " + kind)
def main():
root = repo_root()
template_dir = os.path.join(
root, "roles", "k3s_server_post", "templates"
)
env = Environment(
loader=FileSystemLoader(template_dir), undefined=StrictUndefined
)
# Zero neighbors -> fall back to the single default peer.
output = render(env, {"_cilium_bgp_neighbors": []})
check_common(output)
if "peer-64512-1" not in output:
fail("default single peer name was not rendered")
if "peerAddress: 192.168.30.1" not in output:
fail("default peer address was not rendered")
if 'advertisementType: "PodCIDR"' not in output:
fail("PodCIDR advertisement missing when exportPodCIDR is true")
# One neighbor via the merged list.
output = render(
env,
{"_cilium_bgp_neighbors": [{"peer_address": "10.0.0.1", "peer_asn": "65001"}]},
)
check_common(output)
if "peer-65001-1" not in output:
fail("single merged peer name was not rendered")
if "peerAddress: 10.0.0.1" not in output:
fail("single merged peer address was not rendered")
# Multiple neighbors.
output = render(
env,
{
"_cilium_bgp_neighbors": [
{"peer_address": "10.0.0.1", "peer_asn": "65001"},
{"peer_address": "10.0.0.2", "peer_asn": "65002"},
]
},
)
check_common(output)
if "peer-65001-1" not in output or "peer-65002-2" not in output:
fail("multiple merged peer names were not rendered")
if "peerAddress: 10.0.0.2" not in output:
fail("second merged peer address was not rendered")
# exportPodCIDR false -> no PodCIDR advertisement, service remains.
output = render(
env, {"_cilium_bgp_neighbors": [], "cilium_exportPodCIDR": False}
)
check_common(output)
if 'advertisementType: "PodCIDR"' in output:
fail("PodCIDR advertisement present when exportPodCIDR is false")
if 'advertisementType: "Service"' not in output:
fail("Service advertisement missing when exportPodCIDR is false")
# Load balancer pools: CIDR and start/stop forms.
output = render(env, {"_cilium_bgp_neighbors": []})
if "cidr: 192.168.31.0/24" not in output:
fail("CIDR load balancer pool was not rendered")
output = render(
env,
{
"_cilium_bgp_neighbors": [],
"cilium_bgp_lb_cidr": "192.168.31.80-192.168.31.90",
},
)
check_common(output)
if "start: 192.168.31.80" not in output or "stop: 192.168.31.90" not in output:
fail("start/stop load balancer pool was not rendered")
print("Cilium BGP manifest regression test passed")
if __name__ == "__main__":
main()
-120
View File
@@ -1,120 +0,0 @@
#!/usr/bin/env python3
"""Regression test for the Cilium Envoy toggle.
The `cilium_envoy` variable lets users enable or disable the Cilium Envoy
proxy. The Install/upgrade Cilium task in
roles/k3s_server_post/tasks/cilium.yml passes the value through to Helm as
`envoy.enabled`. This test:
- loads the real "Install Cilium" task and confirms the install/upgrade
command actually contains the `envoy.enabled` Helm value,
- renders the conditional that computes the Helm value and confirms it
produces `true` when cilium_envoy is enabled and `false` when disabled,
- confirms the task stays forward/backward compatible (no raw `true` /
`false` hardcoded in place of the conditional).
"""
from __future__ import print_function
import os
import re
import subprocess
import yaml
from jinja2 import Environment
ENVOY_EXPRESSION = '{{ "true" if cilium_envoy else "false" }}'
def repo_root():
return subprocess.check_output(
["git", "rev-parse", "--show-toplevel"], text=True
).strip()
def fail(message):
raise SystemExit("Cilium Envoy toggle test failed: " + message)
def extract_install_command(path):
"""Return the command string for the 'Install Cilium' task.
Walks both top-level tasks and tasks nested inside a `block`/`always`/
`rescue` list, since the Cilium deploy steps are grouped under the
'Prepare Cilium CLI on first master and deploy CNI' block.
"""
with open(path, encoding="utf-8") as handle:
doc = yaml.safe_load(handle)
def find_command(tasks):
for task in tasks:
if not isinstance(task, dict):
continue
if task.get("name") == "Install Cilium":
command = task.get("ansible.builtin.command")
if command is None:
raise SystemExit(
"Cilium Envoy toggle test failed: "
"'Install Cilium' task has no ansible.builtin.command"
)
return command
# Recurse into block/always/rescue sub-lists.
for key in ("block", "always", "rescue"):
nested = task.get(key)
if isinstance(nested, list):
found = find_command(nested)
if found is not None:
return found
return None
command = find_command(doc)
if command is None:
raise SystemExit(
"Cilium Envoy toggle test failed: could not find 'Install Cilium' task"
)
return command
def assert_envoy_in_command(command):
if "envoy.enabled" not in command:
fail("install command is missing --helm-set envoy.enabled")
if ENVOY_EXPRESSION not in command:
fail(
"install command does not use the cilium_envoy conditional: "
"expected {0!r}".format(ENVOY_EXPRESSION)
)
# The conditional must be a WYSIWYG helm-set value, not a pre-rendered
# true/false literal (which would ignore the cilium_envoy variable).
if re.search(r"--helm-set envoy\.enabled=true(?:$|\s)", command):
fail("install command hardcodes envoy.enabled=true")
if re.search(r"--helm-set envoy\.enabled=false(?:$|\s)", command):
fail("install command hardcodes envoy.enabled=false")
def assert_render():
env = Environment()
def render_for(value):
template = env.from_string(ENVOY_EXPRESSION)
return template.render(cilium_envoy=value)
if render_for(True) != "true":
fail("envoy conditional did not render 'true' when enabled")
if render_for(False) != "false":
fail("envoy conditional did not render 'false' when disabled")
def main():
root = repo_root()
cilium_tasks = os.path.join(
root, "roles", "k3s_server_post", "tasks", "cilium.yml"
)
command = extract_install_command(cilium_tasks)
assert_envoy_in_command(command)
assert_render()
print("Cilium Envoy toggle regression test passed")
if __name__ == "__main__":
main()
@@ -1,129 +0,0 @@
#!/usr/bin/env bash
# shellcheck disable=SC2016
set -Eeuo pipefail
repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
test_root="$(mktemp -d "${TMPDIR:-/tmp}/k3s-ci-cleanup-test.XXXXXX")"
trap 'rm -rf -- "$test_root"' EXIT
molecule_root="$test_root/molecule"
virtualbox_root="$test_root/VirtualBox VMs"
fake_bin="$test_root/bin"
mkdir -p -- "$molecule_root/k3s-ansible/single_node/.vagrant/machines/control1/virtualbox" \
"$molecule_root/k3s-ansible/single_node/.vagrant/machines/control2/virtualbox" \
"$virtualbox_root/control1" "$virtualbox_root/unmarked" "$fake_bin"
printf '%s\n' '11111111-1111-1111-1111-111111111111' \
> "$molecule_root/k3s-ansible/single_node/.vagrant/machines/control1/virtualbox/id"
printf '%s\n' '22222222-2222-2222-2222-222222222222' \
> "$molecule_root/k3s-ansible/single_node/.vagrant/machines/control2/virtualbox/id"
touch "$virtualbox_root/control1/control1.vbox" "$virtualbox_root/control1/disk.vdi" \
"$virtualbox_root/unmarked/unmarked.vbox"
printf '%s\n' \
'#!/usr/bin/env bash' \
'set -Eeuo pipefail' \
'case "${1:-}" in' \
' list)' \
' if [[ "${2:-}" == hostonlyifs && "${FAKE_HOSTONLY_FAIL:-false}" == true ]]; then exit 1; fi' \
' exit 0' \
' ;;' \
' showvminfo)' \
' if [[ "${FAKE_VM_MODE:-normal}" == missing ]]; then' \
' printf '\''VBoxManage: error: Could not find a registered machine named "missing"\n'\'' >&2' \
' exit 1' \
' fi' \
' if [[ "${FAKE_VM_MODE:-normal}" == fault ]]; then' \
' printf '\''VBoxManage: error: VirtualBox service is unavailable\n'\'' >&2' \
' exit 1' \
' fi' \
' printf '\''CfgFile="%s"\n'\'' "${FAKE_VBOX_ROOT}/control1/control1.vbox"' \
' printf '\''SATA-0-0="%s"\n'\'' "${FAKE_VBOX_ROOT}/control1/disk.vdi"' \
' printf '\''VMState="running"\n'\''' \
' ;;' \
' controlvm) printf '\''controlvm %s\n'\'' "$*" >> "${FAKE_LOG}" ;;' \
' unregistervm)' \
' printf '\''unregistervm %s\n'\'' "$*" >> "${FAKE_LOG}"' \
' rm -f -- "${FAKE_VBOX_ROOT}/control1/control1.vbox" "${FAKE_VBOX_ROOT}/control1/disk.vdi"' \
' ;;' \
' *) : ;;' \
'esac' > "$fake_bin/VBoxManage"
chmod 700 "$fake_bin/VBoxManage"
output="$test_root/output.txt"
if PATH="$fake_bin:$PATH" \
HOME="$test_root/home" \
K3S_CI_MOLECULE_ROOT="$molecule_root" \
K3S_CI_MOLECULE_PROJECT=k3s-ansible \
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
FAKE_VM_MODE=fault \
FAKE_VBOX_ROOT="$virtualbox_root" \
FAKE_LOG="$test_root/vbox.log" \
bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --apply > "$output" 2>&1; then
printf '%s\n' 'cleanup unexpectedly accepted a VirtualBox inspection failure' >&2
exit 1
fi
grep -Fq 'cleanup refused: unable to inspect VirtualBox VM' "$output"
[[ -f "$molecule_root/k3s-ansible/single_node/.vagrant/machines/control1/virtualbox/id" ]]
printf '%s\n' 'vboxnet0|192.168.30.1' > "$test_root/hostonly-baseline"
if PATH="$fake_bin:$PATH" \
HOME="$test_root/home" \
K3S_CI_MOLECULE_ROOT="$molecule_root" \
K3S_CI_MOLECULE_PROJECT=k3s-ansible \
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
FAKE_HOSTONLY_FAIL=true \
FAKE_VBOX_ROOT="$virtualbox_root" \
FAKE_LOG="$test_root/vbox.log" \
bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --dry-run > "$output" 2>&1; then
printf '%s\n' 'cleanup unexpectedly accepted a host-only inventory failure' >&2
exit 1
fi
grep -Fq 'cleanup refused: unable to inventory VirtualBox host-only interfaces' "$output"
PATH="$fake_bin:$PATH" \
HOME="$test_root/home" \
K3S_CI_MOLECULE_ROOT="$molecule_root" \
K3S_CI_MOLECULE_PROJECT=k3s-ansible \
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
FAKE_VBOX_ROOT="$virtualbox_root" \
FAKE_LOG="$test_root/vbox.log" \
bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --dry-run > "$output"
grep -Fq 'Would remove VM control1 (11111111-1111-1111-1111-111111111111)' "$output"
[[ ! -e "$test_root/vbox.log" ]]
PATH="$fake_bin:$PATH" \
HOME="$test_root/home" \
K3S_CI_MOLECULE_ROOT="$molecule_root" \
K3S_CI_MOLECULE_PROJECT=k3s-ansible \
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
FAKE_VBOX_ROOT="$virtualbox_root" \
FAKE_LOG="$test_root/vbox.log" \
bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --apply > "$output"
grep -Fq 'controlvm 11111111-1111-1111-1111-111111111111 poweroff' "$test_root/vbox.log"
grep -Fq 'unregistervm unregistervm 11111111-1111-1111-1111-111111111111 --delete' "$test_root/vbox.log"
[[ ! -e "$virtualbox_root/control1/control1.vbox" ]]
[[ -e "$virtualbox_root/unmarked/unmarked.vbox" ]]
PATH="$fake_bin:$PATH" \
HOME="$test_root/home" \
K3S_CI_MOLECULE_ROOT="$molecule_root" \
K3S_CI_MOLECULE_PROJECT=k3s-ansible \
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
FAKE_VM_MODE=missing \
FAKE_VBOX_ROOT="$virtualbox_root" \
FAKE_LOG="$test_root/vbox.log" \
bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --apply > "$output"
grep -Fq 'Stale Vagrant state without a registered VM' "$output"
[[ ! -d "$molecule_root/k3s-ansible/single_node/.vagrant" ]]
printf 'cleanup-runner-resources fixture test passed\n'
-90
View File
@@ -1,90 +0,0 @@
#!/usr/bin/env python3
"""Assert the sample inventory resolves the flannel interface per host.
flannel_iface defaults to the host's default IPv4 interface rather than a
hardcoded eth0. This test extracts the flannel_iface expression from the
sample inventory and proves that a host whose primary interface is not named
eth0 (e.g. enp1s0, ens3) resolves the interface from ansible facts.
"""
from __future__ import print_function
import os
import re
import subprocess
from jinja2 import Environment, StrictUndefined
def repo_root():
return subprocess.check_output(
["git", "rev-parse", "--show-toplevel"], text=True
).strip()
def fail(message):
raise SystemExit("default-interface test failed: " + message)
class FakeAnsibleFacts(object):
"""Stand-in for the per-host ``ansible_facts`` dict."""
def __init__(self, default_iface, iface_ip):
self._default = {"interface": default_iface, "address": iface_ip}
self._ifaces = {
default_iface: {"ipv4": {"address": iface_ip}},
}
@property
def default_ipv4(self):
return self._default
def __getitem__(self, key):
return self._ifaces[key]
def read_all_yml(root):
path = os.path.join(root, "inventory", "sample", "group_vars", "all.yml")
with open(path, "r") as handle:
return handle.read()
def extract_value(content, key):
# Match a quoted value assigned to the key, e.g. flannel_iface: "...".
match = re.search(r"^%s:\s*\"(.+)\"\s*$" % re.escape(key), content, re.M)
if not match:
fail("could not find %s in the sample inventory" % key)
return match.group(1)
def resolve(env, expression, facts):
template = env.from_string(expression)
return template.render(ansible_facts=facts)
def main():
root = repo_root()
content = read_all_yml(root)
env = Environment(undefined=StrictUndefined)
flannel_expr = extract_value(content, "flannel_iface")
if "default_ipv4.interface" not in flannel_expr:
fail("flannel_iface no longer defaults from ansible facts")
# A host whose primary interface is enp1s0 (the core #621 scenario).
facts = FakeAnsibleFacts("enp1s0", "192.168.30.11")
resolved = resolve(env, flannel_expr, facts)
if resolved != "enp1s0":
fail("flannel_iface resolved to %r, expected enp1s0" % resolved)
# A different host with a different interface must resolve independently.
facts2 = FakeAnsibleFacts("ens3", "192.168.30.12")
resolved2 = resolve(env, flannel_expr, facts2)
if resolved2 != "ens3":
fail("flannel_iface resolved to %r, expected ens3" % resolved2)
print("default-interface regression test passed")
if __name__ == "__main__":
main()
-37
View File
@@ -1,37 +0,0 @@
#!/usr/bin/env bash
set -Eeuo pipefail
repo_root="$(git rev-parse --show-toplevel)"
prereq_defaults="$repo_root/roles/prereq/defaults/main.yml"
prereq_tasks="$repo_root/roles/prereq/tasks/main.yml"
# #670: k3s recommends swap be disabled on all nodes. The prereq role must expose
# a disable_swap toggle (defaulting to true) that turns swap off now and comments
# out the /etc/fstab swap entries so swap stays off across reboots.
grep -Eq -- '^disable_swap: true' "$prereq_defaults" || {
printf 'prereq defaults are missing disable_swap: true\n' >&2
exit 1
}
grep -Fq -- 'Disable swap on all cluster nodes' "$prereq_tasks" || {
printf 'prereq tasks are missing the swap-disable block\n' >&2
exit 1
}
grep -Fq -- 'swapoff -a' "$prereq_tasks" || {
printf 'swap-disable block does not run swapoff -a\n' >&2
exit 1
}
grep -Fq -- '/etc/fstab' "$prereq_tasks" || {
printf 'swap-disable block does not comment out /etc/fstab swap entries\n' >&2
exit 1
}
if ! grep -Eq -- 'when: disable_swap' "$prereq_tasks"; then
printf 'swap-disable block is not gated on the disable_swap toggle\n' >&2
exit 1
fi
printf 'Swap disable regression test passed\n'
-4
View File
@@ -1,4 +0,0 @@
#!/usr/bin/env bash
set -Eeuo pipefail
[[ "${1:-}" =~ ^[0-9]+$ ]]
@@ -1,30 +0,0 @@
#!/usr/bin/env bash
set -Eeuo pipefail
printf '%s\n' "$*" >> "$MOCK_VAGRANT_LOG"
case "${1:-}" in
up)
if [[ -n "${MOCK_VAGRANTFILE_CAPTURE:-}" ]]; then
cp -- "$VAGRANT_CWD/Vagrantfile" "$MOCK_VAGRANTFILE_CAPTURE"
fi
counter_file="$MOCK_VBOX_STATE/counter"
counter=0
[[ ! -r "$counter_file" ]] || counter="$(cat "$counter_file")"
counter=$((counter + 1))
printf '%s\n' "$counter" > "$counter_file"
uuid="00000000-0000-4000-8000-$(printf '%012d' "$counter")"
vm_dir="$MOCK_VBOX_ROOT/master-$counter"
mkdir -p -- "$vm_dir"
: > "$vm_dir/master.vbox"
: > "$vm_dir/master.vdi"
printf '%s\n' \
'VMState="poweroff"' \
'groups="/"' \
"CfgFile=\"$vm_dir/master.vbox\"" \
"SATA-0-0=\"$vm_dir/master.vdi\"" > "$MOCK_VBOX_STATE/vm-$uuid"
printf '%s\n' "$uuid" > "$MOCK_BOX_DIR/master_id"
;;
destroy) ;;
*) exit 2 ;;
esac
@@ -1,40 +0,0 @@
#!/usr/bin/env bash
set -Eeuo pipefail
printf '%s\n' "$*" >> "$MOCK_VBOX_LOG"
command_name="${1:-}"
uuid="${2:-}"
case "$command_name" in
showvminfo)
[[ -r "$MOCK_VBOX_STATE/vm-$uuid" ]] || exit 1
cat "$MOCK_VBOX_STATE/vm-$uuid"
;;
getextradata)
key_slug="${3//\//_}"
if [[ ! -r "$MOCK_VBOX_STATE/extra-$uuid-$key_slug" ]]; then
printf '%s\n' 'No value set!'
exit 0
fi
printf 'Value: '
cat "$MOCK_VBOX_STATE/extra-$uuid-$key_slug"
;;
modifyvm)
[[ "${3:-}" == --groups ]]
awk -v groups="${4:-}" '
$1 !~ /^groups=/ { print }
END { printf "groups=\"%s\"\n", groups }
' "$MOCK_VBOX_STATE/vm-$uuid" > "$MOCK_VBOX_STATE/vm-$uuid.tmp"
mv "$MOCK_VBOX_STATE/vm-$uuid.tmp" "$MOCK_VBOX_STATE/vm-$uuid"
;;
setextradata)
key_slug="${3//\//_}"
printf '%s\n' "${4:-}" > "$MOCK_VBOX_STATE/extra-$uuid-$key_slug"
;;
unregistervm|closemedium)
printf '%s\n' 'destructive VirtualBox command invoked' >&2
exit 99
;;
*) exit 2 ;;
esac
@@ -1,35 +0,0 @@
#!/usr/bin/env bash
set -Eeuo pipefail
repo_root="$(git rev-parse --show-toplevel)"
main_tasks="$repo_root/roles/k3s_server/tasks/main.yml"
join_tasks="$repo_root/roles/k3s_server/tasks/join_master.yml"
for task_file in "$main_tasks" "$join_tasks"; do
for property in \
'Delegate=yes' \
'TasksMax=infinity' \
'KillMode=process' \
'LimitNOFILE=1048576' \
'LimitNPROC=infinity' \
'LimitCORE=infinity'; do
grep -Fq -- "$property" "$task_file" || {
printf '%s is missing transient K3s property %s\n' "$task_file" "$property" >&2
exit 1
}
done
done
if grep -Fq -- "node-role.kubernetes.io/master=true' -o=jsonpath" "$main_tasks"; then
printf 'control-plane registration still depends on the optional legacy master role key\n' >&2
exit 1
fi
grep -Fq -- "map('extract', hostvars, 'ansible_hostname')" "$main_tasks"
grep -Fq -- 'difference(nodes.stdout.split())' "$main_tasks"
grep -Fq -- 'crd/addons.k3s.cattle.io' "$main_tasks"
grep -Fq -- 'crd/helmcharts.helm.cattle.io' "$main_tasks"
grep -Fq -- 'crd/helmchartconfigs.helm.cattle.io' "$main_tasks"
printf 'K3s transient bootstrap regression test passed\n'
@@ -1,166 +0,0 @@
#!/usr/bin/env python3
"""Regression test for the kube-vip deploy conditions.
The control-plane VIP (roles/k3s_server/tasks/vip.yml) and the kube-vip
service load balancer (roles/k3s_server/tasks/kube-vip.yml) are both included
from roles/k3s_server/tasks/main.yml. Before this switch existed the VIP
include had no gate and always ran, so a user who wanted no kube-vip (single
node or external LB) could not opt out.
This test loads the real `when` expressions from the k3s_server task file and
evaluates them against representative variable sets, asserting that:
- the control-plane VIP is deployed only when kube_vip_enabled is true;
- the service load balancer is deployed only when kube_vip_enabled is true
AND kube_vip_lb_ip_range is defined.
"""
from __future__ import print_function
import os
import re
import subprocess
import yaml
from jinja2 import Environment
VIP_WHEN = "kube_vip_enabled"
KUBE_VIP_WHEN = "kube_vip_enabled and kube_vip_lb_ip_range is defined"
def repo_root():
return subprocess.check_output(
["git", "rev-parse", "--show-toplevel"], text=True
).strip()
def fail(message):
raise SystemExit("kube-vip deploy condition test failed: " + message)
def extract_when(path, task_name):
"""Return the `when:` expression string for the named task."""
with open(path, encoding="utf-8") as handle:
doc = yaml.safe_load(handle)
for task in doc:
if task.get("name") == task_name:
when = task.get("when")
return (when or "").strip()
return None
def evaluate(when, variables):
"""Evaluate a `when` expression against variables using Jinja2."""
env = Environment()
def fake_bool(value):
# Minimal stand-in for Ansible's truthiness filter used by `| bool`.
if isinstance(value, bool):
return value
if value is None:
return False
return str(value).lower() in ("1", "true", "yes", "on")
env.filters["bool"] = fake_bool
template = env.from_string("{{ " + when + " }}")
rendered = template.render(**variables)
# The expression renders to the literal strings "True"/"False".
if rendered == "True":
return True
if rendered == "False":
return False
fail("condition did not render to a boolean: {0!r}".format(rendered))
def assert_deployment(when, variables, expected, label):
result = evaluate(when, variables)
verdict = "deploy" if result else "skip"
expected_verdict = "deploy" if expected else "skip"
if result != expected:
fail(
"{0}: expected to {1} but the condition chose to {2} "
"(vars: {3})".format(label, expected_verdict, verdict, variables)
)
def scenarios():
"""Yield (variables, expected_vip, expected_service_lb, label) pairs."""
yield (
# Default inventory: kube-vip enabled, no service LB range.
{
"kube_vip_enabled": True,
},
True,
False,
"default: kube-vip VIP only",
)
yield (
# kube-vip owns the service LB range too.
{
"kube_vip_enabled": True,
"kube_vip_lb_ip_range": "192.168.30.80-192.168.30.90",
},
True,
True,
"kube-vip VIP and service LB",
)
yield (
# Explicitly disabled, no LB range.
{
"kube_vip_enabled": False,
},
False,
False,
"kube_vip_enabled: false",
)
yield (
# Explicitly disabled even when the LB range is present.
{
"kube_vip_enabled": False,
"kube_vip_lb_ip_range": "192.168.30.80-192.168.30.90",
},
False,
False,
"kube_vip_enabled: false despite LB range",
)
def main():
root = repo_root()
server_tasks = os.path.join(root, "roles", "k3s_server", "tasks", "main.yml")
vip_when = extract_when(server_tasks, "Deploy vip manifest")
kube_vip_when = extract_when(server_tasks, "Deploy kube-vip manifest")
if vip_when is None:
fail("could not find 'Deploy vip manifest' when condition")
if kube_vip_when is None:
fail("could not find 'Deploy kube-vip manifest' when condition")
if vip_when != VIP_WHEN:
fail(
"k3s_server/tasks/main.yml 'Deploy vip manifest' when condition "
"changed unexpectedly:\n"
" expected: {0}\n got: {1}".format(VIP_WHEN, vip_when)
)
if kube_vip_when != KUBE_VIP_WHEN:
fail(
"k3s_server/tasks/main.yml 'Deploy kube-vip manifest' when "
"condition changed unexpectedly:\n"
" expected: {0}\n got: {1}".format(KUBE_VIP_WHEN, kube_vip_when)
)
for variables, expected_vip, expected_service_lb, label in scenarios():
assert_deployment(vip_when, variables, expected_vip, "vip " + label)
assert_deployment(
kube_vip_when,
variables,
expected_service_lb,
"service_lb " + label,
)
print("kube-vip deploy condition regression test passed for all scenarios")
if __name__ == "__main__":
main()
-162
View File
@@ -1,162 +0,0 @@
#!/usr/bin/env python3
"""Render the kube-vip DaemonSet template and assert env key correctness.
kube-vip v1.2.3 reads `bgp_peers` and `vip_subnet`; it ignores the older
`bgppeers` and `vip_cidr` names. This test proves the rendered manifest uses
the keys the target image actually parses.
"""
from __future__ import print_function
import os
import subprocess
from jinja2 import Environment, FileSystemLoader, StrictUndefined
def repo_root():
return subprocess.check_output(
["git", "rev-parse", "--show-toplevel"], text=True
).strip()
def fail(message):
raise SystemExit("kube-vip manifest test failed: " + message)
def fake_ipsubnet(value):
# ansible.utils.ipsubnet -> network of the address as x.y.z.0/24
parts = value.split(".")
return ".".join(parts[:3]) + ".0/24"
def fake_ipaddr(_value, expr=None):
# ansible.utils.ipaddr('prefix') -> prefix length
return "24"
def fake_bool(value):
# Minimal stand-in for Ansible's truthiness filter used by the template.
if isinstance(value, bool):
return value
return str(value).lower() in ("1", "true", "yes", "on")
def fake_map(seq, *args, **kwargs):
# Minimal stand-in for Ansible's map() filter in the two forms used by the
# template: map(attribute='x') on a list of dicts, and map('join', sep) on
# a list of sequences.
if "attribute" in kwargs:
return [item[kwargs["attribute"]] for item in seq]
if kwargs:
# e.g. map(default='x') not used here; ignore unknown kwargs.
return list(seq)
if args:
filter_name = args[0]
sep = args[1] if len(args) > 1 else ""
if filter_name == "join":
return [sep.join(str(x) for x in item) for item in seq]
return list(seq)
def fake_zip(*seqs):
return list(zip(*seqs))
def render(env, extra_vars):
base_vars = {
"apiserver_endpoint": "192.168.30.222",
"kube_vip_iface": "",
"kube_vip_arp": True,
"kube_vip_bgp": True,
"kube_vip_bgp_routerid": "127.0.0.1",
"_kube_vip_bgp_peers": [
{"peer_address": "192.168.30.1", "peer_asn": "64512"},
{"peer_address": "192.168.30.2", "peer_asn": "64513"},
],
"kube_vip_tag_version": "v1.2.3",
}
base_vars.update(extra_vars)
template = env.get_template("vip.yaml.j2")
return template.render(**base_vars)
def main():
root = repo_root()
template_dir = os.path.join(root, "roles", "k3s_server", "templates")
env = Environment(
loader=FileSystemLoader(template_dir), undefined=StrictUndefined
)
env.filters["ansible.utils.ipsubnet"] = fake_ipsubnet
env.filters["ansible.utils.ipaddr"] = fake_ipaddr
env.filters["bool"] = fake_bool
env.filters["map"] = fake_map
env.filters["zip"] = fake_zip
# Multi-peer BGP armed: must emit bgp_peers, never bgppeers.
output = render(env, {})
if "name: bgp_peers" not in output:
fail("rendered manifest is missing bgp_peers")
if "name: bgppeers" in output:
fail("rendered manifest still uses the ignored bgppeers key")
if "name: vip_subnet" not in output:
fail("rendered manifest is missing vip_subnet")
if "name: vip_cidr" in output:
fail("rendered manifest still uses the ignored vip_cidr key")
if "192.168.30.1:64512,192.168.30.2:64513" not in output:
fail("bgp_peers value is not comma-separated address:ASN entries")
if "ghcr.io/kube-vip/kube-vip:v1.2.3" not in output:
fail("kube-vip image tag is not v1.2.3")
# BGP enabled with no merged peers: single-peer fallback vars, no bgp_peers.
output = render(
env,
{
"_kube_vip_bgp_peers": [],
"kube_vip_bgp_as": "64513",
"kube_vip_bgp_peeraddress": "192.168.30.1",
"kube_vip_bgp_peeras": "64512",
},
)
if "name: bgp_as" not in output:
fail("single-peer bgp_as was not rendered")
if "name: bgp_peers" in output:
fail("bgp_peers present even though the peer list is empty")
# kube_vip_endpoint defaults to null (defined in role defaults): the
# address and subnet must fall back to the apiserver endpoint. default()
# without a truthy flag does NOT fall back on null, only on undefined, so
# this case pins the null runtime condition to prevent that regression.
output = render(
env,
{
"_kube_vip_bgp_peers": [],
"kube_vip_endpoint": None,
"kube_vip_arp": True,
"kube_vip_bgp": False,
},
)
if "value: 192.168.30.222" not in output:
fail("null kube_vip_endpoint does not fall back to apiserver_endpoint")
# kube_vip_endpoint set: overrides the internal listening address AND the
# subnet derivation while the advertised apiserver_endpoint stays separate.
output = render(
env,
{
"_kube_vip_bgp_peers": [],
"kube_vip_endpoint": "10.66.1.5",
"kube_vip_arp": True,
"kube_vip_bgp": False,
},
)
if "value: 10.66.1.5" not in output:
fail("kube_vip_endpoint did not override the address")
if "value: 192.168.30.222" in output:
fail("apiserver_endpoint leaked into address when kube_vip_endpoint set")
print("kube-vip manifest regression test passed")
if __name__ == "__main__":
main()
@@ -1,185 +0,0 @@
#!/usr/bin/env python3
"""Regression test for the MetalLB deploy conditions.
The MetalLB manifest (roles/k3s_server/tasks/main.yml) and the MetalLB pool
(roles/k3s_server_post/tasks/main.yml) are included under a `when` condition
that decides whether MetalLB provides load balancing. A previous change (#683)
guarded `cilium_bgp` but accidentally skipped MetalLB whenever a non-BGP
Cilium CNI was in use (`cilium_iface` defined), breaking the cilium + MetalLB
scenario.
This test loads the real `when` expressions from both task files and evaluates
them against representative variable sets, asserting MetalLB is deployed in
every topology except when MetalLB is explicitly disabled, kube-vip owns the
VIP range, or Cilium BGP is enabled.
"""
from __future__ import print_function
import os
import re
import subprocess
import yaml
from jinja2 import Environment
METALLB_WHEN = (
"metal_lb_enabled and "
"kube_vip_lb_ip_range is not defined and "
"not (cilium_bgp | default(false) | bool)"
)
def repo_root():
return subprocess.check_output(
["git", "rev-parse", "--show-toplevel"], text=True
).strip()
def fail(message):
raise SystemExit("MetalLB deploy condition test failed: " + message)
def extract_when(path, task_name):
"""Return the `when:` expression string for the named task."""
with open(path, encoding="utf-8") as handle:
doc = yaml.safe_load(handle)
for task in doc:
if task.get("name") == task_name:
when = task.get("when")
return (when or "").strip()
return None
def evaluate(when, variables):
"""Evaluate a `when` expression against variables using Jinja2."""
env = Environment()
def fake_bool(value):
# Minimal stand-in for Ansible's truthiness filter used by `| bool`.
if isinstance(value, bool):
return value
if value is None:
return False
return str(value).lower() in ("1", "true", "yes", "on")
env.filters["bool"] = fake_bool
template = env.from_string("{{ " + when + " }}")
rendered = template.render(**variables)
# The expression renders to the literal strings "True"/"False".
if rendered == "True":
return True
if rendered == "False":
return False
fail("condition did not render to a boolean: {0!r}".format(rendered))
def assert_deployment(when, variables, expected, label):
result = evaluate(when, variables)
verdict = "deploy" if result else "skip"
expected_verdict = "deploy" if expected else "skip"
if result != expected:
fail(
"{0}: expected to {1} MetalLB but the condition chose to {2} "
"(vars: {3})".format(label, expected_verdict, verdict, variables)
)
def scenarios():
"""Yield (variables, expected_deploy, label) pairs."""
yield (
# Default Flannel inventory (all.yml sets cilium_bgp: false).
{
"metal_lb_enabled": True,
"cilium_bgp": False,
"cilium_iface": None,
},
True,
"flannel default (cilium_bgp: false)",
)
yield (
# Calico CNI with no Cilium variable in scope (issue #644): cilium_bgp
# is genuinely undefined, so `default(false)` must keep MetalLB on.
{
"metal_lb_enabled": True,
"calico_iface": "eth1",
},
True,
"calico, cilium_bgp undefined (#644)",
)
yield (
# Cilium CNI with BGP disabled: MetalLB must still be deployed.
{
"metal_lb_enabled": True,
"cilium_bgp": False,
"cilium_iface": "eth1",
},
True,
"cilium non-BGP (regression catch)",
)
yield (
# Cilium CNI with BGP enabled: Cilium provides the LB, skip MetalLB.
{
"metal_lb_enabled": True,
"cilium_bgp": True,
"cilium_iface": "eth1",
},
False,
"cilium BGP enabled",
)
yield (
# kube-vip is the load balancer provider: skip MetalLB.
{
"metal_lb_enabled": True,
"kube_vip_lb_ip_range": "192.168.30.80-192.168.30.90",
"cilium_bgp": False,
},
False,
"kube-vip owns the VIP range",
)
yield (
# MetalLB explicitly disabled (external LB / single node): skip.
{
"metal_lb_enabled": False,
},
False,
"metal_lb_enabled: false (external LB)",
)
def main():
root = repo_root()
server_tasks = os.path.join(root, "roles", "k3s_server", "tasks", "main.yml")
server_post_tasks = os.path.join(
root, "roles", "k3s_server_post", "tasks", "main.yml"
)
server_when = extract_when(server_tasks, "Deploy metallb manifest")
server_post_when = extract_when(server_post_tasks, "Deploy metallb pool")
if server_when is None:
fail("could not find 'Deploy metallb manifest' when condition")
if server_post_when is None:
fail("could not find 'Deploy metallb pool' when condition")
for when, source in (
(server_when, "k3s_server/tasks/main.yml"),
(server_post_when, "k3s_server_post/tasks/main.yml"),
):
if when != METALLB_WHEN:
fail(
"{0} when condition changed unexpectedly:\n"
" expected: {1}\n got: {2}".format(source, METALLB_WHEN, when)
)
for variables, expected, label in scenarios():
assert_deployment(server_when, variables, expected, "server " + label)
assert_deployment(
server_post_when, variables, expected, "server_post " + label
)
print("MetalLB deploy condition regression test passed for all scenarios")
if __name__ == "__main__":
main()
@@ -1,88 +0,0 @@
#!/usr/bin/env python3
"""Regression test for the MetalLB L2Advertisement interfaces.
`metal_lb_interfaces` restricts which network interfaces MetalLB announces
load balancer IPs on in layer2 mode. When the list is non-empty, the
L2Advertisement in roles/k3s_server_post/templates/metallb.crs.j2 must render
a `spec.interfaces` block; when it is empty (the default), no spec is rendered
so MetalLB announces on all interfaces.
This renders the template and asserts both cases plus the BGP path (which must
not be affected by the L2 interfaces variable).
"""
from __future__ import print_function
import os
import subprocess
from jinja2 import Environment, FileSystemLoader, StrictUndefined
def repo_root():
return subprocess.check_output(
["git", "rev-parse", "--show-toplevel"], text=True
).strip()
def fail(message):
raise SystemExit("MetalLB interfaces test failed: " + message)
def render(env, extra_vars):
base_vars = {
"metal_lb_mode": "layer2",
"metal_lb_ip_range": "192.168.30.80-192.168.30.90",
}
base_vars.update(extra_vars)
template = env.get_template("metallb.crs.j2")
return template.render(**base_vars)
def main():
root = repo_root()
template_dir = os.path.join(
root, "roles", "k3s_server_post", "templates"
)
env = Environment(
loader=FileSystemLoader(template_dir), undefined=StrictUndefined
)
# Empty list (default): no spec.interfaces in the L2Advertisement.
output = render(env, {"metal_lb_interfaces": []})
if "spec:\n interfaces:" in output:
fail("spec.interfaces rendered with an empty metal_lb_interfaces")
if "kind: L2Advertisement" not in output:
fail("L2Advertisement missing in layer2 mode")
# Single interface.
output = render(env, {"metal_lb_interfaces": ["eth1"]})
if "spec:\n interfaces:\n - eth1" not in output:
fail("single interface was not rendered in spec.interfaces")
# Multiple interfaces.
output = render(env, {"metal_lb_interfaces": ["eth1", "eth2"]})
if "spec:\n interfaces:\n - eth1\n - eth2" not in output:
fail("multiple interfaces were not rendered in spec.interfaces")
# BGP mode must not emit an L2Advertisement spec at all.
output = render(
env,
{
"metal_lb_mode": "bgp",
"metal_lb_interfaces": ["eth1"],
"metal_lb_bgp_my_asn": "64513",
"metal_lb_bgp_peer_asn": "64512",
"metal_lb_bgp_peer_address": "192.168.30.1",
},
)
if "kind: L2Advertisement" in output:
fail("L2Advertisement rendered in bgp mode")
if "interfaces:" in output:
fail("interfaces rendered in bgp mode")
print("MetalLB interfaces regression test passed")
if __name__ == "__main__":
main()
-108
View File
@@ -1,108 +0,0 @@
#!/usr/bin/env python3
"""Regression test for the MetalLB converge checks.
The MetalLB tasks in roles/k3s_server_post/tasks/metallb.yml must actually
verify resources through an explicit kubectl get, and must retry on a
transient kube API error while MetalLB converges.
The "Test metallb-system namespace" task previously ran `k3s kubectl -n
metallb-system` with no subcommand, which only printed a usage page and always
exited 0, so it always succeeded even when the namespace did not exist (issue
#350). It must instead run an explicit `get namespace metallb-system`, which
returns non-zero when the namespace is absent.
An explicit get actually contacts the API server, so these tasks need the same
retry wiring as their siblings (register, until rc == 0, retries, delay). A
bare get with no retry would otherwise abort the converge play on a transient
kube API error while MetalLB converges.
"""
from __future__ import print_function
import os
import subprocess
import yaml
def repo_root():
return subprocess.check_output(
["git", "rev-parse", "--show-toplevel"], text=True
).strip()
def fail(message):
raise SystemExit("MetalLB namespace test failed: " + message)
def find_task(tasks, name):
for entry in tasks:
if entry.get("name") == name:
return entry
fail("could not find the '{0}' task".format(name))
return None
def command_text(task):
cmd = task.get("ansible.builtin.command")
if not cmd:
cmd = task.get("command")
if not cmd:
fail("task does not use ansible.builtin.command")
return cmd if isinstance(cmd, str) else " ".join(cmd)
def check_explicit_get(task, name, needle):
text = command_text(task)
if needle not in text:
fail(
"command does not run '{0}'; the task would only print usage and "
"never verify the resource (got: {1!r})".format(needle, text)
)
def check_retry_wiring(task, name):
# The sibling k3s_server_post metallb tasks retry kubectl because the kube
# API can briefly be unavailable while MetalLB converges. Without the same
# retry, a transient API error aborts the whole converge play.
if not task.get("register"):
fail(
"{0} does not register a result; without retry wiring a transient "
"kube API error aborts the converge play".format(name)
)
if not isinstance(task.get("until"), str) or "rc == 0" not in task["until"]:
fail(
"{0} does not retry on rc == 0; the kube API can transiently fail "
"while MetalLB converges and abort the play".format(name)
)
if task.get("retries") is None:
fail("{0} is missing retries".format(name))
if task.get("delay") is None:
fail("{0} is missing delay".format(name))
def main():
task_file = os.path.join(
repo_root(), "roles", "k3s_server_post", "tasks", "metallb.yml"
)
with open(task_file, encoding="utf-8") as handle:
tasks = yaml.safe_load(handle)
namespace_task = find_task(tasks, "Test metallb-system namespace")
# A bare `-n metallb-system` with no subcommand prints kubectl usage and
# always exits 0, so it never proves the namespace exists. The fix must
# use an explicit get.
check_explicit_get(namespace_task, "Test metallb-system namespace",
"get namespace metallb-system")
check_retry_wiring(namespace_task, "Test metallb-system namespace")
webhook_task = find_task(tasks, "Test metallb-system webhook-service endpoint")
check_explicit_get(webhook_task, "Test metallb-system webhook-service endpoint",
"get endpoints")
check_retry_wiring(webhook_task, "Test metallb-system webhook-service endpoint")
print("MetalLB namespace check regression test passed")
if __name__ == "__main__":
main()
@@ -1,29 +0,0 @@
#!/usr/bin/env bash
set -Eeuo pipefail
repo_root="$(git rev-parse --show-toplevel)"
metallb_task="$repo_root/roles/k3s_server/tasks/metallb.yml"
# The speaker tag verification must read the rendered manifest on the managed
# host with slurp. A controller-side lookup('ansible.builtin.file', ...) would
# read from the Ansible control node, which does not have the file, and would
# fail on every MetalLB scenario.
grep -Fq -- 'ansible.builtin.slurp' "$metallb_task" || {
printf 'MetalLB speaker tag check does not use slurp on the managed host\n' >&2
exit 1
}
grep -Eq -- 'lookup\(.?ansible\.builtin\.file' "$metallb_task" && {
printf 'MetalLB speaker tag check uses a controller-side file lookup\n' >&2
exit 1
}
# The check must reference the full image reference, not just a bare version
# string that could appear anywhere in the manifest.
grep -Fq -- 'quay.io/metallb/speaker:' "$metallb_task" || {
printf 'MetalLB speaker tag check does not match the full image reference\n' >&2
exit 1
}
printf 'MetalLB remote manifest read regression test passed\n'
@@ -1,89 +0,0 @@
#!/usr/bin/env bash
set -Eeuo pipefail
repo_root="$(git rev-parse --show-toplevel)"
fixture="$(mktemp -d)"
trap 'rm -rf -- "$fixture"' EXIT
mock_bin="$fixture/bin"
mock_state="$fixture/state"
mock_home="$fixture/home"
mock_vagrant_home="$fixture/vagrant-home"
mock_box_dir="$mock_vagrant_home/boxes/bento-VAGRANTSLASH-ubuntu-26.04/202606.01.0/amd64/virtualbox"
mock_vbox_root="$mock_home/VirtualBox VMs"
mock_master_root="$mock_home/.cache/k3s-ci/vagrant-masters"
lock_file="$fixture/vagrant-boxes.lock"
mkdir -p -- "$mock_bin" "$mock_state" "$mock_box_dir" "$mock_vbox_root"
printf '%s\n' 'bento/ubuntu-26.04 202606.01.0 amd64' > "$lock_file"
ln -s "$repo_root/.github/scripts/test-fixtures/mock-vboxmanage" "$mock_bin/VBoxManage"
ln -s "$repo_root/.github/scripts/test-fixtures/mock-vagrant" "$mock_bin/vagrant"
ln -s "$repo_root/.github/scripts/test-fixtures/mock-flock" "$mock_bin/flock"
export PATH="$mock_bin:$PATH"
export HOME="$mock_home"
export VAGRANT_HOME="$mock_vagrant_home"
export VAGRANT_BOX_LOCK_FILE="$lock_file"
export K3S_CI_REPOSITORY_ROOT="$repo_root"
export K3S_CI_VAGRANT_MASTER_ROOT="$mock_master_root"
export K3S_CI_VIRTUALBOX_ROOT="$mock_vbox_root"
export MOCK_VBOX_STATE="$mock_state"
export MOCK_VBOX_ROOT="$mock_vbox_root"
export MOCK_BOX_DIR="$mock_box_dir"
export MOCK_VBOX_LOG="$fixture/vbox.log"
export MOCK_VAGRANT_LOG="$fixture/vagrant.log"
export MOCK_VAGRANTFILE_CAPTURE="$fixture/prewarm-Vagrantfile"
: > "$MOCK_VBOX_LOG"
: > "$MOCK_VAGRANT_LOG"
unowned_uuid='99999999-9999-4999-8999-999999999999'
printf '%s\n' "$unowned_uuid" > "$mock_box_dir/master_id"
script="$repo_root/.github/scripts/prepare-vagrant-box-masters.sh"
first_output="$fixture/first-output"
second_output="$fixture/second-output"
third_output="$fixture/third-output"
"$script" > "$first_output"
if grep -Fq 'config.ssh.insert_key' "$MOCK_VAGRANTFILE_CAPTURE"; then
printf 'prewarm Vagrantfile unexpectedly overrides Vagrant SSH key insertion\n' >&2
exit 1
fi
grep -Fq 'virtualbox.memory = 1024' "$MOCK_VAGRANTFILE_CAPTURE"
grep -Fq 'virtualbox.cpus = 2' "$MOCK_VAGRANTFILE_CAPTURE"
grep -Fq 'config.vm.boot_timeout = 600' "$MOCK_VAGRANTFILE_CAPTURE"
mapping_file="$mock_master_root/bento_ubuntu-26.04-202606.01.0-amd64.uuid"
test -s "$mapping_file"
cmp -s "$mapping_file" "$mock_box_dir/master_id"
grep -Fq 'Created and recorded owned master' "$first_output"
grep -Fq 'modifyvm' "$MOCK_VBOX_LOG"
grep -Fq 'setextradata' "$MOCK_VBOX_LOG"
if grep -Fq "$unowned_uuid" "$MOCK_VBOX_LOG"; then
printf 'unowned cached master UUID was unexpectedly inspected or modified\n' >&2
exit 1
fi
if grep -Eq 'unregistervm|closemedium' "$MOCK_VBOX_LOG"; then
printf 'master preparation invoked a destructive VirtualBox command\n' >&2
exit 1
fi
: > "$MOCK_VAGRANT_LOG"
"$script" > "$second_output"
grep -Fq 'Reusing owned master' "$second_output"
if grep -Fq 'up ' "$MOCK_VAGRANT_LOG"; then
printf 'valid owned master was unexpectedly rebuilt\n' >&2
exit 1
fi
stale_uuid="$(tr -d '[:space:]' < "$mapping_file")"
rm -f -- "$mock_state/vm-$stale_uuid"
: > "$MOCK_VAGRANT_LOG"
"$script" > "$third_output"
grep -Fq 'rebuilding without deleting any VM or disk' "$third_output"
grep -Fq 'up ' "$MOCK_VAGRANT_LOG"
if grep -Eq 'unregistervm|closemedium' "$MOCK_VBOX_LOG"; then
printf 'stale master recovery invoked a destructive VirtualBox command\n' >&2
exit 1
fi
printf 'Vagrant box master preparation fixture test passed\n'
@@ -1,28 +0,0 @@
#!/usr/bin/env bash
set -Eeuo pipefail
repo_root="$(git rev-parse --show-toplevel)"
site_play="$repo_root/site.yml"
# #636: verify the "Pre tasks" play asserts that all k3s_cluster hosts have
# unique hostnames, so a duplicate-hostname inventory fails fast instead of
# silently breaking node registration/joining.
grep -Fq -- 'Verify all cluster nodes have unique hostnames' "$site_play" || {
printf 'site.yml is missing the unique-hostname preflight check\n' >&2
exit 1
}
# The check must deduplicate the cluster hostname list via the `unique` filter
# and compare lengths, i.e. groups['k3s_cluster'] must be referenced.
grep -Fq -- "groups['k3s_cluster']" "$site_play" || {
printf 'unique-hostname check does not iterate the k3s_cluster group\n' >&2
exit 1
}
if ! grep -Eq -- 'cluster_hostnames.*\|.*unique|\| unique' "$site_play"; then
printf 'unique-hostname check does not deduplicate the hostname list\n' >&2
exit 1
fi
printf 'Unique hostname preflight regression test passed\n'
@@ -1,92 +0,0 @@
#!/usr/bin/env python3
"""Regression test for the verify_from_outside nginx HTTP assert.
The "Assert that the nginx welcome page is available" task in
molecule/resources/verify_from_outside/tasks/test/deploy-example.yml performs
a plain HTTP GET (ansible.builtin.uri) against the MetalLB VIP that the load
balancer pool assigned, reaching it from the CI runner over the VirtualBox
host-only network.
The MetalLB speaker announces the VIP on the host-only network, and the runner
may briefly not see it in its ARP table yet. That manifests as a single-shot
"status -1 / No route to host" failure that fails the whole verify play even
though the cluster is healthy. Like the sibling load-balancer address wait and
the k3s_server_post metallb waits, the assert must retry until it observes
HTTP 200 with the welcome page.
A bare assert with no retry would otherwise fail an otherwise-green cluster on
a transient announcement miss.
"""
from __future__ import print_function
import os
import subprocess
import yaml
def repo_root():
return subprocess.check_output(
["git", "rev-parse", "--show-toplevel"], text=True
).strip()
def fail(message):
raise SystemExit("verify nginx assert test failed: " + message)
def find_task(play, name):
block = play.get("block")
if not isinstance(block, list):
fail("expected a block list of tasks")
for entry in block:
if entry.get("name") == name:
return entry
fail("could not find the '{0}' task".format(name))
return None
def check_retry_wiring(task, name):
if not task.get("register"):
fail("{0} does not register a result; without retry wiring a transient "
"MetalLB VIP announcement miss fails the verify play".format(name))
until = task.get("until")
if not until:
fail("{0} does not retry; a transient MetalLB VIP announcement miss "
"on the host-only network would fail the verify play".format(name))
if isinstance(until, str):
until = [until]
joined = " | ".join(str(expr) for expr in until)
if "result.status == 200" not in joined:
fail("{0} does not retry until the HTTP status is 200".format(name))
if "Welcome to nginx!" not in joined:
fail("{0} does not verify the welcome page content".format(name))
if task.get("retries") is None:
fail("{0} is missing retries".format(name))
if task.get("delay") is None:
fail("{0} is missing delay".format(name))
def main():
task_file = os.path.join(
repo_root(),
"molecule", "resources", "verify_from_outside", "tasks", "test",
"deploy-example.yml",
)
with open(task_file, encoding="utf-8") as handle:
plays = yaml.safe_load(handle)
play = plays[0]
task = find_task(play, "Assert that the nginx welcome page is available")
if not task.get("ansible.builtin.uri"):
fail("task does not use ansible.builtin.uri to reach the VIP")
check_retry_wiring(task, "Assert that the nginx welcome page is available")
print("verify nginx assert regression test passed")
if __name__ == "__main__":
main()
-24
View File
@@ -1,24 +0,0 @@
#!/usr/bin/env bash
set -Eeuo pipefail
if (($# < 2)); then
printf 'Usage: vagrant-up-timed.sh WORKDIR MACHINE [MACHINE ...]\n' >&2
exit 2
fi
workdir="$1"
shift
timing_log="${K3S_CI_CREATE_TIMING_LOG:-${RUNNER_TEMP:-/tmp}/k3s-ci-create-timing.log}"
mkdir -p -- "${timing_log%/*}"
printf '%s batch-start machines=%s\n' "$(date --iso-8601=ns)" "$*" | tee -a "$timing_log"
set +e
VAGRANT_CWD="$workdir" vagrant up "$@" --provider virtualbox --no-provision 2>&1 |
while IFS= read -r line; do
printf '%s %s\n' "$(date --iso-8601=ns)" "$line"
done | tee -a "$timing_log"
rc=${PIPESTATUS[0]}
set -e
printf '%s batch-end rc=%d machines=%s\n' "$(date --iso-8601=ns)" "$rc" "$*" | tee -a "$timing_log"
exit "$rc"
-98
View File
@@ -1,98 +0,0 @@
#!/usr/bin/env bash
# The single-quoted expressions below are written into fake executables and
# intentionally expand only when those executables run.
# shellcheck disable=SC2016
set -Eeuo pipefail
repo_root=$(git rev-parse --show-toplevel)
test_root=$(mktemp -d)
fake_bin="$test_root/bin"
fake_log="$test_root/vagrant.log"
output="$test_root/output.txt"
mkdir -p "$fake_bin"
trap 'rm -rf "$test_root"' EXIT
printf '%s\n' \
'#!/usr/bin/env bash' \
'set -Eeuo pipefail' \
'printf "%s\n" bento/debian-13 bento/rockylinux-10.1 bento/ubuntu-26.04' \
>"$fake_bin/yq"
printf '%s\n' \
'#!/usr/bin/env bash' \
'set -Eeuo pipefail' \
'emit_box() {' \
' case "$1" in' \
' bento/debian-13) version=202510.26.0 ;;' \
' bento/rockylinux-10.1) version=202512.01.0 ;;' \
' bento/ubuntu-26.04) version=202606.01.0 ;;' \
' *) printf "Unexpected box: %s\n" "$1" >&2; exit 1 ;;' \
' esac' \
' printf "0,,box-name,%s\n" "$1"' \
' printf "0,,box-provider,virtualbox\n"' \
' printf "0,,box-version,%s\n" "$version"' \
' printf "0,,box-architecture,amd64\n"' \
'}' \
'if [[ "${1:-}" == box && "${2:-}" == list ]]; then' \
' emit_box bento/debian-13' \
' emit_box bento/rockylinux-10.1' \
' if [[ "${FAKE_PRESENT_MODE:-all}" == all ]]; then' \
' emit_box bento/ubuntu-26.04' \
' fi' \
'elif [[ "${1:-}" == box && "${2:-}" == add ]]; then' \
' printf "%s\n" "$*" >>"${FAKE_VAGRANT_LOG:?}"' \
'else' \
' printf "Unexpected vagrant arguments: %s\n" "$*" >&2' \
' exit 1' \
'fi' \
>"$fake_bin/vagrant"
chmod +x "$fake_bin/yq" "$fake_bin/vagrant"
PATH="$fake_bin:$PATH" \
FAKE_VAGRANT_LOG="$fake_log" \
"$repo_root/.github/download-boxes.sh" >"$output"
grep -Fq 'All pinned Vagrant boxes are already present.' "$output"
[[ ! -e "$fake_log" ]]
PATH="$fake_bin:$PATH" \
FAKE_PRESENT_MODE=partial \
FAKE_VAGRANT_LOG="$fake_log" \
"$repo_root/.github/download-boxes.sh" >"$output"
grep -Fxq \
'box add --provider virtualbox --box-version 202606.01.0 --architecture amd64 bento/ubuntu-26.04' \
"$fake_log"
incomplete_lock="$test_root/incomplete.lock"
printf '%s\n' \
'bento/debian-13 202510.26.0 amd64' \
'bento/rockylinux-10.1 202512.01.0 amd64' \
>"$incomplete_lock"
if PATH="$fake_bin:$PATH" \
VAGRANT_BOX_LOCK_FILE="$incomplete_lock" \
FAKE_VAGRANT_LOG="$fake_log" \
"$repo_root/.github/download-boxes.sh" >"$output" 2>&1; then
printf 'Download script accepted a lock missing a scenario box.\n' >&2
exit 1
fi
grep -Fq 'Scenario boxes missing from the lock file:' "$output"
grep -Fq 'bento/ubuntu-26.04' "$output"
duplicate_lock="$test_root/duplicate.lock"
printf '%s\n' \
'bento/debian-13 202510.26.0 amd64' \
'bento/debian-13 202508.10.0 amd64' \
'bento/rockylinux-10.1 202512.01.0 amd64' \
'bento/ubuntu-26.04 202606.01.0 amd64' \
>"$duplicate_lock"
if PATH="$fake_bin:$PATH" \
VAGRANT_BOX_LOCK_FILE="$duplicate_lock" \
FAKE_VAGRANT_LOG="$fake_log" \
"$repo_root/.github/download-boxes.sh" >"$output" 2>&1; then
printf 'Download script accepted duplicate box lock entries.\n' >&2
exit 1
fi
grep -Fq 'Duplicate Vagrant box lock entries:' "$output"
printf 'Vagrant box download tests passed.\n'
-4
View File
@@ -1,4 +0,0 @@
# box version architecture
bento/debian-13 202510.26.0 amd64
bento/rockylinux-10.1 202512.01.0 amd64
bento/ubuntu-26.04 202606.01.0 amd64
+14 -29
View File
@@ -5,53 +5,38 @@ on:
jobs: jobs:
molecule: molecule:
name: cache name: cache
runs-on: [self-hosted, linux, x64, k3s-ci, virtualbox, nested-virt] runs-on: self-hosted
env: env:
PYTHON_VERSION: "3.11" PYTHON_VERSION: "3.11"
VAGRANT_DEFAULT_PROVIDER: virtualbox
VAGRANT_HOME: ${{ github.workspace }}/.vagrant-home
steps: steps:
- name: Check out the codebase - name: Check out the codebase
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # 7.0.1 uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
with: with:
ref: ${{ github.event.pull_request.head.sha || github.sha }} ref: ${{ github.event.pull_request.head.sha }}
- name: Check nested VirtualBox platform
run: |
set -Eeuo pipefail
grep -Eq 'vmx|svm' /proc/cpuinfo
test -c /dev/kvm
test -c /dev/vboxdrv
VBoxManage --version
vagrant --version
test -r /etc/vbox/networks.conf
test "$(stat -c '%u' /etc/vbox/networks.conf)" -eq 0
free -h
df -Pk "${RUNNER_TEMP}"
- name: Set up Python ${{ env.PYTHON_VERSION }} - name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # 7.0.0 uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # 5.3.0
with: with:
python-version: ${{ env.PYTHON_VERSION }} python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip' # caching pip dependencies cache: 'pip' # caching pip dependencies
- name: Cache Vagrant boxes - name: Cache Vagrant boxes
id: cache-vagrant id: cache-vagrant
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # 6.1.0 uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # 4.1.0
with: with:
# This producer only needs to know whether the immutable cache exists. lookup-only: true #if it exists, we don't need to restore and can skip the next step
# Molecule jobs restore it after this job completes.
lookup-only: true
path: | path: |
.vagrant-home/boxes ~/.vagrant.d/boxes
key: vagrant-boxes-${{ runner.name }}-${{ runner.os }}-${{ runner.arch }}-virtualbox-7.2-vagrant-2.4-${{ hashFiles('.github/vagrant-boxes.lock') }} # yamllint disable-line rule:line-length key: vagrant-boxes-${{ hashFiles('**/molecule.yml') }}
restore-keys: |
vagrant-boxes
- name: Download Vagrant boxes for all scenarios - name: Download Vagrant boxes for all scenarios
# An exact hit skips both cache restoration and upstream downloads. # To save some cache space, all scenarios share the same cache key.
# A lock change builds and saves one clean, version-pinned cache. # On the other hand, this means that the cache contents should be
if: steps.cache-vagrant.outputs.cache-hit != 'true' # the same across all scenarios. This step ensures that.
if: steps.cache-vagrant.outputs.cache-hit != 'true' # only run if false since this is just a cache step
run: | run: |
./.github/download-boxes.sh ./.github/download-boxes.sh
./.github/scripts/prepare-vagrant-box-masters.sh
vagrant box list vagrant box list
+1 -18
View File
@@ -2,43 +2,26 @@
name: "CI" name: "CI"
on: on:
pull_request: pull_request:
branches:
- master
types: types:
- opened - opened
- synchronize - synchronize
- reopened
- ready_for_review
paths-ignore: paths-ignore:
- '**/.gitignore' - '**/.gitignore'
- '**/FUNDING.yml' - '**/FUNDING.yml'
- '**/host.ini' - '**/host.ini'
- '**/*.md' - '**/*.md'
- '.github/ISSUE_TEMPLATE/**'
- '**/.editorconfig' - '**/.editorconfig'
- '**/ansible.example.cfg' - '**/ansible.example.cfg'
- '**/deploy.sh' - '**/deploy.sh'
- '**/LICENSE' - '**/LICENSE'
- '**/reboot.sh' - '**/reboot.sh'
- '**/reset.sh' - '**/reset.sh'
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ci-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: true
jobs: jobs:
pre: pre:
if: github.actor != 'dependabot[bot]'
uses: ./.github/workflows/cache.yml uses: ./.github/workflows/cache.yml
needs: [lint]
lint: lint:
if: github.actor != 'dependabot[bot]'
uses: ./.github/workflows/lint.yml uses: ./.github/workflows/lint.yml
needs: [pre]
test: test:
if: github.actor != 'dependabot[bot]'
uses: ./.github/workflows/test.yml uses: ./.github/workflows/test.yml
needs: [pre, lint] needs: [pre, lint]
+9 -30
View File
@@ -5,25 +5,24 @@ on:
jobs: jobs:
pre-commit-ci: pre-commit-ci:
name: Pre-Commit name: Pre-Commit
runs-on: ubuntu-latest runs-on: self-hosted
env: env:
PYTHON_VERSION: "3.12" PYTHON_VERSION: "3.11"
steps: steps:
- name: Check out the codebase - name: Check out the codebase
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # 7.0.1 uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
with: with:
ref: ${{ github.event.pull_request.head.sha || github.sha }} ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Python ${{ env.PYTHON_VERSION }} - name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # 7.0.0 uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # 5.3.0
with: with:
python-version: ${{ env.PYTHON_VERSION }} python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip' # caching pip dependencies cache: 'pip' # caching pip dependencies
- name: Restore Ansible cache - name: Restore Ansible cache
id: cache-ansible uses: actions/cache/restore@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # 4.1.0
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # 6.1.0
with: with:
path: ~/.ansible/collections path: ~/.ansible/collections
key: ansible-${{ hashFiles('collections/requirements.yml') }} key: ansible-${{ hashFiles('collections/requirements.yml') }}
@@ -38,37 +37,17 @@ jobs:
python3 -m pip install -r requirements.txt python3 -m pip install -r requirements.txt
echo "::endgroup::" echo "::endgroup::"
- name: Install Ansible collections with retries
if: steps.cache-ansible.outputs.cache-hit != 'true'
run: |
set -Eeuo pipefail
for attempt in 1 2 3 4 5; do
if ansible-galaxy collection install -r collections/requirements.yml; then
exit 0
fi
echo "Ansible Galaxy attempt ${attempt} failed; retrying."
sleep $((attempt * 10))
done
exit 1
- name: Save Ansible collection cache
if: steps.cache-ansible.outputs.cache-hit != 'true'
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # 6.1.0
with:
path: ~/.ansible/collections
key: ansible-${{ hashFiles('collections/requirements.yml') }}
- name: Run pre-commit - name: Run pre-commit
uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # 3.0.1 uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # 3.0.1
ensure-pinned-actions: ensure-pinned-actions:
name: Ensure SHA Pinned Actions name: Ensure SHA Pinned Actions
runs-on: ubuntu-latest runs-on: self-hosted
steps: steps:
- name: Checkout code - name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # 7.0.1 uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
- name: Ensure SHA pinned actions - name: Ensure SHA pinned actions
uses: zgosalvez/github-actions-ensure-sha-pinned-actions@60e3a74c7b74a319e8e53e46bc455205f7173e1a # 5.0.8 uses: zgosalvez/github-actions-ensure-sha-pinned-actions@40ba2d51b6b6d8695f2b6bd74e785172d4f8d00f # 3.0.14
with: with:
allowlist: | allowlist: |
aws-actions/ aws-actions/
+67 -59
View File
@@ -5,64 +5,72 @@ on:
jobs: jobs:
molecule: molecule:
name: Molecule name: Molecule
runs-on: [self-hosted, linux, x64, k3s-ci, virtualbox, nested-virt] runs-on: self-hosted
strategy: strategy:
matrix: matrix:
scenario: scenario:
- default - default
# - ipv6
- single_node - single_node
- calico - calico
- cilium - cilium
- kube-vip - kube-vip
# - ipv6 fail-fast: false
fail-fast: true
max-parallel: 1
env: env:
PYTHON_VERSION: "3.11" PYTHON_VERSION: "3.11"
VAGRANT_DEFAULT_PROVIDER: virtualbox
VAGRANT_HOME: ${{ github.workspace }}/.vagrant-home
steps: steps:
- name: Check out the codebase - name: Check out the codebase
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # 7.0.1 uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
with: with:
ref: ${{ github.event.pull_request.head.sha || github.sha }} ref: ${{ github.event.pull_request.head.sha }}
- name: Clean repository-owned resources before testing # these steps are necessary if not using ephemeral nodes
run: ./.github/scripts/cleanup-runner-resources.sh --apply - name: Delete old Vagrant box versions
if: always() # do this even if a step before has failed
run: vagrant box prune --force
- name: Record host-only network baseline - name: Remove all local Vagrant boxes
run: ./.github/scripts/cleanup-runner-resources.sh --snapshot if: always() # do this even if a step before has failed
run: if vagrant box list 2>/dev/null; then vagrant box list | cut -f 1 -d ' ' | xargs -L 1 vagrant box remove -f 2>/dev/null && echo "All Vagrant boxes removed." || echo "No Vagrant boxes found."; else echo "No Vagrant boxes found."; fi
- name: Check nested VirtualBox platform - name: Remove all Virtualbox VMs
run: | if: always() # do this even if a step before has failed
set -Eeuo pipefail run: VBoxManage list vms | awk -F'"' '{print $2}' | xargs -I {} VBoxManage unregistervm --delete "{}"
grep -Eq 'vmx|svm' /proc/cpuinfo
test -c /dev/kvm - name: Remove all Virtualbox HDs
test -c /dev/vboxdrv if: always() # do this even if a step before has failed
VBoxManage --version run: VBoxManage list hdds | awk -F':' '/^UUID:/ {print $2}' | xargs -I {} VBoxManage closemedium disk "{}" --delete
vagrant --version
test -r /etc/vbox/networks.conf - name: Remove all Virtualbox Networks
test "$(stat -c '%u' /etc/vbox/networks.conf)" -eq 0 if: always() # do this even if a step before has failed
free -h run: VBoxManage list hostonlyifs | grep '^Name:' | awk '{print $2}' | grep '^vboxnet' | xargs -I {} VBoxManage hostonlyif remove {}
df -Pk "${RUNNER_TEMP}"
- name: Remove Virtualbox network config
if: always() # do this even if a step before has failed
run: sudo rm /etc/vbox/networks.conf || true
- name: Configure VirtualBox
run: |-
sudo mkdir -p /etc/vbox
cat <<EOF | sudo tee -a /etc/vbox/networks.conf > /dev/null
* 192.168.30.0/24
* fdad:bad:ba55::/64
EOF
- name: Set up Python ${{ env.PYTHON_VERSION }} - name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # 7.0.0 uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # 5.3.0
with: with:
python-version: ${{ env.PYTHON_VERSION }} python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip' # caching pip dependencies cache: 'pip' # caching pip dependencies
- name: Restore vagrant Boxes cache - name: Restore vagrant Boxes cache
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # 6.1.0 uses: actions/cache/restore@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # 4.1.0
with: with:
path: .vagrant-home/boxes path: ~/.vagrant.d/boxes
key: vagrant-boxes-${{ runner.name }}-${{ runner.os }}-${{ runner.arch }}-virtualbox-7.2-vagrant-2.4-${{ hashFiles('.github/vagrant-boxes.lock') }} # yamllint disable-line rule:line-length key: vagrant-boxes-${{ hashFiles('**/molecule.yml') }}
fail-on-cache-miss: true fail-on-cache-miss: true
- name: Prepare runner-owned Vagrant box masters
run: ./.github/scripts/prepare-vagrant-box-masters.sh
- name: Install dependencies - name: Install dependencies
run: | run: |
echo "::group::Upgrade pip" echo "::group::Upgrade pip"
@@ -74,45 +82,45 @@ jobs:
echo "::endgroup::" echo "::endgroup::"
- name: Test with molecule - name: Test with molecule
run: | run: molecule test --scenario-name ${{ matrix.scenario }}
set -Eeuo pipefail timeout-minutes: 90
resource_dir="${RUNNER_TEMP}/logs/resources/${{ matrix.scenario }}"
timing_file="${RUNNER_TEMP}/logs/timing/${{ matrix.scenario }}.txt"
mkdir -p -- "${timing_file%/*}"
./.github/scripts/monitor-runner-resources.sh "$resource_dir" 10 &
monitor_pid=$!
stop_monitor() {
kill -TERM "$monitor_pid" 2>/dev/null || true
wait "$monitor_pid" 2>/dev/null || true
}
trap stop_monitor EXIT
/usr/bin/time -v -o "$timing_file" \
molecule test --scenario-name ${{ matrix.scenario }}
timeout-minutes: 180
env: env:
ANSIBLE_K3S_LOG_DIR: ${{ runner.temp }}/logs/k3s-ansible/${{ matrix.scenario }} ANSIBLE_K3S_LOG_DIR: ${{ runner.temp }}/logs/k3s-ansible/${{ matrix.scenario }}
ANSIBLE_SSH_RETRIES: 4 ANSIBLE_SSH_RETRIES: 4
ANSIBLE_TIMEOUT: 120 ANSIBLE_TIMEOUT: 120
PY_COLORS: 1 PY_COLORS: 1
ANSIBLE_FORCE_COLOR: 1 ANSIBLE_FORCE_COLOR: 1
K3S_CI_CREATE_TIMING_LOG: ${{ runner.temp }}/logs/timing/${{ matrix.scenario }}-create.log
- name: Collect runner diagnostics # these steps are necessary if not using ephemeral nodes
if: always() - name: Delete old Vagrant box versions
run: ./.github/scripts/collect-runner-diagnostics.sh "${RUNNER_TEMP}/logs/runner" if: always() # do this even if a step before has failed
env: run: vagrant box prune --force
K3S_CI_SCENARIO_NAME: ${{ matrix.scenario }}
- name: Clean repository-owned resources after testing - name: Remove all local Vagrant boxes
if: always() if: always() # do this even if a step before has failed
run: ./.github/scripts/cleanup-runner-resources.sh --apply run: if vagrant box list 2>/dev/null; then vagrant box list | cut -f 1 -d ' ' | xargs -L 1 vagrant box remove -f 2>/dev/null && echo "All Vagrant boxes removed." || echo "No Vagrant boxes found."; else echo "No Vagrant boxes found."; fi
- name: Remove all Virtualbox VMs
if: always() # do this even if a step before has failed
run: VBoxManage list vms | awk -F'"' '{print $2}' | xargs -I {} VBoxManage unregistervm --delete "{}"
- name: Remove all Virtualbox HDs
if: always() # do this even if a step before has failed
run: VBoxManage list hdds | awk -F':' '/^UUID:/ {print $2}' | xargs -I {} VBoxManage closemedium disk "{}" --delete
- name: Remove all Virtualbox Networks
if: always() # do this even if a step before has failed
run: VBoxManage list hostonlyifs | grep '^Name:' | awk '{print $2}' | grep '^vboxnet' | xargs -I {} VBoxManage hostonlyif remove {}
- name: Remove Virtualbox network config
if: always() # do this even if a step before has failed
run: sudo rm /etc/vbox/networks.conf || true
- name: Upload log files - name: Upload log files
if: always() # do this even if a step before has failed if: always() # do this even if a step before has failed
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # 7.0.1 uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # 4.4.3
with: with:
name: logs-${{ matrix.scenario }}-${{ github.run_id }}-${{ github.run_attempt }} name: logs
path: | path: |
${{ runner.temp }}/logs ${{ runner.temp }}/logs
if-no-files-found: warn overwrite: true
retention-days: 14
-2
View File
@@ -1,6 +1,4 @@
.env/ .env/
*.log *.log
ansible.cfg ansible.cfg
.ansible/
kubeconfig kubeconfig
zIgnore/
-122
View File
@@ -20,9 +20,6 @@ repos:
rev: v6.22.2 rev: v6.22.2
hooks: hooks:
- id: ansible-lint - id: ansible-lint
additional_dependencies: [ansible-core==2.18.0]
language_version: python3.12
args: [--offline]
- repo: https://github.com/shellcheck-py/shellcheck-py - repo: https://github.com/shellcheck-py/shellcheck-py
rev: v0.9.0.6 rev: v0.9.0.6
hooks: hooks:
@@ -36,122 +33,3 @@ repos:
rev: 0.6.4 rev: 0.6.4
hooks: hooks:
- id: fix-smartquotes - id: fix-smartquotes
- repo: local
hooks:
- id: cleanup-runner-resources-test
name: cleanup runner resources test
entry: .github/scripts/test-cleanup-runner-resources.sh
language: system
pass_filenames: false
files: ^\.github/scripts/(cleanup-runner-resources|test-cleanup-runner-resources)\.sh$
- id: download-vagrant-boxes-test
name: Vagrant box download test
entry: .github/test-download-boxes.sh
language: system
pass_filenames: false
files: ^\.github/(download-boxes|test-download-boxes)\.sh$|^\.github/vagrant-boxes\.lock$
- id: prepare-vagrant-box-masters-test
name: Vagrant box master preparation test
entry: .github/scripts/test-prepare-vagrant-box-masters.sh
language: system
pass_filenames: false
files: ^\.github/scripts/(prepare-vagrant-box-masters|test-prepare-vagrant-box-masters)\.sh$
- id: k3s-server-bootstrap-test
name: K3s transient bootstrap test
entry: .github/scripts/test-k3s-server-bootstrap.sh
language: system
pass_filenames: false
files: ^roles/k3s_server/tasks/(main|join_master)\.yml$|^\.github/scripts/test-k3s-server-bootstrap\.sh$
- id: unique-hostname-precheck-test
name: Unique hostname precheck test
entry: .github/scripts/test-unique-hostname-precheck.sh
language: system
pass_filenames: false
files: ^site\.yml$|^\.github/scripts/test-unique-hostname-precheck\.sh$
- id: disable-swap-test
name: Disable swap test
entry: .github/scripts/test-disable-swap.sh
language: system
pass_filenames: false
files: ^roles/prereq/(tasks/main|defaults/main)\.yml$|^\.github/scripts/test-disable-swap\.sh$
- id: cilium-bgp-manifest-test
name: Cilium BGP manifest test
entry: python3 .github/scripts/test-cilium-bgp-manifest.py
language: python
additional_dependencies:
- Jinja2>=3.1
pass_filenames: false
files: ^roles/k3s_server_post/templates/cilium\.crs\.j2$|^\.github/scripts/test-cilium-bgp-manifest\.py$
- id: cilium-envoy-toggle-test
name: Cilium Envoy toggle test
entry: python3 .github/scripts/test-cilium-envoy-toggle.py
language: python
additional_dependencies:
- Jinja2>=3.1
- PyYAML
pass_filenames: false
files: ^roles/k3s_server_post/tasks/cilium\.yml$|^\.github/scripts/test-cilium-envoy-toggle\.py$
- id: kube-vip-manifest-test
name: kube-vip manifest test
entry: python3 .github/scripts/test-kube-vip-manifest.py
language: python
additional_dependencies:
- Jinja2>=3.1
pass_filenames: false
files: ^roles/k3s_server/templates/vip\.yaml\.j2$|^\.github/scripts/test-kube-vip-manifest\.py$
- id: metallb-remote-read-test
name: MetalLB remote read test
entry: .github/scripts/test-metallb-remote-read.sh
language: system
pass_filenames: false
files: ^roles/k3s_server/tasks/metallb\.yml$|^\.github/scripts/test-metallb-remote-read\.sh$
- id: metallb-interfaces-test
name: MetalLB interfaces test
entry: python3 .github/scripts/test-metallb-interfaces.py
language: python
additional_dependencies:
- Jinja2>=3.1
pass_filenames: false
files: ^roles/k3s_server_post/templates/metallb\.crs\.j2$|^\.github/scripts/test-metallb-interfaces\.py$
- id: metallb-namespace-test
name: MetalLB namespace test
entry: python3 .github/scripts/test-metallb-namespace.py
language: python
additional_dependencies:
- PyYAML
pass_filenames: false
files: ^roles/k3s_server_post/tasks/metallb\.yml$|^\.github/scripts/test-metallb-namespace\.py$
- id: metallb-deploy-condition-test
name: MetalLB deploy condition test
entry: python3 .github/scripts/test-metallb-deploy-condition.py
language: python
additional_dependencies:
- Jinja2>=3.1
- PyYAML
pass_filenames: false
files: ^roles/k3s_server/tasks/main\.yml$|^roles/k3s_server_post/tasks/main\.yml$|^\.github/scripts/test-metallb-deploy-condition\.py$ # noqa yaml[line-length]
- id: kube-vip-deploy-condition-test
name: kube-vip deploy condition test
entry: python3 .github/scripts/test-kube-vip-deploy-condition.py
language: python
additional_dependencies:
- Jinja2>=3.1
- PyYAML
pass_filenames: false
files: ^roles/k3s_server/tasks/main\.yml$|^\.github/scripts/test-kube-vip-deploy-condition\.py$ # noqa yaml[line-length]
- id: default-interface-test
name: default interface test
entry: python3 .github/scripts/test-default-interface.py
language: python
additional_dependencies:
- Jinja2>=3.1
pass_filenames: false
files: ^inventory/sample/group_vars/all\.yml$|^\.github/scripts/test-default-interface\.py$
- id: verify-nginx-assert-test
name: verify nginx assert test
entry: python3 .github/scripts/test-verify-nginx-assert.py
language: python
additional_dependencies:
- PyYAML
pass_filenames: false
files: ^molecule/resources/verify_from_outside/tasks/test/deploy-example\.yml$|^\.github/scripts/test-verify-nginx-assert\.py$ # noqa yaml[line-length]
-130
View File
@@ -1,130 +0,0 @@
# Agent Guide
This file is the canonical repository guide for coding agents and automated contributors. Read it before making
changes. Human contributors should also review [CONTRIBUTING.md](CONTRIBUTING.md).
## Project overview
This repository is an Ansible collection that provisions and resets highly available k3s clusters. It supports
multiple networking choices, including Flannel, Calico, Cilium, kube-vip, and MetalLB.
The main entry points are:
- `site.yml`: provision or update a cluster.
- `reset.yml`: remove k3s from a cluster.
- `reboot.yml`: reboot cluster nodes.
- `inventory/sample/`: example inventory and variables.
- `roles/`: reusable Ansible roles used by the playbooks.
- `molecule/`: integration scenarios run by CI.
- `.github/scripts/`: CI support scripts and focused regression tests.
## Source of truth
- Role defaults belong in `roles/<role>/defaults/main.yml`.
- Tasks belong in `roles/<role>/tasks/` and handlers in `roles/<role>/handlers/`.
- Example user configuration belongs in `inventory/sample/`.
- User-facing setup and variable documentation belongs in `README.md`.
- Contributor workflows and review expectations belong in `CONTRIBUTING.md`.
- Agent-specific repository instructions belong in this file.
Keep `CLAUDE.md` and `.github/copilot-instructions.md` as small pointers to this file. Do not duplicate these
instructions in tool-specific files.
## Development setup
Use a Python virtual environment. Do not commit the environment, generated logs, inventories, kubeconfigs, or
credentials.
```bash
python3 -m venv .env
source .env/bin/activate
python3 -m pip install -r requirements.txt
ansible-galaxy collection install -r collections/requirements.yml
pre-commit install
```
`ansible.cfg` is intentionally ignored. Copy `ansible.example.cfg` when local configuration is needed.
## Working rules
1. Inspect the current branch and worktree before editing. Preserve unrelated user changes.
2. Keep changes focused. Avoid drive-by formatting or dependency updates.
3. Never add real IP addresses, hostnames, tokens, private keys, kubeconfigs, or inventory secrets.
4. Use placeholders in examples and redact sensitive values from logs and issue reports.
5. Preserve idempotence. An already-converged host should not report changes without a real state transition.
6. Prefer Ansible modules over `ansible.builtin.command` or `ansible.builtin.shell`. When a command is required,
define accurate `changed_when` and `failed_when` behavior.
7. Use fully qualified collection names, such as `ansible.builtin.copy`.
8. Put configurable values in role defaults or inventory variables. Avoid embedding environment-specific values in
tasks and templates.
9. Maintain compatibility with the operating systems and architectures listed in `README.md`.
10. Do not weaken lint rules, tests, or CI checks to make a change pass.
## Change guidance
### Ansible tasks and roles
- Use descriptive task names in sentence case.
- Use YAML booleans (`true` and `false`) rather than aliases.
- Quote file modes, for example `mode: "0644"`.
- Notify handlers only when the managed resource changes.
- Use `become: true` only where privilege escalation is needed.
- Update role defaults, sample inventory, and the README together when adding or renaming user-facing variables.
- Check reset behavior when provisioning introduces persistent services, files, mounts, or network state.
### Templates and manifests
- Keep Jinja logic small and readable. Move complicated decisions into task variables where practical.
- Render valid YAML after Jinja evaluation.
- Preserve explicit handling for optional and undefined variables.
- Add or update a focused test under `.github/scripts/` when changing generated Kubernetes manifests or bootstrap
behavior.
### Molecule scenarios
- Reuse `molecule/resources/` for shared behavior.
- Put scenario-specific inputs in `molecule/<scenario>/overrides.yml` and `verify-vars.yml`.
- Update `molecule/README.md` when adding, removing, or materially changing a scenario.
- Clean up resources created by tests, including failure paths.
## Validation
Run the smallest relevant checks while iterating, then run the complete local validation before considering a change
ready:
```bash
pre-commit run --all-files
```
For playbook or role changes, also run syntax checks with a non-sensitive inventory:
```bash
ansible-playbook site.yml --syntax-check -i inventory/sample/hosts.ini
ansible-playbook reset.yml --syntax-check -i inventory/sample/hosts.ini
```
Run focused regression scripts when their related files change. The mapping is defined in
`.pre-commit-config.yaml`.
Molecule tests require Vagrant, VirtualBox, host-only networking, and substantial local resources. Run the most
relevant scenario when that environment is available:
```bash
molecule test --scenario-name <scenario>
```
If a required test can't be run locally, state exactly which check was skipped and why. Never claim a check passed
unless it was executed.
## Documentation and review
- Keep commands copyable and examples free of secrets.
- Update documentation in the same change as user-visible behavior.
- Explain behavior changes, compatibility concerns, operational risks, and rollback steps in the pull request.
- Use conventional commit messages with a scope, for example `fix(k3s-server): handle an existing token safely`.
- Do not commit, push, open a pull request, or modify remote resources unless the user explicitly requests it.
## Definition of done
A change is ready for review when it is focused, documented, linted, tested in proportion to its risk, and shown in a
clean diff with no secrets or generated artifacts.
-3
View File
@@ -1,3 +0,0 @@
@AGENTS.md
`AGENTS.md` is the canonical repository guide. Follow it for all work in this repository.
-94
View File
@@ -1,94 +0,0 @@
# Contributing to k3s-ansible
Thank you for improving k3s-ansible. Contributions should be focused, safe to apply to existing clusters, and tested
in proportion to their operational impact.
## Before opening an issue
- Search existing issues and discussions for the same behavior.
- Review the [troubleshooting discussion](https://github.com/timothystewart6/k3s-ansible/discussions/20).
- Remove tokens, credentials, public IP addresses, private hostnames, and other sensitive values from logs and
configuration.
- For support requests, include the k3s-ansible revision, Ansible version, target operating system, architecture,
network provider, relevant sanitized variables, and a minimal reproduction.
Use the bug report template for reproducible defects and the feature request template for proposed behavior.
## Development environment
Fork and clone the repository, then create a branch from the latest `master`:
```bash
git switch master
git pull --ff-only
git switch -c <type>/<short-description>
```
Create a Python environment and install the pinned development dependencies:
```bash
python3 -m venv .env
source .env/bin/activate
python3 -m pip install -r requirements.txt
ansible-galaxy collection install -r collections/requirements.yml
pre-commit install
```
Copy `ansible.example.cfg` to the ignored `ansible.cfg` file if local Ansible configuration is needed. Start custom
inventories from `inventory/sample/`, keep them out of Git, and never use production credentials in tests.
## Making changes
- Keep each pull request focused on one problem or feature.
- Follow [AGENTS.md](AGENTS.md) for repository structure, implementation conventions, and safety requirements.
- Preserve idempotence and existing-cluster compatibility.
- Add or update tests for behavior changes and regressions.
- Update role defaults, sample inventory, and documentation when user-facing variables change.
- Consider both provisioning and reset behavior for persistent resources.
- Avoid unrelated reformatting and generated files.
## Validation
Run all pre-commit checks before submitting a pull request:
```bash
pre-commit run --all-files
```
For playbook or role changes, run syntax checks:
```bash
ansible-playbook site.yml --syntax-check -i inventory/sample/hosts.ini
ansible-playbook reset.yml --syntax-check -i inventory/sample/hosts.ini
```
Run the most relevant Molecule scenario when Vagrant, VirtualBox, and the required host networking are available:
```bash
molecule test --scenario-name <scenario>
```
See [molecule/README.md](molecule/README.md) for scenario details and local requirements. Pull requests should list
every check that was run and clearly identify checks that could not be run locally.
## Commits and pull requests
Use a conventional commit subject with a scope:
```text
type(scope): short description
```
Common types are `feat`, `fix`, `refactor`, `test`, `docs`, and `chore`. Write subjects in the imperative mood and
keep commits logically focused.
Pull requests should:
- Describe the problem and the resulting behavior.
- Identify compatibility, security, networking, and upgrade risks.
- Include testing evidence without sensitive data.
- Call out documentation and sample configuration changes.
- Link related issues with `Fixes #<issue>` when applicable.
- Avoid checking boxes for tests that were not run.
Maintainers may ask for a change to be split when unrelated work makes it difficult to review or roll back.
+29 -183
View File
@@ -2,70 +2,51 @@
![Fully Automated K3S etcd High Availability Install](https://img.youtube.com/vi/CbkEWcUZ7zM/0.jpg) ![Fully Automated K3S etcd High Availability Install](https://img.youtube.com/vi/CbkEWcUZ7zM/0.jpg)
This Ansible collection builds a highly available Kubernetes cluster with k3s. It supports kube-vip for the control This playbook will build an HA Kubernetes cluster with `k3s`, `kube-vip` and MetalLB via `ansible`.
plane virtual IP, multiple CNI options, and either MetalLB or kube-vip for service load balancing.
This is based on the work from [this fork](https://github.com/212850a/k3s-ansible) which is based on the work from [k3s-io/k3s-ansible](https://github.com/k3s-io/k3s-ansible). It uses [kube-vip](https://kube-vip.io/) to create a load balancer for control plane, and [metal-lb](https://metallb.universe.tf/installation/) for its service `LoadBalancer`. This is based on the work from [this fork](https://github.com/212850a/k3s-ansible) which is based on the work from [k3s-io/k3s-ansible](https://github.com/k3s-io/k3s-ansible). It uses [kube-vip](https://kube-vip.io/) to create a load balancer for control plane, and [metal-lb](https://metallb.universe.tf/installation/) for its service `LoadBalancer`.
For more context on how it works, see: If you want more context on how this works, see:
📄 [Documentation](https://technotim.com/posts/k3s-etcd-ansible/) (including example commands) 📄 [Documentation](https://technotim.live/posts/k3s-etcd-ansible/) (including example commands)
📺 [Watch the Video](https://www.youtube.com/watch?v=CbkEWcUZ7zM) 📺 [Watch the Video](https://www.youtube.com/watch?v=CbkEWcUZ7zM)
## Project guides
- [Getting started](#-getting-started)
- [Configuration variables](#variables)
- [Upgrading an existing cluster](#-upgrading-an-existing-cluster)
- [Local Molecule testing](molecule/README.md)
- [Contributing guidelines](CONTRIBUTING.md)
- [Repository guide for coding agents](AGENTS.md)
## 📖 k3s Ansible Playbook ## 📖 k3s Ansible Playbook
Build a Kubernetes cluster using Ansible and k3s. The goal is to make a highly available cluster straightforward to Build a Kubernetes cluster using Ansible with k3s. The goal is easily install a HA Kubernetes cluster on machines running:
install on machines running:
- [x] Debian (tested on version 13) - [x] Debian (tested on version 11)
- [x] Ubuntu (tested on version 26.04 LTS) - [x] Ubuntu (tested on version 22.04)
- [x] Rocky (tested on version 10) - [x] Rocky (tested on version 9)
Supported processor architectures are: on processor architecture:
- [x] x64 - [X] x64
- [x] arm64 - [X] arm64
- [x] armhf - [X] armhf
## ✅ System requirements ## ✅ System requirements
- The control node, which runs the Ansible commands, must have Ansible 2.11 or newer. For a quick primer, see - Control Node (the machine you are running `ansible` commands) must have Ansible 2.11+ If you need a quick primer on Ansible [you can check out my docs and setting up Ansible](https://technotim.live/posts/ansible-automation/).
[setting up Ansible](https://technotim.com/posts/ansible-automation/).
- Install the required collections with - You will also need to install collections that this playbook uses by running `ansible-galaxy collection install -r ./collections/requirements.yml` (important❗)
`ansible-galaxy collection install -r ./collections/requirements.yml`.
- [`netaddr` package](https://pypi.org/project/netaddr/) must be available to Ansible. If you have installed Ansible via apt, this is already taken care of. If you have installed Ansible via `pip`, make sure to install `netaddr` into the respective virtual environment. - [`netaddr` package](https://pypi.org/project/netaddr/) must be available to Ansible. If you have installed Ansible via apt, this is already taken care of. If you have installed Ansible via `pip`, make sure to install `netaddr` into the respective virtual environment.
- Server and agent nodes should support passwordless SSH access. Otherwise, pass `--ask-pass --ask-become-pass` to - `server` and `agent` nodes should have passwordless SSH access, if not you can supply arguments to provide credentials `--ask-pass --ask-become-pass` to each command.
each playbook command.
- Every node in the cluster must have a **unique hostname**. k3s registers each node keyed by its hostname, so
two nodes with the same hostname cannot join the cluster. `site.yml` asserts this up front and fails fast if
any duplicate is found.
## 🚀 Getting Started ## 🚀 Getting Started
### 🍴 Preparation ### 🍴 Preparation
Create a cluster-specific inventory from the sample. The `inventory/` directory ignores custom inventory content so First create a new directory based on the `sample` directory within the `inventory` directory:
credentials and environment details aren't committed accidentally.
```bash ```bash
cp -R inventory/sample inventory/my-cluster cp -R inventory/sample inventory/my-cluster
``` ```
Edit `inventory/my-cluster/hosts.ini` to match the target hosts. Second, edit `inventory/my-cluster/hosts.ini` to match the system information gathered above
For example: For example:
@@ -86,10 +67,9 @@ node
If multiple hosts are in the master group, the playbook will automatically set up k3s in [HA mode with etcd](https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/). If multiple hosts are in the master group, the playbook will automatically set up k3s in [HA mode with etcd](https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/).
Copy `ansible.example.cfg` to `ansible.cfg`, then update its inventory path. The local `ansible.cfg` file is ignored by Finally, copy `ansible.example.cfg` to `ansible.cfg` and adapt the inventory path to match the files that you just created.
Git.
The minimum k3s version is `1.19.1`. Select the desired version with the `k3s_version` variable. This requires at least k3s version `1.19.1` however the version is configurable by using the `k3s_version` variable.
If needed, you can also edit `inventory/my-cluster/group_vars/all.yml` to match your environment. If needed, you can also edit `inventory/my-cluster/group_vars/all.yml` to match your environment.
@@ -101,8 +81,7 @@ Start provisioning of the cluster using the following command:
ansible-playbook site.yml -i inventory/my-cluster/hosts.ini ansible-playbook site.yml -i inventory/my-cluster/hosts.ini
``` ```
After deployment, the control plane is accessible through the virtual IP defined by `apiserver_endpoint` in the After deployment control plane will be accessible via virtual ip-address which is defined in inventory/group_vars/all.yml as `apiserver_endpoint`
inventory variables.
### 🔥 Remove k3s cluster ### 🔥 Remove k3s cluster
@@ -110,51 +89,7 @@ inventory variables.
ansible-playbook reset.yml -i inventory/my-cluster/hosts.ini ansible-playbook reset.yml -i inventory/my-cluster/hosts.ini
``` ```
> Reboot the nodes after reset because the virtual IP may remain configured. >You should also reboot these nodes due to the VIP not being destroyed
### ⏻️ Reboot Cluster Nodes
Reboot all cluster nodes at once or stage the reboot across the cluster.
```bash
ansible-playbook reboot.yml -i inventory/my-cluster/hosts.ini
```
To reboot the nodes in batches, set `concurrent_reboots` to the number of nodes
to reboot at a time (or a percentage). Optionally set `wait_seconds_after_reboot`
to pause after each batch so pods in the freshly rebooted batch can settle
before the next batch reboots.
```bash
ansible-playbook reboot.yml -i inventory/my-cluster/hosts.ini \
--extra-vars 'concurrent_reboots=2 wait_seconds_after_reboot=30'
```
## 🔁 Upgrading an existing cluster
These version variables select the components used for a **fresh** installation.
They are not a supported direct in-place upgrade path for an existing cluster.
K3s, Calico, and Cilium each require staged upgrades for long-lived clusters.
- **K3s**: do not jump an embedded-etcd cluster straight to Kubernetes 1.36.
Upgrade one Kubernetes minor version at a time. From the sample default
(`v1.30.2+k3s2`) the sequence is: the latest supported 1.30 patch, then 1.31,
1.32, a 1.33 patch that contains etcd 3.5.26 (for example `v1.33.7+k3s3`),
then 1.34, 1.35, and finally 1.36. Upgrade servers one at a time before
agents. Take backups and confirm cluster health at each step; this playbook
does not automate the upgrade, so those remain manual operational steps. See
[K3s manual upgrades](https://docs.k3s.io/upgrades/manual) and the
[v1.34 release notes](https://docs.k3s.io/release-notes/v1.34.X).
- **Cilium**: upstream supports only consecutive minor upgrades. Update to the
latest patch of the current minor, then upgrade 1.17, 1.18, 1.19, and 1.20 in
order, reading each version's upgrade notes and running preflight checks.
Do not attempt a direct upgrade from an old Cilium to 1.20.
- **Calico**: starting with 3.28 the v3 resource UID behavior changed. If you
have operators with OwnerReferences pointing to `projectcalico.org/v3`
resources, remove and recreate those references around an in-place upgrade.
- **MetalLB**: this project installs application tag `v0.16.0`. A newer
chart-only tag such as `metallb-chart-0.16.1` is not an application or image
release and must not be used as the controller or speaker image tag.
## ⚙️ Kube Config ## ⚙️ Kube Config
@@ -163,21 +98,13 @@ To copy your `kube config` locally so that you can access your **Kubernetes** cl
```bash ```bash
scp debian@master_ip:/etc/rancher/k3s/k3s.yaml ~/.kube/config scp debian@master_ip:/etc/rancher/k3s/k3s.yaml ~/.kube/config
``` ```
If the copy fails with a permission error, grant the SSH user temporary read access using the least permissive method If you get file Permission denied, go into the node and temporarly run:
available for the target system. Restore the original ownership and permissions immediately after copying. Avoid
world-writable permissions on the kubeconfig because it contains cluster credentials.
For example, copy the file to a temporary user-readable path from the control node:
```bash ```bash
ssh debian@master_ip 'sudo install -o "$(id -un)" -m 0600 /etc/rancher/k3s/k3s.yaml /tmp/k3s.yaml' sudo chmod 777 /etc/rancher/k3s/k3s.yaml
``` ```
Then copy with the scp command and reset the permissions back to:
Copy `/tmp/k3s.yaml`, then remove the temporary remote copy:
```bash ```bash
scp debian@master_ip:/tmp/k3s.yaml ~/.kube/config sudo chmod 600 /etc/rancher/k3s/k3s.yaml
ssh debian@master_ip rm -f /tmp/k3s.yaml
``` ```
You'll then want to modify the config to point to master IP by running: You'll then want to modify the config to point to master IP by running:
@@ -188,90 +115,11 @@ Then change `server: https://127.0.0.1:6443` to match your master IP: `server: h
### 🔨 Testing your cluster ### 🔨 Testing your cluster
See the commands [here](https://technotim.com/posts/k3s-etcd-ansible/#testing-your-cluster). See the commands [here](https://technotim.live/posts/k3s-etcd-ansible/#testing-your-cluster).
### Variables
| Role(s) | Variable | Type | Default | Required | Description |
|---|---|---|---|---|---|
| `download` | `k3s_version` | string | ❌ | Required | K3s binaries version |
| `k3s_agent`, `k3s_server`, `k3s_server_post` | `apiserver_endpoint` | string | ❌ | Required | Virtual ip-address configured on each master |
| `k3s_agent` | `extra_agent_args` | string | `null` | Not required | Extra arguments for agents nodes |
| `k3s_agent`, `k3s_server` | `group_name_master` | string | `null` | Not required | Name of the master group |
| `k3s_agent` | `k3s_token` | string | `null` | Not required | Token used to communicate between masters |
| `k3s_agent`, `k3s_server` | `proxy_env` | dict | `null` | Not required | Internet proxy configurations |
| `k3s_agent`, `k3s_server` | `proxy_env.HTTP_PROXY` | string | ❌ | Required | HTTP internet proxy |
| `k3s_agent`, `k3s_server` | `proxy_env.HTTPS_PROXY` | string | ❌ | Required | HTTP internet proxy |
| `k3s_agent`, `k3s_server` | `proxy_env.NO_PROXY` | string | ❌ | Required | Addresses that will not use the proxies |
| `k3s_agent`, `k3s_server`, `reset` | `systemd_dir` | string | `/etc/systemd/system` | Not required | Path to systemd services |
| `k3s_custom_registries` | `custom_registries_yaml` | string | ❌ | Required | YAML block defining custom registries. The following is an example that pulls all images used in this playbook through your private registries. It also allows you to pull your own images from your private registry, without having to use imagePullSecrets in your deployments. If all you need is your own images and you don't care about caching the docker/quay/ghcr.io images, you can just remove those from the mirrors: section. |
| `k3s_server`, `k3s_server_post` | `cilium_bgp` | bool | `~` | Not required | Enable cilium BGP control plane for LB services and pod cidrs. Disables the use of MetalLB. |
| `k3s_server`, `k3s_server_post` | `cilium_iface` | string | ❌ | Not required | The network interface used for when Cilium is enabled |
| `k3s_server` | `extra_server_args` | string | `""` | Not required | Extra arguments for server nodes |
| `k3s_server` | `k3s_create_kubectl_symlink` | bool | `false` | Not required | Create the kubectl -> k3s symlink |
| `k3s_server` | `k3s_create_crictl_symlink` | bool | `true` | Not required | Create the crictl -> k3s symlink |
| `k3s_server` | `kube_vip_arp` | bool | `true` | Not required | Enables kube-vip ARP broadcasts |
| `k3s_server` | `kube_vip_bgp` | bool | `false` | Not required | Enables kube-vip BGP peering |
| `k3s_server` | `kube_vip_bgp_routerid` | string | `"127.0.0.1"` | Not required | Defines the router ID for the kube-vip BGP server |
| `k3s_server` | `kube_vip_bgp_as` | string | `"64513"` | Not required | Defines the AS for the kube-vip BGP server |
| `k3s_server` | `kube_vip_bgp_peeraddress` | string | `"192.168.30.1"` | Not required | Defines the address for the kube-vip BGP peer |
| `k3s_server` | `kube_vip_bgp_peeras` | string | `"64512"` | Not required | Defines the AS for the kube-vip BGP peer |
| `k3s_server` | `kube_vip_bgp_peers` | list | `[]` | Not required | List of BGP peer ASN & address pairs |
| `k3s_server` | `kube_vip_bgp_peers_groups` | list | `['k3s_master']` | Not required | Inventory group in which to search for additional `kube_vip_bgp_peers` parameters to merge. |
| `k3s_server` | `kube_vip_iface` | string | `~` | Not required | Explicitly define an interface that ALL control nodes should use to propagate the VIP, define it here. Otherwise, kube-vip will determine the right interface automatically at runtime. |
| `k3s_server` | `kube_vip_endpoint` | string | `~` | Not required | Overrides the internal address kube-vip binds/listens on, which can differ from the announced apiserver_endpoint for complex routing/tunnels. Defaults to apiserver_endpoint. |
| `k3s_server` | `kube_vip_tag_version` | string | `v1.2.3` | Not required | Image tag for kube-vip |
| `k3s_server` | `kube_vip_cloud_provider_tag_version` | string | `v0.0.12` | Not required | Tag for kube-vip-cloud-provider manifest when enable |
| `k3s_server` | `kube_vip_enabled` | bool | `true` | Not required | Enable kube-vip install, covering both the control-plane VIP and the service load balancer. Set false to skip kube-vip entirely (single node or external LB). |
| `k3s_server`, `k3_server_post` | `kube_vip_lb_ip_range` | string | `~` | Not required | IP range for kube-vip load balancer |
| `k3s_server`, `k3s_server_post` | `metal_lb_enabled` | bool | `true` | Not required | Enable MetalLB install for service load balancing. Set false to skip MetalLB (external LB). |
| `k3s_server`, `k3s_server_post` | `metal_lb_controller_tag_version` | string | `v0.16.0` | Not required | Image tag for MetalLB |
| `k3s_server` | `metal_lb_speaker_tag_version` | string | `v0.16.0` | Not required | Image tag for MetalLB |
| `k3s_server` | `metal_lb_type` | string | `native` | Not required | Use FRR mode or native. Valid values are `frr` and `native` |
| `k3s_server` | `retry_count` | int | `20` | Not required | Amount of retries when verifying that nodes joined |
| `k3s_server` | `server_init_args` | string | ❌ | Not required | Arguments for server nodes |
| `k3s_server_post` | `bpf_lb_algorithm` | string | `maglev` | Not required | BPF lb algorithm |
| `k3s_server_post` | `bpf_lb_mode` | string | `hybrid` | Not required | BPF lb mode |
| `k3s_server_post` | `calico_blocksize` | int | `26` | Not required | IP pool block size |
| `k3s_server_post` | `calico_ebpf` | bool | `false` | Not required | Use eBPF dataplane instead of iptables |
| `k3s_server_post` | `calico_encapsulation` | string | `VXLANCrossSubnet` | Not required | IP pool encapsulation |
| `k3s_server_post` | `calico_natOutgoing` | string | `Enabled` | Not required | IP pool NAT outgoing |
| `k3s_server_post` | `calico_nodeSelector` | string | `all()` | Not required | IP pool node selector |
| `k3s_server_post` | `calico_iface` | string | `~` | Not required | The network interface used for when Calico is enabled |
| `k3s_server_post` | `calico_tag` | string | `v3.32.1` | Not required | Calico version tag |
| `k3s_server_post` | `cilium_bgp_my_asn` | int | `64513` | Not required | Local ASN for BGP peer |
| `k3s_server_post` | `cilium_bgp_peer_asn` | int | `64512` | Not required | BGP peer ASN |
| `k3s_server_post` | `cilium_bgp_peer_address` | string | `~` | Not required | BGP peer address |
| `k3s_server_post` | `cilium_bgp_neighbors` | list | `[]` | Not required | List of BGP peer ASN & address pairs |
| `k3s_server_post` | `cilium_bgp_neighbors_groups` | list | `['k3s_all']` | Not required | Inventory group in which to search for additional `cilium_bgp_neighbors` parameters to merge. |
| `k3s_server_post` | `cilium_bgp_lb_cidr` | string | `192.168.31.0/24` | Not required | BGP load balancer IP range |
| `k3s_server_post` | `cilium_exportPodCIDR` | bool | `true` | Not required | Export pod CIDR |
| `k3s_server_post` | `cilium_hubble` | bool | `true` | Not required | Enable Cilium Hubble |
| `k3s_server_post` | `cilium_mode` | string | `native` | Not required | Inner-node communication mode (choices are `native` and `tunnel`; `routed` is a deprecated alias for `tunnel`) |
| `k3s_server_post` | `cilium_tag` | string | `v1.20.0` | Not required | Cilium version tag |
| `k3s_server_post` | `cilium_cli_tag` | string | `v0.19.7` | Not required | Cilium CLI version tag |
| `k3s_server_post` | `cluster_cidr` | string | `10.52.0.0/16` | Not required | Inner-cluster IP range |
| `k3s_server_post` | `enable_bpf_masquerade` | bool | `true` | Not required | Use IP masquerading |
| `k3s_server_post` | `kube_proxy_replacement` | bool | `true` | Not required | Replace the native kube-proxy with Cilium |
| `k3s_server_post` | `metal_lb_available_timeout` | string | `240s` | Not required | Wait for MetalLB resources |
| `k3s_server_post` | `metal_lb_ip_range` | string | `192.168.30.80-192.168.30.90` | Not required | MetalLB ip range for load balancer |
| `k3s_server_post` | `metal_lb_controller_tag_version` | string | `v0.16.0` | Not required | Image tag for MetalLB |
| `k3s_server_post` | `metal_lb_mode` | string | `layer2` | Not required | Metallb mode (choices are `bgp` and `layer2`) |
| `k3s_server_post` | `metal_lb_bgp_my_asn` | string | `~` | Not required | BGP ASN configurations |
| `k3s_server_post` | `metal_lb_bgp_peer_asn` | string | `~` | Not required | BGP peer ASN configurations |
| `k3s_server_post` | `metal_lb_bgp_peer_address` | string | `~` | Not required | BGP peer address |
| `lxc` | `custom_reboot_command` | string | `~` | Not required | Command to run on reboot |
| `reboot` (playbook) | `concurrent_reboots` | int/string | `100%` | Not required | Number (or percentage) of nodes to reboot at a time for a staggered reboot |
| `reboot` (playbook) | `wait_seconds_after_reboot` | int | `0` | Not required | Pause in seconds between staggered reboot batches |
| `prereq` | `system_timezone` | string | `null` | Not required | Timezone to be set on all nodes |
| `prereq` | `disable_swap` | bool | `true` | Not required | Disable swap on all cluster nodes (swapoff + comment out /etc/fstab swap entries), all-or-nothing |
| `proxmox_lxc`, `reset_proxmox_lxc` | `proxmox_lxc_ct_ids` | list | ❌ | Required | Proxmox container ID list |
| `raspberrypi` | `state` | string | `present` | Not required | Indicates whether the k3s prerequisites for Raspberry Pi should be set up (possible values are `present` and `absent`) |
### Troubleshooting ### Troubleshooting
Be sure to see [this post](https://github.com/timothystewart6/k3s-ansible/discussions/20) on how to troubleshoot common problems Be sure to see [this post](https://github.com/techno-tim/k3s-ansible/discussions/20) on how to troubleshoot common problems
### Testing the playbook using molecule ### Testing the playbook using molecule
@@ -280,11 +128,9 @@ It is run automatically in CI, but you can also run the tests locally.
This might be helpful for quick feedback in a few cases. This might be helpful for quick feedback in a few cases.
You can find more information about it [here](molecule/README.md). You can find more information about it [here](molecule/README.md).
### Pre-commit hooks ### Pre-commit Hooks
This repository uses `pre-commit` to check style, syntax, Ansible content, and shell scripts. Install the Python This repo uses `pre-commit` and `pre-commit-hooks` to lint and fix common style and syntax errors. Be sure to install python packages and then run `pre-commit install`. For more information, see [pre-commit](https://pre-commit.com/)
dependencies, run `pre-commit install` once, and run `pre-commit run --all-files` before submitting a change. See
[CONTRIBUTING.md](CONTRIBUTING.md) for the complete development workflow.
## 🌌 Ansible Galaxy ## 🌌 Ansible Galaxy
@@ -300,7 +146,7 @@ collections:
- name: community.general - name: community.general
- name: ansible.posix - name: ansible.posix
- name: kubernetes.core - name: kubernetes.core
- name: https://github.com/timothystewart6/k3s-ansible.git - name: https://github.com/techno-tim/k3s-ansible.git
type: git type: git
version: master version: master
``` ```
+3 -3
View File
@@ -56,16 +56,16 @@ dependencies:
kubernetes.core: '*' kubernetes.core: '*'
# The URL of the originating SCM repository # The URL of the originating SCM repository
repository: https://github.com/timothystewart6/k3s-ansible repository: https://github.com/techno-tim/k3s-ansible
# The URL to any online docs # The URL to any online docs
documentation: https://github.com/timothystewart6/k3s-ansible documentation: https://github.com/techno-tim/k3s-ansible
# The URL to the homepage of the collection/project # The URL to the homepage of the collection/project
homepage: https://www.youtube.com/watch?v=CbkEWcUZ7zM homepage: https://www.youtube.com/watch?v=CbkEWcUZ7zM
# The URL to the collection issue tracker # The URL to the collection issue tracker
issues: https://github.com/timothystewart6/k3s-ansible/issues issues: https://github.com/techno-tim/k3s-ansible/issues
# A list of file glob-like patterns used to filter any files or directories that should not be included in the build # A list of file glob-like patterns used to filter any files or directories that should not be included in the build
# artifact. A pattern is matched from the relative path of the file or directory of the collection directory. This # artifact. A pattern is matched from the relative path of the file or directory of the collection directory. This
+12 -44
View File
@@ -1,5 +1,5 @@
--- ---
k3s_version: v1.36.4+k3s1 k3s_version: v1.30.2+k3s2
# this is the user that has ssh access to these machines # this is the user that has ssh access to these machines
ansible_user: ansibleuser ansible_user: ansibleuser
systemd_dir: /etc/systemd/system systemd_dir: /etc/systemd/system
@@ -7,31 +7,20 @@ systemd_dir: /etc/systemd/system
# Set your timezone # Set your timezone
system_timezone: Your/Timezone system_timezone: Your/Timezone
# k3s recommends swap be disabled on every cluster node. Applied uniformly to all
# nodes (all-or-nothing) in the prereq role. Set to false to leave swap enabled.
disable_swap: true
# interface which will be used for flannel # interface which will be used for flannel
# Defaults to each host's default IPv4 interface (e.g. eth0, enp1s0, ens3) flannel_iface: eth0
# so KVM/cloud hosts without eth0 work out of the box. Override per-host if needed.
flannel_iface: "{{ ansible_facts.default_ipv4.interface }}"
# uncomment calico_iface to use tigera operator/calico cni instead of flannel https://docs.tigera.io/calico/latest/about # uncomment calico_iface to use tigera operator/calico cni instead of flannel https://docs.tigera.io/calico/latest/about
# calico_iface: "{{ ansible_facts.default_ipv4.interface }}" # calico_iface: "eth0"
calico_ebpf: false # use eBPF dataplane instead of iptables calico_ebpf: false # use eBPF dataplane instead of iptables
calico_tag: v3.32.1 # calico version tag calico_tag: v3.28.0 # calico version tag
# uncomment cilium_iface to use cilium cni instead of flannel or calico # uncomment cilium_iface to use cilium cni instead of flannel or calico
# ensure v4.19.57, v5.1.16, v5.2.0 or more recent kernel # ensure v4.19.57, v5.1.16, v5.2.0 or more recent kernel
# cilium_iface: "{{ ansible_facts.default_ipv4.interface }}" # cilium_iface: "eth0"
cilium_mode: native # native when nodes are on the same subnet or use BGP, otherwise set tunnel cilium_mode: native # native when nodes on same subnet or using bgp, else set routed
cilium_tag: v1.20.0 # cilium version tag cilium_tag: v1.16.0 # cilium version tag
cilium_cli_tag: v0.19.7 # cilium cli version tag
cilium_hubble: true # enable hubble observability relay and ui cilium_hubble: true # enable hubble observability relay and ui
cilium_envoy: true # enable the Envoy proxy for Cilium L7 policies
# disable cilium_envoy to skip the Envoy proxy entirely (e.g. no L7 policies)
# cilium_envoy: false
# if using calico or cilium, you may specify the cluster pod cidr pool # if using calico or cilium, you may specify the cluster pod cidr pool
cluster_cidr: 10.52.0.0/16 cluster_cidr: 10.52.0.0/16
@@ -48,11 +37,6 @@ cilium_bgp_lb_cidr: 192.168.31.0/24 # cidr for cilium loadbalancer ipam
# enable kube-vip ARP broadcasts # enable kube-vip ARP broadcasts
kube_vip_arp: true kube_vip_arp: true
# (optional) overrides the address kube-vip binds/listens on internally, which
# can differ from the announced apiserver_endpoint for complex routing/tunnels.
# Defaults to apiserver_endpoint. Also used to derive the kube-vip subnet.
# kube_vip_endpoint: 10.66.1.5
# enable kube-vip BGP peering # enable kube-vip BGP peering
kube_vip_bgp: false kube_vip_bgp: false
@@ -62,9 +46,7 @@ kube_vip_bgp_as: "64513" # Defines the AS for the BGP server
kube_vip_bgp_peeraddress: "192.168.30.1" # Defines the address for the BGP peer kube_vip_bgp_peeraddress: "192.168.30.1" # Defines the address for the BGP peer
kube_vip_bgp_peeras: "64512" # Defines the AS for the BGP peer kube_vip_bgp_peeras: "64512" # Defines the AS for the BGP peer
# apiserver_endpoint is virtual ip-address which will be configured on each master. # apiserver_endpoint is virtual ip-address which will be configured on each master
# This must be a free, routable IP on your network (not already assigned to a host
# or service), and is used by kube-vip / MetalLB to expose the Kubernetes API.
apiserver_endpoint: 192.168.30.222 apiserver_endpoint: 192.168.30.222
# k3s_token is required masters can talk together securely # k3s_token is required masters can talk together securely
@@ -102,23 +84,15 @@ extra_agent_args: >-
{{ extra_args }} {{ extra_args }}
# image tag for kube-vip # image tag for kube-vip
kube_vip_tag_version: v1.2.3 kube_vip_tag_version: v0.8.2
# enable kube-vip (covers both the control-plane VIP and service load balancing)
# set false for a single node or when an external load balancer is used
kube_vip_enabled: true
# tag for kube-vip-cloud-provider manifest # tag for kube-vip-cloud-provider manifest
# kube_vip_cloud_provider_tag_version: "v0.0.12" # kube_vip_cloud_provider_tag_version: "main"
# kube-vip ip range for load balancer # kube-vip ip range for load balancer
# (uncomment to use kube-vip for services instead of MetalLB) # (uncomment to use kube-vip for services instead of MetalLB)
# kube_vip_lb_ip_range: "192.168.30.80-192.168.30.90" # kube_vip_lb_ip_range: "192.168.30.80-192.168.30.90"
# enable MetalLB for service load balancing
# set false when using an external load balancer
metal_lb_enabled: true
# metallb type frr or native # metallb type frr or native
metal_lb_type: native metal_lb_type: native
@@ -131,18 +105,12 @@ metal_lb_mode: layer2
# metal_lb_bgp_peer_address: "192.168.30.1" # metal_lb_bgp_peer_address: "192.168.30.1"
# image tag for metal lb # image tag for metal lb
metal_lb_speaker_tag_version: v0.16.0 metal_lb_speaker_tag_version: v0.14.8
metal_lb_controller_tag_version: v0.16.0 metal_lb_controller_tag_version: v0.14.8
# metallb ip range for load balancer # metallb ip range for load balancer
metal_lb_ip_range: 192.168.30.80-192.168.30.90 metal_lb_ip_range: 192.168.30.80-192.168.30.90
# (optional) limit MetalLB layer2 announcements to specific network interfaces.
# Leave empty (default) to announce on all interfaces.
# metal_lb_interfaces:
# - eth1
# - eth2
# Only enable if your nodes are proxmox LXC nodes, make sure to configure your proxmox nodes # Only enable if your nodes are proxmox LXC nodes, make sure to configure your proxmox nodes
# in your hosts.ini file. # in your hosts.ini file.
# Please read https://gist.github.com/triangletodd/02f595cd4c0dc9aac5f7763ca2264185 before using this. # Please read https://gist.github.com/triangletodd/02f595cd4c0dc9aac5f7763ca2264185 before using this.
+6 -1
View File
@@ -5,9 +5,14 @@ driver:
name: vagrant name: vagrant
platforms: platforms:
- name: control1 - name: control1
box: bento/ubuntu-26.04 box: generic/ubuntu2204
memory: 4096 memory: 4096
cpus: 4 cpus: 4
config_options:
# We currently can not use public-key based authentication on Ubuntu 22.04,
# see: https://github.com/chef/bento/issues/1405
ssh.username: vagrant
ssh.password: vagrant
groups: groups:
- k3s_cluster - k3s_cluster
- master - master
+1 -3
View File
@@ -1,14 +1,12 @@
--- ---
- name: Apply overrides - name: Apply overrides
hosts: all hosts: all
serial: 1
tasks: tasks:
- name: Override host variables - name: Override host variables
ansible.builtin.set_fact: ansible.builtin.set_fact:
# See: # See:
# https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
calico_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}" calico_iface: eth1
kube_vip_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}"
# The test VMs might be a bit slow, so we give them more time to join the cluster: # The test VMs might be a bit slow, so we give them more time to join the cluster:
retry_count: 45 retry_count: 45
-6
View File
@@ -1,6 +0,0 @@
---
# Durable verify inputs for the calico (Calico CNI + MetalLB) scenario.
verify_cni: calico
verify_lb: metallb
verify_lb_ip_range:
- 192.168.30.100-192.168.30.109
+6 -1
View File
@@ -5,9 +5,14 @@ driver:
name: vagrant name: vagrant
platforms: platforms:
- name: control1 - name: control1
box: bento/ubuntu-26.04 box: generic/ubuntu2204
memory: 4096 memory: 4096
cpus: 4 cpus: 4
config_options:
# We currently can not use public-key based authentication on Ubuntu 22.04,
# see: https://github.com/chef/bento/issues/1405
ssh.username: vagrant
ssh.password: vagrant
groups: groups:
- k3s_cluster - k3s_cluster
- master - master
+1 -3
View File
@@ -1,14 +1,12 @@
--- ---
- name: Apply overrides - name: Apply overrides
hosts: all hosts: all
serial: 1
tasks: tasks:
- name: Override host variables - name: Override host variables
ansible.builtin.set_fact: ansible.builtin.set_fact:
# See: # See:
# https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
cilium_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}" cilium_iface: eth1
kube_vip_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}"
# The test VMs might be a bit slow, so we give them more time to join the cluster: # The test VMs might be a bit slow, so we give them more time to join the cluster:
retry_count: 45 retry_count: 45
-6
View File
@@ -1,6 +0,0 @@
---
# Durable verify inputs for the cilium (Cilium CNI + MetalLB) scenario.
verify_cni: cilium
verify_lb: metallb
verify_lb_ip_range:
- 192.168.30.110-192.168.30.119
-80
View File
@@ -1,80 +0,0 @@
---
- name: Create
hosts: localhost
connection: local
gather_facts: false
no_log: "{{ molecule_no_log }}"
vars:
create_batches:
- [control1, control2]
- [control3, node1]
- [node2]
tasks:
- name: Verify that bounded batches cover the configured platforms exactly once
ansible.builtin.assert:
that:
- create_batches | flatten | sort == molecule_yml.platforms | map(attribute='name') | sort
- create_batches | flatten | length == create_batches | flatten | unique | length
- create_batches | map('length') | max <= 2
fail_msg: Bounded create batches do not match the configured default platforms.
- name: Generate the complete Vagrant configuration # noqa fqcn[action]
vagrant:
instances: "{{ molecule_yml.platforms }}"
default_box: "{{ molecule_yml.driver.default_box | default('generic/alpine316') }}"
provider_name: "{{ molecule_yml.driver.provider.name | default(omit, true) }}"
provision: "{{ molecule_yml.driver.provision | default(omit) }}"
cachier: "{{ molecule_yml.driver.cachier | default(omit) }}"
parallel: false
state: halt
changed_when: false
- name: Start clean Vagrant guests in bounded batches
ansible.builtin.command:
argv: >-
{{
[playbook_dir + '/../../.github/scripts/vagrant-up-timed.sh',
molecule_ephemeral_directory] + item
}}
loop: "{{ create_batches }}"
loop_control:
label: "{{ item | join(', ') }}"
changed_when: true
- name: Reconcile all instances and collect their connection configuration # noqa fqcn[action]
vagrant:
instances: "{{ molecule_yml.platforms }}"
default_box: "{{ molecule_yml.driver.default_box | default('generic/alpine316') }}"
provider_name: "{{ molecule_yml.driver.provider.name | default(omit, true) }}"
provision: "{{ molecule_yml.driver.provision | default(omit) }}"
cachier: "{{ molecule_yml.driver.cachier | default(omit) }}"
parallel: false
state: up
register: server
no_log: false
- name: Populate instance configuration dictionaries
ansible.builtin.set_fact:
instance_conf_dict:
instance: "{{ item.Host }}"
address: "{{ item.HostName }}"
user: "{{ item.User }}"
port: "{{ item.Port }}"
identity_file: "{{ item.IdentityFile }}"
loop: "{{ server.results }}"
register: instance_config_dict
- name: Convert instance configuration dictionaries to a list
ansible.builtin.set_fact:
instance_conf: >-
{{
instance_config_dict.results
| map(attribute='ansible_facts.instance_conf_dict')
| list
}}
- name: Write Molecule instance configuration
ansible.builtin.copy:
content: "{{ instance_conf | to_json | from_json | to_yaml }}"
dest: "{{ molecule_instance_config }}"
mode: "0600"
+15 -21
View File
@@ -3,73 +3,68 @@ dependency:
name: galaxy name: galaxy
driver: driver:
name: vagrant name: vagrant
# The Vagrant driver warns that parallel VirtualBox creation can cause
# platform issues. Keep this five-node, mixed-distribution scenario serial.
parallel: false
platforms: platforms:
- name: control1 - name: control1
box: bento/ubuntu-26.04 box: generic/ubuntu2204
memory: 1024 memory: 1024
cpus: 2 cpus: 2
groups: groups:
- k3s_cluster - k3s_cluster
- master - master
# Keep adapter 2 stable across linked-clone rebuilds so stale host-only
# neighbor state still identifies the current scenario guest.
provider_raw_config_args:
- "customize ['modifyvm', :id, '--mac-address2', '080027A13038']"
interfaces: interfaces:
- network_name: private_network - network_name: private_network
ip: 192.168.30.38 ip: 192.168.30.38
config_options:
# We currently can not use public-key based authentication on Ubuntu 22.04,
# see: https://github.com/chef/bento/issues/1405
ssh.username: vagrant
ssh.password: vagrant
- name: control2 - name: control2
box: bento/debian-13 box: generic/debian12
memory: 1024 memory: 1024
cpus: 2 cpus: 2
groups: groups:
- k3s_cluster - k3s_cluster
- master - master
provider_raw_config_args:
- "customize ['modifyvm', :id, '--mac-address2', '080027A13039']"
interfaces: interfaces:
- network_name: private_network - network_name: private_network
ip: 192.168.30.39 ip: 192.168.30.39
- name: control3 - name: control3
box: bento/rockylinux-10.1 box: generic/rocky9
memory: 1024 memory: 1024
cpus: 2 cpus: 2
groups: groups:
- k3s_cluster - k3s_cluster
- master - master
provider_raw_config_args:
- "customize ['modifyvm', :id, '--mac-address2', '080027A13040']"
interfaces: interfaces:
- network_name: private_network - network_name: private_network
ip: 192.168.30.40 ip: 192.168.30.40
- name: node1 - name: node1
box: bento/ubuntu-26.04 box: generic/ubuntu2204
memory: 1024 memory: 1024
cpus: 2 cpus: 2
groups: groups:
- k3s_cluster - k3s_cluster
- node - node
provider_raw_config_args:
- "customize ['modifyvm', :id, '--mac-address2', '080027A13041']"
interfaces: interfaces:
- network_name: private_network - network_name: private_network
ip: 192.168.30.41 ip: 192.168.30.41
config_options:
# We currently can not use public-key based authentication on Ubuntu 22.04,
# see: https://github.com/chef/bento/issues/1405
ssh.username: vagrant
ssh.password: vagrant
- name: node2 - name: node2
box: bento/rockylinux-10.1 box: generic/rocky9
memory: 1024 memory: 1024
cpus: 2 cpus: 2
groups: groups:
- k3s_cluster - k3s_cluster
- node - node
provider_raw_config_args:
- "customize ['modifyvm', :id, '--mac-address2', '080027A13042']"
interfaces: interfaces:
- network_name: private_network - network_name: private_network
ip: 192.168.30.42 ip: 192.168.30.42
@@ -79,7 +74,6 @@ provisioner:
env: env:
ANSIBLE_VERBOSITY: 1 ANSIBLE_VERBOSITY: 1
playbooks: playbooks:
create: create.yml
converge: ../resources/converge.yml converge: ../resources/converge.yml
side_effect: ../resources/reset.yml side_effect: ../resources/reset.yml
verify: ../resources/verify.yml verify: ../resources/verify.yml
+1 -5
View File
@@ -1,16 +1,12 @@
--- ---
- name: Apply overrides - name: Apply overrides
hosts: all hosts: all
serial: 1
tasks: tasks:
- name: Override host variables - name: Override host variables
ansible.builtin.set_fact: ansible.builtin.set_fact:
# See: # See:
# https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
flannel_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}" flannel_iface: eth1
# kube-vip cannot infer the cluster interface in these multi-NIC
# Vagrant guests because the default route is on eth0.
kube_vip_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}"
# The test VMs might be a bit slow, so we give them more time to join the cluster: # The test VMs might be a bit slow, so we give them more time to join the cluster:
retry_count: 45 retry_count: 45
+7 -126
View File
@@ -5,137 +5,18 @@
- name: Network setup - name: Network setup
hosts: all hosts: all
vars:
primary_master: "{{ groups[group_name_master | default('master')][0] }}"
primary_cluster_ip: >-
{{ hostvars[primary_master].k3s_node_ip | split(',') | first }}
cluster_interface: >-
{{ cilium_iface | default(calico_iface | default(flannel_iface)) }}
primary_cluster_interface: >-
{{ hostvars[primary_master].cilium_iface
| default(hostvars[primary_master].calico_iface
| default(hostvars[primary_master].flannel_iface)) }}
primary_cluster_mac: >-
{{ hostvars[primary_master].ansible_facts[primary_cluster_interface].macaddress }}
tasks: tasks:
- name: Gather service facts - name: Disable firewalld
ansible.builtin.service_facts: when: ansible_distribution == "Rocky"
# Rocky Linux comes with firewalld enabled. It blocks some of the network
- name: Disable guest firewall services # connections needed for our k3s cluster. For our test setup, we just disable
# The disposable test guests use an isolated VirtualBox network. A distro # it since the VM host's firewall is still active for connections to and from
# firewall can allow ICMP while silently blocking the inter-node Kubernetes # the Internet.
# API connection, so disable the known guest firewalls consistently.
# When building your own cluster, please DO NOT blindly copy this. Instead, # When building your own cluster, please DO NOT blindly copy this. Instead,
# please create a custom firewall configuration that fits your network design # please create a custom firewall configuration that fits your network design
# and security needs. # and security needs.
ansible.builtin.systemd: ansible.builtin.systemd:
name: "{{ item }}" name: firewalld
enabled: false enabled: false
state: stopped state: stopped
become: true become: true
loop:
- firewalld.service
- nftables.service
- ufw.service
when: item in ansible_facts.services
- name: Verify the private cluster interface
ansible.builtin.assert:
that:
- flannel_iface in ansible_facts
- ansible_facts[flannel_iface].ipv4 is defined
- ansible_facts[flannel_iface].ipv4.address is defined
fail_msg: >-
The Vagrant private interface {{ flannel_iface }} does not have an
IPv4 address on {{ inventory_hostname }}.
- name: Pin disposable cluster peer neighbor entries
ansible.builtin.command:
argv:
- ip
- neigh
- replace
- "{{ peer_cluster_ip }}"
- lladdr
- "{{ peer_cluster_mac }}"
- nud
- permanent
- dev
- "{{ cluster_interface }}"
become: true
changed_when: false
loop: "{{ groups['k3s_cluster'] }}"
loop_control:
label: "{{ inventory_hostname }} -> {{ item }}"
vars:
peer_cluster_interface: >-
{{ hostvars[item].cilium_iface
| default(hostvars[item].calico_iface
| default(hostvars[item].flannel_iface)) }}
peer_cluster_ip: >-
{{ hostvars[item].k3s_node_ip | split(',') | first }}
peer_cluster_mac: >-
{{ hostvars[item].ansible_facts[peer_cluster_interface].macaddress }}
when: item != inventory_hostname
- name: Verify guest-to-guest cluster network reachability
ansible.builtin.command:
argv:
- ping
- -c
- "1"
- -W
- "1"
- "{{ primary_cluster_ip }}"
register: primary_cluster_ping
until: primary_cluster_ping.rc == 0
retries: 6
delay: 2
changed_when: false
- name: Read the primary neighbor entry
ansible.builtin.command:
argv:
- ip
- neigh
- show
- to
- "{{ primary_cluster_ip }}"
- dev
- "{{ cluster_interface }}"
register: primary_cluster_neighbor
changed_when: false
when: inventory_hostname != primary_master
- name: Verify the primary neighbor identity
ansible.builtin.assert:
that:
- (primary_cluster_mac | lower) in (primary_cluster_neighbor.stdout | lower)
fail_msg: >-
{{ inventory_hostname }} resolved primary {{ primary_cluster_ip }} to
an unexpected MAC on {{ cluster_interface }}. Expected
{{ primary_cluster_mac }}, got: {{ primary_cluster_neighbor.stdout }}
when: inventory_hostname != primary_master
- name: Verify GitHub release host DNS
ansible.builtin.getent:
database: hosts
key: github.com
register: github_dns
retries: 6
delay: 5
until: github_dns is succeeded
- name: Verify k3s checksum URL is reachable
ansible.builtin.uri:
url: >-
https://github.com/k3s-io/k3s/releases/download/{{ k3s_version
}}/sha256sum-amd64.txt
method: HEAD
follow_redirects: safe
status_code: [200, 302]
timeout: 15
register: k3s_checksum_request
retries: 3
delay: 5
until: k3s_checksum_request.status in [200, 302]
-9
View File
@@ -1,9 +0,0 @@
---
# Durable verify inputs for the default (flannel + MetalLB) scenario.
# These are plain inventory vars linked into the shared Molecule inventory so
# the verify play can see them even though the converge play's set_fact values
# are not persisted between the two Ansible processes.
verify_cni: flannel
verify_lb: metallb
verify_lb_ip_range:
- 192.168.30.80-192.168.30.90
+18 -3
View File
@@ -5,7 +5,7 @@ driver:
name: vagrant name: vagrant
platforms: platforms:
- name: control1 - name: control1
box: bento/ubuntu-26.04 box: generic/ubuntu2204
memory: 1024 memory: 1024
cpus: 2 cpus: 2
groups: groups:
@@ -14,9 +14,14 @@ platforms:
interfaces: interfaces:
- network_name: private_network - network_name: private_network
ip: fdad:bad:ba55::de:11 ip: fdad:bad:ba55::de:11
config_options:
# We currently can not use public-key based authentication on Ubuntu 22.04,
# see: https://github.com/chef/bento/issues/1405
ssh.username: vagrant
ssh.password: vagrant
- name: control2 - name: control2
box: bento/ubuntu-26.04 box: generic/ubuntu2204
memory: 1024 memory: 1024
cpus: 2 cpus: 2
groups: groups:
@@ -25,9 +30,14 @@ platforms:
interfaces: interfaces:
- network_name: private_network - network_name: private_network
ip: fdad:bad:ba55::de:12 ip: fdad:bad:ba55::de:12
config_options:
# We currently can not use public-key based authentication on Ubuntu 22.04,
# see: https://github.com/chef/bento/issues/1405
ssh.username: vagrant
ssh.password: vagrant
- name: node1 - name: node1
box: bento/ubuntu-26.04 box: generic/ubuntu2204
memory: 1024 memory: 1024
cpus: 2 cpus: 2
groups: groups:
@@ -36,6 +46,11 @@ platforms:
interfaces: interfaces:
- network_name: private_network - network_name: private_network
ip: fdad:bad:ba55::de:21 ip: fdad:bad:ba55::de:21
config_options:
# We currently can not use public-key based authentication on Ubuntu 22.04,
# see: https://github.com/chef/bento/issues/1405
ssh.username: vagrant
ssh.password: vagrant
provisioner: provisioner:
name: ansible name: ansible
env: env:
+2 -3
View File
@@ -1,18 +1,17 @@
--- ---
- name: Apply overrides - name: Apply overrides
hosts: all hosts: all
serial: 1
tasks: tasks:
- name: Override host variables (1/2) - name: Override host variables (1/2)
ansible.builtin.set_fact: ansible.builtin.set_fact:
# See: # See:
# https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
flannel_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}" flannel_iface: eth1
# In this scenario, we have multiple interfaces that the VIP could be # In this scenario, we have multiple interfaces that the VIP could be
# broadcasted on. Since we have assigned a dedicated private network # broadcasted on. Since we have assigned a dedicated private network
# here, let's make sure that it is used. # here, let's make sure that it is used.
kube_vip_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}" kube_vip_iface: eth1
# The test VMs might be a bit slow, so we give them more time to join the cluster: # The test VMs might be a bit slow, so we give them more time to join the cluster:
retry_count: 45 retry_count: 45
-7
View File
@@ -1,7 +0,0 @@
---
# Durable verify inputs for the ipv6 (flannel CNI + MetalLB) scenario.
verify_cni: flannel
verify_lb: metallb
verify_lb_ip_range:
- fdad:bad:ba55::1b:0/112
- 192.168.123.80-192.168.123.90
+6 -1
View File
@@ -5,9 +5,14 @@ driver:
name: vagrant name: vagrant
platforms: platforms:
- name: control1 - name: control1
box: bento/ubuntu-26.04 box: generic/ubuntu2204
memory: 4096 memory: 4096
cpus: 4 cpus: 4
config_options:
# We currently can not use public-key based authentication on Ubuntu 22.04,
# see: https://github.com/chef/bento/issues/1405
ssh.username: vagrant
ssh.password: vagrant
groups: groups:
- k3s_cluster - k3s_cluster
- master - master
+1 -3
View File
@@ -1,14 +1,12 @@
--- ---
- name: Apply overrides - name: Apply overrides
hosts: all hosts: all
serial: 1
tasks: tasks:
- name: Override host variables - name: Override host variables
ansible.builtin.set_fact: ansible.builtin.set_fact:
# See: # See:
# https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
flannel_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}" flannel_iface: eth1
kube_vip_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}"
# The test VMs might be a bit slow, so we give them more time to join the cluster: # The test VMs might be a bit slow, so we give them more time to join the cluster:
retry_count: 45 retry_count: 45
-9
View File
@@ -1,9 +0,0 @@
---
# Durable verify inputs for the kube-vip (flannel CNI + kube-vip LB) scenario.
verify_cni: flannel
verify_lb: kube-vip
# The kube-vip cloud provider tag is not defined in the linked sample group
# vars (its sample entry is commented out), so it is supplied here.
verify_kube_vip_cloud_provider_tag: v0.0.12
verify_lb_ip_range:
- 192.168.30.110-192.168.30.119
-3
View File
@@ -1,8 +1,5 @@
--- ---
- name: Verify - name: Verify
hosts: all hosts: all
vars_files:
- >-
{{ lookup("ansible.builtin.env", "MOLECULE_SCENARIO_DIRECTORY") }}/verify-vars.yml
roles: roles:
- verify_from_outside - verify_from_outside
@@ -7,8 +7,6 @@
ansible.builtin.import_tasks: kubecfg-fetch.yml ansible.builtin.import_tasks: kubecfg-fetch.yml
- name: "TEST CASE: Get nodes" - name: "TEST CASE: Get nodes"
ansible.builtin.include_tasks: test/get-nodes.yml ansible.builtin.include_tasks: test/get-nodes.yml
- name: "TEST CASE: Verify components"
ansible.builtin.include_tasks: test/verify-components.yml
- name: "TEST CASE: Deploy example" - name: "TEST CASE: Deploy example"
ansible.builtin.include_tasks: test/deploy-example.yml ansible.builtin.include_tasks: test/deploy-example.yml
always: always:
@@ -32,95 +32,17 @@
metallb_port: spec.ports[0].port metallb_port: spec.ports[0].port
register: nginx_services register: nginx_services
- name: Wait for the load balancer address to be assigned
ansible.builtin.set_fact:
nginx_lb_ip: >-
{{
nginx_services.resources[0].status.loadBalancer.ingress[0].ip
if (nginx_services.resources | length > 0) and
(nginx_services.resources[0].status.loadBalancer.ingress is defined) and
(nginx_services.resources[0].status.loadBalancer.ingress | length > 0)
else ''
}}
- name: Retry until the load balancer service has an external IP
block:
- name: Refresh nginx service until it has an assigned address
kubernetes.core.k8s_info:
kind: service
name: nginx
namespace: "{{ testing_namespace }}"
kubeconfig: "{{ kubecfg_path }}"
register: nginx_lb_wait
until: >-
(nginx_lb_wait.resources | length > 0) and
(nginx_lb_wait.resources[0].status.loadBalancer.ingress is defined) and
(nginx_lb_wait.resources[0].status.loadBalancer.ingress | length > 0)
retries: 30
delay: 5
- name: Record the assigned load balancer address
ansible.builtin.set_fact:
nginx_lb_ip: >-
{{ nginx_lb_wait.resources[0].status.loadBalancer.ingress[0].ip }}
- name: Assert that the nginx welcome page is available - name: Assert that the nginx welcome page is available
ansible.builtin.uri: ansible.builtin.uri:
url: http://{{ nginx_lb_ip | ansible.utils.ipwrap }}:{{ port_ }}/ url: http://{{ ip | ansible.utils.ipwrap }}:{{ port_ }}/
return_content: true return_content: true
register: result register: result
# The MetalLB speaker announces the VIP on the host-only network, and the failed_when: "'Welcome to nginx!' not in result.content"
# runner may briefly not see it in its ARP table yet. Retry so a transient
# announcement miss does not fail an otherwise healthy cluster, matching
# the load-balancer address wait above and the k3s_server_post metallb waits.
until:
- result.status is defined
- result.status == 200
- "'Welcome to nginx!' in result.content"
retries: 30
delay: 5
vars: vars:
ip: >-
{{ nginx_services.resources[0].status.loadBalancer.ingress[0].ip }}
port_: >- port_: >-
{{ nginx_services.resources[0].spec.ports[0].port }} {{ nginx_services.resources[0].spec.ports[0].port }}
- name: Initialize load balancer address range check
ansible.builtin.set_fact:
lb_addr_in_range: false
lb_ip_value: "{{ nginx_lb_ip }}"
- name: Check load balancer address against start-end pools
ansible.builtin.set_fact:
lb_addr_in_range: true
loop: "{{ verify_lb_ip_range }}"
loop_control:
label: "{{ item }}"
when:
- "'-' in item"
- "'/' not in item"
- >-
(lb_ip_value | ansible.utils.ipaddr('int') | int) >=
(item.split('-')[0] | ansible.utils.ipaddr('int') | int)
- >-
(lb_ip_value | ansible.utils.ipaddr('int') | int) <=
(item.split('-')[1] | ansible.utils.ipaddr('int') | int)
- name: Check load balancer address against CIDR pools
ansible.builtin.set_fact:
lb_addr_in_range: true
loop: "{{ verify_lb_ip_range }}"
loop_control:
label: "{{ item }}"
when:
- "'/' in item"
- (lb_ip_value | ansible.utils.ipaddr(item)) is string
- name: Assert that the load balancer address is within a configured pool
ansible.builtin.assert:
that: lb_addr_in_range
success_msg: "LoadBalancer address {{ lb_ip_value }} is in a configured range"
fail_msg: >-
LoadBalancer address {{ lb_ip_value }} is not in a configured
range {{ verify_lb_ip_range }}
# Deactivated linter rules: # Deactivated linter rules:
# - jinja[invalid]: As of version 6.6.0, ansible-lint complains that the input to ipwrap # - jinja[invalid]: As of version 6.6.0, ansible-lint complains that the input to ipwrap
# would be undefined. This will not be the case during playbook execution. # would be undefined. This will not be the case during playbook execution.
@@ -1,354 +0,0 @@
---
# Scenario-aware verification of cluster components and their live image tags.
# Scenario identity (verify_cni / verify_lb) and expected address range come
# from each scenario's verify-vars.yml, which is plain inventory data available
# to the verify play. Converge-time set_fact values are not persisted between
# the two Ansible processes, so they are never used here.
- name: Verify cluster components report expected versions
block:
- name: Get all nodes with their kubelet versions
kubernetes.core.k8s_info:
kind: node
kubeconfig: "{{ kubecfg_path }}"
register: verify_nodes
- name: Assert each node reports the expected Kubernetes version
ansible.builtin.assert:
that: item.status.nodeInfo.kubeletVersion == k3s_version
success_msg: "{{ item.metadata.name }} reports {{ k3s_version }}"
fail_msg: >-
{{ item.metadata.name }} reports
{{ item.status.nodeInfo.kubeletVersion }},
expected {{ k3s_version }}
loop: "{{ verify_nodes.resources }}"
loop_control:
label: "{{ item.metadata.name }}"
- name: Verify Flannel is the active CNI
when: verify_cni == 'flannel'
block:
- name: Assert every node reports Ready
ansible.builtin.assert:
that: item.status.conditions
| selectattr('type', 'equalto', 'Ready')
| map(attribute='status') | first | default('') == 'True'
success_msg: "{{ item.metadata.name }} is Ready"
fail_msg: "{{ item.metadata.name }} is not Ready"
loop: "{{ verify_nodes.resources }}"
loop_control:
label: "{{ item.metadata.name }} ready"
- name: Assert every node registered a node IP from its interface
# Each k3s node is launched with --node-ip derived from flannel_iface.
# Confirm every node carries a real InternalIP (not a loopback), which
# proves k3s bound to the cluster interface rather than defaulting to 127.0.0.1.
ansible.builtin.assert:
that: >-
(node_internal_ips | length) >= 1 and
(node_internal_ips | reject('eq', '127.0.0.1') | list | length) == node_internal_ips | length
success_msg: "{{ item.metadata.name }} is bound to {{ node_internal_ips | join(', ') }}"
fail_msg: >-
{{ item.metadata.name }} has no non-loopback InternalIP
(got: {{ node_internal_ips | join(', ') }})
vars:
node_internal_ips: >-
{{
(item.status.addresses | default([]))
| selectattr('type', 'equalto', 'InternalIP')
| map(attribute='address')
| list
}}
loop: "{{ verify_nodes.resources }}"
loop_control:
label: "{{ item.metadata.name }} InternalIP"
- name: Get any Calico namespaces with Flannel enabled
kubernetes.core.k8s_info:
kind: Namespace
name: calico-system
kubeconfig: "{{ kubecfg_path }}"
register: flannel_calico_absent
- name: Assert there is no Calico system namespace
ansible.builtin.assert:
that: flannel_calico_absent.resources | length == 0
success_msg: "No Calico present with Flannel"
fail_msg: "A Calico namespace exists alongside Flannel"
- name: Get the Cilium namespace with Flannel enabled
kubernetes.core.k8s_info:
kind: Namespace
name: cilium
kubeconfig: "{{ kubecfg_path }}"
register: flannel_cilium
- name: Assert the Cilium namespace is absent
ansible.builtin.assert:
that: flannel_cilium.resources | length == 0
success_msg: "No Cilium present with Flannel"
fail_msg: "A Cilium namespace exists alongside Flannel"
- name: Verify Calico is the active CNI
when: verify_cni == 'calico'
block:
- name: Get the Calico node DaemonSet image
kubernetes.core.k8s_info:
kind: DaemonSet
name: calico-node
namespace: calico-system
kubeconfig: "{{ kubecfg_path }}"
register: calico_node_ds
- name: Assert the Calico node image uses the expected tag
ansible.builtin.assert:
that:
- calico_node_ds.resources | length == 1
- calico_node_image | regex_search(':' ~ calico_tag) is not none
success_msg: "Calico node image uses tag {{ calico_tag }}"
fail_msg: >-
Calico node image {{ calico_node_image }},
expected {{ calico_tag }}
vars:
calico_node_image: "{{ calico_node_ds.resources[0].spec.template.spec.containers[0].image }}"
- name: Get Calico TigeraStatus for calico and apiserver
kubernetes.core.k8s_info:
api_version: operator.tigera.io/v1
kind: TigeraStatus
name: "{{ item }}"
kubeconfig: "{{ kubecfg_path }}"
register: calico_tigerastatus
loop:
- calico
- apiserver
loop_control:
label: "Tigerastatus/{{ item }}"
- name: Assert Calico TigeraStatus reports Available
ansible.builtin.assert:
that: >-
item.resources | length == 1 and
(item.resources[0].status.conditions
| selectattr('type', 'equalto', 'Available')
| map(attribute='status') | first | default('')) == 'True'
success_msg: "Tigerastatus {{ item.resources[0].metadata.name }} is Available"
fail_msg: "Tigerastatus is not Available"
loop: "{{ calico_tigerastatus.results }}"
loop_control:
label: "Tigerastatus Available"
- name: Get any Flannel DaemonSets with Calico enabled
kubernetes.core.k8s_info:
kind: DaemonSet
namespace: kube-flannel
kubeconfig: "{{ kubecfg_path }}"
register: no_flannel_ds
- name: Assert there are no Flannel DaemonSets
ansible.builtin.assert:
that: no_flannel_ds.resources | length == 0
success_msg: "No Flannel DaemonSet present with Calico"
fail_msg: "A Flannel DaemonSet exists alongside Calico"
- name: Verify Cilium is the active CNI
when: verify_cni == 'cilium'
block:
- name: Get the Cilium agent and operator images
kubernetes.core.k8s_info:
kind: "{{ item.kind }}"
name: "{{ item.name }}"
namespace: kube-system
kubeconfig: "{{ kubecfg_path }}"
register: cilium_info
loop:
- { kind: DaemonSet, name: cilium }
- { kind: Deployment, name: cilium-operator }
loop_control:
label: "{{ item.kind }}/{{ item.name }}"
- name: Assert Cilium agent and operator use the expected image tag
ansible.builtin.assert:
that:
- cilium_agent_image | regex_search(':' ~ cilium_tag) is not none
- cilium_operator_image | regex_search(':' ~ cilium_tag) is not none
success_msg: "Cilium agent and operator use {{ cilium_tag }}"
fail_msg: >-
Cilium agent {{ cilium_agent_image }},
operator {{ cilium_operator_image }},
expected {{ cilium_tag }}
vars:
cilium_agent_image: >-
{{ (cilium_info.results
| selectattr('resources', 'defined')
| map(attribute='resources')
| list
| map(attribute='0')
| selectattr('kind', 'equalto', 'DaemonSet')
| list)[0].spec.template.spec.containers[0].image }}
cilium_operator_image: >-
{{ (cilium_info.results
| selectattr('resources', 'defined')
| map(attribute='resources')
| list
| map(attribute='0')
| selectattr('kind', 'equalto', 'Deployment')
| list)[0].spec.template.spec.containers[0].image }}
- name: Get Hubble relay and UI deployments when enabled
kubernetes.core.k8s_info:
kind: Deployment
name: "{{ item }}"
namespace: kube-system
kubeconfig: "{{ kubecfg_path }}"
register: hubble_info
loop:
- hubble-relay
- hubble-ui
loop_control:
label: "Deployment/{{ item }}"
when: cilium_hubble | bool
- name: Assert Hubble components are Ready when enabled
ansible.builtin.assert:
that:
- item.resources | length == 1
- item.resources[0].status.readyReplicas | default(0) >= 1
success_msg: "Hubble deployment {{ item.resources[0].metadata.name }} is Ready"
fail_msg: "Hubble deployment is not Ready"
loop: "{{ hubble_info.results }}"
loop_control:
label: "Hubble deployment"
when: cilium_hubble | bool
- name: Get any Flannel DaemonSets with Cilium enabled
kubernetes.core.k8s_info:
kind: DaemonSet
namespace: kube-flannel
kubeconfig: "{{ kubecfg_path }}"
register: no_flannel_ds_cilium
- name: Assert there are no Flannel DaemonSets
ansible.builtin.assert:
that: no_flannel_ds_cilium.resources | length == 0
success_msg: "No Flannel DaemonSet present with Cilium"
fail_msg: "A Flannel DaemonSet exists alongside Cilium"
- name: Verify MetalLB is the active load balancer
when: verify_lb == 'metallb'
block:
- name: Get the MetalLB controller and speaker images
kubernetes.core.k8s_info:
kind: "{{ item.kind }}"
name: "{{ item.name }}"
namespace: metallb-system
kubeconfig: "{{ kubecfg_path }}"
register: metallb_info
until: metallb_info.resources | length > 0
retries: 15
delay: 10
loop:
- { kind: Deployment, name: controller }
- { kind: DaemonSet, name: speaker }
loop_control:
label: "{{ item.kind }}/{{ item.name }}"
- name: Fail with a clear message if MetalLB resources are missing
ansible.builtin.fail:
msg: >-
Did not find {{ item.kind | lower }} {{ item.name }} in
metallb-system. Expected MetalLB to be deployed in this
scenario (verify_lb: {{ verify_lb }}).
when: item.resources | length == 0
loop: "{{ metallb_info.results }}"
loop_control:
label: "{{ item.item.kind }}/{{ item.item.name }}"
- name: Assert MetalLB controller and speaker use the expected image tags
ansible.builtin.assert:
# regex_search returns a string or none; check for a match with `is not
# none` so the assertion is a real boolean (ansible-core 2.19 rejects
# string conditionals and `| bool` deprecates string coercion).
that:
- controller_image | regex_search(metal_lb_controller_tag_version) is not none
- speaker_image | regex_search(metal_lb_speaker_tag_version) is not none
success_msg: >-
MetalLB controller {{ metal_lb_controller_tag_version }},
speaker {{ metal_lb_speaker_tag_version }}
fail_msg: >-
MetalLB controller {{ controller_image }},
speaker {{ speaker_image }}
vars:
controller_image: >-
{{ (metallb_info.results
| selectattr('resources', 'defined')
| map(attribute='resources')
| list
| map(attribute='0')
| selectattr('kind', 'equalto', 'Deployment')
| list)[0].spec.template.spec.containers[0].image }}
speaker_image: >-
{{ (metallb_info.results
| selectattr('resources', 'defined')
| map(attribute='resources')
| list
| map(attribute='0')
| selectattr('kind', 'equalto', 'DaemonSet')
| list)[0].spec.template.spec.containers[0].image }}
- name: Verify kube-vip is the active load balancer
when: verify_lb == 'kube-vip'
block:
- name: Get the kube-vip and cloud provider images
kubernetes.core.k8s_info:
kind: "{{ item.kind }}"
name: "{{ item.name }}"
namespace: kube-system
kubeconfig: "{{ kubecfg_path }}"
register: kubevip_info
loop:
- { kind: DaemonSet, name: kube-vip-ds }
- { kind: Deployment, name: kube-vip-cloud-provider }
loop_control:
label: "{{ item.kind }}/{{ item.name }}"
- name: Assert the kube-vip and cloud provider image tags
ansible.builtin.assert:
that:
- kubevip_image | regex_search(':' ~ kube_vip_tag_version) is not none
- cloud_provider_image | regex_search(verify_kube_vip_cloud_provider_tag) is not none
success_msg: >-
kube-vip {{ kube_vip_tag_version }},
cloud provider {{ verify_kube_vip_cloud_provider_tag }}
fail_msg: >-
kube-vip {{ kubevip_image }},
cloud provider {{ cloud_provider_image }}
vars:
kubevip_image: >-
{{ (kubevip_info.results
| selectattr('resources', 'defined')
| map(attribute='resources')
| list
| map(attribute='0')
| selectattr('kind', 'equalto', 'DaemonSet')
| list)[0].spec.template.spec.containers[0].image }}
cloud_provider_image: >-
{{ (kubevip_info.results
| selectattr('resources', 'defined')
| map(attribute='resources')
| list
| map(attribute='0')
| selectattr('kind', 'equalto', 'Deployment')
| list)[0].spec.template.spec.containers[0].image }}
- name: Get the MetalLB namespace with kube-vip enabled
kubernetes.core.k8s_info:
kind: Namespace
name: metallb-system
kubeconfig: "{{ kubecfg_path }}"
register: metallb_absent
- name: Assert the MetalLB namespace does not exist
ansible.builtin.assert:
that: metallb_absent.resources | length == 0
success_msg: "MetalLB is not installed with kube-vip"
fail_msg: "MetalLB namespace exists alongside kube-vip"
+6 -1
View File
@@ -5,9 +5,14 @@ driver:
name: vagrant name: vagrant
platforms: platforms:
- name: control1 - name: control1
box: bento/ubuntu-26.04 box: generic/ubuntu2204
memory: 4096 memory: 4096
cpus: 4 cpus: 4
config_options:
# We currently can not use public-key based authentication on Ubuntu 22.04,
# see: https://github.com/chef/bento/issues/1405
ssh.username: vagrant
ssh.password: vagrant
groups: groups:
- k3s_cluster - k3s_cluster
- master - master
+1 -3
View File
@@ -1,14 +1,12 @@
--- ---
- name: Apply overrides - name: Apply overrides
hosts: all hosts: all
serial: 1
tasks: tasks:
- name: Override host variables - name: Override host variables
ansible.builtin.set_fact: ansible.builtin.set_fact:
# See: # See:
# https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
flannel_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}" flannel_iface: eth1
kube_vip_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}"
# The test VMs might be a bit slow, so we give them more time to join the cluster: # The test VMs might be a bit slow, so we give them more time to join the cluster:
retry_count: 45 retry_count: 45
-6
View File
@@ -1,6 +0,0 @@
---
# Durable verify inputs for the single_node (flannel + MetalLB) scenario.
verify_cni: flannel
verify_lb: metallb
verify_lb_ip_range:
- 192.168.30.91-192.168.30.99
+1 -21
View File
@@ -2,29 +2,9 @@
- name: Reboot k3s_cluster - name: Reboot k3s_cluster
hosts: k3s_cluster hosts: k3s_cluster
gather_facts: true gather_facts: true
# Stagger the reboot across the cluster when concurrent_reboots is set.
# Defaults to '100%' so the whole cluster reboots at once (backward compatible).
serial: "{{ concurrent_reboots | default('100%') }}"
tasks: tasks:
- name: >- - name: Reboot the nodes (and Wait upto 5 mins max)
{{
'Reboot all nodes at once'
if (concurrent_reboots is not defined)
else 'Reboot nodes with concurrency of ' ~ concurrent_reboots
}}
become: true become: true
ansible.builtin.reboot: ansible.builtin.reboot:
reboot_command: "{{ custom_reboot_command | default(omit) }}" reboot_command: "{{ custom_reboot_command | default(omit) }}"
reboot_timeout: 300 reboot_timeout: 300
test_command: >-
{{ 'kubectl get nodes' if 'master' in group_names else 'whoami' }}
- name: Optional wait before rebooting the next batch of nodes
ansible.builtin.pause:
seconds: "{{ wait_seconds_after_reboot | int }}"
when: >-
concurrent_reboots is defined and
wait_seconds_after_reboot is defined and
wait_seconds_after_reboot | int > 0
+2 -2
View File
@@ -1,5 +1,5 @@
ansible-core>=2.19.13 ansible-core>=2.16.2
jmespath>=1.1.0 jmespath>=1.0.1
jsonpatch>=1.33 jsonpatch>=1.33
kubernetes>=29.0.0 kubernetes>=29.0.0
molecule-plugins[vagrant] molecule-plugins[vagrant]
+5 -5
View File
@@ -2,11 +2,11 @@
# This file is autogenerated by pip-compile with Python 3.11 # This file is autogenerated by pip-compile with Python 3.11
# by the following command: # by the following command:
# #
# pip-compile --output-file=requirements.txt requirements.in # pip-compile requirements.in
# #
ansible-compat==4.1.11 ansible-compat==4.1.11
# via molecule # via molecule
ansible-core==2.19.13 ansible-core==2.17.5
# via # via
# -r requirements.in # -r requirements.in
# ansible-compat # ansible-compat
@@ -53,7 +53,7 @@ jinja2==3.1.3
# via # via
# ansible-core # ansible-core
# molecule # molecule
jmespath==1.1.0 jmespath==1.0.1
# via -r requirements.in # via -r requirements.in
jsonpatch==1.33 jsonpatch==1.33
# via -r requirements.in # via -r requirements.in
@@ -77,7 +77,7 @@ molecule==6.0.3
# via # via
# -r requirements.in # -r requirements.in
# molecule-plugins # molecule-plugins
molecule-plugins[vagrant]==23.6.0 molecule-plugins[vagrant]==23.5.3
# via -r requirements.in # via -r requirements.in
netaddr==0.10.1 netaddr==0.10.1
# via -r requirements.in # via -r requirements.in
@@ -146,7 +146,7 @@ rsa==4.9
# via google-auth # via google-auth
ruamel-yaml==0.18.5 ruamel-yaml==0.18.5
# via pre-commit-hooks # via pre-commit-hooks
ruamel-yaml-clib==0.2.15 ruamel-yaml-clib==0.2.8
# via ruamel-yaml # via ruamel-yaml
six==1.16.0 six==1.16.0
# via # via
-12
View File
@@ -7,10 +7,6 @@
owner: root owner: root
group: root group: root
mode: "0755" mode: "0755"
register: k3s_download_x64
retries: 5
delay: 10
until: k3s_download_x64 is succeeded
when: ansible_facts.architecture == "x86_64" when: ansible_facts.architecture == "x86_64"
- name: Download k3s binary arm64 - name: Download k3s binary arm64
@@ -21,10 +17,6 @@
owner: root owner: root
group: root group: root
mode: "0755" mode: "0755"
register: k3s_download_arm64
retries: 5
delay: 10
until: k3s_download_arm64 is succeeded
when: when:
- ( ansible_facts.architecture is search("arm") and ansible_facts.userspace_bits == "64" ) - ( ansible_facts.architecture is search("arm") and ansible_facts.userspace_bits == "64" )
or ansible_facts.architecture is search("aarch64") or ansible_facts.architecture is search("aarch64")
@@ -37,10 +29,6 @@
owner: root owner: root
group: root group: root
mode: "0755" mode: "0755"
register: k3s_download_armhf
retries: 5
delay: 10
until: k3s_download_armhf is succeeded
when: when:
- ansible_facts.architecture is search("arm") - ansible_facts.architecture is search("arm")
- ansible_facts.userspace_bits == "32" - ansible_facts.userspace_bits == "32"
+1 -6
View File
@@ -19,19 +19,14 @@ argument_specs:
proxy_env: proxy_env:
type: dict type: dict
description: description: Internet proxy configurations
- Internet proxy configurations.
- See https://docs.k3s.io/advanced#configuring-an-http-proxy for details
default: ~ default: ~
options: options:
HTTP_PROXY: HTTP_PROXY:
description: HTTP internet proxy
required: true required: true
HTTPS_PROXY: HTTPS_PROXY:
description: HTTPS internet proxy
required: true required: true
NO_PROXY: NO_PROXY:
description: Addresses that will not use the proxies
required: true required: true
systemd_dir: systemd_dir:
+1 -1
View File
@@ -11,7 +11,7 @@ ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/k3s agent \ ExecStart=/usr/local/bin/k3s agent \
--server https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443 \ --server https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443 \
{% if is_pxe_booted | default(false) %}--snapshotter native \ {% if is_pxe_booted | default(false) %}--snapshotter native \
{% endif %}--token {{ hostvars[groups[group_name_master | default('master')][0]]['token'] | default(k3s_token) | replace( '%', '%%' ) }} \ {% endif %}--token {{ hostvars[groups[group_name_master | default('master')][0]]['token'] | default(k3s_token) }} \
{{ extra_agent_args }} {{ extra_agent_args }}
KillMode=process KillMode=process
Delegate=yes Delegate=yes
+5 -18
View File
@@ -7,9 +7,8 @@ group_name_master: master
kube_vip_arp: true kube_vip_arp: true
kube_vip_iface: kube_vip_iface:
kube_vip_endpoint: kube_vip_cloud_provider_tag_version: main
kube_vip_cloud_provider_tag_version: v0.0.12 kube_vip_tag_version: v0.7.2
kube_vip_tag_version: v1.2.3
kube_vip_bgp: false kube_vip_bgp: false
kube_vip_bgp_routerid: 127.0.0.1 kube_vip_bgp_routerid: 127.0.0.1
@@ -17,22 +16,10 @@ kube_vip_bgp_as: "64513"
kube_vip_bgp_peeraddress: 192.168.30.1 kube_vip_bgp_peeraddress: 192.168.30.1
kube_vip_bgp_peeras: "64512" kube_vip_bgp_peeras: "64512"
kube_vip_bgp_peers: [] metal_lb_controller_tag_version: v0.14.3
kube_vip_bgp_peers_groups: ['k3s_master'] metal_lb_speaker_tag_version: v0.14.3
kube_vip_enabled: true
metal_lb_enabled: true
metal_lb_controller_tag_version: v0.16.0
metal_lb_speaker_tag_version: v0.16.0
metal_lb_type: native metal_lb_type: native
# Shared retry/delay for remote manifest and asset downloads. The CI runner's
# resolver intermittently times out on GitHub-hosted domains (helm.cilium.io,
# raw.githubusercontent.com, github.com), so retry transient DNS/network failures.
download_retries: 5
download_delay: 10
retry_count: 20 retry_count: 20
# yamllint disable rule:line-length # yamllint disable rule:line-length
@@ -43,7 +30,7 @@ server_init_args: >-
{% else %} {% else %}
--server https://{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(",") | first | ansible.utils.ipwrap }}:6443 --server https://{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(",") | first | ansible.utils.ipwrap }}:6443
{% endif %} {% endif %}
--token {{ k3s_token | replace('%', '%%') }} --token {{ k3s_token }}
{% endif %} {% endif %}
{{ extra_server_args }} {{ extra_server_args }}
+5 -43
View File
@@ -62,14 +62,6 @@ argument_specs:
description: Defines the AS for the kube-vip BGP peer description: Defines the AS for the kube-vip BGP peer
default: "64512" default: "64512"
kube_vip_bgp_peers:
description: List of BGP peer ASN & address pairs
default: []
kube_vip_bgp_peers_groups:
description: Inventory group in which to search for additional kube_vip_bgp_peers parameters to merge.
default: ['k3s_master']
kube_vip_iface: kube_vip_iface:
description: description:
- Explicitly define an interface that ALL control nodes - Explicitly define an interface that ALL control nodes
@@ -78,72 +70,42 @@ argument_specs:
- automatically at runtime. - automatically at runtime.
default: ~ default: ~
kube_vip_endpoint:
description:
- Overrides the address kube-vip binds/listens on internally, which
- can differ from the announced apiserver_endpoint for complex
- routing and site-to-site tunnels.
- Defaults to apiserver_endpoint and is used to derive the kube-vip
- subnet.
default: ~
kube_vip_tag_version: kube_vip_tag_version:
description: Image tag for kube-vip description: Image tag for kube-vip
default: v1.2.3 default: v0.7.2
kube_vip_cloud_provider_tag_version: kube_vip_cloud_provider_tag_version:
description: Tag for kube-vip-cloud-provider manifest when enabled description: Tag for kube-vip-cloud-provider manifest when enabled
default: v0.0.12 default: main
kube_vip_enabled:
description:
- Enable installing kube-vip.
- Covers both the control-plane VIP and the service load balancer.
- Set false to skip kube-vip entirely, for example on a single node or when an external load balancer is used.
default: true
type: bool
kube_vip_lb_ip_range: kube_vip_lb_ip_range:
description: IP range for kube-vip load balancer description: IP range for kube-vip load balancer
default: ~ default: ~
metal_lb_enabled:
description:
- Enable installing MetalLB for service load balancing.
- Set false to skip MetalLB, for example when an external load balancer is used.
default: true
type: bool
metal_lb_controller_tag_version: metal_lb_controller_tag_version:
description: Image tag for MetalLB description: Image tag for MetalLB
default: v0.16.0 default: v0.14.3
metal_lb_speaker_tag_version: metal_lb_speaker_tag_version:
description: Image tag for MetalLB description: Image tag for MetalLB
default: v0.16.0 default: v0.14.3
metal_lb_type: metal_lb_type:
choices: choices:
- frr - frr
- native - native
default: native default: native
description: Use FRR mode or native. Valid values are `frr` and `native`
proxy_env: proxy_env:
type: dict type: dict
description: description: Internet proxy configurations
- Internet proxy configurations.
- See https://docs.k3s.io/advanced#configuring-an-http-proxy for details
default: ~ default: ~
options: options:
HTTP_PROXY: HTTP_PROXY:
description: HTTP internet proxy
required: true required: true
HTTPS_PROXY: HTTPS_PROXY:
description: HTTPS internet proxy
required: true required: true
NO_PROXY: NO_PROXY:
description: Addresses that will not use the proxies
required: true required: true
retry_count: retry_count:
+2 -154
View File
@@ -4,110 +4,10 @@
- name: Fetch k3s-init.service logs - name: Fetch k3s-init.service logs
ansible.builtin.command: ansible.builtin.command:
cmd: >- cmd: journalctl --all --unit=k3s-init.service
timeout --signal=TERM --kill-after=5s 30s journalctl --no-pager
--unit=k3s-init.service --since=-30min --lines=5000
changed_when: false changed_when: false
failed_when: false
no_log: true
register: k3s_init_log register: k3s_init_log
- name: Fetch k3s-init.service status
ansible.builtin.command:
cmd: >-
timeout --signal=TERM --kill-after=5s 15s systemctl status
k3s-init.service --no-pager --full
changed_when: false
failed_when: false
no_log: true
register: k3s_init_status
- name: Fetch IP address state
ansible.builtin.command:
cmd: ip -br address
changed_when: false
failed_when: false
no_log: true
register: k3s_ip_address_state
- name: Fetch IP link state
ansible.builtin.command:
cmd: ip -br link
changed_when: false
failed_when: false
no_log: true
register: k3s_ip_link_state
- name: Fetch IP route state
ansible.builtin.command:
cmd: ip route show
changed_when: false
failed_when: false
no_log: true
register: k3s_ip_route_state
- name: Fetch IP neighbor state
ansible.builtin.command:
cmd: ip neigh show
changed_when: false
failed_when: false
no_log: true
register: k3s_ip_neighbor_state
- name: Fetch IP rule state
ansible.builtin.command:
cmd: ip rule show
changed_when: false
failed_when: false
no_log: true
register: k3s_ip_rule_state
- name: Fetch listening TCP sockets
ansible.builtin.command:
cmd: ss -ltn
changed_when: false
failed_when: false
no_log: true
register: k3s_tcp_listener_state
- name: Fetch guest firewall service state
ansible.builtin.service_facts:
no_log: true
register: k3s_service_facts
- name: Fetch input firewall rules
ansible.builtin.command:
cmd: iptables -S INPUT
changed_when: false
failed_when: false
no_log: true
register: k3s_input_firewall_rules
- name: Ping the primary Kubernetes API address from {{ ansible_hostname }}
ansible.builtin.command:
argv:
- ping
- -c
- "1"
- -W
- "1"
- "{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(',') | first }}"
changed_when: false
failed_when: false
no_log: true
register: k3s_primary_api_ping
- name: Probe primary Kubernetes API from {{ ansible_hostname }}
ansible.builtin.wait_for:
host: "{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(',') | first }}"
port: 6443
connect_timeout: 1
timeout: 2
changed_when: false
failed_when: false
no_log: true
register: k3s_primary_api_probe
- name: Create {{ log_destination }} - name: Create {{ log_destination }}
delegate_to: localhost delegate_to: localhost
run_once: true run_once: true
@@ -125,56 +25,4 @@
dest: "{{ log_destination }}/k3s-init@{{ ansible_hostname }}.log" dest: "{{ log_destination }}/k3s-init@{{ ansible_hostname }}.log"
mode: "0644" mode: "0644"
vars: vars:
content: | content: "{{ k3s_init_log.stdout }}"
=== k3s-init.service status ===
{{ k3s_init_status.stdout | regex_replace('--token(?:=| +)[^ ]+', '--token ***') }}
=== k3s-init.service journal ===
{{ k3s_init_log.stdout | regex_replace('--token(?:=| +)[^ ]+', '--token ***') }}
- name: Store network diagnostics to {{ log_destination }}
delegate_to: localhost
become: false
ansible.builtin.template:
src: content.j2
dest: "{{ log_destination }}/network@{{ ansible_hostname }}.log"
mode: "0644"
vars:
content: |
=== ip -br address ===
{{ k3s_ip_address_state.stdout }}
=== ip -br link ===
{{ k3s_ip_link_state.stdout }}
=== ip route show ===
{{ k3s_ip_route_state.stdout }}
=== ip neigh show ===
{{ k3s_ip_neighbor_state.stdout }}
=== ip rule show ===
{{ k3s_ip_rule_state.stdout }}
=== ss -ltn ===
{{ k3s_tcp_listener_state.stdout }}
=== guest firewall services ===
firewalld={{ k3s_service_facts.ansible_facts.services.get('firewalld.service', {}).get('state', 'not-found') }}
nftables={{ k3s_service_facts.ansible_facts.services.get('nftables.service', {}).get('state', 'not-found') }}
ufw={{ k3s_service_facts.ansible_facts.services.get('ufw.service', {}).get('state', 'not-found') }}
=== iptables -S INPUT ===
rc={{ k3s_input_firewall_rules.rc }}
{{ k3s_input_firewall_rules.stdout }}
{{ k3s_input_firewall_rules.stderr }}
=== primary API ping ===
rc={{ k3s_primary_api_ping.rc }}
{{ k3s_primary_api_ping.stdout }}
{{ k3s_primary_api_ping.stderr }}
=== primary API probe ===
failed={{ k3s_primary_api_probe.failed | default(false) }}
elapsed={{ k3s_primary_api_probe.elapsed | default('unknown') }}
msg={{ k3s_primary_api_probe.msg | default('connected') }}
-34
View File
@@ -1,34 +0,0 @@
---
- name: Verify primary Kubernetes API reachability from {{ joining_master }}
ansible.builtin.wait_for:
host: >-
{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip
| split(',') | first }}
port: 6443
connect_timeout: 2
timeout: 30
delegate_to: "{{ joining_master }}"
- name: Join transient k3s-init service for {{ joining_master }}
ansible.builtin.command:
cmd: >-
systemd-run -p RestartSec=2 -p Restart=on-failure
-p Delegate=yes -p TasksMax=infinity -p KillMode=process
-p LimitNOFILE=1048576 -p LimitNPROC=infinity -p LimitCORE=infinity
--unit=k3s-init
k3s server {{ hostvars[joining_master].k3s_server_init_args }}
creates: "{{ systemd_dir }}/k3s-init.service"
delegate_to: "{{ joining_master }}"
no_log: true
- name: Wait for primary registration of {{ joining_master }}
ansible.builtin.command:
cmd: >-
{{ k3s_kubectl_binary | default('k3s kubectl') }} get node
{{ hostvars[joining_master].ansible_hostname }}
delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
register: joined_master
until: joined_master.rc == 0
retries: "{{ retry_count | default(20) }}"
delay: 2
changed_when: false
+1 -5
View File
@@ -10,15 +10,11 @@
- name: Download vip cloud provider manifest to first master - name: Download vip cloud provider manifest to first master
ansible.builtin.get_url: ansible.builtin.get_url:
url: https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/{{ kube_vip_cloud_provider_tag_version | default('v0.0.12') }}/manifest/kube-vip-cloud-controller.yaml # noqa yaml[line-length] url: https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/{{ kube_vip_cloud_provider_tag_version | default('main') }}/manifest/kube-vip-cloud-controller.yaml # noqa yaml[line-length]
dest: /var/lib/rancher/k3s/server/manifests/kube-vip-cloud-controller.yaml dest: /var/lib/rancher/k3s/server/manifests/kube-vip-cloud-controller.yaml
owner: root owner: root
group: root group: root
mode: "0644" mode: "0644"
register: kube_vip_manifest_download
retries: "{{ download_retries }}"
delay: "{{ download_delay }}"
until: kube_vip_manifest_download is succeeded
when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
- name: Copy kubevip configMap manifest to first master - name: Copy kubevip configMap manifest to first master
+17 -91
View File
@@ -24,110 +24,36 @@
- name: Deploy vip manifest - name: Deploy vip manifest
ansible.builtin.include_tasks: vip.yml ansible.builtin.include_tasks: vip.yml
when: kube_vip_enabled
- name: Deploy metallb manifest - name: Deploy metallb manifest
ansible.builtin.include_tasks: metallb.yml ansible.builtin.include_tasks: metallb.yml
tags: metallb tags: metallb
# Deploy MetalLB unless explicitly disabled, kube-vip owns the load balancer when: kube_vip_lb_ip_range is not defined and (not cilium_bgp or cilium_iface is not defined)
# IP range, or Cilium BGP is enabled (Cilium then provides its own load
# balancing). The cilium_bgp default keeps this safe when Cilium variables are
# not in scope at all (#644) while still deploying MetalLB when a non-BGP
# Cilium CNI is in use.
when: metal_lb_enabled and kube_vip_lb_ip_range is not defined and not (cilium_bgp | default(false) | bool)
- name: Deploy kube-vip manifest - name: Deploy kube-vip manifest
ansible.builtin.include_tasks: kube-vip.yml ansible.builtin.include_tasks: kube-vip.yml
tags: kubevip tags: kubevip
when: kube_vip_enabled and kube_vip_lb_ip_range is defined when: kube_vip_lb_ip_range is defined
- name: Initialize and verify the K3s control plane - name: Init cluster inside the transient k3s-init service
any_errors_fatal: true ansible.builtin.command:
cmd: systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server {{ server_init_args }}
creates: "{{ systemd_dir }}/k3s-init.service"
- name: Verification
when: not ansible_check_mode when: not ansible_check_mode
block: block:
- name: Materialize per-host server initialization arguments - name: Verify that all nodes actually joined (check k3s-init.service if this fails)
ansible.builtin.set_fact: ansible.builtin.command:
k3s_server_init_args: "{{ server_init_args }}" cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get nodes -l 'node-role.kubernetes.io/master=true' -o=jsonpath='{.items[*].metadata.name}'" # yamllint disable-line rule:line-length
no_log: true register: nodes
when: groups[group_name_master | default('master')] | length > 1 until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups[group_name_master | default('master')] | length) # yamllint disable-line rule:line-length
retries: "{{ retry_count | default(20) }}"
- name: Orchestrate control-plane initialization from the first master delay: 10
when: inventory_hostname == groups[group_name_master | default('master')][0] changed_when: false
block:
- name: Init the first master inside the transient k3s-init service
ansible.builtin.command:
cmd: >-
systemd-run -p RestartSec=2 -p Restart=on-failure
-p Delegate=yes -p TasksMax=infinity -p KillMode=process
-p LimitNOFILE=1048576 -p LimitNPROC=infinity -p LimitCORE=infinity
--unit=k3s-init k3s server {{ server_init_args }}
creates: "{{ systemd_dir }}/k3s-init.service"
no_log: true
- name: Wait for the first master Kubernetes API
ansible.builtin.command:
cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get --raw=/readyz"
register: first_master_api
until: first_master_api.rc == 0
retries: "{{ retry_count | default(20) }}"
delay: 2
changed_when: false
- name: Verify the first master API listener on its node address
ansible.builtin.wait_for:
host: >-
{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip
| split(',') | first }}
port: 6443
connect_timeout: 2
timeout: "{{ retry_count | default(20) * 2 }}"
- name: Join additional masters one at a time
ansible.builtin.include_tasks: join_master.yml
loop: "{{ groups[group_name_master | default('master')][1:] }}"
loop_control:
loop_var: joining_master
when: groups[group_name_master | default('master')] | length > 1
- name: Verify that all expected control-plane nodes registered
ansible.builtin.command:
cmd: >-
{{ k3s_kubectl_binary | default('k3s kubectl') }} get nodes
-o=jsonpath='{.items[*].metadata.name}'
register: nodes
until:
- nodes.rc == 0
- expected_control_plane_names | difference(nodes.stdout.split()) | length == 0
retries: "{{ retry_count | default(20) }}"
delay: 2
changed_when: false
vars:
expected_control_plane_names: >-
{{
groups[group_name_master | default('master')]
| map('extract', hostvars, 'ansible_hostname')
| list
}}
- name: Wait for K3s bootstrap CRDs to become established
ansible.builtin.command:
cmd: >-
{{ k3s_kubectl_binary | default('k3s kubectl') }} wait
--for=condition=Established --timeout=5s
crd/addons.k3s.cattle.io
crd/helmcharts.helm.cattle.io
crd/helmchartconfigs.helm.cattle.io
register: bootstrap_crds
until: bootstrap_crds.rc == 0
retries: "{{ retry_count | default(20) }}"
delay: 2
changed_when: false
always: always:
- name: Save logs of k3s-init.service - name: Save logs of k3s-init.service
ansible.builtin.include_tasks: fetch_k3s_init_logs.yml ansible.builtin.include_tasks: fetch_k3s_init_logs.yml
# ANSIBLE_K3S_LOG_DIR is a path string when set; evaluate it as a boolean when: log_destination
# so the conditional is a real boolean (ansible-core 2.19 rejects string
# conditionals derived from env vars).
when: log_destination | default('') != ''
vars: vars:
log_destination: >- log_destination: >-
{{ lookup('ansible.builtin.env', 'ANSIBLE_K3S_LOG_DIR', default=False) }} {{ lookup('ansible.builtin.env', 'ANSIBLE_K3S_LOG_DIR', default=False) }}
-30
View File
@@ -15,10 +15,6 @@
owner: root owner: root
group: root group: root
mode: "0644" mode: "0644"
register: metallb_manifest_download
retries: "{{ download_retries }}"
delay: "{{ download_delay }}"
until: metallb_manifest_download is succeeded
when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
- name: Set image versions in manifest for metallb-{{ metal_lb_type }} - name: Set image versions in manifest for metallb-{{ metal_lb_type }}
@@ -32,29 +28,3 @@
loop_control: loop_control:
label: "{{ item.change }} => {{ item.to }}" label: "{{ item.change }} => {{ item.to }}"
when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
- name: Read back MetalLB manifest from first master
ansible.builtin.slurp:
src: /var/lib/rancher/k3s/server/manifests/metallb-crds.yaml
register: metallb_manifest
when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
- name: Check the MetalLB speaker image reference is present
ansible.builtin.set_fact:
metallb_manifest_has_speaker_tag: >-
{{
('quay.io/metallb/speaker:' ~ metal_lb_speaker_tag_version) in
(metallb_manifest.content | default('') | b64decode)
}}
when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
- name: Fail if MetalLB speaker tag was not applied to the manifest
ansible.builtin.fail:
msg: >-
MetalLB speaker image reference
quay.io/metallb/speaker:{{ metal_lb_speaker_tag_version }}
was not found in the downloaded manifest. The upstream image
reference may have changed.
when:
- ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
- not metallb_manifest_has_speaker_tag
+3 -7
View File
@@ -1,8 +1,4 @@
--- ---
- name: Set _kube_vip_bgp_peers fact
ansible.builtin.set_fact:
_kube_vip_bgp_peers: "{{ lookup('community.general.merge_variables', '^kube_vip_bgp_peers__.+$', initial_value=kube_vip_bgp_peers, groups=kube_vip_bgp_peers_groups) }}" # yamllint disable-line rule:line-length
- name: Create manifests directory on first master - name: Create manifests directory on first master
ansible.builtin.file: ansible.builtin.file:
path: /var/lib/rancher/k3s/server/manifests path: /var/lib/rancher/k3s/server/manifests
@@ -12,9 +8,9 @@
mode: "0644" mode: "0644"
when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
- name: Copy kube-vip RBAC manifest to first master - name: Download vip rbac manifest to first master
ansible.builtin.template: ansible.builtin.get_url:
src: kube-vip-rbac.yaml.j2 url: https://kube-vip.io/manifests/rbac.yaml
dest: /var/lib/rancher/k3s/server/manifests/vip-rbac.yaml dest: /var/lib/rancher/k3s/server/manifests/vip-rbac.yaml
owner: root owner: root
group: root group: root
@@ -1,48 +0,0 @@
# Repository-owned kube-vip RBAC derived from the official v1.2.2
# `kube-vip manifest rbac` output (run with `--inCluster`).
# Source: https://github.com/kube-vip/kube-vip/blob/v1.2.2/pkg/kubevip/config_generator.go
# Permissions are kept exactly as broad as the upstream manifest.
apiVersion: v1
kind: ServiceAccount
metadata:
name: kube-vip
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
name: system:kube-vip-role
rules:
- apiGroups: [""]
resources: ["services/status"]
verbs: ["update"]
- apiGroups: [""]
resources: ["services", "endpoints"]
verbs: ["list", "get", "watch", "update"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["list", "get", "watch", "update", "patch"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["list", "get", "watch", "update", "create"]
- apiGroups: ["discovery.k8s.io"]
resources: ["endpointslices"]
verbs: ["list", "get", "watch", "update"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:kube-vip-binding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:kube-vip-role
subjects:
- kind: ServiceAccount
name: kube-vip
namespace: kube-system
+3 -8
View File
@@ -36,8 +36,8 @@ spec:
- name: vip_interface - name: vip_interface
value: {{ kube_vip_iface }} value: {{ kube_vip_iface }}
{% endif %} {% endif %}
- name: vip_subnet - name: vip_cidr
value: "{{ (kube_vip_endpoint | default(apiserver_endpoint, true)) | ansible.utils.ipsubnet | ansible.utils.ipaddr('prefix') }}" value: "{{ apiserver_endpoint | ansible.utils.ipsubnet | ansible.utils.ipaddr('prefix') }}"
- name: cp_enable - name: cp_enable
value: "true" value: "true"
- name: cp_namespace - name: cp_namespace
@@ -55,16 +55,12 @@ spec:
- name: vip_retryperiod - name: vip_retryperiod
value: "2" value: "2"
- name: address - name: address
value: {{ kube_vip_endpoint | default(apiserver_endpoint, true) }} value: {{ apiserver_endpoint }}
{% if kube_vip_bgp | default(false) | bool %} {% if kube_vip_bgp | default(false) | bool %}
{% if kube_vip_bgp_routerid is defined %} {% if kube_vip_bgp_routerid is defined %}
- name: bgp_routerid - name: bgp_routerid
value: "{{ kube_vip_bgp_routerid }}" value: "{{ kube_vip_bgp_routerid }}"
{% endif %} {% endif %}
{% if _kube_vip_bgp_peers | length > 0 %}
- name: bgp_peers
value: "{{ _kube_vip_bgp_peers | map(attribute='peer_address') | zip(_kube_vip_bgp_peers | map(attribute='peer_asn')) | map('join', ':') | join(',') }}" # yamllint disable-line rule:line-length
{% else %}
{% if kube_vip_bgp_as is defined %} {% if kube_vip_bgp_as is defined %}
- name: bgp_as - name: bgp_as
value: "{{ kube_vip_bgp_as }}" value: "{{ kube_vip_bgp_as }}"
@@ -77,7 +73,6 @@ spec:
- name: bgp_peeras - name: bgp_peeras
value: "{{ kube_vip_bgp_peeras }}" value: "{{ kube_vip_bgp_peeras }}"
{% endif %} {% endif %}
{% endif %}
{% endif %} {% endif %}
image: ghcr.io/kube-vip/kube-vip:{{ kube_vip_tag_version }} image: ghcr.io/kube-vip/kube-vip:{{ kube_vip_tag_version }}
imagePullPolicy: Always imagePullPolicy: Always
+2 -14
View File
@@ -9,7 +9,7 @@ calico_ebpf: false
calico_encapsulation: VXLANCrossSubnet calico_encapsulation: VXLANCrossSubnet
calico_natOutgoing: Enabled # noqa var-naming calico_natOutgoing: Enabled # noqa var-naming
calico_nodeSelector: all() # noqa var-naming calico_nodeSelector: all() # noqa var-naming
calico_tag: v3.32.1 calico_tag: v3.27.2
cilium_bgp: false cilium_bgp: false
cilium_exportPodCIDR: true # noqa var-naming cilium_exportPodCIDR: true # noqa var-naming
@@ -18,18 +18,8 @@ cilium_bgp_peer_asn: 64512
cilium_bgp_neighbors: [] cilium_bgp_neighbors: []
cilium_bgp_neighbors_groups: ['k3s_all'] cilium_bgp_neighbors_groups: ['k3s_all']
cilium_bgp_lb_cidr: 192.168.31.0/24 cilium_bgp_lb_cidr: 192.168.31.0/24
cilium_envoy: true
cilium_hubble: true cilium_hubble: true
cilium_mode: native cilium_mode: native
cilium_tag: v1.20.0
cilium_cli_tag: v0.19.7
# Shared retry/delay for remote manifest, asset downloads, and waiting on
# Kubernetes resources. The CI runner's resolver intermittently times out on
# GitHub-hosted domains and the kube API can transiently return
# ServiceUnavailable, so retry transient DNS/network/API failures.
download_retries: 5
download_delay: 10
cluster_cidr: 10.52.0.0/16 cluster_cidr: 10.52.0.0/16
enable_bpf_masquerade: true enable_bpf_masquerade: true
@@ -38,7 +28,5 @@ group_name_master: master
metal_lb_mode: layer2 metal_lb_mode: layer2
metal_lb_available_timeout: 240s metal_lb_available_timeout: 240s
metal_lb_controller_tag_version: v0.16.0 metal_lb_controller_tag_version: v0.14.3
metal_lb_enabled: true
metal_lb_interfaces: []
metal_lb_ip_range: 192.168.30.80-192.168.30.90 metal_lb_ip_range: 192.168.30.80-192.168.30.90
+3 -30
View File
@@ -43,7 +43,7 @@ argument_specs:
calico_tag: calico_tag:
description: Calico version tag description: Calico version tag
default: v3.32.1 default: v3.27.2
cilium_bgp: cilium_bgp:
description: description:
@@ -93,24 +93,12 @@ argument_specs:
default: ~ default: ~
cilium_mode: cilium_mode:
description: description: Inner-node communication mode
- Inner-node communication mode.
- Cilium accepts `native` or `tunnel`. `routed` is accepted as a
legacy alias and is mapped to `tunnel` at install time.
default: native default: native
choices: choices:
- native - native
- tunnel
- routed - routed
cilium_tag:
description: Cilium version tag
default: v1.20.0
cilium_cli_tag:
description: Cilium CLI version tag
default: v0.19.7
cluster_cidr: cluster_cidr:
description: Inner-cluster IP range description: Inner-cluster IP range
default: 10.52.0.0/16 default: 10.52.0.0/16
@@ -133,13 +121,6 @@ argument_specs:
description: IP range for kube-vip load balancer description: IP range for kube-vip load balancer
default: ~ default: ~
metal_lb_enabled:
description:
- Enable installing the MetalLB pool for service load balancing.
- Set false to skip MetalLB, for example when an external load balancer is used.
default: true
type: bool
metal_lb_available_timeout: metal_lb_available_timeout:
description: Wait for MetalLB resources description: Wait for MetalLB resources
default: 240s default: 240s
@@ -148,17 +129,9 @@ argument_specs:
description: MetalLB ip range for load balancer description: MetalLB ip range for load balancer
default: 192.168.30.80-192.168.30.90 default: 192.168.30.80-192.168.30.90
metal_lb_interfaces:
description: >-
List of network interfaces on which MetalLB should announce the
load balancer IPs in layer2 mode. When empty (default), MetalLB
announces on all interfaces.
type: list
default: []
metal_lb_controller_tag_version: metal_lb_controller_tag_version:
description: Image tag for MetalLB description: Image tag for MetalLB
default: v0.16.0 default: v0.14.3
metal_lb_mode: metal_lb_mode:
description: Metallb mode description: Metallb mode
+49 -100
View File
@@ -11,18 +11,6 @@
group: root group: root
mode: "0755" mode: "0755"
- name: "Download to first master: Calico CRD bundle for {{ calico_tag }}"
ansible.builtin.get_url:
url: https://raw.githubusercontent.com/projectcalico/calico/{{ calico_tag }}/manifests/v1_crd_projectcalico_org.yaml # noqa yaml[line-length]
dest: /tmp/k3s/v1_crd_projectcalico_org.yaml
owner: root
group: root
mode: "0755"
register: calico_crd_download
retries: "{{ download_retries }}"
delay: "{{ download_delay }}"
until: calico_crd_download is succeeded
- name: "Download to first master: manifest for Tigera Operator and Calico CRDs" - name: "Download to first master: manifest for Tigera Operator and Calico CRDs"
ansible.builtin.get_url: ansible.builtin.get_url:
url: https://raw.githubusercontent.com/projectcalico/calico/{{ calico_tag }}/manifests/tigera-operator.yaml url: https://raw.githubusercontent.com/projectcalico/calico/{{ calico_tag }}/manifests/tigera-operator.yaml
@@ -30,52 +18,6 @@
owner: root owner: root
group: root group: root
mode: "0755" mode: "0755"
register: tigera_operator_download
retries: "{{ download_retries }}"
delay: "{{ download_delay }}"
until: tigera_operator_download is succeeded
- name: Apply Calico CRD bundle with server-side apply
ansible.builtin.command: >-
{{ k3s_kubectl_binary | default('k3s kubectl') }}
apply --server-side --force-conflicts
-f /tmp/k3s/v1_crd_projectcalico_org.yaml
register: apply_crds
changed_when: >-
'created' in apply_crds.stdout or
'configured' in apply_crds.stdout or
'applied' in apply_crds.stdout
failed_when: apply_crds.rc != 0
- name: Apply Tigera Operator manifest idempotently with server-side apply
ansible.builtin.command: >-
{{ k3s_kubectl_binary | default('k3s kubectl') }}
apply --server-side --force-conflicts
-f /tmp/k3s/tigera-operator.yaml
register: apply_operator
changed_when: >-
'created' in apply_operator.stdout or
'configured' in apply_operator.stdout or
'applied' in apply_operator.stdout
failed_when: apply_operator.rc != 0
- name: Wait for Tigera Operator and managed CRDs to become available
ansible.builtin.command: >-
{{ k3s_kubectl_binary | default('k3s kubectl') }} wait {{ item.type }}/{{ item.name }}
{% if item.namespace is defined %}--namespace='{{ item.namespace }}'{% endif %}
--for=condition={{ item.condition }}
--timeout=30s
register: tigera_result
changed_when: false
until: tigera_result is succeeded
retries: 7
delay: 7
with_items:
- { name: tigera-operator, type: deployment, namespace: tigera-operator, condition: Available=True }
- { name: installations.operator.tigera.io, type: crd, condition: Established }
- { name: apiservers.operator.tigera.io, type: crd, condition: Established }
loop_control:
label: "{{ item.type }}/{{ item.name }}"
- name: Copy Calico custom resources manifest to first master - name: Copy Calico custom resources manifest to first master
ansible.builtin.template: ansible.builtin.template:
@@ -85,16 +27,53 @@
group: root group: root
mode: "0755" mode: "0755"
- name: Apply Calico custom resources - name: Deploy or replace Tigera Operator
block:
- name: Deploy Tigera Operator
ansible.builtin.command:
cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} create -f /tmp/k3s/tigera-operator.yaml"
register: create_operator
changed_when: "'created' in create_operator.stdout"
failed_when: "'Error' in create_operator.stderr and 'already exists' not in create_operator.stderr"
rescue:
- name: Replace existing Tigera Operator
ansible.builtin.command:
cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} replace -f /tmp/k3s/tigera-operator.yaml"
register: replace_operator
changed_when: "'replaced' in replace_operator.stdout"
failed_when: "'Error' in replace_operator.stderr"
- name: Wait for Tigera Operator resources
ansible.builtin.command: >- ansible.builtin.command: >-
{{ k3s_kubectl_binary | default('k3s kubectl') }} {{ k3s_kubectl_binary | default('k3s kubectl') }} wait {{ item.type }}/{{ item.name }}
apply -f /tmp/k3s/custom-resources.yaml --namespace='tigera-operator'
register: apply_cr --for=condition=Available=True
changed_when: >- --timeout=30s
'configured' in apply_cr.stdout or register: tigera_result
'created' in apply_cr.stdout or changed_when: false
'unchanged' in apply_cr.stdout until: tigera_result is succeeded
failed_when: apply_cr.rc != 0 retries: 7
delay: 7
with_items:
- { name: tigera-operator, type: deployment }
loop_control:
label: "{{ item.type }}/{{ item.name }}"
- name: Deploy Calico custom resources
block:
- name: Deploy custom resources for Calico
ansible.builtin.command:
cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} create -f /tmp/k3s/custom-resources.yaml"
register: create_cr
changed_when: "'created' in create_cr.stdout"
failed_when: "'Error' in create_cr.stderr and 'already exists' not in create_cr.stderr"
rescue:
- name: Apply new Calico custom resource manifest
ansible.builtin.command:
cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} apply -f /tmp/k3s/custom-resources.yaml"
register: apply_cr
changed_when: "'configured' in apply_cr.stdout or 'created' in apply_cr.stdout"
failed_when: "'Error' in apply_cr.stderr"
- name: Wait for Calico system resources to be available - name: Wait for Calico system resources to be available
ansible.builtin.command: >- ansible.builtin.command: >-
@@ -112,8 +91,8 @@
register: cr_result register: cr_result
changed_when: false changed_when: false
until: cr_result is succeeded until: cr_result is succeeded
retries: 60 retries: 30
delay: 10 delay: 7
with_items: with_items:
- { name: calico-typha, type: deployment, namespace: calico-system } - { name: calico-typha, type: deployment, namespace: calico-system }
- { name: calico-kube-controllers, type: deployment, namespace: calico-system } - { name: calico-kube-controllers, type: deployment, namespace: calico-system }
@@ -125,40 +104,10 @@
type: daemonset type: daemonset
selector: k8s-app=calico-node selector: k8s-app=calico-node
namespace: calico-system namespace: calico-system
- { name: calico-apiserver, type: deployment, namespace: calico-apiserver }
loop_control: loop_control:
label: "{{ item.type }}/{{ item.name }}" label: "{{ item.type }}/{{ item.name }}"
# The Calico API server is an optional add-on for managing Calico policy via
# kubectl (the projectcalico.org/v3 Kubernetes API). It is not required for
# Calico CNI data plane operation and does not reconcile on K3s with recent
# Calico releases, so it is treated as best-effort here.
- name: Wait for the optional Calico API server (best effort)
ansible.builtin.command: >-
{{ k3s_kubectl_binary | default('k3s kubectl') }} wait deployment/calico-apiserver
--namespace=calico-apiserver --for=condition=Available --timeout=30s
register: cr_apiserver
changed_when: false
until: cr_apiserver is succeeded
retries: 30
delay: 10
ignore_errors: true
- name: Verify Calico TigeraStatus reports Available
ansible.builtin.command: >-
{{ k3s_kubectl_binary | default('k3s kubectl') }}
get tigerastatus {{ item }}
-o jsonpath='{.status.conditions[?(@.type=="Available")].status}'
register: tigera_status
changed_when: false
until: tigera_status.stdout | trim == 'True'
retries: 30
delay: 7
failed_when: tigera_status.rc != 0 or (tigera_status.stdout | trim) != 'True'
with_items:
- calico
loop_control:
label: "tigerastatus/{{ item }}"
- name: Patch Felix configuration for eBPF mode - name: Patch Felix configuration for eBPF mode
ansible.builtin.command: ansible.builtin.command:
cmd: > cmd: >
+32 -62
View File
@@ -30,13 +30,26 @@
}} }}
when: cilium_cli_installed.rc == 0 when: cilium_cli_installed.rc == 0
- name: Get latest stable Cilium CLI version file
ansible.builtin.get_url:
url: https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt
dest: /tmp/k3s/cilium-cli-stable.txt
owner: root
group: root
mode: "0755"
- name: Read Cilium CLI stable version from file
ansible.builtin.command: cat /tmp/k3s/cilium-cli-stable.txt
register: cli_ver
changed_when: false
- name: Log installed Cilium CLI version - name: Log installed Cilium CLI version
ansible.builtin.debug: ansible.builtin.debug:
msg: "Installed Cilium CLI version: {{ installed_cli_version | default('Not installed') }}" msg: "Installed Cilium CLI version: {{ installed_cli_version | default('Not installed') }}"
- name: Log pinned Cilium CLI version - name: Log latest stable Cilium CLI version
ansible.builtin.debug: ansible.builtin.debug:
msg: "Pinned Cilium CLI version: {{ cilium_cli_tag }}" msg: "Latest Cilium CLI version: {{ cli_ver.stdout }}"
- name: Determine if Cilium CLI needs installation or update - name: Determine if Cilium CLI needs installation or update
ansible.builtin.set_fact: ansible.builtin.set_fact:
@@ -44,7 +57,7 @@
{{ {{
cilium_cli_installed.rc != 0 or cilium_cli_installed.rc != 0 or
(cilium_cli_installed.rc == 0 and (cilium_cli_installed.rc == 0 and
installed_cli_version != cilium_cli_tag) installed_cli_version != cli_ver.stdout)
}} }}
- name: Install or update Cilium CLI - name: Install or update Cilium CLI
@@ -65,11 +78,7 @@
- .tar.gz - .tar.gz
- .tar.gz.sha256sum - .tar.gz.sha256sum
vars: vars:
cilium_base_url: https://github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_tag }} cilium_base_url: https://github.com/cilium/cilium-cli/releases/download/{{ cli_ver.stdout }}
register: cilium_cli_download
retries: "{{ download_retries }}"
delay: "{{ download_delay }}"
until: cilium_cli_download is succeeded
- name: Verify the downloaded tarball - name: Verify the downloaded tarball
ansible.builtin.shell: | ansible.builtin.shell: |
@@ -147,13 +156,6 @@
Target Cilium version: {{ cilium_tag }}, Target Cilium version: {{ cilium_tag }},
Update needed: {{ cilium_needs_update }} Update needed: {{ cilium_needs_update }}
- name: Map the legacy routed mode to Cilium tunnel mode
ansible.builtin.set_fact:
# Cilium 1.20 accepts `native` or `tunnel`. `routed` was the name used
# by older releases and is kept only as a backward-compatible alias.
cilium_routing_mode: >-
{{ 'tunnel' if cilium_mode == 'routed' else cilium_mode }}
- name: Install Cilium - name: Install Cilium
ansible.builtin.command: >- ansible.builtin.command: >-
{% if cilium_installed.rc != 0 %} {% if cilium_installed.rc != 0 %}
@@ -165,35 +167,27 @@
--helm-set operator.replicas="1" --helm-set operator.replicas="1"
{{ '--helm-set devices=' + cilium_iface if cilium_iface != 'auto' else '' }} {{ '--helm-set devices=' + cilium_iface if cilium_iface != 'auto' else '' }}
--helm-set ipam.operator.clusterPoolIPv4PodCIDRList={{ cluster_cidr }} --helm-set ipam.operator.clusterPoolIPv4PodCIDRList={{ cluster_cidr }}
{% if cilium_routing_mode == "native" or (cilium_bgp and cilium_exportPodCIDR != 'false') %} {% if cilium_mode == "native" or (cilium_bgp and cilium_exportPodCIDR != 'false') %}
--helm-set ipv4NativeRoutingCIDR={{ cluster_cidr }} --helm-set ipv4NativeRoutingCIDR={{ cluster_cidr }}
{% endif %} {% endif %}
--helm-set k8sServiceHost="127.0.0.1" --helm-set k8sServiceHost="127.0.0.1"
--helm-set k8sServicePort="6444" --helm-set k8sServicePort="6444"
--helm-set routingMode={{ cilium_routing_mode }} --helm-set routingMode={{ cilium_mode }}
--helm-set autoDirectNodeRoutes={{ "true" if cilium_routing_mode == "native" else "false" }} --helm-set autoDirectNodeRoutes={{ "true" if cilium_mode == "native" else "false" }}
--helm-set kubeProxyReplacement={{ kube_proxy_replacement }} --helm-set kubeProxyReplacement={{ kube_proxy_replacement }}
--helm-set bpf.masquerade={{ enable_bpf_masquerade }} --helm-set bpf.masquerade={{ enable_bpf_masquerade }}
--helm-set bgpControlPlane.enabled={{ cilium_bgp | default("false") }} --helm-set bgpControlPlane.enabled={{ cilium_bgp | default("false") }}
--helm-set hubble.enabled={{ "true" if cilium_hubble else "false" }} --helm-set hubble.enabled={{ "true" if cilium_hubble else "false" }}
--helm-set hubble.relay.enabled={{ "true" if cilium_hubble else "false" }} --helm-set hubble.relay.enabled={{ "true" if cilium_hubble else "false" }}
--helm-set hubble.ui.enabled={{ "true" if cilium_hubble else "false" }} --helm-set hubble.ui.enabled={{ "true" if cilium_hubble else "false" }}
--helm-set envoy.enabled={{ "true" if cilium_envoy else "false" }}
{% if kube_proxy_replacement is not false %} {% if kube_proxy_replacement is not false %}
--helm-set loadBalancer.algorithm={{ bpf_lb_algorithm }} --helm-set bpf.loadBalancer.algorithm={{ bpf_lb_algorithm }}
--helm-set loadBalancer.mode={{ bpf_lb_mode }} --helm-set bpf.loadBalancer.mode={{ bpf_lb_mode }}
{% endif %} {% endif %}
environment: environment:
KUBECONFIG: "{{ ansible_user_dir }}/.kube/config" KUBECONFIG: "{{ ansible_user_dir }}/.kube/config"
register: cilium_install_result register: cilium_install_result
changed_when: cilium_install_result.rc == 0 changed_when: cilium_install_result.rc == 0
# cilium install/upgrade fetches the Helm chart from helm.cilium.io, which is
# fronted by GitHub Pages and can transiently fail DNS resolution through the
# host resolver (intermittent "lookup helm.cilium.io ... i/o timeout"). Retry
# so a transient name/network failure does not abort the whole converge play.
until: cilium_install_result.rc == 0
retries: "{{ download_retries }}"
delay: "{{ download_delay }}"
when: cilium_installed.rc != 0 or cilium_needs_update when: cilium_installed.rc != 0 or cilium_needs_update
- name: Wait for Cilium resources - name: Wait for Cilium resources
@@ -224,22 +218,13 @@
when: >- when: >-
not item.check_hubble | default(false) or (item.check_hubble | default(false) and cilium_hubble) not item.check_hubble | default(false) or (item.check_hubble | default(false) and cilium_hubble)
- name: Wait for Cilium status to be healthy
ansible.builtin.command: cilium status --wait
environment:
KUBECONFIG: "{{ ansible_user_dir }}/.kube/config"
register: cilium_status
changed_when: false
until: cilium_status.rc == 0
retries: 30
delay: 7
- name: Configure Cilium BGP - name: Configure Cilium BGP
when: cilium_bgp when: cilium_bgp
block: block:
- name: Set _cilium_bgp_neighbors fact - name: Set _cilium_bgp_neighbors fact
ansible.builtin.set_fact: ansible.builtin.set_fact:
_cilium_bgp_neighbors: "{{ lookup('community.general.merge_variables', '^cilium_bgp_neighbors__.+$', initial_value=cilium_bgp_neighbors, groups=cilium_bgp_neighbors_groups) }}" # yamllint disable-line rule:line-length _cilium_bgp_neighbors: "{{ lookup('community.general.merge_variables', '^cilium_bgp_neighbors__.+$', initial_value=cilium_bgp_neighbors, groups=cilium_bgp_neighbors_groups) }}" # yamllint disable-line rule:line-length
when: cilium_bgp_neighbors | length > 0
- name: Copy BGP manifests to first master - name: Copy BGP manifests to first master
ansible.builtin.template: ansible.builtin.template:
@@ -249,38 +234,23 @@
group: root group: root
mode: "0755" mode: "0755"
- name: Preflight validate rendered BGP manifests against installed CRDs
ansible.builtin.command: >-
{{ k3s_kubectl_binary | default('k3s kubectl') }}
apply --dry-run=server -f /tmp/k3s/cilium-bgp.yaml
register: preflight_cr
changed_when: false
failed_when: preflight_cr.rc != 0
- name: Apply BGP manifests - name: Apply BGP manifests
ansible.builtin.command: >- ansible.builtin.command:
{{ k3s_kubectl_binary | default('k3s kubectl') }} cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} apply -f /tmp/k3s/cilium-bgp.yaml"
apply -f /tmp/k3s/cilium-bgp.yaml
register: apply_cr register: apply_cr
changed_when: "'configured' in apply_cr.stdout or 'created' in apply_cr.stdout" changed_when: "'configured' in apply_cr.stdout or 'created' in apply_cr.stdout"
failed_when: apply_cr.rc != 0 failed_when: "'is invalid' in apply_cr.stderr"
ignore_errors: true
- name: Remove deprecated CiliumBGPPeeringPolicy after v2 resources are accepted - name: Print error message if BGP manifests application fails
ansible.builtin.command: >- ansible.builtin.debug:
{{ k3s_kubectl_binary | default('k3s kubectl') }} msg: "{{ apply_cr.stderr }}"
delete CiliumBGPPeeringPolicy.cilium.io 01-bgp-peering-policy when: "'is invalid' in apply_cr.stderr"
register: delete_old_policy
changed_when: "'deleted' in delete_old_policy.stdout"
# The policy (and possibly its CRD) may already be absent; this is
# intentionally tolerated, not a command whose failure must be hidden.
failed_when: false
- name: Test for BGP config resources - name: Test for BGP config resources
ansible.builtin.command: "{{ item }}" ansible.builtin.command: "{{ item }}"
loop: loop:
- "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPClusterConfig.cilium.io" - "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPPeeringPolicy.cilium.io"
- "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPPeerConfig.cilium.io"
- "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPAdvertisement.cilium.io"
- "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumLoadBalancerIPPool.cilium.io" - "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumLoadBalancerIPPool.cilium.io"
changed_when: false changed_when: false
loop_control: loop_control:
+1 -6
View File
@@ -12,12 +12,7 @@
- name: Deploy metallb pool - name: Deploy metallb pool
ansible.builtin.include_tasks: metallb.yml ansible.builtin.include_tasks: metallb.yml
tags: metallb tags: metallb
# Deploy MetalLB unless explicitly disabled, kube-vip owns the load balancer when: kube_vip_lb_ip_range is not defined and (not cilium_bgp or cilium_iface is not defined)
# IP range, or Cilium BGP is enabled (Cilium then provides its own load
# balancing). The cilium_bgp default keeps this safe when Cilium variables are
# not in scope at all (#644) while still deploying MetalLB when a non-BGP
# Cilium CNI is in use.
when: metal_lb_enabled and kube_vip_lb_ip_range is not defined and not (cilium_bgp | default(false) | bool)
- name: Remove tmp directory used for manifests - name: Remove tmp directory used for manifests
ansible.builtin.file: ansible.builtin.file:
+12 -30
View File
@@ -40,14 +40,8 @@
- name: Test metallb-system namespace - name: Test metallb-system namespace
ansible.builtin.command: >- ansible.builtin.command: >-
{{ k3s_kubectl_binary | default('k3s kubectl') }} get namespace metallb-system {{ k3s_kubectl_binary | default('k3s kubectl') }} -n metallb-system
changed_when: false changed_when: false
# The kube API can briefly return ServiceUnavailable while MetalLB converges,
# which would otherwise abort the whole converge play on a transient error.
register: metallb_namespace_result
until: metallb_namespace_result.rc == 0
retries: "{{ download_retries }}"
delay: "{{ download_delay }}"
with_items: "{{ groups[group_name_master | default('master')] }}" with_items: "{{ groups[group_name_master | default('master')] }}"
run_once: true run_once: true
@@ -60,12 +54,6 @@
{% if item.condition | default(False) -%}{{ item.condition }}{%- endif %} {% if item.condition | default(False) -%}{{ item.condition }}{%- endif %}
--timeout='{{ metal_lb_available_timeout }}' --timeout='{{ metal_lb_available_timeout }}'
changed_when: false changed_when: false
# The kube API can briefly return ServiceUnavailable while MetalLB converges,
# which would otherwise abort the whole converge play on a transient error.
register: metallb_wait_result
until: metallb_wait_result.rc == 0
retries: "{{ download_retries }}"
delay: "{{ download_delay }}"
run_once: true run_once: true
with_items: with_items:
- description: controller - description: controller
@@ -97,20 +85,22 @@
- name: Set metallb webhook service name - name: Set metallb webhook service name
ansible.builtin.set_fact: ansible.builtin.set_fact:
# Inspected the v0.16.0 manifest: the newer webhook Service name is used metallb_webhook_service_name: >-
# by every supported MetalLB release, so the old pre-0.14.4 branch is gone. {{
metallb_webhook_service_name: metallb-webhook-service (
(metal_lb_controller_tag_version | regex_replace('^v', ''))
is
version('0.14.4', '<', version_type='semver')
) | ternary(
'webhook-service',
'metallb-webhook-service'
)
}}
- name: Test metallb-system webhook-service endpoint - name: Test metallb-system webhook-service endpoint
ansible.builtin.command: >- ansible.builtin.command: >-
{{ k3s_kubectl_binary | default('k3s kubectl') }} -n metallb-system get endpoints {{ metallb_webhook_service_name }} {{ k3s_kubectl_binary | default('k3s kubectl') }} -n metallb-system get endpoints {{ metallb_webhook_service_name }}
changed_when: false changed_when: false
# The kube API can briefly return ServiceUnavailable while MetalLB converges,
# which would otherwise abort the whole converge play on a transient error.
register: metallb_webhook_result
until: metallb_webhook_result.rc == 0
retries: "{{ download_retries }}"
delay: "{{ download_delay }}"
with_items: "{{ groups[group_name_master | default('master')] }}" with_items: "{{ groups[group_name_master | default('master')] }}"
run_once: true run_once: true
@@ -130,10 +120,6 @@
changed_when: false changed_when: false
run_once: true run_once: true
when: metal_lb_mode == "layer2" when: metal_lb_mode == "layer2"
register: metallb_l2_test_result
until: metallb_l2_test_result.rc == 0
retries: "{{ download_retries }}"
delay: "{{ download_delay }}"
with_items: with_items:
- IPAddressPool - IPAddressPool
- L2Advertisement - L2Advertisement
@@ -144,10 +130,6 @@
changed_when: false changed_when: false
run_once: true run_once: true
when: metal_lb_mode == "bgp" when: metal_lb_mode == "bgp"
register: metallb_bgp_test_result
until: metallb_bgp_test_result.rc == 0
retries: "{{ download_retries }}"
delay: "{{ download_delay }}"
with_items: with_items:
- IPAddressPool - IPAddressPool
- BGPPeer - BGPPeer
+33 -77
View File
@@ -1,84 +1,40 @@
# Cilium BGP Control Plane v2 resources. apiVersion: "cilium.io/v2alpha1"
# Replace the deprecated v2alpha1 CiliumBGPPeeringPolicy that was removed kind: CiliumBGPPeeringPolicy
# in Cilium 1.19.
{% set _cilium_default_peer = {'peer_address': cilium_bgp_peer_address, 'peer_asn': cilium_bgp_peer_asn} %}
{% set _cilium_peers = _cilium_bgp_neighbors if _cilium_bgp_neighbors | length > 0 else [_cilium_default_peer] %}
apiVersion: cilium.io/v2
kind: CiliumBGPPeerConfig
metadata: metadata:
name: cilium-peer name: 01-bgp-peering-policy
spec: spec: # CiliumBGPPeeringPolicySpec
# Matches the timers and multihop used by the previous v2alpha1 policy. virtualRouters: # []CiliumBGPVirtualRouter
ebgpMultihop: 10 - localASN: {{ cilium_bgp_my_asn }}
timers: exportPodCIDR: {{ cilium_exportPodCIDR | default('true') }}
connectRetryTimeSeconds: 120 neighbors: # []CiliumBGPNeighbor
holdTimeSeconds: 90 {% if _cilium_bgp_neighbors | length > 0 %}
keepAliveTimeSeconds: 30 {% for item in _cilium_bgp_neighbors %}
gracefulRestart: - peerAddress: '{{ item.peer_address + "/32"}}'
enabled: true peerASN: {{ item.peer_asn }}
restartTimeSeconds: 120 eBGPMultihopTTL: 10
families: connectRetryTimeSeconds: 120
- afi: ipv4 holdTimeSeconds: 90
safi: unicast keepAliveTimeSeconds: 30
advertisements: gracefulRestart:
matchLabels: enabled: true
advertise: "bgp" restartTimeSeconds: 120
---
apiVersion: cilium.io/v2
kind: CiliumBGPClusterConfig
metadata:
name: cilium-bgp
spec:
# Explicitly select every node so the BGP instance runs across the cluster.
nodeSelector:
matchExpressions:
- key: somekey
operator: NotIn
values: ['never-used-value']
bgpInstances:
- name: "instance-{{ cilium_bgp_my_asn }}"
localASN: {{ cilium_bgp_my_asn }}
peers:
{% for peer in _cilium_peers %}
- name: "peer-{{ peer.peer_asn }}-{{ loop.index }}"
peerASN: {{ peer.peer_asn }}
peerAddress: {{ peer.peer_address }}
peerConfigRef:
name: cilium-peer
{% endfor %} {% endfor %}
{% if cilium_exportPodCIDR %} {% else %}
--- - peerAddress: '{{ cilium_bgp_peer_address + "/32"}}'
apiVersion: cilium.io/v2 peerASN: {{ cilium_bgp_peer_asn }}
kind: CiliumBGPAdvertisement eBGPMultihopTTL: 10
metadata: connectRetryTimeSeconds: 120
name: cilium-pod-cidrs holdTimeSeconds: 90
labels: keepAliveTimeSeconds: 30
advertise: "bgp" gracefulRestart:
spec: enabled: true
advertisements: restartTimeSeconds: 120
- advertisementType: "PodCIDR"
{% endif %} {% endif %}
serviceSelector:
matchExpressions:
- {key: somekey, operator: NotIn, values: ['never-used-value']}
--- ---
apiVersion: cilium.io/v2 apiVersion: "cilium.io/v2alpha1"
kind: CiliumBGPAdvertisement
metadata:
name: cilium-lb-services
labels:
advertise: "bgp"
spec:
advertisements:
- advertisementType: "Service"
service:
addresses:
- LoadBalancerIP
# Advertise all Services carrying an ingress address from the pool.
selector:
matchExpressions:
- key: somekey
operator: NotIn
values: ['never-used-value']
---
apiVersion: cilium.io/v2
kind: CiliumLoadBalancerIPPool kind: CiliumLoadBalancerIPPool
metadata: metadata:
name: "01-lb-pool" name: "01-lb-pool"
@@ -21,11 +21,6 @@ kind: L2Advertisement
metadata: metadata:
name: default name: default
namespace: metallb-system namespace: metallb-system
{% if metal_lb_interfaces | default([]) | length > 0 %}
spec:
interfaces:{% for iface in metal_lb_interfaces %}
- {{ iface }}{% endfor %}
{% endif %}
{% endif %} {% endif %}
{% if metal_lb_mode == "bgp" %} {% if metal_lb_mode == "bgp" %}
--- ---
-1
View File
@@ -5,4 +5,3 @@ argument_specs:
options: options:
custom_reboot_command: custom_reboot_command:
default: ~ default: ~
description: Command to run on reboot
-2
View File
@@ -1,6 +1,4 @@
--- ---
disable_swap: true
secure_path: secure_path:
RedHat: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin RedHat: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
Suse: /usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin Suse: /usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin
-51
View File
@@ -4,28 +4,6 @@
name: "{{ system_timezone }}" name: "{{ system_timezone }}"
when: (system_timezone is defined) and (system_timezone != "Your/Timezone") when: (system_timezone is defined) and (system_timezone != "Your/Timezone")
# k3s recommends swap be disabled on all nodes. Disabling swap is all-or-nothing
# across the cluster: leaving it enabled on some nodes but not others creates
# uneven scheduling/latency behavior. This block turns swap off and comments out
# the swap entries in /etc/fstab so it stays off across reboots. It is idempotent
# and a no-op when swap is already disabled or swapoff is unavailable.
- name: Disable swap on all cluster nodes
when: disable_swap
block:
- name: Turn off swap now
ansible.builtin.command: swapoff -a
register: swapoff_result
changed_when: false
failed_when: false
- name: Comment out swap entries in fstab
ansible.builtin.replace:
path: /etc/fstab
regexp: '^([^#][^\n]*\s+swap\s+)'
replace: '# \\1'
register: fstab_swap
- name: Set SELinux to disabled state - name: Set SELinux to disabled state
ansible.posix.selinux: ansible.posix.selinux:
state: disabled state: disabled
@@ -55,35 +33,6 @@
reload: true reload: true
tags: sysctl tags: sysctl
- name: Check if br_netfilter module exists
ansible.builtin.shell: |
set -o pipefail
find /lib/modules/$(uname -r) -name "br_netfilter.ko*" | wc -l
args:
executable: /bin/bash
register: br_netfilter_exists
changed_when: false
when: ansible_os_family == "RedHat"
tags: sysctl
- name: Install kernel-modules-extra if br_netfilter missing
ansible.builtin.yum:
name: "kernel-modules-extra-{{ ansible_kernel }}"
state: present
update_cache: true
register: kernel_modules_installed
when:
- ansible_os_family == "RedHat"
- br_netfilter_exists.stdout | int == 0
- name: Reboot node if kernel modules package was installed
ansible.builtin.reboot:
msg: "Rebooting to apply new kernel modules package"
reboot_timeout: 300
when:
- ansible_os_family == "RedHat"
- kernel_modules_installed is changed
- name: Add br_netfilter to /etc/modules-load.d/ - name: Add br_netfilter to /etc/modules-load.d/
ansible.builtin.copy: ansible.builtin.copy:
content: br_netfilter content: br_netfilter
-9
View File
@@ -1,9 +0,0 @@
---
argument_specs:
main:
short_description: Proxmox LXC settings
options:
proxmox_lxc_ct_ids:
description: Proxmox container ID list
type: list
required: true
-10
View File
@@ -1,10 +0,0 @@
---
argument_specs:
main:
short_description: Adjust some Raspberry Pi specific requisites
options:
state:
default: present
description:
- Indicates whether the k3s prerequisites for Raspberry Pi should be
- set up (possible values are `present` and `absent`)
-9
View File
@@ -1,9 +0,0 @@
---
argument_specs:
main:
short_description: Proxmox LXC settings
options:
proxmox_lxc_ct_ids:
description: Proxmox container ID list
type: list
required: true

Some files were not shown because too many files have changed in this diff Show More