mirror of
https://github.com/techno-tim/k3s-ansible.git
synced 2026-09-14 08:03:19 +02:00
Compare commits
4 Commits
master
..
34c0aee31a
| Author | SHA1 | Date | |
|---|---|---|---|
| 34c0aee31a | |||
| e210fc21b6 | |||
| 8e26ba8809 | |||
| 754a379d42 |
@@ -3,7 +3,6 @@ profile: production
|
||||
exclude_paths:
|
||||
# default paths
|
||||
- .cache/
|
||||
- .ansible/
|
||||
- .github/
|
||||
- test/fixtures/formatting-before/
|
||||
- test/fixtures/formatting-prettier/
|
||||
@@ -15,17 +14,8 @@ exclude_paths:
|
||||
- molecule/**/prepare.yml
|
||||
- molecule/**/reset.yml
|
||||
|
||||
# Scenario verify inputs are plain variable files, not playbooks. They are
|
||||
# loaded as vars, not executed, so ansible-lint must not treat them as plays.
|
||||
- molecule/**/verify-vars.yml
|
||||
|
||||
# The file was generated by galaxy ansible - don't mess with it.
|
||||
- galaxy.yml
|
||||
|
||||
skip_list:
|
||||
- var-naming[no-role-prefix]
|
||||
|
||||
# The Molecule Vagrant driver injects this module at runtime. The custom create
|
||||
# playbook is syntax-checked separately against the exact pinned plugin module.
|
||||
mock_modules:
|
||||
- vagrant
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
|
||||
<!-- It's a good idea to check this post first for general troubleshooting https://github.com/techno-tim/k3s-ansible/discussions/19 -->
|
||||
|
||||
<!--- Provide a general summary of the issue in the Title above -->
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
<!--- Tell us what should happen -->
|
||||
|
||||
## Current Behavior
|
||||
<!--- Tell us what happens instead of the expected behavior -->
|
||||
|
||||
## Steps to Reproduce
|
||||
|
||||
<!--- reproduce this bug. Include code to reproduce, if relevant -->
|
||||
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
4.
|
||||
|
||||
## Context (variables)
|
||||
<!--- please include which OS, along with the variables used when running the playbook -->
|
||||
|
||||
Operating system:
|
||||
|
||||
Hardware:
|
||||
|
||||
### Variables Used
|
||||
|
||||
`all.yml`
|
||||
|
||||
```yml
|
||||
k3s_version: ""
|
||||
ansible_user: NA
|
||||
systemd_dir: ""
|
||||
|
||||
flannel_iface: ""
|
||||
|
||||
#calico_iface: ""
|
||||
calico_ebpf: ""
|
||||
calico_cidr: ""
|
||||
calico_tag: ""
|
||||
|
||||
apiserver_endpoint: ""
|
||||
|
||||
k3s_token: "NA"
|
||||
|
||||
extra_server_args: ""
|
||||
extra_agent_args: ""
|
||||
|
||||
kube_vip_tag_version: ""
|
||||
|
||||
kube_vip_cloud_provider_tag_version: ""
|
||||
kube_vip_lb_ip_range: ""
|
||||
|
||||
metal_lb_speaker_tag_version: ""
|
||||
metal_lb_controller_tag_version: ""
|
||||
|
||||
metal_lb_ip_range: ""
|
||||
```
|
||||
|
||||
### Hosts
|
||||
|
||||
`host.ini`
|
||||
|
||||
```ini
|
||||
[master]
|
||||
IP.ADDRESS.ONE
|
||||
IP.ADDRESS.TWO
|
||||
IP.ADDRESS.THREE
|
||||
|
||||
[node]
|
||||
IP.ADDRESS.FOUR
|
||||
IP.ADDRESS.FIVE
|
||||
|
||||
[k3s_cluster:children]
|
||||
master
|
||||
node
|
||||
```
|
||||
|
||||
## Possible Solution
|
||||
<!--- Not obligatory, but suggest a fix/reason for the bug, -->
|
||||
|
||||
- [ ] I've checked the [General Troubleshooting Guide](https://github.com/techno-tim/k3s-ansible/discussions/20)
|
||||
@@ -1,102 +0,0 @@
|
||||
---
|
||||
name: Bug report
|
||||
description: Report a reproducible problem with the playbooks, roles, or generated resources.
|
||||
title: "[Bug]: "
|
||||
labels:
|
||||
- bug
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Thanks for reporting a problem. Search existing issues and review the troubleshooting link first.
|
||||
Remove credentials, tokens, public IP addresses, and private hostnames from all fields and logs.
|
||||
|
||||
- type: checkboxes
|
||||
id: prerequisites
|
||||
attributes:
|
||||
label: Prerequisites
|
||||
options:
|
||||
- label: I searched existing issues and discussions for this problem.
|
||||
required: true
|
||||
- label: I reviewed the troubleshooting guidance linked from the issue chooser.
|
||||
required: true
|
||||
- label: I removed secrets and identifying infrastructure details from this report.
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: summary
|
||||
attributes:
|
||||
label: Problem summary
|
||||
description: Describe what failed and its impact.
|
||||
placeholder: A concise description of the problem and affected nodes or components.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: expected
|
||||
attributes:
|
||||
label: Expected behavior
|
||||
description: What should have happened?
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: reproduction
|
||||
attributes:
|
||||
label: Steps to reproduce
|
||||
description: Provide the smallest reliable sequence that reproduces the issue.
|
||||
placeholder: |
|
||||
1. Configure ...
|
||||
2. Run ...
|
||||
3. Observe ...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: revision
|
||||
attributes:
|
||||
label: Repository revision
|
||||
description: Release, tag, branch, or commit SHA used.
|
||||
placeholder: v1.36.2+k3s1+tt1 or a commit SHA
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: ansible-version
|
||||
attributes:
|
||||
label: Ansible version
|
||||
description: Output of `ansible --version`, shortened to version and Python details.
|
||||
placeholder: ansible-core 2.18.0, Python 3.12
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: environment
|
||||
attributes:
|
||||
label: Environment
|
||||
description: Include target OS and version, architecture, node counts, platform, and network provider.
|
||||
placeholder: Debian 13, amd64, 3 control nodes and 2 agents, bare metal, Cilium
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: configuration
|
||||
attributes:
|
||||
label: Relevant sanitized configuration
|
||||
description: Include only variables and inventory groups needed to reproduce the problem.
|
||||
render: yaml
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: Relevant logs or task output
|
||||
description: Include the failing task and surrounding output. Redact sensitive or identifying values.
|
||||
render: shell
|
||||
|
||||
- type: textarea
|
||||
id: context
|
||||
attributes:
|
||||
label: Additional context
|
||||
description: Add attempted fixes, suspected causes, regressions, or other useful context.
|
||||
@@ -1,9 +0,0 @@
|
||||
---
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: Troubleshooting and support
|
||||
url: https://github.com/timothystewart6/k3s-ansible/discussions/20
|
||||
about: Review common troubleshooting guidance and ask configuration or usage questions.
|
||||
- name: General discussions
|
||||
url: https://github.com/timothystewart6/k3s-ansible/discussions
|
||||
about: Discuss ideas and questions that are not confirmed bugs or concrete feature requests.
|
||||
@@ -1,55 +0,0 @@
|
||||
---
|
||||
name: Feature request
|
||||
description: Propose a focused improvement to supported repository behavior.
|
||||
title: "[Feature]: "
|
||||
labels:
|
||||
- enhancement
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Describe the use case before proposing an implementation. Search existing issues and discussions first.
|
||||
|
||||
- type: checkboxes
|
||||
id: prerequisites
|
||||
attributes:
|
||||
label: Prerequisites
|
||||
options:
|
||||
- label: I searched existing issues and discussions for this request.
|
||||
required: true
|
||||
- label: This request is about reusable project behavior, not support for one private environment.
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: problem
|
||||
attributes:
|
||||
label: Problem or use case
|
||||
description: What limitation exists, who encounters it, and why does it matter?
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: proposal
|
||||
attributes:
|
||||
label: Proposed behavior
|
||||
description: Describe the desired user-visible result. Include example variables or commands when useful.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: alternatives
|
||||
attributes:
|
||||
label: Alternatives considered
|
||||
description: Describe workarounds or other designs and their tradeoffs.
|
||||
|
||||
- type: textarea
|
||||
id: compatibility
|
||||
attributes:
|
||||
label: Compatibility and operational impact
|
||||
description: Note affected operating systems, architectures, CNIs, existing clusters, or reset behavior.
|
||||
|
||||
- type: textarea
|
||||
id: context
|
||||
attributes:
|
||||
label: Additional context
|
||||
description: Add relevant upstream documentation, examples, or prior discussion.
|
||||
@@ -1,46 +1,15 @@
|
||||
## Summary
|
||||
|
||||
<!-- Explain the problem and the resulting behavior. Keep implementation details in the sections below. -->
|
||||
|
||||
## Changes
|
||||
# Proposed Changes
|
||||
<!--- Provide a general summary of your changes -->
|
||||
|
||||
-
|
||||
-
|
||||
-
|
||||
|
||||
## Related issues
|
||||
## Checklist
|
||||
|
||||
<!-- Use "Fixes #123" when this pull request should close an issue. Write "None" when not applicable. -->
|
||||
|
||||
## Testing
|
||||
|
||||
<!-- List exact commands, scenarios, and relevant manual checks. Do not check a box for a test that was not run. -->
|
||||
|
||||
- [ ] `pre-commit run --all-files`
|
||||
- [ ] Relevant Ansible syntax checks
|
||||
- [ ] Relevant focused regression tests
|
||||
- [ ] Relevant Molecule scenario
|
||||
- [ ] Provisioning tested against a non-production cluster
|
||||
- [ ] Reset behavior tested against a non-production cluster
|
||||
|
||||
Not run, with reason:
|
||||
|
||||
## Risk and compatibility
|
||||
|
||||
<!-- Cover existing clusters, upgrades, networking, supported platforms, security, and rollback. Write "None" when a
|
||||
category is not affected. -->
|
||||
|
||||
- Existing cluster or upgrade impact:
|
||||
- Networking or CNI impact:
|
||||
- Security impact:
|
||||
- Rollback plan:
|
||||
|
||||
## Documentation
|
||||
|
||||
<!-- Identify updated docs and sample configuration, or explain why no documentation change is needed. -->
|
||||
|
||||
## Final checklist
|
||||
|
||||
- [ ] The change is focused and contains no unrelated edits.
|
||||
- [ ] Tests cover new behavior or a regression, where applicable.
|
||||
- [ ] User-facing variables are documented in role defaults, sample inventory, and the README.
|
||||
- [ ] Logs, examples, and configuration contain no secrets or identifying infrastructure details.
|
||||
- [ ] Generated files and local environment files are not included.
|
||||
- [ ] Tested locally
|
||||
- [ ] Ran `site.yml` playbook
|
||||
- [ ] Ran `reset.yml` playbook
|
||||
- [ ] Did not add any unnecessary changes
|
||||
- [ ] Ran pre-commit install at least once before committing
|
||||
- [ ] 🚀
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
---
|
||||
self-hosted-runner:
|
||||
labels:
|
||||
- k3s-ci
|
||||
- virtualbox
|
||||
- nested-virt
|
||||
@@ -1,6 +0,0 @@
|
||||
# GitHub Copilot instructions
|
||||
|
||||
Read and follow the repository's root-level `AGENTS.md` before proposing or making changes. It is the canonical guide
|
||||
for architecture, safety, implementation, validation, and documentation expectations.
|
||||
|
||||
Do not duplicate repository guidance here. If instructions need to change, update `AGENTS.md`.
|
||||
+23
-72
@@ -1,91 +1,42 @@
|
||||
#!/bin/bash
|
||||
|
||||
# download-boxes.sh
|
||||
# Validate the pinned Vagrant box set and download exact versions that are not
|
||||
# already present in VAGRANT_HOME.
|
||||
# Check all molecule.yml files for required Vagrant boxes and download the ones that are not
|
||||
# already present on the system.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
GIT_ROOT=$(git rev-parse --show-toplevel)
|
||||
PROVIDER=virtualbox
|
||||
LOCK_FILE="${VAGRANT_BOX_LOCK_FILE:-${GIT_ROOT}/.github/vagrant-boxes.lock}"
|
||||
|
||||
MOLECULE_YML_PATH=("${GIT_ROOT}"/molecule/*/molecule.yml)
|
||||
yq --version
|
||||
|
||||
# Extract the unique boxes referenced by the scenarios.
|
||||
declared_boxes=$(for file in "${MOLECULE_YML_PATH[@]}"; do
|
||||
yq -r '.platforms[].box' "$file"
|
||||
# Define the path to the molecule.yml files
|
||||
MOLECULE_YML_PATH="${GIT_ROOT}/molecule/*/molecule.yml"
|
||||
|
||||
# Extract and sort unique boxes from all molecule.yml files
|
||||
all_boxes=$(for file in $MOLECULE_YML_PATH; do
|
||||
yq eval '.platforms[].box' "$file"
|
||||
done | sort -u)
|
||||
|
||||
if [[ ! -r "$LOCK_FILE" ]]; then
|
||||
printf 'Vagrant box lock file is missing or unreadable: %s\n' "$LOCK_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo all_boxes: "$all_boxes"
|
||||
|
||||
lock_entries=$(awk '
|
||||
/^[[:space:]]*#/ || NF == 0 { next }
|
||||
NF != 3 {
|
||||
printf "Invalid lock entry on line %d: expected box, version, architecture\n", NR > "/dev/stderr"
|
||||
invalid = 1
|
||||
next
|
||||
}
|
||||
{ print $1 " " $2 " " $3 }
|
||||
END { exit invalid }
|
||||
' "$LOCK_FILE")
|
||||
|
||||
duplicate_boxes=$(printf '%s\n' "$lock_entries" | awk '{ print $1 }' | sort | uniq -d)
|
||||
if [[ -n "$duplicate_boxes" ]]; then
|
||||
printf 'Duplicate Vagrant box lock entries:\n%s\n' "$duplicate_boxes" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
locked_boxes=$(printf '%s\n' "$lock_entries" | sort)
|
||||
locked_names=$(printf '%s\n' "$locked_boxes" | awk '{ print $1 }')
|
||||
missing_locks=$(comm -23 <(printf '%s\n' "$declared_boxes") <(printf '%s\n' "$locked_names"))
|
||||
unused_locks=$(comm -13 <(printf '%s\n' "$declared_boxes") <(printf '%s\n' "$locked_names"))
|
||||
|
||||
if [[ -n "$missing_locks" || -n "$unused_locks" ]]; then
|
||||
if [[ -n "$missing_locks" ]]; then
|
||||
printf 'Scenario boxes missing from the lock file:\n%s\n' "$missing_locks" >&2
|
||||
fi
|
||||
if [[ -n "$unused_locks" ]]; then
|
||||
printf 'Lock entries not referenced by a scenario:\n%s\n' "$unused_locks" >&2
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf 'Pinned Vagrant boxes:\n%s\n' "$locked_boxes"
|
||||
|
||||
# Read exact box, provider, version, and architecture tuples already present.
|
||||
# Read the boxes that are currently present on the system (for the current provider)
|
||||
present_boxes=$(
|
||||
vagrant box list --machine-readable |
|
||||
awk -F, -v expected_provider="$PROVIDER" '
|
||||
$3 == "box-name" { name = $4; next }
|
||||
$3 == "box-provider" { provider = $4; next }
|
||||
$3 == "box-version" { version = $4; next }
|
||||
$3 == "box-architecture" {
|
||||
architecture = $4
|
||||
if (provider == expected_provider) {
|
||||
print name " " version " " architecture
|
||||
}
|
||||
name = provider = version = architecture = ""
|
||||
}
|
||||
' |
|
||||
sort -u
|
||||
(vagrant box list |
|
||||
grep "${PROVIDER}" | # Filter by boxes available for the current provider
|
||||
awk '{print $1;}' | # The box name is the first word in each line
|
||||
sort |
|
||||
uniq) ||
|
||||
echo "" # In case any of these commands fails, just use an empty list
|
||||
)
|
||||
|
||||
download_boxes=$(comm -23 \
|
||||
<(printf '%s\n' "$locked_boxes") \
|
||||
<(printf '%s\n' "$present_boxes"))
|
||||
# The boxes that we need to download are the ones present in $all_boxes, but not $present_boxes.
|
||||
download_boxes=$(comm -2 -3 <(echo "${all_boxes}") <(echo "${present_boxes}"))
|
||||
|
||||
if [[ -n "$download_boxes" ]]; then
|
||||
printf '%s\n' "$download_boxes" | while read -r box version architecture; do
|
||||
vagrant box add \
|
||||
--provider "$PROVIDER" \
|
||||
--box-version "$version" \
|
||||
--architecture "$architecture" \
|
||||
"$box"
|
||||
# Actually download the necessary boxes
|
||||
if [ -n "${download_boxes}" ]; then
|
||||
echo "${download_boxes}" | while IFS= read -r box; do
|
||||
vagrant box add --provider "${PROVIDER}" "${box}"
|
||||
done
|
||||
else
|
||||
printf 'All pinned Vagrant boxes are already present.\n'
|
||||
fi
|
||||
|
||||
@@ -1,249 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
usage() {
|
||||
printf '%s\n' \
|
||||
'Usage: cleanup-runner-resources.sh [--snapshot|--dry-run|--apply]' \
|
||||
'' \
|
||||
'Discover and, with --apply, remove only VirtualBox resources referenced by' \
|
||||
'repository-owned Molecule Vagrant state. The default is --dry-run.'
|
||||
}
|
||||
|
||||
mode="dry-run"
|
||||
case "${1:-}" in
|
||||
"") ;;
|
||||
--snapshot) mode="snapshot" ;;
|
||||
--dry-run) mode="dry-run" ;;
|
||||
--apply) mode="apply" ;;
|
||||
--help|-h) usage; exit 0 ;;
|
||||
*) usage >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
home_dir="${HOME:?HOME must be set}"
|
||||
molecule_root="${K3S_CI_MOLECULE_ROOT:-${home_dir}/.cache/molecule}"
|
||||
repository_name="${K3S_CI_MOLECULE_PROJECT:-k3s-ansible}"
|
||||
virtualbox_root="${K3S_CI_VIRTUALBOX_ROOT:-${home_dir}/VirtualBox VMs}"
|
||||
hostonly_marker="${K3S_CI_HOSTONLY_MARKER:-${home_dir}/.cache/k3s-ci/hostonly-interfaces}"
|
||||
|
||||
record_hostonly() {
|
||||
local marker_dir="${hostonly_marker%/*}"
|
||||
local marker_tmp="${hostonly_marker}.tmp"
|
||||
local hostonly_inventory
|
||||
if ! hostonly_inventory="$(VBoxManage list hostonlyifs)"; then
|
||||
fail_closed 'unable to inventory VirtualBox host-only interfaces'
|
||||
fi
|
||||
mkdir -p -- "$marker_dir"
|
||||
awk -F': ' '
|
||||
/^Name:/ { name=$2 }
|
||||
/^IPAddress:/ { print name "|" $2 }
|
||||
' <<< "$hostonly_inventory" > "$marker_tmp"
|
||||
mv -- "$marker_tmp" "$hostonly_marker"
|
||||
chmod 600 "$hostonly_marker"
|
||||
printf 'Recorded host-only interface baseline: %s\n' "$hostonly_marker"
|
||||
}
|
||||
|
||||
cleanup_hostonly() {
|
||||
local hostonly_inventory
|
||||
if [[ ! -f "$hostonly_marker" ]]; then
|
||||
printf 'No host-only interface baseline found; leaving interfaces unchanged.\n'
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! hostonly_inventory="$(VBoxManage list hostonlyifs)"; then
|
||||
fail_closed 'unable to inventory VirtualBox host-only interfaces'
|
||||
fi
|
||||
|
||||
while IFS='|' read -r interface_name interface_ip; do
|
||||
[[ "$interface_name" == vboxnet* ]] || continue
|
||||
[[ "$interface_ip" == 192.168.30.* || "$interface_ip" == fdad:bad:ba55:* ]] || continue
|
||||
if grep -Fqx "${interface_name}|${interface_ip}" "$hostonly_marker"; then
|
||||
continue
|
||||
fi
|
||||
if [[ "$mode" == apply ]]; then
|
||||
VBoxManage hostonlyif remove "$interface_name"
|
||||
printf 'Removed host-only interface %s (%s)\n' "$interface_name" "$interface_ip"
|
||||
else
|
||||
printf 'Would remove host-only interface %s (%s)\n' "$interface_name" "$interface_ip"
|
||||
fi
|
||||
done < <(awk -F': ' '
|
||||
/^Name:/ { name=$2 }
|
||||
/^IPAddress:/ { print name "|" $2 }
|
||||
' <<< "$hostonly_inventory")
|
||||
}
|
||||
|
||||
if [[ "$mode" == snapshot ]]; then
|
||||
record_hostonly
|
||||
exit 0
|
||||
fi
|
||||
|
||||
resolve_existing_dir() {
|
||||
local candidate="$1"
|
||||
if [[ ! -d "$candidate" ]]; then
|
||||
return 1
|
||||
fi
|
||||
readlink -f -- "$candidate"
|
||||
}
|
||||
|
||||
root_contains() {
|
||||
local root="$1"
|
||||
local path="$2"
|
||||
[[ "$path" == "$root"/* ]]
|
||||
}
|
||||
|
||||
is_supported_scenario() {
|
||||
case "$1" in
|
||||
default|single_node|calico|cilium|kube-vip|ipv6) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
is_unregistered_vm_error() {
|
||||
grep -Eq 'Could not find a registered machine|VBOX_E_OBJECT_NOT_FOUND'
|
||||
}
|
||||
|
||||
fail_closed() {
|
||||
printf 'cleanup refused: %s\n' "$1" >&2
|
||||
exit 3
|
||||
}
|
||||
|
||||
if [[ ! "$repository_name" =~ ^[A-Za-z0-9._-]+$ ]]; then
|
||||
fail_closed 'invalid Molecule repository name'
|
||||
fi
|
||||
|
||||
print_inventory() {
|
||||
local phase="$1"
|
||||
printf '%s VirtualBox inventory:\n' "$phase"
|
||||
VBoxManage list vms || true
|
||||
VBoxManage list hdds || true
|
||||
VBoxManage list hostonlyifs || true
|
||||
}
|
||||
|
||||
molecule_root_real="$(resolve_existing_dir "$molecule_root" || true)"
|
||||
if [[ -z "$molecule_root_real" ]]; then
|
||||
printf 'No Molecule root exists: %s\n' "$molecule_root"
|
||||
cleanup_hostonly
|
||||
exit 0
|
||||
fi
|
||||
|
||||
print_inventory before
|
||||
|
||||
repository_root_real="$(resolve_existing_dir "$molecule_root_real/$repository_name" || true)"
|
||||
if [[ -z "$repository_root_real" ]]; then
|
||||
printf 'No repository Molecule state root exists: %s\n' "$molecule_root_real/$repository_name"
|
||||
cleanup_hostonly
|
||||
print_inventory after
|
||||
exit 0
|
||||
fi
|
||||
if ! root_contains "$molecule_root_real" "$repository_root_real"; then
|
||||
fail_closed "repository Molecule state root is outside Molecule root: $repository_root_real"
|
||||
fi
|
||||
|
||||
declare -a state_files=()
|
||||
while IFS= read -r -d '' state_file; do
|
||||
state_files+=("$state_file")
|
||||
done < <(find "$repository_root_real" -mindepth 6 -maxdepth 6 -type f \
|
||||
-path '*/.vagrant/machines/*/virtualbox/id' -print0 2>/dev/null)
|
||||
|
||||
if ((${#state_files[@]} == 0)); then
|
||||
printf 'No repository-owned Molecule Vagrant state found under %s\n' "$repository_root_real"
|
||||
cleanup_hostonly
|
||||
print_inventory after
|
||||
exit 0
|
||||
fi
|
||||
|
||||
virtualbox_root_real="$(resolve_existing_dir "$virtualbox_root" || true)"
|
||||
|
||||
declare -a vm_records=()
|
||||
for state_file in "${state_files[@]}"; do
|
||||
if [[ ! -f "$state_file" ]]; then
|
||||
printf 'Skipping Vagrant state removed with its stale scenario directory: %s\n' "$state_file"
|
||||
continue
|
||||
fi
|
||||
state_file_real="$(readlink -f -- "$state_file")"
|
||||
state_dir="${state_file_real%/.vagrant/machines/*/virtualbox/id}"
|
||||
machine_dir="${state_file_real%/virtualbox/id}"
|
||||
machine_name="${machine_dir##*/}"
|
||||
scenario_name="${state_dir##*/}"
|
||||
|
||||
if ! root_contains "$repository_root_real" "$state_dir"; then
|
||||
fail_closed "state path is outside the repository Molecule root: $state_file_real"
|
||||
fi
|
||||
if ! is_supported_scenario "$scenario_name"; then
|
||||
fail_closed "unexpected Molecule scenario: $scenario_name"
|
||||
fi
|
||||
if [[ "$machine_name" != control* && "$machine_name" != node* ]]; then
|
||||
fail_closed "unexpected Molecule machine name: $machine_name"
|
||||
fi
|
||||
|
||||
vm_uuid="$(tr -d '[:space:]' < "$state_file_real")"
|
||||
if [[ ! "$vm_uuid" =~ ^[0-9a-fA-F-]{36}$ ]]; then
|
||||
fail_closed "invalid VirtualBox UUID in $state_file_real"
|
||||
fi
|
||||
|
||||
if ! vm_info="$(VBoxManage showvminfo "$vm_uuid" --machinereadable 2>&1)"; then
|
||||
if is_unregistered_vm_error <<< "$vm_info"; then
|
||||
printf 'Stale Vagrant state without a registered VM: %s (%s)\n' "$machine_name" "$vm_uuid"
|
||||
if [[ "$mode" == apply ]]; then
|
||||
rm -rf -- "${state_dir}/.vagrant"
|
||||
printf 'Removed stale Vagrant state: %s\n' "${state_dir}/.vagrant"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
fail_closed "unable to inspect VirtualBox VM $vm_uuid: $vm_info"
|
||||
fi
|
||||
|
||||
if [[ -z "$virtualbox_root_real" ]]; then
|
||||
fail_closed "VirtualBox VM root does not exist: $virtualbox_root"
|
||||
fi
|
||||
|
||||
cfg_file="$(awk -F= '$1 == "CfgFile" {gsub(/"/, "", $2); print $2; exit}' <<< "$vm_info")"
|
||||
if [[ -z "$cfg_file" ]]; then
|
||||
fail_closed "VirtualBox configuration path missing for $vm_uuid"
|
||||
fi
|
||||
cfg_file_real="$(readlink -f -- "$cfg_file")"
|
||||
if ! root_contains "$virtualbox_root_real" "$cfg_file_real"; then
|
||||
fail_closed "VM configuration is outside VirtualBox root: $cfg_file_real"
|
||||
fi
|
||||
|
||||
while IFS= read -r disk_path; do
|
||||
[[ -z "$disk_path" ]] && continue
|
||||
disk_path_real="$(readlink -f -- "$disk_path" 2>/dev/null || true)"
|
||||
if [[ -z "$disk_path_real" ]] || ! root_contains "$virtualbox_root_real" "$disk_path_real"; then
|
||||
fail_closed "attached disk is outside VirtualBox root: $disk_path"
|
||||
fi
|
||||
done < <(awk -F= '$1 ~ /^(SATA|IDE|SCSI|SAS|VirtioSCSI|NVMe)-[0-9]+-[0-9]+$/ {gsub(/"/, "", $2); print $2}' <<< "$vm_info")
|
||||
|
||||
vm_records+=("$vm_uuid|$machine_name|$cfg_file_real")
|
||||
done
|
||||
|
||||
if ((${#vm_records[@]} == 0)); then
|
||||
printf 'No live repository-owned VirtualBox resources found\n'
|
||||
cleanup_hostonly
|
||||
print_inventory after
|
||||
exit 0
|
||||
fi
|
||||
|
||||
for record in "${vm_records[@]}"; do
|
||||
IFS='|' read -r vm_uuid machine_name cfg_file_real <<< "$record"
|
||||
if [[ "$mode" == dry-run ]]; then
|
||||
printf 'Would remove VM %s (%s) config=%s\n' "$machine_name" "$vm_uuid" "$cfg_file_real"
|
||||
continue
|
||||
fi
|
||||
|
||||
vm_state="$(VBoxManage showvminfo "$vm_uuid" --machinereadable | awk -F= '$1 == "VMState" {gsub(/"/, "", $2); print $2; exit}')"
|
||||
if [[ "$vm_state" != poweroff && "$vm_state" != saved ]]; then
|
||||
VBoxManage controlvm "$vm_uuid" poweroff
|
||||
fi
|
||||
VBoxManage unregistervm "$vm_uuid" --delete
|
||||
printf 'Removed VM %s (%s)\n' "$machine_name" "$vm_uuid"
|
||||
done
|
||||
|
||||
cleanup_hostonly
|
||||
print_inventory after
|
||||
|
||||
if [[ "$mode" == apply ]]; then
|
||||
printf 'Repository-owned VM and host-only interface cleanup complete.\n'
|
||||
else
|
||||
printf 'Dry run complete. No resources were modified.\n'
|
||||
fi
|
||||
@@ -1,47 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
output_dir="${1:-${RUNNER_TEMP:-/tmp}/k3s-ci-diagnostics}"
|
||||
mkdir -p -- "$output_dir"
|
||||
umask 077
|
||||
|
||||
run_capture() {
|
||||
local output_file="$1"
|
||||
shift
|
||||
{
|
||||
printf '$'
|
||||
printf ' %q' "$@"
|
||||
printf '\n'
|
||||
"$@"
|
||||
} > "$output_dir/$output_file" 2>&1 || true
|
||||
}
|
||||
|
||||
run_capture system.txt uname -a
|
||||
run_capture runner-user.txt id
|
||||
run_capture memory.txt free -h
|
||||
run_capture disk.txt df -h
|
||||
run_capture virtualbox-version VBoxManage --version
|
||||
run_capture virtualbox-vms VBoxManage list vms
|
||||
run_capture virtualbox-running-vms VBoxManage list runningvms
|
||||
run_capture virtualbox-disks VBoxManage list hdds
|
||||
run_capture virtualbox-hostonlyifs VBoxManage list hostonlyifs
|
||||
run_capture virtualbox-groups VBoxManage list groups
|
||||
run_capture vagrant-status vagrant global-status
|
||||
run_capture molecule-state find "${HOME}/.cache/molecule" -maxdepth 6 -type f -path '*/.vagrant/machines/*/virtualbox/id' -print
|
||||
|
||||
scenario_name="${K3S_CI_SCENARIO_NAME:-}"
|
||||
if [[ "$scenario_name" =~ ^[A-Za-z0-9_-]+$ ]]; then
|
||||
molecule_state_dir="${HOME}/.cache/molecule/k3s-ansible/${scenario_name}"
|
||||
for log_name in vagrant.out vagrant.err; do
|
||||
if [[ -r "${molecule_state_dir}/${log_name}" ]]; then
|
||||
cp -- "${molecule_state_dir}/${log_name}" "$output_dir/${scenario_name}-${log_name}"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ -r /etc/vbox/networks.conf ]]; then
|
||||
cp -- /etc/vbox/networks.conf "$output_dir/virtualbox-networks.conf"
|
||||
fi
|
||||
|
||||
printf 'Diagnostics written to %s\n' "$output_dir"
|
||||
@@ -1,44 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
output_dir="${1:?output directory is required}"
|
||||
interval="${2:-10}"
|
||||
[[ "$interval" =~ ^[1-9][0-9]*$ ]] || {
|
||||
printf 'monitor interval must be a positive integer\n' >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
mkdir -p -- "$output_dir"
|
||||
umask 077
|
||||
|
||||
free -h > "$output_dir/memory-before.txt"
|
||||
df -h > "$output_dir/disk-before.txt"
|
||||
vmstat -w "$interval" > "$output_dir/vmstat.txt" &
|
||||
vmstat_pid=$!
|
||||
|
||||
iostat_pid=""
|
||||
if command -v iostat >/dev/null 2>&1; then
|
||||
iostat -dx "$interval" > "$output_dir/iostat.txt" &
|
||||
iostat_pid=$!
|
||||
else
|
||||
printf 'iostat is not installed on this runner\n' > "$output_dir/iostat-unavailable.txt"
|
||||
fi
|
||||
|
||||
cleanup() {
|
||||
local rc=$?
|
||||
trap - EXIT INT TERM
|
||||
kill "$vmstat_pid" 2>/dev/null || true
|
||||
[[ -z "$iostat_pid" ]] || kill "$iostat_pid" 2>/dev/null || true
|
||||
wait "$vmstat_pid" 2>/dev/null || true
|
||||
[[ -z "$iostat_pid" ]] || wait "$iostat_pid" 2>/dev/null || true
|
||||
free -h > "$output_dir/memory-after.txt"
|
||||
df -h > "$output_dir/disk-after.txt"
|
||||
exit "$rc"
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
while :; do
|
||||
sleep 3600 &
|
||||
wait $!
|
||||
done
|
||||
@@ -1,211 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
fail() {
|
||||
printf 'Vagrant box master preparation refused: %s\n' "$1" >&2
|
||||
exit 3
|
||||
}
|
||||
|
||||
root_contains() {
|
||||
local root="$1"
|
||||
local path="$2"
|
||||
[[ "$path" == "$root"/* ]]
|
||||
}
|
||||
|
||||
read_machine_value() {
|
||||
local machine_info="$1"
|
||||
local key="$2"
|
||||
awk -F= -v key="$key" '$1 == key {gsub(/"/, "", $2); print $2; exit}' <<< "$machine_info"
|
||||
}
|
||||
|
||||
read_extra_data() {
|
||||
local uuid="$1"
|
||||
local key="$2"
|
||||
local value
|
||||
value="$(VBoxManage getextradata "$uuid" "$key" 2>/dev/null || true)"
|
||||
[[ "$value" == 'Value: '* ]] || return 1
|
||||
printf '%s\n' "${value#Value: }"
|
||||
}
|
||||
|
||||
validate_owned_master() {
|
||||
local uuid="$1"
|
||||
local box="$2"
|
||||
local version="$3"
|
||||
local architecture="$4"
|
||||
local machine_info cfg_file cfg_file_real vm_state groups disk_path disk_path_real
|
||||
|
||||
[[ "$uuid" =~ ^[0-9a-fA-F-]{36}$ ]] || return 1
|
||||
machine_info="$(VBoxManage showvminfo "$uuid" --machinereadable 2>/dev/null)" || return 1
|
||||
vm_state="$(read_machine_value "$machine_info" VMState)"
|
||||
groups="$(read_machine_value "$machine_info" groups)"
|
||||
cfg_file="$(read_machine_value "$machine_info" CfgFile)"
|
||||
[[ "$vm_state" == poweroff ]] || return 1
|
||||
[[ ",$groups," == *,/k3s-ansible/box-masters,* ]] || return 1
|
||||
[[ -n "$cfg_file" ]] || return 1
|
||||
cfg_file_real="$(readlink -f -- "$cfg_file" 2>/dev/null || true)"
|
||||
[[ -n "$cfg_file_real" ]] || return 1
|
||||
root_contains "$virtualbox_root_real" "$cfg_file_real" || return 1
|
||||
|
||||
[[ "$(read_extra_data "$uuid" k3s-ansible/owner || true)" == box-master ]] || return 1
|
||||
[[ "$(read_extra_data "$uuid" k3s-ansible/box || true)" == "$box" ]] || return 1
|
||||
[[ "$(read_extra_data "$uuid" k3s-ansible/version || true)" == "$version" ]] || return 1
|
||||
[[ "$(read_extra_data "$uuid" k3s-ansible/architecture || true)" == "$architecture" ]] || return 1
|
||||
|
||||
while IFS= read -r disk_path; do
|
||||
[[ -z "$disk_path" || "$disk_path" == none ]] && continue
|
||||
disk_path_real="$(readlink -f -- "$disk_path" 2>/dev/null || true)"
|
||||
[[ -n "$disk_path_real" ]] || return 1
|
||||
root_contains "$virtualbox_root_real" "$disk_path_real" || return 1
|
||||
done < <(awk -F= '$1 ~ /^(SATA|IDE|SCSI|SAS|VirtioSCSI|NVMe)-[0-9]+-[0-9]+$/ {
|
||||
gsub(/"/, "", $2); print $2
|
||||
}' <<< "$machine_info")
|
||||
}
|
||||
|
||||
write_prewarm_vagrantfile() {
|
||||
local destination="$1"
|
||||
local box="$2"
|
||||
local version="$3"
|
||||
{
|
||||
printf '%s\n' "Vagrant.configure('2') do |config|"
|
||||
printf ' config.vm.box = "%s"\n' "$box"
|
||||
printf ' config.vm.box_version = "%s"\n' "$version"
|
||||
printf '%s\n' \
|
||||
' config.vm.synced_folder ".", "/vagrant", disabled: true' \
|
||||
' config.vm.hostname = "k3s-ansible-box-prewarm"' \
|
||||
' config.vm.boot_timeout = 600' \
|
||||
' config.vm.provider "virtualbox" do |virtualbox|' \
|
||||
' virtualbox.linked_clone = true' \
|
||||
' virtualbox.memory = 1024' \
|
||||
' virtualbox.cpus = 2' \
|
||||
' end' \
|
||||
'end'
|
||||
} > "$destination"
|
||||
}
|
||||
|
||||
cleanup_prewarm() {
|
||||
local rc=$?
|
||||
trap - EXIT
|
||||
if [[ -n "${prewarm_dir:-}" && -d "$prewarm_dir" ]]; then
|
||||
VAGRANT_CWD="$prewarm_dir" vagrant destroy --force >/dev/null 2>&1 || true
|
||||
rm -rf -- "$prewarm_dir"
|
||||
fi
|
||||
exit "$rc"
|
||||
}
|
||||
|
||||
create_owned_master() {
|
||||
local box="$1"
|
||||
local version="$2"
|
||||
local architecture="$3"
|
||||
local master_id_file="$4"
|
||||
local mapping_file="$5"
|
||||
local uuid machine_info cfg_file cfg_file_real vm_state mapping_tmp
|
||||
|
||||
# A master_id restored from an immutable cache is only a hint. Without the
|
||||
# runner-local ownership record and matching VirtualBox metadata it is not
|
||||
# trusted, adopted, modified, or deleted.
|
||||
rm -f -- "$master_id_file"
|
||||
|
||||
prewarm_dir="$(mktemp -d "${master_root}/prewarm.XXXXXX")"
|
||||
trap cleanup_prewarm EXIT
|
||||
write_prewarm_vagrantfile "$prewarm_dir/Vagrantfile" "$box" "$version"
|
||||
|
||||
printf 'Creating runner-owned linked-clone master for %s %s %s\n' \
|
||||
"$box" "$version" "$architecture"
|
||||
VAGRANT_CWD="$prewarm_dir" vagrant up --provider virtualbox --no-provision
|
||||
|
||||
[[ -r "$master_id_file" ]] || fail "Vagrant did not record a master UUID for $box"
|
||||
uuid="$(tr -d '[:space:]' < "$master_id_file")"
|
||||
[[ "$uuid" =~ ^[0-9a-fA-F-]{36}$ ]] || fail "Vagrant recorded an invalid master UUID for $box"
|
||||
|
||||
machine_info="$(VBoxManage showvminfo "$uuid" --machinereadable 2>/dev/null)" || \
|
||||
fail "Vagrant master $uuid for $box is not registered"
|
||||
vm_state="$(read_machine_value "$machine_info" VMState)"
|
||||
cfg_file="$(read_machine_value "$machine_info" CfgFile)"
|
||||
cfg_file_real="$(readlink -f -- "$cfg_file" 2>/dev/null || true)"
|
||||
[[ "$vm_state" == poweroff ]] || fail "new Vagrant master $uuid is not powered off"
|
||||
if [[ -z "$cfg_file_real" ]] || ! root_contains "$virtualbox_root_real" "$cfg_file_real"; then
|
||||
fail "new Vagrant master $uuid is outside the runner VirtualBox root"
|
||||
fi
|
||||
|
||||
VAGRANT_CWD="$prewarm_dir" vagrant destroy --force
|
||||
VBoxManage modifyvm "$uuid" --groups /k3s-ansible/box-masters
|
||||
VBoxManage setextradata "$uuid" k3s-ansible/owner box-master
|
||||
VBoxManage setextradata "$uuid" k3s-ansible/box "$box"
|
||||
VBoxManage setextradata "$uuid" k3s-ansible/version "$version"
|
||||
VBoxManage setextradata "$uuid" k3s-ansible/architecture "$architecture"
|
||||
validate_owned_master "$uuid" "$box" "$version" "$architecture" || \
|
||||
fail "new Vagrant master $uuid failed ownership validation"
|
||||
|
||||
rm -rf -- "$prewarm_dir"
|
||||
prewarm_dir=""
|
||||
trap - EXIT
|
||||
|
||||
mapping_tmp="${mapping_file}.tmp"
|
||||
printf '%s\n' "$uuid" > "$mapping_tmp"
|
||||
chmod 600 "$mapping_tmp"
|
||||
mv -- "$mapping_tmp" "$mapping_file"
|
||||
printf '%s\n' "$uuid" > "$master_id_file"
|
||||
chmod 600 "$master_id_file"
|
||||
printf 'Created and recorded owned master %s for %s\n' "$uuid" "$box"
|
||||
}
|
||||
|
||||
repository_root="${K3S_CI_REPOSITORY_ROOT:-$(git rev-parse --show-toplevel)}"
|
||||
lock_file="${VAGRANT_BOX_LOCK_FILE:-${repository_root}/.github/vagrant-boxes.lock}"
|
||||
vagrant_home="${VAGRANT_HOME:?VAGRANT_HOME must be set}"
|
||||
master_root="${K3S_CI_VAGRANT_MASTER_ROOT:-${HOME:?HOME must be set}/.cache/k3s-ci/vagrant-masters}"
|
||||
virtualbox_root="${K3S_CI_VIRTUALBOX_ROOT:-${HOME}/VirtualBox VMs}"
|
||||
|
||||
[[ -r "$lock_file" ]] || fail "box lock file is missing or unreadable: $lock_file"
|
||||
[[ -d "$vagrant_home/boxes" ]] || fail "Vagrant box directory is missing: $vagrant_home/boxes"
|
||||
[[ -d "$virtualbox_root" ]] || fail "VirtualBox root is missing: $virtualbox_root"
|
||||
|
||||
box_root_real="$(readlink -f -- "$vagrant_home/boxes")"
|
||||
virtualbox_root_real="$(readlink -f -- "$virtualbox_root")"
|
||||
mkdir -p -- "$master_root"
|
||||
chmod 700 "$master_root"
|
||||
|
||||
exec 9> "${master_root}/prepare.lock"
|
||||
flock 9
|
||||
|
||||
lock_entries="$(awk '
|
||||
/^[[:space:]]*#/ || NF == 0 { next }
|
||||
NF != 3 { invalid = 1; next }
|
||||
{ print $1 " " $2 " " $3 }
|
||||
END { exit invalid }
|
||||
' "$lock_file")" || fail 'invalid Vagrant box lock entry'
|
||||
[[ -n "$lock_entries" ]] || fail 'Vagrant box lock is empty'
|
||||
|
||||
while read -r box version architecture; do
|
||||
[[ "$box" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || fail "invalid box name: $box"
|
||||
[[ "$version" =~ ^[A-Za-z0-9._-]+$ ]] || fail "invalid box version: $version"
|
||||
[[ "$architecture" =~ ^[A-Za-z0-9._-]+$ ]] || fail "invalid box architecture: $architecture"
|
||||
|
||||
box_slug="${box//\//-VAGRANTSLASH-}"
|
||||
record_slug="${box//\//_}-${version}-${architecture}"
|
||||
box_dir="${vagrant_home}/boxes/${box_slug}/${version}/${architecture}/virtualbox"
|
||||
[[ -d "$box_dir" ]] || fail "pinned box is not installed: $box $version $architecture"
|
||||
box_dir_real="$(readlink -f -- "$box_dir")"
|
||||
root_contains "$box_root_real" "$box_dir_real" || fail "box directory is outside VAGRANT_HOME: $box_dir_real"
|
||||
|
||||
master_id_file="${box_dir_real}/master_id"
|
||||
mapping_file="${master_root}/${record_slug}.uuid"
|
||||
uuid=""
|
||||
if [[ -r "$mapping_file" ]]; then
|
||||
uuid="$(tr -d '[:space:]' < "$mapping_file")"
|
||||
fi
|
||||
|
||||
if [[ -n "$uuid" ]] && validate_owned_master "$uuid" "$box" "$version" "$architecture"; then
|
||||
printf '%s\n' "$uuid" > "$master_id_file"
|
||||
chmod 600 "$master_id_file"
|
||||
printf 'Reusing owned master %s for %s %s %s\n' "$uuid" "$box" "$version" "$architecture"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ -e "$mapping_file" ]]; then
|
||||
printf 'Owned master record is stale for %s; rebuilding without deleting any VM or disk.\n' "$box"
|
||||
fi
|
||||
create_owned_master "$box" "$version" "$architecture" "$master_id_file" "$mapping_file"
|
||||
done <<< "$lock_entries"
|
||||
|
||||
printf 'All pinned Vagrant box masters are ready.\n'
|
||||
@@ -1,134 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render the Cilium BGP CRD template and assert it uses the v2 API.
|
||||
|
||||
This is a manifest-only regression test used where no real BGP peer is
|
||||
available. It renders roles/k3s_server_post/templates/cilium.crs.j2 with
|
||||
zero, one, and multiple neighbors, then checks that the output:
|
||||
- never contains CiliumBGPPeeringPolicy or cilium.io/v2alpha1
|
||||
- emits the Cilium v2 BGP resources
|
||||
- emits deterministic DNS-safe peer and instance names
|
||||
- advertises Pod CIDRs only when cilium_exportPodCIDR is true
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader, StrictUndefined
|
||||
|
||||
|
||||
def repo_root():
|
||||
return subprocess.check_output(
|
||||
["git", "rev-parse", "--show-toplevel"], text=True
|
||||
).strip()
|
||||
|
||||
|
||||
def fail(message):
|
||||
raise SystemExit("Cilium BGP manifest test failed: " + message)
|
||||
|
||||
|
||||
def render(env, extra_vars):
|
||||
base_vars = {
|
||||
"cilium_bgp_my_asn": "64513",
|
||||
"cilium_bgp_peer_asn": "64512",
|
||||
"cilium_bgp_peer_address": "192.168.30.1",
|
||||
"cilium_exportPodCIDR": True,
|
||||
"cilium_bgp_lb_cidr": "192.168.31.0/24",
|
||||
}
|
||||
base_vars.update(extra_vars)
|
||||
template = env.get_template("cilium.crs.j2")
|
||||
return template.render(**base_vars)
|
||||
|
||||
|
||||
def check_common(output):
|
||||
if "cilium.io/v2alpha1" in output:
|
||||
fail("rendered output still contains cilium.io/v2alpha1")
|
||||
if "kind: CiliumBGPPeeringPolicy" in output:
|
||||
fail("rendered output still contains CiliumBGPPeeringPolicy")
|
||||
for kind in (
|
||||
"CiliumBGPPeerConfig",
|
||||
"CiliumBGPClusterConfig",
|
||||
"CiliumBGPAdvertisement",
|
||||
"CiliumLoadBalancerIPPool",
|
||||
):
|
||||
if ("kind: " + kind) not in output:
|
||||
fail("rendered output is missing kind: " + kind)
|
||||
|
||||
|
||||
def main():
|
||||
root = repo_root()
|
||||
template_dir = os.path.join(
|
||||
root, "roles", "k3s_server_post", "templates"
|
||||
)
|
||||
env = Environment(
|
||||
loader=FileSystemLoader(template_dir), undefined=StrictUndefined
|
||||
)
|
||||
|
||||
# Zero neighbors -> fall back to the single default peer.
|
||||
output = render(env, {"_cilium_bgp_neighbors": []})
|
||||
check_common(output)
|
||||
if "peer-64512-1" not in output:
|
||||
fail("default single peer name was not rendered")
|
||||
if "peerAddress: 192.168.30.1" not in output:
|
||||
fail("default peer address was not rendered")
|
||||
if 'advertisementType: "PodCIDR"' not in output:
|
||||
fail("PodCIDR advertisement missing when exportPodCIDR is true")
|
||||
|
||||
# One neighbor via the merged list.
|
||||
output = render(
|
||||
env,
|
||||
{"_cilium_bgp_neighbors": [{"peer_address": "10.0.0.1", "peer_asn": "65001"}]},
|
||||
)
|
||||
check_common(output)
|
||||
if "peer-65001-1" not in output:
|
||||
fail("single merged peer name was not rendered")
|
||||
if "peerAddress: 10.0.0.1" not in output:
|
||||
fail("single merged peer address was not rendered")
|
||||
|
||||
# Multiple neighbors.
|
||||
output = render(
|
||||
env,
|
||||
{
|
||||
"_cilium_bgp_neighbors": [
|
||||
{"peer_address": "10.0.0.1", "peer_asn": "65001"},
|
||||
{"peer_address": "10.0.0.2", "peer_asn": "65002"},
|
||||
]
|
||||
},
|
||||
)
|
||||
check_common(output)
|
||||
if "peer-65001-1" not in output or "peer-65002-2" not in output:
|
||||
fail("multiple merged peer names were not rendered")
|
||||
if "peerAddress: 10.0.0.2" not in output:
|
||||
fail("second merged peer address was not rendered")
|
||||
|
||||
# exportPodCIDR false -> no PodCIDR advertisement, service remains.
|
||||
output = render(
|
||||
env, {"_cilium_bgp_neighbors": [], "cilium_exportPodCIDR": False}
|
||||
)
|
||||
check_common(output)
|
||||
if 'advertisementType: "PodCIDR"' in output:
|
||||
fail("PodCIDR advertisement present when exportPodCIDR is false")
|
||||
if 'advertisementType: "Service"' not in output:
|
||||
fail("Service advertisement missing when exportPodCIDR is false")
|
||||
|
||||
# Load balancer pools: CIDR and start/stop forms.
|
||||
output = render(env, {"_cilium_bgp_neighbors": []})
|
||||
if "cidr: 192.168.31.0/24" not in output:
|
||||
fail("CIDR load balancer pool was not rendered")
|
||||
output = render(
|
||||
env,
|
||||
{
|
||||
"_cilium_bgp_neighbors": [],
|
||||
"cilium_bgp_lb_cidr": "192.168.31.80-192.168.31.90",
|
||||
},
|
||||
)
|
||||
check_common(output)
|
||||
if "start: 192.168.31.80" not in output or "stop: 192.168.31.90" not in output:
|
||||
fail("start/stop load balancer pool was not rendered")
|
||||
|
||||
print("Cilium BGP manifest regression test passed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,120 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regression test for the Cilium Envoy toggle.
|
||||
|
||||
The `cilium_envoy` variable lets users enable or disable the Cilium Envoy
|
||||
proxy. The Install/upgrade Cilium task in
|
||||
roles/k3s_server_post/tasks/cilium.yml passes the value through to Helm as
|
||||
`envoy.enabled`. This test:
|
||||
|
||||
- loads the real "Install Cilium" task and confirms the install/upgrade
|
||||
command actually contains the `envoy.enabled` Helm value,
|
||||
- renders the conditional that computes the Helm value and confirms it
|
||||
produces `true` when cilium_envoy is enabled and `false` when disabled,
|
||||
- confirms the task stays forward/backward compatible (no raw `true` /
|
||||
`false` hardcoded in place of the conditional).
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
import yaml
|
||||
from jinja2 import Environment
|
||||
|
||||
ENVOY_EXPRESSION = '{{ "true" if cilium_envoy else "false" }}'
|
||||
|
||||
|
||||
def repo_root():
|
||||
return subprocess.check_output(
|
||||
["git", "rev-parse", "--show-toplevel"], text=True
|
||||
).strip()
|
||||
|
||||
|
||||
def fail(message):
|
||||
raise SystemExit("Cilium Envoy toggle test failed: " + message)
|
||||
|
||||
|
||||
def extract_install_command(path):
|
||||
"""Return the command string for the 'Install Cilium' task.
|
||||
|
||||
Walks both top-level tasks and tasks nested inside a `block`/`always`/
|
||||
`rescue` list, since the Cilium deploy steps are grouped under the
|
||||
'Prepare Cilium CLI on first master and deploy CNI' block.
|
||||
"""
|
||||
with open(path, encoding="utf-8") as handle:
|
||||
doc = yaml.safe_load(handle)
|
||||
|
||||
def find_command(tasks):
|
||||
for task in tasks:
|
||||
if not isinstance(task, dict):
|
||||
continue
|
||||
if task.get("name") == "Install Cilium":
|
||||
command = task.get("ansible.builtin.command")
|
||||
if command is None:
|
||||
raise SystemExit(
|
||||
"Cilium Envoy toggle test failed: "
|
||||
"'Install Cilium' task has no ansible.builtin.command"
|
||||
)
|
||||
return command
|
||||
# Recurse into block/always/rescue sub-lists.
|
||||
for key in ("block", "always", "rescue"):
|
||||
nested = task.get(key)
|
||||
if isinstance(nested, list):
|
||||
found = find_command(nested)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
command = find_command(doc)
|
||||
if command is None:
|
||||
raise SystemExit(
|
||||
"Cilium Envoy toggle test failed: could not find 'Install Cilium' task"
|
||||
)
|
||||
return command
|
||||
|
||||
|
||||
def assert_envoy_in_command(command):
|
||||
if "envoy.enabled" not in command:
|
||||
fail("install command is missing --helm-set envoy.enabled")
|
||||
if ENVOY_EXPRESSION not in command:
|
||||
fail(
|
||||
"install command does not use the cilium_envoy conditional: "
|
||||
"expected {0!r}".format(ENVOY_EXPRESSION)
|
||||
)
|
||||
# The conditional must be a WYSIWYG helm-set value, not a pre-rendered
|
||||
# true/false literal (which would ignore the cilium_envoy variable).
|
||||
if re.search(r"--helm-set envoy\.enabled=true(?:$|\s)", command):
|
||||
fail("install command hardcodes envoy.enabled=true")
|
||||
if re.search(r"--helm-set envoy\.enabled=false(?:$|\s)", command):
|
||||
fail("install command hardcodes envoy.enabled=false")
|
||||
|
||||
|
||||
def assert_render():
|
||||
env = Environment()
|
||||
|
||||
def render_for(value):
|
||||
template = env.from_string(ENVOY_EXPRESSION)
|
||||
return template.render(cilium_envoy=value)
|
||||
|
||||
if render_for(True) != "true":
|
||||
fail("envoy conditional did not render 'true' when enabled")
|
||||
if render_for(False) != "false":
|
||||
fail("envoy conditional did not render 'false' when disabled")
|
||||
|
||||
|
||||
def main():
|
||||
root = repo_root()
|
||||
cilium_tasks = os.path.join(
|
||||
root, "roles", "k3s_server_post", "tasks", "cilium.yml"
|
||||
)
|
||||
command = extract_install_command(cilium_tasks)
|
||||
assert_envoy_in_command(command)
|
||||
assert_render()
|
||||
|
||||
print("Cilium Envoy toggle regression test passed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,129 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
test_root="$(mktemp -d "${TMPDIR:-/tmp}/k3s-ci-cleanup-test.XXXXXX")"
|
||||
trap 'rm -rf -- "$test_root"' EXIT
|
||||
|
||||
molecule_root="$test_root/molecule"
|
||||
virtualbox_root="$test_root/VirtualBox VMs"
|
||||
fake_bin="$test_root/bin"
|
||||
mkdir -p -- "$molecule_root/k3s-ansible/single_node/.vagrant/machines/control1/virtualbox" \
|
||||
"$molecule_root/k3s-ansible/single_node/.vagrant/machines/control2/virtualbox" \
|
||||
"$virtualbox_root/control1" "$virtualbox_root/unmarked" "$fake_bin"
|
||||
printf '%s\n' '11111111-1111-1111-1111-111111111111' \
|
||||
> "$molecule_root/k3s-ansible/single_node/.vagrant/machines/control1/virtualbox/id"
|
||||
printf '%s\n' '22222222-2222-2222-2222-222222222222' \
|
||||
> "$molecule_root/k3s-ansible/single_node/.vagrant/machines/control2/virtualbox/id"
|
||||
touch "$virtualbox_root/control1/control1.vbox" "$virtualbox_root/control1/disk.vdi" \
|
||||
"$virtualbox_root/unmarked/unmarked.vbox"
|
||||
|
||||
printf '%s\n' \
|
||||
'#!/usr/bin/env bash' \
|
||||
'set -Eeuo pipefail' \
|
||||
'case "${1:-}" in' \
|
||||
' list)' \
|
||||
' if [[ "${2:-}" == hostonlyifs && "${FAKE_HOSTONLY_FAIL:-false}" == true ]]; then exit 1; fi' \
|
||||
' exit 0' \
|
||||
' ;;' \
|
||||
' showvminfo)' \
|
||||
' if [[ "${FAKE_VM_MODE:-normal}" == missing ]]; then' \
|
||||
' printf '\''VBoxManage: error: Could not find a registered machine named "missing"\n'\'' >&2' \
|
||||
' exit 1' \
|
||||
' fi' \
|
||||
' if [[ "${FAKE_VM_MODE:-normal}" == fault ]]; then' \
|
||||
' printf '\''VBoxManage: error: VirtualBox service is unavailable\n'\'' >&2' \
|
||||
' exit 1' \
|
||||
' fi' \
|
||||
' printf '\''CfgFile="%s"\n'\'' "${FAKE_VBOX_ROOT}/control1/control1.vbox"' \
|
||||
' printf '\''SATA-0-0="%s"\n'\'' "${FAKE_VBOX_ROOT}/control1/disk.vdi"' \
|
||||
' printf '\''VMState="running"\n'\''' \
|
||||
' ;;' \
|
||||
' controlvm) printf '\''controlvm %s\n'\'' "$*" >> "${FAKE_LOG}" ;;' \
|
||||
' unregistervm)' \
|
||||
' printf '\''unregistervm %s\n'\'' "$*" >> "${FAKE_LOG}"' \
|
||||
' rm -f -- "${FAKE_VBOX_ROOT}/control1/control1.vbox" "${FAKE_VBOX_ROOT}/control1/disk.vdi"' \
|
||||
' ;;' \
|
||||
' *) : ;;' \
|
||||
'esac' > "$fake_bin/VBoxManage"
|
||||
chmod 700 "$fake_bin/VBoxManage"
|
||||
|
||||
output="$test_root/output.txt"
|
||||
if PATH="$fake_bin:$PATH" \
|
||||
HOME="$test_root/home" \
|
||||
K3S_CI_MOLECULE_ROOT="$molecule_root" \
|
||||
K3S_CI_MOLECULE_PROJECT=k3s-ansible \
|
||||
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
|
||||
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
|
||||
FAKE_VM_MODE=fault \
|
||||
FAKE_VBOX_ROOT="$virtualbox_root" \
|
||||
FAKE_LOG="$test_root/vbox.log" \
|
||||
bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --apply > "$output" 2>&1; then
|
||||
printf '%s\n' 'cleanup unexpectedly accepted a VirtualBox inspection failure' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq 'cleanup refused: unable to inspect VirtualBox VM' "$output"
|
||||
[[ -f "$molecule_root/k3s-ansible/single_node/.vagrant/machines/control1/virtualbox/id" ]]
|
||||
|
||||
printf '%s\n' 'vboxnet0|192.168.30.1' > "$test_root/hostonly-baseline"
|
||||
if PATH="$fake_bin:$PATH" \
|
||||
HOME="$test_root/home" \
|
||||
K3S_CI_MOLECULE_ROOT="$molecule_root" \
|
||||
K3S_CI_MOLECULE_PROJECT=k3s-ansible \
|
||||
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
|
||||
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
|
||||
FAKE_HOSTONLY_FAIL=true \
|
||||
FAKE_VBOX_ROOT="$virtualbox_root" \
|
||||
FAKE_LOG="$test_root/vbox.log" \
|
||||
bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --dry-run > "$output" 2>&1; then
|
||||
printf '%s\n' 'cleanup unexpectedly accepted a host-only inventory failure' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq 'cleanup refused: unable to inventory VirtualBox host-only interfaces' "$output"
|
||||
|
||||
PATH="$fake_bin:$PATH" \
|
||||
HOME="$test_root/home" \
|
||||
K3S_CI_MOLECULE_ROOT="$molecule_root" \
|
||||
K3S_CI_MOLECULE_PROJECT=k3s-ansible \
|
||||
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
|
||||
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
|
||||
FAKE_VBOX_ROOT="$virtualbox_root" \
|
||||
FAKE_LOG="$test_root/vbox.log" \
|
||||
bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --dry-run > "$output"
|
||||
|
||||
grep -Fq 'Would remove VM control1 (11111111-1111-1111-1111-111111111111)' "$output"
|
||||
[[ ! -e "$test_root/vbox.log" ]]
|
||||
|
||||
PATH="$fake_bin:$PATH" \
|
||||
HOME="$test_root/home" \
|
||||
K3S_CI_MOLECULE_ROOT="$molecule_root" \
|
||||
K3S_CI_MOLECULE_PROJECT=k3s-ansible \
|
||||
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
|
||||
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
|
||||
FAKE_VBOX_ROOT="$virtualbox_root" \
|
||||
FAKE_LOG="$test_root/vbox.log" \
|
||||
bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --apply > "$output"
|
||||
|
||||
grep -Fq 'controlvm 11111111-1111-1111-1111-111111111111 poweroff' "$test_root/vbox.log"
|
||||
grep -Fq 'unregistervm unregistervm 11111111-1111-1111-1111-111111111111 --delete' "$test_root/vbox.log"
|
||||
[[ ! -e "$virtualbox_root/control1/control1.vbox" ]]
|
||||
[[ -e "$virtualbox_root/unmarked/unmarked.vbox" ]]
|
||||
|
||||
PATH="$fake_bin:$PATH" \
|
||||
HOME="$test_root/home" \
|
||||
K3S_CI_MOLECULE_ROOT="$molecule_root" \
|
||||
K3S_CI_MOLECULE_PROJECT=k3s-ansible \
|
||||
K3S_CI_VIRTUALBOX_ROOT="$virtualbox_root" \
|
||||
K3S_CI_HOSTONLY_MARKER="$test_root/hostonly-baseline" \
|
||||
FAKE_VM_MODE=missing \
|
||||
FAKE_VBOX_ROOT="$virtualbox_root" \
|
||||
FAKE_LOG="$test_root/vbox.log" \
|
||||
bash "$repo_root/.github/scripts/cleanup-runner-resources.sh" --apply > "$output"
|
||||
|
||||
grep -Fq 'Stale Vagrant state without a registered VM' "$output"
|
||||
[[ ! -d "$molecule_root/k3s-ansible/single_node/.vagrant" ]]
|
||||
|
||||
printf 'cleanup-runner-resources fixture test passed\n'
|
||||
@@ -1,90 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Assert the sample inventory resolves the flannel interface per host.
|
||||
|
||||
flannel_iface defaults to the host's default IPv4 interface rather than a
|
||||
hardcoded eth0. This test extracts the flannel_iface expression from the
|
||||
sample inventory and proves that a host whose primary interface is not named
|
||||
eth0 (e.g. enp1s0, ens3) resolves the interface from ansible facts.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
from jinja2 import Environment, StrictUndefined
|
||||
|
||||
|
||||
def repo_root():
|
||||
return subprocess.check_output(
|
||||
["git", "rev-parse", "--show-toplevel"], text=True
|
||||
).strip()
|
||||
|
||||
|
||||
def fail(message):
|
||||
raise SystemExit("default-interface test failed: " + message)
|
||||
|
||||
|
||||
class FakeAnsibleFacts(object):
|
||||
"""Stand-in for the per-host ``ansible_facts`` dict."""
|
||||
|
||||
def __init__(self, default_iface, iface_ip):
|
||||
self._default = {"interface": default_iface, "address": iface_ip}
|
||||
self._ifaces = {
|
||||
default_iface: {"ipv4": {"address": iface_ip}},
|
||||
}
|
||||
|
||||
@property
|
||||
def default_ipv4(self):
|
||||
return self._default
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self._ifaces[key]
|
||||
|
||||
|
||||
def read_all_yml(root):
|
||||
path = os.path.join(root, "inventory", "sample", "group_vars", "all.yml")
|
||||
with open(path, "r") as handle:
|
||||
return handle.read()
|
||||
|
||||
|
||||
def extract_value(content, key):
|
||||
# Match a quoted value assigned to the key, e.g. flannel_iface: "...".
|
||||
match = re.search(r"^%s:\s*\"(.+)\"\s*$" % re.escape(key), content, re.M)
|
||||
if not match:
|
||||
fail("could not find %s in the sample inventory" % key)
|
||||
return match.group(1)
|
||||
|
||||
|
||||
def resolve(env, expression, facts):
|
||||
template = env.from_string(expression)
|
||||
return template.render(ansible_facts=facts)
|
||||
|
||||
|
||||
def main():
|
||||
root = repo_root()
|
||||
content = read_all_yml(root)
|
||||
env = Environment(undefined=StrictUndefined)
|
||||
|
||||
flannel_expr = extract_value(content, "flannel_iface")
|
||||
if "default_ipv4.interface" not in flannel_expr:
|
||||
fail("flannel_iface no longer defaults from ansible facts")
|
||||
|
||||
# A host whose primary interface is enp1s0 (the core #621 scenario).
|
||||
facts = FakeAnsibleFacts("enp1s0", "192.168.30.11")
|
||||
resolved = resolve(env, flannel_expr, facts)
|
||||
if resolved != "enp1s0":
|
||||
fail("flannel_iface resolved to %r, expected enp1s0" % resolved)
|
||||
|
||||
# A different host with a different interface must resolve independently.
|
||||
facts2 = FakeAnsibleFacts("ens3", "192.168.30.12")
|
||||
resolved2 = resolve(env, flannel_expr, facts2)
|
||||
if resolved2 != "ens3":
|
||||
fail("flannel_iface resolved to %r, expected ens3" % resolved2)
|
||||
|
||||
print("default-interface regression test passed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,37 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
repo_root="$(git rev-parse --show-toplevel)"
|
||||
prereq_defaults="$repo_root/roles/prereq/defaults/main.yml"
|
||||
prereq_tasks="$repo_root/roles/prereq/tasks/main.yml"
|
||||
|
||||
# #670: k3s recommends swap be disabled on all nodes. The prereq role must expose
|
||||
# a disable_swap toggle (defaulting to true) that turns swap off now and comments
|
||||
# out the /etc/fstab swap entries so swap stays off across reboots.
|
||||
grep -Eq -- '^disable_swap: true' "$prereq_defaults" || {
|
||||
printf 'prereq defaults are missing disable_swap: true\n' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
grep -Fq -- 'Disable swap on all cluster nodes' "$prereq_tasks" || {
|
||||
printf 'prereq tasks are missing the swap-disable block\n' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
grep -Fq -- 'swapoff -a' "$prereq_tasks" || {
|
||||
printf 'swap-disable block does not run swapoff -a\n' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
grep -Fq -- '/etc/fstab' "$prereq_tasks" || {
|
||||
printf 'swap-disable block does not comment out /etc/fstab swap entries\n' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if ! grep -Eq -- 'when: disable_swap' "$prereq_tasks"; then
|
||||
printf 'swap-disable block is not gated on the disable_swap toggle\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf 'Swap disable regression test passed\n'
|
||||
@@ -1,4 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
[[ "${1:-}" =~ ^[0-9]+$ ]]
|
||||
@@ -1,30 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
printf '%s\n' "$*" >> "$MOCK_VAGRANT_LOG"
|
||||
case "${1:-}" in
|
||||
up)
|
||||
if [[ -n "${MOCK_VAGRANTFILE_CAPTURE:-}" ]]; then
|
||||
cp -- "$VAGRANT_CWD/Vagrantfile" "$MOCK_VAGRANTFILE_CAPTURE"
|
||||
fi
|
||||
counter_file="$MOCK_VBOX_STATE/counter"
|
||||
counter=0
|
||||
[[ ! -r "$counter_file" ]] || counter="$(cat "$counter_file")"
|
||||
counter=$((counter + 1))
|
||||
printf '%s\n' "$counter" > "$counter_file"
|
||||
uuid="00000000-0000-4000-8000-$(printf '%012d' "$counter")"
|
||||
vm_dir="$MOCK_VBOX_ROOT/master-$counter"
|
||||
mkdir -p -- "$vm_dir"
|
||||
: > "$vm_dir/master.vbox"
|
||||
: > "$vm_dir/master.vdi"
|
||||
printf '%s\n' \
|
||||
'VMState="poweroff"' \
|
||||
'groups="/"' \
|
||||
"CfgFile=\"$vm_dir/master.vbox\"" \
|
||||
"SATA-0-0=\"$vm_dir/master.vdi\"" > "$MOCK_VBOX_STATE/vm-$uuid"
|
||||
printf '%s\n' "$uuid" > "$MOCK_BOX_DIR/master_id"
|
||||
;;
|
||||
destroy) ;;
|
||||
*) exit 2 ;;
|
||||
esac
|
||||
@@ -1,40 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
printf '%s\n' "$*" >> "$MOCK_VBOX_LOG"
|
||||
command_name="${1:-}"
|
||||
uuid="${2:-}"
|
||||
|
||||
case "$command_name" in
|
||||
showvminfo)
|
||||
[[ -r "$MOCK_VBOX_STATE/vm-$uuid" ]] || exit 1
|
||||
cat "$MOCK_VBOX_STATE/vm-$uuid"
|
||||
;;
|
||||
getextradata)
|
||||
key_slug="${3//\//_}"
|
||||
if [[ ! -r "$MOCK_VBOX_STATE/extra-$uuid-$key_slug" ]]; then
|
||||
printf '%s\n' 'No value set!'
|
||||
exit 0
|
||||
fi
|
||||
printf 'Value: '
|
||||
cat "$MOCK_VBOX_STATE/extra-$uuid-$key_slug"
|
||||
;;
|
||||
modifyvm)
|
||||
[[ "${3:-}" == --groups ]]
|
||||
awk -v groups="${4:-}" '
|
||||
$1 !~ /^groups=/ { print }
|
||||
END { printf "groups=\"%s\"\n", groups }
|
||||
' "$MOCK_VBOX_STATE/vm-$uuid" > "$MOCK_VBOX_STATE/vm-$uuid.tmp"
|
||||
mv "$MOCK_VBOX_STATE/vm-$uuid.tmp" "$MOCK_VBOX_STATE/vm-$uuid"
|
||||
;;
|
||||
setextradata)
|
||||
key_slug="${3//\//_}"
|
||||
printf '%s\n' "${4:-}" > "$MOCK_VBOX_STATE/extra-$uuid-$key_slug"
|
||||
;;
|
||||
unregistervm|closemedium)
|
||||
printf '%s\n' 'destructive VirtualBox command invoked' >&2
|
||||
exit 99
|
||||
;;
|
||||
*) exit 2 ;;
|
||||
esac
|
||||
@@ -1,35 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
repo_root="$(git rev-parse --show-toplevel)"
|
||||
main_tasks="$repo_root/roles/k3s_server/tasks/main.yml"
|
||||
join_tasks="$repo_root/roles/k3s_server/tasks/join_master.yml"
|
||||
|
||||
for task_file in "$main_tasks" "$join_tasks"; do
|
||||
for property in \
|
||||
'Delegate=yes' \
|
||||
'TasksMax=infinity' \
|
||||
'KillMode=process' \
|
||||
'LimitNOFILE=1048576' \
|
||||
'LimitNPROC=infinity' \
|
||||
'LimitCORE=infinity'; do
|
||||
grep -Fq -- "$property" "$task_file" || {
|
||||
printf '%s is missing transient K3s property %s\n' "$task_file" "$property" >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
done
|
||||
|
||||
if grep -Fq -- "node-role.kubernetes.io/master=true' -o=jsonpath" "$main_tasks"; then
|
||||
printf 'control-plane registration still depends on the optional legacy master role key\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
grep -Fq -- "map('extract', hostvars, 'ansible_hostname')" "$main_tasks"
|
||||
grep -Fq -- 'difference(nodes.stdout.split())' "$main_tasks"
|
||||
grep -Fq -- 'crd/addons.k3s.cattle.io' "$main_tasks"
|
||||
grep -Fq -- 'crd/helmcharts.helm.cattle.io' "$main_tasks"
|
||||
grep -Fq -- 'crd/helmchartconfigs.helm.cattle.io' "$main_tasks"
|
||||
|
||||
printf 'K3s transient bootstrap regression test passed\n'
|
||||
@@ -1,166 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regression test for the kube-vip deploy conditions.
|
||||
|
||||
The control-plane VIP (roles/k3s_server/tasks/vip.yml) and the kube-vip
|
||||
service load balancer (roles/k3s_server/tasks/kube-vip.yml) are both included
|
||||
from roles/k3s_server/tasks/main.yml. Before this switch existed the VIP
|
||||
include had no gate and always ran, so a user who wanted no kube-vip (single
|
||||
node or external LB) could not opt out.
|
||||
|
||||
This test loads the real `when` expressions from the k3s_server task file and
|
||||
evaluates them against representative variable sets, asserting that:
|
||||
|
||||
- the control-plane VIP is deployed only when kube_vip_enabled is true;
|
||||
- the service load balancer is deployed only when kube_vip_enabled is true
|
||||
AND kube_vip_lb_ip_range is defined.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
import yaml
|
||||
from jinja2 import Environment
|
||||
|
||||
VIP_WHEN = "kube_vip_enabled"
|
||||
KUBE_VIP_WHEN = "kube_vip_enabled and kube_vip_lb_ip_range is defined"
|
||||
|
||||
|
||||
def repo_root():
|
||||
return subprocess.check_output(
|
||||
["git", "rev-parse", "--show-toplevel"], text=True
|
||||
).strip()
|
||||
|
||||
|
||||
def fail(message):
|
||||
raise SystemExit("kube-vip deploy condition test failed: " + message)
|
||||
|
||||
|
||||
def extract_when(path, task_name):
|
||||
"""Return the `when:` expression string for the named task."""
|
||||
with open(path, encoding="utf-8") as handle:
|
||||
doc = yaml.safe_load(handle)
|
||||
for task in doc:
|
||||
if task.get("name") == task_name:
|
||||
when = task.get("when")
|
||||
return (when or "").strip()
|
||||
return None
|
||||
|
||||
|
||||
def evaluate(when, variables):
|
||||
"""Evaluate a `when` expression against variables using Jinja2."""
|
||||
env = Environment()
|
||||
|
||||
def fake_bool(value):
|
||||
# Minimal stand-in for Ansible's truthiness filter used by `| bool`.
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if value is None:
|
||||
return False
|
||||
return str(value).lower() in ("1", "true", "yes", "on")
|
||||
|
||||
env.filters["bool"] = fake_bool
|
||||
template = env.from_string("{{ " + when + " }}")
|
||||
rendered = template.render(**variables)
|
||||
# The expression renders to the literal strings "True"/"False".
|
||||
if rendered == "True":
|
||||
return True
|
||||
if rendered == "False":
|
||||
return False
|
||||
fail("condition did not render to a boolean: {0!r}".format(rendered))
|
||||
|
||||
|
||||
def assert_deployment(when, variables, expected, label):
|
||||
result = evaluate(when, variables)
|
||||
verdict = "deploy" if result else "skip"
|
||||
expected_verdict = "deploy" if expected else "skip"
|
||||
if result != expected:
|
||||
fail(
|
||||
"{0}: expected to {1} but the condition chose to {2} "
|
||||
"(vars: {3})".format(label, expected_verdict, verdict, variables)
|
||||
)
|
||||
|
||||
|
||||
def scenarios():
|
||||
"""Yield (variables, expected_vip, expected_service_lb, label) pairs."""
|
||||
yield (
|
||||
# Default inventory: kube-vip enabled, no service LB range.
|
||||
{
|
||||
"kube_vip_enabled": True,
|
||||
},
|
||||
True,
|
||||
False,
|
||||
"default: kube-vip VIP only",
|
||||
)
|
||||
yield (
|
||||
# kube-vip owns the service LB range too.
|
||||
{
|
||||
"kube_vip_enabled": True,
|
||||
"kube_vip_lb_ip_range": "192.168.30.80-192.168.30.90",
|
||||
},
|
||||
True,
|
||||
True,
|
||||
"kube-vip VIP and service LB",
|
||||
)
|
||||
yield (
|
||||
# Explicitly disabled, no LB range.
|
||||
{
|
||||
"kube_vip_enabled": False,
|
||||
},
|
||||
False,
|
||||
False,
|
||||
"kube_vip_enabled: false",
|
||||
)
|
||||
yield (
|
||||
# Explicitly disabled even when the LB range is present.
|
||||
{
|
||||
"kube_vip_enabled": False,
|
||||
"kube_vip_lb_ip_range": "192.168.30.80-192.168.30.90",
|
||||
},
|
||||
False,
|
||||
False,
|
||||
"kube_vip_enabled: false despite LB range",
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
root = repo_root()
|
||||
server_tasks = os.path.join(root, "roles", "k3s_server", "tasks", "main.yml")
|
||||
|
||||
vip_when = extract_when(server_tasks, "Deploy vip manifest")
|
||||
kube_vip_when = extract_when(server_tasks, "Deploy kube-vip manifest")
|
||||
|
||||
if vip_when is None:
|
||||
fail("could not find 'Deploy vip manifest' when condition")
|
||||
if kube_vip_when is None:
|
||||
fail("could not find 'Deploy kube-vip manifest' when condition")
|
||||
|
||||
if vip_when != VIP_WHEN:
|
||||
fail(
|
||||
"k3s_server/tasks/main.yml 'Deploy vip manifest' when condition "
|
||||
"changed unexpectedly:\n"
|
||||
" expected: {0}\n got: {1}".format(VIP_WHEN, vip_when)
|
||||
)
|
||||
if kube_vip_when != KUBE_VIP_WHEN:
|
||||
fail(
|
||||
"k3s_server/tasks/main.yml 'Deploy kube-vip manifest' when "
|
||||
"condition changed unexpectedly:\n"
|
||||
" expected: {0}\n got: {1}".format(KUBE_VIP_WHEN, kube_vip_when)
|
||||
)
|
||||
|
||||
for variables, expected_vip, expected_service_lb, label in scenarios():
|
||||
assert_deployment(vip_when, variables, expected_vip, "vip " + label)
|
||||
assert_deployment(
|
||||
kube_vip_when,
|
||||
variables,
|
||||
expected_service_lb,
|
||||
"service_lb " + label,
|
||||
)
|
||||
|
||||
print("kube-vip deploy condition regression test passed for all scenarios")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,162 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render the kube-vip DaemonSet template and assert env key correctness.
|
||||
|
||||
kube-vip v1.2.3 reads `bgp_peers` and `vip_subnet`; it ignores the older
|
||||
`bgppeers` and `vip_cidr` names. This test proves the rendered manifest uses
|
||||
the keys the target image actually parses.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader, StrictUndefined
|
||||
|
||||
|
||||
def repo_root():
|
||||
return subprocess.check_output(
|
||||
["git", "rev-parse", "--show-toplevel"], text=True
|
||||
).strip()
|
||||
|
||||
|
||||
def fail(message):
|
||||
raise SystemExit("kube-vip manifest test failed: " + message)
|
||||
|
||||
|
||||
def fake_ipsubnet(value):
|
||||
# ansible.utils.ipsubnet -> network of the address as x.y.z.0/24
|
||||
parts = value.split(".")
|
||||
return ".".join(parts[:3]) + ".0/24"
|
||||
|
||||
|
||||
def fake_ipaddr(_value, expr=None):
|
||||
# ansible.utils.ipaddr('prefix') -> prefix length
|
||||
return "24"
|
||||
|
||||
|
||||
def fake_bool(value):
|
||||
# Minimal stand-in for Ansible's truthiness filter used by the template.
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
return str(value).lower() in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
def fake_map(seq, *args, **kwargs):
|
||||
# Minimal stand-in for Ansible's map() filter in the two forms used by the
|
||||
# template: map(attribute='x') on a list of dicts, and map('join', sep) on
|
||||
# a list of sequences.
|
||||
if "attribute" in kwargs:
|
||||
return [item[kwargs["attribute"]] for item in seq]
|
||||
if kwargs:
|
||||
# e.g. map(default='x') not used here; ignore unknown kwargs.
|
||||
return list(seq)
|
||||
if args:
|
||||
filter_name = args[0]
|
||||
sep = args[1] if len(args) > 1 else ""
|
||||
if filter_name == "join":
|
||||
return [sep.join(str(x) for x in item) for item in seq]
|
||||
return list(seq)
|
||||
|
||||
|
||||
def fake_zip(*seqs):
|
||||
return list(zip(*seqs))
|
||||
|
||||
|
||||
def render(env, extra_vars):
|
||||
base_vars = {
|
||||
"apiserver_endpoint": "192.168.30.222",
|
||||
"kube_vip_iface": "",
|
||||
"kube_vip_arp": True,
|
||||
"kube_vip_bgp": True,
|
||||
"kube_vip_bgp_routerid": "127.0.0.1",
|
||||
"_kube_vip_bgp_peers": [
|
||||
{"peer_address": "192.168.30.1", "peer_asn": "64512"},
|
||||
{"peer_address": "192.168.30.2", "peer_asn": "64513"},
|
||||
],
|
||||
"kube_vip_tag_version": "v1.2.3",
|
||||
}
|
||||
base_vars.update(extra_vars)
|
||||
template = env.get_template("vip.yaml.j2")
|
||||
return template.render(**base_vars)
|
||||
|
||||
|
||||
def main():
|
||||
root = repo_root()
|
||||
template_dir = os.path.join(root, "roles", "k3s_server", "templates")
|
||||
env = Environment(
|
||||
loader=FileSystemLoader(template_dir), undefined=StrictUndefined
|
||||
)
|
||||
env.filters["ansible.utils.ipsubnet"] = fake_ipsubnet
|
||||
env.filters["ansible.utils.ipaddr"] = fake_ipaddr
|
||||
env.filters["bool"] = fake_bool
|
||||
env.filters["map"] = fake_map
|
||||
env.filters["zip"] = fake_zip
|
||||
|
||||
# Multi-peer BGP armed: must emit bgp_peers, never bgppeers.
|
||||
output = render(env, {})
|
||||
if "name: bgp_peers" not in output:
|
||||
fail("rendered manifest is missing bgp_peers")
|
||||
if "name: bgppeers" in output:
|
||||
fail("rendered manifest still uses the ignored bgppeers key")
|
||||
if "name: vip_subnet" not in output:
|
||||
fail("rendered manifest is missing vip_subnet")
|
||||
if "name: vip_cidr" in output:
|
||||
fail("rendered manifest still uses the ignored vip_cidr key")
|
||||
if "192.168.30.1:64512,192.168.30.2:64513" not in output:
|
||||
fail("bgp_peers value is not comma-separated address:ASN entries")
|
||||
if "ghcr.io/kube-vip/kube-vip:v1.2.3" not in output:
|
||||
fail("kube-vip image tag is not v1.2.3")
|
||||
|
||||
# BGP enabled with no merged peers: single-peer fallback vars, no bgp_peers.
|
||||
output = render(
|
||||
env,
|
||||
{
|
||||
"_kube_vip_bgp_peers": [],
|
||||
"kube_vip_bgp_as": "64513",
|
||||
"kube_vip_bgp_peeraddress": "192.168.30.1",
|
||||
"kube_vip_bgp_peeras": "64512",
|
||||
},
|
||||
)
|
||||
if "name: bgp_as" not in output:
|
||||
fail("single-peer bgp_as was not rendered")
|
||||
if "name: bgp_peers" in output:
|
||||
fail("bgp_peers present even though the peer list is empty")
|
||||
|
||||
# kube_vip_endpoint defaults to null (defined in role defaults): the
|
||||
# address and subnet must fall back to the apiserver endpoint. default()
|
||||
# without a truthy flag does NOT fall back on null, only on undefined, so
|
||||
# this case pins the null runtime condition to prevent that regression.
|
||||
output = render(
|
||||
env,
|
||||
{
|
||||
"_kube_vip_bgp_peers": [],
|
||||
"kube_vip_endpoint": None,
|
||||
"kube_vip_arp": True,
|
||||
"kube_vip_bgp": False,
|
||||
},
|
||||
)
|
||||
if "value: 192.168.30.222" not in output:
|
||||
fail("null kube_vip_endpoint does not fall back to apiserver_endpoint")
|
||||
|
||||
# kube_vip_endpoint set: overrides the internal listening address AND the
|
||||
# subnet derivation while the advertised apiserver_endpoint stays separate.
|
||||
output = render(
|
||||
env,
|
||||
{
|
||||
"_kube_vip_bgp_peers": [],
|
||||
"kube_vip_endpoint": "10.66.1.5",
|
||||
"kube_vip_arp": True,
|
||||
"kube_vip_bgp": False,
|
||||
},
|
||||
)
|
||||
if "value: 10.66.1.5" not in output:
|
||||
fail("kube_vip_endpoint did not override the address")
|
||||
if "value: 192.168.30.222" in output:
|
||||
fail("apiserver_endpoint leaked into address when kube_vip_endpoint set")
|
||||
|
||||
print("kube-vip manifest regression test passed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,185 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regression test for the MetalLB deploy conditions.
|
||||
|
||||
The MetalLB manifest (roles/k3s_server/tasks/main.yml) and the MetalLB pool
|
||||
(roles/k3s_server_post/tasks/main.yml) are included under a `when` condition
|
||||
that decides whether MetalLB provides load balancing. A previous change (#683)
|
||||
guarded `cilium_bgp` but accidentally skipped MetalLB whenever a non-BGP
|
||||
Cilium CNI was in use (`cilium_iface` defined), breaking the cilium + MetalLB
|
||||
scenario.
|
||||
|
||||
This test loads the real `when` expressions from both task files and evaluates
|
||||
them against representative variable sets, asserting MetalLB is deployed in
|
||||
every topology except when MetalLB is explicitly disabled, kube-vip owns the
|
||||
VIP range, or Cilium BGP is enabled.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
import yaml
|
||||
from jinja2 import Environment
|
||||
|
||||
METALLB_WHEN = (
|
||||
"metal_lb_enabled and "
|
||||
"kube_vip_lb_ip_range is not defined and "
|
||||
"not (cilium_bgp | default(false) | bool)"
|
||||
)
|
||||
|
||||
|
||||
def repo_root():
|
||||
return subprocess.check_output(
|
||||
["git", "rev-parse", "--show-toplevel"], text=True
|
||||
).strip()
|
||||
|
||||
|
||||
def fail(message):
|
||||
raise SystemExit("MetalLB deploy condition test failed: " + message)
|
||||
|
||||
|
||||
def extract_when(path, task_name):
|
||||
"""Return the `when:` expression string for the named task."""
|
||||
with open(path, encoding="utf-8") as handle:
|
||||
doc = yaml.safe_load(handle)
|
||||
for task in doc:
|
||||
if task.get("name") == task_name:
|
||||
when = task.get("when")
|
||||
return (when or "").strip()
|
||||
return None
|
||||
|
||||
|
||||
def evaluate(when, variables):
|
||||
"""Evaluate a `when` expression against variables using Jinja2."""
|
||||
env = Environment()
|
||||
|
||||
def fake_bool(value):
|
||||
# Minimal stand-in for Ansible's truthiness filter used by `| bool`.
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if value is None:
|
||||
return False
|
||||
return str(value).lower() in ("1", "true", "yes", "on")
|
||||
|
||||
env.filters["bool"] = fake_bool
|
||||
template = env.from_string("{{ " + when + " }}")
|
||||
rendered = template.render(**variables)
|
||||
# The expression renders to the literal strings "True"/"False".
|
||||
if rendered == "True":
|
||||
return True
|
||||
if rendered == "False":
|
||||
return False
|
||||
fail("condition did not render to a boolean: {0!r}".format(rendered))
|
||||
|
||||
|
||||
def assert_deployment(when, variables, expected, label):
|
||||
result = evaluate(when, variables)
|
||||
verdict = "deploy" if result else "skip"
|
||||
expected_verdict = "deploy" if expected else "skip"
|
||||
if result != expected:
|
||||
fail(
|
||||
"{0}: expected to {1} MetalLB but the condition chose to {2} "
|
||||
"(vars: {3})".format(label, expected_verdict, verdict, variables)
|
||||
)
|
||||
|
||||
|
||||
def scenarios():
|
||||
"""Yield (variables, expected_deploy, label) pairs."""
|
||||
yield (
|
||||
# Default Flannel inventory (all.yml sets cilium_bgp: false).
|
||||
{
|
||||
"metal_lb_enabled": True,
|
||||
"cilium_bgp": False,
|
||||
"cilium_iface": None,
|
||||
},
|
||||
True,
|
||||
"flannel default (cilium_bgp: false)",
|
||||
)
|
||||
yield (
|
||||
# Calico CNI with no Cilium variable in scope (issue #644): cilium_bgp
|
||||
# is genuinely undefined, so `default(false)` must keep MetalLB on.
|
||||
{
|
||||
"metal_lb_enabled": True,
|
||||
"calico_iface": "eth1",
|
||||
},
|
||||
True,
|
||||
"calico, cilium_bgp undefined (#644)",
|
||||
)
|
||||
yield (
|
||||
# Cilium CNI with BGP disabled: MetalLB must still be deployed.
|
||||
{
|
||||
"metal_lb_enabled": True,
|
||||
"cilium_bgp": False,
|
||||
"cilium_iface": "eth1",
|
||||
},
|
||||
True,
|
||||
"cilium non-BGP (regression catch)",
|
||||
)
|
||||
yield (
|
||||
# Cilium CNI with BGP enabled: Cilium provides the LB, skip MetalLB.
|
||||
{
|
||||
"metal_lb_enabled": True,
|
||||
"cilium_bgp": True,
|
||||
"cilium_iface": "eth1",
|
||||
},
|
||||
False,
|
||||
"cilium BGP enabled",
|
||||
)
|
||||
yield (
|
||||
# kube-vip is the load balancer provider: skip MetalLB.
|
||||
{
|
||||
"metal_lb_enabled": True,
|
||||
"kube_vip_lb_ip_range": "192.168.30.80-192.168.30.90",
|
||||
"cilium_bgp": False,
|
||||
},
|
||||
False,
|
||||
"kube-vip owns the VIP range",
|
||||
)
|
||||
yield (
|
||||
# MetalLB explicitly disabled (external LB / single node): skip.
|
||||
{
|
||||
"metal_lb_enabled": False,
|
||||
},
|
||||
False,
|
||||
"metal_lb_enabled: false (external LB)",
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
root = repo_root()
|
||||
server_tasks = os.path.join(root, "roles", "k3s_server", "tasks", "main.yml")
|
||||
server_post_tasks = os.path.join(
|
||||
root, "roles", "k3s_server_post", "tasks", "main.yml"
|
||||
)
|
||||
|
||||
server_when = extract_when(server_tasks, "Deploy metallb manifest")
|
||||
server_post_when = extract_when(server_post_tasks, "Deploy metallb pool")
|
||||
|
||||
if server_when is None:
|
||||
fail("could not find 'Deploy metallb manifest' when condition")
|
||||
if server_post_when is None:
|
||||
fail("could not find 'Deploy metallb pool' when condition")
|
||||
|
||||
for when, source in (
|
||||
(server_when, "k3s_server/tasks/main.yml"),
|
||||
(server_post_when, "k3s_server_post/tasks/main.yml"),
|
||||
):
|
||||
if when != METALLB_WHEN:
|
||||
fail(
|
||||
"{0} when condition changed unexpectedly:\n"
|
||||
" expected: {1}\n got: {2}".format(source, METALLB_WHEN, when)
|
||||
)
|
||||
|
||||
for variables, expected, label in scenarios():
|
||||
assert_deployment(server_when, variables, expected, "server " + label)
|
||||
assert_deployment(
|
||||
server_post_when, variables, expected, "server_post " + label
|
||||
)
|
||||
|
||||
print("MetalLB deploy condition regression test passed for all scenarios")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,88 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regression test for the MetalLB L2Advertisement interfaces.
|
||||
|
||||
`metal_lb_interfaces` restricts which network interfaces MetalLB announces
|
||||
load balancer IPs on in layer2 mode. When the list is non-empty, the
|
||||
L2Advertisement in roles/k3s_server_post/templates/metallb.crs.j2 must render
|
||||
a `spec.interfaces` block; when it is empty (the default), no spec is rendered
|
||||
so MetalLB announces on all interfaces.
|
||||
|
||||
This renders the template and asserts both cases plus the BGP path (which must
|
||||
not be affected by the L2 interfaces variable).
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader, StrictUndefined
|
||||
|
||||
|
||||
def repo_root():
|
||||
return subprocess.check_output(
|
||||
["git", "rev-parse", "--show-toplevel"], text=True
|
||||
).strip()
|
||||
|
||||
|
||||
def fail(message):
|
||||
raise SystemExit("MetalLB interfaces test failed: " + message)
|
||||
|
||||
|
||||
def render(env, extra_vars):
|
||||
base_vars = {
|
||||
"metal_lb_mode": "layer2",
|
||||
"metal_lb_ip_range": "192.168.30.80-192.168.30.90",
|
||||
}
|
||||
base_vars.update(extra_vars)
|
||||
template = env.get_template("metallb.crs.j2")
|
||||
return template.render(**base_vars)
|
||||
|
||||
|
||||
def main():
|
||||
root = repo_root()
|
||||
template_dir = os.path.join(
|
||||
root, "roles", "k3s_server_post", "templates"
|
||||
)
|
||||
env = Environment(
|
||||
loader=FileSystemLoader(template_dir), undefined=StrictUndefined
|
||||
)
|
||||
|
||||
# Empty list (default): no spec.interfaces in the L2Advertisement.
|
||||
output = render(env, {"metal_lb_interfaces": []})
|
||||
if "spec:\n interfaces:" in output:
|
||||
fail("spec.interfaces rendered with an empty metal_lb_interfaces")
|
||||
if "kind: L2Advertisement" not in output:
|
||||
fail("L2Advertisement missing in layer2 mode")
|
||||
|
||||
# Single interface.
|
||||
output = render(env, {"metal_lb_interfaces": ["eth1"]})
|
||||
if "spec:\n interfaces:\n - eth1" not in output:
|
||||
fail("single interface was not rendered in spec.interfaces")
|
||||
|
||||
# Multiple interfaces.
|
||||
output = render(env, {"metal_lb_interfaces": ["eth1", "eth2"]})
|
||||
if "spec:\n interfaces:\n - eth1\n - eth2" not in output:
|
||||
fail("multiple interfaces were not rendered in spec.interfaces")
|
||||
|
||||
# BGP mode must not emit an L2Advertisement spec at all.
|
||||
output = render(
|
||||
env,
|
||||
{
|
||||
"metal_lb_mode": "bgp",
|
||||
"metal_lb_interfaces": ["eth1"],
|
||||
"metal_lb_bgp_my_asn": "64513",
|
||||
"metal_lb_bgp_peer_asn": "64512",
|
||||
"metal_lb_bgp_peer_address": "192.168.30.1",
|
||||
},
|
||||
)
|
||||
if "kind: L2Advertisement" in output:
|
||||
fail("L2Advertisement rendered in bgp mode")
|
||||
if "interfaces:" in output:
|
||||
fail("interfaces rendered in bgp mode")
|
||||
|
||||
print("MetalLB interfaces regression test passed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,108 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regression test for the MetalLB converge checks.
|
||||
|
||||
The MetalLB tasks in roles/k3s_server_post/tasks/metallb.yml must actually
|
||||
verify resources through an explicit kubectl get, and must retry on a
|
||||
transient kube API error while MetalLB converges.
|
||||
|
||||
The "Test metallb-system namespace" task previously ran `k3s kubectl -n
|
||||
metallb-system` with no subcommand, which only printed a usage page and always
|
||||
exited 0, so it always succeeded even when the namespace did not exist (issue
|
||||
#350). It must instead run an explicit `get namespace metallb-system`, which
|
||||
returns non-zero when the namespace is absent.
|
||||
|
||||
An explicit get actually contacts the API server, so these tasks need the same
|
||||
retry wiring as their siblings (register, until rc == 0, retries, delay). A
|
||||
bare get with no retry would otherwise abort the converge play on a transient
|
||||
kube API error while MetalLB converges.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
def repo_root():
|
||||
return subprocess.check_output(
|
||||
["git", "rev-parse", "--show-toplevel"], text=True
|
||||
).strip()
|
||||
|
||||
|
||||
def fail(message):
|
||||
raise SystemExit("MetalLB namespace test failed: " + message)
|
||||
|
||||
|
||||
def find_task(tasks, name):
|
||||
for entry in tasks:
|
||||
if entry.get("name") == name:
|
||||
return entry
|
||||
fail("could not find the '{0}' task".format(name))
|
||||
return None
|
||||
|
||||
|
||||
def command_text(task):
|
||||
cmd = task.get("ansible.builtin.command")
|
||||
if not cmd:
|
||||
cmd = task.get("command")
|
||||
if not cmd:
|
||||
fail("task does not use ansible.builtin.command")
|
||||
return cmd if isinstance(cmd, str) else " ".join(cmd)
|
||||
|
||||
|
||||
def check_explicit_get(task, name, needle):
|
||||
text = command_text(task)
|
||||
if needle not in text:
|
||||
fail(
|
||||
"command does not run '{0}'; the task would only print usage and "
|
||||
"never verify the resource (got: {1!r})".format(needle, text)
|
||||
)
|
||||
|
||||
|
||||
def check_retry_wiring(task, name):
|
||||
# The sibling k3s_server_post metallb tasks retry kubectl because the kube
|
||||
# API can briefly be unavailable while MetalLB converges. Without the same
|
||||
# retry, a transient API error aborts the whole converge play.
|
||||
if not task.get("register"):
|
||||
fail(
|
||||
"{0} does not register a result; without retry wiring a transient "
|
||||
"kube API error aborts the converge play".format(name)
|
||||
)
|
||||
if not isinstance(task.get("until"), str) or "rc == 0" not in task["until"]:
|
||||
fail(
|
||||
"{0} does not retry on rc == 0; the kube API can transiently fail "
|
||||
"while MetalLB converges and abort the play".format(name)
|
||||
)
|
||||
if task.get("retries") is None:
|
||||
fail("{0} is missing retries".format(name))
|
||||
if task.get("delay") is None:
|
||||
fail("{0} is missing delay".format(name))
|
||||
|
||||
|
||||
def main():
|
||||
task_file = os.path.join(
|
||||
repo_root(), "roles", "k3s_server_post", "tasks", "metallb.yml"
|
||||
)
|
||||
with open(task_file, encoding="utf-8") as handle:
|
||||
tasks = yaml.safe_load(handle)
|
||||
|
||||
namespace_task = find_task(tasks, "Test metallb-system namespace")
|
||||
# A bare `-n metallb-system` with no subcommand prints kubectl usage and
|
||||
# always exits 0, so it never proves the namespace exists. The fix must
|
||||
# use an explicit get.
|
||||
check_explicit_get(namespace_task, "Test metallb-system namespace",
|
||||
"get namespace metallb-system")
|
||||
check_retry_wiring(namespace_task, "Test metallb-system namespace")
|
||||
|
||||
webhook_task = find_task(tasks, "Test metallb-system webhook-service endpoint")
|
||||
check_explicit_get(webhook_task, "Test metallb-system webhook-service endpoint",
|
||||
"get endpoints")
|
||||
check_retry_wiring(webhook_task, "Test metallb-system webhook-service endpoint")
|
||||
|
||||
print("MetalLB namespace check regression test passed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,29 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
repo_root="$(git rev-parse --show-toplevel)"
|
||||
metallb_task="$repo_root/roles/k3s_server/tasks/metallb.yml"
|
||||
|
||||
# The speaker tag verification must read the rendered manifest on the managed
|
||||
# host with slurp. A controller-side lookup('ansible.builtin.file', ...) would
|
||||
# read from the Ansible control node, which does not have the file, and would
|
||||
# fail on every MetalLB scenario.
|
||||
grep -Fq -- 'ansible.builtin.slurp' "$metallb_task" || {
|
||||
printf 'MetalLB speaker tag check does not use slurp on the managed host\n' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
grep -Eq -- 'lookup\(.?ansible\.builtin\.file' "$metallb_task" && {
|
||||
printf 'MetalLB speaker tag check uses a controller-side file lookup\n' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# The check must reference the full image reference, not just a bare version
|
||||
# string that could appear anywhere in the manifest.
|
||||
grep -Fq -- 'quay.io/metallb/speaker:' "$metallb_task" || {
|
||||
printf 'MetalLB speaker tag check does not match the full image reference\n' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
printf 'MetalLB remote manifest read regression test passed\n'
|
||||
@@ -1,89 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
repo_root="$(git rev-parse --show-toplevel)"
|
||||
fixture="$(mktemp -d)"
|
||||
trap 'rm -rf -- "$fixture"' EXIT
|
||||
|
||||
mock_bin="$fixture/bin"
|
||||
mock_state="$fixture/state"
|
||||
mock_home="$fixture/home"
|
||||
mock_vagrant_home="$fixture/vagrant-home"
|
||||
mock_box_dir="$mock_vagrant_home/boxes/bento-VAGRANTSLASH-ubuntu-26.04/202606.01.0/amd64/virtualbox"
|
||||
mock_vbox_root="$mock_home/VirtualBox VMs"
|
||||
mock_master_root="$mock_home/.cache/k3s-ci/vagrant-masters"
|
||||
lock_file="$fixture/vagrant-boxes.lock"
|
||||
mkdir -p -- "$mock_bin" "$mock_state" "$mock_box_dir" "$mock_vbox_root"
|
||||
printf '%s\n' 'bento/ubuntu-26.04 202606.01.0 amd64' > "$lock_file"
|
||||
ln -s "$repo_root/.github/scripts/test-fixtures/mock-vboxmanage" "$mock_bin/VBoxManage"
|
||||
ln -s "$repo_root/.github/scripts/test-fixtures/mock-vagrant" "$mock_bin/vagrant"
|
||||
ln -s "$repo_root/.github/scripts/test-fixtures/mock-flock" "$mock_bin/flock"
|
||||
|
||||
export PATH="$mock_bin:$PATH"
|
||||
export HOME="$mock_home"
|
||||
export VAGRANT_HOME="$mock_vagrant_home"
|
||||
export VAGRANT_BOX_LOCK_FILE="$lock_file"
|
||||
export K3S_CI_REPOSITORY_ROOT="$repo_root"
|
||||
export K3S_CI_VAGRANT_MASTER_ROOT="$mock_master_root"
|
||||
export K3S_CI_VIRTUALBOX_ROOT="$mock_vbox_root"
|
||||
export MOCK_VBOX_STATE="$mock_state"
|
||||
export MOCK_VBOX_ROOT="$mock_vbox_root"
|
||||
export MOCK_BOX_DIR="$mock_box_dir"
|
||||
export MOCK_VBOX_LOG="$fixture/vbox.log"
|
||||
export MOCK_VAGRANT_LOG="$fixture/vagrant.log"
|
||||
export MOCK_VAGRANTFILE_CAPTURE="$fixture/prewarm-Vagrantfile"
|
||||
: > "$MOCK_VBOX_LOG"
|
||||
: > "$MOCK_VAGRANT_LOG"
|
||||
|
||||
unowned_uuid='99999999-9999-4999-8999-999999999999'
|
||||
printf '%s\n' "$unowned_uuid" > "$mock_box_dir/master_id"
|
||||
|
||||
script="$repo_root/.github/scripts/prepare-vagrant-box-masters.sh"
|
||||
first_output="$fixture/first-output"
|
||||
second_output="$fixture/second-output"
|
||||
third_output="$fixture/third-output"
|
||||
|
||||
"$script" > "$first_output"
|
||||
if grep -Fq 'config.ssh.insert_key' "$MOCK_VAGRANTFILE_CAPTURE"; then
|
||||
printf 'prewarm Vagrantfile unexpectedly overrides Vagrant SSH key insertion\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq 'virtualbox.memory = 1024' "$MOCK_VAGRANTFILE_CAPTURE"
|
||||
grep -Fq 'virtualbox.cpus = 2' "$MOCK_VAGRANTFILE_CAPTURE"
|
||||
grep -Fq 'config.vm.boot_timeout = 600' "$MOCK_VAGRANTFILE_CAPTURE"
|
||||
mapping_file="$mock_master_root/bento_ubuntu-26.04-202606.01.0-amd64.uuid"
|
||||
test -s "$mapping_file"
|
||||
cmp -s "$mapping_file" "$mock_box_dir/master_id"
|
||||
grep -Fq 'Created and recorded owned master' "$first_output"
|
||||
grep -Fq 'modifyvm' "$MOCK_VBOX_LOG"
|
||||
grep -Fq 'setextradata' "$MOCK_VBOX_LOG"
|
||||
if grep -Fq "$unowned_uuid" "$MOCK_VBOX_LOG"; then
|
||||
printf 'unowned cached master UUID was unexpectedly inspected or modified\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -Eq 'unregistervm|closemedium' "$MOCK_VBOX_LOG"; then
|
||||
printf 'master preparation invoked a destructive VirtualBox command\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
: > "$MOCK_VAGRANT_LOG"
|
||||
"$script" > "$second_output"
|
||||
grep -Fq 'Reusing owned master' "$second_output"
|
||||
if grep -Fq 'up ' "$MOCK_VAGRANT_LOG"; then
|
||||
printf 'valid owned master was unexpectedly rebuilt\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
stale_uuid="$(tr -d '[:space:]' < "$mapping_file")"
|
||||
rm -f -- "$mock_state/vm-$stale_uuid"
|
||||
: > "$MOCK_VAGRANT_LOG"
|
||||
"$script" > "$third_output"
|
||||
grep -Fq 'rebuilding without deleting any VM or disk' "$third_output"
|
||||
grep -Fq 'up ' "$MOCK_VAGRANT_LOG"
|
||||
if grep -Eq 'unregistervm|closemedium' "$MOCK_VBOX_LOG"; then
|
||||
printf 'stale master recovery invoked a destructive VirtualBox command\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf 'Vagrant box master preparation fixture test passed\n'
|
||||
@@ -1,28 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
repo_root="$(git rev-parse --show-toplevel)"
|
||||
site_play="$repo_root/site.yml"
|
||||
|
||||
# #636: verify the "Pre tasks" play asserts that all k3s_cluster hosts have
|
||||
# unique hostnames, so a duplicate-hostname inventory fails fast instead of
|
||||
# silently breaking node registration/joining.
|
||||
grep -Fq -- 'Verify all cluster nodes have unique hostnames' "$site_play" || {
|
||||
printf 'site.yml is missing the unique-hostname preflight check\n' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# The check must deduplicate the cluster hostname list via the `unique` filter
|
||||
# and compare lengths, i.e. groups['k3s_cluster'] must be referenced.
|
||||
grep -Fq -- "groups['k3s_cluster']" "$site_play" || {
|
||||
printf 'unique-hostname check does not iterate the k3s_cluster group\n' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if ! grep -Eq -- 'cluster_hostnames.*\|.*unique|\| unique' "$site_play"; then
|
||||
printf 'unique-hostname check does not deduplicate the hostname list\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf 'Unique hostname preflight regression test passed\n'
|
||||
@@ -1,92 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regression test for the verify_from_outside nginx HTTP assert.
|
||||
|
||||
The "Assert that the nginx welcome page is available" task in
|
||||
molecule/resources/verify_from_outside/tasks/test/deploy-example.yml performs
|
||||
a plain HTTP GET (ansible.builtin.uri) against the MetalLB VIP that the load
|
||||
balancer pool assigned, reaching it from the CI runner over the VirtualBox
|
||||
host-only network.
|
||||
|
||||
The MetalLB speaker announces the VIP on the host-only network, and the runner
|
||||
may briefly not see it in its ARP table yet. That manifests as a single-shot
|
||||
"status -1 / No route to host" failure that fails the whole verify play even
|
||||
though the cluster is healthy. Like the sibling load-balancer address wait and
|
||||
the k3s_server_post metallb waits, the assert must retry until it observes
|
||||
HTTP 200 with the welcome page.
|
||||
|
||||
A bare assert with no retry would otherwise fail an otherwise-green cluster on
|
||||
a transient announcement miss.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
def repo_root():
|
||||
return subprocess.check_output(
|
||||
["git", "rev-parse", "--show-toplevel"], text=True
|
||||
).strip()
|
||||
|
||||
|
||||
def fail(message):
|
||||
raise SystemExit("verify nginx assert test failed: " + message)
|
||||
|
||||
|
||||
def find_task(play, name):
|
||||
block = play.get("block")
|
||||
if not isinstance(block, list):
|
||||
fail("expected a block list of tasks")
|
||||
for entry in block:
|
||||
if entry.get("name") == name:
|
||||
return entry
|
||||
fail("could not find the '{0}' task".format(name))
|
||||
return None
|
||||
|
||||
|
||||
def check_retry_wiring(task, name):
|
||||
if not task.get("register"):
|
||||
fail("{0} does not register a result; without retry wiring a transient "
|
||||
"MetalLB VIP announcement miss fails the verify play".format(name))
|
||||
until = task.get("until")
|
||||
if not until:
|
||||
fail("{0} does not retry; a transient MetalLB VIP announcement miss "
|
||||
"on the host-only network would fail the verify play".format(name))
|
||||
if isinstance(until, str):
|
||||
until = [until]
|
||||
joined = " | ".join(str(expr) for expr in until)
|
||||
if "result.status == 200" not in joined:
|
||||
fail("{0} does not retry until the HTTP status is 200".format(name))
|
||||
if "Welcome to nginx!" not in joined:
|
||||
fail("{0} does not verify the welcome page content".format(name))
|
||||
if task.get("retries") is None:
|
||||
fail("{0} is missing retries".format(name))
|
||||
if task.get("delay") is None:
|
||||
fail("{0} is missing delay".format(name))
|
||||
|
||||
|
||||
def main():
|
||||
task_file = os.path.join(
|
||||
repo_root(),
|
||||
"molecule", "resources", "verify_from_outside", "tasks", "test",
|
||||
"deploy-example.yml",
|
||||
)
|
||||
with open(task_file, encoding="utf-8") as handle:
|
||||
plays = yaml.safe_load(handle)
|
||||
|
||||
play = plays[0]
|
||||
task = find_task(play, "Assert that the nginx welcome page is available")
|
||||
|
||||
if not task.get("ansible.builtin.uri"):
|
||||
fail("task does not use ansible.builtin.uri to reach the VIP")
|
||||
|
||||
check_retry_wiring(task, "Assert that the nginx welcome page is available")
|
||||
|
||||
print("verify nginx assert regression test passed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,24 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
if (($# < 2)); then
|
||||
printf 'Usage: vagrant-up-timed.sh WORKDIR MACHINE [MACHINE ...]\n' >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
workdir="$1"
|
||||
shift
|
||||
timing_log="${K3S_CI_CREATE_TIMING_LOG:-${RUNNER_TEMP:-/tmp}/k3s-ci-create-timing.log}"
|
||||
mkdir -p -- "${timing_log%/*}"
|
||||
|
||||
printf '%s batch-start machines=%s\n' "$(date --iso-8601=ns)" "$*" | tee -a "$timing_log"
|
||||
set +e
|
||||
VAGRANT_CWD="$workdir" vagrant up "$@" --provider virtualbox --no-provision 2>&1 |
|
||||
while IFS= read -r line; do
|
||||
printf '%s %s\n' "$(date --iso-8601=ns)" "$line"
|
||||
done | tee -a "$timing_log"
|
||||
rc=${PIPESTATUS[0]}
|
||||
set -e
|
||||
printf '%s batch-end rc=%d machines=%s\n' "$(date --iso-8601=ns)" "$rc" "$*" | tee -a "$timing_log"
|
||||
exit "$rc"
|
||||
@@ -1,98 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# The single-quoted expressions below are written into fake executables and
|
||||
# intentionally expand only when those executables run.
|
||||
# shellcheck disable=SC2016
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
repo_root=$(git rev-parse --show-toplevel)
|
||||
test_root=$(mktemp -d)
|
||||
fake_bin="$test_root/bin"
|
||||
fake_log="$test_root/vagrant.log"
|
||||
output="$test_root/output.txt"
|
||||
mkdir -p "$fake_bin"
|
||||
trap 'rm -rf "$test_root"' EXIT
|
||||
|
||||
printf '%s\n' \
|
||||
'#!/usr/bin/env bash' \
|
||||
'set -Eeuo pipefail' \
|
||||
'printf "%s\n" bento/debian-13 bento/rockylinux-10.1 bento/ubuntu-26.04' \
|
||||
>"$fake_bin/yq"
|
||||
|
||||
printf '%s\n' \
|
||||
'#!/usr/bin/env bash' \
|
||||
'set -Eeuo pipefail' \
|
||||
'emit_box() {' \
|
||||
' case "$1" in' \
|
||||
' bento/debian-13) version=202510.26.0 ;;' \
|
||||
' bento/rockylinux-10.1) version=202512.01.0 ;;' \
|
||||
' bento/ubuntu-26.04) version=202606.01.0 ;;' \
|
||||
' *) printf "Unexpected box: %s\n" "$1" >&2; exit 1 ;;' \
|
||||
' esac' \
|
||||
' printf "0,,box-name,%s\n" "$1"' \
|
||||
' printf "0,,box-provider,virtualbox\n"' \
|
||||
' printf "0,,box-version,%s\n" "$version"' \
|
||||
' printf "0,,box-architecture,amd64\n"' \
|
||||
'}' \
|
||||
'if [[ "${1:-}" == box && "${2:-}" == list ]]; then' \
|
||||
' emit_box bento/debian-13' \
|
||||
' emit_box bento/rockylinux-10.1' \
|
||||
' if [[ "${FAKE_PRESENT_MODE:-all}" == all ]]; then' \
|
||||
' emit_box bento/ubuntu-26.04' \
|
||||
' fi' \
|
||||
'elif [[ "${1:-}" == box && "${2:-}" == add ]]; then' \
|
||||
' printf "%s\n" "$*" >>"${FAKE_VAGRANT_LOG:?}"' \
|
||||
'else' \
|
||||
' printf "Unexpected vagrant arguments: %s\n" "$*" >&2' \
|
||||
' exit 1' \
|
||||
'fi' \
|
||||
>"$fake_bin/vagrant"
|
||||
chmod +x "$fake_bin/yq" "$fake_bin/vagrant"
|
||||
|
||||
PATH="$fake_bin:$PATH" \
|
||||
FAKE_VAGRANT_LOG="$fake_log" \
|
||||
"$repo_root/.github/download-boxes.sh" >"$output"
|
||||
grep -Fq 'All pinned Vagrant boxes are already present.' "$output"
|
||||
[[ ! -e "$fake_log" ]]
|
||||
|
||||
PATH="$fake_bin:$PATH" \
|
||||
FAKE_PRESENT_MODE=partial \
|
||||
FAKE_VAGRANT_LOG="$fake_log" \
|
||||
"$repo_root/.github/download-boxes.sh" >"$output"
|
||||
grep -Fxq \
|
||||
'box add --provider virtualbox --box-version 202606.01.0 --architecture amd64 bento/ubuntu-26.04' \
|
||||
"$fake_log"
|
||||
|
||||
incomplete_lock="$test_root/incomplete.lock"
|
||||
printf '%s\n' \
|
||||
'bento/debian-13 202510.26.0 amd64' \
|
||||
'bento/rockylinux-10.1 202512.01.0 amd64' \
|
||||
>"$incomplete_lock"
|
||||
if PATH="$fake_bin:$PATH" \
|
||||
VAGRANT_BOX_LOCK_FILE="$incomplete_lock" \
|
||||
FAKE_VAGRANT_LOG="$fake_log" \
|
||||
"$repo_root/.github/download-boxes.sh" >"$output" 2>&1; then
|
||||
printf 'Download script accepted a lock missing a scenario box.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq 'Scenario boxes missing from the lock file:' "$output"
|
||||
grep -Fq 'bento/ubuntu-26.04' "$output"
|
||||
|
||||
duplicate_lock="$test_root/duplicate.lock"
|
||||
printf '%s\n' \
|
||||
'bento/debian-13 202510.26.0 amd64' \
|
||||
'bento/debian-13 202508.10.0 amd64' \
|
||||
'bento/rockylinux-10.1 202512.01.0 amd64' \
|
||||
'bento/ubuntu-26.04 202606.01.0 amd64' \
|
||||
>"$duplicate_lock"
|
||||
if PATH="$fake_bin:$PATH" \
|
||||
VAGRANT_BOX_LOCK_FILE="$duplicate_lock" \
|
||||
FAKE_VAGRANT_LOG="$fake_log" \
|
||||
"$repo_root/.github/download-boxes.sh" >"$output" 2>&1; then
|
||||
printf 'Download script accepted duplicate box lock entries.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq 'Duplicate Vagrant box lock entries:' "$output"
|
||||
|
||||
printf 'Vagrant box download tests passed.\n'
|
||||
@@ -1,4 +0,0 @@
|
||||
# box version architecture
|
||||
bento/debian-13 202510.26.0 amd64
|
||||
bento/rockylinux-10.1 202512.01.0 amd64
|
||||
bento/ubuntu-26.04 202606.01.0 amd64
|
||||
+14
-29
@@ -5,53 +5,38 @@ on:
|
||||
jobs:
|
||||
molecule:
|
||||
name: cache
|
||||
runs-on: [self-hosted, linux, x64, k3s-ci, virtualbox, nested-virt]
|
||||
runs-on: self-hosted
|
||||
env:
|
||||
PYTHON_VERSION: "3.11"
|
||||
VAGRANT_DEFAULT_PROVIDER: virtualbox
|
||||
VAGRANT_HOME: ${{ github.workspace }}/.vagrant-home
|
||||
|
||||
steps:
|
||||
- name: Check out the codebase
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # 7.0.1
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Check nested VirtualBox platform
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
grep -Eq 'vmx|svm' /proc/cpuinfo
|
||||
test -c /dev/kvm
|
||||
test -c /dev/vboxdrv
|
||||
VBoxManage --version
|
||||
vagrant --version
|
||||
test -r /etc/vbox/networks.conf
|
||||
test "$(stat -c '%u' /etc/vbox/networks.conf)" -eq 0
|
||||
free -h
|
||||
df -Pk "${RUNNER_TEMP}"
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # 7.0.0
|
||||
uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # 5.3.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip' # caching pip dependencies
|
||||
|
||||
- name: Cache Vagrant boxes
|
||||
id: cache-vagrant
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # 6.1.0
|
||||
uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # 4.1.2
|
||||
with:
|
||||
# This producer only needs to know whether the immutable cache exists.
|
||||
# Molecule jobs restore it after this job completes.
|
||||
lookup-only: true
|
||||
lookup-only: true #if it exists, we don't need to restore and can skip the next step
|
||||
path: |
|
||||
.vagrant-home/boxes
|
||||
key: vagrant-boxes-${{ runner.name }}-${{ runner.os }}-${{ runner.arch }}-virtualbox-7.2-vagrant-2.4-${{ hashFiles('.github/vagrant-boxes.lock') }} # yamllint disable-line rule:line-length
|
||||
~/.vagrant.d/boxes
|
||||
key: vagrant-boxes-${{ hashFiles('**/molecule.yml') }}
|
||||
restore-keys: |
|
||||
vagrant-boxes
|
||||
|
||||
- name: Download Vagrant boxes for all scenarios
|
||||
# An exact hit skips both cache restoration and upstream downloads.
|
||||
# A lock change builds and saves one clean, version-pinned cache.
|
||||
if: steps.cache-vagrant.outputs.cache-hit != 'true'
|
||||
# To save some cache space, all scenarios share the same cache key.
|
||||
# On the other hand, this means that the cache contents should be
|
||||
# the same across all scenarios. This step ensures that.
|
||||
if: steps.cache-vagrant.outputs.cache-hit != 'true' # only run if false since this is just a cache step
|
||||
run: |
|
||||
./.github/download-boxes.sh
|
||||
./.github/scripts/prepare-vagrant-box-masters.sh
|
||||
vagrant box list
|
||||
|
||||
@@ -2,43 +2,26 @@
|
||||
name: "CI"
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
types:
|
||||
- opened
|
||||
- synchronize
|
||||
- reopened
|
||||
- ready_for_review
|
||||
paths-ignore:
|
||||
- '**/.gitignore'
|
||||
- '**/FUNDING.yml'
|
||||
- '**/host.ini'
|
||||
- '**/*.md'
|
||||
- '.github/ISSUE_TEMPLATE/**'
|
||||
- '**/.editorconfig'
|
||||
- '**/ansible.example.cfg'
|
||||
- '**/deploy.sh'
|
||||
- '**/LICENSE'
|
||||
- '**/reboot.sh'
|
||||
- '**/reset.sh'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ci-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
pre:
|
||||
if: github.actor != 'dependabot[bot]'
|
||||
uses: ./.github/workflows/cache.yml
|
||||
needs: [lint]
|
||||
lint:
|
||||
if: github.actor != 'dependabot[bot]'
|
||||
uses: ./.github/workflows/lint.yml
|
||||
needs: [pre]
|
||||
test:
|
||||
if: github.actor != 'dependabot[bot]'
|
||||
uses: ./.github/workflows/test.yml
|
||||
needs: [pre, lint]
|
||||
|
||||
@@ -5,25 +5,24 @@ on:
|
||||
jobs:
|
||||
pre-commit-ci:
|
||||
name: Pre-Commit
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: self-hosted
|
||||
env:
|
||||
PYTHON_VERSION: "3.12"
|
||||
PYTHON_VERSION: "3.11"
|
||||
|
||||
steps:
|
||||
- name: Check out the codebase
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # 7.0.1
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # 7.0.0
|
||||
uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # 5.3.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip' # caching pip dependencies
|
||||
|
||||
- name: Restore Ansible cache
|
||||
id: cache-ansible
|
||||
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # 6.1.0
|
||||
uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # 4.1.2
|
||||
with:
|
||||
path: ~/.ansible/collections
|
||||
key: ansible-${{ hashFiles('collections/requirements.yml') }}
|
||||
@@ -38,37 +37,17 @@ jobs:
|
||||
python3 -m pip install -r requirements.txt
|
||||
echo "::endgroup::"
|
||||
|
||||
- name: Install Ansible collections with retries
|
||||
if: steps.cache-ansible.outputs.cache-hit != 'true'
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
for attempt in 1 2 3 4 5; do
|
||||
if ansible-galaxy collection install -r collections/requirements.yml; then
|
||||
exit 0
|
||||
fi
|
||||
echo "Ansible Galaxy attempt ${attempt} failed; retrying."
|
||||
sleep $((attempt * 10))
|
||||
done
|
||||
exit 1
|
||||
|
||||
- name: Save Ansible collection cache
|
||||
if: steps.cache-ansible.outputs.cache-hit != 'true'
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # 6.1.0
|
||||
with:
|
||||
path: ~/.ansible/collections
|
||||
key: ansible-${{ hashFiles('collections/requirements.yml') }}
|
||||
|
||||
- name: Run pre-commit
|
||||
uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # 3.0.1
|
||||
|
||||
ensure-pinned-actions:
|
||||
name: Ensure SHA Pinned Actions
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # 7.0.1
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
|
||||
- name: Ensure SHA pinned actions
|
||||
uses: zgosalvez/github-actions-ensure-sha-pinned-actions@60e3a74c7b74a319e8e53e46bc455205f7173e1a # 5.0.8
|
||||
uses: zgosalvez/github-actions-ensure-sha-pinned-actions@38608ef4fb69adae7f1eac6eeb88e67b7d083bfd # 3.0.16
|
||||
with:
|
||||
allowlist: |
|
||||
aws-actions/
|
||||
|
||||
+67
-59
@@ -5,64 +5,72 @@ on:
|
||||
jobs:
|
||||
molecule:
|
||||
name: Molecule
|
||||
runs-on: [self-hosted, linux, x64, k3s-ci, virtualbox, nested-virt]
|
||||
runs-on: self-hosted
|
||||
strategy:
|
||||
matrix:
|
||||
scenario:
|
||||
- default
|
||||
# - ipv6
|
||||
- single_node
|
||||
- calico
|
||||
- cilium
|
||||
- kube-vip
|
||||
# - ipv6
|
||||
fail-fast: true
|
||||
max-parallel: 1
|
||||
fail-fast: false
|
||||
env:
|
||||
PYTHON_VERSION: "3.11"
|
||||
VAGRANT_DEFAULT_PROVIDER: virtualbox
|
||||
VAGRANT_HOME: ${{ github.workspace }}/.vagrant-home
|
||||
|
||||
steps:
|
||||
- name: Check out the codebase
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # 7.0.1
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Clean repository-owned resources before testing
|
||||
run: ./.github/scripts/cleanup-runner-resources.sh --apply
|
||||
# these steps are necessary if not using ephemeral nodes
|
||||
- name: Delete old Vagrant box versions
|
||||
if: always() # do this even if a step before has failed
|
||||
run: vagrant box prune --force
|
||||
|
||||
- name: Record host-only network baseline
|
||||
run: ./.github/scripts/cleanup-runner-resources.sh --snapshot
|
||||
- name: Remove all local Vagrant boxes
|
||||
if: always() # do this even if a step before has failed
|
||||
run: if vagrant box list 2>/dev/null; then vagrant box list | cut -f 1 -d ' ' | xargs -L 1 vagrant box remove -f 2>/dev/null && echo "All Vagrant boxes removed." || echo "No Vagrant boxes found."; else echo "No Vagrant boxes found."; fi
|
||||
|
||||
- name: Check nested VirtualBox platform
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
grep -Eq 'vmx|svm' /proc/cpuinfo
|
||||
test -c /dev/kvm
|
||||
test -c /dev/vboxdrv
|
||||
VBoxManage --version
|
||||
vagrant --version
|
||||
test -r /etc/vbox/networks.conf
|
||||
test "$(stat -c '%u' /etc/vbox/networks.conf)" -eq 0
|
||||
free -h
|
||||
df -Pk "${RUNNER_TEMP}"
|
||||
- name: Remove all Virtualbox VMs
|
||||
if: always() # do this even if a step before has failed
|
||||
run: VBoxManage list vms | awk -F'"' '{print $2}' | xargs -I {} VBoxManage unregistervm --delete "{}"
|
||||
|
||||
- name: Remove all Virtualbox HDs
|
||||
if: always() # do this even if a step before has failed
|
||||
run: VBoxManage list hdds | awk -F':' '/^UUID:/ {print $2}' | xargs -I {} VBoxManage closemedium disk "{}" --delete
|
||||
|
||||
- name: Remove all Virtualbox Networks
|
||||
if: always() # do this even if a step before has failed
|
||||
run: VBoxManage list hostonlyifs | grep '^Name:' | awk '{print $2}' | grep '^vboxnet' | xargs -I {} VBoxManage hostonlyif remove {}
|
||||
|
||||
- name: Remove Virtualbox network config
|
||||
if: always() # do this even if a step before has failed
|
||||
run: sudo rm /etc/vbox/networks.conf || true
|
||||
|
||||
- name: Configure VirtualBox
|
||||
run: |-
|
||||
sudo mkdir -p /etc/vbox
|
||||
cat <<EOF | sudo tee -a /etc/vbox/networks.conf > /dev/null
|
||||
* 192.168.30.0/24
|
||||
* fdad:bad:ba55::/64
|
||||
EOF
|
||||
|
||||
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # 7.0.0
|
||||
uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # 5.3.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip' # caching pip dependencies
|
||||
|
||||
- name: Restore vagrant Boxes cache
|
||||
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # 6.1.0
|
||||
uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # 4.1.2
|
||||
with:
|
||||
path: .vagrant-home/boxes
|
||||
key: vagrant-boxes-${{ runner.name }}-${{ runner.os }}-${{ runner.arch }}-virtualbox-7.2-vagrant-2.4-${{ hashFiles('.github/vagrant-boxes.lock') }} # yamllint disable-line rule:line-length
|
||||
path: ~/.vagrant.d/boxes
|
||||
key: vagrant-boxes-${{ hashFiles('**/molecule.yml') }}
|
||||
fail-on-cache-miss: true
|
||||
|
||||
- name: Prepare runner-owned Vagrant box masters
|
||||
run: ./.github/scripts/prepare-vagrant-box-masters.sh
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
echo "::group::Upgrade pip"
|
||||
@@ -74,45 +82,45 @@ jobs:
|
||||
echo "::endgroup::"
|
||||
|
||||
- name: Test with molecule
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
resource_dir="${RUNNER_TEMP}/logs/resources/${{ matrix.scenario }}"
|
||||
timing_file="${RUNNER_TEMP}/logs/timing/${{ matrix.scenario }}.txt"
|
||||
mkdir -p -- "${timing_file%/*}"
|
||||
./.github/scripts/monitor-runner-resources.sh "$resource_dir" 10 &
|
||||
monitor_pid=$!
|
||||
stop_monitor() {
|
||||
kill -TERM "$monitor_pid" 2>/dev/null || true
|
||||
wait "$monitor_pid" 2>/dev/null || true
|
||||
}
|
||||
trap stop_monitor EXIT
|
||||
/usr/bin/time -v -o "$timing_file" \
|
||||
molecule test --scenario-name ${{ matrix.scenario }}
|
||||
timeout-minutes: 180
|
||||
run: molecule test --scenario-name ${{ matrix.scenario }}
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
ANSIBLE_K3S_LOG_DIR: ${{ runner.temp }}/logs/k3s-ansible/${{ matrix.scenario }}
|
||||
ANSIBLE_SSH_RETRIES: 4
|
||||
ANSIBLE_TIMEOUT: 120
|
||||
PY_COLORS: 1
|
||||
ANSIBLE_FORCE_COLOR: 1
|
||||
K3S_CI_CREATE_TIMING_LOG: ${{ runner.temp }}/logs/timing/${{ matrix.scenario }}-create.log
|
||||
|
||||
- name: Collect runner diagnostics
|
||||
if: always()
|
||||
run: ./.github/scripts/collect-runner-diagnostics.sh "${RUNNER_TEMP}/logs/runner"
|
||||
env:
|
||||
K3S_CI_SCENARIO_NAME: ${{ matrix.scenario }}
|
||||
# these steps are necessary if not using ephemeral nodes
|
||||
- name: Delete old Vagrant box versions
|
||||
if: always() # do this even if a step before has failed
|
||||
run: vagrant box prune --force
|
||||
|
||||
- name: Clean repository-owned resources after testing
|
||||
if: always()
|
||||
run: ./.github/scripts/cleanup-runner-resources.sh --apply
|
||||
- name: Remove all local Vagrant boxes
|
||||
if: always() # do this even if a step before has failed
|
||||
run: if vagrant box list 2>/dev/null; then vagrant box list | cut -f 1 -d ' ' | xargs -L 1 vagrant box remove -f 2>/dev/null && echo "All Vagrant boxes removed." || echo "No Vagrant boxes found."; else echo "No Vagrant boxes found."; fi
|
||||
|
||||
- name: Remove all Virtualbox VMs
|
||||
if: always() # do this even if a step before has failed
|
||||
run: VBoxManage list vms | awk -F'"' '{print $2}' | xargs -I {} VBoxManage unregistervm --delete "{}"
|
||||
|
||||
- name: Remove all Virtualbox HDs
|
||||
if: always() # do this even if a step before has failed
|
||||
run: VBoxManage list hdds | awk -F':' '/^UUID:/ {print $2}' | xargs -I {} VBoxManage closemedium disk "{}" --delete
|
||||
|
||||
- name: Remove all Virtualbox Networks
|
||||
if: always() # do this even if a step before has failed
|
||||
run: VBoxManage list hostonlyifs | grep '^Name:' | awk '{print $2}' | grep '^vboxnet' | xargs -I {} VBoxManage hostonlyif remove {}
|
||||
|
||||
- name: Remove Virtualbox network config
|
||||
if: always() # do this even if a step before has failed
|
||||
run: sudo rm /etc/vbox/networks.conf || true
|
||||
|
||||
- name: Upload log files
|
||||
if: always() # do this even if a step before has failed
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # 7.0.1
|
||||
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # 4.4.3
|
||||
with:
|
||||
name: logs-${{ matrix.scenario }}-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
name: logs
|
||||
path: |
|
||||
${{ runner.temp }}/logs
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
overwrite: true
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
.env/
|
||||
*.log
|
||||
ansible.cfg
|
||||
.ansible/
|
||||
kubeconfig
|
||||
zIgnore/
|
||||
|
||||
@@ -20,9 +20,6 @@ repos:
|
||||
rev: v6.22.2
|
||||
hooks:
|
||||
- id: ansible-lint
|
||||
additional_dependencies: [ansible-core==2.18.0]
|
||||
language_version: python3.12
|
||||
args: [--offline]
|
||||
- repo: https://github.com/shellcheck-py/shellcheck-py
|
||||
rev: v0.9.0.6
|
||||
hooks:
|
||||
@@ -36,122 +33,3 @@ repos:
|
||||
rev: 0.6.4
|
||||
hooks:
|
||||
- id: fix-smartquotes
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: cleanup-runner-resources-test
|
||||
name: cleanup runner resources test
|
||||
entry: .github/scripts/test-cleanup-runner-resources.sh
|
||||
language: system
|
||||
pass_filenames: false
|
||||
files: ^\.github/scripts/(cleanup-runner-resources|test-cleanup-runner-resources)\.sh$
|
||||
- id: download-vagrant-boxes-test
|
||||
name: Vagrant box download test
|
||||
entry: .github/test-download-boxes.sh
|
||||
language: system
|
||||
pass_filenames: false
|
||||
files: ^\.github/(download-boxes|test-download-boxes)\.sh$|^\.github/vagrant-boxes\.lock$
|
||||
- id: prepare-vagrant-box-masters-test
|
||||
name: Vagrant box master preparation test
|
||||
entry: .github/scripts/test-prepare-vagrant-box-masters.sh
|
||||
language: system
|
||||
pass_filenames: false
|
||||
files: ^\.github/scripts/(prepare-vagrant-box-masters|test-prepare-vagrant-box-masters)\.sh$
|
||||
- id: k3s-server-bootstrap-test
|
||||
name: K3s transient bootstrap test
|
||||
entry: .github/scripts/test-k3s-server-bootstrap.sh
|
||||
language: system
|
||||
pass_filenames: false
|
||||
files: ^roles/k3s_server/tasks/(main|join_master)\.yml$|^\.github/scripts/test-k3s-server-bootstrap\.sh$
|
||||
- id: unique-hostname-precheck-test
|
||||
name: Unique hostname precheck test
|
||||
entry: .github/scripts/test-unique-hostname-precheck.sh
|
||||
language: system
|
||||
pass_filenames: false
|
||||
files: ^site\.yml$|^\.github/scripts/test-unique-hostname-precheck\.sh$
|
||||
- id: disable-swap-test
|
||||
name: Disable swap test
|
||||
entry: .github/scripts/test-disable-swap.sh
|
||||
language: system
|
||||
pass_filenames: false
|
||||
files: ^roles/prereq/(tasks/main|defaults/main)\.yml$|^\.github/scripts/test-disable-swap\.sh$
|
||||
- id: cilium-bgp-manifest-test
|
||||
name: Cilium BGP manifest test
|
||||
entry: python3 .github/scripts/test-cilium-bgp-manifest.py
|
||||
language: python
|
||||
additional_dependencies:
|
||||
- Jinja2>=3.1
|
||||
pass_filenames: false
|
||||
files: ^roles/k3s_server_post/templates/cilium\.crs\.j2$|^\.github/scripts/test-cilium-bgp-manifest\.py$
|
||||
- id: cilium-envoy-toggle-test
|
||||
name: Cilium Envoy toggle test
|
||||
entry: python3 .github/scripts/test-cilium-envoy-toggle.py
|
||||
language: python
|
||||
additional_dependencies:
|
||||
- Jinja2>=3.1
|
||||
- PyYAML
|
||||
pass_filenames: false
|
||||
files: ^roles/k3s_server_post/tasks/cilium\.yml$|^\.github/scripts/test-cilium-envoy-toggle\.py$
|
||||
- id: kube-vip-manifest-test
|
||||
name: kube-vip manifest test
|
||||
entry: python3 .github/scripts/test-kube-vip-manifest.py
|
||||
language: python
|
||||
additional_dependencies:
|
||||
- Jinja2>=3.1
|
||||
pass_filenames: false
|
||||
files: ^roles/k3s_server/templates/vip\.yaml\.j2$|^\.github/scripts/test-kube-vip-manifest\.py$
|
||||
- id: metallb-remote-read-test
|
||||
name: MetalLB remote read test
|
||||
entry: .github/scripts/test-metallb-remote-read.sh
|
||||
language: system
|
||||
pass_filenames: false
|
||||
files: ^roles/k3s_server/tasks/metallb\.yml$|^\.github/scripts/test-metallb-remote-read\.sh$
|
||||
- id: metallb-interfaces-test
|
||||
name: MetalLB interfaces test
|
||||
entry: python3 .github/scripts/test-metallb-interfaces.py
|
||||
language: python
|
||||
additional_dependencies:
|
||||
- Jinja2>=3.1
|
||||
pass_filenames: false
|
||||
files: ^roles/k3s_server_post/templates/metallb\.crs\.j2$|^\.github/scripts/test-metallb-interfaces\.py$
|
||||
- id: metallb-namespace-test
|
||||
name: MetalLB namespace test
|
||||
entry: python3 .github/scripts/test-metallb-namespace.py
|
||||
language: python
|
||||
additional_dependencies:
|
||||
- PyYAML
|
||||
pass_filenames: false
|
||||
files: ^roles/k3s_server_post/tasks/metallb\.yml$|^\.github/scripts/test-metallb-namespace\.py$
|
||||
- id: metallb-deploy-condition-test
|
||||
name: MetalLB deploy condition test
|
||||
entry: python3 .github/scripts/test-metallb-deploy-condition.py
|
||||
language: python
|
||||
additional_dependencies:
|
||||
- Jinja2>=3.1
|
||||
- PyYAML
|
||||
pass_filenames: false
|
||||
files: ^roles/k3s_server/tasks/main\.yml$|^roles/k3s_server_post/tasks/main\.yml$|^\.github/scripts/test-metallb-deploy-condition\.py$ # noqa yaml[line-length]
|
||||
- id: kube-vip-deploy-condition-test
|
||||
name: kube-vip deploy condition test
|
||||
entry: python3 .github/scripts/test-kube-vip-deploy-condition.py
|
||||
language: python
|
||||
additional_dependencies:
|
||||
- Jinja2>=3.1
|
||||
- PyYAML
|
||||
pass_filenames: false
|
||||
files: ^roles/k3s_server/tasks/main\.yml$|^\.github/scripts/test-kube-vip-deploy-condition\.py$ # noqa yaml[line-length]
|
||||
- id: default-interface-test
|
||||
name: default interface test
|
||||
entry: python3 .github/scripts/test-default-interface.py
|
||||
language: python
|
||||
additional_dependencies:
|
||||
- Jinja2>=3.1
|
||||
pass_filenames: false
|
||||
files: ^inventory/sample/group_vars/all\.yml$|^\.github/scripts/test-default-interface\.py$
|
||||
- id: verify-nginx-assert-test
|
||||
name: verify nginx assert test
|
||||
entry: python3 .github/scripts/test-verify-nginx-assert.py
|
||||
language: python
|
||||
additional_dependencies:
|
||||
- PyYAML
|
||||
pass_filenames: false
|
||||
files: ^molecule/resources/verify_from_outside/tasks/test/deploy-example\.yml$|^\.github/scripts/test-verify-nginx-assert\.py$ # noqa yaml[line-length]
|
||||
|
||||
@@ -1,130 +0,0 @@
|
||||
# Agent Guide
|
||||
|
||||
This file is the canonical repository guide for coding agents and automated contributors. Read it before making
|
||||
changes. Human contributors should also review [CONTRIBUTING.md](CONTRIBUTING.md).
|
||||
|
||||
## Project overview
|
||||
|
||||
This repository is an Ansible collection that provisions and resets highly available k3s clusters. It supports
|
||||
multiple networking choices, including Flannel, Calico, Cilium, kube-vip, and MetalLB.
|
||||
|
||||
The main entry points are:
|
||||
|
||||
- `site.yml`: provision or update a cluster.
|
||||
- `reset.yml`: remove k3s from a cluster.
|
||||
- `reboot.yml`: reboot cluster nodes.
|
||||
- `inventory/sample/`: example inventory and variables.
|
||||
- `roles/`: reusable Ansible roles used by the playbooks.
|
||||
- `molecule/`: integration scenarios run by CI.
|
||||
- `.github/scripts/`: CI support scripts and focused regression tests.
|
||||
|
||||
## Source of truth
|
||||
|
||||
- Role defaults belong in `roles/<role>/defaults/main.yml`.
|
||||
- Tasks belong in `roles/<role>/tasks/` and handlers in `roles/<role>/handlers/`.
|
||||
- Example user configuration belongs in `inventory/sample/`.
|
||||
- User-facing setup and variable documentation belongs in `README.md`.
|
||||
- Contributor workflows and review expectations belong in `CONTRIBUTING.md`.
|
||||
- Agent-specific repository instructions belong in this file.
|
||||
|
||||
Keep `CLAUDE.md` and `.github/copilot-instructions.md` as small pointers to this file. Do not duplicate these
|
||||
instructions in tool-specific files.
|
||||
|
||||
## Development setup
|
||||
|
||||
Use a Python virtual environment. Do not commit the environment, generated logs, inventories, kubeconfigs, or
|
||||
credentials.
|
||||
|
||||
```bash
|
||||
python3 -m venv .env
|
||||
source .env/bin/activate
|
||||
python3 -m pip install -r requirements.txt
|
||||
ansible-galaxy collection install -r collections/requirements.yml
|
||||
pre-commit install
|
||||
```
|
||||
|
||||
`ansible.cfg` is intentionally ignored. Copy `ansible.example.cfg` when local configuration is needed.
|
||||
|
||||
## Working rules
|
||||
|
||||
1. Inspect the current branch and worktree before editing. Preserve unrelated user changes.
|
||||
2. Keep changes focused. Avoid drive-by formatting or dependency updates.
|
||||
3. Never add real IP addresses, hostnames, tokens, private keys, kubeconfigs, or inventory secrets.
|
||||
4. Use placeholders in examples and redact sensitive values from logs and issue reports.
|
||||
5. Preserve idempotence. An already-converged host should not report changes without a real state transition.
|
||||
6. Prefer Ansible modules over `ansible.builtin.command` or `ansible.builtin.shell`. When a command is required,
|
||||
define accurate `changed_when` and `failed_when` behavior.
|
||||
7. Use fully qualified collection names, such as `ansible.builtin.copy`.
|
||||
8. Put configurable values in role defaults or inventory variables. Avoid embedding environment-specific values in
|
||||
tasks and templates.
|
||||
9. Maintain compatibility with the operating systems and architectures listed in `README.md`.
|
||||
10. Do not weaken lint rules, tests, or CI checks to make a change pass.
|
||||
|
||||
## Change guidance
|
||||
|
||||
### Ansible tasks and roles
|
||||
|
||||
- Use descriptive task names in sentence case.
|
||||
- Use YAML booleans (`true` and `false`) rather than aliases.
|
||||
- Quote file modes, for example `mode: "0644"`.
|
||||
- Notify handlers only when the managed resource changes.
|
||||
- Use `become: true` only where privilege escalation is needed.
|
||||
- Update role defaults, sample inventory, and the README together when adding or renaming user-facing variables.
|
||||
- Check reset behavior when provisioning introduces persistent services, files, mounts, or network state.
|
||||
|
||||
### Templates and manifests
|
||||
|
||||
- Keep Jinja logic small and readable. Move complicated decisions into task variables where practical.
|
||||
- Render valid YAML after Jinja evaluation.
|
||||
- Preserve explicit handling for optional and undefined variables.
|
||||
- Add or update a focused test under `.github/scripts/` when changing generated Kubernetes manifests or bootstrap
|
||||
behavior.
|
||||
|
||||
### Molecule scenarios
|
||||
|
||||
- Reuse `molecule/resources/` for shared behavior.
|
||||
- Put scenario-specific inputs in `molecule/<scenario>/overrides.yml` and `verify-vars.yml`.
|
||||
- Update `molecule/README.md` when adding, removing, or materially changing a scenario.
|
||||
- Clean up resources created by tests, including failure paths.
|
||||
|
||||
## Validation
|
||||
|
||||
Run the smallest relevant checks while iterating, then run the complete local validation before considering a change
|
||||
ready:
|
||||
|
||||
```bash
|
||||
pre-commit run --all-files
|
||||
```
|
||||
|
||||
For playbook or role changes, also run syntax checks with a non-sensitive inventory:
|
||||
|
||||
```bash
|
||||
ansible-playbook site.yml --syntax-check -i inventory/sample/hosts.ini
|
||||
ansible-playbook reset.yml --syntax-check -i inventory/sample/hosts.ini
|
||||
```
|
||||
|
||||
Run focused regression scripts when their related files change. The mapping is defined in
|
||||
`.pre-commit-config.yaml`.
|
||||
|
||||
Molecule tests require Vagrant, VirtualBox, host-only networking, and substantial local resources. Run the most
|
||||
relevant scenario when that environment is available:
|
||||
|
||||
```bash
|
||||
molecule test --scenario-name <scenario>
|
||||
```
|
||||
|
||||
If a required test can't be run locally, state exactly which check was skipped and why. Never claim a check passed
|
||||
unless it was executed.
|
||||
|
||||
## Documentation and review
|
||||
|
||||
- Keep commands copyable and examples free of secrets.
|
||||
- Update documentation in the same change as user-visible behavior.
|
||||
- Explain behavior changes, compatibility concerns, operational risks, and rollback steps in the pull request.
|
||||
- Use conventional commit messages with a scope, for example `fix(k3s-server): handle an existing token safely`.
|
||||
- Do not commit, push, open a pull request, or modify remote resources unless the user explicitly requests it.
|
||||
|
||||
## Definition of done
|
||||
|
||||
A change is ready for review when it is focused, documented, linted, tested in proportion to its risk, and shown in a
|
||||
clean diff with no secrets or generated artifacts.
|
||||
@@ -1,3 +0,0 @@
|
||||
@AGENTS.md
|
||||
|
||||
`AGENTS.md` is the canonical repository guide. Follow it for all work in this repository.
|
||||
@@ -1,94 +0,0 @@
|
||||
# Contributing to k3s-ansible
|
||||
|
||||
Thank you for improving k3s-ansible. Contributions should be focused, safe to apply to existing clusters, and tested
|
||||
in proportion to their operational impact.
|
||||
|
||||
## Before opening an issue
|
||||
|
||||
- Search existing issues and discussions for the same behavior.
|
||||
- Review the [troubleshooting discussion](https://github.com/timothystewart6/k3s-ansible/discussions/20).
|
||||
- Remove tokens, credentials, public IP addresses, private hostnames, and other sensitive values from logs and
|
||||
configuration.
|
||||
- For support requests, include the k3s-ansible revision, Ansible version, target operating system, architecture,
|
||||
network provider, relevant sanitized variables, and a minimal reproduction.
|
||||
|
||||
Use the bug report template for reproducible defects and the feature request template for proposed behavior.
|
||||
|
||||
## Development environment
|
||||
|
||||
Fork and clone the repository, then create a branch from the latest `master`:
|
||||
|
||||
```bash
|
||||
git switch master
|
||||
git pull --ff-only
|
||||
git switch -c <type>/<short-description>
|
||||
```
|
||||
|
||||
Create a Python environment and install the pinned development dependencies:
|
||||
|
||||
```bash
|
||||
python3 -m venv .env
|
||||
source .env/bin/activate
|
||||
python3 -m pip install -r requirements.txt
|
||||
ansible-galaxy collection install -r collections/requirements.yml
|
||||
pre-commit install
|
||||
```
|
||||
|
||||
Copy `ansible.example.cfg` to the ignored `ansible.cfg` file if local Ansible configuration is needed. Start custom
|
||||
inventories from `inventory/sample/`, keep them out of Git, and never use production credentials in tests.
|
||||
|
||||
## Making changes
|
||||
|
||||
- Keep each pull request focused on one problem or feature.
|
||||
- Follow [AGENTS.md](AGENTS.md) for repository structure, implementation conventions, and safety requirements.
|
||||
- Preserve idempotence and existing-cluster compatibility.
|
||||
- Add or update tests for behavior changes and regressions.
|
||||
- Update role defaults, sample inventory, and documentation when user-facing variables change.
|
||||
- Consider both provisioning and reset behavior for persistent resources.
|
||||
- Avoid unrelated reformatting and generated files.
|
||||
|
||||
## Validation
|
||||
|
||||
Run all pre-commit checks before submitting a pull request:
|
||||
|
||||
```bash
|
||||
pre-commit run --all-files
|
||||
```
|
||||
|
||||
For playbook or role changes, run syntax checks:
|
||||
|
||||
```bash
|
||||
ansible-playbook site.yml --syntax-check -i inventory/sample/hosts.ini
|
||||
ansible-playbook reset.yml --syntax-check -i inventory/sample/hosts.ini
|
||||
```
|
||||
|
||||
Run the most relevant Molecule scenario when Vagrant, VirtualBox, and the required host networking are available:
|
||||
|
||||
```bash
|
||||
molecule test --scenario-name <scenario>
|
||||
```
|
||||
|
||||
See [molecule/README.md](molecule/README.md) for scenario details and local requirements. Pull requests should list
|
||||
every check that was run and clearly identify checks that could not be run locally.
|
||||
|
||||
## Commits and pull requests
|
||||
|
||||
Use a conventional commit subject with a scope:
|
||||
|
||||
```text
|
||||
type(scope): short description
|
||||
```
|
||||
|
||||
Common types are `feat`, `fix`, `refactor`, `test`, `docs`, and `chore`. Write subjects in the imperative mood and
|
||||
keep commits logically focused.
|
||||
|
||||
Pull requests should:
|
||||
|
||||
- Describe the problem and the resulting behavior.
|
||||
- Identify compatibility, security, networking, and upgrade risks.
|
||||
- Include testing evidence without sensitive data.
|
||||
- Call out documentation and sample configuration changes.
|
||||
- Link related issues with `Fixes #<issue>` when applicable.
|
||||
- Avoid checking boxes for tests that were not run.
|
||||
|
||||
Maintainers may ask for a change to be split when unrelated work makes it difficult to review or roll back.
|
||||
@@ -2,70 +2,51 @@
|
||||
|
||||

|
||||
|
||||
This Ansible collection builds a highly available Kubernetes cluster with k3s. It supports kube-vip for the control
|
||||
plane virtual IP, multiple CNI options, and either MetalLB or kube-vip for service load balancing.
|
||||
This playbook will build an HA Kubernetes cluster with `k3s`, `kube-vip` and MetalLB via `ansible`.
|
||||
|
||||
This is based on the work from [this fork](https://github.com/212850a/k3s-ansible) which is based on the work from [k3s-io/k3s-ansible](https://github.com/k3s-io/k3s-ansible). It uses [kube-vip](https://kube-vip.io/) to create a load balancer for control plane, and [metal-lb](https://metallb.universe.tf/installation/) for its service `LoadBalancer`.
|
||||
|
||||
For more context on how it works, see:
|
||||
If you want more context on how this works, see:
|
||||
|
||||
📄 [Documentation](https://technotim.com/posts/k3s-etcd-ansible/) (including example commands)
|
||||
📄 [Documentation](https://technotim.live/posts/k3s-etcd-ansible/) (including example commands)
|
||||
|
||||
📺 [Watch the Video](https://www.youtube.com/watch?v=CbkEWcUZ7zM)
|
||||
|
||||
## Project guides
|
||||
|
||||
- [Getting started](#-getting-started)
|
||||
- [Configuration variables](#variables)
|
||||
- [Upgrading an existing cluster](#-upgrading-an-existing-cluster)
|
||||
- [Local Molecule testing](molecule/README.md)
|
||||
- [Contributing guidelines](CONTRIBUTING.md)
|
||||
- [Repository guide for coding agents](AGENTS.md)
|
||||
|
||||
## 📖 k3s Ansible Playbook
|
||||
|
||||
Build a Kubernetes cluster using Ansible and k3s. The goal is to make a highly available cluster straightforward to
|
||||
install on machines running:
|
||||
Build a Kubernetes cluster using Ansible with k3s. The goal is easily install a HA Kubernetes cluster on machines running:
|
||||
|
||||
- [x] Debian (tested on version 13)
|
||||
- [x] Ubuntu (tested on version 26.04 LTS)
|
||||
- [x] Rocky (tested on version 10)
|
||||
- [x] Debian (tested on version 11)
|
||||
- [x] Ubuntu (tested on version 22.04)
|
||||
- [x] Rocky (tested on version 9)
|
||||
|
||||
Supported processor architectures are:
|
||||
on processor architecture:
|
||||
|
||||
- [x] x64
|
||||
- [x] arm64
|
||||
- [x] armhf
|
||||
- [X] x64
|
||||
- [X] arm64
|
||||
- [X] armhf
|
||||
|
||||
## ✅ System requirements
|
||||
|
||||
- The control node, which runs the Ansible commands, must have Ansible 2.11 or newer. For a quick primer, see
|
||||
[setting up Ansible](https://technotim.com/posts/ansible-automation/).
|
||||
- Control Node (the machine you are running `ansible` commands) must have Ansible 2.11+ If you need a quick primer on Ansible [you can check out my docs and setting up Ansible](https://technotim.live/posts/ansible-automation/).
|
||||
|
||||
- Install the required collections with
|
||||
`ansible-galaxy collection install -r ./collections/requirements.yml`.
|
||||
- You will also need to install collections that this playbook uses by running `ansible-galaxy collection install -r ./collections/requirements.yml` (important❗)
|
||||
|
||||
- [`netaddr` package](https://pypi.org/project/netaddr/) must be available to Ansible. If you have installed Ansible via apt, this is already taken care of. If you have installed Ansible via `pip`, make sure to install `netaddr` into the respective virtual environment.
|
||||
|
||||
- Server and agent nodes should support passwordless SSH access. Otherwise, pass `--ask-pass --ask-become-pass` to
|
||||
each playbook command.
|
||||
|
||||
- Every node in the cluster must have a **unique hostname**. k3s registers each node keyed by its hostname, so
|
||||
two nodes with the same hostname cannot join the cluster. `site.yml` asserts this up front and fails fast if
|
||||
any duplicate is found.
|
||||
- `server` and `agent` nodes should have passwordless SSH access, if not you can supply arguments to provide credentials `--ask-pass --ask-become-pass` to each command.
|
||||
|
||||
## 🚀 Getting Started
|
||||
|
||||
### 🍴 Preparation
|
||||
|
||||
Create a cluster-specific inventory from the sample. The `inventory/` directory ignores custom inventory content so
|
||||
credentials and environment details aren't committed accidentally.
|
||||
First create a new directory based on the `sample` directory within the `inventory` directory:
|
||||
|
||||
```bash
|
||||
cp -R inventory/sample inventory/my-cluster
|
||||
```
|
||||
|
||||
Edit `inventory/my-cluster/hosts.ini` to match the target hosts.
|
||||
Second, edit `inventory/my-cluster/hosts.ini` to match the system information gathered above
|
||||
|
||||
For example:
|
||||
|
||||
@@ -86,10 +67,9 @@ node
|
||||
|
||||
If multiple hosts are in the master group, the playbook will automatically set up k3s in [HA mode with etcd](https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/).
|
||||
|
||||
Copy `ansible.example.cfg` to `ansible.cfg`, then update its inventory path. The local `ansible.cfg` file is ignored by
|
||||
Git.
|
||||
Finally, copy `ansible.example.cfg` to `ansible.cfg` and adapt the inventory path to match the files that you just created.
|
||||
|
||||
The minimum k3s version is `1.19.1`. Select the desired version with the `k3s_version` variable.
|
||||
This requires at least k3s version `1.19.1` however the version is configurable by using the `k3s_version` variable.
|
||||
|
||||
If needed, you can also edit `inventory/my-cluster/group_vars/all.yml` to match your environment.
|
||||
|
||||
@@ -101,8 +81,7 @@ Start provisioning of the cluster using the following command:
|
||||
ansible-playbook site.yml -i inventory/my-cluster/hosts.ini
|
||||
```
|
||||
|
||||
After deployment, the control plane is accessible through the virtual IP defined by `apiserver_endpoint` in the
|
||||
inventory variables.
|
||||
After deployment control plane will be accessible via virtual ip-address which is defined in inventory/group_vars/all.yml as `apiserver_endpoint`
|
||||
|
||||
### 🔥 Remove k3s cluster
|
||||
|
||||
@@ -110,51 +89,7 @@ inventory variables.
|
||||
ansible-playbook reset.yml -i inventory/my-cluster/hosts.ini
|
||||
```
|
||||
|
||||
> Reboot the nodes after reset because the virtual IP may remain configured.
|
||||
|
||||
### ⏻️ Reboot Cluster Nodes
|
||||
|
||||
Reboot all cluster nodes at once or stage the reboot across the cluster.
|
||||
|
||||
```bash
|
||||
ansible-playbook reboot.yml -i inventory/my-cluster/hosts.ini
|
||||
```
|
||||
|
||||
To reboot the nodes in batches, set `concurrent_reboots` to the number of nodes
|
||||
to reboot at a time (or a percentage). Optionally set `wait_seconds_after_reboot`
|
||||
to pause after each batch so pods in the freshly rebooted batch can settle
|
||||
before the next batch reboots.
|
||||
|
||||
```bash
|
||||
ansible-playbook reboot.yml -i inventory/my-cluster/hosts.ini \
|
||||
--extra-vars 'concurrent_reboots=2 wait_seconds_after_reboot=30'
|
||||
```
|
||||
|
||||
## 🔁 Upgrading an existing cluster
|
||||
|
||||
These version variables select the components used for a **fresh** installation.
|
||||
They are not a supported direct in-place upgrade path for an existing cluster.
|
||||
K3s, Calico, and Cilium each require staged upgrades for long-lived clusters.
|
||||
|
||||
- **K3s**: do not jump an embedded-etcd cluster straight to Kubernetes 1.36.
|
||||
Upgrade one Kubernetes minor version at a time. From the sample default
|
||||
(`v1.30.2+k3s2`) the sequence is: the latest supported 1.30 patch, then 1.31,
|
||||
1.32, a 1.33 patch that contains etcd 3.5.26 (for example `v1.33.7+k3s3`),
|
||||
then 1.34, 1.35, and finally 1.36. Upgrade servers one at a time before
|
||||
agents. Take backups and confirm cluster health at each step; this playbook
|
||||
does not automate the upgrade, so those remain manual operational steps. See
|
||||
[K3s manual upgrades](https://docs.k3s.io/upgrades/manual) and the
|
||||
[v1.34 release notes](https://docs.k3s.io/release-notes/v1.34.X).
|
||||
- **Cilium**: upstream supports only consecutive minor upgrades. Update to the
|
||||
latest patch of the current minor, then upgrade 1.17, 1.18, 1.19, and 1.20 in
|
||||
order, reading each version's upgrade notes and running preflight checks.
|
||||
Do not attempt a direct upgrade from an old Cilium to 1.20.
|
||||
- **Calico**: starting with 3.28 the v3 resource UID behavior changed. If you
|
||||
have operators with OwnerReferences pointing to `projectcalico.org/v3`
|
||||
resources, remove and recreate those references around an in-place upgrade.
|
||||
- **MetalLB**: this project installs application tag `v0.16.0`. A newer
|
||||
chart-only tag such as `metallb-chart-0.16.1` is not an application or image
|
||||
release and must not be used as the controller or speaker image tag.
|
||||
>You should also reboot these nodes due to the VIP not being destroyed
|
||||
|
||||
## ⚙️ Kube Config
|
||||
|
||||
@@ -163,21 +98,13 @@ To copy your `kube config` locally so that you can access your **Kubernetes** cl
|
||||
```bash
|
||||
scp debian@master_ip:/etc/rancher/k3s/k3s.yaml ~/.kube/config
|
||||
```
|
||||
If the copy fails with a permission error, grant the SSH user temporary read access using the least permissive method
|
||||
available for the target system. Restore the original ownership and permissions immediately after copying. Avoid
|
||||
world-writable permissions on the kubeconfig because it contains cluster credentials.
|
||||
|
||||
For example, copy the file to a temporary user-readable path from the control node:
|
||||
|
||||
If you get file Permission denied, go into the node and temporarly run:
|
||||
```bash
|
||||
ssh debian@master_ip 'sudo install -o "$(id -un)" -m 0600 /etc/rancher/k3s/k3s.yaml /tmp/k3s.yaml'
|
||||
sudo chmod 777 /etc/rancher/k3s/k3s.yaml
|
||||
```
|
||||
|
||||
Copy `/tmp/k3s.yaml`, then remove the temporary remote copy:
|
||||
|
||||
Then copy with the scp command and reset the permissions back to:
|
||||
```bash
|
||||
scp debian@master_ip:/tmp/k3s.yaml ~/.kube/config
|
||||
ssh debian@master_ip rm -f /tmp/k3s.yaml
|
||||
sudo chmod 600 /etc/rancher/k3s/k3s.yaml
|
||||
```
|
||||
|
||||
You'll then want to modify the config to point to master IP by running:
|
||||
@@ -188,7 +115,7 @@ Then change `server: https://127.0.0.1:6443` to match your master IP: `server: h
|
||||
|
||||
### 🔨 Testing your cluster
|
||||
|
||||
See the commands [here](https://technotim.com/posts/k3s-etcd-ansible/#testing-your-cluster).
|
||||
See the commands [here](https://technotim.live/posts/k3s-etcd-ansible/#testing-your-cluster).
|
||||
|
||||
### Variables
|
||||
|
||||
@@ -197,7 +124,7 @@ See the commands [here](https://technotim.com/posts/k3s-etcd-ansible/#testing-yo
|
||||
| `download` | `k3s_version` | string | ❌ | Required | K3s binaries version |
|
||||
| `k3s_agent`, `k3s_server`, `k3s_server_post` | `apiserver_endpoint` | string | ❌ | Required | Virtual ip-address configured on each master |
|
||||
| `k3s_agent` | `extra_agent_args` | string | `null` | Not required | Extra arguments for agents nodes |
|
||||
| `k3s_agent`, `k3s_server` | `group_name_master` | string | `null` | Not required | Name of the master group |
|
||||
| `k3s_agent`, `k3s_server` | `group_name_master` | string | `null` | Not required | Name othe master group |
|
||||
| `k3s_agent` | `k3s_token` | string | `null` | Not required | Token used to communicate between masters |
|
||||
| `k3s_agent`, `k3s_server` | `proxy_env` | dict | `null` | Not required | Internet proxy configurations |
|
||||
| `k3s_agent`, `k3s_server` | `proxy_env.HTTP_PROXY` | string | ❌ | Required | HTTP internet proxy |
|
||||
@@ -219,14 +146,11 @@ See the commands [here](https://technotim.com/posts/k3s-etcd-ansible/#testing-yo
|
||||
| `k3s_server` | `kube_vip_bgp_peers` | list | `[]` | Not required | List of BGP peer ASN & address pairs |
|
||||
| `k3s_server` | `kube_vip_bgp_peers_groups` | list | `['k3s_master']` | Not required | Inventory group in which to search for additional `kube_vip_bgp_peers` parameters to merge. |
|
||||
| `k3s_server` | `kube_vip_iface` | string | `~` | Not required | Explicitly define an interface that ALL control nodes should use to propagate the VIP, define it here. Otherwise, kube-vip will determine the right interface automatically at runtime. |
|
||||
| `k3s_server` | `kube_vip_endpoint` | string | `~` | Not required | Overrides the internal address kube-vip binds/listens on, which can differ from the announced apiserver_endpoint for complex routing/tunnels. Defaults to apiserver_endpoint. |
|
||||
| `k3s_server` | `kube_vip_tag_version` | string | `v1.2.3` | Not required | Image tag for kube-vip |
|
||||
| `k3s_server` | `kube_vip_cloud_provider_tag_version` | string | `v0.0.12` | Not required | Tag for kube-vip-cloud-provider manifest when enable |
|
||||
| `k3s_server` | `kube_vip_enabled` | bool | `true` | Not required | Enable kube-vip install, covering both the control-plane VIP and the service load balancer. Set false to skip kube-vip entirely (single node or external LB). |
|
||||
| `k3s_server` | `kube_vip_tag_version` | string | `v0.7.2` | Not required | Image tag for kube-vip |
|
||||
| `k3s_server` | `kube_vip_cloud_provider_tag_version` | string | `main` | Not required | Tag for kube-vip-cloud-provider manifest when enable |
|
||||
| `k3s_server`, `k3_server_post` | `kube_vip_lb_ip_range` | string | `~` | Not required | IP range for kube-vip load balancer |
|
||||
| `k3s_server`, `k3s_server_post` | `metal_lb_enabled` | bool | `true` | Not required | Enable MetalLB install for service load balancing. Set false to skip MetalLB (external LB). |
|
||||
| `k3s_server`, `k3s_server_post` | `metal_lb_controller_tag_version` | string | `v0.16.0` | Not required | Image tag for MetalLB |
|
||||
| `k3s_server` | `metal_lb_speaker_tag_version` | string | `v0.16.0` | Not required | Image tag for MetalLB |
|
||||
| `k3s_server`, `k3s_server_post` | `metal_lb_controller_tag_version` | string | `v0.14.3` | Not required | Image tag for MetalLB |
|
||||
| `k3s_server` | `metal_lb_speaker_tag_version` | string | `v0.14.3` | Not required | Image tag for MetalLB |
|
||||
| `k3s_server` | `metal_lb_type` | string | `native` | Not required | Use FRR mode or native. Valid values are `frr` and `native` |
|
||||
| `k3s_server` | `retry_count` | int | `20` | Not required | Amount of retries when verifying that nodes joined |
|
||||
| `k3s_server` | `server_init_args` | string | ❌ | Not required | Arguments for server nodes |
|
||||
@@ -238,7 +162,7 @@ See the commands [here](https://technotim.com/posts/k3s-etcd-ansible/#testing-yo
|
||||
| `k3s_server_post` | `calico_natOutgoing` | string | `Enabled` | Not required | IP pool NAT outgoing |
|
||||
| `k3s_server_post` | `calico_nodeSelector` | string | `all()` | Not required | IP pool node selector |
|
||||
| `k3s_server_post` | `calico_iface` | string | `~` | Not required | The network interface used for when Calico is enabled |
|
||||
| `k3s_server_post` | `calico_tag` | string | `v3.32.1` | Not required | Calico version tag |
|
||||
| `k3s_server_post` | `calico_tag` | string | `v3.27.2` | Not required | Calico version tag |
|
||||
| `k3s_server_post` | `cilium_bgp_my_asn` | int | `64513` | Not required | Local ASN for BGP peer |
|
||||
| `k3s_server_post` | `cilium_bgp_peer_asn` | int | `64512` | Not required | BGP peer ASN |
|
||||
| `k3s_server_post` | `cilium_bgp_peer_address` | string | `~` | Not required | BGP peer address |
|
||||
@@ -247,31 +171,27 @@ See the commands [here](https://technotim.com/posts/k3s-etcd-ansible/#testing-yo
|
||||
| `k3s_server_post` | `cilium_bgp_lb_cidr` | string | `192.168.31.0/24` | Not required | BGP load balancer IP range |
|
||||
| `k3s_server_post` | `cilium_exportPodCIDR` | bool | `true` | Not required | Export pod CIDR |
|
||||
| `k3s_server_post` | `cilium_hubble` | bool | `true` | Not required | Enable Cilium Hubble |
|
||||
| `k3s_server_post` | `cilium_mode` | string | `native` | Not required | Inner-node communication mode (choices are `native` and `tunnel`; `routed` is a deprecated alias for `tunnel`) |
|
||||
| `k3s_server_post` | `cilium_tag` | string | `v1.20.0` | Not required | Cilium version tag |
|
||||
| `k3s_server_post` | `cilium_cli_tag` | string | `v0.19.7` | Not required | Cilium CLI version tag |
|
||||
| `k3s_server_post` | `cilium_hubble` | bool | `true` | Not required | Enable Cilium Hubble |
|
||||
| `k3s_server_post` | `cilium_mode` | string | `native` | Not required | Inner-node communication mode (choices are `native` and `routed`) |
|
||||
| `k3s_server_post` | `cluster_cidr` | string | `10.52.0.0/16` | Not required | Inner-cluster IP range |
|
||||
| `k3s_server_post` | `enable_bpf_masquerade` | bool | `true` | Not required | Use IP masquerading |
|
||||
| `k3s_server_post` | `kube_proxy_replacement` | bool | `true` | Not required | Replace the native kube-proxy with Cilium |
|
||||
| `k3s_server_post` | `metal_lb_available_timeout` | string | `240s` | Not required | Wait for MetalLB resources |
|
||||
| `k3s_server_post` | `metal_lb_ip_range` | string | `192.168.30.80-192.168.30.90` | Not required | MetalLB ip range for load balancer |
|
||||
| `k3s_server_post` | `metal_lb_controller_tag_version` | string | `v0.16.0` | Not required | Image tag for MetalLB |
|
||||
| `k3s_server_post` | `metal_lb_controller_tag_version` | string | `v0.14.3` | Not required | Image tag for MetalLB |
|
||||
| `k3s_server_post` | `metal_lb_mode` | string | `layer2` | Not required | Metallb mode (choices are `bgp` and `layer2`) |
|
||||
| `k3s_server_post` | `metal_lb_bgp_my_asn` | string | `~` | Not required | BGP ASN configurations |
|
||||
| `k3s_server_post` | `metal_lb_bgp_peer_asn` | string | `~` | Not required | BGP peer ASN configurations |
|
||||
| `k3s_server_post` | `metal_lb_bgp_peer_address` | string | `~` | Not required | BGP peer address |
|
||||
| `lxc` | `custom_reboot_command` | string | `~` | Not required | Command to run on reboot |
|
||||
| `reboot` (playbook) | `concurrent_reboots` | int/string | `100%` | Not required | Number (or percentage) of nodes to reboot at a time for a staggered reboot |
|
||||
| `reboot` (playbook) | `wait_seconds_after_reboot` | int | `0` | Not required | Pause in seconds between staggered reboot batches |
|
||||
| `prereq` | `system_timezone` | string | `null` | Not required | Timezone to be set on all nodes |
|
||||
| `prereq` | `disable_swap` | bool | `true` | Not required | Disable swap on all cluster nodes (swapoff + comment out /etc/fstab swap entries), all-or-nothing |
|
||||
| `proxmox_lxc`, `reset_proxmox_lxc` | `proxmox_lxc_ct_ids` | list | ❌ | Required | Proxmox container ID list |
|
||||
| `raspberrypi` | `state` | string | `present` | Not required | Indicates whether the k3s prerequisites for Raspberry Pi should be set up (possible values are `present` and `absent`) |
|
||||
|
||||
|
||||
### Troubleshooting
|
||||
|
||||
Be sure to see [this post](https://github.com/timothystewart6/k3s-ansible/discussions/20) on how to troubleshoot common problems
|
||||
Be sure to see [this post](https://github.com/techno-tim/k3s-ansible/discussions/20) on how to troubleshoot common problems
|
||||
|
||||
### Testing the playbook using molecule
|
||||
|
||||
@@ -280,11 +200,9 @@ It is run automatically in CI, but you can also run the tests locally.
|
||||
This might be helpful for quick feedback in a few cases.
|
||||
You can find more information about it [here](molecule/README.md).
|
||||
|
||||
### Pre-commit hooks
|
||||
### Pre-commit Hooks
|
||||
|
||||
This repository uses `pre-commit` to check style, syntax, Ansible content, and shell scripts. Install the Python
|
||||
dependencies, run `pre-commit install` once, and run `pre-commit run --all-files` before submitting a change. See
|
||||
[CONTRIBUTING.md](CONTRIBUTING.md) for the complete development workflow.
|
||||
This repo uses `pre-commit` and `pre-commit-hooks` to lint and fix common style and syntax errors. Be sure to install python packages and then run `pre-commit install`. For more information, see [pre-commit](https://pre-commit.com/)
|
||||
|
||||
## 🌌 Ansible Galaxy
|
||||
|
||||
@@ -300,7 +218,7 @@ collections:
|
||||
- name: community.general
|
||||
- name: ansible.posix
|
||||
- name: kubernetes.core
|
||||
- name: https://github.com/timothystewart6/k3s-ansible.git
|
||||
- name: https://github.com/techno-tim/k3s-ansible.git
|
||||
type: git
|
||||
version: master
|
||||
```
|
||||
|
||||
+3
-3
@@ -56,16 +56,16 @@ dependencies:
|
||||
kubernetes.core: '*'
|
||||
|
||||
# The URL of the originating SCM repository
|
||||
repository: https://github.com/timothystewart6/k3s-ansible
|
||||
repository: https://github.com/techno-tim/k3s-ansible
|
||||
|
||||
# The URL to any online docs
|
||||
documentation: https://github.com/timothystewart6/k3s-ansible
|
||||
documentation: https://github.com/techno-tim/k3s-ansible
|
||||
|
||||
# The URL to the homepage of the collection/project
|
||||
homepage: https://www.youtube.com/watch?v=CbkEWcUZ7zM
|
||||
|
||||
# The URL to the collection issue tracker
|
||||
issues: https://github.com/timothystewart6/k3s-ansible/issues
|
||||
issues: https://github.com/techno-tim/k3s-ansible/issues
|
||||
|
||||
# A list of file glob-like patterns used to filter any files or directories that should not be included in the build
|
||||
# artifact. A pattern is matched from the relative path of the file or directory of the collection directory. This
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
---
|
||||
k3s_version: v1.36.4+k3s1
|
||||
k3s_version: v1.30.2+k3s2
|
||||
# this is the user that has ssh access to these machines
|
||||
ansible_user: ansibleuser
|
||||
systemd_dir: /etc/systemd/system
|
||||
@@ -7,31 +7,20 @@ systemd_dir: /etc/systemd/system
|
||||
# Set your timezone
|
||||
system_timezone: Your/Timezone
|
||||
|
||||
# k3s recommends swap be disabled on every cluster node. Applied uniformly to all
|
||||
# nodes (all-or-nothing) in the prereq role. Set to false to leave swap enabled.
|
||||
disable_swap: true
|
||||
|
||||
# interface which will be used for flannel
|
||||
# Defaults to each host's default IPv4 interface (e.g. eth0, enp1s0, ens3)
|
||||
# so KVM/cloud hosts without eth0 work out of the box. Override per-host if needed.
|
||||
flannel_iface: "{{ ansible_facts.default_ipv4.interface }}"
|
||||
flannel_iface: eth0
|
||||
|
||||
# uncomment calico_iface to use tigera operator/calico cni instead of flannel https://docs.tigera.io/calico/latest/about
|
||||
# calico_iface: "{{ ansible_facts.default_ipv4.interface }}"
|
||||
# calico_iface: "eth0"
|
||||
calico_ebpf: false # use eBPF dataplane instead of iptables
|
||||
calico_tag: v3.32.1 # calico version tag
|
||||
calico_tag: v3.28.0 # calico version tag
|
||||
|
||||
# uncomment cilium_iface to use cilium cni instead of flannel or calico
|
||||
# ensure v4.19.57, v5.1.16, v5.2.0 or more recent kernel
|
||||
# cilium_iface: "{{ ansible_facts.default_ipv4.interface }}"
|
||||
cilium_mode: native # native when nodes are on the same subnet or use BGP, otherwise set tunnel
|
||||
cilium_tag: v1.20.0 # cilium version tag
|
||||
cilium_cli_tag: v0.19.7 # cilium cli version tag
|
||||
# cilium_iface: "eth0"
|
||||
cilium_mode: native # native when nodes on same subnet or using bgp, else set routed
|
||||
cilium_tag: v1.16.0 # cilium version tag
|
||||
cilium_hubble: true # enable hubble observability relay and ui
|
||||
cilium_envoy: true # enable the Envoy proxy for Cilium L7 policies
|
||||
|
||||
# disable cilium_envoy to skip the Envoy proxy entirely (e.g. no L7 policies)
|
||||
# cilium_envoy: false
|
||||
|
||||
# if using calico or cilium, you may specify the cluster pod cidr pool
|
||||
cluster_cidr: 10.52.0.0/16
|
||||
@@ -48,11 +37,6 @@ cilium_bgp_lb_cidr: 192.168.31.0/24 # cidr for cilium loadbalancer ipam
|
||||
# enable kube-vip ARP broadcasts
|
||||
kube_vip_arp: true
|
||||
|
||||
# (optional) overrides the address kube-vip binds/listens on internally, which
|
||||
# can differ from the announced apiserver_endpoint for complex routing/tunnels.
|
||||
# Defaults to apiserver_endpoint. Also used to derive the kube-vip subnet.
|
||||
# kube_vip_endpoint: 10.66.1.5
|
||||
|
||||
# enable kube-vip BGP peering
|
||||
kube_vip_bgp: false
|
||||
|
||||
@@ -62,9 +46,7 @@ kube_vip_bgp_as: "64513" # Defines the AS for the BGP server
|
||||
kube_vip_bgp_peeraddress: "192.168.30.1" # Defines the address for the BGP peer
|
||||
kube_vip_bgp_peeras: "64512" # Defines the AS for the BGP peer
|
||||
|
||||
# apiserver_endpoint is virtual ip-address which will be configured on each master.
|
||||
# This must be a free, routable IP on your network (not already assigned to a host
|
||||
# or service), and is used by kube-vip / MetalLB to expose the Kubernetes API.
|
||||
# apiserver_endpoint is virtual ip-address which will be configured on each master
|
||||
apiserver_endpoint: 192.168.30.222
|
||||
|
||||
# k3s_token is required masters can talk together securely
|
||||
@@ -102,23 +84,15 @@ extra_agent_args: >-
|
||||
{{ extra_args }}
|
||||
|
||||
# image tag for kube-vip
|
||||
kube_vip_tag_version: v1.2.3
|
||||
|
||||
# enable kube-vip (covers both the control-plane VIP and service load balancing)
|
||||
# set false for a single node or when an external load balancer is used
|
||||
kube_vip_enabled: true
|
||||
kube_vip_tag_version: v0.8.2
|
||||
|
||||
# tag for kube-vip-cloud-provider manifest
|
||||
# kube_vip_cloud_provider_tag_version: "v0.0.12"
|
||||
# kube_vip_cloud_provider_tag_version: "main"
|
||||
|
||||
# kube-vip ip range for load balancer
|
||||
# (uncomment to use kube-vip for services instead of MetalLB)
|
||||
# kube_vip_lb_ip_range: "192.168.30.80-192.168.30.90"
|
||||
|
||||
# enable MetalLB for service load balancing
|
||||
# set false when using an external load balancer
|
||||
metal_lb_enabled: true
|
||||
|
||||
# metallb type frr or native
|
||||
metal_lb_type: native
|
||||
|
||||
@@ -131,18 +105,12 @@ metal_lb_mode: layer2
|
||||
# metal_lb_bgp_peer_address: "192.168.30.1"
|
||||
|
||||
# image tag for metal lb
|
||||
metal_lb_speaker_tag_version: v0.16.0
|
||||
metal_lb_controller_tag_version: v0.16.0
|
||||
metal_lb_speaker_tag_version: v0.14.8
|
||||
metal_lb_controller_tag_version: v0.14.8
|
||||
|
||||
# metallb ip range for load balancer
|
||||
metal_lb_ip_range: 192.168.30.80-192.168.30.90
|
||||
|
||||
# (optional) limit MetalLB layer2 announcements to specific network interfaces.
|
||||
# Leave empty (default) to announce on all interfaces.
|
||||
# metal_lb_interfaces:
|
||||
# - eth1
|
||||
# - eth2
|
||||
|
||||
# Only enable if your nodes are proxmox LXC nodes, make sure to configure your proxmox nodes
|
||||
# in your hosts.ini file.
|
||||
# Please read https://gist.github.com/triangletodd/02f595cd4c0dc9aac5f7763ca2264185 before using this.
|
||||
|
||||
@@ -5,9 +5,14 @@ driver:
|
||||
name: vagrant
|
||||
platforms:
|
||||
- name: control1
|
||||
box: bento/ubuntu-26.04
|
||||
box: generic/ubuntu2204
|
||||
memory: 4096
|
||||
cpus: 4
|
||||
config_options:
|
||||
# We currently can not use public-key based authentication on Ubuntu 22.04,
|
||||
# see: https://github.com/chef/bento/issues/1405
|
||||
ssh.username: vagrant
|
||||
ssh.password: vagrant
|
||||
groups:
|
||||
- k3s_cluster
|
||||
- master
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
---
|
||||
- name: Apply overrides
|
||||
hosts: all
|
||||
serial: 1
|
||||
tasks:
|
||||
- name: Override host variables
|
||||
ansible.builtin.set_fact:
|
||||
# See:
|
||||
# https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
|
||||
calico_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}"
|
||||
kube_vip_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}"
|
||||
calico_iface: eth1
|
||||
|
||||
# The test VMs might be a bit slow, so we give them more time to join the cluster:
|
||||
retry_count: 45
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
---
|
||||
# Durable verify inputs for the calico (Calico CNI + MetalLB) scenario.
|
||||
verify_cni: calico
|
||||
verify_lb: metallb
|
||||
verify_lb_ip_range:
|
||||
- 192.168.30.100-192.168.30.109
|
||||
@@ -5,9 +5,14 @@ driver:
|
||||
name: vagrant
|
||||
platforms:
|
||||
- name: control1
|
||||
box: bento/ubuntu-26.04
|
||||
box: generic/ubuntu2204
|
||||
memory: 4096
|
||||
cpus: 4
|
||||
config_options:
|
||||
# We currently can not use public-key based authentication on Ubuntu 22.04,
|
||||
# see: https://github.com/chef/bento/issues/1405
|
||||
ssh.username: vagrant
|
||||
ssh.password: vagrant
|
||||
groups:
|
||||
- k3s_cluster
|
||||
- master
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
---
|
||||
- name: Apply overrides
|
||||
hosts: all
|
||||
serial: 1
|
||||
tasks:
|
||||
- name: Override host variables
|
||||
ansible.builtin.set_fact:
|
||||
# See:
|
||||
# https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
|
||||
cilium_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}"
|
||||
kube_vip_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}"
|
||||
cilium_iface: eth1
|
||||
|
||||
# The test VMs might be a bit slow, so we give them more time to join the cluster:
|
||||
retry_count: 45
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
---
|
||||
# Durable verify inputs for the cilium (Cilium CNI + MetalLB) scenario.
|
||||
verify_cni: cilium
|
||||
verify_lb: metallb
|
||||
verify_lb_ip_range:
|
||||
- 192.168.30.110-192.168.30.119
|
||||
@@ -1,80 +0,0 @@
|
||||
---
|
||||
- name: Create
|
||||
hosts: localhost
|
||||
connection: local
|
||||
gather_facts: false
|
||||
no_log: "{{ molecule_no_log }}"
|
||||
vars:
|
||||
create_batches:
|
||||
- [control1, control2]
|
||||
- [control3, node1]
|
||||
- [node2]
|
||||
tasks:
|
||||
- name: Verify that bounded batches cover the configured platforms exactly once
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- create_batches | flatten | sort == molecule_yml.platforms | map(attribute='name') | sort
|
||||
- create_batches | flatten | length == create_batches | flatten | unique | length
|
||||
- create_batches | map('length') | max <= 2
|
||||
fail_msg: Bounded create batches do not match the configured default platforms.
|
||||
|
||||
- name: Generate the complete Vagrant configuration # noqa fqcn[action]
|
||||
vagrant:
|
||||
instances: "{{ molecule_yml.platforms }}"
|
||||
default_box: "{{ molecule_yml.driver.default_box | default('generic/alpine316') }}"
|
||||
provider_name: "{{ molecule_yml.driver.provider.name | default(omit, true) }}"
|
||||
provision: "{{ molecule_yml.driver.provision | default(omit) }}"
|
||||
cachier: "{{ molecule_yml.driver.cachier | default(omit) }}"
|
||||
parallel: false
|
||||
state: halt
|
||||
changed_when: false
|
||||
|
||||
- name: Start clean Vagrant guests in bounded batches
|
||||
ansible.builtin.command:
|
||||
argv: >-
|
||||
{{
|
||||
[playbook_dir + '/../../.github/scripts/vagrant-up-timed.sh',
|
||||
molecule_ephemeral_directory] + item
|
||||
}}
|
||||
loop: "{{ create_batches }}"
|
||||
loop_control:
|
||||
label: "{{ item | join(', ') }}"
|
||||
changed_when: true
|
||||
|
||||
- name: Reconcile all instances and collect their connection configuration # noqa fqcn[action]
|
||||
vagrant:
|
||||
instances: "{{ molecule_yml.platforms }}"
|
||||
default_box: "{{ molecule_yml.driver.default_box | default('generic/alpine316') }}"
|
||||
provider_name: "{{ molecule_yml.driver.provider.name | default(omit, true) }}"
|
||||
provision: "{{ molecule_yml.driver.provision | default(omit) }}"
|
||||
cachier: "{{ molecule_yml.driver.cachier | default(omit) }}"
|
||||
parallel: false
|
||||
state: up
|
||||
register: server
|
||||
no_log: false
|
||||
|
||||
- name: Populate instance configuration dictionaries
|
||||
ansible.builtin.set_fact:
|
||||
instance_conf_dict:
|
||||
instance: "{{ item.Host }}"
|
||||
address: "{{ item.HostName }}"
|
||||
user: "{{ item.User }}"
|
||||
port: "{{ item.Port }}"
|
||||
identity_file: "{{ item.IdentityFile }}"
|
||||
loop: "{{ server.results }}"
|
||||
register: instance_config_dict
|
||||
|
||||
- name: Convert instance configuration dictionaries to a list
|
||||
ansible.builtin.set_fact:
|
||||
instance_conf: >-
|
||||
{{
|
||||
instance_config_dict.results
|
||||
| map(attribute='ansible_facts.instance_conf_dict')
|
||||
| list
|
||||
}}
|
||||
|
||||
- name: Write Molecule instance configuration
|
||||
ansible.builtin.copy:
|
||||
content: "{{ instance_conf | to_json | from_json | to_yaml }}"
|
||||
dest: "{{ molecule_instance_config }}"
|
||||
mode: "0600"
|
||||
@@ -3,73 +3,68 @@ dependency:
|
||||
name: galaxy
|
||||
driver:
|
||||
name: vagrant
|
||||
# The Vagrant driver warns that parallel VirtualBox creation can cause
|
||||
# platform issues. Keep this five-node, mixed-distribution scenario serial.
|
||||
parallel: false
|
||||
platforms:
|
||||
- name: control1
|
||||
box: bento/ubuntu-26.04
|
||||
box: generic/ubuntu2204
|
||||
memory: 1024
|
||||
cpus: 2
|
||||
groups:
|
||||
- k3s_cluster
|
||||
- master
|
||||
# Keep adapter 2 stable across linked-clone rebuilds so stale host-only
|
||||
# neighbor state still identifies the current scenario guest.
|
||||
provider_raw_config_args:
|
||||
- "customize ['modifyvm', :id, '--mac-address2', '080027A13038']"
|
||||
interfaces:
|
||||
- network_name: private_network
|
||||
ip: 192.168.30.38
|
||||
config_options:
|
||||
# We currently can not use public-key based authentication on Ubuntu 22.04,
|
||||
# see: https://github.com/chef/bento/issues/1405
|
||||
ssh.username: vagrant
|
||||
ssh.password: vagrant
|
||||
|
||||
- name: control2
|
||||
box: bento/debian-13
|
||||
box: generic/debian12
|
||||
memory: 1024
|
||||
cpus: 2
|
||||
groups:
|
||||
- k3s_cluster
|
||||
- master
|
||||
provider_raw_config_args:
|
||||
- "customize ['modifyvm', :id, '--mac-address2', '080027A13039']"
|
||||
interfaces:
|
||||
- network_name: private_network
|
||||
ip: 192.168.30.39
|
||||
|
||||
- name: control3
|
||||
box: bento/rockylinux-10.1
|
||||
box: generic/rocky9
|
||||
memory: 1024
|
||||
cpus: 2
|
||||
groups:
|
||||
- k3s_cluster
|
||||
- master
|
||||
provider_raw_config_args:
|
||||
- "customize ['modifyvm', :id, '--mac-address2', '080027A13040']"
|
||||
interfaces:
|
||||
- network_name: private_network
|
||||
ip: 192.168.30.40
|
||||
|
||||
- name: node1
|
||||
box: bento/ubuntu-26.04
|
||||
box: generic/ubuntu2204
|
||||
memory: 1024
|
||||
cpus: 2
|
||||
groups:
|
||||
- k3s_cluster
|
||||
- node
|
||||
provider_raw_config_args:
|
||||
- "customize ['modifyvm', :id, '--mac-address2', '080027A13041']"
|
||||
interfaces:
|
||||
- network_name: private_network
|
||||
ip: 192.168.30.41
|
||||
config_options:
|
||||
# We currently can not use public-key based authentication on Ubuntu 22.04,
|
||||
# see: https://github.com/chef/bento/issues/1405
|
||||
ssh.username: vagrant
|
||||
ssh.password: vagrant
|
||||
|
||||
- name: node2
|
||||
box: bento/rockylinux-10.1
|
||||
box: generic/rocky9
|
||||
memory: 1024
|
||||
cpus: 2
|
||||
groups:
|
||||
- k3s_cluster
|
||||
- node
|
||||
provider_raw_config_args:
|
||||
- "customize ['modifyvm', :id, '--mac-address2', '080027A13042']"
|
||||
interfaces:
|
||||
- network_name: private_network
|
||||
ip: 192.168.30.42
|
||||
@@ -79,7 +74,6 @@ provisioner:
|
||||
env:
|
||||
ANSIBLE_VERBOSITY: 1
|
||||
playbooks:
|
||||
create: create.yml
|
||||
converge: ../resources/converge.yml
|
||||
side_effect: ../resources/reset.yml
|
||||
verify: ../resources/verify.yml
|
||||
|
||||
@@ -1,16 +1,12 @@
|
||||
---
|
||||
- name: Apply overrides
|
||||
hosts: all
|
||||
serial: 1
|
||||
tasks:
|
||||
- name: Override host variables
|
||||
ansible.builtin.set_fact:
|
||||
# See:
|
||||
# https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
|
||||
flannel_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}"
|
||||
# kube-vip cannot infer the cluster interface in these multi-NIC
|
||||
# Vagrant guests because the default route is on eth0.
|
||||
kube_vip_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}"
|
||||
flannel_iface: eth1
|
||||
|
||||
# The test VMs might be a bit slow, so we give them more time to join the cluster:
|
||||
retry_count: 45
|
||||
|
||||
@@ -5,137 +5,18 @@
|
||||
|
||||
- name: Network setup
|
||||
hosts: all
|
||||
vars:
|
||||
primary_master: "{{ groups[group_name_master | default('master')][0] }}"
|
||||
primary_cluster_ip: >-
|
||||
{{ hostvars[primary_master].k3s_node_ip | split(',') | first }}
|
||||
cluster_interface: >-
|
||||
{{ cilium_iface | default(calico_iface | default(flannel_iface)) }}
|
||||
primary_cluster_interface: >-
|
||||
{{ hostvars[primary_master].cilium_iface
|
||||
| default(hostvars[primary_master].calico_iface
|
||||
| default(hostvars[primary_master].flannel_iface)) }}
|
||||
primary_cluster_mac: >-
|
||||
{{ hostvars[primary_master].ansible_facts[primary_cluster_interface].macaddress }}
|
||||
tasks:
|
||||
- name: Gather service facts
|
||||
ansible.builtin.service_facts:
|
||||
|
||||
- name: Disable guest firewall services
|
||||
# The disposable test guests use an isolated VirtualBox network. A distro
|
||||
# firewall can allow ICMP while silently blocking the inter-node Kubernetes
|
||||
# API connection, so disable the known guest firewalls consistently.
|
||||
- name: Disable firewalld
|
||||
when: ansible_distribution == "Rocky"
|
||||
# Rocky Linux comes with firewalld enabled. It blocks some of the network
|
||||
# connections needed for our k3s cluster. For our test setup, we just disable
|
||||
# it since the VM host's firewall is still active for connections to and from
|
||||
# the Internet.
|
||||
# When building your own cluster, please DO NOT blindly copy this. Instead,
|
||||
# please create a custom firewall configuration that fits your network design
|
||||
# and security needs.
|
||||
ansible.builtin.systemd:
|
||||
name: "{{ item }}"
|
||||
name: firewalld
|
||||
enabled: false
|
||||
state: stopped
|
||||
become: true
|
||||
loop:
|
||||
- firewalld.service
|
||||
- nftables.service
|
||||
- ufw.service
|
||||
when: item in ansible_facts.services
|
||||
|
||||
- name: Verify the private cluster interface
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- flannel_iface in ansible_facts
|
||||
- ansible_facts[flannel_iface].ipv4 is defined
|
||||
- ansible_facts[flannel_iface].ipv4.address is defined
|
||||
fail_msg: >-
|
||||
The Vagrant private interface {{ flannel_iface }} does not have an
|
||||
IPv4 address on {{ inventory_hostname }}.
|
||||
|
||||
- name: Pin disposable cluster peer neighbor entries
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- ip
|
||||
- neigh
|
||||
- replace
|
||||
- "{{ peer_cluster_ip }}"
|
||||
- lladdr
|
||||
- "{{ peer_cluster_mac }}"
|
||||
- nud
|
||||
- permanent
|
||||
- dev
|
||||
- "{{ cluster_interface }}"
|
||||
become: true
|
||||
changed_when: false
|
||||
loop: "{{ groups['k3s_cluster'] }}"
|
||||
loop_control:
|
||||
label: "{{ inventory_hostname }} -> {{ item }}"
|
||||
vars:
|
||||
peer_cluster_interface: >-
|
||||
{{ hostvars[item].cilium_iface
|
||||
| default(hostvars[item].calico_iface
|
||||
| default(hostvars[item].flannel_iface)) }}
|
||||
peer_cluster_ip: >-
|
||||
{{ hostvars[item].k3s_node_ip | split(',') | first }}
|
||||
peer_cluster_mac: >-
|
||||
{{ hostvars[item].ansible_facts[peer_cluster_interface].macaddress }}
|
||||
when: item != inventory_hostname
|
||||
|
||||
- name: Verify guest-to-guest cluster network reachability
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- ping
|
||||
- -c
|
||||
- "1"
|
||||
- -W
|
||||
- "1"
|
||||
- "{{ primary_cluster_ip }}"
|
||||
register: primary_cluster_ping
|
||||
until: primary_cluster_ping.rc == 0
|
||||
retries: 6
|
||||
delay: 2
|
||||
changed_when: false
|
||||
|
||||
- name: Read the primary neighbor entry
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- ip
|
||||
- neigh
|
||||
- show
|
||||
- to
|
||||
- "{{ primary_cluster_ip }}"
|
||||
- dev
|
||||
- "{{ cluster_interface }}"
|
||||
register: primary_cluster_neighbor
|
||||
changed_when: false
|
||||
when: inventory_hostname != primary_master
|
||||
|
||||
- name: Verify the primary neighbor identity
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- (primary_cluster_mac | lower) in (primary_cluster_neighbor.stdout | lower)
|
||||
fail_msg: >-
|
||||
{{ inventory_hostname }} resolved primary {{ primary_cluster_ip }} to
|
||||
an unexpected MAC on {{ cluster_interface }}. Expected
|
||||
{{ primary_cluster_mac }}, got: {{ primary_cluster_neighbor.stdout }}
|
||||
when: inventory_hostname != primary_master
|
||||
|
||||
- name: Verify GitHub release host DNS
|
||||
ansible.builtin.getent:
|
||||
database: hosts
|
||||
key: github.com
|
||||
register: github_dns
|
||||
retries: 6
|
||||
delay: 5
|
||||
until: github_dns is succeeded
|
||||
|
||||
- name: Verify k3s checksum URL is reachable
|
||||
ansible.builtin.uri:
|
||||
url: >-
|
||||
https://github.com/k3s-io/k3s/releases/download/{{ k3s_version
|
||||
}}/sha256sum-amd64.txt
|
||||
method: HEAD
|
||||
follow_redirects: safe
|
||||
status_code: [200, 302]
|
||||
timeout: 15
|
||||
register: k3s_checksum_request
|
||||
retries: 3
|
||||
delay: 5
|
||||
until: k3s_checksum_request.status in [200, 302]
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
---
|
||||
# Durable verify inputs for the default (flannel + MetalLB) scenario.
|
||||
# These are plain inventory vars linked into the shared Molecule inventory so
|
||||
# the verify play can see them even though the converge play's set_fact values
|
||||
# are not persisted between the two Ansible processes.
|
||||
verify_cni: flannel
|
||||
verify_lb: metallb
|
||||
verify_lb_ip_range:
|
||||
- 192.168.30.80-192.168.30.90
|
||||
@@ -5,7 +5,7 @@ driver:
|
||||
name: vagrant
|
||||
platforms:
|
||||
- name: control1
|
||||
box: bento/ubuntu-26.04
|
||||
box: generic/ubuntu2204
|
||||
memory: 1024
|
||||
cpus: 2
|
||||
groups:
|
||||
@@ -14,9 +14,14 @@ platforms:
|
||||
interfaces:
|
||||
- network_name: private_network
|
||||
ip: fdad:bad:ba55::de:11
|
||||
config_options:
|
||||
# We currently can not use public-key based authentication on Ubuntu 22.04,
|
||||
# see: https://github.com/chef/bento/issues/1405
|
||||
ssh.username: vagrant
|
||||
ssh.password: vagrant
|
||||
|
||||
- name: control2
|
||||
box: bento/ubuntu-26.04
|
||||
box: generic/ubuntu2204
|
||||
memory: 1024
|
||||
cpus: 2
|
||||
groups:
|
||||
@@ -25,9 +30,14 @@ platforms:
|
||||
interfaces:
|
||||
- network_name: private_network
|
||||
ip: fdad:bad:ba55::de:12
|
||||
config_options:
|
||||
# We currently can not use public-key based authentication on Ubuntu 22.04,
|
||||
# see: https://github.com/chef/bento/issues/1405
|
||||
ssh.username: vagrant
|
||||
ssh.password: vagrant
|
||||
|
||||
- name: node1
|
||||
box: bento/ubuntu-26.04
|
||||
box: generic/ubuntu2204
|
||||
memory: 1024
|
||||
cpus: 2
|
||||
groups:
|
||||
@@ -36,6 +46,11 @@ platforms:
|
||||
interfaces:
|
||||
- network_name: private_network
|
||||
ip: fdad:bad:ba55::de:21
|
||||
config_options:
|
||||
# We currently can not use public-key based authentication on Ubuntu 22.04,
|
||||
# see: https://github.com/chef/bento/issues/1405
|
||||
ssh.username: vagrant
|
||||
ssh.password: vagrant
|
||||
provisioner:
|
||||
name: ansible
|
||||
env:
|
||||
|
||||
@@ -1,18 +1,17 @@
|
||||
---
|
||||
- name: Apply overrides
|
||||
hosts: all
|
||||
serial: 1
|
||||
tasks:
|
||||
- name: Override host variables (1/2)
|
||||
ansible.builtin.set_fact:
|
||||
# See:
|
||||
# https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
|
||||
flannel_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}"
|
||||
flannel_iface: eth1
|
||||
|
||||
# In this scenario, we have multiple interfaces that the VIP could be
|
||||
# broadcasted on. Since we have assigned a dedicated private network
|
||||
# here, let's make sure that it is used.
|
||||
kube_vip_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}"
|
||||
kube_vip_iface: eth1
|
||||
|
||||
# The test VMs might be a bit slow, so we give them more time to join the cluster:
|
||||
retry_count: 45
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
---
|
||||
# Durable verify inputs for the ipv6 (flannel CNI + MetalLB) scenario.
|
||||
verify_cni: flannel
|
||||
verify_lb: metallb
|
||||
verify_lb_ip_range:
|
||||
- fdad:bad:ba55::1b:0/112
|
||||
- 192.168.123.80-192.168.123.90
|
||||
@@ -5,9 +5,14 @@ driver:
|
||||
name: vagrant
|
||||
platforms:
|
||||
- name: control1
|
||||
box: bento/ubuntu-26.04
|
||||
box: generic/ubuntu2204
|
||||
memory: 4096
|
||||
cpus: 4
|
||||
config_options:
|
||||
# We currently can not use public-key based authentication on Ubuntu 22.04,
|
||||
# see: https://github.com/chef/bento/issues/1405
|
||||
ssh.username: vagrant
|
||||
ssh.password: vagrant
|
||||
groups:
|
||||
- k3s_cluster
|
||||
- master
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
---
|
||||
- name: Apply overrides
|
||||
hosts: all
|
||||
serial: 1
|
||||
tasks:
|
||||
- name: Override host variables
|
||||
ansible.builtin.set_fact:
|
||||
# See:
|
||||
# https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
|
||||
flannel_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}"
|
||||
kube_vip_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}"
|
||||
flannel_iface: eth1
|
||||
|
||||
# The test VMs might be a bit slow, so we give them more time to join the cluster:
|
||||
retry_count: 45
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
---
|
||||
# Durable verify inputs for the kube-vip (flannel CNI + kube-vip LB) scenario.
|
||||
verify_cni: flannel
|
||||
verify_lb: kube-vip
|
||||
# The kube-vip cloud provider tag is not defined in the linked sample group
|
||||
# vars (its sample entry is commented out), so it is supplied here.
|
||||
verify_kube_vip_cloud_provider_tag: v0.0.12
|
||||
verify_lb_ip_range:
|
||||
- 192.168.30.110-192.168.30.119
|
||||
@@ -1,8 +1,5 @@
|
||||
---
|
||||
- name: Verify
|
||||
hosts: all
|
||||
vars_files:
|
||||
- >-
|
||||
{{ lookup("ansible.builtin.env", "MOLECULE_SCENARIO_DIRECTORY") }}/verify-vars.yml
|
||||
roles:
|
||||
- verify_from_outside
|
||||
|
||||
@@ -7,8 +7,6 @@
|
||||
ansible.builtin.import_tasks: kubecfg-fetch.yml
|
||||
- name: "TEST CASE: Get nodes"
|
||||
ansible.builtin.include_tasks: test/get-nodes.yml
|
||||
- name: "TEST CASE: Verify components"
|
||||
ansible.builtin.include_tasks: test/verify-components.yml
|
||||
- name: "TEST CASE: Deploy example"
|
||||
ansible.builtin.include_tasks: test/deploy-example.yml
|
||||
always:
|
||||
|
||||
@@ -32,95 +32,17 @@
|
||||
metallb_port: spec.ports[0].port
|
||||
register: nginx_services
|
||||
|
||||
- name: Wait for the load balancer address to be assigned
|
||||
ansible.builtin.set_fact:
|
||||
nginx_lb_ip: >-
|
||||
{{
|
||||
nginx_services.resources[0].status.loadBalancer.ingress[0].ip
|
||||
if (nginx_services.resources | length > 0) and
|
||||
(nginx_services.resources[0].status.loadBalancer.ingress is defined) and
|
||||
(nginx_services.resources[0].status.loadBalancer.ingress | length > 0)
|
||||
else ''
|
||||
}}
|
||||
|
||||
- name: Retry until the load balancer service has an external IP
|
||||
block:
|
||||
- name: Refresh nginx service until it has an assigned address
|
||||
kubernetes.core.k8s_info:
|
||||
kind: service
|
||||
name: nginx
|
||||
namespace: "{{ testing_namespace }}"
|
||||
kubeconfig: "{{ kubecfg_path }}"
|
||||
register: nginx_lb_wait
|
||||
until: >-
|
||||
(nginx_lb_wait.resources | length > 0) and
|
||||
(nginx_lb_wait.resources[0].status.loadBalancer.ingress is defined) and
|
||||
(nginx_lb_wait.resources[0].status.loadBalancer.ingress | length > 0)
|
||||
retries: 30
|
||||
delay: 5
|
||||
|
||||
- name: Record the assigned load balancer address
|
||||
ansible.builtin.set_fact:
|
||||
nginx_lb_ip: >-
|
||||
{{ nginx_lb_wait.resources[0].status.loadBalancer.ingress[0].ip }}
|
||||
|
||||
- name: Assert that the nginx welcome page is available
|
||||
ansible.builtin.uri:
|
||||
url: http://{{ nginx_lb_ip | ansible.utils.ipwrap }}:{{ port_ }}/
|
||||
url: http://{{ ip | ansible.utils.ipwrap }}:{{ port_ }}/
|
||||
return_content: true
|
||||
register: result
|
||||
# The MetalLB speaker announces the VIP on the host-only network, and the
|
||||
# runner may briefly not see it in its ARP table yet. Retry so a transient
|
||||
# announcement miss does not fail an otherwise healthy cluster, matching
|
||||
# the load-balancer address wait above and the k3s_server_post metallb waits.
|
||||
until:
|
||||
- result.status is defined
|
||||
- result.status == 200
|
||||
- "'Welcome to nginx!' in result.content"
|
||||
retries: 30
|
||||
delay: 5
|
||||
failed_when: "'Welcome to nginx!' not in result.content"
|
||||
vars:
|
||||
ip: >-
|
||||
{{ nginx_services.resources[0].status.loadBalancer.ingress[0].ip }}
|
||||
port_: >-
|
||||
{{ nginx_services.resources[0].spec.ports[0].port }}
|
||||
|
||||
- name: Initialize load balancer address range check
|
||||
ansible.builtin.set_fact:
|
||||
lb_addr_in_range: false
|
||||
lb_ip_value: "{{ nginx_lb_ip }}"
|
||||
|
||||
- name: Check load balancer address against start-end pools
|
||||
ansible.builtin.set_fact:
|
||||
lb_addr_in_range: true
|
||||
loop: "{{ verify_lb_ip_range }}"
|
||||
loop_control:
|
||||
label: "{{ item }}"
|
||||
when:
|
||||
- "'-' in item"
|
||||
- "'/' not in item"
|
||||
- >-
|
||||
(lb_ip_value | ansible.utils.ipaddr('int') | int) >=
|
||||
(item.split('-')[0] | ansible.utils.ipaddr('int') | int)
|
||||
- >-
|
||||
(lb_ip_value | ansible.utils.ipaddr('int') | int) <=
|
||||
(item.split('-')[1] | ansible.utils.ipaddr('int') | int)
|
||||
|
||||
- name: Check load balancer address against CIDR pools
|
||||
ansible.builtin.set_fact:
|
||||
lb_addr_in_range: true
|
||||
loop: "{{ verify_lb_ip_range }}"
|
||||
loop_control:
|
||||
label: "{{ item }}"
|
||||
when:
|
||||
- "'/' in item"
|
||||
- (lb_ip_value | ansible.utils.ipaddr(item)) is string
|
||||
|
||||
- name: Assert that the load balancer address is within a configured pool
|
||||
ansible.builtin.assert:
|
||||
that: lb_addr_in_range
|
||||
success_msg: "LoadBalancer address {{ lb_ip_value }} is in a configured range"
|
||||
fail_msg: >-
|
||||
LoadBalancer address {{ lb_ip_value }} is not in a configured
|
||||
range {{ verify_lb_ip_range }}
|
||||
# Deactivated linter rules:
|
||||
# - jinja[invalid]: As of version 6.6.0, ansible-lint complains that the input to ipwrap
|
||||
# would be undefined. This will not be the case during playbook execution.
|
||||
|
||||
@@ -1,354 +0,0 @@
|
||||
---
|
||||
# Scenario-aware verification of cluster components and their live image tags.
|
||||
# Scenario identity (verify_cni / verify_lb) and expected address range come
|
||||
# from each scenario's verify-vars.yml, which is plain inventory data available
|
||||
# to the verify play. Converge-time set_fact values are not persisted between
|
||||
# the two Ansible processes, so they are never used here.
|
||||
- name: Verify cluster components report expected versions
|
||||
block:
|
||||
- name: Get all nodes with their kubelet versions
|
||||
kubernetes.core.k8s_info:
|
||||
kind: node
|
||||
kubeconfig: "{{ kubecfg_path }}"
|
||||
register: verify_nodes
|
||||
|
||||
- name: Assert each node reports the expected Kubernetes version
|
||||
ansible.builtin.assert:
|
||||
that: item.status.nodeInfo.kubeletVersion == k3s_version
|
||||
success_msg: "{{ item.metadata.name }} reports {{ k3s_version }}"
|
||||
fail_msg: >-
|
||||
{{ item.metadata.name }} reports
|
||||
{{ item.status.nodeInfo.kubeletVersion }},
|
||||
expected {{ k3s_version }}
|
||||
loop: "{{ verify_nodes.resources }}"
|
||||
loop_control:
|
||||
label: "{{ item.metadata.name }}"
|
||||
|
||||
- name: Verify Flannel is the active CNI
|
||||
when: verify_cni == 'flannel'
|
||||
block:
|
||||
- name: Assert every node reports Ready
|
||||
ansible.builtin.assert:
|
||||
that: item.status.conditions
|
||||
| selectattr('type', 'equalto', 'Ready')
|
||||
| map(attribute='status') | first | default('') == 'True'
|
||||
success_msg: "{{ item.metadata.name }} is Ready"
|
||||
fail_msg: "{{ item.metadata.name }} is not Ready"
|
||||
loop: "{{ verify_nodes.resources }}"
|
||||
loop_control:
|
||||
label: "{{ item.metadata.name }} ready"
|
||||
|
||||
- name: Assert every node registered a node IP from its interface
|
||||
# Each k3s node is launched with --node-ip derived from flannel_iface.
|
||||
# Confirm every node carries a real InternalIP (not a loopback), which
|
||||
# proves k3s bound to the cluster interface rather than defaulting to 127.0.0.1.
|
||||
ansible.builtin.assert:
|
||||
that: >-
|
||||
(node_internal_ips | length) >= 1 and
|
||||
(node_internal_ips | reject('eq', '127.0.0.1') | list | length) == node_internal_ips | length
|
||||
success_msg: "{{ item.metadata.name }} is bound to {{ node_internal_ips | join(', ') }}"
|
||||
fail_msg: >-
|
||||
{{ item.metadata.name }} has no non-loopback InternalIP
|
||||
(got: {{ node_internal_ips | join(', ') }})
|
||||
vars:
|
||||
node_internal_ips: >-
|
||||
{{
|
||||
(item.status.addresses | default([]))
|
||||
| selectattr('type', 'equalto', 'InternalIP')
|
||||
| map(attribute='address')
|
||||
| list
|
||||
}}
|
||||
loop: "{{ verify_nodes.resources }}"
|
||||
loop_control:
|
||||
label: "{{ item.metadata.name }} InternalIP"
|
||||
|
||||
- name: Get any Calico namespaces with Flannel enabled
|
||||
kubernetes.core.k8s_info:
|
||||
kind: Namespace
|
||||
name: calico-system
|
||||
kubeconfig: "{{ kubecfg_path }}"
|
||||
register: flannel_calico_absent
|
||||
|
||||
- name: Assert there is no Calico system namespace
|
||||
ansible.builtin.assert:
|
||||
that: flannel_calico_absent.resources | length == 0
|
||||
success_msg: "No Calico present with Flannel"
|
||||
fail_msg: "A Calico namespace exists alongside Flannel"
|
||||
|
||||
- name: Get the Cilium namespace with Flannel enabled
|
||||
kubernetes.core.k8s_info:
|
||||
kind: Namespace
|
||||
name: cilium
|
||||
kubeconfig: "{{ kubecfg_path }}"
|
||||
register: flannel_cilium
|
||||
|
||||
- name: Assert the Cilium namespace is absent
|
||||
ansible.builtin.assert:
|
||||
that: flannel_cilium.resources | length == 0
|
||||
success_msg: "No Cilium present with Flannel"
|
||||
fail_msg: "A Cilium namespace exists alongside Flannel"
|
||||
|
||||
- name: Verify Calico is the active CNI
|
||||
when: verify_cni == 'calico'
|
||||
block:
|
||||
- name: Get the Calico node DaemonSet image
|
||||
kubernetes.core.k8s_info:
|
||||
kind: DaemonSet
|
||||
name: calico-node
|
||||
namespace: calico-system
|
||||
kubeconfig: "{{ kubecfg_path }}"
|
||||
register: calico_node_ds
|
||||
|
||||
- name: Assert the Calico node image uses the expected tag
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- calico_node_ds.resources | length == 1
|
||||
- calico_node_image | regex_search(':' ~ calico_tag) is not none
|
||||
success_msg: "Calico node image uses tag {{ calico_tag }}"
|
||||
fail_msg: >-
|
||||
Calico node image {{ calico_node_image }},
|
||||
expected {{ calico_tag }}
|
||||
vars:
|
||||
calico_node_image: "{{ calico_node_ds.resources[0].spec.template.spec.containers[0].image }}"
|
||||
|
||||
- name: Get Calico TigeraStatus for calico and apiserver
|
||||
kubernetes.core.k8s_info:
|
||||
api_version: operator.tigera.io/v1
|
||||
kind: TigeraStatus
|
||||
name: "{{ item }}"
|
||||
kubeconfig: "{{ kubecfg_path }}"
|
||||
register: calico_tigerastatus
|
||||
loop:
|
||||
- calico
|
||||
- apiserver
|
||||
loop_control:
|
||||
label: "Tigerastatus/{{ item }}"
|
||||
|
||||
- name: Assert Calico TigeraStatus reports Available
|
||||
ansible.builtin.assert:
|
||||
that: >-
|
||||
item.resources | length == 1 and
|
||||
(item.resources[0].status.conditions
|
||||
| selectattr('type', 'equalto', 'Available')
|
||||
| map(attribute='status') | first | default('')) == 'True'
|
||||
success_msg: "Tigerastatus {{ item.resources[0].metadata.name }} is Available"
|
||||
fail_msg: "Tigerastatus is not Available"
|
||||
loop: "{{ calico_tigerastatus.results }}"
|
||||
loop_control:
|
||||
label: "Tigerastatus Available"
|
||||
|
||||
- name: Get any Flannel DaemonSets with Calico enabled
|
||||
kubernetes.core.k8s_info:
|
||||
kind: DaemonSet
|
||||
namespace: kube-flannel
|
||||
kubeconfig: "{{ kubecfg_path }}"
|
||||
register: no_flannel_ds
|
||||
|
||||
- name: Assert there are no Flannel DaemonSets
|
||||
ansible.builtin.assert:
|
||||
that: no_flannel_ds.resources | length == 0
|
||||
success_msg: "No Flannel DaemonSet present with Calico"
|
||||
fail_msg: "A Flannel DaemonSet exists alongside Calico"
|
||||
|
||||
- name: Verify Cilium is the active CNI
|
||||
when: verify_cni == 'cilium'
|
||||
block:
|
||||
- name: Get the Cilium agent and operator images
|
||||
kubernetes.core.k8s_info:
|
||||
kind: "{{ item.kind }}"
|
||||
name: "{{ item.name }}"
|
||||
namespace: kube-system
|
||||
kubeconfig: "{{ kubecfg_path }}"
|
||||
register: cilium_info
|
||||
loop:
|
||||
- { kind: DaemonSet, name: cilium }
|
||||
- { kind: Deployment, name: cilium-operator }
|
||||
loop_control:
|
||||
label: "{{ item.kind }}/{{ item.name }}"
|
||||
|
||||
- name: Assert Cilium agent and operator use the expected image tag
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- cilium_agent_image | regex_search(':' ~ cilium_tag) is not none
|
||||
- cilium_operator_image | regex_search(':' ~ cilium_tag) is not none
|
||||
success_msg: "Cilium agent and operator use {{ cilium_tag }}"
|
||||
fail_msg: >-
|
||||
Cilium agent {{ cilium_agent_image }},
|
||||
operator {{ cilium_operator_image }},
|
||||
expected {{ cilium_tag }}
|
||||
vars:
|
||||
cilium_agent_image: >-
|
||||
{{ (cilium_info.results
|
||||
| selectattr('resources', 'defined')
|
||||
| map(attribute='resources')
|
||||
| list
|
||||
| map(attribute='0')
|
||||
| selectattr('kind', 'equalto', 'DaemonSet')
|
||||
| list)[0].spec.template.spec.containers[0].image }}
|
||||
cilium_operator_image: >-
|
||||
{{ (cilium_info.results
|
||||
| selectattr('resources', 'defined')
|
||||
| map(attribute='resources')
|
||||
| list
|
||||
| map(attribute='0')
|
||||
| selectattr('kind', 'equalto', 'Deployment')
|
||||
| list)[0].spec.template.spec.containers[0].image }}
|
||||
|
||||
- name: Get Hubble relay and UI deployments when enabled
|
||||
kubernetes.core.k8s_info:
|
||||
kind: Deployment
|
||||
name: "{{ item }}"
|
||||
namespace: kube-system
|
||||
kubeconfig: "{{ kubecfg_path }}"
|
||||
register: hubble_info
|
||||
loop:
|
||||
- hubble-relay
|
||||
- hubble-ui
|
||||
loop_control:
|
||||
label: "Deployment/{{ item }}"
|
||||
when: cilium_hubble | bool
|
||||
|
||||
- name: Assert Hubble components are Ready when enabled
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- item.resources | length == 1
|
||||
- item.resources[0].status.readyReplicas | default(0) >= 1
|
||||
success_msg: "Hubble deployment {{ item.resources[0].metadata.name }} is Ready"
|
||||
fail_msg: "Hubble deployment is not Ready"
|
||||
loop: "{{ hubble_info.results }}"
|
||||
loop_control:
|
||||
label: "Hubble deployment"
|
||||
when: cilium_hubble | bool
|
||||
|
||||
- name: Get any Flannel DaemonSets with Cilium enabled
|
||||
kubernetes.core.k8s_info:
|
||||
kind: DaemonSet
|
||||
namespace: kube-flannel
|
||||
kubeconfig: "{{ kubecfg_path }}"
|
||||
register: no_flannel_ds_cilium
|
||||
|
||||
- name: Assert there are no Flannel DaemonSets
|
||||
ansible.builtin.assert:
|
||||
that: no_flannel_ds_cilium.resources | length == 0
|
||||
success_msg: "No Flannel DaemonSet present with Cilium"
|
||||
fail_msg: "A Flannel DaemonSet exists alongside Cilium"
|
||||
|
||||
- name: Verify MetalLB is the active load balancer
|
||||
when: verify_lb == 'metallb'
|
||||
block:
|
||||
- name: Get the MetalLB controller and speaker images
|
||||
kubernetes.core.k8s_info:
|
||||
kind: "{{ item.kind }}"
|
||||
name: "{{ item.name }}"
|
||||
namespace: metallb-system
|
||||
kubeconfig: "{{ kubecfg_path }}"
|
||||
register: metallb_info
|
||||
until: metallb_info.resources | length > 0
|
||||
retries: 15
|
||||
delay: 10
|
||||
loop:
|
||||
- { kind: Deployment, name: controller }
|
||||
- { kind: DaemonSet, name: speaker }
|
||||
loop_control:
|
||||
label: "{{ item.kind }}/{{ item.name }}"
|
||||
|
||||
- name: Fail with a clear message if MetalLB resources are missing
|
||||
ansible.builtin.fail:
|
||||
msg: >-
|
||||
Did not find {{ item.kind | lower }} {{ item.name }} in
|
||||
metallb-system. Expected MetalLB to be deployed in this
|
||||
scenario (verify_lb: {{ verify_lb }}).
|
||||
when: item.resources | length == 0
|
||||
loop: "{{ metallb_info.results }}"
|
||||
loop_control:
|
||||
label: "{{ item.item.kind }}/{{ item.item.name }}"
|
||||
|
||||
- name: Assert MetalLB controller and speaker use the expected image tags
|
||||
ansible.builtin.assert:
|
||||
# regex_search returns a string or none; check for a match with `is not
|
||||
# none` so the assertion is a real boolean (ansible-core 2.19 rejects
|
||||
# string conditionals and `| bool` deprecates string coercion).
|
||||
that:
|
||||
- controller_image | regex_search(metal_lb_controller_tag_version) is not none
|
||||
- speaker_image | regex_search(metal_lb_speaker_tag_version) is not none
|
||||
success_msg: >-
|
||||
MetalLB controller {{ metal_lb_controller_tag_version }},
|
||||
speaker {{ metal_lb_speaker_tag_version }}
|
||||
fail_msg: >-
|
||||
MetalLB controller {{ controller_image }},
|
||||
speaker {{ speaker_image }}
|
||||
vars:
|
||||
controller_image: >-
|
||||
{{ (metallb_info.results
|
||||
| selectattr('resources', 'defined')
|
||||
| map(attribute='resources')
|
||||
| list
|
||||
| map(attribute='0')
|
||||
| selectattr('kind', 'equalto', 'Deployment')
|
||||
| list)[0].spec.template.spec.containers[0].image }}
|
||||
speaker_image: >-
|
||||
{{ (metallb_info.results
|
||||
| selectattr('resources', 'defined')
|
||||
| map(attribute='resources')
|
||||
| list
|
||||
| map(attribute='0')
|
||||
| selectattr('kind', 'equalto', 'DaemonSet')
|
||||
| list)[0].spec.template.spec.containers[0].image }}
|
||||
|
||||
- name: Verify kube-vip is the active load balancer
|
||||
when: verify_lb == 'kube-vip'
|
||||
block:
|
||||
- name: Get the kube-vip and cloud provider images
|
||||
kubernetes.core.k8s_info:
|
||||
kind: "{{ item.kind }}"
|
||||
name: "{{ item.name }}"
|
||||
namespace: kube-system
|
||||
kubeconfig: "{{ kubecfg_path }}"
|
||||
register: kubevip_info
|
||||
loop:
|
||||
- { kind: DaemonSet, name: kube-vip-ds }
|
||||
- { kind: Deployment, name: kube-vip-cloud-provider }
|
||||
loop_control:
|
||||
label: "{{ item.kind }}/{{ item.name }}"
|
||||
|
||||
- name: Assert the kube-vip and cloud provider image tags
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- kubevip_image | regex_search(':' ~ kube_vip_tag_version) is not none
|
||||
- cloud_provider_image | regex_search(verify_kube_vip_cloud_provider_tag) is not none
|
||||
success_msg: >-
|
||||
kube-vip {{ kube_vip_tag_version }},
|
||||
cloud provider {{ verify_kube_vip_cloud_provider_tag }}
|
||||
fail_msg: >-
|
||||
kube-vip {{ kubevip_image }},
|
||||
cloud provider {{ cloud_provider_image }}
|
||||
vars:
|
||||
kubevip_image: >-
|
||||
{{ (kubevip_info.results
|
||||
| selectattr('resources', 'defined')
|
||||
| map(attribute='resources')
|
||||
| list
|
||||
| map(attribute='0')
|
||||
| selectattr('kind', 'equalto', 'DaemonSet')
|
||||
| list)[0].spec.template.spec.containers[0].image }}
|
||||
cloud_provider_image: >-
|
||||
{{ (kubevip_info.results
|
||||
| selectattr('resources', 'defined')
|
||||
| map(attribute='resources')
|
||||
| list
|
||||
| map(attribute='0')
|
||||
| selectattr('kind', 'equalto', 'Deployment')
|
||||
| list)[0].spec.template.spec.containers[0].image }}
|
||||
|
||||
- name: Get the MetalLB namespace with kube-vip enabled
|
||||
kubernetes.core.k8s_info:
|
||||
kind: Namespace
|
||||
name: metallb-system
|
||||
kubeconfig: "{{ kubecfg_path }}"
|
||||
register: metallb_absent
|
||||
|
||||
- name: Assert the MetalLB namespace does not exist
|
||||
ansible.builtin.assert:
|
||||
that: metallb_absent.resources | length == 0
|
||||
success_msg: "MetalLB is not installed with kube-vip"
|
||||
fail_msg: "MetalLB namespace exists alongside kube-vip"
|
||||
@@ -5,9 +5,14 @@ driver:
|
||||
name: vagrant
|
||||
platforms:
|
||||
- name: control1
|
||||
box: bento/ubuntu-26.04
|
||||
box: generic/ubuntu2204
|
||||
memory: 4096
|
||||
cpus: 4
|
||||
config_options:
|
||||
# We currently can not use public-key based authentication on Ubuntu 22.04,
|
||||
# see: https://github.com/chef/bento/issues/1405
|
||||
ssh.username: vagrant
|
||||
ssh.password: vagrant
|
||||
groups:
|
||||
- k3s_cluster
|
||||
- master
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
---
|
||||
- name: Apply overrides
|
||||
hosts: all
|
||||
serial: 1
|
||||
tasks:
|
||||
- name: Override host variables
|
||||
ansible.builtin.set_fact:
|
||||
# See:
|
||||
# https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
|
||||
flannel_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}"
|
||||
kube_vip_iface: "{{ 'eth1' if 'eth1' in ansible_facts.interfaces else 'enp0s8' }}"
|
||||
flannel_iface: eth1
|
||||
|
||||
# The test VMs might be a bit slow, so we give them more time to join the cluster:
|
||||
retry_count: 45
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
---
|
||||
# Durable verify inputs for the single_node (flannel + MetalLB) scenario.
|
||||
verify_cni: flannel
|
||||
verify_lb: metallb
|
||||
verify_lb_ip_range:
|
||||
- 192.168.30.91-192.168.30.99
|
||||
+1
-21
@@ -2,29 +2,9 @@
|
||||
- name: Reboot k3s_cluster
|
||||
hosts: k3s_cluster
|
||||
gather_facts: true
|
||||
|
||||
# Stagger the reboot across the cluster when concurrent_reboots is set.
|
||||
# Defaults to '100%' so the whole cluster reboots at once (backward compatible).
|
||||
serial: "{{ concurrent_reboots | default('100%') }}"
|
||||
|
||||
tasks:
|
||||
- name: >-
|
||||
{{
|
||||
'Reboot all nodes at once'
|
||||
if (concurrent_reboots is not defined)
|
||||
else 'Reboot nodes with concurrency of ' ~ concurrent_reboots
|
||||
}}
|
||||
- name: Reboot the nodes (and Wait upto 5 mins max)
|
||||
become: true
|
||||
ansible.builtin.reboot:
|
||||
reboot_command: "{{ custom_reboot_command | default(omit) }}"
|
||||
reboot_timeout: 300
|
||||
test_command: >-
|
||||
{{ 'kubectl get nodes' if 'master' in group_names else 'whoami' }}
|
||||
|
||||
- name: Optional wait before rebooting the next batch of nodes
|
||||
ansible.builtin.pause:
|
||||
seconds: "{{ wait_seconds_after_reboot | int }}"
|
||||
when: >-
|
||||
concurrent_reboots is defined and
|
||||
wait_seconds_after_reboot is defined and
|
||||
wait_seconds_after_reboot | int > 0
|
||||
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
ansible-core>=2.19.13
|
||||
jmespath>=1.1.0
|
||||
ansible-core>=2.16.2
|
||||
jmespath>=1.0.1
|
||||
jsonpatch>=1.33
|
||||
kubernetes>=29.0.0
|
||||
molecule-plugins[vagrant]
|
||||
|
||||
+5
-5
@@ -2,11 +2,11 @@
|
||||
# This file is autogenerated by pip-compile with Python 3.11
|
||||
# by the following command:
|
||||
#
|
||||
# pip-compile --output-file=requirements.txt requirements.in
|
||||
# pip-compile requirements.in
|
||||
#
|
||||
ansible-compat==4.1.11
|
||||
# via molecule
|
||||
ansible-core==2.19.13
|
||||
ansible-core==2.18.0
|
||||
# via
|
||||
# -r requirements.in
|
||||
# ansible-compat
|
||||
@@ -53,7 +53,7 @@ jinja2==3.1.3
|
||||
# via
|
||||
# ansible-core
|
||||
# molecule
|
||||
jmespath==1.1.0
|
||||
jmespath==1.0.1
|
||||
# via -r requirements.in
|
||||
jsonpatch==1.33
|
||||
# via -r requirements.in
|
||||
@@ -77,7 +77,7 @@ molecule==6.0.3
|
||||
# via
|
||||
# -r requirements.in
|
||||
# molecule-plugins
|
||||
molecule-plugins[vagrant]==23.6.0
|
||||
molecule-plugins[vagrant]==23.5.3
|
||||
# via -r requirements.in
|
||||
netaddr==0.10.1
|
||||
# via -r requirements.in
|
||||
@@ -146,7 +146,7 @@ rsa==4.9
|
||||
# via google-auth
|
||||
ruamel-yaml==0.18.5
|
||||
# via pre-commit-hooks
|
||||
ruamel-yaml-clib==0.2.15
|
||||
ruamel-yaml-clib==0.2.8
|
||||
# via ruamel-yaml
|
||||
six==1.16.0
|
||||
# via
|
||||
|
||||
@@ -7,10 +7,6 @@
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
register: k3s_download_x64
|
||||
retries: 5
|
||||
delay: 10
|
||||
until: k3s_download_x64 is succeeded
|
||||
when: ansible_facts.architecture == "x86_64"
|
||||
|
||||
- name: Download k3s binary arm64
|
||||
@@ -21,10 +17,6 @@
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
register: k3s_download_arm64
|
||||
retries: 5
|
||||
delay: 10
|
||||
until: k3s_download_arm64 is succeeded
|
||||
when:
|
||||
- ( ansible_facts.architecture is search("arm") and ansible_facts.userspace_bits == "64" )
|
||||
or ansible_facts.architecture is search("aarch64")
|
||||
@@ -37,10 +29,6 @@
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
register: k3s_download_armhf
|
||||
retries: 5
|
||||
delay: 10
|
||||
until: k3s_download_armhf is succeeded
|
||||
when:
|
||||
- ansible_facts.architecture is search("arm")
|
||||
- ansible_facts.userspace_bits == "32"
|
||||
|
||||
@@ -11,7 +11,7 @@ ExecStartPre=-/sbin/modprobe overlay
|
||||
ExecStart=/usr/local/bin/k3s agent \
|
||||
--server https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443 \
|
||||
{% if is_pxe_booted | default(false) %}--snapshotter native \
|
||||
{% endif %}--token {{ hostvars[groups[group_name_master | default('master')][0]]['token'] | default(k3s_token) | replace( '%', '%%' ) }} \
|
||||
{% endif %}--token {{ hostvars[groups[group_name_master | default('master')][0]]['token'] | default(k3s_token) }} \
|
||||
{{ extra_agent_args }}
|
||||
KillMode=process
|
||||
Delegate=yes
|
||||
|
||||
@@ -7,9 +7,8 @@ group_name_master: master
|
||||
|
||||
kube_vip_arp: true
|
||||
kube_vip_iface:
|
||||
kube_vip_endpoint:
|
||||
kube_vip_cloud_provider_tag_version: v0.0.12
|
||||
kube_vip_tag_version: v1.2.3
|
||||
kube_vip_cloud_provider_tag_version: main
|
||||
kube_vip_tag_version: v0.7.2
|
||||
|
||||
kube_vip_bgp: false
|
||||
kube_vip_bgp_routerid: 127.0.0.1
|
||||
@@ -20,19 +19,10 @@ kube_vip_bgp_peeras: "64512"
|
||||
kube_vip_bgp_peers: []
|
||||
kube_vip_bgp_peers_groups: ['k3s_master']
|
||||
|
||||
kube_vip_enabled: true
|
||||
|
||||
metal_lb_enabled: true
|
||||
metal_lb_controller_tag_version: v0.16.0
|
||||
metal_lb_speaker_tag_version: v0.16.0
|
||||
metal_lb_controller_tag_version: v0.14.3
|
||||
metal_lb_speaker_tag_version: v0.14.3
|
||||
metal_lb_type: native
|
||||
|
||||
# Shared retry/delay for remote manifest and asset downloads. The CI runner's
|
||||
# resolver intermittently times out on GitHub-hosted domains (helm.cilium.io,
|
||||
# raw.githubusercontent.com, github.com), so retry transient DNS/network failures.
|
||||
download_retries: 5
|
||||
download_delay: 10
|
||||
|
||||
retry_count: 20
|
||||
|
||||
# yamllint disable rule:line-length
|
||||
@@ -43,7 +33,7 @@ server_init_args: >-
|
||||
{% else %}
|
||||
--server https://{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(",") | first | ansible.utils.ipwrap }}:6443
|
||||
{% endif %}
|
||||
--token {{ k3s_token | replace('%', '%%') }}
|
||||
--token {{ k3s_token }}
|
||||
{% endif %}
|
||||
{{ extra_server_args }}
|
||||
|
||||
|
||||
@@ -78,49 +78,25 @@ argument_specs:
|
||||
- automatically at runtime.
|
||||
default: ~
|
||||
|
||||
kube_vip_endpoint:
|
||||
description:
|
||||
- Overrides the address kube-vip binds/listens on internally, which
|
||||
- can differ from the announced apiserver_endpoint for complex
|
||||
- routing and site-to-site tunnels.
|
||||
- Defaults to apiserver_endpoint and is used to derive the kube-vip
|
||||
- subnet.
|
||||
default: ~
|
||||
|
||||
kube_vip_tag_version:
|
||||
description: Image tag for kube-vip
|
||||
default: v1.2.3
|
||||
default: v0.7.2
|
||||
|
||||
kube_vip_cloud_provider_tag_version:
|
||||
description: Tag for kube-vip-cloud-provider manifest when enabled
|
||||
default: v0.0.12
|
||||
|
||||
kube_vip_enabled:
|
||||
description:
|
||||
- Enable installing kube-vip.
|
||||
- Covers both the control-plane VIP and the service load balancer.
|
||||
- Set false to skip kube-vip entirely, for example on a single node or when an external load balancer is used.
|
||||
default: true
|
||||
type: bool
|
||||
default: main
|
||||
|
||||
kube_vip_lb_ip_range:
|
||||
description: IP range for kube-vip load balancer
|
||||
default: ~
|
||||
|
||||
metal_lb_enabled:
|
||||
description:
|
||||
- Enable installing MetalLB for service load balancing.
|
||||
- Set false to skip MetalLB, for example when an external load balancer is used.
|
||||
default: true
|
||||
type: bool
|
||||
|
||||
metal_lb_controller_tag_version:
|
||||
description: Image tag for MetalLB
|
||||
default: v0.16.0
|
||||
default: v0.14.3
|
||||
|
||||
metal_lb_speaker_tag_version:
|
||||
description: Image tag for MetalLB
|
||||
default: v0.16.0
|
||||
default: v0.14.3
|
||||
|
||||
metal_lb_type:
|
||||
choices:
|
||||
|
||||
@@ -4,110 +4,10 @@
|
||||
|
||||
- name: Fetch k3s-init.service logs
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
timeout --signal=TERM --kill-after=5s 30s journalctl --no-pager
|
||||
--unit=k3s-init.service --since=-30min --lines=5000
|
||||
cmd: journalctl --all --unit=k3s-init.service
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
no_log: true
|
||||
register: k3s_init_log
|
||||
|
||||
- name: Fetch k3s-init.service status
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
timeout --signal=TERM --kill-after=5s 15s systemctl status
|
||||
k3s-init.service --no-pager --full
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
no_log: true
|
||||
register: k3s_init_status
|
||||
|
||||
- name: Fetch IP address state
|
||||
ansible.builtin.command:
|
||||
cmd: ip -br address
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
no_log: true
|
||||
register: k3s_ip_address_state
|
||||
|
||||
- name: Fetch IP link state
|
||||
ansible.builtin.command:
|
||||
cmd: ip -br link
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
no_log: true
|
||||
register: k3s_ip_link_state
|
||||
|
||||
- name: Fetch IP route state
|
||||
ansible.builtin.command:
|
||||
cmd: ip route show
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
no_log: true
|
||||
register: k3s_ip_route_state
|
||||
|
||||
- name: Fetch IP neighbor state
|
||||
ansible.builtin.command:
|
||||
cmd: ip neigh show
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
no_log: true
|
||||
register: k3s_ip_neighbor_state
|
||||
|
||||
- name: Fetch IP rule state
|
||||
ansible.builtin.command:
|
||||
cmd: ip rule show
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
no_log: true
|
||||
register: k3s_ip_rule_state
|
||||
|
||||
- name: Fetch listening TCP sockets
|
||||
ansible.builtin.command:
|
||||
cmd: ss -ltn
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
no_log: true
|
||||
register: k3s_tcp_listener_state
|
||||
|
||||
- name: Fetch guest firewall service state
|
||||
ansible.builtin.service_facts:
|
||||
no_log: true
|
||||
register: k3s_service_facts
|
||||
|
||||
- name: Fetch input firewall rules
|
||||
ansible.builtin.command:
|
||||
cmd: iptables -S INPUT
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
no_log: true
|
||||
register: k3s_input_firewall_rules
|
||||
|
||||
- name: Ping the primary Kubernetes API address from {{ ansible_hostname }}
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- ping
|
||||
- -c
|
||||
- "1"
|
||||
- -W
|
||||
- "1"
|
||||
- "{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(',') | first }}"
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
no_log: true
|
||||
register: k3s_primary_api_ping
|
||||
|
||||
- name: Probe primary Kubernetes API from {{ ansible_hostname }}
|
||||
ansible.builtin.wait_for:
|
||||
host: "{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(',') | first }}"
|
||||
port: 6443
|
||||
connect_timeout: 1
|
||||
timeout: 2
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
no_log: true
|
||||
register: k3s_primary_api_probe
|
||||
|
||||
- name: Create {{ log_destination }}
|
||||
delegate_to: localhost
|
||||
run_once: true
|
||||
@@ -125,56 +25,4 @@
|
||||
dest: "{{ log_destination }}/k3s-init@{{ ansible_hostname }}.log"
|
||||
mode: "0644"
|
||||
vars:
|
||||
content: |
|
||||
=== k3s-init.service status ===
|
||||
{{ k3s_init_status.stdout | regex_replace('--token(?:=| +)[^ ]+', '--token ***') }}
|
||||
|
||||
=== k3s-init.service journal ===
|
||||
{{ k3s_init_log.stdout | regex_replace('--token(?:=| +)[^ ]+', '--token ***') }}
|
||||
|
||||
- name: Store network diagnostics to {{ log_destination }}
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
ansible.builtin.template:
|
||||
src: content.j2
|
||||
dest: "{{ log_destination }}/network@{{ ansible_hostname }}.log"
|
||||
mode: "0644"
|
||||
vars:
|
||||
content: |
|
||||
=== ip -br address ===
|
||||
{{ k3s_ip_address_state.stdout }}
|
||||
|
||||
=== ip -br link ===
|
||||
{{ k3s_ip_link_state.stdout }}
|
||||
|
||||
=== ip route show ===
|
||||
{{ k3s_ip_route_state.stdout }}
|
||||
|
||||
=== ip neigh show ===
|
||||
{{ k3s_ip_neighbor_state.stdout }}
|
||||
|
||||
=== ip rule show ===
|
||||
{{ k3s_ip_rule_state.stdout }}
|
||||
|
||||
=== ss -ltn ===
|
||||
{{ k3s_tcp_listener_state.stdout }}
|
||||
|
||||
=== guest firewall services ===
|
||||
firewalld={{ k3s_service_facts.ansible_facts.services.get('firewalld.service', {}).get('state', 'not-found') }}
|
||||
nftables={{ k3s_service_facts.ansible_facts.services.get('nftables.service', {}).get('state', 'not-found') }}
|
||||
ufw={{ k3s_service_facts.ansible_facts.services.get('ufw.service', {}).get('state', 'not-found') }}
|
||||
|
||||
=== iptables -S INPUT ===
|
||||
rc={{ k3s_input_firewall_rules.rc }}
|
||||
{{ k3s_input_firewall_rules.stdout }}
|
||||
{{ k3s_input_firewall_rules.stderr }}
|
||||
|
||||
=== primary API ping ===
|
||||
rc={{ k3s_primary_api_ping.rc }}
|
||||
{{ k3s_primary_api_ping.stdout }}
|
||||
{{ k3s_primary_api_ping.stderr }}
|
||||
|
||||
=== primary API probe ===
|
||||
failed={{ k3s_primary_api_probe.failed | default(false) }}
|
||||
elapsed={{ k3s_primary_api_probe.elapsed | default('unknown') }}
|
||||
msg={{ k3s_primary_api_probe.msg | default('connected') }}
|
||||
content: "{{ k3s_init_log.stdout }}"
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
---
|
||||
- name: Verify primary Kubernetes API reachability from {{ joining_master }}
|
||||
ansible.builtin.wait_for:
|
||||
host: >-
|
||||
{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip
|
||||
| split(',') | first }}
|
||||
port: 6443
|
||||
connect_timeout: 2
|
||||
timeout: 30
|
||||
delegate_to: "{{ joining_master }}"
|
||||
|
||||
- name: Join transient k3s-init service for {{ joining_master }}
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
systemd-run -p RestartSec=2 -p Restart=on-failure
|
||||
-p Delegate=yes -p TasksMax=infinity -p KillMode=process
|
||||
-p LimitNOFILE=1048576 -p LimitNPROC=infinity -p LimitCORE=infinity
|
||||
--unit=k3s-init
|
||||
k3s server {{ hostvars[joining_master].k3s_server_init_args }}
|
||||
creates: "{{ systemd_dir }}/k3s-init.service"
|
||||
delegate_to: "{{ joining_master }}"
|
||||
no_log: true
|
||||
|
||||
- name: Wait for primary registration of {{ joining_master }}
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
{{ k3s_kubectl_binary | default('k3s kubectl') }} get node
|
||||
{{ hostvars[joining_master].ansible_hostname }}
|
||||
delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
|
||||
register: joined_master
|
||||
until: joined_master.rc == 0
|
||||
retries: "{{ retry_count | default(20) }}"
|
||||
delay: 2
|
||||
changed_when: false
|
||||
@@ -10,15 +10,11 @@
|
||||
|
||||
- name: Download vip cloud provider manifest to first master
|
||||
ansible.builtin.get_url:
|
||||
url: https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/{{ kube_vip_cloud_provider_tag_version | default('v0.0.12') }}/manifest/kube-vip-cloud-controller.yaml # noqa yaml[line-length]
|
||||
url: https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/{{ kube_vip_cloud_provider_tag_version | default('main') }}/manifest/kube-vip-cloud-controller.yaml # noqa yaml[line-length]
|
||||
dest: /var/lib/rancher/k3s/server/manifests/kube-vip-cloud-controller.yaml
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
register: kube_vip_manifest_download
|
||||
retries: "{{ download_retries }}"
|
||||
delay: "{{ download_delay }}"
|
||||
until: kube_vip_manifest_download is succeeded
|
||||
when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
|
||||
|
||||
- name: Copy kubevip configMap manifest to first master
|
||||
|
||||
@@ -24,110 +24,36 @@
|
||||
|
||||
- name: Deploy vip manifest
|
||||
ansible.builtin.include_tasks: vip.yml
|
||||
when: kube_vip_enabled
|
||||
- name: Deploy metallb manifest
|
||||
ansible.builtin.include_tasks: metallb.yml
|
||||
tags: metallb
|
||||
# Deploy MetalLB unless explicitly disabled, kube-vip owns the load balancer
|
||||
# IP range, or Cilium BGP is enabled (Cilium then provides its own load
|
||||
# balancing). The cilium_bgp default keeps this safe when Cilium variables are
|
||||
# not in scope at all (#644) while still deploying MetalLB when a non-BGP
|
||||
# Cilium CNI is in use.
|
||||
when: metal_lb_enabled and kube_vip_lb_ip_range is not defined and not (cilium_bgp | default(false) | bool)
|
||||
when: kube_vip_lb_ip_range is not defined and (not cilium_bgp or cilium_iface is not defined)
|
||||
|
||||
- name: Deploy kube-vip manifest
|
||||
ansible.builtin.include_tasks: kube-vip.yml
|
||||
tags: kubevip
|
||||
when: kube_vip_enabled and kube_vip_lb_ip_range is defined
|
||||
when: kube_vip_lb_ip_range is defined
|
||||
|
||||
- name: Initialize and verify the K3s control plane
|
||||
any_errors_fatal: true
|
||||
- name: Init cluster inside the transient k3s-init service
|
||||
ansible.builtin.command:
|
||||
cmd: systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server {{ server_init_args }}
|
||||
creates: "{{ systemd_dir }}/k3s-init.service"
|
||||
|
||||
- name: Verification
|
||||
when: not ansible_check_mode
|
||||
block:
|
||||
- name: Materialize per-host server initialization arguments
|
||||
ansible.builtin.set_fact:
|
||||
k3s_server_init_args: "{{ server_init_args }}"
|
||||
no_log: true
|
||||
when: groups[group_name_master | default('master')] | length > 1
|
||||
|
||||
- name: Orchestrate control-plane initialization from the first master
|
||||
when: inventory_hostname == groups[group_name_master | default('master')][0]
|
||||
block:
|
||||
- name: Init the first master inside the transient k3s-init service
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
systemd-run -p RestartSec=2 -p Restart=on-failure
|
||||
-p Delegate=yes -p TasksMax=infinity -p KillMode=process
|
||||
-p LimitNOFILE=1048576 -p LimitNPROC=infinity -p LimitCORE=infinity
|
||||
--unit=k3s-init k3s server {{ server_init_args }}
|
||||
creates: "{{ systemd_dir }}/k3s-init.service"
|
||||
no_log: true
|
||||
|
||||
- name: Wait for the first master Kubernetes API
|
||||
ansible.builtin.command:
|
||||
cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get --raw=/readyz"
|
||||
register: first_master_api
|
||||
until: first_master_api.rc == 0
|
||||
retries: "{{ retry_count | default(20) }}"
|
||||
delay: 2
|
||||
changed_when: false
|
||||
|
||||
- name: Verify the first master API listener on its node address
|
||||
ansible.builtin.wait_for:
|
||||
host: >-
|
||||
{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip
|
||||
| split(',') | first }}
|
||||
port: 6443
|
||||
connect_timeout: 2
|
||||
timeout: "{{ retry_count | default(20) * 2 }}"
|
||||
|
||||
- name: Join additional masters one at a time
|
||||
ansible.builtin.include_tasks: join_master.yml
|
||||
loop: "{{ groups[group_name_master | default('master')][1:] }}"
|
||||
loop_control:
|
||||
loop_var: joining_master
|
||||
when: groups[group_name_master | default('master')] | length > 1
|
||||
|
||||
- name: Verify that all expected control-plane nodes registered
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
{{ k3s_kubectl_binary | default('k3s kubectl') }} get nodes
|
||||
-o=jsonpath='{.items[*].metadata.name}'
|
||||
register: nodes
|
||||
until:
|
||||
- nodes.rc == 0
|
||||
- expected_control_plane_names | difference(nodes.stdout.split()) | length == 0
|
||||
retries: "{{ retry_count | default(20) }}"
|
||||
delay: 2
|
||||
changed_when: false
|
||||
vars:
|
||||
expected_control_plane_names: >-
|
||||
{{
|
||||
groups[group_name_master | default('master')]
|
||||
| map('extract', hostvars, 'ansible_hostname')
|
||||
| list
|
||||
}}
|
||||
|
||||
- name: Wait for K3s bootstrap CRDs to become established
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
{{ k3s_kubectl_binary | default('k3s kubectl') }} wait
|
||||
--for=condition=Established --timeout=5s
|
||||
crd/addons.k3s.cattle.io
|
||||
crd/helmcharts.helm.cattle.io
|
||||
crd/helmchartconfigs.helm.cattle.io
|
||||
register: bootstrap_crds
|
||||
until: bootstrap_crds.rc == 0
|
||||
retries: "{{ retry_count | default(20) }}"
|
||||
delay: 2
|
||||
changed_when: false
|
||||
- name: Verify that all nodes actually joined (check k3s-init.service if this fails)
|
||||
ansible.builtin.command:
|
||||
cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get nodes -l 'node-role.kubernetes.io/master=true' -o=jsonpath='{.items[*].metadata.name}'" # yamllint disable-line rule:line-length
|
||||
register: nodes
|
||||
until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups[group_name_master | default('master')] | length) # yamllint disable-line rule:line-length
|
||||
retries: "{{ retry_count | default(20) }}"
|
||||
delay: 10
|
||||
changed_when: false
|
||||
always:
|
||||
- name: Save logs of k3s-init.service
|
||||
ansible.builtin.include_tasks: fetch_k3s_init_logs.yml
|
||||
# ANSIBLE_K3S_LOG_DIR is a path string when set; evaluate it as a boolean
|
||||
# so the conditional is a real boolean (ansible-core 2.19 rejects string
|
||||
# conditionals derived from env vars).
|
||||
when: log_destination | default('') != ''
|
||||
when: log_destination
|
||||
vars:
|
||||
log_destination: >-
|
||||
{{ lookup('ansible.builtin.env', 'ANSIBLE_K3S_LOG_DIR', default=False) }}
|
||||
|
||||
@@ -15,10 +15,6 @@
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
register: metallb_manifest_download
|
||||
retries: "{{ download_retries }}"
|
||||
delay: "{{ download_delay }}"
|
||||
until: metallb_manifest_download is succeeded
|
||||
when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
|
||||
|
||||
- name: Set image versions in manifest for metallb-{{ metal_lb_type }}
|
||||
@@ -32,29 +28,3 @@
|
||||
loop_control:
|
||||
label: "{{ item.change }} => {{ item.to }}"
|
||||
when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
|
||||
|
||||
- name: Read back MetalLB manifest from first master
|
||||
ansible.builtin.slurp:
|
||||
src: /var/lib/rancher/k3s/server/manifests/metallb-crds.yaml
|
||||
register: metallb_manifest
|
||||
when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
|
||||
|
||||
- name: Check the MetalLB speaker image reference is present
|
||||
ansible.builtin.set_fact:
|
||||
metallb_manifest_has_speaker_tag: >-
|
||||
{{
|
||||
('quay.io/metallb/speaker:' ~ metal_lb_speaker_tag_version) in
|
||||
(metallb_manifest.content | default('') | b64decode)
|
||||
}}
|
||||
when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
|
||||
|
||||
- name: Fail if MetalLB speaker tag was not applied to the manifest
|
||||
ansible.builtin.fail:
|
||||
msg: >-
|
||||
MetalLB speaker image reference
|
||||
quay.io/metallb/speaker:{{ metal_lb_speaker_tag_version }}
|
||||
was not found in the downloaded manifest. The upstream image
|
||||
reference may have changed.
|
||||
when:
|
||||
- ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
|
||||
- not metallb_manifest_has_speaker_tag
|
||||
|
||||
@@ -12,9 +12,9 @@
|
||||
mode: "0644"
|
||||
when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
|
||||
|
||||
- name: Copy kube-vip RBAC manifest to first master
|
||||
ansible.builtin.template:
|
||||
src: kube-vip-rbac.yaml.j2
|
||||
- name: Download vip rbac manifest to first master
|
||||
ansible.builtin.get_url:
|
||||
url: https://kube-vip.io/manifests/rbac.yaml
|
||||
dest: /var/lib/rancher/k3s/server/manifests/vip-rbac.yaml
|
||||
owner: root
|
||||
group: root
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
# Repository-owned kube-vip RBAC derived from the official v1.2.2
|
||||
# `kube-vip manifest rbac` output (run with `--inCluster`).
|
||||
# Source: https://github.com/kube-vip/kube-vip/blob/v1.2.2/pkg/kubevip/config_generator.go
|
||||
# Permissions are kept exactly as broad as the upstream manifest.
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: kube-vip
|
||||
namespace: kube-system
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
annotations:
|
||||
rbac.authorization.kubernetes.io/autoupdate: "true"
|
||||
name: system:kube-vip-role
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["services/status"]
|
||||
verbs: ["update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["services", "endpoints"]
|
||||
verbs: ["list", "get", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["list", "get", "watch", "update", "patch"]
|
||||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources: ["leases"]
|
||||
verbs: ["list", "get", "watch", "update", "create"]
|
||||
- apiGroups: ["discovery.k8s.io"]
|
||||
resources: ["endpointslices"]
|
||||
verbs: ["list", "get", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["list"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: system:kube-vip-binding
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: system:kube-vip-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: kube-vip
|
||||
namespace: kube-system
|
||||
@@ -36,8 +36,8 @@ spec:
|
||||
- name: vip_interface
|
||||
value: {{ kube_vip_iface }}
|
||||
{% endif %}
|
||||
- name: vip_subnet
|
||||
value: "{{ (kube_vip_endpoint | default(apiserver_endpoint, true)) | ansible.utils.ipsubnet | ansible.utils.ipaddr('prefix') }}"
|
||||
- name: vip_cidr
|
||||
value: "{{ apiserver_endpoint | ansible.utils.ipsubnet | ansible.utils.ipaddr('prefix') }}"
|
||||
- name: cp_enable
|
||||
value: "true"
|
||||
- name: cp_namespace
|
||||
@@ -55,15 +55,15 @@ spec:
|
||||
- name: vip_retryperiod
|
||||
value: "2"
|
||||
- name: address
|
||||
value: {{ kube_vip_endpoint | default(apiserver_endpoint, true) }}
|
||||
value: {{ apiserver_endpoint }}
|
||||
{% if kube_vip_bgp | default(false) | bool %}
|
||||
{% if kube_vip_bgp_routerid is defined %}
|
||||
- name: bgp_routerid
|
||||
value: "{{ kube_vip_bgp_routerid }}"
|
||||
{% endif %}
|
||||
{% if _kube_vip_bgp_peers | length > 0 %}
|
||||
- name: bgp_peers
|
||||
value: "{{ _kube_vip_bgp_peers | map(attribute='peer_address') | zip(_kube_vip_bgp_peers | map(attribute='peer_asn')) | map('join', ':') | join(',') }}" # yamllint disable-line rule:line-length
|
||||
- name: bgppeers
|
||||
value: "{{ _kube_vip_bgp_peers | map(attribute='peer_address') | zip(_kube_vip_bgp_peers| map(attribute='peer_asn')) | map('join', ',') | join(':') }}" # yamllint disable-line rule:line-length
|
||||
{% else %}
|
||||
{% if kube_vip_bgp_as is defined %}
|
||||
- name: bgp_as
|
||||
|
||||
@@ -9,7 +9,7 @@ calico_ebpf: false
|
||||
calico_encapsulation: VXLANCrossSubnet
|
||||
calico_natOutgoing: Enabled # noqa var-naming
|
||||
calico_nodeSelector: all() # noqa var-naming
|
||||
calico_tag: v3.32.1
|
||||
calico_tag: v3.27.2
|
||||
|
||||
cilium_bgp: false
|
||||
cilium_exportPodCIDR: true # noqa var-naming
|
||||
@@ -18,18 +18,8 @@ cilium_bgp_peer_asn: 64512
|
||||
cilium_bgp_neighbors: []
|
||||
cilium_bgp_neighbors_groups: ['k3s_all']
|
||||
cilium_bgp_lb_cidr: 192.168.31.0/24
|
||||
cilium_envoy: true
|
||||
cilium_hubble: true
|
||||
cilium_mode: native
|
||||
cilium_tag: v1.20.0
|
||||
cilium_cli_tag: v0.19.7
|
||||
|
||||
# Shared retry/delay for remote manifest, asset downloads, and waiting on
|
||||
# Kubernetes resources. The CI runner's resolver intermittently times out on
|
||||
# GitHub-hosted domains and the kube API can transiently return
|
||||
# ServiceUnavailable, so retry transient DNS/network/API failures.
|
||||
download_retries: 5
|
||||
download_delay: 10
|
||||
|
||||
cluster_cidr: 10.52.0.0/16
|
||||
enable_bpf_masquerade: true
|
||||
@@ -38,7 +28,6 @@ group_name_master: master
|
||||
|
||||
metal_lb_mode: layer2
|
||||
metal_lb_available_timeout: 240s
|
||||
metal_lb_controller_tag_version: v0.16.0
|
||||
metal_lb_enabled: true
|
||||
metal_lb_interfaces: []
|
||||
metal_lb_controller_tag_version: v0.14.3
|
||||
metal_lb_ip_range: 192.168.30.80-192.168.30.90
|
||||
metal_lb_interfaces:
|
||||
|
||||
@@ -43,7 +43,7 @@ argument_specs:
|
||||
|
||||
calico_tag:
|
||||
description: Calico version tag
|
||||
default: v3.32.1
|
||||
default: v3.27.2
|
||||
|
||||
cilium_bgp:
|
||||
description:
|
||||
@@ -93,24 +93,12 @@ argument_specs:
|
||||
default: ~
|
||||
|
||||
cilium_mode:
|
||||
description:
|
||||
- Inner-node communication mode.
|
||||
- Cilium accepts `native` or `tunnel`. `routed` is accepted as a
|
||||
legacy alias and is mapped to `tunnel` at install time.
|
||||
description: Inner-node communication mode
|
||||
default: native
|
||||
choices:
|
||||
- native
|
||||
- tunnel
|
||||
- routed
|
||||
|
||||
cilium_tag:
|
||||
description: Cilium version tag
|
||||
default: v1.20.0
|
||||
|
||||
cilium_cli_tag:
|
||||
description: Cilium CLI version tag
|
||||
default: v0.19.7
|
||||
|
||||
cluster_cidr:
|
||||
description: Inner-cluster IP range
|
||||
default: 10.52.0.0/16
|
||||
@@ -133,13 +121,6 @@ argument_specs:
|
||||
description: IP range for kube-vip load balancer
|
||||
default: ~
|
||||
|
||||
metal_lb_enabled:
|
||||
description:
|
||||
- Enable installing the MetalLB pool for service load balancing.
|
||||
- Set false to skip MetalLB, for example when an external load balancer is used.
|
||||
default: true
|
||||
type: bool
|
||||
|
||||
metal_lb_available_timeout:
|
||||
description: Wait for MetalLB resources
|
||||
default: 240s
|
||||
@@ -148,17 +129,9 @@ argument_specs:
|
||||
description: MetalLB ip range for load balancer
|
||||
default: 192.168.30.80-192.168.30.90
|
||||
|
||||
metal_lb_interfaces:
|
||||
description: >-
|
||||
List of network interfaces on which MetalLB should announce the
|
||||
load balancer IPs in layer2 mode. When empty (default), MetalLB
|
||||
announces on all interfaces.
|
||||
type: list
|
||||
default: []
|
||||
|
||||
metal_lb_controller_tag_version:
|
||||
description: Image tag for MetalLB
|
||||
default: v0.16.0
|
||||
default: v0.14.3
|
||||
|
||||
metal_lb_mode:
|
||||
description: Metallb mode
|
||||
|
||||
@@ -11,18 +11,6 @@
|
||||
group: root
|
||||
mode: "0755"
|
||||
|
||||
- name: "Download to first master: Calico CRD bundle for {{ calico_tag }}"
|
||||
ansible.builtin.get_url:
|
||||
url: https://raw.githubusercontent.com/projectcalico/calico/{{ calico_tag }}/manifests/v1_crd_projectcalico_org.yaml # noqa yaml[line-length]
|
||||
dest: /tmp/k3s/v1_crd_projectcalico_org.yaml
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
register: calico_crd_download
|
||||
retries: "{{ download_retries }}"
|
||||
delay: "{{ download_delay }}"
|
||||
until: calico_crd_download is succeeded
|
||||
|
||||
- name: "Download to first master: manifest for Tigera Operator and Calico CRDs"
|
||||
ansible.builtin.get_url:
|
||||
url: https://raw.githubusercontent.com/projectcalico/calico/{{ calico_tag }}/manifests/tigera-operator.yaml
|
||||
@@ -30,52 +18,6 @@
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
register: tigera_operator_download
|
||||
retries: "{{ download_retries }}"
|
||||
delay: "{{ download_delay }}"
|
||||
until: tigera_operator_download is succeeded
|
||||
|
||||
- name: Apply Calico CRD bundle with server-side apply
|
||||
ansible.builtin.command: >-
|
||||
{{ k3s_kubectl_binary | default('k3s kubectl') }}
|
||||
apply --server-side --force-conflicts
|
||||
-f /tmp/k3s/v1_crd_projectcalico_org.yaml
|
||||
register: apply_crds
|
||||
changed_when: >-
|
||||
'created' in apply_crds.stdout or
|
||||
'configured' in apply_crds.stdout or
|
||||
'applied' in apply_crds.stdout
|
||||
failed_when: apply_crds.rc != 0
|
||||
|
||||
- name: Apply Tigera Operator manifest idempotently with server-side apply
|
||||
ansible.builtin.command: >-
|
||||
{{ k3s_kubectl_binary | default('k3s kubectl') }}
|
||||
apply --server-side --force-conflicts
|
||||
-f /tmp/k3s/tigera-operator.yaml
|
||||
register: apply_operator
|
||||
changed_when: >-
|
||||
'created' in apply_operator.stdout or
|
||||
'configured' in apply_operator.stdout or
|
||||
'applied' in apply_operator.stdout
|
||||
failed_when: apply_operator.rc != 0
|
||||
|
||||
- name: Wait for Tigera Operator and managed CRDs to become available
|
||||
ansible.builtin.command: >-
|
||||
{{ k3s_kubectl_binary | default('k3s kubectl') }} wait {{ item.type }}/{{ item.name }}
|
||||
{% if item.namespace is defined %}--namespace='{{ item.namespace }}'{% endif %}
|
||||
--for=condition={{ item.condition }}
|
||||
--timeout=30s
|
||||
register: tigera_result
|
||||
changed_when: false
|
||||
until: tigera_result is succeeded
|
||||
retries: 7
|
||||
delay: 7
|
||||
with_items:
|
||||
- { name: tigera-operator, type: deployment, namespace: tigera-operator, condition: Available=True }
|
||||
- { name: installations.operator.tigera.io, type: crd, condition: Established }
|
||||
- { name: apiservers.operator.tigera.io, type: crd, condition: Established }
|
||||
loop_control:
|
||||
label: "{{ item.type }}/{{ item.name }}"
|
||||
|
||||
- name: Copy Calico custom resources manifest to first master
|
||||
ansible.builtin.template:
|
||||
@@ -85,16 +27,53 @@
|
||||
group: root
|
||||
mode: "0755"
|
||||
|
||||
- name: Apply Calico custom resources
|
||||
- name: Deploy or replace Tigera Operator
|
||||
block:
|
||||
- name: Deploy Tigera Operator
|
||||
ansible.builtin.command:
|
||||
cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} create -f /tmp/k3s/tigera-operator.yaml"
|
||||
register: create_operator
|
||||
changed_when: "'created' in create_operator.stdout"
|
||||
failed_when: "'Error' in create_operator.stderr and 'already exists' not in create_operator.stderr"
|
||||
rescue:
|
||||
- name: Replace existing Tigera Operator
|
||||
ansible.builtin.command:
|
||||
cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} replace -f /tmp/k3s/tigera-operator.yaml"
|
||||
register: replace_operator
|
||||
changed_when: "'replaced' in replace_operator.stdout"
|
||||
failed_when: "'Error' in replace_operator.stderr"
|
||||
|
||||
- name: Wait for Tigera Operator resources
|
||||
ansible.builtin.command: >-
|
||||
{{ k3s_kubectl_binary | default('k3s kubectl') }}
|
||||
apply -f /tmp/k3s/custom-resources.yaml
|
||||
register: apply_cr
|
||||
changed_when: >-
|
||||
'configured' in apply_cr.stdout or
|
||||
'created' in apply_cr.stdout or
|
||||
'unchanged' in apply_cr.stdout
|
||||
failed_when: apply_cr.rc != 0
|
||||
{{ k3s_kubectl_binary | default('k3s kubectl') }} wait {{ item.type }}/{{ item.name }}
|
||||
--namespace='tigera-operator'
|
||||
--for=condition=Available=True
|
||||
--timeout=30s
|
||||
register: tigera_result
|
||||
changed_when: false
|
||||
until: tigera_result is succeeded
|
||||
retries: 7
|
||||
delay: 7
|
||||
with_items:
|
||||
- { name: tigera-operator, type: deployment }
|
||||
loop_control:
|
||||
label: "{{ item.type }}/{{ item.name }}"
|
||||
|
||||
- name: Deploy Calico custom resources
|
||||
block:
|
||||
- name: Deploy custom resources for Calico
|
||||
ansible.builtin.command:
|
||||
cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} create -f /tmp/k3s/custom-resources.yaml"
|
||||
register: create_cr
|
||||
changed_when: "'created' in create_cr.stdout"
|
||||
failed_when: "'Error' in create_cr.stderr and 'already exists' not in create_cr.stderr"
|
||||
rescue:
|
||||
- name: Apply new Calico custom resource manifest
|
||||
ansible.builtin.command:
|
||||
cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} apply -f /tmp/k3s/custom-resources.yaml"
|
||||
register: apply_cr
|
||||
changed_when: "'configured' in apply_cr.stdout or 'created' in apply_cr.stdout"
|
||||
failed_when: "'Error' in apply_cr.stderr"
|
||||
|
||||
- name: Wait for Calico system resources to be available
|
||||
ansible.builtin.command: >-
|
||||
@@ -112,8 +91,8 @@
|
||||
register: cr_result
|
||||
changed_when: false
|
||||
until: cr_result is succeeded
|
||||
retries: 60
|
||||
delay: 10
|
||||
retries: 30
|
||||
delay: 7
|
||||
with_items:
|
||||
- { name: calico-typha, type: deployment, namespace: calico-system }
|
||||
- { name: calico-kube-controllers, type: deployment, namespace: calico-system }
|
||||
@@ -125,40 +104,10 @@
|
||||
type: daemonset
|
||||
selector: k8s-app=calico-node
|
||||
namespace: calico-system
|
||||
- { name: calico-apiserver, type: deployment, namespace: calico-apiserver }
|
||||
loop_control:
|
||||
label: "{{ item.type }}/{{ item.name }}"
|
||||
|
||||
# The Calico API server is an optional add-on for managing Calico policy via
|
||||
# kubectl (the projectcalico.org/v3 Kubernetes API). It is not required for
|
||||
# Calico CNI data plane operation and does not reconcile on K3s with recent
|
||||
# Calico releases, so it is treated as best-effort here.
|
||||
- name: Wait for the optional Calico API server (best effort)
|
||||
ansible.builtin.command: >-
|
||||
{{ k3s_kubectl_binary | default('k3s kubectl') }} wait deployment/calico-apiserver
|
||||
--namespace=calico-apiserver --for=condition=Available --timeout=30s
|
||||
register: cr_apiserver
|
||||
changed_when: false
|
||||
until: cr_apiserver is succeeded
|
||||
retries: 30
|
||||
delay: 10
|
||||
ignore_errors: true
|
||||
|
||||
- name: Verify Calico TigeraStatus reports Available
|
||||
ansible.builtin.command: >-
|
||||
{{ k3s_kubectl_binary | default('k3s kubectl') }}
|
||||
get tigerastatus {{ item }}
|
||||
-o jsonpath='{.status.conditions[?(@.type=="Available")].status}'
|
||||
register: tigera_status
|
||||
changed_when: false
|
||||
until: tigera_status.stdout | trim == 'True'
|
||||
retries: 30
|
||||
delay: 7
|
||||
failed_when: tigera_status.rc != 0 or (tigera_status.stdout | trim) != 'True'
|
||||
with_items:
|
||||
- calico
|
||||
loop_control:
|
||||
label: "tigerastatus/{{ item }}"
|
||||
|
||||
- name: Patch Felix configuration for eBPF mode
|
||||
ansible.builtin.command:
|
||||
cmd: >
|
||||
|
||||
@@ -30,13 +30,26 @@
|
||||
}}
|
||||
when: cilium_cli_installed.rc == 0
|
||||
|
||||
- name: Get latest stable Cilium CLI version file
|
||||
ansible.builtin.get_url:
|
||||
url: https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt
|
||||
dest: /tmp/k3s/cilium-cli-stable.txt
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
|
||||
- name: Read Cilium CLI stable version from file
|
||||
ansible.builtin.command: cat /tmp/k3s/cilium-cli-stable.txt
|
||||
register: cli_ver
|
||||
changed_when: false
|
||||
|
||||
- name: Log installed Cilium CLI version
|
||||
ansible.builtin.debug:
|
||||
msg: "Installed Cilium CLI version: {{ installed_cli_version | default('Not installed') }}"
|
||||
|
||||
- name: Log pinned Cilium CLI version
|
||||
- name: Log latest stable Cilium CLI version
|
||||
ansible.builtin.debug:
|
||||
msg: "Pinned Cilium CLI version: {{ cilium_cli_tag }}"
|
||||
msg: "Latest Cilium CLI version: {{ cli_ver.stdout }}"
|
||||
|
||||
- name: Determine if Cilium CLI needs installation or update
|
||||
ansible.builtin.set_fact:
|
||||
@@ -44,7 +57,7 @@
|
||||
{{
|
||||
cilium_cli_installed.rc != 0 or
|
||||
(cilium_cli_installed.rc == 0 and
|
||||
installed_cli_version != cilium_cli_tag)
|
||||
installed_cli_version != cli_ver.stdout)
|
||||
}}
|
||||
|
||||
- name: Install or update Cilium CLI
|
||||
@@ -65,11 +78,7 @@
|
||||
- .tar.gz
|
||||
- .tar.gz.sha256sum
|
||||
vars:
|
||||
cilium_base_url: https://github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_tag }}
|
||||
register: cilium_cli_download
|
||||
retries: "{{ download_retries }}"
|
||||
delay: "{{ download_delay }}"
|
||||
until: cilium_cli_download is succeeded
|
||||
cilium_base_url: https://github.com/cilium/cilium-cli/releases/download/{{ cli_ver.stdout }}
|
||||
|
||||
- name: Verify the downloaded tarball
|
||||
ansible.builtin.shell: |
|
||||
@@ -147,13 +156,6 @@
|
||||
Target Cilium version: {{ cilium_tag }},
|
||||
Update needed: {{ cilium_needs_update }}
|
||||
|
||||
- name: Map the legacy routed mode to Cilium tunnel mode
|
||||
ansible.builtin.set_fact:
|
||||
# Cilium 1.20 accepts `native` or `tunnel`. `routed` was the name used
|
||||
# by older releases and is kept only as a backward-compatible alias.
|
||||
cilium_routing_mode: >-
|
||||
{{ 'tunnel' if cilium_mode == 'routed' else cilium_mode }}
|
||||
|
||||
- name: Install Cilium
|
||||
ansible.builtin.command: >-
|
||||
{% if cilium_installed.rc != 0 %}
|
||||
@@ -165,35 +167,27 @@
|
||||
--helm-set operator.replicas="1"
|
||||
{{ '--helm-set devices=' + cilium_iface if cilium_iface != 'auto' else '' }}
|
||||
--helm-set ipam.operator.clusterPoolIPv4PodCIDRList={{ cluster_cidr }}
|
||||
{% if cilium_routing_mode == "native" or (cilium_bgp and cilium_exportPodCIDR != 'false') %}
|
||||
{% if cilium_mode == "native" or (cilium_bgp and cilium_exportPodCIDR != 'false') %}
|
||||
--helm-set ipv4NativeRoutingCIDR={{ cluster_cidr }}
|
||||
{% endif %}
|
||||
--helm-set k8sServiceHost="127.0.0.1"
|
||||
--helm-set k8sServicePort="6444"
|
||||
--helm-set routingMode={{ cilium_routing_mode }}
|
||||
--helm-set autoDirectNodeRoutes={{ "true" if cilium_routing_mode == "native" else "false" }}
|
||||
--helm-set routingMode={{ cilium_mode }}
|
||||
--helm-set autoDirectNodeRoutes={{ "true" if cilium_mode == "native" else "false" }}
|
||||
--helm-set kubeProxyReplacement={{ kube_proxy_replacement }}
|
||||
--helm-set bpf.masquerade={{ enable_bpf_masquerade }}
|
||||
--helm-set bgpControlPlane.enabled={{ cilium_bgp | default("false") }}
|
||||
--helm-set hubble.enabled={{ "true" if cilium_hubble else "false" }}
|
||||
--helm-set hubble.relay.enabled={{ "true" if cilium_hubble else "false" }}
|
||||
--helm-set hubble.ui.enabled={{ "true" if cilium_hubble else "false" }}
|
||||
--helm-set envoy.enabled={{ "true" if cilium_envoy else "false" }}
|
||||
{% if kube_proxy_replacement is not false %}
|
||||
--helm-set loadBalancer.algorithm={{ bpf_lb_algorithm }}
|
||||
--helm-set loadBalancer.mode={{ bpf_lb_mode }}
|
||||
--helm-set bpf.loadBalancer.algorithm={{ bpf_lb_algorithm }}
|
||||
--helm-set bpf.loadBalancer.mode={{ bpf_lb_mode }}
|
||||
{% endif %}
|
||||
environment:
|
||||
KUBECONFIG: "{{ ansible_user_dir }}/.kube/config"
|
||||
register: cilium_install_result
|
||||
changed_when: cilium_install_result.rc == 0
|
||||
# cilium install/upgrade fetches the Helm chart from helm.cilium.io, which is
|
||||
# fronted by GitHub Pages and can transiently fail DNS resolution through the
|
||||
# host resolver (intermittent "lookup helm.cilium.io ... i/o timeout"). Retry
|
||||
# so a transient name/network failure does not abort the whole converge play.
|
||||
until: cilium_install_result.rc == 0
|
||||
retries: "{{ download_retries }}"
|
||||
delay: "{{ download_delay }}"
|
||||
when: cilium_installed.rc != 0 or cilium_needs_update
|
||||
|
||||
- name: Wait for Cilium resources
|
||||
@@ -224,16 +218,6 @@
|
||||
when: >-
|
||||
not item.check_hubble | default(false) or (item.check_hubble | default(false) and cilium_hubble)
|
||||
|
||||
- name: Wait for Cilium status to be healthy
|
||||
ansible.builtin.command: cilium status --wait
|
||||
environment:
|
||||
KUBECONFIG: "{{ ansible_user_dir }}/.kube/config"
|
||||
register: cilium_status
|
||||
changed_when: false
|
||||
until: cilium_status.rc == 0
|
||||
retries: 30
|
||||
delay: 7
|
||||
|
||||
- name: Configure Cilium BGP
|
||||
when: cilium_bgp
|
||||
block:
|
||||
@@ -249,38 +233,23 @@
|
||||
group: root
|
||||
mode: "0755"
|
||||
|
||||
- name: Preflight validate rendered BGP manifests against installed CRDs
|
||||
ansible.builtin.command: >-
|
||||
{{ k3s_kubectl_binary | default('k3s kubectl') }}
|
||||
apply --dry-run=server -f /tmp/k3s/cilium-bgp.yaml
|
||||
register: preflight_cr
|
||||
changed_when: false
|
||||
failed_when: preflight_cr.rc != 0
|
||||
|
||||
- name: Apply BGP manifests
|
||||
ansible.builtin.command: >-
|
||||
{{ k3s_kubectl_binary | default('k3s kubectl') }}
|
||||
apply -f /tmp/k3s/cilium-bgp.yaml
|
||||
ansible.builtin.command:
|
||||
cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} apply -f /tmp/k3s/cilium-bgp.yaml"
|
||||
register: apply_cr
|
||||
changed_when: "'configured' in apply_cr.stdout or 'created' in apply_cr.stdout"
|
||||
failed_when: apply_cr.rc != 0
|
||||
failed_when: "'is invalid' in apply_cr.stderr"
|
||||
ignore_errors: true
|
||||
|
||||
- name: Remove deprecated CiliumBGPPeeringPolicy after v2 resources are accepted
|
||||
ansible.builtin.command: >-
|
||||
{{ k3s_kubectl_binary | default('k3s kubectl') }}
|
||||
delete CiliumBGPPeeringPolicy.cilium.io 01-bgp-peering-policy
|
||||
register: delete_old_policy
|
||||
changed_when: "'deleted' in delete_old_policy.stdout"
|
||||
# The policy (and possibly its CRD) may already be absent; this is
|
||||
# intentionally tolerated, not a command whose failure must be hidden.
|
||||
failed_when: false
|
||||
- name: Print error message if BGP manifests application fails
|
||||
ansible.builtin.debug:
|
||||
msg: "{{ apply_cr.stderr }}"
|
||||
when: "'is invalid' in apply_cr.stderr"
|
||||
|
||||
- name: Test for BGP config resources
|
||||
ansible.builtin.command: "{{ item }}"
|
||||
loop:
|
||||
- "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPClusterConfig.cilium.io"
|
||||
- "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPPeerConfig.cilium.io"
|
||||
- "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPAdvertisement.cilium.io"
|
||||
- "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPPeeringPolicy.cilium.io"
|
||||
- "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumLoadBalancerIPPool.cilium.io"
|
||||
changed_when: false
|
||||
loop_control:
|
||||
|
||||
@@ -12,12 +12,7 @@
|
||||
- name: Deploy metallb pool
|
||||
ansible.builtin.include_tasks: metallb.yml
|
||||
tags: metallb
|
||||
# Deploy MetalLB unless explicitly disabled, kube-vip owns the load balancer
|
||||
# IP range, or Cilium BGP is enabled (Cilium then provides its own load
|
||||
# balancing). The cilium_bgp default keeps this safe when Cilium variables are
|
||||
# not in scope at all (#644) while still deploying MetalLB when a non-BGP
|
||||
# Cilium CNI is in use.
|
||||
when: metal_lb_enabled and kube_vip_lb_ip_range is not defined and not (cilium_bgp | default(false) | bool)
|
||||
when: kube_vip_lb_ip_range is not defined and (not cilium_bgp or cilium_iface is not defined)
|
||||
|
||||
- name: Remove tmp directory used for manifests
|
||||
ansible.builtin.file:
|
||||
|
||||
@@ -40,14 +40,8 @@
|
||||
|
||||
- name: Test metallb-system namespace
|
||||
ansible.builtin.command: >-
|
||||
{{ k3s_kubectl_binary | default('k3s kubectl') }} get namespace metallb-system
|
||||
{{ k3s_kubectl_binary | default('k3s kubectl') }} -n metallb-system
|
||||
changed_when: false
|
||||
# The kube API can briefly return ServiceUnavailable while MetalLB converges,
|
||||
# which would otherwise abort the whole converge play on a transient error.
|
||||
register: metallb_namespace_result
|
||||
until: metallb_namespace_result.rc == 0
|
||||
retries: "{{ download_retries }}"
|
||||
delay: "{{ download_delay }}"
|
||||
with_items: "{{ groups[group_name_master | default('master')] }}"
|
||||
run_once: true
|
||||
|
||||
@@ -60,12 +54,6 @@
|
||||
{% if item.condition | default(False) -%}{{ item.condition }}{%- endif %}
|
||||
--timeout='{{ metal_lb_available_timeout }}'
|
||||
changed_when: false
|
||||
# The kube API can briefly return ServiceUnavailable while MetalLB converges,
|
||||
# which would otherwise abort the whole converge play on a transient error.
|
||||
register: metallb_wait_result
|
||||
until: metallb_wait_result.rc == 0
|
||||
retries: "{{ download_retries }}"
|
||||
delay: "{{ download_delay }}"
|
||||
run_once: true
|
||||
with_items:
|
||||
- description: controller
|
||||
@@ -97,20 +85,22 @@
|
||||
|
||||
- name: Set metallb webhook service name
|
||||
ansible.builtin.set_fact:
|
||||
# Inspected the v0.16.0 manifest: the newer webhook Service name is used
|
||||
# by every supported MetalLB release, so the old pre-0.14.4 branch is gone.
|
||||
metallb_webhook_service_name: metallb-webhook-service
|
||||
metallb_webhook_service_name: >-
|
||||
{{
|
||||
(
|
||||
(metal_lb_controller_tag_version | regex_replace('^v', ''))
|
||||
is
|
||||
version('0.14.4', '<', version_type='semver')
|
||||
) | ternary(
|
||||
'webhook-service',
|
||||
'metallb-webhook-service'
|
||||
)
|
||||
}}
|
||||
|
||||
- name: Test metallb-system webhook-service endpoint
|
||||
ansible.builtin.command: >-
|
||||
{{ k3s_kubectl_binary | default('k3s kubectl') }} -n metallb-system get endpoints {{ metallb_webhook_service_name }}
|
||||
changed_when: false
|
||||
# The kube API can briefly return ServiceUnavailable while MetalLB converges,
|
||||
# which would otherwise abort the whole converge play on a transient error.
|
||||
register: metallb_webhook_result
|
||||
until: metallb_webhook_result.rc == 0
|
||||
retries: "{{ download_retries }}"
|
||||
delay: "{{ download_delay }}"
|
||||
with_items: "{{ groups[group_name_master | default('master')] }}"
|
||||
run_once: true
|
||||
|
||||
@@ -130,10 +120,6 @@
|
||||
changed_when: false
|
||||
run_once: true
|
||||
when: metal_lb_mode == "layer2"
|
||||
register: metallb_l2_test_result
|
||||
until: metallb_l2_test_result.rc == 0
|
||||
retries: "{{ download_retries }}"
|
||||
delay: "{{ download_delay }}"
|
||||
with_items:
|
||||
- IPAddressPool
|
||||
- L2Advertisement
|
||||
@@ -144,10 +130,6 @@
|
||||
changed_when: false
|
||||
run_once: true
|
||||
when: metal_lb_mode == "bgp"
|
||||
register: metallb_bgp_test_result
|
||||
until: metallb_bgp_test_result.rc == 0
|
||||
retries: "{{ download_retries }}"
|
||||
delay: "{{ download_delay }}"
|
||||
with_items:
|
||||
- IPAddressPool
|
||||
- BGPPeer
|
||||
|
||||
@@ -1,84 +1,40 @@
|
||||
# Cilium BGP Control Plane v2 resources.
|
||||
# Replace the deprecated v2alpha1 CiliumBGPPeeringPolicy that was removed
|
||||
# in Cilium 1.19.
|
||||
{% set _cilium_default_peer = {'peer_address': cilium_bgp_peer_address, 'peer_asn': cilium_bgp_peer_asn} %}
|
||||
{% set _cilium_peers = _cilium_bgp_neighbors if _cilium_bgp_neighbors | length > 0 else [_cilium_default_peer] %}
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumBGPPeerConfig
|
||||
apiVersion: "cilium.io/v2alpha1"
|
||||
kind: CiliumBGPPeeringPolicy
|
||||
metadata:
|
||||
name: cilium-peer
|
||||
spec:
|
||||
# Matches the timers and multihop used by the previous v2alpha1 policy.
|
||||
ebgpMultihop: 10
|
||||
timers:
|
||||
connectRetryTimeSeconds: 120
|
||||
holdTimeSeconds: 90
|
||||
keepAliveTimeSeconds: 30
|
||||
gracefulRestart:
|
||||
enabled: true
|
||||
restartTimeSeconds: 120
|
||||
families:
|
||||
- afi: ipv4
|
||||
safi: unicast
|
||||
advertisements:
|
||||
matchLabels:
|
||||
advertise: "bgp"
|
||||
---
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumBGPClusterConfig
|
||||
metadata:
|
||||
name: cilium-bgp
|
||||
spec:
|
||||
# Explicitly select every node so the BGP instance runs across the cluster.
|
||||
nodeSelector:
|
||||
matchExpressions:
|
||||
- key: somekey
|
||||
operator: NotIn
|
||||
values: ['never-used-value']
|
||||
bgpInstances:
|
||||
- name: "instance-{{ cilium_bgp_my_asn }}"
|
||||
localASN: {{ cilium_bgp_my_asn }}
|
||||
peers:
|
||||
{% for peer in _cilium_peers %}
|
||||
- name: "peer-{{ peer.peer_asn }}-{{ loop.index }}"
|
||||
peerASN: {{ peer.peer_asn }}
|
||||
peerAddress: {{ peer.peer_address }}
|
||||
peerConfigRef:
|
||||
name: cilium-peer
|
||||
name: 01-bgp-peering-policy
|
||||
spec: # CiliumBGPPeeringPolicySpec
|
||||
virtualRouters: # []CiliumBGPVirtualRouter
|
||||
- localASN: {{ cilium_bgp_my_asn }}
|
||||
exportPodCIDR: {{ cilium_exportPodCIDR | default('true') }}
|
||||
neighbors: # []CiliumBGPNeighbor
|
||||
{% if _cilium_bgp_neighbors | length > 0 %}
|
||||
{% for item in _cilium_bgp_neighbors %}
|
||||
- peerAddress: '{{ item.peer_address + "/32"}}'
|
||||
peerASN: {{ item.peer_asn }}
|
||||
eBGPMultihopTTL: 10
|
||||
connectRetryTimeSeconds: 120
|
||||
holdTimeSeconds: 90
|
||||
keepAliveTimeSeconds: 30
|
||||
gracefulRestart:
|
||||
enabled: true
|
||||
restartTimeSeconds: 120
|
||||
{% endfor %}
|
||||
{% if cilium_exportPodCIDR %}
|
||||
---
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumBGPAdvertisement
|
||||
metadata:
|
||||
name: cilium-pod-cidrs
|
||||
labels:
|
||||
advertise: "bgp"
|
||||
spec:
|
||||
advertisements:
|
||||
- advertisementType: "PodCIDR"
|
||||
{% else %}
|
||||
- peerAddress: '{{ cilium_bgp_peer_address + "/32"}}'
|
||||
peerASN: {{ cilium_bgp_peer_asn }}
|
||||
eBGPMultihopTTL: 10
|
||||
connectRetryTimeSeconds: 120
|
||||
holdTimeSeconds: 90
|
||||
keepAliveTimeSeconds: 30
|
||||
gracefulRestart:
|
||||
enabled: true
|
||||
restartTimeSeconds: 120
|
||||
{% endif %}
|
||||
serviceSelector:
|
||||
matchExpressions:
|
||||
- {key: somekey, operator: NotIn, values: ['never-used-value']}
|
||||
---
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumBGPAdvertisement
|
||||
metadata:
|
||||
name: cilium-lb-services
|
||||
labels:
|
||||
advertise: "bgp"
|
||||
spec:
|
||||
advertisements:
|
||||
- advertisementType: "Service"
|
||||
service:
|
||||
addresses:
|
||||
- LoadBalancerIP
|
||||
# Advertise all Services carrying an ingress address from the pool.
|
||||
selector:
|
||||
matchExpressions:
|
||||
- key: somekey
|
||||
operator: NotIn
|
||||
values: ['never-used-value']
|
||||
---
|
||||
apiVersion: cilium.io/v2
|
||||
apiVersion: "cilium.io/v2alpha1"
|
||||
kind: CiliumLoadBalancerIPPool
|
||||
metadata:
|
||||
name: "01-lb-pool"
|
||||
|
||||
@@ -21,10 +21,12 @@ kind: L2Advertisement
|
||||
metadata:
|
||||
name: default
|
||||
namespace: metallb-system
|
||||
{% if metal_lb_interfaces | default([]) | length > 0 %}
|
||||
{% if metal_lb_interfaces %}
|
||||
spec:
|
||||
interfaces:{% for iface in metal_lb_interfaces %}
|
||||
- {{ iface }}{% endfor %}
|
||||
interfaces:
|
||||
{% for interface in metal_lb_interfaces %}
|
||||
- {{ interface }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% if metal_lb_mode == "bgp" %}
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
---
|
||||
disable_swap: true
|
||||
|
||||
secure_path:
|
||||
RedHat: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
|
||||
Suse: /usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin
|
||||
|
||||
@@ -4,28 +4,6 @@
|
||||
name: "{{ system_timezone }}"
|
||||
when: (system_timezone is defined) and (system_timezone != "Your/Timezone")
|
||||
|
||||
# k3s recommends swap be disabled on all nodes. Disabling swap is all-or-nothing
|
||||
# across the cluster: leaving it enabled on some nodes but not others creates
|
||||
# uneven scheduling/latency behavior. This block turns swap off and comments out
|
||||
# the swap entries in /etc/fstab so it stays off across reboots. It is idempotent
|
||||
# and a no-op when swap is already disabled or swapoff is unavailable.
|
||||
- name: Disable swap on all cluster nodes
|
||||
when: disable_swap
|
||||
block:
|
||||
- name: Turn off swap now
|
||||
ansible.builtin.command: swapoff -a
|
||||
register: swapoff_result
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
|
||||
- name: Comment out swap entries in fstab
|
||||
ansible.builtin.replace:
|
||||
path: /etc/fstab
|
||||
regexp: '^([^#][^\n]*\s+swap\s+)'
|
||||
replace: '# \\1'
|
||||
register: fstab_swap
|
||||
|
||||
|
||||
- name: Set SELinux to disabled state
|
||||
ansible.posix.selinux:
|
||||
state: disabled
|
||||
@@ -55,35 +33,6 @@
|
||||
reload: true
|
||||
tags: sysctl
|
||||
|
||||
- name: Check if br_netfilter module exists
|
||||
ansible.builtin.shell: |
|
||||
set -o pipefail
|
||||
find /lib/modules/$(uname -r) -name "br_netfilter.ko*" | wc -l
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: br_netfilter_exists
|
||||
changed_when: false
|
||||
when: ansible_os_family == "RedHat"
|
||||
tags: sysctl
|
||||
|
||||
- name: Install kernel-modules-extra if br_netfilter missing
|
||||
ansible.builtin.yum:
|
||||
name: "kernel-modules-extra-{{ ansible_kernel }}"
|
||||
state: present
|
||||
update_cache: true
|
||||
register: kernel_modules_installed
|
||||
when:
|
||||
- ansible_os_family == "RedHat"
|
||||
- br_netfilter_exists.stdout | int == 0
|
||||
|
||||
- name: Reboot node if kernel modules package was installed
|
||||
ansible.builtin.reboot:
|
||||
msg: "Rebooting to apply new kernel modules package"
|
||||
reboot_timeout: 300
|
||||
when:
|
||||
- ansible_os_family == "RedHat"
|
||||
- kernel_modules_installed is changed
|
||||
|
||||
- name: Add br_netfilter to /etc/modules-load.d/
|
||||
ansible.builtin.copy:
|
||||
content: br_netfilter
|
||||
|
||||
@@ -6,24 +6,7 @@
|
||||
ansible.builtin.assert:
|
||||
that: ansible_version.full is version_compare('2.11', '>=')
|
||||
msg: >
|
||||
"Ansible is out of date. See here for more info: https://docs.technotim.com/posts/ansible-automation/"
|
||||
|
||||
- name: Verify all cluster nodes have unique hostnames
|
||||
ansible.builtin.assert:
|
||||
that: (cluster_hostnames | unique | length) == (cluster_hostnames | length)
|
||||
msg: >-
|
||||
k3s nodes must have unique hostnames. Found a duplicate in:
|
||||
{{ cluster_hostnames | unique }}. Each node registers in the cluster
|
||||
keyed by its hostname, so matching hostnames prevent nodes from joining.
|
||||
vars:
|
||||
cluster_hostnames: >-
|
||||
{{
|
||||
groups['k3s_cluster']
|
||||
| map('extract', hostvars, 'ansible_hostname')
|
||||
| list
|
||||
}}
|
||||
run_once: true
|
||||
when: "'k3s_cluster' in groups"
|
||||
"Ansible is out of date. See here for more info: https://docs.technotim.live/posts/ansible-automation/"
|
||||
|
||||
- name: Prepare Proxmox cluster
|
||||
hosts: proxmox
|
||||
|
||||
Reference in New Issue
Block a user