fix(metallb): verify the metallb-system namespace actually exists

- change the Test metallb-system namespace task to run
  k3s kubectl get namespace metallb-system instead of the bare
  -n metallb-system, which printed kubectl usage and always exited 0
- add a regression test that asserts the task uses an explicit get
  and fails if it ever regresses to the usage-only form
- wire the new test into pre-commit

.github/scripts/test-cilium-envoy-toggle.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+"""Regression test for the Cilium Envoy toggle.
+
+The `cilium_envoy` variable lets users enable or disable the Cilium Envoy
+proxy. The Install/upgrade Cilium task in
+roles/k3s_server_post/tasks/cilium.yml passes the value through to Helm as
+`envoy.enabled`. This test:
+
+  - loads the real "Install Cilium" task and confirms the install/upgrade
+    command actually contains the `envoy.enabled` Helm value,
+  - renders the conditional that computes the Helm value and confirms it
+    produces `true` when cilium_envoy is enabled and `false` when disabled,
+  - confirms the task stays forward/backward compatible (no raw `true` /
+    `false` hardcoded in place of the conditional).
+"""
+
+from __future__ import print_function
+
+import os
+import re
+import subprocess
+
+import yaml
+from jinja2 import Environment
+
+ENVOY_EXPRESSION = '{{ "true" if cilium_envoy else "false" }}'
+
+
+def repo_root():
+    return subprocess.check_output(
+        ["git", "rev-parse", "--show-toplevel"], text=True
+    ).strip()
+
+
+def fail(message):
+    raise SystemExit("Cilium Envoy toggle test failed: " + message)
+
+
+def extract_install_command(path):
+    """Return the command string for the 'Install Cilium' task.
+
+    Walks both top-level tasks and tasks nested inside a `block`/`always`/
+    `rescue` list, since the Cilium deploy steps are grouped under the
+    'Prepare Cilium CLI on first master and deploy CNI' block.
+    """
+    with open(path, encoding="utf-8") as handle:
+        doc = yaml.safe_load(handle)
+
+    def find_command(tasks):
+        for task in tasks:
+            if not isinstance(task, dict):
+                continue
+            if task.get("name") == "Install Cilium":
+                command = task.get("ansible.builtin.command")
+                if command is None:
+                    raise SystemExit(
+                        "Cilium Envoy toggle test failed: "
+                        "'Install Cilium' task has no ansible.builtin.command"
+                    )
+                return command
+            # Recurse into block/always/rescue sub-lists.
+            for key in ("block", "always", "rescue"):
+                nested = task.get(key)
+                if isinstance(nested, list):
+                    found = find_command(nested)
+                    if found is not None:
+                        return found
+        return None
+
+    command = find_command(doc)
+    if command is None:
+        raise SystemExit(
+            "Cilium Envoy toggle test failed: could not find 'Install Cilium' task"
+        )
+    return command
+
+
+def assert_envoy_in_command(command):
+    if "envoy.enabled" not in command:
+        fail("install command is missing --helm-set envoy.enabled")
+    if ENVOY_EXPRESSION not in command:
+        fail(
+            "install command does not use the cilium_envoy conditional: "
+            "expected {0!r}".format(ENVOY_EXPRESSION)
+        )
+    # The conditional must be a WYSIWYG helm-set value, not a pre-rendered
+    # true/false literal (which would ignore the cilium_envoy variable).
+    if re.search(r"--helm-set envoy\.enabled=true(?:$|\s)", command):
+        fail("install command hardcodes envoy.enabled=true")
+    if re.search(r"--helm-set envoy\.enabled=false(?:$|\s)", command):
+        fail("install command hardcodes envoy.enabled=false")
+
+
+def assert_render():
+    env = Environment()
+
+    def render_for(value):
+        template = env.from_string(ENVOY_EXPRESSION)
+        return template.render(cilium_envoy=value)
+
+    if render_for(True) != "true":
+        fail("envoy conditional did not render 'true' when enabled")
+    if render_for(False) != "false":
+        fail("envoy conditional did not render 'false' when disabled")
+
+
+def main():
+    root = repo_root()
+    cilium_tasks = os.path.join(
+        root, "roles", "k3s_server_post", "tasks", "cilium.yml"
+    )
+    command = extract_install_command(cilium_tasks)
+    assert_envoy_in_command(command)
+    assert_render()
+
+    print("Cilium Envoy toggle regression test passed")
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/test-disable-swap.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+repo_root="$(git rev-parse --show-toplevel)"
+prereq_defaults="$repo_root/roles/prereq/defaults/main.yml"
+prereq_tasks="$repo_root/roles/prereq/tasks/main.yml"
+
+# #670: k3s recommends swap be disabled on all nodes. The prereq role must expose
+# a disable_swap toggle (defaulting to true) that turns swap off now and comments
+# out the /etc/fstab swap entries so swap stays off across reboots.
+grep -Eq -- '^disable_swap: true' "$prereq_defaults" || {
+  printf 'prereq defaults are missing disable_swap: true\n' >&2
+  exit 1
+}
+
+grep -Fq -- 'Disable swap on all cluster nodes' "$prereq_tasks" || {
+  printf 'prereq tasks are missing the swap-disable block\n' >&2
+  exit 1
+}
+
+grep -Fq -- 'swapoff -a' "$prereq_tasks" || {
+  printf 'swap-disable block does not run swapoff -a\n' >&2
+  exit 1
+}
+
+grep -Fq -- '/etc/fstab' "$prereq_tasks" || {
+  printf 'swap-disable block does not comment out /etc/fstab swap entries\n' >&2
+  exit 1
+}
+
+if ! grep -Eq -- 'when: disable_swap' "$prereq_tasks"; then
+  printf 'swap-disable block is not gated on the disable_swap toggle\n' >&2
+  exit 1
+fi
+
+printf 'Swap disable regression test passed\n'

.github/scripts/test-kube-vip-manifest.py

@@ -123,6 +123,38 @@ def main():
     if "name: bgp_peers" in output:
         fail("bgp_peers present even though the peer list is empty")
 
+    # kube_vip_endpoint defaults to null (defined in role defaults): the
+    # address and subnet must fall back to the apiserver endpoint. default()
+    # without a truthy flag does NOT fall back on null, only on undefined, so
+    # this case pins the null runtime condition to prevent that regression.
+    output = render(
+        env,
+        {
+            "_kube_vip_bgp_peers": [],
+            "kube_vip_endpoint": None,
+            "kube_vip_arp": True,
+            "kube_vip_bgp": False,
+        },
+    )
+    if "value: 192.168.30.222" not in output:
+        fail("null kube_vip_endpoint does not fall back to apiserver_endpoint")
+
+    # kube_vip_endpoint set: overrides the internal listening address AND the
+    # subnet derivation while the advertised apiserver_endpoint stays separate.
+    output = render(
+        env,
+        {
+            "_kube_vip_bgp_peers": [],
+            "kube_vip_endpoint": "10.66.1.5",
+            "kube_vip_arp": True,
+            "kube_vip_bgp": False,
+        },
+    )
+    if "value: 10.66.1.5" not in output:
+        fail("kube_vip_endpoint did not override the address")
+    if "value: 192.168.30.222" in output:
+        fail("apiserver_endpoint leaked into address when kube_vip_endpoint set")
+
     print("kube-vip manifest regression test passed")
 
 

.github/scripts/test-metallb-interfaces.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""Regression test for the MetalLB L2Advertisement interfaces.
+
+`metal_lb_interfaces` restricts which network interfaces MetalLB announces
+load balancer IPs on in layer2 mode. When the list is non-empty, the
+L2Advertisement in roles/k3s_server_post/templates/metallb.crs.j2 must render
+a `spec.interfaces` block; when it is empty (the default), no spec is rendered
+so MetalLB announces on all interfaces.
+
+This renders the template and asserts both cases plus the BGP path (which must
+not be affected by the L2 interfaces variable).
+"""
+
+from __future__ import print_function
+
+import os
+import subprocess
+
+from jinja2 import Environment, FileSystemLoader, StrictUndefined
+
+
+def repo_root():
+    return subprocess.check_output(
+        ["git", "rev-parse", "--show-toplevel"], text=True
+    ).strip()
+
+
+def fail(message):
+    raise SystemExit("MetalLB interfaces test failed: " + message)
+
+
+def render(env, extra_vars):
+    base_vars = {
+        "metal_lb_mode": "layer2",
+        "metal_lb_ip_range": "192.168.30.80-192.168.30.90",
+    }
+    base_vars.update(extra_vars)
+    template = env.get_template("metallb.crs.j2")
+    return template.render(**base_vars)
+
+
+def main():
+    root = repo_root()
+    template_dir = os.path.join(
+        root, "roles", "k3s_server_post", "templates"
+    )
+    env = Environment(
+        loader=FileSystemLoader(template_dir), undefined=StrictUndefined
+    )
+
+    # Empty list (default): no spec.interfaces in the L2Advertisement.
+    output = render(env, {"metal_lb_interfaces": []})
+    if "spec:\n  interfaces:" in output:
+        fail("spec.interfaces rendered with an empty metal_lb_interfaces")
+    if "kind: L2Advertisement" not in output:
+        fail("L2Advertisement missing in layer2 mode")
+
+    # Single interface.
+    output = render(env, {"metal_lb_interfaces": ["eth1"]})
+    if "spec:\n  interfaces:\n    - eth1" not in output:
+        fail("single interface was not rendered in spec.interfaces")
+
+    # Multiple interfaces.
+    output = render(env, {"metal_lb_interfaces": ["eth1", "eth2"]})
+    if "spec:\n  interfaces:\n    - eth1\n    - eth2" not in output:
+        fail("multiple interfaces were not rendered in spec.interfaces")
+
+    # BGP mode must not emit an L2Advertisement spec at all.
+    output = render(
+        env,
+        {
+            "metal_lb_mode": "bgp",
+            "metal_lb_interfaces": ["eth1"],
+            "metal_lb_bgp_my_asn": "64513",
+            "metal_lb_bgp_peer_asn": "64512",
+            "metal_lb_bgp_peer_address": "192.168.30.1",
+        },
+    )
+    if "kind: L2Advertisement" in output:
+        fail("L2Advertisement rendered in bgp mode")
+    if "interfaces:" in output:
+        fail("interfaces rendered in bgp mode")
+
+    print("MetalLB interfaces regression test passed")
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/test-metallb-namespace.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Regression test for the MetalLB namespace existence check.
+
+The "Test metallb-system namespace" task in
+roles/k3s_server_post/tasks/metallb.yml must actually verify the namespace
+exists. A previous version ran `k3s kubectl -n metallb-system` with no
+subcommand, which only printed a usage page and always exited 0, so the task
+always succeeded even when the namespace did not exist (issue #350).
+
+This test loads the real task and asserts the command performs an explicit
+`get namespace metallb-system`, which returns non-zero when the namespace is
+absent.
+"""
+
+from __future__ import print_function
+
+import os
+import subprocess
+
+import yaml
+
+
+def repo_root():
+    return subprocess.check_output(
+        ["git", "rev-parse", "--show-toplevel"], text=True
+    ).strip()
+
+
+def fail(message):
+    raise SystemExit(
+        "MetalLB namespace test failed: " + message
+    )
+
+
+def main():
+    task_file = os.path.join(
+        repo_root(), "roles", "k3s_server_post", "tasks", "metallb.yml"
+    )
+    with open(task_file, encoding="utf-8") as handle:
+        tasks = yaml.safe_load(handle)
+
+    task = None
+    for entry in tasks:
+        if entry.get("name") == "Test metallb-system namespace":
+            task = entry
+            break
+    if task is None:
+        fail("could not find the 'Test metallb-system namespace' task")
+
+    cmd = task.get("ansible.builtin.command")
+    if not cmd:
+        cmd = task.get("command")
+    if not cmd:
+        fail("task does not use ansible.builtin.command")
+
+    command_text = cmd if isinstance(cmd, str) else " ".join(cmd)
+
+    # A bare `-n metallb-system` with no subcommand prints kubectl usage and
+    # always exits 0, so it never proves the namespace exists. The fix must
+    # use an explicit get.
+    if "get namespace metallb-system" not in command_text:
+        fail(
+            "command does not run 'get namespace metallb-system'; "
+            "the task would only print usage and never verify the namespace "
+            "(got: {0!r})".format(command_text)
+        )
+
+    print("MetalLB namespace check regression test passed")
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/test-unique-hostname-precheck.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+repo_root="$(git rev-parse --show-toplevel)"
+site_play="$repo_root/site.yml"
+
+# #636: verify the "Pre tasks" play asserts that all k3s_cluster hosts have
+# unique hostnames, so a duplicate-hostname inventory fails fast instead of
+# silently breaking node registration/joining.
+grep -Fq -- 'Verify all cluster nodes have unique hostnames' "$site_play" || {
+  printf 'site.yml is missing the unique-hostname preflight check\n' >&2
+  exit 1
+}
+
+# The check must deduplicate the cluster hostname list via the `unique` filter
+# and compare lengths, i.e. groups['k3s_cluster'] must be referenced.
+grep -Fq -- "groups['k3s_cluster']" "$site_play" || {
+  printf 'unique-hostname check does not iterate the k3s_cluster group\n' >&2
+  exit 1
+}
+
+if ! grep -Eq -- 'cluster_hostnames.*\|.*unique|\| unique' "$site_play"; then
+  printf 'unique-hostname check does not deduplicate the hostname list\n' >&2
+  exit 1
+fi
+
+printf 'Unique hostname preflight regression test passed\n'

.github/workflows/lint.yml

@@ -68,7 +68,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # 7.0.1
       - name: Ensure SHA pinned actions
-        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@3db98c0363e2fa5df3e1c4c471777a7c10b24cc9 # 5.0.5
+        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@46cfe808a5f1588656ef299eedd0ce2fd7ec0dcc # 5.0.6
         with:
           allowlist: |
             aws-actions/

.github/workflows/test.yml

@@ -88,7 +88,7 @@ jobs:
           trap stop_monitor EXIT
           /usr/bin/time -v -o "$timing_file" \
             molecule test --scenario-name ${{ matrix.scenario }}
-        timeout-minutes: 150
+        timeout-minutes: 180
         env:
           ANSIBLE_K3S_LOG_DIR: ${{ runner.temp }}/logs/k3s-ansible/${{ matrix.scenario }}
           ANSIBLE_SSH_RETRIES: 4

.pre-commit-config.yaml

@@ -62,6 +62,18 @@ repos:
         language: system
         pass_filenames: false
         files: ^roles/k3s_server/tasks/(main|join_master)\.yml$|^\.github/scripts/test-k3s-server-bootstrap\.sh$
+      - id: unique-hostname-precheck-test
+        name: Unique hostname precheck test
+        entry: .github/scripts/test-unique-hostname-precheck.sh
+        language: system
+        pass_filenames: false
+        files: ^site\.yml$|^\.github/scripts/test-unique-hostname-precheck\.sh$
+      - id: disable-swap-test
+        name: Disable swap test
+        entry: .github/scripts/test-disable-swap.sh
+        language: system
+        pass_filenames: false
+        files: ^roles/prereq/(tasks/main|defaults/main)\.yml$|^\.github/scripts/test-disable-swap\.sh$
       - id: cilium-bgp-manifest-test
         name: Cilium BGP manifest test
         entry: python3 .github/scripts/test-cilium-bgp-manifest.py
@@ -70,6 +82,15 @@ repos:
           - Jinja2>=3.1
         pass_filenames: false
         files: ^roles/k3s_server_post/templates/cilium\.crs\.j2$|^\.github/scripts/test-cilium-bgp-manifest\.py$
+      - id: cilium-envoy-toggle-test
+        name: Cilium Envoy toggle test
+        entry: python3 .github/scripts/test-cilium-envoy-toggle.py
+        language: python
+        additional_dependencies:
+          - Jinja2>=3.1
+          - PyYAML
+        pass_filenames: false
+        files: ^roles/k3s_server_post/tasks/cilium\.yml$|^\.github/scripts/test-cilium-envoy-toggle\.py$
       - id: kube-vip-manifest-test
         name: kube-vip manifest test
         entry: python3 .github/scripts/test-kube-vip-manifest.py
@@ -84,6 +105,22 @@ repos:
         language: system
         pass_filenames: false
         files: ^roles/k3s_server/tasks/metallb\.yml$|^\.github/scripts/test-metallb-remote-read\.sh$
+      - id: metallb-interfaces-test
+        name: MetalLB interfaces test
+        entry: python3 .github/scripts/test-metallb-interfaces.py
+        language: python
+        additional_dependencies:
+          - Jinja2>=3.1
+        pass_filenames: false
+        files: ^roles/k3s_server_post/templates/metallb\.crs\.j2$|^\.github/scripts/test-metallb-interfaces\.py$
+      - id: metallb-namespace-test
+        name: MetalLB namespace test
+        entry: python3 .github/scripts/test-metallb-namespace.py
+        language: python
+        additional_dependencies:
+          - PyYAML
+        pass_filenames: false
+        files: ^roles/k3s_server_post/tasks/metallb\.yml$|^\.github/scripts/test-metallb-namespace\.py$
       - id: metallb-deploy-condition-test
         name: MetalLB deploy condition test
         entry: python3 .github/scripts/test-metallb-deploy-condition.py

README.md

@@ -50,6 +50,10 @@ Supported processor architectures are:
 - Server and agent nodes should support passwordless SSH access. Otherwise, pass `--ask-pass --ask-become-pass` to
   each playbook command.
 
+- Every node in the cluster must have a **unique hostname**. k3s registers each node keyed by its hostname, so
+  two nodes with the same hostname cannot join the cluster. `site.yml` asserts this up front and fails fast if
+  any duplicate is found.
+
 ## 🚀 Getting Started
 
 ### 🍴 Preparation
@@ -108,6 +112,24 @@ ansible-playbook reset.yml -i inventory/my-cluster/hosts.ini
 
 > Reboot the nodes after reset because the virtual IP may remain configured.
 
+### ⏻️ Reboot Cluster Nodes
+
+Reboot all cluster nodes at once or stage the reboot across the cluster.
+
+```bash
+ansible-playbook reboot.yml -i inventory/my-cluster/hosts.ini
+```
+
+To reboot the nodes in batches, set `concurrent_reboots` to the number of nodes
+to reboot at a time (or a percentage). Optionally set `wait_seconds_after_reboot`
+to pause after each batch so pods in the freshly rebooted batch can settle
+before the next batch reboots.
+
+```bash
+ansible-playbook reboot.yml -i inventory/my-cluster/hosts.ini \
+  --extra-vars 'concurrent_reboots=2 wait_seconds_after_reboot=30'
+```
+
 ## 🔁 Upgrading an existing cluster
 
 These version variables select the components used for a **fresh** installation.
@@ -197,6 +219,7 @@ See the commands [here](https://technotim.com/posts/k3s-etcd-ansible/#testing-yo
 | `k3s_server` | `kube_vip_bgp_peers` | list | `[]` | Not required | List of BGP peer ASN & address pairs |
 | `k3s_server` | `kube_vip_bgp_peers_groups` | list | `['k3s_master']` | Not required | Inventory group in which to search for additional `kube_vip_bgp_peers` parameters to merge. |
 | `k3s_server` | `kube_vip_iface` | string | `~` | Not required | Explicitly define an interface that ALL control nodes should use to propagate the VIP, define it here. Otherwise, kube-vip will determine the right interface automatically at runtime. |
+| `k3s_server` | `kube_vip_endpoint` | string | `~` | Not required | Overrides the internal address kube-vip binds/listens on, which can differ from the announced apiserver_endpoint for complex routing/tunnels. Defaults to apiserver_endpoint. |
 | `k3s_server` | `kube_vip_tag_version` | string | `v1.2.2` | Not required | Image tag for kube-vip |
 | `k3s_server` | `kube_vip_cloud_provider_tag_version` | string | `v0.0.12` | Not required | Tag for kube-vip-cloud-provider manifest when enable |
 | `k3s_server`, `k3_server_post` | `kube_vip_lb_ip_range` | string | `~` | Not required | IP range for kube-vip load balancer |
@@ -236,7 +259,10 @@ See the commands [here](https://technotim.com/posts/k3s-etcd-ansible/#testing-yo
 | `k3s_server_post` | `metal_lb_bgp_peer_asn` | string | `~` | Not required | BGP peer ASN configurations |
 | `k3s_server_post` | `metal_lb_bgp_peer_address` | string | `~` | Not required | BGP peer address |
 | `lxc` | `custom_reboot_command` | string | `~` | Not required | Command to run on reboot |
+| `reboot` (playbook) | `concurrent_reboots` | int/string | `100%` | Not required | Number (or percentage) of nodes to reboot at a time for a staggered reboot |
+| `reboot` (playbook) | `wait_seconds_after_reboot` | int | `0` | Not required | Pause in seconds between staggered reboot batches |
 | `prereq` | `system_timezone` | string | `null` | Not required | Timezone to be set on all nodes |
+| `prereq` | `disable_swap` | bool | `true` | Not required | Disable swap on all cluster nodes (swapoff + comment out /etc/fstab swap entries), all-or-nothing |
 | `proxmox_lxc`, `reset_proxmox_lxc` | `proxmox_lxc_ct_ids` | list | ❌ | Required | Proxmox container ID list |
 | `raspberrypi` | `state` | string | `present` | Not required | Indicates whether the k3s prerequisites for Raspberry Pi should be set up (possible values are `present` and `absent`) |
 

inventory/sample/group_vars/all.yml

@@ -7,6 +7,10 @@ systemd_dir: /etc/systemd/system
 # Set your timezone
 system_timezone: Your/Timezone
 
+# k3s recommends swap be disabled on every cluster node. Applied uniformly to all
+# nodes (all-or-nothing) in the prereq role. Set to false to leave swap enabled.
+disable_swap: true
+
 # interface which will be used for flannel
 # Defaults to each host's default IPv4 interface (e.g. eth0, enp1s0, ens3)
 # so KVM/cloud hosts without eth0 work out of the box. Override per-host if needed.
@@ -24,6 +28,10 @@ cilium_mode: native # native when nodes are on the same subnet or use BGP, other
 cilium_tag: v1.20.0 # cilium version tag
 cilium_cli_tag: v0.19.7 # cilium cli version tag
 cilium_hubble: true # enable hubble observability relay and ui
+cilium_envoy: true # enable the Envoy proxy for Cilium L7 policies
+
+# disable cilium_envoy to skip the Envoy proxy entirely (e.g. no L7 policies)
+# cilium_envoy: false
 
 # if using calico or cilium, you may specify the cluster pod cidr pool
 cluster_cidr: 10.52.0.0/16
@@ -40,6 +48,11 @@ cilium_bgp_lb_cidr: 192.168.31.0/24 # cidr for cilium loadbalancer ipam
 # enable kube-vip ARP broadcasts
 kube_vip_arp: true
 
+# (optional) overrides the address kube-vip binds/listens on internally, which
+# can differ from the announced apiserver_endpoint for complex routing/tunnels.
+# Defaults to apiserver_endpoint. Also used to derive the kube-vip subnet.
+# kube_vip_endpoint: 10.66.1.5
+
 # enable kube-vip BGP peering
 kube_vip_bgp: false
 
@@ -116,6 +129,12 @@ metal_lb_controller_tag_version: v0.16.0
 # metallb ip range for load balancer
 metal_lb_ip_range: 192.168.30.80-192.168.30.90
 
+# (optional) limit MetalLB layer2 announcements to specific network interfaces.
+# Leave empty (default) to announce on all interfaces.
+# metal_lb_interfaces:
+#   - eth1
+#   - eth2
+
 # Only enable if your nodes are proxmox LXC nodes, make sure to configure your proxmox nodes
 # in your hosts.ini file.
 # Please read https://gist.github.com/triangletodd/02f595cd4c0dc9aac5f7763ca2264185 before using this.

molecule/resources/verify_from_outside/tasks/test/verify-components.yml

@@ -103,7 +103,7 @@
           ansible.builtin.assert:
             that:
               - calico_node_ds.resources | length == 1
-              - calico_node_image | regex_search(':' ~ calico_tag)
+              - calico_node_image | regex_search(':' ~ calico_tag) is not none
             success_msg: "Calico node image uses tag {{ calico_tag }}"
             fail_msg: >-
               Calico node image {{ calico_node_image }},
@@ -169,8 +169,8 @@
         - name: Assert Cilium agent and operator use the expected image tag
           ansible.builtin.assert:
             that:
-              - cilium_agent_image | regex_search(':' ~ cilium_tag)
-              - cilium_operator_image | regex_search(':' ~ cilium_tag)
+              - cilium_agent_image | regex_search(':' ~ cilium_tag) is not none
+              - cilium_operator_image | regex_search(':' ~ cilium_tag) is not none
             success_msg: "Cilium agent and operator use {{ cilium_tag }}"
             fail_msg: >-
               Cilium agent {{ cilium_agent_image }},
@@ -265,9 +265,12 @@
 
         - name: Assert MetalLB controller and speaker use the expected image tags
           ansible.builtin.assert:
+            # regex_search returns a string or none; check for a match with `is not
+            # none` so the assertion is a real boolean (ansible-core 2.19 rejects
+            # string conditionals and `| bool` deprecates string coercion).
             that:
-              - controller_image | regex_search(metal_lb_controller_tag_version)
-              - speaker_image | regex_search(metal_lb_speaker_tag_version)
+              - controller_image | regex_search(metal_lb_controller_tag_version) is not none
+              - speaker_image | regex_search(metal_lb_speaker_tag_version) is not none
             success_msg: >-
               MetalLB controller {{ metal_lb_controller_tag_version }},
               speaker {{ metal_lb_speaker_tag_version }}
@@ -311,8 +314,8 @@
         - name: Assert the kube-vip and cloud provider image tags
           ansible.builtin.assert:
             that:
-              - kubevip_image | regex_search(':' ~ kube_vip_tag_version)
-              - cloud_provider_image | regex_search(verify_kube_vip_cloud_provider_tag)
+              - kubevip_image | regex_search(':' ~ kube_vip_tag_version) is not none
+              - cloud_provider_image | regex_search(verify_kube_vip_cloud_provider_tag) is not none
             success_msg: >-
               kube-vip {{ kube_vip_tag_version }},
               cloud provider {{ verify_kube_vip_cloud_provider_tag }}

reboot.yml

@@ -2,9 +2,29 @@
 - name: Reboot k3s_cluster
   hosts: k3s_cluster
   gather_facts: true
+
+  # Stagger the reboot across the cluster when concurrent_reboots is set.
+  # Defaults to '100%' so the whole cluster reboots at once (backward compatible).
+  serial: "{{ concurrent_reboots | default('100%') }}"
+
   tasks:
-    - name: Reboot the nodes (and Wait upto 5 mins max)
+    - name: >-
+        {{
+          'Reboot all nodes at once'
+          if (concurrent_reboots is not defined)
+          else 'Reboot nodes with concurrency of ' ~ concurrent_reboots
+        }}
       become: true
       ansible.builtin.reboot:
         reboot_command: "{{ custom_reboot_command | default(omit) }}"
         reboot_timeout: 300
+        test_command: >-
+          {{ 'kubectl get nodes' if 'master' in group_names else 'whoami' }}
+
+    - name: Optional wait before rebooting the next batch of nodes
+      ansible.builtin.pause:
+        seconds: "{{ wait_seconds_after_reboot | int }}"
+      when: >-
+        concurrent_reboots is defined and
+        wait_seconds_after_reboot is defined and
+        wait_seconds_after_reboot | int > 0

requirements.in

@@ -1,5 +1,5 @@
-ansible-core>=2.16.2
-jmespath>=1.0.1
+ansible-core>=2.19.11
+jmespath>=1.1.0
 jsonpatch>=1.33
 kubernetes>=29.0.0
 molecule-plugins[vagrant]

requirements.txt

@@ -2,11 +2,11 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile requirements.in
+#    pip-compile --output-file=requirements.txt requirements.in
 #
 ansible-compat==4.1.11
     # via molecule
-ansible-core==2.18.0
+ansible-core==2.19.11
     # via
     #   -r requirements.in
     #   ansible-compat
@@ -53,7 +53,7 @@ jinja2==3.1.3
     # via
     #   ansible-core
     #   molecule
-jmespath==1.0.1
+jmespath==1.1.0
     # via -r requirements.in
 jsonpatch==1.33
     # via -r requirements.in
@@ -77,7 +77,7 @@ molecule==6.0.3
     # via
     #   -r requirements.in
     #   molecule-plugins
-molecule-plugins[vagrant]==23.5.3
+molecule-plugins[vagrant]==23.6.0
     # via -r requirements.in
 netaddr==0.10.1
     # via -r requirements.in
@@ -146,7 +146,7 @@ rsa==4.9
     # via google-auth
 ruamel-yaml==0.18.5
     # via pre-commit-hooks
-ruamel-yaml-clib==0.2.8
+ruamel-yaml-clib==0.2.15
     # via ruamel-yaml
 six==1.16.0
     # via

roles/k3s_server/defaults/main.yml

@@ -7,6 +7,7 @@ group_name_master: master
 
 kube_vip_arp: true
 kube_vip_iface:
+kube_vip_endpoint:
 kube_vip_cloud_provider_tag_version: v0.0.12
 kube_vip_tag_version: v1.2.2
 
@@ -23,6 +24,12 @@ metal_lb_controller_tag_version: v0.16.0
 metal_lb_speaker_tag_version: v0.16.0
 metal_lb_type: native
 
+# Shared retry/delay for remote manifest and asset downloads. The CI runner's
+# resolver intermittently times out on GitHub-hosted domains (helm.cilium.io,
+# raw.githubusercontent.com, github.com), so retry transient DNS/network failures.
+download_retries: 5
+download_delay: 10
+
 retry_count: 20
 
 # yamllint disable rule:line-length

roles/k3s_server/meta/main.yml

@@ -78,6 +78,15 @@ argument_specs:
           - automatically at runtime.
         default: ~
 
+      kube_vip_endpoint:
+        description:
+          - Overrides the address kube-vip binds/listens on internally, which
+          - can differ from the announced apiserver_endpoint for complex
+          - routing and site-to-site tunnels.
+          - Defaults to apiserver_endpoint and is used to derive the kube-vip
+          - subnet.
+        default: ~
+
       kube_vip_tag_version:
         description: Image tag for kube-vip
         default: v1.2.2

roles/k3s_server/tasks/kube-vip.yml

@@ -15,6 +15,10 @@
     owner: root
     group: root
     mode: "0644"
+  register: kube_vip_manifest_download
+  retries: "{{ download_retries }}"
+  delay: "{{ download_delay }}"
+  until: kube_vip_manifest_download is succeeded
   when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
 
 - name: Copy kubevip configMap manifest to first master

roles/k3s_server/tasks/main.yml

@@ -122,7 +122,10 @@
   always:
     - name: Save logs of k3s-init.service
       ansible.builtin.include_tasks: fetch_k3s_init_logs.yml
-      when: log_destination
+      # ANSIBLE_K3S_LOG_DIR is a path string when set; evaluate it as a boolean
+      # so the conditional is a real boolean (ansible-core 2.19 rejects string
+      # conditionals derived from env vars).
+      when: log_destination | default('') != ''
       vars:
         log_destination: >-
           {{ lookup('ansible.builtin.env', 'ANSIBLE_K3S_LOG_DIR', default=False) }}

roles/k3s_server/tasks/metallb.yml

@@ -15,6 +15,10 @@
     owner: root
     group: root
     mode: "0644"
+  register: metallb_manifest_download
+  retries: "{{ download_retries }}"
+  delay: "{{ download_delay }}"
+  until: metallb_manifest_download is succeeded
   when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
 
 - name: Set image versions in manifest for metallb-{{ metal_lb_type }}

roles/k3s_server/templates/vip.yaml.j2

@@ -37,7 +37,7 @@ spec:
           value: {{ kube_vip_iface }}
 {% endif %}
         - name: vip_subnet
-          value: "{{ apiserver_endpoint | ansible.utils.ipsubnet | ansible.utils.ipaddr('prefix') }}"
+          value: "{{ (kube_vip_endpoint | default(apiserver_endpoint, true)) | ansible.utils.ipsubnet | ansible.utils.ipaddr('prefix') }}"
         - name: cp_enable
           value: "true"
         - name: cp_namespace
@@ -55,7 +55,7 @@ spec:
         - name: vip_retryperiod
           value: "2"
         - name: address
-          value: {{ apiserver_endpoint }}
+          value: {{ kube_vip_endpoint | default(apiserver_endpoint, true) }}
 {% if kube_vip_bgp | default(false) | bool %}
 {% if kube_vip_bgp_routerid is defined %}
         - name: bgp_routerid

roles/k3s_server_post/defaults/main.yml

@@ -18,6 +18,7 @@ cilium_bgp_peer_asn: 64512
 cilium_bgp_neighbors: []
 cilium_bgp_neighbors_groups: ['k3s_all']
 cilium_bgp_lb_cidr: 192.168.31.0/24
+cilium_envoy: true
 cilium_hubble: true
 cilium_mode: native
 cilium_tag: v1.20.0
@@ -38,4 +39,5 @@ group_name_master: master
 metal_lb_mode: layer2
 metal_lb_available_timeout: 240s
 metal_lb_controller_tag_version: v0.16.0
+metal_lb_interfaces: []
 metal_lb_ip_range: 192.168.30.80-192.168.30.90

roles/k3s_server_post/meta/main.yml

@@ -141,6 +141,14 @@ argument_specs:
         description: MetalLB ip range for load balancer
         default: 192.168.30.80-192.168.30.90
 
+      metal_lb_interfaces:
+        description: >-
+          List of network interfaces on which MetalLB should announce the
+          load balancer IPs in layer2 mode. When empty (default), MetalLB
+          announces on all interfaces.
+        type: list
+        default: []
+
       metal_lb_controller_tag_version:
         description: Image tag for MetalLB
         default: v0.16.0

roles/k3s_server_post/tasks/calico.yml

@@ -18,6 +18,10 @@
         owner: root
         group: root
         mode: "0755"
+      register: calico_crd_download
+      retries: "{{ download_retries }}"
+      delay: "{{ download_delay }}"
+      until: calico_crd_download is succeeded
 
     - name: "Download to first master: manifest for Tigera Operator and Calico CRDs"
       ansible.builtin.get_url:
@@ -26,6 +30,10 @@
         owner: root
         group: root
         mode: "0755"
+      register: tigera_operator_download
+      retries: "{{ download_retries }}"
+      delay: "{{ download_delay }}"
+      until: tigera_operator_download is succeeded
 
     - name: Apply Calico CRD bundle with server-side apply
       ansible.builtin.command: >-

roles/k3s_server_post/tasks/cilium.yml

@@ -66,6 +66,10 @@
             - .tar.gz.sha256sum
           vars:
             cilium_base_url: https://github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_tag }}
+          register: cilium_cli_download
+          retries: "{{ download_retries }}"
+          delay: "{{ download_delay }}"
+          until: cilium_cli_download is succeeded
 
         - name: Verify the downloaded tarball
           ansible.builtin.shell: |
@@ -174,6 +178,7 @@
         --helm-set hubble.enabled={{ "true" if cilium_hubble else "false" }}
         --helm-set hubble.relay.enabled={{ "true" if cilium_hubble else "false" }}
         --helm-set hubble.ui.enabled={{ "true" if cilium_hubble else "false" }}
+        --helm-set envoy.enabled={{ "true" if cilium_envoy else "false" }}
         {% if kube_proxy_replacement is not false %}
         --helm-set loadBalancer.algorithm={{ bpf_lb_algorithm }}
         --helm-set loadBalancer.mode={{ bpf_lb_mode }}
@@ -182,6 +187,13 @@
         KUBECONFIG: "{{ ansible_user_dir }}/.kube/config"
       register: cilium_install_result
       changed_when: cilium_install_result.rc == 0
+      # cilium install/upgrade fetches the Helm chart from helm.cilium.io, which is
+      # fronted by GitHub Pages and can transiently fail DNS resolution through the
+      # host resolver (intermittent "lookup helm.cilium.io ... i/o timeout"). Retry
+      # so a transient name/network failure does not abort the whole converge play.
+      until: cilium_install_result.rc == 0
+      retries: "{{ download_retries }}"
+      delay: "{{ download_delay }}"
       when: cilium_installed.rc != 0 or cilium_needs_update
 
     - name: Wait for Cilium resources

roles/k3s_server_post/tasks/metallb.yml

@@ -40,7 +40,7 @@
 
 - name: Test metallb-system namespace
   ansible.builtin.command: >-
-    {{ k3s_kubectl_binary | default('k3s kubectl') }} -n metallb-system
+    {{ k3s_kubectl_binary | default('k3s kubectl') }} get namespace metallb-system
   changed_when: false
   with_items: "{{ groups[group_name_master | default('master')] }}"
   run_once: true
@@ -118,6 +118,10 @@
   changed_when: false
   run_once: true
   when: metal_lb_mode == "layer2"
+  register: metallb_l2_test_result
+  until: metallb_l2_test_result.rc == 0
+  retries: "{{ download_retries }}"
+  delay: "{{ download_delay }}"
   with_items:
     - IPAddressPool
     - L2Advertisement
@@ -128,6 +132,10 @@
   changed_when: false
   run_once: true
   when: metal_lb_mode == "bgp"
+  register: metallb_bgp_test_result
+  until: metallb_bgp_test_result.rc == 0
+  retries: "{{ download_retries }}"
+  delay: "{{ download_delay }}"
   with_items:
     - IPAddressPool
     - BGPPeer

roles/k3s_server_post/templates/metallb.crs.j2

@@ -21,6 +21,11 @@ kind: L2Advertisement
 metadata:
   name: default
   namespace: metallb-system
+{% if metal_lb_interfaces | default([]) | length > 0 %}
+spec:
+  interfaces:{% for iface in metal_lb_interfaces %}
+    - {{ iface }}{% endfor %}
+{% endif %}
 {% endif %}
 {% if metal_lb_mode == "bgp" %}
 ---

roles/prereq/defaults/main.yml

@@ -1,4 +1,6 @@
 ---
+disable_swap: true
+
 secure_path:
   RedHat: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
   Suse: /usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin

roles/prereq/tasks/main.yml

@@ -4,6 +4,28 @@
     name: "{{ system_timezone }}"
   when: (system_timezone is defined) and (system_timezone != "Your/Timezone")
 
+# k3s recommends swap be disabled on all nodes. Disabling swap is all-or-nothing
+# across the cluster: leaving it enabled on some nodes but not others creates
+# uneven scheduling/latency behavior. This block turns swap off and comments out
+# the swap entries in /etc/fstab so it stays off across reboots. It is idempotent
+# and a no-op when swap is already disabled or swapoff is unavailable.
+- name: Disable swap on all cluster nodes
+  when: disable_swap
+  block:
+    - name: Turn off swap now
+      ansible.builtin.command: swapoff -a
+      register: swapoff_result
+      changed_when: false
+      failed_when: false
+
+    - name: Comment out swap entries in fstab
+      ansible.builtin.replace:
+        path: /etc/fstab
+        regexp: '^([^#][^\n]*\s+swap\s+)'
+        replace: '# \\1'
+      register: fstab_swap
+
+
 - name: Set SELinux to disabled state
   ansible.posix.selinux:
     state: disabled

site.yml

@@ -8,6 +8,23 @@
         msg: >
           "Ansible is out of date. See here for more info: https://docs.technotim.com/posts/ansible-automation/"
 
+    - name: Verify all cluster nodes have unique hostnames
+      ansible.builtin.assert:
+        that: (cluster_hostnames | unique | length) == (cluster_hostnames | length)
+        msg: >-
+          k3s nodes must have unique hostnames. Found a duplicate in:
+          {{ cluster_hostnames | unique }}. Each node registers in the cluster
+          keyed by its hostname, so matching hostnames prevent nodes from joining.
+      vars:
+        cluster_hostnames: >-
+          {{
+            groups['k3s_cluster']
+            | map('extract', hostvars, 'ansible_hostname')
+            | list
+          }}
+      run_once: true
+      when: "'k3s_cluster' in groups"
+
 - name: Prepare Proxmox cluster
   hosts: proxmox
   gather_facts: true