fix(verify): retry the nginx VIP assert on transient ARP miss

- convert the verify_from_outside URI assert to register/until/retries/delay
- retry until the MetalLB VIP returns HTTP 200 with the welcome page
- cover the host-only L2 announcement race that failed CI on the shared runner
- add a regression test asserting the retry wiring and hook it into pre-commit

.github/scripts/test-kube-vip-deploy-condition.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+"""Regression test for the kube-vip deploy conditions.
+
+The control-plane VIP (roles/k3s_server/tasks/vip.yml) and the kube-vip
+service load balancer (roles/k3s_server/tasks/kube-vip.yml) are both included
+from roles/k3s_server/tasks/main.yml. Before this switch existed the VIP
+include had no gate and always ran, so a user who wanted no kube-vip (single
+node or external LB) could not opt out.
+
+This test loads the real `when` expressions from the k3s_server task file and
+evaluates them against representative variable sets, asserting that:
+
+- the control-plane VIP is deployed only when kube_vip_enabled is true;
+- the service load balancer is deployed only when kube_vip_enabled is true
+  AND kube_vip_lb_ip_range is defined.
+"""
+
+from __future__ import print_function
+
+import os
+import re
+import subprocess
+
+import yaml
+from jinja2 import Environment
+
+VIP_WHEN = "kube_vip_enabled"
+KUBE_VIP_WHEN = "kube_vip_enabled and kube_vip_lb_ip_range is defined"
+
+
+def repo_root():
+    return subprocess.check_output(
+        ["git", "rev-parse", "--show-toplevel"], text=True
+    ).strip()
+
+
+def fail(message):
+    raise SystemExit("kube-vip deploy condition test failed: " + message)
+
+
+def extract_when(path, task_name):
+    """Return the `when:` expression string for the named task."""
+    with open(path, encoding="utf-8") as handle:
+        doc = yaml.safe_load(handle)
+    for task in doc:
+        if task.get("name") == task_name:
+            when = task.get("when")
+            return (when or "").strip()
+    return None
+
+
+def evaluate(when, variables):
+    """Evaluate a `when` expression against variables using Jinja2."""
+    env = Environment()
+
+    def fake_bool(value):
+        # Minimal stand-in for Ansible's truthiness filter used by `| bool`.
+        if isinstance(value, bool):
+            return value
+        if value is None:
+            return False
+        return str(value).lower() in ("1", "true", "yes", "on")
+
+    env.filters["bool"] = fake_bool
+    template = env.from_string("{{ " + when + " }}")
+    rendered = template.render(**variables)
+    # The expression renders to the literal strings "True"/"False".
+    if rendered == "True":
+        return True
+    if rendered == "False":
+        return False
+    fail("condition did not render to a boolean: {0!r}".format(rendered))
+
+
+def assert_deployment(when, variables, expected, label):
+    result = evaluate(when, variables)
+    verdict = "deploy" if result else "skip"
+    expected_verdict = "deploy" if expected else "skip"
+    if result != expected:
+        fail(
+            "{0}: expected to {1} but the condition chose to {2} "
+            "(vars: {3})".format(label, expected_verdict, verdict, variables)
+        )
+
+
+def scenarios():
+    """Yield (variables, expected_vip, expected_service_lb, label) pairs."""
+    yield (
+        # Default inventory: kube-vip enabled, no service LB range.
+        {
+            "kube_vip_enabled": True,
+        },
+        True,
+        False,
+        "default: kube-vip VIP only",
+    )
+    yield (
+        # kube-vip owns the service LB range too.
+        {
+            "kube_vip_enabled": True,
+            "kube_vip_lb_ip_range": "192.168.30.80-192.168.30.90",
+        },
+        True,
+        True,
+        "kube-vip VIP and service LB",
+    )
+    yield (
+        # Explicitly disabled, no LB range.
+        {
+            "kube_vip_enabled": False,
+        },
+        False,
+        False,
+        "kube_vip_enabled: false",
+    )
+    yield (
+        # Explicitly disabled even when the LB range is present.
+        {
+            "kube_vip_enabled": False,
+            "kube_vip_lb_ip_range": "192.168.30.80-192.168.30.90",
+        },
+        False,
+        False,
+        "kube_vip_enabled: false despite LB range",
+    )
+
+
+def main():
+    root = repo_root()
+    server_tasks = os.path.join(root, "roles", "k3s_server", "tasks", "main.yml")
+
+    vip_when = extract_when(server_tasks, "Deploy vip manifest")
+    kube_vip_when = extract_when(server_tasks, "Deploy kube-vip manifest")
+
+    if vip_when is None:
+        fail("could not find 'Deploy vip manifest' when condition")
+    if kube_vip_when is None:
+        fail("could not find 'Deploy kube-vip manifest' when condition")
+
+    if vip_when != VIP_WHEN:
+        fail(
+            "k3s_server/tasks/main.yml 'Deploy vip manifest' when condition "
+            "changed unexpectedly:\n"
+            "  expected: {0}\n  got:      {1}".format(VIP_WHEN, vip_when)
+        )
+    if kube_vip_when != KUBE_VIP_WHEN:
+        fail(
+            "k3s_server/tasks/main.yml 'Deploy kube-vip manifest' when "
+            "condition changed unexpectedly:\n"
+            "  expected: {0}\n  got:      {1}".format(KUBE_VIP_WHEN, kube_vip_when)
+        )
+
+    for variables, expected_vip, expected_service_lb, label in scenarios():
+        assert_deployment(vip_when, variables, expected_vip, "vip " + label)
+        assert_deployment(
+            kube_vip_when,
+            variables,
+            expected_service_lb,
+            "service_lb " + label,
+        )
+
+    print("kube-vip deploy condition regression test passed for all scenarios")
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/test-metallb-deploy-condition.py

@@ -10,7 +10,8 @@ scenario.
 
 This test loads the real `when` expressions from both task files and evaluates
 them against representative variable sets, asserting MetalLB is deployed in
-every topology except when kube-vip owns the VIP range or Cilium BGP is enabled.
+every topology except when MetalLB is explicitly disabled, kube-vip owns the
+VIP range, or Cilium BGP is enabled.
 """
 
 from __future__ import print_function
@@ -23,6 +24,7 @@ import yaml
 from jinja2 import Environment
 
 METALLB_WHEN = (
+    "metal_lb_enabled and "
     "kube_vip_lb_ip_range is not defined and "
     "not (cilium_bgp | default(false) | bool)"
 )
@@ -88,6 +90,7 @@ def scenarios():
     yield (
         # Default Flannel inventory (all.yml sets cilium_bgp: false).
         {
+            "metal_lb_enabled": True,
             "cilium_bgp": False,
             "cilium_iface": None,
         },
@@ -98,6 +101,7 @@ def scenarios():
         # Calico CNI with no Cilium variable in scope (issue #644): cilium_bgp
         # is genuinely undefined, so `default(false)` must keep MetalLB on.
         {
+            "metal_lb_enabled": True,
             "calico_iface": "eth1",
         },
         True,
@@ -106,6 +110,7 @@ def scenarios():
     yield (
         # Cilium CNI with BGP disabled: MetalLB must still be deployed.
         {
+            "metal_lb_enabled": True,
             "cilium_bgp": False,
             "cilium_iface": "eth1",
         },
@@ -115,6 +120,7 @@ def scenarios():
     yield (
         # Cilium CNI with BGP enabled: Cilium provides the LB, skip MetalLB.
         {
+            "metal_lb_enabled": True,
             "cilium_bgp": True,
             "cilium_iface": "eth1",
         },
@@ -124,12 +130,21 @@ def scenarios():
     yield (
         # kube-vip is the load balancer provider: skip MetalLB.
         {
+            "metal_lb_enabled": True,
             "kube_vip_lb_ip_range": "192.168.30.80-192.168.30.90",
             "cilium_bgp": False,
         },
         False,
         "kube-vip owns the VIP range",
     )
+    yield (
+        # MetalLB explicitly disabled (external LB / single node): skip.
+        {
+            "metal_lb_enabled": False,
+        },
+        False,
+        "metal_lb_enabled: false (external LB)",
+    )
 
 
 def main():

.github/scripts/test-verify-nginx-assert.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Regression test for the verify_from_outside nginx HTTP assert.
+
+The "Assert that the nginx welcome page is available" task in
+molecule/resources/verify_from_outside/tasks/test/deploy-example.yml performs
+a plain HTTP GET (ansible.builtin.uri) against the MetalLB VIP that the load
+balancer pool assigned, reaching it from the CI runner over the VirtualBox
+host-only network.
+
+The MetalLB speaker announces the VIP on the host-only network, and the runner
+may briefly not see it in its ARP table yet. That manifests as a single-shot
+"status -1 / No route to host" failure that fails the whole verify play even
+though the cluster is healthy. Like the sibling load-balancer address wait and
+the k3s_server_post metallb waits, the assert must retry until it observes
+HTTP 200 with the welcome page.
+
+A bare assert with no retry would otherwise fail an otherwise-green cluster on
+a transient announcement miss.
+"""
+
+from __future__ import print_function
+
+import os
+import subprocess
+
+import yaml
+
+
+def repo_root():
+    return subprocess.check_output(
+        ["git", "rev-parse", "--show-toplevel"], text=True
+    ).strip()
+
+
+def fail(message):
+    raise SystemExit("verify nginx assert test failed: " + message)
+
+
+def find_task(play, name):
+    block = play.get("block")
+    if not isinstance(block, list):
+        fail("expected a block list of tasks")
+    for entry in block:
+        if entry.get("name") == name:
+            return entry
+    fail("could not find the '{0}' task".format(name))
+    return None
+
+
+def check_retry_wiring(task, name):
+    if not task.get("register"):
+        fail("{0} does not register a result; without retry wiring a transient "
+             "MetalLB VIP announcement miss fails the verify play".format(name))
+    until = task.get("until")
+    if not until:
+        fail("{0} does not retry; a transient MetalLB VIP announcement miss "
+             "on the host-only network would fail the verify play".format(name))
+    if isinstance(until, str):
+        until = [until]
+    joined = " | ".join(str(expr) for expr in until)
+    if "result.status == 200" not in joined:
+        fail("{0} does not retry until the HTTP status is 200".format(name))
+    if "Welcome to nginx!" not in joined:
+        fail("{0} does not verify the welcome page content".format(name))
+    if task.get("retries") is None:
+        fail("{0} is missing retries".format(name))
+    if task.get("delay") is None:
+        fail("{0} is missing delay".format(name))
+
+
+def main():
+    task_file = os.path.join(
+        repo_root(),
+        "molecule", "resources", "verify_from_outside", "tasks", "test",
+        "deploy-example.yml",
+    )
+    with open(task_file, encoding="utf-8") as handle:
+        plays = yaml.safe_load(handle)
+
+    play = plays[0]
+    task = find_task(play, "Assert that the nginx welcome page is available")
+
+    if not task.get("ansible.builtin.uri"):
+        fail("task does not use ansible.builtin.uri to reach the VIP")
+
+    check_retry_wiring(task, "Assert that the nginx welcome page is available")
+
+    print("verify nginx assert regression test passed")
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/lint.yml

@@ -68,7 +68,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # 7.0.1
       - name: Ensure SHA pinned actions
-        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@46cfe808a5f1588656ef299eedd0ce2fd7ec0dcc # 5.0.6
+        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@c5fc58bd0be7a4b94b73ce40250322d5b838a108 # 5.0.7
         with:
           allowlist: |
             aws-actions/

.pre-commit-config.yaml

@@ -130,6 +130,15 @@ repos:
           - PyYAML
         pass_filenames: false
         files: ^roles/k3s_server/tasks/main\.yml$|^roles/k3s_server_post/tasks/main\.yml$|^\.github/scripts/test-metallb-deploy-condition\.py$ # noqa yaml[line-length]
+      - id: kube-vip-deploy-condition-test
+        name: kube-vip deploy condition test
+        entry: python3 .github/scripts/test-kube-vip-deploy-condition.py
+        language: python
+        additional_dependencies:
+          - Jinja2>=3.1
+          - PyYAML
+        pass_filenames: false
+        files: ^roles/k3s_server/tasks/main\.yml$|^\.github/scripts/test-kube-vip-deploy-condition\.py$ # noqa yaml[line-length]
       - id: default-interface-test
         name: default interface test
         entry: python3 .github/scripts/test-default-interface.py
@@ -138,3 +147,11 @@ repos:
           - Jinja2>=3.1
         pass_filenames: false
         files: ^inventory/sample/group_vars/all\.yml$|^\.github/scripts/test-default-interface\.py$
+      - id: verify-nginx-assert-test
+        name: verify nginx assert test
+        entry: python3 .github/scripts/test-verify-nginx-assert.py
+        language: python
+        additional_dependencies:
+          - PyYAML
+        pass_filenames: false
+        files: ^molecule/resources/verify_from_outside/tasks/test/deploy-example\.yml$|^\.github/scripts/test-verify-nginx-assert\.py$ # noqa yaml[line-length]

README.md

@@ -222,7 +222,9 @@ See the commands [here](https://technotim.com/posts/k3s-etcd-ansible/#testing-yo
 | `k3s_server` | `kube_vip_endpoint` | string | `~` | Not required | Overrides the internal address kube-vip binds/listens on, which can differ from the announced apiserver_endpoint for complex routing/tunnels. Defaults to apiserver_endpoint. |
 | `k3s_server` | `kube_vip_tag_version` | string | `v1.2.2` | Not required | Image tag for kube-vip |
 | `k3s_server` | `kube_vip_cloud_provider_tag_version` | string | `v0.0.12` | Not required | Tag for kube-vip-cloud-provider manifest when enable |
+| `k3s_server` | `kube_vip_enabled` | bool | `true` | Not required | Enable kube-vip install, covering both the control-plane VIP and the service load balancer. Set false to skip kube-vip entirely (single node or external LB). |
 | `k3s_server`, `k3_server_post` | `kube_vip_lb_ip_range` | string | `~` | Not required | IP range for kube-vip load balancer |
+| `k3s_server`, `k3s_server_post` | `metal_lb_enabled` | bool | `true` | Not required | Enable MetalLB install for service load balancing. Set false to skip MetalLB (external LB). |
 | `k3s_server`, `k3s_server_post` | `metal_lb_controller_tag_version` | string | `v0.16.0` | Not required | Image tag for MetalLB |
 | `k3s_server` | `metal_lb_speaker_tag_version` | string | `v0.16.0` | Not required | Image tag for MetalLB |
 | `k3s_server` | `metal_lb_type` | string | `native` | Not required | Use FRR mode or native. Valid values are `frr` and `native` |

inventory/sample/group_vars/all.yml

@@ -104,6 +104,10 @@ extra_agent_args: >-
 # image tag for kube-vip
 kube_vip_tag_version: v1.2.2
 
+# enable kube-vip (covers both the control-plane VIP and service load balancing)
+# set false for a single node or when an external load balancer is used
+kube_vip_enabled: true
+
 # tag for kube-vip-cloud-provider manifest
 # kube_vip_cloud_provider_tag_version: "v0.0.12"
 
@@ -111,6 +115,10 @@ kube_vip_tag_version: v1.2.2
 # (uncomment to use kube-vip for services instead of MetalLB)
 # kube_vip_lb_ip_range: "192.168.30.80-192.168.30.90"
 
+# enable MetalLB for service load balancing
+# set false when using an external load balancer
+metal_lb_enabled: true
+
 # metallb type frr or native
 metal_lb_type: native
 

molecule/resources/verify_from_outside/tasks/test/deploy-example.yml

@@ -69,7 +69,16 @@
         url: http://{{ nginx_lb_ip | ansible.utils.ipwrap }}:{{ port_ }}/
         return_content: true
       register: result
-      failed_when: "'Welcome to nginx!' not in result.content"
+      # The MetalLB speaker announces the VIP on the host-only network, and the
+      # runner may briefly not see it in its ARP table yet. Retry so a transient
+      # announcement miss does not fail an otherwise healthy cluster, matching
+      # the load-balancer address wait above and the k3s_server_post metallb waits.
+      until:
+        - result.status is defined
+        - result.status == 200
+        - "'Welcome to nginx!' in result.content"
+      retries: 30
+      delay: 5
       vars:
         port_: >-
           {{ nginx_services.resources[0].spec.ports[0].port }}

requirements.in

@@ -1,4 +1,4 @@
-ansible-core>=2.19.11
+ansible-core>=2.19.12
 jmespath>=1.1.0
 jsonpatch>=1.33
 kubernetes>=29.0.0

requirements.txt

@@ -6,7 +6,7 @@
 #
 ansible-compat==4.1.11
     # via molecule
-ansible-core==2.19.11
+ansible-core==2.19.12
     # via
     #   -r requirements.in
     #   ansible-compat

roles/k3s_server/defaults/main.yml

@@ -20,6 +20,9 @@ kube_vip_bgp_peeras: "64512"
 kube_vip_bgp_peers: []
 kube_vip_bgp_peers_groups: ['k3s_master']
 
+kube_vip_enabled: true
+
+metal_lb_enabled: true
 metal_lb_controller_tag_version: v0.16.0
 metal_lb_speaker_tag_version: v0.16.0
 metal_lb_type: native

roles/k3s_server/meta/main.yml

@@ -95,10 +95,25 @@ argument_specs:
         description: Tag for kube-vip-cloud-provider manifest when enabled
         default: v0.0.12
 
+      kube_vip_enabled:
+        description:
+          - Enable installing kube-vip.
+          - Covers both the control-plane VIP and the service load balancer.
+          - Set false to skip kube-vip entirely, for example on a single node or when an external load balancer is used.
+        default: true
+        type: bool
+
       kube_vip_lb_ip_range:
         description: IP range for kube-vip load balancer
         default: ~
 
+      metal_lb_enabled:
+        description:
+          - Enable installing MetalLB for service load balancing.
+          - Set false to skip MetalLB, for example when an external load balancer is used.
+        default: true
+        type: bool
+
       metal_lb_controller_tag_version:
         description: Image tag for MetalLB
         default: v0.16.0

roles/k3s_server/tasks/main.yml

@@ -24,19 +24,21 @@
 
 - name: Deploy vip manifest
   ansible.builtin.include_tasks: vip.yml
+  when: kube_vip_enabled
 - name: Deploy metallb manifest
   ansible.builtin.include_tasks: metallb.yml
   tags: metallb
-  # Deploy MetalLB unless kube-vip owns the load balancer IP range, or Cilium
-  # BGP is enabled (Cilium then provides its own load balancing). The cilium_bgp
-  # default keeps this safe when Cilium variables are not in scope at all (#644)
-  # while still deploying MetalLB when a non-BGP Cilium CNI is in use.
-  when: kube_vip_lb_ip_range is not defined and not (cilium_bgp | default(false) | bool)
+  # Deploy MetalLB unless explicitly disabled, kube-vip owns the load balancer
+  # IP range, or Cilium BGP is enabled (Cilium then provides its own load
+  # balancing). The cilium_bgp default keeps this safe when Cilium variables are
+  # not in scope at all (#644) while still deploying MetalLB when a non-BGP
+  # Cilium CNI is in use.
+  when: metal_lb_enabled and kube_vip_lb_ip_range is not defined and not (cilium_bgp | default(false) | bool)
 
 - name: Deploy kube-vip manifest
   ansible.builtin.include_tasks: kube-vip.yml
   tags: kubevip
-  when: kube_vip_lb_ip_range is defined
+  when: kube_vip_enabled and kube_vip_lb_ip_range is defined
 
 - name: Initialize and verify the K3s control plane
   any_errors_fatal: true

roles/k3s_server_post/defaults/main.yml

@@ -39,5 +39,6 @@ group_name_master: master
 metal_lb_mode: layer2
 metal_lb_available_timeout: 240s
 metal_lb_controller_tag_version: v0.16.0
+metal_lb_enabled: true
 metal_lb_interfaces: []
 metal_lb_ip_range: 192.168.30.80-192.168.30.90

roles/k3s_server_post/meta/main.yml

@@ -133,6 +133,13 @@ argument_specs:
         description: IP range for kube-vip load balancer
         default: ~
 
+      metal_lb_enabled:
+        description:
+          - Enable installing the MetalLB pool for service load balancing.
+          - Set false to skip MetalLB, for example when an external load balancer is used.
+        default: true
+        type: bool
+
       metal_lb_available_timeout:
         description: Wait for MetalLB resources
         default: 240s

roles/k3s_server_post/tasks/main.yml

@@ -12,11 +12,12 @@
 - name: Deploy metallb pool
   ansible.builtin.include_tasks: metallb.yml
   tags: metallb
-  # Deploy MetalLB unless kube-vip owns the load balancer IP range, or Cilium
-  # BGP is enabled (Cilium then provides its own load balancing). The cilium_bgp
-  # default keeps this safe when Cilium variables are not in scope at all (#644)
-  # while still deploying MetalLB when a non-BGP Cilium CNI is in use.
-  when: kube_vip_lb_ip_range is not defined and not (cilium_bgp | default(false) | bool)
+  # Deploy MetalLB unless explicitly disabled, kube-vip owns the load balancer
+  # IP range, or Cilium BGP is enabled (Cilium then provides its own load
+  # balancing). The cilium_bgp default keeps this safe when Cilium variables are
+  # not in scope at all (#644) while still deploying MetalLB when a non-BGP
+  # Cilium CNI is in use.
+  when: metal_lb_enabled and kube_vip_lb_ip_range is not defined and not (cilium_bgp | default(false) | bool)
 
 - name: Remove tmp directory used for manifests
   ansible.builtin.file: