test(molecule): verify upgraded cluster components

- Assert every node reports the expected K3s kubelet version
- Verify the active CNI (Flannel / Calico / Cilium) is Ready and runs
  the expected image tag, including Calico TigeraStatus Available
- Verify the active load balancer (MetalLB / kube-vip) runs the expected
  image tags and that MetalLB is absent when kube-vip is active
- Assert no Flannel DaemonSet remains when Calico or Cilium is enabled
- Assert the example LoadBalancer address falls inside the configured
  pool range
- Add a manifest-only Cilium BGP regression test that renders the v2
  template with zero, one, and multiple neighbors and rejects any v2alpha1
  or CiliumBGPPeeringPolicy output

.github/scripts/test-cilium-bgp-manifest.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""Render the Cilium BGP CRD template and assert it uses the v2 API.
+
+This is a manifest-only regression test used where no real BGP peer is
+available. It renders roles/k3s_server_post/templates/cilium.crs.j2 with
+zero, one, and multiple neighbors, then checks that the output:
+  - never contains CiliumBGPPeeringPolicy or cilium.io/v2alpha1
+  - emits the Cilium v2 BGP resources
+  - emits deterministic DNS-safe peer and instance names
+  - advertises Pod CIDRs only when cilium_exportPodCIDR is true
+"""
+
+from __future__ import print_function
+
+import os
+import subprocess
+
+from jinja2 import Environment, FileSystemLoader, StrictUndefined
+
+
+def repo_root():
+    return subprocess.check_output(
+        ["git", "rev-parse", "--show-toplevel"], text=True
+    ).strip()
+
+
+def fail(message):
+    raise SystemExit("Cilium BGP manifest test failed: " + message)
+
+
+def render(env, extra_vars):
+    base_vars = {
+        "cilium_bgp_my_asn": "64513",
+        "cilium_bgp_peer_asn": "64512",
+        "cilium_bgp_peer_address": "192.168.30.1",
+        "cilium_exportPodCIDR": True,
+        "cilium_bgp_lb_cidr": "192.168.31.0/24",
+    }
+    base_vars.update(extra_vars)
+    template = env.get_template("cilium.crs.j2")
+    return template.render(**base_vars)
+
+
+def check_common(output):
+    if "cilium.io/v2alpha1" in output:
+        fail("rendered output still contains cilium.io/v2alpha1")
+    if "kind: CiliumBGPPeeringPolicy" in output:
+        fail("rendered output still contains CiliumBGPPeeringPolicy")
+    for kind in (
+        "CiliumBGPPeerConfig",
+        "CiliumBGPClusterConfig",
+        "CiliumBGPAdvertisement",
+        "CiliumLoadBalancerIPPool",
+    ):
+        if ("kind: " + kind) not in output:
+            fail("rendered output is missing kind: " + kind)
+
+
+def main():
+    root = repo_root()
+    template_dir = os.path.join(
+        root, "roles", "k3s_server_post", "templates"
+    )
+    env = Environment(
+        loader=FileSystemLoader(template_dir), undefined=StrictUndefined
+    )
+
+    # Zero neighbors -> fall back to the single default peer.
+    output = render(env, {"_cilium_bgp_neighbors": []})
+    check_common(output)
+    if "peer-64512-1" not in output:
+        fail("default single peer name was not rendered")
+    if "peerAddress: 192.168.30.1" not in output:
+        fail("default peer address was not rendered")
+    if 'advertisementType: "PodCIDR"' not in output:
+        fail("PodCIDR advertisement missing when exportPodCIDR is true")
+
+    # One neighbor via the merged list.
+    output = render(
+        env,
+        {"_cilium_bgp_neighbors": [{"peer_address": "10.0.0.1", "peer_asn": "65001"}]},
+    )
+    check_common(output)
+    if "peer-65001-1" not in output:
+        fail("single merged peer name was not rendered")
+    if "peerAddress: 10.0.0.1" not in output:
+        fail("single merged peer address was not rendered")
+
+    # Multiple neighbors.
+    output = render(
+        env,
+        {
+            "_cilium_bgp_neighbors": [
+                {"peer_address": "10.0.0.1", "peer_asn": "65001"},
+                {"peer_address": "10.0.0.2", "peer_asn": "65002"},
+            ]
+        },
+    )
+    check_common(output)
+    if "peer-65001-1" not in output or "peer-65002-2" not in output:
+        fail("multiple merged peer names were not rendered")
+    if "peerAddress: 10.0.0.2" not in output:
+        fail("second merged peer address was not rendered")
+
+    # exportPodCIDR false -> no PodCIDR advertisement, service remains.
+    output = render(
+        env, {"_cilium_bgp_neighbors": [], "cilium_exportPodCIDR": False}
+    )
+    check_common(output)
+    if 'advertisementType: "PodCIDR"' in output:
+        fail("PodCIDR advertisement present when exportPodCIDR is false")
+    if 'advertisementType: "Service"' not in output:
+        fail("Service advertisement missing when exportPodCIDR is false")
+
+    # Load balancer pools: CIDR and start/stop forms.
+    output = render(env, {"_cilium_bgp_neighbors": []})
+    if "cidr: 192.168.31.0/24" not in output:
+        fail("CIDR load balancer pool was not rendered")
+    output = render(
+        env,
+        {
+            "_cilium_bgp_neighbors": [],
+            "cilium_bgp_lb_cidr": "192.168.31.80-192.168.31.90",
+        },
+    )
+    check_common(output)
+    if "start: 192.168.31.80" not in output or "stop: 192.168.31.90" not in output:
+        fail("start/stop load balancer pool was not rendered")
+
+    print("Cilium BGP manifest regression test passed")
+
+
+if __name__ == "__main__":
+    main()

.pre-commit-config.yaml

@@ -62,3 +62,11 @@ repos:
         language: system
         pass_filenames: false
         files: ^roles/k3s_server/tasks/(main|join_master)\.yml$|^\.github/scripts/test-k3s-server-bootstrap\.sh$
+      - id: cilium-bgp-manifest-test
+        name: Cilium BGP manifest test
+        entry: python3 .github/scripts/test-cilium-bgp-manifest.py
+        language: python
+        additional_dependencies:
+          - Jinja2>=3.1
+        pass_filenames: false
+        files: ^roles/k3s_server_post/templates/cilium\.crs\.j2$|^\.github/scripts/test-cilium-bgp-manifest\.py$

molecule/resources/verify_from_outside/tasks/main.yml

@@ -7,6 +7,8 @@
       ansible.builtin.import_tasks: kubecfg-fetch.yml
     - name: "TEST CASE: Get nodes"
       ansible.builtin.include_tasks: test/get-nodes.yml
+    - name: "TEST CASE: Verify components"
+      ansible.builtin.include_tasks: test/verify-components.yml
     - name: "TEST CASE: Deploy example"
       ansible.builtin.include_tasks: test/deploy-example.yml
   always:

molecule/resources/verify_from_outside/tasks/test/deploy-example.yml

@@ -43,6 +43,28 @@
           {{ nginx_services.resources[0].status.loadBalancer.ingress[0].ip }}
         port_: >-
           {{ nginx_services.resources[0].spec.ports[0].port }}
+
+    - name: Assert that the load balancer address is within the configured pool
+      ansible.builtin.assert:
+        that: lb_addr_in_range
+        success_msg: "LoadBalancer address {{ ip_value }} is in the configured range"
+        fail_msg: >-
+          LoadBalancer address {{ ip_value }} is not in the configured
+          range {{ lb_range }}
+      vars:
+        ip_value: >-
+          {{ nginx_services.resources[0].status.loadBalancer.ingress[0].ip }}
+        lb_range: >-
+          {{ kube_vip_lb_ip_range | default(metal_lb_ip_range, true) }}
+        lb_int: "{{ ip_value | ansible.utils.ipaddr('int') }}"
+        start_int: "{{ lb_range.split('-')[0] | ansible.utils.ipaddr('int') }}"
+        end_int: "{{ lb_range.split('-')[1] | ansible.utils.ipaddr('int') }}"
+        lb_addr_in_range: >-
+          {{
+            (lb_range.split('-') | length == 2) and
+            (lb_int >= start_int) and
+            (lb_int <= end_int)
+          }}
   # Deactivated linter rules:
   #   - jinja[invalid]: As of version 6.6.0, ansible-lint complains that the input to ipwrap
   #                     would be undefined. This will not be the case during playbook execution.

molecule/resources/verify_from_outside/tasks/test/verify-components.yml

@@ -0,0 +1,288 @@
+---
+# Scenario-aware verification of cluster components and their live image tags.
+# Branches on the active CNI (flannel / calico / cilium) and load balancer
+# (MetalLB / kube-vip) using the same variable names the install roles use.
+- name: Verify cluster components report expected versions
+  block:
+    - name: Get all nodes with their kubelet versions
+      kubernetes.core.k8s_info:
+        kind: node
+        kubeconfig: "{{ kubecfg_path }}"
+      register: verify_nodes
+
+    - name: Assert each node reports the expected Kubernetes version
+      ansible.builtin.assert:
+        that: item.status.nodeInfo.kubeletVersion == k3s_version
+        success_msg: "{{ item.metadata.name }} reports {{ k3s_version }}"
+        fail_msg: >-
+          {{ item.metadata.name }} reports
+          {{ item.status.nodeInfo.kubeletVersion }},
+          expected {{ k3s_version }}
+      loop: "{{ verify_nodes.resources }}"
+      loop_control:
+        label: "{{ item.metadata.name }}"
+
+    - name: Verify Flannel is the active CNI
+      when: cilium_iface is not defined and calico_iface is not defined
+      block:
+        - name: Get the Flannel DaemonSet
+          kubernetes.core.k8s_info:
+            kind: DaemonSet
+            name: kube-flannel-ds
+            namespace: kube-flannel
+            kubeconfig: "{{ kubecfg_path }}"
+          register: flannel_ds
+
+        - name: Assert the Flannel DaemonSet is scheduled and Ready
+          ansible.builtin.assert:
+            that:
+              - flannel_ds.resources | length == 1
+              - flannel_ds.resources[0].status.desiredNumberScheduled > 0
+              - flannel_ds.resources[0].status.numberReady == flannel_ds.resources[0].status.desiredNumberScheduled
+            success_msg: "Flannel DaemonSet is Ready"
+            fail_msg: "Flannel DaemonSet is not fully Ready"
+
+    - name: Verify Calico is the active CNI
+      when: calico_iface is defined
+      block:
+        - name: Get the Calico node DaemonSet image
+          kubernetes.core.k8s_info:
+            kind: DaemonSet
+            name: calico-node
+            namespace: calico-system
+            kubeconfig: "{{ kubecfg_path }}"
+          register: calico_node_ds
+
+        - name: Assert the Calico node image uses the expected tag
+          ansible.builtin.assert:
+            that:
+              - calico_node_ds.resources | length == 1
+              - calico_node_image | regex_search(':' ~ calico_tag)
+            success_msg: "Calico node image uses tag {{ calico_tag }}"
+            fail_msg: >-
+              Calico node image {{ calico_node_image }},
+              expected {{ calico_tag }}
+          vars:
+            calico_node_image: "{{ calico_node_ds.resources[0].spec.template.spec.containers[0].image }}"
+
+        - name: Get Calico TigeraStatus for calico and apiserver
+          kubernetes.core.k8s_info:
+            api_version: operator.tigera.io/v1
+            kind: Tigerastatus
+            name: "{{ item }}"
+            kubeconfig: "{{ kubecfg_path }}"
+          register: calico_tigerastatus
+          loop:
+            - calico
+            - apiserver
+          loop_control:
+            label: "Tigerastatus/{{ item }}"
+
+        - name: Assert Calico TigeraStatus reports Available
+          ansible.builtin.assert:
+            that: >-
+              item.resources | length == 1 and
+              (item.resources[0].status.conditions
+               | selectattr('type', 'equalto', 'Available')
+               | map(attribute='status') | first | default('')) == 'True'
+            success_msg: "Tigerastatus {{ item.resources[0].metadata.name }} is Available"
+            fail_msg: "Tigerastatus is not Available"
+          loop: "{{ calico_tigerastatus.results }}"
+          loop_control:
+            label: "Tigerastatus Available"
+
+        - name: Get any Flannel DaemonSets with Calico enabled
+          kubernetes.core.k8s_info:
+            kind: DaemonSet
+            namespace: kube-flannel
+            kubeconfig: "{{ kubecfg_path }}"
+          register: no_flannel_ds
+        - name: Assert there are no Flannel DaemonSets
+          ansible.builtin.assert:
+            that: no_flannel_ds.resources | length == 0
+            success_msg: "No Flannel DaemonSet present with Calico"
+            fail_msg: "A Flannel DaemonSet exists alongside Calico"
+
+    - name: Verify Cilium is the active CNI
+      when: cilium_iface is defined
+      block:
+        - name: Get the Cilium agent and operator images
+          kubernetes.core.k8s_info:
+            kind: "{{ item.kind }}"
+            name: "{{ item.name }}"
+            namespace: kube-system
+            kubeconfig: "{{ kubecfg_path }}"
+          register: cilium_info
+          loop:
+            - { kind: DaemonSet, name: cilium }
+            - { kind: Deployment, name: cilium-operator }
+          loop_control:
+            label: "{{ item.kind }}/{{ item.name }}"
+
+        - name: Assert Cilium agent and operator use the expected image tag
+          ansible.builtin.assert:
+            that:
+              - cilium_agent_image | regex_search(':' ~ cilium_tag)
+              - cilium_operator_image | regex_search(':' ~ cilium_tag)
+            success_msg: "Cilium agent and operator use {{ cilium_tag }}"
+            fail_msg: >-
+              Cilium agent {{ cilium_agent_image }},
+              operator {{ cilium_operator_image }},
+              expected {{ cilium_tag }}
+          vars:
+            cilium_agent_image: >-
+              {{ (cilium_info.results
+                  | selectattr('resources', 'defined')
+                  | map(attribute='resources')
+                  | list
+                  | map(attribute='0')
+                  | selectattr('kind', 'equalto', 'DaemonSet')
+                  | list)[0].spec.template.spec.containers[0].image }}
+            cilium_operator_image: >-
+              {{ (cilium_info.results
+                  | selectattr('resources', 'defined')
+                  | map(attribute='resources')
+                  | list
+                  | map(attribute='0')
+                  | selectattr('kind', 'equalto', 'Deployment')
+                  | list)[0].spec.template.spec.containers[0].image }}
+
+        - name: Get Hubble relay and UI deployments when enabled
+          kubernetes.core.k8s_info:
+            kind: Deployment
+            name: "{{ item }}"
+            namespace: kube-system
+            kubeconfig: "{{ kubecfg_path }}"
+          register: hubble_info
+          loop:
+            - hubble-relay
+            - hubble-ui
+          loop_control:
+            label: "Deployment/{{ item }}"
+          when: cilium_hubble | bool
+
+        - name: Assert Hubble components are Ready when enabled
+          ansible.builtin.assert:
+            that:
+              - item.resources | length == 1
+              - item.resources[0].status.readyReplicas | default(0) >= 1
+            success_msg: "Hubble deployment {{ item.resources[0].metadata.name }} is Ready"
+            fail_msg: "Hubble deployment is not Ready"
+          loop: "{{ hubble_info.results }}"
+          loop_control:
+            label: "Hubble deployment"
+          when: cilium_hubble | bool
+
+        - name: Get any Flannel DaemonSets with Cilium enabled
+          kubernetes.core.k8s_info:
+            kind: DaemonSet
+            namespace: kube-flannel
+            kubeconfig: "{{ kubecfg_path }}"
+          register: no_flannel_ds_cilium
+        - name: Assert there are no Flannel DaemonSets
+          ansible.builtin.assert:
+            that: no_flannel_ds_cilium.resources | length == 0
+            success_msg: "No Flannel DaemonSet present with Cilium"
+            fail_msg: "A Flannel DaemonSet exists alongside Cilium"
+
+    - name: Verify MetalLB is the active load balancer
+      when: kube_vip_lb_ip_range is not defined
+      block:
+        - name: Get the MetalLB controller and speaker images
+          kubernetes.core.k8s_info:
+            kind: "{{ item.kind }}"
+            name: "{{ item.name }}"
+            namespace: metallb-system
+            kubeconfig: "{{ kubecfg_path }}"
+          register: metallb_info
+          loop:
+            - { kind: Deployment, name: controller }
+            - { kind: DaemonSet, name: speaker }
+          loop_control:
+            label: "{{ item.kind }}/{{ item.name }}"
+
+        - name: Assert MetalLB controller and speaker use the expected image tags
+          ansible.builtin.assert:
+            that:
+              - controller_image | regex_search(metal_lb_controller_tag_version)
+              - speaker_image | regex_search(metal_lb_speaker_tag_version)
+            success_msg: >-
+              MetalLB controller {{ metal_lb_controller_tag_version }},
+              speaker {{ metal_lb_speaker_tag_version }}
+            fail_msg: >-
+              MetalLB controller {{ controller_image }},
+              speaker {{ speaker_image }}
+          vars:
+            controller_image: >-
+              {{ (metallb_info.results
+                  | selectattr('resources', 'defined')
+                  | map(attribute='resources')
+                  | list
+                  | map(attribute='0')
+                  | selectattr('kind', 'equalto', 'Deployment')
+                  | list)[0].spec.template.spec.containers[0].image }}
+            speaker_image: >-
+              {{ (metallb_info.results
+                  | selectattr('resources', 'defined')
+                  | map(attribute='resources')
+                  | list
+                  | map(attribute='0')
+                  | selectattr('kind', 'equalto', 'DaemonSet')
+                  | list)[0].spec.template.spec.containers[0].image }}
+
+    - name: Verify kube-vip is the active load balancer
+      when: kube_vip_lb_ip_range is defined
+      block:
+        - name: Get the kube-vip and cloud provider images
+          kubernetes.core.k8s_info:
+            kind: "{{ item.kind }}"
+            name: "{{ item.name }}"
+            namespace: kube-system
+            kubeconfig: "{{ kubecfg_path }}"
+          register: kubevip_info
+          loop:
+            - { kind: DaemonSet, name: kube-vip-ds }
+            - { kind: Deployment, name: kube-vip-cloud-provider }
+          loop_control:
+            label: "{{ item.kind }}/{{ item.name }}"
+
+        - name: Assert the kube-vip and cloud provider image tags
+          ansible.builtin.assert:
+            that:
+              - kubevip_image | regex_search(':' ~ kube_vip_tag_version)
+              - cloud_provider_image | regex_search(kube_vip_cloud_provider_tag_version)
+            success_msg: >-
+              kube-vip {{ kube_vip_tag_version }},
+              cloud provider {{ kube_vip_cloud_provider_tag_version }}
+            fail_msg: >-
+              kube-vip {{ kubevip_image }},
+              cloud provider {{ cloud_provider_image }}
+          vars:
+            kubevip_image: >-
+              {{ (kubevip_info.results
+                  | selectattr('resources', 'defined')
+                  | map(attribute='resources')
+                  | list
+                  | map(attribute='0')
+                  | selectattr('kind', 'equalto', 'DaemonSet')
+                  | list)[0].spec.template.spec.containers[0].image }}
+            cloud_provider_image: >-
+              {{ (kubevip_info.results
+                  | selectattr('resources', 'defined')
+                  | map(attribute='resources')
+                  | list
+                  | map(attribute='0')
+                  | selectattr('kind', 'equalto', 'Deployment')
+                  | list)[0].spec.template.spec.containers[0].image }}
+
+        - name: Get the MetalLB namespace with kube-vip enabled
+          kubernetes.core.k8s_info:
+            kind: Namespace
+            name: metallb-system
+            kubeconfig: "{{ kubecfg_path }}"
+          register: metallb_absent
+        - name: Assert the MetalLB namespace does not exist
+          ansible.builtin.assert:
+            that: metallb_absent.resources | length == 0
+            success_msg: "MetalLB is not installed with kube-vip"
+            fail_msg: "MetalLB namespace exists alongside kube-vip"