From ae250652f907b003a004b84c9365c045b675e2e7 Mon Sep 17 00:00:00 2001 From: Timothy Stewart Date: Sat, 1 Aug 2026 21:33:45 -0500 Subject: [PATCH] test(molecule): verify upgraded cluster components - Assert every node reports the expected K3s kubelet version - Verify the active CNI (Flannel / Calico / Cilium) is Ready and runs the expected image tag, including Calico TigeraStatus Available - Verify the active load balancer (MetalLB / kube-vip) runs the expected image tags and that MetalLB is absent when kube-vip is active - Assert no Flannel DaemonSet remains when Calico or Cilium is enabled - Assert the example LoadBalancer address falls inside the configured pool range - Add a manifest-only Cilium BGP regression test that renders the v2 template with zero, one, and multiple neighbors and rejects any v2alpha1 or CiliumBGPPeeringPolicy output --- .github/scripts/test-cilium-bgp-manifest.py | 134 ++++++++ .pre-commit-config.yaml | 8 + .../verify_from_outside/tasks/main.yml | 2 + .../tasks/test/deploy-example.yml | 22 ++ .../tasks/test/verify-components.yml | 288 ++++++++++++++++++ 5 files changed, 454 insertions(+) create mode 100644 .github/scripts/test-cilium-bgp-manifest.py create mode 100644 molecule/resources/verify_from_outside/tasks/test/verify-components.yml diff --git a/.github/scripts/test-cilium-bgp-manifest.py b/.github/scripts/test-cilium-bgp-manifest.py new file mode 100644 index 0000000..9701385 --- /dev/null +++ b/.github/scripts/test-cilium-bgp-manifest.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Render the Cilium BGP CRD template and assert it uses the v2 API. + +This is a manifest-only regression test used where no real BGP peer is +available. It renders roles/k3s_server_post/templates/cilium.crs.j2 with +zero, one, and multiple neighbors, then checks that the output: + - never contains CiliumBGPPeeringPolicy or cilium.io/v2alpha1 + - emits the Cilium v2 BGP resources + - emits deterministic DNS-safe peer and instance names + - advertises Pod CIDRs only when cilium_exportPodCIDR is true +""" + +from __future__ import print_function + +import os +import subprocess + +from jinja2 import Environment, FileSystemLoader, StrictUndefined + + +def repo_root(): + return subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], text=True + ).strip() + + +def fail(message): + raise SystemExit("Cilium BGP manifest test failed: " + message) + + +def render(env, extra_vars): + base_vars = { + "cilium_bgp_my_asn": "64513", + "cilium_bgp_peer_asn": "64512", + "cilium_bgp_peer_address": "192.168.30.1", + "cilium_exportPodCIDR": True, + "cilium_bgp_lb_cidr": "192.168.31.0/24", + } + base_vars.update(extra_vars) + template = env.get_template("cilium.crs.j2") + return template.render(**base_vars) + + +def check_common(output): + if "cilium.io/v2alpha1" in output: + fail("rendered output still contains cilium.io/v2alpha1") + if "kind: CiliumBGPPeeringPolicy" in output: + fail("rendered output still contains CiliumBGPPeeringPolicy") + for kind in ( + "CiliumBGPPeerConfig", + "CiliumBGPClusterConfig", + "CiliumBGPAdvertisement", + "CiliumLoadBalancerIPPool", + ): + if ("kind: " + kind) not in output: + fail("rendered output is missing kind: " + kind) + + +def main(): + root = repo_root() + template_dir = os.path.join( + root, "roles", "k3s_server_post", "templates" + ) + env = Environment( + loader=FileSystemLoader(template_dir), undefined=StrictUndefined + ) + + # Zero neighbors -> fall back to the single default peer. + output = render(env, {"_cilium_bgp_neighbors": []}) + check_common(output) + if "peer-64512-1" not in output: + fail("default single peer name was not rendered") + if "peerAddress: 192.168.30.1" not in output: + fail("default peer address was not rendered") + if 'advertisementType: "PodCIDR"' not in output: + fail("PodCIDR advertisement missing when exportPodCIDR is true") + + # One neighbor via the merged list. + output = render( + env, + {"_cilium_bgp_neighbors": [{"peer_address": "10.0.0.1", "peer_asn": "65001"}]}, + ) + check_common(output) + if "peer-65001-1" not in output: + fail("single merged peer name was not rendered") + if "peerAddress: 10.0.0.1" not in output: + fail("single merged peer address was not rendered") + + # Multiple neighbors. + output = render( + env, + { + "_cilium_bgp_neighbors": [ + {"peer_address": "10.0.0.1", "peer_asn": "65001"}, + {"peer_address": "10.0.0.2", "peer_asn": "65002"}, + ] + }, + ) + check_common(output) + if "peer-65001-1" not in output or "peer-65002-2" not in output: + fail("multiple merged peer names were not rendered") + if "peerAddress: 10.0.0.2" not in output: + fail("second merged peer address was not rendered") + + # exportPodCIDR false -> no PodCIDR advertisement, service remains. + output = render( + env, {"_cilium_bgp_neighbors": [], "cilium_exportPodCIDR": False} + ) + check_common(output) + if 'advertisementType: "PodCIDR"' in output: + fail("PodCIDR advertisement present when exportPodCIDR is false") + if 'advertisementType: "Service"' not in output: + fail("Service advertisement missing when exportPodCIDR is false") + + # Load balancer pools: CIDR and start/stop forms. + output = render(env, {"_cilium_bgp_neighbors": []}) + if "cidr: 192.168.31.0/24" not in output: + fail("CIDR load balancer pool was not rendered") + output = render( + env, + { + "_cilium_bgp_neighbors": [], + "cilium_bgp_lb_cidr": "192.168.31.80-192.168.31.90", + }, + ) + check_common(output) + if "start: 192.168.31.80" not in output or "stop: 192.168.31.90" not in output: + fail("start/stop load balancer pool was not rendered") + + print("Cilium BGP manifest regression test passed") + + +if __name__ == "__main__": + main() diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b3433e1..e94cb72 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -62,3 +62,11 @@ repos: language: system pass_filenames: false files: ^roles/k3s_server/tasks/(main|join_master)\.yml$|^\.github/scripts/test-k3s-server-bootstrap\.sh$ + - id: cilium-bgp-manifest-test + name: Cilium BGP manifest test + entry: python3 .github/scripts/test-cilium-bgp-manifest.py + language: python + additional_dependencies: + - Jinja2>=3.1 + pass_filenames: false + files: ^roles/k3s_server_post/templates/cilium\.crs\.j2$|^\.github/scripts/test-cilium-bgp-manifest\.py$ diff --git a/molecule/resources/verify_from_outside/tasks/main.yml b/molecule/resources/verify_from_outside/tasks/main.yml index 2f43a27..9528073 100644 --- a/molecule/resources/verify_from_outside/tasks/main.yml +++ b/molecule/resources/verify_from_outside/tasks/main.yml @@ -7,6 +7,8 @@ ansible.builtin.import_tasks: kubecfg-fetch.yml - name: "TEST CASE: Get nodes" ansible.builtin.include_tasks: test/get-nodes.yml + - name: "TEST CASE: Verify components" + ansible.builtin.include_tasks: test/verify-components.yml - name: "TEST CASE: Deploy example" ansible.builtin.include_tasks: test/deploy-example.yml always: diff --git a/molecule/resources/verify_from_outside/tasks/test/deploy-example.yml b/molecule/resources/verify_from_outside/tasks/test/deploy-example.yml index 13a1c4b..13104ca 100644 --- a/molecule/resources/verify_from_outside/tasks/test/deploy-example.yml +++ b/molecule/resources/verify_from_outside/tasks/test/deploy-example.yml @@ -43,6 +43,28 @@ {{ nginx_services.resources[0].status.loadBalancer.ingress[0].ip }} port_: >- {{ nginx_services.resources[0].spec.ports[0].port }} + + - name: Assert that the load balancer address is within the configured pool + ansible.builtin.assert: + that: lb_addr_in_range + success_msg: "LoadBalancer address {{ ip_value }} is in the configured range" + fail_msg: >- + LoadBalancer address {{ ip_value }} is not in the configured + range {{ lb_range }} + vars: + ip_value: >- + {{ nginx_services.resources[0].status.loadBalancer.ingress[0].ip }} + lb_range: >- + {{ kube_vip_lb_ip_range | default(metal_lb_ip_range, true) }} + lb_int: "{{ ip_value | ansible.utils.ipaddr('int') }}" + start_int: "{{ lb_range.split('-')[0] | ansible.utils.ipaddr('int') }}" + end_int: "{{ lb_range.split('-')[1] | ansible.utils.ipaddr('int') }}" + lb_addr_in_range: >- + {{ + (lb_range.split('-') | length == 2) and + (lb_int >= start_int) and + (lb_int <= end_int) + }} # Deactivated linter rules: # - jinja[invalid]: As of version 6.6.0, ansible-lint complains that the input to ipwrap # would be undefined. This will not be the case during playbook execution. diff --git a/molecule/resources/verify_from_outside/tasks/test/verify-components.yml b/molecule/resources/verify_from_outside/tasks/test/verify-components.yml new file mode 100644 index 0000000..3572b80 --- /dev/null +++ b/molecule/resources/verify_from_outside/tasks/test/verify-components.yml @@ -0,0 +1,288 @@ +--- +# Scenario-aware verification of cluster components and their live image tags. +# Branches on the active CNI (flannel / calico / cilium) and load balancer +# (MetalLB / kube-vip) using the same variable names the install roles use. +- name: Verify cluster components report expected versions + block: + - name: Get all nodes with their kubelet versions + kubernetes.core.k8s_info: + kind: node + kubeconfig: "{{ kubecfg_path }}" + register: verify_nodes + + - name: Assert each node reports the expected Kubernetes version + ansible.builtin.assert: + that: item.status.nodeInfo.kubeletVersion == k3s_version + success_msg: "{{ item.metadata.name }} reports {{ k3s_version }}" + fail_msg: >- + {{ item.metadata.name }} reports + {{ item.status.nodeInfo.kubeletVersion }}, + expected {{ k3s_version }} + loop: "{{ verify_nodes.resources }}" + loop_control: + label: "{{ item.metadata.name }}" + + - name: Verify Flannel is the active CNI + when: cilium_iface is not defined and calico_iface is not defined + block: + - name: Get the Flannel DaemonSet + kubernetes.core.k8s_info: + kind: DaemonSet + name: kube-flannel-ds + namespace: kube-flannel + kubeconfig: "{{ kubecfg_path }}" + register: flannel_ds + + - name: Assert the Flannel DaemonSet is scheduled and Ready + ansible.builtin.assert: + that: + - flannel_ds.resources | length == 1 + - flannel_ds.resources[0].status.desiredNumberScheduled > 0 + - flannel_ds.resources[0].status.numberReady == flannel_ds.resources[0].status.desiredNumberScheduled + success_msg: "Flannel DaemonSet is Ready" + fail_msg: "Flannel DaemonSet is not fully Ready" + + - name: Verify Calico is the active CNI + when: calico_iface is defined + block: + - name: Get the Calico node DaemonSet image + kubernetes.core.k8s_info: + kind: DaemonSet + name: calico-node + namespace: calico-system + kubeconfig: "{{ kubecfg_path }}" + register: calico_node_ds + + - name: Assert the Calico node image uses the expected tag + ansible.builtin.assert: + that: + - calico_node_ds.resources | length == 1 + - calico_node_image | regex_search(':' ~ calico_tag) + success_msg: "Calico node image uses tag {{ calico_tag }}" + fail_msg: >- + Calico node image {{ calico_node_image }}, + expected {{ calico_tag }} + vars: + calico_node_image: "{{ calico_node_ds.resources[0].spec.template.spec.containers[0].image }}" + + - name: Get Calico TigeraStatus for calico and apiserver + kubernetes.core.k8s_info: + api_version: operator.tigera.io/v1 + kind: Tigerastatus + name: "{{ item }}" + kubeconfig: "{{ kubecfg_path }}" + register: calico_tigerastatus + loop: + - calico + - apiserver + loop_control: + label: "Tigerastatus/{{ item }}" + + - name: Assert Calico TigeraStatus reports Available + ansible.builtin.assert: + that: >- + item.resources | length == 1 and + (item.resources[0].status.conditions + | selectattr('type', 'equalto', 'Available') + | map(attribute='status') | first | default('')) == 'True' + success_msg: "Tigerastatus {{ item.resources[0].metadata.name }} is Available" + fail_msg: "Tigerastatus is not Available" + loop: "{{ calico_tigerastatus.results }}" + loop_control: + label: "Tigerastatus Available" + + - name: Get any Flannel DaemonSets with Calico enabled + kubernetes.core.k8s_info: + kind: DaemonSet + namespace: kube-flannel + kubeconfig: "{{ kubecfg_path }}" + register: no_flannel_ds + - name: Assert there are no Flannel DaemonSets + ansible.builtin.assert: + that: no_flannel_ds.resources | length == 0 + success_msg: "No Flannel DaemonSet present with Calico" + fail_msg: "A Flannel DaemonSet exists alongside Calico" + + - name: Verify Cilium is the active CNI + when: cilium_iface is defined + block: + - name: Get the Cilium agent and operator images + kubernetes.core.k8s_info: + kind: "{{ item.kind }}" + name: "{{ item.name }}" + namespace: kube-system + kubeconfig: "{{ kubecfg_path }}" + register: cilium_info + loop: + - { kind: DaemonSet, name: cilium } + - { kind: Deployment, name: cilium-operator } + loop_control: + label: "{{ item.kind }}/{{ item.name }}" + + - name: Assert Cilium agent and operator use the expected image tag + ansible.builtin.assert: + that: + - cilium_agent_image | regex_search(':' ~ cilium_tag) + - cilium_operator_image | regex_search(':' ~ cilium_tag) + success_msg: "Cilium agent and operator use {{ cilium_tag }}" + fail_msg: >- + Cilium agent {{ cilium_agent_image }}, + operator {{ cilium_operator_image }}, + expected {{ cilium_tag }} + vars: + cilium_agent_image: >- + {{ (cilium_info.results + | selectattr('resources', 'defined') + | map(attribute='resources') + | list + | map(attribute='0') + | selectattr('kind', 'equalto', 'DaemonSet') + | list)[0].spec.template.spec.containers[0].image }} + cilium_operator_image: >- + {{ (cilium_info.results + | selectattr('resources', 'defined') + | map(attribute='resources') + | list + | map(attribute='0') + | selectattr('kind', 'equalto', 'Deployment') + | list)[0].spec.template.spec.containers[0].image }} + + - name: Get Hubble relay and UI deployments when enabled + kubernetes.core.k8s_info: + kind: Deployment + name: "{{ item }}" + namespace: kube-system + kubeconfig: "{{ kubecfg_path }}" + register: hubble_info + loop: + - hubble-relay + - hubble-ui + loop_control: + label: "Deployment/{{ item }}" + when: cilium_hubble | bool + + - name: Assert Hubble components are Ready when enabled + ansible.builtin.assert: + that: + - item.resources | length == 1 + - item.resources[0].status.readyReplicas | default(0) >= 1 + success_msg: "Hubble deployment {{ item.resources[0].metadata.name }} is Ready" + fail_msg: "Hubble deployment is not Ready" + loop: "{{ hubble_info.results }}" + loop_control: + label: "Hubble deployment" + when: cilium_hubble | bool + + - name: Get any Flannel DaemonSets with Cilium enabled + kubernetes.core.k8s_info: + kind: DaemonSet + namespace: kube-flannel + kubeconfig: "{{ kubecfg_path }}" + register: no_flannel_ds_cilium + - name: Assert there are no Flannel DaemonSets + ansible.builtin.assert: + that: no_flannel_ds_cilium.resources | length == 0 + success_msg: "No Flannel DaemonSet present with Cilium" + fail_msg: "A Flannel DaemonSet exists alongside Cilium" + + - name: Verify MetalLB is the active load balancer + when: kube_vip_lb_ip_range is not defined + block: + - name: Get the MetalLB controller and speaker images + kubernetes.core.k8s_info: + kind: "{{ item.kind }}" + name: "{{ item.name }}" + namespace: metallb-system + kubeconfig: "{{ kubecfg_path }}" + register: metallb_info + loop: + - { kind: Deployment, name: controller } + - { kind: DaemonSet, name: speaker } + loop_control: + label: "{{ item.kind }}/{{ item.name }}" + + - name: Assert MetalLB controller and speaker use the expected image tags + ansible.builtin.assert: + that: + - controller_image | regex_search(metal_lb_controller_tag_version) + - speaker_image | regex_search(metal_lb_speaker_tag_version) + success_msg: >- + MetalLB controller {{ metal_lb_controller_tag_version }}, + speaker {{ metal_lb_speaker_tag_version }} + fail_msg: >- + MetalLB controller {{ controller_image }}, + speaker {{ speaker_image }} + vars: + controller_image: >- + {{ (metallb_info.results + | selectattr('resources', 'defined') + | map(attribute='resources') + | list + | map(attribute='0') + | selectattr('kind', 'equalto', 'Deployment') + | list)[0].spec.template.spec.containers[0].image }} + speaker_image: >- + {{ (metallb_info.results + | selectattr('resources', 'defined') + | map(attribute='resources') + | list + | map(attribute='0') + | selectattr('kind', 'equalto', 'DaemonSet') + | list)[0].spec.template.spec.containers[0].image }} + + - name: Verify kube-vip is the active load balancer + when: kube_vip_lb_ip_range is defined + block: + - name: Get the kube-vip and cloud provider images + kubernetes.core.k8s_info: + kind: "{{ item.kind }}" + name: "{{ item.name }}" + namespace: kube-system + kubeconfig: "{{ kubecfg_path }}" + register: kubevip_info + loop: + - { kind: DaemonSet, name: kube-vip-ds } + - { kind: Deployment, name: kube-vip-cloud-provider } + loop_control: + label: "{{ item.kind }}/{{ item.name }}" + + - name: Assert the kube-vip and cloud provider image tags + ansible.builtin.assert: + that: + - kubevip_image | regex_search(':' ~ kube_vip_tag_version) + - cloud_provider_image | regex_search(kube_vip_cloud_provider_tag_version) + success_msg: >- + kube-vip {{ kube_vip_tag_version }}, + cloud provider {{ kube_vip_cloud_provider_tag_version }} + fail_msg: >- + kube-vip {{ kubevip_image }}, + cloud provider {{ cloud_provider_image }} + vars: + kubevip_image: >- + {{ (kubevip_info.results + | selectattr('resources', 'defined') + | map(attribute='resources') + | list + | map(attribute='0') + | selectattr('kind', 'equalto', 'DaemonSet') + | list)[0].spec.template.spec.containers[0].image }} + cloud_provider_image: >- + {{ (kubevip_info.results + | selectattr('resources', 'defined') + | map(attribute='resources') + | list + | map(attribute='0') + | selectattr('kind', 'equalto', 'Deployment') + | list)[0].spec.template.spec.containers[0].image }} + + - name: Get the MetalLB namespace with kube-vip enabled + kubernetes.core.k8s_info: + kind: Namespace + name: metallb-system + kubeconfig: "{{ kubecfg_path }}" + register: metallb_absent + - name: Assert the MetalLB namespace does not exist + ansible.builtin.assert: + that: metallb_absent.resources | length == 0 + success_msg: "MetalLB is not installed with kube-vip" + fail_msg: "MetalLB namespace exists alongside kube-vip"