diff --git a/.ansible-lint b/.ansible-lint index c535afc1..51884c60 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -15,6 +15,10 @@ exclude_paths: - molecule/**/prepare.yml - molecule/**/reset.yml + # Scenario verify inputs are plain variable files, not playbooks. They are + # loaded as vars, not executed, so ansible-lint must not treat them as plays. + - molecule/**/verify-vars.yml + # The file was generated by galaxy ansible - don't mess with it. - galaxy.yml diff --git a/.github/scripts/test-cilium-bgp-manifest.py b/.github/scripts/test-cilium-bgp-manifest.py new file mode 100644 index 00000000..97013855 --- /dev/null +++ b/.github/scripts/test-cilium-bgp-manifest.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Render the Cilium BGP CRD template and assert it uses the v2 API. + +This is a manifest-only regression test used where no real BGP peer is +available. It renders roles/k3s_server_post/templates/cilium.crs.j2 with +zero, one, and multiple neighbors, then checks that the output: + - never contains CiliumBGPPeeringPolicy or cilium.io/v2alpha1 + - emits the Cilium v2 BGP resources + - emits deterministic DNS-safe peer and instance names + - advertises Pod CIDRs only when cilium_exportPodCIDR is true +""" + +from __future__ import print_function + +import os +import subprocess + +from jinja2 import Environment, FileSystemLoader, StrictUndefined + + +def repo_root(): + return subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], text=True + ).strip() + + +def fail(message): + raise SystemExit("Cilium BGP manifest test failed: " + message) + + +def render(env, extra_vars): + base_vars = { + "cilium_bgp_my_asn": "64513", + "cilium_bgp_peer_asn": "64512", + "cilium_bgp_peer_address": "192.168.30.1", + "cilium_exportPodCIDR": True, + "cilium_bgp_lb_cidr": "192.168.31.0/24", + } + base_vars.update(extra_vars) + template = env.get_template("cilium.crs.j2") + return template.render(**base_vars) + + +def check_common(output): + if "cilium.io/v2alpha1" in output: + fail("rendered output still contains cilium.io/v2alpha1") + if "kind: CiliumBGPPeeringPolicy" in output: + fail("rendered output still contains CiliumBGPPeeringPolicy") + for kind in ( + "CiliumBGPPeerConfig", + "CiliumBGPClusterConfig", + "CiliumBGPAdvertisement", + "CiliumLoadBalancerIPPool", + ): + if ("kind: " + kind) not in output: + fail("rendered output is missing kind: " + kind) + + +def main(): + root = repo_root() + template_dir = os.path.join( + root, "roles", "k3s_server_post", "templates" + ) + env = Environment( + loader=FileSystemLoader(template_dir), undefined=StrictUndefined + ) + + # Zero neighbors -> fall back to the single default peer. + output = render(env, {"_cilium_bgp_neighbors": []}) + check_common(output) + if "peer-64512-1" not in output: + fail("default single peer name was not rendered") + if "peerAddress: 192.168.30.1" not in output: + fail("default peer address was not rendered") + if 'advertisementType: "PodCIDR"' not in output: + fail("PodCIDR advertisement missing when exportPodCIDR is true") + + # One neighbor via the merged list. + output = render( + env, + {"_cilium_bgp_neighbors": [{"peer_address": "10.0.0.1", "peer_asn": "65001"}]}, + ) + check_common(output) + if "peer-65001-1" not in output: + fail("single merged peer name was not rendered") + if "peerAddress: 10.0.0.1" not in output: + fail("single merged peer address was not rendered") + + # Multiple neighbors. + output = render( + env, + { + "_cilium_bgp_neighbors": [ + {"peer_address": "10.0.0.1", "peer_asn": "65001"}, + {"peer_address": "10.0.0.2", "peer_asn": "65002"}, + ] + }, + ) + check_common(output) + if "peer-65001-1" not in output or "peer-65002-2" not in output: + fail("multiple merged peer names were not rendered") + if "peerAddress: 10.0.0.2" not in output: + fail("second merged peer address was not rendered") + + # exportPodCIDR false -> no PodCIDR advertisement, service remains. + output = render( + env, {"_cilium_bgp_neighbors": [], "cilium_exportPodCIDR": False} + ) + check_common(output) + if 'advertisementType: "PodCIDR"' in output: + fail("PodCIDR advertisement present when exportPodCIDR is false") + if 'advertisementType: "Service"' not in output: + fail("Service advertisement missing when exportPodCIDR is false") + + # Load balancer pools: CIDR and start/stop forms. + output = render(env, {"_cilium_bgp_neighbors": []}) + if "cidr: 192.168.31.0/24" not in output: + fail("CIDR load balancer pool was not rendered") + output = render( + env, + { + "_cilium_bgp_neighbors": [], + "cilium_bgp_lb_cidr": "192.168.31.80-192.168.31.90", + }, + ) + check_common(output) + if "start: 192.168.31.80" not in output or "stop: 192.168.31.90" not in output: + fail("start/stop load balancer pool was not rendered") + + print("Cilium BGP manifest regression test passed") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/test-kube-vip-manifest.py b/.github/scripts/test-kube-vip-manifest.py new file mode 100644 index 00000000..c74cf005 --- /dev/null +++ b/.github/scripts/test-kube-vip-manifest.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Render the kube-vip DaemonSet template and assert env key correctness. + +kube-vip v1.2.2 reads `bgp_peers` and `vip_subnet`; it ignores the older +`bgppeers` and `vip_cidr` names. This test proves the rendered manifest uses +the keys the target image actually parses. +""" + +from __future__ import print_function + +import os +import subprocess + +from jinja2 import Environment, FileSystemLoader, StrictUndefined + + +def repo_root(): + return subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], text=True + ).strip() + + +def fail(message): + raise SystemExit("kube-vip manifest test failed: " + message) + + +def fake_ipsubnet(value): + # ansible.utils.ipsubnet -> network of the address as x.y.z.0/24 + parts = value.split(".") + return ".".join(parts[:3]) + ".0/24" + + +def fake_ipaddr(_value, expr=None): + # ansible.utils.ipaddr('prefix') -> prefix length + return "24" + + +def fake_bool(value): + # Minimal stand-in for Ansible's truthiness filter used by the template. + if isinstance(value, bool): + return value + return str(value).lower() in ("1", "true", "yes", "on") + + +def fake_map(seq, *args, **kwargs): + # Minimal stand-in for Ansible's map() filter in the two forms used by the + # template: map(attribute='x') on a list of dicts, and map('join', sep) on + # a list of sequences. + if "attribute" in kwargs: + return [item[kwargs["attribute"]] for item in seq] + if kwargs: + # e.g. map(default='x') not used here; ignore unknown kwargs. + return list(seq) + if args: + filter_name = args[0] + sep = args[1] if len(args) > 1 else "" + if filter_name == "join": + return [sep.join(str(x) for x in item) for item in seq] + return list(seq) + + +def fake_zip(*seqs): + return list(zip(*seqs)) + + +def render(env, extra_vars): + base_vars = { + "apiserver_endpoint": "192.168.30.222", + "kube_vip_iface": "", + "kube_vip_arp": True, + "kube_vip_bgp": True, + "kube_vip_bgp_routerid": "127.0.0.1", + "_kube_vip_bgp_peers": [ + {"peer_address": "192.168.30.1", "peer_asn": "64512"}, + {"peer_address": "192.168.30.2", "peer_asn": "64513"}, + ], + "kube_vip_tag_version": "v1.2.2", + } + base_vars.update(extra_vars) + template = env.get_template("vip.yaml.j2") + return template.render(**base_vars) + + +def main(): + root = repo_root() + template_dir = os.path.join(root, "roles", "k3s_server", "templates") + env = Environment( + loader=FileSystemLoader(template_dir), undefined=StrictUndefined + ) + env.filters["ansible.utils.ipsubnet"] = fake_ipsubnet + env.filters["ansible.utils.ipaddr"] = fake_ipaddr + env.filters["bool"] = fake_bool + env.filters["map"] = fake_map + env.filters["zip"] = fake_zip + + # Multi-peer BGP armed: must emit bgp_peers, never bgppeers. + output = render(env, {}) + if "name: bgp_peers" not in output: + fail("rendered manifest is missing bgp_peers") + if "name: bgppeers" in output: + fail("rendered manifest still uses the ignored bgppeers key") + if "name: vip_subnet" not in output: + fail("rendered manifest is missing vip_subnet") + if "name: vip_cidr" in output: + fail("rendered manifest still uses the ignored vip_cidr key") + if "192.168.30.1:64512,192.168.30.2:64513" not in output: + fail("bgp_peers value is not comma-separated address:ASN entries") + if "ghcr.io/kube-vip/kube-vip:v1.2.2" not in output: + fail("kube-vip image tag is not v1.2.2") + + # BGP enabled with no merged peers: single-peer fallback vars, no bgp_peers. + output = render( + env, + { + "_kube_vip_bgp_peers": [], + "kube_vip_bgp_as": "64513", + "kube_vip_bgp_peeraddress": "192.168.30.1", + "kube_vip_bgp_peeras": "64512", + }, + ) + if "name: bgp_as" not in output: + fail("single-peer bgp_as was not rendered") + if "name: bgp_peers" in output: + fail("bgp_peers present even though the peer list is empty") + + print("kube-vip manifest regression test passed") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/test-metallb-remote-read.sh b/.github/scripts/test-metallb-remote-read.sh new file mode 100755 index 00000000..5a1b9b0f --- /dev/null +++ b/.github/scripts/test-metallb-remote-read.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +repo_root="$(git rev-parse --show-toplevel)" +metallb_task="$repo_root/roles/k3s_server/tasks/metallb.yml" + +# The speaker tag verification must read the rendered manifest on the managed +# host with slurp. A controller-side lookup('ansible.builtin.file', ...) would +# read from the Ansible control node, which does not have the file, and would +# fail on every MetalLB scenario. +grep -Fq -- 'ansible.builtin.slurp' "$metallb_task" || { + printf 'MetalLB speaker tag check does not use slurp on the managed host\n' >&2 + exit 1 +} + +grep -Eq -- 'lookup\(.?ansible\.builtin\.file' "$metallb_task" && { + printf 'MetalLB speaker tag check uses a controller-side file lookup\n' >&2 + exit 1 +} + +# The check must reference the full image reference, not just a bare version +# string that could appear anywhere in the manifest. +grep -Fq -- 'quay.io/metallb/speaker:' "$metallb_task" || { + printf 'MetalLB speaker tag check does not match the full image reference\n' >&2 + exit 1 +} + +printf 'MetalLB remote manifest read regression test passed\n' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1f1932d4..b418bd3d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -88,7 +88,7 @@ jobs: trap stop_monitor EXIT /usr/bin/time -v -o "$timing_file" \ molecule test --scenario-name ${{ matrix.scenario }} - timeout-minutes: 90 + timeout-minutes: 150 env: ANSIBLE_K3S_LOG_DIR: ${{ runner.temp }}/logs/k3s-ansible/${{ matrix.scenario }} ANSIBLE_SSH_RETRIES: 4 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b3433e1b..0c289028 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -62,3 +62,25 @@ repos: language: system pass_filenames: false files: ^roles/k3s_server/tasks/(main|join_master)\.yml$|^\.github/scripts/test-k3s-server-bootstrap\.sh$ + - id: cilium-bgp-manifest-test + name: Cilium BGP manifest test + entry: python3 .github/scripts/test-cilium-bgp-manifest.py + language: python + additional_dependencies: + - Jinja2>=3.1 + pass_filenames: false + files: ^roles/k3s_server_post/templates/cilium\.crs\.j2$|^\.github/scripts/test-cilium-bgp-manifest\.py$ + - id: kube-vip-manifest-test + name: kube-vip manifest test + entry: python3 .github/scripts/test-kube-vip-manifest.py + language: python + additional_dependencies: + - Jinja2>=3.1 + pass_filenames: false + files: ^roles/k3s_server/templates/vip\.yaml\.j2$|^\.github/scripts/test-kube-vip-manifest\.py$ + - id: metallb-remote-read-test + name: MetalLB remote read test + entry: .github/scripts/test-metallb-remote-read.sh + language: system + pass_filenames: false + files: ^roles/k3s_server/tasks/metallb\.yml$|^\.github/scripts/test-metallb-remote-read\.sh$ diff --git a/README.md b/README.md index bbf1dc1b..87063ec0 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,32 @@ ansible-playbook reset.yml -i inventory/my-cluster/hosts.ini >You should also reboot these nodes due to the VIP not being destroyed +## 🔁 Upgrading an existing cluster + +These version variables select the components used for a **fresh** installation. +They are not a supported direct in-place upgrade path for an existing cluster. +K3s, Calico, and Cilium each require staged upgrades for long-lived clusters. + +- **K3s**: do not jump an embedded-etcd cluster straight to Kubernetes 1.36. + Upgrade one Kubernetes minor version at a time. From the sample default + (`v1.30.2+k3s2`) the sequence is: the latest supported 1.30 patch, then 1.31, + 1.32, a 1.33 patch that contains etcd 3.5.26 (for example `v1.33.7+k3s3`), + then 1.34, 1.35, and finally 1.36. Upgrade servers one at a time before + agents. Take backups and confirm cluster health at each step; this playbook + does not automate the upgrade, so those remain manual operational steps. See + [K3s manual upgrades](https://docs.k3s.io/upgrades/manual) and the + [v1.34 release notes](https://docs.k3s.io/release-notes/v1.34.X). +- **Cilium**: upstream supports only consecutive minor upgrades. Update to the + latest patch of the current minor, then upgrade 1.17, 1.18, 1.19, and 1.20 in + order, reading each version's upgrade notes and running preflight checks. + Do not attempt a direct upgrade from an old Cilium to 1.20. +- **Calico**: starting with 3.28 the v3 resource UID behavior changed. If you + have operators with OwnerReferences pointing to `projectcalico.org/v3` + resources, remove and recreate those references around an in-place upgrade. +- **MetalLB**: this project installs application tag `v0.16.0`. A newer + chart-only tag such as `metallb-chart-0.16.1` is not an application or image + release and must not be used as the controller or speaker image tag. + ## ⚙️ Kube Config To copy your `kube config` locally so that you can access your **Kubernetes** cluster run: @@ -146,11 +172,11 @@ See the commands [here](https://technotim.com/posts/k3s-etcd-ansible/#testing-yo | `k3s_server` | `kube_vip_bgp_peers` | list | `[]` | Not required | List of BGP peer ASN & address pairs | | `k3s_server` | `kube_vip_bgp_peers_groups` | list | `['k3s_master']` | Not required | Inventory group in which to search for additional `kube_vip_bgp_peers` parameters to merge. | | `k3s_server` | `kube_vip_iface` | string | `~` | Not required | Explicitly define an interface that ALL control nodes should use to propagate the VIP, define it here. Otherwise, kube-vip will determine the right interface automatically at runtime. | -| `k3s_server` | `kube_vip_tag_version` | string | `v0.7.2` | Not required | Image tag for kube-vip | -| `k3s_server` | `kube_vip_cloud_provider_tag_version` | string | `main` | Not required | Tag for kube-vip-cloud-provider manifest when enable | +| `k3s_server` | `kube_vip_tag_version` | string | `v1.2.2` | Not required | Image tag for kube-vip | +| `k3s_server` | `kube_vip_cloud_provider_tag_version` | string | `v0.0.12` | Not required | Tag for kube-vip-cloud-provider manifest when enable | | `k3s_server`, `k3_server_post` | `kube_vip_lb_ip_range` | string | `~` | Not required | IP range for kube-vip load balancer | -| `k3s_server`, `k3s_server_post` | `metal_lb_controller_tag_version` | string | `v0.14.3` | Not required | Image tag for MetalLB | -| `k3s_server` | `metal_lb_speaker_tag_version` | string | `v0.14.3` | Not required | Image tag for MetalLB | +| `k3s_server`, `k3s_server_post` | `metal_lb_controller_tag_version` | string | `v0.16.0` | Not required | Image tag for MetalLB | +| `k3s_server` | `metal_lb_speaker_tag_version` | string | `v0.16.0` | Not required | Image tag for MetalLB | | `k3s_server` | `metal_lb_type` | string | `native` | Not required | Use FRR mode or native. Valid values are `frr` and `native` | | `k3s_server` | `retry_count` | int | `20` | Not required | Amount of retries when verifying that nodes joined | | `k3s_server` | `server_init_args` | string | ❌ | Not required | Arguments for server nodes | @@ -162,7 +188,7 @@ See the commands [here](https://technotim.com/posts/k3s-etcd-ansible/#testing-yo | `k3s_server_post` | `calico_natOutgoing` | string | `Enabled` | Not required | IP pool NAT outgoing | | `k3s_server_post` | `calico_nodeSelector` | string | `all()` | Not required | IP pool node selector | | `k3s_server_post` | `calico_iface` | string | `~` | Not required | The network interface used for when Calico is enabled | -| `k3s_server_post` | `calico_tag` | string | `v3.27.2` | Not required | Calico version tag | +| `k3s_server_post` | `calico_tag` | string | `v3.32.1` | Not required | Calico version tag | | `k3s_server_post` | `cilium_bgp_my_asn` | int | `64513` | Not required | Local ASN for BGP peer | | `k3s_server_post` | `cilium_bgp_peer_asn` | int | `64512` | Not required | BGP peer ASN | | `k3s_server_post` | `cilium_bgp_peer_address` | string | `~` | Not required | BGP peer address | @@ -171,14 +197,15 @@ See the commands [here](https://technotim.com/posts/k3s-etcd-ansible/#testing-yo | `k3s_server_post` | `cilium_bgp_lb_cidr` | string | `192.168.31.0/24` | Not required | BGP load balancer IP range | | `k3s_server_post` | `cilium_exportPodCIDR` | bool | `true` | Not required | Export pod CIDR | | `k3s_server_post` | `cilium_hubble` | bool | `true` | Not required | Enable Cilium Hubble | -| `k3s_server_post` | `cilium_hubble` | bool | `true` | Not required | Enable Cilium Hubble | -| `k3s_server_post` | `cilium_mode` | string | `native` | Not required | Inner-node communication mode (choices are `native` and `routed`) | +| `k3s_server_post` | `cilium_mode` | string | `native` | Not required | Inner-node communication mode (choices are `native` and `tunnel`; `routed` is a deprecated alias for `tunnel`) | +| `k3s_server_post` | `cilium_tag` | string | `v1.20.0` | Not required | Cilium version tag | +| `k3s_server_post` | `cilium_cli_tag` | string | `v0.19.7` | Not required | Cilium CLI version tag | | `k3s_server_post` | `cluster_cidr` | string | `10.52.0.0/16` | Not required | Inner-cluster IP range | | `k3s_server_post` | `enable_bpf_masquerade` | bool | `true` | Not required | Use IP masquerading | | `k3s_server_post` | `kube_proxy_replacement` | bool | `true` | Not required | Replace the native kube-proxy with Cilium | | `k3s_server_post` | `metal_lb_available_timeout` | string | `240s` | Not required | Wait for MetalLB resources | | `k3s_server_post` | `metal_lb_ip_range` | string | `192.168.30.80-192.168.30.90` | Not required | MetalLB ip range for load balancer | -| `k3s_server_post` | `metal_lb_controller_tag_version` | string | `v0.14.3` | Not required | Image tag for MetalLB | +| `k3s_server_post` | `metal_lb_controller_tag_version` | string | `v0.16.0` | Not required | Image tag for MetalLB | | `k3s_server_post` | `metal_lb_mode` | string | `layer2` | Not required | Metallb mode (choices are `bgp` and `layer2`) | | `k3s_server_post` | `metal_lb_bgp_my_asn` | string | `~` | Not required | BGP ASN configurations | | `k3s_server_post` | `metal_lb_bgp_peer_asn` | string | `~` | Not required | BGP peer ASN configurations | diff --git a/inventory/sample/group_vars/all.yml b/inventory/sample/group_vars/all.yml index 8ddeb05a..ac0cb71b 100644 --- a/inventory/sample/group_vars/all.yml +++ b/inventory/sample/group_vars/all.yml @@ -1,5 +1,5 @@ --- -k3s_version: v1.30.2+k3s2 +k3s_version: v1.36.2+k3s1 # this is the user that has ssh access to these machines ansible_user: ansibleuser systemd_dir: /etc/systemd/system @@ -13,13 +13,14 @@ flannel_iface: eth0 # uncomment calico_iface to use tigera operator/calico cni instead of flannel https://docs.tigera.io/calico/latest/about # calico_iface: "eth0" calico_ebpf: false # use eBPF dataplane instead of iptables -calico_tag: v3.28.0 # calico version tag +calico_tag: v3.32.1 # calico version tag # uncomment cilium_iface to use cilium cni instead of flannel or calico # ensure v4.19.57, v5.1.16, v5.2.0 or more recent kernel # cilium_iface: "eth0" -cilium_mode: native # native when nodes on same subnet or using bgp, else set routed -cilium_tag: v1.16.0 # cilium version tag +cilium_mode: native # native when nodes are on the same subnet or use BGP, otherwise set tunnel +cilium_tag: v1.20.0 # cilium version tag +cilium_cli_tag: v0.19.7 # cilium cli version tag cilium_hubble: true # enable hubble observability relay and ui # if using calico or cilium, you may specify the cluster pod cidr pool @@ -84,10 +85,10 @@ extra_agent_args: >- {{ extra_args }} # image tag for kube-vip -kube_vip_tag_version: v0.8.2 +kube_vip_tag_version: v1.2.2 # tag for kube-vip-cloud-provider manifest -# kube_vip_cloud_provider_tag_version: "main" +# kube_vip_cloud_provider_tag_version: "v0.0.12" # kube-vip ip range for load balancer # (uncomment to use kube-vip for services instead of MetalLB) @@ -105,8 +106,8 @@ metal_lb_mode: layer2 # metal_lb_bgp_peer_address: "192.168.30.1" # image tag for metal lb -metal_lb_speaker_tag_version: v0.14.8 -metal_lb_controller_tag_version: v0.14.8 +metal_lb_speaker_tag_version: v0.16.0 +metal_lb_controller_tag_version: v0.16.0 # metallb ip range for load balancer metal_lb_ip_range: 192.168.30.80-192.168.30.90 diff --git a/molecule/calico/verify-vars.yml b/molecule/calico/verify-vars.yml new file mode 100644 index 00000000..7b801d88 --- /dev/null +++ b/molecule/calico/verify-vars.yml @@ -0,0 +1,6 @@ +--- +# Durable verify inputs for the calico (Calico CNI + MetalLB) scenario. +verify_cni: calico +verify_lb: metallb +verify_lb_ip_range: + - 192.168.30.100-192.168.30.109 diff --git a/molecule/cilium/verify-vars.yml b/molecule/cilium/verify-vars.yml new file mode 100644 index 00000000..4ea59803 --- /dev/null +++ b/molecule/cilium/verify-vars.yml @@ -0,0 +1,6 @@ +--- +# Durable verify inputs for the cilium (Cilium CNI + MetalLB) scenario. +verify_cni: cilium +verify_lb: metallb +verify_lb_ip_range: + - 192.168.30.110-192.168.30.119 diff --git a/molecule/default/verify-vars.yml b/molecule/default/verify-vars.yml new file mode 100644 index 00000000..d0988c24 --- /dev/null +++ b/molecule/default/verify-vars.yml @@ -0,0 +1,9 @@ +--- +# Durable verify inputs for the default (flannel + MetalLB) scenario. +# These are plain inventory vars linked into the shared Molecule inventory so +# the verify play can see them even though the converge play's set_fact values +# are not persisted between the two Ansible processes. +verify_cni: flannel +verify_lb: metallb +verify_lb_ip_range: + - 192.168.30.80-192.168.30.90 diff --git a/molecule/ipv6/verify-vars.yml b/molecule/ipv6/verify-vars.yml new file mode 100644 index 00000000..868654b6 --- /dev/null +++ b/molecule/ipv6/verify-vars.yml @@ -0,0 +1,7 @@ +--- +# Durable verify inputs for the ipv6 (flannel CNI + MetalLB) scenario. +verify_cni: flannel +verify_lb: metallb +verify_lb_ip_range: + - fdad:bad:ba55::1b:0/112 + - 192.168.123.80-192.168.123.90 diff --git a/molecule/kube-vip/verify-vars.yml b/molecule/kube-vip/verify-vars.yml new file mode 100644 index 00000000..b693ece0 --- /dev/null +++ b/molecule/kube-vip/verify-vars.yml @@ -0,0 +1,9 @@ +--- +# Durable verify inputs for the kube-vip (flannel CNI + kube-vip LB) scenario. +verify_cni: flannel +verify_lb: kube-vip +# The kube-vip cloud provider tag is not defined in the linked sample group +# vars (its sample entry is commented out), so it is supplied here. +verify_kube_vip_cloud_provider_tag: v0.0.12 +verify_lb_ip_range: + - 192.168.30.110-192.168.30.119 diff --git a/molecule/resources/verify.yml b/molecule/resources/verify.yml index ef7ea526..d56be849 100644 --- a/molecule/resources/verify.yml +++ b/molecule/resources/verify.yml @@ -1,5 +1,8 @@ --- - name: Verify hosts: all + vars_files: + - >- + {{ lookup("ansible.builtin.env", "MOLECULE_SCENARIO_DIRECTORY") }}/verify-vars.yml roles: - verify_from_outside diff --git a/molecule/resources/verify_from_outside/tasks/main.yml b/molecule/resources/verify_from_outside/tasks/main.yml index 2f43a277..95280735 100644 --- a/molecule/resources/verify_from_outside/tasks/main.yml +++ b/molecule/resources/verify_from_outside/tasks/main.yml @@ -7,6 +7,8 @@ ansible.builtin.import_tasks: kubecfg-fetch.yml - name: "TEST CASE: Get nodes" ansible.builtin.include_tasks: test/get-nodes.yml + - name: "TEST CASE: Verify components" + ansible.builtin.include_tasks: test/verify-components.yml - name: "TEST CASE: Deploy example" ansible.builtin.include_tasks: test/deploy-example.yml always: diff --git a/molecule/resources/verify_from_outside/tasks/test/deploy-example.yml b/molecule/resources/verify_from_outside/tasks/test/deploy-example.yml index 13a1c4be..3781afd2 100644 --- a/molecule/resources/verify_from_outside/tasks/test/deploy-example.yml +++ b/molecule/resources/verify_from_outside/tasks/test/deploy-example.yml @@ -32,17 +32,86 @@ metallb_port: spec.ports[0].port register: nginx_services + - name: Wait for the load balancer address to be assigned + ansible.builtin.set_fact: + nginx_lb_ip: >- + {{ + nginx_services.resources[0].status.loadBalancer.ingress[0].ip + if (nginx_services.resources | length > 0) and + (nginx_services.resources[0].status.loadBalancer.ingress is defined) and + (nginx_services.resources[0].status.loadBalancer.ingress | length > 0) + else '' + }} + + - name: Retry until the load balancer service has an external IP + block: + - name: Refresh nginx service until it has an assigned address + kubernetes.core.k8s_info: + kind: service + name: nginx + namespace: "{{ testing_namespace }}" + kubeconfig: "{{ kubecfg_path }}" + register: nginx_lb_wait + until: >- + (nginx_lb_wait.resources | length > 0) and + (nginx_lb_wait.resources[0].status.loadBalancer.ingress is defined) and + (nginx_lb_wait.resources[0].status.loadBalancer.ingress | length > 0) + retries: 30 + delay: 5 + + - name: Record the assigned load balancer address + ansible.builtin.set_fact: + nginx_lb_ip: >- + {{ nginx_lb_wait.resources[0].status.loadBalancer.ingress[0].ip }} + - name: Assert that the nginx welcome page is available ansible.builtin.uri: - url: http://{{ ip | ansible.utils.ipwrap }}:{{ port_ }}/ + url: http://{{ nginx_lb_ip | ansible.utils.ipwrap }}:{{ port_ }}/ return_content: true register: result failed_when: "'Welcome to nginx!' not in result.content" vars: - ip: >- - {{ nginx_services.resources[0].status.loadBalancer.ingress[0].ip }} port_: >- {{ nginx_services.resources[0].spec.ports[0].port }} + + - name: Initialize load balancer address range check + ansible.builtin.set_fact: + lb_addr_in_range: false + lb_ip_value: "{{ nginx_lb_ip }}" + + - name: Check load balancer address against start-end pools + ansible.builtin.set_fact: + lb_addr_in_range: true + loop: "{{ verify_lb_ip_range }}" + loop_control: + label: "{{ item }}" + when: + - "'-' in item" + - "'/' not in item" + - >- + (lb_ip_value | ansible.utils.ipaddr('int') | int) >= + (item.split('-')[0] | ansible.utils.ipaddr('int') | int) + - >- + (lb_ip_value | ansible.utils.ipaddr('int') | int) <= + (item.split('-')[1] | ansible.utils.ipaddr('int') | int) + + - name: Check load balancer address against CIDR pools + ansible.builtin.set_fact: + lb_addr_in_range: true + loop: "{{ verify_lb_ip_range }}" + loop_control: + label: "{{ item }}" + when: + - "'/' in item" + - (lb_ip_value | ansible.utils.ipaddr(item)) is string + + - name: Assert that the load balancer address is within a configured pool + ansible.builtin.assert: + that: lb_addr_in_range + success_msg: "LoadBalancer address {{ lb_ip_value }} is in a configured range" + fail_msg: >- + LoadBalancer address {{ lb_ip_value }} is not in a configured + range {{ verify_lb_ip_range }} # Deactivated linter rules: # - jinja[invalid]: As of version 6.6.0, ansible-lint complains that the input to ipwrap # would be undefined. This will not be the case during playbook execution. diff --git a/molecule/resources/verify_from_outside/tasks/test/verify-components.yml b/molecule/resources/verify_from_outside/tasks/test/verify-components.yml new file mode 100644 index 00000000..debc2989 --- /dev/null +++ b/molecule/resources/verify_from_outside/tasks/test/verify-components.yml @@ -0,0 +1,313 @@ +--- +# Scenario-aware verification of cluster components and their live image tags. +# Scenario identity (verify_cni / verify_lb) and expected address range come +# from each scenario's verify-vars.yml, which is plain inventory data available +# to the verify play. Converge-time set_fact values are not persisted between +# the two Ansible processes, so they are never used here. +- name: Verify cluster components report expected versions + block: + - name: Get all nodes with their kubelet versions + kubernetes.core.k8s_info: + kind: node + kubeconfig: "{{ kubecfg_path }}" + register: verify_nodes + + - name: Assert each node reports the expected Kubernetes version + ansible.builtin.assert: + that: item.status.nodeInfo.kubeletVersion == k3s_version + success_msg: "{{ item.metadata.name }} reports {{ k3s_version }}" + fail_msg: >- + {{ item.metadata.name }} reports + {{ item.status.nodeInfo.kubeletVersion }}, + expected {{ k3s_version }} + loop: "{{ verify_nodes.resources }}" + loop_control: + label: "{{ item.metadata.name }}" + + - name: Verify Flannel is the active CNI + when: verify_cni == 'flannel' + block: + - name: Assert every node reports Ready + ansible.builtin.assert: + that: item.status.conditions + | selectattr('type', 'equalto', 'Ready') + | map(attribute='status') | first | default('') == 'True' + success_msg: "{{ item.metadata.name }} is Ready" + fail_msg: "{{ item.metadata.name }} is not Ready" + loop: "{{ verify_nodes.resources }}" + loop_control: + label: "{{ item.metadata.name }} ready" + + - name: Get any Calico namespaces with Flannel enabled + kubernetes.core.k8s_info: + kind: Namespace + name: calico-system + kubeconfig: "{{ kubecfg_path }}" + register: flannel_calico_absent + + - name: Assert there is no Calico system namespace + ansible.builtin.assert: + that: flannel_calico_absent.resources | length == 0 + success_msg: "No Calico present with Flannel" + fail_msg: "A Calico namespace exists alongside Flannel" + + - name: Get the Cilium namespace with Flannel enabled + kubernetes.core.k8s_info: + kind: Namespace + name: cilium + kubeconfig: "{{ kubecfg_path }}" + register: flannel_cilium + + - name: Assert the Cilium namespace is absent + ansible.builtin.assert: + that: flannel_cilium.resources | length == 0 + success_msg: "No Cilium present with Flannel" + fail_msg: "A Cilium namespace exists alongside Flannel" + + - name: Verify Calico is the active CNI + when: verify_cni == 'calico' + block: + - name: Get the Calico node DaemonSet image + kubernetes.core.k8s_info: + kind: DaemonSet + name: calico-node + namespace: calico-system + kubeconfig: "{{ kubecfg_path }}" + register: calico_node_ds + + - name: Assert the Calico node image uses the expected tag + ansible.builtin.assert: + that: + - calico_node_ds.resources | length == 1 + - calico_node_image | regex_search(':' ~ calico_tag) + success_msg: "Calico node image uses tag {{ calico_tag }}" + fail_msg: >- + Calico node image {{ calico_node_image }}, + expected {{ calico_tag }} + vars: + calico_node_image: "{{ calico_node_ds.resources[0].spec.template.spec.containers[0].image }}" + + - name: Get Calico TigeraStatus for calico and apiserver + kubernetes.core.k8s_info: + api_version: operator.tigera.io/v1 + kind: TigeraStatus + name: "{{ item }}" + kubeconfig: "{{ kubecfg_path }}" + register: calico_tigerastatus + loop: + - calico + - apiserver + loop_control: + label: "Tigerastatus/{{ item }}" + + - name: Assert Calico TigeraStatus reports Available + ansible.builtin.assert: + that: >- + item.resources | length == 1 and + (item.resources[0].status.conditions + | selectattr('type', 'equalto', 'Available') + | map(attribute='status') | first | default('')) == 'True' + success_msg: "Tigerastatus {{ item.resources[0].metadata.name }} is Available" + fail_msg: "Tigerastatus is not Available" + loop: "{{ calico_tigerastatus.results }}" + loop_control: + label: "Tigerastatus Available" + + - name: Get any Flannel DaemonSets with Calico enabled + kubernetes.core.k8s_info: + kind: DaemonSet + namespace: kube-flannel + kubeconfig: "{{ kubecfg_path }}" + register: no_flannel_ds + + - name: Assert there are no Flannel DaemonSets + ansible.builtin.assert: + that: no_flannel_ds.resources | length == 0 + success_msg: "No Flannel DaemonSet present with Calico" + fail_msg: "A Flannel DaemonSet exists alongside Calico" + + - name: Verify Cilium is the active CNI + when: verify_cni == 'cilium' + block: + - name: Get the Cilium agent and operator images + kubernetes.core.k8s_info: + kind: "{{ item.kind }}" + name: "{{ item.name }}" + namespace: kube-system + kubeconfig: "{{ kubecfg_path }}" + register: cilium_info + loop: + - { kind: DaemonSet, name: cilium } + - { kind: Deployment, name: cilium-operator } + loop_control: + label: "{{ item.kind }}/{{ item.name }}" + + - name: Assert Cilium agent and operator use the expected image tag + ansible.builtin.assert: + that: + - cilium_agent_image | regex_search(':' ~ cilium_tag) + - cilium_operator_image | regex_search(':' ~ cilium_tag) + success_msg: "Cilium agent and operator use {{ cilium_tag }}" + fail_msg: >- + Cilium agent {{ cilium_agent_image }}, + operator {{ cilium_operator_image }}, + expected {{ cilium_tag }} + vars: + cilium_agent_image: >- + {{ (cilium_info.results + | selectattr('resources', 'defined') + | map(attribute='resources') + | list + | map(attribute='0') + | selectattr('kind', 'equalto', 'DaemonSet') + | list)[0].spec.template.spec.containers[0].image }} + cilium_operator_image: >- + {{ (cilium_info.results + | selectattr('resources', 'defined') + | map(attribute='resources') + | list + | map(attribute='0') + | selectattr('kind', 'equalto', 'Deployment') + | list)[0].spec.template.spec.containers[0].image }} + + - name: Get Hubble relay and UI deployments when enabled + kubernetes.core.k8s_info: + kind: Deployment + name: "{{ item }}" + namespace: kube-system + kubeconfig: "{{ kubecfg_path }}" + register: hubble_info + loop: + - hubble-relay + - hubble-ui + loop_control: + label: "Deployment/{{ item }}" + when: cilium_hubble | bool + + - name: Assert Hubble components are Ready when enabled + ansible.builtin.assert: + that: + - item.resources | length == 1 + - item.resources[0].status.readyReplicas | default(0) >= 1 + success_msg: "Hubble deployment {{ item.resources[0].metadata.name }} is Ready" + fail_msg: "Hubble deployment is not Ready" + loop: "{{ hubble_info.results }}" + loop_control: + label: "Hubble deployment" + when: cilium_hubble | bool + + - name: Get any Flannel DaemonSets with Cilium enabled + kubernetes.core.k8s_info: + kind: DaemonSet + namespace: kube-flannel + kubeconfig: "{{ kubecfg_path }}" + register: no_flannel_ds_cilium + + - name: Assert there are no Flannel DaemonSets + ansible.builtin.assert: + that: no_flannel_ds_cilium.resources | length == 0 + success_msg: "No Flannel DaemonSet present with Cilium" + fail_msg: "A Flannel DaemonSet exists alongside Cilium" + + - name: Verify MetalLB is the active load balancer + when: verify_lb == 'metallb' + block: + - name: Get the MetalLB controller and speaker images + kubernetes.core.k8s_info: + kind: "{{ item.kind }}" + name: "{{ item.name }}" + namespace: metallb-system + kubeconfig: "{{ kubecfg_path }}" + register: metallb_info + loop: + - { kind: Deployment, name: controller } + - { kind: DaemonSet, name: speaker } + loop_control: + label: "{{ item.kind }}/{{ item.name }}" + + - name: Assert MetalLB controller and speaker use the expected image tags + ansible.builtin.assert: + that: + - controller_image | regex_search(metal_lb_controller_tag_version) + - speaker_image | regex_search(metal_lb_speaker_tag_version) + success_msg: >- + MetalLB controller {{ metal_lb_controller_tag_version }}, + speaker {{ metal_lb_speaker_tag_version }} + fail_msg: >- + MetalLB controller {{ controller_image }}, + speaker {{ speaker_image }} + vars: + controller_image: >- + {{ (metallb_info.results + | selectattr('resources', 'defined') + | map(attribute='resources') + | list + | map(attribute='0') + | selectattr('kind', 'equalto', 'Deployment') + | list)[0].spec.template.spec.containers[0].image }} + speaker_image: >- + {{ (metallb_info.results + | selectattr('resources', 'defined') + | map(attribute='resources') + | list + | map(attribute='0') + | selectattr('kind', 'equalto', 'DaemonSet') + | list)[0].spec.template.spec.containers[0].image }} + + - name: Verify kube-vip is the active load balancer + when: verify_lb == 'kube-vip' + block: + - name: Get the kube-vip and cloud provider images + kubernetes.core.k8s_info: + kind: "{{ item.kind }}" + name: "{{ item.name }}" + namespace: kube-system + kubeconfig: "{{ kubecfg_path }}" + register: kubevip_info + loop: + - { kind: DaemonSet, name: kube-vip-ds } + - { kind: Deployment, name: kube-vip-cloud-provider } + loop_control: + label: "{{ item.kind }}/{{ item.name }}" + + - name: Assert the kube-vip and cloud provider image tags + ansible.builtin.assert: + that: + - kubevip_image | regex_search(':' ~ kube_vip_tag_version) + - cloud_provider_image | regex_search(verify_kube_vip_cloud_provider_tag) + success_msg: >- + kube-vip {{ kube_vip_tag_version }}, + cloud provider {{ verify_kube_vip_cloud_provider_tag }} + fail_msg: >- + kube-vip {{ kubevip_image }}, + cloud provider {{ cloud_provider_image }} + vars: + kubevip_image: >- + {{ (kubevip_info.results + | selectattr('resources', 'defined') + | map(attribute='resources') + | list + | map(attribute='0') + | selectattr('kind', 'equalto', 'DaemonSet') + | list)[0].spec.template.spec.containers[0].image }} + cloud_provider_image: >- + {{ (kubevip_info.results + | selectattr('resources', 'defined') + | map(attribute='resources') + | list + | map(attribute='0') + | selectattr('kind', 'equalto', 'Deployment') + | list)[0].spec.template.spec.containers[0].image }} + + - name: Get the MetalLB namespace with kube-vip enabled + kubernetes.core.k8s_info: + kind: Namespace + name: metallb-system + kubeconfig: "{{ kubecfg_path }}" + register: metallb_absent + + - name: Assert the MetalLB namespace does not exist + ansible.builtin.assert: + that: metallb_absent.resources | length == 0 + success_msg: "MetalLB is not installed with kube-vip" + fail_msg: "MetalLB namespace exists alongside kube-vip" diff --git a/molecule/single_node/verify-vars.yml b/molecule/single_node/verify-vars.yml new file mode 100644 index 00000000..55b794bd --- /dev/null +++ b/molecule/single_node/verify-vars.yml @@ -0,0 +1,6 @@ +--- +# Durable verify inputs for the single_node (flannel + MetalLB) scenario. +verify_cni: flannel +verify_lb: metallb +verify_lb_ip_range: + - 192.168.30.91-192.168.30.99 diff --git a/roles/k3s_server/defaults/main.yml b/roles/k3s_server/defaults/main.yml index 1d18efd9..8fe55fb9 100644 --- a/roles/k3s_server/defaults/main.yml +++ b/roles/k3s_server/defaults/main.yml @@ -7,8 +7,8 @@ group_name_master: master kube_vip_arp: true kube_vip_iface: -kube_vip_cloud_provider_tag_version: main -kube_vip_tag_version: v0.7.2 +kube_vip_cloud_provider_tag_version: v0.0.12 +kube_vip_tag_version: v1.2.2 kube_vip_bgp: false kube_vip_bgp_routerid: 127.0.0.1 @@ -19,8 +19,8 @@ kube_vip_bgp_peeras: "64512" kube_vip_bgp_peers: [] kube_vip_bgp_peers_groups: ['k3s_master'] -metal_lb_controller_tag_version: v0.14.3 -metal_lb_speaker_tag_version: v0.14.3 +metal_lb_controller_tag_version: v0.16.0 +metal_lb_speaker_tag_version: v0.16.0 metal_lb_type: native retry_count: 20 diff --git a/roles/k3s_server/meta/main.yml b/roles/k3s_server/meta/main.yml index 7d9fbfda..7a5c0f78 100644 --- a/roles/k3s_server/meta/main.yml +++ b/roles/k3s_server/meta/main.yml @@ -80,11 +80,11 @@ argument_specs: kube_vip_tag_version: description: Image tag for kube-vip - default: v0.7.2 + default: v1.2.2 kube_vip_cloud_provider_tag_version: description: Tag for kube-vip-cloud-provider manifest when enabled - default: main + default: v0.0.12 kube_vip_lb_ip_range: description: IP range for kube-vip load balancer @@ -92,11 +92,11 @@ argument_specs: metal_lb_controller_tag_version: description: Image tag for MetalLB - default: v0.14.3 + default: v0.16.0 metal_lb_speaker_tag_version: description: Image tag for MetalLB - default: v0.14.3 + default: v0.16.0 metal_lb_type: choices: diff --git a/roles/k3s_server/tasks/kube-vip.yml b/roles/k3s_server/tasks/kube-vip.yml index f8b53e6a..29d3f03d 100644 --- a/roles/k3s_server/tasks/kube-vip.yml +++ b/roles/k3s_server/tasks/kube-vip.yml @@ -10,7 +10,7 @@ - name: Download vip cloud provider manifest to first master ansible.builtin.get_url: - url: https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/{{ kube_vip_cloud_provider_tag_version | default('main') }}/manifest/kube-vip-cloud-controller.yaml # noqa yaml[line-length] + url: https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/{{ kube_vip_cloud_provider_tag_version | default('v0.0.12') }}/manifest/kube-vip-cloud-controller.yaml # noqa yaml[line-length] dest: /var/lib/rancher/k3s/server/manifests/kube-vip-cloud-controller.yaml owner: root group: root diff --git a/roles/k3s_server/tasks/metallb.yml b/roles/k3s_server/tasks/metallb.yml index 7624d16d..4dfb72f8 100644 --- a/roles/k3s_server/tasks/metallb.yml +++ b/roles/k3s_server/tasks/metallb.yml @@ -28,3 +28,29 @@ loop_control: label: "{{ item.change }} => {{ item.to }}" when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] + +- name: Read back MetalLB manifest from first master + ansible.builtin.slurp: + src: /var/lib/rancher/k3s/server/manifests/metallb-crds.yaml + register: metallb_manifest + when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] + +- name: Check the MetalLB speaker image reference is present + ansible.builtin.set_fact: + metallb_manifest_has_speaker_tag: >- + {{ + ('quay.io/metallb/speaker:' ~ metal_lb_speaker_tag_version) in + (metallb_manifest.content | default('') | b64decode) + }} + when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] + +- name: Fail if MetalLB speaker tag was not applied to the manifest + ansible.builtin.fail: + msg: >- + MetalLB speaker image reference + quay.io/metallb/speaker:{{ metal_lb_speaker_tag_version }} + was not found in the downloaded manifest. The upstream image + reference may have changed. + when: + - ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] + - not metallb_manifest_has_speaker_tag diff --git a/roles/k3s_server/tasks/vip.yml b/roles/k3s_server/tasks/vip.yml index aba5b4f0..857ed90f 100644 --- a/roles/k3s_server/tasks/vip.yml +++ b/roles/k3s_server/tasks/vip.yml @@ -12,9 +12,9 @@ mode: "0644" when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] -- name: Download vip rbac manifest to first master - ansible.builtin.get_url: - url: https://kube-vip.io/manifests/rbac.yaml +- name: Copy kube-vip RBAC manifest to first master + ansible.builtin.template: + src: kube-vip-rbac.yaml.j2 dest: /var/lib/rancher/k3s/server/manifests/vip-rbac.yaml owner: root group: root diff --git a/roles/k3s_server/templates/kube-vip-rbac.yaml.j2 b/roles/k3s_server/templates/kube-vip-rbac.yaml.j2 new file mode 100644 index 00000000..1f15df25 --- /dev/null +++ b/roles/k3s_server/templates/kube-vip-rbac.yaml.j2 @@ -0,0 +1,48 @@ +# Repository-owned kube-vip RBAC derived from the official v1.2.2 +# `kube-vip manifest rbac` output (run with `--inCluster`). +# Source: https://github.com/kube-vip/kube-vip/blob/v1.2.2/pkg/kubevip/config_generator.go +# Permissions are kept exactly as broad as the upstream manifest. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list", "get", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list", "get", "watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list", "get", "watch", "update"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role +subjects: + - kind: ServiceAccount + name: kube-vip + namespace: kube-system diff --git a/roles/k3s_server/templates/vip.yaml.j2 b/roles/k3s_server/templates/vip.yaml.j2 index 44469a69..39e16de9 100644 --- a/roles/k3s_server/templates/vip.yaml.j2 +++ b/roles/k3s_server/templates/vip.yaml.j2 @@ -36,7 +36,7 @@ spec: - name: vip_interface value: {{ kube_vip_iface }} {% endif %} - - name: vip_cidr + - name: vip_subnet value: "{{ apiserver_endpoint | ansible.utils.ipsubnet | ansible.utils.ipaddr('prefix') }}" - name: cp_enable value: "true" @@ -62,8 +62,8 @@ spec: value: "{{ kube_vip_bgp_routerid }}" {% endif %} {% if _kube_vip_bgp_peers | length > 0 %} - - name: bgppeers - value: "{{ _kube_vip_bgp_peers | map(attribute='peer_address') | zip(_kube_vip_bgp_peers| map(attribute='peer_asn')) | map('join', ',') | join(':') }}" # yamllint disable-line rule:line-length + - name: bgp_peers + value: "{{ _kube_vip_bgp_peers | map(attribute='peer_address') | zip(_kube_vip_bgp_peers | map(attribute='peer_asn')) | map('join', ':') | join(',') }}" # yamllint disable-line rule:line-length {% else %} {% if kube_vip_bgp_as is defined %} - name: bgp_as diff --git a/roles/k3s_server_post/defaults/main.yml b/roles/k3s_server_post/defaults/main.yml index 578e5571..6ac138a4 100644 --- a/roles/k3s_server_post/defaults/main.yml +++ b/roles/k3s_server_post/defaults/main.yml @@ -9,7 +9,7 @@ calico_ebpf: false calico_encapsulation: VXLANCrossSubnet calico_natOutgoing: Enabled # noqa var-naming calico_nodeSelector: all() # noqa var-naming -calico_tag: v3.27.2 +calico_tag: v3.32.1 cilium_bgp: false cilium_exportPodCIDR: true # noqa var-naming @@ -20,6 +20,8 @@ cilium_bgp_neighbors_groups: ['k3s_all'] cilium_bgp_lb_cidr: 192.168.31.0/24 cilium_hubble: true cilium_mode: native +cilium_tag: v1.20.0 +cilium_cli_tag: v0.19.7 cluster_cidr: 10.52.0.0/16 enable_bpf_masquerade: true @@ -28,5 +30,5 @@ group_name_master: master metal_lb_mode: layer2 metal_lb_available_timeout: 240s -metal_lb_controller_tag_version: v0.14.3 +metal_lb_controller_tag_version: v0.16.0 metal_lb_ip_range: 192.168.30.80-192.168.30.90 diff --git a/roles/k3s_server_post/meta/main.yml b/roles/k3s_server_post/meta/main.yml index f9fc83d5..93c2cfb2 100644 --- a/roles/k3s_server_post/meta/main.yml +++ b/roles/k3s_server_post/meta/main.yml @@ -43,7 +43,7 @@ argument_specs: calico_tag: description: Calico version tag - default: v3.27.2 + default: v3.32.1 cilium_bgp: description: @@ -93,12 +93,24 @@ argument_specs: default: ~ cilium_mode: - description: Inner-node communication mode + description: + - Inner-node communication mode. + - Cilium accepts `native` or `tunnel`. `routed` is accepted as a + legacy alias and is mapped to `tunnel` at install time. default: native choices: - native + - tunnel - routed + cilium_tag: + description: Cilium version tag + default: v1.20.0 + + cilium_cli_tag: + description: Cilium CLI version tag + default: v0.19.7 + cluster_cidr: description: Inner-cluster IP range default: 10.52.0.0/16 @@ -131,7 +143,7 @@ argument_specs: metal_lb_controller_tag_version: description: Image tag for MetalLB - default: v0.14.3 + default: v0.16.0 metal_lb_mode: description: Metallb mode diff --git a/roles/k3s_server_post/tasks/calico.yml b/roles/k3s_server_post/tasks/calico.yml index 2a9302f3..70dac411 100644 --- a/roles/k3s_server_post/tasks/calico.yml +++ b/roles/k3s_server_post/tasks/calico.yml @@ -11,6 +11,14 @@ group: root mode: "0755" + - name: "Download to first master: Calico CRD bundle for {{ calico_tag }}" + ansible.builtin.get_url: + url: https://raw.githubusercontent.com/projectcalico/calico/{{ calico_tag }}/manifests/v1_crd_projectcalico_org.yaml # noqa yaml[line-length] + dest: /tmp/k3s/v1_crd_projectcalico_org.yaml + owner: root + group: root + mode: "0755" + - name: "Download to first master: manifest for Tigera Operator and Calico CRDs" ansible.builtin.get_url: url: https://raw.githubusercontent.com/projectcalico/calico/{{ calico_tag }}/manifests/tigera-operator.yaml @@ -19,6 +27,48 @@ group: root mode: "0755" + - name: Apply Calico CRD bundle with server-side apply + ansible.builtin.command: >- + {{ k3s_kubectl_binary | default('k3s kubectl') }} + apply --server-side --force-conflicts + -f /tmp/k3s/v1_crd_projectcalico_org.yaml + register: apply_crds + changed_when: >- + 'created' in apply_crds.stdout or + 'configured' in apply_crds.stdout or + 'applied' in apply_crds.stdout + failed_when: apply_crds.rc != 0 + + - name: Apply Tigera Operator manifest idempotently with server-side apply + ansible.builtin.command: >- + {{ k3s_kubectl_binary | default('k3s kubectl') }} + apply --server-side --force-conflicts + -f /tmp/k3s/tigera-operator.yaml + register: apply_operator + changed_when: >- + 'created' in apply_operator.stdout or + 'configured' in apply_operator.stdout or + 'applied' in apply_operator.stdout + failed_when: apply_operator.rc != 0 + + - name: Wait for Tigera Operator and managed CRDs to become available + ansible.builtin.command: >- + {{ k3s_kubectl_binary | default('k3s kubectl') }} wait {{ item.type }}/{{ item.name }} + {% if item.namespace is defined %}--namespace='{{ item.namespace }}'{% endif %} + --for=condition={{ item.condition }} + --timeout=30s + register: tigera_result + changed_when: false + until: tigera_result is succeeded + retries: 7 + delay: 7 + with_items: + - { name: tigera-operator, type: deployment, namespace: tigera-operator, condition: Available=True } + - { name: installations.operator.tigera.io, type: crd, condition: Established } + - { name: apiservers.operator.tigera.io, type: crd, condition: Established } + loop_control: + label: "{{ item.type }}/{{ item.name }}" + - name: Copy Calico custom resources manifest to first master ansible.builtin.template: src: calico.crs.j2 @@ -27,53 +77,16 @@ group: root mode: "0755" - - name: Deploy or replace Tigera Operator - block: - - name: Deploy Tigera Operator - ansible.builtin.command: - cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} create -f /tmp/k3s/tigera-operator.yaml" - register: create_operator - changed_when: "'created' in create_operator.stdout" - failed_when: "'Error' in create_operator.stderr and 'already exists' not in create_operator.stderr" - rescue: - - name: Replace existing Tigera Operator - ansible.builtin.command: - cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} replace -f /tmp/k3s/tigera-operator.yaml" - register: replace_operator - changed_when: "'replaced' in replace_operator.stdout" - failed_when: "'Error' in replace_operator.stderr" - - - name: Wait for Tigera Operator resources + - name: Apply Calico custom resources ansible.builtin.command: >- - {{ k3s_kubectl_binary | default('k3s kubectl') }} wait {{ item.type }}/{{ item.name }} - --namespace='tigera-operator' - --for=condition=Available=True - --timeout=30s - register: tigera_result - changed_when: false - until: tigera_result is succeeded - retries: 7 - delay: 7 - with_items: - - { name: tigera-operator, type: deployment } - loop_control: - label: "{{ item.type }}/{{ item.name }}" - - - name: Deploy Calico custom resources - block: - - name: Deploy custom resources for Calico - ansible.builtin.command: - cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} create -f /tmp/k3s/custom-resources.yaml" - register: create_cr - changed_when: "'created' in create_cr.stdout" - failed_when: "'Error' in create_cr.stderr and 'already exists' not in create_cr.stderr" - rescue: - - name: Apply new Calico custom resource manifest - ansible.builtin.command: - cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} apply -f /tmp/k3s/custom-resources.yaml" - register: apply_cr - changed_when: "'configured' in apply_cr.stdout or 'created' in apply_cr.stdout" - failed_when: "'Error' in apply_cr.stderr" + {{ k3s_kubectl_binary | default('k3s kubectl') }} + apply -f /tmp/k3s/custom-resources.yaml + register: apply_cr + changed_when: >- + 'configured' in apply_cr.stdout or + 'created' in apply_cr.stdout or + 'unchanged' in apply_cr.stdout + failed_when: apply_cr.rc != 0 - name: Wait for Calico system resources to be available ansible.builtin.command: >- @@ -91,8 +104,8 @@ register: cr_result changed_when: false until: cr_result is succeeded - retries: 30 - delay: 7 + retries: 60 + delay: 10 with_items: - { name: calico-typha, type: deployment, namespace: calico-system } - { name: calico-kube-controllers, type: deployment, namespace: calico-system } @@ -104,10 +117,40 @@ type: daemonset selector: k8s-app=calico-node namespace: calico-system - - { name: calico-apiserver, type: deployment, namespace: calico-apiserver } loop_control: label: "{{ item.type }}/{{ item.name }}" + # The Calico API server is an optional add-on for managing Calico policy via + # kubectl (the projectcalico.org/v3 Kubernetes API). It is not required for + # Calico CNI data plane operation and does not reconcile on K3s with recent + # Calico releases, so it is treated as best-effort here. + - name: Wait for the optional Calico API server (best effort) + ansible.builtin.command: >- + {{ k3s_kubectl_binary | default('k3s kubectl') }} wait deployment/calico-apiserver + --namespace=calico-apiserver --for=condition=Available --timeout=30s + register: cr_apiserver + changed_when: false + until: cr_apiserver is succeeded + retries: 30 + delay: 10 + ignore_errors: true + + - name: Verify Calico TigeraStatus reports Available + ansible.builtin.command: >- + {{ k3s_kubectl_binary | default('k3s kubectl') }} + get tigerastatus {{ item }} + -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' + register: tigera_status + changed_when: false + until: tigera_status.stdout | trim == 'True' + retries: 30 + delay: 7 + failed_when: tigera_status.rc != 0 or (tigera_status.stdout | trim) != 'True' + with_items: + - calico + loop_control: + label: "tigerastatus/{{ item }}" + - name: Patch Felix configuration for eBPF mode ansible.builtin.command: cmd: > diff --git a/roles/k3s_server_post/tasks/cilium.yml b/roles/k3s_server_post/tasks/cilium.yml index d7a48b01..9b7f1ea2 100644 --- a/roles/k3s_server_post/tasks/cilium.yml +++ b/roles/k3s_server_post/tasks/cilium.yml @@ -30,26 +30,13 @@ }} when: cilium_cli_installed.rc == 0 - - name: Get latest stable Cilium CLI version file - ansible.builtin.get_url: - url: https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt - dest: /tmp/k3s/cilium-cli-stable.txt - owner: root - group: root - mode: "0755" - - - name: Read Cilium CLI stable version from file - ansible.builtin.command: cat /tmp/k3s/cilium-cli-stable.txt - register: cli_ver - changed_when: false - - name: Log installed Cilium CLI version ansible.builtin.debug: msg: "Installed Cilium CLI version: {{ installed_cli_version | default('Not installed') }}" - - name: Log latest stable Cilium CLI version + - name: Log pinned Cilium CLI version ansible.builtin.debug: - msg: "Latest Cilium CLI version: {{ cli_ver.stdout }}" + msg: "Pinned Cilium CLI version: {{ cilium_cli_tag }}" - name: Determine if Cilium CLI needs installation or update ansible.builtin.set_fact: @@ -57,7 +44,7 @@ {{ cilium_cli_installed.rc != 0 or (cilium_cli_installed.rc == 0 and - installed_cli_version != cli_ver.stdout) + installed_cli_version != cilium_cli_tag) }} - name: Install or update Cilium CLI @@ -78,7 +65,7 @@ - .tar.gz - .tar.gz.sha256sum vars: - cilium_base_url: https://github.com/cilium/cilium-cli/releases/download/{{ cli_ver.stdout }} + cilium_base_url: https://github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_tag }} - name: Verify the downloaded tarball ansible.builtin.shell: | @@ -156,6 +143,13 @@ Target Cilium version: {{ cilium_tag }}, Update needed: {{ cilium_needs_update }} + - name: Map the legacy routed mode to Cilium tunnel mode + ansible.builtin.set_fact: + # Cilium 1.20 accepts `native` or `tunnel`. `routed` was the name used + # by older releases and is kept only as a backward-compatible alias. + cilium_routing_mode: >- + {{ 'tunnel' if cilium_mode == 'routed' else cilium_mode }} + - name: Install Cilium ansible.builtin.command: >- {% if cilium_installed.rc != 0 %} @@ -167,13 +161,13 @@ --helm-set operator.replicas="1" {{ '--helm-set devices=' + cilium_iface if cilium_iface != 'auto' else '' }} --helm-set ipam.operator.clusterPoolIPv4PodCIDRList={{ cluster_cidr }} - {% if cilium_mode == "native" or (cilium_bgp and cilium_exportPodCIDR != 'false') %} + {% if cilium_routing_mode == "native" or (cilium_bgp and cilium_exportPodCIDR != 'false') %} --helm-set ipv4NativeRoutingCIDR={{ cluster_cidr }} {% endif %} --helm-set k8sServiceHost="127.0.0.1" --helm-set k8sServicePort="6444" - --helm-set routingMode={{ cilium_mode }} - --helm-set autoDirectNodeRoutes={{ "true" if cilium_mode == "native" else "false" }} + --helm-set routingMode={{ cilium_routing_mode }} + --helm-set autoDirectNodeRoutes={{ "true" if cilium_routing_mode == "native" else "false" }} --helm-set kubeProxyReplacement={{ kube_proxy_replacement }} --helm-set bpf.masquerade={{ enable_bpf_masquerade }} --helm-set bgpControlPlane.enabled={{ cilium_bgp | default("false") }} @@ -181,8 +175,8 @@ --helm-set hubble.relay.enabled={{ "true" if cilium_hubble else "false" }} --helm-set hubble.ui.enabled={{ "true" if cilium_hubble else "false" }} {% if kube_proxy_replacement is not false %} - --helm-set bpf.loadBalancer.algorithm={{ bpf_lb_algorithm }} - --helm-set bpf.loadBalancer.mode={{ bpf_lb_mode }} + --helm-set loadBalancer.algorithm={{ bpf_lb_algorithm }} + --helm-set loadBalancer.mode={{ bpf_lb_mode }} {% endif %} environment: KUBECONFIG: "{{ ansible_user_dir }}/.kube/config" @@ -218,6 +212,16 @@ when: >- not item.check_hubble | default(false) or (item.check_hubble | default(false) and cilium_hubble) + - name: Wait for Cilium status to be healthy + ansible.builtin.command: cilium status --wait + environment: + KUBECONFIG: "{{ ansible_user_dir }}/.kube/config" + register: cilium_status + changed_when: false + until: cilium_status.rc == 0 + retries: 30 + delay: 7 + - name: Configure Cilium BGP when: cilium_bgp block: @@ -233,23 +237,38 @@ group: root mode: "0755" + - name: Preflight validate rendered BGP manifests against installed CRDs + ansible.builtin.command: >- + {{ k3s_kubectl_binary | default('k3s kubectl') }} + apply --dry-run=server -f /tmp/k3s/cilium-bgp.yaml + register: preflight_cr + changed_when: false + failed_when: preflight_cr.rc != 0 + - name: Apply BGP manifests - ansible.builtin.command: - cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} apply -f /tmp/k3s/cilium-bgp.yaml" + ansible.builtin.command: >- + {{ k3s_kubectl_binary | default('k3s kubectl') }} + apply -f /tmp/k3s/cilium-bgp.yaml register: apply_cr changed_when: "'configured' in apply_cr.stdout or 'created' in apply_cr.stdout" - failed_when: "'is invalid' in apply_cr.stderr" - ignore_errors: true + failed_when: apply_cr.rc != 0 - - name: Print error message if BGP manifests application fails - ansible.builtin.debug: - msg: "{{ apply_cr.stderr }}" - when: "'is invalid' in apply_cr.stderr" + - name: Remove deprecated CiliumBGPPeeringPolicy after v2 resources are accepted + ansible.builtin.command: >- + {{ k3s_kubectl_binary | default('k3s kubectl') }} + delete CiliumBGPPeeringPolicy.cilium.io 01-bgp-peering-policy + register: delete_old_policy + changed_when: "'deleted' in delete_old_policy.stdout" + # The policy (and possibly its CRD) may already be absent; this is + # intentionally tolerated, not a command whose failure must be hidden. + failed_when: false - name: Test for BGP config resources ansible.builtin.command: "{{ item }}" loop: - - "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPPeeringPolicy.cilium.io" + - "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPClusterConfig.cilium.io" + - "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPPeerConfig.cilium.io" + - "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumBGPAdvertisement.cilium.io" - "{{ k3s_kubectl_binary | default('k3s kubectl') }} get CiliumLoadBalancerIPPool.cilium.io" changed_when: false loop_control: diff --git a/roles/k3s_server_post/tasks/metallb.yml b/roles/k3s_server_post/tasks/metallb.yml index 4a3279c6..63afcea7 100644 --- a/roles/k3s_server_post/tasks/metallb.yml +++ b/roles/k3s_server_post/tasks/metallb.yml @@ -85,17 +85,9 @@ - name: Set metallb webhook service name ansible.builtin.set_fact: - metallb_webhook_service_name: >- - {{ - ( - (metal_lb_controller_tag_version | regex_replace('^v', '')) - is - version('0.14.4', '<', version_type='semver') - ) | ternary( - 'webhook-service', - 'metallb-webhook-service' - ) - }} + # Inspected the v0.16.0 manifest: the newer webhook Service name is used + # by every supported MetalLB release, so the old pre-0.14.4 branch is gone. + metallb_webhook_service_name: metallb-webhook-service - name: Test metallb-system webhook-service endpoint ansible.builtin.command: >- diff --git a/roles/k3s_server_post/templates/cilium.crs.j2 b/roles/k3s_server_post/templates/cilium.crs.j2 index 5a9e81c8..c0cfbea6 100644 --- a/roles/k3s_server_post/templates/cilium.crs.j2 +++ b/roles/k3s_server_post/templates/cilium.crs.j2 @@ -1,40 +1,84 @@ -apiVersion: "cilium.io/v2alpha1" -kind: CiliumBGPPeeringPolicy +# Cilium BGP Control Plane v2 resources. +# Replace the deprecated v2alpha1 CiliumBGPPeeringPolicy that was removed +# in Cilium 1.19. +{% set _cilium_default_peer = {'peer_address': cilium_bgp_peer_address, 'peer_asn': cilium_bgp_peer_asn} %} +{% set _cilium_peers = _cilium_bgp_neighbors if _cilium_bgp_neighbors | length > 0 else [_cilium_default_peer] %} +apiVersion: cilium.io/v2 +kind: CiliumBGPPeerConfig metadata: - name: 01-bgp-peering-policy -spec: # CiliumBGPPeeringPolicySpec - virtualRouters: # []CiliumBGPVirtualRouter - - localASN: {{ cilium_bgp_my_asn }} - exportPodCIDR: {{ cilium_exportPodCIDR | default('true') }} - neighbors: # []CiliumBGPNeighbor -{% if _cilium_bgp_neighbors | length > 0 %} -{% for item in _cilium_bgp_neighbors %} - - peerAddress: '{{ item.peer_address + "/32"}}' - peerASN: {{ item.peer_asn }} - eBGPMultihopTTL: 10 - connectRetryTimeSeconds: 120 - holdTimeSeconds: 90 - keepAliveTimeSeconds: 30 - gracefulRestart: - enabled: true - restartTimeSeconds: 120 -{% endfor %} -{% else %} - - peerAddress: '{{ cilium_bgp_peer_address + "/32"}}' - peerASN: {{ cilium_bgp_peer_asn }} - eBGPMultihopTTL: 10 - connectRetryTimeSeconds: 120 - holdTimeSeconds: 90 - keepAliveTimeSeconds: 30 - gracefulRestart: - enabled: true - restartTimeSeconds: 120 -{% endif %} - serviceSelector: - matchExpressions: - - {key: somekey, operator: NotIn, values: ['never-used-value']} + name: cilium-peer +spec: + # Matches the timers and multihop used by the previous v2alpha1 policy. + ebgpMultihop: 10 + timers: + connectRetryTimeSeconds: 120 + holdTimeSeconds: 90 + keepAliveTimeSeconds: 30 + gracefulRestart: + enabled: true + restartTimeSeconds: 120 + families: + - afi: ipv4 + safi: unicast + advertisements: + matchLabels: + advertise: "bgp" --- -apiVersion: "cilium.io/v2alpha1" +apiVersion: cilium.io/v2 +kind: CiliumBGPClusterConfig +metadata: + name: cilium-bgp +spec: + # Explicitly select every node so the BGP instance runs across the cluster. + nodeSelector: + matchExpressions: + - key: somekey + operator: NotIn + values: ['never-used-value'] + bgpInstances: + - name: "instance-{{ cilium_bgp_my_asn }}" + localASN: {{ cilium_bgp_my_asn }} + peers: +{% for peer in _cilium_peers %} + - name: "peer-{{ peer.peer_asn }}-{{ loop.index }}" + peerASN: {{ peer.peer_asn }} + peerAddress: {{ peer.peer_address }} + peerConfigRef: + name: cilium-peer +{% endfor %} +{% if cilium_exportPodCIDR %} +--- +apiVersion: cilium.io/v2 +kind: CiliumBGPAdvertisement +metadata: + name: cilium-pod-cidrs + labels: + advertise: "bgp" +spec: + advertisements: + - advertisementType: "PodCIDR" +{% endif %} +--- +apiVersion: cilium.io/v2 +kind: CiliumBGPAdvertisement +metadata: + name: cilium-lb-services + labels: + advertise: "bgp" +spec: + advertisements: + - advertisementType: "Service" + service: + addresses: + - LoadBalancerIP + # Advertise all Services carrying an ingress address from the pool. + selector: + matchExpressions: + - key: somekey + operator: NotIn + values: ['never-used-value'] +--- +apiVersion: cilium.io/v2 kind: CiliumLoadBalancerIPPool metadata: name: "01-lb-pool"