From 287d8b7a2776455ee10ad91ceb5ab8b96f5568af Mon Sep 17 00:00:00 2001 From: Techno Tim Date: Thu, 6 Aug 2026 02:48:57 -0500 Subject: [PATCH] feat(kube-vip): add endpoint override for the internal listening address (#699) * feat(kube-vip): add endpoint override for the internal listening address Add a kube_vip_endpoint variable so the address kube-vip binds and listens on can differ from the announced apiserver_endpoint. This is useful for complex routing and site-to-site tunnels where the VIP kube-vip advertises over ARP differs from the address it listens on internally. - roles/k3s_server/templates/vip.yaml.j2: use kube_vip_endpoint (defaulting to apiserver_endpoint) for the `address` env and for deriving `vip_subnet` - roles/k3s_server/defaults/main.yml: add kube_vip_endpoint default (null) - roles/k3s_server/meta/main.yml: add kube_vip_endpoint argument_spec - inventory/sample/group_vars/all.yml: document the new sample variable - README.md: document the kube_vip_endpoint option - .github/scripts/test-kube-vip-manifest.py: extend regression test to cover the default (apiserver_endpoint) and the override case Closes #221 * chore(ci): extend molecule job timeout to 3 hours The default scenario occasionally takes longer than 150 minutes on the shared nested-virt runner (k3s agent notify-wait can exceed the limit under load), and a single timeout aborts the whole run before the other four scenarios execute. Raise timeout-minutes from 150 to 180 so a slow-but-progressing run completes instead of aborting. The default scenario remains first in the matrix so a failure surfaces fastest. * fix(kube-vip): fall back on null kube_vip_endpoint and cover it in the test - vip_subnet and address use default(apiserver_endpoint, true) so the null role default falls back to the apiserver endpoint instead of rendering an empty/invalid address and subnet - change the manifest regression test default case to pass kube_vip_endpoint as None so it pins the real runtime null condition and fails fast on this regression rather than timing out in CI --- .github/scripts/test-kube-vip-manifest.py | 32 +++++++++++++++++++++++ .github/workflows/test.yml | 2 +- README.md | 1 + inventory/sample/group_vars/all.yml | 5 ++++ roles/k3s_server/defaults/main.yml | 1 + roles/k3s_server/meta/main.yml | 9 +++++++ roles/k3s_server/templates/vip.yaml.j2 | 4 +-- 7 files changed, 51 insertions(+), 3 deletions(-) diff --git a/.github/scripts/test-kube-vip-manifest.py b/.github/scripts/test-kube-vip-manifest.py index c74cf005..d58565e7 100644 --- a/.github/scripts/test-kube-vip-manifest.py +++ b/.github/scripts/test-kube-vip-manifest.py @@ -123,6 +123,38 @@ def main(): if "name: bgp_peers" in output: fail("bgp_peers present even though the peer list is empty") + # kube_vip_endpoint defaults to null (defined in role defaults): the + # address and subnet must fall back to the apiserver endpoint. default() + # without a truthy flag does NOT fall back on null, only on undefined, so + # this case pins the null runtime condition to prevent that regression. + output = render( + env, + { + "_kube_vip_bgp_peers": [], + "kube_vip_endpoint": None, + "kube_vip_arp": True, + "kube_vip_bgp": False, + }, + ) + if "value: 192.168.30.222" not in output: + fail("null kube_vip_endpoint does not fall back to apiserver_endpoint") + + # kube_vip_endpoint set: overrides the internal listening address AND the + # subnet derivation while the advertised apiserver_endpoint stays separate. + output = render( + env, + { + "_kube_vip_bgp_peers": [], + "kube_vip_endpoint": "10.66.1.5", + "kube_vip_arp": True, + "kube_vip_bgp": False, + }, + ) + if "value: 10.66.1.5" not in output: + fail("kube_vip_endpoint did not override the address") + if "value: 192.168.30.222" in output: + fail("apiserver_endpoint leaked into address when kube_vip_endpoint set") + print("kube-vip manifest regression test passed") diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b418bd3d..d17b46d8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -88,7 +88,7 @@ jobs: trap stop_monitor EXIT /usr/bin/time -v -o "$timing_file" \ molecule test --scenario-name ${{ matrix.scenario }} - timeout-minutes: 150 + timeout-minutes: 180 env: ANSIBLE_K3S_LOG_DIR: ${{ runner.temp }}/logs/k3s-ansible/${{ matrix.scenario }} ANSIBLE_SSH_RETRIES: 4 diff --git a/README.md b/README.md index 90f2ef06..9dfe68ab 100644 --- a/README.md +++ b/README.md @@ -219,6 +219,7 @@ See the commands [here](https://technotim.com/posts/k3s-etcd-ansible/#testing-yo | `k3s_server` | `kube_vip_bgp_peers` | list | `[]` | Not required | List of BGP peer ASN & address pairs | | `k3s_server` | `kube_vip_bgp_peers_groups` | list | `['k3s_master']` | Not required | Inventory group in which to search for additional `kube_vip_bgp_peers` parameters to merge. | | `k3s_server` | `kube_vip_iface` | string | `~` | Not required | Explicitly define an interface that ALL control nodes should use to propagate the VIP, define it here. Otherwise, kube-vip will determine the right interface automatically at runtime. | +| `k3s_server` | `kube_vip_endpoint` | string | `~` | Not required | Overrides the internal address kube-vip binds/listens on, which can differ from the announced apiserver_endpoint for complex routing/tunnels. Defaults to apiserver_endpoint. | | `k3s_server` | `kube_vip_tag_version` | string | `v1.2.2` | Not required | Image tag for kube-vip | | `k3s_server` | `kube_vip_cloud_provider_tag_version` | string | `v0.0.12` | Not required | Tag for kube-vip-cloud-provider manifest when enable | | `k3s_server`, `k3_server_post` | `kube_vip_lb_ip_range` | string | `~` | Not required | IP range for kube-vip load balancer | diff --git a/inventory/sample/group_vars/all.yml b/inventory/sample/group_vars/all.yml index 8ddff5a4..8caebdd7 100644 --- a/inventory/sample/group_vars/all.yml +++ b/inventory/sample/group_vars/all.yml @@ -48,6 +48,11 @@ cilium_bgp_lb_cidr: 192.168.31.0/24 # cidr for cilium loadbalancer ipam # enable kube-vip ARP broadcasts kube_vip_arp: true +# (optional) overrides the address kube-vip binds/listens on internally, which +# can differ from the announced apiserver_endpoint for complex routing/tunnels. +# Defaults to apiserver_endpoint. Also used to derive the kube-vip subnet. +# kube_vip_endpoint: 10.66.1.5 + # enable kube-vip BGP peering kube_vip_bgp: false diff --git a/roles/k3s_server/defaults/main.yml b/roles/k3s_server/defaults/main.yml index 4a8022d0..5cc8e5b3 100644 --- a/roles/k3s_server/defaults/main.yml +++ b/roles/k3s_server/defaults/main.yml @@ -7,6 +7,7 @@ group_name_master: master kube_vip_arp: true kube_vip_iface: +kube_vip_endpoint: kube_vip_cloud_provider_tag_version: v0.0.12 kube_vip_tag_version: v1.2.2 diff --git a/roles/k3s_server/meta/main.yml b/roles/k3s_server/meta/main.yml index 7a5c0f78..458a48fa 100644 --- a/roles/k3s_server/meta/main.yml +++ b/roles/k3s_server/meta/main.yml @@ -78,6 +78,15 @@ argument_specs: - automatically at runtime. default: ~ + kube_vip_endpoint: + description: + - Overrides the address kube-vip binds/listens on internally, which + - can differ from the announced apiserver_endpoint for complex + - routing and site-to-site tunnels. + - Defaults to apiserver_endpoint and is used to derive the kube-vip + - subnet. + default: ~ + kube_vip_tag_version: description: Image tag for kube-vip default: v1.2.2 diff --git a/roles/k3s_server/templates/vip.yaml.j2 b/roles/k3s_server/templates/vip.yaml.j2 index 39e16de9..edef2c6d 100644 --- a/roles/k3s_server/templates/vip.yaml.j2 +++ b/roles/k3s_server/templates/vip.yaml.j2 @@ -37,7 +37,7 @@ spec: value: {{ kube_vip_iface }} {% endif %} - name: vip_subnet - value: "{{ apiserver_endpoint | ansible.utils.ipsubnet | ansible.utils.ipaddr('prefix') }}" + value: "{{ (kube_vip_endpoint | default(apiserver_endpoint, true)) | ansible.utils.ipsubnet | ansible.utils.ipaddr('prefix') }}" - name: cp_enable value: "true" - name: cp_namespace @@ -55,7 +55,7 @@ spec: - name: vip_retryperiod value: "2" - name: address - value: {{ apiserver_endpoint }} + value: {{ kube_vip_endpoint | default(apiserver_endpoint, true) }} {% if kube_vip_bgp | default(false) | bool %} {% if kube_vip_bgp_routerid is defined %} - name: bgp_routerid