k3s_server: add kube_vip_arp parameter (#550)

With the kube_vip_arp parameter it is possible to set or unset the
vip_arp environment variable of the kube-vip-ds daemonset. The value of
the kube_vip_arp is true by default to not change the existing default.

Signed-off-by: Christian Berendt <berendt@osism.tech>
Co-authored-by: Techno Tim <timothystewart6@gmail.com>

.github/workflows/cache.yml

@@ -16,7 +16,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # 5.1.0
+        uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # 5.1.1
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies

.github/workflows/lint.yml

@@ -16,7 +16,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # 5.1.0
+        uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # 5.1.1
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies
@@ -47,7 +47,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # 4.1.7
       - name: Ensure SHA pinned actions
-        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@2f2ebc6d914ab515939dc13f570f91baeb2c194c # 3.0.6
+        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@b88cd0aad2c36a63e42c71f81cb1958fed95ac87 # 3.0.10
         with:
           allowlist: |
             aws-actions/

.github/workflows/test.yml

@@ -59,7 +59,7 @@ jobs:
           EOF
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # 5.1.0
+        uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # 5.1.1
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies
@@ -118,7 +118,7 @@ jobs:
 
       - name: Upload log files
         if: always() # do this even if a step before has failed
-        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # 4.3.3
+        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # 4.3.4
         with:
           name: logs
           path: |

inventory/sample/group_vars/all.yml

@@ -1,5 +1,5 @@
 ---
-k3s_version: v1.29.2+k3s1
+k3s_version: v1.30.2+k3s2
 # this is the user that has ssh access to these machines
 ansible_user: ansibleuser
 systemd_dir: /etc/systemd/system
@@ -13,13 +13,13 @@ flannel_iface: "eth0"
 # uncomment calico_iface to use tigera operator/calico cni instead of flannel https://docs.tigera.io/calico/latest/about
 # calico_iface: "eth0"
 calico_ebpf: false           # use eBPF dataplane instead of iptables
-calico_tag: "v3.27.2"        # calico version tag
+calico_tag: "v3.28.0"        # calico version tag
 
 # uncomment cilium_iface to use cilium cni instead of flannel or calico
 # ensure v4.19.57, v5.1.16, v5.2.0 or more recent kernel
 # cilium_iface: "eth0"
 cilium_mode: "native"        # native when nodes on same subnet or using bgp, else set routed
-cilium_tag: "v1.15.2"        # cilium version tag
+cilium_tag: "v1.16.0"        # cilium version tag
 cilium_hubble: true          # enable hubble observability relay and ui
 
 # if using calico or cilium, you may specify the cluster pod cidr pool
@@ -72,7 +72,7 @@ extra_agent_args: >-
   {{ extra_args }}
 
 # image tag for kube-vip
-kube_vip_tag_version: "v0.7.2"
+kube_vip_tag_version: "v0.8.2"
 
 # tag for kube-vip-cloud-provider manifest
 # kube_vip_cloud_provider_tag_version: "main"
@@ -93,8 +93,8 @@ metal_lb_mode: "layer2"
 # metal_lb_bgp_peer_address: "192.168.30.1"
 
 # image tag for metal lb
-metal_lb_speaker_tag_version: "v0.14.3"
+metal_lb_speaker_tag_version: "v0.14.8"
-metal_lb_controller_tag_version: "v0.14.3"
+metal_lb_controller_tag_version: "v0.14.8"
 
 # metallb ip range for load balancer
 metal_lb_ip_range: "192.168.30.80-192.168.30.90"

requirements.txt

@@ -6,7 +6,7 @@
 #
 ansible-compat==4.1.11
     # via molecule
-ansible-core==2.17.0
+ansible-core==2.17.2
     # via
     #   -r requirements.in
     #   ansible-compat
@@ -96,7 +96,7 @@ platformdirs==4.1.0
     # via virtualenv
 pluggy==1.3.0
     # via molecule
-pre-commit==3.7.1
+pre-commit==3.8.0
     # via -r requirements.in
 pre-commit-hooks==4.6.0
     # via -r requirements.in

roles/download/meta/main.yml

@@ -1,8 +0,0 @@
----
-argument_specs:
-  main:
-    short_description: Manage the downloading of K3S binaries
-    options:
-      k3s_version:
-        description: The desired version of K3S
-        required: true

roles/k3s_agent/defaults/main.yml

@@ -1,4 +0,0 @@
----
-extra_agent_args: ""
-group_name_master: master
-systemd_dir: /etc/systemd/system

roles/k3s_agent/meta/main.yml

@@ -1,35 +0,0 @@
----
-argument_specs:
-  main:
-    short_description: Setup k3s agents
-    options:
-      apiserver_endpoint:
-        description: Virtual ip-address configured on each master
-        required: true
-
-      extra_agent_args:
-        description: Extra arguments for agents nodes
-
-      group_name_master:
-        description: Name of the master group
-        default: master
-
-      k3s_token:
-        description: Token used to communicate between masters
-
-      proxy_env:
-        type: dict
-        description: Internet proxy configurations
-        default: ~
-        options:
-          HTTP_PROXY:
-            required: true
-          HTTPS_PROXY:
-            required: true
-          NO_PROXY:
-            required: true
-
-      systemd_dir:
-        description: Path to systemd services
-        default: /etc/systemd/system
-        required: true

roles/k3s_agent/templates/k3s.service.j2

@@ -12,7 +12,7 @@ ExecStart=/usr/local/bin/k3s agent \
   --server https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443 \
   {% if is_pxe_booted | default(false) %}--snapshotter native \
   {% endif %}--token {{ hostvars[groups[group_name_master | default('master')][0]]['token'] | default(k3s_token) }} \
-  {{ extra_agent_args }}
+  {{ extra_agent_args | default("") }}
 KillMode=process
 Delegate=yes
 LimitNOFILE=1048576

roles/k3s_custom_registries/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+# Indicates whether custom registries for k3s should be configured
+# Possible values:
+#   - present
+#   - absent
+state: present

roles/k3s_custom_registries/meta/main.yml

@@ -1,20 +0,0 @@
----
-argument_specs:
-  main:
-    short_description: Configure the use of a custom container registry
-    options:
-      custom_registries_yaml:
-        description:
-          - YAML block defining custom registries.
-          - >
-            The following is an example that pulls all images used in
-            this playbook through your private registries.
-          - >
-            It also allows you to pull your own images from your private
-            registry, without having to use imagePullSecrets in your
-            deployments.
-          - >
-            If all you need is your own images and you don't care about
-            caching the docker/quay/ghcr.io images, you can just remove
-            those from the mirrors: section.
-        required: true

roles/k3s_server/defaults/main.yml

@@ -1,18 +1,15 @@
 ---
-extra_server_args: ""
+# If you want to explicitly define an interface that ALL control nodes
+# should use to propagate the VIP, define it here. Otherwise, kube-vip
+# will determine the right interface automatically at runtime.
+kube_vip_iface: null
 
+# Enables ARP broadcasts from Leader
+kube_vip_arp: true
+
+# Name of the master group
 group_name_master: master
 
-kube_vip_iface: ~
-kube_vip_cloud_provider_tag_version: main
-kube_vip_tag_version: v0.7.2
-
-metal_lb_controller_tag_version: v0.14.3
-metal_lb_speaker_tag_version: v0.14.3
-metal_lb_type: native
-
-retry_count: 20
-
 # yamllint disable rule:line-length
 server_init_args: >-
   {% if groups[group_name_master | default('master')] | length > 1 %}
@@ -23,6 +20,4 @@ server_init_args: >-
     {% endif %}
     --token {{ k3s_token }}
   {% endif %}
-  {{ extra_server_args }}
+  {{ extra_server_args | default('') }}
-
-systemd_dir: /etc/systemd/system

roles/k3s_server/meta/main.yml

@@ -1,86 +0,0 @@
----
-argument_specs:
-  main:
-    short_description: Setup k3s servers
-    options:
-      apiserver_endpoint:
-        description: Virtual ip-address configured on each master
-        required: true
-
-      cilium_bgp:
-        description:
-          - Enable cilium BGP control plane for LB services and pod cidrs.
-          - Disables the use of MetalLB.
-        type: bool
-        default: ~
-
-      cilium_iface:
-        description: The network interface used for when Cilium is enabled
-        default: ~
-
-      extra_server_args:
-        description: Extra arguments for server nodes
-        default: ""
-
-      group_name_master:
-        description: Name of the master group
-        default: master
-
-      kube_vip_iface:
-        description:
-          - Explicitly define an interface that ALL control nodes
-          - should use to propagate the VIP, define it here.
-          - Otherwise, kube-vip will determine the right interface
-          - automatically at runtime.
-        default: ~
-
-      kube_vip_tag_version:
-        description: Image tag for kube-vip
-        default: v0.7.2
-
-      kube_vip_cloud_provider_tag_version:
-        description: Tag for kube-vip-cloud-provider manifest when enabled
-        default: main
-
-      kube_vip_lb_ip_range:
-        description: IP range for kube-vip load balancer
-        default: ~
-
-      metal_lb_controller_tag_version:
-        description: Image tag for MetalLB
-        default: v0.14.3
-
-      metal_lb_speaker_tag_version:
-        description: Image tag for MetalLB
-        default: v0.14.3
-
-      metal_lb_type:
-        choices:
-          - frr
-          - native
-        default: native
-
-      proxy_env:
-        type: dict
-        description: Internet proxy configurations
-        default: ~
-        options:
-          HTTP_PROXY:
-            required: true
-          HTTPS_PROXY:
-            required: true
-          NO_PROXY:
-            required: true
-
-      retry_count:
-        description: Amount of retries when verifying that nodes joined
-        type: int
-        default: 20
-
-      server_init_args:
-        description: Arguments for server nodes
-
-      systemd_dir:
-        description: Path to systemd services
-        default: /etc/systemd/system
-        required: true

roles/k3s_server/tasks/http_proxy.yml

@@ -1,4 +1,5 @@
 ---
+
 - name: Create k3s.service.d directory
   file:
     path: '{{ systemd_dir }}/k3s.service.d'

roles/k3s_server/tasks/main.yml

@@ -1,4 +1,5 @@
 ---
+
 - name: Stop k3s-init
   systemd:
     name: k3s-init
@@ -28,7 +29,7 @@
 - name: Deploy metallb manifest
   include_tasks: metallb.yml
   tags: metallb
-  when: kube_vip_lb_ip_range is not defined and (cilium_bgp is not defined or cilium_iface is not defined)
+  when: kube_vip_lb_ip_range is not defined and (not cilium_bgp or cilium_iface is not defined)
 
 - name: Deploy kube-vip manifest
   include_tasks: kube-vip.yml

roles/k3s_server/templates/vip.yaml.j2

@@ -27,7 +27,7 @@ spec:
         - manager
         env:
         - name: vip_arp
-          value: "true"
+          value: "{{ 'true' if kube_vip_arp | bool else 'false' }}"
         - name: port
           value: "6443"
 {% if kube_vip_iface %}

roles/k3s_server_post/defaults/main.yml

@@ -1,28 +1,6 @@
 ---
-bpf_lb_algorithm: maglev
+# Timeout to wait for MetalLB services to come up
-bpf_lb_mode: hybrid
-
-calico_blockSize: 26
-calico_ebpf: false
-calico_encapsulation: VXLANCrossSubnet
-calico_natOutgoing: Enabled
-calico_nodeSelector: all()
-calico_tag: v3.27.2
-
-cilium_bgp: false
-cilium_exportPodCIDR: true
-cilium_bgp_my_asn: 64513
-cilium_bgp_peer_asn: 64512
-cilium_bgp_lb_cidr: 192.168.31.0/24
-cilium_hubble: true
-cilium_mode: native
-
-cluster_cidr: 10.52.0.0/16
-enable_bpf_masquerade: true
-kube_proxy_replacement: true
-group_name_master: master
-
-metal_lb_mode: layer2
 metal_lb_available_timeout: 240s
-metal_lb_controller_tag_version: v0.14.3
+
-metal_lb_ip_range: 192.168.30.80-192.168.30.90
+# Name of the master group
+group_name_master: master

roles/k3s_server_post/meta/main.yml

@@ -1,145 +0,0 @@
----
-argument_specs:
-  main:
-    short_description: Configure k3s cluster
-    options:
-      apiserver_endpoint:
-        description: Virtual ip-address configured on each master
-        required: true
-
-      bpf_lb_algorithm:
-        description: BPF lb algorithm
-        default: maglev
-
-      bpf_lb_mode:
-        description: BPF lb mode
-        default: hybrid
-
-      calico_blockSize:
-        description: IP pool block size
-        type: int
-        default: 26
-
-      calico_ebpf:
-        description: Use eBPF dataplane instead of iptables
-        type: bool
-        default: false
-
-      calico_encapsulation:
-        description: IP pool encapsulation
-        default: VXLANCrossSubnet
-
-      calico_natOutgoing:
-        description: IP pool NAT outgoing
-        default: Enabled
-
-      calico_nodeSelector:
-        description: IP pool node selector
-        default: all()
-
-      calico_iface:
-        description: The network interface used for when Calico is enabled
-        default: ~
-
-      calico_tag:
-        description: Calico version tag
-        default: v3.27.2
-
-      cilium_bgp:
-        description:
-          - Enable cilium BGP control plane for LB services and pod cidrs.
-          - Disables the use of MetalLB.
-        type: bool
-        default: false
-
-      cilium_bgp_my_asn:
-        description: Local ASN for BGP peer
-        type: int
-        default: 64513
-
-      cilium_bgp_peer_asn:
-        description: BGP peer ASN
-        type: int
-        default: 64512
-
-      cilium_bgp_peer_address:
-        description: BGP peer address
-        default: ~
-
-      cilium_bgp_lb_cidr:
-        description: BGP load balancer IP range
-        default: 192.168.31.0/24
-
-      cilium_exportPodCIDR:
-        description: Export pod CIDR
-        type: bool
-        default: true
-
-      cilium_hubble:
-        description: Enable Cilium Hubble
-        type: bool
-        default: true
-
-      cilium_iface:
-        description: The network interface used for when Cilium is enabled
-        default: ~
-
-      cilium_mode:
-        description: Inner-node communication mode
-        default: native
-        choices:
-          - native
-          - routed
-
-      cluster_cidr:
-        description: Inner-cluster IP range
-        default: 10.52.0.0/16
-
-      enable_bpf_masquerade:
-        description: Use IP masquerading
-        type: bool
-        default: true
-
-      group_name_master:
-        description: Name of the master group
-        default: master
-
-      kube_proxy_replacement:
-        description: Replace the native kube-proxy with Cilium
-        type: bool
-        default: true
-
-      kube_vip_lb_ip_range:
-        description: IP range for kube-vip load balancer
-        default: ~
-
-      metal_lb_available_timeout:
-        description: Wait for MetalLB resources
-        default: 240s
-
-      metal_lb_ip_range:
-        description: MetalLB ip range for load balancer
-        default: 192.168.30.80-192.168.30.90
-
-      metal_lb_controller_tag_version:
-        description: Image tag for MetalLB
-        default: v0.14.3
-
-      metal_lb_mode:
-        description: Metallb mode
-        default: layer2
-        choices:
-          - bgp
-          - layer2
-
-      metal_lb_bgp_my_asn:
-        description: BGP ASN configurations
-        default: ~
-
-      metal_lb_bgp_peer_asn:
-        description: BGP peer ASN configurations
-        default: ~
-
-      metal_lb_bgp_peer_address:
-        description: BGP peer address
-        default: ~

roles/k3s_server_post/tasks/cilium.yml

@@ -172,17 +172,17 @@
         {% endif %}
         --helm-set k8sServiceHost="127.0.0.1"
         --helm-set k8sServicePort="6444"
-        --helm-set routingMode={{ cilium_mode }}
+        --helm-set routingMode={{ cilium_mode | default("native") }}
         --helm-set autoDirectNodeRoutes={{ "true" if cilium_mode == "native" else "false" }}
-        --helm-set kubeProxyReplacement={{ kube_proxy_replacement }}
+        --helm-set kubeProxyReplacement={{ kube_proxy_replacement | default("true") }}
-        --helm-set bpf.masquerade={{ enable_bpf_masquerade }}
+        --helm-set bpf.masquerade={{ enable_bpf_masquerade | default("true") }}
         --helm-set bgpControlPlane.enabled={{ cilium_bgp | default("false") }}
         --helm-set hubble.enabled={{ "true" if cilium_hubble else "false" }}
         --helm-set hubble.relay.enabled={{ "true" if cilium_hubble else "false" }}
         --helm-set hubble.ui.enabled={{ "true" if cilium_hubble else "false" }}
         {% if kube_proxy_replacement is not false %}
-        --helm-set bpf.loadBalancer.algorithm={{ bpf_lb_algorithm }}
+        --helm-set bpf.loadBalancer.algorithm={{ bpf_lb_algorithm | default("maglev") }}
-        --helm-set bpf.loadBalancer.mode={{ bpf_lb_mode }}
+        --helm-set bpf.loadBalancer.mode={{ bpf_lb_mode | default("hybrid") }}
         {% endif %}
       environment:
         KUBECONFIG: "{{ ansible_user_dir }}/.kube/config"

roles/k3s_server_post/tasks/main.yml

@@ -12,7 +12,7 @@
 - name: Deploy metallb pool
   include_tasks: metallb.yml
   tags: metallb
-  when: kube_vip_lb_ip_range is not defined and (cilium_bgp is not defined or cilium_iface is not defined)
+  when: kube_vip_lb_ip_range is not defined and (not cilium_bgp or cilium_iface is not defined)
 
 - name: Remove tmp directory used for manifests
   file:

roles/k3s_server_post/tasks/metallb.yml

@@ -83,9 +83,23 @@
   loop_control:
     label: "{{ item.description }}"
 
+- name: Set metallb webhook service name
+  set_fact:
+    metallb_webhook_service_name: >-
+      {{
+        (
+          (metal_lb_controller_tag_version | regex_replace('^v', ''))
+          is
+          version('0.14.4', '<', version_type='semver')
+        ) | ternary(
+          'webhook-service',
+          'metallb-webhook-service'
+        )
+      }}
+
 - name: Test metallb-system webhook-service endpoint
   command: >-
-    k3s kubectl -n metallb-system get endpoints webhook-service
+    k3s kubectl -n metallb-system get endpoints {{ metallb_webhook_service_name }}
   changed_when: false
   with_items: "{{ groups[group_name_master | default('master')] }}"
   run_once: true

roles/k3s_server_post/templates/calico.crs.j2

@@ -9,11 +9,11 @@ spec:
   calicoNetwork:
     # Note: The ipPools section cannot be modified post-install.
     ipPools:
-    - blockSize: {{ calico_blockSize }}
+    - blockSize: {{ calico_blockSize | default('26') }}
-      cidr: {{ cluster_cidr }}
+      cidr: {{ cluster_cidr | default('10.52.0.0/16') }}
-      encapsulation: {{ calico_encapsulation }}
+      encapsulation: {{ calico_encapsulation | default('VXLANCrossSubnet') }}
-      natOutgoing: {{ calico_natOutgoing }}
+      natOutgoing: {{ calico_natOutgoing | default('Enabled') }}
-      nodeSelector: {{ calico_nodeSelector }}
+      nodeSelector: {{ calico_nodeSelector | default('all()') }}
     nodeAddressAutodetectionV4:
       interface: {{ calico_iface  }}
     linuxDataplane: {{ 'BPF' if calico_ebpf else 'Iptables' }}

roles/lxc/meta/main.yml

@@ -1,7 +0,0 @@
----
-argument_specs:
-  main:
-    short_description: Configure LXC
-    options:
-      custom_reboot_command:
-        default: ~

roles/prereq/defaults/main.yml

@@ -1,4 +1,4 @@
 ---
 secure_path:
-  RedHat: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
+  RedHat: '/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin'
-  Suse: /usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin
+  Suse: '/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin'

roles/prereq/meta/main.yml

@@ -1,7 +0,0 @@
----
-argument_specs:
-  main:
-    short_description: Prerequisites
-    options:
-      system_timezone:
-        description: Timezone to be set on all nodes

roles/reset/defaults/main.yml

@@ -1,2 +0,0 @@
----
-systemd_dir: /etc/systemd/system

roles/reset/meta/main.yml

@@ -1,9 +0,0 @@
----
-argument_specs:
-  main:
-    short_description: Reset all nodes
-    options:
-      systemd_dir:
-        description: Path to systemd services
-        default: /etc/systemd/system
-        required: true