k3s_server: add kube_vip_arp parameter (#550)

With the kube_vip_arp parameter it is possible to set or unset the
vip_arp environment variable of the kube-vip-ds daemonset. The value of
the kube_vip_arp is true by default to not change the existing default.

Signed-off-by: Christian Berendt <berendt@osism.tech>
Co-authored-by: Techno Tim <timothystewart6@gmail.com>

.github/download-boxes.sh

@@ -9,12 +9,17 @@ set -euo pipefail
 GIT_ROOT=$(git rev-parse --show-toplevel)
 PROVIDER=virtualbox
 
-# Read all boxes for all platforms from the "molecule.yml" files
-all_boxes=$(cat "${GIT_ROOT}"/molecule/*/molecule.yml |
-    yq -r '.platforms[].box' |         # Read the "box" property of each node under "platforms"
-    grep --invert-match --regexp=--- | # Filter out file separators
-    sort |
-    uniq)
+yq --version
+
+# Define the path to the molecule.yml files
+MOLECULE_YML_PATH="${GIT_ROOT}/molecule/*/molecule.yml"
+
+# Extract and sort unique boxes from all molecule.yml files
+all_boxes=$(for file in $MOLECULE_YML_PATH; do
+    yq eval '.platforms[].box' "$file"
+done | sort -u)
+
+echo all_boxes: "$all_boxes"
 
 # Read the boxes that are currently present on the system (for the current provider)
 present_boxes=$(

.github/workflows/cache.yml

@@ -11,19 +11,19 @@ jobs:
 
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # 4.1.7
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # 5.0.0
+        uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # 5.1.1
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies
 
       - name: Cache Vagrant boxes
         id: cache-vagrant
-        uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # 4.0
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # 4.0.2
         with:
           lookup-only: true #if it exists, we don't need to restore and can skip the next step
           path: |

.github/workflows/lint.yml

@@ -11,18 +11,18 @@ jobs:
 
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # 4.1.7
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # 5.0.0
+        uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # 5.1.1
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies
 
       - name: Restore Ansible cache
-        uses: actions/cache/restore@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # 4.0
+        uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # 4.0.2
         with:
           path: ~/.ansible/collections
           key: ansible-${{ hashFiles('collections/requirements.yml') }}
@@ -38,16 +38,16 @@ jobs:
           echo "::endgroup::"
 
       - name: Run pre-commit
-        uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507 # 3.0.0
+        uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # 3.0.1
 
   ensure-pinned-actions:
     name: Ensure SHA Pinned Actions
     runs-on: self-hosted
     steps:
       - name: Checkout code
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # 4.1.7
       - name: Ensure SHA pinned actions
-        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@ba37328d4ea95eaf8b3bd6c6cef308f709a5f2ec # 3.0.3
+        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@b88cd0aad2c36a63e42c71f81cb1958fed95ac87 # 3.0.10
         with:
           allowlist: |
             aws-actions/

.github/workflows/test.yml

@@ -10,7 +10,7 @@ jobs:
       matrix:
         scenario:
           - default
-          - ipv6
+          # - ipv6
           - single_node
           - calico
           - cilium
@@ -21,7 +21,7 @@ jobs:
 
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # 4.1.7
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
@@ -59,13 +59,13 @@ jobs:
           EOF
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # 5.0.0
+        uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # 5.1.1
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies
 
       - name: Restore vagrant Boxes cache
-        uses: actions/cache/restore@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # 4.0
+        uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # 4.0.2
         with:
           path: ~/.vagrant.d/boxes
           key: vagrant-boxes-${{ hashFiles('**/molecule.yml') }}
@@ -118,7 +118,7 @@ jobs:
 
       - name: Upload log files
         if: always() # do this even if a step before has failed
-        uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # 4.3.0
+        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # 4.3.4
         with:
           name: logs
           path: |

README.md

@@ -96,8 +96,22 @@ ansible-playbook reset.yml -i inventory/my-cluster/hosts.ini
 To copy your `kube config` locally so that you can access your **Kubernetes** cluster run:
 
 ```bash
-scp debian@master_ip:~/.kube/config ~/.kube/config
+scp debian@master_ip:/etc/rancher/k3s/k3s.yaml ~/.kube/config
 ```
+If you get file Permission denied, go into the node and temporarly run:
+```bash
+sudo chmod 777 /etc/rancher/k3s/k3s.yaml
+```
+Then copy with the scp command and reset the permissions back to:
+```bash
+sudo chmod 600 /etc/rancher/k3s/k3s.yaml
+```
+
+You'll then want to modify the config to point to master IP by running:
+```bash
+sudo nano ~/.kube/config
+```
+Then change `server: https://127.0.0.1:6443` to match your master IP: `server: https://192.168.1.222:6443`
 
 ### 🔨 Testing your cluster
 

inventory/sample/group_vars/all.yml

@@ -1,5 +1,5 @@
 ---
-k3s_version: v1.29.0+k3s1
+k3s_version: v1.30.2+k3s2
 # this is the user that has ssh access to these machines
 ansible_user: ansibleuser
 systemd_dir: /etc/systemd/system
@@ -13,13 +13,13 @@ flannel_iface: "eth0"
 # uncomment calico_iface to use tigera operator/calico cni instead of flannel https://docs.tigera.io/calico/latest/about
 # calico_iface: "eth0"
 calico_ebpf: false           # use eBPF dataplane instead of iptables
-calico_tag: "v3.27.0"        # calico version tag
+calico_tag: "v3.28.0"        # calico version tag
 
 # uncomment cilium_iface to use cilium cni instead of flannel or calico
 # ensure v4.19.57, v5.1.16, v5.2.0 or more recent kernel
 # cilium_iface: "eth0"
 cilium_mode: "native"        # native when nodes on same subnet or using bgp, else set routed
-cilium_tag: "v1.14.6"        # cilium version tag
+cilium_tag: "v1.16.0"        # cilium version tag
 cilium_hubble: true          # enable hubble observability relay and ui
 
 # if using calico or cilium, you may specify the cluster pod cidr pool
@@ -72,7 +72,7 @@ extra_agent_args: >-
   {{ extra_args }}
 
 # image tag for kube-vip
-kube_vip_tag_version: "v0.6.4"
+kube_vip_tag_version: "v0.8.2"
 
 # tag for kube-vip-cloud-provider manifest
 # kube_vip_cloud_provider_tag_version: "main"
@@ -93,8 +93,8 @@ metal_lb_mode: "layer2"
 # metal_lb_bgp_peer_address: "192.168.30.1"
 
 # image tag for metal lb
-metal_lb_speaker_tag_version: "v0.13.12"
-metal_lb_controller_tag_version: "v0.13.12"
+metal_lb_speaker_tag_version: "v0.14.8"
+metal_lb_controller_tag_version: "v0.14.8"
 
 # metallb ip range for load balancer
 metal_lb_ip_range: "192.168.30.80-192.168.30.90"
@@ -160,6 +160,10 @@ custom_registries_yaml: |
         username: yourusername
         password: yourpassword
 
+# On some distros like Diet Pi, there is no dbus installed. dbus required by the default reboot command.
+# Uncomment if you need a custom reboot command
+# custom_reboot_command: /usr/sbin/shutdown -r now
+
 # Only enable and configure these if you access the internet through a proxy
 # proxy_env:
 #   HTTP_PROXY: "http://proxy.domain.local:3128"

reboot.yml

@@ -6,4 +6,5 @@
     - name: Reboot the nodes (and Wait upto 5 mins max)
       become: true
       reboot:
+        reboot_command: "{{ custom_reboot_command | default(omit) }}"
         reboot_timeout: 300

requirements.txt

@@ -6,7 +6,7 @@
 #
 ansible-compat==4.1.11
     # via molecule
-ansible-core==2.16.3
+ansible-core==2.17.2
     # via
     #   -r requirements.in
     #   ansible-compat
@@ -77,7 +77,7 @@ molecule==6.0.3
     # via
     #   -r requirements.in
     #   molecule-plugins
-molecule-plugins[vagrant]==23.5.0
+molecule-plugins[vagrant]==23.5.3
     # via -r requirements.in
 netaddr==0.10.1
     # via -r requirements.in
@@ -96,9 +96,9 @@ platformdirs==4.1.0
     # via virtualenv
 pluggy==1.3.0
     # via molecule
-pre-commit==3.6.0
+pre-commit==3.8.0
     # via -r requirements.in
-pre-commit-hooks==4.5.0
+pre-commit-hooks==4.6.0
     # via -r requirements.in
 pyasn1==0.5.1
     # via

reset.yml

@@ -12,6 +12,7 @@
     - name: Reboot and wait for node to come back up
       become: true
       reboot:
+        reboot_command: "{{ custom_reboot_command | default(omit) }}"
         reboot_timeout: 3600
 
 - name: Revert changes to Proxmox cluster

roles/k3s_agent/tasks/http_proxy.yml

@@ -1,5 +1,4 @@
 ---
-
 - name: Create k3s-node.service.d directory
   file:
     path: '{{ systemd_dir }}/k3s-node.service.d'
@@ -7,7 +6,7 @@
     owner: root
     group: root
     mode: '0755'
-
+  when: proxy_env is defined
 
 - name: Copy K3s http_proxy conf file
   template:
@@ -16,3 +15,4 @@
     owner: root
     group: root
     mode: '0755'
+  when: proxy_env is defined

roles/k3s_agent/tasks/main.yml

@@ -1,19 +1,35 @@
 ---
+- name: Check for PXE-booted system
+  block:
+    - name: Check if system is PXE-booted
+      ansible.builtin.command:
+        cmd: cat /proc/cmdline
+      register: boot_cmdline
+      changed_when: false
+      check_mode: false
+
+    - name: Set fact for PXE-booted system
+      ansible.builtin.set_fact:
+        is_pxe_booted: "{{ 'root=/dev/nfs' in boot_cmdline.stdout }}"
+      when: boot_cmdline.stdout is defined
+
+    - name: Include http_proxy configuration tasks
+      ansible.builtin.include_tasks: http_proxy.yml
 
 - name: Deploy K3s http_proxy conf
   include_tasks: http_proxy.yml
   when: proxy_env is defined
 
-- name: Copy K3s service file
-  template:
+- name: Configure the k3s service
+  ansible.builtin.template:
     src: "k3s.service.j2"
     dest: "{{ systemd_dir }}/k3s-node.service"
     owner: root
     group: root
-    mode: 0755
+    mode: '0755'
 
-- name: Enable and check K3s service
-  systemd:
+- name: Manage k3s service
+  ansible.builtin.systemd:
     name: k3s-node
     daemon_reload: true
     state: restarted

roles/k3s_agent/templates/k3s.service.j2

@@ -7,11 +7,14 @@ After=network-online.target
 Type=notify
 ExecStartPre=-/sbin/modprobe br_netfilter
 ExecStartPre=-/sbin/modprobe overlay
-ExecStart=/usr/local/bin/k3s agent --server https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443 --token {{ hostvars[groups[group_name_master | default('master')][0]]['token'] | default(k3s_token) }} {{ extra_agent_args | default("") }}
+# Conditional snapshotter based on PXE boot status
+ExecStart=/usr/local/bin/k3s agent \
+  --server https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443 \
+  {% if is_pxe_booted | default(false) %}--snapshotter native \
+  {% endif %}--token {{ hostvars[groups[group_name_master | default('master')][0]]['token'] | default(k3s_token) }} \
+  {{ extra_agent_args | default("") }}
 KillMode=process
 Delegate=yes
-# Having non-zero Limit*s causes performance problems due to accounting overhead
-# in the kernel. We recommend using cgroups to do container-local accounting.
 LimitNOFILE=1048576
 LimitNPROC=infinity
 LimitCORE=infinity

roles/k3s_server/defaults/main.yml

@@ -4,6 +4,9 @@
 # will determine the right interface automatically at runtime.
 kube_vip_iface: null
 
+# Enables ARP broadcasts from Leader
+kube_vip_arp: true
+
 # Name of the master group
 group_name_master: master
 

roles/k3s_server/tasks/vip.yml

@@ -10,7 +10,7 @@
 
 - name: Download vip rbac manifest to first master
   ansible.builtin.get_url:
-    url: "https://raw.githubusercontent.com/kube-vip/kube-vip/{{ kube_vip_tag_version }}/docs/manifests/rbac.yaml"
+    url: "https://kube-vip.io/manifests/rbac.yaml"
     dest: "/var/lib/rancher/k3s/server/manifests/vip-rbac.yaml"
     owner: root
     group: root

roles/k3s_server/templates/vip.yaml.j2

@@ -27,7 +27,7 @@ spec:
         - manager
         env:
         - name: vip_arp
-          value: "true"
+          value: "{{ 'true' if kube_vip_arp | bool else 'false' }}"
         - name: port
           value: "6443"
 {% if kube_vip_iface %}

roles/k3s_server_post/tasks/calico.yml

@@ -48,7 +48,7 @@
         k3s kubectl wait {{ item.type }}/{{ item.name }}
         --namespace='tigera-operator'
         --for=condition=Available=True
-        --timeout=7s
+        --timeout=30s
       register: tigera_result
       changed_when: false
       until: tigera_result is succeeded
@@ -87,7 +87,7 @@
         --namespace='{{ item.namespace }}'
         --for=condition=Available
         {% endif %}
-        --timeout=7s
+        --timeout=30s
       register: cr_result
       changed_when: false
       until: cr_result is succeeded

roles/k3s_server_post/tasks/cilium.yml

@@ -170,8 +170,8 @@
         {% if cilium_mode == "native" or (cilium_bgp and cilium_exportPodCIDR != 'false') %}
         --helm-set ipv4NativeRoutingCIDR={{ cluster_cidr }}
         {% endif %}
-        --helm-set k8sServiceHost={{ apiserver_endpoint }}
-        --helm-set k8sServicePort="6443"
+        --helm-set k8sServiceHost="127.0.0.1"
+        --helm-set k8sServicePort="6444"
         --helm-set routingMode={{ cilium_mode | default("native") }}
         --helm-set autoDirectNodeRoutes={{ "true" if cilium_mode == "native" else "false" }}
         --helm-set kubeProxyReplacement={{ kube_proxy_replacement | default("true") }}
@@ -185,7 +185,7 @@
         --helm-set bpf.loadBalancer.mode={{ bpf_lb_mode | default("hybrid") }}
         {% endif %}
       environment:
-        KUBECONFIG: /home/{{ ansible_user }}/.kube/config
+        KUBECONFIG: "{{ ansible_user_dir }}/.kube/config"
       register: cilium_install_result
       changed_when: cilium_install_result.rc == 0
       when: cilium_installed.rc != 0 or cilium_needs_update
@@ -202,7 +202,7 @@
         --namespace=kube-system
         --for=condition=Available
         {% endif %}
-        --timeout=7s
+        --timeout=30s
       register: cr_result
       changed_when: false
       until: cr_result is succeeded

roles/k3s_server_post/tasks/metallb.yml

@@ -83,9 +83,23 @@
   loop_control:
     label: "{{ item.description }}"
 
+- name: Set metallb webhook service name
+  set_fact:
+    metallb_webhook_service_name: >-
+      {{
+        (
+          (metal_lb_controller_tag_version | regex_replace('^v', ''))
+          is
+          version('0.14.4', '<', version_type='semver')
+        ) | ternary(
+          'webhook-service',
+          'metallb-webhook-service'
+        )
+      }}
+
 - name: Test metallb-system webhook-service endpoint
   command: >-
-    k3s kubectl -n metallb-system get endpoints webhook-service
+    k3s kubectl -n metallb-system get endpoints {{ metallb_webhook_service_name }}
   changed_when: false
   with_items: "{{ groups[group_name_master | default('master')] }}"
   run_once: true

roles/k3s_server_post/templates/cilium.crs.j2

@@ -25,5 +25,10 @@ kind: CiliumLoadBalancerIPPool
 metadata:
   name: "01-lb-pool"
 spec:
-  cidrs:
-  - cidr: "{{ cilium_bgp_lb_cidr }}"
+  blocks:
+{% if "/" in cilium_bgp_lb_cidr %}
+  - cidr: {{ cilium_bgp_lb_cidr }}
+{% else %}
+  - start: {{ cilium_bgp_lb_cidr.split('-')[0] }}
+    stop: {{ cilium_bgp_lb_cidr.split('-')[1] }}
+{% endif %}

roles/lxc/handlers/main.yml

@@ -2,4 +2,5 @@
 - name: Reboot server
   become: true
   reboot:
+    reboot_command: "{{ custom_reboot_command | default(omit) }}"
   listen: reboot server

roles/raspberrypi/handlers/main.yml

@@ -1,4 +1,5 @@
 ---
 - name: Reboot
   reboot:
+    reboot_command: "{{ custom_reboot_command | default(omit) }}"
   listen: reboot

roles/raspberrypi/tasks/setup/Raspbian.yml

@@ -1,7 +1,27 @@
 ---
+- name: Test for cmdline path
+  stat:
+    path: /boot/firmware/cmdline.txt
+  register: boot_cmdline_path
+  failed_when: false
+  changed_when: false
+
+- name: Set cmdline path based on Debian version and command result
+  set_fact:
+    cmdline_path: >-
+      {{
+        (
+          boot_cmdline_path.stat.exists and
+          ansible_facts.lsb.description | default('') is match('Debian.*(?!(bookworm|sid))')
+        ) | ternary(
+          '/boot/firmware/cmdline.txt',
+          '/boot/cmdline.txt'
+        )
+      }}
+
 - name: Activating cgroup support
   lineinfile:
-    path: /boot/cmdline.txt
+    path: "{{ cmdline_path }}"
     regexp: '^((?!.*\bcgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory\b).*)$'
     line: '\1 cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory'
     backrefs: true

site.yml

@@ -1,4 +1,13 @@
 ---
+- name: Pre tasks
+  hosts: all
+  pre_tasks:
+    - name: Verify Ansible is version 2.11 or above. (If this fails you may need to update Ansible)
+      assert:
+        that: "ansible_version.full is version_compare('2.11', '>=')"
+        msg: >
+          "Ansible is out of date. See here for more info: https://docs.technotim.live/posts/ansible-automation/"
+
 - name: Prepare Proxmox cluster
   hosts: proxmox
   gather_facts: true