k3s_server: add kube_vip_arp parameter (#550)

With the kube_vip_arp parameter it is possible to set or unset the
vip_arp environment variable of the kube-vip-ds daemonset. The value of
the kube_vip_arp is true by default to not change the existing default.

Signed-off-by: Christian Berendt <berendt@osism.tech>
Co-authored-by: Techno Tim <timothystewart6@gmail.com>

.ansible-lint

@@ -18,3 +18,4 @@ exclude_paths:
 
 skip_list:
   - 'fqcn-builtins'
+  - var-naming[no-role-prefix]

.github/ISSUE_TEMPLATE.md

@@ -37,6 +37,11 @@ systemd_dir: ""
 
 flannel_iface: ""
 
+#calico_iface: ""
+calico_ebpf: ""
+calico_cidr: ""
+calico_tag: ""
+
 apiserver_endpoint: ""
 
 k3s_token: "NA"
@@ -46,6 +51,9 @@ extra_agent_args: ""
 
 kube_vip_tag_version: ""
 
+kube_vip_cloud_provider_tag_version: ""
+kube_vip_lb_ip_range: ""
+
 metal_lb_speaker_tag_version: ""
 metal_lb_controller_tag_version: ""
 

.github/download-boxes.sh

@@ -9,12 +9,17 @@ set -euo pipefail
 GIT_ROOT=$(git rev-parse --show-toplevel)
 PROVIDER=virtualbox
 
-# Read all boxes for all platforms from the "molecule.yml" files
-all_boxes=$(cat "${GIT_ROOT}"/molecule/*/molecule.yml |
-    yq -r '.platforms[].box' |         # Read the "box" property of each node under "platforms"
-    grep --invert-match --regexp=--- | # Filter out file separators
-    sort |
-    uniq)
+yq --version
+
+# Define the path to the molecule.yml files
+MOLECULE_YML_PATH="${GIT_ROOT}/molecule/*/molecule.yml"
+
+# Extract and sort unique boxes from all molecule.yml files
+all_boxes=$(for file in $MOLECULE_YML_PATH; do
+    yq eval '.platforms[].box' "$file"
+done | sort -u)
+
+echo all_boxes: "$all_boxes"
 
 # Read the boxes that are currently present on the system (for the current provider)
 present_boxes=$(

.github/workflows/cache.yml

@@ -11,19 +11,19 @@ jobs:
 
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v3 2.5.0
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # 4.1.7
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # 2.3.3
+        uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # 5.1.1
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies
 
       - name: Cache Vagrant boxes
         id: cache-vagrant
-        uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # 4.0
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # 4.0.2
         with:
           lookup-only: true #if it exists, we don't need to restore and can skip the next step
           path: |

.github/workflows/ci.yml

@@ -2,11 +2,20 @@
 name: "CI"
 on:
   pull_request:
-  push:
-    branches:
-      - master
+    types:
+      - opened
+      - synchronize
     paths-ignore:
-      - '**/README.md'
+      - '**/.gitignore'
+      - '**/FUNDING.yml'
+      - '**/host.ini'
+      - '**/*.md'
+      - '**/.editorconfig'
+      - '**/ansible.example.cfg'
+      - '**/deploy.sh'
+      - '**/LICENSE'
+      - '**/reboot.sh'
+      - '**/reset.sh'
 jobs:
   pre:
     uses: ./.github/workflows/cache.yml

.github/workflows/lint.yml

@@ -11,18 +11,18 @@ jobs:
 
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v3 2.5.0
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # 4.1.7
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # 2.3.3
+        uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # 5.1.1
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies
 
       - name: Restore Ansible cache
-        uses: actions/cache/restore@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # 4.0
+        uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # 4.0.2
         with:
           path: ~/.ansible/collections
           key: ansible-${{ hashFiles('collections/requirements.yml') }}
@@ -38,16 +38,16 @@ jobs:
           echo "::endgroup::"
 
       - name: Run pre-commit
-        uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507 # 3.0.0
+        uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # 3.0.1
 
   ensure-pinned-actions:
     name: Ensure SHA Pinned Actions
     runs-on: self-hosted
     steps:
       - name: Checkout code
-        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v3 2.5.0
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # 4.1.7
       - name: Ensure SHA pinned actions
-        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@af2eb3226618e2494e3d9084f515ad6dcf16e229 # 2.0.1
+        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@b88cd0aad2c36a63e42c71f81cb1958fed95ac87 # 3.0.10
         with:
           allowlist: |
             aws-actions/

.github/workflows/test.yml

@@ -10,18 +10,46 @@ jobs:
       matrix:
         scenario:
           - default
-          - ipv6
+          # - ipv6
           - single_node
+          - calico
+          - cilium
+          - kube-vip
       fail-fast: false
     env:
       PYTHON_VERSION: "3.11"
 
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v3 2.5.0
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # 4.1.7
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
+      # these steps are necessary if not using ephemeral nodes
+      - name: Delete old Vagrant box versions
+        if: always() # do this even if a step before has failed
+        run: vagrant box prune --force
+
+      - name: Remove all local Vagrant boxes
+        if: always() # do this even if a step before has failed
+        run: if vagrant box list 2>/dev/null; then vagrant box list | cut -f 1 -d ' ' | xargs -L 1 vagrant box remove -f 2>/dev/null && echo "All Vagrant boxes removed." || echo "No Vagrant boxes found."; else echo "No Vagrant boxes found."; fi
+
+      - name: Remove all Virtualbox VMs
+        if: always() # do this even if a step before has failed
+        run: VBoxManage list vms | awk -F'"' '{print $2}' | xargs -I {} VBoxManage unregistervm --delete "{}"
+
+      - name: Remove all Virtualbox HDs
+        if: always() # do this even if a step before has failed
+        run: VBoxManage list hdds | awk -F':' '/^UUID:/ {print $2}' | xargs -I {} VBoxManage closemedium disk "{}" --delete
+
+      - name: Remove all Virtualbox Networks
+        if: always() # do this even if a step before has failed
+        run: VBoxManage list hostonlyifs | grep '^Name:' | awk '{print $2}' | grep '^vboxnet' | xargs -I {} VBoxManage hostonlyif remove {}
+
+      - name: Remove Virtualbox network config
+        if: always() # do this even if a step before has failed
+        run: sudo rm /etc/vbox/networks.conf || true
+
       - name: Configure VirtualBox
         run: |-
           sudo mkdir -p /etc/vbox
@@ -31,13 +59,13 @@ jobs:
           EOF
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # 2.3.3
+        uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # 5.1.1
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies
 
       - name: Restore vagrant Boxes cache
-        uses: actions/cache/restore@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # 4.0
+        uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # 4.0.2
         with:
           path: ~/.vagrant.d/boxes
           key: vagrant-boxes-${{ hashFiles('**/molecule.yml') }}
@@ -70,7 +98,7 @@ jobs:
 
       - name: Remove all local Vagrant boxes
         if: always() # do this even if a step before has failed
-        run: vagrant box list | cut -f 1 -d ' ' | xargs -L 1 vagrant box remove -f
+        run: if vagrant box list 2>/dev/null; then vagrant box list | cut -f 1 -d ' ' | xargs -L 1 vagrant box remove -f 2>/dev/null && echo "All Vagrant boxes removed." || echo "No Vagrant boxes found."; else echo "No Vagrant boxes found."; fi
 
       - name: Remove all Virtualbox VMs
         if: always() # do this even if a step before has failed
@@ -86,12 +114,13 @@ jobs:
 
       - name: Remove Virtualbox network config
         if: always() # do this even if a step before has failed
-        run: sudo rm /etc/vbox/networks.conf
+        run: sudo rm /etc/vbox/networks.conf || true
 
       - name: Upload log files
         if: always() # do this even if a step before has failed
-        uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # 3.1.1
+        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # 4.3.4
         with:
           name: logs
           path: |
             ${{ runner.temp }}/logs
+          overwrite: true

.gitignore

@@ -1,3 +1,4 @@
 .env/
 *.log
 ansible.cfg
+kubeconfig

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: f71fa2c1f9cf5cb705f73dffe4b21f7c61470ba9  # frozen: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: requirements-txt-fixer
       - id: sort-simple-yaml
@@ -12,24 +12,24 @@ repos:
       - id: trailing-whitespace
         args: [--markdown-linebreak-ext=md]
   - repo: https://github.com/adrienverge/yamllint.git
-    rev: b05e028c5881819161d11cb543fd96a30c06cceb  # frozen: v1.32.0
+    rev: v1.33.0
     hooks:
       - id: yamllint
         args: [-c=.yamllint]
   - repo: https://github.com/ansible-community/ansible-lint.git
-    rev: 3293b64b939c0de16ef8cb81dd49255e475bf89a  # frozen: v6.17.2
+    rev: v6.22.2
     hooks:
       - id: ansible-lint
   - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: 375289a39f5708101b1f916eb729e8d6da96993f  # frozen: v0.9.0.5
+    rev: v0.9.0.6
     hooks:
       - id: shellcheck
   - repo: https://github.com/Lucas-C/pre-commit-hooks
-    rev: 12885e376b93dc4536ad68d156065601e4433665  # frozen: v1.5.1
+    rev: v1.5.4
     hooks:
       - id: remove-crlf
       - id: remove-tabs
   - repo: https://github.com/sirosen/texthooks
-    rev: c4ffd3e31669dd4fa4d31a23436cc13839730084  # frozen: 0.5.0
+    rev: 0.6.4
     hooks:
       - id: fix-smartquotes

README.md

@@ -96,8 +96,22 @@ ansible-playbook reset.yml -i inventory/my-cluster/hosts.ini
 To copy your `kube config` locally so that you can access your **Kubernetes** cluster run:
 
 ```bash
-scp debian@master_ip:~/.kube/config ~/.kube/config
+scp debian@master_ip:/etc/rancher/k3s/k3s.yaml ~/.kube/config
 ```
+If you get file Permission denied, go into the node and temporarly run:
+```bash
+sudo chmod 777 /etc/rancher/k3s/k3s.yaml
+```
+Then copy with the scp command and reset the permissions back to:
+```bash
+sudo chmod 600 /etc/rancher/k3s/k3s.yaml
+```
+
+You'll then want to modify the config to point to master IP by running:
+```bash
+sudo nano ~/.kube/config
+```
+Then change `server: https://127.0.0.1:6443` to match your master IP: `server: https://192.168.1.222:6443`
 
 ### 🔨 Testing your cluster
 

inventory/sample/group_vars/all.yml

@@ -1,5 +1,5 @@
 ---
-k3s_version: v1.28.5+k3s1
+k3s_version: v1.30.2+k3s2
 # this is the user that has ssh access to these machines
 ansible_user: ansibleuser
 systemd_dir: /etc/systemd/system
@@ -10,6 +10,30 @@ system_timezone: "Your/Timezone"
 # interface which will be used for flannel
 flannel_iface: "eth0"
 
+# uncomment calico_iface to use tigera operator/calico cni instead of flannel https://docs.tigera.io/calico/latest/about
+# calico_iface: "eth0"
+calico_ebpf: false           # use eBPF dataplane instead of iptables
+calico_tag: "v3.28.0"        # calico version tag
+
+# uncomment cilium_iface to use cilium cni instead of flannel or calico
+# ensure v4.19.57, v5.1.16, v5.2.0 or more recent kernel
+# cilium_iface: "eth0"
+cilium_mode: "native"        # native when nodes on same subnet or using bgp, else set routed
+cilium_tag: "v1.16.0"        # cilium version tag
+cilium_hubble: true          # enable hubble observability relay and ui
+
+# if using calico or cilium, you may specify the cluster pod cidr pool
+cluster_cidr: "10.52.0.0/16"
+
+# enable cilium bgp control plane for lb services and pod cidrs. disables metallb.
+cilium_bgp: false
+
+# bgp parameters for cilium cni. only active when cilium_iface is defined and cilium_bgp is true.
+cilium_bgp_my_asn: "64513"
+cilium_bgp_peer_asn: "64512"
+cilium_bgp_peer_address: "192.168.30.1"
+cilium_bgp_lb_cidr: "192.168.31.0/24"   # cidr for cilium loadbalancer ipam
+
 # apiserver_endpoint is virtual ip-address which will be configured on each master
 apiserver_endpoint: "192.168.30.222"
 
@@ -20,28 +44,42 @@ k3s_token: "some-SUPER-DEDEUPER-secret-password"
 # The IP on which the node is reachable in the cluster.
 # Here, a sensible default is provided, you can still override
 # it for each of your hosts, though.
-k3s_node_ip: '{{ ansible_facts[flannel_iface]["ipv4"]["address"] }}'
+k3s_node_ip: "{{ ansible_facts[(cilium_iface | default(calico_iface | default(flannel_iface)))]['ipv4']['address'] }}"
 
 # Disable the taint manually by setting: k3s_master_taint = false
 k3s_master_taint: "{{ true if groups['node'] | default([]) | length >= 1 else false }}"
 
 # these arguments are recommended for servers as well as agents:
 extra_args: >-
-  --flannel-iface={{ flannel_iface }}
+  {{ '--flannel-iface=' + flannel_iface if calico_iface is not defined and cilium_iface is not defined else '' }}
   --node-ip={{ k3s_node_ip }}
 
 # change these to your liking, the only required are: --disable servicelb, --tls-san {{ apiserver_endpoint }}
+# the contents of the if block is also required if using calico or cilium
 extra_server_args: >-
   {{ extra_args }}
   {{ '--node-taint node-role.kubernetes.io/master=true:NoSchedule' if k3s_master_taint else '' }}
+  {% if calico_iface is defined or cilium_iface is defined %}
+  --flannel-backend=none
+  --disable-network-policy
+  --cluster-cidr={{ cluster_cidr | default('10.52.0.0/16') }}
+  {% endif %}
   --tls-san {{ apiserver_endpoint }}
   --disable servicelb
   --disable traefik
+
 extra_agent_args: >-
   {{ extra_args }}
 
 # image tag for kube-vip
-kube_vip_tag_version: "v0.5.12"
+kube_vip_tag_version: "v0.8.2"
+
+# tag for kube-vip-cloud-provider manifest
+# kube_vip_cloud_provider_tag_version: "main"
+
+# kube-vip ip range for load balancer
+# (uncomment to use kube-vip for services instead of MetalLB)
+# kube_vip_lb_ip_range: "192.168.30.80-192.168.30.90"
 
 # metallb type frr or native
 metal_lb_type: "native"
@@ -55,8 +93,8 @@ metal_lb_mode: "layer2"
 # metal_lb_bgp_peer_address: "192.168.30.1"
 
 # image tag for metal lb
-metal_lb_speaker_tag_version: "v0.13.9"
-metal_lb_controller_tag_version: "v0.13.9"
+metal_lb_speaker_tag_version: "v0.14.8"
+metal_lb_controller_tag_version: "v0.14.8"
 
 # metallb ip range for load balancer
 metal_lb_ip_range: "192.168.30.80-192.168.30.90"
@@ -122,6 +160,10 @@ custom_registries_yaml: |
         username: yourusername
         password: yourpassword
 
+# On some distros like Diet Pi, there is no dbus installed. dbus required by the default reboot command.
+# Uncomment if you need a custom reboot command
+# custom_reboot_command: /usr/sbin/shutdown -r now
+
 # Only enable and configure these if you access the internet through a proxy
 # proxy_env:
 #   HTTP_PROXY: "http://proxy.domain.local:3128"

molecule/README.md

@@ -13,6 +13,12 @@ We have these scenarios:
   To save a bit of test time, this cluster is _not_ highly available, it consists of only one control and one worker node.
 - **single_node**:
   Very similar to the default scenario, but uses only a single node for all cluster functionality.
+- **calico**:
+  The same as single node, but uses calico cni instead of flannel.
+- **cilium**:
+  The same as single node, but uses cilium cni instead of flannel.
+- **kube-vip**
+  The same as single node, but uses kube-vip as service loadbalancer instead of MetalLB
 
 ## How to execute
 

molecule/calico/molecule.yml

@@ -0,0 +1,49 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: vagrant
+platforms:
+  - name: control1
+    box: generic/ubuntu2204
+    memory: 4096
+    cpus: 4
+    config_options:
+      # We currently can not use public-key based authentication on Ubuntu 22.04,
+      # see: https://github.com/chef/bento/issues/1405
+      ssh.username: "vagrant"
+      ssh.password: "vagrant"
+    groups:
+      - k3s_cluster
+      - master
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.62
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_VERBOSITY: 1
+  playbooks:
+    converge: ../resources/converge.yml
+    side_effect: ../resources/reset.yml
+    verify: ../resources/verify.yml
+  inventory:
+    links:
+      group_vars: ../../inventory/sample/group_vars
+scenario:
+  test_sequence:
+    - dependency
+    - cleanup
+    - destroy
+    - syntax
+    - create
+    - prepare
+    - converge
+    # idempotence is not possible with the playbook in its current form.
+    - verify
+    # We are repurposing side_effect here to test the reset playbook.
+    # This is why we do not run it before verify (which tests the cluster),
+    # but after the verify step.
+    - side_effect
+    - cleanup
+    - destroy

molecule/calico/overrides.yml

@@ -0,0 +1,16 @@
+---
+- name: Apply overrides
+  hosts: all
+  tasks:
+    - name: Override host variables
+      ansible.builtin.set_fact:
+        # See:
+        # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
+        calico_iface: eth1
+
+        # The test VMs might be a bit slow, so we give them more time to join the cluster:
+        retry_count: 45
+
+        # Make sure that our IP ranges do not collide with those of the other scenarios
+        apiserver_endpoint: "192.168.30.224"
+        metal_lb_ip_range: "192.168.30.100-192.168.30.109"

molecule/cilium/molecule.yml

@@ -0,0 +1,49 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: vagrant
+platforms:
+  - name: control1
+    box: generic/ubuntu2204
+    memory: 4096
+    cpus: 4
+    config_options:
+      # We currently can not use public-key based authentication on Ubuntu 22.04,
+      # see: https://github.com/chef/bento/issues/1405
+      ssh.username: "vagrant"
+      ssh.password: "vagrant"
+    groups:
+      - k3s_cluster
+      - master
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.63
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_VERBOSITY: 1
+  playbooks:
+    converge: ../resources/converge.yml
+    side_effect: ../resources/reset.yml
+    verify: ../resources/verify.yml
+  inventory:
+    links:
+      group_vars: ../../inventory/sample/group_vars
+scenario:
+  test_sequence:
+    - dependency
+    - cleanup
+    - destroy
+    - syntax
+    - create
+    - prepare
+    - converge
+    # idempotence is not possible with the playbook in its current form.
+    - verify
+    # We are repurposing side_effect here to test the reset playbook.
+    # This is why we do not run it before verify (which tests the cluster),
+    # but after the verify step.
+    - side_effect
+    - cleanup
+    - destroy

molecule/cilium/overrides.yml

@@ -0,0 +1,16 @@
+---
+- name: Apply overrides
+  hosts: all
+  tasks:
+    - name: Override host variables
+      ansible.builtin.set_fact:
+        # See:
+        # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
+        cilium_iface: eth1
+
+        # The test VMs might be a bit slow, so we give them more time to join the cluster:
+        retry_count: 45
+
+        # Make sure that our IP ranges do not collide with those of the other scenarios
+        apiserver_endpoint: "192.168.30.225"
+        metal_lb_ip_range: "192.168.30.110-192.168.30.119"

molecule/kube-vip/molecule.yml

@@ -0,0 +1,49 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: vagrant
+platforms:
+  - name: control1
+    box: generic/ubuntu2204
+    memory: 4096
+    cpus: 4
+    config_options:
+      # We currently can not use public-key based authentication on Ubuntu 22.04,
+      # see: https://github.com/chef/bento/issues/1405
+      ssh.username: "vagrant"
+      ssh.password: "vagrant"
+    groups:
+      - k3s_cluster
+      - master
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.62
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_VERBOSITY: 1
+  playbooks:
+    converge: ../resources/converge.yml
+    side_effect: ../resources/reset.yml
+    verify: ../resources/verify.yml
+  inventory:
+    links:
+      group_vars: ../../inventory/sample/group_vars
+scenario:
+  test_sequence:
+    - dependency
+    - cleanup
+    - destroy
+    - syntax
+    - create
+    - prepare
+    - converge
+    # idempotence is not possible with the playbook in its current form.
+    - verify
+    # We are repurposing side_effect here to test the reset playbook.
+    # This is why we do not run it before verify (which tests the cluster),
+    # but after the verify step.
+    - side_effect
+    - cleanup
+    - destroy

molecule/kube-vip/overrides.yml

@@ -0,0 +1,17 @@
+---
+- name: Apply overrides
+  hosts: all
+  tasks:
+    - name: Override host variables
+      ansible.builtin.set_fact:
+        # See:
+        # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
+        flannel_iface: eth1
+
+        # The test VMs might be a bit slow, so we give them more time to join the cluster:
+        retry_count: 45
+
+        # Make sure that our IP ranges do not collide with those of the other scenarios
+        apiserver_endpoint: "192.168.30.225"
+        # Use kube-vip instead of MetalLB
+        kube_vip_lb_ip_range: "192.168.30.110-192.168.30.119"

reboot.yml

@@ -6,4 +6,5 @@
     - name: Reboot the nodes (and Wait upto 5 mins max)
       become: true
       reboot:
+        reboot_command: "{{ custom_reboot_command | default(omit) }}"
         reboot_timeout: 300

requirements.txt

@@ -6,7 +6,7 @@
 #
 ansible-compat==4.1.11
     # via molecule
-ansible-core==2.16.2
+ansible-core==2.17.2
     # via
     #   -r requirements.in
     #   ansible-compat
@@ -77,7 +77,7 @@ molecule==6.0.3
     # via
     #   -r requirements.in
     #   molecule-plugins
-molecule-plugins[vagrant]==23.5.0
+molecule-plugins[vagrant]==23.5.3
     # via -r requirements.in
 netaddr==0.10.1
     # via -r requirements.in
@@ -96,9 +96,9 @@ platformdirs==4.1.0
     # via virtualenv
 pluggy==1.3.0
     # via molecule
-pre-commit==3.6.0
+pre-commit==3.8.0
     # via -r requirements.in
-pre-commit-hooks==4.5.0
+pre-commit-hooks==4.6.0
     # via -r requirements.in
 pyasn1==0.5.1
     # via

reset.yml

@@ -12,6 +12,7 @@
     - name: Reboot and wait for node to come back up
       become: true
       reboot:
+        reboot_command: "{{ custom_reboot_command | default(omit) }}"
         reboot_timeout: 3600
 
 - name: Revert changes to Proxmox cluster

roles/k3s_agent/tasks/http_proxy.yml

@@ -1,5 +1,4 @@
 ---
-
 - name: Create k3s-node.service.d directory
   file:
     path: '{{ systemd_dir }}/k3s-node.service.d'
@@ -7,7 +6,7 @@
     owner: root
     group: root
     mode: '0755'
-
+  when: proxy_env is defined
 
 - name: Copy K3s http_proxy conf file
   template:
@@ -16,3 +15,4 @@
     owner: root
     group: root
     mode: '0755'
+  when: proxy_env is defined

roles/k3s_agent/tasks/main.yml

@@ -1,19 +1,35 @@
 ---
+- name: Check for PXE-booted system
+  block:
+    - name: Check if system is PXE-booted
+      ansible.builtin.command:
+        cmd: cat /proc/cmdline
+      register: boot_cmdline
+      changed_when: false
+      check_mode: false
+
+    - name: Set fact for PXE-booted system
+      ansible.builtin.set_fact:
+        is_pxe_booted: "{{ 'root=/dev/nfs' in boot_cmdline.stdout }}"
+      when: boot_cmdline.stdout is defined
+
+    - name: Include http_proxy configuration tasks
+      ansible.builtin.include_tasks: http_proxy.yml
 
 - name: Deploy K3s http_proxy conf
   include_tasks: http_proxy.yml
   when: proxy_env is defined
 
-- name: Copy K3s service file
-  template:
+- name: Configure the k3s service
+  ansible.builtin.template:
     src: "k3s.service.j2"
     dest: "{{ systemd_dir }}/k3s-node.service"
     owner: root
     group: root
-    mode: 0755
+    mode: '0755'
 
-- name: Enable and check K3s service
-  systemd:
+- name: Manage k3s service
+  ansible.builtin.systemd:
     name: k3s-node
     daemon_reload: true
     state: restarted

roles/k3s_agent/templates/k3s.service.j2

@@ -7,11 +7,14 @@ After=network-online.target
 Type=notify
 ExecStartPre=-/sbin/modprobe br_netfilter
 ExecStartPre=-/sbin/modprobe overlay
-ExecStart=/usr/local/bin/k3s agent --server https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443 --token {{ hostvars[groups[group_name_master | default('master')][0]]['token'] | default(k3s_token) }} {{ extra_agent_args | default("") }}
+# Conditional snapshotter based on PXE boot status
+ExecStart=/usr/local/bin/k3s agent \
+  --server https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443 \
+  {% if is_pxe_booted | default(false) %}--snapshotter native \
+  {% endif %}--token {{ hostvars[groups[group_name_master | default('master')][0]]['token'] | default(k3s_token) }} \
+  {{ extra_agent_args | default("") }}
 KillMode=process
 Delegate=yes
-# Having non-zero Limit*s causes performance problems due to accounting overhead
-# in the kernel. We recommend using cgroups to do container-local accounting.
 LimitNOFILE=1048576
 LimitNPROC=infinity
 LimitCORE=infinity

roles/k3s_server/defaults/main.yml

@@ -4,6 +4,9 @@
 # will determine the right interface automatically at runtime.
 kube_vip_iface: null
 
+# Enables ARP broadcasts from Leader
+kube_vip_arp: true
+
 # Name of the master group
 group_name_master: master
 

roles/k3s_server/tasks/kube-vip.yml

@@ -0,0 +1,27 @@
+---
+- name: Create manifests directory on first master
+  file:
+    path: /var/lib/rancher/k3s/server/manifests
+    state: directory
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+
+- name: Download vip cloud provider manifest to first master
+  ansible.builtin.get_url:
+    url: "https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/{{ kube_vip_cloud_provider_tag_version | default('main') }}/manifest/kube-vip-cloud-controller.yaml"  # noqa yaml[line-length]
+    dest: "/var/lib/rancher/k3s/server/manifests/kube-vip-cloud-controller.yaml"
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+
+- name: Copy kubevip configMap manifest to first master
+  template:
+    src: "kubevip.yaml.j2"
+    dest: "/var/lib/rancher/k3s/server/manifests/kubevip.yaml"
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']

roles/k3s_server/tasks/main.yml

@@ -29,6 +29,12 @@
 - name: Deploy metallb manifest
   include_tasks: metallb.yml
   tags: metallb
+  when: kube_vip_lb_ip_range is not defined and (not cilium_bgp or cilium_iface is not defined)
+
+- name: Deploy kube-vip manifest
+  include_tasks: kube-vip.yml
+  tags: kubevip
+  when: kube_vip_lb_ip_range is defined
 
 - name: Init cluster inside the transient k3s-init service
   command:

roles/k3s_server/tasks/vip.yml

@@ -10,7 +10,7 @@
 
 - name: Download vip rbac manifest to first master
   ansible.builtin.get_url:
-    url: "https://raw.githubusercontent.com/kube-vip/kube-vip/{{ kube_vip_tag_version }}/docs/manifests/rbac.yaml"
+    url: "https://kube-vip.io/manifests/rbac.yaml"
     dest: "/var/lib/rancher/k3s/server/manifests/vip-rbac.yaml"
     owner: root
     group: root

roles/k3s_server/templates/kubevip.yaml.j2

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kubevip
+  namespace: kube-system
+data:
+{% if kube_vip_lb_ip_range is string %}
+{# kube_vip_lb_ip_range was used in the legacy way: single string instead of a list #}
+{#   => transform to list with single element #}
+{% set kube_vip_lb_ip_range = [kube_vip_lb_ip_range] %}
+{% endif %}
+  range-global: {{ kube_vip_lb_ip_range | join(',') }}

roles/k3s_server/templates/vip.yaml.j2

@@ -27,7 +27,7 @@ spec:
         - manager
         env:
         - name: vip_arp
-          value: "true"
+          value: "{{ 'true' if kube_vip_arp | bool else 'false' }}"
         - name: port
           value: "6443"
 {% if kube_vip_iface %}
@@ -43,7 +43,7 @@ spec:
         - name: vip_ddns
           value: "false"
         - name: svc_enable
-          value: "false"
+          value: "{{ 'true' if kube_vip_lb_ip_range is defined else 'false' }}"
         - name: vip_leaderelection
           value: "true"
         - name: vip_leaseduration

roles/k3s_server_post/tasks/calico.yml

@@ -0,0 +1,114 @@
+---
+- name: Deploy Calico to cluster
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+  run_once: true
+  block:
+    - name: Create manifests directory on first master
+      file:
+        path: /tmp/k3s
+        state: directory
+        owner: root
+        group: root
+        mode: 0755
+
+    - name: "Download to first master: manifest for Tigera Operator and Calico CRDs"
+      ansible.builtin.get_url:
+        url: "https://raw.githubusercontent.com/projectcalico/calico/{{ calico_tag }}/manifests/tigera-operator.yaml"
+        dest: "/tmp/k3s/tigera-operator.yaml"
+        owner: root
+        group: root
+        mode: 0755
+
+    - name: Copy Calico custom resources manifest to first master
+      ansible.builtin.template:
+        src: "calico.crs.j2"
+        dest: /tmp/k3s/custom-resources.yaml
+        owner: root
+        group: root
+        mode: 0755
+
+    - name: Deploy or replace Tigera Operator
+      block:
+        - name: Deploy Tigera Operator
+          ansible.builtin.command:
+            cmd: kubectl create -f /tmp/k3s/tigera-operator.yaml
+          register: create_operator
+          changed_when: "'created' in create_operator.stdout"
+          failed_when: "'Error' in create_operator.stderr and 'already exists' not in create_operator.stderr"
+      rescue:
+        - name: Replace existing Tigera Operator
+          ansible.builtin.command:
+            cmd: kubectl replace -f /tmp/k3s/tigera-operator.yaml
+          register: replace_operator
+          changed_when: "'replaced' in replace_operator.stdout"
+          failed_when: "'Error' in replace_operator.stderr"
+
+    - name: Wait for Tigera Operator resources
+      command: >-
+        k3s kubectl wait {{ item.type }}/{{ item.name }}
+        --namespace='tigera-operator'
+        --for=condition=Available=True
+        --timeout=30s
+      register: tigera_result
+      changed_when: false
+      until: tigera_result is succeeded
+      retries: 7
+      delay: 7
+      with_items:
+        - {name: tigera-operator, type: deployment}
+      loop_control:
+        label: "{{ item.type }}/{{ item.name }}"
+
+    - name: Deploy Calico custom resources
+      block:
+        - name: Deploy custom resources for Calico
+          ansible.builtin.command:
+            cmd: kubectl create -f /tmp/k3s/custom-resources.yaml
+          register: create_cr
+          changed_when: "'created' in create_cr.stdout"
+          failed_when: "'Error' in create_cr.stderr and 'already exists' not in create_cr.stderr"
+      rescue:
+        - name: Apply new Calico custom resource manifest
+          ansible.builtin.command:
+            cmd: kubectl apply -f /tmp/k3s/custom-resources.yaml
+          register: apply_cr
+          changed_when: "'configured' in apply_cr.stdout or 'created' in apply_cr.stdout"
+          failed_when: "'Error' in apply_cr.stderr"
+
+    - name: Wait for Calico system resources to be available
+      command: >-
+        {% if item.type == 'daemonset' %}
+        k3s kubectl wait pods
+        --namespace='{{ item.namespace }}'
+        --selector={{ item.selector }}
+        --for=condition=Ready
+        {% else %}
+        k3s kubectl wait {{ item.type }}/{{ item.name }}
+        --namespace='{{ item.namespace }}'
+        --for=condition=Available
+        {% endif %}
+        --timeout=30s
+      register: cr_result
+      changed_when: false
+      until: cr_result is succeeded
+      retries: 30
+      delay: 7
+      with_items:
+        - {name: calico-typha, type: deployment, namespace: calico-system}
+        - {name: calico-kube-controllers, type: deployment, namespace: calico-system}
+        - {name: csi-node-driver, type: daemonset, selector: 'k8s-app=csi-node-driver', namespace: calico-system}
+        - {name: calico-node, type: daemonset, selector: 'k8s-app=calico-node', namespace: calico-system}
+        - {name: calico-apiserver, type: deployment, namespace: calico-apiserver}
+      loop_control:
+        label: "{{ item.type }}/{{ item.name }}"
+
+    - name: Patch Felix configuration for eBPF mode
+      ansible.builtin.command:
+        cmd: >
+          kubectl patch felixconfiguration default
+          --type='merge'
+          --patch='{"spec": {"bpfKubeProxyIptablesCleanupEnabled": false}}'
+      register: patch_result
+      changed_when: "'felixconfiguration.projectcalico.org/default patched' in patch_result.stdout"
+      failed_when: "'Error' in patch_result.stderr"
+      when: calico_ebpf

roles/k3s_server_post/tasks/cilium.yml

@@ -0,0 +1,253 @@
+---
+- name: Prepare Cilium CLI on first master and deploy CNI
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+  run_once: true
+  block:
+    - name: Create tmp directory on first master
+      file:
+        path: /tmp/k3s
+        state: directory
+        owner: root
+        group: root
+        mode: 0755
+
+    - name: Check if Cilium CLI is installed
+      ansible.builtin.command: cilium version
+      register: cilium_cli_installed
+      failed_when: false
+      changed_when: false
+      ignore_errors: true
+
+    - name: Check for Cilium CLI version in command output
+      set_fact:
+        installed_cli_version: >-
+          {{
+            cilium_cli_installed.stdout_lines
+            | join(' ')
+            | regex_findall('cilium-cli: (v\d+\.\d+\.\d+)')
+            | first
+            | default('unknown')
+          }}
+      when: cilium_cli_installed.rc == 0
+
+    - name: Get latest stable Cilium CLI version file
+      ansible.builtin.get_url:
+        url: "https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt"
+        dest: "/tmp/k3s/cilium-cli-stable.txt"
+        owner: root
+        group: root
+        mode: 0755
+
+    - name: Read Cilium CLI stable version from file
+      ansible.builtin.command: cat /tmp/k3s/cilium-cli-stable.txt
+      register: cli_ver
+      changed_when: false
+
+    - name: Log installed Cilium CLI version
+      ansible.builtin.debug:
+        msg: "Installed Cilium CLI version: {{ installed_cli_version | default('Not installed') }}"
+
+    - name: Log latest stable Cilium CLI version
+      ansible.builtin.debug:
+        msg: "Latest Cilium CLI version: {{ cli_ver.stdout }}"
+
+    - name: Determine if Cilium CLI needs installation or update
+      set_fact:
+        cilium_cli_needs_update: >-
+          {{
+            cilium_cli_installed.rc != 0 or
+            (cilium_cli_installed.rc == 0 and
+            installed_cli_version != cli_ver.stdout)
+          }}
+
+    - name: Install or update Cilium CLI
+      when: cilium_cli_needs_update
+      block:
+        - name: Set architecture variable
+          ansible.builtin.set_fact:
+            cli_arch: "{{ 'arm64' if ansible_architecture == 'aarch64' else 'amd64' }}"
+
+        - name: Download Cilium CLI and checksum
+          ansible.builtin.get_url:
+            url: "{{ cilium_base_url }}/cilium-linux-{{ cli_arch }}{{ item }}"
+            dest: "/tmp/k3s/cilium-linux-{{ cli_arch }}{{ item }}"
+            owner: root
+            group: root
+            mode: 0755
+          loop:
+            - ".tar.gz"
+            - ".tar.gz.sha256sum"
+          vars:
+            cilium_base_url: "https://github.com/cilium/cilium-cli/releases/download/{{ cli_ver.stdout }}"
+
+        - name: Verify the downloaded tarball
+          ansible.builtin.shell: |
+            cd /tmp/k3s && sha256sum --check cilium-linux-{{ cli_arch }}.tar.gz.sha256sum
+          args:
+            executable: /bin/bash
+          changed_when: false
+
+        - name: Extract Cilium CLI to /usr/local/bin
+          ansible.builtin.unarchive:
+            src: "/tmp/k3s/cilium-linux-{{ cli_arch }}.tar.gz"
+            dest: /usr/local/bin
+            remote_src: true
+
+        - name: Remove downloaded tarball and checksum file
+          ansible.builtin.file:
+            path: "{{ item }}"
+            state: absent
+          loop:
+            - "/tmp/k3s/cilium-linux-{{ cli_arch }}.tar.gz"
+            - "/tmp/k3s/cilium-linux-{{ cli_arch }}.tar.gz.sha256sum"
+
+    - name: Wait for connectivity to kube VIP
+      ansible.builtin.command: ping -c 1 {{ apiserver_endpoint }}
+      register: ping_result
+      until: ping_result.rc == 0
+      retries: 21
+      delay: 1
+      ignore_errors: true
+      changed_when: false
+
+    - name: Fail if kube VIP not reachable
+      ansible.builtin.fail:
+        msg: "API endpoint {{ apiserver_endpoint }} is not reachable"
+      when: ping_result.rc != 0
+
+    - name: Test for existing Cilium install
+      ansible.builtin.command: k3s kubectl -n kube-system get daemonsets cilium
+      register: cilium_installed
+      failed_when: false
+      changed_when: false
+      ignore_errors: true
+
+    - name: Check existing Cilium install
+      when: cilium_installed.rc == 0
+      block:
+
+        - name: Check Cilium version
+          ansible.builtin.command: cilium version
+          register: cilium_version
+          failed_when: false
+          changed_when: false
+          ignore_errors: true
+
+        - name: Parse installed Cilium version
+          set_fact:
+            installed_cilium_version: >-
+              {{
+                cilium_version.stdout_lines
+                | join(' ')
+                | regex_findall('cilium image.+(\d+\.\d+\.\d+)')
+                | first
+                | default('unknown')
+              }}
+
+        - name: Determine if Cilium needs update
+          set_fact:
+            cilium_needs_update: >-
+              {{ 'v' + installed_cilium_version != cilium_tag }}
+
+        - name: Log result
+          ansible.builtin.debug:
+            msg: >
+              Installed Cilium version: {{ installed_cilium_version }},
+              Target Cilium version: {{ cilium_tag }},
+              Update needed: {{ cilium_needs_update }}
+
+    - name: Install Cilium
+      ansible.builtin.command: >-
+        {% if cilium_installed.rc != 0 %}
+        cilium install
+        {% else %}
+        cilium upgrade
+        {% endif %}
+        --version "{{ cilium_tag }}"
+        --helm-set operator.replicas="1"
+        {{ '--helm-set devices=' + cilium_iface if cilium_iface != 'auto' else '' }}
+        --helm-set ipam.operator.clusterPoolIPv4PodCIDRList={{ cluster_cidr }}
+        {% if cilium_mode == "native" or (cilium_bgp and cilium_exportPodCIDR != 'false') %}
+        --helm-set ipv4NativeRoutingCIDR={{ cluster_cidr }}
+        {% endif %}
+        --helm-set k8sServiceHost="127.0.0.1"
+        --helm-set k8sServicePort="6444"
+        --helm-set routingMode={{ cilium_mode | default("native") }}
+        --helm-set autoDirectNodeRoutes={{ "true" if cilium_mode == "native" else "false" }}
+        --helm-set kubeProxyReplacement={{ kube_proxy_replacement | default("true") }}
+        --helm-set bpf.masquerade={{ enable_bpf_masquerade | default("true") }}
+        --helm-set bgpControlPlane.enabled={{ cilium_bgp | default("false") }}
+        --helm-set hubble.enabled={{ "true" if cilium_hubble else "false" }}
+        --helm-set hubble.relay.enabled={{ "true" if cilium_hubble else "false" }}
+        --helm-set hubble.ui.enabled={{ "true" if cilium_hubble else "false" }}
+        {% if kube_proxy_replacement is not false %}
+        --helm-set bpf.loadBalancer.algorithm={{ bpf_lb_algorithm | default("maglev") }}
+        --helm-set bpf.loadBalancer.mode={{ bpf_lb_mode | default("hybrid") }}
+        {% endif %}
+      environment:
+        KUBECONFIG: "{{ ansible_user_dir }}/.kube/config"
+      register: cilium_install_result
+      changed_when: cilium_install_result.rc == 0
+      when: cilium_installed.rc != 0 or cilium_needs_update
+
+    - name: Wait for Cilium resources
+      command: >-
+        {% if item.type == 'daemonset' %}
+        k3s kubectl wait pods
+        --namespace=kube-system
+        --selector='k8s-app=cilium'
+        --for=condition=Ready
+        {% else %}
+        k3s kubectl wait {{ item.type }}/{{ item.name }}
+        --namespace=kube-system
+        --for=condition=Available
+        {% endif %}
+        --timeout=30s
+      register: cr_result
+      changed_when: false
+      until: cr_result is succeeded
+      retries: 30
+      delay: 7
+      with_items:
+        - {name: cilium-operator, type: deployment}
+        - {name: cilium, type: daemonset, selector: 'k8s-app=cilium'}
+        - {name: hubble-relay, type: deployment, check_hubble: true}
+        - {name: hubble-ui, type: deployment, check_hubble: true}
+      loop_control:
+        label: "{{ item.type }}/{{ item.name }}"
+      when: >-
+        not item.check_hubble | default(false) or (item.check_hubble | default(false) and cilium_hubble)
+
+    - name: Configure Cilium BGP
+      when: cilium_bgp
+      block:
+
+        - name: Copy BGP manifests to first master
+          ansible.builtin.template:
+            src: "cilium.crs.j2"
+            dest: /tmp/k3s/cilium-bgp.yaml
+            owner: root
+            group: root
+            mode: 0755
+
+        - name: Apply BGP manifests
+          ansible.builtin.command:
+            cmd: kubectl apply -f /tmp/k3s/cilium-bgp.yaml
+          register: apply_cr
+          changed_when: "'configured' in apply_cr.stdout or 'created' in apply_cr.stdout"
+          failed_when: "'is invalid' in apply_cr.stderr"
+          ignore_errors: true
+
+        - name: Print error message if BGP manifests application fails
+          ansible.builtin.debug:
+            msg: "{{ apply_cr.stderr }}"
+          when: "'is invalid' in apply_cr.stderr"
+
+        - name: Test for BGP config resources
+          ansible.builtin.command: "{{ item }}"
+          loop:
+            - k3s kubectl get CiliumBGPPeeringPolicy.cilium.io
+            - k3s kubectl get CiliumLoadBalancerIPPool.cilium.io
+          changed_when: false
+          loop_control:
+            label: "{{ item }}"

roles/k3s_server_post/tasks/main.yml

@@ -1,7 +1,18 @@
 ---
+- name: Deploy calico
+  include_tasks: calico.yml
+  tags: calico
+  when: calico_iface is defined and cilium_iface is not defined
+
+- name: Deploy cilium
+  include_tasks: cilium.yml
+  tags: cilium
+  when: cilium_iface is defined
+
 - name: Deploy metallb pool
   include_tasks: metallb.yml
   tags: metallb
+  when: kube_vip_lb_ip_range is not defined and (not cilium_bgp or cilium_iface is not defined)
 
 - name: Remove tmp directory used for manifests
   file:

roles/k3s_server_post/tasks/metallb.yml

@@ -8,6 +8,27 @@
   with_items: "{{ groups[group_name_master | default('master')] }}"
   run_once: true
 
+- name: Delete outdated metallb replicas
+  shell: |-
+    set -o pipefail
+
+    REPLICAS=$(k3s kubectl --namespace='metallb-system' get replicasets \
+      -l 'component=controller,app=metallb' \
+      -o jsonpath='{.items[0].spec.template.spec.containers[0].image}, {.items[0].metadata.name}' 2>/dev/null || true)
+    REPLICAS_SETS=$(echo ${REPLICAS} | grep -v '{{ metal_lb_controller_tag_version }}' | sed -e "s/^.*\s//g")
+    if [ -n "${REPLICAS_SETS}" ] ; then
+      for REPLICAS in "${REPLICAS_SETS}"
+      do
+        k3s kubectl --namespace='metallb-system' \
+          delete rs "${REPLICAS}"
+      done
+    fi
+  args:
+    executable: /bin/bash
+  changed_when: false
+  run_once: true
+  with_items: "{{ groups[group_name_master | default('master')] }}"
+
 - name: Copy metallb CRs manifest to first master
   template:
     src: "metallb.crs.j2"
@@ -62,9 +83,23 @@
   loop_control:
     label: "{{ item.description }}"
 
+- name: Set metallb webhook service name
+  set_fact:
+    metallb_webhook_service_name: >-
+      {{
+        (
+          (metal_lb_controller_tag_version | regex_replace('^v', ''))
+          is
+          version('0.14.4', '<', version_type='semver')
+        ) | ternary(
+          'webhook-service',
+          'metallb-webhook-service'
+        )
+      }}
+
 - name: Test metallb-system webhook-service endpoint
   command: >-
-    k3s kubectl -n metallb-system get endpoints webhook-service
+    k3s kubectl -n metallb-system get endpoints {{ metallb_webhook_service_name }}
   changed_when: false
   with_items: "{{ groups[group_name_master | default('master')] }}"
   run_once: true

roles/k3s_server_post/templates/calico.crs.j2

@@ -0,0 +1,41 @@
+# This section includes base Calico installation configuration.
+# For more information, see: https://docs.tigera.io/calico/latest/reference/installation/api#operator.tigera.io/v1.Installation
+apiVersion: operator.tigera.io/v1
+kind: Installation
+metadata:
+  name: default
+spec:
+  # Configures Calico networking.
+  calicoNetwork:
+    # Note: The ipPools section cannot be modified post-install.
+    ipPools:
+    - blockSize: {{ calico_blockSize | default('26') }}
+      cidr: {{ cluster_cidr | default('10.52.0.0/16') }}
+      encapsulation: {{ calico_encapsulation | default('VXLANCrossSubnet') }}
+      natOutgoing: {{ calico_natOutgoing | default('Enabled') }}
+      nodeSelector: {{ calico_nodeSelector | default('all()') }}
+    nodeAddressAutodetectionV4:
+      interface: {{ calico_iface  }}
+    linuxDataplane: {{ 'BPF' if calico_ebpf else 'Iptables' }}
+
+---
+
+# This section configures the Calico API server.
+# For more information, see: https://docs.tigera.io/calico/latest/reference/installation/api#operator.tigera.io/v1.APIServer
+apiVersion: operator.tigera.io/v1
+kind: APIServer
+metadata:
+  name: default
+spec: {}
+
+{% if calico_ebpf %}
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kubernetes-services-endpoint
+  namespace: tigera-operator
+data:
+  KUBERNETES_SERVICE_HOST: '{{ apiserver_endpoint }}'
+  KUBERNETES_SERVICE_PORT: '6443'
+{% endif %}

roles/k3s_server_post/templates/cilium.crs.j2

@@ -0,0 +1,34 @@
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPPeeringPolicy
+metadata:
+ name: 01-bgp-peering-policy
+spec: # CiliumBGPPeeringPolicySpec
+ virtualRouters: # []CiliumBGPVirtualRouter
+ - localASN: {{ cilium_bgp_my_asn }}
+   exportPodCIDR: {{ cilium_exportPodCIDR | default('true') }}
+   neighbors: # []CiliumBGPNeighbor
+    - peerAddress: '{{ cilium_bgp_peer_address + "/32"}}'
+      peerASN: {{ cilium_bgp_peer_asn }}
+      eBGPMultihopTTL: 10
+      connectRetryTimeSeconds: 120
+      holdTimeSeconds: 90
+      keepAliveTimeSeconds: 30
+      gracefulRestart:
+        enabled: true
+        restartTimeSeconds: 120
+   serviceSelector:
+      matchExpressions:
+         - {key: somekey, operator: NotIn, values: ['never-used-value']}
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: "01-lb-pool"
+spec:
+  blocks:
+{% if "/" in cilium_bgp_lb_cidr %}
+  - cidr: {{ cilium_bgp_lb_cidr }}
+{% else %}
+  - start: {{ cilium_bgp_lb_cidr.split('-')[0] }}
+    stop: {{ cilium_bgp_lb_cidr.split('-')[1] }}
+{% endif %}

roles/lxc/handlers/main.yml

@@ -2,4 +2,5 @@
 - name: Reboot server
   become: true
   reboot:
+    reboot_command: "{{ custom_reboot_command | default(omit) }}"
   listen: reboot server

roles/raspberrypi/handlers/main.yml

@@ -1,4 +1,5 @@
 ---
 - name: Reboot
   reboot:
+    reboot_command: "{{ custom_reboot_command | default(omit) }}"
   listen: reboot

roles/raspberrypi/tasks/main.yml

@@ -31,6 +31,14 @@
     - raspberry_pi|default(false)
     - ansible_facts.lsb.description|default("") is match(allowed_descriptions | join('|'))
 
+- name: Set detected_distribution to Raspbian (ARM64 on Debian Bookworm)
+  set_fact:
+    detected_distribution: Raspbian
+  when:
+    - ansible_facts.architecture is search("aarch64")
+    - raspberry_pi|default(false)
+    - ansible_facts.lsb.description|default("") is match("Debian.*bookworm")
+
 - name: Set detected_distribution_major_version
   set_fact:
     detected_distribution_major_version: "{{ ansible_facts.lsb.major_release }}"

roles/raspberrypi/tasks/setup/Raspbian.yml

@@ -1,7 +1,27 @@
 ---
+- name: Test for cmdline path
+  stat:
+    path: /boot/firmware/cmdline.txt
+  register: boot_cmdline_path
+  failed_when: false
+  changed_when: false
+
+- name: Set cmdline path based on Debian version and command result
+  set_fact:
+    cmdline_path: >-
+      {{
+        (
+          boot_cmdline_path.stat.exists and
+          ansible_facts.lsb.description | default('') is match('Debian.*(?!(bookworm|sid))')
+        ) | ternary(
+          '/boot/firmware/cmdline.txt',
+          '/boot/cmdline.txt'
+        )
+      }}
+
 - name: Activating cgroup support
   lineinfile:
-    path: /boot/cmdline.txt
+    path: "{{ cmdline_path }}"
     regexp: '^((?!.*\bcgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory\b).*)$'
     line: '\1 cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory'
     backrefs: true

roles/reset/tasks/main.yml

@@ -45,6 +45,7 @@
     - /var/lib/rancher/k3s
     - /var/lib/rancher/
     - /var/lib/cni/
+    - /etc/cni/net.d
 
 - name: Remove K3s http_proxy files
   file:

site.yml

@@ -1,4 +1,13 @@
 ---
+- name: Pre tasks
+  hosts: all
+  pre_tasks:
+    - name: Verify Ansible is version 2.11 or above. (If this fails you may need to update Ansible)
+      assert:
+        that: "ansible_version.full is version_compare('2.11', '>=')"
+        msg: >
+          "Ansible is out of date. See here for more info: https://docs.technotim.live/posts/ansible-automation/"
+
 - name: Prepare Proxmox cluster
   hosts: proxmox
   gather_facts: true
@@ -46,3 +55,14 @@
   roles:
     - role: k3s_server_post
       become: true
+
+- name: Storing kubeconfig in the playbook directory
+  hosts: master
+  environment: "{{ proxy_env | default({}) }}"
+  tasks:
+    - name: Copying kubeconfig from {{ hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] }}
+      ansible.builtin.fetch:
+        src: "{{ ansible_user_dir }}/.kube/config"
+        dest: ./kubeconfig
+        flat: true
+      when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']