k3s_server: add kube-vip BGP support

With the kube_vip_bgp parameter it is possible to enable the kube-vip
BGP support (https://kube-vip.io/docs/modes/bgp/).

The configuration is possible with the following new parameters:

* kube_vip_bgp_routerid
* kube_vip_bgp_as
* kube_vip_bgp_peeraddress
* kube_vip_bgp_peeras

Signed-off-by: Christian Berendt <berendt@osism.tech>

.github/workflows/test.yml

@@ -118,7 +118,7 @@ jobs:
 
       - name: Upload log files
         if: always() # do this even if a step before has failed
-        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # 4.3.4
+        uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # 4.3.5
         with:
           name: logs
           path: |

inventory/sample/group_vars/all.yml

@@ -34,6 +34,18 @@ cilium_bgp_peer_asn: "64512"
 cilium_bgp_peer_address: "192.168.30.1"
 cilium_bgp_lb_cidr: "192.168.31.0/24"   # cidr for cilium loadbalancer ipam
 
+# enable kube-vip ARP broadcasts
+kube_vip_arp: true
+
+# enable kube-vip BGP peering
+kube_vip_bgp: false
+
+# bgp parameters for kube-vip
+kube_vip_bgp_routerid: "127.0.0.1" # Defines the router ID for the BGP server
+kube_vip_bgp_as: "64513" # Defines the AS for the BGP server
+kube_vip_bgp_peeraddress: "192.168.30.1" # Defines the address for the BGP peer
+kube_vip_bgp_peeras: "64512" # Defines the AS for the BGP peer
+
 # apiserver_endpoint is virtual ip-address which will be configured on each master
 apiserver_endpoint: "192.168.30.222"
 

roles/download/meta/main.yml

@@ -0,0 +1,8 @@
+---
+argument_specs:
+  main:
+    short_description: Manage the downloading of K3S binaries
+    options:
+      k3s_version:
+        description: The desired version of K3S
+        required: true

roles/k3s_agent/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+extra_agent_args: ""
+group_name_master: master
+systemd_dir: /etc/systemd/system

roles/k3s_agent/meta/main.yml

@@ -0,0 +1,34 @@
+---
+argument_specs:
+  main:
+    short_description: Setup k3s agents
+    options:
+      apiserver_endpoint:
+        description: Virtual ip-address configured on each master
+        required: true
+
+      extra_agent_args:
+        description: Extra arguments for agents nodes
+
+      group_name_master:
+        description: Name of the master group
+        default: master
+
+      k3s_token:
+        description: Token used to communicate between masters
+
+      proxy_env:
+        type: dict
+        description: Internet proxy configurations
+        default: ~
+        options:
+          HTTP_PROXY:
+            required: true
+          HTTPS_PROXY:
+            required: true
+          NO_PROXY:
+            required: true
+
+      systemd_dir:
+        description: Path to systemd services
+        default: /etc/systemd/system

roles/k3s_agent/templates/k3s.service.j2

@@ -12,7 +12,7 @@ ExecStart=/usr/local/bin/k3s agent \
   --server https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443 \
   {% if is_pxe_booted | default(false) %}--snapshotter native \
   {% endif %}--token {{ hostvars[groups[group_name_master | default('master')][0]]['token'] | default(k3s_token) }} \
-  {{ extra_agent_args | default("") }}
+  {{ extra_agent_args }}
 KillMode=process
 Delegate=yes
 LimitNOFILE=1048576

roles/k3s_custom_registries/defaults/main.yml

@@ -1,6 +0,0 @@
----
-# Indicates whether custom registries for k3s should be configured
-# Possible values:
-#   - present
-#   - absent
-state: present

roles/k3s_custom_registries/meta/main.yml

@@ -0,0 +1,20 @@
+---
+argument_specs:
+  main:
+    short_description: Configure the use of a custom container registry
+    options:
+      custom_registries_yaml:
+        description:
+          - YAML block defining custom registries.
+          - >
+            The following is an example that pulls all images used in
+            this playbook through your private registries.
+          - >
+            It also allows you to pull your own images from your private
+            registry, without having to use imagePullSecrets in your
+            deployments.
+          - >
+            If all you need is your own images and you don't care about
+            caching the docker/quay/ghcr.io images, you can just remove
+            those from the mirrors: section.
+        required: true

roles/k3s_server/defaults/main.yml

@@ -1,15 +1,25 @@
 ---
-# If you want to explicitly define an interface that ALL control nodes
-# should use to propagate the VIP, define it here. Otherwise, kube-vip
-# will determine the right interface automatically at runtime.
-kube_vip_iface: null
+extra_server_args: ""
 
-# Enables ARP broadcasts from Leader
-kube_vip_arp: true
-
-# Name of the master group
 group_name_master: master
 
+kube_vip_arp: true
+kube_vip_iface: ~
+kube_vip_cloud_provider_tag_version: main
+kube_vip_tag_version: v0.7.2
+
+kube_vip_bgp: false
+kube_vip_bgp_routerid: "127.0.0.1"
+kube_vip_bgp_as: "64513"
+kube_vip_bgp_peeraddress: "192.168.30.1"
+kube_vip_bgp_peeras: "64512"
+
+metal_lb_controller_tag_version: v0.14.3
+metal_lb_speaker_tag_version: v0.14.3
+metal_lb_type: native
+
+retry_count: 20
+
 # yamllint disable rule:line-length
 server_init_args: >-
   {% if groups[group_name_master | default('master')] | length > 1 %}
@@ -20,4 +30,6 @@ server_init_args: >-
     {% endif %}
     --token {{ k3s_token }}
   {% endif %}
-  {{ extra_server_args | default('') }}
+  {{ extra_server_args }}
+
+systemd_dir: /etc/systemd/system

roles/k3s_server/meta/main.yml

@@ -0,0 +1,111 @@
+---
+argument_specs:
+  main:
+    short_description: Setup k3s servers
+    options:
+      apiserver_endpoint:
+        description: Virtual ip-address configured on each master
+        required: true
+
+      cilium_bgp:
+        description:
+          - Enable cilium BGP control plane for LB services and pod cidrs.
+          - Disables the use of MetalLB.
+        type: bool
+        default: ~
+
+      cilium_iface:
+        description: The network interface used for when Cilium is enabled
+        default: ~
+
+      extra_server_args:
+        description: Extra arguments for server nodes
+        default: ""
+
+      group_name_master:
+        description: Name of the master group
+        default: master
+
+      kube_vip_arp:
+        description: Enables kube-vip ARP broadcasts
+        default: true
+        type: bool
+
+      kube_vip_bgp:
+        description: Enables kube-vip BGP peering
+        default: false
+        type: bool
+
+      kube_vip_bgp_routerid:
+        description: Defines the router ID for the kube-vip BGP server
+        default: "127.0.0.1"
+
+      kube_vip_bgp_as:
+        description: Defines the AS for the kube-vip BGP server
+        default: "64513"
+
+      kube_vip_bgp_peeraddress:
+        description: Defines the address for the kube-vip BGP peer
+        default: "192.168.30.1"
+
+      kube_vip_bgp_peeras:
+        description: Defines the AS for the kube-vip BGP peer
+        default: "64512"
+
+      kube_vip_iface:
+        description:
+          - Explicitly define an interface that ALL control nodes
+          - should use to propagate the VIP, define it here.
+          - Otherwise, kube-vip will determine the right interface
+          - automatically at runtime.
+        default: ~
+
+      kube_vip_tag_version:
+        description: Image tag for kube-vip
+        default: v0.7.2
+
+      kube_vip_cloud_provider_tag_version:
+        description: Tag for kube-vip-cloud-provider manifest when enabled
+        default: main
+
+      kube_vip_lb_ip_range:
+        description: IP range for kube-vip load balancer
+        default: ~
+
+      metal_lb_controller_tag_version:
+        description: Image tag for MetalLB
+        default: v0.14.3
+
+      metal_lb_speaker_tag_version:
+        description: Image tag for MetalLB
+        default: v0.14.3
+
+      metal_lb_type:
+        choices:
+          - frr
+          - native
+        default: native
+
+      proxy_env:
+        type: dict
+        description: Internet proxy configurations
+        default: ~
+        options:
+          HTTP_PROXY:
+            required: true
+          HTTPS_PROXY:
+            required: true
+          NO_PROXY:
+            required: true
+
+      retry_count:
+        description: Amount of retries when verifying that nodes joined
+        type: int
+        default: 20
+
+      server_init_args:
+        description: Arguments for server nodes
+
+      systemd_dir:
+        description: Path to systemd services
+        default: /etc/systemd/system

roles/k3s_server/tasks/http_proxy.yml

@@ -1,5 +1,4 @@
 ---
-
 - name: Create k3s.service.d directory
   file:
     path: '{{ systemd_dir }}/k3s.service.d'

roles/k3s_server/tasks/main.yml

@@ -1,5 +1,4 @@
 ---
-
 - name: Stop k3s-init
   systemd:
     name: k3s-init
@@ -146,12 +145,14 @@
     src: /usr/local/bin/k3s
     dest: /usr/local/bin/kubectl
     state: link
+  when: k3s_create_kubectl_symlink | default(true) | bool
 
 - name: Create crictl symlink
   file:
     src: /usr/local/bin/k3s
     dest: /usr/local/bin/crictl
     state: link
+  when: k3s_create_crictl_symlink | default(true) | bool
 
 - name: Get contents of manifests folder
   find:

roles/k3s_server/templates/vip.yaml.j2

@@ -27,7 +27,9 @@ spec:
         - manager
         env:
         - name: vip_arp
-          value: "{{ 'true' if kube_vip_arp | bool else 'false' }}"
+          value: "{{ 'true' if kube_vip_arp | default(true) | bool else 'false' }}"
+        - name: bgp_enable
+          value: "{{ 'true' if kube_vip_bgp | default(false) | bool else 'false' }}"
         - name: port
           value: "6443"
 {% if kube_vip_iface %}
@@ -54,6 +56,24 @@ spec:
           value: "2"
         - name: address
           value: {{ apiserver_endpoint }}
+{% if kube_vip_bgp | default(false) | bool %}
+{% if kube_vip_bgp_routerid is defined %}
+        - name: bgp_routerid
+          value: "{{ kube_vip_bgp_routerid }}"
+{% endif %}
+{% if kube_vip_bgp_as is defined %}
+        - name: bgp_as
+          value: "{{ kube_vip_bgp_as }}"
+{% endif %}
+{% if kube_vip_bgp_peeraddress is defined %}
+        - name: bgp_peeraddress
+          value: "{{ kube_vip_bgp_peeraddress }}"
+{% endif %}
+{% if kube_vip_bgp_peeras is defined %}
+        - name: bgp_peeras
+          value: "{{ kube_vip_bgp_peeras }}"
+{% endif %}
+{% endif %}
         image: ghcr.io/kube-vip/kube-vip:{{ kube_vip_tag_version }}
         imagePullPolicy: Always
         name: kube-vip

roles/k3s_server_post/defaults/main.yml

@@ -1,6 +1,28 @@
 ---
-# Timeout to wait for MetalLB services to come up
-metal_lb_available_timeout: 240s
+bpf_lb_algorithm: maglev
+bpf_lb_mode: hybrid
 
-# Name of the master group
+calico_blockSize: 26  # noqa var-naming
+calico_ebpf: false
+calico_encapsulation: VXLANCrossSubnet
+calico_natOutgoing: Enabled  # noqa var-naming
+calico_nodeSelector: all()  # noqa var-naming
+calico_tag: v3.27.2
+
+cilium_bgp: false
+cilium_exportPodCIDR: true  # noqa var-naming
+cilium_bgp_my_asn: 64513
+cilium_bgp_peer_asn: 64512
+cilium_bgp_lb_cidr: 192.168.31.0/24
+cilium_hubble: true
+cilium_mode: native
+
+cluster_cidr: 10.52.0.0/16
+enable_bpf_masquerade: true
+kube_proxy_replacement: true
 group_name_master: master
+
+metal_lb_mode: layer2
+metal_lb_available_timeout: 240s
+metal_lb_controller_tag_version: v0.14.3
+metal_lb_ip_range: 192.168.30.80-192.168.30.90

roles/k3s_server_post/meta/main.yml

@@ -0,0 +1,145 @@
+---
+argument_specs:
+  main:
+    short_description: Configure k3s cluster
+    options:
+      apiserver_endpoint:
+        description: Virtual ip-address configured on each master
+        required: true
+
+      bpf_lb_algorithm:
+        description: BPF lb algorithm
+        default: maglev
+
+      bpf_lb_mode:
+        description: BPF lb mode
+        default: hybrid
+
+      calico_blockSize:
+        description: IP pool block size
+        type: int
+        default: 26
+
+      calico_ebpf:
+        description: Use eBPF dataplane instead of iptables
+        type: bool
+        default: false
+
+      calico_encapsulation:
+        description: IP pool encapsulation
+        default: VXLANCrossSubnet
+
+      calico_natOutgoing:
+        description: IP pool NAT outgoing
+        default: Enabled
+
+      calico_nodeSelector:
+        description: IP pool node selector
+        default: all()
+
+      calico_iface:
+        description: The network interface used for when Calico is enabled
+        default: ~
+
+      calico_tag:
+        description: Calico version tag
+        default: v3.27.2
+
+      cilium_bgp:
+        description:
+          - Enable cilium BGP control plane for LB services and pod cidrs.
+          - Disables the use of MetalLB.
+        type: bool
+        default: false
+
+      cilium_bgp_my_asn:
+        description: Local ASN for BGP peer
+        type: int
+        default: 64513
+
+      cilium_bgp_peer_asn:
+        description: BGP peer ASN
+        type: int
+        default: 64512
+
+      cilium_bgp_peer_address:
+        description: BGP peer address
+        default: ~
+
+      cilium_bgp_lb_cidr:
+        description: BGP load balancer IP range
+        default: 192.168.31.0/24
+
+      cilium_exportPodCIDR:
+        description: Export pod CIDR
+        type: bool
+        default: true
+
+      cilium_hubble:
+        description: Enable Cilium Hubble
+        type: bool
+        default: true
+
+      cilium_iface:
+        description: The network interface used for when Cilium is enabled
+        default: ~
+
+      cilium_mode:
+        description: Inner-node communication mode
+        default: native
+        choices:
+          - native
+          - routed
+
+      cluster_cidr:
+        description: Inner-cluster IP range
+        default: 10.52.0.0/16
+
+      enable_bpf_masquerade:
+        description: Use IP masquerading
+        type: bool
+        default: true
+
+      group_name_master:
+        description: Name of the master group
+        default: master
+
+      kube_proxy_replacement:
+        description: Replace the native kube-proxy with Cilium
+        type: bool
+        default: true
+
+      kube_vip_lb_ip_range:
+        description: IP range for kube-vip load balancer
+        default: ~
+
+      metal_lb_available_timeout:
+        description: Wait for MetalLB resources
+        default: 240s
+
+      metal_lb_ip_range:
+        description: MetalLB ip range for load balancer
+        default: 192.168.30.80-192.168.30.90
+
+      metal_lb_controller_tag_version:
+        description: Image tag for MetalLB
+        default: v0.14.3
+
+      metal_lb_mode:
+        description: Metallb mode
+        default: layer2
+        choices:
+          - bgp
+          - layer2
+
+      metal_lb_bgp_my_asn:
+        description: BGP ASN configurations
+        default: ~
+
+      metal_lb_bgp_peer_asn:
+        description: BGP peer ASN configurations
+        default: ~
+
+      metal_lb_bgp_peer_address:
+        description: BGP peer address
+        default: ~

roles/k3s_server_post/tasks/cilium.yml

@@ -172,17 +172,17 @@
         {% endif %}
         --helm-set k8sServiceHost="127.0.0.1"
         --helm-set k8sServicePort="6444"
-        --helm-set routingMode={{ cilium_mode | default("native") }}
+        --helm-set routingMode={{ cilium_mode }}
         --helm-set autoDirectNodeRoutes={{ "true" if cilium_mode == "native" else "false" }}
-        --helm-set kubeProxyReplacement={{ kube_proxy_replacement | default("true") }}
-        --helm-set bpf.masquerade={{ enable_bpf_masquerade | default("true") }}
+        --helm-set kubeProxyReplacement={{ kube_proxy_replacement }}
+        --helm-set bpf.masquerade={{ enable_bpf_masquerade }}
         --helm-set bgpControlPlane.enabled={{ cilium_bgp | default("false") }}
         --helm-set hubble.enabled={{ "true" if cilium_hubble else "false" }}
         --helm-set hubble.relay.enabled={{ "true" if cilium_hubble else "false" }}
         --helm-set hubble.ui.enabled={{ "true" if cilium_hubble else "false" }}
         {% if kube_proxy_replacement is not false %}
-        --helm-set bpf.loadBalancer.algorithm={{ bpf_lb_algorithm | default("maglev") }}
-        --helm-set bpf.loadBalancer.mode={{ bpf_lb_mode | default("hybrid") }}
+        --helm-set bpf.loadBalancer.algorithm={{ bpf_lb_algorithm }}
+        --helm-set bpf.loadBalancer.mode={{ bpf_lb_mode }}
         {% endif %}
       environment:
         KUBECONFIG: "{{ ansible_user_dir }}/.kube/config"

roles/k3s_server_post/templates/calico.crs.j2

@@ -9,11 +9,11 @@ spec:
   calicoNetwork:
     # Note: The ipPools section cannot be modified post-install.
     ipPools:
-    - blockSize: {{ calico_blockSize | default('26') }}
-      cidr: {{ cluster_cidr | default('10.52.0.0/16') }}
-      encapsulation: {{ calico_encapsulation | default('VXLANCrossSubnet') }}
-      natOutgoing: {{ calico_natOutgoing | default('Enabled') }}
-      nodeSelector: {{ calico_nodeSelector | default('all()') }}
+    - blockSize: {{ calico_blockSize }}
+      cidr: {{ cluster_cidr }}
+      encapsulation: {{ calico_encapsulation }}
+      natOutgoing: {{ calico_natOutgoing }}
+      nodeSelector: {{ calico_nodeSelector }}
     nodeAddressAutodetectionV4:
       interface: {{ calico_iface  }}
     linuxDataplane: {{ 'BPF' if calico_ebpf else 'Iptables' }}

roles/lxc/meta/main.yml

@@ -0,0 +1,7 @@
+---
+argument_specs:
+  main:
+    short_description: Configure LXC
+    options:
+      custom_reboot_command:
+        default: ~

roles/prereq/defaults/main.yml

@@ -1,4 +1,4 @@
 ---
 secure_path:
-  RedHat: '/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin'
-  Suse: '/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin'
+  RedHat: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
+  Suse: /usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin

roles/prereq/meta/main.yml

@@ -0,0 +1,7 @@
+---
+argument_specs:
+  main:
+    short_description: Prerequisites
+    options:
+      system_timezone:
+        description: Timezone to be set on all nodes

roles/reset/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+systemd_dir: /etc/systemd/system

roles/reset/meta/main.yml

@@ -0,0 +1,8 @@
+---
+argument_specs:
+  main:
+    short_description: Reset all nodes
+    options:
+      systemd_dir:
+        description: Path to systemd services
+        default: /etc/systemd/system