add kube-vip as a service load balancer (#432)

* add kube-vip as a service load balancer

* add molecule scenario kube-vip

---------

Co-authored-by: Techno Tim <timothystewart6@gmail.com>

.github/ISSUE_TEMPLATE.md

@@ -51,6 +51,9 @@ extra_agent_args: ""
 
 kube_vip_tag_version: ""
 
+kube_vip_cloud_provider_tag_version: ""
+kube_vip_lb_ip_range: ""
+
 metal_lb_speaker_tag_version: ""
 metal_lb_controller_tag_version: ""
 

.github/workflows/cache.yml

@@ -11,12 +11,12 @@ jobs:
 
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v3 2.5.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 4.1.1
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # 2.3.3
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # 5.0.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies

.github/workflows/ci.yml

@@ -6,7 +6,16 @@ on:
       - opened
       - synchronize
     paths-ignore:
-      - '**/README.md'
+      - '**/.gitignore'
+      - '**/FUNDING.yml'
+      - '**/host.ini'
+      - '**/*.md'
+      - '**/.editorconfig'
+      - '**/ansible.example.cfg'
+      - '**/deploy.sh'
+      - '**/LICENSE'
+      - '**/reboot.sh'
+      - '**/reset.sh'
 jobs:
   pre:
     uses: ./.github/workflows/cache.yml

.github/workflows/lint.yml

@@ -11,12 +11,12 @@ jobs:
 
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v3 2.5.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 4.1.1
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # 2.3.3
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # 5.0.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies
@@ -45,9 +45,9 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Checkout code
-        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v3 2.5.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 4.1.1
       - name: Ensure SHA pinned actions
-        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@af2eb3226618e2494e3d9084f515ad6dcf16e229 # 2.0.1
+        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@ba37328d4ea95eaf8b3bd6c6cef308f709a5f2ec # 3.0.3
         with:
           allowlist: |
             aws-actions/

.github/workflows/test.yml

@@ -13,13 +13,14 @@ jobs:
           - ipv6
           - single_node
           - calico
+          - kube-vip
       fail-fast: false
     env:
       PYTHON_VERSION: "3.11"
 
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v3 2.5.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 4.1.1
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
@@ -57,7 +58,7 @@ jobs:
           EOF
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # 2.3.3
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # 5.0.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies
@@ -116,8 +117,9 @@ jobs:
 
       - name: Upload log files
         if: always() # do this even if a step before has failed
-        uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # 3.1.1
+        uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # 4.3.0
         with:
           name: logs
           path: |
             ${{ runner.temp }}/logs
+          overwrite: true

.gitignore

@@ -1,3 +1,4 @@
 .env/
 *.log
 ansible.cfg
+kubeconfig

inventory/sample/group_vars/all.yml

@@ -56,6 +56,13 @@ extra_agent_args: >-
 # image tag for kube-vip
 kube_vip_tag_version: "v0.6.4"
 
+# tag for kube-vip-cloud-provider manifest
+# kube_vip_cloud_provider_tag_version: "main"
+
+# kube-vip ip range for load balancer
+# (uncomment to use kube-vip for services instead of MetalLB)
+# kube_vip_lb_ip_range: "192.168.30.80-192.168.30.90"
+
 # metallb type frr or native
 metal_lb_type: "native"
 

molecule/README.md

@@ -15,6 +15,8 @@ We have these scenarios:
   Very similar to the default scenario, but uses only a single node for all cluster functionality.
 - **calico**:
   The same as single node, but uses calico cni instead of flannel.
+- **kube-vip**
+  The same as single node, but uses kube-vip as service loadbalancer instead of MetalLB
 
 ## How to execute
 

molecule/kube-vip/molecule.yml

@@ -0,0 +1,49 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: vagrant
+platforms:
+  - name: control1
+    box: generic/ubuntu2204
+    memory: 4096
+    cpus: 4
+    config_options:
+      # We currently can not use public-key based authentication on Ubuntu 22.04,
+      # see: https://github.com/chef/bento/issues/1405
+      ssh.username: "vagrant"
+      ssh.password: "vagrant"
+    groups:
+      - k3s_cluster
+      - master
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.62
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_VERBOSITY: 1
+  playbooks:
+    converge: ../resources/converge.yml
+    side_effect: ../resources/reset.yml
+    verify: ../resources/verify.yml
+  inventory:
+    links:
+      group_vars: ../../inventory/sample/group_vars
+scenario:
+  test_sequence:
+    - dependency
+    - cleanup
+    - destroy
+    - syntax
+    - create
+    - prepare
+    - converge
+    # idempotence is not possible with the playbook in its current form.
+    - verify
+    # We are repurposing side_effect here to test the reset playbook.
+    # This is why we do not run it before verify (which tests the cluster),
+    # but after the verify step.
+    - side_effect
+    - cleanup
+    - destroy

molecule/kube-vip/overrides.yml

@@ -0,0 +1,17 @@
+---
+- name: Apply overrides
+  hosts: all
+  tasks:
+    - name: Override host variables
+      ansible.builtin.set_fact:
+        # See:
+        # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
+        flannel_iface: eth1
+
+        # The test VMs might be a bit slow, so we give them more time to join the cluster:
+        retry_count: 45
+
+        # Make sure that our IP ranges do not collide with those of the other scenarios
+        apiserver_endpoint: "192.168.30.225"
+        # Use kube-vip instead of MetalLB
+        kube_vip_lb_ip_range: "192.168.30.110-192.168.30.119"

roles/k3s_server/tasks/kube-vip.yml

@@ -0,0 +1,27 @@
+---
+- name: Create manifests directory on first master
+  file:
+    path: /var/lib/rancher/k3s/server/manifests
+    state: directory
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+
+- name: Download vip cloud provider manifest to first master
+  ansible.builtin.get_url:
+    url: "https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/{{ kube_vip_cloud_provider_tag_version | default('main') }}/manifest/kube-vip-cloud-controller.yaml"  # noqa yaml[line-length]
+    dest: "/var/lib/rancher/k3s/server/manifests/kube-vip-cloud-controller.yaml"
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+
+- name: Copy kubevip configMap manifest to first master
+  template:
+    src: "kubevip.yaml.j2"
+    dest: "/var/lib/rancher/k3s/server/manifests/kubevip.yaml"
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']

roles/k3s_server/tasks/main.yml

@@ -29,6 +29,12 @@
 - name: Deploy metallb manifest
   include_tasks: metallb.yml
   tags: metallb
+  when: kube_vip_lb_ip_range is not defined
+
+- name: Deploy kube-vip manifest
+  include_tasks: kube-vip.yml
+  tags: kubevip
+  when: kube_vip_lb_ip_range is defined
 
 - name: Init cluster inside the transient k3s-init service
   command:

roles/k3s_server/templates/kubevip.yaml.j2

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kubevip
+  namespace: kube-system
+data:
+{% if kube_vip_lb_ip_range is string %}
+{# kube_vip_lb_ip_range was used in the legacy way: single string instead of a list #}
+{#   => transform to list with single element #}
+{% set kube_vip_lb_ip_range = [kube_vip_lb_ip_range] %}
+{% endif %}
+  range-global: {{ kube_vip_lb_ip_range | join(',') }}

roles/k3s_server/templates/vip.yaml.j2

@@ -43,7 +43,7 @@ spec:
         - name: vip_ddns
           value: "false"
         - name: svc_enable
-          value: "false"
+          value: "{{ 'true' if kube_vip_lb_ip_range is defined else 'false' }}"
         - name: vip_leaderelection
           value: "true"
         - name: vip_leaseduration

roles/k3s_server_post/tasks/main.yml

@@ -7,6 +7,7 @@
 - name: Deploy metallb pool
   include_tasks: metallb.yml
   tags: metallb
+  when: kube_vip_lb_ip_range is not defined
 
 - name: Remove tmp directory used for manifests
   file:

roles/k3s_server_post/tasks/metallb.yml

@@ -8,6 +8,27 @@
   with_items: "{{ groups[group_name_master | default('master')] }}"
   run_once: true
 
+- name: Delete outdated metallb replicas
+  shell: |-
+    set -o pipefail
+
+    REPLICAS=$(k3s kubectl --namespace='metallb-system' get replicasets \
+      -l 'component=controller,app=metallb' \
+      -o jsonpath='{.items[0].spec.template.spec.containers[0].image}, {.items[0].metadata.name}' 2>/dev/null || true)
+    REPLICAS_SETS=$(echo ${REPLICAS} | grep -v '{{ metal_lb_controller_tag_version }}' | sed -e "s/^.*\s//g")
+    if [ -n "${REPLICAS_SETS}" ] ; then
+      for REPLICAS in "${REPLICAS_SETS}"
+      do
+        k3s kubectl --namespace='metallb-system' \
+          delete rs "${REPLICAS}"
+      done
+    fi
+  args:
+    executable: /bin/bash
+  changed_when: false
+  run_once: true
+  with_items: "{{ groups[group_name_master | default('master')] }}"
+
 - name: Copy metallb CRs manifest to first master
   template:
     src: "metallb.crs.j2"

site.yml

@@ -46,3 +46,14 @@
   roles:
     - role: k3s_server_post
       become: true
+
+- name: Storing kubeconfig in the playbook directory
+  hosts: master
+  environment: "{{ proxy_env | default({}) }}"
+  tasks:
+    - name: Copying kubeconfig from {{ hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] }}
+      ansible.builtin.fetch:
+        src: "{{ ansible_user_dir }}/.kube/config"
+        dest: ./kubeconfig
+        flat: true
+      when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']