add kube-vip as a service load balancer (#432)

* add kube-vip as a service load balancer

* add molecule scenario kube-vip

---------

Co-authored-by: Techno Tim <timothystewart6@gmail.com>

.github/ISSUE_TEMPLATE.md

@@ -35,7 +35,12 @@ k3s_version: ""
 ansible_user: NA
 systemd_dir: ""
 
-container_iface: ""
+flannel_iface: ""
+
+#calico_iface: ""
+calico_ebpf: ""
+calico_cidr: ""
+calico_tag: ""
 
 apiserver_endpoint: ""
 
@@ -46,6 +51,9 @@ extra_agent_args: ""
 
 kube_vip_tag_version: ""
 
+kube_vip_cloud_provider_tag_version: ""
+kube_vip_lb_ip_range: ""
+
 metal_lb_speaker_tag_version: ""
 metal_lb_controller_tag_version: ""
 

.github/dependabot.yml

@@ -9,3 +9,18 @@ updates:
     ignore:
       - dependency-name: "*"
         update-types: ["version-update:semver-major"]
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    rebase-strategy: "auto"
+
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    rebase-strategy: "auto"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]

.github/workflows/cache.yml

@@ -0,0 +1,42 @@
+---
+name: "Cache"
+on:
+  workflow_call:
+jobs:
+  molecule:
+    name: cache
+    runs-on: self-hosted
+    env:
+      PYTHON_VERSION: "3.11"
+
+    steps:
+      - name: Check out the codebase
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 4.1.1
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # 5.0.0
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip' # caching pip dependencies
+
+      - name: Cache Vagrant boxes
+        id: cache-vagrant
+        uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # 4.0
+        with:
+          lookup-only: true #if it exists, we don't need to restore and can skip the next step
+          path: |
+            ~/.vagrant.d/boxes
+          key: vagrant-boxes-${{ hashFiles('**/molecule.yml') }}
+          restore-keys: |
+            vagrant-boxes
+
+      - name: Download Vagrant boxes for all scenarios
+        # To save some cache space, all scenarios share the same cache key.
+        # On the other hand, this means that the cache contents should be
+        # the same across all scenarios. This step ensures that.
+        if: steps.cache-vagrant.outputs.cache-hit != 'true' # only run if false since this is just a cache step
+        run: |
+          ./.github/download-boxes.sh
+          vagrant box list

.github/workflows/ci.yml

@@ -2,14 +2,26 @@
 name: "CI"
 on:
   pull_request:
-  push:
+    types:
-    branches:
+      - opened
-      - master
+      - synchronize
     paths-ignore:
-      - '**/README.md'
+      - '**/.gitignore'
+      - '**/FUNDING.yml'
+      - '**/host.ini'
+      - '**/*.md'
+      - '**/.editorconfig'
+      - '**/ansible.example.cfg'
+      - '**/deploy.sh'
+      - '**/LICENSE'
+      - '**/reboot.sh'
+      - '**/reset.sh'
 jobs:
+  pre:
+    uses: ./.github/workflows/cache.yml
   lint:
     uses: ./.github/workflows/lint.yml
+    needs: [pre]
   test:
     uses: ./.github/workflows/test.yml
-    needs: [lint]
+    needs: [pre, lint]

.github/workflows/lint.yml

@@ -5,37 +5,27 @@ on:
 jobs:
   pre-commit-ci:
     name: Pre-Commit
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     env:
       PYTHON_VERSION: "3.11"
 
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v3 2.5.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 4.1.1
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # 2.3.3
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # 5.0.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies
 
-      - name: Cache pip
+      - name: Restore Ansible cache
-        uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # 3.0.11
+        uses: actions/cache/restore@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # 4.0
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('./requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-
-      - name: Cache Ansible
-        uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # 3.0.11
         with:
           path: ~/.ansible/collections
-          key: ${{ runner.os }}-ansible-${{ hashFiles('collections/requirements.txt') }}
+          key: ansible-${{ hashFiles('collections/requirements.yml') }}
-          restore-keys: |
-            ${{ runner.os }}-ansible-
 
       - name: Install dependencies
         run: |
@@ -47,21 +37,17 @@ jobs:
           python3 -m pip install -r requirements.txt
           echo "::endgroup::"
 
-          echo "::group::Install Ansible role requirements from collections/requirements.yml"
-          ansible-galaxy install -r collections/requirements.yml
-          echo "::endgroup::"
-
       - name: Run pre-commit
         uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507 # 3.0.0
 
   ensure-pinned-actions:
     name: Ensure SHA Pinned Actions
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     steps:
       - name: Checkout code
-        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v3 2.5.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 4.1.1
       - name: Ensure SHA pinned actions
-        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@af2eb3226618e2494e3d9084f515ad6dcf16e229 # 2.0.1
+        uses: zgosalvez/github-actions-ensure-sha-pinned-actions@ba37328d4ea95eaf8b3bd6c6cef308f709a5f2ec # 3.0.3
         with:
           allowlist: |
             aws-actions/

.github/workflows/test.yml

@@ -5,23 +5,50 @@ on:
 jobs:
   molecule:
     name: Molecule
-    runs-on: macos-12
+    runs-on: self-hosted
     strategy:
       matrix:
         scenario:
           - default
           - ipv6
           - single_node
+          - calico
+          - kube-vip
       fail-fast: false
     env:
       PYTHON_VERSION: "3.11"
 
     steps:
       - name: Check out the codebase
-        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v3 2.5.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 4.1.1
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
+      # these steps are necessary if not using ephemeral nodes
+      - name: Delete old Vagrant box versions
+        if: always() # do this even if a step before has failed
+        run: vagrant box prune --force
+
+      - name: Remove all local Vagrant boxes
+        if: always() # do this even if a step before has failed
+        run: if vagrant box list 2>/dev/null; then vagrant box list | cut -f 1 -d ' ' | xargs -L 1 vagrant box remove -f 2>/dev/null && echo "All Vagrant boxes removed." || echo "No Vagrant boxes found."; else echo "No Vagrant boxes found."; fi
+
+      - name: Remove all Virtualbox VMs
+        if: always() # do this even if a step before has failed
+        run: VBoxManage list vms | awk -F'"' '{print $2}' | xargs -I {} VBoxManage unregistervm --delete "{}"
+
+      - name: Remove all Virtualbox HDs
+        if: always() # do this even if a step before has failed
+        run: VBoxManage list hdds | awk -F':' '/^UUID:/ {print $2}' | xargs -I {} VBoxManage closemedium disk "{}" --delete
+
+      - name: Remove all Virtualbox Networks
+        if: always() # do this even if a step before has failed
+        run: VBoxManage list hostonlyifs | grep '^Name:' | awk '{print $2}' | grep '^vboxnet' | xargs -I {} VBoxManage hostonlyif remove {}
+
+      - name: Remove Virtualbox network config
+        if: always() # do this even if a step before has failed
+        run: sudo rm /etc/vbox/networks.conf || true
+
       - name: Configure VirtualBox
         run: |-
           sudo mkdir -p /etc/vbox
@@ -30,35 +57,19 @@ jobs:
           * fdad:bad:ba55::/64
           EOF
 
-      - name: Cache pip
-        uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # 3.0.11
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('./requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-
-      - name: Cache Vagrant boxes
-        uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # 3.0.11
-        with:
-          path: |
-            ~/.vagrant.d/boxes
-          key: vagrant-boxes-${{ hashFiles('**/molecule.yml') }}
-          restore-keys: |
-            vagrant-boxes
-
-      - name: Download Vagrant boxes for all scenarios
-        # To save some cache space, all scenarios share the same cache key.
-        # On the other hand, this means that the cache contents should be
-        # the same across all scenarios. This step ensures that.
-        run: ./.github/download-boxes.sh
-
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # 2.3.3
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # 5.0.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip' # caching pip dependencies
 
+      - name: Restore vagrant Boxes cache
+        uses: actions/cache/restore@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # 4.0
+        with:
+          path: ~/.vagrant.d/boxes
+          key: vagrant-boxes-${{ hashFiles('**/molecule.yml') }}
+          fail-on-cache-miss: true
+
       - name: Install dependencies
         run: |
           echo "::group::Upgrade pip"
@@ -75,18 +86,40 @@ jobs:
         env:
           ANSIBLE_K3S_LOG_DIR: ${{ runner.temp }}/logs/k3s-ansible/${{ matrix.scenario }}
           ANSIBLE_SSH_RETRIES: 4
-          ANSIBLE_TIMEOUT: 60
+          ANSIBLE_TIMEOUT: 120
           PY_COLORS: 1
           ANSIBLE_FORCE_COLOR: 1
 
+      # these steps are necessary if not using ephemeral nodes
+      - name: Delete old Vagrant box versions
+        if: always() # do this even if a step before has failed
+        run: vagrant box prune --force
+
+      - name: Remove all local Vagrant boxes
+        if: always() # do this even if a step before has failed
+        run: if vagrant box list 2>/dev/null; then vagrant box list | cut -f 1 -d ' ' | xargs -L 1 vagrant box remove -f 2>/dev/null && echo "All Vagrant boxes removed." || echo "No Vagrant boxes found."; else echo "No Vagrant boxes found."; fi
+
+      - name: Remove all Virtualbox VMs
+        if: always() # do this even if a step before has failed
+        run: VBoxManage list vms | awk -F'"' '{print $2}' | xargs -I {} VBoxManage unregistervm --delete "{}"
+
+      - name: Remove all Virtualbox HDs
+        if: always() # do this even if a step before has failed
+        run: VBoxManage list hdds | awk -F':' '/^UUID:/ {print $2}' | xargs -I {} VBoxManage closemedium disk "{}" --delete
+
+      - name: Remove all Virtualbox Networks
+        if: always() # do this even if a step before has failed
+        run: VBoxManage list hostonlyifs | grep '^Name:' | awk '{print $2}' | grep '^vboxnet' | xargs -I {} VBoxManage hostonlyif remove {}
+
+      - name: Remove Virtualbox network config
+        if: always() # do this even if a step before has failed
+        run: sudo rm /etc/vbox/networks.conf || true
+
       - name: Upload log files
         if: always() # do this even if a step before has failed
-        uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # 3.1.1
+        uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # 4.3.0
         with:
           name: logs
           path: |
             ${{ runner.temp }}/logs
-
+          overwrite: true
-      - name: Delete old box versions
-        if: always() # do this even if a step before has failed
-        run: vagrant box prune --force

.gitignore

@@ -1,3 +1,4 @@
 .env/
 *.log
 ansible.cfg
+kubeconfig

inventory/sample/group_vars/all.yml

@@ -1,5 +1,5 @@
 ---
-k3s_version: v1.25.16+k3s4
+k3s_version: v1.29.0+k3s1
 # this is the user that has ssh access to these machines
 ansible_user: ansibleuser
 systemd_dir: /etc/systemd/system
@@ -7,15 +7,14 @@ systemd_dir: /etc/systemd/system
 # Set your timezone
 system_timezone: "Your/Timezone"
 
-# node interface which will be used for the container network interface (flannel or calico)
+# interface which will be used for flannel
-container_iface: "eth0"
+flannel_iface: "eth0"
 
-# set use_calico to true to use tigera operator/calico instead of the default CNI flannel
+# uncomment calico_iface to use tigera operator/calico cni instead of flannel https://docs.tigera.io/calico/latest/about
-# install reference: https://docs.tigera.io/calico/latest/getting-started/kubernetes/k3s/multi-node-install#install-calico
+# calico_iface: "eth0"
-use_calico: false
+calico_ebpf: false           # use eBPF dataplane instead of iptables
-calico_ebpf: false # use eBPF dataplane instead of iptables https://docs.tigera.io/calico/latest/operations/ebpf
+calico_cidr: "10.52.0.0/16"  # calico cluster pod cidr pool
-calico_cidr: "10.52.0.0/16" # pod cidr pool
+calico_tag: "v3.27.0"        # calico version tag
-calico_tag: "v3.27.0" # calico version tag
 
 # apiserver_endpoint is virtual ip-address which will be configured on each master
 apiserver_endpoint: "192.168.30.222"
@@ -27,14 +26,14 @@ k3s_token: "some-SUPER-DEDEUPER-secret-password"
 # The IP on which the node is reachable in the cluster.
 # Here, a sensible default is provided, you can still override
 # it for each of your hosts, though.
-k3s_node_ip: '{{ ansible_facts[container_iface]["ipv4"]["address"] }}'
+k3s_node_ip: "{{ ansible_facts[(calico_iface | default(flannel_iface))]['ipv4']['address'] }}"
 
 # Disable the taint manually by setting: k3s_master_taint = false
 k3s_master_taint: "{{ true if groups['node'] | default([]) | length >= 1 else false }}"
 
 # these arguments are recommended for servers as well as agents:
 extra_args: >-
-  {{ '--flannel-iface=' + container_iface if not use_calico else '' }}
+  {{ '--flannel-iface=' + flannel_iface if calico_iface is not defined else '' }}
   --node-ip={{ k3s_node_ip }}
 
 # change these to your liking, the only required are: --disable servicelb, --tls-san {{ apiserver_endpoint }}
@@ -42,10 +41,10 @@ extra_args: >-
 extra_server_args: >-
   {{ extra_args }}
   {{ '--node-taint node-role.kubernetes.io/master=true:NoSchedule' if k3s_master_taint else '' }}
-  {% if use_calico %}
+  {% if calico_iface is defined %}
   --flannel-backend=none
   --disable-network-policy
-  --cluster-cidr={{ calico_cidr }}
+  --cluster-cidr={{ calico_cidr | default('10.52.0.0/16') }}
   {% endif %}
   --tls-san {{ apiserver_endpoint }}
   --disable servicelb
@@ -55,7 +54,14 @@ extra_agent_args: >-
   {{ extra_args }}
 
 # image tag for kube-vip
-kube_vip_tag_version: "v0.5.12"
+kube_vip_tag_version: "v0.6.4"
+
+# tag for kube-vip-cloud-provider manifest
+# kube_vip_cloud_provider_tag_version: "main"
+
+# kube-vip ip range for load balancer
+# (uncomment to use kube-vip for services instead of MetalLB)
+# kube_vip_lb_ip_range: "192.168.30.80-192.168.30.90"
 
 # metallb type frr or native
 metal_lb_type: "native"
@@ -69,8 +75,8 @@ metal_lb_mode: "layer2"
 # metal_lb_bgp_peer_address: "192.168.30.1"
 
 # image tag for metal lb
-metal_lb_speaker_tag_version: "v0.13.9"
+metal_lb_speaker_tag_version: "v0.13.12"
-metal_lb_controller_tag_version: "v0.13.9"
+metal_lb_controller_tag_version: "v0.13.12"
 
 # metallb ip range for load balancer
 metal_lb_ip_range: "192.168.30.80-192.168.30.90"
@@ -80,9 +86,9 @@ metal_lb_ip_range: "192.168.30.80-192.168.30.90"
 # Please read https://gist.github.com/triangletodd/02f595cd4c0dc9aac5f7763ca2264185 before using this.
 # Most notably, your containers must be privileged, and must not have nesting set to true.
 # Please note this script disables most of the security of lxc containers, with the trade off being that lxc
-# containers are significantly more resource efficent compared to full VMs.
+# containers are significantly more resource efficient compared to full VMs.
 # Mixing and matching VMs and lxc containers is not supported, ymmv if you want to do this.
-# I would only really recommend using this if you have partiularly low powered proxmox nodes where the overhead of
+# I would only really recommend using this if you have particularly low powered proxmox nodes where the overhead of
 # VMs would use a significant portion of your available resources.
 proxmox_lxc_configure: false
 # the user that you would use to ssh into the host, for example if you run ssh some-user@my-proxmox-host,

molecule/README.md

@@ -13,6 +13,10 @@ We have these scenarios:
   To save a bit of test time, this cluster is _not_ highly available, it consists of only one control and one worker node.
 - **single_node**:
   Very similar to the default scenario, but uses only a single node for all cluster functionality.
+- **calico**:
+  The same as single node, but uses calico cni instead of flannel.
+- **kube-vip**
+  The same as single node, but uses kube-vip as service loadbalancer instead of MetalLB
 
 ## How to execute
 

molecule/calico/molecule.yml

@@ -0,0 +1,49 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: vagrant
+platforms:
+  - name: control1
+    box: generic/ubuntu2204
+    memory: 4096
+    cpus: 4
+    config_options:
+      # We currently can not use public-key based authentication on Ubuntu 22.04,
+      # see: https://github.com/chef/bento/issues/1405
+      ssh.username: "vagrant"
+      ssh.password: "vagrant"
+    groups:
+      - k3s_cluster
+      - master
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.62
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_VERBOSITY: 1
+  playbooks:
+    converge: ../resources/converge.yml
+    side_effect: ../resources/reset.yml
+    verify: ../resources/verify.yml
+  inventory:
+    links:
+      group_vars: ../../inventory/sample/group_vars
+scenario:
+  test_sequence:
+    - dependency
+    - cleanup
+    - destroy
+    - syntax
+    - create
+    - prepare
+    - converge
+    # idempotence is not possible with the playbook in its current form.
+    - verify
+    # We are repurposing side_effect here to test the reset playbook.
+    # This is why we do not run it before verify (which tests the cluster),
+    # but after the verify step.
+    - side_effect
+    - cleanup
+    - destroy

molecule/calico/overrides.yml

@@ -0,0 +1,16 @@
+---
+- name: Apply overrides
+  hosts: all
+  tasks:
+    - name: Override host variables
+      ansible.builtin.set_fact:
+        # See:
+        # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
+        calico_iface: eth1
+
+        # The test VMs might be a bit slow, so we give them more time to join the cluster:
+        retry_count: 45
+
+        # Make sure that our IP ranges do not collide with those of the other scenarios
+        apiserver_endpoint: "192.168.30.224"
+        metal_lb_ip_range: "192.168.30.100-192.168.30.109"

molecule/default/molecule.yml

@@ -7,7 +7,7 @@ platforms:
 
   - name: control1
     box: generic/ubuntu2204
-    memory: 2048
+    memory: 1024
     cpus: 2
     groups:
       - k3s_cluster
@@ -22,8 +22,8 @@ platforms:
       ssh.password: "vagrant"
 
   - name: control2
-    box: generic/debian11
+    box: generic/debian12
-    memory: 2048
+    memory: 1024
     cpus: 2
     groups:
       - k3s_cluster
@@ -34,7 +34,7 @@ platforms:
 
   - name: control3
     box: generic/rocky9
-    memory: 2048
+    memory: 1024
     cpus: 2
     groups:
       - k3s_cluster
@@ -45,7 +45,7 @@ platforms:
 
   - name: node1
     box: generic/ubuntu2204
-    memory: 2048
+    memory: 1024
     cpus: 2
     groups:
       - k3s_cluster
@@ -61,7 +61,7 @@ platforms:
 
   - name: node2
     box: generic/rocky9
-    memory: 2048
+    memory: 1024
     cpus: 2
     groups:
       - k3s_cluster
@@ -72,6 +72,8 @@ platforms:
 
 provisioner:
   name: ansible
+  env:
+    ANSIBLE_VERBOSITY: 1
   playbooks:
     converge: ../resources/converge.yml
     side_effect: ../resources/reset.yml
@@ -82,7 +84,6 @@ provisioner:
 scenario:
   test_sequence:
     - dependency
-    - lint
     - cleanup
     - destroy
     - syntax

molecule/default/overrides.yml

@@ -6,7 +6,7 @@
       ansible.builtin.set_fact:
         # See:
         # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
-        container_iface: eth1
+        flannel_iface: eth1
 
         # The test VMs might be a bit slow, so we give them more time to join the cluster:
         retry_count: 45

molecule/ipv6/molecule.yml

@@ -6,7 +6,7 @@ driver:
 platforms:
   - name: control1
     box: generic/ubuntu2204
-    memory: 2048
+    memory: 1024
     cpus: 2
     groups:
       - k3s_cluster
@@ -22,7 +22,7 @@ platforms:
 
   - name: control2
     box: generic/ubuntu2204
-    memory: 2048
+    memory: 1024
     cpus: 2
     groups:
       - k3s_cluster
@@ -38,7 +38,7 @@ platforms:
 
   - name: node1
     box: generic/ubuntu2204
-    memory: 2048
+    memory: 1024
     cpus: 2
     groups:
       - k3s_cluster
@@ -53,6 +53,8 @@ platforms:
       ssh.password: "vagrant"
 provisioner:
   name: ansible
+  env:
+    ANSIBLE_VERBOSITY: 1
   playbooks:
     converge: ../resources/converge.yml
     side_effect: ../resources/reset.yml
@@ -63,7 +65,6 @@ provisioner:
 scenario:
   test_sequence:
     - dependency
-    - lint
     - cleanup
     - destroy
     - syntax

molecule/ipv6/overrides.yml

@@ -6,7 +6,7 @@
       ansible.builtin.set_fact:
         # See:
         # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
-        container_iface: eth1
+        flannel_iface: eth1
 
         # In this scenario, we have multiple interfaces that the VIP could be
         # broadcasted on. Since we have assigned a dedicated private network
@@ -27,13 +27,13 @@
           - fdad:bad:ba55::1b:0/112
           - 192.168.123.80-192.168.123.90
 
-        # k3s_node_ip is by default set to the IPv4 address of container_iface.
+        # k3s_node_ip is by default set to the IPv4 address of flannel_iface.
         # We want IPv6 addresses here of course, so we just specify them
         # manually below.
         k3s_node_ip: "{{ node_ipv4 }},{{ node_ipv6 }}"
 
     - name: Override host variables (2/2)
-      # Since "extra_args" depends on "k3s_node_ip" and "container_iface" we have
+      # Since "extra_args" depends on "k3s_node_ip" and "flannel_iface" we have
       # to set this AFTER overriding the both of them.
       ansible.builtin.set_fact:
         # A few extra server args are necessary:

molecule/ipv6/prepare.yml

@@ -30,7 +30,7 @@
         name: net.ipv6.conf.{{ item }}.accept_dad
         value: "0"
       with_items:
-        - "{{ container_iface }}"
+        - "{{ flannel_iface }}"
 
     - name: Write IPv4 configuration
       ansible.builtin.template:

molecule/ipv6/templates/55-flannel-ipv4.yaml.j2

@@ -3,6 +3,6 @@ network:
   version: 2
   renderer: networkd
   ethernets:
-    {{ container_iface }}:
+    {{ flannel_iface }}:
       addresses:
       - {{ node_ipv4 }}/24

molecule/kube-vip/molecule.yml

@@ -0,0 +1,49 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: vagrant
+platforms:
+  - name: control1
+    box: generic/ubuntu2204
+    memory: 4096
+    cpus: 4
+    config_options:
+      # We currently can not use public-key based authentication on Ubuntu 22.04,
+      # see: https://github.com/chef/bento/issues/1405
+      ssh.username: "vagrant"
+      ssh.password: "vagrant"
+    groups:
+      - k3s_cluster
+      - master
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.62
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_VERBOSITY: 1
+  playbooks:
+    converge: ../resources/converge.yml
+    side_effect: ../resources/reset.yml
+    verify: ../resources/verify.yml
+  inventory:
+    links:
+      group_vars: ../../inventory/sample/group_vars
+scenario:
+  test_sequence:
+    - dependency
+    - cleanup
+    - destroy
+    - syntax
+    - create
+    - prepare
+    - converge
+    # idempotence is not possible with the playbook in its current form.
+    - verify
+    # We are repurposing side_effect here to test the reset playbook.
+    # This is why we do not run it before verify (which tests the cluster),
+    # but after the verify step.
+    - side_effect
+    - cleanup
+    - destroy

molecule/kube-vip/overrides.yml

@@ -0,0 +1,17 @@
+---
+- name: Apply overrides
+  hosts: all
+  tasks:
+    - name: Override host variables
+      ansible.builtin.set_fact:
+        # See:
+        # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
+        flannel_iface: eth1
+
+        # The test VMs might be a bit slow, so we give them more time to join the cluster:
+        retry_count: 45
+
+        # Make sure that our IP ranges do not collide with those of the other scenarios
+        apiserver_endpoint: "192.168.30.225"
+        # Use kube-vip instead of MetalLB
+        kube_vip_lb_ip_range: "192.168.30.110-192.168.30.119"

molecule/single_node/molecule.yml

@@ -21,6 +21,8 @@ platforms:
         ip: 192.168.30.50
 provisioner:
   name: ansible
+  env:
+    ANSIBLE_VERBOSITY: 1
   playbooks:
     converge: ../resources/converge.yml
     side_effect: ../resources/reset.yml
@@ -31,7 +33,6 @@ provisioner:
 scenario:
   test_sequence:
     - dependency
-    - lint
     - cleanup
     - destroy
     - syntax

molecule/single_node/overrides.yml

@@ -6,7 +6,7 @@
       ansible.builtin.set_fact:
         # See:
         # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
-        container_iface: eth1
+        flannel_iface: eth1
 
         # The test VMs might be a bit slow, so we give them more time to join the cluster:
         retry_count: 45

requirements.in

@@ -1,10 +1,10 @@
-ansible-core>=2.13.5
+ansible-core>=2.16.2
 jmespath>=1.0.1
-jsonpatch>=1.32
+jsonpatch>=1.33
-kubernetes>=25.3.0
+kubernetes>=29.0.0
-molecule-vagrant>=1.0.0
+molecule-plugins[vagrant]
-molecule>=4.0.3
+molecule>=6.0.3
-netaddr>=0.8.0
+netaddr>=0.10.1
-pre-commit>=2.20.0
+pre-commit>=3.6.0
-pre-commit-hooks>=1.3.1
+pre-commit-hooks>=4.5.0
-pyyaml>=6.0
+pyyaml>=6.0.1

requirements.txt

@@ -4,174 +4,165 @@
 #
 #    pip-compile requirements.in
 #
-ansible-compat==3.0.1
+ansible-compat==4.1.11
     # via molecule
-ansible-core==2.15.4
+ansible-core==2.16.2
     # via
     #   -r requirements.in
     #   ansible-compat
-arrow==1.2.3
+    #   molecule
-    # via jinja2-time
+attrs==23.2.0
-attrs==22.1.0
+    # via
-    # via jsonschema
+    #   jsonschema
-binaryornot==0.4.4
+    #   referencing
-    # via cookiecutter
+bracex==2.4
-cachetools==5.2.0
+    # via wcmatch
+cachetools==5.3.2
     # via google-auth
-certifi==2022.9.24
+certifi==2023.11.17
     # via
     #   kubernetes
     #   requests
-cffi==1.15.1
+cffi==1.16.0
     # via cryptography
-cfgv==3.3.1
+cfgv==3.4.0
     # via pre-commit
-chardet==5.0.0
+charset-normalizer==3.3.2
-    # via binaryornot
-charset-normalizer==2.1.1
     # via requests
-click==8.1.3
+click==8.1.7
     # via
     #   click-help-colors
-    #   cookiecutter
     #   molecule
-click-help-colors==0.9.1
+click-help-colors==0.9.4
     # via molecule
-commonmark==0.9.1
+cryptography==41.0.7
-    # via rich
-cookiecutter==2.1.1
-    # via molecule
-cryptography==38.0.3
     # via ansible-core
-distlib==0.3.6
+distlib==0.3.8
     # via virtualenv
-distro==1.8.0
-    # via selinux
 enrich==1.2.7
     # via molecule
-filelock==3.8.0
+filelock==3.13.1
     # via virtualenv
-google-auth==2.14.0
+google-auth==2.26.2
     # via kubernetes
-identify==2.5.8
+identify==2.5.33
     # via pre-commit
-idna==3.4
+idna==3.6
     # via requests
-jinja2==3.1.2
+jinja2==3.1.3
     # via
     #   ansible-core
-    #   cookiecutter
-    #   jinja2-time
     #   molecule
-    #   molecule-vagrant
-jinja2-time==0.2.0
-    # via cookiecutter
 jmespath==1.0.1
     # via -r requirements.in
 jsonpatch==1.33
     # via -r requirements.in
-jsonpointer==2.3
+jsonpointer==2.4
     # via jsonpatch
-jsonschema==4.17.0
+jsonschema==4.21.1
     # via
     #   ansible-compat
     #   molecule
-kubernetes==25.3.0
+jsonschema-specifications==2023.12.1
+    # via jsonschema
+kubernetes==29.0.0
     # via -r requirements.in
-markupsafe==2.1.1
+markdown-it-py==3.0.0
+    # via rich
+markupsafe==2.1.4
     # via jinja2
-molecule==4.0.4
+mdurl==0.1.2
+    # via markdown-it-py
+molecule==6.0.3
     # via
     #   -r requirements.in
-    #   molecule-vagrant
+    #   molecule-plugins
-molecule-vagrant==1.0.0
+molecule-plugins[vagrant]==23.5.0
     # via -r requirements.in
-netaddr==0.10.0
+netaddr==0.10.1
     # via -r requirements.in
-nodeenv==1.7.0
+nodeenv==1.8.0
     # via pre-commit
 oauthlib==3.2.2
-    # via requests-oauthlib
+    # via
-packaging==21.3
+    #   kubernetes
+    #   requests-oauthlib
+packaging==23.2
     # via
     #   ansible-compat
     #   ansible-core
     #   molecule
-platformdirs==2.5.2
+platformdirs==4.1.0
     # via virtualenv
-pluggy==1.0.0
+pluggy==1.3.0
     # via molecule
-pre-commit==2.21.0
+pre-commit==3.6.0
     # via -r requirements.in
 pre-commit-hooks==4.5.0
     # via -r requirements.in
-pyasn1==0.4.8
+pyasn1==0.5.1
     # via
     #   pyasn1-modules
     #   rsa
-pyasn1-modules==0.2.8
+pyasn1-modules==0.3.0
     # via google-auth
 pycparser==2.21
     # via cffi
-pygments==2.13.0
+pygments==2.17.2
     # via rich
-pyparsing==3.0.9
-    # via packaging
-pyrsistent==0.19.2
-    # via jsonschema
 python-dateutil==2.8.2
-    # via
+    # via kubernetes
-    #   arrow
-    #   kubernetes
-python-slugify==6.1.2
-    # via cookiecutter
 python-vagrant==1.0.0
-    # via molecule-vagrant
+    # via molecule-plugins
 pyyaml==6.0.1
     # via
     #   -r requirements.in
     #   ansible-compat
     #   ansible-core
-    #   cookiecutter
     #   kubernetes
     #   molecule
-    #   molecule-vagrant
     #   pre-commit
-requests==2.28.1
+referencing==0.32.1
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+requests==2.31.0
     # via
-    #   cookiecutter
     #   kubernetes
     #   requests-oauthlib
 requests-oauthlib==1.3.1
     # via kubernetes
-resolvelib==0.8.1
+resolvelib==1.0.1
     # via ansible-core
-rich==12.6.0
+rich==13.7.0
     # via
     #   enrich
     #   molecule
+rpds-py==0.17.1
+    # via
+    #   jsonschema
+    #   referencing
 rsa==4.9
     # via google-auth
-ruamel-yaml==0.17.21
+ruamel-yaml==0.18.5
     # via pre-commit-hooks
-selinux==0.2.1
+ruamel-yaml-clib==0.2.8
-    # via molecule-vagrant
+    # via ruamel-yaml
 six==1.16.0
     # via
-    #   google-auth
     #   kubernetes
     #   python-dateutil
 subprocess-tee==0.4.1
     # via ansible-compat
-text-unidecode==1.3
+urllib3==2.1.0
-    # via python-slugify
-urllib3==1.26.12
     # via
     #   kubernetes
     #   requests
-virtualenv==20.16.6
+virtualenv==20.25.0
     # via pre-commit
-websocket-client==1.4.2
+wcmatch==8.5
+    # via molecule
+websocket-client==1.7.0
     # via kubernetes
 
 # The following packages are considered to be unsafe in a requirements file:

roles/k3s_agent/tasks/http_proxy.yml

@@ -1,8 +1,8 @@
 ---
 
-- name: Create k3s.service.d directory
+- name: Create k3s-node.service.d directory
   file:
-    path: '{{ systemd_dir }}/k3s.service.d'
+    path: '{{ systemd_dir }}/k3s-node.service.d'
     state: directory
     owner: root
     group: root
@@ -12,7 +12,7 @@
 - name: Copy K3s http_proxy conf file
   template:
     src: "http_proxy.conf.j2"
-    dest: "{{ systemd_dir }}/k3s.service.d/http_proxy.conf"
+    dest: "{{ systemd_dir }}/k3s-node.service.d/http_proxy.conf"
     owner: root
     group: root
     mode: '0755'

roles/k3s_server/tasks/kube-vip.yml

@@ -0,0 +1,27 @@
+---
+- name: Create manifests directory on first master
+  file:
+    path: /var/lib/rancher/k3s/server/manifests
+    state: directory
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+
+- name: Download vip cloud provider manifest to first master
+  ansible.builtin.get_url:
+    url: "https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/{{ kube_vip_cloud_provider_tag_version | default('main') }}/manifest/kube-vip-cloud-controller.yaml"  # noqa yaml[line-length]
+    dest: "/var/lib/rancher/k3s/server/manifests/kube-vip-cloud-controller.yaml"
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+
+- name: Copy kubevip configMap manifest to first master
+  template:
+    src: "kubevip.yaml.j2"
+    dest: "/var/lib/rancher/k3s/server/manifests/kubevip.yaml"
+    owner: root
+    group: root
+    mode: 0644
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']

roles/k3s_server/tasks/main.yml

@@ -29,6 +29,12 @@
 - name: Deploy metallb manifest
   include_tasks: metallb.yml
   tags: metallb
+  when: kube_vip_lb_ip_range is not defined
+
+- name: Deploy kube-vip manifest
+  include_tasks: kube-vip.yml
+  tags: kubevip
+  when: kube_vip_lb_ip_range is defined
 
 - name: Init cluster inside the transient k3s-init service
   command:

roles/k3s_server/templates/kubevip.yaml.j2

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kubevip
+  namespace: kube-system
+data:
+{% if kube_vip_lb_ip_range is string %}
+{# kube_vip_lb_ip_range was used in the legacy way: single string instead of a list #}
+{#   => transform to list with single element #}
+{% set kube_vip_lb_ip_range = [kube_vip_lb_ip_range] %}
+{% endif %}
+  range-global: {{ kube_vip_lb_ip_range | join(',') }}

roles/k3s_server/templates/vip.yaml.j2

@@ -43,7 +43,7 @@ spec:
         - name: vip_ddns
           value: "false"
         - name: svc_enable
-          value: "false"
+          value: "{{ 'true' if kube_vip_lb_ip_range is defined else 'false' }}"
         - name: vip_leaderelection
           value: "true"
         - name: vip_leaseduration

roles/k3s_server_post/defaults/main.yml

@@ -1,6 +1,6 @@
 ---
 # Timeout to wait for MetalLB services to come up
-metal_lb_available_timeout: 120s
+metal_lb_available_timeout: 240s
 
 # Name of the master group
 group_name_master: master

roles/k3s_server_post/tasks/calico.yml

@@ -1,5 +1,8 @@
 ---
-- block:
+- name: Deploy Calico to cluster
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+  run_once: true
+  block:
     - name: Create manifests directory on first master
       file:
         path: /tmp/k3s
@@ -20,6 +23,9 @@
       ansible.builtin.template:
         src: "calico.crs.j2"
         dest: /tmp/k3s/custom-resources.yaml
+        owner: root
+        group: root
+        mode: 0755
 
     - name: Deploy or replace Tigera Operator
       block:
@@ -49,7 +55,7 @@
       retries: 7
       delay: 7
       with_items:
-        - { name: tigera-operator, type: deployment }
+        - {name: tigera-operator, type: deployment}
       loop_control:
         label: "{{ item.type }}/{{ item.name }}"
 
@@ -88,11 +94,11 @@
       retries: 30
       delay: 7
       with_items:
-        - { name: calico-typha, type: deployment, namespace: calico-system }
+        - {name: calico-typha, type: deployment, namespace: calico-system}
-        - { name: calico-kube-controllers, type: deployment, namespace: calico-system }
+        - {name: calico-kube-controllers, type: deployment, namespace: calico-system}
-        - { name: csi-node-driver, type: daemonset, selector: 'k8s-app=csi-node-driver', namespace: calico-system }
+        - {name: csi-node-driver, type: daemonset, selector: 'k8s-app=csi-node-driver', namespace: calico-system}
-        - { name: calico-node, type: daemonset, selector: 'k8s-app=calico-node', namespace: calico-system }
+        - {name: calico-node, type: daemonset, selector: 'k8s-app=calico-node', namespace: calico-system}
-        - { name: calico-apiserver, type: deployment, selector: 'k8s-app=calico-apiserver', namespace: calico-apiserver }
+        - {name: calico-apiserver, type: deployment, namespace: calico-apiserver}
       loop_control:
         label: "{{ item.type }}/{{ item.name }}"
 
@@ -105,7 +111,4 @@
       register: patch_result
       changed_when: "'felixconfiguration.projectcalico.org/default patched' in patch_result.stdout"
       failed_when: "'Error' in patch_result.stderr"
-      when: calico_ebpf == true
+      when: calico_ebpf
-
-  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
-  run_once: true # stops "skipped" log spam

roles/k3s_server_post/tasks/main.yml

@@ -2,11 +2,12 @@
 - name: Deploy calico
   include_tasks: calico.yml
   tags: calico
-  when: use_calico == true
+  when: calico_iface is defined
 
 - name: Deploy metallb pool
   include_tasks: metallb.yml
   tags: metallb
+  when: kube_vip_lb_ip_range is not defined
 
 - name: Remove tmp directory used for manifests
   file:

roles/k3s_server_post/tasks/metallb.yml

@@ -8,6 +8,27 @@
   with_items: "{{ groups[group_name_master | default('master')] }}"
   run_once: true
 
+- name: Delete outdated metallb replicas
+  shell: |-
+    set -o pipefail
+
+    REPLICAS=$(k3s kubectl --namespace='metallb-system' get replicasets \
+      -l 'component=controller,app=metallb' \
+      -o jsonpath='{.items[0].spec.template.spec.containers[0].image}, {.items[0].metadata.name}' 2>/dev/null || true)
+    REPLICAS_SETS=$(echo ${REPLICAS} | grep -v '{{ metal_lb_controller_tag_version }}' | sed -e "s/^.*\s//g")
+    if [ -n "${REPLICAS_SETS}" ] ; then
+      for REPLICAS in "${REPLICAS_SETS}"
+      do
+        k3s kubectl --namespace='metallb-system' \
+          delete rs "${REPLICAS}"
+      done
+    fi
+  args:
+    executable: /bin/bash
+  changed_when: false
+  run_once: true
+  with_items: "{{ groups[group_name_master | default('master')] }}"
+
 - name: Copy metallb CRs manifest to first master
   template:
     src: "metallb.crs.j2"

roles/k3s_server_post/templates/calico.crs.j2

@@ -9,13 +9,13 @@ spec:
   calicoNetwork:
     # Note: The ipPools section cannot be modified post-install.
     ipPools:
-    - blockSize: {{ calico_blockSize if calico_blockSize is defined else '26' }}
+    - blockSize: {{ calico_blockSize | default('26') }}
-      cidr: {{ calico_cidr if calico_cidr is defined else '10.52.0.0/16' }}
+      cidr: {{ calico_cidr | default('10.52.0.0/16') }}
-      encapsulation: {{ calico_encapsulation if calico_encapsulation is defined else 'VXLANCrossSubnet' }}
+      encapsulation: {{ calico_encapsulation | default('VXLANCrossSubnet') }}
-      natOutgoing: {{ calico_natOutgoing if calico_natOutgoing is defined else 'Enabled' }}
+      natOutgoing: {{ calico_natOutgoing | default('Enabled') }}
-      nodeSelector: {{ calico_nodeSelector if calico_nodeSelector is defined else 'all()' }}
+      nodeSelector: {{ calico_nodeSelector | default('all()') }}
     nodeAddressAutodetectionV4:
-      interface: {{ container_iface if container_iface is defined else 'eth0' }}
+      interface: {{ calico_iface  }}
     linuxDataplane: {{ 'BPF' if calico_ebpf else 'Iptables' }}
 
 ---
@@ -27,3 +27,15 @@ kind: APIServer
 metadata:
   name: default
 spec: {}
+
+{% if calico_ebpf %}
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kubernetes-services-endpoint
+  namespace: tigera-operator
+data:
+  KUBERNETES_SERVICE_HOST: '{{ apiserver_endpoint }}'
+  KUBERNETES_SERVICE_PORT: '6443'
+{% endif %}

roles/raspberrypi/tasks/main.yml

@@ -17,21 +17,27 @@
   when:
     grep_cpuinfo_raspberrypi.rc == 0 or grep_device_tree_model_raspberrypi.rc == 0
 
-- name: Set detected_distribution to Raspbian
+- name: Set detected_distribution to Raspbian (ARM64 on Raspbian, Debian Buster/Bullseye/Bookworm)
   set_fact:
     detected_distribution: Raspbian
-  when: >
+  vars:
-    raspberry_pi|default(false) and
+    allowed_descriptions:
-    ( ansible_facts.lsb.id|default("") == "Raspbian" or
+      - "[Rr]aspbian.*"
-      ansible_facts.lsb.description|default("") is match("[Rr]aspbian.*") )
+      - "Debian.*buster"
+      - "Debian.*bullseye"
+      - "Debian.*bookworm"
+  when:
+    - ansible_facts.architecture is search("aarch64")
+    - raspberry_pi|default(false)
+    - ansible_facts.lsb.description|default("") is match(allowed_descriptions | join('|'))
 
-- name: Set detected_distribution to Raspbian (ARM64 on Debian Buster)
+- name: Set detected_distribution to Raspbian (ARM64 on Debian Bookworm)
   set_fact:
     detected_distribution: Raspbian
   when:
     - ansible_facts.architecture is search("aarch64")
     - raspberry_pi|default(false)
-    - ansible_facts.lsb.description|default("") is match("Debian.*buster")
+    - ansible_facts.lsb.description|default("") is match("Debian.*bookworm")
 
 - name: Set detected_distribution_major_version
   set_fact:
@@ -39,14 +45,6 @@
   when:
     - detected_distribution | default("") == "Raspbian"
 
-- name: Set detected_distribution to Raspbian (ARM64 on Debian Bullseye)
-  set_fact:
-    detected_distribution: Raspbian
-  when:
-    - ansible_facts.architecture is search("aarch64")
-    - raspberry_pi|default(false)
-    - ansible_facts.lsb.description|default("") is match("Debian.*bullseye")
-
 - name: Execute OS related tasks on the Raspberry Pi - {{ action_ }}
   include_tasks: "{{ item }}"
   with_first_found:

roles/reset/tasks/main.yml

@@ -45,13 +45,16 @@
     - /var/lib/rancher/k3s
     - /var/lib/rancher/
     - /var/lib/cni/
+    - /etc/cni/net.d
 
 - name: Remove K3s http_proxy files
   file:
     name: "{{ item }}"
     state: absent
   with_items:
+    - "{{ systemd_dir }}/k3s.service.d/http_proxy.conf"
     - "{{ systemd_dir }}/k3s.service.d"
+    - "{{ systemd_dir }}/k3s-node.service.d/http_proxy.conf"
     - "{{ systemd_dir }}/k3s-node.service.d"
   when: proxy_env is defined
 

site.yml

@@ -46,3 +46,14 @@
   roles:
     - role: k3s_server_post
       become: true
+
+- name: Storing kubeconfig in the playbook directory
+  hosts: master
+  environment: "{{ proxy_env | default({}) }}"
+  tasks:
+    - name: Copying kubeconfig from {{ hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] }}
+      ansible.builtin.fetch:
+        src: "{{ ansible_user_dir }}/.kube/config"
+        dest: ./kubeconfig
+        flat: true
+      when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']