Merge branch 'master' into verify

* Add Tigera Operator/Calico CNI option

Small tweak to reduce delta from head

Set calico option to be disabled by default

Add rescue blocks in case updating existing

Refactor items and update comments

Refactor and consolidate calico.yml into block

Refactor to use template for Calico CRs

Revert use_calico to false

Template blockSize

Align default cidr in template with all.yml sample

Apply upstream version tags

Revert to current ver tags. Upstream's don't work.

Update template address detection

Add Tigera Operator/Calico CNI option

* Add calico-apiserver check

* Add eBPF dataplane option

* Add kube svc endpoint configmap when ebpf enabled

* Add /etc/cni/net.d to reset task

* Refactor based on comments

* Add molecule scenario

* Fix lint

---------

Co-authored-by: Techno Tim <timothystewart6@gmail.com>

.github/ISSUE_TEMPLATE.md

@@ -37,6 +37,11 @@ systemd_dir: ""
 
 flannel_iface: ""
 
+#calico_iface: ""
+calico_ebpf: ""
+calico_cidr: ""
+calico_tag: ""
+
 apiserver_endpoint: ""
 
 k3s_token: "NA"

.github/workflows/test.yml

@@ -12,6 +12,7 @@ jobs:
           - default
           - ipv6
           - single_node
+          - calico
       fail-fast: false
     env:
       PYTHON_VERSION: "3.11"
@@ -22,6 +23,31 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
+      # these steps are necessary if not using ephemeral nodes
+      - name: Delete old Vagrant box versions
+        if: always() # do this even if a step before has failed
+        run: vagrant box prune --force
+
+      - name: Remove all local Vagrant boxes
+        if: always() # do this even if a step before has failed
+        run: if vagrant box list 2>/dev/null; then vagrant box list | cut -f 1 -d ' ' | xargs -L 1 vagrant box remove -f 2>/dev/null && echo "All Vagrant boxes removed." || echo "No Vagrant boxes found."; else echo "No Vagrant boxes found."; fi
+
+      - name: Remove all Virtualbox VMs
+        if: always() # do this even if a step before has failed
+        run: VBoxManage list vms | awk -F'"' '{print $2}' | xargs -I {} VBoxManage unregistervm --delete "{}"
+
+      - name: Remove all Virtualbox HDs
+        if: always() # do this even if a step before has failed
+        run: VBoxManage list hdds | awk -F':' '/^UUID:/ {print $2}' | xargs -I {} VBoxManage closemedium disk "{}" --delete
+
+      - name: Remove all Virtualbox Networks
+        if: always() # do this even if a step before has failed
+        run: VBoxManage list hostonlyifs | grep '^Name:' | awk '{print $2}' | grep '^vboxnet' | xargs -I {} VBoxManage hostonlyif remove {}
+
+      - name: Remove Virtualbox network config
+        if: always() # do this even if a step before has failed
+        run: sudo rm /etc/vbox/networks.conf || true
+
       - name: Configure VirtualBox
         run: |-
           sudo mkdir -p /etc/vbox
@@ -70,7 +96,7 @@ jobs:
 
       - name: Remove all local Vagrant boxes
         if: always() # do this even if a step before has failed
-        run: vagrant box list | cut -f 1 -d ' ' | xargs -L 1 vagrant box remove -f
+        run: if vagrant box list 2>/dev/null; then vagrant box list | cut -f 1 -d ' ' | xargs -L 1 vagrant box remove -f 2>/dev/null && echo "All Vagrant boxes removed." || echo "No Vagrant boxes found."; else echo "No Vagrant boxes found."; fi
 
       - name: Remove all Virtualbox VMs
         if: always() # do this even if a step before has failed
@@ -86,7 +112,7 @@ jobs:
 
       - name: Remove Virtualbox network config
         if: always() # do this even if a step before has failed
-        run: sudo rm /etc/vbox/networks.conf
+        run: sudo rm /etc/vbox/networks.conf || true
 
       - name: Upload log files
         if: always() # do this even if a step before has failed

.gitignore

@@ -1,3 +1,6 @@
 .env/
 *.log
 ansible.cfg
+
+__pycache__/
+*.py[cod]

README.md

@@ -73,6 +73,21 @@ This requires at least k3s version `1.19.1` however the version is configurable
 
 If needed, you can also edit `inventory/my-cluster/group_vars/all.yml` to match your environment.
 
+#### verify routing prefixes and ip addresses in all.yml
+Ensure that the ip routing prefix matches the the one from the flannel_iface (`eth0` by default) for each host.
+Example `192.168.30` is the default routing prefix, and it is used in the following variables:
+- apiserver_endpoint
+- metal_lb_bgp_peer_address (commented by default)
+- metal_lb_ip_range
+Also Ensure that the apiserver_endpoint is not in the metal_lb_ip_range
+
+For your convience The playbook site.yml verifies the above elements of the config are valid.
+* Optionally to skip these verifications set, in all.yml `verify_config: false`  
+* Optionally to just verify the config Run
+    `ansible-playbook -i inventory/my-cluster/hosts.ini verify_config.yml`
+* Optionally manually recover ip routing prefix based on inet and netmask from
+    `ansible -i inventory/my-cluster/hosts.ini --become -m shell -a 'ifconfig eth0 | grep "inet "' all`
+
 ### ☸️ Create Cluster
 
 Start provisioning of the cluster using the following command:

inventory/sample/group_vars/all.yml

@@ -1,5 +1,5 @@
 ---
-k3s_version: v1.27.9+k3s1
+k3s_version: v1.29.0+k3s1
 # this is the user that has ssh access to these machines
 ansible_user: ansibleuser
 systemd_dir: /etc/systemd/system
@@ -10,6 +10,12 @@ system_timezone: "Your/Timezone"
 # interface which will be used for flannel
 flannel_iface: "eth0"
 
+# uncomment calico_iface to use tigera operator/calico cni instead of flannel https://docs.tigera.io/calico/latest/about
+# calico_iface: "eth0"
+calico_ebpf: false           # use eBPF dataplane instead of iptables
+calico_cidr: "10.52.0.0/16"  # calico cluster pod cidr pool
+calico_tag: "v3.27.0"        # calico version tag
+
 # apiserver_endpoint is virtual ip-address which will be configured on each master
 apiserver_endpoint: "192.168.30.222"
 
@@ -20,28 +26,35 @@ k3s_token: "some-SUPER-DEDEUPER-secret-password"
 # The IP on which the node is reachable in the cluster.
 # Here, a sensible default is provided, you can still override
 # it for each of your hosts, though.
-k3s_node_ip: '{{ ansible_facts[flannel_iface]["ipv4"]["address"] }}'
+k3s_node_ip: "{{ ansible_facts[(calico_iface | default(flannel_iface))]['ipv4']['address'] }}"
 
 # Disable the taint manually by setting: k3s_master_taint = false
 k3s_master_taint: "{{ true if groups['node'] | default([]) | length >= 1 else false }}"
 
 # these arguments are recommended for servers as well as agents:
 extra_args: >-
-  --flannel-iface={{ flannel_iface }}
+  {{ '--flannel-iface=' + flannel_iface if calico_iface is not defined else '' }}
   --node-ip={{ k3s_node_ip }}
 
 # change these to your liking, the only required are: --disable servicelb, --tls-san {{ apiserver_endpoint }}
+# the contents of the if block is also required if using calico
 extra_server_args: >-
   {{ extra_args }}
   {{ '--node-taint node-role.kubernetes.io/master=true:NoSchedule' if k3s_master_taint else '' }}
+  {% if calico_iface is defined %}
+  --flannel-backend=none
+  --disable-network-policy
+  --cluster-cidr={{ calico_cidr | default('10.52.0.0/16') }}
+  {% endif %}
   --tls-san {{ apiserver_endpoint }}
   --disable servicelb
   --disable traefik
+
 extra_agent_args: >-
   {{ extra_args }}
 
 # image tag for kube-vip
-kube_vip_tag_version: "v0.5.12"
+kube_vip_tag_version: "v0.6.4"
 
 # metallb type frr or native
 metal_lb_type: "native"
@@ -55,8 +68,8 @@ metal_lb_mode: "layer2"
 # metal_lb_bgp_peer_address: "192.168.30.1"
 
 # image tag for metal lb
-metal_lb_speaker_tag_version: "v0.13.9"
+metal_lb_speaker_tag_version: "v0.13.12"
-metal_lb_controller_tag_version: "v0.13.9"
+metal_lb_controller_tag_version: "v0.13.12"
 
 # metallb ip range for load balancer
 metal_lb_ip_range: "192.168.30.80-192.168.30.90"
@@ -127,3 +140,5 @@ custom_registries_yaml: |
 #   HTTP_PROXY: "http://proxy.domain.local:3128"
 #   HTTPS_PROXY: "http://proxy.domain.local:3128"
 #   NO_PROXY: "*.domain.local,127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
+
+verify_config: true

molecule/README.md

@@ -13,6 +13,8 @@ We have these scenarios:
   To save a bit of test time, this cluster is _not_ highly available, it consists of only one control and one worker node.
 - **single_node**:
   Very similar to the default scenario, but uses only a single node for all cluster functionality.
+- **calico**:
+  The same as single node, but uses calico cni instead of flannel.
 
 ## How to execute
 

molecule/calico/molecule.yml

@@ -0,0 +1,49 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: vagrant
+platforms:
+  - name: control1
+    box: generic/ubuntu2204
+    memory: 4096
+    cpus: 4
+    config_options:
+      # We currently can not use public-key based authentication on Ubuntu 22.04,
+      # see: https://github.com/chef/bento/issues/1405
+      ssh.username: "vagrant"
+      ssh.password: "vagrant"
+    groups:
+      - k3s_cluster
+      - master
+    interfaces:
+      - network_name: private_network
+        ip: 192.168.30.62
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_VERBOSITY: 1
+  playbooks:
+    converge: ../resources/converge.yml
+    side_effect: ../resources/reset.yml
+    verify: ../resources/verify.yml
+  inventory:
+    links:
+      group_vars: ../../inventory/sample/group_vars
+scenario:
+  test_sequence:
+    - dependency
+    - cleanup
+    - destroy
+    - syntax
+    - create
+    - prepare
+    - converge
+    # idempotence is not possible with the playbook in its current form.
+    - verify
+    # We are repurposing side_effect here to test the reset playbook.
+    # This is why we do not run it before verify (which tests the cluster),
+    # but after the verify step.
+    - side_effect
+    - cleanup
+    - destroy

molecule/calico/overrides.yml

@@ -0,0 +1,16 @@
+---
+- name: Apply overrides
+  hosts: all
+  tasks:
+    - name: Override host variables
+      ansible.builtin.set_fact:
+        # See:
+        # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
+        calico_iface: eth1
+
+        # The test VMs might be a bit slow, so we give them more time to join the cluster:
+        retry_count: 45
+
+        # Make sure that our IP ranges do not collide with those of the other scenarios
+        apiserver_endpoint: "192.168.30.224"
+        metal_lb_ip_range: "192.168.30.100-192.168.30.109"

roles/k3s_server_post/tasks/calico.yml

@@ -0,0 +1,114 @@
+---
+- name: Deploy Calico to cluster
+  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
+  run_once: true
+  block:
+    - name: Create manifests directory on first master
+      file:
+        path: /tmp/k3s
+        state: directory
+        owner: root
+        group: root
+        mode: 0755
+
+    - name: "Download to first master: manifest for Tigera Operator and Calico CRDs"
+      ansible.builtin.get_url:
+        url: "https://raw.githubusercontent.com/projectcalico/calico/{{ calico_tag }}/manifests/tigera-operator.yaml"
+        dest: "/tmp/k3s/tigera-operator.yaml"
+        owner: root
+        group: root
+        mode: 0755
+
+    - name: Copy Calico custom resources manifest to first master
+      ansible.builtin.template:
+        src: "calico.crs.j2"
+        dest: /tmp/k3s/custom-resources.yaml
+        owner: root
+        group: root
+        mode: 0755
+
+    - name: Deploy or replace Tigera Operator
+      block:
+        - name: Deploy Tigera Operator
+          ansible.builtin.command:
+            cmd: kubectl create -f /tmp/k3s/tigera-operator.yaml
+          register: create_operator
+          changed_when: "'created' in create_operator.stdout"
+          failed_when: "'Error' in create_operator.stderr and 'already exists' not in create_operator.stderr"
+      rescue:
+        - name: Replace existing Tigera Operator
+          ansible.builtin.command:
+            cmd: kubectl replace -f /tmp/k3s/tigera-operator.yaml
+          register: replace_operator
+          changed_when: "'replaced' in replace_operator.stdout"
+          failed_when: "'Error' in replace_operator.stderr"
+
+    - name: Wait for Tigera Operator resources
+      command: >-
+        k3s kubectl wait {{ item.type }}/{{ item.name }}
+        --namespace='tigera-operator'
+        --for=condition=Available=True
+        --timeout=7s
+      register: tigera_result
+      changed_when: false
+      until: tigera_result is succeeded
+      retries: 7
+      delay: 7
+      with_items:
+        - {name: tigera-operator, type: deployment}
+      loop_control:
+        label: "{{ item.type }}/{{ item.name }}"
+
+    - name: Deploy Calico custom resources
+      block:
+        - name: Deploy custom resources for Calico
+          ansible.builtin.command:
+            cmd: kubectl create -f /tmp/k3s/custom-resources.yaml
+          register: create_cr
+          changed_when: "'created' in create_cr.stdout"
+          failed_when: "'Error' in create_cr.stderr and 'already exists' not in create_cr.stderr"
+      rescue:
+        - name: Apply new Calico custom resource manifest
+          ansible.builtin.command:
+            cmd: kubectl apply -f /tmp/k3s/custom-resources.yaml
+          register: apply_cr
+          changed_when: "'configured' in apply_cr.stdout or 'created' in apply_cr.stdout"
+          failed_when: "'Error' in apply_cr.stderr"
+
+    - name: Wait for Calico system resources to be available
+      command: >-
+        {% if item.type == 'daemonset' %}
+        k3s kubectl wait pods
+        --namespace='{{ item.namespace }}'
+        --selector={{ item.selector }}
+        --for=condition=Ready
+        {% else %}
+        k3s kubectl wait {{ item.type }}/{{ item.name }}
+        --namespace='{{ item.namespace }}'
+        --for=condition=Available
+        {% endif %}
+        --timeout=7s
+      register: cr_result
+      changed_when: false
+      until: cr_result is succeeded
+      retries: 30
+      delay: 7
+      with_items:
+        - {name: calico-typha, type: deployment, namespace: calico-system}
+        - {name: calico-kube-controllers, type: deployment, namespace: calico-system}
+        - {name: csi-node-driver, type: daemonset, selector: 'k8s-app=csi-node-driver', namespace: calico-system}
+        - {name: calico-node, type: daemonset, selector: 'k8s-app=calico-node', namespace: calico-system}
+        - {name: calico-apiserver, type: deployment, namespace: calico-apiserver}
+      loop_control:
+        label: "{{ item.type }}/{{ item.name }}"
+
+    - name: Patch Felix configuration for eBPF mode
+      ansible.builtin.command:
+        cmd: >
+          kubectl patch felixconfiguration default
+          --type='merge'
+          --patch='{"spec": {"bpfKubeProxyIptablesCleanupEnabled": false}}'
+      register: patch_result
+      changed_when: "'felixconfiguration.projectcalico.org/default patched' in patch_result.stdout"
+      failed_when: "'Error' in patch_result.stderr"
+      when: calico_ebpf

roles/k3s_server_post/tasks/main.yml

@@ -1,4 +1,9 @@
 ---
+- name: Deploy calico
+  include_tasks: calico.yml
+  tags: calico
+  when: calico_iface is defined
+
 - name: Deploy metallb pool
   include_tasks: metallb.yml
   tags: metallb

roles/k3s_server_post/templates/calico.crs.j2

@@ -0,0 +1,41 @@
+# This section includes base Calico installation configuration.
+# For more information, see: https://docs.tigera.io/calico/latest/reference/installation/api#operator.tigera.io/v1.Installation
+apiVersion: operator.tigera.io/v1
+kind: Installation
+metadata:
+  name: default
+spec:
+  # Configures Calico networking.
+  calicoNetwork:
+    # Note: The ipPools section cannot be modified post-install.
+    ipPools:
+    - blockSize: {{ calico_blockSize | default('26') }}
+      cidr: {{ calico_cidr | default('10.52.0.0/16') }}
+      encapsulation: {{ calico_encapsulation | default('VXLANCrossSubnet') }}
+      natOutgoing: {{ calico_natOutgoing | default('Enabled') }}
+      nodeSelector: {{ calico_nodeSelector | default('all()') }}
+    nodeAddressAutodetectionV4:
+      interface: {{ calico_iface  }}
+    linuxDataplane: {{ 'BPF' if calico_ebpf else 'Iptables' }}
+
+---
+
+# This section configures the Calico API server.
+# For more information, see: https://docs.tigera.io/calico/latest/reference/installation/api#operator.tigera.io/v1.APIServer
+apiVersion: operator.tigera.io/v1
+kind: APIServer
+metadata:
+  name: default
+spec: {}
+
+{% if calico_ebpf %}
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kubernetes-services-endpoint
+  namespace: tigera-operator
+data:
+  KUBERNETES_SERVICE_HOST: '{{ apiserver_endpoint }}'
+  KUBERNETES_SERVICE_PORT: '6443'
+{% endif %}

roles/raspberrypi/tasks/main.yml

@@ -31,6 +31,14 @@
     - raspberry_pi|default(false)
     - ansible_facts.lsb.description|default("") is match(allowed_descriptions | join('|'))
 
+- name: Set detected_distribution to Raspbian (ARM64 on Debian Bookworm)
+  set_fact:
+    detected_distribution: Raspbian
+  when:
+    - ansible_facts.architecture is search("aarch64")
+    - raspberry_pi|default(false)
+    - ansible_facts.lsb.description|default("") is match("Debian.*bookworm")
+
 - name: Set detected_distribution_major_version
   set_fact:
     detected_distribution_major_version: "{{ ansible_facts.lsb.major_release }}"

roles/reset/tasks/main.yml

@@ -45,6 +45,7 @@
     - /var/lib/rancher/k3s
     - /var/lib/rancher/
     - /var/lib/cni/
+    - /etc/cni/net.d
 
 - name: Remove K3s http_proxy files
   file:

roles/verify_config/tasks/main.yml

@@ -0,0 +1,13 @@
+---
+# To be ran on localhost after verify_config_gather role.
+# Verifies gatherd facts.
+- name: Collect all routing4_prefixes into a list
+  set_fact:
+    all_routing4_prefixes: "{{ groups['all'] | map('extract', hostvars, 'routing4_prefix') | list }}"
+    routing4_prefix: "{{ groups['all'] | map('extract', hostvars, 'routing4_prefix') | list | first }}"
+
+- name: Ensure all hosts have the same routing4_prefix
+  assert:
+    that: all_routing4_prefixes | unique | length == 1
+    fail_msg: "Not all hosts have the same routing4_prefix."
+    success_msg: "Using verified routing prefix {{ routing4_prefix }} across all hosts"

roles/verify_config_gather/filter_plugins/range_to_ips.py

@@ -0,0 +1,53 @@
+import netaddr
+
+''' returns True if ip is in range
+    examples:
+    - see test functions below
+'''
+def test(ip, range, expected):
+    assert netaddr_ip_in_dash_range(ip, range) == expected
+
+def test_netaddr_ip_in_dash_range():
+    # ipv4
+    test('192.168.1.1', '192.168.1.1-192.168.1.2', True)
+    test('192.168.1.1', '192.168.1.1', True)
+    test('192.168.1.1', '192.168.1.2-192.168.1.3', False)
+    test('192.168.1.1', '192.168.1.2', False)
+
+    # ipv6 style
+    test('::ffff:192.168.1.1', '::ffff:192.168.1.1-::ffff:192.168.1.8', True)
+    test('::ffff:192.168.1.1', '::ffff:192.168.1.1', True)
+    test('::ffff:192.168.1.1', '::ffff:192.168.1.2-::ffff:192.168.1.8', False)
+    test('::ffff:192.168.1.1', '::ffff:192.168.1.2', False)
+
+    # Note I expedted true but apperently the netaddr library does not support this?? or I don't understand ipv6 :)
+    test('::2:1', '::2:1-::2:2', False)
+    '''
+    todo: ?
+    - netaddr_ip_in_dash_range('192.168.1.1',  '192.168.1.0/24')             => True (TODO: test, implement)
+    - netaddr_ip_in_dash_range('192.168.99.1', '192.168.1.0/24')            => False (TODO: test, implement)
+    '''
+
+def netaddr_ip_in_dash_range(ip, range):
+    # return False early if range is invalid
+    if '-' not in range:
+        ip_start = range
+        ip_end = range
+    else:
+        ip_start = range.split('-')[0]
+        ip_end = range.split('-')[1]
+    return ip in [str(ip) for ip in netaddr.iter_iprange(ip_start, ip_end)]
+
+
+class FilterModule(object):
+        ''' Ansible filters. Interface to custom netaddr methods.
+            https://pypi.org/project/netaddr/
+        '''
+
+        def filters(self):
+            return {
+                'netaddr_ip_in_dash_range' : netaddr_ip_in_dash_range
+                }
+
+if __name__ == '__main__':
+    test_netaddr_ip_in_dash_range()

roles/verify_config_gather/tasks/main.yml

@@ -0,0 +1,117 @@
+---
+# To be ran on all hosts before role verify_config.
+# Gathers facts and ensures they are set
+
+- name: Set routing4_prefix from regex
+  set_fact:
+    routing4_prefix: "{{ hostvars[inventory_hostname]['ansible_' ~ flannel_iface]['ipv4']['broadcast']
+      | regex_replace('\\.?255', '') }}"
+    routing6_cidr: "{{ hostvars[inventory_hostname]['ansible_' ~ flannel_iface]['ipv6'][0]['address'] }}/{{
+      hostvars[inventory_hostname]['ansible_' ~ flannel_iface]['ipv6'][0]['prefix'] }}"
+
+- name: Check if fact routing4_prefix exists and is not empty
+  assert:
+    that:
+      - routing4_prefix is defined
+      - routing4_prefix is not none
+      - routing4_prefix != ''
+    fail_msg: >-
+      The fact 'routing_4prefix' is not defined, is null, or is empty
+      (based on flannel_iface: {{ flannel_iface }} ipv4 broadcast).
+
+# metal_lb_bgp_peer_address
+- name: Assert that metal_lb_bgp_peer_address starts with routing4_prefix, or is ipv6
+  assert:
+    that:
+      - >
+        metal_lb_bgp_peer_address.startswith(routing4_prefix)
+        or metal_lb_bgp_peer_address.matches('.*:.*')
+    fail_msg: >
+      The fact 'metal_lb_bgp_peer_address' <{{ metal_lb_bgp_peer_address }}>
+      doesn't start with the routing prefix <{{ routing4_prefix }}>
+  when: metal_lb_bgp_peer_address is defined
+
+# metal_lb_ip_range
+- name: >
+    Assert that metal_lb_ip_range (when string) contains <ipv4>-<ipv4>
+    and both ips start with routing4_prefix, skip any containing ':' (ipv6)
+  assert:
+    that:
+      - metal_lb_ip_range | regex_search('^{{ routing4_prefix }}\.[0-9]{1,3}-{{ routing4_prefix }}\.[0-9]{1,3}$|.*:.*')
+    fail_msg: >
+      metal_lb_ip_range <{{ metal_lb_ip_range }}> has one or more ipv4s
+      that don't start with the routing prefix <{{ routing4_prefix }}>
+  when: metal_lb_ip_range is string
+
+- name: Assert that metal_lb_ip_ranges (when list) has only strings that match the regexes in the task above
+  assert:
+    that:
+      - >
+        ( metal_lb_ip_range
+        | select('match', '^{{ routing4_prefix }}\.[0-9]{1,3}-{{ routing4_prefix }}\.[0-9]{1,3}$|.*:.*')
+        | list | length
+        )
+        ==
+        (metal_lb_ip_range
+        | length
+        )
+    fail_msg: >
+      metal_lb_ip_range <{{ metal_lb_ip_range }}> has one or more values with ipv4s
+      that don't start with the routing prefix <{{ routing4_prefix }}>
+  when: metal_lb_ip_range is not string and metal_lb_ip_range is not mapping and metal_lb_ip_range is iterable
+
+# apiserver_endpoint
+- name: Assert that apiserver_endpoint is not in metal_lb_ip_range (when string) using network_in_usable
+  # For /<mask> ranges
+  assert:
+    that:
+      - not ( metal_lb_ip_range | ansible.utils.network_in_usable( apiserver_endpoint ))
+    fail_msg: "apiserver_endpoint {{ apiserver_endpoint }} cannot be in the metal_lb_ip_range {{ metal_lb_ip_range }}"
+    success_msg: >
+      apiserver_endpoint {{ apiserver_endpoint }} is *probably* not in the metal_lb_ip_range {{ metal_lb_ip_range }}
+  when: metal_lb_ip_range is string
+
+- name: >
+    Assert that apiserver_endpoint is not in metal_lb_ip_range (when string)
+    using custom filter netaddr_ip_in_dash_range
+  # For <ip>-<ip> ranges. Not sure this works for ipv6
+  assert:
+    that:
+      - not (apiserver_endpoint | netaddr_ip_in_dash_range(metal_lb_ip_range))
+    fail_msg: "apiserver_endpoint {{ apiserver_endpoint }} cannot be in the metal_lb_ip_range {{ metal_lb_ip_range }}"
+    success_msg: >
+      apiserver_endpoint {{ apiserver_endpoint }} is *probably* not in the metal_lb_ip_range {{ metal_lb_ip_range }}
+  when: metal_lb_ip_range is string
+  # *probably* in the success_msg sections of the prior two tasks because not all cases may work
+
+- name: Assert that apiserver_endpoint is not in metal_lb_ip_ranges (when list) using logic of the task above
+  assert:
+    that:
+      - not (apiserver_endpoint | netaddr_ip_in_dash_range(item))
+    # this probably fails on an ipv6 range ''::1-::2', and on <ipv4or6>/<mask> ranges
+    fail_msg: "apiserver_endpoint {{ apiserver_endpoint }} cannot be in the metal_lb_ip_range item {{ item }}"
+    success_msg: >
+      apiserver_endpoint {{ apiserver_endpoint }} is *probably* not in the metal_lb_ip_range item {{ item }}
+  loop: "{{ metal_lb_ip_range | list }}"
+  when: metal_lb_ip_range is not string   and metal_lb_ip_range is not mapping   and metal_lb_ip_range is iterable
+  # these (when string, when list tasks) smell funny.
+  # It seems there should be a way in one task to handle an object that is a string or a list
+  #    and loop on {{ [metal_lb_ip_range] }}  or {{ metal_lb_ip_range }} respectively
+  # when it is a string it skips each char :(
+  # I tried the select pattern like in the task
+  #    'assert that metal_lb_ip_ranges (when list) has only strings that match the regexes in the task above'
+  #  but it didn't work
+
+- name: Assert that apiserver_endpoint, is ipv4 and starts with routing4_prefix, or is ipv6 and is in routing6_cidr
+  assert:
+    that:
+      - apiserver_endpoint is defined
+      - apiserver_endpoint is not none
+      - >-
+        ( apiserver_endpoint | ansible.utils.ipv4 and apiserver_endpoint.startswith(routing4_prefix) )
+        or
+        ( apiserver_endpoint | ansible.utils.ipv6 and apiserver_endpoint | ansible.utils.ipaddr(routing6_cidr))
+    fail_msg: >
+      The fact 'apiserver_endpoint' <{{ apiserver_endpoint }}>
+      doesn't start with the routing prefix <{{ routing4_prefix }}>
+      or is not in the routing6_cidr <{{ routing6_cidr }}>

site.yml

@@ -1,4 +1,7 @@
 ---
+- name: Import verify_config playbook
+  ansible.builtin.import_playbook: verify_config.yml
+
 - name: Prepare Proxmox cluster
   hosts: proxmox
   gather_facts: true

verify_config.yml

@@ -0,0 +1,14 @@
+---
+- name: Gather config for verify
+  hosts: all
+  gather_facts: true
+  roles:
+    - role: verify_config_gather
+      when: verify_config is not defined or verify_config
+
+- name: Verify config
+  hosts: localhost
+  gather_facts: false
+  roles:
+    - role: verify_config
+      when: verify_config is not defined or verify_config