fix(k3s-server): harden isolated control-plane bootstrap

- pin kube-vip and cluster traffic to the private guest interface\n- disable disposable guest firewalls and verify API reachability before joins\n- keep control-plane orchestration on the primary and preserve failure diagnostics

molecule/calico/overrides.yml

@@ -7,6 +7,7 @@
         # See:
         # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
         calico_iface: eth1
+        kube_vip_iface: eth1
 
         # The test VMs might be a bit slow, so we give them more time to join the cluster:
         retry_count: 45

molecule/cilium/overrides.yml

@@ -7,6 +7,7 @@
         # See:
         # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
         cilium_iface: eth1
+        kube_vip_iface: eth1
 
         # The test VMs might be a bit slow, so we give them more time to join the cluster:
         retry_count: 45

molecule/default/overrides.yml

@@ -7,6 +7,9 @@
         # See:
         # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
         flannel_iface: eth1
+        # kube-vip cannot infer the cluster interface in these multi-NIC
+        # Vagrant guests because the default route is on eth0.
+        kube_vip_iface: eth1
 
         # The test VMs might be a bit slow, so we give them more time to join the cluster:
         retry_count: 45

molecule/default/prepare.yml

@@ -6,20 +6,26 @@
 - name: Network setup
   hosts: all
   tasks:
-    - name: Disable firewalld
+    - name: Gather service facts
-      when: ansible_distribution == "Rocky"
+      ansible.builtin.service_facts:
-      # Rocky Linux comes with firewalld enabled. It blocks some of the network
+
-      # connections needed for our k3s cluster. For our test setup, we just disable
+    - name: Disable guest firewall services
-      # it since the VM host's firewall is still active for connections to and from
+      # The disposable test guests use an isolated VirtualBox network. A distro
-      # the Internet.
+      # firewall can allow ICMP while silently blocking the inter-node Kubernetes
+      # API connection, so disable the known guest firewalls consistently.
       # When building your own cluster, please DO NOT blindly copy this. Instead,
       # please create a custom firewall configuration that fits your network design
       # and security needs.
       ansible.builtin.systemd:
-        name: firewalld
+        name: "{{ item }}"
         enabled: false
         state: stopped
       become: true
+      loop:
+        - firewalld.service
+        - nftables.service
+        - ufw.service
+      when: item in ansible_facts.services
 
     - name: Verify the private cluster interface
       ansible.builtin.assert:

molecule/kube-vip/overrides.yml

@@ -7,6 +7,7 @@
         # See:
         # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
         flannel_iface: eth1
+        kube_vip_iface: eth1
 
         # The test VMs might be a bit slow, so we give them more time to join the cluster:
         retry_count: 45

molecule/single_node/overrides.yml

@@ -7,6 +7,7 @@
         # See:
         # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant
         flannel_iface: eth1
+        kube_vip_iface: eth1
 
         # The test VMs might be a bit slow, so we give them more time to join the cluster:
         retry_count: 45

roles/k3s_server/tasks/fetch_k3s_init_logs.yml

@@ -62,6 +62,19 @@
   no_log: true
   register: k3s_tcp_listener_state
 
+- name: Fetch guest firewall service state
+  ansible.builtin.service_facts:
+  no_log: true
+  register: k3s_service_facts
+
+- name: Fetch input firewall rules
+  ansible.builtin.command:
+    cmd: iptables -S INPUT
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_input_firewall_rules
+
 - name: Ping the primary Kubernetes API address from {{ ansible_hostname }}
   ansible.builtin.command:
     argv:
@@ -135,6 +148,16 @@
       === ss -ltn ===
       {{ k3s_tcp_listener_state.stdout }}
 
+      === guest firewall services ===
+      firewalld={{ k3s_service_facts.ansible_facts.services.get('firewalld.service', {}).get('state', 'not-found') }}
+      nftables={{ k3s_service_facts.ansible_facts.services.get('nftables.service', {}).get('state', 'not-found') }}
+      ufw={{ k3s_service_facts.ansible_facts.services.get('ufw.service', {}).get('state', 'not-found') }}
+
+      === iptables -S INPUT ===
+      rc={{ k3s_input_firewall_rules.rc }}
+      {{ k3s_input_firewall_rules.stdout }}
+      {{ k3s_input_firewall_rules.stderr }}
+
       === primary API ping ===
       rc={{ k3s_primary_api_ping.rc }}
       {{ k3s_primary_api_ping.stdout }}

roles/k3s_server/tasks/join_master.yml

@@ -1,4 +1,14 @@
 ---
+- name: Verify primary Kubernetes API reachability from {{ joining_master }}
+  ansible.builtin.wait_for:
+    host: >-
+      {{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip
+      | split(',') | first }}
+    port: 6443
+    connect_timeout: 2
+    timeout: 30
+  delegate_to: "{{ joining_master }}"
+
 - name: Join transient k3s-init service for {{ joining_master }}
   ansible.builtin.command:
     cmd: >-

roles/k3s_server/tasks/main.yml

@@ -35,6 +35,7 @@
   when: kube_vip_lb_ip_range is defined
 
 - name: Initialize and verify the K3s control plane
+  any_errors_fatal: true
   when: not ansible_check_mode
   block:
     - name: Materialize per-host server initialization arguments
@@ -43,55 +44,50 @@
       no_log: true
       when: groups[group_name_master | default('master')] | length > 1
 
-    - name: Init the first master inside the transient k3s-init service
+    - name: Orchestrate control-plane initialization from the first master
-      ansible.builtin.command:
-        cmd: >-
-          systemd-run -p RestartSec=2 -p Restart=on-failure
-          --unit=k3s-init k3s server {{ server_init_args }}
-        creates: "{{ systemd_dir }}/k3s-init.service"
-      no_log: true
       when: inventory_hostname == groups[group_name_master | default('master')][0]
+      block:
+        - name: Init the first master inside the transient k3s-init service
+          ansible.builtin.command:
+            cmd: >-
+              systemd-run -p RestartSec=2 -p Restart=on-failure
+              --unit=k3s-init k3s server {{ server_init_args }}
+            creates: "{{ systemd_dir }}/k3s-init.service"
+          no_log: true
 
-    - name: Wait for the first master Kubernetes API
+        - name: Wait for the first master Kubernetes API
-      ansible.builtin.command:
+          ansible.builtin.command:
-        cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get --raw=/readyz"
+            cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get --raw=/readyz"
-      delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
+          register: first_master_api
-      register: first_master_api
+          until: first_master_api.rc == 0
-      until: first_master_api.rc == 0
+          retries: "{{ retry_count | default(20) }}"
-      retries: "{{ retry_count | default(20) }}"
+          delay: 2
-      delay: 2
+          changed_when: false
-      changed_when: false
-      run_once: true
 
-    - name: Verify the first master API listener on its node address
+        - name: Verify the first master API listener on its node address
-      ansible.builtin.wait_for:
+          ansible.builtin.wait_for:
-        host: >-
+            host: >-
-          {{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip
+              {{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip
-          | split(',') | first }}
+              | split(',') | first }}
-        port: 6443
+            port: 6443
-        connect_timeout: 2
+            connect_timeout: 2
-        timeout: "{{ retry_count | default(20) * 2 }}"
+            timeout: "{{ retry_count | default(20) * 2 }}"
-      delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
-      run_once: true
 
-    - name: Join additional masters one at a time
+        - name: Join additional masters one at a time
-      ansible.builtin.include_tasks: join_master.yml
+          ansible.builtin.include_tasks: join_master.yml
-      loop: "{{ groups[group_name_master | default('master')][1:] }}"
+          loop: "{{ groups[group_name_master | default('master')][1:] }}"
-      loop_control:
+          loop_control:
-        loop_var: joining_master
+            loop_var: joining_master
-      run_once: true
+          when: groups[group_name_master | default('master')] | length > 1
-      when: groups[group_name_master | default('master')] | length > 1
 
-    - name: Verify that all nodes actually joined (check k3s-init.service if this fails)
+        - name: Verify that all nodes actually joined (check k3s-init.service if this fails)
-      ansible.builtin.command:
+          ansible.builtin.command:
-        cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get nodes -l 'node-role.kubernetes.io/master=true' -o=jsonpath='{.items[*].metadata.name}'" # yamllint disable-line rule:line-length
+            cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get nodes -l 'node-role.kubernetes.io/master=true' -o=jsonpath='{.items[*].metadata.name}'" # yamllint disable-line rule:line-length
-      delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
+          register: nodes
-      register: nodes
+          until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups[group_name_master | default('master')] | length) # yamllint disable-line rule:line-length
-      until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups[group_name_master | default('master')] | length) # yamllint disable-line rule:line-length
+          retries: "{{ retry_count | default(20) }}"
-      retries: "{{ retry_count | default(20) }}"
+          delay: 10
-      delay: 10
+          changed_when: false
-      changed_when: false
-      run_once: true
   always:
     - name: Save logs of k3s-init.service
       ansible.builtin.include_tasks: fetch_k3s_init_logs.yml