fix(k3s-server): serialize master joins and capture diagnostics

- validate host-only connectivity before cluster bootstrap\n- materialize per-host join arguments and wait for sequential registration\n- collect bounded network diagnostics when k3s initialization fails

.github/workflows/test.yml

@@ -9,13 +9,13 @@ jobs:
     strategy:
       matrix:
         scenario:
+          - default
           - single_node
           - calico
           - cilium
           - kube-vip
-          - default
           # - ipv6
-      fail-fast: false
+      fail-fast: true
       max-parallel: 1
     env:
       PYTHON_VERSION: "3.11"

molecule/default/prepare.yml

@@ -21,6 +21,33 @@
         state: stopped
       become: true
 
+    - name: Verify the private cluster interface
+      ansible.builtin.assert:
+        that:
+          - flannel_iface in ansible_facts
+          - ansible_facts[flannel_iface].ipv4 is defined
+          - ansible_facts[flannel_iface].ipv4.address is defined
+        fail_msg: >-
+          The Vagrant private interface {{ flannel_iface }} does not have an
+          IPv4 address on {{ inventory_hostname }}.
+
+    - name: Verify guest-to-guest cluster network reachability
+      ansible.builtin.command:
+        argv:
+          - ping
+          - -c
+          - "1"
+          - -W
+          - "1"
+          - >-
+            {{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip
+            | split(',') | first }}
+      register: primary_cluster_ping
+      until: primary_cluster_ping.rc == 0
+      retries: 6
+      delay: 2
+      changed_when: false
+
     - name: Verify GitHub release host DNS
       ansible.builtin.getent:
         database: hosts

roles/k3s_server/tasks/fetch_k3s_init_logs.yml

@@ -5,12 +5,88 @@
 - name: Fetch k3s-init.service logs
   ansible.builtin.command:
     cmd: >-
-      timeout --signal=TERM 30s journalctl --no-pager
+      timeout --signal=TERM --kill-after=5s 30s journalctl --no-pager
       --unit=k3s-init.service --since=-30min --lines=5000
   changed_when: false
   failed_when: false
+  no_log: true
   register: k3s_init_log
 
+- name: Fetch k3s-init.service status
+  ansible.builtin.command:
+    cmd: >-
+      timeout --signal=TERM --kill-after=5s 15s systemctl status
+      k3s-init.service --no-pager --full
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_init_status
+
+- name: Fetch IP address state
+  ansible.builtin.command:
+    cmd: ip -br address
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_ip_address_state
+
+- name: Fetch IP route state
+  ansible.builtin.command:
+    cmd: ip route show
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_ip_route_state
+
+- name: Fetch IP neighbor state
+  ansible.builtin.command:
+    cmd: ip neigh show
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_ip_neighbor_state
+
+- name: Fetch IP rule state
+  ansible.builtin.command:
+    cmd: ip rule show
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_ip_rule_state
+
+- name: Fetch listening TCP sockets
+  ansible.builtin.command:
+    cmd: ss -ltn
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_tcp_listener_state
+
+- name: Ping the primary Kubernetes API address from {{ ansible_hostname }}
+  ansible.builtin.command:
+    argv:
+      - ping
+      - -c
+      - "1"
+      - -W
+      - "1"
+      - "{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(',') | first }}"
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_primary_api_ping
+
+- name: Probe primary Kubernetes API from {{ ansible_hostname }}
+  ansible.builtin.wait_for:
+    host: "{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(',') | first }}"
+    port: 6443
+    connect_timeout: 1
+    timeout: 2
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_primary_api_probe
+
 - name: Create {{ log_destination }}
   delegate_to: localhost
   run_once: true
@@ -28,4 +104,43 @@
     dest: "{{ log_destination }}/k3s-init@{{ ansible_hostname }}.log"
     mode: "0644"
   vars:
-    content: "{{ k3s_init_log.stdout }}"
+    content: |
+      === k3s-init.service status ===
+      {{ k3s_init_status.stdout | regex_replace('--token(?:=| +)[^ ]+', '--token ***') }}
+
+      === k3s-init.service journal ===
+      {{ k3s_init_log.stdout | regex_replace('--token(?:=| +)[^ ]+', '--token ***') }}
+
+- name: Store network diagnostics to {{ log_destination }}
+  delegate_to: localhost
+  become: false
+  ansible.builtin.template:
+    src: content.j2
+    dest: "{{ log_destination }}/network@{{ ansible_hostname }}.log"
+    mode: "0644"
+  vars:
+    content: |
+      === ip -br address ===
+      {{ k3s_ip_address_state.stdout }}
+
+      === ip route show ===
+      {{ k3s_ip_route_state.stdout }}
+
+      === ip neigh show ===
+      {{ k3s_ip_neighbor_state.stdout }}
+
+      === ip rule show ===
+      {{ k3s_ip_rule_state.stdout }}
+
+      === ss -ltn ===
+      {{ k3s_tcp_listener_state.stdout }}
+
+      === primary API ping ===
+      rc={{ k3s_primary_api_ping.rc }}
+      {{ k3s_primary_api_ping.stdout }}
+      {{ k3s_primary_api_ping.stderr }}
+
+      === primary API probe ===
+      failed={{ k3s_primary_api_probe.failed | default(false) }}
+      elapsed={{ k3s_primary_api_probe.elapsed | default('unknown') }}
+      msg={{ k3s_primary_api_probe.msg | default('connected') }}

roles/k3s_server/tasks/join_master.yml

@@ -0,0 +1,21 @@
+---
+- name: Join transient k3s-init service for {{ joining_master }}
+  ansible.builtin.command:
+    cmd: >-
+      systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init
+      k3s server {{ hostvars[joining_master].k3s_server_init_args }}
+    creates: "{{ systemd_dir }}/k3s-init.service"
+  delegate_to: "{{ joining_master }}"
+  no_log: true
+
+- name: Wait for primary registration of {{ joining_master }}
+  ansible.builtin.command:
+    cmd: >-
+      {{ k3s_kubectl_binary | default('k3s kubectl') }} get node
+      {{ hostvars[joining_master].ansible_hostname }}
+  delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
+  register: joined_master
+  until: joined_master.rc == 0
+  retries: "{{ retry_count | default(20) }}"
+  delay: 2
+  changed_when: false

roles/k3s_server/tasks/main.yml

@@ -34,22 +34,64 @@
   tags: kubevip
   when: kube_vip_lb_ip_range is defined
 
-- name: Init cluster inside the transient k3s-init service
-  ansible.builtin.command:
-    cmd: systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server {{ server_init_args }}
-    creates: "{{ systemd_dir }}/k3s-init.service"
-
-- name: Verification
+- name: Initialize and verify the K3s control plane
   when: not ansible_check_mode
   block:
+    - name: Materialize per-host server initialization arguments
+      ansible.builtin.set_fact:
+        k3s_server_init_args: "{{ server_init_args }}"
+      no_log: true
+      when: groups[group_name_master | default('master')] | length > 1
+
+    - name: Init the first master inside the transient k3s-init service
+      ansible.builtin.command:
+        cmd: >-
+          systemd-run -p RestartSec=2 -p Restart=on-failure
+          --unit=k3s-init k3s server {{ server_init_args }}
+        creates: "{{ systemd_dir }}/k3s-init.service"
+      no_log: true
+      when: inventory_hostname == groups[group_name_master | default('master')][0]
+
+    - name: Wait for the first master Kubernetes API
+      ansible.builtin.command:
+        cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get --raw=/readyz"
+      delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
+      register: first_master_api
+      until: first_master_api.rc == 0
+      retries: "{{ retry_count | default(20) }}"
+      delay: 2
+      changed_when: false
+      run_once: true
+
+    - name: Verify the first master API listener on its node address
+      ansible.builtin.wait_for:
+        host: >-
+          {{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip
+          | split(',') | first }}
+        port: 6443
+        connect_timeout: 2
+        timeout: "{{ retry_count | default(20) * 2 }}"
+      delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
+      run_once: true
+
+    - name: Join additional masters one at a time
+      ansible.builtin.include_tasks: join_master.yml
+      loop: "{{ groups[group_name_master | default('master')][1:] }}"
+      loop_control:
+        loop_var: joining_master
+      run_once: true
+      when: groups[group_name_master | default('master')] | length > 1
+
     - name: Verify that all nodes actually joined (check k3s-init.service if this fails)
       ansible.builtin.command:
         cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get nodes -l 'node-role.kubernetes.io/master=true' -o=jsonpath='{.items[*].metadata.name}'" # yamllint disable-line rule:line-length
+      delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
       register: nodes
       until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups[group_name_master | default('master')] | length) # yamllint disable-line rule:line-length
       retries: "{{ retry_count | default(20) }}"
       delay: 10
       changed_when: false
+      run_once: true
   always:
     - name: Save logs of k3s-init.service
       ansible.builtin.include_tasks: fetch_k3s_init_logs.yml