fix(k3s-server): serialize master joins and capture diagnostics

- validate host-only connectivity before cluster bootstrap\n- materialize per-host join arguments and wait for sequential registration\n- collect bounded network diagnostics when k3s initialization fails

molecule/default/prepare.yml

@@ -21,6 +21,33 @@
         state: stopped
       become: true
 
+    - name: Verify the private cluster interface
+      ansible.builtin.assert:
+        that:
+          - flannel_iface in ansible_facts
+          - ansible_facts[flannel_iface].ipv4 is defined
+          - ansible_facts[flannel_iface].ipv4.address is defined
+        fail_msg: >-
+          The Vagrant private interface {{ flannel_iface }} does not have an
+          IPv4 address on {{ inventory_hostname }}.
+
+    - name: Verify guest-to-guest cluster network reachability
+      ansible.builtin.command:
+        argv:
+          - ping
+          - -c
+          - "1"
+          - -W
+          - "1"
+          - >-
+            {{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip
+            | split(',') | first }}
+      register: primary_cluster_ping
+      until: primary_cluster_ping.rc == 0
+      retries: 6
+      delay: 2
+      changed_when: false
+
     - name: Verify GitHub release host DNS
       ansible.builtin.getent:
         database: hosts

roles/k3s_server/tasks/fetch_k3s_init_logs.yml

@@ -22,6 +22,71 @@
   no_log: true
   register: k3s_init_status
 
+- name: Fetch IP address state
+  ansible.builtin.command:
+    cmd: ip -br address
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_ip_address_state
+
+- name: Fetch IP route state
+  ansible.builtin.command:
+    cmd: ip route show
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_ip_route_state
+
+- name: Fetch IP neighbor state
+  ansible.builtin.command:
+    cmd: ip neigh show
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_ip_neighbor_state
+
+- name: Fetch IP rule state
+  ansible.builtin.command:
+    cmd: ip rule show
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_ip_rule_state
+
+- name: Fetch listening TCP sockets
+  ansible.builtin.command:
+    cmd: ss -ltn
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_tcp_listener_state
+
+- name: Ping the primary Kubernetes API address from {{ ansible_hostname }}
+  ansible.builtin.command:
+    argv:
+      - ping
+      - -c
+      - "1"
+      - -W
+      - "1"
+      - "{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(',') | first }}"
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_primary_api_ping
+
+- name: Probe primary Kubernetes API from {{ ansible_hostname }}
+  ansible.builtin.wait_for:
+    host: "{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(',') | first }}"
+    port: 6443
+    connect_timeout: 1
+    timeout: 2
+  changed_when: false
+  failed_when: false
+  no_log: true
+  register: k3s_primary_api_probe
+
 - name: Create {{ log_destination }}
   delegate_to: localhost
   run_once: true
@@ -45,3 +110,37 @@
 
       === k3s-init.service journal ===
       {{ k3s_init_log.stdout | regex_replace('--token(?:=| +)[^ ]+', '--token ***') }}
+
+- name: Store network diagnostics to {{ log_destination }}
+  delegate_to: localhost
+  become: false
+  ansible.builtin.template:
+    src: content.j2
+    dest: "{{ log_destination }}/network@{{ ansible_hostname }}.log"
+    mode: "0644"
+  vars:
+    content: |
+      === ip -br address ===
+      {{ k3s_ip_address_state.stdout }}
+
+      === ip route show ===
+      {{ k3s_ip_route_state.stdout }}
+
+      === ip neigh show ===
+      {{ k3s_ip_neighbor_state.stdout }}
+
+      === ip rule show ===
+      {{ k3s_ip_rule_state.stdout }}
+
+      === ss -ltn ===
+      {{ k3s_tcp_listener_state.stdout }}
+
+      === primary API ping ===
+      rc={{ k3s_primary_api_ping.rc }}
+      {{ k3s_primary_api_ping.stdout }}
+      {{ k3s_primary_api_ping.stderr }}
+
+      === primary API probe ===
+      failed={{ k3s_primary_api_probe.failed | default(false) }}
+      elapsed={{ k3s_primary_api_probe.elapsed | default('unknown') }}
+      msg={{ k3s_primary_api_probe.msg | default('connected') }}

roles/k3s_server/tasks/join_master.yml

@@ -1,34 +1,18 @@
 ---
-- name: Verify primary Kubernetes API is reachable from {{ joining_master }}
-  ansible.builtin.wait_for:
-    host: "{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(',') | first }}"
-    port: 6443
-    connect_timeout: 5
-    timeout: "{{ retry_count | default(20) * 2 }}"
-  delegate_to: "{{ joining_master }}"
-
 - name: Join transient k3s-init service for {{ joining_master }}
   ansible.builtin.command:
     cmd: >-
       systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init
-      k3s server {{ joining_server_args }}
+      k3s server {{ hostvars[joining_master].k3s_server_init_args }}
     creates: "{{ systemd_dir }}/k3s-init.service"
   delegate_to: "{{ joining_master }}"
   no_log: true
-  vars:
-    primary_master: "{{ groups[group_name_master | default('master')][0] }}"
-    primary_master_api: >-
-      {{ hostvars[primary_master].k3s_node_ip | split(',') | first | ansible.utils.ipwrap }}
-    joining_server_args: >-
-      {{ hostvars[joining_master].server_init_args | default('--server https://' +
-      primary_master_api + ':6443 --token ' +
-      (hostvars[joining_master].k3s_token | default(k3s_token)) + ' ' +
-      (hostvars[joining_master].extra_server_args | default(''))) }}
 
 - name: Wait for primary registration of {{ joining_master }}
   ansible.builtin.command:
     cmd: >-
-      {{ k3s_kubectl_binary | default('k3s kubectl') }} get node {{ joining_master }}
+      {{ k3s_kubectl_binary | default('k3s kubectl') }} get node
+      {{ hostvars[joining_master].ansible_hostname }}
   delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
   register: joined_master
   until: joined_master.rc == 0

roles/k3s_server/tasks/main.yml

@@ -34,43 +34,64 @@
   tags: kubevip
   when: kube_vip_lb_ip_range is defined
 
-- name: Init the first master inside the transient k3s-init service
+- name: Initialize and verify the K3s control plane
-  ansible.builtin.command:
-    cmd: systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server {{ server_init_args }}
-    creates: "{{ systemd_dir }}/k3s-init.service"
-  no_log: true
-  when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
-
-- name: Wait for the first master Kubernetes API before joining other masters
-  ansible.builtin.command:
-    cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get --raw=/readyz"
-  register: first_master_api
-  until: first_master_api.rc == 0
-  retries: "{{ retry_count | default(20) }}"
-  delay: 2
-  changed_when: false
-  run_once: true
-  when: groups[group_name_master | default('master')] | length > 1
-
-- name: Join additional masters one at a time
-  ansible.builtin.include_tasks: join_master.yml
-  loop: "{{ groups[group_name_master | default('master')][1:] }}"
-  loop_control:
-    loop_var: joining_master
-  run_once: true
-  when: groups[group_name_master | default('master')] | length > 1
-
-- name: Verification
   when: not ansible_check_mode
   block:
+    - name: Materialize per-host server initialization arguments
+      ansible.builtin.set_fact:
+        k3s_server_init_args: "{{ server_init_args }}"
+      no_log: true
+      when: groups[group_name_master | default('master')] | length > 1
+
+    - name: Init the first master inside the transient k3s-init service
+      ansible.builtin.command:
+        cmd: >-
+          systemd-run -p RestartSec=2 -p Restart=on-failure
+          --unit=k3s-init k3s server {{ server_init_args }}
+        creates: "{{ systemd_dir }}/k3s-init.service"
+      no_log: true
+      when: inventory_hostname == groups[group_name_master | default('master')][0]
+
+    - name: Wait for the first master Kubernetes API
+      ansible.builtin.command:
+        cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get --raw=/readyz"
+      delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
+      register: first_master_api
+      until: first_master_api.rc == 0
+      retries: "{{ retry_count | default(20) }}"
+      delay: 2
+      changed_when: false
+      run_once: true
+
+    - name: Verify the first master API listener on its node address
+      ansible.builtin.wait_for:
+        host: >-
+          {{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip
+          | split(',') | first }}
+        port: 6443
+        connect_timeout: 2
+        timeout: "{{ retry_count | default(20) * 2 }}"
+      delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
+      run_once: true
+
+    - name: Join additional masters one at a time
+      ansible.builtin.include_tasks: join_master.yml
+      loop: "{{ groups[group_name_master | default('master')][1:] }}"
+      loop_control:
+        loop_var: joining_master
+      run_once: true
+      when: groups[group_name_master | default('master')] | length > 1
+
     - name: Verify that all nodes actually joined (check k3s-init.service if this fails)
       ansible.builtin.command:
         cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get nodes -l 'node-role.kubernetes.io/master=true' -o=jsonpath='{.items[*].metadata.name}'" # yamllint disable-line rule:line-length
+      delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
       register: nodes
       until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups[group_name_master | default('master')] | length) # yamllint disable-line rule:line-length
       retries: "{{ retry_count | default(20) }}"
       delay: 10
       changed_when: false
+      run_once: true
   always:
     - name: Save logs of k3s-init.service
       ansible.builtin.include_tasks: fetch_k3s_init_logs.yml