forked from tim/k3s-ansible
fix(k3s-server): serialize master joins and capture diagnostics
- validate host-only connectivity before cluster bootstrap\n- materialize per-host join arguments and wait for sequential registration\n- collect bounded network diagnostics when k3s initialization fails
This commit is contained in:
Timothy Stewart
committed by
Techno Tim
Techno Tim
parent
a0d78ff317
commit
9b220c1629
@@ -21,6 +21,33 @@
|
||||
state: stopped
|
||||
become: true
|
||||
|
||||
- name: Verify the private cluster interface
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- flannel_iface in ansible_facts
|
||||
- ansible_facts[flannel_iface].ipv4 is defined
|
||||
- ansible_facts[flannel_iface].ipv4.address is defined
|
||||
fail_msg: >-
|
||||
The Vagrant private interface {{ flannel_iface }} does not have an
|
||||
IPv4 address on {{ inventory_hostname }}.
|
||||
|
||||
- name: Verify guest-to-guest cluster network reachability
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- ping
|
||||
- -c
|
||||
- "1"
|
||||
- -W
|
||||
- "1"
|
||||
- >-
|
||||
{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip
|
||||
| split(',') | first }}
|
||||
register: primary_cluster_ping
|
||||
until: primary_cluster_ping.rc == 0
|
||||
retries: 6
|
||||
delay: 2
|
||||
changed_when: false
|
||||
|
||||
- name: Verify GitHub release host DNS
|
||||
ansible.builtin.getent:
|
||||
database: hosts
|
||||
|
||||
@@ -22,6 +22,71 @@
|
||||
no_log: true
|
||||
register: k3s_init_status
|
||||
|
||||
- name: Fetch IP address state
|
||||
ansible.builtin.command:
|
||||
cmd: ip -br address
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
no_log: true
|
||||
register: k3s_ip_address_state
|
||||
|
||||
- name: Fetch IP route state
|
||||
ansible.builtin.command:
|
||||
cmd: ip route show
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
no_log: true
|
||||
register: k3s_ip_route_state
|
||||
|
||||
- name: Fetch IP neighbor state
|
||||
ansible.builtin.command:
|
||||
cmd: ip neigh show
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
no_log: true
|
||||
register: k3s_ip_neighbor_state
|
||||
|
||||
- name: Fetch IP rule state
|
||||
ansible.builtin.command:
|
||||
cmd: ip rule show
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
no_log: true
|
||||
register: k3s_ip_rule_state
|
||||
|
||||
- name: Fetch listening TCP sockets
|
||||
ansible.builtin.command:
|
||||
cmd: ss -ltn
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
no_log: true
|
||||
register: k3s_tcp_listener_state
|
||||
|
||||
- name: Ping the primary Kubernetes API address from {{ ansible_hostname }}
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- ping
|
||||
- -c
|
||||
- "1"
|
||||
- -W
|
||||
- "1"
|
||||
- "{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(',') | first }}"
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
no_log: true
|
||||
register: k3s_primary_api_ping
|
||||
|
||||
- name: Probe primary Kubernetes API from {{ ansible_hostname }}
|
||||
ansible.builtin.wait_for:
|
||||
host: "{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(',') | first }}"
|
||||
port: 6443
|
||||
connect_timeout: 1
|
||||
timeout: 2
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
no_log: true
|
||||
register: k3s_primary_api_probe
|
||||
|
||||
- name: Create {{ log_destination }}
|
||||
delegate_to: localhost
|
||||
run_once: true
|
||||
@@ -45,3 +110,37 @@
|
||||
|
||||
=== k3s-init.service journal ===
|
||||
{{ k3s_init_log.stdout | regex_replace('--token(?:=| +)[^ ]+', '--token ***') }}
|
||||
|
||||
- name: Store network diagnostics to {{ log_destination }}
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
ansible.builtin.template:
|
||||
src: content.j2
|
||||
dest: "{{ log_destination }}/network@{{ ansible_hostname }}.log"
|
||||
mode: "0644"
|
||||
vars:
|
||||
content: |
|
||||
=== ip -br address ===
|
||||
{{ k3s_ip_address_state.stdout }}
|
||||
|
||||
=== ip route show ===
|
||||
{{ k3s_ip_route_state.stdout }}
|
||||
|
||||
=== ip neigh show ===
|
||||
{{ k3s_ip_neighbor_state.stdout }}
|
||||
|
||||
=== ip rule show ===
|
||||
{{ k3s_ip_rule_state.stdout }}
|
||||
|
||||
=== ss -ltn ===
|
||||
{{ k3s_tcp_listener_state.stdout }}
|
||||
|
||||
=== primary API ping ===
|
||||
rc={{ k3s_primary_api_ping.rc }}
|
||||
{{ k3s_primary_api_ping.stdout }}
|
||||
{{ k3s_primary_api_ping.stderr }}
|
||||
|
||||
=== primary API probe ===
|
||||
failed={{ k3s_primary_api_probe.failed | default(false) }}
|
||||
elapsed={{ k3s_primary_api_probe.elapsed | default('unknown') }}
|
||||
msg={{ k3s_primary_api_probe.msg | default('connected') }}
|
||||
|
||||
@@ -1,34 +1,18 @@
|
||||
---
|
||||
- name: Verify primary Kubernetes API is reachable from {{ joining_master }}
|
||||
ansible.builtin.wait_for:
|
||||
host: "{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(',') | first }}"
|
||||
port: 6443
|
||||
connect_timeout: 5
|
||||
timeout: "{{ retry_count | default(20) * 2 }}"
|
||||
delegate_to: "{{ joining_master }}"
|
||||
|
||||
- name: Join transient k3s-init service for {{ joining_master }}
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init
|
||||
k3s server {{ joining_server_args }}
|
||||
k3s server {{ hostvars[joining_master].k3s_server_init_args }}
|
||||
creates: "{{ systemd_dir }}/k3s-init.service"
|
||||
delegate_to: "{{ joining_master }}"
|
||||
no_log: true
|
||||
vars:
|
||||
primary_master: "{{ groups[group_name_master | default('master')][0] }}"
|
||||
primary_master_api: >-
|
||||
{{ hostvars[primary_master].k3s_node_ip | split(',') | first | ansible.utils.ipwrap }}
|
||||
joining_server_args: >-
|
||||
{{ hostvars[joining_master].server_init_args | default('--server https://' +
|
||||
primary_master_api + ':6443 --token ' +
|
||||
(hostvars[joining_master].k3s_token | default(k3s_token)) + ' ' +
|
||||
(hostvars[joining_master].extra_server_args | default(''))) }}
|
||||
|
||||
- name: Wait for primary registration of {{ joining_master }}
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
{{ k3s_kubectl_binary | default('k3s kubectl') }} get node {{ joining_master }}
|
||||
{{ k3s_kubectl_binary | default('k3s kubectl') }} get node
|
||||
{{ hostvars[joining_master].ansible_hostname }}
|
||||
delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
|
||||
register: joined_master
|
||||
until: joined_master.rc == 0
|
||||
|
||||
@@ -34,43 +34,64 @@
|
||||
tags: kubevip
|
||||
when: kube_vip_lb_ip_range is defined
|
||||
|
||||
- name: Init the first master inside the transient k3s-init service
|
||||
ansible.builtin.command:
|
||||
cmd: systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server {{ server_init_args }}
|
||||
creates: "{{ systemd_dir }}/k3s-init.service"
|
||||
no_log: true
|
||||
when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname']
|
||||
|
||||
- name: Wait for the first master Kubernetes API before joining other masters
|
||||
ansible.builtin.command:
|
||||
cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get --raw=/readyz"
|
||||
register: first_master_api
|
||||
until: first_master_api.rc == 0
|
||||
retries: "{{ retry_count | default(20) }}"
|
||||
delay: 2
|
||||
changed_when: false
|
||||
run_once: true
|
||||
when: groups[group_name_master | default('master')] | length > 1
|
||||
|
||||
- name: Join additional masters one at a time
|
||||
ansible.builtin.include_tasks: join_master.yml
|
||||
loop: "{{ groups[group_name_master | default('master')][1:] }}"
|
||||
loop_control:
|
||||
loop_var: joining_master
|
||||
run_once: true
|
||||
when: groups[group_name_master | default('master')] | length > 1
|
||||
|
||||
- name: Verification
|
||||
- name: Initialize and verify the K3s control plane
|
||||
when: not ansible_check_mode
|
||||
block:
|
||||
- name: Materialize per-host server initialization arguments
|
||||
ansible.builtin.set_fact:
|
||||
k3s_server_init_args: "{{ server_init_args }}"
|
||||
no_log: true
|
||||
when: groups[group_name_master | default('master')] | length > 1
|
||||
|
||||
- name: Init the first master inside the transient k3s-init service
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
systemd-run -p RestartSec=2 -p Restart=on-failure
|
||||
--unit=k3s-init k3s server {{ server_init_args }}
|
||||
creates: "{{ systemd_dir }}/k3s-init.service"
|
||||
no_log: true
|
||||
when: inventory_hostname == groups[group_name_master | default('master')][0]
|
||||
|
||||
- name: Wait for the first master Kubernetes API
|
||||
ansible.builtin.command:
|
||||
cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get --raw=/readyz"
|
||||
delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
|
||||
register: first_master_api
|
||||
until: first_master_api.rc == 0
|
||||
retries: "{{ retry_count | default(20) }}"
|
||||
delay: 2
|
||||
changed_when: false
|
||||
run_once: true
|
||||
|
||||
- name: Verify the first master API listener on its node address
|
||||
ansible.builtin.wait_for:
|
||||
host: >-
|
||||
{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip
|
||||
| split(',') | first }}
|
||||
port: 6443
|
||||
connect_timeout: 2
|
||||
timeout: "{{ retry_count | default(20) * 2 }}"
|
||||
delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
|
||||
run_once: true
|
||||
|
||||
- name: Join additional masters one at a time
|
||||
ansible.builtin.include_tasks: join_master.yml
|
||||
loop: "{{ groups[group_name_master | default('master')][1:] }}"
|
||||
loop_control:
|
||||
loop_var: joining_master
|
||||
run_once: true
|
||||
when: groups[group_name_master | default('master')] | length > 1
|
||||
|
||||
- name: Verify that all nodes actually joined (check k3s-init.service if this fails)
|
||||
ansible.builtin.command:
|
||||
cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get nodes -l 'node-role.kubernetes.io/master=true' -o=jsonpath='{.items[*].metadata.name}'" # yamllint disable-line rule:line-length
|
||||
delegate_to: "{{ groups[group_name_master | default('master')][0] }}"
|
||||
register: nodes
|
||||
until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups[group_name_master | default('master')] | length) # yamllint disable-line rule:line-length
|
||||
retries: "{{ retry_count | default(20) }}"
|
||||
delay: 10
|
||||
changed_when: false
|
||||
run_once: true
|
||||
always:
|
||||
- name: Save logs of k3s-init.service
|
||||
ansible.builtin.include_tasks: fetch_k3s_init_logs.yml
|
||||
|
||||
Reference in New Issue
Block a user