diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml index 0100002..08852f5 100644 --- a/molecule/default/prepare.yml +++ b/molecule/default/prepare.yml @@ -21,6 +21,33 @@ state: stopped become: true + - name: Verify the private cluster interface + ansible.builtin.assert: + that: + - flannel_iface in ansible_facts + - ansible_facts[flannel_iface].ipv4 is defined + - ansible_facts[flannel_iface].ipv4.address is defined + fail_msg: >- + The Vagrant private interface {{ flannel_iface }} does not have an + IPv4 address on {{ inventory_hostname }}. + + - name: Verify guest-to-guest cluster network reachability + ansible.builtin.command: + argv: + - ping + - -c + - "1" + - -W + - "1" + - >- + {{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip + | split(',') | first }} + register: primary_cluster_ping + until: primary_cluster_ping.rc == 0 + retries: 6 + delay: 2 + changed_when: false + - name: Verify GitHub release host DNS ansible.builtin.getent: database: hosts diff --git a/roles/k3s_server/tasks/fetch_k3s_init_logs.yml b/roles/k3s_server/tasks/fetch_k3s_init_logs.yml index a41db11..4f08f4f 100644 --- a/roles/k3s_server/tasks/fetch_k3s_init_logs.yml +++ b/roles/k3s_server/tasks/fetch_k3s_init_logs.yml @@ -22,6 +22,71 @@ no_log: true register: k3s_init_status +- name: Fetch IP address state + ansible.builtin.command: + cmd: ip -br address + changed_when: false + failed_when: false + no_log: true + register: k3s_ip_address_state + +- name: Fetch IP route state + ansible.builtin.command: + cmd: ip route show + changed_when: false + failed_when: false + no_log: true + register: k3s_ip_route_state + +- name: Fetch IP neighbor state + ansible.builtin.command: + cmd: ip neigh show + changed_when: false + failed_when: false + no_log: true + register: k3s_ip_neighbor_state + +- name: Fetch IP rule state + ansible.builtin.command: + cmd: ip rule show + changed_when: false + failed_when: false + no_log: true + register: k3s_ip_rule_state + +- name: Fetch listening TCP sockets + ansible.builtin.command: + cmd: ss -ltn + changed_when: false + failed_when: false + no_log: true + register: k3s_tcp_listener_state + +- name: Ping the primary Kubernetes API address from {{ ansible_hostname }} + ansible.builtin.command: + argv: + - ping + - -c + - "1" + - -W + - "1" + - "{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(',') | first }}" + changed_when: false + failed_when: false + no_log: true + register: k3s_primary_api_ping + +- name: Probe primary Kubernetes API from {{ ansible_hostname }} + ansible.builtin.wait_for: + host: "{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(',') | first }}" + port: 6443 + connect_timeout: 1 + timeout: 2 + changed_when: false + failed_when: false + no_log: true + register: k3s_primary_api_probe + - name: Create {{ log_destination }} delegate_to: localhost run_once: true @@ -45,3 +110,37 @@ === k3s-init.service journal === {{ k3s_init_log.stdout | regex_replace('--token(?:=| +)[^ ]+', '--token ***') }} + +- name: Store network diagnostics to {{ log_destination }} + delegate_to: localhost + become: false + ansible.builtin.template: + src: content.j2 + dest: "{{ log_destination }}/network@{{ ansible_hostname }}.log" + mode: "0644" + vars: + content: | + === ip -br address === + {{ k3s_ip_address_state.stdout }} + + === ip route show === + {{ k3s_ip_route_state.stdout }} + + === ip neigh show === + {{ k3s_ip_neighbor_state.stdout }} + + === ip rule show === + {{ k3s_ip_rule_state.stdout }} + + === ss -ltn === + {{ k3s_tcp_listener_state.stdout }} + + === primary API ping === + rc={{ k3s_primary_api_ping.rc }} + {{ k3s_primary_api_ping.stdout }} + {{ k3s_primary_api_ping.stderr }} + + === primary API probe === + failed={{ k3s_primary_api_probe.failed | default(false) }} + elapsed={{ k3s_primary_api_probe.elapsed | default('unknown') }} + msg={{ k3s_primary_api_probe.msg | default('connected') }} diff --git a/roles/k3s_server/tasks/join_master.yml b/roles/k3s_server/tasks/join_master.yml index 2b42d91..930489e 100644 --- a/roles/k3s_server/tasks/join_master.yml +++ b/roles/k3s_server/tasks/join_master.yml @@ -1,34 +1,18 @@ --- -- name: Verify primary Kubernetes API is reachable from {{ joining_master }} - ansible.builtin.wait_for: - host: "{{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip | split(',') | first }}" - port: 6443 - connect_timeout: 5 - timeout: "{{ retry_count | default(20) * 2 }}" - delegate_to: "{{ joining_master }}" - - name: Join transient k3s-init service for {{ joining_master }} ansible.builtin.command: cmd: >- systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init - k3s server {{ joining_server_args }} + k3s server {{ hostvars[joining_master].k3s_server_init_args }} creates: "{{ systemd_dir }}/k3s-init.service" delegate_to: "{{ joining_master }}" no_log: true - vars: - primary_master: "{{ groups[group_name_master | default('master')][0] }}" - primary_master_api: >- - {{ hostvars[primary_master].k3s_node_ip | split(',') | first | ansible.utils.ipwrap }} - joining_server_args: >- - {{ hostvars[joining_master].server_init_args | default('--server https://' + - primary_master_api + ':6443 --token ' + - (hostvars[joining_master].k3s_token | default(k3s_token)) + ' ' + - (hostvars[joining_master].extra_server_args | default(''))) }} - name: Wait for primary registration of {{ joining_master }} ansible.builtin.command: cmd: >- - {{ k3s_kubectl_binary | default('k3s kubectl') }} get node {{ joining_master }} + {{ k3s_kubectl_binary | default('k3s kubectl') }} get node + {{ hostvars[joining_master].ansible_hostname }} delegate_to: "{{ groups[group_name_master | default('master')][0] }}" register: joined_master until: joined_master.rc == 0 diff --git a/roles/k3s_server/tasks/main.yml b/roles/k3s_server/tasks/main.yml index 5a459b7..c1c6088 100644 --- a/roles/k3s_server/tasks/main.yml +++ b/roles/k3s_server/tasks/main.yml @@ -34,43 +34,64 @@ tags: kubevip when: kube_vip_lb_ip_range is defined -- name: Init the first master inside the transient k3s-init service - ansible.builtin.command: - cmd: systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server {{ server_init_args }} - creates: "{{ systemd_dir }}/k3s-init.service" - no_log: true - when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] - -- name: Wait for the first master Kubernetes API before joining other masters - ansible.builtin.command: - cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get --raw=/readyz" - register: first_master_api - until: first_master_api.rc == 0 - retries: "{{ retry_count | default(20) }}" - delay: 2 - changed_when: false - run_once: true - when: groups[group_name_master | default('master')] | length > 1 - -- name: Join additional masters one at a time - ansible.builtin.include_tasks: join_master.yml - loop: "{{ groups[group_name_master | default('master')][1:] }}" - loop_control: - loop_var: joining_master - run_once: true - when: groups[group_name_master | default('master')] | length > 1 - -- name: Verification +- name: Initialize and verify the K3s control plane when: not ansible_check_mode block: + - name: Materialize per-host server initialization arguments + ansible.builtin.set_fact: + k3s_server_init_args: "{{ server_init_args }}" + no_log: true + when: groups[group_name_master | default('master')] | length > 1 + + - name: Init the first master inside the transient k3s-init service + ansible.builtin.command: + cmd: >- + systemd-run -p RestartSec=2 -p Restart=on-failure + --unit=k3s-init k3s server {{ server_init_args }} + creates: "{{ systemd_dir }}/k3s-init.service" + no_log: true + when: inventory_hostname == groups[group_name_master | default('master')][0] + + - name: Wait for the first master Kubernetes API + ansible.builtin.command: + cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get --raw=/readyz" + delegate_to: "{{ groups[group_name_master | default('master')][0] }}" + register: first_master_api + until: first_master_api.rc == 0 + retries: "{{ retry_count | default(20) }}" + delay: 2 + changed_when: false + run_once: true + + - name: Verify the first master API listener on its node address + ansible.builtin.wait_for: + host: >- + {{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip + | split(',') | first }} + port: 6443 + connect_timeout: 2 + timeout: "{{ retry_count | default(20) * 2 }}" + delegate_to: "{{ groups[group_name_master | default('master')][0] }}" + run_once: true + + - name: Join additional masters one at a time + ansible.builtin.include_tasks: join_master.yml + loop: "{{ groups[group_name_master | default('master')][1:] }}" + loop_control: + loop_var: joining_master + run_once: true + when: groups[group_name_master | default('master')] | length > 1 + - name: Verify that all nodes actually joined (check k3s-init.service if this fails) ansible.builtin.command: cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get nodes -l 'node-role.kubernetes.io/master=true' -o=jsonpath='{.items[*].metadata.name}'" # yamllint disable-line rule:line-length + delegate_to: "{{ groups[group_name_master | default('master')][0] }}" register: nodes until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups[group_name_master | default('master')] | length) # yamllint disable-line rule:line-length retries: "{{ retry_count | default(20) }}" delay: 10 changed_when: false + run_once: true always: - name: Save logs of k3s-init.service ansible.builtin.include_tasks: fetch_k3s_init_logs.yml