diff --git a/molecule/calico/overrides.yml b/molecule/calico/overrides.yml index a63ec44..bfecf73 100644 --- a/molecule/calico/overrides.yml +++ b/molecule/calico/overrides.yml @@ -7,6 +7,7 @@ # See: # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant calico_iface: eth1 + kube_vip_iface: eth1 # The test VMs might be a bit slow, so we give them more time to join the cluster: retry_count: 45 diff --git a/molecule/cilium/overrides.yml b/molecule/cilium/overrides.yml index c602a28..dbab142 100644 --- a/molecule/cilium/overrides.yml +++ b/molecule/cilium/overrides.yml @@ -7,6 +7,7 @@ # See: # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant cilium_iface: eth1 + kube_vip_iface: eth1 # The test VMs might be a bit slow, so we give them more time to join the cluster: retry_count: 45 diff --git a/molecule/default/overrides.yml b/molecule/default/overrides.yml index 4eea472..73c950f 100644 --- a/molecule/default/overrides.yml +++ b/molecule/default/overrides.yml @@ -7,6 +7,9 @@ # See: # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant flannel_iface: eth1 + # kube-vip cannot infer the cluster interface in these multi-NIC + # Vagrant guests because the default route is on eth0. + kube_vip_iface: eth1 # The test VMs might be a bit slow, so we give them more time to join the cluster: retry_count: 45 diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml index 08852f5..912016c 100644 --- a/molecule/default/prepare.yml +++ b/molecule/default/prepare.yml @@ -6,20 +6,26 @@ - name: Network setup hosts: all tasks: - - name: Disable firewalld - when: ansible_distribution == "Rocky" - # Rocky Linux comes with firewalld enabled. It blocks some of the network - # connections needed for our k3s cluster. For our test setup, we just disable - # it since the VM host's firewall is still active for connections to and from - # the Internet. + - name: Gather service facts + ansible.builtin.service_facts: + + - name: Disable guest firewall services + # The disposable test guests use an isolated VirtualBox network. A distro + # firewall can allow ICMP while silently blocking the inter-node Kubernetes + # API connection, so disable the known guest firewalls consistently. # When building your own cluster, please DO NOT blindly copy this. Instead, # please create a custom firewall configuration that fits your network design # and security needs. ansible.builtin.systemd: - name: firewalld + name: "{{ item }}" enabled: false state: stopped become: true + loop: + - firewalld.service + - nftables.service + - ufw.service + when: item in ansible_facts.services - name: Verify the private cluster interface ansible.builtin.assert: diff --git a/molecule/kube-vip/overrides.yml b/molecule/kube-vip/overrides.yml index 4577afc..936fb90 100644 --- a/molecule/kube-vip/overrides.yml +++ b/molecule/kube-vip/overrides.yml @@ -7,6 +7,7 @@ # See: # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant flannel_iface: eth1 + kube_vip_iface: eth1 # The test VMs might be a bit slow, so we give them more time to join the cluster: retry_count: 45 diff --git a/molecule/single_node/overrides.yml b/molecule/single_node/overrides.yml index 2cb8ec7..bac6ec8 100644 --- a/molecule/single_node/overrides.yml +++ b/molecule/single_node/overrides.yml @@ -7,6 +7,7 @@ # See: # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant flannel_iface: eth1 + kube_vip_iface: eth1 # The test VMs might be a bit slow, so we give them more time to join the cluster: retry_count: 45 diff --git a/roles/k3s_server/tasks/fetch_k3s_init_logs.yml b/roles/k3s_server/tasks/fetch_k3s_init_logs.yml index 4f08f4f..efd955c 100644 --- a/roles/k3s_server/tasks/fetch_k3s_init_logs.yml +++ b/roles/k3s_server/tasks/fetch_k3s_init_logs.yml @@ -62,6 +62,19 @@ no_log: true register: k3s_tcp_listener_state +- name: Fetch guest firewall service state + ansible.builtin.service_facts: + no_log: true + register: k3s_service_facts + +- name: Fetch input firewall rules + ansible.builtin.command: + cmd: iptables -S INPUT + changed_when: false + failed_when: false + no_log: true + register: k3s_input_firewall_rules + - name: Ping the primary Kubernetes API address from {{ ansible_hostname }} ansible.builtin.command: argv: @@ -135,6 +148,16 @@ === ss -ltn === {{ k3s_tcp_listener_state.stdout }} + === guest firewall services === + firewalld={{ k3s_service_facts.ansible_facts.services.get('firewalld.service', {}).get('state', 'not-found') }} + nftables={{ k3s_service_facts.ansible_facts.services.get('nftables.service', {}).get('state', 'not-found') }} + ufw={{ k3s_service_facts.ansible_facts.services.get('ufw.service', {}).get('state', 'not-found') }} + + === iptables -S INPUT === + rc={{ k3s_input_firewall_rules.rc }} + {{ k3s_input_firewall_rules.stdout }} + {{ k3s_input_firewall_rules.stderr }} + === primary API ping === rc={{ k3s_primary_api_ping.rc }} {{ k3s_primary_api_ping.stdout }} diff --git a/roles/k3s_server/tasks/join_master.yml b/roles/k3s_server/tasks/join_master.yml index 930489e..ca327ef 100644 --- a/roles/k3s_server/tasks/join_master.yml +++ b/roles/k3s_server/tasks/join_master.yml @@ -1,4 +1,14 @@ --- +- name: Verify primary Kubernetes API reachability from {{ joining_master }} + ansible.builtin.wait_for: + host: >- + {{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip + | split(',') | first }} + port: 6443 + connect_timeout: 2 + timeout: 30 + delegate_to: "{{ joining_master }}" + - name: Join transient k3s-init service for {{ joining_master }} ansible.builtin.command: cmd: >- diff --git a/roles/k3s_server/tasks/main.yml b/roles/k3s_server/tasks/main.yml index c1c6088..79482cb 100644 --- a/roles/k3s_server/tasks/main.yml +++ b/roles/k3s_server/tasks/main.yml @@ -35,6 +35,7 @@ when: kube_vip_lb_ip_range is defined - name: Initialize and verify the K3s control plane + any_errors_fatal: true when: not ansible_check_mode block: - name: Materialize per-host server initialization arguments @@ -43,55 +44,50 @@ no_log: true when: groups[group_name_master | default('master')] | length > 1 - - name: Init the first master inside the transient k3s-init service - ansible.builtin.command: - cmd: >- - systemd-run -p RestartSec=2 -p Restart=on-failure - --unit=k3s-init k3s server {{ server_init_args }} - creates: "{{ systemd_dir }}/k3s-init.service" - no_log: true + - name: Orchestrate control-plane initialization from the first master when: inventory_hostname == groups[group_name_master | default('master')][0] + block: + - name: Init the first master inside the transient k3s-init service + ansible.builtin.command: + cmd: >- + systemd-run -p RestartSec=2 -p Restart=on-failure + --unit=k3s-init k3s server {{ server_init_args }} + creates: "{{ systemd_dir }}/k3s-init.service" + no_log: true - - name: Wait for the first master Kubernetes API - ansible.builtin.command: - cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get --raw=/readyz" - delegate_to: "{{ groups[group_name_master | default('master')][0] }}" - register: first_master_api - until: first_master_api.rc == 0 - retries: "{{ retry_count | default(20) }}" - delay: 2 - changed_when: false - run_once: true + - name: Wait for the first master Kubernetes API + ansible.builtin.command: + cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get --raw=/readyz" + register: first_master_api + until: first_master_api.rc == 0 + retries: "{{ retry_count | default(20) }}" + delay: 2 + changed_when: false - - name: Verify the first master API listener on its node address - ansible.builtin.wait_for: - host: >- - {{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip - | split(',') | first }} - port: 6443 - connect_timeout: 2 - timeout: "{{ retry_count | default(20) * 2 }}" - delegate_to: "{{ groups[group_name_master | default('master')][0] }}" - run_once: true + - name: Verify the first master API listener on its node address + ansible.builtin.wait_for: + host: >- + {{ hostvars[groups[group_name_master | default('master')][0]].k3s_node_ip + | split(',') | first }} + port: 6443 + connect_timeout: 2 + timeout: "{{ retry_count | default(20) * 2 }}" - - name: Join additional masters one at a time - ansible.builtin.include_tasks: join_master.yml - loop: "{{ groups[group_name_master | default('master')][1:] }}" - loop_control: - loop_var: joining_master - run_once: true - when: groups[group_name_master | default('master')] | length > 1 + - name: Join additional masters one at a time + ansible.builtin.include_tasks: join_master.yml + loop: "{{ groups[group_name_master | default('master')][1:] }}" + loop_control: + loop_var: joining_master + when: groups[group_name_master | default('master')] | length > 1 - - name: Verify that all nodes actually joined (check k3s-init.service if this fails) - ansible.builtin.command: - cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get nodes -l 'node-role.kubernetes.io/master=true' -o=jsonpath='{.items[*].metadata.name}'" # yamllint disable-line rule:line-length - delegate_to: "{{ groups[group_name_master | default('master')][0] }}" - register: nodes - until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups[group_name_master | default('master')] | length) # yamllint disable-line rule:line-length - retries: "{{ retry_count | default(20) }}" - delay: 10 - changed_when: false - run_once: true + - name: Verify that all nodes actually joined (check k3s-init.service if this fails) + ansible.builtin.command: + cmd: "{{ k3s_kubectl_binary | default('k3s kubectl') }} get nodes -l 'node-role.kubernetes.io/master=true' -o=jsonpath='{.items[*].metadata.name}'" # yamllint disable-line rule:line-length + register: nodes + until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups[group_name_master | default('master')] | length) # yamllint disable-line rule:line-length + retries: "{{ retry_count | default(20) }}" + delay: 10 + changed_when: false always: - name: Save logs of k3s-init.service ansible.builtin.include_tasks: fetch_k3s_init_logs.yml